[DNM] Release 1.2.1 staging

.github/workflows/bot.yml

@@ -67,7 +67,7 @@ jobs:
         include:
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.5"
-            flinkProfile: "flink1.18"
+            flinkProfile: "flink2.1"
 
     steps:
       - uses: actions/checkout@v5
@@ -126,7 +126,7 @@ jobs:
         include:
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.5"
-            flinkProfile: "flink1.18"
+            flinkProfile: "flink2.1"
 
     steps:
       - uses: actions/checkout@v5
@@ -178,7 +178,7 @@ jobs:
         include:
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.5"
-            flinkProfile: "flink1.18"
+            flinkProfile: "flink2.1"
 
     env:
       UT_MODULES: >-
@@ -563,7 +563,7 @@ jobs:
         include:
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.5"
-            flinkProfile: "flink1.20"
+            flinkProfile: "flink2.1"
 
     steps:
       - uses: actions/checkout@v5
@@ -580,14 +580,14 @@ jobs:
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           FLINK_PROFILE: ${{ matrix.flinkProfile }}
         run:
-          mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client
+          mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client
       - name: UT - hudi-hadoop-mr and hudi-client/hudi-java-client
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
           FLINK_PROFILE: ${{ matrix.flinkProfile }}
         run:
-          mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS -Djacoco.skip=false
+          mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS -Djacoco.skip=false
       - name: Generate merged coverage report
         if: always()
         run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE
@@ -938,19 +938,30 @@ jobs:
           FLINK_PROFILE: ${{ matrix.flinkProfile }}
           FLINK_AVRO_VERSION: ${{ matrix.flinkAvroVersion }}
           FLINK_PARQUET_VERSION: ${{ matrix.flinkParquetVersion }}
-        if: ${{ endsWith(env.FLINK_PROFILE, '1.20') }}
+        if: ${{ endsWith(env.FLINK_PROFILE, '2.1') }}
         run: |
           mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS
-          mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER1 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS
+          mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER1 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false
+      - name: Generate merged coverage report
+        if: always() && endsWith(matrix.flinkProfile, '2.1')
+        run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE
+      - name: Upload coverage to Codecov
+        if: always() && endsWith(matrix.flinkProfile, '2.1')
+        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5
+        with:
+          files: ./jacoco-report.xml
+          disable_search: true
+          flags: flink-integration-tests
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   test-flink-2:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         include:
-          - flinkProfile: "flink1.20"
+          - flinkProfile: "flink2.1"
             flinkAvroVersion: "1.11.4"
-            flinkParquetVersion: '1.13.1'
+            flinkParquetVersion: '1.15.2'
     steps:
       - uses: actions/checkout@v5
       - name: Set up JDK 11
@@ -974,30 +985,41 @@ jobs:
           FLINK_PROFILE: ${{ matrix.flinkProfile }}
           FLINK_AVRO_VERSION: ${{ matrix.flinkAvroVersion }}
           FLINK_PARQUET_VERSION: ${{ matrix.flinkParquetVersion }}
-        if: ${{ endsWith(env.FLINK_PROFILE, '1.20') }}
+        if: ${{ endsWith(env.FLINK_PROFILE, '2.1') }}
         run: |
           mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS
-          mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER2 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS
+          mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER2 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false
+      - name: Generate merged coverage report
+        if: always() && endsWith(matrix.flinkProfile, '2.1')
+        run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE
+      - name: Upload coverage to Codecov
+        if: always() && endsWith(matrix.flinkProfile, '2.1')
+        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5
+        with:
+          files: ./jacoco-report.xml
+          disable_search: true
+          flags: flink-integration-tests
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   docker-java17-test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         include:
           - scalaProfile: 'scala-2.13'
-            flinkProfile: 'flink1.20'
+            flinkProfile: 'flink2.1'
             sparkProfile: 'spark3.5'
             sparkRuntime: 'spark3.5.0'
           - scalaProfile: 'scala-2.12'
-            flinkProfile: 'flink1.20'
+            flinkProfile: 'flink2.1'
             sparkProfile: 'spark3.5'
             sparkRuntime: 'spark3.5.0'
           - scalaProfile: 'scala-2.13'
-            flinkProfile: 'flink1.20'
+            flinkProfile: 'flink2.1'
             sparkProfile: 'spark4.0'
             sparkRuntime: 'spark4.0.0'
           - scalaProfile: 'scala-2.13'
-            flinkProfile: 'flink1.20'
+            flinkProfile: 'flink2.1'
             sparkProfile: 'spark4.1'
             sparkRuntime: 'spark4.1.1'
 
@@ -1074,7 +1096,7 @@ jobs:
             mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS
             # TODO remove the sudo below. It's a needed workaround as detailed in HUDI-5708.
             sudo chown -R "$USER:$(id -g -n)" hudi-platform-service/hudi-metaserver/target/generated-sources
-            mvn package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION"
+            mvn package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION"
           fi
       - name: IT - Bundle Validation - OpenJDK 11
         env:
@@ -1201,7 +1223,7 @@ jobs:
       matrix:
         include:
           - sparkProfile: 'spark3.5'
-            flinkProfile: 'flink1.20'
+            flinkProfile: 'flink2.1'
             sparkArchive: 'spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz'
     steps:
       - uses: actions/checkout@v5
@@ -1251,6 +1273,81 @@ jobs:
           rm -f $GITHUB_WORKSPACE/$SPARK_ARCHIVE
           mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -pl !hudi-flink-datasource/hudi-flink $MVN_ARGS
 
+  integration-tests-hive-sync:
+    # Testcontainers-based E2E hive sync coverage for Hudi's custom logical types
+    # (VECTOR, BLOB) and the Spark 4.0+ VARIANT type. Runs the integ2 testcontainers
+    # suite (ITTestCustomTypeHiveSync) against a real Hive metastore on Spark 3.5.3,
+    # 4.0.2, and 4.1.1 stacks.
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sparkProfile: 'spark3.5'
+            scalaProfile: '-Dscala-2.12 -Dscala.binary.version=2.12'
+            flinkProfile: 'flink1.20'
+            jdkVersion: '11'
+            composePrefix: 'docker-compose_hadoop284_hive2310_spark353'
+            sparkAdhocImage: 'apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.3:latest'
+          - sparkProfile: 'spark4.0'
+            scalaProfile: '-Dscala-2.13 -Dscala.binary.version=2.13'
+            flinkProfile: 'flink1.20'
+            jdkVersion: '17'
+            composePrefix: 'docker-compose_hadoop340_hive2310_spark402'
+            sparkAdhocImage: 'apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.0.2:latest'
+          - sparkProfile: 'spark4.1'
+            scalaProfile: '-Dscala-2.13 -Dscala.binary.version=2.13'
+            flinkProfile: 'flink1.20'
+            jdkVersion: '17'
+            composePrefix: 'docker-compose_hadoop340_hive2310_spark411'
+            sparkAdhocImage: 'apachehudi/hudi-hadoop_3.4.0-hive_2.3.10-sparkadhoc_4.1.1:latest'
+    steps:
+      - uses: actions/checkout@v5
+      - name: Set up JDK ${{ matrix.jdkVersion }}
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ matrix.jdkVersion }}
+          distribution: 'temurin'
+          architecture: x64
+          cache: maven
+      - name: Free disk space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          docker system prune --all --force --volumes
+      - name: Pre-pull compose images (fails fast if not published)
+        env:
+          SPARK_ADHOC_IMAGE: ${{ matrix.sparkAdhocImage }}
+        run: |
+          # Surface missing Spark 4.0.2 images before the 15-minute Maven install.
+          # The remaining images in the compose stack are pulled by docker-compose at
+          # test time.
+          docker pull "$SPARK_ADHOC_IMAGE"
+      - name: Build and install Hudi artifacts
+        env:
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+        run:
+          mvn clean install -T 2 $SCALA_PROFILE -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Pintegration-tests -DskipTests=true -Ddocker.compose.skip=true $MVN_ARGS
+      - name: Run integ2 testcontainers suite
+        env:
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+          COMPOSE_PREFIX: ${{ matrix.composePrefix }}
+        run: |
+          # -DskipITs=false overrides the spark4.0 profile's skipITs=true default
+          # (see root pom.xml). Without it, failsafe skips all ITs on the spark4.0 matrix row.
+          mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests \
+            -pl hudi-integ-test \
+            -DskipITs=false \
+            -Ddocker.compose.skip=true \
+            -Dit.test='ITTestCustomTypeHiveSync' \
+            -Dspark.docker.compose.prefix=$COMPOSE_PREFIX \
+            $MVN_ARGS
+
   build-spark-java17:
     runs-on: ubuntu-latest
     strategy:
@@ -1297,9 +1394,9 @@ jobs:
       matrix:
         include:
           - scalaProfile: "scala-2.12"
-            flinkProfile: "flink1.20"
+            flinkProfile: "flink2.1"
             flinkAvroVersion: '1.11.4'
-            flinkParquetVersion: '1.13.1'
+            flinkParquetVersion: '1.15.2'
     steps:
       - uses: actions/checkout@v5
       - name: Set up JDK 17
@@ -1316,33 +1413,11 @@ jobs:
           FLINK_AVRO_VERSION: ${{ matrix.flinkAvroVersion }}
           FLINK_PARQUET_VERSION: ${{ matrix.flinkParquetVersion }}
         run:
-          mvn clean install -T 2 -Djava17 -Djava.version=17 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS
+          mvn clean install -T 2 -Djava17 -Djava.version=17 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS
       - name: Quickstart Test
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
           FLINK_PROFILE: ${{ matrix.flinkProfile }}
         run:
           mvn test -Punit-tests -Djava17 -Djava.version=17 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink $MVN_ARGS
 
-  test-hudi-trino-plugin:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v5
-      - name: Set up JDK 23
-        uses: actions/setup-java@v5
-        with:
-          # Note: We are not caching here again, as we want to use the .m2 repository populated by
-          # the previous step
-          java-version: '23'
-          distribution: 'temurin'
-          architecture: x64
-          cache: maven
-      - name: Build hudi-trino-plugin with JDK 23
-        working-directory: ./hudi-trino-plugin
-        run:
-          mvn clean install -DskipTests
-      - name: Test hudi-trino-plugin with JDK 23
-        working-directory: ./hudi-trino-plugin
-        run:
-          mvn test -Dapi.version=1.44