Upgrade Elasticsearch to 9.5.0

.github/workflows/build.yaml

@@ -83,6 +83,8 @@ jobs:
     timeout-minutes: 30
     outputs:
       version: ${{ steps.version.outputs.version }}
+      elasticsearch_image_tag: ${{ steps.elasticsearch_image.outputs.tag }}
+      build_elasticsearch_image: ${{ steps.elasticsearch_image.outputs.build_locally }}
       should_publish: ${{ (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'dev-preview')) && secrets.DOCKER_USERNAME != '' && secrets.DOCKER_PASSWORD != '' }}
       is_prod_deploy: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name != 'pull_request' }}
       is_dev_deploy: ${{ (github.event_name == 'repository_dispatch' && github.event.action == 'preview') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'dev-preview')) }}
@@ -129,16 +131,77 @@ jobs:
           echo "version=$version" >> $GITHUB_OUTPUT
           echo "### $version" >> $GITHUB_STEP_SUMMARY
 
+      - name: Resolve Elasticsearch image
+        id: elasticsearch_image
+        env:
+          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }}
+          PREVIEW_BASE_SHA: ${{ github.event.client_payload.base_sha }}
+          PUSH_BEFORE_SHA: ${{ github.event.before }}
+        run: |
+          version=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile)
+          tag=$version
+          image_changed=false
+          build_locally=false
+
+          if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] &&
+            ! git diff --quiet "$PR_BASE_SHA"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then
+            image_changed=true
+          elif [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" == "refs/heads/main" ]] &&
+            ! git diff --quiet "$PUSH_BEFORE_SHA"..HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then
+            image_changed=true
+          elif [[ "$GITHUB_EVENT_NAME" == "repository_dispatch" ]] &&
+            ! git diff --quiet "${PREVIEW_BASE_SHA:-origin/main}"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then
+            image_changed=true
+          elif [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]] &&
+            ! git diff --quiet origin/main...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then
+            image_changed=true
+          fi
+
+          if [[ "$image_changed" == "true" ]]; then
+            image_sha=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1)
+            tag="$version-sha256-$image_sha"
+            image="exceptionless/elasticsearch:$tag"
+
+            if [[ "$GITHUB_EVENT_NAME" == "pull_request" && "$PR_HEAD_REPOSITORY" != "$GITHUB_REPOSITORY" ]]; then
+              build_locally=true
+            else
+              for attempt in {1..150}; do
+                if docker manifest inspect "$image" > /dev/null 2>&1; then
+                  break
+                fi
+
+                if [[ "$attempt" -eq 150 ]]; then
+                  echo "::error::Timed out waiting for $image to be published."
+                  exit 1
+                fi
+
+                sleep 10
+              done
+            fi
+          fi
+
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"
+          echo "build_locally=$build_locally" >> "$GITHUB_OUTPUT"
+          echo "### Elasticsearch image: exceptionless/elasticsearch:$tag" >> "$GITHUB_STEP_SUMMARY"
+
   test-api:
+    needs: version
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    env:
+      Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }}
 
     steps:
       - name: Checkout
         uses: actions/checkout@v7
         with:
           ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }}
 
+      - name: Build fork Elasticsearch candidate locally
+        if: ${{ needs.version.outputs.build_elasticsearch_image == 'true' }}
+        run: docker build --tag "exceptionless/elasticsearch:${{ needs.version.outputs.elasticsearch_image_tag }}" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x
+
       - name: Setup .NET Core
         uses: actions/setup-dotnet@v6
         with:
@@ -223,12 +286,21 @@ jobs:
         run: echo "npm run test:integration"
 
   test-e2e:
+    needs: version
     runs-on: ubuntu-latest
     timeout-minutes: 45
+    env:
+      Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }}
 
     steps:
       - name: Checkout
         uses: actions/checkout@v6
+        with:
+          ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }}
+
+      - name: Build fork Elasticsearch candidate locally
+        if: ${{ needs.version.outputs.build_elasticsearch_image == 'true' }}
+        run: docker build --tag "exceptionless/elasticsearch:${{ needs.version.outputs.elasticsearch_image_tag }}" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x
 
       - name: Setup .NET Core
         uses: actions/setup-dotnet@v6

.github/workflows/elasticsearch-docker-8.yml

@@ -43,4 +43,4 @@ jobs:
         working-directory: build/docker/elasticsearch/8.x
         run: |
           VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile)
-          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION --tag exceptionless/elasticsearch:latest
+          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION

.github/workflows/elasticsearch-docker-9.yml

@@ -0,0 +1,52 @@
+name: Elasticsearch 9.x Docker Image CI
+
+on:
+  push:
+    paths:
+      - "build/docker/elasticsearch/9.x/**"
+      - ".github/workflows/elasticsearch-docker-9.yml"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v') != true
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Setup .NET Core
+        uses: actions/setup-dotnet@v5
+        with:
+          dotnet-version: 10.0.301
+      - name: Build Reason
+        env:
+          GITHUB_EVENT: ${{ toJson(github) }}
+        run: "echo ref: ${{github.ref}} event: ${{github.event_name}}"
+      - name: Build Version
+        run: |
+          dotnet tool install --global minver-cli --version 7.0.0
+          version=$(minver --tag-prefix v)
+          echo "MINVERVERSIONOVERRIDE=$version" >> $GITHUB_ENV
+          echo "VERSION=$version" >> $GITHUB_ENV
+          echo "### Version: $version" >> $GITHUB_STEP_SUMMARY
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+      - name: Login to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+        with:
+          platforms: linux/amd64,linux/arm64
+      - name: Build custom Elasticsearch 9.x docker image
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile)
+          IMAGE_SHA=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1)
+          TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA")
+          if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then
+            TAGS+=(--tag "exceptionless/elasticsearch:$VERSION" --tag "exceptionless/elasticsearch:latest")
+          fi
+          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x "${TAGS[@]}"

.github/workflows/preview-command.yml

@@ -77,6 +77,7 @@ jobs:
             const headRepository = pullRequest.head.repo.full_name;
             const headRef = pullRequest.head.ref;
             const headSha = pullRequest.head.sha;
+            const baseSha = pullRequest.base.sha;
             const headLabel = pullRequest.head.label;
             const headShortSha = headSha.slice(0, 12);
 
@@ -95,6 +96,7 @@ jobs:
             core.setOutput("head-ref", headRef);
             core.setOutput("head-label", headLabel);
             core.setOutput("head-sha", headSha);
+            core.setOutput("base-sha", baseSha);
             core.setOutput("head-short-sha", headShortSha);
 
             const previewLabel = "dev-preview";
@@ -142,6 +144,7 @@ jobs:
           HEAD_REF: ${{ steps.preview.outputs.head-ref }}
           HEAD_LABEL: ${{ steps.preview.outputs.head-label }}
           HEAD_SHA: ${{ steps.preview.outputs.head-sha }}
+          BASE_SHA: ${{ steps.preview.outputs.base-sha }}
         with:
           script: |
             await github.rest.repos.createDispatchEvent({
@@ -152,7 +155,8 @@ jobs:
                 pr_number: Number(process.env.PR_NUMBER),
                 head_ref: process.env.HEAD_REF,
                 head_label: process.env.HEAD_LABEL,
-                head_sha: process.env.HEAD_SHA
+                head_sha: process.env.HEAD_SHA,
+                base_sha: process.env.BASE_SHA
               }
             });
 

Dockerfile

@@ -100,7 +100,7 @@ ENTRYPOINT ["/app/app-docker-entrypoint.sh"]
 
 # completely self-contained
 
-FROM exceptionless/elasticsearch:8.19.15 AS exceptionless
+FROM exceptionless/elasticsearch:9.5.0 AS exceptionless
 
 WORKDIR /app
 COPY --from=job-publish /app/src/Exceptionless.Job/out ./
@@ -113,21 +113,23 @@ COPY ./build/supervisord.conf /etc/
 USER root
 
 # install dotnet and supervisor
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
-    supervisor \
+RUN microdnf install -y \
     wget \
     dos2unix \
     ca-certificates \
+    python3-pip \
     \
     # .NET dependencies
-    libc6 \
-    libgcc-s1 \
-    libicu74 \
-    libssl3 \
-    libstdc++6 \
+    glibc \
+    gzip \
+    libgcc \
+    libicu \
+    openssl-libs \
+    libstdc++ \
+    tar \
     tzdata \
-    && rm -rf /var/lib/apt/lists/* \
+    && pip3 install --no-cache-dir supervisor==4.3.0 \
+    && microdnf clean all \
     && dos2unix /app/docker-entrypoint.sh
 
 ENV discovery.type=single-node \

build/docker/elasticsearch/9.x/Dockerfile

@@ -0,0 +1,4 @@
+# https://www.docker.elastic.co/
+FROM docker.elastic.co/elasticsearch/elasticsearch:9.5.0
+
+RUN elasticsearch-plugin install -b mapper-size

docker/docker-compose.apm.yml

@@ -2,7 +2,7 @@ version: "2.2"
 
 services:
   setup:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.5.0
     volumes:
       - certs:/usr/share/elasticsearch/config/certs
     user: "0"
@@ -53,7 +53,7 @@ services:
     depends_on:
       setup:
         condition: service_healthy
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.5.0
     volumes:
       - certs:/usr/share/elasticsearch/config/certs
       - esdata:/usr/share/elasticsearch/data
@@ -98,7 +98,7 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.5.0
     volumes:
       - certs:/usr/share/kibana/config/certs
     ports:
@@ -124,7 +124,7 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    image: docker.elastic.co/apm/apm-server:8.19.15
+    image: docker.elastic.co/apm/apm-server:9.5.0
     volumes:
       - certs:/usr/share/apm-server/certs
     ports:

docker/docker-compose.dev.yml

@@ -50,7 +50,7 @@ services:
       - appdata:/app/storage
 
   elasticsearch:
-    image: exceptionless/elasticsearch:8.19.15
+    image: exceptionless/elasticsearch:9.5.0
     environment:
       discovery.type: single-node
       xpack.security.enabled: "false"
@@ -59,6 +59,8 @@ services:
       - 9200:9200
       - 9300:9300
     volumes:
+      # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9.
+      # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance
       - esdata7:/usr/share/elasticsearch/data
     healthcheck:
       test:
@@ -74,7 +76,7 @@ services:
   kibana:
     depends_on:
       - elasticsearch
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.5.0
     ports:
       - 5601:5601
 

docker/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   elasticsearch:
-    image: exceptionless/elasticsearch:8.19.15
+    image: exceptionless/elasticsearch:9.5.0
     environment:
       node.name: elasticsearch
       cluster.name: exceptionless
@@ -11,6 +11,8 @@ services:
     ports:
       - 9200:9200
     volumes:
+      # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9.
+      # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance
       - esdata:/usr/share/elasticsearch/data
     healthcheck:
       test:
@@ -26,7 +28,7 @@ services:
   kibana:
     depends_on:
       - elasticsearch
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.5.0
     environment:
       xpack.security.enabled: "false"
     ports:

docs/docs/self-hosting/upgrading-self-hosted-instance.md

@@ -8,6 +8,41 @@ title: "Upgrading"
 
 **If you are upgrading from v1 or [v2](https://github.com/exceptionless/Exceptionless/releases/tag/v2.0.0) you will need to upgrade to [v3.0](https://github.com/exceptionless/Exceptionless/releases/tag/v3.0.0) before upgrading to the latest release.**
 
+## Upgrading from v8 to v9
+
+Exceptionless v9 uses Elasticsearch 9. Do not point an Elasticsearch 9 node at an existing data volume until the cluster has been prepared with Elasticsearch 8.19. Elasticsearch 9 can fail to start when incompatible indices created before Elasticsearch 8 remain.
+
+Use this upgrade path for an existing self-hosted installation:
+
+1. Take a current Elasticsearch snapshot or other verified backup and test that it can be restored. Elasticsearch does not support downgrading a data directory after it has been upgraded.
+2. Upgrade Elasticsearch and Kibana to the latest 8.19.x patch release first, using the existing data volume. Do not start Elasticsearch 9 yet.
+3. Stop the Exceptionless app and job services, but leave Elasticsearch and Kibana 8.19 running. This prevents writes while legacy indices are reindexed.
+4. Open Kibana's **Upgrade Assistant** and resolve every critical issue. Reindex every active Exceptionless index created before Elasticsearch 8. Delete only indices you have confirmed are no longer needed; do not mark active Exceptionless indices as read-only.
+5. If this data volume previously ran Elasticsearch 7, temporarily disable the GeoIP downloader while still on Elasticsearch 8.19. Elasticsearch deletes its downloaded `.geoip_databases` system index when this setting is disabled; Exceptionless data is not affected.
+
+   ```bash
+   curl -fsS -X PUT "http://localhost:9200/_cluster/settings" \
+     -H "Content-Type: application/json" \
+     -d '{"persistent":{"ingest.geoip.downloader.enabled":false}}'
+   ```
+
+6. Confirm that the deprecation API reports no critical issues:
+
+   ```bash
+   curl -fsS "http://localhost:9200/_migration/deprecations?pretty"
+   ```
+
+7. Stop Elasticsearch and Kibana 8.19 without deleting their data volume. Update the Elasticsearch and Kibana images to the v9 versions, start them, and verify cluster health before restarting the Exceptionless app and jobs. In Docker Compose, do not run `docker compose down -v` because `-v` deletes the data volume.
+8. After Elasticsearch 9 is healthy, restore the default GeoIP downloader behavior:
+
+   ```bash
+   curl -fsS -X PUT "http://localhost:9200/_cluster/settings" \
+     -H "Content-Type: application/json" \
+     -d '{"persistent":{"ingest.geoip.downloader.enabled":null}}'
+   ```
+
+See Elastic's [prepare-to-upgrade guide](https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade) for the supported 8.x to 9.x upgrade requirements and Upgrade Assistant details.
+
 ## Upgrading from v7.1 to v8
 
 We simplified the self hosting process by integrating the UI into the existing app images. As such `exceptionless/ui` docker images are deprecated and we recommend using `exceptionless/app`.