Upgraded Triton to v2.70.0

t/triton-inference-server/fil_backend.patch

@@ -1,16 +1,116 @@
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 9aff54c..6ea6a77 100644
+index f14add1..0a7eb88 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -78,6 +78,10 @@ else()
+@@ -78,7 +78,10 @@ else()
    # - Target names -------------------------------------------------------------
    set(BACKEND_NAME "fil")
    set(BACKEND_TARGET "triton_${BACKEND_NAME}")
+-
 +  install(
 +    FILES ${CMAKE_CURRENT_BINARY_DIR}/libtriton_fil.so
 +    DESTINATION ${CMAKE_INSTALL_PREFIX}/backends/fil
 +  )
 
+   ##############################################################################
+   # - Prepare rapids-cmake -----------------------------------------------------
+@@ -274,7 +277,19 @@ else()
+     TARGETS ${BACKEND_TARGET}
+     LIBRARY DESTINATION ${BACKEND_FOLDER}/${BACKEND_NAME}
+   )
++  # nvforest++ is a CPM sub-project target; install(TARGETS) cannot reach it
++  # from the top-level project. Install the built shared library directly.
++  install(
++    FILES $<TARGET_FILE:nvforest::nvforest++>
++    DESTINATION ${BACKEND_FOLDER}/${BACKEND_NAME}
++  )
 
++  # rapids_logger is a header-only / shared lib that nvforest++ links publicly.
++  # It must sit next to libtriton_fil.so so the $ORIGIN RPATH resolves it.
++  install(
++    FILES $<TARGET_FILE:rapids_logger::rapids_logger>
++    DESTINATION ${BACKEND_FOLDER}/${BACKEND_NAME}
++  )
    ##############################################################################
-
+   # - build test executable ----------------------------------------------------
+
+diff --git a/ops/Dockerfile b/ops/Dockerfile
+index e4b4115..18defa2 100644
+--- a/ops/Dockerfile
++++ b/ops/Dockerfile
+@@ -46,14 +46,21 @@ RUN gpuci_mamba_retry create -n triton_test \
+     && rm /environment.yml
+
+ FROM base-test-install as wheel-install-0
+-RUN apt-get update \
+-    && apt-get install --no-install-recommends -y \
+-      build-essential \
+-      ca-certificates \
+-      git \
+-    && apt-get clean \
+-    && rm -rf /var/lib/apt/lists/* \
+-    && conda run --no-capture-output -n triton_test pip install tritonclient[all]
++#RUN apt-get update \
++#    && apt-get install --no-install-recommends -y \
++#      build-essential \
++#      ca-certificates \
++#      git \
++#    && apt-get clean \
++#    && rm -rf /var/lib/apt/lists/* \
++#    && conda run --no-capture-output -n triton_test pip install tritonclient[all]
++RUN yum install -y \
++    gcc \
++    gcc-c++ \
++    make \
++    patch \
++    ca-certificates \
++    git
+
+ FROM ${SDK_IMAGE} as sdk-image
+
+@@ -80,22 +87,29 @@ ENV PATH="/root/miniconda3/bin:${PATH}"
+
+ # In CI, CPU base image may not have curl, but it also does not need to update
+ # the cuda keys
+-RUN  if command -v curl; \
+- then [ $(uname -m) = 'x86_64' ] \
+- && curl -L -o /tmp/cuda-keyring.deb \
+-      https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb \
+- || curl -L -o /tmp/cuda-keyring.deb \
+-      https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/sbsa/cuda-keyring_1.0-1_all.deb; \
+- dpkg -i /tmp/cuda-keyring.deb \
+- && rm /tmp/cuda-keyring.deb; fi
+-
+-RUN apt-get update \
+-    && apt-get install --no-install-recommends -y \
+-      build-essential \
+-      ca-certificates \
+-      git \
+-    && apt-get clean \
+-    && rm -rf /var/lib/apt/lists/*
++#RUN  if command -v curl; \
++# then [ $(uname -m) = 'x86_64' ] \
++# && curl -L -o /tmp/cuda-keyring.deb \
++#      https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb \
++# || curl -L -o /tmp/cuda-keyring.deb \
++#      https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/sbsa/cuda-keyring_1.0-1_all.deb; \
++# dpkg -i /tmp/cuda-keyring.deb \
++# && rm /tmp/cuda-keyring.deb; fi
++
++#RUN apt-get update \
++#    && apt-get install --no-install-recommends -y \
++#      build-essential \
++#      ca-certificates \
++#      git \
++#    && apt-get clean \
++#    && rm -rf /var/lib/apt/lists/*
++RUN yum install -y \
++    gcc \
++    gcc-c++ \
++    make \
++    patch \
++    ca-certificates \
++    git
+
+ # Stage immediately before building; useful for build iteration
+ FROM base as build-prep

t/triton-inference-server/onnxruntime_backend.patch

@@ -23,50 +23,38 @@ index bf374cd..c53c6b9 100644
        COMMAND docker create --name onnxruntime_backend_ort ${TRITON_ONNXRUNTIME_DOCKER_IMAGE}
        COMMAND rm -fr onnxruntime
 diff --git a/tools/gen_ort_dockerfile.py b/tools/gen_ort_dockerfile.py
-index c462558..c3d58a9 100755
+index cdb4253..dde34db 100755
 --- a/tools/gen_ort_dockerfile.py
 +++ b/tools/gen_ort_dockerfile.py
-@@ -134,27 +134,31 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1
+@@ -170,17 +170,27 @@ ENV PIP_BREAK_SYSTEM_PACKAGES=1
+     # if the changes become more substantial.
+     if target_platform() == "rhel":
          df += """
- # The manylinux container defaults to Python 3.7, but some feature installation
- # requires a higher version.
--ARG PYVER=3.12
--ENV PYTHONPATH=/opt/python/v
--RUN ln -sf /opt/python/cp${PYVER/./}* ${PYTHONPATH}
-
--ENV PYBIN=${PYTHONPATH}/bin
--ENV PYTHON_BIN_PATH=${PYBIN}/python${PYVER} \
--    PATH=${PYBIN}:${PATH}
+-
 +RUN yum install -y wget && \
 +   dnf install -y https://mirror.stream.centos.org/9-stream/BaseOS/`arch`/os/Packages/centos-gpg-keys-9.0-35.el9.noarch.rpm \
 +        https://mirror.stream.centos.org/9-stream/BaseOS/`arch`/os/Packages/centos-stream-repos-9.0-35.el9.noarch.rpm \
 +        https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
 +        dnf config-manager --set-enabled crb &&\
 +        dnf install gperf libarchive-devel numactl-devel readline-devel libusb patch -y && \
 +        dnf remove -y centos-gpg-keys-9.0-35.el9.noarch centos-stream-repos-9.0-35.el9.noarch
-
  RUN dnf install -y \\
          ca-certificates \\
 -        curl \\
          git \\
 -        gnupg \\
          openssl-devel \\
--        python3-pip \\
++	python3.12 \\
+         python3.12-devel \\
+         python3.12-pip \\
++	python3.12-numpy \\
          wget \\
 -        zip
 +        zip \\
-+        python3.12 \\
-+        python3.12-pip \\
-+        python3.12-devel \\
-+        python3.12-numpy \\ 
-+        cmake \\
-+        g++  
-
--RUN pipx install cmake==4.0.3 --force
++	cmake \\
++	g++
++
 +RUN ln -sf $(which python3.12) /usr/bin/python3 && ln -sf $(which pip3.12) /usr/bin/pip3 && ln -sf $(which pip3.12) /usr/bin/pip
-+RUN pip3 install patchelf==0.17.2 cmake==4.0.3 packaging wheel
 
--RUN pip3 install patchelf==0.17.2 numpy>=2.0.0
- """
-     else:
-         if os.getenv("CCACHE_REMOTE_ONLY") and os.getenv("CCACHE_REMOTE_STORAGE"):
+ RUN pip3 install \\
+        cmake==4.0.3 \\

t/triton-inference-server/pytorch_backend.patch

@@ -1,20 +1,20 @@
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 13c32d9..b4d1c74 100644
+index 49536a6..3ed3360 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -90,8 +90,9 @@ find_package(Python3 REQUIRED COMPONENTS Development.Module)
+@@ -91,8 +91,9 @@ find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module)
 
  set(RHEL_BUILD OFF)
  set(LIB_DIR "lib")
 -set(LIBTORCH_LIBS_PATH "/usr/local/lib")
 -set(PY_INSTALL_PATH "/usr/local/lib/python3.12/dist-packages")
 +set(LIBTORCH_LIBS_PATH "/usr/local/lib64")
 +set(PY_INSTALL_PATH "/usr/local/lib64/python3.12/site-packages")
-+message(WARNING "USING UBUNTU INSTEAD OF RHEL")
++message(WARNING "USING RHEL INSTEAD OF UBUNTU")
  if(LINUX)
    file(STRINGS "/etc/os-release" DISTRO_ID_LIKE REGEX "ID_LIKE")
    if(${DISTRO_ID_LIKE} MATCHES "rhel|centos")
-@@ -175,26 +176,16 @@ configure_file(src/libtriton_pytorch.ldscript libtriton_pytorch.ldscript COPYONL
+@@ -218,27 +219,16 @@ configure_file(src/libtriton_pytorch.ldscript libtriton_pytorch.ldscript COPYONL
 
  set(PT_LIBS
      "libc10.so"
@@ -38,10 +38,11 @@ index 13c32d9..b4d1c74 100644
 -      "libtorch_nvshmem.so"
 -  )
 -endif() # TRITON_PYTORCH_NVSHMEM
- 
+-
  if (${TRITON_PYTORCH_ENABLE_TORCHVISION})
    set(PT_LIBS
-@@ -226,19 +217,6 @@ if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
+       ${PT_LIBS}
+@@ -271,19 +261,6 @@ if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
        "libnvpl_lapack_lp64_seq.so.0"
    )
  else()
@@ -60,8 +61,8 @@ index 13c32d9..b4d1c74 100644
 -  )
  endif()
  set(TORCHVISION_LIBS
-     $<IF:$<BOOL:${RHEL_BUILD}>,libjpeg.so.62,libjpeg.so>
-@@ -269,41 +247,15 @@ if (${TRITON_PYTORCH_DOCKER_BUILD})
+     libjpeg.so.62
+@@ -314,41 +291,15 @@ if (${TRITON_PYTORCH_DOCKER_BUILD})
      COMMAND docker create --name pytorch_backend_ptlib ${TRITON_PYTORCH_DOCKER_IMAGE}
      COMMAND /bin/sh -c "for i in ${LIBTORCH_LIBS_STR} ; do echo copying $i && docker cp -L pytorch_backend_ptlib:${LIBTORCH_LIBS_PATH}/$i $i ; done"
      COMMAND docker cp pytorch_backend_ptlib:${PY_INSTALL_PATH}/torch/lib/libc10.so libc10.so
@@ -78,17 +79,16 @@ index 13c32d9..b4d1c74 100644
      COMMAND docker cp pytorch_backend_ptlib:${PY_INSTALL_PATH}/torch/lib/libtorch_python.so libtorch_python.so
 -    COMMAND /bin/sh -c "if [ ${TRITON_PYTORCH_NVSHMEM} = 'ON' ]; then docker cp pytorch_backend_ptlib:${PY_INSTALL_PATH}/torch/lib/libtorch_nvshmem.so libtorch_nvshmem.so; fi"
 -    COMMAND /bin/sh -c "if [ ${TRITON_PYTORCH_ENABLE_TORCHVISION} = 'ON' ]; then docker cp -a -L pytorch_backend_ptlib:/usr/local/${LIB_DIR}/libtorchvision.so.1 libtorchvision.so.1; fi;"
-+    COMMAND /bin/sh -c "if [ ${TRITON_PYTORCH_ENABLE_TORCHVISION} = 'ON' ]; then if [ ${RHEL_BUILD} = 'ON' ]; then docker cp -a -L pytorch_backend_ptlib:/usr/local/lib64/libtorchvision.so libtorchvision.so; else docker cp -a -L pytorch_backend_ptlib:/usr/local/lib64/libtorchvision.so.1 libtorchvision.so.1; fi; fi"    
-     COMMAND /bin/sh -c "if [ ${TRITON_PYTORCH_ENABLE_TORCHVISION} = 'ON' ]; then docker cp pytorch_backend_ptlib:/opt/pytorch/vision/torchvision/csrc include/torchvision/torchvision; fi"
+-    COMMAND /bin/sh -c "if [ ${TRITON_PYTORCH_ENABLE_TORCHVISION} = 'ON' ]; then docker cp pytorch_backend_ptlib:/opt/pytorch/vision/torchvision/csrc include/torchvision/torchvision; fi"
 -    COMMAND /bin/sh -c "if [ ${TRITON_PYTORCH_ENABLE_TORCHTRT} = 'ON' ]; then docker cp pytorch_backend_ptlib:${PY_INSTALL_PATH}/torch_tensorrt/lib/libtorchtrt.so libtorchtrt.so; fi"
 -    COMMAND /bin/sh -c "if [ ${TRITON_PYTORCH_ENABLE_TORCHTRT} = 'ON' ]; then docker cp pytorch_backend_ptlib:${PY_INSTALL_PATH}/torch_tensorrt/lib/libtorchtrt_runtime.so libtorchtrt_runtime.so; fi"
 -    COMMAND /bin/sh -c "if [ ${TRITON_PYTORCH_ENABLE_TORCHTRT} = 'ON' ]; then docker cp pytorch_backend_ptlib:${PY_INSTALL_PATH}/torch_tensorrt/bin/torchtrtc torchtrtc; fi"
 -    COMMAND docker cp pytorch_backend_ptlib:/opt/pytorch/pytorch/LICENSE LICENSE.pytorch
 -    COMMAND docker cp pytorch_backend_ptlib:${PY_INSTALL_PATH}/torch/include include/torch
 -    COMMAND docker cp pytorch_backend_ptlib:/opt/pytorch/pytorch/torch/csrc/jit/codegen include/torch/torch/csrc/jit/.
 -
--    COMMAND /bin/sh -c "if [ ${RHEL_BUILD} = 'ON' ]; then docker cp -L pytorch_backend_ptlib:/usr/lib64/libjpeg.so.62 libjpeg.so.62; else docker cp -L pytorch_backend_ptlib:/usr/local/lib/libjpeg.so.62 libjpeg.so.62 && docker cp pytorch_backend_ptlib:/usr/lib/${LIBS_ARCH}-linux-gnu/libjpeg.so.8.2.2 libjpeg.so; fi;"
--    COMMAND /bin/sh -c "if [ ${RHEL_BUILD} = 'ON' ]; then docker cp -L pytorch_backend_ptlib:/usr/lib64/libpng16.so.16 libpng16.so.16; else docker cp -L pytorch_backend_ptlib:/usr/lib/${LIBS_ARCH}-linux-gnu/libpng16.so libpng16.so; fi;"
+-    COMMAND /bin/sh -c "if [ ${RHEL_BUILD} = 'ON' ]; then docker cp -L pytorch_backend_ptlib:/usr/lib64/libjpeg.so.62 libjpeg.so.62; else docker cp -L pytorch_backend_ptlib:/usr/local/lib/libjpeg.so.62 libjpeg.so.62 ; fi;"
+-    COMMAND /bin/sh -c "if [ ${RHEL_BUILD} = 'ON' ]; then docker cp -L pytorch_backend_ptlib:/usr/lib64/libpng16.so.16 libpng16.so.16; else docker cp -L pytorch_backend_ptlib:/usr/lib/${LIBS_ARCH}-linux-gnu/libpng16.so.16 libpng16.so.16; fi;"
 -    COMMAND /bin/sh -c "if [ -f libmkl_def.so.1 ]; then patchelf --add-needed libmkl_gnu_thread.so.1 libmkl_def.so.1; fi"
 -    COMMAND /bin/sh -c "if [ -f libmkl_def.so.1 ]; then patchelf --add-needed libmkl_core.so.1 libmkl_def.so.1; fi"
 -    COMMAND /bin/sh -c "if [ -f libmkl_avx2.so.1 ]; then patchelf --add-needed libmkl_gnu_thread.so.1 libmkl_avx2.so.1; fi"
@@ -101,14 +101,16 @@ index 13c32d9..b4d1c74 100644
 -    COMMAND /bin/sh -c "if [ -f libmkl_intel_thread.so.1 ]; then patchelf --add-needed libmkl_intel_lp64.so.1 libmkl_intel_thread.so.1; fi"
 -    COMMAND /bin/sh -c "if [ ${TRITON_PYTORCH_ENABLE_TORCHVISION} = 'ON' ]; then ln -s libtorchvision.so.1 libtorchvision.so; fi;"
 -    COMMAND docker rm pytorch_backend_ptlib
++    COMMAND /bin/sh -c "if [ ${TRITON_PYTORCH_ENABLE_TORCHVISION} = 'ON' ]; then if [ ${RHEL_BUILD} = 'ON' ]; then docker cp -a -L pytorch_backend_ptlib:/usr/local/lib64/torchvision.so libtorchvision.so; else edocker cp -a -L pytorch_backend_ptlib:/usr/local/lib64/libtorchvision.so.1 libtorchvision.so.1; fi; fi"
++
      COMMENT "Extracting pytorch and torchvision libraries and includes from ${TRITON_PYTORCH_DOCKER_IMAGE}"
      VERBATIM
    )
 diff --git a/src/libtorch_utils.h b/src/libtorch_utils.h
-index 6ec325b..0ac84bc 100644
+index 4ced3dc..481dac1 100644
 --- a/src/libtorch_utils.h
 +++ b/src/libtorch_utils.h
-@@ -34,7 +34,6 @@
+@@ -38,7 +38,6 @@
  #pragma GCC diagnostic ignored "-Wsign-compare"
  #pragma warning(push, 0)
  #include <torch/csrc/jit/codegen/cuda/interface.h>