From 9063aed50ffa8b0e806670bd5258225c328d32a3 Mon Sep 17 00:00:00 2001 From: Ethan Wee Date: Fri, 21 Nov 2025 10:09:06 -0800 Subject: [PATCH 01/18] Update install_rocm.sh --- .ci/docker/common/install_rocm.sh | 54 +++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.ci/docker/common/install_rocm.sh b/.ci/docker/common/install_rocm.sh index 988347e28e9d8..b2eb09069ab90 100644 --- a/.ci/docker/common/install_rocm.sh +++ b/.ci/docker/common/install_rocm.sh @@ -27,6 +27,60 @@ install_ubuntu() { apt-get install -y libc++abi1 # Make sure rocm packages from repo.radeon.com have highest priority +# When USE_THEROCK_NIGHTLY=1, install ROCm from TheRock nightly wheels + +if [[ "${USE_THEROCK_NIGHTLY:-0}" == "1" ]]; then + echo "install_rocm.sh: installing ROCm from TheRock nightly wheels" + + # Clean any previous ROCm installation in the base CI image. + if [[ -d /opt/rocm ]]; then + echo "Removing existing /opt/rocm from base image" + rm -rf /opt/rocm + fi + + # Allow overriding the index URL via env; default to gfx110X Linux nightlies. + : "${THEROCK_NIGHTLY_INDEX_URL:=https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/}" + + # Ensure python/pip are present; most CI images already have them. + if ! command -v python3 >/dev/null 2>&1; then + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-pip + fi + + python3 -m pip install --upgrade pip + + # Install TheRock ROCm SDK from nightlies. The 'rocm-sdk' package and + # its CLI are documented in TheRock issues/README for PyTorch builds. + python3 -m pip install \ + --index-url "${THEROCK_NIGHTLY_INDEX_URL}" \ + "rocm-sdk[libraries,devel]" + + # Use the rocm-sdk CLI helper to populate environment defaults + # that PyTorch's ROCm build expects. + ROCM_HOME="$(rocm-sdk path --root)" + ROCM_BIN="$(rocm-sdk path --bin)" + ROCM_CMAKE_PREFIX="$(rocm-sdk path --cmake)" + + echo "ROCM_HOME=${ROCM_HOME}" + echo "ROCM_BIN=${ROCM_BIN}" + echo "ROCM_CMAKE_PREFIX=${ROCM_CMAKE_PREFIX}" + + # Export these in a profile file so interactive shells and subsequent + # Dockerfile RUN steps see them. + cat >/etc/profile.d/rocm-sdk.sh < /etc/apt/preferences.d/rocm-pin-600 Package: * Pin: release o=repo.radeon.com From e84c36e258fd0083fea43aea179aab8a7c06fe7b Mon Sep 17 00:00:00 2001 From: Ethan Wee Date: Fri, 21 Nov 2025 10:13:00 -0800 Subject: [PATCH 02/18] Create therock-nightly.yml --- .github/workflows/therock-nightly.yml | 98 +++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 .github/workflows/therock-nightly.yml diff --git a/.github/workflows/therock-nightly.yml b/.github/workflows/therock-nightly.yml new file mode 100644 index 0000000000000..7a65c1e226f49 --- /dev/null +++ b/.github/workflows/therock-nightly.yml @@ -0,0 +1,98 @@ +name: therock-nightly + +on: + # Run at midnight UTC every day + schedule: + - cron: "0 0 * * *" + # Also allow manual triggering + workflow_dispatch: + +env: + PYTORCH_NIGHTLY_REF: nightly + + THEROCK_NIGHTLY_INDEX_URL: https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ + + # Whether to install ROCm via TheRock nightly wheels in install_rocm.sh + USE_THEROCK_NIGHTLY: "1" + +jobs: + therock-nightly-build: + # You can change this to a self-hosted GPU runner if desired. + runs-on: ubuntu-latest + + steps: + - name: Checkout pytorch + uses: actions/checkout@v4 + + - name: Show runner info + run: | + uname -a + docker version + + - name: Build TheRock-based ROCm CI image + env: + THEROCK_NIGHTLY_INDEX_URL: ${{ env.THEROCK_NIGHTLY_INDEX_URL }} + USE_THEROCK_NIGHTLY: ${{ env.USE_THEROCK_NIGHTLY }} + run: | + set -euxo pipefail + # Tag is local to this runner; we don't push anywhere in this workflow. + IMAGE_TAG="pytorch-therock-rocm-ci:nightly" + + # Build using the existing ROCm CI Dockerfile, but flip on TheRock mode. + docker build \ + -f .ci/docker/ubuntu-rocm/Dockerfile \ + --build-arg USE_THEROCK_NIGHTLY="${USE_THEROCK_NIGHTLY}" \ + --build-arg THEROCK_NIGHTLY_INDEX_URL="${THEROCK_NIGHTLY_INDEX_URL}" \ + -t "${IMAGE_TAG}" . + + echo "BUILT_IMAGE=${IMAGE_TAG}" >> "$GITHUB_ENV" + + - name: Build PyTorch nightly using TheRock ROCm CI image + env: + PYTORCH_NIGHTLY_REF: ${{ env.PYTORCH_NIGHTLY_REF }} + THEROCK_NIGHTLY_INDEX_URL: ${{ env.THEROCK_NIGHTLY_INDEX_URL }} + run: | + set -euxo pipefail + IMAGE_TAG="${BUILT_IMAGE:-pytorch-therock-rocm-ci:nightly}" + + # Build PyTorch inside the container + docker run --rm \ + -e PYTORCH_NIGHTLY_REF="${PYTORCH_NIGHTLY_REF}" \ + -e THEROCK_NIGHTLY_INDEX_URL="${THEROCK_NIGHTLY_INDEX_URL}" \ + "${IMAGE_TAG}" \ + bash -lc ' + set -euxo pipefail + + # Basic build deps (adjust as needed if your ROCm CI image already has them) + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y git python3 python3-pip python3-venv + + # Clone the nightly ref of PyTorch + git clone --depth 1 --branch "$PYTORCH_NIGHTLY_REF" https://github.com/pytorch/pytorch.git /pytorch + cd /pytorch + + python3 -m venv .venv + . .venv/bin/activate + + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt + + # Configure ROCm + TheRock SDK environment inside the container. + # The rocm-sdk CLI is provided by the TheRock nightly wheels. + export ROCM_HOME="$(rocm-sdk path --root)" + export CMAKE_PREFIX_PATH="$(rocm-sdk path --cmake)" + export PATH="$(rocm-sdk path --bin):$PATH" + + # Standard ROCm build flags for PyTorch; adjust ARCH / FLAGS as needed. + export USE_ROCM=1 + : "${PYTORCH_ROCM_ARCH:=gfx1100}" + export PYTORCH_ROCM_ARCH + : "${MAX_JOBS:=8}" + export MAX_JOBS + + # Build PyTorch wheel + python3 setup.py bdist_wheel + + # Stash built artifacts somewhere predictable + mkdir -p /artifacts + cp dist/*.whl /artifacts/ From 668fd91020584c8d43076438714c3c7c30944083 Mon Sep 17 00:00:00 2001 From: Ethan Wee Date: Fri, 21 Nov 2025 10:19:45 -0800 Subject: [PATCH 03/18] Update therock-nightly.yml --- .github/workflows/therock-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/therock-nightly.yml b/.github/workflows/therock-nightly.yml index 7a65c1e226f49..9a3416f17f665 100644 --- a/.github/workflows/therock-nightly.yml +++ b/.github/workflows/therock-nightly.yml @@ -85,7 +85,7 @@ jobs: # Standard ROCm build flags for PyTorch; adjust ARCH / FLAGS as needed. export USE_ROCM=1 - : "${PYTORCH_ROCM_ARCH:=gfx1100}" + : "${PYTORCH_ROCM_ARCH:=gfx942}" export PYTORCH_ROCM_ARCH : "${MAX_JOBS:=8}" export MAX_JOBS From bd618cd8fc80347e384c6aa938789cae26a26ac4 Mon Sep 17 00:00:00 2001 From: Ethan Wee Date: Fri, 21 Nov 2025 10:21:28 -0800 Subject: [PATCH 04/18] Update install_rocm.sh --- .ci/docker/common/install_rocm.sh | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.ci/docker/common/install_rocm.sh b/.ci/docker/common/install_rocm.sh index b2eb09069ab90..6fffe821ec9a7 100644 --- a/.ci/docker/common/install_rocm.sh +++ b/.ci/docker/common/install_rocm.sh @@ -38,25 +38,12 @@ if [[ "${USE_THEROCK_NIGHTLY:-0}" == "1" ]]; then rm -rf /opt/rocm fi - # Allow overriding the index URL via env; default to gfx110X Linux nightlies. : "${THEROCK_NIGHTLY_INDEX_URL:=https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/}" - - # Ensure python/pip are present; most CI images already have them. - if ! command -v python3 >/dev/null 2>&1; then - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-pip - fi - - python3 -m pip install --upgrade pip - - # Install TheRock ROCm SDK from nightlies. The 'rocm-sdk' package and - # its CLI are documented in TheRock issues/README for PyTorch builds. python3 -m pip install \ --index-url "${THEROCK_NIGHTLY_INDEX_URL}" \ "rocm-sdk[libraries,devel]" # Use the rocm-sdk CLI helper to populate environment defaults - # that PyTorch's ROCm build expects. ROCM_HOME="$(rocm-sdk path --root)" ROCM_BIN="$(rocm-sdk path --bin)" ROCM_CMAKE_PREFIX="$(rocm-sdk path --cmake)" @@ -65,15 +52,12 @@ if [[ "${USE_THEROCK_NIGHTLY:-0}" == "1" ]]; then echo "ROCM_BIN=${ROCM_BIN}" echo "ROCM_CMAKE_PREFIX=${ROCM_CMAKE_PREFIX}" - # Export these in a profile file so interactive shells and subsequent - # Dockerfile RUN steps see them. cat >/etc/profile.d/rocm-sdk.sh < Date: Fri, 21 Nov 2025 18:33:42 +0000 Subject: [PATCH 05/18] Add TheRock nightly CI workflow for PyTorch ROCm builds --- .ci/docker/common/install_rocm.sh | 58 ++++----- .ci/docker/ubuntu-rocm/Dockerfile | 4 + .github/workflows/therock-nightly.yml | 171 ++++++++++++++++++++------ 3 files changed, 164 insertions(+), 69 deletions(-) diff --git a/.ci/docker/common/install_rocm.sh b/.ci/docker/common/install_rocm.sh index 6fffe821ec9a7..02fa6910c8412 100644 --- a/.ci/docker/common/install_rocm.sh +++ b/.ci/docker/common/install_rocm.sh @@ -26,45 +26,45 @@ install_ubuntu() { apt-get install -y libc++1 apt-get install -y libc++abi1 - # Make sure rocm packages from repo.radeon.com have highest priority -# When USE_THEROCK_NIGHTLY=1, install ROCm from TheRock nightly wheels - -if [[ "${USE_THEROCK_NIGHTLY:-0}" == "1" ]]; then - echo "install_rocm.sh: installing ROCm from TheRock nightly wheels" - - # Clean any previous ROCm installation in the base CI image. - if [[ -d /opt/rocm ]]; then - echo "Removing existing /opt/rocm from base image" - rm -rf /opt/rocm - fi + # When USE_THEROCK_NIGHTLY=1, install ROCm from TheRock nightly wheels + if [[ "${USE_THEROCK_NIGHTLY:-0}" == "1" ]]; then + echo "install_rocm.sh: installing ROCm from TheRock nightly wheels" + + # Clean any previous ROCm installation in the base CI image. + if [[ -d /opt/rocm ]]; then + echo "Removing existing /opt/rocm from base image" + rm -rf /opt/rocm + fi - : "${THEROCK_NIGHTLY_INDEX_URL:=https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/}" - python3 -m pip install \ - --index-url "${THEROCK_NIGHTLY_INDEX_URL}" \ - "rocm-sdk[libraries,devel]" + : "${THEROCK_NIGHTLY_INDEX_URL:=https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/}" + python3 -m pip install \ + --index-url "${THEROCK_NIGHTLY_INDEX_URL}" \ + "rocm-sdk[libraries,devel]" - # Use the rocm-sdk CLI helper to populate environment defaults - ROCM_HOME="$(rocm-sdk path --root)" - ROCM_BIN="$(rocm-sdk path --bin)" - ROCM_CMAKE_PREFIX="$(rocm-sdk path --cmake)" + # Use the rocm-sdk CLI helper to populate environment defaults + ROCM_HOME="$(rocm-sdk path --root)" + ROCM_BIN="$(rocm-sdk path --bin)" + ROCM_CMAKE_PREFIX="$(rocm-sdk path --cmake)" - echo "ROCM_HOME=${ROCM_HOME}" - echo "ROCM_BIN=${ROCM_BIN}" - echo "ROCM_CMAKE_PREFIX=${ROCM_CMAKE_PREFIX}" + echo "ROCM_HOME=${ROCM_HOME}" + echo "ROCM_BIN=${ROCM_BIN}" + echo "ROCM_CMAKE_PREFIX=${ROCM_CMAKE_PREFIX}" - cat >/etc/profile.d/rocm-sdk.sh </etc/profile.d/rocm-sdk.sh < /etc/apt/preferences.d/rocm-pin-600 Package: * Pin: release o=repo.radeon.com diff --git a/.ci/docker/ubuntu-rocm/Dockerfile b/.ci/docker/ubuntu-rocm/Dockerfile index b517a990a057b..28dedf962844d 100644 --- a/.ci/docker/ubuntu-rocm/Dockerfile +++ b/.ci/docker/ubuntu-rocm/Dockerfile @@ -52,6 +52,10 @@ ENV INSTALLED_VISION ${VISION} # Install rocm ARG ROCM_VERSION +ARG USE_THEROCK_NIGHTLY +ARG THEROCK_NIGHTLY_INDEX_URL +ENV USE_THEROCK_NIGHTLY=${USE_THEROCK_NIGHTLY} +ENV THEROCK_NIGHTLY_INDEX_URL=${THEROCK_NIGHTLY_INDEX_URL} RUN mkdir ci_commit_pins COPY ./common/common_utils.sh common_utils.sh COPY ./ci_commit_pins/rocm-composable-kernel.txt ci_commit_pins/rocm-composable-kernel.txt diff --git a/.github/workflows/therock-nightly.yml b/.github/workflows/therock-nightly.yml index 9a3416f17f665..f57759be1f909 100644 --- a/.github/workflows/therock-nightly.yml +++ b/.github/workflows/therock-nightly.yml @@ -9,90 +9,181 @@ on: env: PYTORCH_NIGHTLY_REF: nightly - THEROCK_NIGHTLY_INDEX_URL: https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ - - # Whether to install ROCm via TheRock nightly wheels in install_rocm.sh USE_THEROCK_NIGHTLY: "1" + # Docker image configuration + UBUNTU_VERSION: "22.04" + ROCM_VERSION: "6.4" + ANACONDA_PYTHON_VERSION: "3.11" + PYTORCH_ROCM_ARCH: "gfx942" jobs: - therock-nightly-build: - # You can change this to a self-hosted GPU runner if desired. + # Job 1: Build TheRock-based Docker image at midnight + build-therock-docker: runs-on: ubuntu-latest + outputs: + image-tag: ${{ steps.build-image.outputs.image-tag }} steps: - name: Checkout pytorch uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Show runner info run: | uname -a docker version + df -h - name: Build TheRock-based ROCm CI image + id: build-image env: THEROCK_NIGHTLY_INDEX_URL: ${{ env.THEROCK_NIGHTLY_INDEX_URL }} USE_THEROCK_NIGHTLY: ${{ env.USE_THEROCK_NIGHTLY }} + UBUNTU_VERSION: ${{ env.UBUNTU_VERSION }} + ROCM_VERSION: ${{ env.ROCM_VERSION }} + ANACONDA_PYTHON_VERSION: ${{ env.ANACONDA_PYTHON_VERSION }} + PYTORCH_ROCM_ARCH: ${{ env.PYTORCH_ROCM_ARCH }} run: | set -euxo pipefail - # Tag is local to this runner; we don't push anywhere in this workflow. - IMAGE_TAG="pytorch-therock-rocm-ci:nightly" + + # Tag with date for tracking + DATE_TAG=$(date +%Y%m%d) + IMAGE_TAG="pytorch-therock-rocm-ci:nightly-${DATE_TAG}" + + echo "Building TheRock nightly CI image: ${IMAGE_TAG}" - # Build using the existing ROCm CI Dockerfile, but flip on TheRock mode. + # Build using the existing ROCm CI Dockerfile with TheRock nightly mode docker build \ -f .ci/docker/ubuntu-rocm/Dockerfile \ + --build-arg UBUNTU_VERSION="${UBUNTU_VERSION}" \ + --build-arg ROCM_VERSION="${ROCM_VERSION}" \ + --build-arg ANACONDA_PYTHON_VERSION="${ANACONDA_PYTHON_VERSION}" \ + --build-arg PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH}" \ --build-arg USE_THEROCK_NIGHTLY="${USE_THEROCK_NIGHTLY}" \ --build-arg THEROCK_NIGHTLY_INDEX_URL="${THEROCK_NIGHTLY_INDEX_URL}" \ - -t "${IMAGE_TAG}" . + --build-arg BUILD_ENVIRONMENT="therock-nightly-rocm${ROCM_VERSION}" \ + -t "${IMAGE_TAG}" \ + . + + # Also tag as latest + docker tag "${IMAGE_TAG}" pytorch-therock-rocm-ci:latest + + echo "image-tag=${IMAGE_TAG}" >> "$GITHUB_OUTPUT" + + # Save docker image as artifact for next job + docker save "${IMAGE_TAG}" | gzip > therock-docker-image.tar.gz + + - name: Upload Docker image as artifact + uses: actions/upload-artifact@v4 + with: + name: therock-docker-image + path: therock-docker-image.tar.gz + retention-days: 1 + + # Job 2: Build PyTorch nightly using the TheRock image + build-pytorch-nightly: + needs: build-therock-docker + runs-on: ubuntu-latest + + steps: + - name: Checkout pytorch nightly branch + uses: actions/checkout@v4 + with: + repository: pytorch/pytorch + ref: ${{ env.PYTORCH_NIGHTLY_REF }} + submodules: recursive + + - name: Download Docker image artifact + uses: actions/download-artifact@v4 + with: + name: therock-docker-image - echo "BUILT_IMAGE=${IMAGE_TAG}" >> "$GITHUB_ENV" + - name: Load Docker image + run: | + gunzip -c therock-docker-image.tar.gz | docker load + docker images - name: Build PyTorch nightly using TheRock ROCm CI image env: - PYTORCH_NIGHTLY_REF: ${{ env.PYTORCH_NIGHTLY_REF }} - THEROCK_NIGHTLY_INDEX_URL: ${{ env.THEROCK_NIGHTLY_INDEX_URL }} + IMAGE_TAG: ${{ needs.build-therock-docker.outputs.image-tag }} + PYTORCH_ROCM_ARCH: ${{ env.PYTORCH_ROCM_ARCH }} run: | set -euxo pipefail - IMAGE_TAG="${BUILT_IMAGE:-pytorch-therock-rocm-ci:nightly}" + + # Create output directory on host + mkdir -p ${{ github.workspace }}/artifacts # Build PyTorch inside the container docker run --rm \ - -e PYTORCH_NIGHTLY_REF="${PYTORCH_NIGHTLY_REF}" \ - -e THEROCK_NIGHTLY_INDEX_URL="${THEROCK_NIGHTLY_INDEX_URL}" \ + -v ${{ github.workspace }}:/workspace \ + -v ${{ github.workspace }}/artifacts:/artifacts \ + -w /workspace \ + -e PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH}" \ + -e MAX_JOBS=8 \ "${IMAGE_TAG}" \ - bash -lc ' + bash -c ' set -euxo pipefail - # Basic build deps (adjust as needed if your ROCm CI image already has them) - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y git python3 python3-pip python3-venv + # Source the rocm-sdk environment (set up by install_rocm.sh) + if [ -f /etc/profile.d/rocm-sdk.sh ]; then + source /etc/profile.d/rocm-sdk.sh + fi - # Clone the nightly ref of PyTorch - git clone --depth 1 --branch "$PYTORCH_NIGHTLY_REF" https://github.com/pytorch/pytorch.git /pytorch - cd /pytorch + # Verify ROCm is available + echo "ROCM_HOME=${ROCM_HOME}" + rocm-sdk path --root + hipcc --version || true - python3 -m venv .venv - . .venv/bin/activate + # Activate conda environment (already present in CI image) + source /opt/conda/etc/profile.d/conda.sh + conda activate py_${ANACONDA_PYTHON_VERSION:-3.11} - python3 -m pip install --upgrade pip - python3 -m pip install -r requirements.txt + # Install PyTorch dependencies + python -m pip install --upgrade pip + python -m pip install -r requirements.txt - # Configure ROCm + TheRock SDK environment inside the container. - # The rocm-sdk CLI is provided by the TheRock nightly wheels. - export ROCM_HOME="$(rocm-sdk path --root)" - export CMAKE_PREFIX_PATH="$(rocm-sdk path --cmake)" - export PATH="$(rocm-sdk path --bin):$PATH" + # Initialize submodules if not already done + git submodule sync + git submodule update --init --recursive - # Standard ROCm build flags for PyTorch; adjust ARCH / FLAGS as needed. + # Configure ROCm build flags export USE_ROCM=1 - : "${PYTORCH_ROCM_ARCH:=gfx942}" - export PYTORCH_ROCM_ARCH - : "${MAX_JOBS:=8}" - export MAX_JOBS - + export CMAKE_PREFIX_PATH="${ROCM_HOME}:${CMAKE_PREFIX_PATH:-}" + export PATH="${ROCM_HOME}/bin:${PATH}" + # Build PyTorch wheel - python3 setup.py bdist_wheel + python setup.py bdist_wheel - # Stash built artifacts somewhere predictable - mkdir -p /artifacts + # Copy artifacts to host-mounted directory cp dist/*.whl /artifacts/ + + echo "Build complete! Artifacts:" + ls -lh /artifacts/ + ' + + - name: Upload PyTorch wheel artifacts + uses: actions/upload-artifact@v4 + with: + name: pytorch-therock-nightly-wheels + path: artifacts/*.whl + retention-days: 7 + + - name: Show build summary + run: | + echo "## PyTorch TheRock Nightly Build Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Built using TheRock nightly wheels from: ${{ env.THEROCK_NIGHTLY_INDEX_URL }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Build Configuration" >> $GITHUB_STEP_SUMMARY + echo "- ROCm Version: ${{ env.ROCM_VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- Ubuntu Version: ${{ env.UBUNTU_VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- Python Version: ${{ env.ANACONDA_PYTHON_VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- PyTorch ROCM Arch: ${{ env.PYTORCH_ROCM_ARCH }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Artifacts" >> $GITHUB_STEP_SUMMARY + ls -lh artifacts/*.whl >> $GITHUB_STEP_SUMMARY || echo "No wheels found" From 0caf833f9395dc83ff7827ac29d3b246a151afdb Mon Sep 17 00:00:00 2001 From: ethanwee1 Date: Fri, 21 Nov 2025 18:44:53 +0000 Subject: [PATCH 06/18] updates --- .github/workflows/therock-nightly.yml | 159 +++++++++++++++++--------- PR_TITLE_AND_DESCRIPTION.txt | 117 +++++++++++++++++++ THEROCK_NIGHTLY_PR_SUMMARY.md | 148 ++++++++++++++++++++++++ 3 files changed, 372 insertions(+), 52 deletions(-) create mode 100644 PR_TITLE_AND_DESCRIPTION.txt create mode 100644 THEROCK_NIGHTLY_PR_SUMMARY.md diff --git a/.github/workflows/therock-nightly.yml b/.github/workflows/therock-nightly.yml index f57759be1f909..2033ba8489000 100644 --- a/.github/workflows/therock-nightly.yml +++ b/.github/workflows/therock-nightly.yml @@ -10,28 +10,23 @@ on: env: PYTORCH_NIGHTLY_REF: nightly THEROCK_NIGHTLY_INDEX_URL: https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ - USE_THEROCK_NIGHTLY: "1" - # Docker image configuration - UBUNTU_VERSION: "22.04" - ROCM_VERSION: "6.4" - ANACONDA_PYTHON_VERSION: "3.11" + # Base CI image to modify (same as rocm-mi300.yml uses) + BASE_CI_IMAGE: pytorch/pytorch-linux-noble-rocm-n-py3:latest + # New image name after installing TheRock + THEROCK_IMAGE: pytorch-linux-noble-rocm-alpha-py3 PYTORCH_ROCM_ARCH: "gfx942" + PYTHON_VERSION: "3.12" jobs: - # Job 1: Build TheRock-based Docker image at midnight - build-therock-docker: + # Job 1: Modify existing CI image to use TheRock nightly + create-therock-image: runs-on: ubuntu-latest outputs: - image-tag: ${{ steps.build-image.outputs.image-tag }} + image-tag: ${{ steps.create-image.outputs.image-tag }} steps: - name: Checkout pytorch uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - name: Show runner info run: | @@ -39,43 +34,88 @@ jobs: docker version df -h - - name: Build TheRock-based ROCm CI image - id: build-image + - name: Pull base CI image + run: | + echo "Pulling base CI image: ${{ env.BASE_CI_IMAGE }}" + docker pull ${{ env.BASE_CI_IMAGE }} || echo "Warning: Could not pull image, may need authentication" + + - name: Create TheRock-based image + id: create-image env: THEROCK_NIGHTLY_INDEX_URL: ${{ env.THEROCK_NIGHTLY_INDEX_URL }} - USE_THEROCK_NIGHTLY: ${{ env.USE_THEROCK_NIGHTLY }} - UBUNTU_VERSION: ${{ env.UBUNTU_VERSION }} - ROCM_VERSION: ${{ env.ROCM_VERSION }} - ANACONDA_PYTHON_VERSION: ${{ env.ANACONDA_PYTHON_VERSION }} - PYTORCH_ROCM_ARCH: ${{ env.PYTORCH_ROCM_ARCH }} + BASE_CI_IMAGE: ${{ env.BASE_CI_IMAGE }} + THEROCK_IMAGE: ${{ env.THEROCK_IMAGE }} run: | set -euxo pipefail # Tag with date for tracking DATE_TAG=$(date +%Y%m%d) - IMAGE_TAG="pytorch-therock-rocm-ci:nightly-${DATE_TAG}" - - echo "Building TheRock nightly CI image: ${IMAGE_TAG}" - - # Build using the existing ROCm CI Dockerfile with TheRock nightly mode - docker build \ - -f .ci/docker/ubuntu-rocm/Dockerfile \ - --build-arg UBUNTU_VERSION="${UBUNTU_VERSION}" \ - --build-arg ROCM_VERSION="${ROCM_VERSION}" \ - --build-arg ANACONDA_PYTHON_VERSION="${ANACONDA_PYTHON_VERSION}" \ - --build-arg PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH}" \ - --build-arg USE_THEROCK_NIGHTLY="${USE_THEROCK_NIGHTLY}" \ - --build-arg THEROCK_NIGHTLY_INDEX_URL="${THEROCK_NIGHTLY_INDEX_URL}" \ - --build-arg BUILD_ENVIRONMENT="therock-nightly-rocm${ROCM_VERSION}" \ - -t "${IMAGE_TAG}" \ - . - + IMAGE_TAG="${THEROCK_IMAGE}:nightly-${DATE_TAG}" + + echo "Creating TheRock nightly image from base CI image" + echo "Base: ${BASE_CI_IMAGE}" + echo "Target: ${IMAGE_TAG}" + + # Run the base image and modify it + docker run --name therock-setup "${BASE_CI_IMAGE}" bash -c ' + set -euxo pipefail + + echo "=== Removing existing /opt/rocm ===" + if [ -d /opt/rocm ]; then + rm -rf /opt/rocm + echo "/opt/rocm removed successfully" + else + echo "/opt/rocm does not exist" + fi + + echo "=== Installing TheRock nightly wheels ===" + python3 -m pip install --upgrade pip + python3 -m pip install \ + --index-url "'"${THEROCK_NIGHTLY_INDEX_URL}"'" \ + "rocm-sdk[libraries,devel]" + + echo "=== Setting up ROCm environment ===" + ROCM_HOME="$(rocm-sdk path --root)" + ROCM_BIN="$(rocm-sdk path --bin)" + ROCM_CMAKE_PREFIX="$(rocm-sdk path --cmake)" + + echo "ROCM_HOME=${ROCM_HOME}" + echo "ROCM_BIN=${ROCM_BIN}" + echo "ROCM_CMAKE_PREFIX=${ROCM_CMAKE_PREFIX}" + + # Create environment setup script + cat > /etc/profile.d/rocm-sdk.sh <> "$GITHUB_OUTPUT" + # Show image info + docker images | grep "${THEROCK_IMAGE}" || true + # Save docker image as artifact for next job + echo "Saving image as artifact..." docker save "${IMAGE_TAG}" | gzip > therock-docker-image.tar.gz - name: Upload Docker image as artifact @@ -87,7 +127,7 @@ jobs: # Job 2: Build PyTorch nightly using the TheRock image build-pytorch-nightly: - needs: build-therock-docker + needs: create-therock-image runs-on: ubuntu-latest steps: @@ -105,13 +145,15 @@ jobs: - name: Load Docker image run: | + echo "Loading TheRock Docker image..." gunzip -c therock-docker-image.tar.gz | docker load docker images - - name: Build PyTorch nightly using TheRock ROCm CI image + - name: Build PyTorch nightly using TheRock image env: - IMAGE_TAG: ${{ needs.build-therock-docker.outputs.image-tag }} + IMAGE_TAG: ${{ needs.create-therock-image.outputs.image-tag }} PYTORCH_ROCM_ARCH: ${{ env.PYTORCH_ROCM_ARCH }} + PYTHON_VERSION: ${{ env.PYTHON_VERSION }} run: | set -euxo pipefail @@ -129,25 +171,32 @@ jobs: bash -c ' set -euxo pipefail - # Source the rocm-sdk environment (set up by install_rocm.sh) + # Source the rocm-sdk environment if [ -f /etc/profile.d/rocm-sdk.sh ]; then source /etc/profile.d/rocm-sdk.sh + echo "Sourced /etc/profile.d/rocm-sdk.sh" fi # Verify ROCm is available + echo "=== ROCm Environment ===" echo "ROCM_HOME=${ROCM_HOME}" rocm-sdk path --root - hipcc --version || true + rocm-sdk --version || true + which hipcc && hipcc --version || echo "hipcc not found" - # Activate conda environment (already present in CI image) + # Activate conda environment for Python '"${PYTHON_VERSION}"' source /opt/conda/etc/profile.d/conda.sh - conda activate py_${ANACONDA_PYTHON_VERSION:-3.11} + conda activate py_'"${PYTHON_VERSION}"' || conda activate py_3.12 + + echo "=== Python Environment ===" + which python + python --version # Install PyTorch dependencies python -m pip install --upgrade pip python -m pip install -r requirements.txt - # Initialize submodules if not already done + # Initialize submodules if needed git submodule sync git submodule update --init --recursive @@ -156,20 +205,26 @@ jobs: export CMAKE_PREFIX_PATH="${ROCM_HOME}:${CMAKE_PREFIX_PATH:-}" export PATH="${ROCM_HOME}/bin:${PATH}" + echo "=== Build Configuration ===" + echo "USE_ROCM=${USE_ROCM}" + echo "PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH}" + echo "MAX_JOBS=${MAX_JOBS}" + # Build PyTorch wheel + echo "=== Starting PyTorch Build ===" python setup.py bdist_wheel # Copy artifacts to host-mounted directory cp dist/*.whl /artifacts/ - echo "Build complete! Artifacts:" + echo "=== Build Complete ===" ls -lh /artifacts/ ' - name: Upload PyTorch wheel artifacts uses: actions/upload-artifact@v4 with: - name: pytorch-therock-nightly-wheels + name: pytorch-therock-nightly-wheels-py${{ env.PYTHON_VERSION }} path: artifacts/*.whl retention-days: 7 @@ -180,9 +235,9 @@ jobs: echo "Built using TheRock nightly wheels from: ${{ env.THEROCK_NIGHTLY_INDEX_URL }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "### Build Configuration" >> $GITHUB_STEP_SUMMARY - echo "- ROCm Version: ${{ env.ROCM_VERSION }}" >> $GITHUB_STEP_SUMMARY - echo "- Ubuntu Version: ${{ env.UBUNTU_VERSION }}" >> $GITHUB_STEP_SUMMARY - echo "- Python Version: ${{ env.ANACONDA_PYTHON_VERSION }}" >> $GITHUB_STEP_SUMMARY + echo "- Base Image: ${{ env.BASE_CI_IMAGE }}" >> $GITHUB_STEP_SUMMARY + echo "- TheRock Image: ${{ env.THEROCK_IMAGE }}" >> $GITHUB_STEP_SUMMARY + echo "- Python Version: ${{ env.PYTHON_VERSION }}" >> $GITHUB_STEP_SUMMARY echo "- PyTorch ROCM Arch: ${{ env.PYTORCH_ROCM_ARCH }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "### Artifacts" >> $GITHUB_STEP_SUMMARY diff --git a/PR_TITLE_AND_DESCRIPTION.txt b/PR_TITLE_AND_DESCRIPTION.txt new file mode 100644 index 0000000000000..5270e01a1a535 --- /dev/null +++ b/PR_TITLE_AND_DESCRIPTION.txt @@ -0,0 +1,117 @@ +============================================================================== +PR TITLE: +============================================================================== + +Add TheRock nightly CI workflow for PyTorch ROCm builds + +============================================================================== +PR DESCRIPTION: +============================================================================== + +## Overview + +This PR introduces a new nightly CI workflow to build PyTorch using [TheRock](https://github.com/ROCm/TheRock) nightly wheels for ROCm. The workflow modifies an existing CI image to replace traditional ROCm with TheRock's pip-installable ROCm packages. + +## Motivation + +- Enable automated testing of PyTorch with the latest TheRock nightly ROCm wheels +- Catch compatibility issues early between PyTorch nightly and TheRock nightly +- Leverage existing CI infrastructure without rebuilding Docker images from scratch + +## Approach + +Instead of building a new Docker image from scratch, this workflow: + +1. **Pulls existing CI image**: Uses `pytorch/pytorch-linux-noble-rocm-n-py3` (same as `rocm-mi300.yml`) +2. **Removes `/opt/rocm`**: Clears the traditional ROCm installation +3. **Installs TheRock nightly**: Installs `rocm-sdk[libraries,devel]` via pip from TheRock nightly index +4. **Commits new image**: Creates `pytorch-linux-noble-rocm-alpha-py3` with TheRock ROCm +5. **Builds PyTorch**: Compiles PyTorch nightly using the TheRock-enabled image + +This approach is much faster and more maintainable than rebuilding from Dockerfile. + +## Changes + +### 1. New Workflow (`.github/workflows/therock-nightly.yml`) + +- **Trigger**: Runs daily at midnight UTC via cron, with manual dispatch option +- **Job 1 - Create TheRock Image**: Modifies existing CI image to use TheRock nightly +- **Job 2 - Build PyTorch**: Builds PyTorch `nightly` branch inside the TheRock image +- **Configuration**: Python 3.12, gfx942 architecture +- **Artifacts**: Uploads built PyTorch wheels (7-day retention) + +**Key Features:** +```yaml +env: + BASE_CI_IMAGE: pytorch/pytorch-linux-noble-rocm-n-py3:latest + THEROCK_IMAGE: pytorch-linux-noble-rocm-alpha-py3 + PYTHON_VERSION: "3.12" + PYTORCH_ROCM_ARCH: "gfx942" +``` + +### 2. Dockerfile Enhancement (`.ci/docker/ubuntu-rocm/Dockerfile`) + +Added optional build arguments to enable TheRock mode (for other use cases): +- `USE_THEROCK_NIGHTLY`: Flag to enable TheRock installation during Docker build +- `THEROCK_NIGHTLY_INDEX_URL`: URL for TheRock nightly wheel index + +**Note**: The nightly workflow doesn't use these args - it modifies an existing image instead. + +### 3. ROCm Installation Script (`.ci/docker/common/install_rocm.sh`) + +Added conditional logic for TheRock installation (Ubuntu only): +- When `USE_THEROCK_NIGHTLY=1`, installs `rocm-sdk[libraries,devel]` from TheRock index +- Removes existing `/opt/rocm` and uses pip-installed ROCm +- Sets up environment variables using `rocm-sdk` CLI helper +- Fully backwards compatible - existing builds unaffected + +## Testing + +**Manual workflow trigger:** +``` +Actions → therock-nightly → Run workflow +``` + +**Local test to create TheRock image:** +```bash +# Pull base image +docker pull pytorch/pytorch-linux-noble-rocm-n-py3:latest + +# Modify it with TheRock +docker run --name therock-test pytorch/pytorch-linux-noble-rocm-n-py3:latest bash -c ' + rm -rf /opt/rocm + pip install --index-url https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ "rocm-sdk[libraries,devel]" + rocm-sdk path --root +' + +# Commit to new image +docker commit therock-test pytorch-linux-noble-rocm-alpha-py3:test +docker rm therock-test + +# Test the new image +docker run --rm pytorch-linux-noble-rocm-alpha-py3:test bash -c 'rocm-sdk --version' +``` + +## Impact + +- **No breaking changes**: Existing CI workflows are unaffected +- **Fast execution**: Reuses existing CI image (no 30+ min Docker rebuild) +- **Isolated**: Runs independently on nightly schedule +- **Efficient**: Creates new image by modification, not full rebuild + +## Benefits over building from scratch + +1. ✅ **Faster**: Modifying an image takes ~5 minutes vs 30+ minutes for full rebuild +2. ✅ **Simpler**: No need to maintain separate Dockerfile configurations +3. ✅ **Consistent**: Uses exact same base as production CI (rocm-mi300.yml) +4. ✅ **Flexible**: Easy to switch between base images as PyTorch CI evolves + +## Related + +- TheRock repository: https://github.com/ROCm/TheRock +- TheRock nightly index: https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ +- Base image workflow: `.github/workflows/rocm-mi300.yml` + +--- + +cc: @pytorch/rocm (please review) diff --git a/THEROCK_NIGHTLY_PR_SUMMARY.md b/THEROCK_NIGHTLY_PR_SUMMARY.md new file mode 100644 index 0000000000000..e33e969791192 --- /dev/null +++ b/THEROCK_NIGHTLY_PR_SUMMARY.md @@ -0,0 +1,148 @@ +# PyTorch TheRock Nightly CI Workflow + +## Summary + +This PR adds a new GitHub Actions workflow to build PyTorch nightly using TheRock nightly wheels for ROCm. The workflow runs at midnight UTC daily and consists of two jobs: + +1. **Build TheRock-based Docker image** - Creates a CI Docker image with TheRock nightly ROCm wheels +2. **Build PyTorch nightly** - Builds PyTorch from the `nightly` branch using the TheRock Docker image + +## Changes Made + +### 1. New Workflow: `.github/workflows/therock-nightly.yml` + +**Features:** +- Scheduled to run at midnight UTC daily via cron trigger +- Manual workflow dispatch capability +- Two-job pipeline: + - `build-therock-docker`: Builds Docker image with TheRock nightly ROCm + - `build-pytorch-nightly`: Builds PyTorch using the TheRock image +- Artifact management: Docker image passed between jobs, PyTorch wheels uploaded +- Configurable via environment variables (ROCm version, Python version, GPU arch) + +**Key Configuration:** +```yaml +env: + PYTORCH_NIGHTLY_REF: nightly + THEROCK_NIGHTLY_INDEX_URL: https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ + USE_THEROCK_NIGHTLY: "1" + UBUNTU_VERSION: "22.04" + ROCM_VERSION: "6.4" + ANACONDA_PYTHON_VERSION: "3.11" + PYTORCH_ROCM_ARCH: "gfx942" +``` + +### 2. Dockerfile Updates: `.ci/docker/ubuntu-rocm/Dockerfile` + +**Added build arguments:** +- `USE_THEROCK_NIGHTLY` - Flag to enable TheRock nightly installation +- `THEROCK_NIGHTLY_INDEX_URL` - URL for TheRock nightly wheel index + +**Changes:** +```dockerfile +ARG USE_THEROCK_NIGHTLY +ARG THEROCK_NIGHTLY_INDEX_URL +ENV USE_THEROCK_NIGHTLY=${USE_THEROCK_NIGHTLY} +ENV THEROCK_NIGHTLY_INDEX_URL=${THEROCK_NIGHTLY_INDEX_URL} +``` + +These environment variables are passed to `install_rocm.sh` during the Docker build. + +### 3. ROCm Installation Script: `.ci/docker/common/install_rocm.sh` + +**Added TheRock nightly support:** +- Checks for `USE_THEROCK_NIGHTLY=1` environment variable +- When enabled: + - Removes existing `/opt/rocm` directory + - Installs `rocm-sdk[libraries,devel]` from TheRock nightly index + - Sets up ROCm environment using `rocm-sdk` CLI helper + - Creates `/etc/profile.d/rocm-sdk.sh` for persistent environment configuration + - Early exits to skip traditional ROCm installation + +**Logic flow:** +```bash +if [[ "${USE_THEROCK_NIGHTLY:-0}" == "1" ]]; then + # Install TheRock nightly wheels + python3 -m pip install --index-url "${THEROCK_NIGHTLY_INDEX_URL}" "rocm-sdk[libraries,devel]" + # Configure environment + export ROCM_HOME="$(rocm-sdk path --root)" + # Exit early + exit 0 +fi +# Traditional ROCm installation continues... +``` + +## Workflow Details + +### Job 1: build-therock-docker + +1. Checks out PyTorch repository +2. Sets up Docker Buildx +3. Builds Docker image using `.ci/docker/ubuntu-rocm/Dockerfile` with TheRock build args +4. Tags image with date (e.g., `nightly-20251121`) and `latest` +5. Saves and uploads Docker image as artifact + +### Job 2: build-pytorch-nightly + +1. Checks out PyTorch nightly branch with submodules +2. Downloads Docker image artifact from previous job +3. Runs PyTorch build inside the TheRock container: + - Sources ROCm SDK environment from `/etc/profile.d/rocm-sdk.sh` + - Activates conda environment + - Installs PyTorch dependencies + - Builds PyTorch wheel with `USE_ROCM=1` +4. Uploads built wheel artifacts (retained for 7 days) +5. Generates build summary in GitHub Actions UI + +## Benefits + +1. **Automated nightly testing** of PyTorch with TheRock nightly ROCm wheels +2. **Early detection** of compatibility issues between PyTorch and TheRock +3. **Reproducible builds** via Docker containerization +4. **Artifact preservation** for debugging and testing +5. **No impact** on existing CI workflows (opt-in via build args) + +## Testing + +To test this workflow: + +1. **Manual trigger**: Go to Actions → therock-nightly → Run workflow +2. **Local Docker build**: + ```bash + docker build \ + -f .ci/docker/ubuntu-rocm/Dockerfile \ + --build-arg UBUNTU_VERSION=22.04 \ + --build-arg ROCM_VERSION=6.4 \ + --build-arg ANACONDA_PYTHON_VERSION=3.11 \ + --build-arg PYTORCH_ROCM_ARCH=gfx942 \ + --build-arg USE_THEROCK_NIGHTLY=1 \ + --build-arg THEROCK_NIGHTLY_INDEX_URL=https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ \ + --build-arg BUILD_ENVIRONMENT=therock-nightly-rocm6.4 \ + -t pytorch-therock-rocm-ci:test \ + . + ``` + +## Future Enhancements + +- Add test execution step after build +- Push Docker images to a registry for reuse +- Support multiple GPU architectures (gfx908, gfx90a, etc.) +- Add Slack/email notifications on build failures +- Create separate workflow for TheRock stable releases + +## Files Changed + +- `.github/workflows/therock-nightly.yml` (new) +- `.ci/docker/ubuntu-rocm/Dockerfile` (modified) +- `.ci/docker/common/install_rocm.sh` (modified) + +--- + +## PR Checklist + +- [ ] Workflow tested with manual trigger +- [ ] Docker image builds successfully with TheRock nightly wheels +- [ ] PyTorch builds successfully inside TheRock Docker image +- [ ] Artifacts are properly uploaded +- [ ] No breaking changes to existing CI workflows +- [ ] Documentation updated (if needed) From cd63b977b975458a97755f0106baca9a956c9673 Mon Sep 17 00:00:00 2001 From: Ethan Wee Date: Fri, 21 Nov 2025 10:45:38 -0800 Subject: [PATCH 07/18] Delete PR_TITLE_AND_DESCRIPTION.txt --- PR_TITLE_AND_DESCRIPTION.txt | 117 ----------------------------------- 1 file changed, 117 deletions(-) delete mode 100644 PR_TITLE_AND_DESCRIPTION.txt diff --git a/PR_TITLE_AND_DESCRIPTION.txt b/PR_TITLE_AND_DESCRIPTION.txt deleted file mode 100644 index 5270e01a1a535..0000000000000 --- a/PR_TITLE_AND_DESCRIPTION.txt +++ /dev/null @@ -1,117 +0,0 @@ -============================================================================== -PR TITLE: -============================================================================== - -Add TheRock nightly CI workflow for PyTorch ROCm builds - -============================================================================== -PR DESCRIPTION: -============================================================================== - -## Overview - -This PR introduces a new nightly CI workflow to build PyTorch using [TheRock](https://github.com/ROCm/TheRock) nightly wheels for ROCm. The workflow modifies an existing CI image to replace traditional ROCm with TheRock's pip-installable ROCm packages. - -## Motivation - -- Enable automated testing of PyTorch with the latest TheRock nightly ROCm wheels -- Catch compatibility issues early between PyTorch nightly and TheRock nightly -- Leverage existing CI infrastructure without rebuilding Docker images from scratch - -## Approach - -Instead of building a new Docker image from scratch, this workflow: - -1. **Pulls existing CI image**: Uses `pytorch/pytorch-linux-noble-rocm-n-py3` (same as `rocm-mi300.yml`) -2. **Removes `/opt/rocm`**: Clears the traditional ROCm installation -3. **Installs TheRock nightly**: Installs `rocm-sdk[libraries,devel]` via pip from TheRock nightly index -4. **Commits new image**: Creates `pytorch-linux-noble-rocm-alpha-py3` with TheRock ROCm -5. **Builds PyTorch**: Compiles PyTorch nightly using the TheRock-enabled image - -This approach is much faster and more maintainable than rebuilding from Dockerfile. - -## Changes - -### 1. New Workflow (`.github/workflows/therock-nightly.yml`) - -- **Trigger**: Runs daily at midnight UTC via cron, with manual dispatch option -- **Job 1 - Create TheRock Image**: Modifies existing CI image to use TheRock nightly -- **Job 2 - Build PyTorch**: Builds PyTorch `nightly` branch inside the TheRock image -- **Configuration**: Python 3.12, gfx942 architecture -- **Artifacts**: Uploads built PyTorch wheels (7-day retention) - -**Key Features:** -```yaml -env: - BASE_CI_IMAGE: pytorch/pytorch-linux-noble-rocm-n-py3:latest - THEROCK_IMAGE: pytorch-linux-noble-rocm-alpha-py3 - PYTHON_VERSION: "3.12" - PYTORCH_ROCM_ARCH: "gfx942" -``` - -### 2. Dockerfile Enhancement (`.ci/docker/ubuntu-rocm/Dockerfile`) - -Added optional build arguments to enable TheRock mode (for other use cases): -- `USE_THEROCK_NIGHTLY`: Flag to enable TheRock installation during Docker build -- `THEROCK_NIGHTLY_INDEX_URL`: URL for TheRock nightly wheel index - -**Note**: The nightly workflow doesn't use these args - it modifies an existing image instead. - -### 3. ROCm Installation Script (`.ci/docker/common/install_rocm.sh`) - -Added conditional logic for TheRock installation (Ubuntu only): -- When `USE_THEROCK_NIGHTLY=1`, installs `rocm-sdk[libraries,devel]` from TheRock index -- Removes existing `/opt/rocm` and uses pip-installed ROCm -- Sets up environment variables using `rocm-sdk` CLI helper -- Fully backwards compatible - existing builds unaffected - -## Testing - -**Manual workflow trigger:** -``` -Actions → therock-nightly → Run workflow -``` - -**Local test to create TheRock image:** -```bash -# Pull base image -docker pull pytorch/pytorch-linux-noble-rocm-n-py3:latest - -# Modify it with TheRock -docker run --name therock-test pytorch/pytorch-linux-noble-rocm-n-py3:latest bash -c ' - rm -rf /opt/rocm - pip install --index-url https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ "rocm-sdk[libraries,devel]" - rocm-sdk path --root -' - -# Commit to new image -docker commit therock-test pytorch-linux-noble-rocm-alpha-py3:test -docker rm therock-test - -# Test the new image -docker run --rm pytorch-linux-noble-rocm-alpha-py3:test bash -c 'rocm-sdk --version' -``` - -## Impact - -- **No breaking changes**: Existing CI workflows are unaffected -- **Fast execution**: Reuses existing CI image (no 30+ min Docker rebuild) -- **Isolated**: Runs independently on nightly schedule -- **Efficient**: Creates new image by modification, not full rebuild - -## Benefits over building from scratch - -1. ✅ **Faster**: Modifying an image takes ~5 minutes vs 30+ minutes for full rebuild -2. ✅ **Simpler**: No need to maintain separate Dockerfile configurations -3. ✅ **Consistent**: Uses exact same base as production CI (rocm-mi300.yml) -4. ✅ **Flexible**: Easy to switch between base images as PyTorch CI evolves - -## Related - -- TheRock repository: https://github.com/ROCm/TheRock -- TheRock nightly index: https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ -- Base image workflow: `.github/workflows/rocm-mi300.yml` - ---- - -cc: @pytorch/rocm (please review) From e55d5a3a372c044558323323f83826c98596a7d3 Mon Sep 17 00:00:00 2001 From: Ethan Wee Date: Fri, 21 Nov 2025 10:46:21 -0800 Subject: [PATCH 08/18] Delete THEROCK_NIGHTLY_PR_SUMMARY.md --- THEROCK_NIGHTLY_PR_SUMMARY.md | 148 ---------------------------------- 1 file changed, 148 deletions(-) delete mode 100644 THEROCK_NIGHTLY_PR_SUMMARY.md diff --git a/THEROCK_NIGHTLY_PR_SUMMARY.md b/THEROCK_NIGHTLY_PR_SUMMARY.md deleted file mode 100644 index e33e969791192..0000000000000 --- a/THEROCK_NIGHTLY_PR_SUMMARY.md +++ /dev/null @@ -1,148 +0,0 @@ -# PyTorch TheRock Nightly CI Workflow - -## Summary - -This PR adds a new GitHub Actions workflow to build PyTorch nightly using TheRock nightly wheels for ROCm. The workflow runs at midnight UTC daily and consists of two jobs: - -1. **Build TheRock-based Docker image** - Creates a CI Docker image with TheRock nightly ROCm wheels -2. **Build PyTorch nightly** - Builds PyTorch from the `nightly` branch using the TheRock Docker image - -## Changes Made - -### 1. New Workflow: `.github/workflows/therock-nightly.yml` - -**Features:** -- Scheduled to run at midnight UTC daily via cron trigger -- Manual workflow dispatch capability -- Two-job pipeline: - - `build-therock-docker`: Builds Docker image with TheRock nightly ROCm - - `build-pytorch-nightly`: Builds PyTorch using the TheRock image -- Artifact management: Docker image passed between jobs, PyTorch wheels uploaded -- Configurable via environment variables (ROCm version, Python version, GPU arch) - -**Key Configuration:** -```yaml -env: - PYTORCH_NIGHTLY_REF: nightly - THEROCK_NIGHTLY_INDEX_URL: https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ - USE_THEROCK_NIGHTLY: "1" - UBUNTU_VERSION: "22.04" - ROCM_VERSION: "6.4" - ANACONDA_PYTHON_VERSION: "3.11" - PYTORCH_ROCM_ARCH: "gfx942" -``` - -### 2. Dockerfile Updates: `.ci/docker/ubuntu-rocm/Dockerfile` - -**Added build arguments:** -- `USE_THEROCK_NIGHTLY` - Flag to enable TheRock nightly installation -- `THEROCK_NIGHTLY_INDEX_URL` - URL for TheRock nightly wheel index - -**Changes:** -```dockerfile -ARG USE_THEROCK_NIGHTLY -ARG THEROCK_NIGHTLY_INDEX_URL -ENV USE_THEROCK_NIGHTLY=${USE_THEROCK_NIGHTLY} -ENV THEROCK_NIGHTLY_INDEX_URL=${THEROCK_NIGHTLY_INDEX_URL} -``` - -These environment variables are passed to `install_rocm.sh` during the Docker build. - -### 3. ROCm Installation Script: `.ci/docker/common/install_rocm.sh` - -**Added TheRock nightly support:** -- Checks for `USE_THEROCK_NIGHTLY=1` environment variable -- When enabled: - - Removes existing `/opt/rocm` directory - - Installs `rocm-sdk[libraries,devel]` from TheRock nightly index - - Sets up ROCm environment using `rocm-sdk` CLI helper - - Creates `/etc/profile.d/rocm-sdk.sh` for persistent environment configuration - - Early exits to skip traditional ROCm installation - -**Logic flow:** -```bash -if [[ "${USE_THEROCK_NIGHTLY:-0}" == "1" ]]; then - # Install TheRock nightly wheels - python3 -m pip install --index-url "${THEROCK_NIGHTLY_INDEX_URL}" "rocm-sdk[libraries,devel]" - # Configure environment - export ROCM_HOME="$(rocm-sdk path --root)" - # Exit early - exit 0 -fi -# Traditional ROCm installation continues... -``` - -## Workflow Details - -### Job 1: build-therock-docker - -1. Checks out PyTorch repository -2. Sets up Docker Buildx -3. Builds Docker image using `.ci/docker/ubuntu-rocm/Dockerfile` with TheRock build args -4. Tags image with date (e.g., `nightly-20251121`) and `latest` -5. Saves and uploads Docker image as artifact - -### Job 2: build-pytorch-nightly - -1. Checks out PyTorch nightly branch with submodules -2. Downloads Docker image artifact from previous job -3. Runs PyTorch build inside the TheRock container: - - Sources ROCm SDK environment from `/etc/profile.d/rocm-sdk.sh` - - Activates conda environment - - Installs PyTorch dependencies - - Builds PyTorch wheel with `USE_ROCM=1` -4. Uploads built wheel artifacts (retained for 7 days) -5. Generates build summary in GitHub Actions UI - -## Benefits - -1. **Automated nightly testing** of PyTorch with TheRock nightly ROCm wheels -2. **Early detection** of compatibility issues between PyTorch and TheRock -3. **Reproducible builds** via Docker containerization -4. **Artifact preservation** for debugging and testing -5. **No impact** on existing CI workflows (opt-in via build args) - -## Testing - -To test this workflow: - -1. **Manual trigger**: Go to Actions → therock-nightly → Run workflow -2. **Local Docker build**: - ```bash - docker build \ - -f .ci/docker/ubuntu-rocm/Dockerfile \ - --build-arg UBUNTU_VERSION=22.04 \ - --build-arg ROCM_VERSION=6.4 \ - --build-arg ANACONDA_PYTHON_VERSION=3.11 \ - --build-arg PYTORCH_ROCM_ARCH=gfx942 \ - --build-arg USE_THEROCK_NIGHTLY=1 \ - --build-arg THEROCK_NIGHTLY_INDEX_URL=https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ \ - --build-arg BUILD_ENVIRONMENT=therock-nightly-rocm6.4 \ - -t pytorch-therock-rocm-ci:test \ - . - ``` - -## Future Enhancements - -- Add test execution step after build -- Push Docker images to a registry for reuse -- Support multiple GPU architectures (gfx908, gfx90a, etc.) -- Add Slack/email notifications on build failures -- Create separate workflow for TheRock stable releases - -## Files Changed - -- `.github/workflows/therock-nightly.yml` (new) -- `.ci/docker/ubuntu-rocm/Dockerfile` (modified) -- `.ci/docker/common/install_rocm.sh` (modified) - ---- - -## PR Checklist - -- [ ] Workflow tested with manual trigger -- [ ] Docker image builds successfully with TheRock nightly wheels -- [ ] PyTorch builds successfully inside TheRock Docker image -- [ ] Artifacts are properly uploaded -- [ ] No breaking changes to existing CI workflows -- [ ] Documentation updated (if needed) From 9de27421e3ec536740ee0c720872e63c543b612c Mon Sep 17 00:00:00 2001 From: ethanwee1 Date: Fri, 21 Nov 2025 19:09:11 +0000 Subject: [PATCH 09/18] test --- .github/workflows/docker-builds.yml | 170 ++++++++++++++++++++++++++ .github/workflows/therock-nightly.yml | 4 +- 2 files changed, 173 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index fa1f083800fe0..eada60528f760 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -90,6 +90,8 @@ jobs: # Docker uploads fail from LF runners, see https://github.com/pytorch/pytorch/pull/137358 # runs-on: "${{ needs.get-label-type.outputs.label-type }}${{ matrix.runner }}" runs-on: "${{ matrix.runner }}" + outputs: + pytorch-linux-noble-rocm-n-py3: ${{ steps.build-docker-image.outputs.docker-image }} steps: - name: Clean workspace shell: bash @@ -164,3 +166,171 @@ jobs: - name: Teardown Linux uses: pytorch/test-infra/.github/actions/teardown-linux@main if: always() + + # New job: Build TheRock nightly variant of pytorch-linux-noble-rocm-n-py3 + docker-build-therock-nightly: + needs: docker-build + if: github.repository_owner == 'pytorch' + runs-on: linux.12xlarge + environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} + env: + THEROCK_NIGHTLY_INDEX_URL: https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ + ROCM_VERSION: nightly + + steps: + - name: Clean workspace + shell: bash + run: | + echo "${GITHUB_WORKSPACE}" + sudo rm -rf "${GITHUB_WORKSPACE}" + mkdir "${GITHUB_WORKSPACE}" + + - name: Checkout PyTorch + uses: pytorch/pytorch/.github/actions/checkout-pytorch@main + + - name: Setup Linux + uses: ./.github/actions/setup-linux + + - name: Pull base ROCm image + run: | + BASE_IMAGE="${{ needs.docker-build.outputs.pytorch-linux-noble-rocm-n-py3 }}" + echo "Pulling base image: ${BASE_IMAGE}" + docker pull "${BASE_IMAGE}" + echo "BASE_IMAGE=${BASE_IMAGE}" >> $GITHUB_ENV + + - name: Create TheRock nightly variant + id: create-therock-image + env: + THEROCK_NIGHTLY_INDEX_URL: ${{ env.THEROCK_NIGHTLY_INDEX_URL }} + run: | + set -euxo pipefail + + BASE_IMAGE="${{ env.BASE_IMAGE }}" + DATE_TAG=$(date +%Y%m%d) + THEROCK_IMAGE="ci-image:pytorch-linux-noble-rocm-nightly-py3-${DATE_TAG}" + + echo "Creating TheRock nightly image from: ${BASE_IMAGE}" + echo "Target image: ${THEROCK_IMAGE}" + + # Run the base image and modify it + docker run --name therock-setup "${BASE_IMAGE}" bash -c ' + set -euxo pipefail + + echo "=== Removing existing /opt/rocm ===" + if [ -d /opt/rocm ]; then + rm -rf /opt/rocm + echo "/opt/rocm removed successfully" + else + echo "/opt/rocm does not exist" + fi + + echo "=== Installing TheRock nightly wheels ===" + python3 -m pip install --upgrade pip + python3 -m pip install \ + --index-url "'"${THEROCK_NIGHTLY_INDEX_URL}"'" \ + "rocm-sdk[libraries,devel]" + + echo "=== Setting up ROCm environment ===" + ROCM_HOME="$(rocm-sdk path --root)" + ROCM_BIN="$(rocm-sdk path --bin)" + ROCM_CMAKE_PREFIX="$(rocm-sdk path --cmake)" + + echo "ROCM_HOME=${ROCM_HOME}" + echo "ROCM_BIN=${ROCM_BIN}" + echo "ROCM_CMAKE_PREFIX=${ROCM_CMAKE_PREFIX}" + + # Create environment setup script + cat > /etc/profile.d/rocm-sdk.sh <> "$GITHUB_OUTPUT" + + # Show image info + docker images | grep "rocm-nightly" || true + + - name: Generate output + id: generate_output + run: | + docker_image_name="pytorch-linux-noble-rocm-nightly-py3" + docker_image_tag="${{ steps.create-therock-image.outputs.docker-image }}" + echo "${docker_image_name}=${docker_image_tag}" >> docker-builds-output-${docker_image_name}.txt + cat docker-builds-output-${docker_image_name}.txt + + - name: Upload artifacts + uses: actions/upload-artifact@v4.4.0 + with: + name: docker-builds-artifacts-pytorch-linux-noble-rocm-nightly-py3 + retention-days: 14 + path: ./docker-builds-output-pytorch-linux-noble-rocm-nightly-py3.txt + + - uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 + name: Push to ECR + id: push-to-ecr + if: ${{ github.event_name == 'push' }} + env: + DOCKER_IMAGE: ${{ steps.create-therock-image.outputs.docker-image }} + with: + shell: bash + timeout_minutes: 60 + max_attempts: 5 + retry_wait_seconds: 90 + command: | + # Login is handled by setup-linux action + docker push "${DOCKER_IMAGE}" + docker push "ci-image:pytorch-linux-noble-rocm-nightly-py3" + + - uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 + name: Push to https://ghcr.io/ + id: push-to-ghcr-io + if: ${{ github.event_name == 'push' }} + env: + DOCKER_IMAGE: ${{ steps.create-therock-image.outputs.docker-image }} + GHCR_PAT: ${{ secrets.GHCR_PAT }} + with: + shell: bash + timeout_minutes: 60 + max_attempts: 5 + retry_wait_seconds: 90 + command: | + ghcr_image="ghcr.io/pytorch/ci-image" + tag=${DOCKER_IMAGE##*:} + # Push docker image to ghcr.io + echo $GHCR_PAT | docker login ghcr.io -u pytorch --password-stdin + docker tag "${DOCKER_IMAGE}" "${ghcr_image}:${tag}" + docker push "${ghcr_image}:${tag}" + # Also push the latest tag + docker tag "${DOCKER_IMAGE}" "${ghcr_image}:pytorch-linux-noble-rocm-nightly-py3" + docker push "${ghcr_image}:pytorch-linux-noble-rocm-nightly-py3" + + - name: Chown workspace + uses: ./.github/actions/chown-workspace + with: + ALPINE_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine + if: always() + + - name: Teardown Linux + uses: pytorch/test-infra/.github/actions/teardown-linux@main + if: always() diff --git a/.github/workflows/therock-nightly.yml b/.github/workflows/therock-nightly.yml index 2033ba8489000..33eff8eaa019f 100644 --- a/.github/workflows/therock-nightly.yml +++ b/.github/workflows/therock-nightly.yml @@ -10,10 +10,11 @@ on: env: PYTORCH_NIGHTLY_REF: nightly THEROCK_NIGHTLY_INDEX_URL: https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ + ROCM_VERSION: nightly # Base CI image to modify (same as rocm-mi300.yml uses) BASE_CI_IMAGE: pytorch/pytorch-linux-noble-rocm-n-py3:latest # New image name after installing TheRock - THEROCK_IMAGE: pytorch-linux-noble-rocm-alpha-py3 + THEROCK_IMAGE: pytorch-linux-noble-rocm-nightly-py3 PYTORCH_ROCM_ARCH: "gfx942" PYTHON_VERSION: "3.12" @@ -89,6 +90,7 @@ export ROCM_HOME="${ROCM_HOME}" export ROCM_PATH="${ROCM_HOME}" export PATH="${ROCM_BIN}:\$PATH" export CMAKE_PREFIX_PATH="${ROCM_CMAKE_PREFIX}:\${CMAKE_PREFIX_PATH:-}" +export ROCM_VERSION=nightly EOF echo "=== Verifying TheRock installation ===" From c2985447c86f7f80e5c5d201ad800b95fda9afe8 Mon Sep 17 00:00:00 2001 From: ethanwee1 Date: Fri, 21 Nov 2025 19:14:35 +0000 Subject: [PATCH 10/18] test --- .ci/docker/ubuntu-rocm/Dockerfile | 14 ++++++++------ .github/workflows/docker-builds.yml | 22 +++++++++++++++++++--- .github/workflows/therock-nightly.yml | 22 +++++++++++++++++++--- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/.ci/docker/ubuntu-rocm/Dockerfile b/.ci/docker/ubuntu-rocm/Dockerfile index 28dedf962844d..e50a3d25bfc00 100644 --- a/.ci/docker/ubuntu-rocm/Dockerfile +++ b/.ci/docker/ubuntu-rocm/Dockerfile @@ -54,6 +54,7 @@ ENV INSTALLED_VISION ${VISION} ARG ROCM_VERSION ARG USE_THEROCK_NIGHTLY ARG THEROCK_NIGHTLY_INDEX_URL +ARG ROCM_PATH=/opt/rocm ENV USE_THEROCK_NIGHTLY=${USE_THEROCK_NIGHTLY} ENV THEROCK_NIGHTLY_INDEX_URL=${THEROCK_NIGHTLY_INDEX_URL} RUN mkdir ci_commit_pins @@ -68,12 +69,13 @@ RUN bash ./install_rocm_magma.sh ${ROCM_VERSION} RUN rm install_rocm_magma.sh ADD ./common/install_miopen.sh install_miopen.sh RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh -ENV ROCM_PATH /opt/rocm -ENV PATH /opt/rocm/bin:$PATH -ENV PATH /opt/rocm/hcc/bin:$PATH -ENV PATH /opt/rocm/hip/bin:$PATH -ENV PATH /opt/rocm/opencl/bin:$PATH -ENV PATH /opt/rocm/llvm/bin:$PATH +ENV ROCM_PATH ${ROCM_PATH} +ENV PATH ${ROCM_PATH}/bin:$PATH +ENV PATH ${ROCM_PATH}/hcc/bin:$PATH +ENV PATH ${ROCM_PATH}/hip/bin:$PATH +ENV PATH ${ROCM_PATH}/opencl/bin:$PATH +ENV PATH ${ROCM_PATH}/llvm/bin:$PATH +ENV MAGMA_HOME ${ROCM_PATH}/magma ENV MAGMA_HOME /opt/rocm/magma ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index eada60528f760..c6bd4a62adb33 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -239,6 +239,10 @@ jobs: echo "ROCM_BIN=${ROCM_BIN}" echo "ROCM_CMAKE_PREFIX=${ROCM_CMAKE_PREFIX}" + # Save paths for docker commit + echo "${ROCM_HOME}" > /tmp/rocm_home.txt + echo "${ROCM_BIN}" > /tmp/rocm_bin.txt + # Create environment setup script cat > /etc/profile.d/rocm-sdk.sh < /tmp/rocm_home.txt + echo "${ROCM_BIN}" > /tmp/rocm_bin.txt + # Create environment setup script cat > /etc/profile.d/rocm-sdk.sh < Date: Fri, 21 Nov 2025 19:19:22 +0000 Subject: [PATCH 11/18] test --- .ci/docker/ubuntu-rocm/Dockerfile | 15 ++-- .github/workflows/docker-builds.yml | 84 ++++---------------- .github/workflows/therock-nightly.yml | 108 ++++++-------------------- 3 files changed, 46 insertions(+), 161 deletions(-) diff --git a/.ci/docker/ubuntu-rocm/Dockerfile b/.ci/docker/ubuntu-rocm/Dockerfile index e50a3d25bfc00..11013580accd5 100644 --- a/.ci/docker/ubuntu-rocm/Dockerfile +++ b/.ci/docker/ubuntu-rocm/Dockerfile @@ -54,7 +54,6 @@ ENV INSTALLED_VISION ${VISION} ARG ROCM_VERSION ARG USE_THEROCK_NIGHTLY ARG THEROCK_NIGHTLY_INDEX_URL -ARG ROCM_PATH=/opt/rocm ENV USE_THEROCK_NIGHTLY=${USE_THEROCK_NIGHTLY} ENV THEROCK_NIGHTLY_INDEX_URL=${THEROCK_NIGHTLY_INDEX_URL} RUN mkdir ci_commit_pins @@ -69,13 +68,13 @@ RUN bash ./install_rocm_magma.sh ${ROCM_VERSION} RUN rm install_rocm_magma.sh ADD ./common/install_miopen.sh install_miopen.sh RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh -ENV ROCM_PATH ${ROCM_PATH} -ENV PATH ${ROCM_PATH}/bin:$PATH -ENV PATH ${ROCM_PATH}/hcc/bin:$PATH -ENV PATH ${ROCM_PATH}/hip/bin:$PATH -ENV PATH ${ROCM_PATH}/opencl/bin:$PATH -ENV PATH ${ROCM_PATH}/llvm/bin:$PATH -ENV MAGMA_HOME ${ROCM_PATH}/magma +ENV ROCM_PATH /opt/rocm +ENV PATH /opt/rocm/bin:$PATH +ENV PATH /opt/rocm/hcc/bin:$PATH +ENV PATH /opt/rocm/hip/bin:$PATH +ENV PATH /opt/rocm/opencl/bin:$PATH +ENV PATH /opt/rocm/llvm/bin:$PATH +ENV MAGMA_HOME /opt/rocm/magma ENV MAGMA_HOME /opt/rocm/magma ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index c6bd4a62adb33..558d476dafb54 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -209,75 +209,25 @@ jobs: DATE_TAG=$(date +%Y%m%d) THEROCK_IMAGE="ci-image:pytorch-linux-noble-rocm-nightly-py3-${DATE_TAG}" - echo "Creating TheRock nightly image from: ${BASE_IMAGE}" - echo "Target image: ${THEROCK_IMAGE}" + echo "Building TheRock nightly image from Dockerfile" + echo "Base: ${BASE_IMAGE}" + echo "Target: ${THEROCK_IMAGE}" - # Run the base image and modify it - docker run --name therock-setup "${BASE_IMAGE}" bash -c ' - set -euxo pipefail - - echo "=== Removing existing /opt/rocm ===" - if [ -d /opt/rocm ]; then - rm -rf /opt/rocm - echo "/opt/rocm removed successfully" - else - echo "/opt/rocm does not exist" - fi - - echo "=== Installing TheRock nightly wheels ===" - python3 -m pip install --upgrade pip - python3 -m pip install \ - --index-url "'"${THEROCK_NIGHTLY_INDEX_URL}"'" \ - "rocm-sdk[libraries,devel]" - - echo "=== Setting up ROCm environment ===" - ROCM_HOME="$(rocm-sdk path --root)" - ROCM_BIN="$(rocm-sdk path --bin)" - ROCM_CMAKE_PREFIX="$(rocm-sdk path --cmake)" - - echo "ROCM_HOME=${ROCM_HOME}" - echo "ROCM_BIN=${ROCM_BIN}" - echo "ROCM_CMAKE_PREFIX=${ROCM_CMAKE_PREFIX}" - - # Save paths for docker commit - echo "${ROCM_HOME}" > /tmp/rocm_home.txt - echo "${ROCM_BIN}" > /tmp/rocm_bin.txt - - # Create environment setup script - cat > /etc/profile.d/rocm-sdk.sh < /tmp/rocm_home.txt - echo "${ROCM_BIN}" > /tmp/rocm_bin.txt - - # Create environment setup script - cat > /etc/profile.d/rocm-sdk.sh <> "$GITHUB_OUTPUT" - # Show image info - docker images | grep "${THEROCK_IMAGE}" || true - # Save docker image as artifact for next job - echo "Saving image as artifact..." docker save "${IMAGE_TAG}" | gzip > therock-docker-image.tar.gz - name: Upload Docker image as artifact From 204070185ad619955d8c8213e44eb647022780ac Mon Sep 17 00:00:00 2001 From: ethanwee1 Date: Fri, 21 Nov 2025 19:27:47 +0000 Subject: [PATCH 12/18] test --- .ci/docker/build.sh | 14 +++ .github/workflows/docker-builds.yml | 135 +--------------------------- 2 files changed, 15 insertions(+), 134 deletions(-) diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index 748608005e622..67f019bee0467 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -189,6 +189,20 @@ case "$tag" in INDUCTOR_BENCHMARKS=yes fi ;; + pytorch-linux-noble-rocm-nightly-py3) + ANACONDA_PYTHON_VERSION=3.12 + GCC_VERSION=11 + VISION=yes + ROCM_VERSION=nightly + NINJA_VERSION=1.9.0 + TRITON=yes + KATEX=yes + UCX_COMMIT=${_UCX_COMMIT} + UCC_COMMIT=${_UCC_COMMIT} + PYTORCH_ROCM_ARCH="gfx942" + USE_THEROCK_NIGHTLY=1 + THEROCK_NIGHTLY_INDEX_URL=https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ + ;; pytorch-linux-jammy-xpu-n-1-py3) ANACONDA_PYTHON_VERSION=3.10 GCC_VERSION=11 diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index 558d476dafb54..97cc3746c5458 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -61,6 +61,7 @@ jobs: pytorch-linux-jammy-py3.14-clang12, pytorch-linux-jammy-rocm-n-py3, pytorch-linux-noble-rocm-n-py3, + pytorch-linux-noble-rocm-nightly-py3, pytorch-linux-jammy-rocm-n-py3-benchmarks, pytorch-linux-jammy-cuda12.8-cudnn9-py3.10-clang12, pytorch-linux-jammy-py3.10-gcc11, @@ -166,137 +167,3 @@ jobs: - name: Teardown Linux uses: pytorch/test-infra/.github/actions/teardown-linux@main if: always() - - # New job: Build TheRock nightly variant of pytorch-linux-noble-rocm-n-py3 - docker-build-therock-nightly: - needs: docker-build - if: github.repository_owner == 'pytorch' - runs-on: linux.12xlarge - environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'docker-build' || '' }} - env: - THEROCK_NIGHTLY_INDEX_URL: https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/ - ROCM_VERSION: nightly - - steps: - - name: Clean workspace - shell: bash - run: | - echo "${GITHUB_WORKSPACE}" - sudo rm -rf "${GITHUB_WORKSPACE}" - mkdir "${GITHUB_WORKSPACE}" - - - name: Checkout PyTorch - uses: pytorch/pytorch/.github/actions/checkout-pytorch@main - - - name: Setup Linux - uses: ./.github/actions/setup-linux - - - name: Pull base ROCm image - run: | - BASE_IMAGE="${{ needs.docker-build.outputs.pytorch-linux-noble-rocm-n-py3 }}" - echo "Pulling base image: ${BASE_IMAGE}" - docker pull "${BASE_IMAGE}" - echo "BASE_IMAGE=${BASE_IMAGE}" >> $GITHUB_ENV - - - name: Create TheRock nightly variant - id: create-therock-image - env: - THEROCK_NIGHTLY_INDEX_URL: ${{ env.THEROCK_NIGHTLY_INDEX_URL }} - run: | - set -euxo pipefail - - BASE_IMAGE="${{ env.BASE_IMAGE }}" - DATE_TAG=$(date +%Y%m%d) - THEROCK_IMAGE="ci-image:pytorch-linux-noble-rocm-nightly-py3-${DATE_TAG}" - - echo "Building TheRock nightly image from Dockerfile" - echo "Base: ${BASE_IMAGE}" - echo "Target: ${THEROCK_IMAGE}" - - # Build TheRock image from Dockerfile with USE_THEROCK_NIGHTLY=1 - cd .ci/docker - docker build \ - -f ubuntu-rocm/Dockerfile \ - --build-arg UBUNTU_VERSION=24.04 \ - --build-arg ROCM_VERSION=nightly \ - --build-arg ANACONDA_PYTHON_VERSION=3.12 \ - --build-arg PYTORCH_ROCM_ARCH=gfx942 \ - --build-arg USE_THEROCK_NIGHTLY=1 \ - --build-arg THEROCK_NIGHTLY_INDEX_URL="${THEROCK_NIGHTLY_INDEX_URL}" \ - --build-arg BUILD_ENVIRONMENT="therock-nightly-rocm" \ - -t "${THEROCK_IMAGE}" \ - ../.. - - cd ../.. - - # Also tag as latest - docker tag "${THEROCK_IMAGE}" "ci-image:pytorch-linux-noble-rocm-nightly-py3" - - echo "docker-image=${THEROCK_IMAGE}" >> "$GITHUB_OUTPUT" - - # Show image info - docker images | grep "rocm-nightly" || true - - - name: Generate output - id: generate_output - run: | - docker_image_name="pytorch-linux-noble-rocm-nightly-py3" - docker_image_tag="${{ steps.create-therock-image.outputs.docker-image }}" - echo "${docker_image_name}=${docker_image_tag}" >> docker-builds-output-${docker_image_name}.txt - cat docker-builds-output-${docker_image_name}.txt - - - name: Upload artifacts - uses: actions/upload-artifact@v4.4.0 - with: - name: docker-builds-artifacts-pytorch-linux-noble-rocm-nightly-py3 - retention-days: 14 - path: ./docker-builds-output-pytorch-linux-noble-rocm-nightly-py3.txt - - - uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 - name: Push to ECR - id: push-to-ecr - if: ${{ github.event_name == 'push' }} - env: - DOCKER_IMAGE: ${{ steps.create-therock-image.outputs.docker-image }} - with: - shell: bash - timeout_minutes: 60 - max_attempts: 5 - retry_wait_seconds: 90 - command: | - # Login is handled by setup-linux action - docker push "${DOCKER_IMAGE}" - docker push "ci-image:pytorch-linux-noble-rocm-nightly-py3" - - - uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 - name: Push to https://ghcr.io/ - id: push-to-ghcr-io - if: ${{ github.event_name == 'push' }} - env: - DOCKER_IMAGE: ${{ steps.create-therock-image.outputs.docker-image }} - GHCR_PAT: ${{ secrets.GHCR_PAT }} - with: - shell: bash - timeout_minutes: 60 - max_attempts: 5 - retry_wait_seconds: 90 - command: | - ghcr_image="ghcr.io/pytorch/ci-image" - tag=${DOCKER_IMAGE##*:} - # Push docker image to ghcr.io - echo $GHCR_PAT | docker login ghcr.io -u pytorch --password-stdin - docker tag "${DOCKER_IMAGE}" "${ghcr_image}:${tag}" - docker push "${ghcr_image}:${tag}" - # Also push the latest tag - docker tag "${DOCKER_IMAGE}" "${ghcr_image}:pytorch-linux-noble-rocm-nightly-py3" - docker push "${ghcr_image}:pytorch-linux-noble-rocm-nightly-py3" - - - name: Chown workspace - uses: ./.github/actions/chown-workspace - with: - ALPINE_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine - if: always() - - - name: Teardown Linux - uses: pytorch/test-infra/.github/actions/teardown-linux@main - if: always() From 9d1cb6b228b908fe4f53eb9f7ee8b1869187b253 Mon Sep 17 00:00:00 2001 From: ethanwee1 Date: Fri, 21 Nov 2025 19:34:43 +0000 Subject: [PATCH 13/18] test --- .ci/docker/ubuntu-rocm/Dockerfile | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.ci/docker/ubuntu-rocm/Dockerfile b/.ci/docker/ubuntu-rocm/Dockerfile index 11013580accd5..c37ec8a5b703f 100644 --- a/.ci/docker/ubuntu-rocm/Dockerfile +++ b/.ci/docker/ubuntu-rocm/Dockerfile @@ -68,6 +68,18 @@ RUN bash ./install_rocm_magma.sh ${ROCM_VERSION} RUN rm install_rocm_magma.sh ADD ./common/install_miopen.sh install_miopen.sh RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh + +# Detect actual ROCM_PATH and set in /etc/environment for all processes +RUN if [ "${USE_THEROCK_NIGHTLY}" = "1" ]; then \ + ROCM_PATH_DETECTED=$(python3 -c "from pathlib import Path; import subprocess; print(subprocess.check_output(['python3', '-m', 'rocm_sdk', 'path', '--root']).decode().strip())"); \ + else \ + ROCM_PATH_DETECTED=/opt/rocm; \ + fi && \ + echo "ROCM_PATH=${ROCM_PATH_DETECTED}" >> /etc/environment && \ + echo "PATH=${ROCM_PATH_DETECTED}/bin:${ROCM_PATH_DETECTED}/hcc/bin:${ROCM_PATH_DETECTED}/hip/bin:${ROCM_PATH_DETECTED}/opencl/bin:${ROCM_PATH_DETECTED}/llvm/bin:\${PATH}" >> /etc/environment && \ + echo "MAGMA_HOME=${ROCM_PATH_DETECTED}/magma" >> /etc/environment + +# Set default ENV for traditional builds - will be overridden by /etc/environment for TheRock ENV ROCM_PATH /opt/rocm ENV PATH /opt/rocm/bin:$PATH ENV PATH /opt/rocm/hcc/bin:$PATH @@ -145,5 +157,19 @@ ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT} # Install LLVM dev version (Defined in the pytorch/builder github repository) COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm +# Create entrypoint that sources /etc/environment for dynamic ROCm paths +RUN echo '#!/bin/bash' > /docker-entrypoint.sh && \ + echo 'set -e' >> /docker-entrypoint.sh && \ + echo '# Source environment to get dynamic ROCM_PATH' >> /docker-entrypoint.sh && \ + echo 'if [ -f /etc/environment ]; then' >> /docker-entrypoint.sh && \ + echo ' set -a' >> /docker-entrypoint.sh && \ + echo ' source /etc/environment' >> /docker-entrypoint.sh && \ + echo ' set +a' >> /docker-entrypoint.sh && \ + echo 'fi' >> /docker-entrypoint.sh && \ + echo 'exec "$@"' >> /docker-entrypoint.sh && \ + chmod +x /docker-entrypoint.sh + +ENTRYPOINT ["/docker-entrypoint.sh"] + USER jenkins CMD ["bash"] From 4278dabb1baf71909ccbb4a19c21a21ee8d013ae Mon Sep 17 00:00:00 2001 From: ethanwee1 Date: Fri, 21 Nov 2025 19:53:26 +0000 Subject: [PATCH 14/18] test --- .ci/docker/build.sh | 2 ++ .ci/docker/common/install_miopen.sh | 6 ++++++ .ci/docker/common/install_rocm_magma.sh | 7 +++++++ 3 files changed, 15 insertions(+) diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index 67f019bee0467..f15fbd2a9fdac 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -399,6 +399,8 @@ docker build \ --build-arg "SKIP_SCCACHE_INSTALL=${SKIP_SCCACHE_INSTALL:-}" \ --build-arg "SKIP_LLVM_SRC_BUILD_INSTALL=${SKIP_LLVM_SRC_BUILD_INSTALL:-}" \ --build-arg "INSTALL_MINGW=${INSTALL_MINGW:-}" \ + --build-arg "USE_THEROCK_NIGHTLY=${USE_THEROCK_NIGHTLY:-}" \ + --build-arg "THEROCK_NIGHTLY_INDEX_URL=${THEROCK_NIGHTLY_INDEX_URL:-}" \ -f $(dirname ${DOCKERFILE})/Dockerfile \ -t "$tmp_tag" \ "$@" \ diff --git a/.ci/docker/common/install_miopen.sh b/.ci/docker/common/install_miopen.sh index 3dbc67b90abaf..df05a765b4e06 100644 --- a/.ci/docker/common/install_miopen.sh +++ b/.ci/docker/common/install_miopen.sh @@ -10,6 +10,12 @@ if [[ -z $ROCM_VERSION ]]; then exit 1; fi +# Skip MIOpen build for TheRock nightly builds +if [[ "${USE_THEROCK_NIGHTLY:-0}" == "1" ]]; then + echo "Skipping MIOpen custom build for TheRock nightly" + exit 0 +fi + IS_UBUNTU=0 ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') case "$ID" in diff --git a/.ci/docker/common/install_rocm_magma.sh b/.ci/docker/common/install_rocm_magma.sh index 9bf45e6f1b0a9..c390ebbc89d6b 100644 --- a/.ci/docker/common/install_rocm_magma.sh +++ b/.ci/docker/common/install_rocm_magma.sh @@ -5,6 +5,13 @@ set -eou pipefail function do_install() { rocm_version=$1 + + # Skip magma install for TheRock nightly builds + if [[ "${USE_THEROCK_NIGHTLY:-0}" == "1" ]]; then + echo "Skipping MAGMA install for TheRock nightly build" + return 0 + fi + if [[ ${rocm_version} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then # chop off any patch version rocm_version="${rocm_version%.*}" From d4f11f321ec76f208052325c4538e8cb1b663263 Mon Sep 17 00:00:00 2001 From: Ethan Wee Date: Fri, 21 Nov 2025 11:55:13 -0800 Subject: [PATCH 15/18] Update Dockerfile --- .ci/docker/ubuntu-rocm/Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.ci/docker/ubuntu-rocm/Dockerfile b/.ci/docker/ubuntu-rocm/Dockerfile index c37ec8a5b703f..c36dc1829fc21 100644 --- a/.ci/docker/ubuntu-rocm/Dockerfile +++ b/.ci/docker/ubuntu-rocm/Dockerfile @@ -71,10 +71,8 @@ RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh # Detect actual ROCM_PATH and set in /etc/environment for all processes RUN if [ "${USE_THEROCK_NIGHTLY}" = "1" ]; then \ - ROCM_PATH_DETECTED=$(python3 -c "from pathlib import Path; import subprocess; print(subprocess.check_output(['python3', '-m', 'rocm_sdk', 'path', '--root']).decode().strip())"); \ - else \ - ROCM_PATH_DETECTED=/opt/rocm; \ - fi && \ + ROCM_PATH_DETECTED=$(python3 -c "from pathlib import Path; import subprocess; print(subprocess.check_output(['python3', '-m', 'rocm_sdk', 'path', '--root']).decode().strip())"); + fi echo "ROCM_PATH=${ROCM_PATH_DETECTED}" >> /etc/environment && \ echo "PATH=${ROCM_PATH_DETECTED}/bin:${ROCM_PATH_DETECTED}/hcc/bin:${ROCM_PATH_DETECTED}/hip/bin:${ROCM_PATH_DETECTED}/opencl/bin:${ROCM_PATH_DETECTED}/llvm/bin:\${PATH}" >> /etc/environment && \ echo "MAGMA_HOME=${ROCM_PATH_DETECTED}/magma" >> /etc/environment @@ -87,7 +85,6 @@ ENV PATH /opt/rocm/hip/bin:$PATH ENV PATH /opt/rocm/opencl/bin:$PATH ENV PATH /opt/rocm/llvm/bin:$PATH ENV MAGMA_HOME /opt/rocm/magma -ENV MAGMA_HOME /opt/rocm/magma ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 From 24b27cbcf8db7ddb5a5086fac53db47ec9b882f6 Mon Sep 17 00:00:00 2001 From: Ethan Wee Date: Fri, 21 Nov 2025 12:09:07 -0800 Subject: [PATCH 16/18] Update install_rocm.sh --- .ci/docker/common/install_rocm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker/common/install_rocm.sh b/.ci/docker/common/install_rocm.sh index 02fa6910c8412..78d9691d94902 100644 --- a/.ci/docker/common/install_rocm.sh +++ b/.ci/docker/common/install_rocm.sh @@ -39,7 +39,7 @@ install_ubuntu() { : "${THEROCK_NIGHTLY_INDEX_URL:=https://rocm.nightlies.amd.com/v2/gfx94X-dcgpu/}" python3 -m pip install \ --index-url "${THEROCK_NIGHTLY_INDEX_URL}" \ - "rocm-sdk[libraries,devel]" + "rocm[libraries,devel]" # Use the rocm-sdk CLI helper to populate environment defaults ROCM_HOME="$(rocm-sdk path --root)" From 6aeb8818cccdd826a5376e159e175c6fb57f89b1 Mon Sep 17 00:00:00 2001 From: Ethan Wee Date: Fri, 21 Nov 2025 12:37:03 -0800 Subject: [PATCH 17/18] Update Dockerfile --- .ci/docker/ubuntu-rocm/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.ci/docker/ubuntu-rocm/Dockerfile b/.ci/docker/ubuntu-rocm/Dockerfile index c36dc1829fc21..25e6d2dc32603 100644 --- a/.ci/docker/ubuntu-rocm/Dockerfile +++ b/.ci/docker/ubuntu-rocm/Dockerfile @@ -69,10 +69,11 @@ RUN rm install_rocm_magma.sh ADD ./common/install_miopen.sh install_miopen.sh RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh -# Detect actual ROCM_PATH and set in /etc/environment for all processes RUN if [ "${USE_THEROCK_NIGHTLY}" = "1" ]; then \ - ROCM_PATH_DETECTED=$(python3 -c "from pathlib import Path; import subprocess; print(subprocess.check_output(['python3', '-m', 'rocm_sdk', 'path', '--root']).decode().strip())"); - fi + ROCM_PATH_DETECTED="$(python3 -m rocm_sdk path --root)"; \ + else \ + ROCM_PATH_DETECTED="/opt/rocm"; \ + fi; \ echo "ROCM_PATH=${ROCM_PATH_DETECTED}" >> /etc/environment && \ echo "PATH=${ROCM_PATH_DETECTED}/bin:${ROCM_PATH_DETECTED}/hcc/bin:${ROCM_PATH_DETECTED}/hip/bin:${ROCM_PATH_DETECTED}/opencl/bin:${ROCM_PATH_DETECTED}/llvm/bin:\${PATH}" >> /etc/environment && \ echo "MAGMA_HOME=${ROCM_PATH_DETECTED}/magma" >> /etc/environment From eb23cc28421ab5e149e9ba9c02c1fde912783ac0 Mon Sep 17 00:00:00 2001 From: Ethan Wee Date: Fri, 21 Nov 2025 12:45:06 -0800 Subject: [PATCH 18/18] Update Dockerfile --- .ci/docker/ubuntu-rocm/Dockerfile | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.ci/docker/ubuntu-rocm/Dockerfile b/.ci/docker/ubuntu-rocm/Dockerfile index 25e6d2dc32603..1eee25a222e99 100644 --- a/.ci/docker/ubuntu-rocm/Dockerfile +++ b/.ci/docker/ubuntu-rocm/Dockerfile @@ -155,19 +155,5 @@ ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT} # Install LLVM dev version (Defined in the pytorch/builder github repository) COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm -# Create entrypoint that sources /etc/environment for dynamic ROCm paths -RUN echo '#!/bin/bash' > /docker-entrypoint.sh && \ - echo 'set -e' >> /docker-entrypoint.sh && \ - echo '# Source environment to get dynamic ROCM_PATH' >> /docker-entrypoint.sh && \ - echo 'if [ -f /etc/environment ]; then' >> /docker-entrypoint.sh && \ - echo ' set -a' >> /docker-entrypoint.sh && \ - echo ' source /etc/environment' >> /docker-entrypoint.sh && \ - echo ' set +a' >> /docker-entrypoint.sh && \ - echo 'fi' >> /docker-entrypoint.sh && \ - echo 'exec "$@"' >> /docker-entrypoint.sh && \ - chmod +x /docker-entrypoint.sh - -ENTRYPOINT ["/docker-entrypoint.sh"] - USER jenkins CMD ["bash"]