From ccd810bb73f767386f094b50f266cacc1cb57beb Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Wed, 19 Aug 2026 19:07:30 +0000
Subject: [PATCH 1/3] CI: install ccache from the staged bundle and bound every
 apt call

ccache-setup ran a raw apt-get update, so jobs whose ghcr .deb bundle had
already installed cleanly still reached the mirror. On 2026-08-19 that step
stalled on archive.ubuntu.com for 9-43 min and took out 11 jobs. The .deb is
already in /var/cache/apt/archives - install-apt-deps stages the whole bundle
- so install it with --no-download first and only fall back to the mirror.
ccache was missing from the 22.04-minimal and linuxkm bundles; add it.

Neither the fallback in install-apt-deps nor any of the raw apt sites had a
timeout, so a wedged mirror hung instead of failing and the retry loops never
fired. ci-deps-image already solved this for itself (Acquire timeouts plus
`timeout`); apply the same to the consumers, and to the linuxkm producer job
that was still missing it. Also recover from a dpkg interrupted by a kill,
and bound the bundle pull.
---
 .github/actions/ccache-setup/action.yml       | 34 +++++++++++++--
 .github/actions/install-apt-deps/action.yml   | 18 ++++++--
 .../ci-deps/packages-ubuntu-22.04-minimal.txt |  1 +
 .github/scripts/zephyr-4.x/zephyr-test.sh     |  8 +++-
 .github/workflows/ci-deps-image.yml           | 14 ++++---
 .github/workflows/cross-library.yml           | 42 +++++++++++++++----
 .github/workflows/falcon-interop.yml          | 42 +++++++++++++++++--
 .github/workflows/sbom.yml                    | 21 +++++++++-
 8 files changed, 154 insertions(+), 26 deletions(-)

diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml
index f68734209fa..7e11c1f1159 100644
--- a/.github/actions/ccache-setup/action.yml
+++ b/.github/actions/ccache-setup/action.yml
@@ -42,9 +42,37 @@ runs:
         if command -v ccache >/dev/null 2>&1; then
           echo "ccache already installed: $(ccache --version | head -1)"
         elif [ "${{ runner.os }}" = "Linux" ]; then
-          sudo apt-get update -q
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
-            --no-install-recommends ccache
+          export DEBIAN_FRONTEND=noninteractive
+          # install-apt-deps stages the WHOLE ghcr bundle into
+          # /var/cache/apt/archives, and ccache is in the -minimal/-full
+          # lists, so in a job that ran it first the .deb is already on disk.
+          # Take it offline (--no-download): no apt-get update, nothing to
+          # stall on. Every other path here reaches the mirror, which is what
+          # used to hang these jobs for 10-40 min after the bundle had
+          # already installed cleanly.
+          if sudo apt-get install -y --no-install-recommends \
+               --no-download ccache; then
+            echo "ccache installed offline from the staged .deb bundle"
+          else
+            # Same defence in depth as install-apt-deps: Acquire timeouts drop
+            # a stalled connection, `timeout` hard-kills a wedged apt-get, and
+            # only then does the retry loop get a non-zero exit to act on.
+            APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                      -o Acquire::https::Timeout=30)
+            ok=""
+            for i in 1 2 3; do
+              sudo dpkg --configure -a >/dev/null 2>&1 || true
+              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
+                   --no-install-recommends ccache; then
+                ok=1
+                break
+              fi
+              echo "::warning::ccache apt install failed (attempt $i/3)"
+              sleep $((5 * i))
+            done
+            [ -n "$ok" ] || { echo "::error::could not install ccache"; exit 1; }
+          fi
         elif [ "${{ runner.os }}" = "macOS" ]; then
           brew install ccache
         else
diff --git a/.github/actions/install-apt-deps/action.yml b/.github/actions/install-apt-deps/action.yml
index b8cb68b4e4a..c279dca0e75 100644
--- a/.github/actions/install-apt-deps/action.yml
+++ b/.github/actions/install-apt-deps/action.yml
@@ -50,7 +50,7 @@ runs:
         # PRs read the public upstream image too rather than a nonexistent
         # ghcr.io/<fork>/wolfssl-ci-debs.
         IMG="ghcr.io/wolfssl/wolfssl-ci-debs:${{ inputs.ghcr-debs-tag }}"
-        if ! docker pull -q "$IMG" >/dev/null 2>&1; then
+        if ! timeout -k 10 300 docker pull -q "$IMG" >/dev/null 2>&1; then
           echo "::notice::ghcr bundle $IMG unavailable; using apt"
           exit 0
         fi
@@ -85,9 +85,21 @@ runs:
           NO_REC="--no-install-recommends"
         fi
 
+        # A wedged mirror hangs apt rather than failing it, so the retry loop
+        # below never fired and the job burned its whole budget instead.
+        # Defend in depth: apt drops a stalled connection after 30s and retries
+        # it (Acquire timeouts), `timeout` hard-kills an apt-get that wedged
+        # anyway, then the loop re-runs - re-reading apt-mirrors.txt, so a
+        # retry can land on a different mirror.
+        APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                  -o Acquire::https::Timeout=30)
+
         for i in $(seq 1 $RETRIES); do
-          if sudo apt-get update -q && \
-             sudo apt-get install -y $NO_REC ${{ inputs.packages }}; then
+          # A previous attempt killed mid-unpack leaves dpkg needing this.
+          sudo dpkg --configure -a >/dev/null 2>&1 || true
+          if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+             sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+               $NO_REC ${{ inputs.packages }}; then
             exit 0
           fi
           if [ "$i" -eq "$RETRIES" ]; then
diff --git a/.github/ci-deps/packages-ubuntu-22.04-minimal.txt b/.github/ci-deps/packages-ubuntu-22.04-minimal.txt
index c32e3ccb9cf..6fdbf8cfbba 100644
--- a/.github/ci-deps/packages-ubuntu-22.04-minimal.txt
+++ b/.github/ci-deps/packages-ubuntu-22.04-minimal.txt
@@ -4,6 +4,7 @@
 autoconf
 automake
 build-essential
+ccache
 crossbuild-essential-arm64
 crossbuild-essential-armel
 crossbuild-essential-armhf
diff --git a/.github/scripts/zephyr-4.x/zephyr-test.sh b/.github/scripts/zephyr-4.x/zephyr-test.sh
index c225277cacc..66a2930d64c 100755
--- a/.github/scripts/zephyr-4.x/zephyr-test.sh
+++ b/.github/scripts/zephyr-4.x/zephyr-test.sh
@@ -201,7 +201,13 @@ echo "==> [container] Exporting Zephyr..."
 west zephyr-export
 
 echo "==> [container] Installing host packages (newlib, python3-venv)..."
-sudo apt-get update -qq && sudo apt-get install -y -qq python3-venv libnewlib-dev >/dev/null 2>&1 || true
+# `|| true` keeps this best-effort, but without a timeout a wedged mirror
+# stalls here silently until the job budget runs out.
+APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
+sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -qq >/dev/null 2>&1 \
+  && sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y -qq \
+       python3-venv libnewlib-dev >/dev/null 2>&1 \
+  || echo "==> [container] host package install skipped (apt unavailable)"
 python3 -m venv .venv
 source .venv/bin/activate
 pip3 install west
diff --git a/.github/workflows/ci-deps-image.yml b/.github/workflows/ci-deps-image.yml
index d91ac4fb877..2fa1b1130b1 100644
--- a/.github/workflows/ci-deps-image.yml
+++ b/.github/workflows/ci-deps-image.yml
@@ -183,21 +183,25 @@ jobs:
           set -euo pipefail
           K="${{ steps.check.outputs.kernel }}"
           # linuxkm.yml installs only the headers; the membrowse linuxkm targets
-          # also need the build toolchain. Bundle the union - each consumer
-          # installs its own subset offline.
-          PKGS=(build-essential autoconf automake libtool "linux-headers-$K")
+          # also need the build toolchain, and ccache-setup installs ccache
+          # offline from whatever this bundle staged. Bundle the union - each
+          # consumer installs its own subset offline.
+          PKGS=(build-essential autoconf automake libtool ccache
+                "linux-headers-$K")
           echo "Packages: ${PKGS[*]}"
           export DEBIAN_FRONTEND=noninteractive
           rm -rf debs && mkdir -p debs
           sudo apt-get clean
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
           retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; }
-          retry sudo apt-get update -q
+          retry sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q
           # The whole set is required and this bundle is small, so resolve it as
           # one closure and let any download failure fail the job. We push only
           # on success, so a transient mirror error keeps the last good bundle
           # rather than publishing a partial one - which the kernel-label skip
           # would then pin in place until the kernel next changes (~monthly).
-          retry sudo apt-get install -y --download-only "${PKGS[@]}"
+          retry sudo timeout -k 10 600 apt-get "${APT_OPTS[@]}" install -y \
+            --download-only "${PKGS[@]}"
           sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true
           echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files"
           test -n "$(ls debs/*.deb 2>/dev/null)"  # headers are never preinstalled
diff --git a/.github/workflows/cross-library.yml b/.github/workflows/cross-library.yml
index af25dd52cf1..3d0463dfc78 100644
--- a/.github/workflows/cross-library.yml
+++ b/.github/workflows/cross-library.yml
@@ -64,10 +64,23 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool pkg-config \
-            git ca-certificates ${{ inputs.apt_packages }}
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          for i in 1 2 3; do
+            if timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+               timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+                 --no-install-recommends \
+                 build-essential autoconf automake libtool pkg-config \
+                 git ca-certificates ${{ inputs.apt_packages }}; then
+              break
+            fi
+            test "$i" -lt 3 || { echo "::error::apt-get failed after 3 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/3)"
+            sleep $((5 * i))
+          done
 
       # Building only needs the commit under test, not history. The break check
       # that needs history runs in the compile job, not here.
@@ -129,10 +142,23 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool pkg-config \
-            git ca-certificates ${{ inputs.apt_packages }}
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          for i in 1 2 3; do
+            if timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+               timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+                 --no-install-recommends \
+                 build-essential autoconf automake libtool pkg-config \
+                 git ca-certificates ${{ inputs.apt_packages }}; then
+              break
+            fi
+            test "$i" -lt 3 || { echo "::error::apt-get failed after 3 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/3)"
+            sleep $((5 * i))
+          done
 
       # This job does not build wolfSSL, but the latest leg still checks out
       # wolfSSL history because check-break.sh scans commit messages here. The
diff --git a/.github/workflows/falcon-interop.yml b/.github/workflows/falcon-interop.yml
index ad6bc8f0137..a6b1e6ffe46 100644
--- a/.github/workflows/falcon-interop.yml
+++ b/.github/workflows/falcon-interop.yml
@@ -91,8 +91,25 @@ jobs:
     steps:
       - name: Install build tools
         run: |
-          sudo apt-get update
-          sudo apt-get install -y ninja-build
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          apt_retry() {
+            local i
+            for i in 1 2 3; do
+              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+                return 0
+              fi
+              echo "::warning::apt-get failed (attempt $i/3)"
+              sleep $((5 * i))
+            done
+            echo "::error::apt-get failed after 3 attempts"
+            return 1
+          }
+          apt_retry ninja-build
 
       # Check out wolfSSL first: actions/checkout runs "git clean -ffdx", which
       # would delete an untracked oqs-install/ placed in the workspace by the
@@ -161,8 +178,25 @@ jobs:
     steps:
       - name: Install build tools
         run: |
-          sudo apt-get update
-          sudo apt-get install -y autoconf automake libtool
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          apt_retry() {
+            local i
+            for i in 1 2 3; do
+              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+                return 0
+              fi
+              echo "::warning::apt-get failed (attempt $i/3)"
+              sleep $((5 * i))
+            done
+            echo "::error::apt-get failed after 3 attempts"
+            return 1
+          }
+          apt_retry autoconf automake libtool
 
       - name: Checkout wolfSSL
         uses: actions/checkout@v5
diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml
index 4e2b3fcd950..98adbe5ffad 100644
--- a/.github/workflows/sbom.yml
+++ b/.github/workflows/sbom.yml
@@ -822,7 +822,24 @@ jobs:
 
       - name: Install build deps + SBOM validators
         run: |
-          sudo apt-get update
+          # A wedged mirror hangs apt instead of failing it. Acquire timeouts
+          # drop a stalled connection, `timeout` hard-kills apt-get if it
+          # wedges anyway, and the loop then retries against a fresh mirror.
+          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
+                    -o Acquire::https::Timeout=30)
+          apt_retry() {
+            local i
+            for i in 1 2 3; do
+              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+                return 0
+              fi
+              echo "::warning::apt-get failed (attempt $i/3)"
+              sleep $((5 * i))
+            done
+            echo "::error::apt-get failed after 3 attempts"
+            return 1
+          }
           # bison + autotools-dev are required by strace's ./bootstrap.
           # gcc-multilib + g++-multilib give strace's --enable-mpers=check
           # the 32-bit/x32 compilers it needs - without them mpers is
@@ -830,7 +847,7 @@ jobs:
           # syscalls, diverging from what bomsh's devcontainer produces.
           # The rest mirror bomsh's .devcontainer/Dockerfile bomtrace3
           # stage.
-          sudo apt-get install -y build-essential autoconf automake libtool \
+          apt_retry build-essential autoconf automake libtool \
             bison autotools-dev gcc-multilib g++-multilib \
             python3 python3-pip git
           python3 -m pip install --user --upgrade pip

From 16d9378f808cab1ff549c526f203a7053d7b3db8 Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Thu, 20 Aug 2026 09:21:33 +0000
Subject: [PATCH 2/3] CI: keep the apt retry loops POSIX and inside the job
 budget

cross-library's containers are bare images with no bash, so Actions runs the
step under sh; the bash array APT_OPTS=(...) was a syntax error and took out
all six cross-library builds. Use a plain word-splitting variable.

The retry budget also has to fit the caller's timeout-minutes - install-apt-deps
is used by jobs with 4 minutes - or the last attempt is cut off before it can
report. Two attempts at 60s/300s instead of three at 120s/900s. The Acquire
timeouts are what actually detect a wedge, so the outer bound only backstops
apt wedging outside its own I/O loop, and apt resumes from archives/partial/.

Also run dpkg --configure -a before the offline ccache probe, not only in the
fallback loop: an interrupted dpkg would otherwise push it to the mirror.
---
 .github/actions/ccache-setup/action.yml     | 11 ++---
 .github/actions/install-apt-deps/action.yml | 21 ++++++---
 .github/workflows/ci-deps-image.yml         |  8 ++--
 .github/workflows/cross-library.yml         | 50 ++++++++++++---------
 .github/workflows/falcon-interop.yml        | 26 ++++++-----
 .github/workflows/sbom.yml                  | 13 +++---
 6 files changed, 76 insertions(+), 53 deletions(-)

diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml
index 7e11c1f1159..31cea8d4686 100644
--- a/.github/actions/ccache-setup/action.yml
+++ b/.github/actions/ccache-setup/action.yml
@@ -50,6 +50,7 @@ runs:
           # stall on. Every other path here reaches the mirror, which is what
           # used to hang these jobs for 10-40 min after the bundle had
           # already installed cleanly.
+          sudo dpkg --configure -a >/dev/null 2>&1 || true
           if sudo apt-get install -y --no-install-recommends \
                --no-download ccache; then
             echo "ccache installed offline from the staged .deb bundle"
@@ -60,16 +61,16 @@ runs:
             APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                       -o Acquire::https::Timeout=30)
             ok=""
-            for i in 1 2 3; do
+            for i in 1 2; do
               sudo dpkg --configure -a >/dev/null 2>&1 || true
-              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
+              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 180 apt-get "${APT_OPTS[@]}" install -y \
                    --no-install-recommends ccache; then
                 ok=1
                 break
               fi
-              echo "::warning::ccache apt install failed (attempt $i/3)"
-              sleep $((5 * i))
+              echo "::warning::ccache apt install failed (attempt $i/2)"
+              sleep 5
             done
             [ -n "$ok" ] || { echo "::error::could not install ccache"; exit 1; }
           fi
diff --git a/.github/actions/install-apt-deps/action.yml b/.github/actions/install-apt-deps/action.yml
index c279dca0e75..f61e81febf3 100644
--- a/.github/actions/install-apt-deps/action.yml
+++ b/.github/actions/install-apt-deps/action.yml
@@ -5,9 +5,12 @@ inputs:
     description: 'Space-separated list of apt packages to install'
     required: true
   retries:
-    description: 'Number of retry attempts'
+    description: >
+      Number of retry attempts. Keep attempts x (60s update + 300s install)
+      within the calling job's timeout-minutes, or the last attempt is cut
+      off before it can report.
     required: false
-    default: '3'
+    default: '2'
   retry-delay:
     description: 'Initial delay between retries (seconds, doubles each attempt)'
     required: false
@@ -88,17 +91,21 @@ runs:
         # A wedged mirror hangs apt rather than failing it, so the retry loop
         # below never fired and the job burned its whole budget instead.
         # Defend in depth: apt drops a stalled connection after 30s and retries
-        # it (Acquire timeouts), `timeout` hard-kills an apt-get that wedged
-        # anyway, then the loop re-runs - re-reading apt-mirrors.txt, so a
-        # retry can land on a different mirror.
+        # it (Acquire timeouts - this is what actually detects a wedge, in
+        # ~90s), `timeout` hard-kills an apt-get that wedged outside its own
+        # I/O loop, then the loop re-runs - re-reading apt-mirrors.txt, so a
+        # retry can land on a different mirror. The timeouts stay tight so
+        # every attempt fits the caller's timeout-minutes (as low as 4 min);
+        # apt resumes from archives/partial/, so a killed transfer is not
+        # restarted from scratch.
         APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                   -o Acquire::https::Timeout=30)
 
         for i in $(seq 1 $RETRIES); do
           # A previous attempt killed mid-unpack leaves dpkg needing this.
           sudo dpkg --configure -a >/dev/null 2>&1 || true
-          if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-             sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+          if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+             sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
                $NO_REC ${{ inputs.packages }}; then
             exit 0
           fi
diff --git a/.github/workflows/ci-deps-image.yml b/.github/workflows/ci-deps-image.yml
index 2fa1b1130b1..76f1a65489d 100644
--- a/.github/workflows/ci-deps-image.yml
+++ b/.github/workflows/ci-deps-image.yml
@@ -193,14 +193,16 @@ jobs:
           rm -rf debs && mkdir -p debs
           sudo apt-get clean
           APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30)
-          retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; }
-          retry sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q
+          # 2 attempts, not 5: this job's timeout-minutes is 20, and an
+          # attempt cut off mid-flight reports nothing.
+          retry() { local i; for i in 1 2; do "$@" && return 0; sleep 5; done; "$@"; }
+          retry sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q
           # The whole set is required and this bundle is small, so resolve it as
           # one closure and let any download failure fail the job. We push only
           # on success, so a transient mirror error keeps the last good bundle
           # rather than publishing a partial one - which the kernel-label skip
           # would then pin in place until the kernel next changes (~monthly).
-          retry sudo timeout -k 10 600 apt-get "${APT_OPTS[@]}" install -y \
+          retry sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
             --download-only "${PKGS[@]}"
           sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true
           echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files"
diff --git a/.github/workflows/cross-library.yml b/.github/workflows/cross-library.yml
index 3d0463dfc78..34bb4da8c41 100644
--- a/.github/workflows/cross-library.yml
+++ b/.github/workflows/cross-library.yml
@@ -64,22 +64,27 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
+          # These containers are bare images with no bash, so this step runs
+          # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it
+          # word-splits.
           # A wedged mirror hangs apt instead of failing it. Acquire timeouts
-          # drop a stalled connection, `timeout` hard-kills apt-get if it
-          # wedges anyway, and the loop then retries against a fresh mirror.
-          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
-                    -o Acquire::https::Timeout=30)
-          for i in 1 2 3; do
-            if timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-               timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+          # drop a stalled connection (and are what actually detects a wedge),
+          # `timeout` hard-kills apt-get if it wedges outside its own I/O loop,
+          # and the loop then retries - re-reading the mirror list. Two
+          # attempts at 60s+300s fit inside this job's timeout-minutes; apt
+          # resumes from archives/partial/, so a killed transfer is not lost.
+          APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30"
+          for i in 1 2; do
+            if timeout -k 10 60 apt-get $APT_OPTS update -q && \
+               timeout -k 10 300 apt-get $APT_OPTS install -y \
                  --no-install-recommends \
                  build-essential autoconf automake libtool pkg-config \
                  git ca-certificates ${{ inputs.apt_packages }}; then
               break
             fi
-            test "$i" -lt 3 || { echo "::error::apt-get failed after 3 attempts"; exit 1; }
-            echo "::warning::apt-get failed (attempt $i/3)"
-            sleep $((5 * i))
+            test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/2)"
+            sleep 5
           done
 
       # Building only needs the commit under test, not history. The break check
@@ -142,22 +147,27 @@ jobs:
         run: |
           set -eux
           export DEBIAN_FRONTEND=noninteractive
+          # These containers are bare images with no bash, so this step runs
+          # under `sh` - keep it POSIX. $APT_OPTS is unquoted on purpose so it
+          # word-splits.
           # A wedged mirror hangs apt instead of failing it. Acquire timeouts
-          # drop a stalled connection, `timeout` hard-kills apt-get if it
-          # wedges anyway, and the loop then retries against a fresh mirror.
-          APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
-                    -o Acquire::https::Timeout=30)
-          for i in 1 2 3; do
-            if timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-               timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y \
+          # drop a stalled connection (and are what actually detects a wedge),
+          # `timeout` hard-kills apt-get if it wedges outside its own I/O loop,
+          # and the loop then retries - re-reading the mirror list. Two
+          # attempts at 60s+300s fit inside this job's timeout-minutes; apt
+          # resumes from archives/partial/, so a killed transfer is not lost.
+          APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30"
+          for i in 1 2; do
+            if timeout -k 10 60 apt-get $APT_OPTS update -q && \
+               timeout -k 10 300 apt-get $APT_OPTS install -y \
                  --no-install-recommends \
                  build-essential autoconf automake libtool pkg-config \
                  git ca-certificates ${{ inputs.apt_packages }}; then
               break
             fi
-            test "$i" -lt 3 || { echo "::error::apt-get failed after 3 attempts"; exit 1; }
-            echo "::warning::apt-get failed (attempt $i/3)"
-            sleep $((5 * i))
+            test "$i" -lt 2 || { echo "::error::apt-get failed after 2 attempts"; exit 1; }
+            echo "::warning::apt-get failed (attempt $i/2)"
+            sleep 5
           done
 
       # This job does not build wolfSSL, but the latest leg still checks out
diff --git a/.github/workflows/falcon-interop.yml b/.github/workflows/falcon-interop.yml
index a6b1e6ffe46..1cf613e2ff4 100644
--- a/.github/workflows/falcon-interop.yml
+++ b/.github/workflows/falcon-interop.yml
@@ -94,19 +94,20 @@ jobs:
           # A wedged mirror hangs apt instead of failing it. Acquire timeouts
           # drop a stalled connection, `timeout` hard-kills apt-get if it
           # wedges anyway, and the loop then retries against a fresh mirror.
+          # Two attempts at 60s+300s stay inside this job's timeout-minutes.
           APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                     -o Acquire::https::Timeout=30)
           apt_retry() {
             local i
-            for i in 1 2 3; do
-              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+            for i in 1 2; do
+              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then
                 return 0
               fi
-              echo "::warning::apt-get failed (attempt $i/3)"
-              sleep $((5 * i))
+              echo "::warning::apt-get failed (attempt $i/2)"
+              sleep 5
             done
-            echo "::error::apt-get failed after 3 attempts"
+            echo "::error::apt-get failed after 2 attempts"
             return 1
           }
           apt_retry ninja-build
@@ -181,19 +182,20 @@ jobs:
           # A wedged mirror hangs apt instead of failing it. Acquire timeouts
           # drop a stalled connection, `timeout` hard-kills apt-get if it
           # wedges anyway, and the loop then retries against a fresh mirror.
+          # Two attempts at 60s+300s stay inside this job's timeout-minutes.
           APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                     -o Acquire::https::Timeout=30)
           apt_retry() {
             local i
-            for i in 1 2 3; do
-              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+            for i in 1 2; do
+              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then
                 return 0
               fi
-              echo "::warning::apt-get failed (attempt $i/3)"
-              sleep $((5 * i))
+              echo "::warning::apt-get failed (attempt $i/2)"
+              sleep 5
             done
-            echo "::error::apt-get failed after 3 attempts"
+            echo "::error::apt-get failed after 2 attempts"
             return 1
           }
           apt_retry autoconf automake libtool
diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml
index 98adbe5ffad..8a6389a6c91 100644
--- a/.github/workflows/sbom.yml
+++ b/.github/workflows/sbom.yml
@@ -825,19 +825,20 @@ jobs:
           # A wedged mirror hangs apt instead of failing it. Acquire timeouts
           # drop a stalled connection, `timeout` hard-kills apt-get if it
           # wedges anyway, and the loop then retries against a fresh mirror.
+          # Two attempts at 60s+300s stay inside this job's timeout-minutes.
           APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                     -o Acquire::https::Timeout=30)
           apt_retry() {
             local i
-            for i in 1 2 3; do
-              if sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q && \
-                 sudo timeout -k 10 900 apt-get "${APT_OPTS[@]}" install -y "$@"; then
+            for i in 1 2; do
+              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y "$@"; then
                 return 0
               fi
-              echo "::warning::apt-get failed (attempt $i/3)"
-              sleep $((5 * i))
+              echo "::warning::apt-get failed (attempt $i/2)"
+              sleep 5
             done
-            echo "::error::apt-get failed after 3 attempts"
+            echo "::error::apt-get failed after 2 attempts"
             return 1
           }
           # bison + autotools-dev are required by strace's ./bootstrap.

From e1a2dd18628528226de0ee435b1b2e11aaf9d1d3 Mon Sep 17 00:00:00 2001
From: Juliusz Sosinowicz <juliusz@wolfssl.com>
Date: Fri, 21 Aug 2026 02:48:32 +0000
Subject: [PATCH 3/3] Address review: budget the apt loop and keep
 DEBIAN_FRONTEND across sudo

The per-attempt deadlines did not fit callers with a four minute
timeout-minutes, so a wedged mirror cancelled the job before the loop
could report it. Spend a configurable budget across the attempts instead
and set one that fits on the short jobs.

sudo resets the environment, so an exported DEBIAN_FRONTEND never
reached apt-get and a debconf prompt could block until the timeout.
---
 .github/actions/ccache-setup/action.yml     | 12 +++--
 .github/actions/install-apt-deps/action.yml | 57 +++++++++++++++------
 .github/workflows/check-source-text.yml     |  3 ++
 .github/workflows/cyrus-sasl.yml            |  3 ++
 .github/workflows/ipmitool.yml              |  3 ++
 .github/workflows/jwt-cpp.yml               |  3 ++
 .github/workflows/linuxkm.yml               |  3 ++
 .github/workflows/mosquitto.yml             |  3 ++
 .github/workflows/pam-ipmi.yml              |  3 ++
 .github/workflows/rng-tools.yml             |  3 ++
 10 files changed, 73 insertions(+), 20 deletions(-)

diff --git a/.github/actions/ccache-setup/action.yml b/.github/actions/ccache-setup/action.yml
index 31cea8d4686..a39b93567d3 100644
--- a/.github/actions/ccache-setup/action.yml
+++ b/.github/actions/ccache-setup/action.yml
@@ -51,8 +51,8 @@ runs:
           # used to hang these jobs for 10-40 min after the bundle had
           # already installed cleanly.
           sudo dpkg --configure -a >/dev/null 2>&1 || true
-          if sudo apt-get install -y --no-install-recommends \
-               --no-download ccache; then
+          if sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
+               --no-install-recommends --no-download ccache; then
             echo "ccache installed offline from the staged .deb bundle"
           else
             # Same defence in depth as install-apt-deps: Acquire timeouts drop
@@ -63,9 +63,11 @@ runs:
             ok=""
             for i in 1 2; do
               sudo dpkg --configure -a >/dev/null 2>&1 || true
-              if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
-                 sudo timeout -k 10 180 apt-get "${APT_OPTS[@]}" install -y \
-                   --no-install-recommends ccache; then
+              if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 60 \
+                     apt-get "${APT_OPTS[@]}" update -q && \
+                 sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 180 \
+                     apt-get "${APT_OPTS[@]}" install -y \
+                     --no-install-recommends ccache; then
                 ok=1
                 break
               fi
diff --git a/.github/actions/install-apt-deps/action.yml b/.github/actions/install-apt-deps/action.yml
index f61e81febf3..03417164832 100644
--- a/.github/actions/install-apt-deps/action.yml
+++ b/.github/actions/install-apt-deps/action.yml
@@ -5,12 +5,24 @@ inputs:
     description: 'Space-separated list of apt packages to install'
     required: true
   retries:
-    description: >
-      Number of retry attempts. Keep attempts x (60s update + 300s install)
-      within the calling job's timeout-minutes, or the last attempt is cut
-      off before it can report.
+    description: 'Number of retry attempts.'
     required: false
     default: '2'
+  budget-seconds:
+    description: >
+      Wall-clock the whole retry loop may spend. Split across the attempts,
+      so a wedged mirror is reported by this action instead of the job being
+      cancelled around it. Must fit inside the caller's timeout-minutes
+      alongside pull-timeout; the default suits jobs of 15 minutes or more,
+      short jobs have to lower it.
+    required: false
+    default: '600'
+  pull-timeout:
+    description: >
+      Deadline for the ghcr bundle pull. Counts against the same
+      timeout-minutes as budget-seconds.
+    required: false
+    default: '300'
   retry-delay:
     description: 'Initial delay between retries (seconds, doubles each attempt)'
     required: false
@@ -53,7 +65,7 @@ runs:
         # PRs read the public upstream image too rather than a nonexistent
         # ghcr.io/<fork>/wolfssl-ci-debs.
         IMG="ghcr.io/wolfssl/wolfssl-ci-debs:${{ inputs.ghcr-debs-tag }}"
-        if ! timeout -k 10 300 docker pull -q "$IMG" >/dev/null 2>&1; then
+        if ! timeout -k 10 ${{ inputs.pull-timeout }} docker pull -q "$IMG" >/dev/null 2>&1; then
           echo "::notice::ghcr bundle $IMG unavailable; using apt"
           exit 0
         fi
@@ -80,9 +92,9 @@ runs:
       if: steps.ghcr.outputs.satisfied != 'true'
       shell: bash
       run: |
-        export DEBIAN_FRONTEND=noninteractive
         RETRIES=${{ inputs.retries }}
         DELAY=${{ inputs.retry-delay }}
+        BUDGET=${{ inputs.budget-seconds }}
         NO_REC=""
         if [ "${{ inputs.no-install-recommends }}" = "true" ]; then
           NO_REC="--no-install-recommends"
@@ -94,23 +106,38 @@ runs:
         # it (Acquire timeouts - this is what actually detects a wedge, in
         # ~90s), `timeout` hard-kills an apt-get that wedged outside its own
         # I/O loop, then the loop re-runs - re-reading apt-mirrors.txt, so a
-        # retry can land on a different mirror. The timeouts stay tight so
-        # every attempt fits the caller's timeout-minutes (as low as 4 min);
-        # apt resumes from archives/partial/, so a killed transfer is not
-        # restarted from scratch.
+        # retry can land on a different mirror. apt resumes from
+        # archives/partial/, so a killed transfer is not restarted from
+        # scratch.
         APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30
                   -o Acquire::https::Timeout=30)
 
+        # Spend budget-seconds over the attempts rather than a fixed
+        # per-attempt deadline: a caller with a short timeout-minutes would
+        # otherwise be cancelled mid-attempt, before the loop could report
+        # the failure. update gets a sixth of an attempt, install the rest,
+        # with floors so a small budget still leaves apt time to work.
+        PER=$((BUDGET / RETRIES))
+        UPD=$((PER / 6))
+        [ "$UPD" -ge 20 ] || UPD=20
+        INS=$((PER - UPD))
+        [ "$INS" -ge 40 ] || INS=40
+        DEADLINE=$(($(date +%s) + BUDGET))
+
+        # sudo resets the environment, so DEBIAN_FRONTEND has to ride along
+        # on each privileged command rather than being exported once.
         for i in $(seq 1 $RETRIES); do
           # A previous attempt killed mid-unpack leaves dpkg needing this.
           sudo dpkg --configure -a >/dev/null 2>&1 || true
-          if sudo timeout -k 10 60 apt-get "${APT_OPTS[@]}" update -q && \
-             sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y \
-               $NO_REC ${{ inputs.packages }}; then
+          if sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $UPD \
+                 apt-get "${APT_OPTS[@]}" update -q && \
+             sudo DEBIAN_FRONTEND=noninteractive timeout -k 10 $INS \
+                 apt-get "${APT_OPTS[@]}" install -y \
+                 $NO_REC ${{ inputs.packages }}; then
             exit 0
           fi
-          if [ "$i" -eq "$RETRIES" ]; then
-            echo "::error::apt-get failed after $RETRIES attempts"
+          if [ "$i" -eq "$RETRIES" ] || [ "$(date +%s)" -ge "$DEADLINE" ]; then
+            echo "::error::apt-get failed after $i attempt(s) in ${BUDGET}s"
             exit 1
           fi
           echo "::warning::apt-get failed (attempt $i/$RETRIES), retrying in ${DELAY}s..."
diff --git a/.github/workflows/check-source-text.yml b/.github/workflows/check-source-text.yml
index cf3efb95da2..7f9524563b8 100644
--- a/.github/workflows/check-source-text.yml
+++ b/.github/workflows/check-source-text.yml
@@ -45,6 +45,9 @@ jobs:
       - name: Install shellcheck
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: shellcheck python3-yaml
           ghcr-debs-tag: ubuntu-24.04-full
 
diff --git a/.github/workflows/cyrus-sasl.yml b/.github/workflows/cyrus-sasl.yml
index 5c1ad746e0a..c4a5bc3c97e 100644
--- a/.github/workflows/cyrus-sasl.yml
+++ b/.github/workflows/cyrus-sasl.yml
@@ -66,6 +66,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: krb5-kdc krb5-otp libkrb5-dev libsocket-wrapper libnss-wrapper krb5-admin-server libdb5.3-dev
           ghcr-debs-tag: ubuntu-24.04-full
 
diff --git a/.github/workflows/ipmitool.yml b/.github/workflows/ipmitool.yml
index b5df5fbe124..6db463a4e63 100644
--- a/.github/workflows/ipmitool.yml
+++ b/.github/workflows/ipmitool.yml
@@ -63,6 +63,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libreadline-dev
           ghcr-debs-tag: ubuntu-24.04-full
       - name: Download lib
diff --git a/.github/workflows/jwt-cpp.yml b/.github/workflows/jwt-cpp.yml
index e3263f3f996..df9a56247f0 100644
--- a/.github/workflows/jwt-cpp.yml
+++ b/.github/workflows/jwt-cpp.yml
@@ -67,6 +67,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libgtest-dev
           ghcr-debs-tag: ubuntu-24.04-full
 
diff --git a/.github/workflows/linuxkm.yml b/.github/workflows/linuxkm.yml
index 8c404a4b0c8..a7194f2065d 100644
--- a/.github/workflows/linuxkm.yml
+++ b/.github/workflows/linuxkm.yml
@@ -36,6 +36,9 @@ jobs:
       - name: Install linux-headers
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: linux-headers-$(uname -r)
           ghcr-debs-tag: ubuntu-24.04-linuxkm
 
diff --git a/.github/workflows/mosquitto.yml b/.github/workflows/mosquitto.yml
index 49b515d274b..dbea1074939 100644
--- a/.github/workflows/mosquitto.yml
+++ b/.github/workflows/mosquitto.yml
@@ -78,6 +78,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: build-essential libev-dev libssl-dev automake python3-docutils libcunit1 libcunit1-doc libcunit1-dev pkg-config make python3-psutil
           ghcr-debs-tag: ubuntu-24.04-full
 
diff --git a/.github/workflows/pam-ipmi.yml b/.github/workflows/pam-ipmi.yml
index de907d05aff..faa19001a5a 100644
--- a/.github/workflows/pam-ipmi.yml
+++ b/.github/workflows/pam-ipmi.yml
@@ -64,6 +64,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libpam-dev ninja-build meson
           ghcr-debs-tag: ubuntu-24.04-full
 
diff --git a/.github/workflows/rng-tools.yml b/.github/workflows/rng-tools.yml
index 1027ab80fc6..f14877d0124 100644
--- a/.github/workflows/rng-tools.yml
+++ b/.github/workflows/rng-tools.yml
@@ -65,6 +65,9 @@ jobs:
       - name: Install dependencies
         uses: ./.github/actions/install-apt-deps
         with:
+          # Fit the loop inside this job's timeout-minutes.
+          budget-seconds: '120'
+          pull-timeout: '60'
           packages: libcurl4-openssl-dev libjansson-dev libp11-dev librtlsdr-dev libcap-dev
           ghcr-debs-tag: ubuntu-24.04-full