arm: validate archive and retry on all errors for FVP and toolchain downloads (#19309)

### Summary

The Test ARM Backend workflow has been intermittently failing with
`curl: (92) HTTP/2 stream 0 was not closed cleanly: INTERNAL_ERROR`
during the FVP corstone download from developer.arm.com's CDN. The
toolchain download in the same setup uses the same bare-curl pattern and
fails the same way when the CDN flakes. In both cases the previous flow
was a single `curl --output ...` followed by a fatal `verify_md5`, so
neither a transient HTTP/2 reset nor a short error body that curl treats
as a successful 200 was retried.

Factor out a `download_with_retry` helper in utils.sh that wraps the
download in a five-attempt outer loop using
`curl --fail --retry-all-errors` and validates each attempt against the
published MD5 before proceeding, with the on-disk file size logged on
failure for diagnosis. Switch verify_md5's mismatch path from `exit 2`
to `return 2` so the helper can treat a bad checksum as a retryable
failure; existing callers (`verify_md5 ... || exit 1`) keep the same
fatal-on-mismatch behavior since the function still returns non-zero on
a bad checksum.

Use the helper from both fvp_utils.sh and toolchain_utils.sh in place of
the bare `curl` + `verify_md5` pair.

Authored with Claude Code.

### Test plan
CI

cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils
@Sebastian-Larsson @robell

Author: rascani

backends/arm/scripts/fvp_utils.sh

@@ -67,10 +67,9 @@ function install_fvp() {
             log_step "fvp" "Downloading FVP ${fvp}"
             url_variable=${fvp}_url
             fvp_url=${!url_variable}
-            curl --output "FVP_${fvp}.tgz" "${fvp_url}"
             md5_variable=${fvp}_md5_checksum
             fvp_md5_checksum=${!md5_variable}
-            verify_md5 ${fvp_md5_checksum} FVP_${fvp}.tgz || exit 1
+            download_with_retry "fvp" "${fvp_url}" "FVP_${fvp}.tgz" "${fvp_md5_checksum}" || exit 1
         fi
 
         log_step "fvp" "Installing FVP ${fvp}"

backends/arm/scripts/toolchain_utils.sh

@@ -107,8 +107,7 @@ function setup_toolchain() {
 
     if [[ ! -e "${toolchain_archive}" ]]; then
         log_step "toolchain" "Downloading ${toolchain_dir} toolchain"
-        curl --output "${toolchain_archive}" -L "${toolchain_url}"
-        verify_md5 ${toolchain_md5_checksum} "${toolchain_archive}" || exit 1
+        download_with_retry "toolchain" "${toolchain_url}" "${toolchain_archive}" "${toolchain_md5_checksum}" || exit 1
     fi
 
     log_step "toolchain" "Installing ${toolchain_dir} toolchain"

backends/arm/scripts/utils.sh

@@ -47,7 +47,10 @@ function verify_md5() {
     # Arg 1: Expected checksum for file
     # Arg 2: Path to file
     # Exits with return code 1 if the number of arguments is incorrect.
-    # Exits with return code 2 if the calculated mf5 does not match the given. 
+    # Returns 2 if the calculated md5 does not match the given. Returning
+    # rather than exiting lets callers like download_with_retry treat a bad
+    # checksum as a retryable failure (e.g. truncated download) instead of
+    # tearing down the whole script.
 
     [[ $# -ne 2 ]]  \
         && { echo "[${FUNCNAME[0]}] Invalid number of args, expecting 2, but got $#"; exit 1; }
@@ -60,11 +63,50 @@ function verify_md5() {
         local file_checksum="$(md5sum $file | awk '{print $1}')"
     fi
     if [[ ${ref_checksum} != ${file_checksum} ]]; then
-        echo "Mismatched MD5 checksum for file: ${file}. Expecting ${ref_checksum} but got ${file_checksum}. Exiting."
-        exit 2
+        echo "Mismatched MD5 checksum for file: ${file}. Expecting ${ref_checksum} but got ${file_checksum}."
+        return 2
     fi
 }
 
+function download_with_retry() {
+    # Download a URL to a path and validate its MD5, retrying on transport
+    # or checksum errors. developer.arm.com's CDN intermittently aborts the
+    # download mid-stream with HTTP/2 INTERNAL_ERROR (curl exit 92), and
+    # rare cases return a short error body that curl treats as success;
+    # both are caught here. --fail rejects HTTP errors,
+    # --retry-all-errors handles transport errors, and verify_md5 catches
+    # truncation / wrong-content via the published archive checksum.
+
+    # Arg 1: log context (passed to log_step)
+    # Arg 2: URL to download
+    # Arg 3: Output path
+    # Arg 4: Expected MD5 checksum
+
+    [[ $# -ne 4 ]] \
+        && { echo "[${FUNCNAME[0]}] Invalid number of args, expecting 4, but got $#"; exit 1; }
+    local context="${1}"
+    local url="${2}"
+    local output="${3}"
+    local expected_md5="${4}"
+
+    local max_attempts=5
+    for attempt in $(seq 1 ${max_attempts}); do
+        rm -f "${output}"
+        if curl --fail --retry 3 --retry-delay 5 --retry-connrefused --retry-all-errors \
+             -L --output "${output}" "${url}" \
+           && verify_md5 "${expected_md5}" "${output}"; then
+            return 0
+        fi
+        ls -l "${output}" 2>&1 || true
+        if [[ "${attempt}" = "${max_attempts}" ]]; then
+            log_step "${context}" "ERROR: download of ${url} failed after ${attempt} attempts"
+            return 1
+        fi
+        log_step "${context}" "download attempt ${attempt} failed; retrying in $((attempt * 10))s..."
+        sleep $((attempt * 10))
+    done
+}
+
 function patch_repo() {
     # Patch git repo found in $repo_dir, starting from patch $base_rev and applying patches found in $patch_dir/$name.