Retarget x64 CI to the NVMe 1ES pool and images

.pipeline/OneBranch/PublishFeeds-Sandbox.yml

@@ -110,9 +110,9 @@ extends:
         pool:
           type: windows
           isCustom: true
-          name: RUST-1ES-POOL-WUS3
+          name: RUST-1ES-POOL-WUS3-NVME
           demands:
-            - imageOverride -equals RUST-Win22-Sql25-1P
+            - imageOverride -equals RUST-Win22-Sql25-1P-NVME
         variables:
           ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
         steps:
@@ -144,9 +144,9 @@ extends:
         pool:
           type: linux
           isCustom: true
-          name: RUST-1ES-POOL-WUS3
+          name: RUST-1ES-POOL-WUS3-NVME
           demands:
-            - imageOverride -equals RUST-1ES-UBUSLIM
+            - imageOverride -equals RUST-1ES-UBUSLIM-NVME
         variables:
           ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
           ob_artifactSuffix: '_mock_linux_x64'
@@ -248,9 +248,9 @@ extends:
         pool:
           type: linux
           isCustom: true
-          name: RUST-1ES-POOL-WUS3
+          name: RUST-1ES-POOL-WUS3-NVME
           demands:
-            - imageOverride -equals RUST-1ES-UBUSLIM
+            - imageOverride -equals RUST-1ES-UBUSLIM-NVME
         variables:
           ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
           ob_artifactSuffix: '_mock_linux_musl_x64'
@@ -353,9 +353,9 @@ extends:
         pool:
           type: windows
           isCustom: true
-          name: RUST-1ES-POOL-WUS3
+          name: RUST-1ES-POOL-WUS3-NVME
           demands:
-            - imageOverride -equals RUST-Win22-Sql25-1P
+            - imageOverride -equals RUST-Win22-Sql25-1P-NVME
         variables:
           ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
           ob_artifactSuffix: '_mock_windows_x64'
@@ -522,9 +522,9 @@ extends:
         pool:
           type: windows
           isCustom: true
-          name: RUST-1ES-POOL-WUS3
+          name: RUST-1ES-POOL-WUS3-NVME
           demands:
-            - imageOverride -equals RUST-Win22-Sql25-1P
+            - imageOverride -equals RUST-Win22-Sql25-1P-NVME
         variables:
           ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
         steps:
@@ -576,9 +576,9 @@ extends:
         pool:
           type: windows
           isCustom: true
-          name: RUST-1ES-POOL-WUS3
+          name: RUST-1ES-POOL-WUS3-NVME
           demands:
-            - imageOverride -equals RUST-Win22-Sql25-1P
+            - imageOverride -equals RUST-Win22-Sql25-1P-NVME
         variables:
           ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
           CARGO_TARGET_DIR: C:\cargo_target_dir

.pipeline/OneBranch/stages.yml

@@ -48,9 +48,9 @@ stages:
     pool:
       type: windows
       isCustom: true
-      name: RUST-1ES-POOL-WUS3
+      name: RUST-1ES-POOL-WUS3-NVME
       demands:
-        - imageOverride -equals RUST-Win22-Sql25-1P
+        - imageOverride -equals RUST-Win22-Sql25-1P-NVME
     variables:
     - ${{ if eq(parameters.isOfficial, true) }}:
       - group: 'ESRP Federated Creds (AME)'
@@ -148,9 +148,9 @@ stages:
     pool:
       type: linux
       isCustom: true
-      name: RUST-1ES-POOL-WUS3
+      name: RUST-1ES-POOL-WUS3-NVME
       demands:
-        - imageOverride -equals RUST-1ES-UBUSLIM
+        - imageOverride -equals RUST-1ES-UBUSLIM-NVME
     variables:
       ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
       ob_artifactSuffix: '_linux_x64'

.pipeline/benchmark-pipeline.yml

@@ -25,9 +25,9 @@ stages:
         displayName: 'Criterion Benchmark'
         timeoutInMinutes: 30
         pool:
-          name: RUST-1ES-POOL-WUS3
+          name: RUST-1ES-POOL-WUS3-NVME
           demands:
-            - imageOverride -equals RUST-1ES-UBUSLIM
+            - imageOverride -equals RUST-1ES-UBUSLIM-NVME
         variables:
           ob_outputDirectory: '$(Build.ArtifactStagingDirectory)'
 

.pipeline/docs/arm-sql-host-design.md

@@ -341,15 +341,15 @@ unchanged.
 
 ## Open assumptions
 
-- Cross-pool reachability between `RUST-1ES-POOL-WUS3` (x64) and
+- Cross-pool reachability between `RUST-1ES-POOL-WUS3-NVME` (x64) and
   `RUST-1ES-POOL-ARM-WUS3` (ARM) on the SQL host's private IPv4 and the
   published port. Confirmed at design time; a connectivity regression here
   would surface as `poll-for-endpoint.sh` succeeds (the test job sees the
   endpoint) but the subsequent test connect times out.
 - `mcr.microsoft.com/mssql/server:<sqlImageTag>` ships
   `/opt/mssql-tools18/bin/sqlcmd`, which `start.sh` uses for the readiness
   probe. The current default tag (`2025-latest`) does.
-- The ubuntu image used by the x64 SQL host (`RUST-1ES-UBUSLIM`) provides
+- The ubuntu image used by the x64 SQL host (`RUST-1ES-UBUSLIM-NVME`) provides
   `curl` and `docker`. The latter is enforced by `DockerInstaller@0` and
   `install-ubuntu-dependency.yaml`; the former is standard in the image.
 - Both ARM and x64 1ES pool agents have `python3` available (used by

.pipeline/scripts/Generate-SqlCertificates.ps1

@@ -23,6 +23,39 @@ function Copy-To-Root-Store($cert) {
     }
 }
 
+function Restart-SqlServiceSafely($serviceName) {
+    # Freshly-provisioned images can leave SQL Server briefly in a start-pending /
+    # not-yet-stoppable state (startup database recovery), which makes a plain
+    # Restart-Service fail intermittently with "cannot be stopped"
+    # (CouldNotStopService). Wait for a steady state, then stop/start with retries.
+    $svc = Get-Service -Name $serviceName -ErrorAction Stop
+
+    for ($i = 1; $i -le 30; $i++) {
+        $svc.Refresh()
+        if ($svc.Status -eq 'Running' -or $svc.Status -eq 'Stopped') { break }
+        Write-Host "Waiting for $serviceName to reach a steady state (current: $($svc.Status))..."
+        Start-Sleep -Seconds 5
+    }
+
+    for ($attempt = 1; $attempt -le 5; $attempt++) {
+        try {
+            $svc.Refresh()
+            if ($svc.Status -ne 'Stopped') {
+                Stop-Service -Name $serviceName -Force -ErrorAction Stop
+                $svc.WaitForStatus('Stopped', [TimeSpan]::FromSeconds(120))
+            }
+            Start-Service -Name $serviceName -ErrorAction Stop
+            $svc.WaitForStatus('Running', [TimeSpan]::FromSeconds(120))
+            Write-Host "SQL service '$serviceName' restarted (attempt $attempt)."
+            return
+        } catch {
+            Write-Host "Restart attempt $attempt for '$serviceName' failed: $($_.Exception.Message)"
+            Start-Sleep -Seconds 10
+        }
+    }
+    throw "Failed to restart SQL service '$serviceName' after multiple attempts."
+}
+
 function New-And-Install-Certificates($instanceName) {
     Write-Output "Instance name received is " + $instanceName
     $certStorePath  = "Cert:\LocalMachine\My"
@@ -70,7 +103,7 @@ function New-And-Install-Certificates($instanceName) {
 
     Set-ItemProperty -Path $registryPath -Name "Certificate" -Value $thumbprint
 
-    Restart-Service -Name "MSSQLSERVER"
+    Restart-SqlServiceSafely -serviceName $instanceName
     Copy-To-Root-Store -cert $cert
 }
 

.pipeline/scripts/PrintAgentDebugInfo.ps1

@@ -17,3 +17,26 @@ if (Test-Path $pythonDir) {
 } else {
     Write-Host "Directory $pythonDir does NOT exist"
 }
+
+Write-Host ""
+Write-Host "=== Physical Disks (BusType shows NVMe / SATA / SAS) ==="
+try {
+    Get-PhysicalDisk | Select-Object DeviceId, FriendlyName, MediaType, BusType,
+        @{N='SizeGB';E={[math]::Round($_.Size/1GB,1)}} |
+        Format-Table -AutoSize | Out-String | Write-Host
+} catch { Write-Host "Get-PhysicalDisk failed: $_" }
+
+Write-Host "=== Disks ==="
+try {
+    Get-Disk | Select-Object Number, FriendlyName, BusType,
+        @{N='SizeGB';E={[math]::Round($_.Size/1GB,1)}}, PartitionStyle, OperationalStatus |
+        Format-Table -AutoSize | Out-String | Write-Host
+} catch { Write-Host "Get-Disk failed: $_" }
+
+Write-Host "=== Volumes (drive letters + free space) ==="
+try {
+    Get-Volume | Where-Object DriveLetter | Select-Object DriveLetter, FileSystemLabel, FileSystem,
+        @{N='SizeGB';E={[math]::Round($_.Size/1GB,1)}},
+        @{N='FreeGB';E={[math]::Round($_.SizeRemaining/1GB,1)}} |
+        Format-Table -AutoSize | Out-String | Write-Host
+} catch { Write-Host "Get-Volume failed: $_" }

.pipeline/scripts/print-disk-info.sh

@@ -4,6 +4,34 @@
 echo "=== Disk Space - All Mount Points ==="
 df -h
 
+echo ""
+echo "=== Block Devices (TYPE / TRAN=transport / ROTA=1 spinning,0 SSD) ==="
+# TRAN column shows 'nvme' for NVMe-attached devices; ROTA=0 confirms SSD/NVMe.
+lsblk -o NAME,SIZE,TYPE,TRAN,ROTA,FSTYPE,MOUNTPOINT,MODEL 2>/dev/null || lsblk
+
+echo ""
+echo "=== NVMe devices present ==="
+if ls /dev/nvme* >/dev/null 2>&1; then
+  ls -l /dev/nvme*
+  command -v nvme >/dev/null 2>&1 && nvme list 2>/dev/null || true
+else
+  echo "No /dev/nvme* devices found (agent is NOT on NVMe)."
+fi
+
+echo ""
+echo "=== Backing device + transport for key mount points ==="
+for m in / /mnt /mnt/vss /mnt/azure_nvme_temp "${AGENT_WORKFOLDER:-/mnt/vss/_work}"; do
+  [ -e "$m" ] || continue
+  src=$(findmnt -no SOURCE --target "$m" 2>/dev/null)
+  fstype=$(findmnt -no FSTYPE --target "$m" 2>/dev/null)
+  # Resolve the parent block device and read its transport from sysfs.
+  dev=$(lsblk -no PKNAME "$src" 2>/dev/null | head -1)
+  [ -z "$dev" ] && dev=$(basename "$src" 2>/dev/null)
+  tran=$(cat "/sys/class/block/$dev/device/../transport" 2>/dev/null)
+  case "$src" in *nvme*) tran="${tran:-nvme}";; esac
+  echo "$m -> ${src:-?} (${fstype:-?}) transport=${tran:-unknown}"
+done
+
 echo ""
 echo "=== Agent Work Folder Disk Space ==="
 WORK_DIR="${AGENT_WORKFOLDER:-/mnt/vss/_work}"