From 95b9dd653abc6c3745077e6c10da75898af86e8b Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Thu, 16 Jul 2026 13:50:58 +0000 Subject: [PATCH 1/3] socat.yml: prevent rare shard hang in inverted OPENSSL tests The inverted testserversec OPENSSL tests (OPENSSLCERTCLIENT, the commonname test and the fips test) run a foreground one-shot SSL listener with no port-wait; their background connector exits after 2s (retry=2, 1s apart). On a loaded runner the listener can bind after the connector is already gone and then blocks in accept() forever, stalling the shard until the job-level 15-minute timeout kills the run with no diagnosable output. Seen once on a run whose re-run passed; reproduced deterministically by delaying the listener past the retry budget. - Widen the connector budget to retry=30. Retries only happen while the listener is not yet up and testserversec kills the connector as soon as the client returns, so passing runs are unaffected. - Bound each shard with timeout(1) at 10 minutes (normal shard time is ~2 min) so any future hang fails fast and the shard output names the test that was running. --- .github/workflows/socat.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/socat.yml b/.github/workflows/socat.yml index 6b285653d0b..ea08e6c8895 100644 --- a/.github/workflows/socat.yml +++ b/.github/workflows/socat.yml @@ -126,10 +126,22 @@ jobs: # numeric non-loopback addresses (e.g. the multicast tests). Best-effort # (|| true), errors left visible so a runner without IPv6 still runs the # IPv4 tests and any failure stays diagnosable in the log. + # + # The inverted OPENSSL testserversec tests (OPENSSLCERTCLIENT etc.) + # run a foreground one-shot SSL listener with no port-wait; their + # background connector gives up after 2s (retry=2, 1s apart). On a + # loaded runner the listener can bind after the connector is gone + # and then blocks in accept() forever. retry=30 widens that window; + # a healthy run still connects on the first or second attempt, and + # testserversec kills the connector right after the client returns, + # so passing runs pay nothing. timeout(1) bounds each shard + # (normally ~2 min) so any remaining hang fails fast with the + # culprit test named in the shard output instead of stalling the + # job into the 15-min kill, which discards all output. cat > socat-configs.json <<'EOF' [{ "name": "socat", "build": false, "netns": true, "shards": __SHARDS__, - "run": [["bash", "-c", "set -e; ip link set lo up || true; sysctl -wq net.ipv6.conf.lo.disable_ipv6=0 || true; ip addr add ::1/128 dev lo || true; ip addr add fc00::1/128 dev lo || true; ip addr add 192.0.2.1/32 dev lo || true; sysctl -wq net.ipv6.bindv6only=0 || true; cp -a \"$SOCAT_SRC/.\" .; tests=$(seq \"$SHARD\" \"$SHARDS\" 999); SOCAT=\"$PWD/socat\" SHELL=/bin/bash ./test.sh -t 1.0 --expect-fail \"$EXPECT_FAIL\" ${tests:-0}"]] + "run": [["bash", "-c", "set -e; ip link set lo up || true; sysctl -wq net.ipv6.conf.lo.disable_ipv6=0 || true; ip addr add ::1/128 dev lo || true; ip addr add fc00::1/128 dev lo || true; ip addr add 192.0.2.1/32 dev lo || true; sysctl -wq net.ipv6.bindv6only=0 || true; cp -a \"$SOCAT_SRC/.\" .; sed -i 's/fork,retry=2,/fork,retry=30,/' test.sh; tests=$(seq \"$SHARD\" \"$SHARDS\" 999); SOCAT=\"$PWD/socat\" SHELL=/bin/bash timeout -k 10 600 ./test.sh -t 1.0 --expect-fail \"$EXPECT_FAIL\" ${tests:-0}"]] }] EOF sed -i "s/__SHARDS__/$(( 6 * $(nproc) ))/" socat-configs.json From 2ed0369ecc7bc7488286171cbf1db86d1f0644fc Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Thu, 16 Jul 2026 14:16:08 +0000 Subject: [PATCH 2/3] socat.yml: tighten comment --- .github/workflows/socat.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/socat.yml b/.github/workflows/socat.yml index ea08e6c8895..9c7be3b98ef 100644 --- a/.github/workflows/socat.yml +++ b/.github/workflows/socat.yml @@ -127,17 +127,12 @@ jobs: # (|| true), errors left visible so a runner without IPv6 still runs the # IPv4 tests and any failure stays diagnosable in the log. # - # The inverted OPENSSL testserversec tests (OPENSSLCERTCLIENT etc.) - # run a foreground one-shot SSL listener with no port-wait; their - # background connector gives up after 2s (retry=2, 1s apart). On a - # loaded runner the listener can bind after the connector is gone - # and then blocks in accept() forever. retry=30 widens that window; - # a healthy run still connects on the first or second attempt, and - # testserversec kills the connector right after the client returns, - # so passing runs pay nothing. timeout(1) bounds each shard - # (normally ~2 min) so any remaining hang fails fast with the - # culprit test named in the shard output instead of stalling the - # job into the 15-min kill, which discards all output. + # The inverted OPENSSL testserversec tests run a one-shot foreground + # listener with no port-wait; the background connector quits after + # 2s (retry=2), and a later listener then hangs in accept() forever. + # retry=30 widens the window at no cost to passing runs. timeout + # makes any remaining hang fail fast with the culprit test named in + # the shard output instead of stalling into the 15-min job kill. cat > socat-configs.json <<'EOF' [{ "name": "socat", "build": false, "netns": true, "shards": __SHARDS__, From b0c0ceed3d186cf4c5edd66c0469cba78257e699 Mon Sep 17 00:00:00 2001 From: Juliusz Sosinowicz Date: Thu, 16 Jul 2026 17:34:05 +0000 Subject: [PATCH 3/3] socat.yml: assert retry=2 pattern before patching test.sh Fail the shard loudly if socat's test.sh no longer contains the fork,retry=2 pattern, instead of letting sed silently no-op and reintroduce the hang while the run still looks green. --- .github/workflows/socat.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/socat.yml b/.github/workflows/socat.yml index 9c7be3b98ef..b679c883b07 100644 --- a/.github/workflows/socat.yml +++ b/.github/workflows/socat.yml @@ -133,10 +133,13 @@ jobs: # retry=30 widens the window at no cost to passing runs. timeout # makes any remaining hang fail fast with the culprit test named in # the shard output instead of stalling into the 15-min job kill. + # Assert the retry=2 pattern is present first: a silent sed no-op + # (upstream test.sh changed) would otherwise reintroduce the hang + # while the run still looked green. cat > socat-configs.json <<'EOF' [{ "name": "socat", "build": false, "netns": true, "shards": __SHARDS__, - "run": [["bash", "-c", "set -e; ip link set lo up || true; sysctl -wq net.ipv6.conf.lo.disable_ipv6=0 || true; ip addr add ::1/128 dev lo || true; ip addr add fc00::1/128 dev lo || true; ip addr add 192.0.2.1/32 dev lo || true; sysctl -wq net.ipv6.bindv6only=0 || true; cp -a \"$SOCAT_SRC/.\" .; sed -i 's/fork,retry=2,/fork,retry=30,/' test.sh; tests=$(seq \"$SHARD\" \"$SHARDS\" 999); SOCAT=\"$PWD/socat\" SHELL=/bin/bash timeout -k 10 600 ./test.sh -t 1.0 --expect-fail \"$EXPECT_FAIL\" ${tests:-0}"]] + "run": [["bash", "-c", "set -e; ip link set lo up || true; sysctl -wq net.ipv6.conf.lo.disable_ipv6=0 || true; ip addr add ::1/128 dev lo || true; ip addr add fc00::1/128 dev lo || true; ip addr add 192.0.2.1/32 dev lo || true; sysctl -wq net.ipv6.bindv6only=0 || true; cp -a \"$SOCAT_SRC/.\" .; grep -q 'fork,retry=2,' test.sh || { echo 'socat test.sh: fork,retry=2 pattern gone, retry-widen would silently no-op (upstream changed?)' >&2; exit 1; }; sed -i 's/fork,retry=2,/fork,retry=30,/' test.sh; tests=$(seq \"$SHARD\" \"$SHARDS\" 999); SOCAT=\"$PWD/socat\" SHELL=/bin/bash timeout -k 10 600 ./test.sh -t 1.0 --expect-fail \"$EXPECT_FAIL\" ${tests:-0}"]] }] EOF sed -i "s/__SHARDS__/$(( 6 * $(nproc) ))/" socat-configs.json