diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e9781f4..217a5108 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,8 +75,12 @@ jobs: with: submodules: recursive + # patchelf is for test-bundle-linux-runtime, which exercises the release + # packaging scripts. Without it that test SKIPs, and a skip is + # indistinguishable from a pass in the summary — which is how the release + # packaging went unobserved in the first place. - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y cmake ninja-build g++ libopenblas-dev + run: sudo apt-get update && sudo apt-get install -y cmake ninja-build g++ libopenblas-dev patchelf # Examples/server stay at their defaults: some tests link `common`, which # lives under examples/, so switching them off would break configure. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 86b55196..c03abe81 100755 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,7 +41,12 @@ jobs: with: submodules: recursive ref: ${{ inputs.tag || github.ref }} + # A dry run (empty tag, dispatched from a branch) has no tag to match, so + # this compared VERSION against a branch name and failed every time — + # which made the run red and the one job you dispatched it for unreadable + # among the noise. There is nothing to verify when nothing is published. - name: Verify VERSION matches tag + if: inputs.tag != '' || startsWith(github.ref, 'refs/tags/') run: | tag_version="${{ inputs.tag || github.ref_name }}" tag_version="${tag_version#v}" @@ -126,6 +131,12 @@ jobs: # base-system means the loader dies before main() and the process # exits 127 silently. python3 scripts/check-bundled-deps.py release/crispasr-linux-x86_64 + # Ship the licence text with the binaries. Every Windows, Android and + # libcrispasr artifact already did; no CLI tarball did, and these bundle + # third-party shared libraries (libgomp, and libomp on HIP) whose terms + # require the notice to travel with them. + cp LICENSE release/crispasr-linux-x86_64/ + cp THIRD_PARTY_NOTICES.txt release/crispasr-linux-x86_64/ cd release && tar czf crispasr-linux-x86_64.tar.gz crispasr-linux-x86_64/ - uses: actions/upload-artifact@v4 @@ -225,6 +236,12 @@ jobs: # Then assert it: any DT_NEEDED that is neither bundled nor base-system # means the loader dies before main() and the process exits 127 silently. python3 scripts/check-bundled-deps.py release/crispasr-linux-x86_64-avx512 + # Ship the licence text with the binaries. Every Windows, Android and + # libcrispasr artifact already did; no CLI tarball did, and these bundle + # third-party shared libraries (libgomp, and libomp on HIP) whose terms + # require the notice to travel with them. + cp LICENSE release/crispasr-linux-x86_64-avx512/ + cp THIRD_PARTY_NOTICES.txt release/crispasr-linux-x86_64-avx512/ cd release && tar czf crispasr-linux-x86_64-avx512.tar.gz \ crispasr-linux-x86_64-avx512/ @@ -302,6 +319,12 @@ jobs: # means the loader dies before main() and the process exits 127 silently. python3 scripts/check-bundled-deps.py release/crispasr-linux-x86_64-cuda \ --allow 'libcuda.so.*' --allow 'libcudart.so.*' --allow 'libcublas*.so.*' --allow 'libnv*.so.*' + # Ship the licence text with the binaries. Every Windows, Android and + # libcrispasr artifact already did; no CLI tarball did, and these bundle + # third-party shared libraries (libgomp, and libomp on HIP) whose terms + # require the notice to travel with them. + cp LICENSE release/crispasr-linux-x86_64-cuda/ + cp THIRD_PARTY_NOTICES.txt release/crispasr-linux-x86_64-cuda/ cd release && tar czf crispasr-linux-x86_64-cuda.tar.gz crispasr-linux-x86_64-cuda/ ls -lh crispasr-linux-x86_64-cuda.tar.gz @@ -378,6 +401,12 @@ jobs: # means the loader dies before main() and the process exits 127 silently. python3 scripts/check-bundled-deps.py release/crispasr-linux-x86_64-cuda13 \ --allow 'libcuda.so.*' --allow 'libcudart.so.*' --allow 'libcublas*.so.*' --allow 'libnv*.so.*' + # Ship the licence text with the binaries. Every Windows, Android and + # libcrispasr artifact already did; no CLI tarball did, and these bundle + # third-party shared libraries (libgomp, and libomp on HIP) whose terms + # require the notice to travel with them. + cp LICENSE release/crispasr-linux-x86_64-cuda13/ + cp THIRD_PARTY_NOTICES.txt release/crispasr-linux-x86_64-cuda13/ cd release && tar czf crispasr-linux-x86_64-cuda13.tar.gz crispasr-linux-x86_64-cuda13/ ls -lh crispasr-linux-x86_64-cuda13.tar.gz @@ -452,6 +481,12 @@ jobs: # means the loader dies before main() and the process exits 127 silently. python3 scripts/check-bundled-deps.py release/crispasr-linux-x86_64-hip \ --allow 'libamdhip64.so.*' --allow 'librocblas.so.*' --allow 'libhsa*.so.*' + # Ship the licence text with the binaries. Every Windows, Android and + # libcrispasr artifact already did; no CLI tarball did, and these bundle + # third-party shared libraries (libgomp, and libomp on HIP) whose terms + # require the notice to travel with them. + cp LICENSE release/crispasr-linux-x86_64-hip/ + cp THIRD_PARTY_NOTICES.txt release/crispasr-linux-x86_64-hip/ cd release && tar czf crispasr-linux-x86_64-hip.tar.gz crispasr-linux-x86_64-hip/ ls -lh crispasr-linux-x86_64-hip.tar.gz @@ -534,6 +569,13 @@ jobs: # because the gate only checked dependencies were PRESENT, never that # RUNPATH could find them. Fails the release rather than the consumer. bash tools/package-lib-bundle.sh "$OUT" + # And assert the result. verify-lib-bundle.sh (inside the script above) + # gates INTRA-bundle resolvability only — a dependency absent from the + # bundle entirely is outside what it asks, which is how the HIP bundle + # shipped needing an unbundled libomp.so (#341). This is the same gate + # the CLI tarballs have had since #296. + python3 scripts/check-bundled-deps.py "$OUT/lib" \ + --allow 'libamdhip64.so.*' --allow 'librocblas.so.*' --allow 'libhsa*.so.*' cp include/crispasr.h "$OUT/include/" cp ggml/include/*.h "$OUT/ggml/include/" 2>/dev/null || true cp LICENSE "$OUT/" @@ -610,6 +652,12 @@ jobs: # means the loader dies before main() and the process exits 127 silently. python3 scripts/check-bundled-deps.py release/crispasr-linux-x86_64-vulkan \ --allow 'libvulkan.so.*' + # Ship the licence text with the binaries. Every Windows, Android and + # libcrispasr artifact already did; no CLI tarball did, and these bundle + # third-party shared libraries (libgomp, and libomp on HIP) whose terms + # require the notice to travel with them. + cp LICENSE release/crispasr-linux-x86_64-vulkan/ + cp THIRD_PARTY_NOTICES.txt release/crispasr-linux-x86_64-vulkan/ cd release && tar czf crispasr-linux-x86_64-vulkan.tar.gz crispasr-linux-x86_64-vulkan/ ls -lh crispasr-linux-x86_64-vulkan.tar.gz @@ -673,6 +721,12 @@ jobs: # base-system means the loader dies before main() and the process # exits 127 silently. python3 scripts/check-bundled-deps.py release/crispasr-linux-arm64 + # Ship the licence text with the binaries. Every Windows, Android and + # libcrispasr artifact already did; no CLI tarball did, and these bundle + # third-party shared libraries (libgomp, and libomp on HIP) whose terms + # require the notice to travel with them. + cp LICENSE release/crispasr-linux-arm64/ + cp THIRD_PARTY_NOTICES.txt release/crispasr-linux-arm64/ cd release && tar czf crispasr-linux-arm64.tar.gz crispasr-linux-arm64/ - uses: actions/upload-artifact@v4 @@ -744,10 +798,14 @@ jobs: cp build/ggml/src/libggml-cpu.so "$PKG/" 2>/dev/null || true cp build/libcrispasr_helpers.so "$PKG/" - # Patch RPATH to $ORIGIN so libs resolve siblings without LD_LIBRARY_PATH - for f in "$PKG"/*.so; do - patchelf --set-rpath '$ORIGIN' "$f" 2>/dev/null || true - done + # Bundle the external dependency closure and patch RPATH to $ORIGIN, + # in that order (CrispASR #339). The inline loop this replaces did only + # the RUNPATH half, so the shipped tarball needed libgomp.so.1 and + # libblas.so.3 from the user's machine and `import crispasr` died in + # the loader on any host without them. + bash scripts/bundle-linux-runtime.sh "$PKG" + # Then assert it — the gate the other Linux legs have had since #296. + python3 scripts/check-bundled-deps.py "$PKG" cd release && tar czf crispasr-python-linux-x86_64.tar.gz crispasr-python-linux-x86_64/ @@ -817,10 +875,14 @@ jobs: cp build/ggml/src/libggml-cpu.so "$PKG/" 2>/dev/null || true cp build/libcrispasr_helpers.so "$PKG/" - # RPATH = $ORIGIN so siblings resolve without LD_LIBRARY_PATH - for f in "$PKG"/*.so; do - patchelf --set-rpath '$ORIGIN' "$f" 2>/dev/null || true - done + # Bundle the external dependency closure and patch RPATH to $ORIGIN, + # in that order (CrispASR #339). The inline loop this replaces did only + # the RUNPATH half, so the shipped tarball needed libgomp.so.1 and + # libblas.so.3 from the user's machine and `import crispasr` died in + # the loader on any host without them. + bash scripts/bundle-linux-runtime.sh "$PKG" + # Then assert it — the gate the other Linux legs have had since #296. + python3 scripts/check-bundled-deps.py "$PKG" cd release && tar czf crispasr-python-linux-arm64.tar.gz crispasr-python-linux-arm64/ @@ -858,6 +920,12 @@ jobs: cp build/bin/crispasr release/crispasr-macos/ cp build/bin/crispasr-quantize release/crispasr-macos/ bash scripts/bundle-c2pa.sh release/crispasr-macos || true + # Ship the licence text with the binaries. Every Windows, Android and + # libcrispasr artifact already did; no CLI tarball did, and these carry + # bundled third-party libraries whose terms require the notice to + # travel with them. + cp LICENSE release/crispasr-macos/ + cp THIRD_PARTY_NOTICES.txt release/crispasr-macos/ cd release && tar czf crispasr-macos.tar.gz crispasr-macos/ - uses: actions/upload-artifact@v4 @@ -1097,6 +1165,12 @@ jobs: # because the gate only checked dependencies were PRESENT, never that # RUNPATH could find them. Fails the release rather than the consumer. bash tools/package-lib-bundle.sh "$OUT" + # And assert the result. verify-lib-bundle.sh (inside the script above) + # gates INTRA-bundle resolvability only — a dependency absent from the + # bundle entirely is outside what it asks, which is how the HIP bundle + # shipped needing an unbundled libomp.so (#341). This is the same gate + # the CLI tarballs have had since #296. + python3 scripts/check-bundled-deps.py "$OUT/lib" # Headers so binding authors can extend the FFI without # vendoring the whole CrispASR repo. cp include/crispasr.h "$OUT/include/" @@ -1169,6 +1243,12 @@ jobs: # because the gate only checked dependencies were PRESENT, never that # RUNPATH could find them. Fails the release rather than the consumer. bash tools/package-lib-bundle.sh "$OUT" + # And assert the result. verify-lib-bundle.sh (inside the script above) + # gates INTRA-bundle resolvability only — a dependency absent from the + # bundle entirely is outside what it asks, which is how the HIP bundle + # shipped needing an unbundled libomp.so (#341). This is the same gate + # the CLI tarballs have had since #296. + python3 scripts/check-bundled-deps.py "$OUT/lib" cp include/crispasr.h "$OUT/include/" cp ggml/include/*.h "$OUT/ggml/include/" 2>/dev/null || true cp LICENSE "$OUT/" @@ -1243,6 +1323,12 @@ jobs: # because the gate only checked dependencies were PRESENT, never that # RUNPATH could find them. Fails the release rather than the consumer. bash tools/package-lib-bundle.sh "$OUT" + # And assert the result. verify-lib-bundle.sh (inside the script above) + # gates INTRA-bundle resolvability only — a dependency absent from the + # bundle entirely is outside what it asks, which is how the HIP bundle + # shipped needing an unbundled libomp.so (#341). This is the same gate + # the CLI tarballs have had since #296. + python3 scripts/check-bundled-deps.py "$OUT/lib" # Headers so binding authors can extend the FFI without # vendoring the whole CrispASR repo. cp include/crispasr.h "$OUT/include/" @@ -1502,6 +1588,13 @@ jobs: # because the gate only checked dependencies were PRESENT, never that # RUNPATH could find them. Fails the release rather than the consumer. bash tools/package-lib-bundle.sh "$OUT" + # And assert the result. verify-lib-bundle.sh (inside the script above) + # gates INTRA-bundle resolvability only — a dependency absent from the + # bundle entirely is outside what it asks, which is how the HIP bundle + # shipped needing an unbundled libomp.so (#341). This is the same gate + # the CLI tarballs have had since #296. + python3 scripts/check-bundled-deps.py "$OUT/lib" \ + --allow 'libcuda.so.*' --allow 'libcudart.so.*' --allow 'libcublas*.so.*' --allow 'libnv*.so.*' cp include/crispasr.h "$OUT/include/" cp ggml/include/*.h "$OUT/ggml/include/" 2>/dev/null || true cp LICENSE "$OUT/" diff --git a/LEARNINGS.md b/LEARNINGS.md index 0cb0fba5..18ec28a2 100755 --- a/LEARNINGS.md +++ b/LEARNINGS.md @@ -11,12 +11,69 @@ If a lesson is still "live" (affects current work), it's linked from --- -**Do not read this file top to bottom** — it is ~800 KB across 275 sections. +**Do not read this file top to bottom** — it is ~800 KB across 276 sections. The generated topic + per-model index is [`docs/LEARNINGS-INDEX.md`](docs/LEARNINGS-INDEX.md). Find the lesson there, then jump with `grep -n "" LEARNINGS.md`. Regenerate both with `python tools/gen-learnings-index.py` after adding a section. +## A packaging step that destroys the state a later step reads — and scripts that only run in a release job (#339 HIP, 2026-08-10) + +`bundle-linux-runtime.sh` makes a staged Linux directory self-contained in two +moves: rewrite `RUNPATH` to `$ORIGIN`, and copy in the non-system dependency +closure. It did them in that order for two releases, and the order is the bug. + +**1. `ldd` answers through the RUNPATH you just erased.** Resolution is not a +property of the binary alone. Rewriting `RUNPATH` first turns every dependency +that was reachable *only* that way into `=> not found`, so the copy loop — +`ldd … | awk '/=>/ {print $3}' | grep '^/'` — dropped them along with the blank +lines. The libraries most in need of bundling are precisely the ones in +non-default directories, i.e. precisely the ones this loses. **If step A mutates +the state step B queries, B goes first**, and this is worth stating in the +script's header rather than leaving as an ordering accident. + +The concrete casualty: ROCm's clang links OpenMP against LLVM's `libomp.so` in +`/opt/rocm/lib/llvm/lib`. gcc's `libgomp.so.1` is in the default loader path, so +six Linux legs worked and only HIP failed — a one-leg failure that looks +GPU-specific and is actually generic. + +Corollary for the copy itself: **scan a copied library at its ORIGINAL path**, +not at the staged copy. A `$ORIGIN`-relative RUNPATH resolves somewhere else +once the file has moved, so `ldd` on the copy answers a different question. + +**2. `grep '^/'` is a filter that eats its own error message.** `ldd` reports a +failure as `libfoo.so => not found` — a line with no path, indistinguishable +from noise to a path filter. The script then printed `rpaths normalised, 0 +librar(ies) bundled`: a success line, with a count, over a dropped dependency. +A count of what was done is not a statement about what was missed. Anything the +resolver could not resolve is now fatal **and named** — with the same exclusion +list the copy loop uses, since `libcuda.so.1` is legitimately absent from a +driverless CI runner and would otherwise fail every CUDA leg. + +**3. The log printed the answer and it read as progress.** The failing run +contains `rpath crispasr: '$ORIGIN:…:/opt/rocm-6.3.0/lib/llvm/lib:…' -> +'$ORIGIN'`. The directory holding the missing library is inside the string being +discarded, one line above the count that should have been non-zero. Both lines +describe work completed. **When a step reports what it did rather than what it +found, its output cannot be scanned for a problem** — you have to know the +expected value to notice a wrong one. + +**4. A script that only ever runs inside a release job cannot be observed +without publishing a release.** That is why two defects in these packaging +scripts shipped: v0.8.18's unloadable lib bundles, and this. The fix is not more +care at release time — it is a test in the ordinary tier. `cc`, a private +directory and `-Wl,-rpath` reproduce the exact condition with no ROCm, no GPU +and no release; the acceptance check is the staged binary *running* with its +build-time libdir deleted, not the file being present (presence is not +resolvability — v0.8.18 shipped bundles with every dependency beside them and +one unreachable). Red-verify against the shipped script before trusting the +green: the control printed `0 librar(ies) bundled` and failed, which is the +whole defect in one line. + +⚠ And install the tool the test needs on the CI job — `patchelf` was absent, and +a ctest SKIP reads exactly like a PASS in the summary. A gate that cannot run is +a gate that cannot go red. + ## A watermarked TTS output cannot be A/B'd with `cmp` — and whisper LID is not an accent metric (omnivoice, 2026-08-07) Four transferable things from SubtitleEdit-13273, where OmniVoice's target-language diff --git a/PLAN.md b/PLAN.md index e2222680..64327aef 100755 --- a/PLAN.md +++ b/PLAN.md @@ -245,6 +245,47 @@ None tracked for cohere. Two things deliberately NOT done, with reasons: extra model. Threshold sits below one int16 LSB so one non-zero sample disables it; the quietest real speech to hand peaks ~3800x higher. +## LANDED 2026-08-10 — #339 HIP: the bundler erased the RUNPATH it needed to read + +v0.8.27 restored five of the six missing Linux tarballs and HIP still did not +package. Separate defect, and one the first fix could not have caught: the build +succeeds, then `check-bundled-deps.py` refuses the staged directory with +`crispasr needs libomp.so`. + +`bundle-linux-runtime.sh` did its two jobs in the wrong order — rewrite RUNPATH +to `$ORIGIN`, THEN ask `ldd` what the binaries need. `ldd` resolves through the +binary's own RUNPATH, so erasing it first turns exactly those dependencies into +`=> not found`, and the copy loop's `grep '^/'` dropped them with the blank +lines. ROCm's clang links OpenMP against LLVM's `libomp.so` under +`/opt/rocm/lib/llvm/lib`, reachable only that way; gcc's `libgomp.so.1` is in +the default loader path, which is why six legs were unaffected and this +survived v0.8.27. + +⚠ **The failing line printed the evidence and was read as progress.** The log +says `rpath crispasr: '$ORIGIN:…:/opt/rocm-6.3.0/lib/llvm/lib:…' -> '$ORIGIN'` +and then `rpaths normalised, 0 librar(ies) bundled`. The directory it needed +was in the string being discarded, and "0 bundled" was a count nobody had a +reason to expect to be non-zero. + +⚠ **A green summary line over a dropped dependency.** The bundler reported how +many libraries it had copied and said nothing about the one it could not find; +`grep '^/'` filtered `=> not found` out with the blank lines. It is now fatal +there, naming the library, and consults the same exclusion list the copy loop +uses — otherwise `libcuda.so.1`, legitimately absent from a driverless CI +runner, would take down every CUDA leg. + +⚠ **These scripts only ever ran inside a release job.** That is why two defects +in them shipped: there was no way to observe one without publishing a release. +`tests/test-bundle-linux-runtime.sh` now reproduces the whole thing with `cc` +and a private directory plus `-Wl,-rpath` — no ROCm, no GPU, no release — and +sits in the `unit` tier on every push. Red-verified against the v0.8.27 script +before being trusted. `patchelf` was added to the CI unit job for it, because a +SKIP reads exactly like a PASS in the ctest summary. + +Dry runs of a single leg are now readable: `validate-version` compared VERSION +against the branch name when `tag` was empty and failed on every dry run, which +is the mode the input's own documentation recommends. + ## LANDED 2026-08-10 — #339 fallout: a red Release run silently killed every GPU wheel The reported bug was six of seven Linux tarballs failing in v0.8.26 (two shell diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index a7649774..d18feea2 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -336,9 +336,56 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -NOTE: OpenBLAS is dynamically linked (not bundled in the release tarball) -on Linux builds compiled with -DGGML_OPENBLAS=ON. Users must have -libopenblas installed on their system. +NOTE: OpenBLAS IS bundled in the Linux release archives, alongside its own +dependency closure. It used to be dynamically linked and left to the user, +which is what CrispASR #296 fixed — the loader failed before main() on any +machine without it. This paragraph said otherwise until 2026-08-10. + + +──────────────────────────────────────────────────────────────────────────────── +GNU OpenMP runtime — libgomp (https://gcc.gnu.org/) +GPL-3.0-or-later WITH GCC-exception-3.1 (GCC Runtime Library Exception) +──────────────────────────────────────────────────────────────────────────────── + +The Linux release archives bundle `libgomp.so.1`, the OpenMP runtime that ships +with GCC, because GGML is compiled with OpenMP enabled and the library is absent +from minimal installs. + +CrispASR's own source remains under the MIT licence in LICENSE. The GCC Runtime +Library Exception grants permission to propagate the compiled result of +combining this runtime with independent modules under any licence, provided all +target code was produced by Eligible Compilation Processes — which GCC is. The +exception is what makes an MIT-licensed binary linking libgomp unproblematic; +it does not change the licence of libgomp itself. + +Redistributing the `libgomp.so.1` FILE is separately a conveyance of a +GPL-3.0-or-later work, and carries the corresponding-source obligation of +GPLv3 section 6. The bundled copy is the unmodified library from the GCC +package of the distribution the release was built on (Ubuntu 22.04, `gcc-12`). +Complete corresponding source for that exact build is available from Ubuntu's +source archive (`apt-get source gcc-12`) and from https://gcc.gnu.org/, and the +CrispASR maintainers will provide it on request for three years from the date +of the release that carried it — open an issue at +https://github.com/CrispStrobe/CrispASR. + +Licence texts: https://www.gnu.org/licenses/gpl-3.0.html and +https://www.gnu.org/licenses/gcc-exception-3.1.html + + +──────────────────────────────────────────────────────────────────────────────── +LLVM OpenMP runtime — libomp (https://openmp.llvm.org/) +Apache License 2.0 WITH LLVM-exception +──────────────────────────────────────────────────────────────────────────────── + +The Linux HIP/ROCm release archives bundle `libomp.so`, the LLVM OpenMP runtime. +ROCm's compiler is clang, so `-fopenmp` resolves to LLVM's runtime rather than +GCC's, and it lives under `/opt/rocm/lib/llvm/lib` — a directory the dynamic +loader does not search, so it must travel with the binaries (CrispASR #339). + +Unmodified. Apache-2.0 with the LLVM exception is a permissive licence; it +imposes no copyleft on CrispASR's MIT-licensed source and no source-availability +obligation on this redistribution. Licence text and the exception: +https://llvm.org/LICENSE.txt ──────────────────────────────────────────────────────────────────────────────── diff --git a/docs/LEARNINGS-INDEX.md b/docs/LEARNINGS-INDEX.md index 8e7ec441..55be33ff 100644 --- a/docs/LEARNINGS-INDEX.md +++ b/docs/LEARNINGS-INDEX.md @@ -17,324 +17,325 @@ per-backend porting detail (Per-model port notes + the family cross-reference). Reach for the topic groups when you are stuck on a *technique*, and the cross-reference when you already know which model you are touching. -## Index by topic (275 lessons) +## Index by topic (276 lessons) **Security & untrusted input** (2) -- L1277 — Two more untrusted-input lessons: an HTTP server reads the body before your auth check, and a parser that abort()s on malformed input is a DoS a fuzzer finds in seconds -- L1304 — Every hand-rolled file parser sizes a buffer from an untrusted length field — that is the #1 memory-safety bug class, and a multi-agent audit finds them fast +- L1334 — Two more untrusted-input lessons: an HTTP server reads the body before your auth check, and a parser that abort()s on malformed input is a DoS a fuzzer finds in seconds +- L1361 — Every hand-rolled file parser sizes a buffer from an untrusted length field — that is the #1 memory-safety bug class, and a multi-agent audit finds them fast **Compliance, provenance & marking (EU AI Act)** (7) -- L20 — A watermarked TTS output cannot be A/B'd with `cmp` — and whisper LID is not an accent metric -- L580 — Moving a release tag is CLEAN when the fix touches only release TOOLING — the test is provenance, not "never move a tag" -- L10652 — Always rebuild the box-under-test before benchmarking -- L15741 — EU AI Act audit — provenance not suffix -- L15799 — EU AI Act — a PRESET voice can be a real person -- L16249 — consent record — BIND to the audio, don't chain the log -- L17544 — watermark detector — swap the STATISTIC, not the threshold - -**Build, CI, release & packaging** (20) - -- L228 — A guard job that runs ONE compiler family guards one compiler family -- L516 — A green release job is not a shipped artifact — verify what you DELIVER, not what the workflow says -- L645 — "Linked in CMake" is NOT evidence the code SHIPS — the linker drops an object nothing references -- L865 — Path-filtered CI workflows ROT — the first re-triggering change inherits ALL the debt accumulated since the last run -- L885 — Scoped `ctest -R` on a PARTIALLY BUILT tree has two name-collision traps: configure-time add_test and catch_discover NOT_BUILT placeholders -- L1170 — Porting a libllama-hosted reference onto our in-house Qwen3 was clean on first contact; and Kaggle `kernels_logs()` is the escape hatch when the output is page-capped -- L1228 — ggml disables CUDA graphs below Ampere (cc < 800) — an Ampere-only re-warm bug can't be reproduced on Kaggle's P100/T4 -- L1527 — Two concurrent HuggingFace uploads from one machine cause spurious mid-batch failures — serialize them (#192 aligner re-ship) -- L1996 — Never benchmark on a near-full disk — SIGBUS + nondeterminism masquerade as a code signal (§192) -- L2072 — Kaggle: datasets are per-account, and the ccache dataset has a required shape (§213) -- L2494 — Kaggle regression kernels: write output incrementally, crash-guard everything -- L2621 — Regenerate Go cgo LDFLAGS from a *linux-equivalent* config, never macOS -- L2636 — cmake-js defaults its build dir to `build/` — point it elsewhere (§166) -- L2790 — HF Space is a separate, FLAT repo that silently drifts from `hf-space/` -- L4093 — Windows / MSVC portability -- L5331 — GitHub Actions workflow triggers — `master` → `main` rename gotcha -- L5477 — libcrispasr.a + libcommon.a both define stb_vorbis / miniaudio impl (Linux ld dies) -- L10716 — Kaggle as a batch-rebake target: seven fragilities the script has to work around -- L14160 — CI has no NVIDIA GPU — a CUDA-only graph path (mimo RVQ) is guarded by an on-Kaggle exact-parity smoke, not CI (#309) -- L14987 — #313 Rust crates on crates.io via git dep +- L77 — A watermarked TTS output cannot be A/B'd with `cmp` — and whisper LID is not an accent metric +- L637 — Moving a release tag is CLEAN when the fix touches only release TOOLING — the test is provenance, not "never move a tag" +- L10709 — Always rebuild the box-under-test before benchmarking +- L15798 — EU AI Act audit — provenance not suffix +- L15856 — EU AI Act — a PRESET voice can be a real person +- L16306 — consent record — BIND to the audio, don't chain the log +- L17601 — watermark detector — swap the STATISTIC, not the threshold + +**Build, CI, release & packaging** (21) + +- L20 — A packaging step that destroys the state a later step reads — and scripts that only run in a release job +- L285 — A guard job that runs ONE compiler family guards one compiler family +- L573 — A green release job is not a shipped artifact — verify what you DELIVER, not what the workflow says +- L702 — "Linked in CMake" is NOT evidence the code SHIPS — the linker drops an object nothing references +- L922 — Path-filtered CI workflows ROT — the first re-triggering change inherits ALL the debt accumulated since the last run +- L942 — Scoped `ctest -R` on a PARTIALLY BUILT tree has two name-collision traps: configure-time add_test and catch_discover NOT_BUILT placeholders +- L1227 — Porting a libllama-hosted reference onto our in-house Qwen3 was clean on first contact; and Kaggle `kernels_logs()` is the escape hatch when the output is page-capped +- L1285 — ggml disables CUDA graphs below Ampere (cc < 800) — an Ampere-only re-warm bug can't be reproduced on Kaggle's P100/T4 +- L1584 — Two concurrent HuggingFace uploads from one machine cause spurious mid-batch failures — serialize them (#192 aligner re-ship) +- L2053 — Never benchmark on a near-full disk — SIGBUS + nondeterminism masquerade as a code signal (§192) +- L2129 — Kaggle: datasets are per-account, and the ccache dataset has a required shape (§213) +- L2551 — Kaggle regression kernels: write output incrementally, crash-guard everything +- L2678 — Regenerate Go cgo LDFLAGS from a *linux-equivalent* config, never macOS +- L2693 — cmake-js defaults its build dir to `build/` — point it elsewhere (§166) +- L2847 — HF Space is a separate, FLAT repo that silently drifts from `hf-space/` +- L4150 — Windows / MSVC portability +- L5388 — GitHub Actions workflow triggers — `master` → `main` rename gotcha +- L5534 — libcrispasr.a + libcommon.a both define stb_vorbis / miniaudio impl (Linux ld dies) +- L10773 — Kaggle as a batch-rebake target: seven fragilities the script has to work around +- L14217 — CI has no NVIDIA GPU — a CUDA-only graph path (mimo RVQ) is guarded by an on-Kaggle exact-parity smoke, not CI (#309) +- L15044 — #313 Rust crates on crates.io via git dep **Multi-surface wiring — CLI / C ABI / bindings / server** (24) -- L149 — A gate that ANDs "both sides known" is only as alive as its weakest detector -- L388 — A new REQUIRED request field is a breaking API change on a schedule you don't control — deny the sub-feature, don't refuse the request -- L476 — A hardcoded decode cap silently ignores --max-new-tokens — and forwarding it naively SHRINKS a backend -- L553 — A capability flag is a PROMISE the backend must keep — an unimplemented cap disables the safety nets keyed off it -- L847 — Two writers on one output field at different pipeline stages = an ordering bug — fix the ORDER, don't bolt on provenance -- L906 — A binding wrapper that has never been EXECUTED is untested code — and parallel agents produce cross-cutting conflicts only the verifier can see -- L1496 — A persistent KV cache reused from a larger allocation makes local `max_ctx` the wrong stride (#171 VibeVoice server leak) -- L2403 — Backend-name guards must match the *registered* name, by prefix when aliased (#171, #174) -- L2537 — A feature has ~8 front-ends — wiring it into one isn't "done" (§166) -- L2563 — A dry-run "preview" must mirror the real resolver, or it lies (§166) -- L2648 — A WebSocket server that "works in the browser" can still be RFC-broken (§166) -- L2662 — Server warmup is the launch-time differentiator vs the CLI (#165) -- L2675 — Don't throw on the per-request server path; surface the real error (#165) -- L6002 — Chatterbox CAMPPlus phase 1 — Kaldi fbank front-end -- L8544 — voxcpm2 VAE — Python wrapper that captures kwargs without forwarding -- L13080 — 2026-07-02 — `--gpu-backend` was silently ignored (#214) -- L13768 — Before starting a "top-priority, profiled" perf target: check whether it's already been tried-and-rejected, and whether a parallel session owns it -- L14249 — A public API with no caller is a harness-blind zone announcing itself — and an ONNX export may FOLD a BatchNorm your C++ still expects as tensors -- L14921 — #311 --strict-pipeline SHIPPED -- L15064 — #315 vibevoice long-form ASR budget: PR fixed only session; CLI+server needed the adapter… -- L15100 — #324 server diarize DROPPED 59% of the transcript -- L17250 — Parakeet #208 long-audio + enc-cache collapse -- L17357 — session-state fixes + moonshine CPU-only SOLVED -- L17581 — Wyoming TTS marked nothing — FIXED 8f4197ec +- L206 — A gate that ANDs "both sides known" is only as alive as its weakest detector +- L445 — A new REQUIRED request field is a breaking API change on a schedule you don't control — deny the sub-feature, don't refuse the request +- L533 — A hardcoded decode cap silently ignores --max-new-tokens — and forwarding it naively SHRINKS a backend +- L610 — A capability flag is a PROMISE the backend must keep — an unimplemented cap disables the safety nets keyed off it +- L904 — Two writers on one output field at different pipeline stages = an ordering bug — fix the ORDER, don't bolt on provenance +- L963 — A binding wrapper that has never been EXECUTED is untested code — and parallel agents produce cross-cutting conflicts only the verifier can see +- L1553 — A persistent KV cache reused from a larger allocation makes local `max_ctx` the wrong stride (#171 VibeVoice server leak) +- L2460 — Backend-name guards must match the *registered* name, by prefix when aliased (#171, #174) +- L2594 — A feature has ~8 front-ends — wiring it into one isn't "done" (§166) +- L2620 — A dry-run "preview" must mirror the real resolver, or it lies (§166) +- L2705 — A WebSocket server that "works in the browser" can still be RFC-broken (§166) +- L2719 — Server warmup is the launch-time differentiator vs the CLI (#165) +- L2732 — Don't throw on the per-request server path; surface the real error (#165) +- L6059 — Chatterbox CAMPPlus phase 1 — Kaldi fbank front-end +- L8601 — voxcpm2 VAE — Python wrapper that captures kwargs without forwarding +- L13137 — 2026-07-02 — `--gpu-backend` was silently ignored (#214) +- L13825 — Before starting a "top-priority, profiled" perf target: check whether it's already been tried-and-rejected, and whether a parallel session owns it +- L14306 — A public API with no caller is a harness-blind zone announcing itself — and an ONNX export may FOLD a BatchNorm your C++ still expects as tensors +- L14978 — #311 --strict-pipeline SHIPPED +- L15121 — #315 vibevoice long-form ASR budget: PR fixed only session; CLI+server needed the adapter… +- L15157 — #324 server diarize DROPPED 59% of the transcript +- L17307 — Parakeet #208 long-audio + enc-cache collapse +- L17414 — session-state fixes + moonshine CPU-only SOLVED +- L17638 — Wyoming TTS marked nothing — FIXED 8f4197ec **Parity, the diff harness & ground truth** (36) -- L193 — The diff harness starts where its INPUT starts — check what it is fed before trusting parity -- L420 — When a clone/zero-shot path misbehaves but the BAKED/preset path is clean, diff the two paths' DATA byte-for-byte — the difference IS the bug -- L683 — Cosine, correlation and peak-match are ALL scale-invariant — a uniform gain error passes every one of them -- L751 — A logits-level diff harness at cos 1.000000 says nothing about the TABLE that turns logits into labels -- L826 — When you can't run the acceptance roundtrip, ship the path gated default-OFF + a provable-equivalence argument — don't block on the model -- L923 — Deterministic DROPOUT of whole words/tokens (not garble) = a zeroed weight BLOCK in the shipped GGUF — scan converted tensors for zero-norm rows BEFORE debugging the port -- L968 — A TTS AR model that "never stops" is often a WRONG-SAMPLING-PARAMS bug, not a stop-head/port bug — read the model card's generation params -- L1263 — Validating a TTS perf change by WAV correlation is invalid — melotts `--seed` isn't deterministic; use ASR round-trip or a deterministic-ASR bit-check -- L1345 — MelsTime mel layout IS ggml Conv1d input layout — transposing corrupts with cos ~0.3, not a gradual drift -- L1390 — Baked mel filterbank layout MUST match core_mel's `fb_layout` — a transpose produces cos_min ≈ −0.15, not a gradual drift -- L1593 — A production default that diverges from the upstream reference is a whole class of bug the stage-cosine diff can't see (#192 TADA) -- L1633 — A forced aligner is not an ASR model — its free CTC argmax is all-blank (#192) -- L1642 — When the engine and the PyTorch reference produce the SAME wrong output, the engine is right — the harness input (usually the prompt) is the bug (moss-transcribe) -- L1667 — An input feature the checkpoint wasn't trained with is worse than nothing — the moss-transcribe time-marker experiment (#218) -- L1692 — Stage the PyTorch reference load — encoder and LM as separate standalone modules, never the monolithic `*ForCausalLM` (moss-transcribe) -- L1712 — Parakeet single-pass DROPS whole sections of long audio — chunk-and-merge, and pick the reference clip carefully (#208b) -- L2008 — Always A/B against the *upstream reference*, not just your own option matrix (§216) -- L2059 — TTS WAV md5 is only a valid parity gate with a pinned seed -- L2480 — Regression transcripts need WER tolerance, not byte-exact match (#92) -- L3490 — Methodical debugging of ported models against ground truth -- L5597 — Chatterbox base T3 sampler parity — Gumbel-max → torch.multinomial -- L5636 — Chatterbox HiFT vocoder parity nits -- L8667 — Diff-harness "drift" is mostly the GGUF quant, not a code bug -- L11942 — Pocket TTS — 12 bugs from stub to cos=0.999997 -- L13357 — ASR roundtrip does not validate audio ONSETS; reproduce with the original pipeline before debugging your runtime -- L13401 — Porting the missing sub-model is necessary but not sufficient — hunt the hardcoded "unconditional" defaults that ignore it -- L13474 — Python reference dumpers must apply identical audio conditioning as the C++ runtime -- L13491 — A `frames/token` (or chars/sec) length heuristic silently truncates diffusion TTS — run the duration predictor the model already ships -- L13596 — A faithful port can still loop: get the bf16 blueprint's behaviour on the SAME audio before hunting a runtime bug -- L13656 — Generation-config defaults are part of the blueprint contract — a masked-iterative TTS with the wrong sampler knobs degenerates to SILENCE, not to bad audio -- L13800 — CosyVoice3 CFM Euler steps: 6 is the sweet spot, and validate it with log-mel corr vs full-steps — NOT ASR roundtrip -- L13992 — Exact greedy code-parity is unachievable for a QUANTIZED AR audio LM — confirm the port with a step-0 logit-rank probe, not byte-exact codes -- L14042 — A GPU-vs-reference RTF gap that survives kernel parity is per-step HOST detours — fuse them into the step graph; unified-memory Metal hides exactly this class of cost -- L14135 — A normalization stage amplifies upstream float error — input-align it before trusting its cos (§248 MBR) -- L15273 — #333 madlad400 quants + T5 parity -- L16145 — chatterbox hift_pcm(ref_mel) cos≈0.879 was a diff-harness layout bug (source_stft fed transposed), not a… +- L250 — The diff harness starts where its INPUT starts — check what it is fed before trusting parity +- L477 — When a clone/zero-shot path misbehaves but the BAKED/preset path is clean, diff the two paths' DATA byte-for-byte — the difference IS the bug +- L740 — Cosine, correlation and peak-match are ALL scale-invariant — a uniform gain error passes every one of them +- L808 — A logits-level diff harness at cos 1.000000 says nothing about the TABLE that turns logits into labels +- L883 — When you can't run the acceptance roundtrip, ship the path gated default-OFF + a provable-equivalence argument — don't block on the model +- L980 — Deterministic DROPOUT of whole words/tokens (not garble) = a zeroed weight BLOCK in the shipped GGUF — scan converted tensors for zero-norm rows BEFORE debugging the port +- L1025 — A TTS AR model that "never stops" is often a WRONG-SAMPLING-PARAMS bug, not a stop-head/port bug — read the model card's generation params +- L1320 — Validating a TTS perf change by WAV correlation is invalid — melotts `--seed` isn't deterministic; use ASR round-trip or a deterministic-ASR bit-check +- L1402 — MelsTime mel layout IS ggml Conv1d input layout — transposing corrupts with cos ~0.3, not a gradual drift +- L1447 — Baked mel filterbank layout MUST match core_mel's `fb_layout` — a transpose produces cos_min ≈ −0.15, not a gradual drift +- L1650 — A production default that diverges from the upstream reference is a whole class of bug the stage-cosine diff can't see (#192 TADA) +- L1690 — A forced aligner is not an ASR model — its free CTC argmax is all-blank (#192) +- L1699 — When the engine and the PyTorch reference produce the SAME wrong output, the engine is right — the harness input (usually the prompt) is the bug (moss-transcribe) +- L1724 — An input feature the checkpoint wasn't trained with is worse than nothing — the moss-transcribe time-marker experiment (#218) +- L1749 — Stage the PyTorch reference load — encoder and LM as separate standalone modules, never the monolithic `*ForCausalLM` (moss-transcribe) +- L1769 — Parakeet single-pass DROPS whole sections of long audio — chunk-and-merge, and pick the reference clip carefully (#208b) +- L2065 — Always A/B against the *upstream reference*, not just your own option matrix (§216) +- L2116 — TTS WAV md5 is only a valid parity gate with a pinned seed +- L2537 — Regression transcripts need WER tolerance, not byte-exact match (#92) +- L3547 — Methodical debugging of ported models against ground truth +- L5654 — Chatterbox base T3 sampler parity — Gumbel-max → torch.multinomial +- L5693 — Chatterbox HiFT vocoder parity nits +- L8724 — Diff-harness "drift" is mostly the GGUF quant, not a code bug +- L11999 — Pocket TTS — 12 bugs from stub to cos=0.999997 +- L13414 — ASR roundtrip does not validate audio ONSETS; reproduce with the original pipeline before debugging your runtime +- L13458 — Porting the missing sub-model is necessary but not sufficient — hunt the hardcoded "unconditional" defaults that ignore it +- L13531 — Python reference dumpers must apply identical audio conditioning as the C++ runtime +- L13548 — A `frames/token` (or chars/sec) length heuristic silently truncates diffusion TTS — run the duration predictor the model already ships +- L13653 — A faithful port can still loop: get the bf16 blueprint's behaviour on the SAME audio before hunting a runtime bug +- L13713 — Generation-config defaults are part of the blueprint contract — a masked-iterative TTS with the wrong sampler knobs degenerates to SILENCE, not to bad audio +- L13857 — CosyVoice3 CFM Euler steps: 6 is the sweet spot, and validate it with log-mel corr vs full-steps — NOT ASR roundtrip +- L14049 — Exact greedy code-parity is unachievable for a QUANTIZED AR audio LM — confirm the port with a step-0 logit-rank probe, not byte-exact codes +- L14099 — A GPU-vs-reference RTF gap that survives kernel parity is per-step HOST detours — fuse them into the step graph; unified-memory Metal hides exactly this class of cost +- L14192 — A normalization stage amplifies upstream float error — input-align it before trusting its cos (§248 MBR) +- L15330 — #333 madlad400 quants + T5 parity +- L16202 — chatterbox hift_pcm(ref_mel) cos≈0.879 was a diff-harness layout bug (source_stft fed transposed), not a… **ggml graphs, allocation & caching** (43) -- L459 — Reusing a cached scheduler graph across `sched_reset`/`alloc` cycles is CPU-safe but SIGSEGVs on GPU — the reused input tensor is bound to the prior cycle's freed buffer -- L1029 — Metal-validated GPU TTS graph paths DID transfer cleanly to CUDA — the stricter-CUDA risk is real but not universal -- L1065 — BatchNorm-fold must respect the conv tensor's dtype — and a per-backend fix usually has siblings -- L1099 — "Migrate host KV to device-resident" is a ~1% win on a compute-bound transformer decoder — measure the fraction before the rewrite -- L1399 — Never cache a ggml compute graph across calls that share a scheduler with a larger graph — the allocator regrows and frees the cached graph's GPU buffers -- L1410 — Flow-matching DiT with JointAttention REQUIRES attention masking even for unconditional generation — skipping masked KV tokens silently corrupts output (2026-07) -- L1745 — A cached cgraph is NOT re-entrant with ggml_backend_sched — the 2nd reuse silently corrupts (#208) -- L1787 — On Metal a cached decode graph is dispatch-bound, not alloc-bound — measure the step's parts, don't trust wall time (§210, PR #207) -- L1944 — Precomputed attention masks in a GGUF are dead weight if the runtime rebuilds them (§192) -- L2112 — Cross-attention KV is a free F16 win — encoder-decoder backends (§176i) -- L2140 — Localizing a GPU miscompute with CPU-vs-Metal diffing — the ggml_backend_sched weight-less-first-op trap (§206) -- L2294 — A byte-identical standalone reproducer is the only way to (dis)prove a "ggml bug" (§203) -- L2324 — In-place device-KV decode graphs on Metal: three hazards (§176b+c, §203) -- L2432 — Never pass `ggml_graph_get_tensor` directly to `ggml_backend_tensor_set` (#164) -- L2452 — `rope_theta=0` means "no positional encoding" — skip RoPE entirely (#164) -- L2461 — `ggml_siglu` ≠ PyTorch `glu` — the gate/value halves are SWAPPED (#81) -- L2530 — Asymmetric rel-pos shift uses the same formula as symmetric (#81) -- L2575 — Graph input tensors that nothing consumes are invisible to `ggml_graph_get_tensor` (#164) -- L2608 — `ggml_backend_cpu_set_n_threads` asserts CPU — guard it after `init_best()` -- L2721 — Decompose ConvTranspose1d → mul_mat + col2im_1d; port col2im per-backend (#155) -- L2761 — Beam-search KV snapshots must stay on-device (#161) -- L3044 — ggml / inference engine -- L3616 — ggml graph allocation: gallocr vs compute_with_ctx -- L4964 — 2026-04-22 - No-gpu mode must gate `ggml_backend_load_all()` -- L7069 — 2026-05-05 — ggml fork patches we carry (must re-apply on every ggml bump) -- L8340 — voxcpm2 perf — per-step ggml graphs, Metal, SIMD layouts -- L8624 — ggml broadcast hides size-mismatch bugs -- L10844 — ggml scheduler tightened cross-backend tensor resolution between §56 and 2026-05-26 (PLAN #115) -- L12098 — Pocket TTS — manual CPU → ggml compute graph rewrite -- L12337 — AudioSeal ggml port — ggml_pad_ext convention trap -- L12414 — GPU weight mirrors for mixed legacy / graph codebases -- L12538 — ggml_graph_get_tensor hash invalidated by ggml_gallocr -- L12731 — read_tensor_f32 weight pre-cache for VITS-family TTS -- L12744 — Single-token embed graph elimination for LLM-ASR -- L13258 — A once-allocated-then-reused bucket step graph is a CUDA-graph-capture hazard: reset+alloc the sched EVERY step -- L13317 — A cached cgraph is invalidated by ANY other graph on its scheduler — reset+alloc per step is NOT enough -- L13417 — TTS code predictor: the dispatch overhead is real, but "fuse into one graph" and "skip the reset" are BOTH wrong — the fix is a sched-free persistent graph -- L13486 — Any T-sized `ggml_view` into a fixed-length weight table must be bounded by the tensor's real length — long inputs overflow it and abort -- L13628 — gallocr does NOT protect input-flagged tensors across computes — re-set EVERY input before EVERY compute of a persistent graph -- L13689 — Vocoder/codec conv graphs hide three graph-construction wastes that dwarf the actual conv FLOPs — profile per-node before assuming it's inherent -- L13736 — Before optimizing a graph's *dispatch*, split host-encode vs GPU-execute — and never trust a GPU benchmark taken under load -- L13831 — A "GPU" model can be silently running on CPU — audit for hot graphs pinned to backend_cpu and CLI adapters that never forward use_gpu -- L17319 — PR #244 dequant-cache fix MERGED +- L516 — Reusing a cached scheduler graph across `sched_reset`/`alloc` cycles is CPU-safe but SIGSEGVs on GPU — the reused input tensor is bound to the prior cycle's freed buffer +- L1086 — Metal-validated GPU TTS graph paths DID transfer cleanly to CUDA — the stricter-CUDA risk is real but not universal +- L1122 — BatchNorm-fold must respect the conv tensor's dtype — and a per-backend fix usually has siblings +- L1156 — "Migrate host KV to device-resident" is a ~1% win on a compute-bound transformer decoder — measure the fraction before the rewrite +- L1456 — Never cache a ggml compute graph across calls that share a scheduler with a larger graph — the allocator regrows and frees the cached graph's GPU buffers +- L1467 — Flow-matching DiT with JointAttention REQUIRES attention masking even for unconditional generation — skipping masked KV tokens silently corrupts output (2026-07) +- L1802 — A cached cgraph is NOT re-entrant with ggml_backend_sched — the 2nd reuse silently corrupts (#208) +- L1844 — On Metal a cached decode graph is dispatch-bound, not alloc-bound — measure the step's parts, don't trust wall time (§210, PR #207) +- L2001 — Precomputed attention masks in a GGUF are dead weight if the runtime rebuilds them (§192) +- L2169 — Cross-attention KV is a free F16 win — encoder-decoder backends (§176i) +- L2197 — Localizing a GPU miscompute with CPU-vs-Metal diffing — the ggml_backend_sched weight-less-first-op trap (§206) +- L2351 — A byte-identical standalone reproducer is the only way to (dis)prove a "ggml bug" (§203) +- L2381 — In-place device-KV decode graphs on Metal: three hazards (§176b+c, §203) +- L2489 — Never pass `ggml_graph_get_tensor` directly to `ggml_backend_tensor_set` (#164) +- L2509 — `rope_theta=0` means "no positional encoding" — skip RoPE entirely (#164) +- L2518 — `ggml_siglu` ≠ PyTorch `glu` — the gate/value halves are SWAPPED (#81) +- L2587 — Asymmetric rel-pos shift uses the same formula as symmetric (#81) +- L2632 — Graph input tensors that nothing consumes are invisible to `ggml_graph_get_tensor` (#164) +- L2665 — `ggml_backend_cpu_set_n_threads` asserts CPU — guard it after `init_best()` +- L2778 — Decompose ConvTranspose1d → mul_mat + col2im_1d; port col2im per-backend (#155) +- L2818 — Beam-search KV snapshots must stay on-device (#161) +- L3101 — ggml / inference engine +- L3673 — ggml graph allocation: gallocr vs compute_with_ctx +- L5021 — 2026-04-22 - No-gpu mode must gate `ggml_backend_load_all()` +- L7126 — 2026-05-05 — ggml fork patches we carry (must re-apply on every ggml bump) +- L8397 — voxcpm2 perf — per-step ggml graphs, Metal, SIMD layouts +- L8681 — ggml broadcast hides size-mismatch bugs +- L10901 — ggml scheduler tightened cross-backend tensor resolution between §56 and 2026-05-26 (PLAN #115) +- L12155 — Pocket TTS — manual CPU → ggml compute graph rewrite +- L12394 — AudioSeal ggml port — ggml_pad_ext convention trap +- L12471 — GPU weight mirrors for mixed legacy / graph codebases +- L12595 — ggml_graph_get_tensor hash invalidated by ggml_gallocr +- L12788 — read_tensor_f32 weight pre-cache for VITS-family TTS +- L12801 — Single-token embed graph elimination for LLM-ASR +- L13315 — A once-allocated-then-reused bucket step graph is a CUDA-graph-capture hazard: reset+alloc the sched EVERY step +- L13374 — A cached cgraph is invalidated by ANY other graph on its scheduler — reset+alloc per step is NOT enough +- L13474 — TTS code predictor: the dispatch overhead is real, but "fuse into one graph" and "skip the reset" are BOTH wrong — the fix is a sched-free persistent graph +- L13543 — Any T-sized `ggml_view` into a fixed-length weight table must be bounded by the tensor's real length — long inputs overflow it and abort +- L13685 — gallocr does NOT protect input-flagged tensors across computes — re-set EVERY input before EVERY compute of a persistent graph +- L13746 — Vocoder/codec conv graphs hide three graph-construction wastes that dwarf the actual conv FLOPs — profile per-node before assuming it's inherent +- L13793 — Before optimizing a graph's *dispatch*, split host-encode vs GPU-execute — and never trust a GPU benchmark taken under load +- L13888 — A "GPU" model can be silently running on CPU — audit for hot graphs pinned to backend_cpu and CLI adapters that never forward use_gpu +- L17376 — PR #244 dequant-cache fix MERGED **GPU portability — Metal / CUDA / Vulkan** (24) -- L1004 — When the full system needs an unavailable resource (model / GPU), factor the risky logic into a pure helper and prove IT on synthetic data -- L1119 — Verify a roadmap "broken/OPEN" claim empirically before implementing it — the codebase may have outgrown the note -- L1199 — A Metal masked-attention-padding penalty does NOT transfer to CUDA, and a bucket-width change is byte-identical on Metal/CPU but not on CUDA -- L1246 — ggml-metal im2col starves batch-1 convs — 3–11 threads/threadgroup at N=1, ~40× below bandwidth -- L1908 — A "garbled GPU output" bug can live entirely downstream — A/B the GPUs before localizing (§192) -- L1962 — MoltenVK `mul_mm`/`mul_mat_vec` downconvert src0 to f16 regardless of stored dtype (§192) -- L1973 — Vulkan has no `REPEAT f16→f16` — cast K/V to F32 *before* the GQA repeat (§192) -- L1986 — On macOS, GGML_METAL is default-ON and silently wins over Vulkan (§192) -- L2036 — §214 Metal batched (B=2) quantized mat-vec ≠ the single-token PREC_F32 path -- L2247 — Metal's q8_0 mat-vec kernel requantizes activations to q8 and ignores the F32 prec hint (§205) -- L2745 — Measuring GPU perf on a thermally-throttling laptop GPU -- L8990 — WDDM idle-clock-state hysteresis on consumer/laptop NVIDIA SKUs -- L9127 — Chatterbox #83 Round 9 — S3Gen UNet GPU drift on Metal -- L10380 — FA per-head additive mask CUDA kernel — what the upstream signature already gave us -- L10998 — funasr CUDA !-loop — all-NaN prefill logits (issue #125, §136) -- L12308 — Cohere flash-attn crossover -- L12368 — conv_transpose_1d GPU TDR — naive loop is O(IL), not O(K/s0) -- L12570 — flash_attn_ext vs scalar attention — numerical divergence -- L12681 — VibeVoice TTS garbles only on AMD RDNA4: coopmat2 flash-attention -- L13866 — Micro-optimizing an already-BLAS'd path with a GPU-ggml port is usually a dud — check for existing Accelerate/cblas and measure encode-vs-execute BEFORE porting -- L14189 — "Backend miscomputes my pipeline" ≠ "op X is broken": arbitrate with test-backend-ops, and beware aggregate precision (#304 native-Vulkan post-mortem) -- L16107 — Chatterbox GPU UNet performance fix -- L16346 — cosyvoice3 \"CUDA test FAIL\" was a registry-alias + flow-quant discovery bug, NOT a CUDA bug -- L17637 — "GPU picks a different token than CPU" is usually NOT a miscompute in an AR audio model — dump the LOGITS, and don't reach for the repetition detector +- L1061 — When the full system needs an unavailable resource (model / GPU), factor the risky logic into a pure helper and prove IT on synthetic data +- L1176 — Verify a roadmap "broken/OPEN" claim empirically before implementing it — the codebase may have outgrown the note +- L1256 — A Metal masked-attention-padding penalty does NOT transfer to CUDA, and a bucket-width change is byte-identical on Metal/CPU but not on CUDA +- L1303 — ggml-metal im2col starves batch-1 convs — 3–11 threads/threadgroup at N=1, ~40× below bandwidth +- L1965 — A "garbled GPU output" bug can live entirely downstream — A/B the GPUs before localizing (§192) +- L2019 — MoltenVK `mul_mm`/`mul_mat_vec` downconvert src0 to f16 regardless of stored dtype (§192) +- L2030 — Vulkan has no `REPEAT f16→f16` — cast K/V to F32 *before* the GQA repeat (§192) +- L2043 — On macOS, GGML_METAL is default-ON and silently wins over Vulkan (§192) +- L2093 — §214 Metal batched (B=2) quantized mat-vec ≠ the single-token PREC_F32 path +- L2304 — Metal's q8_0 mat-vec kernel requantizes activations to q8 and ignores the F32 prec hint (§205) +- L2802 — Measuring GPU perf on a thermally-throttling laptop GPU +- L9047 — WDDM idle-clock-state hysteresis on consumer/laptop NVIDIA SKUs +- L9184 — Chatterbox #83 Round 9 — S3Gen UNet GPU drift on Metal +- L10437 — FA per-head additive mask CUDA kernel — what the upstream signature already gave us +- L11055 — funasr CUDA !-loop — all-NaN prefill logits (issue #125, §136) +- L12365 — Cohere flash-attn crossover +- L12425 — conv_transpose_1d GPU TDR — naive loop is O(IL), not O(K/s0) +- L12627 — flash_attn_ext vs scalar attention — numerical divergence +- L12738 — VibeVoice TTS garbles only on AMD RDNA4: coopmat2 flash-attention +- L13923 — Micro-optimizing an already-BLAS'd path with a GPU-ggml port is usually a dud — check for existing Accelerate/cblas and measure encode-vs-execute BEFORE porting +- L14246 — "Backend miscomputes my pipeline" ≠ "op X is broken": arbitrate with test-backend-ops, and beware aggregate precision (#304 native-Vulkan post-mortem) +- L16164 — Chatterbox GPU UNet performance fix +- L16403 — cosyvoice3 \"CUDA test FAIL\" was a registry-alias + flow-quant discovery bug, NOT a CUDA bug +- L17694 — "GPU picks a different token than CPU" is usually NOT a miscompute in an AR audio model — dump the LOGITS, and don't reach for the repetition detector **Quantization** (10) -- L1463 — Auditing CrispASR against CrispEmbed's bug classes: a weight reader with no quantized branch fails silently (2026-07) -- L1543 — Quantizing a forced-aligner: q8 everywhere (incl. lm_head) is bit-identical; q4 on the encoder is not (#192 TADA aligner) -- L2593 — Orpheus/SNAC TTS: `token_embd` must stay F16 for sub-Q8 quants -- L3195 — Quantisation and memory -- L3470 — Quantization -- L12299 — F16 precision loss in deep WaveNet stacks -- L12316 — MeloTTS + BERT quantization — what works and what doesn't -- L13410 — Re-quantized GGUFs on HF must be re-baked when the quantizer adds carve-outs -- L13498 — Sub-8-bit quantization of an audio tower fails as BEHAVIORAL collapse (loops, empty output) — and there is no per-block cliff to bisect -- L17134 — imatrix quant SHIPPED +- L1520 — Auditing CrispASR against CrispEmbed's bug classes: a weight reader with no quantized branch fails silently (2026-07) +- L1600 — Quantizing a forced-aligner: q8 everywhere (incl. lm_head) is bit-identical; q4 on the encoder is not (#192 TADA aligner) +- L2650 — Orpheus/SNAC TTS: `token_embd` must stay F16 for sub-Q8 quants +- L3252 — Quantisation and memory +- L3527 — Quantization +- L12356 — F16 precision loss in deep WaveNet stacks +- L12373 — MeloTTS + BERT quantization — what works and what doesn't +- L13467 — Re-quantized GGUFs on HF must be re-baked when the quantizer adds carve-outs +- L13555 — Sub-8-bit quantization of an audio tower fails as BEHAVIORAL collapse (loops, empty output) — and there is no per-block cliff to bisect +- L17191 — imatrix quant SHIPPED **Perf measurement & A/B discipline** (8) -- L1137 — Per-step matvec dispatch overhead is LOAD-DEPENDENT — a "win" measured on a busy box is mostly a contention artifact -- L1607 — Best-of-N only helps if the scorer measures what you care about — TADA's reconstruction scorer is blind to duration outliers (#192) -- L3231 — CPU vs ONNX vs PyTorch baselines -- L3674 — Performance: what faster-whisper / insanely-fast-whisper do -- L4971 — 2026-04-23 - FireRed decoder optimization triage -- L10679 — Distinguishing "slow run" from "hung run" — CPU time ≪ wall time is the signal -- L10814 — Cross-backend bug-sweep methodology — pair the cap survey with an empirical A/B -- L12752 — §176 runtime optimization audit methodology +- L1194 — Per-step matvec dispatch overhead is LOAD-DEPENDENT — a "win" measured on a busy box is mostly a contention artifact +- L1664 — Best-of-N only helps if the scorer measures what you care about — TADA's reconstruction scorer is blind to duration outliers (#192) +- L3288 — CPU vs ONNX vs PyTorch baselines +- L3731 — Performance: what faster-whisper / insanely-fast-whisper do +- L5028 — 2026-04-23 - FireRed decoder optimization triage +- L10736 — Distinguishing "slow run" from "hung run" — CPU time ≪ wall time is the signal +- L10871 — Cross-backend bug-sweep methodology — pair the cap survey with an empirical A/B +- L12809 — §176 runtime optimization audit methodology **Long-form audio, VAD & chunking** (8) -- L1368 — LLM-ASR (SALM) instruction-echo on short windows is genuine model behaviour, not a port bug — gate the input, don't chase the prompt -- L1421 — The Mimi codec transformer must be causal — non-causal silently truncates long audio; the >250-frame WER A/B settled it (2026-07) -- L2814 — VAD + chunking -- L3747 — VAD integration and long audio -- L10503 — TDT single-pass over a full long utterance is numerically fragile to codec-level audio noise -- L14611 — #227 "--vad-import wants ggml-tiny" = LID not VAD -- L14833 — #300 streaming diarization SHIPPED -- L15429 — #89 parakeet-ja long-form FIXED +- L1425 — LLM-ASR (SALM) instruction-echo on short windows is genuine model behaviour, not a port bug — gate the input, don't chase the prompt +- L1478 — The Mimi codec transformer must be causal — non-causal silently truncates long audio; the >250-frame WER A/B settled it (2026-07) +- L2871 — VAD + chunking +- L3804 — VAD integration and long audio +- L10560 — TDT single-pass over a full long utterance is numerically fragile to codec-level audio noise +- L14668 — #227 "--vad-import wants ggml-tiny" = LID not VAD +- L14890 — #300 streaming diarization SHIPPED +- L15486 — #89 parakeet-ja long-form FIXED **Tokenizers, prompts, language & text** (17) -- L280 — MOSS-TTS-Local 4B stop runaway was a PROMPT-TOKENIZATION bug — non-compositional BPE, not the forward (#249, 2026-07) -- L330 — Cross-lingual TTS needs the target language plumbed through /v1/audio/speech (#249/#304, 2026-07) -- L1619 — A converter that embeds a tokenizer MUST embed the merges — a silent `try/except: pass` byte-fallback breaks everything downstream (#192) -- L3317 — Language handling -- L5138 — FireRedPunc / fullstop-punc — BERT punctuation restoration -- L6143 — Chatterbox 24 kHz prompt mel — module 4 phase 3 -- L7302 — Text LID via fastText — GlotLID-V3 + LID-176 -- L7513 — Text LID via CLD3 — Google compact language detector -- L12603 — VibeVoice TTS: missing BPE merges still need BPE behavior -- L13133 — 2026-07 — SentencePiece tokenizer taxonomy: greedy longest-match is wrong for BOTH Unigram and BPE -- L13390 — SentencePiece `byte_fallback` is not optional decoration — OOV emoji/symbols must become `<0xHH>` byte tokens, and a `utf8_aligned` Viterbi dead-ends at a multi-byte lead without it -- L13539 — The prompt contract is part of the port: a specials-only tokenizer silently sent EVERY glm-asr prompt instruction-less -- L13574 — Unigram run-length loop metrics pass 2-gram cycles — and a "raw" baseline is only raw if the disable gate actually exists -- L14747 — #249 MOSS-Local 4B stop-runaway = PROMPT tokenization bug -- L15126 — #329 TTS target language -- L16200 — cohere language whitelist + probe LID -- L16318 — CV3 phase 6 speech_tokenizer_v3 ggml port +- L337 — MOSS-TTS-Local 4B stop runaway was a PROMPT-TOKENIZATION bug — non-compositional BPE, not the forward (#249, 2026-07) +- L387 — Cross-lingual TTS needs the target language plumbed through /v1/audio/speech (#249/#304, 2026-07) +- L1676 — A converter that embeds a tokenizer MUST embed the merges — a silent `try/except: pass` byte-fallback breaks everything downstream (#192) +- L3374 — Language handling +- L5195 — FireRedPunc / fullstop-punc — BERT punctuation restoration +- L6200 — Chatterbox 24 kHz prompt mel — module 4 phase 3 +- L7359 — Text LID via fastText — GlotLID-V3 + LID-176 +- L7570 — Text LID via CLD3 — Google compact language detector +- L12660 — VibeVoice TTS: missing BPE merges still need BPE behavior +- L13190 — 2026-07 — SentencePiece tokenizer taxonomy: greedy longest-match is wrong for BOTH Unigram and BPE +- L13447 — SentencePiece `byte_fallback` is not optional decoration — OOV emoji/symbols must become `<0xHH>` byte tokens, and a `utf8_aligned` Viterbi dead-ends at a multi-byte lead without it +- L13596 — The prompt contract is part of the port: a specials-only tokenizer silently sent EVERY glm-asr prompt instruction-less +- L13631 — Unigram run-length loop metrics pass 2-gram cycles — and a "raw" baseline is only raw if the disable gate actually exists +- L14804 — #249 MOSS-Local 4B stop-runaway = PROMPT tokenization bug +- L15183 — #329 TTS target language +- L16257 — cohere language whitelist + probe LID +- L16375 — CV3 phase 6 speech_tokenizer_v3 ggml port **Mel, codecs & the audio front-end** (15) -- L1357 — A one-frame error in a codec ENCODER shifts every RoPE position and shows up as an onset artifact, not a gradual drift — and a padding scheme faithful to one codec can be wrong for another -- L2221 — A glibc-only crash reproduces on macOS under AddressSanitizer — and the bug is often a non-power-of-two FFT (§205) -- L2512 — Streaming conv modules need cached left context, not zero-padding (#81) -- L2690 — FFT size must match upstream exactly (Mini-Omni2 / Whisper mel) -- L2999 — mel / preprocessor -- L3147 — Mel spectrograms -- L3285 — Audio format lessons -- L4256 — Kyutai STT: causal padding, interleaved RoPE, and codec-based ASR -- L4390 — FireRedVAD: FSMN Conv1d replication -- L6191 — Chatterbox atomic native voice clone — the resampler + 5-cond install -- L11168 — moshi / Mimi RVQ codebooks: decode uses embed_sum / cluster_usage -- L12191 — MeloTTS (VITS2) — from zero to BERT conditioning -- L14708 — #245/§232 qwen3-tts CP_DIRECT + codec FASTCONV DONE -- L17033 — glint MP3+AAC TTS output §225 SHIPPED -- L17516 — voxcpm2 CausalTransposeConv1d kwargs gotcha +- L1414 — A one-frame error in a codec ENCODER shifts every RoPE position and shows up as an onset artifact, not a gradual drift — and a padding scheme faithful to one codec can be wrong for another +- L2278 — A glibc-only crash reproduces on macOS under AddressSanitizer — and the bug is often a non-power-of-two FFT (§205) +- L2569 — Streaming conv modules need cached left context, not zero-padding (#81) +- L2747 — FFT size must match upstream exactly (Mini-Omni2 / Whisper mel) +- L3056 — mel / preprocessor +- L3204 — Mel spectrograms +- L3342 — Audio format lessons +- L4313 — Kyutai STT: causal padding, interleaved RoPE, and codec-based ASR +- L4447 — FireRedVAD: FSMN Conv1d replication +- L6248 — Chatterbox atomic native voice clone — the resampler + 5-cond install +- L11225 — moshi / Mimi RVQ codebooks: decode uses embed_sum / cluster_usage +- L12248 — MeloTTS (VITS2) — from zero to BERT conditioning +- L14765 — #245/§232 qwen3-tts CP_DIRECT + codec FASTCONV DONE +- L17090 — glint MP3+AAC TTS output §225 SHIPPED +- L17573 — voxcpm2 CausalTransposeConv1d kwargs gotcha **Process, triage & documentation discipline** (18) -- L252 — When instrumenting the suspect file produces NO output, you are editing the wrong file — look for a second copy -- L345 — "The model emits it inline as text" is a claim to VERIFY, not to document — structured data you never parsed looks identical to a model limitation -- L602 — A positional arg landing on the WRONG parameter is invisible in review and baked into the weights -- L807 — PLAN "OPEN" items are frequently already shipped — audit against the CODE, never the prose -- L1045 — Audit a whole roadmap CLUSTER in one measure-first pass before implementing any of it -- L1351 — Handover docs can be dangerously stale — read the actual Python source, not the handover -- L2100 — DRY shared headers can exist for months before callers migrate (§175) -- L3393 — Regression testing discipline -- L3420 — Specific bugs that cost us a day each -- L3883 — CLI ↔ library DRY refactor -- L5447 — Audit script ≠ behavior test -- L10588 — Long-form ASR has three distinct failure classes, not one (2026-05-25, generalising issue #89) -- L12514 — beam_size default — greedy vs beam-5 -- L12638 — VibeVoice TTS start clicks: distinguish decoder PCM from CLI post-processing -- L13186 — Issue #89 close-out: four transferable lessons -- L13481 — Issue triage discipline: check the codebase before leaving issues open -- L14108 — `git apply --3way` STAGES its result — a later `git add X && git commit` sweeps it up -- L16386 — CrispEmbed #31 WASM OCR e2e +- L309 — When instrumenting the suspect file produces NO output, you are editing the wrong file — look for a second copy +- L402 — "The model emits it inline as text" is a claim to VERIFY, not to document — structured data you never parsed looks identical to a model limitation +- L659 — A positional arg landing on the WRONG parameter is invisible in review and baked into the weights +- L864 — PLAN "OPEN" items are frequently already shipped — audit against the CODE, never the prose +- L1102 — Audit a whole roadmap CLUSTER in one measure-first pass before implementing any of it +- L1408 — Handover docs can be dangerously stale — read the actual Python source, not the handover +- L2157 — DRY shared headers can exist for months before callers migrate (§175) +- L3450 — Regression testing discipline +- L3477 — Specific bugs that cost us a day each +- L3940 — CLI ↔ library DRY refactor +- L5504 — Audit script ≠ behavior test +- L10645 — Long-form ASR has three distinct failure classes, not one (2026-05-25, generalising issue #89) +- L12571 — beam_size default — greedy vs beam-5 +- L12695 — VibeVoice TTS start clicks: distinguish decoder PCM from CLI post-processing +- L13243 — Issue #89 close-out: four transferable lessons +- L13538 — Issue triage discipline: check the codebase before leaving issues open +- L14165 — `git apply --3way` STAGES its result — a later `git add X && git commit` sweeps it up +- L16443 — CrispEmbed #31 WASM OCR e2e **Per-model port notes** (43) -- L100 — A model's capability list cannot be inferred from its vocabulary — and forcing a fake list does not simulate having the capability -- L2705 — Multi-stream token architecture (Mini-Omni2) -- L3355 — Model architecture comparisons -- L4299 — FireRedASR: Conformer encoder debugging -- L5252 — TTS / Vocoder (Chatterbox HiFTGenerator) -- L5514 — Chatterbox-Turbo conformer encoder — ggml layout traps -- L5565 — Chatterbox repaired GGUF split — stage-specific regen -- L5740 — Chatterbox voice cloning — bake to GGUF, load via `--voice` -- L5793 — Chatterbox VoiceEncoder native port — module 2 of voice cloning -- L5899 — Chatterbox S3Tokenizer V2 native port — module 3 of voice cloning -- L6068 — Chatterbox CAMPPlus phase 2 — TDNN forward -- L6964 — T5-family translation runtime traps (May 2026, MADLAD-400 debugging) -- L7670 — IndexTTS-1.5 TTS backend -- L8137 — Speaker verification — TitaNet -- L8185 — Parakeet-TDT greedy decode — blank + duration=0 -- L8251 — VibeVoice 1.5B TTS voice cloning: acoustic + semantic dual encoder -- L8734 — SANM-encoder family (FunASR / SenseVoice / CosyVoice) -- L11127 — Round 10 — SpeechT5 + Dia TTS backend ports (2026-05-31/06-01) -- L11202 — Dia 1.6B TTS — what ACTUALLY fixed it -- L11256 — FastPitch TTS — non-autoregressive parallel TTS port (§133, 2026-06-02/03) -- L11422 — SpeechT5 TTS decoder — what ACTUALLY fixed it -- L11496 — Parler TTS — T5 + MusicGen decoder + DAC 44 kHz -- L11628 — MAES beam search for TDT transducers -- L11730 — Bark TTS — what ACTUALLY fixed it -- L12277 — OpenVoice2 voice cloning — ggml data layout -- L13015 — TADA encoder port: staged model loading on constrained RAM (§221) -- L14329 — #195 parakeet-ctc-1.1b-ja CTC routing -- L14372 — #205 --max-len text-split + granite-plus -- L14489 — #221 irodori voice cloning SHIPPED -- L14652 — #231 cohere-arabic loop = corrupt GGUF -- L14765 — #249 MOSS voice cloning SHIPPED -- L14813 — vibevoice #299 runtime-reproduced -- L14878 — #308 whisper subtitles double-caps -- L15359 — #334 cosyvoice3 clone bugs -- L15497 — #93 voxtral-tts SHIPPED -- L15977 — ARK-ASR-3B port SHIPPED -- L16481 — CSM-1B TTS §135 FIXED -- L16525 — Dia 1.6B TTS port -- L16701 — dots.tts #200 SHIPPED -- L17074 — higgs-audio-v3-stt SHIPPED -- L17205 — indextts long-ref crash FIXED -- L17444 — TADA TTS time embedding bugs -- L17492 — Four bugs that together caused empty/garbage transcripts; all patched in src/vibevoice.cpp +… +- L157 — A model's capability list cannot be inferred from its vocabulary — and forcing a fake list does not simulate having the capability +- L2762 — Multi-stream token architecture (Mini-Omni2) +- L3412 — Model architecture comparisons +- L4356 — FireRedASR: Conformer encoder debugging +- L5309 — TTS / Vocoder (Chatterbox HiFTGenerator) +- L5571 — Chatterbox-Turbo conformer encoder — ggml layout traps +- L5622 — Chatterbox repaired GGUF split — stage-specific regen +- L5797 — Chatterbox voice cloning — bake to GGUF, load via `--voice` +- L5850 — Chatterbox VoiceEncoder native port — module 2 of voice cloning +- L5956 — Chatterbox S3Tokenizer V2 native port — module 3 of voice cloning +- L6125 — Chatterbox CAMPPlus phase 2 — TDNN forward +- L7021 — T5-family translation runtime traps (May 2026, MADLAD-400 debugging) +- L7727 — IndexTTS-1.5 TTS backend +- L8194 — Speaker verification — TitaNet +- L8242 — Parakeet-TDT greedy decode — blank + duration=0 +- L8308 — VibeVoice 1.5B TTS voice cloning: acoustic + semantic dual encoder +- L8791 — SANM-encoder family (FunASR / SenseVoice / CosyVoice) +- L11184 — Round 10 — SpeechT5 + Dia TTS backend ports (2026-05-31/06-01) +- L11259 — Dia 1.6B TTS — what ACTUALLY fixed it +- L11313 — FastPitch TTS — non-autoregressive parallel TTS port (§133, 2026-06-02/03) +- L11479 — SpeechT5 TTS decoder — what ACTUALLY fixed it +- L11553 — Parler TTS — T5 + MusicGen decoder + DAC 44 kHz +- L11685 — MAES beam search for TDT transducers +- L11787 — Bark TTS — what ACTUALLY fixed it +- L12334 — OpenVoice2 voice cloning — ggml data layout +- L13072 — TADA encoder port: staged model loading on constrained RAM (§221) +- L14386 — #195 parakeet-ctc-1.1b-ja CTC routing +- L14429 — #205 --max-len text-split + granite-plus +- L14546 — #221 irodori voice cloning SHIPPED +- L14709 — #231 cohere-arabic loop = corrupt GGUF +- L14822 — #249 MOSS voice cloning SHIPPED +- L14870 — vibevoice #299 runtime-reproduced +- L14935 — #308 whisper subtitles double-caps +- L15416 — #334 cosyvoice3 clone bugs +- L15554 — #93 voxtral-tts SHIPPED +- L16034 — ARK-ASR-3B port SHIPPED +- L16538 — CSM-1B TTS §135 FIXED +- L16582 — Dia 1.6B TTS port +- L16758 — dots.tts #200 SHIPPED +- L17131 — higgs-audio-v3-stt SHIPPED +- L17262 — indextts long-ref crash FIXED +- L17501 — TADA TTS time embedding bugs +- L17549 — Four bugs that together caused empty/garbage transcripts; all patched in src/vibevoice.cpp +… ## Cross-reference by model / family @@ -342,42 +343,42 @@ A section is listed when its heading names the family, or its body mentions it at least 3 times. Sections appear under several families; this is a lookup, not a partition. -- **chatterbox** (25) — L2036, L2221, L5252, L5514, L5565, L5597, L5636, L5740, L5793, L5899, L6002, L6068, L6143, L6191, L8340, L9127, L12752, L13800, L14189, L14249, L14708, L15497, L16107, L16145, L16701 -- **cosyvoice3** (9) — L420, L8734, L12752, L13800, L14249, L15126, L15359, L16318, L16346 -- **vibevoice** (14) — L345, L1496, L2403, L4971, L5740, L8251, L12603, L12638, L12681, L13357, L14813, L14833, L15064, L17492 -- **tada** (9) — L826, L1199, L1543, L1593, L1607, L12752, L13015, L15741, L17444 -- **whisper** (22) — L20, L1712, L2690, L3490, L3674, L3747, L3883, L4390, L5331, L7302, L7670, L12514, L14372, L14878, L14921, L14987, L15429, L15977, L16318, L17074, L17134, L17250 -- **parakeet / TDT** (16) — L553, L1712, L1745, L2814, L3747, L4093, L6191, L8185, L10503, L10588, L11628, L13186, L14329, L14372, L15429, L17250 -- **moss** (13) — L280, L476, L968, L1170, L1345, L1351, L1642, L1667, L1692, L2814, L13992, L14747, L14765 -- **firered** (9) — L1137, L4299, L4390, L4971, L5138, L6002, L13133, L14878, L14921 -- **funasr / sensevoice / SANM** (2) — L8734, L10998 -- **kyutai / moshi / mimi** (7) — L1357, L1421, L4256, L11168, L11942, L12098, L16481 -- **dia** (5) — L1099, L11127, L11202, L11422, L16525 -- **parler** (1) — L11496 -- **speecht5** (2) — L11127, L11422 -- **bark** (1) — L11730 -- **pocket-tts** (3) — L11942, L12098, L17319 -- **melotts / VITS** (5) — L1263, L12191, L12316, L12731, L15799 -- **fastpitch** (1) — L11256 -- **indextts** (1) — L7670 -- **orpheus / SNAC** (1) — L2593 -- **voxcpm2** (4) — L8340, L8544, L8667, L17516 -- **openvoice2** (1) — L12277 -- **glm-asr** (1) — L13539 -- **qwen3** (17) — L1170, L3044, L3747, L4390, L7069, L7670, L8340, L8734, L10716, L13498, L13596, L13689, L14372, L14708, L15126, L17134, L17637 -- **madlad / T5** (1) — L6964 -- **mini-omni2** (2) — L2690, L2705 -- **titanet / speaker-id** (1) — L8137 -- **audioseal** (1) — L12337 -- **cohere** (7) — L100, L3147, L10652, L12308, L14372, L14652, L16200 -- **voxtral** (11) — L3044, L3147, L3195, L3355, L3747, L10588, L10652, L10679, L10814, L15497, L16701 -- **granite** (3) — L1787, L3044, L14372 -- **ark-asr** (2) — L15977, L16701 -- **higgs-stt** (2) — L476, L17074 -- **dots-tts** (2) — L1351, L16701 -- **irodori** (2) — L1228, L14489 -- **f5-tts** (1) — L13831 -- **kokoro** (4) — L193, L7069, L15126, L15799 -- **zonos** (1) — L11496 -- **omniasr / omnivoice** (6) — L20, L923, L4390, L13628, L13656, L14042 -- **glm / mimo / mega** (3) — L10844, L13539, L13596 +- **chatterbox** (25) — L2093, L2278, L5309, L5571, L5622, L5654, L5693, L5797, L5850, L5956, L6059, L6125, L6200, L6248, L8397, L9184, L12809, L13857, L14246, L14306, L14765, L15554, L16164, L16202, L16758 +- **cosyvoice3** (9) — L477, L8791, L12809, L13857, L14306, L15183, L15416, L16375, L16403 +- **vibevoice** (14) — L402, L1553, L2460, L5028, L5797, L8308, L12660, L12695, L12738, L13414, L14870, L14890, L15121, L17549 +- **tada** (9) — L883, L1256, L1600, L1650, L1664, L12809, L13072, L15798, L17501 +- **whisper** (22) — L77, L1769, L2747, L3547, L3731, L3804, L3940, L4447, L5388, L7359, L7727, L12571, L14429, L14935, L14978, L15044, L15486, L16034, L16375, L17131, L17191, L17307 +- **parakeet / TDT** (16) — L610, L1769, L1802, L2871, L3804, L4150, L6248, L8242, L10560, L10645, L11685, L13243, L14386, L14429, L15486, L17307 +- **moss** (13) — L337, L533, L1025, L1227, L1402, L1408, L1699, L1724, L1749, L2871, L14049, L14804, L14822 +- **firered** (9) — L1194, L4356, L4447, L5028, L5195, L6059, L13190, L14935, L14978 +- **funasr / sensevoice / SANM** (2) — L8791, L11055 +- **kyutai / moshi / mimi** (7) — L1414, L1478, L4313, L11225, L11999, L12155, L16538 +- **dia** (5) — L1156, L11184, L11259, L11479, L16582 +- **parler** (1) — L11553 +- **speecht5** (2) — L11184, L11479 +- **bark** (1) — L11787 +- **pocket-tts** (3) — L11999, L12155, L17376 +- **melotts / VITS** (5) — L1320, L12248, L12373, L12788, L15856 +- **fastpitch** (1) — L11313 +- **indextts** (1) — L7727 +- **orpheus / SNAC** (1) — L2650 +- **voxcpm2** (4) — L8397, L8601, L8724, L17573 +- **openvoice2** (1) — L12334 +- **glm-asr** (1) — L13596 +- **qwen3** (17) — L1227, L3101, L3804, L4447, L7126, L7727, L8397, L8791, L10773, L13555, L13653, L13746, L14429, L14765, L15183, L17191, L17694 +- **madlad / T5** (1) — L7021 +- **mini-omni2** (2) — L2747, L2762 +- **titanet / speaker-id** (1) — L8194 +- **audioseal** (1) — L12394 +- **cohere** (7) — L157, L3204, L10709, L12365, L14429, L14709, L16257 +- **voxtral** (11) — L3101, L3204, L3252, L3412, L3804, L10645, L10709, L10736, L10871, L15554, L16758 +- **granite** (3) — L1844, L3101, L14429 +- **ark-asr** (2) — L16034, L16758 +- **higgs-stt** (2) — L533, L17131 +- **dots-tts** (2) — L1408, L16758 +- **irodori** (2) — L1285, L14546 +- **f5-tts** (1) — L13888 +- **kokoro** (4) — L250, L7126, L15183, L15856 +- **zonos** (1) — L11553 +- **omniasr / omnivoice** (6) — L77, L980, L4447, L13685, L13713, L14099 +- **glm / mimo / mega** (3) — L10901, L13596, L13653 diff --git a/scripts/bundle-linux-runtime.sh b/scripts/bundle-linux-runtime.sh index 1cda4922..dafc4f8d 100644 --- a/scripts/bundle-linux-runtime.sh +++ b/scripts/bundle-linux-runtime.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # bundle-linux-runtime.sh — make a staged Linux release directory self-contained. # -# Two jobs, in this order: +# Two jobs, and THE ORDER IS PART OF THE CONTRACT: # -# 1. Rewrite RUNPATH to `$ORIGIN` on every ELF in the directory. -# 2. Copy in every non-system shared library the binaries still need. +# 1. Copy in every non-system shared library the binaries need. +# 2. Rewrite RUNPATH to `$ORIGIN` on every ELF in the directory. # -# Both are required, and step 1 is the one that was missing. The published -# v0.8.25 artifact carried: +# Both are required. Step 2 was the one originally missing: the published +# v0.8.25 artifact carried # # crispasr RUNPATH=$ORIGIN:/home/runner/work/CrispASR/.../c2pa/lib # crispasr-quantize RUNPATH=/home/runner/work/CrispASR/.../c2pa/lib @@ -17,15 +17,36 @@ # `libc2pa_c.so` sitting right beside it. Bundling a library the loader has no # way to look for buys nothing, which is why this script does both. # -# Supersedes the openblas-only `bundle-openblas.sh`: driving the copy from the -# actual `ldd` output picks up libgomp (OpenMP, shipped with gcc and absent on -# minimal installs) and anything a future flag introduces, instead of naming one -# library and rotting the moment a second appears. +# WHY RESOLVE BEFORE REWRITING. The first version did these in the opposite +# order, and that silently dropped exactly the libraries most in need of +# bundling. `ldd` resolves through the binary's own RUNPATH; erasing it first +# turns every such dependency into `=> not found`, which the copy loop then +# filtered out along with the blank lines. The HIP leg of v0.8.27 is what this +# cost: ROCm's clang links OpenMP against `libomp.so` in +# /opt/rocm-6.3.0/lib/llvm/lib, a directory reachable only via the RUNPATH this +# script had just erased, so it was never copied. (gcc's `libgomp.so.1` sits in +# the default loader path, which is why every non-HIP leg was unaffected and the +# bug stayed hidden.) Resolve first, rewrite second, and a dependency's +# discoverability no longer depends on this script's own side effects. +# +# Copied libraries are scanned at their ORIGINAL path, not at the staged copy, +# for the same reason: a `$ORIGIN`-relative RUNPATH means something different +# once the file has moved. +# +# WHY AN UNRESOLVED DEPENDENCY IS FATAL HERE. It used to be silent, and the +# script still reported how many libraries it had bundled — a green line over a +# missing one. Now anything the loader cannot find, and that is not on the +# excluded list below, stops the release. check-bundled-deps.py remains the +# authority on the finished directory (it reads DT_NEEDED rather than trusting +# ldd), but a failure there names a symptom; a failure here names the library +# the bundler could not reach. # # GPU runtimes are deliberately NOT bundled — libcuda/libcudart/libcublas, # libamdhip64/librocblas and libvulkan belong to the host's driver or toolkit -# install. Pass them to check-bundled-deps.py with --allow so the contract is -# recorded rather than assumed. +# install. They are also legitimately absent from CI runners (no driver), so +# they are excluded from the fatal check as well. Pass them to +# check-bundled-deps.py with --allow so the contract is recorded rather than +# assumed. # # Usage: scripts/bundle-linux-runtime.sh set -euo pipefail @@ -38,42 +59,83 @@ command -v patchelf >/dev/null 2>&1 || { is_elf() { head -c 4 "$1" 2>/dev/null | grep -q $'\x7fELF'; } -# ── 1. RUNPATH -> $ORIGIN ──────────────────────────────────────────────────── -# Unconditional: an inherited build-tree RUNPATH is at best useless and at worst -# points somewhere that exists on the build machine only. -for f in "$DEST"/*; do - [ -f "$f" ] || continue +# One policy, consulted by both the copy loop and the unresolved-dependency +# check. Splitting them is how a library ends up excluded from the copy but +# still fatal, or bundled but not required. +skip_lib() { + case "$1" in + libc.so.*|libm.so.*|libdl.so.*|libpthread.so.*|librt.so.*|libutil.so.*) return 0 ;; + libgcc_s.so.*|libstdc++.so.*|libresolv.so.*|ld-linux*) return 0 ;; + libcuda.so.*|libcudart.so.*|libcublas*.so.*|libnv*.so.*) return 0 ;; + libamdhip64.so.*|librocblas.so.*|libhsa*.so.*|libvulkan.so.*) return 0 ;; + esac + return 1 +} + +# ── 1. copy the non-system dependency closure ─────────────────────────────── +# A work queue rather than a single pass: `ldd` gives the transitive closure of +# each file it is pointed at, but a library copied in during the sweep may pull +# in something the original binaries never named directly. +queue=() +while IFS= read -r f; do [ -L "$f" ] && continue is_elf "$f" || continue - before=$(patchelf --print-rpath "$f" 2>/dev/null || echo "") - patchelf --set-rpath '$ORIGIN' "$f" 2>/dev/null || { - echo "bundle-linux-runtime: patchelf failed on $(basename "$f")" >&2; exit 1; } - echo " rpath $(basename "$f"): '${before}' -> '\$ORIGIN'" -done + queue+=("$f") +done < <(find "$DEST" -maxdepth 1 -type f | sort) -# ── 2. copy the non-system dependency closure ─────────────────────────────── -# `ldd` prints the full transitive set already resolved, so one pass per binary -# is enough. Skip the C/C++ runtime (present everywhere) and the GPU runtimes. copied=0 -for f in "$DEST"/*; do - [ -f "$f" ] || continue - [ -L "$f" ] && continue - is_elf "$f" || continue +missing="" +i=0 +while [ "$i" -lt "${#queue[@]}" ]; do + f="${queue[$i]}" + i=$((i + 1)) + ldd_out=$(ldd "$f" 2>/dev/null || true) + + while read -r base; do + [ -n "$base" ] || continue + skip_lib "$base" && continue + # Already staged beside the binary. This is not hypothetical: + # bundle-c2pa.sh drops libc2pa_c.so in before this script runs, and a + # binary whose RUNPATH lacks $ORIGIN (crispasr-quantize, pre-v0.8.26) + # reports it as `not found` even though it is sitting right there — and + # will resolve it once step 2 gives it $ORIGIN. + [ -e "$DEST/$base" ] && continue + case "$missing" in *" $base "*) continue ;; esac + missing="$missing $base " + echo " MISSING $base (needed by $(basename "$f"), unresolvable on this machine)" >&2 + done < <(printf '%s\n' "$ldd_out" | awk '/not found/ {print $1}' | sort -u) + while read -r lib; do [ -n "$lib" ] || continue base=$(basename "$lib") - case "$base" in - libc.so.*|libm.so.*|libdl.so.*|libpthread.so.*|librt.so.*|libutil.so.*) continue ;; - libgcc_s.so.*|libstdc++.so.*|libresolv.so.*|ld-linux*) continue ;; - libcuda.so.*|libcudart.so.*|libcublas*.so.*|libnv*.so.*) continue ;; - libamdhip64.so.*|librocblas.so.*|libhsa*.so.*|libvulkan.so.*) continue ;; - esac + skip_lib "$base" && continue [ -e "$DEST/$base" ] && continue cp -Lf "$lib" "$DEST/$base" - patchelf --set-rpath '$ORIGIN' "$DEST/$base" 2>/dev/null || true echo " bundle $base (from $lib)" copied=$((copied + 1)) - done < <(ldd "$f" 2>/dev/null | awk '/=>/ {print $3}' | grep '^/' | sort -u) + # Scan the source, whose RUNPATH still means what it meant when the + # library was built. + queue+=("$lib") + done < <(printf '%s\n' "$ldd_out" | awk '/=>/ {print $3}' | grep '^/' | sort -u) +done + +if [ -n "$missing" ]; then + echo "bundle-linux-runtime: cannot bundle the dependencies listed above." >&2 + echo "They are neither resolvable on this machine nor host-provided by contract." >&2 + exit 1 +fi + +# ── 2. RUNPATH -> $ORIGIN ──────────────────────────────────────────────────── +# Unconditional, and last: an inherited build-tree RUNPATH is at best useless +# and at worst points somewhere that exists on the build machine only. +for f in "$DEST"/*; do + [ -f "$f" ] || continue + [ -L "$f" ] && continue + is_elf "$f" || continue + before=$(patchelf --print-rpath "$f" 2>/dev/null || echo "") + patchelf --set-rpath '$ORIGIN' "$f" 2>/dev/null || { + echo "bundle-linux-runtime: patchelf failed on $(basename "$f")" >&2; exit 1; } + echo " rpath $(basename "$f"): '${before}' -> '\$ORIGIN'" done -echo "bundle-linux-runtime: $DEST — rpaths normalised, $copied librar(ies) bundled" +echo "bundle-linux-runtime: $DEST — $copied librar(ies) bundled, rpaths normalised" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 26f5dd92..5bc14606 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -4231,6 +4231,31 @@ set_tests_properties(test-dry-run-resolve PROPERTIES LABELS "cli" TIMEOUT 60) +# ─── test-bundle-linux-runtime — the Linux packaging bundler. Registered as a +# unit test on purpose: these scripts previously ran only inside a release job, +# so a defect in them could not be observed without publishing a release, and +# two shipped that way. Needs no models and no GPU — cc + patchelf + ldd, which +# is why it can run on every push. SKIPs (77) off Linux. +add_test(NAME test-bundle-linux-runtime + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/test-bundle-linux-runtime.sh + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) +set_tests_properties(test-bundle-linux-runtime PROPERTIES + LABELS "unit" + SKIP_RETURN_CODE 77 + TIMEOUT 120) + +# ─── test-package-lib-bundle — the libcrispasr bundle packager (#341). Its own +# verifier gates INTRA-bundle resolvability only, and its dlopen tolerance +# exits 0 on the first externally-provided soname, so a dependency missing from +# the bundle entirely was never a question anyone posed. This asks it. +add_test(NAME test-package-lib-bundle + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/test-package-lib-bundle.sh + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) +set_tests_properties(test-package-lib-bundle PROPERTIES + LABELS "unit" + SKIP_RETURN_CODE 77 + TIMEOUT 120) + # ─── test-server-ws-stream — live: --ws-port real-time WebSocket ASR streaming. # Raw-socket client (stdlib only) verifies the RFC 6455 handshake + a streaming # transcription. Needs a whisper ggml model; SKIPs otherwise. diff --git a/tests/test-bundle-linux-runtime.sh b/tests/test-bundle-linux-runtime.sh new file mode 100755 index 00000000..721cddc9 --- /dev/null +++ b/tests/test-bundle-linux-runtime.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# test-bundle-linux-runtime.sh — regression test for scripts/bundle-linux-runtime.sh. +# +# The bug this pins down shipped twice before anyone could see it, because the +# packaging scripts only ever ran inside a release job: `bundle-linux-runtime.sh` +# rewrote RUNPATH to $ORIGIN *before* asking `ldd` what the binaries needed, so +# any dependency reachable only through the binary's own RUNPATH became +# `=> not found` and was quietly filtered out with the blank lines. On the HIP +# leg that was `libomp.so` (ROCm's clang links OpenMP against LLVM's, which +# lives in /opt/rocm/lib/llvm/lib); the tarball failed to package at all, and +# only because check-bundled-deps.py happened to be downstream. +# +# Nothing here needs ROCm, or a GPU, or a release. A private directory plus +# -Wl,-rpath reproduces the exact condition: a library the loader can find only +# via the RUNPATH the bundler is about to erase. +# +# Skips (exit 77) anywhere the tools are missing, so it is inert on macOS. + +set -euo pipefail + +SKIP=77 +[ "$(uname -s)" = "Linux" ] || { echo "SKIP: Linux-only (ELF/patchelf/ldd)"; exit $SKIP; } +for tool in cc patchelf ldd; do + command -v "$tool" >/dev/null 2>&1 || { echo "SKIP: $tool not installed"; exit $SKIP; } +done + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUNDLE="$HERE/../scripts/bundle-linux-runtime.sh" +[ -f "$BUNDLE" ] || { echo "FAIL: no $BUNDLE"; exit 1; } + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +fail() { echo "FAIL: $*" >&2; exit 1; } + +cat > "$WORK/dep.c" <<'EOF' +int crispasr_test_dep(void) { return 42; } +EOF +cat > "$WORK/app.c" <<'EOF' +#include +int crispasr_test_dep(void); +int main(void) { printf("dep=%d\n", crispasr_test_dep()); return 0; } +EOF + +# ── 1. a dependency reachable only via RUNPATH must be bundled ─────────────── +# This is the regression. Pre-fix the bundler reported success and copied +# nothing, and the staged binary could not start once the private dir was gone. +priv="$WORK/private-lib" +stage="$WORK/stage" +mkdir -p "$priv" "$stage" +cc -shared -fPIC -o "$priv/libcrispasrtestdep.so" "$WORK/dep.c" +cc -o "$WORK/app" "$WORK/app.c" -L"$priv" -lcrispasrtestdep -Wl,-rpath,"$priv" +cp "$WORK/app" "$stage/app" + +bash "$BUNDLE" "$stage" > "$WORK/bundle.log" 2>&1 || { + cat "$WORK/bundle.log"; fail "bundler exited nonzero on a resolvable dependency"; } + +[ -f "$stage/libcrispasrtestdep.so" ] || { + cat "$WORK/bundle.log" + fail "libcrispasrtestdep.so was not bundled — the RUNPATH-only dependency was dropped"; } + +rp="$(patchelf --print-rpath "$stage/app")" +[ "$rp" = '$ORIGIN' ] || fail "staged app RUNPATH is '$rp', expected \$ORIGIN" + +# The acceptance test is the binary starting with the original directory gone, +# not the file being present. v0.8.18 shipped bundles that had every dependency +# beside them and still could not load one. +mv "$priv" "$priv.gone" +out="$("$stage/app" 2>&1)" || { echo "$out"; fail "staged app does not run once its build-time libdir is gone"; } +[ "$out" = "dep=42" ] || fail "staged app printed '$out', expected 'dep=42'" +echo " ok: RUNPATH-only dependency bundled, and the relocated binary runs" + +# ── 2. a genuinely unresolvable dependency must fail the release ───────────── +# Without this arm arm 1 would also pass against a bundler that simply reports +# everything as fine. +priv2="$WORK/private-lib2" +stage2="$WORK/stage2" +mkdir -p "$priv2" "$stage2" +cc -shared -fPIC -o "$priv2/libcrispasrtestgone.so" "$WORK/dep.c" +cc -o "$WORK/app2" "$WORK/app.c" -L"$priv2" -lcrispasrtestgone -Wl,-rpath,"$priv2" +cp "$WORK/app2" "$stage2/app2" +rm -rf "$priv2" # the library no longer exists anywhere + +if bash "$BUNDLE" "$stage2" > "$WORK/bundle2.log" 2>&1; then + cat "$WORK/bundle2.log" + fail "bundler reported success with an unresolvable dependency" +fi +grep -q "libcrispasrtestgone.so" "$WORK/bundle2.log" || { + cat "$WORK/bundle2.log"; fail "failure did not name the missing library"; } +echo " ok: an unresolvable dependency fails the bundler, by name" + +# ── 3. host-provided GPU runtimes must NOT fail it ─────────────────────────── +# CI runners have no GPU driver, so `libcuda.so.1 => not found` is the normal +# state of the CUDA legs. If arm 2's check did not consult the same exclusion +# list the copy loop uses, this is where it would take the whole release down. +priv3="$WORK/private-lib3" +stage3="$WORK/stage3" +mkdir -p "$priv3" "$stage3" +cc -shared -fPIC -Wl,-soname,libcuda.so.1 -o "$priv3/libcuda.so.1" "$WORK/dep.c" +cc -o "$WORK/app3" "$WORK/app.c" "$priv3/libcuda.so.1" -Wl,-rpath,"$priv3" +cp "$WORK/app3" "$stage3/app3" +rm -rf "$priv3" + +bash "$BUNDLE" "$stage3" > "$WORK/bundle3.log" 2>&1 || { + cat "$WORK/bundle3.log"; fail "an absent host-provided runtime must not fail the bundler"; } +# `[ … ] && fail` would take the whole script down under `set -e` on the +# success path, since the AND-list itself then returns 1. +if [ -f "$stage3/libcuda.so.1" ]; then fail "libcuda.so.1 must never be bundled"; fi +echo " ok: an absent host-provided runtime is tolerated and not bundled" + +# ── 4. a dependency already staged, but not yet reachable ─────────────────── +# bundle-c2pa.sh drops libc2pa_c.so into the directory before this script runs, +# and crispasr-quantize used to have no $ORIGIN in its RUNPATH at all — so the +# library sits right there and `ldd` still says `not found`. Making an +# unresolved dependency fatal without this exemption would have failed every +# Linux leg. It resolves once step 2 grants $ORIGIN, which is what the run +# check below proves. +priv4="$WORK/private-lib4" +stage4="$WORK/stage4" +mkdir -p "$priv4" "$stage4" +cc -shared -fPIC -o "$priv4/libcrispasrteststaged.so" "$WORK/dep.c" +cc -o "$WORK/app4" "$WORK/app.c" -L"$priv4" -lcrispasrteststaged -Wl,-rpath,"$priv4" +cp "$WORK/app4" "$stage4/app4" +cp "$priv4/libcrispasrteststaged.so" "$stage4/" +rm -rf "$priv4" + +bash "$BUNDLE" "$stage4" > "$WORK/bundle4.log" 2>&1 || { + cat "$WORK/bundle4.log"; fail "a dependency already staged must not be treated as missing"; } +out4="$("$stage4/app4" 2>&1)" || { echo "$out4"; fail "staged app4 does not run"; } +[ "$out4" = "dep=42" ] || fail "staged app4 printed '$out4', expected 'dep=42'" +echo " ok: an already-staged dependency is not missing, and resolves once \$ORIGIN is set" + +echo "PASS: bundle-linux-runtime" diff --git a/tests/test-package-lib-bundle.sh b/tests/test-package-lib-bundle.sh new file mode 100755 index 00000000..4b5c58d3 --- /dev/null +++ b/tests/test-package-lib-bundle.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# test-package-lib-bundle.sh — regression test for tools/package-lib-bundle.sh. +# +# CrispASR #341. The published libcrispasr-linux-x86_64-hip bundle had +# `libomp.so` in DT_NEEDED and did not carry it: ROCm clang's OpenMP runtime, +# which lives in /opt/rocm/lib/llvm/lib and is on no loader search path. Nothing +# caught it, because `verify-lib-bundle.sh` gates INTRA-bundle resolvability — +# for every library the bundle ships, every dependency whose soname is ALSO +# shipped must be reachable via rpath — and a dependency absent from the bundle +# entirely is outside the question it asks. +# +# So this checks the other half: an external dependency reachable only through +# the build-time RUNPATH must end up in the bundle, and the bundle must still +# load once that directory is gone. Needs no GPU and no release. +# +# Skips (exit 77) anywhere the tools are missing, so it is inert on macOS — +# the packaging script's macOS branch is deliberately not changed by #341. + +set -euo pipefail + +SKIP=77 +[ "$(uname -s)" = "Linux" ] || { echo "SKIP: Linux-only (ELF/patchelf/ldd)"; exit $SKIP; } +for tool in cc patchelf ldd python3; do + command -v "$tool" >/dev/null 2>&1 || { echo "SKIP: $tool not installed"; exit $SKIP; } +done + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PKG="$HERE/../tools/package-lib-bundle.sh" +CHECK="$HERE/../scripts/check-bundled-deps.py" +[ -f "$PKG" ] || { echo "FAIL: no $PKG"; exit 1; } + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +fail() { echo "FAIL: $*" >&2; exit 1; } + +cat > "$WORK/ext.c" <<'EOF' +int crispasr_test_ext(void) { return 7; } +EOF +# verify-lib-bundle.sh insists the concrete libcrispasr dlopens AND exports the +# session ABI, so the stand-in has to carry those two symbols. That check is the +# point of the script and is not worth weakening for a test. +cat > "$WORK/main.c" <<'EOF' +int crispasr_test_ext(void); +int crispasr_test_main(void) { return crispasr_test_ext(); } +void *crispasr_session_open_explicit(void) { return 0; } +void crispasr_session_close(void *s) { (void)s; } +EOF +cat > "$WORK/ggml.c" <<'EOF' +int crispasr_test_ggml(void) { return 1; } +EOF + +# A bundle shaped the way release.yml stages one: real libraries in src/ and +# ggml/src/, SOVERSION symlinks beside them, and one dependency living outside +# in a directory only the RUNPATH knows about. +OUT="$WORK/bundle" +priv="$WORK/toolchain-lib" +mkdir -p "$OUT/src" "$OUT/ggml/src" "$priv" + +cc -shared -fPIC -o "$priv/libcrispasrtestext.so" "$WORK/ext.c" +cc -shared -fPIC -Wl,-soname,libcrispasr.so.1 \ + -o "$OUT/src/libcrispasr.so.1.0.0" "$WORK/main.c" \ + -L"$priv" -lcrispasrtestext -Wl,-rpath,"$priv" +ln -s libcrispasr.so.1.0.0 "$OUT/src/libcrispasr.so.1" +ln -s libcrispasr.so.1 "$OUT/src/libcrispasr.so" +cc -shared -fPIC -o "$OUT/ggml/src/libcrispasrtestggml.so" "$WORK/ggml.c" + +bash "$PKG" "$OUT" > "$WORK/pkg.log" 2>&1 || { cat "$WORK/pkg.log"; fail "packaging failed"; } + +# 1. the external dependency is in the bundle +[ -f "$OUT/lib/libcrispasrtestext.so" ] || { + cat "$WORK/pkg.log" + fail "libcrispasrtestext.so was not bundled — the RUNPATH-only dependency was dropped"; } + +# 2. the existing contract is intact: flattened, with src/ and ggml/src/ as +# symlinks, and the SOVERSION aliases still resolving +[ -L "$OUT/src" ] || fail "src/ is not a symlink to lib/" +[ -L "$OUT/ggml/src" ] || fail "ggml/src/ is not a symlink to lib/" +[ -f "$OUT/lib/libcrispasr.so.1.0.0" ] || fail "the real library did not reach lib/" +[ -e "$OUT/lib/libcrispasr.so.1" ] || fail "SOVERSION alias did not survive the flatten" +[ -f "$OUT/lib/libcrispasrtestggml.so" ] || fail "the ggml library did not reach lib/" + +# 3. nothing unbundled is left over. This is the gate release.yml now runs. +rm -rf "$priv" +python3 "$CHECK" "$OUT/lib" > "$WORK/check.log" 2>&1 || { + cat "$WORK/check.log"; fail "check-bundled-deps rejects the packaged bundle"; } + +# 4. and it loads, relocated, with the build-time toolchain dir deleted — +# presence is not resolvability (v0.8.18 shipped bundles that had every +# dependency beside them and one unreachable). +moved="$WORK/relocated" +cp -R "$OUT" "$moved" +python3 - "$moved" <<'PY' || exit 1 +import ctypes, sys +lib = ctypes.CDLL(sys.argv[1] + "/lib/libcrispasr.so.1") +lib.crispasr_test_main.restype = ctypes.c_int +v = lib.crispasr_test_main() +if v != 7: + print(f"FAIL: relocated bundle returned {v}, expected 7"); sys.exit(1) +print(" ok: relocated bundle dlopens and calls through to the bundled dependency") +PY + +echo "PASS: package-lib-bundle" diff --git a/tools/package-lib-bundle.sh b/tools/package-lib-bundle.sh index fcf8e454..05112ffe 100755 --- a/tools/package-lib-bundle.sh +++ b/tools/package-lib-bundle.sh @@ -82,9 +82,22 @@ Darwin) echo " rpaths → @loader_path (macOS), re-signed" ;; Linux) + # Copy the external dependency closure FIRST, while the libraries still + # carry the RUNPATH that can find it (CrispASR #341, and #339 for why the + # order is not a detail). The published HIP bundle needed libomp.so — ROCm + # clang's OpenMP runtime, which lives in /opt/rocm/lib/llvm/lib and is on no + # loader search path — and shipped without it, because nothing here ever + # asked about a dependency the bundle did not already provide. + # + # Safe to point at the flattened lib/: anything the bundle already provides + # is skipped by name, so the ggml sonames the libraries resolve through the + # build tree are not re-copied over the SOVERSION symlinks. + bash "$HERE/../scripts/bundle-linux-runtime.sh" "$OUT/lib" for lib in "$OUT"/lib/*.so*; do [ -f "$lib" ] || continue [ -L "$lib" ] && continue + # Overwrites the plain $ORIGIN the bundler just set: a consumer reaching + # the bundle through the src/ symlink sees $ORIGIN as the symlinked dir. patchelf --set-rpath '$ORIGIN:$ORIGIN/../lib' "$lib" 2>/dev/null || true done echo " rpaths → \$ORIGIN (Linux)" diff --git a/tools/verify-lib-bundle.sh b/tools/verify-lib-bundle.sh index d7e047c4..bec1b6cc 100755 --- a/tools/verify-lib-bundle.sh +++ b/tools/verify-lib-bundle.sh @@ -52,10 +52,23 @@ import ctypes, glob, os, re, subprocess, sys root = sys.argv[1] IS_MAC = sys.platform == "darwin" -# Driver / loader-provided sonames that are correctly NOT in any bundle: GPU -# drivers, plus toolchain and OpenMP runtimes that ship with the compiler/system -# (libomp/libgomp are the LLVM/GNU OpenMP runtimes — same category as libstdc++, -# pulled in by GGML_OPENMP and provided by the user's toolchain, never bundled). +# Driver / loader-provided sonames that may legitimately be absent here. +# +# ⚠ This list is a TOLERANCE, and the dlopen below `sys.exit(0)`s on the FIRST +# match — so anything named here stops the check rather than passing it. That is +# how #341 shipped: the HIP bundle's dlopen failed on `libomp.so`, matched this +# list, printed "dlopen deferred: external driver absent in CI" and exited 0, +# and the bundle went out needing a library that exists only under +# /opt/rocm/lib/llvm/lib. The claim that libomp/libgomp are "provided by the +# user's toolchain" holds for gcc's versioned libgomp.so.1 on the default loader +# path; it does not hold for ROCm clang's unversioned libomp.so. +# +# On Linux those two are now bundled (package-lib-bundle.sh copies the +# dependency closure), so they land in `provided` and the tolerance no longer +# applies to them — the `not in provided` guard below is what makes that work. +# They stay listed for macOS, where OpenMP really does come from the toolchain. +# Add nothing here without asking what a bundle would look like if the entry +# were wrong. EXTERNAL = re.compile( r"^(libcuda|libamdhip64|libhsa-runtime|librocm|libnvidia|libcudart|libcublas|" r"libomp|libgomp|librt|libdl|libpthread|libm|libc|libstdc\+\+|libgcc_s|libSystem)\b"