diff --git a/docs/USAGE.md b/docs/USAGE.md index 345cf8577..9f17e6085 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -331,13 +331,17 @@ In particular, the encoders being present does NOT mean image, keyframe, reference-video or reference-audio conditioning is usable: the video engine still refuses every one of those by name, because the request-side work between a file on disk and a tensor the encoder accepts — image decode, aspect-fill -resize, and the H.264 CRF re-compression upstream performs before encoding — is -not ported. Two encoder-level limits are worth stating in advance because they -are refusals rather than approximations. A reference waveform whose sample rate -differs from the audio VAE's is refused rather than resampled, since upstream -uses a polyphase kaiser resampler this project does not carry. And a VAE -configured with `latent_log_var: none` is refused, because upstream itself -raises on it. +resize, and the H.264 CRF re-compression upstream performs before encoding +whenever the resolved CRF is not `0` and the image is at least 2 pixels on its +shorter side — is not ported. The engine also holds no +encoder to call: it materializes the VAE DECODER key filters only, so no +encoder weights are ever in memory, and the refusal names that rather than +claiming the encoder itself is missing. Two encoder-level limits are worth +stating in advance because they are refusals rather than approximations. A +reference waveform whose sample rate differs from the audio VAE's is refused +rather than resampled, since upstream uses a polyphase kaiser resampler this +project does not carry. And a VAE configured with `latent_log_var: none` is +refused, because upstream itself raises on it. One behaviour is worth stating in advance, because it decides what you get when the entrypoint does arrive. LTX-2.5 ships two video decoders behind one @@ -1557,6 +1561,17 @@ python3 scripts/gen-ltx2-pipeline-goldens.py \ cmake --build build --target test_ltx2_pipeline && ./build/tests/test_ltx2_pipeline ``` +If you regenerate that `.inc` against a moved upstream, expect the goldens to +carry the change rather than only the pin cases. The pipeline goldens reach the +GroupNorm eps and group count in the latent upsampler, the connector's +`rms_norm` eps, the `BlurDownsample` width (on the 1.5 arm only, since the blur +runs on the rational denominator) and the Res2s `sigma_up` clamp — that last one +on the eta = 1 arm, where the clamp binds on every step. A regeneration that +moves one of those constants alone reds a value comparison; one that moves the +constant AND the tensors together passes it, and is caught only by the cases that +compare each constant against upstream's own signature. Both layers are there +deliberately, and neither is redundant. + Recipes resolve on an EXACT `(pipeline_kind, model_version)` pair and refuse anything else by name rather than defaulting, because a plausible but wrong sigma schedule or guidance scale renders a video instead of failing. The pairs that diff --git a/include/vllm/model_executor/models/ltx2.h b/include/vllm/model_executor/models/ltx2.h index 2799e1169..f71a1a404 100644 --- a/include/vllm/model_executor/models/ltx2.h +++ b/include/vllm/model_executor/models/ltx2.h @@ -362,6 +362,14 @@ struct Ltx2AttentionArgs { int64_t context_dim = 0; int64_t heads = 0; int64_t dim_head = 0; + // The q/k RMSNorm eps: `Attention.__init__`'s `norm_eps: float = 1e-6` + // (attention.py:485), handed to both RMSNorms (attention.py:505-506). + // + // This DEFAULT is read by nothing today — every construction of this struct + // assigns it — and a 10^6 mutation of it leaves every suite green. It is a + // latent trap rather than live code, so the only instrument that can hold it is + // the pin in tests/vllm/models/test_ltx2.cpp. Keep it equal to + // `Ltx2DitParams::norm_eps`, which is where every real call site sources it. double norm_eps = 1e-6; Ltx2RopeType rope_type = Ltx2RopeType::kSplit; const Ltx2FreqsCis* pe = nullptr; diff --git a/include/vllm/model_executor/models/ltx2_audio_vae.h b/include/vllm/model_executor/models/ltx2_audio_vae.h index 58751d556..e8823f252 100644 --- a/include/vllm/model_executor/models/ltx2_audio_vae.h +++ b/include/vllm/model_executor/models/ltx2_audio_vae.h @@ -97,7 +97,22 @@ struct Ltx2AudioDecoderConfig { // The decoder's TARGET mel-bin count. 0 keeps whatever the latent carried, // mirroring `mel_bins=None` (audio_vae.py:422). int64_t mel_bins = 0; - // Only read on the GroupNorm arm; PixelNorm has no parameters at all. + // Only read on the GroupNorm arm; PixelNorm has no parameters at all. Neither + // is reachable from a checkpoint: `build_normalization_layer` passes `eps=1e-6` + // as a LITERAL and forwards its own `num_groups` keyword, whose default is 32 + // (normalization.py:44, 56), and no audio_vae call site passes `num_groups`. + // They are fields here so the gate can pin them. + // + // `norm_type = kGroup` is not a dead arm, but it is not what pure defaults give + // you either. `AudioDecoder.__init__` declares `norm_type = GROUP` + // (audio_vae.py:294) and, on the very next line, `causality_axis = WIDTH` + // (audio_vae.py:295) — and `ResnetBlock` refuses that combination with + // `ValueError: Causal ResnetBlock with GroupNorm is not supported` + // (resnet.py:130-131), verified by construction against the pinned upstream. A + // group-norm checkpoint is therefore one that declares `causality_axis: none` + // alongside it, which is legal and is what the group-norm golden in + // test_ltx2_vae.cpp runs. Before that arm existed this eps was never READ on + // any path, and a 100x change moved nothing. int64_t num_groups = 32; double norm_eps = 1e-6; // The audio VAE reaches PixelNorm through `build_normalization_layer`, which @@ -188,12 +203,18 @@ std::vector Ltx2VocoderForward(const Ltx2VocoderConfig& config, // --------------------------------------------------------------------------- // The floor under the BWE mel BEFORE its log: `torch.clamp(mel, min=1e-5)` -// (vocoder.py:516). Named so it can be pinned, because it is the member of the -// invisible-constant class that actually bites in production: it sets the floor of -// the log-mel fed to the bwe_generator, and REAL SILENCE reaches it. A -// reduced-dimension golden built from the deterministic stream cannot, because -// that stream's mel_basis is non-negative and well-scaled and nothing saturates — -// mutation proves 1e-5 -> 1e-8 leaves every tensor golden green. +// (vocoder.py:515). Named so it can be pinned, because it sets the floor of the +// log-mel fed to the bwe_generator and REAL SILENCE reaches it in production. +// +// It is NOT a member of the invisible-constant class described in +// ltx2_video_vae.h, and the line here that said it was is corrected rather than +// carried: the ORDINARY BWE arm's mel_basis is non-negative and well-scaled and +// its raw minimum is ~4.4e-3, so that arm alone cannot move under a mutation. +// "ltx2 vae: the BWE mel log clamp is gated where it actually binds" attenuates +// mel_basis until every bin saturates the floor, and against it 1e-5 -> 1e-8 REDS +// at max|diff| = 0.144965 versus the 5e-6 band. The pin below stays anyway, for +// what no golden can see: a regeneration that moves the constant and the expected +// tensors together. inline constexpr double kLtx2BweMelLogClamp = 1e-5; struct Ltx2VocoderBweConfig { diff --git a/include/vllm/model_executor/models/ltx2_audio_vae_encoder.h b/include/vllm/model_executor/models/ltx2_audio_vae_encoder.h index 020a2db39..7e88063e0 100644 --- a/include/vllm/model_executor/models/ltx2_audio_vae_encoder.h +++ b/include/vllm/model_executor/models/ltx2_audio_vae_encoder.h @@ -90,7 +90,22 @@ struct Ltx2AudioEncoderConfig { bool mid_block_add_attention = true; Ltx2NormType norm_type = Ltx2NormType::kPixel; Ltx2CausalityAxis causality_axis = Ltx2CausalityAxis::kHeight; - // Only read on the GroupNorm arm; PixelNorm is parameter-free. + // Only read on the GroupNorm arm; PixelNorm is parameter-free. Neither is + // reachable from a checkpoint: `build_normalization_layer` passes `eps=1e-6` as + // a LITERAL and forwards its own `num_groups` keyword, whose default is 32 + // (normalization.py:44, 56), and no audio_vae call site passes `num_groups` at + // all — so they are fields only so the gate can pin them. + // + // `norm_type = kGroup` is `AudioEncoder.__init__`'s declared default + // (audio_vae.py:82), but it is NOT what pure defaults give you: the paired + // default is `causality_axis = WIDTH` (audio_vae.py:83), and `ResnetBlock` + // refuses GroupNorm on any causal axis with + // `ValueError: Causal ResnetBlock with GroupNorm is not supported` + // (resnet.py:130-131) — verified by construction against the pinned upstream. + // So a group-norm checkpoint is one that declares `causality_axis: none` + // alongside it, which is legal and is what the group-norm golden in + // test_ltx2_vae.cpp runs. That arm is what stopped this eps from being a + // constant no arm ever read. int64_t num_groups = 32; double norm_eps = 1e-6; // Reached through `build_normalization_layer`, which passes eps=1e-6 diff --git a/include/vllm/model_executor/models/ltx2_connector.h b/include/vllm/model_executor/models/ltx2_connector.h index ba86c901d..9b1a9bab0 100644 --- a/include/vllm/model_executor/models/ltx2_connector.h +++ b/include/vllm/model_executor/models/ltx2_connector.h @@ -62,11 +62,16 @@ namespace vllm { // utils.py:7-12 — `torch.nn.functional.rms_norm`'s eps as `rms_norm` passes it. // The connector uses the WEIGHTLESS form, so this is the only stabilizer in the -// residual path. A member of the invisible-constant class (the fixture's rows are -// never near-zero), so the value comparison does NOT gate it. What gates it is -// test_ltx2_pipeline.cpp, case "the constants the headers call pinned are -// actually pinned", which compares this against upstream's own `rms_norm` -// signature default rather than a retyped literal. +// residual path. NOT a member of the invisible-constant class, however near-zero +// the fixture's rows are: `rms_norm` adds the epsilon to the MEAN SQUARE, not to +// a row minimum, so it perturbs every row it normalizes. At the class's own 100x +// bar (1e-6 -> 1e-4) it REDS 5 of the arms in "ltx2 the Embeddings1DConnector +// reproduces upstream on every arm" — Split 0.0558581, Interleaved 0.104284, +// Float64 0.140343, NoRegisters 0.000542641, GatedNoBias 0.0892045. +// It is pinned as well as gated, in test_ltx2_pipeline.cpp, case "the constants +// the headers call pinned are actually pinned", which compares this against +// upstream's own `rms_norm` signature default rather than a retyped literal — +// the one check a regenerated golden cannot satisfy by moving with it. inline constexpr double kLtx2ConnectorRmsNormEps = 1e-6; // Embeddings1DConnector.__init__ defaults (:95-108), which are also both diff --git a/include/vllm/model_executor/models/ltx2_pipeline.h b/include/vllm/model_executor/models/ltx2_pipeline.h index 8af6875d3..264668416 100644 --- a/include/vllm/model_executor/models/ltx2_pipeline.h +++ b/include/vllm/model_executor/models/ltx2_pipeline.h @@ -171,10 +171,19 @@ struct Ltx2SdeCoeff { double sigma_up = 0.0; }; -// diffusion_steps.py:138. The fallback that keeps `sqrt(sigma_next^2 - sigma_up^2)` -// off zero. Pinned because it never binds on a well-formed schedule (eta <= 1 -// keeps sigma_up <= sigma_next), so no value comparison can see it — a member of -// the invisible-constant class the spec's section 7.0(a) names. +// diffusion_steps.py:138. What keeps `sqrt(sigma_next^2 - sigma_up^2)` off zero, +// and it BINDS on the ordinary eta = 1 schedule rather than only on a malformed +// one. `step` forms `sigma_up = sigma_next * eta`, so eta <= 1 gives +// sigma_up <= sigma_next — but <= includes ==, and at equality `min` takes +// `sigma_next * 0.9999`, which is the whole point: without it the residual is +// exactly 0 and `sigma_down` collapses. The earlier note reasoned from the +// inequality and skipped its boundary, calling the constant invisible; it is not. +// A 1% move (0.9999 -> 0.99) REDS the Eta1 arm the suite already runs, at +// max|diff| = 0.086 (index 0) and 0.130563 (index 1), because the residual scales +// as sqrt(1 - clamp^2) and that is 10x larger at 0.99. The EtaHalf arm stays green +// (0.5 * sigma_next is below the clamp) and Eta1 index 2 stays green +// (sigma_next == 0 returns the denoised prediction unchanged, :181-182). Pinned +// as well, because a regenerated golden would move with the constant. inline constexpr double kLtx2Res2sSigmaUpClamp = 0.9999; Ltx2SdeCoeff Ltx2Res2sSdeCoeff(double sigma_next, double sigma_up); diff --git a/include/vllm/model_executor/models/ltx2_text_encoder.h b/include/vllm/model_executor/models/ltx2_text_encoder.h index c64262197..dc1bb7ad9 100644 --- a/include/vllm/model_executor/models/ltx2_text_encoder.h +++ b/include/vllm/model_executor/models/ltx2_text_encoder.h @@ -63,11 +63,20 @@ // `Ltx2TextFeatureExtractorForward`. // // ─── AND THE EPSILONS, WHICH ARE A CLASS AND NOT ONE INSTANCE ──────────────── -// A constant that only changes the answer on a DEGENERATE input is invisible to -// any golden built from random values. Both normalizations have one, they are -// named below, and each is held two ways: its VALUE against upstream measured by -// probe, and the degenerate input on which it is the only thing between the port -// and a division by zero. When a fourth epsilon arrives, it owes the same pair — +// A constant that only changes the answer on a DEGENERATE input is invisible to a +// golden built from random values — but NEITHER of the two below is that +// constant, and the detailed note at kLtx2TextNormV1Eps already says why for the +// V1 half. Both are additive terms on an O(1) denominator against a 1e-5 band, so +// at the 100x bar they move ORDINARY random-value goldens: V1 1e-6 -> 1e-4 REDS +// "`_norm_and_concat_padded_batch`, both padding sides" at 0.000524044 and +// "FeatureExtractorV1" at 7.53999e-05 / 6.61612e-05; V2 1e-6 -> 1e-4 REDS +// "`norm_and_concat_per_token_rms`, both padding sides" at 0.00232971 and +// carries into "FeatureExtractorV2" and the hand-off at 0.000344872 / +// 0.000259042 / 0.00039053. They are held two ways all the same: their VALUE +// against upstream measured by probe, and the degenerate input on which each is +// the only thing between the port and a division by zero — that input is what +// makes a division-by-zero visible, not what makes the constant visible at all. +// When a fourth epsilon arrives, it owes the same pair — // not a comment saying it matches upstream. // // ─── DTYPE ─────────────────────────────────────────────────────────────────── @@ -161,10 +170,23 @@ struct Ltx2TextHiddenStates { // lands around 2e-12 at the output — small, but not the "any input" the earlier // comment claimed. Both the value and the arithmetic width are gated on the // degenerate inputs, where they are visible. +// +// That is the ONE-ULP dtype perturbation. A VALUE move of the size the pin +// exists to catch is far louder and needs no degenerate input at all: 1e-6 -> +// 1e-4 REDS "`_norm_and_concat_padded_batch`, both padding sides" at 0.000524044 +// and "FeatureExtractorV1" at 7.53999e-05 / 6.61612e-05, on the ordinary +// random-value goldens. inline constexpr double kLtx2TextNormV1Eps = 1e-6; -// feature_extractor.py:61 — `torch.rsqrt(variance + 1e-6)`. Reachable only when a -// token's whole hidden slice is zero. +// feature_extractor.py:61 — `torch.rsqrt(variance + 1e-6)`. DIVISION-BY-ZERO- +// reachable only when a token's whole hidden slice is zero, which is the narrow +// claim the earlier "reachable only when" was making and the wrong one to state +// alone: the epsilon is added to an O(1) variance, so it is OUTPUT-observable on +// ordinary random values too. 1e-6 -> 1e-4 REDS +// "`norm_and_concat_per_token_rms`, both padding sides" at 0.00232971, and the +// perturbation carries through the projections into "FeatureExtractorV2" and +// "the encoder -> conditioning hand-off" at 0.000344872 / 0.000259042 / +// 0.00039053 against the suite's 1e-5 kTol. Same shape as the V1 half above. inline constexpr float kLtx2TextNormV2Eps = 1e-6f; // feature_extractor.py:12-64. Which of the two normalizations runs. diff --git a/include/vllm/model_executor/models/ltx2_upsampler.h b/include/vllm/model_executor/models/ltx2_upsampler.h index 0df83a8f5..f172ba247 100644 --- a/include/vllm/model_executor/models/ltx2_upsampler.h +++ b/include/vllm/model_executor/models/ltx2_upsampler.h @@ -59,10 +59,14 @@ namespace vllm { // literal on all three sites. inline constexpr int64_t kLtx2UpsamplerNormGroups = 32; // torch's `nn.GroupNorm` default `eps` (it is not passed at any of the three -// construction sites). A member of the invisible-constant class: the reduced -// fixture's variances are O(1), so a mutation of it leaves every golden green, -// and it is therefore pinned here against the upstream default rather than by the -// value comparison. +// construction sites). NOT a member of the invisible-constant class. It was +// recorded as one on a mutation that happened not to move anything, and a +// mutation that moves nothing proves nothing; at the class's OWN 100x bar +// (1e-5 -> 1e-3) it REDS all three arms of +// "ltx2 the latent spatial upsampler reproduces upstream" — +// PixelShuffle 0.0289409, Rational2 0.0347079, Rational1p5 0.0649014. The pin +// below still earns its place, because a golden regenerated with a moved eps +// moves with it and only the pin compares against torch's own default. inline constexpr double kLtx2UpsamplerNormEps = 1e-5; // LatentUpsampler.__init__ defaults (model.py:25-35), which are also @@ -99,7 +103,10 @@ std::vector Ltx2BlurKernel(int64_t kernel_size); // `SpatialRationalResampler` never overrides (:38) — so this default IS the // shipped kernel width. Gated against upstream's own signature by // test_ltx2_pipeline.cpp, case "the constants the headers call pinned are -// actually pinned". +// actually pinned", and reached NUMERICALLY as well: 5 -> 3 REDS the Rational1p5 +// arm of "ltx2 the latent spatial upsampler reproduces upstream" at 0.689782. +// Only that arm, because the blur runs on the rational `den` and 1.5 -> {3, 2} is +// the one scale the suite covers with den != 1. inline constexpr int64_t kLtx2BlurKernelSize = 5; // The parameter contract: every tensor `LatentUpsampler(config)` creates, in diff --git a/include/vllm/model_executor/models/ltx2_video_vae.h b/include/vllm/model_executor/models/ltx2_video_vae.h index bd8e5b7db..bfb7022a5 100644 --- a/include/vllm/model_executor/models/ltx2_video_vae.h +++ b/include/vllm/model_executor/models/ltx2_video_vae.h @@ -77,8 +77,10 @@ enum class Ltx2VideoDecoderKind { kConv, kDiffusion }; // normalize, not a mean-square RMS, and not this project's usual rms_norm epsilon. // Named so it can be pinned: mutation proves 1e-12 -> 0.0 leaves every golden // green, because the reduced-dimension activations are O(1) and the floor never -// binds. It still decides whether an all-zero channel vector divides or produces -// NaN. +// binds at that magnitude. It is still READ on every element, so the goldens are +// not blind to it in the other direction — 1e-12 -> 1.0 reds two encoder goldens +// at 0.000525832. And it decides whether an all-zero channel vector divides or +// produces NaN. inline constexpr double kLtx2RmsNorm2dEps = 1e-12; // `config.vae._class_name` -> the decoder kind, mirroring @@ -102,24 +104,63 @@ struct Ltx2VideoDecoderBlock { // ─── THE INVISIBLE-CONSTANT CLASS ──────────────────────────────────────────── // An HONEST LIMIT of these goldens, and it is a CLASS, not one instance. Any // epsilon or floor that exists to stabilize a division is, by construction, -// invisible to a reduced-dimension parity gate: the deterministic stream produces -// O(1) activations, the term it guards never binds, and the tensor comparison -// therefore accepts any value at all — including 0.0, and including one 100x off. -// MEASURED, by mutating each in turn with EVERY golden staying green: +// hard for a reduced-dimension parity gate to reach DOWNWARD: the deterministic +// stream produces O(1) activations, so shrinking the term it guards changes +// nothing the tensor comparison can see. That is the honest form of the claim. +// "Accepts any value at all" is what this paragraph used to say, and it is FALSE +// even of its own members — the epsilon is still READ on every element, so a +// large enough value moves the output. Only a probe that FAILS TO REACH proves +// unreachable; a mutation that happens not to move anything proves nothing, and +// the direction and MAGNITUDE of the mutation are therefore part of the verdict. +// MEASURED, by mutating each in turn, with the bound each number actually holds: // -// Ltx2ConvVideoDecoderConfig::norm_eps 1e-6 -> 1e-4 green -// Ltx2ConvVideoDecoderConfig::pixel_norm_eps 1e-8 -> 1e-6 green -// kLtx2BweMelLogClamp 1e-5 -> 1e-8 green -// kMiniMaxH3SnakeEps 1e-9 -> 0.0 green -// kLtx2RmsNorm2dEps 1e-12 -> 0.0 green +// kMiniMaxH3SnakeEps 1e-9 -> 0.0 green +// kLtx2RmsNorm2dEps 1e-12 -> 0.0 green ...but NOT green upward: +// escalating it to 1.0 REDS "the video ENCODER (*_res family)" and "the video +// encoder CROPS a frame count that is not 1 + k*factor", both at 0.000525832 +// against the 5e-6 band. It never BINDS at the shipped value, and it is read +// regardless — the two are different statements and only the first is true +// of this constant. +// +// `kLtx2BweMelLogClamp` was listed with them and NO LONGER belongs — the third +// entry to leave this list for the same reason, which is why the verdict is now +// stated per-entry with the number that proves it. The arm that made it +// reachable, "ltx2 vae: the BWE mel log clamp is gated where it actually binds", +// landed with the pin itself; the line calling it invisible was written in the +// same change and was false the moment it shipped. 1e-5 -> 1e-8 REDS that arm at +// max|diff| = 0.144965 against the 5e-6 band (36 cases: 34 passed, 2 failed — +// the golden, and the constant assertion below it). What made it look invisible +// was the SCALE of the ordinary arm, not the constant's nature: that arm's raw +// mel minimum is ~4.4e-3 and never approaches the floor, so the reachable arm +// attenuates mel_basis by 1e-4 until every bin lands under it — and asserts the +// saturated-bin count rather than assuming it. +// +// `Ltx2ConvVideoDecoderConfig::pixel_norm_eps` was listed with them and NO LONGER +// belongs. The arm added to make `norm_eps` reachable — "ltx2 vae: the video +// decoder's norm_eps is gated where it BINDS" — runs its latent at a tenth of the +// usual scale, and that makes this epsilon a first-order term too: 1e-8 -> 1e-6 +// now REDS that arm at max|diff| = 1.69305e-04 against the 5e-6 band. The fixture +// built to close one hole closed its neighbour with it, and the line claiming +// otherwise survived the change that falsified it. It stays pinned, in "ltx2 vae: +// the two PixelNorm epsilons stay different", for the reason a pin always earns: +// a regeneration that moves the constant and the goldens together. +// +// `Ltx2ConvVideoDecoderConfig::norm_eps` was listed here and DOES NOT BELONG. It +// is read on every arm that has a `res_x_y` block, PixelNorm included, because +// `norm3` is a GroupNorm built whenever `in_channels != out_channels` +// (resnet.py:93-97) and applied at resnet.py:178. Its 1e-6 -> 1e-4 mutation +// stayed green only because the norm3 in the shipped fixture divides by a +// variance of ~0.2 five blocks deep; at 1e-6 -> 1.0 the same golden moves 1.6e-2. +// That is a sensitivity property of one fixture, not invisibility, and it is now +// gated numerically by a fixture where the epsilon is a first-order term. // // So every member of the class is held by a SOURCE-ANCHORED CONSTANT ASSERTION in // tests/vllm/models/test_ltx2_vae.cpp, cited to the upstream line that sets it, // rather than by the tensor comparison — and a constant that is added later and -// left unpinned is a new hole, not a covered one. The BWE clamp additionally gets -// a golden whose input SATURATES it, because that is the one the reduced-dimension -// stream can be pushed into reaching and the one real silence reaches in -// production. +// left unpinned is a new hole, not a covered one. The three names above that LEFT +// the class keep their assertions as well: their goldens now move under a +// mutation, but a golden still cannot catch a regeneration that shifts the +// constant and the expected tensors together, and only the pin can. struct Ltx2ConvVideoDecoderConfig { // Defaults mirror `_build_conv_video_decoder` // (video_vae/model_configurator.py:81-94). @@ -141,6 +182,14 @@ struct Ltx2ConvVideoDecoderConfig { // `ResnetBlock3D.__init__` declares `eps: float = 1e-6` (video_vae/resnet.py:31) // and hands it to every nn.GroupNorm it builds (resnet.py:44, 65, 94); // `UNetMidBlock3D` carries the same value as `resnet_eps` (resnet.py:216). + // + // norm3 is the reason this is LIVE on a PixelNorm checkpoint too: it is built + // whenever `in_channels != out_channels` (resnet.py:93-97) and applied to the + // residual at resnet.py:178, and `norm_layer` does not gate it. Neither does a + // checkpoint key — `_make_decoder_block` passes `eps=1e-6` / `resnet_eps=1e-6` + // literally (conv_video_decoder.py:78, 103), so this field exists to be pinned + // to that literal, and is gated numerically by the norm_eps arm in + // tests/vllm/models/test_ltx2_vae.cpp. double norm_eps = 1e-6; // `PixelNorm()`'s DEFAULT (normalization.py:22), reached bare from // video_vae/resnet.py:46 and conv_video_decoder.py:243 — NOT the 1e-6 the audio diff --git a/include/vllm/model_executor/models/ltx2_video_vae_encoder.h b/include/vllm/model_executor/models/ltx2_video_vae_encoder.h index e137d7543..65faa1a29 100644 --- a/include/vllm/model_executor/models/ltx2_video_vae_encoder.h +++ b/include/vllm/model_executor/models/ltx2_video_vae_encoder.h @@ -122,6 +122,27 @@ struct Ltx2ConvVideoEncoderConfig { // `eps=1e-6` literally (video_vae.py:56, 66) and `conv_norm_out` takes // `eps=1e-6` (video_vae.py:240). It is a field here only so the gate can pin // it; there is no checkpoint key that moves it. + // + // And norm3 is the reason it is LIVE on a PixelNorm checkpoint here too, for + // the identical reason it is on the decoder's `norm_eps`: `ResnetBlock3D` + // builds `norm3 = nn.GroupNorm(num_groups=1, ..., eps=eps)` whenever + // `in_channels != out_channels` (resnet.py:93-97) and applies it to the + // residual (resnet.py:178), and `norm_layer` does not gate that. So every + // `res_x_y` encoder block reads this value even though `conv_norm_out` and + // `ApplyNorm` take their PixelNorm branches. + // + // Both halves route through ONE line in the port — ltx2_video_vae.cpp:1051,1056 + // reach :405, the same line the decoder reaches from :693,700 — but a SHARED + // LINE IS NOT AN ARGUMENT FOR LIVENESS, and this file previously offered it as + // one. :405 sits behind the `input.channels != out_channels` guard at :400, so + // even entering ResnetBlock3d is not reaching it — `res_x` passes `x.channels` + // as `out_channels` at :1051 and the guard is false. Encoder arm B does not + // reach it at all: all four blocks it holds are plain strided CausalConv3d + // (:1060-1068), so it never enters ResnetBlock3d and stays green under every + // mutation of this value. Liveness is per-arm and MEASURED — the field default + // 1e-6 -> 1e-4 reds two encoder goldens at max|diff| = 4.38839e-05 — which makes + // the numerical coverage real but PARTIAL, and is why the pin still carries the + // arms the goldens do not. double norm_eps = 1e-6; // `PixelNorm()`'s bare DEFAULT (normalization.py:22), same as the decoder's. double pixel_norm_eps = 1e-8; diff --git a/scripts/gen-ltx2-vae-goldens.py b/scripts/gen-ltx2-vae-goldens.py index 1f1151bc1..9dabbb3ee 100644 --- a/scripts/gen-ltx2-vae-goldens.py +++ b/scripts/gen-ltx2-vae-goldens.py @@ -71,6 +71,7 @@ import argparse import math +import re import subprocess import sys from pathlib import Path @@ -80,6 +81,42 @@ _MASK64 = (1 << 64) - 1 +# --------------------------------------------------------------------------- +# THE GOLDEN BAND, READ from the C++ suite rather than repeated here. +# `kLtx2GoldenTol` (tests/vllm/models/test_ltx2_vae.cpp) is the ONE authority on +# what "green" means for these goldens, and section 5d asserts against it before +# emitting — an arm that exists to make a constant reachable is worthless if it +# does not clear the band the C++ side actually applies. A literal `5e-6` here +# would be a second definition of one number in a second language, and the two +# would drift the moment either moved: a widened C++ band would leave this +# generator certifying arms against a band nobody uses, and a tightened one would +# let it emit arms the suite already rejects. +# +# So the value is PARSED from that file, and a parse that does not find EXACTLY +# ONE definition is fatal. An anchor that silently stops matching is how a gate +# goes quiet, which is the failure this whole section of the suite exists to +# prevent. +# --------------------------------------------------------------------------- + +_GOLDEN_TOL_SOURCE = ( + Path(__file__).resolve().parents[1] / "tests" / "vllm" / "models" / "test_ltx2_vae.cpp" +) + + +def _read_golden_tol() -> float: + text = _GOLDEN_TOL_SOURCE.read_text(encoding="utf-8") + hits = re.findall(r"^constexpr double kLtx2GoldenTol = ([0-9eE.+-]+);", text, re.M) + assert len(hits) == 1, ( + f"expected EXACTLY ONE `constexpr double kLtx2GoldenTol = ...;` in " + f"{_GOLDEN_TOL_SOURCE}, found {len(hits)} — the generator cannot assert " + f"against a band it cannot resolve" + ) + return float(hits[0]) + + +GOLDEN_TOL = _read_golden_tol() + + # --------------------------------------------------------------------------- # Deterministic weight/input stream, mirrored bit-for-bit by the C++ suite # (tests/vllm/models/test_ltx2_vae.cpp :: Ltx2Rand). A per-tensor FNV-1a seed plus @@ -262,6 +299,41 @@ def emit_manifest(out, name: str, manifest: list[tuple[str, int]]) -> None: AUDIO_DEC_LATENT_T = 3 AUDIO_DEC_LATENT_F = 2 # z_channels * mel_bins(latent) must equal `ch` (patchified width) +# Sections 1d / 7d — the GROUP-NORM arms, which exist so `norm_eps` is REACHABLE. +# +# `build_normalization_layer` has two branches (common/normalization.py:56-59) and +# every arm above takes the PIXEL one, which is parameter-free and reads +# `pixel_norm_eps`. On those arms `norm_eps` is not merely inert, it is never +# LOADED: nothing in the audio VAE ever touches the GroupNorm branch, so mutating +# the constant 100x moved no golden at all. +# +# `norm_type=group` is legal upstream, but NOT on defaults alone: both +# `AudioEncoder.__init__` and `AudioDecoder.__init__` declare `norm_type = GROUP` +# (audio_vae.py:82, :294) and, on the very next line, `causality_axis = WIDTH` +# (audio_vae.py:83, :295) — a pair `ResnetBlock` refuses with +# `ValueError: Causal ResnetBlock with GroupNorm is not supported` +# (audio_vae/resnet.py:130-131). Constructing either class on pure defaults raises, +# verified by construction against the pinned tree. What a checkpoint may legally +# declare, and what these arms gate, is GROUP alongside `causality_axis=NONE`. +# +# TWO REDUCED DIMENSIONS MOVE, and both are forced, not chosen: +# * `ch` becomes 32 because `build_normalization_layer` forwards its own +# `num_groups` keyword, whose default is 32 (normalization.py:44, 56), and no +# audio_vae call site passes one — so `torch.nn.GroupNorm` refuses any channel +# count that 32 does not divide. Every level is then 32/64/128. +# * `z_channels` becomes 16 because `PerChannelStatistics(latent_channels=ch)` +# indexes the patchified `(c, f)` axis, so `z_channels * mel_bins(latent)` has +# to equal the new `ch` (audio_vae.py:118, 318). +# No other DIMENSION changes; the rest of the diff against the pixel arms is the +# norm type and the causality axis it forces, which is what the arms are for. +AUDIO_GROUP_DEC = {**AUDIO_DEC, "ch": 32, "z_channels": 16} +AUDIO_GROUP_DEC_LATENT_T = 3 +AUDIO_GROUP_DEC_LATENT_F = 2 +# The mel-bin count the network itself produces (latent F doubled once per level +# transition), so `_adjust_output_shape` neither crops nor pads and the golden is +# the decoder's own output. +AUDIO_GROUP_DEC_MEL_BINS = 8 + # Section 2 — BigVGAN v2 vocoder (resblock "AMP1", snakebeta). conv_pre's input is # hardcoded to 128 upstream (2 stereo channels x 64 mel bins), so the reduced arm # keeps 64 mel bins and shrinks everything else. @@ -339,6 +411,44 @@ def emit_manifest(out, name: str, manifest: list[tuple[str, int]]) -> None: ) VIDEO_LATENT = (1, 6, 3, 2, 2) +# Section 5d — the arm on which `norm_eps` is a FIRST-ORDER term. +# +# `norm_eps` is NOT unreachable on the video decoder, and the earlier record that +# called it "invisible" was wrong about the reason. `ResnetBlock3D.__init__` +# builds `norm3 = nn.GroupNorm(num_groups=1, num_channels=in_channels, eps=eps)` +# whenever `in_channels != out_channels` (resnet.py:93-97) — REGARDLESS of +# `norm_layer` — and `forward` applies it to the residual (resnet.py:178). So +# every `res_x_y` block reads the constant even on the PIXEL_NORM arms, and +# section 5 above has one. +# +# What was actually true is a sensitivity statement about the FIXTURE, not a +# reachability statement about the constant. norm3 divides by +# `sqrt(var + eps)` over ALL of (C, T, H, W), and on section 5's arm that +# variance is ~0.2 at a norm3 that sits five blocks deep, so 1e-6 -> 1e-4 moves +# the golden 1.8e-6 — under the 5e-6 band — while 1e-6 -> 1.0 moves it 1.6e-2. +# The band accepted a 100x error only because the denominator was large. +# +# This arm removes that accident instead of recording it. ONE `res_x_y` block, so +# norm3 sits directly behind conv_in, and a latent at a tenth of the usual scale, +# so the variance it competes with is ~5e-3 rather than ~0.2. Every mutation of +# the constant then moves the golden by 1e-5 or more, INCLUDING eps -> 0, which +# no other arm in this file can see. Nothing else about the arm is unusual: the +# weights come from the same stream, the padding mode and causality match section +# 5, and `timestep_conditioning` is off only because the epsilon is what is under +# test and the noise stream is not. +VIDEO_EPS_BLOCKS = [("res_x_y", {"num_layers": 1, "multiplier": 2})] +VIDEO_EPS_DEC = dict( + convolution_dimensions=3, + in_channels=6, + out_channels=3, + patch_size=2, + causal=True, + timestep_conditioning=False, + base_channels=8, +) +VIDEO_EPS_LATENT = (1, 6, 3, 2, 2) +VIDEO_EPS_LATENT_SCALE = 0.1 + def section_audio_decoder(out) -> None: from ltx_core.model.audio_vae.audio_vae import AudioDecoder @@ -447,6 +557,34 @@ def section_audio_decoder(out) -> None: emit_manifest(out, f"kLtx2AudioDec{label}Param", arm_manifest) emit_f32(out, f"kLtx2AudioDec{label}Golden", y_axis.numpy()) + # --- section 1d: norm_type = GROUP, the arm that READS `norm_eps` --- + # See AUDIO_GROUP_DEC for why this arm exists and why its two dimensions move. + # GroupNorm is the only consumer of `norm_eps` in the audio VAE, and it also + # makes `num_groups` load-bearing: the norms carry `weight`/`bias` here, which + # PixelNorm does not, so the parameter manifest differs too. + group = AudioDecoder( + norm_type=NormType.GROUP, + causality_axis=CausalityAxis.NONE, + mel_bins=AUDIO_GROUP_DEC_MEL_BINS, + **AUDIO_GROUP_DEC, + ).eval() + group_manifest = fill_from_stream(group, prefix="ltx2.audiodecgroup.") + group_latent = make_input( + "ltx2.audiodecgroup.input", + (1, AUDIO_GROUP_DEC["z_channels"], AUDIO_GROUP_DEC_LATENT_T, AUDIO_GROUP_DEC_LATENT_F), + 1.0, + ) + y_group = group(group_latent) + out.write("// --- section 1d: norm_type = GROUP (normalization.py:56-57) ---\n") + emit_scalar(out, "kLtx2AudioDecGroupLatentC", AUDIO_GROUP_DEC["z_channels"]) + emit_scalar(out, "kLtx2AudioDecGroupLatentT", AUDIO_GROUP_DEC_LATENT_T) + emit_scalar(out, "kLtx2AudioDecGroupLatentF", AUDIO_GROUP_DEC_LATENT_F) + emit_scalar(out, "kLtx2AudioDecGroupOutFrames", y_group.shape[2]) + emit_scalar(out, "kLtx2AudioDecGroupOutMelBins", y_group.shape[3]) + out.write("\n") + emit_manifest(out, "kLtx2AudioDecGroupParam", group_manifest) + emit_f32(out, "kLtx2AudioDecGroupGolden", y_group.numpy()) + def _vocoder(cfg): from ltx_core.model.audio_vae.vocoder import Vocoder @@ -723,6 +861,62 @@ def noncausal_randn(*args, **kwargs): emit_manifest(out, "kLtx2VideoDecNcParam", noncausal_manifest) emit_f32(out, "kLtx2VideoDecNcGolden", y_nc.numpy()) + # --- section 5d: the arm where `norm_eps` actually BINDS --- + # See VIDEO_EPS_BLOCKS for why this arm exists. No torch.randn patch: with + # `timestep_conditioning=False` and no `inject_noise` block, the decoder draws + # nothing, which the C++ side asserts by requiring an EMPTY draw list. + eps_dec = ConvVideoDecoder( + decoder_blocks=VIDEO_EPS_BLOCKS, + norm_layer=NormLayerType.PIXEL_NORM, + decoder_spatial_padding_mode=PaddingModeType.REFLECT, + **VIDEO_EPS_DEC, + ).eval() + eps_manifest = fill_from_stream(eps_dec, prefix="ltx2.videodeceps.") + eps_latent = make_input("ltx2.videodeceps.input", VIDEO_EPS_LATENT, VIDEO_EPS_LATENT_SCALE) + + norm3 = [m for m in eps_dec.modules() if isinstance(m, torch.nn.GroupNorm)] + assert len(norm3) == 1, ( + f"the eps arm must have EXACTLY ONE nn.GroupNorm (norm3), found {len(norm3)}" + ) + assert norm3[0].eps == 1e-6, "norm3 must carry the constant this arm gates" + y_eps = eps_dec(eps_latent) + + # PROVEN SENSITIVE, not assumed — the same discipline section 5b applies to its + # causality probe and section 4b to the mel clamp. eps -> 0 is the mutation the + # OTHER arms are blindest to (section 5's golden moves 5.4e-7 under it, a tenth + # of the band), so it is the one this arm has to catch. + norm3[0].eps = 0.0 + try: + y_zero = eps_dec(eps_latent) + finally: + norm3[0].eps = 1e-6 + zero_move = float((y_eps - y_zero).abs().max()) + assert zero_move > 10 * GOLDEN_TOL, ( + f"the eps arm must move well past the {GOLDEN_TOL:g} band when the " + f"constant is removed, moved only {zero_move:g}" + ) + print(f"[eps arm] norm3 in {tuple(y_eps.shape)}; eps 1e-6 -> 0 moves {zero_move:g}", + file=sys.stderr) + + out.write("// --- section 5d: the arm on which `norm_eps` BINDS (resnet.py:93-97) ---\n") + emit_scalar(out, "kLtx2VideoDecEpsLatentC", VIDEO_EPS_LATENT[1]) + emit_scalar(out, "kLtx2VideoDecEpsLatentT", VIDEO_EPS_LATENT[2]) + emit_scalar(out, "kLtx2VideoDecEpsLatentH", VIDEO_EPS_LATENT[3]) + emit_scalar(out, "kLtx2VideoDecEpsLatentW", VIDEO_EPS_LATENT[4]) + out.write("inline constexpr double kLtx2VideoDecEpsLatentScale = " + + _cxx_float(VIDEO_EPS_LATENT_SCALE, 17) + ";\n") + emit_scalar(out, "kLtx2VideoDecEpsOutC", y_eps.shape[1]) + emit_scalar(out, "kLtx2VideoDecEpsOutT", y_eps.shape[2]) + emit_scalar(out, "kLtx2VideoDecEpsOutH", y_eps.shape[3]) + emit_scalar(out, "kLtx2VideoDecEpsOutW", y_eps.shape[4]) + # How far the golden travels when the constant is REMOVED, measured on the + # oracle. The C++ arm requires it to clear the band by a wide margin, so the + # sensitivity this arm exists for is gated rather than narrated. + out.write("inline constexpr double kLtx2VideoDecEpsZeroMove = " + + _cxx_float(zero_move, 9) + ";\n\n") + emit_manifest(out, "kLtx2VideoDecEpsParam", eps_manifest) + emit_f32(out, "kLtx2VideoDecEpsGolden", y_eps.numpy()) + # --------------------------------------------------------------------------- # Sections 6-8 — the ENCODER halves (phase L11), which L4 recorded as owed. @@ -800,6 +994,13 @@ def noncausal_randn(*args, **kwargs): ) AUDIO_ENC_FRAMES = 8 AUDIO_ENC_MEL = 8 +# Section 7d — the encoder's GROUP-NORM arm. Same two forced dimension changes as +# AUDIO_GROUP_DEC, for the same two reasons: `build_normalization_layer`'s own +# `num_groups` default is 32 and nothing overrides it (normalization.py:44, 56), +# and `z_channels * mel_bins(latent)` must equal `ch` +# because `PerChannelStatistics` indexes the patchified `(c, f)` axis. The encoder +# reaches its deepest level at 8 -> 4 -> 2 mel bins, so 16 * 2 = 32. +AUDIO_GROUP_ENC = {**AUDIO_ENC, "ch": 32, "z_channels": 16} # Section 8 — the mel front-end. n_fft is small so the direct DFT the C++ side # uses stays cheap, but every parameter that decides a value is preserved. @@ -1027,6 +1228,26 @@ def build(prefix, **overrides): emit_manifest(out, "kLtx2AudioEncPoolParam", manifest_pool) emit_f32(out, "kLtx2AudioEncPoolGolden", y_pool.numpy()) + # --- 7d: norm_type = GROUP, the arm that READS `norm_eps` on the encoder half. + # See AUDIO_GROUP_ENC. `ResnetBlock` only permits GroupNorm at causality NONE + # (resnet.py:130-131), which is a configuration a checkpoint may legally + # declare and which the pixel arms above leave entirely unexecuted. + enc_group = AudioEncoder( + norm_type=NormType.GROUP, + causality_axis=CausalityAxis.NONE, + attn_type=AttentionType.VANILLA, + **AUDIO_GROUP_ENC, + ).eval() + manifest_group = fill_from_stream(enc_group, prefix="ltx2.audioencgroup.") + y_group = enc_group(spectrogram) + out.write("// --- section 7d: AudioEncoder, norm_type = GROUP, causality NONE ---\n") + emit_scalar(out, "kLtx2AudioEncGroupOutC", y_group.shape[1]) + emit_scalar(out, "kLtx2AudioEncGroupOutT", y_group.shape[2]) + emit_scalar(out, "kLtx2AudioEncGroupOutF", y_group.shape[3]) + out.write("\n") + emit_manifest(out, "kLtx2AudioEncGroupParam", manifest_group) + emit_f32(out, "kLtx2AudioEncGroupGolden", y_group.numpy()) + def section_audio_mel(out) -> None: import torch diff --git a/src/vllm/multimodal/ltx2_video.cpp b/src/vllm/multimodal/ltx2_video.cpp index 743c19e9d..42f7a8897 100644 --- a/src/vllm/multimodal/ltx2_video.cpp +++ b/src/vllm/multimodal/ltx2_video.cpp @@ -668,17 +668,26 @@ VideoResult Ltx2VideoEngine::Generate(const VideoGenParams& gen) { std::string(kLtx2AudioPromptEmbedsExtra) + "' extra"); } // Image / reference conditioning is `ImageConditioner` upstream - // (distilled.py:245-258) and needs the video VAE's ENCODER, which phase L4 - // ported the DECODER of. Refused by name rather than dropped: a keyframe that - // is silently ignored renders an unconditioned clip that looks like the + // (ltx-pipelines/utils/blocks.py:936-993, called at distilled.py:212). The + // ENCODER it needs is no longer what is missing — phase L11 ported it as + // `Ltx2ConvVideoEncode` — so the refusal names what actually is: this engine + // holds no encoder to call. Refused by name rather than dropped: a keyframe + // that is silently ignored renders an unconditioned clip that looks like the // feature not working. if (!gen.first_frame_path.empty() || !gen.first_frame_ppm.empty() || !gen.last_frame_path.empty() || !gen.ref_image_paths.empty() || !gen.ref_video_dir.empty() || !gen.ref_audio_path.empty() || !gen.ref_audio_wav.empty()) { Fail( - "keyframe / reference conditioning is not ported for this family: it runs through " - "upstream's ImageConditioner (distilled.py:245-258), which encodes the images with " - "the video VAE's ENCODER, and phase L4 ported the DECODER only. Recorded as owed."); + "keyframe / reference conditioning is not ported for this family. The video VAE " + "ENCODER itself landed in phase L11 (Ltx2ConvVideoEncode), but nothing can reach it " + "from here: this engine materializes the DECODER key filter only, so no " + "VAE_ENCODER_COMFY_KEYS_FILTER / VideoEncoderConfigurator path " + "(video_vae/model_configurator.py:72, 267) puts encoder weights in memory, and " + "upstream resolves each image conditioning's CRF against the checkpoint's " + "default_image_crf when the caller left it unset (ImageConditioner.resolve_crf, " + "ltx-pipelines/utils/blocks.py:977-983) and then re-compresses through an H.264 " + "round trip unless that CRF is 0 (media_io/decode.py:413-435, from " + "load_image_and_preprocess :75), which this build does not do. Recorded as owed."); } // ── geometry ────────────────────────────────────────────────────────────── diff --git a/tests/vllm/models/ltx2_vae_goldens.inc b/tests/vllm/models/ltx2_vae_goldens.inc index 93ce8b246..aeb670c31 100644 --- a/tests/vllm/models/ltx2_vae_goldens.inc +++ b/tests/vllm/models/ltx2_vae_goldens.inc @@ -581,6 +581,174 @@ inline constexpr float kLtx2AudioDecWidthCompatGolden[] = { -0.425801784f, -0.165738866f, -0.35139966f, 0.229771987f, 0.17241101f, 0.109865874f, }; +// --- section 1d: norm_type = GROUP (normalization.py:56-57) --- +inline constexpr int64_t kLtx2AudioDecGroupLatentC = 16; +inline constexpr int64_t kLtx2AudioDecGroupLatentT = 3; +inline constexpr int64_t kLtx2AudioDecGroupLatentF = 2; +inline constexpr int64_t kLtx2AudioDecGroupOutFrames = 12; +inline constexpr int64_t kLtx2AudioDecGroupOutMelBins = 8; + +inline constexpr const char* kLtx2AudioDecGroupParamNames[] = { + "ltx2.audiodecgroup.per_channel_statistics.std-of-means", + "ltx2.audiodecgroup.per_channel_statistics.mean-of-means", + "ltx2.audiodecgroup.conv_in.conv.weight", + "ltx2.audiodecgroup.conv_in.conv.bias", + "ltx2.audiodecgroup.mid.block_1.norm1.weight", + "ltx2.audiodecgroup.mid.block_1.norm1.bias", + "ltx2.audiodecgroup.mid.block_1.conv1.conv.weight", + "ltx2.audiodecgroup.mid.block_1.conv1.conv.bias", + "ltx2.audiodecgroup.mid.block_1.norm2.weight", + "ltx2.audiodecgroup.mid.block_1.norm2.bias", + "ltx2.audiodecgroup.mid.block_1.conv2.conv.weight", + "ltx2.audiodecgroup.mid.block_1.conv2.conv.bias", + "ltx2.audiodecgroup.mid.attn_1.norm.weight", + "ltx2.audiodecgroup.mid.attn_1.norm.bias", + "ltx2.audiodecgroup.mid.attn_1.q.weight", + "ltx2.audiodecgroup.mid.attn_1.q.bias", + "ltx2.audiodecgroup.mid.attn_1.k.weight", + "ltx2.audiodecgroup.mid.attn_1.k.bias", + "ltx2.audiodecgroup.mid.attn_1.v.weight", + "ltx2.audiodecgroup.mid.attn_1.v.bias", + "ltx2.audiodecgroup.mid.attn_1.proj_out.weight", + "ltx2.audiodecgroup.mid.attn_1.proj_out.bias", + "ltx2.audiodecgroup.mid.block_2.norm1.weight", + "ltx2.audiodecgroup.mid.block_2.norm1.bias", + "ltx2.audiodecgroup.mid.block_2.conv1.conv.weight", + "ltx2.audiodecgroup.mid.block_2.conv1.conv.bias", + "ltx2.audiodecgroup.mid.block_2.norm2.weight", + "ltx2.audiodecgroup.mid.block_2.norm2.bias", + "ltx2.audiodecgroup.mid.block_2.conv2.conv.weight", + "ltx2.audiodecgroup.mid.block_2.conv2.conv.bias", + "ltx2.audiodecgroup.up.0.block.0.norm1.weight", + "ltx2.audiodecgroup.up.0.block.0.norm1.bias", + "ltx2.audiodecgroup.up.0.block.0.conv1.conv.weight", + "ltx2.audiodecgroup.up.0.block.0.conv1.conv.bias", + "ltx2.audiodecgroup.up.0.block.0.norm2.weight", + "ltx2.audiodecgroup.up.0.block.0.norm2.bias", + "ltx2.audiodecgroup.up.0.block.0.conv2.conv.weight", + "ltx2.audiodecgroup.up.0.block.0.conv2.conv.bias", + "ltx2.audiodecgroup.up.0.block.0.nin_shortcut.conv.weight", + "ltx2.audiodecgroup.up.0.block.0.nin_shortcut.conv.bias", + "ltx2.audiodecgroup.up.0.block.1.norm1.weight", + "ltx2.audiodecgroup.up.0.block.1.norm1.bias", + "ltx2.audiodecgroup.up.0.block.1.conv1.conv.weight", + "ltx2.audiodecgroup.up.0.block.1.conv1.conv.bias", + "ltx2.audiodecgroup.up.0.block.1.norm2.weight", + "ltx2.audiodecgroup.up.0.block.1.norm2.bias", + "ltx2.audiodecgroup.up.0.block.1.conv2.conv.weight", + "ltx2.audiodecgroup.up.0.block.1.conv2.conv.bias", + "ltx2.audiodecgroup.up.1.block.0.norm1.weight", + "ltx2.audiodecgroup.up.1.block.0.norm1.bias", + "ltx2.audiodecgroup.up.1.block.0.conv1.conv.weight", + "ltx2.audiodecgroup.up.1.block.0.conv1.conv.bias", + "ltx2.audiodecgroup.up.1.block.0.norm2.weight", + "ltx2.audiodecgroup.up.1.block.0.norm2.bias", + "ltx2.audiodecgroup.up.1.block.0.conv2.conv.weight", + "ltx2.audiodecgroup.up.1.block.0.conv2.conv.bias", + "ltx2.audiodecgroup.up.1.block.0.nin_shortcut.conv.weight", + "ltx2.audiodecgroup.up.1.block.0.nin_shortcut.conv.bias", + "ltx2.audiodecgroup.up.1.block.1.norm1.weight", + "ltx2.audiodecgroup.up.1.block.1.norm1.bias", + "ltx2.audiodecgroup.up.1.block.1.conv1.conv.weight", + "ltx2.audiodecgroup.up.1.block.1.conv1.conv.bias", + "ltx2.audiodecgroup.up.1.block.1.norm2.weight", + "ltx2.audiodecgroup.up.1.block.1.norm2.bias", + "ltx2.audiodecgroup.up.1.block.1.conv2.conv.weight", + "ltx2.audiodecgroup.up.1.block.1.conv2.conv.bias", + "ltx2.audiodecgroup.up.1.upsample.conv.conv.weight", + "ltx2.audiodecgroup.up.1.upsample.conv.conv.bias", + "ltx2.audiodecgroup.up.2.block.0.norm1.weight", + "ltx2.audiodecgroup.up.2.block.0.norm1.bias", + "ltx2.audiodecgroup.up.2.block.0.conv1.conv.weight", + "ltx2.audiodecgroup.up.2.block.0.conv1.conv.bias", + "ltx2.audiodecgroup.up.2.block.0.norm2.weight", + "ltx2.audiodecgroup.up.2.block.0.norm2.bias", + "ltx2.audiodecgroup.up.2.block.0.conv2.conv.weight", + "ltx2.audiodecgroup.up.2.block.0.conv2.conv.bias", + "ltx2.audiodecgroup.up.2.block.1.norm1.weight", + "ltx2.audiodecgroup.up.2.block.1.norm1.bias", + "ltx2.audiodecgroup.up.2.block.1.conv1.conv.weight", + "ltx2.audiodecgroup.up.2.block.1.conv1.conv.bias", + "ltx2.audiodecgroup.up.2.block.1.norm2.weight", + "ltx2.audiodecgroup.up.2.block.1.norm2.bias", + "ltx2.audiodecgroup.up.2.block.1.conv2.conv.weight", + "ltx2.audiodecgroup.up.2.block.1.conv2.conv.bias", + "ltx2.audiodecgroup.up.2.attn.0.norm.weight", + "ltx2.audiodecgroup.up.2.attn.0.norm.bias", + "ltx2.audiodecgroup.up.2.attn.0.q.weight", + "ltx2.audiodecgroup.up.2.attn.0.q.bias", + "ltx2.audiodecgroup.up.2.attn.0.k.weight", + "ltx2.audiodecgroup.up.2.attn.0.k.bias", + "ltx2.audiodecgroup.up.2.attn.0.v.weight", + "ltx2.audiodecgroup.up.2.attn.0.v.bias", + "ltx2.audiodecgroup.up.2.attn.0.proj_out.weight", + "ltx2.audiodecgroup.up.2.attn.0.proj_out.bias", + "ltx2.audiodecgroup.up.2.attn.1.norm.weight", + "ltx2.audiodecgroup.up.2.attn.1.norm.bias", + "ltx2.audiodecgroup.up.2.attn.1.q.weight", + "ltx2.audiodecgroup.up.2.attn.1.q.bias", + "ltx2.audiodecgroup.up.2.attn.1.k.weight", + "ltx2.audiodecgroup.up.2.attn.1.k.bias", + "ltx2.audiodecgroup.up.2.attn.1.v.weight", + "ltx2.audiodecgroup.up.2.attn.1.v.bias", + "ltx2.audiodecgroup.up.2.attn.1.proj_out.weight", + "ltx2.audiodecgroup.up.2.attn.1.proj_out.bias", + "ltx2.audiodecgroup.up.2.upsample.conv.conv.weight", + "ltx2.audiodecgroup.up.2.upsample.conv.conv.bias", + "ltx2.audiodecgroup.norm_out.weight", + "ltx2.audiodecgroup.norm_out.bias", + "ltx2.audiodecgroup.conv_out.conv.weight", + "ltx2.audiodecgroup.conv_out.conv.bias", +}; +inline constexpr int64_t kLtx2AudioDecGroupParamCounts[] = { + 32, 32, 18432, 128, 128, 128, 147456, 128, 128, 128, + 147456, 128, 128, 128, 16384, 128, 16384, 128, 16384, 128, + 16384, 128, 128, 128, 147456, 128, 128, 128, 147456, 128, + 64, 64, 18432, 32, 32, 32, 9216, 32, 2048, 32, + 32, 32, 9216, 32, 32, 32, 9216, 32, 128, 128, + 73728, 64, 64, 64, 36864, 64, 8192, 64, 64, 64, + 36864, 64, 64, 64, 36864, 64, 36864, 64, 128, 128, + 147456, 128, 128, 128, 147456, 128, 128, 128, 147456, 128, + 128, 128, 147456, 128, 128, 128, 16384, 128, 16384, 128, + 16384, 128, 16384, 128, 128, 128, 16384, 128, 16384, 128, + 16384, 128, 16384, 128, 147456, 128, 32, 32, 576, 2, +}; + +inline constexpr float kLtx2AudioDecGroupGolden[] = { + 0.254412442f, 0.49378264f, 0.0873959363f, 0.348718852f, -0.253762603f, -0.489824593f, + -0.300619036f, -0.463318408f, 0.149628371f, 1.16802025f, 1.48224032f, 1.0298543f, + -0.0187939089f, -0.819010794f, -0.37197268f, -0.389468431f, 0.401003003f, 0.97302568f, + 1.20360398f, 1.06986356f, 0.0523178875f, -0.314367473f, -1.14971173f, -0.182393938f, + 0.485824734f, 1.26803982f, 1.27902699f, 0.942487121f, 0.19047153f, -0.184167236f, + -1.08607805f, -0.491619676f, 0.403063893f, 0.491142809f, 0.64327538f, 0.567441344f, + -0.443947434f, 0.313590586f, 0.160350904f, 0.0115117226f, 0.208626717f, 0.230058447f, + 0.604961634f, 0.0561831966f, -0.670129716f, -0.210171491f, -0.0955527574f, -0.137300774f, + 0.602567255f, 0.461462826f, -0.208059341f, -0.578993678f, -1.04529738f, -0.812146187f, + -0.284006566f, -0.29690671f, 0.258820534f, 0.408568859f, -0.714232922f, -0.216012269f, + -0.711150706f, -0.947372496f, -0.385223389f, -0.283973873f, 0.134870112f, 0.228138492f, + -1.03829408f, -1.07293987f, -0.841648579f, -1.29970074f, -0.195218742f, 0.224846989f, + 0.246901765f, 0.588590562f, -0.687967956f, -0.549966812f, -1.3084718f, -1.51328862f, + -0.223140895f, 0.314098179f, -0.35306868f, 0.212790459f, -0.341865778f, -0.344777197f, + -0.125321314f, -0.124309748f, 0.224438816f, 0.432502985f, 0.21273914f, 0.695779979f, + 0.363164991f, -0.0352303199f, 0.161505416f, 0.0672856122f, -0.140146047f, 0.206123248f, + -0.0863174573f, 0.656345129f, 0.408471972f, 0.299332559f, 0.0457959026f, -0.059676446f, + 0.319880575f, -0.100198418f, 0.227355108f, 1.12942576f, 0.687801898f, 0.695969284f, + 0.0259915255f, -0.0682759956f, 1.12081027f, -0.262945384f, -0.102140926f, 0.89244324f, + 0.942369938f, 1.13111162f, 0.292847127f, 0.529573977f, 0.953980505f, -0.483644307f, + 0.00290121161f, 0.378790051f, 0.708401501f, 0.809208333f, 1.0562253f, 0.502600849f, + 0.473319173f, -0.307291478f, -0.303855419f, -0.292235255f, -0.07925082f, 0.19784902f, + 0.920972288f, -0.0381647311f, 0.397911161f, 0.0606064051f, -0.259182662f, -0.0612378754f, + 0.466430038f, 0.00969763845f, 0.119928256f, -0.981307805f, -0.573017716f, -0.0237864982f, + 0.105015419f, 0.128595352f, 1.0032419f, 0.79477042f, 0.482512534f, -1.01709831f, + -1.1132195f, -0.297602236f, 0.100264065f, 0.0468156487f, 0.760492325f, 0.570661664f, + 0.38296774f, -0.424678147f, -0.778518498f, -0.110242471f, 0.170706347f, 0.172453597f, + 0.27163747f, -0.17842795f, -0.366048217f, 0.148340106f, -0.596942127f, -0.0184304342f, + 0.379511833f, -0.119235635f, -0.184574112f, -0.369306892f, -0.113753468f, -0.117913209f, + -0.248044834f, 0.198247313f, 0.0500604808f, -0.462488145f, -0.028434718f, 0.517962575f, + 0.283285022f, -0.225237638f, -0.362094849f, 0.0134697817f, 0.381380945f, 0.314627439f, + 0.202479482f, 0.0234818589f, -0.717592657f, -0.905055881f, -0.871151805f, -0.341808558f, +}; + // --- section 2: Vocoder, BigVGAN v2 arm (vocoder.py:293-438) --- inline constexpr int64_t kLtx2VocFrames = 5; inline constexpr int64_t kLtx2VocMelBins = 64; @@ -3791,6 +3959,66 @@ inline constexpr float kLtx2VideoDecNcGolden[] = { -0.284617603f, -0.397902191f, -0.13071467f, -0.841662765f, 0.062342234f, 0.121501237f, }; +// --- section 5d: the arm on which `norm_eps` BINDS (resnet.py:93-97) --- +inline constexpr int64_t kLtx2VideoDecEpsLatentC = 6; +inline constexpr int64_t kLtx2VideoDecEpsLatentT = 3; +inline constexpr int64_t kLtx2VideoDecEpsLatentH = 2; +inline constexpr int64_t kLtx2VideoDecEpsLatentW = 2; +inline constexpr double kLtx2VideoDecEpsLatentScale = 0.10000000000000001; +inline constexpr int64_t kLtx2VideoDecEpsOutC = 3; +inline constexpr int64_t kLtx2VideoDecEpsOutT = 3; +inline constexpr int64_t kLtx2VideoDecEpsOutH = 4; +inline constexpr int64_t kLtx2VideoDecEpsOutW = 4; +inline constexpr double kLtx2VideoDecEpsZeroMove = 0.00010240078; + +inline constexpr const char* kLtx2VideoDecEpsParamNames[] = { + "ltx2.videodeceps.per_channel_statistics.std-of-means", + "ltx2.videodeceps.per_channel_statistics.mean-of-means", + "ltx2.videodeceps.conv_in.conv.weight", + "ltx2.videodeceps.conv_in.conv.bias", + "ltx2.videodeceps.up_blocks.0.conv1.conv.weight", + "ltx2.videodeceps.up_blocks.0.conv1.conv.bias", + "ltx2.videodeceps.up_blocks.0.conv2.conv.weight", + "ltx2.videodeceps.up_blocks.0.conv2.conv.bias", + "ltx2.videodeceps.up_blocks.0.conv_shortcut.weight", + "ltx2.videodeceps.up_blocks.0.conv_shortcut.bias", + "ltx2.videodeceps.up_blocks.0.norm3.weight", + "ltx2.videodeceps.up_blocks.0.norm3.bias", + "ltx2.videodeceps.conv_out.conv.weight", + "ltx2.videodeceps.conv_out.conv.bias", +}; +inline constexpr int64_t kLtx2VideoDecEpsParamCounts[] = { + 6, 6, 2592, 16, 3456, 8, 1728, 8, 128, 8, + 16, 16, 2592, 12, +}; + +inline constexpr float kLtx2VideoDecEpsGolden[] = { + -0.090099901f, -0.190385759f, 0.380734861f, 0.718365669f, -0.31643483f, 0.147556275f, + -0.0505845137f, -0.789054036f, -0.11956919f, -0.145736665f, 0.434144616f, 0.35374558f, + -0.499161243f, 0.0523100793f, -0.120252334f, -0.878462672f, -0.16313675f, -0.163570255f, + 0.486694992f, 0.785750449f, -0.228940845f, 0.12952736f, -0.112032175f, -0.843012929f, + -0.0985285193f, -0.171556711f, 0.395003319f, 0.435430497f, -0.60490793f, 0.00392973423f, + -0.0427067131f, -0.878887415f, -0.256884098f, -0.198793769f, 0.632248998f, 0.69324261f, + -0.274890065f, 0.390317559f, -0.294240803f, -0.995928526f, -0.190123811f, -0.107693791f, + 0.427072853f, 0.395498425f, -0.587177634f, 0.138619155f, 0.0856989399f, -1.01287937f, + -0.397891402f, 0.402391732f, 0.0976787806f, -0.943622291f, -0.0623765476f, 0.504308522f, + -0.515643835f, -0.530895472f, -0.83202219f, 0.712152362f, 0.155863255f, -0.767041802f, + -0.400780559f, 0.23786135f, -0.750451028f, -0.787663937f, -0.299554139f, 0.338808537f, + 0.133289099f, -0.993130922f, -0.0635531694f, 0.565576851f, -0.559501112f, -0.437328607f, + -0.915952027f, 0.752729118f, 0.0779756084f, -0.813403726f, -0.445726335f, 0.196311533f, + -0.612994373f, -0.705531895f, -0.54736352f, 0.387349099f, 0.215518951f, -0.881484926f, + -0.0314500444f, 0.279632688f, -0.549191356f, -0.49227643f, -0.832705796f, 0.583334982f, + 0.124879047f, -0.890190005f, -0.261227369f, 0.205473766f, -0.546670079f, -0.696205318f, + -0.362264246f, 0.0967199802f, 0.162463546f, -0.0424314439f, 0.541362107f, -0.0823153034f, + -0.102280006f, 0.618953407f, -0.492172986f, -0.107823014f, 0.0281953812f, -0.214136556f, + -0.184159517f, -0.305839151f, -0.408877313f, 0.31305331f, -0.431710958f, 0.127348065f, + 0.219748765f, -0.125161737f, 0.573620319f, -0.0725017786f, -0.167064339f, 0.606333792f, + -0.431218207f, -0.0944405794f, 0.0440774485f, -0.306593657f, -0.0629140884f, -0.260454893f, + -0.473808587f, 0.28820464f, -0.319598079f, 0.0228947997f, 0.0474464335f, -0.0519712567f, + 0.500710905f, -0.0979719311f, -0.372783929f, 0.51922673f, -0.388694286f, 0.330318749f, + 0.0460506156f, -0.246671095f, -0.131759092f, -0.448623538f, -0.278485179f, 0.350407869f, +}; + // --- section 6a: VideoEncoder, the *_res family (video_vae.py:148-336) --- inline constexpr int64_t kLtx2VideoEncInC = 3; inline constexpr int64_t kLtx2VideoEncInT = 5; @@ -4152,6 +4380,114 @@ inline constexpr float kLtx2AudioEncPoolGolden[] = { 0.408448756f, 1.19915688f, 0.244993225f, 0.149357542f, }; +// --- section 7d: AudioEncoder, norm_type = GROUP, causality NONE --- +inline constexpr int64_t kLtx2AudioEncGroupOutC = 16; +inline constexpr int64_t kLtx2AudioEncGroupOutT = 2; +inline constexpr int64_t kLtx2AudioEncGroupOutF = 2; + +inline constexpr const char* kLtx2AudioEncGroupParamNames[] = { + "ltx2.audioencgroup.per_channel_statistics.std-of-means", + "ltx2.audioencgroup.per_channel_statistics.mean-of-means", + "ltx2.audioencgroup.conv_in.conv.weight", + "ltx2.audioencgroup.conv_in.conv.bias", + "ltx2.audioencgroup.down.0.block.0.norm1.weight", + "ltx2.audioencgroup.down.0.block.0.norm1.bias", + "ltx2.audioencgroup.down.0.block.0.conv1.conv.weight", + "ltx2.audioencgroup.down.0.block.0.conv1.conv.bias", + "ltx2.audioencgroup.down.0.block.0.norm2.weight", + "ltx2.audioencgroup.down.0.block.0.norm2.bias", + "ltx2.audioencgroup.down.0.block.0.conv2.conv.weight", + "ltx2.audioencgroup.down.0.block.0.conv2.conv.bias", + "ltx2.audioencgroup.down.0.downsample.conv.weight", + "ltx2.audioencgroup.down.0.downsample.conv.bias", + "ltx2.audioencgroup.down.1.block.0.norm1.weight", + "ltx2.audioencgroup.down.1.block.0.norm1.bias", + "ltx2.audioencgroup.down.1.block.0.conv1.conv.weight", + "ltx2.audioencgroup.down.1.block.0.conv1.conv.bias", + "ltx2.audioencgroup.down.1.block.0.norm2.weight", + "ltx2.audioencgroup.down.1.block.0.norm2.bias", + "ltx2.audioencgroup.down.1.block.0.conv2.conv.weight", + "ltx2.audioencgroup.down.1.block.0.conv2.conv.bias", + "ltx2.audioencgroup.down.1.block.0.nin_shortcut.conv.weight", + "ltx2.audioencgroup.down.1.block.0.nin_shortcut.conv.bias", + "ltx2.audioencgroup.down.1.downsample.conv.weight", + "ltx2.audioencgroup.down.1.downsample.conv.bias", + "ltx2.audioencgroup.down.2.block.0.norm1.weight", + "ltx2.audioencgroup.down.2.block.0.norm1.bias", + "ltx2.audioencgroup.down.2.block.0.conv1.conv.weight", + "ltx2.audioencgroup.down.2.block.0.conv1.conv.bias", + "ltx2.audioencgroup.down.2.block.0.norm2.weight", + "ltx2.audioencgroup.down.2.block.0.norm2.bias", + "ltx2.audioencgroup.down.2.block.0.conv2.conv.weight", + "ltx2.audioencgroup.down.2.block.0.conv2.conv.bias", + "ltx2.audioencgroup.down.2.block.0.nin_shortcut.conv.weight", + "ltx2.audioencgroup.down.2.block.0.nin_shortcut.conv.bias", + "ltx2.audioencgroup.down.2.attn.0.norm.weight", + "ltx2.audioencgroup.down.2.attn.0.norm.bias", + "ltx2.audioencgroup.down.2.attn.0.q.weight", + "ltx2.audioencgroup.down.2.attn.0.q.bias", + "ltx2.audioencgroup.down.2.attn.0.k.weight", + "ltx2.audioencgroup.down.2.attn.0.k.bias", + "ltx2.audioencgroup.down.2.attn.0.v.weight", + "ltx2.audioencgroup.down.2.attn.0.v.bias", + "ltx2.audioencgroup.down.2.attn.0.proj_out.weight", + "ltx2.audioencgroup.down.2.attn.0.proj_out.bias", + "ltx2.audioencgroup.mid.block_1.norm1.weight", + "ltx2.audioencgroup.mid.block_1.norm1.bias", + "ltx2.audioencgroup.mid.block_1.conv1.conv.weight", + "ltx2.audioencgroup.mid.block_1.conv1.conv.bias", + "ltx2.audioencgroup.mid.block_1.norm2.weight", + "ltx2.audioencgroup.mid.block_1.norm2.bias", + "ltx2.audioencgroup.mid.block_1.conv2.conv.weight", + "ltx2.audioencgroup.mid.block_1.conv2.conv.bias", + "ltx2.audioencgroup.mid.attn_1.norm.weight", + "ltx2.audioencgroup.mid.attn_1.norm.bias", + "ltx2.audioencgroup.mid.attn_1.q.weight", + "ltx2.audioencgroup.mid.attn_1.q.bias", + "ltx2.audioencgroup.mid.attn_1.k.weight", + "ltx2.audioencgroup.mid.attn_1.k.bias", + "ltx2.audioencgroup.mid.attn_1.v.weight", + "ltx2.audioencgroup.mid.attn_1.v.bias", + "ltx2.audioencgroup.mid.attn_1.proj_out.weight", + "ltx2.audioencgroup.mid.attn_1.proj_out.bias", + "ltx2.audioencgroup.mid.block_2.norm1.weight", + "ltx2.audioencgroup.mid.block_2.norm1.bias", + "ltx2.audioencgroup.mid.block_2.conv1.conv.weight", + "ltx2.audioencgroup.mid.block_2.conv1.conv.bias", + "ltx2.audioencgroup.mid.block_2.norm2.weight", + "ltx2.audioencgroup.mid.block_2.norm2.bias", + "ltx2.audioencgroup.mid.block_2.conv2.conv.weight", + "ltx2.audioencgroup.mid.block_2.conv2.conv.bias", + "ltx2.audioencgroup.norm_out.weight", + "ltx2.audioencgroup.norm_out.bias", + "ltx2.audioencgroup.conv_out.conv.weight", + "ltx2.audioencgroup.conv_out.conv.bias", +}; +inline constexpr int64_t kLtx2AudioEncGroupParamCounts[] = { + 32, 32, 576, 32, 32, 32, 9216, 32, 32, 32, + 9216, 32, 9216, 32, 32, 32, 18432, 64, 64, 64, + 36864, 64, 2048, 64, 36864, 64, 64, 64, 73728, 128, + 128, 128, 147456, 128, 8192, 128, 128, 128, 16384, 128, + 16384, 128, 16384, 128, 16384, 128, 128, 128, 147456, 128, + 128, 128, 147456, 128, 128, 128, 16384, 128, 16384, 128, + 16384, 128, 16384, 128, 128, 128, 147456, 128, 128, 128, + 147456, 128, 128, 128, 36864, 32, +}; + +inline constexpr float kLtx2AudioEncGroupGolden[] = { + -0.415124387f, 1.29275453f, 0.498907506f, -0.575730801f, -1.62029707f, 0.399742216f, + -1.27433813f, -1.5076828f, 0.576601684f, 1.38621402f, -0.103683658f, 0.666440964f, + -0.92569983f, 0.178702965f, 0.852021933f, 0.0271965712f, -1.39139009f, -0.653059721f, + -0.517684519f, -0.429321557f, 1.28704107f, -0.217622474f, 0.189814135f, 0.85940665f, + -0.0855252966f, 1.32952476f, -0.160250455f, 1.28190529f, 0.0534490608f, 0.143534899f, + 1.44125962f, 0.0584254004f, 1.22335052f, 0.0162700787f, 0.78545785f, -1.07138836f, + 0.861104965f, -0.100202091f, 0.890008628f, 0.723289728f, 1.05474973f, -0.793653429f, + -0.027898185f, 0.278479785f, 0.0288700331f, -0.315794587f, -0.912896693f, -1.25509763f, + 0.0351633877f, 0.0417222045f, 0.242204696f, 0.663830519f, -0.553536654f, 0.182536632f, + -1.92017734f, -1.89745152f, -0.609760642f, -1.01902819f, -1.61708963f, -1.8382777f, + -0.639630437f, 0.923621953f, 0.585529745f, -0.39709264f, +}; + // --- section 8a: the slaney mel filterbank (torchaudio melscale_fbanks) --- inline constexpr int64_t kLtx2MelRate = 16000; inline constexpr int64_t kLtx2MelFreqs = 33; diff --git a/tests/vllm/models/test_ltx2.cpp b/tests/vllm/models/test_ltx2.cpp index 6b2a29593..dd6a8e077 100644 --- a/tests/vllm/models/test_ltx2.cpp +++ b/tests/vllm/models/test_ltx2.cpp @@ -362,6 +362,26 @@ TEST_CASE("ltx2 config: ParseLtx2DitParams mirrors LTXModelConfigurator") { CHECK(parsed.rope_type == Ltx2RopeType::kSplit); CHECK_FALSE(parsed.double_precision_rope); + // THE INVISIBLE-CONSTANT CLASS, in the DiT. `norm_eps` feeds the q/k RMSNorm + // (attention.py:505-506) and every AdaLN, but every arm in this suite passes it + // EXPLICITLY through ReducedParams, so nothing here reads the FIELD DEFAULT and + // a 100x mutation of it left all six LTX suites green. The default is not dead + // code: `ReducedConfig()` carries no `norm_eps` key, which is the shape of a + // checkpoint that omits it, and upstream's own fallback is + // `config.get("norm_eps", 1e-06)` (transformer/model_configurator.py:54, 124, + // 181). So the parse below is exactly the path the default binds on, and this + // pins it there rather than in a list far from its use. + CHECK(parsed.norm_eps == doctest::Approx(1e-6).epsilon(1e-12).scale(0.0)); + { + // Not a SUBCASE deliberately: doctest re-enters the whole case body once per + // subcase, so adding one here would multiply every assertion above it and + // move this suite's recorded count for a reason unrelated to coverage. + nlohmann::json explicit_eps = ReducedConfig(); + explicit_eps["config"]["transformer"]["norm_eps"] = 1e-5; + CHECK(ParseLtx2DitParams(explicit_eps).norm_eps == + doctest::Approx(1e-5).epsilon(1e-12).scale(0.0)); + } + SUBCASE("frequencies_precision selects the float64 ladder") { nlohmann::json cfg = ReducedConfig(); cfg["config"]["transformer"]["frequencies_precision"] = "float64"; @@ -394,6 +414,31 @@ TEST_CASE("ltx2 config: ParseLtx2DitParams mirrors LTXModelConfigurator") { } } +TEST_CASE("ltx2 dit: Ltx2AttentionArgs::norm_eps is a LATENT default, so it is pinned") { + // The sixth instance the constant sweep turned up, and the most inert of them. + // `Ltx2AttentionArgs::norm_eps` is the eps of the q/k RMSNorm — upstream's + // `Attention.__init__` declares `norm_eps: float = 1e-6` (attention.py:485) and + // hands it to both `torch.nn.RMSNorm`s (attention.py:505-506) — but EVERY + // construction of the struct assigns it before use: ltx2_dit.cpp:188, :244, + // :280, :338, :366 from `Ltx2DitParams::norm_eps`, ltx2_connector.cpp:253 from + // `kLtx2ConnectorRmsNormEps`, and each of this suite's own arms from its + // ReducedParams. + // + // Measured, not assumed: mutating this default 1e-6 -> 1.0, a 10^6 change, + // leaves every suite green. That is not the invisible-epsilon story the other + // five tell — those are read and merely never bind. This one is never READ, so + // no fixture, however scaled, can reach it. It is a latent trap: the value a + // future call site inherits on the day someone adds one and forgets the + // assignment, at which point 1.0 would be silently applied inside an RMSNorm. + // A pin is the only instrument that can hold it, and this records that limit + // rather than dressing it up as coverage. + CHECK(vllm::Ltx2AttentionArgs{}.norm_eps == doctest::Approx(1e-6).epsilon(1e-12).scale(0.0)); + // ...and it must agree with the DiT parameter that every real call site feeds + // it from, so the two cannot drift apart unnoticed. + CHECK(vllm::Ltx2AttentionArgs{}.norm_eps == + doctest::Approx(vllm::Ltx2DitParams{}.norm_eps).epsilon(1e-12).scale(0.0)); +} + TEST_CASE("ltx2 layout: the shapes recover the geometry") { const Ltx2DitParams p = ReducedParams(Ltx2RopeType::kSplit, false); const Ltx2DitParams derived = ParseLtx2DitParamsFromManifest(EnumerateLtx2DitTensors(p)); diff --git a/tests/vllm/models/test_ltx2_pipeline.cpp b/tests/vllm/models/test_ltx2_pipeline.cpp index 9baba554c..5076148ea 100644 --- a/tests/vllm/models/test_ltx2_pipeline.cpp +++ b/tests/vllm/models/test_ltx2_pipeline.cpp @@ -473,10 +473,14 @@ TEST_CASE("ltx2 Res2s diffusion step reproduces upstream") { LTX2_RES2S_ARM(Eta1); #undef LTX2_RES2S_ARM - // The sigma_up clamp (diffusion_steps.py:138). A member of the - // invisible-constant class: on any well-formed schedule eta <= 1 keeps sigma_up - // at or below sigma_next, so the clamp never binds and no value comparison can - // see it. Pinned against the upstream literal instead. + // The sigma_up clamp (diffusion_steps.py:138), and NOT a member of the + // invisible-constant class. The old reasoning here — "eta <= 1 keeps sigma_up at + // or below sigma_next, so the clamp never binds" — reads its own inequality + // wrongly: <= includes ==, and the Eta1 arm run five lines up sits exactly on + // that boundary, where `min` takes `sigma_next * 0.9999` on every step. A 1% + // move (0.9999 -> 0.99) REDS Eta1 index 0 at max|diff| = 0.086 and index 1 at + // 0.130563; EtaHalf and the terminal index stay green. Pinned against the + // upstream literal AS WELL, since a regenerated golden moves with the constant. CHECK(vllm::kLtx2Res2sSigmaUpClamp == 0.9999); // ...and the clamp is proved to be a clamp, on an input that DOES exceed it. const vllm::Ltx2SdeCoeff clamped = vllm::Ltx2Res2sSdeCoeff(0.5, 2.0); @@ -1293,12 +1297,29 @@ vllm::Ltx2LatentVolume ReducedUpsamplerLatent() { } // namespace TEST_CASE("ltx2 the constants the headers call pinned are actually pinned") { - // Both headers said "pinned", and NEITHER constant had a single test reference, - // so either could be edited with every golden still green — exactly the - // invisible-constant class of spec §7.0(a), where the fixture never enters the - // regime the constant governs. The connector's rows are never near-zero, so its - // eps is inert in the value comparison; the blur width is only reachable through - // a default upstream never passes explicitly. + // Both headers said "pinned" and NEITHER constant had a test reference, so this + // case exists because an unreferenced constant can be edited without the suite + // naming it. What it does NOT do is make either one invisible to the goldens: + // that was inferred rather than measured, and both are in fact reached. + // + // kLtx2ConnectorRmsNormEps 1e-6 -> 1e-4 (100x) REDS 5 arms of "ltx2 the + // Embeddings1DConnector reproduces upstream on every arm" — Split + // 0.0558581, Interleaved 0.104284, Float64 0.140343, NoRegisters + // 0.000542641, GatedNoBias 0.0892045. `rms_norm` adds the epsilon to the + // MEAN SQUARE, so "the rows are never near-zero" was never the question. + // kLtx2BlurKernelSize 5 -> 3 REDS "ltx2 the latent spatial upsampler + // reproduces upstream", arm Rational1p5, at 0.689782. Upstream not passing + // the argument does not make the default unreachable — the default IS the + // shipped width, which ltx2_upsampler.h has said all along. Only the + // Rational1p5 arm reaches it, because `BlurDownsample` runs on the + // rational `den` (ltx2_upsampler.cpp:439) and 1.5 -> {3, 2} is the only + // one of the THREE ARMS with den != 1. (0.75 -> {3, 4} would reach it too + // and no arm covers it, so this is arm coverage, not a property of the + // supported-scale map.) + // + // So the reason to keep this case is the narrower, real one: a golden + // regenerated from a moved constant moves with it, and these two lines are the + // only comparison against upstream's own signature. // // Each expected value is READ OFF upstream's own signature by the generator // (utils.py:7, blur_downsample.py:14), not retyped here, so upstream moving @@ -1375,8 +1396,17 @@ TEST_CASE("ltx2 the latent spatial upsampler reproduces upstream") { // GroupNorm's group count is a LITERAL upstream (res_block.py:24,26; model.py:50), // not a config key a checkpoint could move, and its eps is torch's default - // because no site passes one. Both are members of the invisible-constant class - // at this fixture's scale, so both are pinned rather than left to the tensors. + // because no site passes one. NEITHER is a member of the invisible-constant + // class: the three arms above reach both, MEASURED on this tree. + // + // kLtx2UpsamplerNormEps 1e-5 -> 1e-3 (100x) REDS PixelShuffle 0.0289409, + // Rational2 0.0347079, Rational1p5 0.0649014 + // kLtx2UpsamplerNormGroups 32 -> 16 REDS PixelShuffle 0.63738, + // Rational2 0.633718, Rational1p5 0.874346 + // + // Both stay pinned anyway, for the one thing the tensors cannot do: a + // regeneration that moves the constant and the goldens together still passes + // the value comparison, and only these two lines compare against upstream. CHECK(vllm::kLtx2UpsamplerNormGroups == vllm_test::kLtx2UpsNormGroups); CHECK(vllm::kLtx2UpsamplerNormEps == 1e-5); } diff --git a/tests/vllm/models/test_ltx2_vae.cpp b/tests/vllm/models/test_ltx2_vae.cpp index 379b25b97..1667f58d3 100644 --- a/tests/vllm/models/test_ltx2_vae.cpp +++ b/tests/vllm/models/test_ltx2_vae.cpp @@ -203,23 +203,53 @@ vllm::Ltx2AudioDecoderConfig ReducedAudioDecoderConfig(int64_t mel_bins) { return cfg; } +// The reduced GROUP-NORM audio decoder the generator built (AUDIO_GROUP_DEC). +// `ch` is 32 because `build_normalization_layer` forwards its own `num_groups` +// keyword, whose default is 32 and which no audio_vae call site passes +// (normalization.py:44, 56), and torch's GroupNorm refuses a channel count 32 +// does not divide; `z_channels` is 16 because `PerChannelStatistics` indexes the +// patchified `(c, f)` axis, so z_channels x latent mel bins must equal `ch`. +vllm::Ltx2AudioDecoderConfig ReducedAudioDecoderGroupConfig() { + vllm::Ltx2AudioDecoderConfig cfg = ReducedAudioDecoderConfig( + vllm_test::kLtx2AudioDecGroupOutMelBins); + cfg.ch = 32; + cfg.z_channels = vllm_test::kLtx2AudioDecGroupLatentC; + cfg.norm_type = vllm::Ltx2NormType::kGroup; + // ResnetBlock REFUSES GroupNorm on any causal axis (resnet.py:130-131), so this + // is the only causality a group-norm checkpoint can legally declare. + cfg.causality_axis = vllm::Ltx2CausalityAxis::kNone; + cfg.prefix = "ltx2.audiodecgroup."; + return cfg; +} + // Build the audio decoder's parameters in upstream state_dict ORDER: // per_channel_statistics, conv_in, mid, up (block / attn / upsample per level), -// conv_out. PixelNorm carries no parameters, which is why no norm tensor appears. +// norm_out, conv_out. PixelNorm carries no parameters, which is why no norm +// tensor appears on the pixel arms; GroupNorm is affine, so on `kGroup` every +// `norm1` / `norm2` / `attn.norm` / `norm_out` contributes a weight and a bias +// AHEAD of the convolution it precedes (resnet.py:136-146, attention.py:25-29). ParamBag BuildAudioDecoderParams(const vllm::Ltx2AudioDecoderConfig& cfg) { ParamBag bag; const std::string p = cfg.prefix; const int64_t levels = static_cast(cfg.ch_mult.size()); const int64_t base = cfg.ch * cfg.ch_mult[static_cast(levels - 1)]; + const bool group = cfg.norm_type == vllm::Ltx2NormType::kGroup; bag.Put(p + "per_channel_statistics.std-of-means", {cfg.ch}); bag.Put(p + "per_channel_statistics.mean-of-means", {cfg.ch}); bag.Put(p + "conv_in.conv.weight", {base, cfg.z_channels, 3, 3}); bag.Put(p + "conv_in.conv.bias", {base}); + auto put_norm = [&](const std::string& prefix, int64_t channels) { + if (!group) return; + bag.Put(prefix + ".weight", {channels}); + bag.Put(prefix + ".bias", {channels}); + }; auto put_resnet = [&](const std::string& prefix, int64_t in_ch, int64_t out_ch) { + put_norm(prefix + ".norm1", in_ch); bag.Put(prefix + ".conv1.conv.weight", {out_ch, in_ch, 3, 3}); bag.Put(prefix + ".conv1.conv.bias", {out_ch}); + put_norm(prefix + ".norm2", out_ch); bag.Put(prefix + ".conv2.conv.weight", {out_ch, out_ch, 3, 3}); bag.Put(prefix + ".conv2.conv.bias", {out_ch}); if (in_ch != out_ch) { @@ -228,6 +258,7 @@ ParamBag BuildAudioDecoderParams(const vllm::Ltx2AudioDecoderConfig& cfg) { } }; auto put_attn = [&](const std::string& prefix, int64_t channels) { + put_norm(prefix + ".norm", channels); for (const char* leaf : {"q", "k", "v", "proj_out"}) { bag.Put(prefix + "." + leaf + ".weight", {channels, channels, 1, 1}); bag.Put(prefix + "." + leaf + ".bias", {channels}); @@ -280,6 +311,7 @@ ParamBag BuildAudioDecoderParams(const vllm::Ltx2AudioDecoderConfig& cfg) { } } + put_norm(p + "norm_out", block_in); bag.Put(p + "conv_out.conv.weight", {cfg.out_ch, block_in, 3, 3}); bag.Put(p + "conv_out.conv.bias", {cfg.out_ch}); return bag; @@ -416,6 +448,61 @@ TEST_CASE("ltx2 vae: the audio decoder matches upstream ltx_core") { } } +TEST_CASE("ltx2 vae: the GROUP-NORM audio decoder matches upstream ltx_core") { + // THE ARM THAT MAKES `Ltx2AudioDecoderConfig::norm_eps` REACHABLE. Every other + // audio arm in this file runs `norm_type = kPixel`, so `ApplyNorm` never enters + // the GroupNorm branch and `norm_eps` is not merely inert but never READ: + // mutating it 1e-6 -> 1e-4, a 100x change, left all 33 cases green. + // + // `norm_type = group` is not hypothetical, and it is not free either. + // `AudioDecoder.__init__` declares `norm_type = GROUP` (audio_vae.py:294) and + // on the next line `causality_axis = WIDTH` (audio_vae.py:295) — a pair + // `ResnetBlock` REFUSES, with `ValueError: Causal ResnetBlock with GroupNorm is + // not supported` (resnet.py:130-131), so constructing the upstream decoder on + // pure defaults raises rather than group-normalizing. What is legal, and what + // this arm runs, is a checkpoint that declares `causality_axis: none` alongside + // it — the other half of `build_normalization_layer` (normalization.py:56-57). + // Without this arm such a checkpoint would run a 100x-wrong stabilizer and + // still produce a spectrogram. + const vllm::Ltx2AudioDecoderConfig cfg = ReducedAudioDecoderGroupConfig(); + ParamBag bag = BuildAudioDecoderParams(cfg); + CheckManifest(bag, vllm_test::kLtx2AudioDecGroupParamNames, + vllm_test::kLtx2AudioDecGroupParamCounts, + std::size(vllm_test::kLtx2AudioDecGroupParamNames)); + + const int64_t c = vllm_test::kLtx2AudioDecGroupLatentC; + const int64_t t = vllm_test::kLtx2AudioDecGroupLatentT; + const int64_t f = vllm_test::kLtx2AudioDecGroupLatentF; + const std::vector latent = Ltx2Input("ltx2.audiodecgroup.input", c * t * f, 1.0); + + const vllm::Ltx2AudioSpectrogram mel = + vllm::Ltx2AudioDecoderForward(cfg, bag.weights, latent, c, t, f); + CHECK(mel.channels == cfg.out_ch); + CHECK(mel.frames == vllm_test::kLtx2AudioDecGroupOutFrames); + CHECK(mel.mel_bins == vllm_test::kLtx2AudioDecGroupOutMelBins); + + const double err = MaxAbsDiff(mel.data, vllm_test::kLtx2AudioDecGroupGolden, + std::size(vllm_test::kLtx2AudioDecGroupGolden)); + INFO("group-norm audio decoder max|diff| = " << err); + CHECK(err <= kLtx2GoldenTol); + + // The combination upstream REFUSES must be refused here too, not silently + // group-normalized on a causal axis. + vllm::Ltx2AudioDecoderConfig bad = cfg; + bad.causality_axis = vllm::Ltx2CausalityAxis::kHeight; + bool threw = false; + std::string message; + try { + vllm::Ltx2AudioDecoderForward(bad, bag.weights, latent, c, t, f); + } catch (const std::exception& error) { + threw = true; + message = error.what(); + } + REQUIRE(threw); + INFO("refusal message: " << message); + CHECK(message.find("GroupNorm") != std::string::npos); +} + TEST_CASE("ltx2 vae: the audio decoder's CONVOLUTIONS are one-sided in time") { // The trap: padding the time axis symmetrically instead of on the LEFT still // produces a plausible spectrogram that merely peeks into the future. Isolate @@ -971,6 +1058,81 @@ TEST_CASE("ltx2 vae: the NON-causal Conv video decoder matches upstream ltx_core CHECK(causal_frames.data != frames.data); } +TEST_CASE("ltx2 vae: the video decoder's norm_eps is gated where it BINDS") { + // THE ARM THAT MAKES `Ltx2ConvVideoDecoderConfig::norm_eps` NUMERICALLY + // REACHABLE, and the correction of a record that said it was not. + // + // The earlier claim — that this constant is invisible because upstream discards + // it on the PixelNorm arm — is FALSE. `ResnetBlock3D.__init__` builds + // `norm3 = nn.GroupNorm(num_groups=1, num_channels=in_channels, eps=eps)` + // whenever `in_channels != out_channels` (resnet.py:93-97), REGARDLESS of + // `norm_layer`, and `forward` applies it to the residual (resnet.py:178). Every + // `res_x_y` block therefore reads it, and the shipped section-5 arm above has + // one, at `up_blocks.4.norm3`. + // + // What was true is a statement about the FIXTURE, not the constant. norm3 + // divides by `sqrt(var + eps)` over all of (C, T, H, W); five blocks deep that + // variance is ~0.2, so on section 5's golden 1e-6 -> 1e-4 moves 1.8e-6 — under + // the 5e-6 band — while 1e-6 -> 1.0 moves 1.6e-2. A 100x error passed because + // the denominator was large, which is an accident of scale and not a property + // worth recording as coverage. + // + // So this arm removes the accident. ONE `res_x_y` block puts norm3 directly + // behind conv_in, and a latent at a tenth of the usual scale leaves it a + // variance of ~5e-3 to compete with. `kLtx2VideoDecEpsZeroMove` is what the + // ORACLE measured for the mutation the other arms are blindest to — removing + // the epsilon entirely, which moves section 5's golden by 5.4e-7, a tenth of + // the band — so the sensitivity is gated here rather than narrated. + REQUIRE(vllm_test::kLtx2VideoDecEpsZeroMove > 10.0 * kLtx2GoldenTol); + + vllm::Ltx2ConvVideoDecoderConfig cfg; + cfg.in_channels = 6; + cfg.out_channels = 3; + cfg.patch_size = 2; + cfg.norm_layer = vllm::Ltx2NormLayer::kPixelNorm; + cfg.causal = true; + cfg.timestep_conditioning = false; + cfg.spatial_padding_mode = vllm::Ltx2PaddingMode::kReflect; + cfg.base_channels = 8; + cfg.prefix = "ltx2.videodeceps."; + cfg.decoder_blocks = {{"res_x_y", 1, 2, false, false}}; + // The constant under test is the FIELD DEFAULT, never an override — an arm that + // set it explicitly would gate the plumbing and not the value. + CHECK(cfg.norm_eps == doctest::Approx(1e-6).epsilon(1e-12).scale(0.0)); + + ParamBag bag = BuildVideoDecoderParams(cfg); + CheckManifest(bag, vllm_test::kLtx2VideoDecEpsParamNames, + vllm_test::kLtx2VideoDecEpsParamCounts, + std::size(vllm_test::kLtx2VideoDecEpsParamNames)); + // norm3 exists only because the block changes channel count; without these two + // parameters the manifest would match a decoder that never reads the epsilon. + CHECK(bag.weights.Has("ltx2.videodeceps.up_blocks.0.norm3.weight")); + CHECK(bag.weights.Has("ltx2.videodeceps.up_blocks.0.norm3.bias")); + + const int64_t lc = vllm_test::kLtx2VideoDecEpsLatentC; + const int64_t lt = vllm_test::kLtx2VideoDecEpsLatentT; + const int64_t lh = vllm_test::kLtx2VideoDecEpsLatentH; + const int64_t lw = vllm_test::kLtx2VideoDecEpsLatentW; + const std::vector latent = Ltx2Input("ltx2.videodeceps.input", lc * lt * lh * lw, + vllm_test::kLtx2VideoDecEpsLatentScale); + + GoldenNoise noise("ltx2.videodeceps."); + const vllm::Ltx2VideoFrames frames = + vllm::Ltx2ConvVideoDecode(cfg, bag.weights, latent, lc, lt, lh, lw, &noise); + CHECK(frames.channels == vllm_test::kLtx2VideoDecEpsOutC); + CHECK(frames.frames == vllm_test::kLtx2VideoDecEpsOutT); + CHECK(frames.height == vllm_test::kLtx2VideoDecEpsOutH); + CHECK(frames.width == vllm_test::kLtx2VideoDecEpsOutW); + // Neither timestep conditioning nor an inject_noise block, so upstream calls + // torch.randn zero times and this port must draw nothing either. + CHECK(noise.counts().empty()); + + const double err = MaxAbsDiff(frames.data, vllm_test::kLtx2VideoDecEpsGolden, + std::size(vllm_test::kLtx2VideoDecEpsGolden)); + INFO("norm_eps-binding video decoder max|diff| = " << err); + CHECK(err <= kLtx2GoldenTol); +} + TEST_CASE("ltx2 vae: video temporal causality is one-sided, proven by perturbation") { // The trap this catches: putting temporal padding on BOTH sides of a causal // Conv3d — or zero-padding it the way MiniMax-H3's Conv3d does instead of @@ -1073,18 +1235,39 @@ TEST_CASE("ltx2 vae: the goldens carry the upstream revision they came from") { } TEST_CASE("ltx2 vae: every stabilizing epsilon is pinned to its upstream line") { - // THE INVISIBLE-CONSTANT CLASS. An epsilon that exists to stabilize a division - // is by construction invisible to a reduced-dimension parity gate: the - // deterministic stream produces O(1) activations, the guarded term never binds, - // and the tensor comparison accepts ANY value — including 0.0 and including one - // 100x off. Each of these was mutated with every golden staying green, so each - // is held HERE, cited to the upstream line that sets it. Adding a new constant - // without adding it to this list reopens the hole. + // THE INVISIBLE-CONSTANT CLASS, and the PIN LIST that holds it. An epsilon that + // exists to stabilize a division CAN be invisible to a reduced-dimension parity + // gate: the deterministic stream produces O(1) activations, the guarded term + // never binds, and the tensor comparison accepts ANY value — including 0.0 and + // including one 100x off. + // + // Membership is a MEASURED, PER-ENTRY property, and it is NOT a property of + // this list. Some entries below were mutated with every golden staying green. + // Others are gated numerically by an arm that reaches them, and are pinned + // anyway, because a pin catches the edit a golden cannot: a regeneration that + // moves the constant and the goldens TOGETHER. Each entry says which it is, and + // says it because the mutation was RUN, on this tree, with the numbers recorded + // beside it. + // + // Adding a new constant without adding it here reopens the hole. Recording one + // as invisible without mutating it reopens a worse one — this case has now + // carried a wrong reachability verdict twice, which is the whole reason the + // claim is per-entry and quantified rather than a sentence at the top. // ResnetBlock3D's `eps: float = 1e-6` (video_vae/resnet.py:31), handed to every // nn.GroupNorm it builds (resnet.py:44, 65, 94) and carried by UNetMidBlock3D as // `resnet_eps` (resnet.py:216). This is the norm `res_x_y`'s shortcut uses. - // Mutation: 1e-6 -> 1e-4, a 100x change, left every golden green. + // + // CORRECTED. This one is NOT a member of the invisible class, and the record + // that put it here said so for a reason that does not hold: `norm3` is built at + // resnet.py:93-97 whenever `in_channels != out_channels`, REGARDLESS of + // `norm_layer`, and applied at resnet.py:178 — so a PixelNorm decoder reads it + // too, at every `res_x_y`. The 1e-6 -> 1e-4 mutation stayed green because + // section 5's norm3 sits five blocks deep behind a variance of ~0.2, not + // because nothing read the value. "ltx2 vae: the video decoder's norm_eps is + // gated where it BINDS" is the arm that removes that accident of scale; the pin + // stays because it still catches the edit a golden cannot, a regeneration that + // moves the constant and the goldens together. CHECK(vllm::Ltx2ConvVideoDecoderConfig{}.norm_eps == doctest::Approx(1e-6).epsilon(1e-12).scale(0.0)); @@ -1107,6 +1290,58 @@ TEST_CASE("ltx2 vae: every stabilizing epsilon is pinned to its upstream line") // left every golden green; it decides whether an all-zero channel vector // divides or produces NaN. CHECK(vllm::kLtx2RmsNorm2dEps == doctest::Approx(1e-12).epsilon(1e-12).scale(0.0)); + + // The AUDIO VAE's GroupNorm eps, on BOTH halves. `build_normalization_layer` + // passes `eps=1e-6` literally to `torch.nn.GroupNorm` (normalization.py:56), + // the same line that gives PixelNorm its 1e-6 — so the two fields agree here + // and, unlike the video VAE's pair below, are NOT deliberately different. + // + // These two were the fourth recurrence of this class, and worse than inert: + // every audio arm ran `norm_type = kPixel`, so the GroupNorm branch was never + // entered and the constant was never READ. 1e-6 -> 1e-4 on both left all 33 + // cases green. They are now reachable — the two group-norm arms above execute + // the branch numerically — and pinned here as well, because a pin catches the + // edit a golden cannot: replacing 1e-6 with the video VAE's 1e-8 while + // regenerating would move the goldens and the arms would follow it. + CHECK(vllm::Ltx2AudioDecoderConfig{}.norm_eps == + doctest::Approx(1e-6).epsilon(1e-12).scale(0.0)); + CHECK(vllm::Ltx2AudioEncoderConfig{}.norm_eps == + doctest::Approx(1e-6).epsilon(1e-12).scale(0.0)); + + // The VIDEO VAE's GroupNorm eps on the ENCODER half, which phase L11 added and + // this list did not grow to match. `_make_encoder_block` passes `resnet_eps=1e-6` + // / `eps=1e-6` literally (video_vae.py:56, 66) and `conv_norm_out` takes + // `eps=1e-6` (video_vae.py:240); there is no checkpoint key that moves it. + // + // NOT INVISIBLE — this entry arrived carrying the same wrong verdict the decoder + // entry above did, for the same reason, and is corrected the same way. Encoder + // arm A has a `res_x_y` block, so ResnetBlock3d builds norm3 (resnet.py:93-97, + // applied at :178) exactly as the decoder does, and our port reads it at ONE + // line for both halves: ltx2_video_vae.cpp:1051,1056 reach :405, which the + // decoder reaches from :693,700. Measured on this tree: the field default + // 1e-6 -> 1e-4 REDS two goldens at max|diff| = 4.38839e-05 against the 5e-6 + // band — "the video ENCODER (*_res family)" and "the video encoder CROPS a + // frame count that is not 1 + k*factor". Forcing :405 to 1.0 reds those same + // two at 0.150858, which is what IDENTIFIES norm3 as the reader: `norm_layer` + // is kPixelNorm on both encoder arms, so neither ApplyNorm nor conv_norm_out + // (ltx2_video_vae.cpp:1081-1087) ever enters a GroupNorm branch. Arm B holds no + // `res_x_y` and therefore no norm3, and stays green throughout — the coverage + // is real but PARTIAL, which is exactly what the pin is still for. + CHECK(vllm::Ltx2ConvVideoEncoderConfig{}.norm_eps == + doctest::Approx(1e-6).epsilon(1e-12).scale(0.0)); + + // And its PixelNorm eps, which is the video VAE's bare `PixelNorm()` DEFAULT of + // 1e-8 (normalization.py:22) — NOT the audio VAE's 1e-6. The decoder-side pair + // has its own case below; the encoder was missing from both. + // + // Also NOT INVISIBLE, and the more clearly so: PixelNorm IS the encoder's norm + // on every arm, so this epsilon is a first-order term the whole way down rather + // than a guard that never binds. Measured: 1e-8 -> 1e-6 REDS four encoder + // goldens — 1.02744e-05 on the `*_res` family and on the frame-count crop, + // 8.10623e-06 on encoder temporal causality, and 0.000175595 on "the video + // ENCODER (strided convs, per_channel, reflect)". + CHECK(vllm::Ltx2ConvVideoEncoderConfig{}.pixel_norm_eps == + doctest::Approx(1e-8).epsilon(1e-12).scale(0.0)); } TEST_CASE("ltx2 vae: the BWE mel log clamp is gated where it actually binds") { @@ -1166,10 +1401,24 @@ TEST_CASE("ltx2 vae: the BWE mel log clamp is gated where it actually binds") { } TEST_CASE("ltx2 vae: the two PixelNorm epsilons stay different") { - // This is a SOURCE-ANCHORED CONSTANT guard, not a numerical gate, and it exists - // because mutation proved the numerical gate cannot do the job: flipping the - // video decoder's eps from 1e-8 to 1e-6 leaves every golden green, since the - // normalized activations are O(1) and the difference is ~1e-7 relative. + // This is a SOURCE-ANCHORED CONSTANT guard, and it is NOT the only thing holding + // either epsilon: both are also reached numerically, MEASURED on this tree by + // mutating the field defaults every arm runs. + // + // Ltx2ConvVideoDecoderConfig::pixel_norm_eps 1e-8 -> 1e-6 (100x) + // REDS "ltx2 vae: the video decoder's norm_eps is gated where it BINDS" + // at max|diff| = 0.000169305 against the 5e-6 band. + // Ltx2AudioDecoderConfig::pixel_norm_eps 1e-6 -> 1e-4 (100x) + // REDS 5 goldens across three arms — "the audio decoder matches upstream + // ltx_core" (0.0120053), "the other three causality axes" (0.00461239, + // 0.00302449, 0.00245912) and "pads the frequency axis" (0.0120053). + // + // An earlier revision of this comment said the video half "leaves every golden + // green". That was true when it was written and false when the low-scale + // norm_eps arm landed: at a tenth of the usual latent scale this epsilon is a + // first-order term too, which ltx2_video_vae.h already records. The pin stays + // for the reason a pin always earns — a regeneration that moves the constant + // and the goldens together, which no value comparison can see. // // The values are not interchangeable upstream. The audio VAE reaches PixelNorm // through build_normalization_layer, which passes eps=1e-6 @@ -1182,6 +1431,17 @@ TEST_CASE("ltx2 vae: the two PixelNorm epsilons stay different") { doctest::Approx(1e-8).epsilon(1e-12).scale(0.0)); CHECK(vllm::Ltx2AudioDecoderConfig{}.pixel_norm_eps != vllm::Ltx2ConvVideoDecoderConfig{}.pixel_norm_eps); + + // The two ENCODER halves phase L11 added carry the SAME split for the SAME + // reason, and this case only ever held the decoder pair. All FOUR are reachable + // by their own goldens — the encoder pair per d45bcb5fb, the decoder pair per + // the numbers above — so this is not closing a silent hole. It keeps the case's + // claim ("the two PixelNorm epsilons stay different") true of every config that + // has the field, rather than of the two it happened to be written for. + CHECK(vllm::Ltx2AudioEncoderConfig{}.pixel_norm_eps == + doctest::Approx(1e-6).epsilon(1e-12).scale(0.0)); + CHECK(vllm::Ltx2AudioEncoderConfig{}.pixel_norm_eps != + vllm::Ltx2ConvVideoEncoderConfig{}.pixel_norm_eps); } TEST_CASE("ltx2 vae: the diffusion video decoder is refused by name, never downgraded") { @@ -1369,6 +1629,18 @@ vllm::Ltx2AudioEncoderConfig ReducedAudioEncoderConfig() { return cfg; } +// The reduced GROUP-NORM audio encoder the generator built (AUDIO_GROUP_ENC). +// Same two forced dimensions as the decoder's group arm, for the same reasons. +vllm::Ltx2AudioEncoderConfig ReducedAudioEncoderGroupConfig() { + vllm::Ltx2AudioEncoderConfig cfg = ReducedAudioEncoderConfig(); + cfg.ch = 32; + cfg.z_channels = vllm_test::kLtx2AudioEncGroupOutC; + cfg.norm_type = vllm::Ltx2NormType::kGroup; + cfg.causality_axis = vllm::Ltx2CausalityAxis::kNone; + cfg.prefix = "ltx2.audioencgroup."; + return cfg; +} + // Build the AudioEncoder's parameters in upstream state_dict ORDER // (audio_vae.py:118-188): per_channel_statistics, conv_in, then per level the // `block` ModuleList, then that level's `attn` ModuleList, then `downsample`; @@ -1377,15 +1649,25 @@ ParamBag BuildAudioEncoderParams(const vllm::Ltx2AudioEncoderConfig& cfg) { ParamBag bag; const std::string p = cfg.prefix; const int64_t levels = cfg.num_resolutions(); + const bool group = cfg.norm_type == vllm::Ltx2NormType::kGroup; bag.Put(p + "per_channel_statistics.std-of-means", {cfg.ch}); bag.Put(p + "per_channel_statistics.mean-of-means", {cfg.ch}); bag.Put(p + "conv_in.conv.weight", {cfg.ch, cfg.in_channels, 3, 3}); bag.Put(p + "conv_in.conv.bias", {cfg.ch}); + // GroupNorm is affine and PixelNorm is parameter-free, so the `kGroup` arm + // carries a weight/bias pair AHEAD of each convolution the norm precedes. + auto put_norm = [&](const std::string& prefix, int64_t channels) { + if (!group) return; + bag.Put(prefix + ".weight", {channels}); + bag.Put(prefix + ".bias", {channels}); + }; auto put_resnet = [&](const std::string& prefix, int64_t in_ch, int64_t out_ch) { + put_norm(prefix + ".norm1", in_ch); bag.Put(prefix + ".conv1.conv.weight", {out_ch, in_ch, 3, 3}); bag.Put(prefix + ".conv1.conv.bias", {out_ch}); + put_norm(prefix + ".norm2", out_ch); bag.Put(prefix + ".conv2.conv.weight", {out_ch, out_ch, 3, 3}); bag.Put(prefix + ".conv2.conv.bias", {out_ch}); if (in_ch != out_ch) { @@ -1394,6 +1676,7 @@ ParamBag BuildAudioEncoderParams(const vllm::Ltx2AudioEncoderConfig& cfg) { } }; auto put_attn = [&](const std::string& prefix, int64_t channels) { + put_norm(prefix + ".norm", channels); for (const char* leaf : {"q", "k", "v", "proj_out"}) { bag.Put(prefix + "." + leaf + ".weight", {channels, channels, 1, 1}); bag.Put(prefix + "." + leaf + ".bias", {channels}); @@ -1433,6 +1716,7 @@ ParamBag BuildAudioEncoderParams(const vllm::Ltx2AudioEncoderConfig& cfg) { if (cfg.mid_block_add_attention) put_attn(p + "mid.attn_1", block_in); put_resnet(p + "mid.block_2", block_in, block_in); + put_norm(p + "norm_out", block_in); const int64_t conv_out_channels = cfg.double_z ? 2 * cfg.z_channels : cfg.z_channels; bag.Put(p + "conv_out.conv.weight", {conv_out_channels, block_in, 3, 3}); bag.Put(p + "conv_out.conv.bias", {conv_out_channels}); @@ -1764,6 +2048,50 @@ TEST_CASE("ltx2 vae: the AUDIO encoder's average-pool downsample arm") { CHECK(message.find("with_conv") != std::string::npos); } +TEST_CASE("ltx2 vae: the GROUP-NORM audio encoder matches upstream ltx_core") { + // The encoder half of the same hole: `Ltx2AudioEncoderConfig::norm_eps` was + // added by phase L11 and, like the decoder's, could not be read by any arm, + // because 7a/7b/7c all run `norm_type = kPixel`. Mutating it 1e-6 -> 1e-4 left + // all 33 cases green. This arm executes the GroupNorm branch of + // `build_normalization_layer` (normalization.py:56-57) at the only causality + // `ResnetBlock` permits it on (resnet.py:130-131). + const vllm::Ltx2AudioEncoderConfig cfg = ReducedAudioEncoderGroupConfig(); + ParamBag bag = BuildAudioEncoderParams(cfg); + CheckManifest(bag, vllm_test::kLtx2AudioEncGroupParamNames, + vllm_test::kLtx2AudioEncGroupParamCounts, + std::size(vllm_test::kLtx2AudioEncGroupParamNames)); + + const int64_t c = vllm_test::kLtx2AudioEncInC; + const int64_t t = vllm_test::kLtx2AudioEncInT; + const int64_t f = vllm_test::kLtx2AudioEncInF; + const std::vector spec = Ltx2Input("ltx2.audioenc.input", c * t * f, 1.0); + + const vllm::Ltx2AudioSpectrogram latent = + vllm::Ltx2AudioEncoderForward(cfg, bag.weights, spec, c, t, f); + CHECK(latent.channels == vllm_test::kLtx2AudioEncGroupOutC); + CHECK(latent.frames == vllm_test::kLtx2AudioEncGroupOutT); + CHECK(latent.mel_bins == vllm_test::kLtx2AudioEncGroupOutF); + + const double err = MaxAbsDiff(latent.data, vllm_test::kLtx2AudioEncGroupGolden, + std::size(vllm_test::kLtx2AudioEncGroupGolden)); + INFO("group-norm audio encoder max|diff| = " << err); + CHECK(err <= kLtx2GoldenTol); + + vllm::Ltx2AudioEncoderConfig bad = cfg; + bad.causality_axis = vllm::Ltx2CausalityAxis::kHeight; + bool threw = false; + std::string message; + try { + vllm::Ltx2AudioEncoderForward(bad, bag.weights, spec, c, t, f); + } catch (const std::exception& error) { + threw = true; + message = error.what(); + } + REQUIRE(threw); + INFO("refusal message: " << message); + CHECK(message.find("GroupNorm") != std::string::npos); +} + TEST_CASE("ltx2 vae: the slaney mel filterbank matches torchaudio") { // The filterbank is gated on its own because a wrong one makes every mel bin // wrong at once, and the resulting mismatch is impossible to localize from the diff --git a/tests/vllm/multimodal/test_ltx2_video.cpp b/tests/vllm/multimodal/test_ltx2_video.cpp index af5c6e268..405d81fd2 100644 --- a/tests/vllm/multimodal/test_ltx2_video.cpp +++ b/tests/vllm/multimodal/test_ltx2_video.cpp @@ -650,11 +650,29 @@ TEST_CASE("ltx2 video: keyframe and reference conditioning is refused by name") gen.first_frame_path = ws.paths.video_embeds; // any path: the refusal precedes the read try { (void)engine->Generate(gen); - FAIL("keyframe conditioning must be refused while the VAE encoder is unported"); + FAIL("keyframe conditioning must be refused while no encoder is reachable from here"); } catch (const std::exception& e) { const std::string msg = e.what(); INFO(msg); CHECK(msg.find("ImageConditioner") != std::string::npos); + // A refusal whose stated REASON has gone stale is worse than a vague one: it + // sends the next reader to build something that already exists. Phase L11 + // ported the video VAE encoder, so the message may no longer claim the + // encoder is missing, and these two assertions hold it to the pieces that + // actually are — the loader path that would put encoder weights in memory, + // and the CRF re-compression upstream applies before encoding. + CHECK(msg.find("VAE_ENCODER_COMFY_KEYS_FILTER") != std::string::npos); + CHECK(msg.find("default_image_crf") != std::string::npos); + // And the QUALIFIER on that re-compression, which the two substrings above do + // not reach: `preprocess` returns the image UNTOUCHED at `crf == 0` + // (media_io/decode.py:413-435, the `if crf == 0:` early return at :425-426 — + // NOT the one at :427-428, which is the degenerate-size guard), so "re-compresses + // before encoding" is only true of a nonzero resolved CRF. Naming the round + // trip without naming its exception overstates what is unported and sends the + // next reader to build an H.264 path for a case that needs none — the same + // failure mode as a stale reason, one step subtler. Gated here so deleting the + // qualifier goes RED rather than quietly restoring the overstatement. + CHECK(msg.find("unless that CRF is 0") != std::string::npos); } }