diff --git a/PLAN.md b/PLAN.md index 82e50107..6102dbe2 100755 --- a/PLAN.md +++ b/PLAN.md @@ -27,6 +27,18 @@ full-frame replay still diverges at prefill. Reproduce with the diff harness, bisect the first divergent talker tensor/op under identical teacher-forced history, then fix and validate on the real HIP path via the Kaggle regime. +## CLAIMED 2026-08-13 — #344 MOSS valid-frame metadata in stable C ABI + +Worktree: `.claude/worktrees/fix-344-moss-valid-frame-metadata` +(branch `fix/344-moss-valid-frame-metadata`). +Additive MOSS encoder/tap/adapter valid-frame metadata for the downstream +MOSS-Music-8B-Thinking feature pipeline: `moss_audio_plan_chunks`, +`moss_audio_compute_mel_meta` (preserves pre-pad `T_mel_actual` — never inferred +from padded zeros/floor), and `moss_audio_run_encoder_meta` (existing chunk loop +reused, caller-allocated per-chunk valid counts, adapter output dim reported from +GGUF weights, fail-closed llm_hidden vs adapter-row check). Existing +`moss_audio_*` symbols unchanged. Hermetic CPU test + live differential test. + ## LANDED 2026-08-10 — voxtral-tts pre-tokenizer parity (c69ac61b, from #338) Carried out of #338 after it closed. Reporter measured 5/57 vs `mistral-common`; diff --git a/src/moss_audio.cpp b/src/moss_audio.cpp index 331d6728..d60607d1 100644 --- a/src/moss_audio.cpp +++ b/src/moss_audio.cpp @@ -408,6 +408,23 @@ static bool moss_audio_load_model(moss_audio_model& model, moss_audio_vocab& voc model.adapter.up_w = require(model, "adapter.up.weight"); model.adapter.down_w = require(model, "adapter.down.weight"); + // Issue #344: the adapter's output-row shape comes from the weights, not a + // hardcoded 2560. down_w is (adapter_hidden, llm_hidden) in ggml ne terms, + // so ne[1] is the output feature count. Fail closed if the GGUF kv + // moss_audio.llm.hidden_size disagrees — never silently accept a 2560 + // fallback for a 4096-D MOSS-Music-8B-Thinking checkpoint. + { + const int adapter_out_rows = (int)model.adapter.down_w->ne[1]; + if ((int)model.hparams.llm_hidden != adapter_out_rows) { + fprintf(stderr, + "moss_audio: llm.hidden_size=%u disagrees with adapter down_proj " + "output rows=%d — refusing to load (no silent 2560 fallback for a " + "4096-D checkpoint)\n", + model.hparams.llm_hidden, adapter_out_rows); + return false; + } + } + // DeepStack mergers for (uint32_t i = 0; i < model.hparams.ds_num_taps && i < 3; i++) { char buf[64]; @@ -528,8 +545,8 @@ static void moss_audio_fft(float* in, int N, float* out) { // Mel spectrogram (Whisper-style, 128-bin) // =========================================================================== -extern "C" float* moss_audio_compute_mel(struct moss_audio_context* ctx, const float* samples, int n_samples, - int* out_n_mels, int* out_T_mel) { +static float* moss_audio_compute_mel_impl(struct moss_audio_context* ctx, const float* samples, int n_samples, + int* out_n_mels, int* out_T_mel, int* out_T_mel_actual) { if (!ctx || !samples || n_samples <= 0) return nullptr; const auto& hp = ctx->model.hparams; @@ -567,9 +584,14 @@ extern "C" float* moss_audio_compute_mel(struct moss_audio_context* ctx, const f mel_params.center_pad = true; mel_params.center_pad_reflect = true; // WhisperFeatureExtractor uses reflect padding + // T_mel_actual is the OBSERVED pre-pad frame count. It must survive to the + // caller so a downstream pipeline can separate real audio frames from the + // 3000-frame Whisper pad (issue #344); it is never re-derived from padded + // zeros or a global floor. int T_mel_actual = 0; std::vector mel_out = core_mel::compute(samples, n_samples, hann.data(), n_fft, mel_filters.data(), n_freqs, fft_fn, mel_params, T_mel_actual); + const int T_mel_pre_pad = T_mel_actual; // Pad to 3000 frames (30s Whisper convention) — WhisperFeatureExtractor // always pads to nb_max_frames=3000. The encoder chunks this into 400-frame @@ -594,9 +616,21 @@ extern "C" float* moss_audio_compute_mel(struct moss_audio_context* ctx, const f *out_n_mels = n_mels_val; if (out_T_mel) *out_T_mel = T_mel_actual; + if (out_T_mel_actual) + *out_T_mel_actual = T_mel_pre_pad; return result; } +extern "C" float* moss_audio_compute_mel(struct moss_audio_context* ctx, const float* samples, int n_samples, + int* out_n_mels, int* out_T_mel) { + return moss_audio_compute_mel_impl(ctx, samples, n_samples, out_n_mels, out_T_mel, nullptr); +} + +extern "C" float* moss_audio_compute_mel_meta(struct moss_audio_context* ctx, const float* samples, int n_samples, + int* out_n_mels, int* out_T_mel, int* out_T_mel_actual) { + return moss_audio_compute_mel_impl(ctx, samples, n_samples, out_n_mels, out_T_mel, out_T_mel_actual); +} + // =========================================================================== // Audio encoder graph // =========================================================================== @@ -606,6 +640,35 @@ static int conv_out_len(int L) { return (L - 1) / 2 + 1; } +// Per-chunk valid-frame bookkeeping for the encoder chunk loop (issue #344). +// chunk_frames=400 is fixed by the encoder architecture; a chunk's valid token +// count is 3× stride-2 conv downsampling of its real length. This is the ONLY +// source of per-chunk valid counts for both moss_audio_run_encoder and +// moss_audio_run_encoder_meta, so metadata can never drift from the loop. +// Returns the number of chunks (0 when T_mel <= 0). If valid_counts is +// non-null it must have capacity >= (T_mel + chunk_frames - 1) / chunk_frames. +extern "C" int moss_audio_plan_chunks(int T_mel, int* valid_counts, int* out_total_valid) { + if (T_mel <= 0) { + if (out_total_valid) + *out_total_valid = 0; + return 0; + } + const int chunk_frames = 400; + const int num_chunks = (T_mel + chunk_frames - 1) / chunk_frames; + + int total_valid = 0; + for (int c = 0; c < num_chunks; c++) { + const int chunk_len = std::min(chunk_frames, T_mel - c * chunk_frames); + const int valid = conv_out_len(conv_out_len(conv_out_len(chunk_len))); + if (valid_counts) + valid_counts[c] = valid; + total_valid += valid; + } + if (out_total_valid) + *out_total_valid = total_valid; + return num_chunks; +} + static ggml_cgraph* moss_audio_build_encoder_graph(moss_audio_context* ctx, int T_mel, bool capture_deepstack, ggml_context* arena_ctx = nullptr) { const auto& hp = ctx->model.hparams; @@ -895,14 +958,27 @@ static ggml_cgraph* moss_audio_build_adapter_graph(moss_audio_context* ctx, int // Encoder + Adapter execution // =========================================================================== -extern "C" float* moss_audio_run_encoder(struct moss_audio_context* ctx, const float* mel, int n_mels, int T_mel, - int* out_T_enc, int* out_d, float** ds_tap_0, float** ds_tap_1, - float** ds_tap_2) { - if (!ctx || !mel) +// Shared encoder executor for moss_audio_run_encoder and +// moss_audio_run_encoder_meta (issue #344). The chunk/valid bookkeeping is +// delegated to moss_audio_plan_chunks so the two entrypoints can never drift. +// +// mel : (n_mels, T_mel_stride) F32 row-major; T_mel_stride is the +// buffer's row stride (the padded length as returned by +// moss_audio_compute_mel/meta). +// T_chunk_end : mel columns to process — the FULL length for the reference- +// faithful moss_audio_run_encoder, or the pre-pad T_mel_actual +// for the content-only moss_audio_run_encoder_meta. +// want_ds[3] : per-tap capture flags. +// ds_results : out slots; entries with want_ds[t] are malloc'd + filled. +// valid_counts : optional caller buffer (capacity >= ceil(T_chunk_end/400)). +static float* moss_audio_run_encoder_impl(moss_audio_context* ctx, const float* mel, int n_mels, int T_mel_stride, + int T_chunk_end, const bool want_ds[3], float* ds_results[3], int* out_T_enc, + int* out_d, int* valid_counts, int* out_num_chunks, int* out_total_valid) { + if (!ctx || !mel || n_mels <= 0 || T_mel_stride <= 0 || T_chunk_end <= 0) return nullptr; const auto& hp = ctx->model.hparams; const int d = (int)hp.enc_d_model; - const bool want_ds = (ds_tap_0 || ds_tap_1 || ds_tap_2); + const bool capture_ds = (want_ds[0] || want_ds[1] || want_ds[2]); // Invalidate the §176s cached encoder graph across invocations (#215). // Reuse within the chunk loop below stays safe (identical allocs, no @@ -919,21 +995,31 @@ extern "C" float* moss_audio_run_encoder(struct moss_audio_context* ctx, const f // chunk (no cross-chunk attention), then selects valid output tokens. const int chunk_frames = 400; // n_window(200) * 2 - // Compute chunk boundaries - int num_chunks = (T_mel + chunk_frames - 1) / chunk_frames; - std::vector chunk_lengths(num_chunks, chunk_frames); - int tail = T_mel % chunk_frames; - if (tail > 0) - chunk_lengths[num_chunks - 1] = tail; - // If tail == 0, last chunk is full (chunk_frames) - - // For each chunk, compute valid output length after 3× stride-2 conv - std::vector valid_lens(num_chunks); + // Per-chunk valid-frame counts come from the single shared planner. + std::vector valid_lens; + int num_chunks = 0; int total_valid = 0; - for (int c = 0; c < num_chunks; c++) { - valid_lens[c] = conv_out_len(conv_out_len(conv_out_len(chunk_lengths[c]))); - total_valid += valid_lens[c]; + if (T_chunk_end > 0) { + valid_lens.assign((size_t)((T_chunk_end + chunk_frames - 1) / chunk_frames), 0); + num_chunks = moss_audio_plan_chunks(T_chunk_end, valid_lens.data(), &total_valid); + } + if (num_chunks == 0) { + if (out_T_enc) + *out_T_enc = 0; + if (out_d) + *out_d = d; + if (out_num_chunks) + *out_num_chunks = 0; + if (out_total_valid) + *out_total_valid = 0; + return nullptr; } + if (valid_counts) + memcpy(valid_counts, valid_lens.data(), (size_t)num_chunks * sizeof(int)); + if (out_num_chunks) + *out_num_chunks = num_chunks; + if (out_total_valid) + *out_total_valid = total_valid; // Padded chunk conv output length (all chunks padded to chunk_frames) const int T_chunk_down = conv_out_len(conv_out_len(conv_out_len(chunk_frames))); @@ -946,14 +1032,14 @@ extern "C" float* moss_audio_run_encoder(struct moss_audio_context* ctx, const f // Allocate output buffers float* result = (float*)malloc((size_t)d * total_valid * sizeof(float)); - float* ds_results[3] = {nullptr, nullptr, nullptr}; - if (want_ds) { - if (ds_tap_0) - ds_results[0] = (float*)malloc((size_t)d * total_valid * sizeof(float)); - if (ds_tap_1) - ds_results[1] = (float*)malloc((size_t)d * total_valid * sizeof(float)); - if (ds_tap_2) - ds_results[2] = (float*)malloc((size_t)d * total_valid * sizeof(float)); + float* ds_results_alloc[3] = {nullptr, nullptr, nullptr}; + if (capture_ds) { + for (int t = 0; t < 3; t++) { + if (want_ds[t]) { + ds_results_alloc[t] = (float*)malloc((size_t)d * total_valid * sizeof(float)); + ds_results[t] = ds_results_alloc[t]; + } + } } int out_offset = 0; // write position in output buffers @@ -961,17 +1047,17 @@ extern "C" float* moss_audio_run_encoder(struct moss_audio_context* ctx, const f for (int c = 0; c < num_chunks; c++) { // Prepare padded mel chunk for ggml ne=(T=chunk_frames, n_mels). // ggml ne[0]=T varies fastest: data[t + chunk_frames * f]. - // Input mel is (n_mels, T_mel) row-major: mel[f * T_mel + t]. + // Input mel is (n_mels, T_mel_stride) row-major: mel[f * T_mel_stride + t]. std::vector chunk_mel((size_t)n_mels * chunk_frames, 0.0f); int t_start = c * chunk_frames; - int t_len = chunk_lengths[c]; + int t_len = std::min(chunk_frames, T_chunk_end - t_start); // Pack mel with freq (n_mels) as ne[0] (fastest) and time as ne[1]. // This transposes from (n_mels, T) row-major to (T, n_mels) ne-order // = ggml ne=(n_mels, T). ggml conv2d applies KW (kernel ne[0]) along // data ne[0]=n_mels and KH (ne[1]) along data ne[1]=T. for (int t = 0; t < t_len; t++) { for (int f = 0; f < n_mels; f++) { - chunk_mel[(size_t)f + n_mels * (size_t)t] = mel[(size_t)f * T_mel + t_start + t]; + chunk_mel[(size_t)f + n_mels * (size_t)t] = mel[(size_t)f * T_mel_stride + t_start + t]; } } @@ -986,7 +1072,7 @@ extern "C" float* moss_audio_run_encoder(struct moss_audio_context* ctx, const f ctx->cached_enc_meta.assign(ctx->compute_meta.size(), 0); ggml_init_params aip = {ctx->cached_enc_meta.size(), ctx->cached_enc_meta.data(), true}; ctx->cached_enc_ctx = ggml_init(aip); - gf = moss_audio_build_encoder_graph(ctx, chunk_frames, want_ds, ctx->cached_enc_ctx); + gf = moss_audio_build_encoder_graph(ctx, chunk_frames, capture_ds, ctx->cached_enc_ctx); ctx->cached_enc_gf = gf; ctx->cached_enc_T_mel = chunk_frames; } @@ -994,8 +1080,10 @@ extern "C" float* moss_audio_run_encoder(struct moss_audio_context* ctx, const f if (!ggml_backend_sched_alloc_graph(ctx->sched, gf)) { fprintf(stderr, "moss_audio: encoder graph alloc failed (chunk %d)\n", c); free(result); - for (int t = 0; t < 3; t++) - free(ds_results[t]); + for (int t = 0; t < 3; t++) { + free(ds_results_alloc[t]); + ds_results[t] = nullptr; // never leave a freed pointer behind (issue #344 B1) + } return nullptr; } @@ -1046,8 +1134,10 @@ extern "C" float* moss_audio_run_encoder(struct moss_audio_context* ctx, const f if (ggml_backend_sched_graph_compute(ctx->sched, gf) != GGML_STATUS_SUCCESS) { fprintf(stderr, "moss_audio: encoder graph compute failed (chunk %d)\n", c); free(result); - for (int t = 0; t < 3; t++) - free(ds_results[t]); + for (int t = 0; t < 3; t++) { + free(ds_results_alloc[t]); + ds_results[t] = nullptr; // never leave a freed pointer behind (issue #344 B1) + } return nullptr; } @@ -1071,8 +1161,10 @@ extern "C" float* moss_audio_run_encoder(struct moss_audio_context* ctx, const f if (!enc_out) { fprintf(stderr, "moss_audio: missing encoder_output (chunk %d)\n", c); free(result); - for (int t = 0; t < 3; t++) - free(ds_results[t]); + for (int t = 0; t < 3; t++) { + free(ds_results_alloc[t]); + ds_results[t] = nullptr; // never leave a freed pointer behind (issue #344 B1) + } return nullptr; } @@ -1082,15 +1174,15 @@ extern "C" float* moss_audio_run_encoder(struct moss_audio_context* ctx, const f ggml_backend_tensor_get(enc_out, result + (size_t)out_offset * d, 0, (size_t)valid * d * sizeof(float)); // Extract deepstack taps (same valid subset) - if (want_ds) { + if (capture_ds) { for (int t = 0; t < 3; t++) { - if (!ds_results[t]) + if (!ds_results_alloc[t]) continue; char name[32]; snprintf(name, sizeof(name), "ds_tap_%d", t); ggml_tensor* tap = ggml_graph_get_tensor(gf, name); if (tap) { - ggml_backend_tensor_get(tap, ds_results[t] + (size_t)out_offset * d, 0, + ggml_backend_tensor_get(tap, ds_results_alloc[t] + (size_t)out_offset * d, 0, (size_t)valid * d * sizeof(float)); } } @@ -1103,22 +1195,67 @@ extern "C" float* moss_audio_run_encoder(struct moss_audio_context* ctx, const f *out_T_enc = total_valid; if (out_d) *out_d = d; - if (ds_tap_0) - *ds_tap_0 = ds_results[0]; - if (ds_tap_1) - *ds_tap_1 = ds_results[1]; - if (ds_tap_2) - *ds_tap_2 = ds_results[2]; return result; } +extern "C" float* moss_audio_run_encoder(struct moss_audio_context* ctx, const float* mel, int n_mels, int T_mel, + int* out_T_enc, int* out_d, float** ds_tap_0, float** ds_tap_1, + float** ds_tap_2) { + const bool want_ds[3] = {ds_tap_0 != nullptr, ds_tap_1 != nullptr, ds_tap_2 != nullptr}; + float* ds_results[3] = {nullptr, nullptr, nullptr}; + float* r = moss_audio_run_encoder_impl(ctx, mel, n_mels, T_mel, T_mel, want_ds, ds_results, out_T_enc, out_d, + nullptr, nullptr, nullptr); + // On failure the impl frees any allocated tap buffers; write the tap + // out-pointers ONLY on success so *ds_tap_x can never dangle (issue #344, + // B1). This matches the pre-refactor contract: taps were only ever + // published after the chunk loop completed. + if (r != nullptr) { + if (ds_tap_0) + *ds_tap_0 = ds_results[0]; + if (ds_tap_1) + *ds_tap_1 = ds_results[1]; + if (ds_tap_2) + *ds_tap_2 = ds_results[2]; + } + return r; +} + +extern "C" float* moss_audio_run_encoder_meta(struct moss_audio_context* ctx, const float* mel, int n_mels, int T_mel, + int T_mel_actual, int* out_T_enc, int* out_d, int* valid_counts, + int* out_num_chunks, int* out_T_mel_actual, int* out_total_valid, + float** ds_tap_0, float** ds_tap_1, float** ds_tap_2) { + // Fail closed: T_mel_actual must be the caller's observed pre-pad length. + // It is NEVER inferred from padded zeros or a global floor here. + if (!ctx || !mel || n_mels <= 0 || T_mel <= 0 || T_mel_actual <= 0 || T_mel_actual > T_mel) + return nullptr; + if (out_T_mel_actual) + *out_T_mel_actual = T_mel_actual; + + const bool want_ds[3] = {ds_tap_0 != nullptr, ds_tap_1 != nullptr, ds_tap_2 != nullptr}; + float* ds_results[3] = {nullptr, nullptr, nullptr}; + float* r = moss_audio_run_encoder_impl(ctx, mel, n_mels, T_mel, T_mel_actual, want_ds, ds_results, out_T_enc, out_d, + valid_counts, out_num_chunks, out_total_valid); + // Same failure contract as moss_audio_run_encoder: never publish tap + // out-pointers on failure (the impl has freed them). + if (r != nullptr) { + if (ds_tap_0) + *ds_tap_0 = ds_results[0]; + if (ds_tap_1) + *ds_tap_1 = ds_results[1]; + if (ds_tap_2) + *ds_tap_2 = ds_results[2]; + } + return r; +} + extern "C" float* moss_audio_run_adapter(struct moss_audio_context* ctx, const float* encoder_out, int T_enc, int d_enc, int* out_T, int* out_d) { if (!ctx || !encoder_out) return nullptr; - const auto& hp = ctx->model.hparams; - const int d_llm = (int)hp.llm_hidden; + // Adapter output dim reported from the GGUF weight shape (ne[1] = output + // rows); == moss_audio.llm.hidden_size, enforced at load time (#344). + const int d_llm = (int)ctx->model.adapter.down_w->ne[1]; ggml_cgraph* gf = moss_audio_build_adapter_graph(ctx, T_enc); ggml_backend_sched_reset(ctx->sched); diff --git a/src/moss_audio.h b/src/moss_audio.h index 44a2d7ef..80dc01cf 100644 --- a/src/moss_audio.h +++ b/src/moss_audio.h @@ -1,15 +1,34 @@ -// moss_audio.h — public C API for MOSS-Audio-4B-Instruct ggml runtime +// moss_audio.h — public C API for the MOSS-Audio / MOSS-Music ggml runtime // // Audio understanding (ASR + audio QA + scene description) using a 32-layer // Whisper-style encoder with DeepStack 3-tap cross-layer injection + -// 36-layer Qwen3 LLM. Models loaded from GGUF files produced by: +// Qwen3 LLM. Models loaded from GGUF files produced by: // `python models/convert-moss-audio-to-gguf.py --input --output X.gguf` // -// Architecture: OpenMOSS-Team/MOSS-Audio-4B-Instruct (Apache-2.0) +// Architecture: OpenMOSS-Team/MOSS-Audio-4B-Instruct (Apache-2.0); +// the same encoder/tap/adapter structure serves +// MOSS-Music-8B-Thinking (the adapter's llm.hidden_size is then 4096). // Audio encoder: 128-mel → 3×Conv2d(stride 2) → stem_proj → 32 WhisperEncoderLayer // DeepStack: taps at L8/L16/L24 → 3× GatedMLP → residual inject at LM L0/L1/L2 -// Audio adapter: GatedMLP(1280→8192→2560) for final encoder output -// LM: 36-layer Qwen3 (2560d, 32Q/8KV, head_dim=128, QK-norm, SwiGLU, RoPE θ=1M) +// Audio adapter: GatedMLP(1280→8192→llm.hidden_size) for final encoder output +// LM: 36-layer Qwen3 (d=llm.hidden_size, 32Q/8KV, head_dim=128, QK-norm, +// SwiGLU, RoPE θ=1M) +// +// The GGUF kv `moss_audio.llm.hidden_size` is 2560 for MOSS-Audio-4B and 4096 +// for MOSS-Music-8B-Thinking. It is never hardcoded here: load fails closed if +// it disagrees with the adapter's output-row shape. +// +// Ownership & threading contract (all stage helpers below): +// * Returned buffers are malloc'd deep copies; the caller owns them and +// must free() them. Nothing aliases ggml graph or backend memory. +// * Copies are synchronous: the helper returns only after the stage is +// fully computed and copied to the returned buffer. +// * One context = one in-flight stage run. A context is not re-entrant and +// not thread-safe: callbacks run synchronously on the caller thread and +// must not re-enter the same context; the encoder's cached graph is +// invalidated across invocations (see below), so a second call from a +// callback or another thread corrupts state. Use one context per +// concurrent consumer. #pragma once @@ -49,15 +68,85 @@ char* moss_audio_transcribe(struct moss_audio_context* ctx, const float* samples // Compute 128-bin log-mel spectrogram (Whisper-style). // Output: malloc'd (n_mels, T_mel) F32 row-major. Caller frees. +// T_mel is the mel length the encoder consumes, padded to the 3000-frame +// (30s Whisper) convention. float* moss_audio_compute_mel(struct moss_audio_context* ctx, const float* samples, int n_samples, int* out_n_mels, int* out_T_mel); +// Like moss_audio_compute_mel(), but also reports the PRE-PAD mel length. +// The returned tensor and *out_T_mel are identical to the plain variant; +// *out_T_mel_actual is the true frame count before the 3000-frame pad +// (0 means the input produced no frames). This is the only truthful way to +// distinguish real audio frames from frames created purely by the pad — +// T_mel_actual is observed during computation, never inferred from padded +// zeros or a global floor. Pass it straight to moss_audio_run_encoder_meta(). +float* moss_audio_compute_mel_meta(struct moss_audio_context* ctx, const float* samples, int n_samples, int* out_n_mels, + int* out_T_mel, int* out_T_mel_actual); + // Run audio encoder only. Returns (T_enc, d_model=1280) F32 row-major. // Also fills deepstack taps if ds_tap_0/1/2 are non-null (each T_enc × 1280). +// The encoder chunks the FULL mel buffer (typically the 3000-frame padded +// length passed as T_mel) into 400-frame pieces (3× stride-2 conv +// downsampling, 50 valid tokens per full chunk) and selects only valid +// output tokens — so for a padded input this includes the pad chunks, exactly +// as the Python reference does. Use moss_audio_run_encoder_meta() to get only +// real-content frames with truthful valid-frame metadata. +// On failure (NULL return) *ds_tap_0/1/2 are NOT modified: they never receive +// a dangling pointer. Check the return value before using or freeing taps. float* moss_audio_run_encoder(struct moss_audio_context* ctx, const float* mel, int n_mels, int T_mel, int* out_T_enc, int* out_d, float** ds_tap_0, float** ds_tap_1, float** ds_tap_2); -// Run audio adapter on encoder output. Returns (T_enc, llm_dim=2560) F32. +// Pure valid-frame bookkeeping for the encoder chunk loop — no model state. +// chunk_frames = 400; each chunk's valid token count is 3× stride-2 conv +// downsampling of its real length. Returns the number of chunks (0 when +// T_mel <= 0). When valid_counts is non-null it must have capacity at least +// (T_mel + 399) / 400; the first num_chunks entries are filled with per-chunk +// valid counts. *out_total_valid receives sum(valid_counts). This is the ONLY +// source of per-chunk valid counts for both moss_audio_run_encoder and +// moss_audio_run_encoder_meta, so their metadata can never drift from the +// counts the chunk loop actually computes. +int moss_audio_plan_chunks(int T_mel, int* valid_counts, int* out_total_valid); + +// Metadata-returning encoder entrypoint. Same 1280-D encoder and DeepStack +// taps 8/16/24 as moss_audio_run_encoder(), but it EXECUTES AND TRIMS using +// exactly T_mel_actual: only real-content frames are computed and returned. +// The 3000-frame Whisper pad never contributes a frame, an attention key, or +// a valid count — consumers need no post-hoc cutting. +// +// mel, n_mels, T_mel : the padded (n_mels, T_mel) mel buffer as returned by +// moss_audio_compute_mel_meta() — T_mel is the buffer's +// row stride (typically the padded 3000). +// T_mel_actual : the PRE-PAD content length (the same value +// moss_audio_compute_mel_meta() reports). Validated to +// 1 <= T_mel_actual <= T_mel; anything else fails closed +// (returns NULL). Never inferred from padded zeros or a +// global floor. +// valid_counts : caller-allocated int array with capacity at least +// ceil(T_mel_actual / 400). Filled with the per-chunk +// valid-frame counts of the REAL chunks; may be NULL. +// *out_num_chunks : == ceil(T_mel_actual / 400), the number of valid_counts +// entries written. +// *out_total_valid : == sum(valid_counts) == *out_T_enc. +// *out_T_enc : real-content frame count, == out_total_valid. +// *out_d : encoder dim (1280). +// *out_T_mel_actual : receives the validated T_mel_actual actually used. +// ds_tap_0/1/2 : real-content DeepStack taps, each *out_T_enc × 1280. +// +// When the input is genuinely >= 30 s (T_mel_actual == T_mel, no pad), this is +// byte-identical to moss_audio_run_encoder(). For shorter inputs it differs by +// design: run_encoder preserves the reference's padded full-length behaviour, +// while this entrypoint returns only content. +// On failure (NULL return, including fail-closed T_mel_actual validation) +// *ds_tap_0/1/2 are NOT modified: they never receive a dangling pointer. +float* moss_audio_run_encoder_meta(struct moss_audio_context* ctx, const float* mel, int n_mels, int T_mel, + int T_mel_actual, int* out_T_enc, int* out_d, int* valid_counts, int* out_num_chunks, + int* out_T_mel_actual, int* out_total_valid, float** ds_tap_0, float** ds_tap_1, + float** ds_tap_2); + +// Run audio adapter on encoder output. Returns (T_enc, llm_dim) F32 where +// llm_dim is the adapter output-row count read from the GGUF weights +// (== moss_audio.llm.hidden_size; 2560 for MOSS-Audio-4B, 4096 for +// MOSS-Music-8B-Thinking). *out_d reports that same weight-derived value. float* moss_audio_run_adapter(struct moss_audio_context* ctx, const float* encoder_out, int T_enc, int d_enc, int* out_T, int* out_d); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f3430aef..dba1238d 100755 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1302,6 +1302,23 @@ catch_discover_tests(test-moss-audio WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" ) +# ─── test-moss-audio-valid-meta — hermetic (no model) valid-frame metadata (issue #344) +add_executable(test-moss-audio-valid-meta + test-moss-audio-valid-meta.cpp +) +target_include_directories(test-moss-audio-valid-meta PRIVATE + ${PROJECT_SOURCE_DIR}/src +) +target_link_libraries(test-moss-audio-valid-meta PRIVATE + Catch2::Catch2WithMain + moss_audio +) +catch_discover_tests(test-moss-audio-valid-meta + TEST_SPEC "[moss-audio-valid-meta]" + PROPERTIES + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" +) + # ─── test - moss-transcribe — live ASR integration test for MOSS-Transcribe-preview-2B add_executable(test-moss-transcribe test_moss_transcribe_live.cpp diff --git a/tests/test-moss-audio-valid-meta.cpp b/tests/test-moss-audio-valid-meta.cpp new file mode 100644 index 00000000..029c288f --- /dev/null +++ b/tests/test-moss-audio-valid-meta.cpp @@ -0,0 +1,160 @@ +// Hermetic CPU test for the issue #344 valid-frame metadata. +// +// moss_audio_plan_chunks() is pure arithmetic (no model state): per-chunk +// valid-frame counts for the MOSS encoder's 400-frame chunk loop, where each +// chunk's valid token count is 3× stride-2 conv downsampling of its real +// length. These tests pin the invariants the downstream MOSS-Music feature +// pipeline relies on: +// +// 1. sum(per-chunk valid) == out_total_valid (metadata is internally +// consistent with what the encoder loop actually computes). +// 2. num_chunks == ceil(T_mel / 400) and per-chunk values == conv_len³ +// of the chunk's real length. +// 3. Sub-30s pad exclusion: a content length under 3000 yields strictly +// fewer valid frames than the same mel run through the 3000-frame +// Whisper pad — the pad never fabricates frames that a consumer would +// mistake for content. The valid count is derived from the real chunk +// lengths, never from a padded-zeros heuristic or a global floor. + +#include + +#include "moss_audio.h" + +#include + +namespace { + +// 3× stride-2 conv downsampling (conv_len(L) = (L-1)/2 + 1), replicated here +// ONLY to independently predict plan_chunks' output — the function under test +// is the production source of truth, this is the oracle. +int conv_len(int L) { + return (L - 1) / 2 + 1; +} +int conv_len3(int L) { + return conv_len(conv_len(conv_len(L))); +} + +} // namespace + +TEST_CASE("plan_chunks: sum(valid) == total and per-chunk math", "[moss-audio-valid-meta]") { + const int kChunk = 400; + const int cases[] = {1, 2, 50, 399, 400, 401, 799, 800, 801, 1199, 1200, 3000, 3001, 3999, 4000, 4001, 12345}; + + for (int T_mel : cases) { + const int expect_chunks = (T_mel + kChunk - 1) / kChunk; + std::vector counts(expect_chunks, -1); + int total = -1; + CAPTURE(T_mel); + REQUIRE(moss_audio_plan_chunks(T_mel, counts.data(), &total) == expect_chunks); + + int sum = 0; + for (int c = 0; c < expect_chunks; c++) { + const int chunk_len = std::min(kChunk, T_mel - c * kChunk); + INFO("chunk " << c << " len " << chunk_len); + REQUIRE(counts[c] == conv_len3(chunk_len)); + sum += counts[c]; + } + REQUIRE(sum == total); + } +} + +TEST_CASE("plan_chunks: invalid input fails closed, null counts allowed", "[moss-audio-valid-meta]") { + int total = 999; + int counts[4] = {-1, -1, -1, -1}; + + REQUIRE(moss_audio_plan_chunks(0, counts, &total) == 0); + REQUIRE(total == 0); + REQUIRE(moss_audio_plan_chunks(-5, nullptr, &total) == 0); + REQUIRE(total == 0); + + // null valid_counts is permitted: only num_chunks / total matter. + REQUIRE(moss_audio_plan_chunks(800, nullptr, &total) == 2); + REQUIRE(total == 100); +} + +TEST_CASE("sub-30s pad exclusion: plan_chunks over T_mel_actual, never the pad", "[moss-audio-valid-meta]") { + // The encoder mel buffer is padded to 3000 frames. Planning over the + // PADDED length reports valid frames that include zero-content pad chunks. + std::vector padded_counts(8, -1); + int padded_total = -1; + REQUIRE(moss_audio_plan_chunks(3000, padded_counts.data(), &padded_total) == 8); + REQUIRE(padded_total == 375); + + // conv_len³ collapses like ceil(L/8), so near 30s a 1-7 frame pad tail can + // land on the same token count as the real tail (e.g. 2999 -> 375, equal + // to the padded path). The universal invariant is therefore "never more + // than the padded path"; a strictly-lower count is guaranteed whenever a + // whole 400-frame pad chunk exists (T_actual <= 2800). + for (int T_actual : {1, 2, 100, 399, 400, 401, 799, 800, 801, 1001, 1500, 2400, 2800, 2999}) { + const int n = (T_actual + 399) / 400; + std::vector counts(n, -1); + int total = -1; + INFO("T_actual=" << T_actual); + REQUIRE(moss_audio_plan_chunks(T_actual, counts.data(), &total) == n); + REQUIRE(total > 0); + REQUIRE(total <= padded_total); // pad must never fabricate a frame the meta path counts + if (T_actual <= 2800) + REQUIRE(total < padded_total); // whole pad chunks exist and are excluded + if (T_actual >= 400) + REQUIRE(counts[0] == 50); // every full chunk is 50 valid tokens + } + + // >= 30 s (no pad) must NOT fabricate pad-only frames. + int full = -1; + REQUIRE(moss_audio_plan_chunks(3000, nullptr, &full) == 8); + REQUIRE(full == 375); + REQUIRE(full == padded_total); +} + +TEST_CASE("encoder wrappers: ds_tap out-pointers untouched on failure (issue #344 B1)", "[moss-audio-valid-meta]") { + // B1 regression guard. The run_encoder/run_encoder_meta wrappers must NEVER + // write *ds_tap_x when the encoder impl fails: pre-fix, both wrappers + // published the impl's (possibly freed) tap slots on EVERY failure path, + // leaving callers with dangling non-NULL pointers on a NULL return. + // + // The deep failure paths (graph alloc / graph compute / missing + // encoder_output) require a loaded model and a ggml backend and cannot be + // forced hermetically. This pins the wrapper invariant at the observable + // boundary instead: any NULL-returning call must leave caller-provided tap + // out-pointers byte-for-byte untouched. Pre-fix this test fails (the + // sentinels are overwritten with NULL); post-fix it passes. + float dummy0 = 0.0f, dummy1 = 0.0f, dummy2 = 0.0f; + int T_enc = -1, d = -1; + + // run_encoder: NULL ctx => impl early-validation failure => NULL return. + float* ds0 = &dummy0; + float* ds1 = &dummy1; + float* ds2 = &dummy2; + float* r = moss_audio_run_encoder(nullptr, nullptr, 0, 0, &T_enc, &d, &ds0, &ds1, &ds2); + REQUIRE(r == nullptr); + REQUIRE(ds0 == &dummy0); + REQUIRE(ds1 == &dummy1); + REQUIRE(ds2 == &dummy2); + + // Same, with tap requests nominally enabled and garbage mel args. + ds0 = &dummy0; + ds1 = &dummy1; + ds2 = &dummy2; + T_enc = -1; + d = -1; + float sample = 0.0f; + r = moss_audio_run_encoder(nullptr, &sample, 1, 400, &T_enc, &d, &ds0, &ds1, &ds2); + REQUIRE(r == nullptr); + REQUIRE(ds0 == &dummy0); + REQUIRE(ds1 == &dummy1); + REQUIRE(ds2 == &dummy2); + + // run_encoder_meta: fail-closed NULL return => taps untouched. + ds0 = &dummy0; + ds1 = &dummy1; + ds2 = &dummy2; + T_enc = -1; + d = -1; + int nc = -1, tot = -1, echo = -1; + r = moss_audio_run_encoder_meta(nullptr, &sample, 1, 3000, 100, &T_enc, &d, nullptr, &nc, &echo, &tot, &ds0, &ds1, + &ds2); + REQUIRE(r == nullptr); + REQUIRE(ds0 == &dummy0); + REQUIRE(ds1 == &dummy1); + REQUIRE(ds2 == &dummy2); +} diff --git a/tests/test_moss_audio_live.cpp b/tests/test_moss_audio_live.cpp index c3038348..5bd255e2 100644 --- a/tests/test_moss_audio_live.cpp +++ b/tests/test_moss_audio_live.cpp @@ -85,3 +85,95 @@ TEST_CASE("moss-audio custom prompt", "[integration][moss-audio]") { moss_audio_free(ctx); } + +TEST_CASE("moss-audio run_encoder_meta valid-frame metadata (issue #344)", "[integration][moss-audio]") { + const char* model_path = std::getenv("CRISPASR_MODEL_MOSS_AUDIO"); + if (!model_path || !*model_path) { + SKIP("CRISPASR_MODEL_MOSS_AUDIO not set"); + } + + auto params = moss_audio_context_default_params(); + params.verbosity = 0; + auto* ctx = moss_audio_init_from_file(model_path, params); + REQUIRE(ctx != nullptr); + + auto pcm = load_wav_16k("samples/jfk.wav"); + REQUIRE(!pcm.empty()); + + // 1. mel with truthful pre-pad length + int n_mels = 0, T_mel = 0, T_mel_actual = 0; + float* mel = moss_audio_compute_mel_meta(ctx, pcm.data(), (int)pcm.size(), &n_mels, &T_mel, &T_mel_actual); + REQUIRE(mel != nullptr); + REQUIRE(n_mels == 128); + REQUIRE(T_mel_actual > 0); + // jfk.wav is ~10s, so the pre-pad length must be strictly below the pad. + REQUIRE(T_mel_actual < 3000); + REQUIRE(T_mel == 3000); + + // 2. metadata entrypoint: content-only frames + per-chunk valid counts + std::vector valid_counts((size_t)((T_mel_actual + 399) / 400), -1); + int T_enc = 0, d_enc = 0, num_chunks = -1, echo = -1, total_valid = -1; + float *ds0 = nullptr, *ds1 = nullptr, *ds2 = nullptr; + float* enc = moss_audio_run_encoder_meta(ctx, mel, n_mels, T_mel, T_mel_actual, &T_enc, &d_enc, valid_counts.data(), + &num_chunks, &echo, &total_valid, &ds0, &ds1, &ds2); + REQUIRE(enc != nullptr); + REQUIRE(d_enc == 1280); + REQUIRE(echo == T_mel_actual); + REQUIRE(num_chunks == (int)((T_mel_actual + 399) / 400)); + + int sum_valid = 0; + for (int c = 0; c < num_chunks; c++) { + REQUIRE(valid_counts[c] > 0); + sum_valid += valid_counts[c]; + } + // core invariant: sum(per-chunk valid) == out_total_valid == out_T_enc + REQUIRE(sum_valid == total_valid); + REQUIRE(total_valid == T_enc); + + // taps are content-only too + REQUIRE(ds0 != nullptr); + REQUIRE(ds1 != nullptr); + REQUIRE(ds2 != nullptr); + + // 3. differential vs the reference-faithful entrypoint: the padded path + // must expose MORE frames (pad chunks), never fewer. + int T_enc_ref = 0, d_ref = 0; + float* enc_ref = moss_audio_run_encoder(ctx, mel, n_mels, T_mel, &T_enc_ref, &d_ref, nullptr, nullptr, nullptr); + REQUIRE(enc_ref != nullptr); + REQUIRE(T_enc_ref > T_enc); // pad-only frames excluded from the meta path + + // 4. differential: no-pad input (T_mel_actual == T_mel) must be + // byte-identical to moss_audio_run_encoder. + int T_enc_full = 0, d_full = 0, num_full = -1, total_full = -1; + float* enc_full = moss_audio_run_encoder_meta(ctx, mel, n_mels, T_mel, T_mel, &T_enc_full, &d_full, nullptr, + &num_full, &echo, &total_full, nullptr, nullptr, nullptr); + REQUIRE(enc_full != nullptr); + REQUIRE(T_enc_full == T_enc_ref); + REQUIRE(total_full == T_enc_full); + REQUIRE(num_full == (int)((T_mel + 399) / 400)); + REQUIRE(memcmp(enc_full, enc_ref, (size_t)T_enc_full * d_full * sizeof(float)) == 0); + + // 5. adapter dims report from the GGUF (2560 for MOSS-Audio-4B) and stay + // row-consistent with the meta output. + int adapt_T = 0, adapt_d = 0; + float* adapted = moss_audio_run_adapter(ctx, enc, T_enc, d_enc, &adapt_T, &adapt_d); + REQUIRE(adapted != nullptr); + REQUIRE(adapt_T == T_enc); + REQUIRE(adapt_d > 0); + + // 6. fail closed on out-of-range T_mel_actual (never inferred here) + REQUIRE(moss_audio_run_encoder_meta(ctx, mel, n_mels, T_mel, 0, nullptr, nullptr, nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr) == nullptr); + REQUIRE(moss_audio_run_encoder_meta(ctx, mel, n_mels, T_mel, T_mel + 1, nullptr, nullptr, nullptr, nullptr, nullptr, + nullptr, nullptr, nullptr, nullptr) == nullptr); + + free(mel); + free(enc); + free(enc_ref); + free(enc_full); + free(adapted); + free(ds0); + free(ds1); + free(ds2); + moss_audio_free(ctx); +}