diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..45c86b6c8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,140 @@ +name: Bug report +description: Report a reproducible Hebrus runtime, build, API, model-loading, or compatibility problem. +title: "[Bug]: " +body: + - type: markdown + attributes: + value: | + Thanks for helping improve Hebrus. Search existing issues before filing a report. + + For a possible vulnerability, stop here and follow the [security policy](https://github.com/andreaborio/ds4/blob/main/SECURITY.md). Do not post vulnerability details, secrets, private data, model weights, or model blobs in an issue. + + - type: checkboxes + id: prerequisites + attributes: + label: Prerequisites + options: + - label: I searched the open and closed issues for an existing report. + required: true + - label: This is not an undisclosed security vulnerability. + required: true + - label: I will not attach model weights, model blobs, secrets, or private data. + required: true + + - type: textarea + id: summary + attributes: + label: Problem summary + description: Describe the failure and its impact in a few precise sentences. + placeholder: What failed, where, and how does it affect you? + validations: + required: true + + - type: dropdown + id: component + attributes: + label: Affected area + options: + - CLI or inference engine + - Server or API + - Agent mode + - Metal backend + - SSD streaming or ExpertMajor + - GGUF, conversion, or model download + - Build, tests, or documentation + - Compatibility alias or legacy interface + - Other + validations: + required: true + + - type: textarea + id: build + attributes: + label: Build identity + description: Include the exact commit, whether the tree is dirty, and `./hebrus --build-info` when available. + render: shell + placeholder: | + git rev-parse HEAD + git status --short + ./hebrus --build-info + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Environment and resolved execution plan + description: Include Mac model, Apple chip, RAM, macOS build, power mode, selected backend, storage, and the resolved resident/SSD plan. For non-production reference paths, identify the backend explicitly. + placeholder: | + Hardware: + Memory: + OS build: + Power mode: + Storage/model volume: + Backend and resolved plan: + validations: + required: true + + - type: textarea + id: model + attributes: + label: Model artifact identity + description: If model-backed, include family, exact filename, byte size, SHA-256, provenance, and quantization. Do not upload or link redistributed weights unless you are authorized to do so. Write "Not model-backed" when inapplicable. + placeholder: | + Family: + Filename: + Size in bytes: + SHA-256: + Provenance/conversion: + Quantization: + validations: + required: true + + - type: textarea + id: reproduce + attributes: + label: Minimal reproduction + description: Provide the smallest sanitized command or request that reproduces the problem, including relevant flags and environment variables. + render: shell + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected behavior + description: What should have happened? + validations: + required: true + + - type: textarea + id: actual + attributes: + label: Actual behavior and diagnostics + description: Include the exact error and the smallest useful log excerpt. Redact tokens, paths containing personal information, prompts, and private data. + validations: + required: true + + - type: dropdown + id: executable + attributes: + label: Entry point used + options: + - hebrus + - ds4 compatibility alias + - Library or embedded API + - Other + validations: + required: true + + - type: textarea + id: compatibility + attributes: + label: Compatibility observations + description: Note whether the same problem occurs through the canonical Hebrus entry point and any applicable `ds4*` compatibility alias. Include API, exit-code, environment-variable, or on-disk compatibility details when relevant. + + - type: textarea + id: additional + attributes: + label: Additional context + description: Add links to minimal source patches or public logs if useful. Do not attach model files, binaries, large generated outputs, or sensitive data. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..8ab4777ee --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Security policy + url: https://github.com/andreaborio/ds4/blob/main/SECURITY.md + about: Read the private-reporting status before reporting a vulnerability. Do not disclose vulnerability details in a public issue. + - name: Contribution guide + url: https://github.com/andreaborio/ds4/blob/main/CONTRIBUTING.md + about: Review the contribution, validation, evidence, and compatibility requirements before opening a pull request. diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml new file mode 100644 index 000000000..9aba42c6a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -0,0 +1,26 @@ +name: Documentation +description: Report missing, incorrect, or unclear Hebrus documentation +title: "[Docs]: " +labels: ["documentation"] +body: + - type: input + id: location + attributes: + label: Page or section + validations: + required: true + - type: textarea + id: problem + attributes: + label: What is wrong or unclear? + validations: + required: true + - type: textarea + id: proposed + attributes: + label: Suggested correction + description: Paste-ready wording is welcome, but not required. + - type: input + id: version + attributes: + label: Hebrus version or commit diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..20076297b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,109 @@ +name: Feature request +description: Propose a focused Hebrus capability or support-contract change. +title: "[Feature]: " +body: + - type: markdown + attributes: + value: | + Explain the user problem before the implementation. Hebrus prioritizes Apple Metal inference and SSD streaming while preserving deliberate compatibility and upstream attribution. + + Performance claims belong in the dedicated performance report. Security reports must follow [SECURITY.md](https://github.com/andreaborio/ds4/blob/main/SECURITY.md) and must not be disclosed here. + + - type: checkboxes + id: prerequisites + attributes: + label: Prerequisites + options: + - label: I searched existing issues and discussions for this proposal. + required: true + - label: This request contains no undisclosed vulnerability details, secrets, or model blobs. + required: true + + - type: textarea + id: problem + attributes: + label: Problem and use case + description: Who needs this, what are they trying to do, and what limitation exists today? + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed behavior + description: Describe observable behavior and acceptance criteria. Keep implementation details separate when possible. + validations: + required: true + + - type: dropdown + id: component + attributes: + label: Primary area + options: + - CLI or inference engine + - Server or API + - Agent mode + - Metal backend + - SSD streaming or ExpertMajor + - GGUF, conversion, or model distribution + - Build, tests, or documentation + - Compatibility or migration + - Other + validations: + required: true + + - type: dropdown + id: contract + attributes: + label: Expected support-contract impact + options: + - No contract change + - Additive behavior within the current production path + - Experimental or reference-only path + - Compatibility or serialized-format change + - Backend or model-family support change + - Unsure + validations: + required: true + + - type: dropdown + id: upstreamability + attributes: + label: Upstreamability classification + description: Choose the classification defined by the contribution policy. + options: + - General, reproducible, and safe for the affected backends + - Potentially general, but not yet proven across the affected paths + - Model-, quant-, or hardware-specific research + - Equivalent change already exists upstream + - Measured regression without a necessary correctness fix + validations: + required: true + + - type: textarea + id: compatibility + attributes: + label: Compatibility and migration impact + description: Cover canonical `hebrus*` interfaces, `ds4*` aliases, `DS4_*` variables, serialized identifiers, GGUF metadata, disk-KV data, API behavior, and exit codes as applicable. State "None" when unaffected. + validations: + required: true + + - type: textarea + id: evidence + attributes: + label: Validation and evidence plan + description: List model-free tests, required model artifacts and hashes, correctness checks, performance frontiers, documentation, and migration coverage needed to accept this feature. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Describe workarounds or smaller designs and why they do not solve the use case. + + - type: textarea + id: references + attributes: + label: References + description: Link specifications, prior art, source code, or public measurements. Do not attach model weights, local benchmark residue, or large binaries. diff --git a/.github/ISSUE_TEMPLATE/model-support.yml b/.github/ISSUE_TEMPLATE/model-support.yml new file mode 100644 index 000000000..1e19382c8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/model-support.yml @@ -0,0 +1,44 @@ +name: Model support +description: Request or report compatibility for a model artifact +title: "[Model]: " +labels: ["model-support"] +body: + - type: markdown + attributes: + value: | + Hebrus supports only explicitly qualified artifacts. Do not attach weights, private prompts, secrets, traces, or KV snapshots. + - type: input + id: artifact + attributes: + label: Artifact identity + description: Repository, immutable revision, exact filename, byte size, and SHA-256. + validations: + required: true + - type: input + id: build + attributes: + label: Hebrus build + description: Commit plus `hebrus --build-info` and `hebrus --capabilities=json`. + validations: + required: true + - type: input + id: hardware + attributes: + label: Hardware and OS + placeholder: Mac model, chip, unified memory, macOS version + validations: + required: true + - type: textarea + id: result + attributes: + label: Observed behavior + description: Include the exact admission or runtime error and the command with personal paths removed. + validations: + required: true + - type: checkboxes + id: hygiene + attributes: + label: Privacy check + options: + - label: I removed prompts, tokens, personal paths, model weights, traces, and cache payloads. + required: true diff --git a/.github/ISSUE_TEMPLATE/performance_report.yml b/.github/ISSUE_TEMPLATE/performance_report.yml new file mode 100644 index 000000000..8d2cdadb0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/performance_report.yml @@ -0,0 +1,131 @@ +name: Performance report +description: Report a reproducible performance regression, result, or optimization opportunity with controlled evidence. +title: "[Performance]: " +body: + - type: markdown + attributes: + value: | + Performance reports must be reproducible and correctness-preserving. Separate cold and warm runs, identify exact artifacts, and disclose memory pressure or competing workloads. + + Do not attach model weights, model blobs, large binaries, secrets, or private prompts. Possible vulnerabilities belong under [SECURITY.md](https://github.com/andreaborio/ds4/blob/main/SECURITY.md), not in a public issue. + + - type: checkboxes + id: prerequisites + attributes: + label: Evidence prerequisites + options: + - label: The result uses an exact build and model artifact identity. + required: true + - label: Correctness was checked before comparing speed or memory. + required: true + - label: Cold and warm measurements are separated, and competing workloads are disclosed. + required: true + - label: I will not attach model weights, model blobs, secrets, or private data. + required: true + + - type: dropdown + id: report_type + attributes: + label: Report type + options: + - Performance regression + - Optimization proposal + - Reproducible benchmark result + - Memory or storage-pressure problem + - Other + validations: + required: true + + - type: textarea + id: summary + attributes: + label: Summary and hypothesis + description: State the observed behavior, the baseline, the candidate, and your current explanation. + validations: + required: true + + - type: textarea + id: build + attributes: + label: Build identities + description: Provide exact baseline and candidate commits, dirty-tree state, compiler/build options, and `./hebrus --build-info` output. + render: shell + validations: + required: true + + - type: textarea + id: artifact + attributes: + label: Model artifact identity + description: Include family, exact filename, size in bytes, SHA-256, provenance/conversion, quantization, and manifest identity if applicable. Do not upload the artifact. + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Hardware and operating environment + description: Include Mac model, Apple chip, RAM, macOS build, power mode, storage device/filesystem, free space, thermal state if known, and all competing processes. + validations: + required: true + + - type: textarea + id: workload + attributes: + label: Workload and resolved plan + description: Include exact sanitized commands, backend, resident/SSD mode, context, prompt/decode lengths, batch settings, thread settings, cache policy, and relevant environment variables. + render: shell + validations: + required: true + + - type: checkboxes + id: frontier + attributes: + label: Performance frontiers covered + description: Select completed lanes. Explain omissions below; do not claim a general promotion from a single lane. + options: + - label: Short, context 128 with at least 128 decoded tokens + - label: Medium, context 2048 with at least 128 decoded tokens + - label: Large, context 8192 with at least 128 decoded tokens + - label: Long, context 32768 with at least 128 decoded tokens + - label: Extended 65K/100K lane when the change affects long-context memory or scheduling + + - type: textarea + id: procedure + attributes: + label: Measurement procedure + description: Describe A/B/B/A ordering or why it is inapplicable, repetition count, warm-up, cold-cache method, sampling tools, and how inherited `DS4_*` variables were excluded or recorded. + validations: + required: true + + - type: textarea + id: correctness + attributes: + label: Correctness and decode evidence + description: Provide parity/cosine/logit checks as appropriate, successful decoded-token evidence, and any quality gates used. Performance without valid output is not a pass. + validations: + required: true + + - type: textarea + id: results + attributes: + label: Results + description: Separate cold and warm values. Include prefill/decode throughput, time to first token, wall time, peak RSS, swap, memory pressure, bytes read, and variance where available. + placeholder: | + Lane | Order | Cold/warm | Baseline | Candidate | Delta | Correctness + validations: + required: true + + - type: textarea + id: conclusion + attributes: + label: Interpretation and requested decision + description: State what the evidence supports, important limitations, and the change or follow-up you propose. + validations: + required: true + + - type: textarea + id: evidence + attributes: + label: Public evidence links + description: Link compact public logs, scripts, or patches. Remove absolute personal paths and sensitive prompts; do not attach model files or large benchmark residue. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..9c61cd88c --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,83 @@ +## Summary + +Explain the user-visible outcome and why this change belongs in Hebrus. + +## Scope + +- [ ] Behavioral or runtime change +- [ ] Mechanical rename or compatibility change +- [ ] Documentation, tests, tooling, or build-only change +- [ ] Performance-sensitive change + +Primary components changed: + +## Upstreamability + +Select one and explain the boundary: + +- [ ] General, reproducible, and safe for the affected backends +- [ ] Potentially general, but not yet proven across the affected paths +- [ ] Model-, quant-, or hardware-specific research +- [ ] Equivalent change already exists upstream +- [ ] Measured regression without a necessary correctness fix + +Rationale and any upstream issue/patch reference: + +## Compatibility contract + +- [ ] `hebrus*` remains the canonical public interface, or the intentional change is documented. +- [ ] Applicable `ds4*` aliases and `DS4_*` environment variables remain compatible, or migration coverage is included. +- [ ] ExpertMajor/GGUF serialized identifiers and metadata remain compatible, or the format change has an ADR, versioning plan, and migration tests. +- [ ] Applicable disk-KV data, server/API behavior, and exit codes remain compatible, or the break is explicitly documented and tested. +- [ ] Not applicable; this change cannot affect public or serialized compatibility. + +Compatibility notes: + +## Validation + +List exact commands and results. Do not write only “tests pass.” + +```text +# command +# result +``` + +Model-free coverage: + +- [ ] Targeted unit/integration tests +- [ ] Compatibility or CLI checks where applicable +- [ ] Documentation links/context audit where applicable +- [ ] `git diff --check` + +## Model-backed evidence + +- [ ] Required and provided below +- [ ] Not applicable; explain why + +When required, include: + +- Exact model family, filename, byte size, SHA-256, provenance/conversion, quantization, and manifest identity. +- Exact hardware, OS build, power state, build identity, backend, and resolved resident/SSD plan. +- Successful decoded-token evidence plus parity/cosine/logit or other appropriate correctness checks. +- Short (128), medium (2048), large (8192), and long (32768) lanes with at least 128 decoded tokens at each frontier when making a general performance claim. +- 65K/100K evidence when changing long-context memory behavior or scheduling. +- A/B/B/A ordering, repeated samples, separate cold/warm results, and memory-pressure/swap observations for performance promotion. + +Evidence or reason it is not applicable: + +## Documentation and release impact + +- [ ] User-facing behavior and current support status are documented. +- [ ] Release QA or support-matrix updates are included when this changes a release contract. +- [ ] No documentation or release change is needed; explain why. + +## Repository hygiene and security + +- [ ] No model weights/blobs, local logs, benchmark scratch, binaries, secrets, private prompts, or personal absolute paths are committed. +- [ ] Generated files have reproducible provenance and are intentionally tracked. +- [ ] This pull request does not publicly disclose an uncoordinated vulnerability; security reports follow [`SECURITY.md`](../SECURITY.md). +- [ ] I reviewed the complete diff, including generated and compatibility changes. + +## Reviewer notes + +Call out risky assumptions, deferred work, and the smallest useful review order. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..b1ed6e38e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,88 @@ +name: Engine CI + +on: + pull_request: + push: + branches: + - main + tags: + - "v*" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: engine-ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + linux-premerge: + name: Linux CPU model-free premerge + runs-on: ubuntu-24.04 + timeout-minutes: 20 + + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Record toolchain + run: | + python3 --version + cc --version + make --version + + # This is the complete Linux CPU/model-free gate from the Makefile. It + # does not replace the Apple Metal, qualified-model, or manual release + # evidence required by QA_BEFORE_RELEASES.md. + - name: Run repository and model-free premerge gates + env: + PYTHONDONTWRITEBYTECODE: "1" + run: make premerge + + macos-build: + name: macOS arm64 build and CPU model-free gates + runs-on: macos-15 + timeout-minutes: 30 + + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Record toolchain and runner + run: | + uname -a + python3 --version + cc --version + make --version + + - name: Run CPU model-free gates on macOS + env: + PYTHONDONTWRITEBYTECODE: "1" + run: make UNAME_S=Linux NATIVE_CPU_FLAG= premerge + + - name: Prove clean Metal and CPU build isolation + run: | + make clean + make build-isolation-test + + - name: Verify the Metal command and install surfaces + run: make capabilities-test command-alias-test install-test + + # A standard hosted macOS job proves compilation and model-free command + # contracts only. Metal kernel execution and qualified-GGUF evidence stay + # in the explicit hardware release gate in QA_BEFORE_RELEASES.md. diff --git a/.gitignore b/.gitignore index e1e8b8577..115613c23 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,11 @@ /ds4-bench /ds4-eval /ds4-agent +/hebrus +/hebrus-server +/hebrus-bench +/hebrus-eval +/hebrus-agent /ds4-qwen-pack /ds4_native /ds4_server_test diff --git a/ACKNOWLEDGMENTS.md b/ACKNOWLEDGMENTS.md new file mode 100644 index 000000000..b34e0136a --- /dev/null +++ b/ACKNOWLEDGMENTS.md @@ -0,0 +1,98 @@ +# Acknowledgments + +Hebrus is the result of an open-source lineage, not a clean-room rewrite. This +document records the principal projects and implementations that materially +shaped the current repository. It describes technical provenance; it does not +claim endorsement by any upstream author, project, company, or model provider. + +## Project origin: `antirez/ds4` + +Hebrus began as a fork of +[`antirez/ds4`](https://github.com/antirez/ds4) and retains substantial +core-engine implementation, architecture, utilities, and Git history from that +project. Salvatore Sanfilippo (antirez) and the upstream contributors created +the foundation on which this fork continues to build. + +The fork has since concentrated on Apple Metal execution, embedded ExpertMajor +stores, adaptive residency, SSD streaming, additional qualified model families, +and the evidence and tooling required to maintain those paths. The repositories +have diverged, but the contribution policy still asks whether general, +reproducible improvements should also be proposed upstream. See +[`FORK_NOTES.md`](FORK_NOTES.md) for the current boundary. + +## GGML and llama.cpp + +[`llama.cpp`](https://github.com/ggml-org/llama.cpp) and +[`GGML`](https://github.com/ggml-org/ggml) are essential engineering references +for this repository. Their work informs GGUF handling, quantization formats and +tables, CPU quantized dot products, Metal kernels, converters, test methods, and +many implementation decisions. + +Hebrus does not link against GGML or llama.cpp as a runtime dependency. Some +source-level pieces are retained or adapted under the MIT license, including +quant layouts and tables, CPU quant/dot logic, and selected kernel techniques. +The GGML authors' copyright notice remains in [`LICENSE`](LICENSE), and +source-specific attribution remains in the relevant files. + +Pinned llama.cpp converter implementations are also used as independent Qwen +validation references. A reference implementation or fixture oracle is not the +same thing as a linked dependency. + +## MLX and MLX-LM + +[`MLX`](https://github.com/ml-explore/mlx) and +[`MLX-LM`](https://github.com/ml-explore/mlx-lm) informed the current Qwen MLX +affine4/group-64 storage layout, scheduling comparisons, and validation work. +The published Qwen release repacks routed weights from an MLX-format source +into the existing checksummed ExpertMajor v2 GGUF container. + +Hebrus does not link the MLX libraries at runtime. This credit does not imply +Apple authorship, sponsorship, or endorsement of Hebrus. + +## YaRN + +The Metal RoPE implementation in `metal/dsv4_rope.metal` contains an algorithm +based on [`YaRN`](https://github.com/jquesnelle/yarn), with the source header's +MIT attribution to Jeffrey Quesnelle and Bowen Peng preserved. + +## Bundled utilities + +The repository includes: + +- [`linenoise`](https://github.com/antirez/linenoise), by Salvatore Sanfilippo + and Pieter Noordhuis, for terminal line editing; +- Rax, by Salvatore Sanfilippo, for radix-tree storage used by the server path. + +These files carry BSD-style terms, including the applicable no-endorsement +language. Their exact bundled notices are collected in +[`THIRD_PARTY_NOTICES.md`](THIRD_PARTY_NOTICES.md) and remain in the source +headers. + +## Validation references and model ecosystems + +Qwen numerical fixtures use a pinned +[Hugging Face Transformers implementation](https://github.com/huggingface/transformers/blob/4626421dc6b741a329300682a6408246ee465490/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py) +and a pinned +[llama.cpp Qwen converter](https://github.com/ggml-org/llama.cpp/blob/657e01125aa49577a62a5531fde24cbcc007006d/conversion/qwen.py) +as independent validation references. The collectors use local deterministic +fixtures and do not make those projects runtime dependencies. + +The project also depends intellectually on the researchers and open-weight +model teams behind the DeepSeek, Qwen, and GLM families, and on the wider GGUF, +Metal, and local-inference communities. Exact model, artifact, revision, and +benchmark provenance belongs in the corresponding model documentation and +dated evidence records rather than in a general credit list. + +## Development process + +The long-form engine reference discloses substantial AI assistance during +development, with humans directing design, testing, benchmarking, and release +decisions. That assistance does not replace the authorship, copyright, or +license notices attached to upstream and third-party work. + +## Licenses and model terms + +The project-level software license is [`LICENSE`](LICENSE). Bundled third-party +files retain their own notices and conditions. Model weights, datasets, model +cards, and external repositories have their own terms; this repository's +software license does not grant rights to those assets. diff --git a/AGENTS.md b/AGENTS.md index 011dfdf3a..9a5e2b575 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,9 +31,9 @@ and resolve the contradiction in the same change. The production path is Apple Metal inference with an embedded ExpertMajor v2 store. DeepSeek V4 and Qwen3.6 have qualified AUTO/resident/SSD modes. GLM 5.2 is SSD-only: normal AUTO resolves to SSD streaming and a resident request is -rejected. The minimum actively qualified host class is Apple Silicon with 64 GB -unified memory. See the runtime support contract and accepted ADRs for exact -boundaries. +rejected. DeepSeek and GLM require at least 64 GiB of unified memory; Qwen has a +separate hardware-aware 16 GiB minimum and necessarily uses SSD at that tier. +See the runtime support contract and accepted ADRs for exact boundaries. CUDA and ROCm are frozen and their backend source is intentionally absent from the active tree. Do not restore or extend it without a new accepted decision; diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..77f445abc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,55 @@ +# Changelog + +This file records user-visible changes. Historical benchmark documents and +release notes remain evidence for their exact commits; they are not rewritten +into current support claims here. + +## Unreleased + +The entries below describe the local Hebrus compatibility bridge after the +current published history. They have no release version or release date and do +not represent a published release. + +### Added + +- Added deterministic, model-free `--capabilities=json` output to the CLI, + server, agent, benchmark, and evaluation executables. The versioned document + reports build identity, backend, executable role, supported model-family IDs, + and the immutable ExpertMajor v2 storage contract. +- Added canonical `hebrus`, `hebrus-server`, `hebrus-agent`, `hebrus-bench`, and + `hebrus-eval` commands. Existing `ds4*` command names remain symlink aliases + to the same binaries for command-surface parity. +- Added model-free tests for capability-schema determinism, cross-executable + consistency, command aliases, build-profile isolation, and canonical/legacy + output parity. +- Added Proposed ADR 0005 to document the intended Hebrus naming and + compatibility boundary without changing durable model, cache, or historical + identifiers. +- Added the maintainer-supplied Hebrus logo as a hash-frozen, unchanged RGBA + master shared with Hebrus Studio. Repository tests reject pixel or encoding + drift; web presentation effects remain CSS-only. + +### Changed + +- Help headings, usage lines, examples, and retired-option diagnostics now use + the executable name that was invoked. Canonical `hebrus*` commands present + the Hebrus name, while the `ds4*` compatibility aliases preserve their + legacy command identity and the same options, defaults, streams, and exit + codes. +- Canonical commands report `engine_id: "hebrus"` and `hebrus build`; legacy + aliases retain `engine_id: "ds4"` and `ds4 build`. Both identities use the + same schema-1 capability fields and immutable ExpertMajor wire contract. +- Current contributor, release, and engine-reference documentation now presents + Hebrus commands first and links an explicit brand compatibility contract. + Canonical CLI/agent/evaluation prompts and new benchmark evidence use the + invoked Hebrus identity; legacy aliases preserve their existing labels. + +### Compatibility + +- The repository remains at until an + administrative rename is actually performed. +- Existing `DS4_*` environment variables and `ds4`-owned serialized identifiers + remain unchanged. This bridge does not introduce `HEBRUS_*` environment + aliases or rename source-level C symbols. +- ExpertMajor tensor names, wire values, model artifact identities, disk-KV + formats, Git history, and historical links are unchanged. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..e305a849b --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,20 @@ +cff-version: 1.2.0 +message: >- + If you use this software, please cite it using the metadata in this file and + preserve the upstream acknowledgements in the repository documentation. +title: Hebrus +type: software +authors: + - name: The ds4.c authors +abstract: >- + An open-source inference engine focused on Apple Metal, embedded ExpertMajor + model stores, and SSD streaming for large mixture-of-experts models. +license: MIT +repository-code: "https://github.com/andreaborio/ds4" +url: "https://github.com/andreaborio/ds4" +keywords: + - Apple Metal + - GGUF + - local inference + - mixture of experts + - SSD streaming diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..753e029a6 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,132 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic +status, nationality, personal appearance, race, caste, color, religion, or +sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances + of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, + without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Project maintainers are responsible for clarifying and enforcing our standards +of acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned with this Code of Conduct, and will communicate reasons +for moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email +address, posting via an official social media account, or acting as an +appointed representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported through GitHub's official abuse-reporting mechanism: + +[Reporting abuse or spam on GitHub][github-abuse] + +The project does not yet publish a dedicated private maintainer contact for +conduct reports. That contact must be established before public launch. Until +then, do not put sensitive conduct reports or personal information in a public +issue. Project maintainers will review conduct matters they receive or observe +fairly and apply the guidelines below. + +All project maintainers are obligated to respect the privacy and security of +the reporter of any incident. + +## Enforcement Guidelines + +Project maintainers will follow these Community Impact Guidelines in +determining the consequences for any action they deem in violation of this +Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from project maintainers, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external +channels like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited +interaction with those enforcing the Code of Conduct, is allowed during this +period. Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available from the [official version 2.1 source][version]. The +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][mozilla]. + +[homepage]: https://www.contributor-covenant.org +[version]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[mozilla]: https://github.com/mozilla/diversity +[github-abuse]: https://docs.github.com/en/communities/maintaining-your-safety-on-github/reporting-abuse-or-spam diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83d509d99..8a2fc5a50 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,17 +1,20 @@ # Contributing -DwarfStar4 changes should be tested against the failure mode they can realistically +Hebrus changes should be tested against the failure mode they can realistically affect. The project has two regression tracks: correctness and speed. Please include the commands you ran, the machine/backend, the model quant, and any notable failures in the PR or commit notes. ## Co-development with `antirez/ds4` -This repository is a transparent research fork of -[`antirez/ds4`](https://github.com/antirez/ds4), not a replacement for it. The -goal is to co-develop DwarfStar: use the fork to investigate complementary -hardware and model paths without blocking on review latency, then contribute -general improvements back upstream. +Hebrus is an increasingly independent inference engine that began as a +transparent research fork of +[`antirez/ds4`](https://github.com/antirez/ds4). It retains substantial upstream +implementation and history while focusing on Apple Metal, embedded ExpertMajor +storage, and SSD streaming across several model families. It is not a +replacement for the upstream project. General improvements that remain +applicable to upstream-supported paths continue to be contributed back after +validation. Every change applicable to an upstream-supported path **must be opened as an upstream PR** once it is scoped and validated. This includes model- or @@ -98,6 +101,34 @@ DS4_TEST_VECTOR_FILE=/path/to/official.vec ./ds4_test --logprob-vectors DS4_TEST_LONG_PROMPT=/path/to/prompt.txt ./ds4_test --long-context ``` +### Artifact publication boundary + +Runtime qualification does not make a local artifact publicly downloadable. +A runtime download target may be enabled only after an immutable repository +revision records the exact filename, byte count, complete output SHA-256, +manifest contract, and compatible runtime commit. Until then, contributor QA +may use an explicitly labelled local candidate, but release and download checks +must report public distribution as unavailable. + +The machine-readable +[Qwen release contract](docs/contracts/qwen-release.json) is the canonical +source for every repeated identity below. The current Qwen release is +`published` as +`Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf`, 20,808,566,880 +bytes, SHA-256 +`dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d`. +It is published at immutable repository revision +`7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02`, and +its manifest requires runtime commit +`73a332fef82a0bcdd567d17e0de17aa004cad85d` or a compatible descendant. +`download_model.sh qwen-v2` must pin and validate that exact identity. The older +`Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-Q4_K_S.gguf` object is `negative-only`, not +a runnable fallback. See +[`docs/qwen-expert-major-store.md`](docs/qwen-expert-major-store.md). +Change the JSON contract first, update every intentional human-readable mirror, +then run `make release-contract release-contract-test`; do not update a mirror +as an independent release authority. + CUDA and ROCm are frozen and their backend source and build targets are absent from the active tree. Ordinary changes must not restore them accidentally. A change that intentionally reactivates either backend must restore and pass the @@ -119,6 +150,12 @@ On macOS, also run `make build-isolation-test`. The CPU-only binaries live in `build/cpu-$(uname -m)/bin`; `make cpu` intentionally leaves the root Metal commands unchanged. +The GitHub Actions workflow runs the full Linux CPU/model-free premerge gate +and a macOS arm64 lane that repeats the CPU suite, builds the Metal command +surface, and checks Metal/CPU artifact isolation. Hosted compilation is not +Metal kernel or qualified-model evidence; the hardware release gates below +remain mandatory. + The CPU backend is a reference/debug path, not the production performance target. Remember that executing the CPU path on Metal can crash the system because of a kernel bug in macOS. diff --git a/FORK_NOTES.md b/FORK_NOTES.md index b15478baa..05bfd836f 100644 --- a/FORK_NOTES.md +++ b/FORK_NOTES.md @@ -17,7 +17,7 @@ equivalent implementation, this fork converges on it and removes the duplicate. Ledger snapshot: fork-wide ExpertMajor v2-only migration for DeepSeek, GLM, and Qwen, including the earlier GLM promotion and Metal lifecycle milestones; -upstream `main` `6d5572d`, audited 2026-07-20. +upstream `main` `f9602e5`, audited 2026-07-21. Commit links and live branch differences remain authoritative if this dated snapshot drifts. @@ -41,8 +41,7 @@ snapshot drifts. | Mixed-precision routed-expert streaming (`fefa426`, later sync) | serve per-layer boosted expert quants under SSD streaming | **CONVERGED UPSTREAM** — the fork takes upstream's implementation after `5800f15`; no competing delta should be preserved. | | Fork-wide ExpertMajor v2-only admission (2026-07-20 tranche) | use the family-tagged, checksummed `ds4.expert_major.v2` store as the only physical routed-expert layout for DeepSeek, GLM, and Qwen; reconstruct logical tensor identity while translating all Metal mappings and reads through the manifest | **FORK MAIN CONTRACT** — local Apple Metal only. Canonical GGUFs are offline converter inputs, not inference artifacts. ExpertMajor v1, sidecars, CPU, CUDA, ROCm, and distributed execution have no compatibility path. Normal startup is `./ds4 -m MODEL.gguf --ctx N`; invalid family, inventory, geometry, or manifest state fails before inference. | | GLM 5.2 ExpertMajor v2 (`55d2bab` release gate) | #520 correctness + #528 indexed prepare, compact DSA KV, physical grouped prefill, one-record selected decode reads, translated advisory I/O, tokenizer/prompt routing, measured 601-expert 64 GiB policy, embedded single-payload GGUF, and removal of the retired predicted-install path | **FORK MAIN, EXPERTMAJOR-ONLY METAL CONTRACT** — the broken port moved from 1.27 to 1.77-1.81 decode t/s; the final simple gate was 1.79 t/s and the old/new same-condition A/B was 1.75/1.74, restoring runtime parity with the qualified path. The earlier rested-storage median remains 11.08/1.90 t/s. Model-backed output is coherent and deterministic; synthetic canonical/native direct, scalar-batch, and grouped-batch math passes. #520/#528 remain open upstream. GLM canonical/non-Metal/distributed execution and old GLM layouts have no compatibility path. | -| Qwen3.6-35B-A3B ExpertMajor v2 (`qwen35moe`, 2026-07-20 release gate) | distinct v2 family ID, generic converter, embedded single-layout Q4_K_S store, AUTO resident/SSD admission, model-backed output, and existing resident prefill/parallel decode schedules | **FORK MAIN; UPSTREAM ISSUE [#462](https://github.com/antirez/ds4/issues/462)** — the experimental guard, v1 loader, sidecar, and canonical inference paths are retired. The 20,808,566,880-byte v2 artifact has SHA-256 `d7c43a6388ec20e6fe5530850350f96fdb0ac37c5ce36d3e5f92b172c447f56b`. Resident v2/v1/v2 at 2K measured 318.96/29.54, 320.59/29.59, and 318.83/29.54 prefill/decode t/s; evidence was byte-identical and no new swapout occurred. The v1 arm is historical control evidence only. | -| Qwen unified MLX-affine v2 replacement (2026-07-21) | replace the equal-size routed Q4_K_S payload with MLX affine 4-bit/group-64 records while retaining the embedded ExpertMajor v2 GGUF container and one AUTO resident/SSD configuration | **FORK MAIN CONTRACT; SUPERSEDES THE 2026-07-20 Q4_K_S ARTIFACT** — normal inference accepts only `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf`, 20,808,566,880 bytes, SHA-256 `dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d`. The old Q4_K_S artifact remains historical evidence and is deliberately rejected rather than decoded through compatibility code. | +| Qwen3.6-35B-A3B ExpertMajor v2 (`qwen35moe`, current affine contract) | distinct v2 family ID, generic converter, one embedded MLX affine4/group-64 routed store, hardware-aware AUTO resident/SSD admission, affine Metal kernels, and model-backed resident/SSD qualification | **FORK MAIN RUNTIME; PUBLISHED ARTIFACT; UPSTREAM ISSUE [#462](https://github.com/antirez/ds4/issues/462)** — the only runnable release is `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf`, 20,808,566,880 bytes, SHA-256 `dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d`, pinned at repository revision `7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02`. The previously published Q4_K_S v2 artifact and its SHA-256 `d7c43a6388ec20e6fe5530850350f96fdb0ac37c5ce36d3e5f92b172c447f56b` remain dated 2026-07-20 measurement/control evidence only; the current runtime rejects that store before inference. The experimental guard, v1 loader, sidecar, and canonical inference paths remain retired. | | Qwen hardware-aware Metal policy (2026-07-21) | named 16/24/32/36/48/64/96/128 GiB profiles, continuous resident reserve, normal-pressure cold/warm file-cache equivalence, and live resident/SSD fallback | **FORK MAIN CONTRACT** — `make premerge` passes; AUTO resolves to SSD on M1 Pro 16 GiB and resident on AC-powered M1 Pro 32 GiB with zero swap. Split-K transfer is independently confirmed at 2K on 16 GiB (15.00 versus 9.70 decode t/s) and 32K on 32 GiB (19.02 versus 1.63 decode t/s), with identical greedy token IDs in each A/B. | | Qwen exact K/V pair-blit (2026-07-22) | validate both full-attention K/V cache copies transactionally, then record them in one ordered Metal blit encoder in resident and SSD modes | **FORK MAIN, OWNER-AUTHORIZED STRUCTURAL MERGE; RELEASE MATRIX INCOMPLETE** — removes exactly 50% of K/V blit-encoder creation without changing copy commands, bytes, commits, SSD I/O, logits, or decode evidence. M5 resident 2K decode improved 56.52 to 58.06 t/s (+2.72%); M5 SSD 2K decode improved +0.75%; M1 Pro 32 GiB measured +1.37% at 2K and -0.20% at contaminated 8K. Every valid mean end-to-end inference regression stayed below 2%; the M1 Pro 16 GiB timing gate is explicitly invalid due alternating 88--189 s external host stalls, while its correctness and zero-swap gates passed. The 128/32K/65K/100K release arms remain unqualified, so this merge is not recorded as completion of the normal performance-promotion gate. | | Exact paired Q8 Metal decode (2026-07-20 long-context experiment) | compute two independent Q8 matvec outputs in one exact Metal dispatch while preserving bit-identical accumulation order | **REJECTED AND REMOVED** — the final zero-swap Qwen 32K A/B/B/A preserved exact evidence but did not show a speed win. Scalar means were 67.03 prefill t/s, 40,217.705 ms decode wall, 314.818 ms p50, and 317.563 ms p95; paired means were 40,273.322 ms, 315.395 ms, and 317.665 ms. The differences were neutral-to-worse and smaller than cohort spread, so the M5 kernel, dispatch, and dedicated test were removed. The clean post-removal source then measured 65.76 prefill t/s, about 3.18 decode t/s, and 315.193/317.518 ms p50/p95 with exact evidence and zero swap. | diff --git a/GOLD_METAL_SSD.md b/GOLD_METAL_SSD.md index 7a0115669..ffc289bd1 100644 --- a/GOLD_METAL_SSD.md +++ b/GOLD_METAL_SSD.md @@ -8,14 +8,17 @@ its qualified SSD-streaming path. ## User-facing contract ```sh -make # namespaced Metal build; publishes ./ds4* -./ds4 -m DEEPSEEK-OR-QWEN-DS4-ExpertMajor-v2.gguf # AUTO -./ds4 -m DEEPSEEK-OR-QWEN-DS4-ExpertMajor-v2.gguf --resident -./ds4 -m DEEPSEEK-OR-QWEN-DS4-ExpertMajor-v2.gguf --ssd-streaming -./ds4 -m GLM-5.2-DS4-ExpertMajor-v2-Q2_K.gguf --ctx 8192 # AUTO -> SSD -./ds4 --build-info +make # publishes ./hebrus* plus legacy ./ds4* aliases +./hebrus -m DEEPSEEK-OR-QWEN-DS4-ExpertMajor-v2.gguf # AUTO +./hebrus -m DEEPSEEK-OR-QWEN-DS4-ExpertMajor-v2.gguf --resident +./hebrus -m DEEPSEEK-OR-QWEN-DS4-ExpertMajor-v2.gguf --ssd-streaming +./hebrus -m GLM-5.2-DS4-ExpertMajor-v2-Q2_K.gguf --ctx 8192 # AUTO -> SSD +./hebrus --build-info ``` +The `./ds4` and `./ds4-server` names remain byte-identical compatibility +aliases throughout the bridge and 1.x compatibility window. + DeepSeek V4, GLM 5.2, and Qwen3.6 inference accepts only a validated embedded `ds4.expert_major.v2` store on Apple Metal. Canonical GGUFs remain offline converter inputs; ExpertMajor v1, external sidecars, CPU, CUDA, ROCm, and diff --git a/GOVERNANCE.md b/GOVERNANCE.md new file mode 100644 index 000000000..4894c7374 --- /dev/null +++ b/GOVERNANCE.md @@ -0,0 +1,81 @@ +# Governance + +## Project Model + +This repository uses a lightweight maintainer model. There is no foundation, +steering committee, technical committee, or voting body represented by this +document. Maintainer status is determined by the repository's actual write and +release permissions; this document does not create honorary roles. + +Maintainers have final responsibility for: + +- reviewing, merging, and rejecting changes; +- protecting correctness, compatibility, security, and release quality; +- accepting or rejecting architectural decisions; +- changing the runtime support contract; +- coordinating private security reports; and +- creating tags and publishing releases or artifacts. + +That authority does not override project licenses, preserve a right to remove +authorship or attribution, or weaken the repository's documented validation +gates. + +## Contributions And Review + +Anyone may propose a change through a pull request. The contribution and test +requirements in [`CONTRIBUTING.md`](CONTRIBUTING.md) apply to every change. +Maintainers may request narrower scope, more evidence, changes to ownership +boundaries, or an independent review before merging. + +Reviewers and contributors influence decisions through technical evidence and +discussion, but review activity alone does not grant merge or release +authority. Repository permissions remain the factual source of maintainer +authority. + +## Decision Records And Runtime Authority + +Routine, reversible implementation choices may be decided in pull-request +review. Changes to architecture, compatibility, supported models or backends, +serialized formats, or release policy require a written decision record and +the validation required by the repository guides. + +Accepted records under [`docs/adr/`](docs/adr/) define architectural decisions. +[`docs/contracts/RUNTIME_SUPPORT.md`](docs/contracts/RUNTIME_SUPPORT.md) defines +the current production support boundary. Proposed ADRs describe proposals, not +active policy. In particular, +[`ADR 0005`](docs/adr/0005-hebrus-naming-and-compatibility-boundary.md) remains +Proposed until its acceptance conditions are met. + +When normative documents disagree, the contradiction must be resolved in the +same change. A PR description, benchmark note, or temporary handoff cannot +silently override an accepted ADR, the runtime contract, `CONTRIBUTING.md`, or +`QA_BEFORE_RELEASES.md`. + +## Upstream Relationship + +The repository remains a fork of +[`antirez/ds4`](https://github.com/antirez/ds4). Changes that are general, +reproducible, and applicable to an upstream-supported path follow the +upstream-first classification and submission policy in +[`CONTRIBUTING.md`](CONTRIBUTING.md). Upstream review does not transfer +maintainer authority over this repository, and this governance document does +not imply endorsement by or formal affiliation with upstream. + +## Releases And Security + +A maintainer may publish a release only after the applicable gates in +[`QA_BEFORE_RELEASES.md`](QA_BEFORE_RELEASES.md) are satisfied and the release +scope is documented. Local qualification, an unmerged branch, or an entry under +`Unreleased` is not a published release. + +Undisclosed vulnerabilities follow [`SECURITY.md`](SECURITY.md) and must not be +debated in public issues. GitHub private vulnerability reporting must be +enabled before launch; once it is available, maintainers coordinate +investigation and disclosure through that advisory flow. No response-time or +release-frequency guarantee is created by this document. + +## Changing Governance + +Governance changes use the same public pull-request and review process as other +repository changes. Only a maintainer with the corresponding repository +permissions can merge a governance change or alter access and release rights. diff --git a/Makefile b/Makefile index aed00f571..611e57f81 100644 --- a/Makefile +++ b/Makefile @@ -25,18 +25,57 @@ LDLIBS ?= -lm -pthread METAL_SRCS := $(wildcard metal/*.metal) BUILD_ROOT ?= build -PROGRAMS := ds4 ds4-server ds4-bench ds4-eval ds4-agent +HEBRUS_PROGRAMS := hebrus hebrus-server hebrus-bench hebrus-eval hebrus-agent +DS4_PROGRAMS := ds4 ds4-server ds4-bench ds4-eval ds4-agent +PROGRAMS := $(HEBRUS_PROGRAMS) $(DS4_PROGRAMS) + +PREFIX ?= /usr/local +BINDIR ?= $(PREFIX)/bin +INSTALL ?= install +INSTALL_MODE ?= 0755 +INSTALL_DEST_BINDIR = $(DESTDIR)$(BINDIR) +INSTALL_SOURCE_PROGRAMS = $(addprefix $(INSTALL_SOURCE_BINDIR)/,$(HEBRUS_PROGRAMS)) +SERVER_ALIAS_EXPECTED_BACKEND ?= metal +SERVER_ALIAS_EXPECTED_BUILD_SHA ?= $(BUILD_GIT_SHA) +SERVER_ALIAS_PORT ?= 0 .PHONY: all help clean test model-free-test premerge context-audit doc-links \ + brand-boundary-audit brand-boundary-test brand-asset-test \ + release-contract release-contract-test \ imatrix-dataset-check prompt-fixture-check cpu FORCE \ metal build-isolation-test q4k-dot-test qwen-metadata-test \ qwen-reference-test qwen-unicode-test qwen-tokenizer-test \ qwen-expert-group-test expert-store-test metal-ssd-profile-test \ - download-model-test $(PROGRAMS) ds4_test ds4_agent_test - -download-model-test: tests/test_download_model.sh download_model.sh + download-model-test capabilities-test command-alias-test \ + visible-identity-test server-alias-model-unit-test \ + server-alias-model-test \ + install uninstall install-test $(PROGRAMS) \ + ds4_test ds4_agent_test + +download-model-test: tests/test_download_model.sh download_model.sh \ + docs/contracts/qwen-release.json sh tests/test_download_model.sh +release-contract: tools/qwen_release_contract.py \ + docs/contracts/qwen-release.json README.md CONTRIBUTING.md \ + QA_BEFORE_RELEASES.md docs/contracts/RUNTIME_SUPPORT.md \ + docs/qwen-expert-major-store.md download_model.sh \ + tests/test_download_model.sh + python3 tools/qwen_release_contract.py + +release-contract-test: release-contract tools/qwen_release_contract.py \ + tests/test_qwen_release_contract.py + python3 tests/test_qwen_release_contract.py + +brand-boundary-audit: tools/brand_boundary_audit.py tools/brand_boundary.json + python3 tools/brand_boundary_audit.py --check + +brand-boundary-test: tools/brand_boundary_audit.py tests/test_brand_boundary_audit.py + python3 tests/test_brand_boundary_audit.py + +brand-asset-test: tests/test_brand_asset.py docs/media/hebrus-logo.png README.md + python3 tests/test_brand_asset.py + ifeq ($(UNAME_S),Darwin) # A build profile owns every object and binary it produces. In particular, a @@ -47,6 +86,8 @@ METAL_OBJDIR := $(BUILD_ROOT)/$(METAL_PROFILE)/obj METAL_BINDIR := $(BUILD_ROOT)/$(METAL_PROFILE)/bin CPU_OBJDIR := $(BUILD_ROOT)/$(CPU_PROFILE)/obj CPU_BINDIR := $(BUILD_ROOT)/$(CPU_PROFILE)/bin +INSTALL_SOURCE_BINDIR := $(METAL_BINDIR) +INSTALL_BACKEND := metal METAL_LDLIBS := $(LDLIBS) -framework Foundation -framework Metal @@ -70,92 +111,128 @@ METAL_TEST_BINS := \ $(METAL_BINDIR)/test_qwen_expert_group \ $(METAL_BINDIR)/test_expert_store \ $(METAL_BINDIR)/test_metal_ssd_profile \ - $(METAL_BINDIR)/test_ssd_residency + $(METAL_BINDIR)/test_ssd_residency \ + $(METAL_BINDIR)/test_visible_identity all: metal help: - @echo "DS4 build targets:" - @echo " make / make metal Build Metal and publish ./ds4* -> $(METAL_BINDIR)" - @echo " make cpu Build CPU-only binaries in $(CPU_BINDIR) (never changes ./ds4*)" + @echo "Hebrus build targets:" + @echo " make / make metal Build Metal and publish ./hebrus* plus ./ds4* aliases" + @echo " make cpu Build CPU-only commands in $(CPU_BINDIR); keep root Metal links" @echo " make test Build and run the Metal test suite" @echo " make model-free-test" @echo " Run all Metal gates that do not require a GGUF" @echo " make build-isolation-test" @echo " Prove Metal -> CPU -> Metal cannot mix artifacts" + @echo " make install Install commands under DESTDIR+$(BINDIR)" + @echo " make uninstall Remove only the ten installed command paths" + @echo " make install-test Verify staged install layout and capabilities" + @echo " make brand-boundary-audit" + @echo " Reject unclassified or increased legacy brand tokens" + @echo " make release-contract" + @echo " Reject Qwen release identity drift" + @echo " make server-alias-model-test QWEN_V2=/absolute/model.gguf" + @echo " Run the opt-in model-backed server alias release gate" @echo " make premerge Run context/docs, isolation, and model-free gates" @echo " make clean Remove build outputs and published root binaries" # Root binaries are a Metal-only compatibility surface on macOS. These targets # are phony so an old regular CPU binary is replaced even when its timestamp is # newer than the namespaced Metal binary. -metal: $(PROGRAMS) +metal: $(PROGRAMS) $(METAL_BINS) + +hebrus ds4: $(METAL_BINDIR)/hebrus +hebrus-server ds4-server: $(METAL_BINDIR)/hebrus-server +hebrus-bench ds4-bench: $(METAL_BINDIR)/hebrus-bench +hebrus-eval ds4-eval: $(METAL_BINDIR)/hebrus-eval +hebrus-agent ds4-agent: $(METAL_BINDIR)/hebrus-agent -$(PROGRAMS): %: $(METAL_BINDIR)/% +$(PROGRAMS): @rm -f "$@" @ln -s "$<" "$@" cpu: $(CPU_BINS) @echo "CPU-only binaries: $(CPU_BINDIR)" -$(METAL_BINDIR)/ds4: \ +$(METAL_BINDIR)/hebrus: \ $(METAL_OBJDIR)/ds4_cli.o $(METAL_OBJDIR)/ds4_help.o \ $(METAL_OBJDIR)/linenoise.o $(METAL_CORE_OBJS) @mkdir -p "$(@D)" $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) -$(METAL_BINDIR)/ds4-server: \ +$(METAL_BINDIR)/hebrus-server: \ $(METAL_OBJDIR)/ds4_server.o $(METAL_OBJDIR)/ds4_help.o \ $(METAL_OBJDIR)/ds4_kvstore.o $(METAL_OBJDIR)/rax.o $(METAL_CORE_OBJS) @mkdir -p "$(@D)" $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) -$(METAL_BINDIR)/ds4-bench: \ +$(METAL_BINDIR)/hebrus-bench: \ $(METAL_OBJDIR)/ds4_bench.o $(METAL_OBJDIR)/ds4_help.o $(METAL_CORE_OBJS) @mkdir -p "$(@D)" $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) -$(METAL_BINDIR)/ds4-eval: \ +$(METAL_BINDIR)/hebrus-eval: \ $(METAL_OBJDIR)/ds4_eval.o $(METAL_OBJDIR)/ds4_help.o $(METAL_CORE_OBJS) @mkdir -p "$(@D)" $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) -$(METAL_BINDIR)/ds4-agent: \ +$(METAL_BINDIR)/hebrus-agent: \ $(METAL_OBJDIR)/ds4_agent.o $(METAL_OBJDIR)/ds4_help.o \ $(METAL_OBJDIR)/ds4_web.o $(METAL_OBJDIR)/ds4_kvstore.o \ $(METAL_OBJDIR)/linenoise.o $(METAL_CORE_OBJS) @mkdir -p "$(@D)" $(CC) $(CFLAGS) -o $@ $^ $(METAL_LDLIBS) -$(CPU_BINDIR)/ds4: \ +$(METAL_BINDIR)/ds4: $(METAL_BINDIR)/hebrus +$(METAL_BINDIR)/ds4-server: $(METAL_BINDIR)/hebrus-server +$(METAL_BINDIR)/ds4-bench: $(METAL_BINDIR)/hebrus-bench +$(METAL_BINDIR)/ds4-eval: $(METAL_BINDIR)/hebrus-eval +$(METAL_BINDIR)/ds4-agent: $(METAL_BINDIR)/hebrus-agent + +$(addprefix $(METAL_BINDIR)/,$(DS4_PROGRAMS)): + @rm -f "$@" + @ln -s "$(notdir $<)" "$@" + +$(CPU_BINDIR)/hebrus: \ $(CPU_OBJDIR)/ds4_cli.o $(CPU_OBJDIR)/ds4_help.o \ $(CPU_OBJDIR)/linenoise.o $(CPU_CORE_OBJS) @mkdir -p "$(@D)" $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) -$(CPU_BINDIR)/ds4-server: \ +$(CPU_BINDIR)/hebrus-server: \ $(CPU_OBJDIR)/ds4_server.o $(CPU_OBJDIR)/ds4_help.o \ $(CPU_OBJDIR)/ds4_kvstore.o $(CPU_OBJDIR)/rax.o $(CPU_CORE_OBJS) @mkdir -p "$(@D)" $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) -$(CPU_BINDIR)/ds4-bench: \ +$(CPU_BINDIR)/hebrus-bench: \ $(CPU_OBJDIR)/ds4_bench.o $(CPU_OBJDIR)/ds4_help.o $(CPU_CORE_OBJS) @mkdir -p "$(@D)" $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) -$(CPU_BINDIR)/ds4-eval: \ +$(CPU_BINDIR)/hebrus-eval: \ $(CPU_OBJDIR)/ds4_eval.o $(CPU_OBJDIR)/ds4_help.o $(CPU_CORE_OBJS) @mkdir -p "$(@D)" $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) -$(CPU_BINDIR)/ds4-agent: \ +$(CPU_BINDIR)/hebrus-agent: \ $(CPU_OBJDIR)/ds4_agent.o $(CPU_OBJDIR)/ds4_help.o \ $(CPU_OBJDIR)/ds4_web.o $(CPU_OBJDIR)/ds4_kvstore.o \ $(CPU_OBJDIR)/linenoise.o $(CPU_CORE_OBJS) @mkdir -p "$(@D)" $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) +$(CPU_BINDIR)/ds4: $(CPU_BINDIR)/hebrus +$(CPU_BINDIR)/ds4-server: $(CPU_BINDIR)/hebrus-server +$(CPU_BINDIR)/ds4-bench: $(CPU_BINDIR)/hebrus-bench +$(CPU_BINDIR)/ds4-eval: $(CPU_BINDIR)/hebrus-eval +$(CPU_BINDIR)/ds4-agent: $(CPU_BINDIR)/hebrus-agent + +$(addprefix $(CPU_BINDIR)/,$(DS4_PROGRAMS)): + @rm -f "$@" + @ln -s "$(notdir $<)" "$@" + $(METAL_OBJDIR)/%.o: %.c @mkdir -p "$(@D)" $(CC) $(CFLAGS) $(DEPFLAGS) -c -o $@ $< @@ -176,11 +253,11 @@ $(CPU_OBJDIR)/%.o: %.c # Build provenance is intentionally refreshed on every invocation. Keeping it # in this tiny object prevents a clean/dirty transition from forcing the giant # engine translation unit to rebuild while ensuring --build-info is truthful. -$(METAL_OBJDIR)/ds4_build.o: ds4_build.c ds4.h FORCE +$(METAL_OBJDIR)/ds4_build.o: ds4_build.c ds4.h ds4_expert_store.h FORCE @mkdir -p "$(@D)" $(CC) $(CFLAGS) $(DEPFLAGS) -c -o $@ $< -$(CPU_OBJDIR)/ds4_build.o: ds4_build.c ds4.h FORCE +$(CPU_OBJDIR)/ds4_build.o: ds4_build.c ds4.h ds4_expert_store.h FORCE @mkdir -p "$(@D)" $(CC) $(CFLAGS) -DDS4_NO_GPU $(DEPFLAGS) -c -o $@ $< @@ -355,6 +432,21 @@ $(METAL_BINDIR)/test_ssd_residency: \ @mkdir -p "$(@D)" $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) +$(METAL_BINDIR)/test_visible_identity: \ + tests/test_visible_identity.c hebrus_identity.h + @mkdir -p "$(@D)" + $(CC) $(CFLAGS) -I. -o $@ $< + +$(CPU_BINDIR)/test_visible_identity: \ + tests/test_visible_identity.c hebrus_identity.h + @mkdir -p "$(@D)" + $(CC) $(CFLAGS) -DDS4_NO_GPU -I. -o $@ $< + +visible-identity-test: $(METAL_BINDIR)/test_visible_identity \ + $(CPU_BINDIR)/test_visible_identity + $(METAL_BINDIR)/test_visible_identity + $(CPU_BINDIR)/test_visible_identity + $(METAL_BINDIR)/test_qwen_gdn_ref: \ $(METAL_OBJDIR)/test_qwen_gdn_ref.o $(METAL_OBJDIR)/ds4_qwen_ref.o \ $(METAL_OBJDIR)/ds4_qwen.o @@ -439,6 +531,13 @@ qwen-unicode-test: $(METAL_BINDIR)/test_qwen_unicode qwen-tokenizer-test: $(METAL_BINDIR)/test_qwen_tokenizer $(METAL_BINDIR)/test_qwen_tokenizer +capabilities-test: metal tests/test_capabilities.py + python3 tests/test_capabilities.py --bin-dir $(METAL_BINDIR) --backend metal + +command-alias-test: metal tests/test_command_aliases.py + python3 tests/test_command_aliases.py --bin-dir $(METAL_BINDIR) \ + --backend metal --layout profile + model-free-test: metal ds4_test ds4_agent_test $(METAL_BINDIR)/test_q4k_dot \ $(METAL_BINDIR)/test_q4k_top8 \ $(METAL_BINDIR)/test_qwen_session \ @@ -450,7 +549,13 @@ model-free-test: metal ds4_test ds4_agent_test $(METAL_BINDIR)/test_q4k_dot \ $(METAL_BINDIR)/test_qwen_expert_group \ $(METAL_BINDIR)/test_expert_store \ $(METAL_BINDIR)/test_metal_ssd_profile \ - $(METAL_BINDIR)/test_ssd_residency download-model-test + $(METAL_BINDIR)/test_ssd_residency download-model-test \ + visible-identity-test \ + server-alias-model-unit-test \ + tests/test_capabilities.py tests/test_command_aliases.py + python3 tests/test_capabilities.py --bin-dir $(METAL_BINDIR) --backend metal + python3 tests/test_command_aliases.py --bin-dir $(METAL_BINDIR) \ + --backend metal --layout profile DS4_BIN_DIR=$(METAL_BINDIR) sh tests/test_retired_distributed_flags.sh sh tests/test_benchmark_env_guard.sh $(METAL_BINDIR)/ds4-eval --self-test-extractors @@ -495,11 +600,15 @@ prompt-fixture-check: # Build isolation removes and rebuilds BUILD_ROOT, so model-free-test must start # only after it completes even when an agent invokes `make -j premerge`. -premerge: context-audit doc-links imatrix-dataset-check prompt-fixture-check build-isolation-test +premerge: context-audit doc-links brand-boundary-audit brand-boundary-test brand-asset-test \ + release-contract release-contract-test \ + imatrix-dataset-check prompt-fixture-check build-isolation-test $(MAKE) model-free-test + $(MAKE) install-test git diff --check -build-isolation-test: tests/test_build_isolation.sh +build-isolation-test: tests/test_build_isolation.sh tests/test_capabilities.py \ + tests/test_command_aliases.py MAKE="$(MAKE)" sh tests/test_build_isolation.sh -include $(wildcard $(METAL_OBJDIR)/*.d $(CPU_OBJDIR)/*.d) @@ -509,34 +618,55 @@ else CFLAGS += -D_GNU_SOURCE -fno-finite-math-only CPU_CORE_OBJS := ds4_cpu.o ds4_build_cpu.o ds4_ssd.o \ ds4_profile.o ds4_expert_store.o ds4_qwen.o ds4_qwen_unicode.o +INSTALL_SOURCE_BINDIR := . +INSTALL_BACKEND := cpu all: cpu help: - @echo "DS4 build targets:" - @echo " make / make cpu Build CPU-only ./ds4* binaries" + @echo "Hebrus build targets:" + @echo " make / make cpu Build ./hebrus* plus ./ds4* aliases" @echo " make test Build and run tests" @echo " make model-free-test Run all tests that do not require a GGUF" + @echo " make install Install commands under DESTDIR+$(BINDIR)" + @echo " make uninstall Remove only the ten installed command paths" + @echo " make install-test Verify staged install layout and capabilities" + @echo " make brand-boundary-audit" + @echo " Reject unclassified or increased legacy brand tokens" + @echo " make release-contract Reject Qwen release identity drift" + @echo " make server-alias-model-test QWEN_V2=/absolute/model.gguf" + @echo " Run the opt-in model-backed server alias gate" + @echo " make premerge Run repository audits and Linux CPU/model-free gates" @echo " make clean Remove build outputs" cpu: $(PROGRAMS) -ds4: ds4_cli_cpu.o ds4_help.o linenoise.o $(CPU_CORE_OBJS) +hebrus: ds4_cli_cpu.o ds4_help.o linenoise.o $(CPU_CORE_OBJS) $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) -ds4-server: ds4_server_cpu.o ds4_help.o ds4_kvstore.o rax.o $(CPU_CORE_OBJS) +hebrus-server: ds4_server_cpu.o ds4_help.o ds4_kvstore.o rax.o $(CPU_CORE_OBJS) $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) -ds4-bench: ds4_bench_cpu.o ds4_help.o $(CPU_CORE_OBJS) +hebrus-bench: ds4_bench_cpu.o ds4_help.o $(CPU_CORE_OBJS) $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) -ds4-eval: ds4_eval_cpu.o ds4_help.o $(CPU_CORE_OBJS) +hebrus-eval: ds4_eval_cpu.o ds4_help.o $(CPU_CORE_OBJS) $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) -ds4-agent: ds4_agent_cpu.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o $(CPU_CORE_OBJS) +hebrus-agent: ds4_agent_cpu.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o $(CPU_CORE_OBJS) $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) -ds4_build_cpu.o: ds4_build.c ds4.h FORCE +ds4: hebrus +ds4-server: hebrus-server +ds4-bench: hebrus-bench +ds4-eval: hebrus-eval +ds4-agent: hebrus-agent + +$(DS4_PROGRAMS): + @rm -f "$@" + @ln -s "$(notdir $<)" "$@" + +ds4_build_cpu.o: ds4_build.c ds4.h ds4_expert_store.h FORCE $(CC) $(CFLAGS) -DDS4_NO_GPU -c -o $@ ds4_build.c ds4_ssd.o: ds4_ssd.c ds4_ssd.h @@ -552,22 +682,22 @@ ds4_qwen_unicode.o: ds4_qwen_unicode.c ds4_qwen_unicode.h \ ds4_qwen_unicode_data.inc $(CC) $(CFLAGS) -c -o $@ ds4_qwen_unicode.c -ds4_cli.o: ds4_cli.c ds4.h ds4_ssd.h ds4_help.h linenoise.h +ds4_cli.o: ds4_cli.c ds4.h ds4_ssd.h ds4_help.h hebrus_identity.h linenoise.h $(CC) $(CFLAGS) -c -o $@ ds4_cli.c -ds4_help.o: ds4_help.c ds4_help.h +ds4_help.o: ds4_help.c ds4_help.h hebrus_identity.h $(CC) $(CFLAGS) -c -o $@ ds4_help.c ds4_server.o: ds4_server.c ds4.h ds4_ssd.h ds4_help.h ds4_kvstore.h rax.h $(CC) $(CFLAGS) -c -o $@ ds4_server.c -ds4_bench.o: ds4_bench.c ds4.h ds4_ssd.h ds4_help.h +ds4_bench.o: ds4_bench.c ds4.h ds4_ssd.h ds4_help.h hebrus_identity.h $(CC) $(CFLAGS) -c -o $@ ds4_bench.c -ds4_eval.o: ds4_eval.c ds4.h ds4_ssd.h ds4_help.h +ds4_eval.o: ds4_eval.c ds4.h ds4_ssd.h ds4_help.h hebrus_identity.h $(CC) $(CFLAGS) -c -o $@ ds4_eval.c -ds4_agent.o: ds4_agent.c ds4.h ds4_ssd.h ds4_help.h ds4_kvstore.h ds4_web.h linenoise.h +ds4_agent.o: ds4_agent.c ds4.h ds4_ssd.h ds4_help.h hebrus_identity.h ds4_kvstore.h ds4_web.h linenoise.h $(CC) $(CFLAGS) -c -o $@ ds4_agent.c ds4_web.o: ds4_web.c ds4_web.h @@ -583,7 +713,7 @@ ds4_agent_test.o: tests/ds4_agent_test.c tests/internal/ds4_agent_unit.h $(CC) $(CFLAGS) -DDS4_NO_GPU -c -o $@ tests/ds4_agent_test.c ds4_agent_test_impl.o: ds4_agent.c ds4.h ds4_ssd.h \ - ds4_help.h ds4_kvstore.h ds4_web.h linenoise.h \ + ds4_help.h hebrus_identity.h ds4_kvstore.h ds4_web.h linenoise.h \ tests/internal/ds4_agent_unit.h tests/internal/ds4_agent_unit.inc $(CC) $(CFLAGS) -DDS4_NO_GPU -DDS4_AGENT_TEST \ -DDS4_AGENT_TEST_NO_MAIN -Wno-unused-function -c -o $@ ds4_agent.c @@ -607,19 +737,19 @@ ds4_test_core.o: ds4.c ds4.h ds4_ssd.h ds4_profile.h \ $(CC) $(CFLAGS) $(QWEN_CFLAGS) -DDS4_NO_GPU -DDS4_TEST_HOOKS \ -Wno-unused-function -Wno-unused-parameter -c -o $@ ds4.c -ds4_cli_cpu.o: ds4_cli.c ds4.h ds4_ssd.h ds4_help.h linenoise.h +ds4_cli_cpu.o: ds4_cli.c ds4.h ds4_ssd.h ds4_help.h hebrus_identity.h linenoise.h $(CC) $(CFLAGS) -DDS4_NO_GPU -c -o $@ ds4_cli.c ds4_server_cpu.o: ds4_server.c ds4.h ds4_ssd.h ds4_help.h ds4_kvstore.h rax.h $(CC) $(CFLAGS) -DDS4_NO_GPU -c -o $@ ds4_server.c -ds4_bench_cpu.o: ds4_bench.c ds4.h ds4_ssd.h ds4_help.h +ds4_bench_cpu.o: ds4_bench.c ds4.h ds4_ssd.h ds4_help.h hebrus_identity.h $(CC) $(CFLAGS) -DDS4_NO_GPU -c -o $@ ds4_bench.c -ds4_eval_cpu.o: ds4_eval.c ds4.h ds4_ssd.h ds4_help.h +ds4_eval_cpu.o: ds4_eval.c ds4.h ds4_ssd.h ds4_help.h hebrus_identity.h $(CC) $(CFLAGS) -DDS4_NO_GPU -c -o $@ ds4_eval.c -ds4_agent_cpu.o: ds4_agent.c ds4.h ds4_ssd.h ds4_help.h ds4_kvstore.h ds4_web.h linenoise.h +ds4_agent_cpu.o: ds4_agent.c ds4.h ds4_ssd.h ds4_help.h hebrus_identity.h ds4_kvstore.h ds4_web.h linenoise.h $(CC) $(CFLAGS) -DDS4_NO_GPU -c -o $@ ds4_agent.c ds4_test: ds4_test.o ds4_help.o ds4_kvstore.o rax.o $(CPU_CORE_OBJS) @@ -629,7 +759,7 @@ ds4_agent_test: ds4_agent_test.o ds4_agent_test_impl.o ds4_help.o \ ds4_web.o ds4_kvstore.o linenoise.o $(CPU_CORE_OBJS) $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) -model-free-test: ds4 ds4_test ds4_agent_test ds4-eval q4k-dot-test \ +model-free-test: $(PROGRAMS) ds4_test ds4_agent_test q4k-dot-test \ tests/test_q4k_top8 \ tests/test_qwen_session \ tests/test_qwen_tokenizer \ @@ -638,7 +768,12 @@ model-free-test: ds4 ds4_test ds4_agent_test ds4-eval q4k-dot-test \ tests/test_qwen_expert_group \ tests/test_expert_store \ tests/test_metal_ssd_profile \ - tests/test_ssd_residency download-model-test + tests/test_ssd_residency download-model-test visible-identity-test \ + server-alias-model-unit-test \ + tests/test_capabilities.py \ + tests/test_command_aliases.py + python3 tests/test_capabilities.py --bin-dir . --backend cpu + python3 tests/test_command_aliases.py --bin-dir . --backend cpu --layout profile sh tests/test_retired_distributed_flags.sh sh tests/test_benchmark_env_guard.sh ./ds4-eval --self-test-extractors @@ -662,6 +797,18 @@ model-free-test: ds4 ds4_test ds4_agent_test ds4-eval q4k-dot-test \ ./tests/test_ssd_residency python3 tests/test_qwen_metadata.py ./ds4 +capabilities-test: $(PROGRAMS) tests/test_capabilities.py + python3 tests/test_capabilities.py --bin-dir . --backend cpu + +command-alias-test: $(PROGRAMS) tests/test_command_aliases.py + python3 tests/test_command_aliases.py --bin-dir . --backend cpu --layout profile + +tests/test_visible_identity: tests/test_visible_identity.c hebrus_identity.h + $(CC) $(CFLAGS) -DDS4_NO_GPU -I. -o $@ $< + +visible-identity-test: tests/test_visible_identity + ./tests/test_visible_identity + test: model-free-test ./ds4_test @@ -677,7 +824,10 @@ imatrix-dataset-check: prompt-fixture-check: python3 speed-bench/build_long_context_prompt.py --check -premerge: context-audit doc-links imatrix-dataset-check prompt-fixture-check model-free-test +premerge: context-audit doc-links brand-boundary-audit brand-boundary-test brand-asset-test \ + release-contract release-contract-test \ + imatrix-dataset-check prompt-fixture-check model-free-test + $(MAKE) install-test git diff --check q4k-dot-test: tests/test_q4k_dot.c @@ -696,24 +846,23 @@ tests/test_q4k_top8: test_q4k_top8.o ds4_test_core.o ds4_build_cpu.o \ tests/test_qwen_session: tests/test_qwen_session.c ds4.c ds4.h ds4_ssd.h ds4_profile.h \ ds4_gpu.h ds4_qwen.h ds4_qwen_unicode.h \ - ds4_build.c ds4_ssd.c ds4_profile.c ds4_qwen.c \ + ds4_build.c ds4_expert_store.o ds4_ssd.c \ + ds4_profile.c ds4_qwen.c \ ds4_qwen_unicode.c ds4_qwen_unicode_data.inc \ ds4_streaming_hotlist.inc runtime/ds4_glm_graph.inc \ runtime/ds4_deepseek_cache_phase.inc $(CC) $(CFLAGS) $(QWEN_CFLAGS) -DDS4_NO_GPU \ -Wno-unused-function -Wno-unused-parameter -I. -o $@ \ - tests/test_qwen_session.c ds4_build.c ds4_ssd.c \ - ds4_profile.c ds4_qwen.c ds4_qwen_unicode.c $(LDLIBS) + $(filter-out ds4.c,$(filter %.c %.o,$^)) $(LDLIBS) tests/test_qwen_tokenizer: tests/test_qwen_tokenizer.c ds4.c ds4.h \ ds4_kvstore.c ds4_kvstore.h ds4_ssd.h ds4_profile.c ds4_profile.h ds4_gpu.h ds4_qwen.h \ - ds4_qwen_unicode.h ds4_build.c ds4_ssd.c \ + ds4_qwen_unicode.h ds4_build.c ds4_expert_store.o ds4_ssd.c \ ds4_qwen.c ds4_qwen_unicode.c ds4_qwen_unicode_data.inc \ ds4_streaming_hotlist.inc tests/qwen/qwen36_tokenizer_fixture.inc $(CC) $(CFLAGS) $(QWEN_CFLAGS) -DDS4_NO_GPU \ -Wno-unused-function -Wno-unused-parameter -I. -o $@ \ - tests/test_qwen_tokenizer.c ds4_kvstore.c ds4_build.c ds4_ssd.c \ - ds4_profile.c ds4_qwen.c ds4_qwen_unicode.c $(LDLIBS) + $(filter-out ds4.c,$(filter %.c %.o,$^)) $(LDLIBS) qwen-metadata-test: ds4 tests/test_qwen_metadata.py python3 tests/test_qwen_metadata.py ./ds4 @@ -782,9 +931,94 @@ metal-ssd-profile-test: tests/test_metal_ssd_profile endif +server-alias-model-unit-test: tests/test_server_alias_model.py \ + tests/test_server_alias_model_unit.py docs/contracts/qwen-release.json + python3 tests/test_server_alias_model_unit.py + +# Deliberately opt-in and excluded from premerge: this loads the complete +# published Qwen artifact twice and must run only on an isolated qualified host. +server-alias-model-test: $(INSTALL_SOURCE_BINDIR)/hebrus-server \ + $(INSTALL_SOURCE_BINDIR)/ds4-server \ + tests/test_server_alias_model.py docs/contracts/qwen-release.json + @test -n "$(QWEN_V2)" || { \ + echo "server-alias-model-test: set QWEN_V2 to the absolute published artifact path" >&2; \ + exit 2; \ + } + @set -eu; \ + set --; \ + if test -n "$(SERVER_ALIAS_EVIDENCE_DIR)"; then \ + set -- --evidence-dir "$(SERVER_ALIAS_EVIDENCE_DIR)"; \ + fi; \ + python3 tests/test_server_alias_model.py \ + --model "$(QWEN_V2)" \ + --bin-dir "$(INSTALL_SOURCE_BINDIR)" \ + --expected-backend "$(SERVER_ALIAS_EXPECTED_BACKEND)" \ + --expected-build-sha "$(SERVER_ALIAS_EXPECTED_BUILD_SHA)" \ + --port "$(SERVER_ALIAS_PORT)" "$$@" + +install: $(INSTALL_SOURCE_PROGRAMS) + @set -eu; \ + case "$(BINDIR)" in \ + /*) ;; \ + *) echo "install: BINDIR must be absolute: $(BINDIR)" >&2; exit 2 ;; \ + esac; \ + dest="$(INSTALL_DEST_BINDIR)"; \ + mkdir -p "$$dest"; \ + for name in $(PROGRAMS); do \ + path="$$dest/$$name"; \ + if [ -d "$$path" ] && [ ! -L "$$path" ]; then \ + echo "install: refusing to replace directory $$path" >&2; \ + exit 2; \ + fi; \ + done; \ + tmp=; \ + trap 'test -z "$$tmp" || rm -f "$$tmp"' 0 1 2 3 15; \ + for name in $(HEBRUS_PROGRAMS); do \ + tmp="$$dest/.$$name.install.$$$$"; \ + rm -f "$$tmp"; \ + $(INSTALL) -m "$(INSTALL_MODE)" \ + "$(INSTALL_SOURCE_BINDIR)/$$name" "$$tmp"; \ + rm -f "$$dest/$$name"; \ + mv "$$tmp" "$$dest/$$name"; \ + tmp=; \ + done; \ + for canonical in $(HEBRUS_PROGRAMS); do \ + legacy=ds4$${canonical#hebrus}; \ + tmp="$$dest/.$$legacy.install.$$$$"; \ + rm -f "$$tmp"; \ + ln -s "$$canonical" "$$tmp"; \ + rm -f "$$dest/$$legacy"; \ + mv "$$tmp" "$$dest/$$legacy"; \ + tmp=; \ + done + +uninstall: + @set -eu; \ + case "$(BINDIR)" in \ + /*) ;; \ + *) echo "uninstall: BINDIR must be absolute: $(BINDIR)" >&2; exit 2 ;; \ + esac; \ + dest="$(INSTALL_DEST_BINDIR)"; \ + for name in $(PROGRAMS); do \ + path="$$dest/$$name"; \ + if [ -d "$$path" ] && [ ! -L "$$path" ]; then \ + echo "uninstall: refusing to remove directory $$path" >&2; \ + exit 2; \ + fi; \ + done; \ + for name in $(PROGRAMS); do \ + rm -f "$$dest/$$name"; \ + done + +install-test: $(INSTALL_SOURCE_PROGRAMS) tests/test_install.sh \ + tests/test_capabilities.py tests/test_command_aliases.py + HEBRUS_INSTALL_BACKEND="$(INSTALL_BACKEND)" MAKE="$(MAKE)" \ + sh tests/test_install.sh + clean: rm -rf "$(BUILD_ROOT)" - rm -f ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native \ + rm -f hebrus hebrus-server hebrus-bench hebrus-eval hebrus-agent \ + ds4 ds4-server ds4-bench ds4-eval ds4-agent ds4_cpu ds4_native \ ds4_server_test ds4_test ds4_agent_test tests/test_q4k_dot \ tests/test_q4k_top8 \ tests/test_qwen_session \ @@ -793,4 +1027,6 @@ clean: tests/test_qwen_state tests/test_qwen_unicode \ tests/test_qwen_expert_group \ tests/test_expert_store \ - tests/test_ssd_residency *.o + tests/test_metal_ssd_profile \ + tests/test_ssd_residency \ + tests/test_visible_identity *.o diff --git a/QA_BEFORE_RELEASES.md b/QA_BEFORE_RELEASES.md index 7261f8b98..9f8e84af6 100644 --- a/QA_BEFORE_RELEASES.md +++ b/QA_BEFORE_RELEASES.md @@ -1,6 +1,6 @@ # QA Before Releases -This is the release gate for DwarfStar. Run it before tagging or pushing a +This is the release gate for Hebrus. Run it before tagging or pushing a release build. The goal is not to prove every code path exhaustively; it is to exercise the paths that have historically regressed: Metal graph inference, model-family residency, SSD streaming, distributed rejection, disk KV cache, @@ -30,28 +30,76 @@ target, or an artifact whose complete output hash is missing. | `DEEPSEEK_V2` | `DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-DS4-ExpertMajor-v2.gguf`; 86,720,114,272 bytes; SHA-256 `8378080263eb9224f7228d72e2afa4ac3cf74a116023fdec2c596ff228a33e3f` | | `DEEPSEEK_MIXED_V2` | Non-applicable until a mixed-quant DeepSeek ExpertMajor v2 artifact has a publication record with exact filename, bytes, and complete output SHA-256; do not resolve or use this variable before qualification | | `GLM_V2` | `GLM-5.2-DS4-ExpertMajor-v2-Q2_K.gguf`; 262,147,193,504 bytes; SHA-256 `7f5017e3076e706c78f2a5322b035a9e2f6519c65ff5b6be8b2d91aeff61505d` | -| `QWEN_V2` | `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf`; 20,808,566,880 bytes; SHA-256 `dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d` | +| `QWEN_V2` | Status `published`; `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf`; immutable repository revision `7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02`; 20,808,566,880 bytes; SHA-256 `dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d`; MLX affine4/group-64 routed storage; minimum compatible runtime commit `73a332fef82a0bcdd567d17e0de17aa004cad85d` | +| `QWEN_RETIRED_Q4_NEGATIVE` | Status `negative-only`; rejection-only input `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-Q4_K_S.gguf`; 20,808,566,880 bytes; SHA-256 `d7c43a6388ec20e6fe5530850350f96fdb0ac37c5ce36d3e5f92b172c447f56b`; it must fail before inference | + +The machine-readable +[Qwen release contract](docs/contracts/qwen-release.json) is the canonical +source for both Qwen rows. `make release-contract` rejects drift in this table, +the public documentation, the downloader, and its model-free tests. Record the test machine by hardware model, unified memory, OS build, and power state in the release evidence. Do not encode local hostnames, addresses, or network routes in this checklist. +The Qwen runtime and downloader paths must both resolve the exact `QWEN_V2` +identity above. The immutable revision, filename, byte count, complete output +SHA-256, manifest contract, and compatible runtime commit are one release gate. + ## 1. Repository And Build Sanity -- Start from a clean tree except intentional release notes: - `git status --short`. +- Start from the exact clean committed release tree. Release notes and every + other intentional change must be committed first; untracked files are not + allowed: `test -z "$(git status --porcelain)"`. - Build the normal local target: `make clean && make`. - Prove macOS Metal/CPU artifact isolation: `make build-isolation-test`. +- Confirm the GitHub Linux and macOS jobs passed on the exact commit. Treat the + hosted macOS job as compile/model-free evidence only, never as a substitute + for the Metal kernel and qualified-model lanes on release hardware. - Build CPU-only binaries as a compile check. On macOS they remain under `build/cpu-$(uname -m)/bin` and must not replace the root Metal commands: `make cpu`. -- Record `./ds4 --build-info` and the CPU binary's `--build-info` output. +- Record `./hebrus --build-info`, `./ds4 --build-info`, and the CPU profile's + canonical and compatibility `--build-info` output. +- Run `--capabilities=json` on all five canonical and all five compatibility + Metal commands and the complete CPU profile; + validate schema version 1, executable roles, backend identity, model-family + claims, and ExpertMajor wire values with `make capabilities-test` and + `python3 tests/test_capabilities.py --bin-dir build/cpu-$(uname -m)/bin --backend cpu`. +- Run `make command-alias-test` and confirm each `ds4*` profile command is a + symlink to its single real `hebrus*` binary, with exact build-info, help, + capability, invalid-option, and retired-option parity. +- Run `make install-test` and confirm a temporary `DESTDIR` receives five real + canonical executables, five relative compatibility aliases, valid capability + documents, no embedded checkout/staging paths, and an explicit-only, + idempotent uninstall. - Run whitespace checks before committing: `git diff --check`. -- Confirm `./ds4 --help`, `./ds4-server --help`, and `./ds4-agent --help` render - cleanly, with readable section colors and no broken wrapping. +- Confirm both names of the CLI, server, and agent render help cleanly, with + readable section colors and no broken wrapping. +- On a qualified Qwen host, run the two server names sequentially against the + exact published artifact and compare model discovery, one seeded greedy chat + completion, graceful shutdown, and exit status. Set + `RELEASE_EVIDENCE_ROOT` explicitly to a persistent, release-owned absolute + directory outside the checkout, then run: + `QWEN_V2="$QWEN_V2" SERVER_ALIAS_EVIDENCE_DIR="$RELEASE_EVIDENCE_ROOT/hebrus-server-alias-$(git rev-parse --short=12 HEAD)" make server-alias-model-test`. + The opt-in target rejects a dirty or wrong engine build, validates Metal + capabilities before loading the model, selects an unused local port by + default, and retains its JSON report and logs. The evidence directory must + not already exist. Archive the complete evidence directory only when the + report is final `PASS` and + `server-alias-evidence-manifest.json` is present; verify every artifact hash + and the manifest's deterministic `bundle_sha256`, defined as SHA-256 over + compact sorted-key JSON containing only `schema_version` and `artifacts`. + `RUNNING`, a missing or invalid manifest, and every `FAIL` report are invalid + release evidence. The report records non-sensitive architecture, memory, + OS/build, and available + power metadata, but the release record must still identify the hardware + model and explain unavailable fields. This is model-backed correctness + evidence, not a performance benchmark, and the target is intentionally + excluded from `premerge`. ## 2. Core Regression Tests @@ -188,13 +236,14 @@ context. Do not add an explicit residency, cache, preload, or ExpertMajor flag. The Qwen path on `main` follows the same repository, build, core-test, and regression rules as the other model paths. Use the verified normalized -`QWEN_V2` artifact above and run the relevant model-backed smoke; +`QWEN_V2` above and run the relevant model-backed smoke; canonical, v1, sidecar, and community GGUFs are not equivalent inputs. - Run `make model-free-test` and `./ds4_test --metal-kernels`. The latter must retain resident/SSD top-8 output equivalence, zero resident cache/`pread` accounting, malformed-route fail-closed behavior, and slab-growth checks. -- Run `./ds4 -m "$QWEN_V2" --ctx 8192` for the normal flag-free AUTO smoke. +- Run `./ds4 -m "$QWEN_V2" --ctx 8192` for the normal flag-free + AUTO smoke. - Run AUTO with the normal flag-free startup command; record both admission plans, their point-in-time inputs, resolved mode, cache tier, configured 321-expert slab target, cache `buffer_allocs`, task physical footprint, and system swap @@ -209,6 +258,10 @@ canonical, v1, sidecar, and community GGUFs are not equivalent inputs. - Resident mode proves complete model mapping and full-tensor Metal execution, not that every mapped GGUF page remained physically resident. Measure the stronger claim separately if it is used in release language. +- Run the model-free ExpertMajor admission fixture for a Qwen storage value of + GGML/Q4, and, when the exact retired file is available, run + `./ds4 -m "$QWEN_RETIRED_Q4_NEGATIVE" --ctx 8192`. Both must reject before + inference; a command that produces tokens is a release blocker. - Physical 16 GiB measurements and normalized-vs-source research comparisons improve hardware and artifact characterization, but are not additional release gates beyond the standard model/backend checks above. Do not claim @@ -272,8 +325,8 @@ before model loading: `sh tests/test_retired_distributed_flags.sh` -The gate covers `ds4`, `ds4-server`, `ds4-agent`, `ds4-bench`, and `ds4-eval`, -and all nine retired flags: `--role`, `--layers`, `--listen`, `--coordinator`, +The gate covers all five `hebrus*` commands and their five `ds4*` aliases, and +all nine retired flags: `--role`, `--layers`, `--listen`, `--coordinator`, `--dist-prefill-chunk`, `--dist-prefill-window`, `--dist-activation-bits`, `--dist-replay-check`, and `--debug`. Also confirm: @@ -352,8 +405,10 @@ The agent is the most stateful component. Test it manually, not only by build. - Test `download_model.sh` in a temporary directory so local weights are not overwritten. -- Verify the `deepseek-v2`, `glm-v2`, and `qwen-v2` targets resolve to the exact - qualified repository and ExpertMajor v2 filename. +- Verify `deepseek-v2`, `glm-v2`, and `qwen-v2` resolve to their exact qualified + repository, immutable revision, and ExpertMajor v2 filename. Qwen must pin + revision `7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02` and must not resolve to the + retired Q4_K_S object. - Treat every `offline-*` target as a converter input, not a runnable artifact. Verify resume and file naming without launching inference from the source. - Verify the script never creates or changes `./ds4flash.gguf`, exposes no @@ -379,8 +434,13 @@ The agent is the most stateful component. Test it manually, not only by build. Do not sign off until: - macOS Metal Flash passed. -- The qualified DeepSeek, GLM, and Qwen artifacts passed their model-backed - lanes with the residency modes defined by `docs/contracts/RUNTIME_SUPPORT.md`. +- The qualified DeepSeek, GLM, and Qwen release artifacts passed their + model-backed lanes with the residency modes defined by + `docs/contracts/RUNTIME_SUPPORT.md`. +- The Qwen immutable publication record and downloader gate matched the same + exact affine artifact used by the runtime lane. +- The retired Qwen Q4_K_S store remained fail-closed and was not presented as a + runnable or downloadable fallback. - CUDA source/tests/build targets were confirmed absent and recorded as frozen; if any were restored, the complete section 6 reactivation gate passed. - ROCm source/tests/build targets were confirmed absent and recorded as frozen; diff --git a/README.md b/README.md index 23b8dc1f8..8c18b87e9 100644 --- a/README.md +++ b/README.md @@ -1,573 +1,292 @@ -

DwarfStar

- -

ExpertMajor v2 inference for large MoE models on Apple Metal.

-

- A transparent research and co-development fork of - antirez/ds4, focused on Metal, - adaptive SSD streaming, Apple Silicon systems, and measured experimentation. + Hebrus logo

-

- MIT license - Apple Metal primary target - ExpertMajor v2 model format - Project status beta -

+# Hebrus -

- Quick start - · Benchmarks - · Models - · DSBox - · Upstream diff - · Documentation -

+**Local inference built first for Apple Metal and SSD streaming.** + +Hebrus is a focused inference engine for running a small, explicitly qualified +set of large mixture-of-experts models on Apple Silicon. It combines mmap-backed +GGUF loading, an embedded ExpertMajor v2 routed-weight store, Metal execution, +and adaptive SSD streaming so unified memory is a performance tier rather than +the only capacity boundary. + +[Quick start](#quick-start) · [Supported models](#supported-models) · +[Measured results](#measured-results) · [Documentation](#documentation) · +[Contributing](CONTRIBUTING.md) + +> [!NOTE] +> Hebrus is the working public engine name and the canonical executable +> namespace in the current bridge. The repository is still +> [`andreaborio/ds4`](https://github.com/andreaborio/ds4), and +> [ADR 0005](docs/adr/0005-hebrus-naming-and-compatibility-boundary.md) remains +> **Proposed** while namespace and legal screening remain incomplete. The +> companion application is named **Hebrus Studio** and its bridge release is +> implemented locally but not yet published. This repository has not been +> administratively renamed. > [!IMPORTANT] -> This is [`andreaborio/ds4`](https://github.com/andreaborio/ds4), a fork of -> [`antirez/ds4`](https://github.com/antirez/ds4). It does **not** aim to replace -> upstream. The goal is to co-develop DwarfStar: explore complementary hardware -> and model paths here, then propose every general, reproducible improvement back -> to upstream when it clears the correctness and performance bar. - -DwarfStar is a small, self-contained inference engine optimized around a narrow -set of very large models. It includes native model loading, prompt rendering, -tool calling, RAM/on-disk KV state, an HTTP server, a coding agent, GGUF tooling, -and correctness and speed tests. It is intentionally **not** a generic GGUF -runner; arbitrary GGUF files are not expected to work. - -## Why this fork exists - -Upstream DwarfStar provides the core engine and leads the high-memory and -distributed paths. This fork asks a complementary question: - -**How far can the same specialized design be pushed across Apple Silicon memory -tiers, when the SSD becomes an active model-memory tier?** - -The current work concentrates on: - -- adaptive Metal residency and routed-expert cache policies across memory tiers; -- SSD streaming that accounts for page cache, wired memory, swap, I/O, and - throughput together; -- safer model-backed experiments near macOS memory limits; -- one validated ExpertMajor v2 storage contract for DeepSeek, GLM, and Qwen; -- GGUF calibration, incremental quantization, and expert-analysis tooling; -- keeping useful changes small enough to validate and send upstream. - -This is primarily a learning and systems-research project. The fork lets work -continue while upstream changes are under review; it is not a parallel rewrite -or a competing inference ecosystem. - -## Co-development with upstream - -The boundary is explicit and reviewable: - -| Change type | Where it belongs | -| --- | --- | -| General, reproducible, backend-safe improvement | Open a PR against [`antirez/ds4`](https://github.com/antirez/ds4) | -| Model- or hardware-specific experiment | Keep it isolated while evidence is incomplete; open an upstream PR too whenever it applies to an upstream-supported path | -| Change that regresses an existing path | Do not promote it; isolate or revise it first | -| Change already solved upstream | Take the upstream implementation and remove the fork delta | - -This is mandatory, not aspirational: every fork change applicable to an -upstream-supported path will be opened upstream once its scope, correctness, -and performance evidence are ready. Fork development can continue while that -review is in progress. - -Current upstream work includes [#434](https://github.com/antirez/ds4/pull/434) -(quality-score build fix), [#520](https://github.com/antirez/ds4/pull/520) -(GLM streamed-prefill correctness), and -[#528](https://github.com/antirez/ds4/pull/528) (GLM indexed-prefill prepare). -The DeepSeek regression found on the GLM line is tracked in -[#532](https://github.com/antirez/ds4/issues/532). See -[`FORK_NOTES.md`](FORK_NOTES.md) for the status of each fork change and -[`MERGE_LOG.md`](MERGE_LOG.md) for sync history. The same policy is part of -[`CONTRIBUTING.md`](CONTRIBUTING.md). The current `main` delta is always -inspectable in GitHub's -[upstream/fork comparison](https://github.com/antirez/ds4/compare/main...andreaborio:ds4:main). +> Hebrus began as a fork of +> [`antirez/ds4`](https://github.com/antirez/ds4) and retains substantial +> core-engine implementation, Git history, and design work from that project. +> The fork has since diverged toward Apple Metal, embedded ExpertMajor storage, +> and SSD-first execution. This attribution does not imply endorsement by or an +> official partnership with the upstream maintainer. See +> [Acknowledgments](ACKNOWLEDGMENTS.md) and +> [Third-party notices](THIRD_PARTY_NOTICES.md) for the precise source scopes. + +## What Hebrus is for + +Hebrus is designed around one practical question: how effectively can large +MoE models run locally when Metal and the SSD are treated as one coordinated +memory system? + +- **Metal-first inference.** Production model execution targets Apple Metal. +- **SSD streaming as a first-class path.** Routed experts can be cached and + streamed without keeping the complete model in unified memory. +- **Hardware-aware AUTO policy.** Residency and cache plans use model geometry, + context memory, Metal's working-set recommendation, and live memory pressure. +- **One validated routed-weight container.** Supported artifacts embed the + checksummed `ds4.expert_major.v2` store directly in GGUF. +- **Narrow support, explicit failure.** Unsupported model families, old stores, + sidecars, and unqualified backend combinations fail closed instead of taking + an untested fallback. +- **Evidence before claims.** Correctness and performance decisions are tied to + exact commits, model hashes, hardware, prompts, memory state, and context + frontiers. + +Hebrus is not a general GGUF runner. Arbitrary community GGUFs are not expected +to load, even when their model family name looks compatible. + +## Supported models + +The current production contract is Apple Silicon + Metal + embedded +ExpertMajor v2. AUTO is the normal startup mode. + +| Model family | Minimum unified memory | Qualified execution | Artifact availability | +| --- | ---: | --- | --- | +| DeepSeek V4 Flash | 64 GiB | AUTO resolving to resident or SSD; explicit modes for qualification | Published ExpertMajor v2 artifact; `download_model.sh deepseek-v2` | +| GLM 5.2 | 64 GiB | AUTO resolving to SSD streaming only; resident requests are rejected | Published ExpertMajor v2 artifact; `download_model.sh glm-v2` | +| Qwen3.6-35B-A3B | 16 GiB | AUTO resolving to resident or SSD; 16 GiB necessarily uses SSD | Published MLX affine4/group-64 artifact; `download_model.sh qwen-v2` | + +The canonical machine-readable +[Qwen release contract](docs/contracts/qwen-release.json) records the current +artifact as `published`. The release is exactly +`Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf`, +20,808,566,880 bytes, SHA-256 +`dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d`. +It is pinned to immutable repository revision +`7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02` and requires runtime commit +`73a332fef82a0bcdd567d17e0de17aa004cad85d` or a compatible descendant. The +older `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-Q4_K_S.gguf` store is incompatible +with the current runtime and remains `negative-only`, never a download or +inference fallback. + +DeepSeek V4 PRO has no qualified runtime artifact in the current contract. +Canonical model files are converter inputs, not inference fallbacks. CPU is a +reference and build-isolation path; CUDA and ROCm are frozen with source absent; +distributed inference is retired. + +The authoritative details are in the +[runtime support contract](docs/contracts/RUNTIME_SUPPORT.md). ## Quick start -Requirements: Apple Silicon, Xcode Command Line Tools, an ExpertMajor v2 GGUF, -and enough SSD space for the selected model. DeepSeek and GLM currently require -at least 64 GiB. Qwen's hardware-aware AUTO path starts at 16 GiB and chooses -resident or SSD from the active Metal and live-memory budgets. +Requirements: + +- Apple Silicon Mac; +- Xcode Command Line Tools; +- enough unified memory for the selected supported family; +- enough SSD space for the exact ExpertMajor v2 artifact. + +Build and verify the engine without loading model weights: ```sh xcode-select --install git clone https://github.com/andreaborio/ds4.git cd ds4 -make -j8 -./ds4 --build-info -./download_model.sh qwen-v2 -./ds4 \ - -m gguf/Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf \ - --ctx 8192 +make -j +./hebrus --build-info +./hebrus --capabilities=json +make model-free-test ``` -`download_model.sh deepseek-v2`, `download_model.sh glm-v2`, and -`download_model.sh qwen-v2` install revision-pinned release artifacts and -verify their complete byte size and SHA-256 before returning success. - -On macOS, AUTO residency keeps the model resident when it safely fits. -Otherwise it selects SSD streaming and derives an expert-cache budget from the -model geometry and live host memory. Force the SSD path only when you need a -controlled run: +For a user-local installation that does not require `sudo`: ```sh -./ds4 -m /absolute/path/to/MODEL-DS4-ExpertMajor-v2.gguf \ - --ssd-streaming --ctx 8192 +make install PREFIX="$HOME/.local" +export PATH="$HOME/.local/bin:$PATH" + +hebrus --build-info ``` -Start the local API with: +`make install` copies the five canonical `hebrus*` executables and creates the +five `ds4*` compatibility names as relative symlinks to those files. Package +builders can stage the same layout without touching the host filesystem: ```sh -./ds4-server -m /absolute/path/to/MODEL-DS4-ExpertMajor-v2.gguf --ctx 8192 +make install DESTDIR="$PWD/package-root" PREFIX=/usr/local +make uninstall DESTDIR="$PWD/package-root" PREFIX=/usr/local ``` -## How SSD streaming uses memory - -```mermaid -flowchart LR - GGUF["Model GGUF on SSD"] --> AUTO["AUTO memory planner"] - AUTO -->|"safe fit"| RES["Resident model"] - AUTO -->|"model exceeds budget"| STREAM["SSD-streamed model"] - STREAM --> FIXED["Mapped fixed / non-routed state"] - STREAM --> CACHE["Adaptive routed-expert cache"] - GGUF -->|"cache miss"| CACHE - FIXED --> METAL["Metal graph"] - CACHE --> METAL - METAL --> TOKEN["Next token"] -``` +`PREFIX` defaults to `/usr/local`; `BINDIR` may override its `bin` directory. +Uninstall removes only the ten explicit command paths and leaves the directory +and every unrelated file intact. `make install-test` verifies the staged +layout, aliases, capabilities, and uninstall boundary without a model or +privileged access. -The fixed model state, KV cache, graph scratch, and macOS file-backed cache all -need headroom. The routed-expert cache is the variable tier; making it larger -can help only until it starts displacing the pages and allocations the rest of -the runtime needs. - -## Model status - -| Model | Location | Status | Current focus | -| --- | --- | --- | --- | -| DeepSeek V4 Flash ExpertMajor v2 | `main` | Supported Apple Metal resident/SSD path | Adaptive SSD streaming and grouped prefill | -| DeepSeek V4 PRO ExpertMajor v2 | offline tooling only | Not runtime-qualified; no release artifact identity | Converter and future per-artifact qualification | -| GLM 5.2 ExpertMajor v2 | `main` | Qualified Apple Metal SSD path; 64 GB minimum | Embedded expert store, grouped prefill, compact DSA KV, 601-record DS4 cache plus adaptive macOS file caching | -| Qwen3.6-35B-A3B ExpertMajor v2 affine4/group-64 | `main` | Supported Apple Metal resident/SSD path; 16 GiB minimum | One AUTO configuration, MLX-affine resident/SSD kernels, and read-once macro prefill | - -### DeepSeek expert-major v2 format - -DS4 uses the self-describing `ds4.expert_major.v2` layout for DeepSeek V4. It -stores each layer as adjacent gate/up/down expert records without requantizing -and without keeping a second routed-weight copy. The runtime accepts only the -native v2 artifact on Apple Metal. Canonical GGUFs are converter inputs, not an -inference fallback; CPU, CUDA, ROCm, distributed, v1, and sidecar paths fail -closed. - -Conversion, full byte-level verification, admission limits, and the -model-backed promotion gate are in -[`docs/deepseek-expert-major-v2.md`](docs/deepseek-expert-major-v2.md). The first -M5 Pro SSD tranche is recorded in -[`docs/benchmarks/2026-07-17-deepseek-native-expert-major.md`](docs/benchmarks/2026-07-17-deepseek-native-expert-major.md). -The distinctly named v2 artifact is -[`DeepSeek-V4-Flash-DS4-GGUF`](https://huggingface.co/andreaborio/DeepSeek-V4-Flash-DS4-GGUF), -with full conversion provenance and runtime limits in its model card. - -### GLM 5.2 ExpertMajor v2 - -The 262,147,193,504-byte GLM release stores its 76 × 256 routed experts once, -inside the GGUF, in the physical order used by Metal prefill and decode. It -does not need a sidecar or ExpertMajor environment variables. The qualified -tier is an Apple Silicon Mac with at least 64 GiB of unified memory. GLM in -this repo intentionally supports only the ExpertMajor v2 Metal SSD path: +For a published DeepSeek V4 Flash artifact on a qualified 64 GiB-or-larger +host: ```sh -make -j8 -./ds4 \ - -m /absolute/path/to/GLM-5.2-DS4-ExpertMajor-v2-Q2_K.gguf \ - --ctx 8192 -``` +./download_model.sh deepseek-v2 -The artifact is -[`andreaborio/GLM-5.2-DS4-GGUF`](https://huggingface.co/andreaborio/GLM-5.2-DS4-GGUF), -SHA-256 -`7f5017e3076e706c78f2a5322b035a9e2f6519c65ff5b6be8b2d91aeff61505d`. -The simple command automatically selects Metal, SSD residency, the GLM Gold -profile, the measured 601-expert cache on the 64 GiB tier, indexed preparation, -grouped native prefill, contiguous-record decode reads, and compact DSA KV. -Canonical GLM GGUFs and non-Metal/distributed execution fail closed. Do not add -cache, preload, full-layer, or ExpertMajor flags. Format, memory flow, tier -boundaries, and current limits are in -[`docs/glm52-expert-major-v2.md`](docs/glm52-expert-major-v2.md). -Older GLM files, sidecars, layout revisions, and retired tuning modes have no -backward-compatibility contract in this fork. - -### Qwen3.6 ExpertMajor v2 AUTO path - -The supported release is the single-layout -`Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf`. It stores the 40 -layers of routed weights once, activates automatically, and is 20,808,566,880 -bytes with SHA-256 -`dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d`. -It is still a GGUF with the shared `ds4.expert_major.v2` container; only the -routed payload uses MLX affine 4-bit groups (`32 packed bytes + BF16 scale + -BF16 bias` per group of 64). The previous v2 GGML/Q4 payload is deliberately -rejected rather than retained as a slower compatibility mode. -No environment guard, sidecar variable, backend flag, resident flag, or power -flag is part of normal startup: - -```sh -./ds4 \ - -m /absolute/path/to/Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf \ - --ctx 8192 +./hebrus \ + -m gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-DS4-ExpertMajor-v2.gguf \ + --ctx 8192 \ + -p "Explain why SSD streaming changes the memory limit for MoE inference." ``` -This is a narrow Qwen3.6 artifact contract, not arbitrary Qwen or community -GGUF support. Current inference rejects canonical Qwen, ExpertMajor v1, -sidecars, CPU, CUDA, ROCm, and distributed execution. The canonical GGUF may be -used offline with `gguf-tools/ds4-expert-major.py` to build and verify v2. -Format details and the current resident/SSD promotion gate are in -[`docs/qwen-expert-major-store.md`](docs/qwen-expert-major-store.md) and -[`docs/benchmarks/2026-07-21-qwen-unified-affine-auto-ssd.md`](docs/benchmarks/2026-07-21-qwen-unified-affine-auto-ssd.md). - -Qwen AUTO reports a named 16/24/32/36/48/64/96/128 GiB profile, then computes -from the exact physical RAM and `recommendedMaxWorkingSetSize`; the labels do -not hardcode a cache count. Its resident reserve is -`max(2 GiB, RAM/16) + max(0.25 GiB, RAM/64)`. Full-model mapped Metal mode is -selected only when both that fixed Metal plan and a point-in-time host-memory -pressure check pass. The 19.37 GiB tensor payload cannot be resident on 16 GiB, -so AUTO uses SSD there. A 32 GiB Mac can select resident for shorter contexts -when both gates pass and falls back to SSD when current pressure or context -runtime no longer fits. - -In SSD mode the planner charges static pages, context/runtime memory, and system -headroom, then selects the largest complete expert-cache cycle admitted by the -remaining live and platform budgets. Under normal macOS pressure, equivalent -free and bounded file-backed GGUF pages receive equal credit on every Qwen -profile; warming the model file therefore cannot by itself reduce AUTO cache. -Unknown or elevated pressure remains conservative, and the 16 GiB class -requires an affirmative normal-pressure signal. Its measured AUTO ceiling is -eleven complete routes plus the safety slot, or 3,521 experts for this 40x8 -model; the next tier produced swap and is not retained. `--resident` fails -unless both admission checks pass. `--ssd-streaming` remains a controlled-test -override. Cache storage grows in 321-expert slabs (about 0.529 GiB). - -Here `resident` means that DS4 maps the complete tensor payload, disables its -explicit SSD expert cache, and executes full-tensor Metal kernels. Metal's -residency request is a budgeting hint: it neither pre-faults every GGUF page nor -proves that every page remains physically resident as later pressure changes. -That stronger physical-residency claim requires separate runtime measurement. -All neural math in the supported Qwen path is on Metal. The CPU still performs -tokenization, sampling, route readback, cache bookkeeping, and streamed GGUF -I/O; a CPU+GPU split of layers or experts is not implemented in this path. - -The hard SSD cache floor is 321 complete routed experts (about 0.53 GiB); 640 -(about 1.06 GiB) is a useful controlled small-cache tier. Startup and the -per-layer path fail closed if the effective locked cache falls below the floor. -The affine runtime has completed model-backed resident and SSD generation on an -M5 Pro with 64 GiB, including allocation at the model-declared 262,144-token -maximum. A final 128+16 lane produced identical resident/SSD greedy tokens. -Performance numbers remain tied to each exact host and workload; policy tests -for an unmeasured RAM cut are not presented as a throughput claim. - -## Measured results - -The current table uses the mandatory 32K long-context lane rather than mixing -short-context bests. These rows are not cross-model rankings: every model uses -a different artifact and runtime path. - -| Model | M5 Pro 64 GB setup | 32K prefill | 32K generation / decode | Status | -| --- | --- | ---: | ---: | --- | -| Qwen3.6-35B-A3B ExpertMajor v2 MLX affine4/group-64, 20.81 GB | Metal resident AUTO | **877.34 t/s** | **57.43 t/s** at the qualified 128+16 decode lane, 17.352 ms p50, 18.753 ms p95 | Same GGUF resident/SSD; forced SSD 32K is 83.69 t/s with 1.000x read amplification | -| DeepSeek V4 Flash IQ2XXS, 86.72 GB | Metal SSD, phase-adaptive 259→4,129 through 32K, prose prompt | **164.43 t/s** | **7.27 t/s**, 105.410 ms p50, 149.943 ms p95 | Exact final-stack evidence and zero swap; old AUTO controls can swap, so no 32K speedup percentage | -| GLM 5.2 ExpertMajor v2 Q2_K, 244.14 GiB | Metal AUTO→SSD, 601 records, fixed compact-indexer transition, prose prompt | **44.73 t/s** | **1.87 t/s** overall; about **2.12 t/s** at p50 | Same-prompt output matches the earlier corrected arm; zero swap; original baseline crashes before logits | - -Full identities, invalidations, control drift and remaining release gates are -in the -[`current Qwen split-K and hardware-policy record`](docs/benchmarks/2026-07-21-qwen-split-k-hardware-policy.md) -and the earlier -[`long-context Metal stack record`](docs/benchmarks/2026-07-20-long-context-metal-stack.md). -DeepSeek also completes isolated 65K+128 and 100K+128 AUTO→SSD lanes with the -2,065-expert extended-context tier at 137.29/6.59 and 145.11/6.44 prefill/decode -t/s respectively, with zero swapout. GLM's separate 32K security/coding prompt -lane completes at 45.53/1.33 t/s; its lower decode rate comes from a 13.15% -expert-cache hit rate versus 36.90% for the prose lane and is not a -same-condition regression. -Earlier short-context bests remain historical evidence in the dated records; -they are not substituted for the long-context promotion gate. - -Historical upstream DeepSeek hardware reference bests from the standard -`speed-bench` sweep follow. They are context for comparison, not a statement -that this v2-only fork runtime supports those non-Metal hosts: - -| Host | Model | Prefill | Generation | -| --- | --- | ---: | ---: | -| MacBook Pro M5 Max, 128 GB | Flash q2, 11,707-token context | 463.44 t/s | 25.90 t/s | -| Mac Studio M3 Ultra, 512 GB | Flash q2, 11,709-token context | 468.03 t/s | 27.39 t/s | -| Mac Studio M3 Ultra, 512 GB | PRO q2, 32,768-token context | 138.82 t/s | 9.56 t/s | -| DGX Spark GB10, 128 GB | Flash q2, 7,047-token context | 343.81 t/s | 13.75 t/s | - -The [`benchmark evidence index`](docs/benchmarks/README.md) classifies current, -historical, and rejected records. Full commands and caveats live in those dated -records; the latest pre-policy cross-model correctness/artifact baseline is the -[`2026-07-20 validation record`](docs/benchmarks/2026-07-20-agent-friendly-refactor-validation.md). -See also [`SSD_STREAMING_VERIFICATION.md`](SSD_STREAMING_VERIFICATION.md) and -[`docs/ENGINE_REFERENCE.md`](docs/ENGINE_REFERENCE.md). - -## Memory safety is part of performance - -More expert-cache RAM is not automatically faster. On memory-constrained Macs, -an oversized cache can evict the file-backed pages SSD streaming needs and make -decode slower even when Activity Monitor appears to show free memory. AUTO -therefore treats the routed-expert cache as variable and preserves headroom for -fixed weights, KV, scratch, Metal allocations, and the macOS page cache. - -During development, a model-backed test bypassed SSD streaming and attempted to -make an 80.76 GiB GGUF resident with a 100,000-token context on a 64 GiB Mac. -Global wired memory reached roughly 61.36 GiB before a watchdog kernel panic. -Crashing the host is not an acceptable test outcome. - -Current `main` includes hardware-aware AUTO residency, fail-closed cache -admission, bounded benchmark guards, and GPU cleanup before model mappings are -released (`1523b26`). A stricter guard that rejects resident mappings larger -than 90% of physical RAM is tested and published on -`fix/refuse-oversized-resident-maps` at `06fd005`, but is **not yet on `main`**. -Until it is merged, it must not be described as a mainline guarantee. - -## Prefer a desktop interface? - -[DSBox](https://github.com/andreaborio/dsbox) is the companion desktop -interface, inspired by Unsloth Studio: discover compatible models, manage ds4, -chat locally, connect coding agents, and observe memory, swap, disk, and token -throughput without hand-assembling every command. DSBox is a separate project -and still a work in progress. - -

- - DSBox local chat interface controlling a ds4 server - -
- DSBox is an optional companion UI, maintained in a separate repository. -

- -## Documentation - -- [`docs/releases/v0.2.0.md`](docs/releases/v0.2.0.md): current ExpertMajor - v2-only release boundary, startup, measured results, and upgrade notes. -- [`docs/ENGINE_REFERENCE.md`](docs/ENGINE_REFERENCE.md): complete model, - runtime, server, agent, KV-cache, supported-backend, retired-feature, and - debugging reference. -- [`docs/qwen-expert-major-store.md`](docs/qwen-expert-major-store.md): - Qwen v2 layout, converter, runtime contract, and parity evidence. -- [`docs/glm52-expert-major-v2.md`](docs/glm52-expert-major-v2.md): embedded GLM - artifact, one-command startup, prefill/decode memory flow, qualification, and - supported memory tiers. -- [`docs/expert-major-v2-roadmap.md`](docs/expert-major-v2-roadmap.md): shared - ExpertMajor v2 contract and family-specific qualification status. -- [`CONTRIBUTING.md`](CONTRIBUTING.md): upstream-first contribution policy and - correctness/performance gates. -- [`FORK_NOTES.md`](FORK_NOTES.md): fork delta and upstreamability ledger. -- [`MERGE_LOG.md`](MERGE_LOG.md): upstream synchronization history. -- [`GOLD_METAL_SSD.md`](GOLD_METAL_SSD.md): Metal build identity, AUTO residency, - and benchmark promotion gates. -- [`SSD_STREAMING_VERIFICATION.md`](SSD_STREAMING_VERIFICATION.md): superseded - July 2026 SSD-streaming campaign evidence; not a current runtime guide. -- [`ONEDGE_IMATRIX.md`](ONEDGE_IMATRIX.md): live, privacy-preserving imatrix - collection. -- [`EXPERT_PRUNE.md`](EXPERT_PRUNE.md): expert profiling and prune-mask research. -- [`gguf-tools/README.md`](gguf-tools/README.md): GGUF, imatrix, quantization, and - quality tooling. - ---- - -
-Detailed fork additions and research notes - -## Fork feature details - -The sections below preserve the longer design notes for the fork's research -features. They are not an exhaustive commit count: adaptive residency, cache -hardening, benchmark guardrails, telemetry, and safe Metal teardown have also -evolved since the original five-feature summary was written. The authoritative -per-change ledger is [`FORK_NOTES.md`](FORK_NOTES.md); upstream syncs are recorded -in [`MERGE_LOG.md`](MERGE_LOG.md). - -### The GLM 5.2 SSD-streaming path - -GLM 5.2 is promoted as a narrow Apple Metal path for the published embedded -ExpertMajor v2 artifact. It includes the streamed-prefill correctness and -indexed-prepare work proposed as [#520](https://github.com/antirez/ds4/pull/520) -and [#528](https://github.com/antirez/ds4/pull/528), plus model-specific compact -DSA KV, grouped Q2_K prefill, physical expert-record addressing, NextN metadata -binding, tokenizer/prompt routing, and a pressure-sized cache policy for the -64 GiB reference tier. - -All admitted families use ExpertMajor v2, but GLM does not become a compute or -cache fallback for DeepSeek or Qwen. Family checks select their independent -Metal schedules, while CPU, CUDA, ROCm, and distributed execution fail closed. -The DeepSeek regression discovered on the historical upstream GLM line remains -tracked in [#532](https://github.com/antirez/ds4/issues/532). - -Speculative streamed decode is still a measured no-go: the existing NextN -probe reached about 55% acceptance, below the level needed to repay its extra -expert I/O. Current release evidence and the remaining decode targets are in -[`docs/benchmarks/2026-07-20-glm52-expert-major-v2.md`](docs/benchmarks/2026-07-20-glm52-expert-major-v2.md). - -### 1. On-edge / real-time imatrix collection: `ds4-server --imatrix-out` - -Upstream collects the routed-MoE importance matrix (imatrix) **offline** from a fixed corpus -(`ds4 --imatrix-dataset … --imatrix-out …`). This fork lets **`ds4-server` collect it from the -live prompt stream on the device**, so a quantized model can be **re-calibrated to its actual -workload**, without ever storing a single user prompt. The only artifact is the imatrix: -aggregate per-(layer, expert) activation statistics (squared activations + hit counts), a -structure that cannot hold prompt text. +Start the local HTTP server with the same artifact: ```sh -ds4-server -m model.gguf --imatrix-out edge.dat # collect from live traffic -ds4-server -m model.gguf --imatrix-out edge.dat --imatrix-every 128 --imatrix-min-requests 32 +./hebrus-server \ + -m gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-DS4-ExpertMajor-v2.gguf \ + --ctx 8192 ``` -Default **off** (zero behavioral change); opt-in via `--imatrix-out`, with periodic snapshots -(`--imatrix-every`) and a minimum-requests guard (`--imatrix-min-requests`). Full design, -wiring, limits and privacy verification in [`ONEDGE_IMATRIX.md`](ONEDGE_IMATRIX.md). +The canonical commands are `hebrus`, `hebrus-server`, `hebrus-agent`, +`hebrus-bench`, and `hebrus-eval`. Their `ds4*` counterparts remain symlinks to +the same binaries for compatibility; formats, model filenames, environment +variables, and serialized `DS4` identifiers are not renamed by this bridge. -### 2. Incremental re-quantization: `deepseek4-quantize --reuse PRIOR.gguf` +## How the runtime fits together -Re-forging a *variant* (say, adding a per-layer Q4 "boost" on top of an IQ2 build that used the -same imatrix) normally regenerates **every** routed-expert tensor from the FP weights, even the -ones that don't change. But quantization is deterministic in (FP weights, target type, imatrix -slice), so an unchanged tensor is **byte-identical** to the one already sitting in a prior -build. Recomputing it is pure waste. +![Hebrus runtime flow from mmap GGUF through ExpertMajor validation and AUTO to resident Metal or SSD streaming](docs/architecture/hebrus-runtime-flow.svg) -`--reuse PRIOR.gguf` copies a planned output tensor straight from PRIOR when its **name, target -type and shape** match, and quantizes only the tensors that actually changed (the boosted -layers, at their new type). +1. The engine mmaps the GGUF and validates the embedded + `ds4.expert_major.v2` manifest, tensor inventory, geometry, ranges, and + digest before inference. +2. AUTO calculates fixed model and context memory, Metal headroom, live host + pressure, and the model-specific residency policy. +3. When the complete admitted working set fits, Metal uses full mapped tensors. + Otherwise, the runtime keeps non-routed state mapped and streams routed + expert records through a bounded cache. +4. CLI, HTTP server, agent, benchmark, and evaluation frontends share the same + engine and capability contract. -```sh -# 1. build the 2-bit base once -gguf-tools/deepseek4-quantize --hf FP --template base.gguf --imatrix coder.dat \ - --out coder-iq2.gguf - -# 2. every boost variant reuses the base's unchanged layers, re-quantizing only the boosted ones -gguf-tools/deepseek4-quantize --hf FP --template base.gguf --imatrix coder.dat \ - --reuse coder-iq2.gguf \ - --tensor-type blk.30.ffn_gate_exps.weight=q4_k … --out coder-q4boost.gguf -``` - -**Measured** (DeepSeek-V4-Flash, a 6-of-43-layer Q4 boost over an IQ2 base): a full build is -~80 minutes; the same variant via `--reuse` took **5.5 minutes** (1,310 of 1,328 tensors copied, -18 regenerated), about a **14× speedup**. The output was verified **byte-for-byte identical** to -a from-scratch build across all 1,328 tensors. The fast build is not an approximation, it is the -same file. - -**Correctness.** Every build stamps a `quantize.reuse_key` GGUF KV: an fnv1a64 over the -safetensors index, each weight shard's size and mtime, the imatrix content, and a template -structural salt. `--reuse` copies a tensor only when PRIOR's key matches this build **and** the -per-tensor type and shape match, so a boosted tensor (different target type) is regenerated, and -a stale / foreign / keyless prior (changed weights, imatrix, or recipe) safely falls back to a -full quantize. Copied bytes are size-checked against the plan (a hard error on any mismatch), -and `--reuse` refuses to alias `--out`. This is **not** present in llama.cpp, which always -requantizes from the source weights; the closest prior art is splicing GGUF tensors by hand. - -### 3. Re-calibration reuse: `quantize.reuse_key_weights` - -Changing the *imatrix* only changes the tensors the -imatrix actually steers (the routed expert families: the importance vectors re-allocate bits -inside those tensors). Everything else — attention, shared experts, norms, embeddings, output — -is byte-identical across builds that share the same FP weights and template. So every build now -also stamps `quantize.reuse_key_weights`: the same fnv1a64 **without** the imatrix folded in. -When PRIOR matches the full key, behavior is unchanged; when it matches only the weights key -(same weights, different imatrix — the re-calibration case), `--reuse` copies the -imatrix-independent tensors and regenerates only the steered ones: - -``` -reuse: PRIOR.gguf shares the weights key (…) but not the imatrix — copying - imatrix-independent tensors, regenerating the steered ones -``` - -The dependence test is conservative and mirrors the generators' own imatrix lookups (routed -`*_exps.*` families always count as steered; regular tensors are probed with the exact same -name resolution `generate_regular()` uses), so over-approximation can only cost an unneeded -regeneration, never a stale byte. Priors built before this change carry only the old key and -keep the old all-or-nothing behavior. - -Measured (DeepSeek-V4-Flash, 1,328 tensors, M5 Pro): a full re-calibration — same recipe, -`coder.dat` → `general.dat` — copied **1,199 of 1,328 tensors** and regenerated the 129 -routed-expert tensors with the new imatrix, in **~45 minutes vs ~80 for the full quantize**. -Byte-level verification: 40/40 sampled imatrix-independent tensors identical to the prior, -16/16 sampled expert tensors changed, tensor tables identical. The change went through an -adversarial 3-lens review that rejected the first cut (two stale-byte paths, one strict-mode -abort — all reachable, all fixed before this exercise: the no-imatrix gate, the coverage -fingerprint, the I32 probe exclusion). - -### 4. Historical mixed-precision routed-expert work - -Upstream `--ssd-streaming` assumes routed-expert tensors are quantized uniformly across -layers. A GGUF with a few layers boosted to Q4_K over an IQ2 base (the forgequant boost -recipe) failed **every** request under streaming (`model range … is not covered by mapped -model views`) while serving fine with full residency. Two compounding uniformity -assumptions are fixed: the streaming prefill span set now also maps the exps tensors of -off-class ("boosted") layers, so they are read through mmap'd no-copy views; and the -single-size-class expert cache pre-seeds its slab size at startup and **rejects** off-size -layers (which use the mapped path) instead of silently adopting their size and corrupting -the slot accounting. - -Uniform models were verified **byte-identical** under the change (3/3 builds), and mixed -models were validated with the canary benchmark plus the evaluation suites. This is -historical research evidence, not an extension of the current ExpertMajor v2 admission -contract. The original diagnosis and workaround were reported in -[antirez/ds4#388](https://github.com/antirez/ds4/issues/388). - -**Update (upstream converged):** antirez has since implemented equivalent mixed-precision -streaming upstream. After the latest sync this fork **takes upstream's implementation** of -`weights_streaming_layer_experts_uniform` (the only merge conflict; the two designs converged) — -see [`MERGE_LOG.md`](MERGE_LOG.md). This addition is effectively now upstream. - -### 5. Coding-eval expert tooling: prune mask + full expert profile - -Two small, opt-in hooks for studying *which* experts a domain actually needs, used by the -forgequant layer/expert A/B work: - -- **`DS4_EXPERT_PROFILE_FULL`** — the expert profiler (`ds4_expert_profile_write_layer`) emits - the *full* per-expert ranking instead of the top-16, so a static prune/keep set can be chosen - per layer from real routing statistics. -- **`DS4_EXPERT_PRUNE_MASK`** — point it at a `43 × N_EXPERT` grid of `'0'/'1'` (`'1'` = prune). - The mask is applied to the CPU router's `probs` **before top-k** (masked experts get a - large-negative sentinel so they never win), letting each token route to its next-best - surviving expert. This measures "how much of the domain lives in a few experts" without - re-quantizing anything. +ExpertMajor v2 is a disk ABI, not a brand label. The tensor identifier +`ds4.expert_major.v2`, storage wire values, published filenames and hashes, and +disk-KV format remain unchanged. -```sh -# the mask lives in the CPU router, so enable it (streaming-IQ2 path), then prune: -DS4_METAL_ENABLE_STREAMING_IQ2_CPU_ROUTER=1 DS4_EXPERT_PRUNE_MASK=mask.txt \ - ds4 -m coder-iq2.gguf -p "…" --ssd-streaming -# -> "ds4: expert prune mask ACTIVE (N experts pruned) from mask.txt" -``` - -Both default **off** (zero behavioral change). The mask affects only routed (non-hash) layers, -and only when the CPU router is active (streaming-IQ2 or PRO-Q4 paths). Details in -[`EXPERT_PRUNE.md`](EXPERT_PRUNE.md). +## Measured results -
+These are current durable measurements from one Apple M5 Pro with 64 GiB of +unified memory. They are not cross-model rankings: artifacts, prompts, +residency modes, and decode lanes differ. Follow the evidence links for exact +commands, hashes, invalidations, and memory telemetry. -## Full engine reference +| Path | Evidence lane | Measured result | +| --- | --- | --- | +| Qwen3.6 MLX affine4/group-64 | 32K pure prefill; separate 128+16 decode, resident and forced SSD | Resident: 877.34 prefill t/s and 57.43 decode t/s. Forced SSD: 83.69 prefill t/s and 23.94 decode t/s. | +| DeepSeek V4 Flash ExpertMajor v2 | 32K prose + 128 decode, Metal SSD | 164.43 prefill t/s and 7.27 decode t/s, with zero swapout. | +| GLM 5.2 ExpertMajor v2 | 32K prose + 128 decode, AUTO resolving to SSD | 44.73 prefill t/s and 1.87 overall decode t/s, with zero swapout. | -The long-form guide now lives in -[`docs/ENGINE_REFERENCE.md`](docs/ENGINE_REFERENCE.md). It covers model -downloads, full-resident and SSD-streamed operation, the native agent, -benchmarking, capability evaluation, CLI, server/tool calling, disk KV cache, -historical upstream backend controls, steering, test vectors, and debugging. +Sources: -Keeping the manual separate makes this README a reviewable landing page while -preserving the full operational reference. +- [Qwen unified affine AUTO and SSD promotion](docs/benchmarks/2026-07-21-qwen-unified-affine-auto-ssd.md) +- [DeepSeek and GLM long-context Metal stack](docs/benchmarks/2026-07-20-long-context-metal-stack.md) +- [Benchmark evidence index](docs/benchmarks/README.md) -## Status, credit, and license +Performance acceptance requires isolated short, medium, large, and 32K +long-context lanes; memory-sensitive changes may require 65K and 100K as well. +A short-context best is not substituted for the long-context gate. -DwarfStar is beta software and `ds4-agent` remains alpha. The core engine and -upstream direction come from [`antirez/ds4`](https://github.com/antirez/ds4). -The project also exists thanks to the kernels, formats, and engineering work -of [`llama.cpp`](https://github.com/ggml-org/llama.cpp) and GGML. +## Documentation -Released under the [MIT license](LICENSE). Contributions follow the -[upstream-first policy](CONTRIBUTING.md). +### Start and operate + +- [Migration guide](docs/guides/MIGRATING_TO_HEBRUS.md) — install the bridge, + validate both command identities, move integrations safely, and roll back + without rewriting models or user data. +- [Runtime support contract](docs/contracts/RUNTIME_SUPPORT.md) — supported + families, hardware floors, backends, and fail-closed boundaries. +- [Brand compatibility contract](docs/contracts/BRAND_COMPATIBILITY.md) — + canonical commands, legacy aliases, stable formats, and migration rules. +- [Engine reference](docs/ENGINE_REFERENCE.md) — CLI, server, agent, disk KV, + tracing, evaluation, and operational details. +- [Metal and SSD policy](GOLD_METAL_SSD.md) — build identity, AUTO residency, + cache planning, and benchmark gates. + +### Models and formats + +- [ExpertMajor v2 roadmap](docs/expert-major-v2-roadmap.md) +- [DeepSeek ExpertMajor v2](docs/deepseek-expert-major-v2.md) +- [GLM 5.2 ExpertMajor v2](docs/glm52-expert-major-v2.md) +- [Qwen ExpertMajor v2 MLX-affine store](docs/qwen-expert-major-store.md) +- [GGUF and quantization tools](gguf-tools/README.md) + +### Architecture and evidence + +- [Runtime flow diagram](docs/architecture/hebrus-runtime-flow.svg) — accessible, + versioned view of mmap GGUF, ExpertMajor validation, AUTO admission, and the + resident Metal or SSD-streaming paths. +- [Code map](docs/architecture/CODEMAP.md) +- [Hebrus naming and compatibility decision](docs/adr/0005-hebrus-naming-and-compatibility-boundary.md) +- [Benchmark methodology and records](docs/benchmarks/README.md) +- [Release checklist](QA_BEFORE_RELEASES.md) +- [Fork/upstream ledger](FORK_NOTES.md) + +### Project and community + +- [Cross-repository rebrand rollout plan](docs/REBRAND_ROLLOUT_PLAN.md) — + current engine, Studio, and website status; non-breaking rename order; CI, + compatibility, repository, release, and launch gates. +- [Hebrus bridge launch-candidate notes](docs/releases/hebrus-bridge-launch-candidate.md) — + implemented scope, preserved compatibility, release evidence, and external + gates that remain pending. +- [Contribution guide](CONTRIBUTING.md) +- [Security policy](SECURITY.md) +- [Code of conduct](CODE_OF_CONDUCT.md) +- [Governance](GOVERNANCE.md) +- [Changelog](CHANGELOG.md) +- [Citation metadata](CITATION.cff) +- [Acknowledgments](ACKNOWLEDGMENTS.md) and + [third-party notices](THIRD_PARTY_NOTICES.md) + +## Project status + +The engine is beta software; `hebrus-agent` remains alpha. Large-model inference +can create substantial memory pressure and I/O. Follow AUTO, use only exact +qualified artifacts, monitor swap and memory pressure, and do not expose a +local server to untrusted networks without an appropriate security boundary. + +The separate companion desktop application is **Hebrus Studio**, currently +developed in the existing +[`andreaborio/dsbox`](https://github.com/andreaborio/dsbox) fork until its +administrative rename. Its bridge release preserves the DSBox bundle ID, data +locations, environment namespace, storage keys, and rollback path so the public +rename does not split existing installations. + +## Contributing and security + +Read [CONTRIBUTING.md](CONTRIBUTING.md) before opening a change. Contributions +must identify realistic failure modes, run the applicable correctness and +performance gates, and classify whether the work should also be proposed to +`antirez/ds4`. + +Do not publish vulnerability details in an ordinary issue. Until a dedicated +private reporting channel is enabled, follow the zero-disclosure contact +procedure in [SECURITY.md](SECURITY.md). Enabling GitHub private vulnerability +reporting remains a pre-launch administrative gate. + +## License and credit + +The project-level license is [MIT](LICENSE). Bundled or adapted third-party +code retains its own notices and conditions; model weights and datasets are not +granted rights by this repository's software license. + +Read [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md) for engineering provenance and +[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for bundled-code notices. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..3de3d555a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,54 @@ +# Security Policy + +## Reporting A Vulnerability + +The project intends to receive vulnerability reports through GitHub's private +vulnerability reporting flow: + + + +Private vulnerability reporting is not enabled for this repository yet. +Enabling it is a required administrative step before public launch; until then, +the advisory URL above will not accept a report. A reporter may open a minimal +public issue requesting private security contact, but that issue must contain +no vulnerability details, reproduction steps, affected versions, logs, or +attachments. + +Never disclose an undisclosed vulnerability in a public issue, discussion, or +pull request. Public details can expose users before a fix or mitigation is +available. + +A useful private report includes: + +- the affected commit, tag, executable, and runtime configuration; +- a concise description of the impact and the conditions required to trigger + it; +- reproduction steps or a minimal proof of concept, with secrets and model + data removed; +- any known workaround or mitigation; and +- whether the issue has already been disclosed elsewhere. + +Once private reporting is enabled, the maintainers will use the advisory to +investigate, coordinate a fix, and agree on disclosure details with the +reporter. This project does not promise a fixed acknowledgement or remediation +time. Complexity, hardware access, model availability, and impact can all +affect the investigation. + +## Supported Versions + +The project does not currently publish a version-by-version security support +matrix. Include the exact Git commit or tag in every report. Historical tags, +benchmark records, and model artifacts are not by themselves a promise of +ongoing security support. + +## Scope And Disclosure + +Security reports may cover the engine, command-line tools, HTTP server, agent, +model conversion and download tooling, file-format parsing, or build and +release infrastructure in this repository. Model quality problems without a +security impact belong in the normal issue tracker. + +Please keep vulnerability details in the private advisory until the +maintainers and reporter have coordinated a public disclosure. Never attach +private model weights, API keys, access tokens, personal data, or unrelated +system contents to a report. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 000000000..639312416 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,125 @@ +# Third-party notices + +This file collects notices for third-party code bundled in or materially +adapted by this repository. Source headers remain authoritative and must not be +removed. The project-level [`LICENSE`](LICENSE) does not replace the terms +below. + +## linenoise + +Files: `linenoise.c`, `linenoise.h` + +Upstream: + +```text +Copyright (c) 2010-2023, Salvatore Sanfilippo +Copyright (c) 2010-2013, Pieter Noordhuis + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` + +## Rax + +Bundled copyright lines: + +`rax.c`: + +```text +Copyright (c) 2017-2026, Salvatore Sanfilippo +``` + +`rax.h`: + +```text +Copyright (c) 2017-2018, Salvatore Sanfilippo +``` + +`rax_malloc.h`: + +```text +Copyright (c) 2017, Salvatore Sanfilippo +``` + +All three files carry the following terms: + +```text +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Redis nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +``` + +## GGML and llama.cpp-derived implementation + +The root MIT license retains: + +```text +Copyright (c) 2023-2026 The ggml authors +``` + +The repository retains or adapts narrow pieces of MIT-licensed GGML/llama.cpp +work, including quant layouts and tables, CPU quantization and dot-product +logic, selected kernel techniques, and GGUF/converter behavior. See +[`ACKNOWLEDGMENTS.md`](ACKNOWLEDGMENTS.md), [`LICENSE`](LICENSE), and the +source-file comments for scope. + +## YaRN-derived RoPE algorithm + +File: `metal/dsv4_rope.metal` + +The source contains the following notice: + +```text +YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn +MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng. +``` + +## Preservation + +When redistributing source or binaries, preserve the notices and conditions +required by the relevant source headers. When copying or moving one of these +files, carry its attribution and license text with it. No project or contributor +listed here endorses Hebrus merely because their work is credited. diff --git a/docs/ENGINE_REFERENCE.md b/docs/ENGINE_REFERENCE.md index d196753b2..e2315b1c3 100644 --- a/docs/ENGINE_REFERENCE.md +++ b/docs/ENGINE_REFERENCE.md @@ -1,4 +1,4 @@ -# DwarfStar engine reference +# Hebrus engine reference > [!NOTE] > This is the long-form engine guide inherited and evolved from @@ -26,8 +26,8 @@ That said, a few important things about this project: * The local inference landscape contains many excellent projects, but new models are released continuously, and the attention immediately gets captured by the next model to implement. This project takes a deliberately narrow bet: one model at a time, official-vector validation (logits obtained with the official implementation), long-context tests, and enough agent integration to know if it really works. The exact model may change as the landscape evolves, but the constraint remains: credible local Apple Metal inference, with family-specific memory limits. DeepSeek and GLM currently start at 64 GiB; Qwen's SSD-backed policy starts at 16 GiB. * This software is developed with **strong assistance from GPT 5.5** and with humans leading the ideas, testing, and debugging. We say this openly because it shaped how the project was built. If you are not happy with AI-developed code, this software is not for you. The acknowledgement below is equally important: this would not exist without `llama.cpp` and GGML, largely written by hand. * This implementation is based on the idea that compressed KV caches like the one of DeepSeek v4 and the fast SSD disks of modern MacBooks should change our idea that KV cache belongs to RAM. **The KV cache is actually a first-class disk citizen**. Fast SSD disks also changed the inference game from the point of view of "model needs to fit RAM": while having more RAM than the model size is still preferred, SSD streaming turns the available amount of RAM from a hard cutoff (can I run this model or not?) into a continuous spectrum of speed levels. -* Our vision is that local inference should be a set of three things working well together, out of the box: A) inference engine with HTTP API + B) GGUF specially crafted to run well under a given engine and given assumptions + C) testing and validation with coding agents implementations. D) Purpose built agents for specific models and execution environments. DwarfStar only runs with the GGUF files provided. It gets tested against officially obtained logits at different context sizes. This project exists because we wanted to make one local model feel finished end to end, not just runnable. However this is beta quality code, so probably we are not still there, especially since SSD streaming and the additional model families are recent additions. -* The production graph path targets **Metal on macOS**. The CPU path is only for correctness checks and model/tokenizer diagnostics. For CPU-only Linux builds, use `make cpu`; it builds the normal `./ds4` and `./ds4-server` binaries without a GPU backend. On macOS, **warning: current macOS versions have a bug in the virtual memory implementation that can crash the kernel** if you try to run very large CPU model inference. Do not use CPU as a production fallback. +* Our vision is that local inference should be a set of three things working well together, out of the box: A) inference engine with HTTP API + B) GGUF specially crafted to run well under a given engine and given assumptions + C) testing and validation with coding agents implementations. D) Purpose built agents for specific models and execution environments. Hebrus only runs with the GGUF files provided. It gets tested against officially obtained logits at different context sizes. This project exists because we wanted to make one local model feel finished end to end, not just runnable. However this is beta quality code, so probably we are not still there, especially since SSD streaming and the additional model families are recent additions. +* The production graph path targets **Metal on macOS**. The CPU path is only for correctness checks and model/tokenizer diagnostics. For CPU-only Linux builds, use `make cpu`; it builds canonical `hebrus` commands plus their `ds4*` compatibility aliases without a GPU backend. On macOS, **warning: current macOS versions have a bug in the virtual memory implementation that can crash the kernel** if you try to run very large CPU model inference. Do not use CPU as a production fallback. ## Acknowledgements to llama.cpp and GGML @@ -50,7 +50,7 @@ backend receives the same validation at the same time. We try to keep tested paths usable. If you hit an issue, use `--trace` to log the session and include the full trace in the report. -The `ds4-agent` was added later and remains alpha quality. +The `hebrus-agent` was added later and remains alpha quality. ## More Documentation @@ -81,10 +81,11 @@ next sections. ## Model Weights -This implementation only works with the ExpertMajor v2 GGUFs published for -this project. It is not a general GGUF loader, and arbitrary Qwen, DeepSeek, -GLM, or community GGUF files will not have the validated embedded store, -tensor layout, quantization mix, or metadata expected by the engine. The 2 bit +This implementation only works with the explicitly qualified ExpertMajor v2 +GGUFs published for this project. It is not a general GGUF loader, and +arbitrary Qwen, DeepSeek, GLM, or community GGUF files will not have the +validated embedded store, tensor layout, quantization mix, or metadata expected +by the engine. The 2 bit DeepSeek quantizations provided here are not a joke: they behave well, work under coding agents, call tools in a reliable way. The 2 bit quants use a very asymmetrical quantization: only the routed MoE @@ -100,15 +101,18 @@ complete output SHA-256 in its publication record: [`deepseek-expert-major-v2.md`](deepseek-expert-major-v2.md). The current canonical mirror is an offline converter source, not the runtime artifact. - [GLM 5.2 DS4 GGUF](https://huggingface.co/andreaborio/GLM-5.2-DS4-GGUF) -- [Qwen3.6-35B-A3B DS4 GGUF](https://huggingface.co/andreaborio/Qwen3.6-35B-A3B-DS4-GGUF) +- [Qwen3.6-35B-A3B DS4 GGUF](https://huggingface.co/andreaborio/Qwen3.6-35B-A3B-DS4-GGUF): + use only the MLX affine4/group-64 artifact identified in + [`qwen-expert-major-store.md`](qwen-expert-major-store.md). The older Q4_K_S + object is incompatible with this runtime. Use the exact artifact, size, and SHA-256 recorded in the family documentation. -`download_model.sh` exposes the three qualified runtime artifacts as explicit -`*-v2` targets. Its clearly named `offline-*` targets download only complete -canonical converter inputs; distributed slices and legacy runtime targets are -not exposed. The script never creates `./ds4flash.gguf`. Convert and verify an -offline source with `gguf-tools/ds4-expert-major.py` before publication or -inference. +`download_model.sh` exposes all three published runtime artifacts as explicit +`*-v2` targets. The clearly named `offline-*` targets download +only complete canonical converter inputs; distributed slices and legacy +runtime targets are not exposed. The script never creates `./ds4flash.gguf`. +Convert and verify an offline source with `gguf-tools/ds4-expert-major.py` +before publication or inference. If you want to regenerate GGUF files or collect a new imatrix, see [gguf-tools/README.md](../gguf-tools/README.md). Those tools are meant for offline @@ -129,14 +133,77 @@ make cpu # CPU-only diagnostics in build/cpu-/bin on macOS ``` On macOS, Metal and CPU objects/binaries live in separate build profiles. -`make cpu` never replaces the root Metal commands. Use -`build/cpu-$(uname -m)/bin/ds4` for the CPU-only binary and `./ds4 --build-info` -to verify build provenance. +`make cpu` never replaces the root Metal commands. Use +`build/cpu-$(uname -m)/bin/hebrus` for the canonical CPU-only binary and +`./hebrus --build-info` to verify build provenance. + +Each build profile links one real executable per role under the canonical +`hebrus*` name. The matching `ds4*` command is a relative symlink to that file; +it is not a separately compiled or linked wrapper. On macOS both root command +sets resolve to the same namespaced Metal binaries. The bridge deliberately +keeps options, defaults, output streams, exit codes, and runtime behavior +identical while presenting the invoked command identity. Canonical `hebrus*` +commands use Hebrus in help and build metadata; compatibility `ds4*` aliases +retain their legacy command identity. Existing runtime diagnostics remain in +the compatibility namespace during this bridge. + +The Makefile can install that same single-binary-per-role layout locally or +into a package staging root. `PREFIX` defaults to `/usr/local`, `BINDIR` +defaults to `$(PREFIX)/bin`, and `DESTDIR` prepends a packaging root without +being embedded in the executables. For example: + +```sh +make install PREFIX="$HOME/.local" +make install DESTDIR="$PWD/package-root" PREFIX=/usr/local +make uninstall DESTDIR="$PWD/package-root" PREFIX=/usr/local +make install-test +``` + +Installation copies only the five canonical executables and creates each +legacy command as a relative symlink to its canonical peer. Uninstall removes +only those ten named paths; it does not remove the containing directory or any +other file. The test uses a temporary `DESTDIR`, validates all capability +documents and aliases without loading a model, exercises `BINDIR` overriding, +and proves the explicit uninstall boundary. + +Every canonical and compatibility executable accepts `--capabilities=json` +without loading a model. +The deterministic schema reports the build revision, compiled backend, +executable role, supported model families, and the ExpertMajor v2 tensor and +storage wire contract. Consumers should reject unknown `schema_version` values +instead of inferring support from source files, binary strings, or visible +product copy. The bare `--capabilities` spelling is intentionally invalid. + +```json +{ + "schema_version": 1, + "engine_id": "hebrus", + "build_git_sha": "", + "backend": "metal", + "executable_role": "server", + "model_families": ["deepseek4", "glm-dsa", "qwen35moe"], + "expert_major": { + "version": 2, + "tensor": "ds4.expert_major.v2", + "storage_formats": [ + {"id": "ggml", "wire_value": 0, "group_sizes": []}, + {"id": "mlx-affine4", "wire_value": 1, "group_sizes": [64]} + ] + } +} +``` + +`backend` is `metal` or `cpu`; `executable_role` is `cli`, `server`, `agent`, +`bench`, or `eval`. Canonical commands report `engine_id: "hebrus"`; legacy +aliases report `engine_id: "ds4"`. Consumers in the bridge window must accept +both values and use the structured fields rather than visible product copy. +The normal build identity is the 12-character Git revision, optionally followed +by `-dirty`; non-Git builds may report `unknown`. Do not rely on the historical `./ds4flash.gguf` symlink for runtime identity. Pass `-m` with an absolute qualified ExpertMajor v2 path and verify its complete -published output SHA-256 before inference. Run `./ds4 --help` and -`./ds4-server --help` for the full flag list. +published output SHA-256 before inference. Run `./hebrus --help` and +`./hebrus-server --help` for the full flag list. ## Speed @@ -185,7 +252,7 @@ routes through experts again. Start with AUTO residency and the automatic cache budget: ```sh -./ds4 -m /absolute/path/to/QUALIFIED-DEEPSEEK-OR-QWEN-DS4-ExpertMajor-v2.gguf +./hebrus -m /absolute/path/to/QUALIFIED-DEEPSEEK-OR-QWEN-DS4-ExpertMajor-v2.gguf ``` For DeepSeek and Qwen qualification, use `--ssd-streaming` to force streaming @@ -222,13 +289,13 @@ If startup reports that the expert cache is too large, or if you want to reserve more memory for context, set the routed expert cache explicitly: ```sh -./ds4 \ +./hebrus \ -m /absolute/path/to/QUALIFIED-DEEPSEEK-OR-QWEN-DS4-ExpertMajor-v2.gguf \ --ssd-streaming --ssd-streaming-cache-experts 32GB ``` The `32GB` value is a memory budget for complete routed experts, not a generic -byte cache. DwarfStar converts it to the number of full experts that fit for the +byte cache. Hebrus converts it to the number of full experts that fit for the current GGUF. Non-routed weights, KV cache, graph scratch, and activations need additional memory. Only the automatic cache budget does the full subtraction for you: it reserves context/KV/scratch, external pressure, and 20% backend @@ -243,7 +310,7 @@ On a qualified 64 GB host, run the published DeepSeek Flash ExpertMajor v2 artifact with its recorded complete output SHA-256 and a moderate expert cache: ```sh -./ds4 \ +./hebrus \ -m /absolute/path/to/QUALIFIED-DEEPSEEK-FLASH-DS4-ExpertMajor-v2.gguf \ --ssd-streaming \ --ssd-streaming-cache-experts 32GB \ @@ -326,9 +393,9 @@ Long local inference runs can keep the GPU busy for extended periods. If you care more about heat, fan noise, battery life on MacBooks, or reducing thermal stress on the hardware than about maximum throughput, use `--power N`. -`--power 100` is the default and means full speed. Lower values ask DwarfStar to target +`--power 100` is the default and means full speed. Lower values ask Hebrus to target that percentage of GPU usage: `--power 70` targets about 70%, `--power 50` -targets about half usage, and so forth. DwarfStar does this by measuring GPU work time +targets about half usage, and so forth. Hebrus does this by measuring GPU work time and inserting small sleeps between work units: during prefill it sleeps between layers, and during generation it sleeps between decoded tokens. This reduces sustained load without changing model output. @@ -337,14 +404,14 @@ The option is available on the CLI, server, agent, eval, and benchmark tools, for example: ```sh -./ds4 -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --power 50 -./ds4-agent -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --power 70 -./ds4-server -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --power 40 --ctx 100000 +./hebrus -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --power 50 +./hebrus-agent -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --power 70 +./hebrus-server -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --power 40 --ctx 100000 ``` ## Native agent -DwarfStar features a native coding agent that works in a different way +Hebrus features a native coding agent that works in a different way than most other systems: the inference is controlled from within the agent itself, without socket/API boundaries, so the session is represented by the on-disk KV cache itself. Moreover the tools and the system prompt @@ -376,7 +443,7 @@ Private compaction instructions never enter the visible transcript or execute tools. `/compact` requests the same operation manually, and the terminal shows `COMPACTING` while the summary and rebuilt prefill are in progress. -Use `--chdir /path/to/ds4` when launching `ds4-agent` from another directory, +Use `--chdir /path/to/engine-checkout` when launching `hebrus-agent` from another directory, so relative runtime files such as `metal/*.metal` resolve from the project tree. However while the system already works, there is a lot of work to do @@ -386,7 +453,7 @@ session-based protocol that can recreate all that in a client-server way. ## Benchmarking -`ds4-bench` measures instantaneous prefill and generation throughput at context +`hebrus-bench` measures instantaneous prefill and generation throughput at context frontiers instead of reporting one whole-run average. It loads the model once, walks a fixed token sequence to frontiers such as 2048, 4096, 6144, and uses incremental prefill so each row measures only the newly-added token interval. @@ -394,7 +461,7 @@ After each frontier it saves the live KV state to memory, generates a fixed greedy non-EOS probe, restores the memory snapshot, and continues prefill. ```sh -./ds4-bench \ +./hebrus-bench \ -m /absolute/path/to/QUALIFIED-DEEPSEEK-FLASH-DS4-ExpertMajor-v2.gguf \ --prompt-file speed-bench/promessi_sposi.txt \ --ctx-start 2048 \ @@ -424,7 +491,7 @@ per-layer chunk dispatch path. ## Capability Evaluation -`ds4-eval` is a small real-model integration benchmark. It is not a leaderboard +`hebrus-eval` is a small real-model integration benchmark. It is not a leaderboard runner and should not be reported as an official GPQA, SuperGPQA, AIME, or security benchmark score: the questions are an embedded 92-item subset chosen to make local regression testing useful and visually inspectable. The program @@ -433,14 +500,14 @@ the final answer, and prints a per-question report with prompt tokens, generated tokens, pass/fail state, the model answer, and the correct answer. ```sh -./ds4-eval \ +./hebrus-eval \ -m /absolute/path/to/QUALIFIED-DEEPSEEK-FLASH-DS4-ExpertMajor-v2.gguf \ --trace /tmp/ds4-eval.txt ``` The default run uses `--tokens 16000`, thinking mode enabled, and a soft/hard `` budget cutoff so the model has room to produce a visible answer. -`ds4-eval` sizes the context internally from the largest selected prompt plus +`hebrus-eval` sizes the context internally from the largest selected prompt plus the generation budget, and refuses runs that would need more than 1M context tokens. Press `p` to pause, `q` to exit and print the report, Up/Down to inspect or select another question, and Enter to run the selected question next. @@ -456,7 +523,7 @@ For inference changes that can affect generation drift, keep this deterministic q1..q4 token-count gate in the test plan: ```sh -./ds4-eval \ +./hebrus-eval \ -m /absolute/path/to/QUALIFIED-DEEPSEEK-FLASH-DS4-ExpertMajor-v2.gguf \ --plain \ --questions 4 \ @@ -508,7 +575,7 @@ a pass/fail unit test. to identify the best source line where the defensive code flaw is introduced, or return `0` for a safe function. -In practice this means `ds4-eval` should not be expected to produce a perfect +In practice this means `hebrus-eval` should not be expected to produce a perfect 92/92 run. It is meant to answer a more useful engineering question: after a kernel, quantization, prompt-rendering, KV-cache, or tool-streaming change, does DeepSeek V4 Flash still solve a representative mix of hard science, broad @@ -520,7 +587,7 @@ path users run? One-shot prompt: ```sh -./ds4 \ +./hebrus \ -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf \ -p "Explain Redis streams in one paragraph." ``` @@ -528,8 +595,8 @@ One-shot prompt: No `-p` starts the interactive prompt: ```sh -./ds4 -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf -ds4> +./hebrus -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf +hebrus> ``` The interactive CLI is a real multi-turn chat. It keeps the rendered chat @@ -549,12 +616,12 @@ experimental slight-speedup path. Start a local OpenAI/Anthropic-compatible server: ```sh -./ds4-server \ +./hebrus-server \ -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf \ --ctx 100000 --kv-disk-dir /tmp/ds4-kv --kv-disk-space-mb 8192 ``` -Use `--chdir /path/to/ds4` when launching `ds4-server` from another directory, +Use `--chdir /path/to/engine-checkout` when launching `hebrus-server` from another directory, so relative runtime files such as `metal/*.metal` resolve from the project tree. The server keeps one mutable backend/KV checkpoint in memory, @@ -605,7 +672,7 @@ assistant `tool_use` blocks or it is rejected. Default sampled API generation uses `temperature=1`, `top_p=1`, and `min_p=0.05`, so the default filter is relative probability rather than -nucleus mass. In thinking mode DwarfStar uses those fixed sampling defaults and +nucleus mass. In thinking mode Hebrus uses those fixed sampling defaults and ignores client sampling knobs, matching DeepSeek's fixed-thinking API behavior. The chat, Responses, and Anthropic endpoints support SSE streaming. In thinking @@ -679,12 +746,12 @@ curl http://127.0.0.1:8000/v1/chat/completions \ ### Agent Client Usage -`ds4-server` can be used by local coding agents that speak OpenAI-compatible +`hebrus-server` can be used by local coding agents that speak OpenAI-compatible chat completions. Start the server first, and set the client context limit no higher than the `--ctx` value you started the server with: ```sh -./ds4-server \ +./hebrus-server \ -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf \ --ctx 100000 --kv-disk-dir /tmp/ds4-kv --kv-disk-space-mb 8192 ``` @@ -857,7 +924,7 @@ non-thinking model alias such as `deepseek-chat`. ## Disk KV Cache Chat/completion APIs are stateless: agent clients usually resend the whole -conversation every request. `ds4-server` first tries the cheap exact token-prefix +conversation every request. `hebrus-server` first tries the cheap exact token-prefix check, then falls back to comparing rendered prompt bytes with decoded checkpoint bytes. The live in-memory checkpoint covers the current session; the disk KV cache makes useful prefixes survive session switches and server @@ -872,7 +939,7 @@ different sessions. Enable it with: ```sh -./ds4-server \ +./hebrus-server \ -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf \ --kv-disk-dir /tmp/ds4-kv --kv-disk-space-mb 8192 ``` @@ -1040,7 +1107,7 @@ Production inference for Qwen, DeepSeek, and GLM ExpertMajor v2 artifacts is local Apple Metal: ```sh -./ds4 -m /absolute/path/to/MODEL-DS4-ExpertMajor-v2.gguf -p "Hello" +./hebrus -m /absolute/path/to/MODEL-DS4-ExpertMajor-v2.gguf -p "Hello" ``` CPU remains useful for build and model-free diagnostics: @@ -1076,12 +1143,12 @@ willingness to provide dual-use or offensive security guidance. captured from the official DeepSeek V4 Flash API. The requests use `deepseek-v4-flash`, greedy decoding, thinking disabled, and the maximum `top_logprobs` slice exposed by the API. Local vectors are generated with -`./ds4 --dump-logprobs` and compared by token bytes, so tokenizer/template or +`./hebrus --dump-logprobs` and compared by token bytes, so tokenizer/template or attention regressions show up before they become long generation failures. The C runner pins `DS4_METAL_PREFILL_CHUNK=2048` for this strict API-vector comparison. -All project tests are driven by the C runner, with a small `ds4-eval` +All project tests are driven by the C runner, with a small `hebrus-eval` extractor self-test run first: ```sh @@ -1096,10 +1163,10 @@ When a generation looks wrong, three small tools are usually enough to get a first answer: ```sh -./ds4 -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --dump-tokens -p "..." -./ds4 -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --dump-logprobs /tmp/out.json --logprobs-top-k 20 --temp 0 -p "..." -./ds4 -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --dump-logits /tmp/logits.json --nothink --prompt-file prompt.txt -./ds4-server -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --trace /tmp/ds4-trace.txt ... +./hebrus -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --dump-tokens -p "..." +./hebrus -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --dump-logprobs /tmp/out.json --logprobs-top-k 20 --temp 0 -p "..." +./hebrus -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --dump-logits /tmp/logits.json --nothink --prompt-file prompt.txt +./hebrus-server -m /absolute/path/to/QUALIFIED-DS4-ExpertMajor-v2.gguf --trace /tmp/ds4-trace.txt ... ``` - `--dump-tokens` tokenizes the `-p` or `--prompt-file` string exactly as diff --git a/docs/HF_MODEL_RENAME_INVENTORY.md b/docs/HF_MODEL_RENAME_INVENTORY.md new file mode 100644 index 000000000..0a0228ba0 --- /dev/null +++ b/docs/HF_MODEL_RENAME_INVENTORY.md @@ -0,0 +1,57 @@ +# Hugging Face Model Rename Inventory + +- Snapshot time: 2026-07-22, before any repository move +- Namespace: `andreaborio` +- Visibility: all three repositories are public and ungated +- Purpose: establish a reviewable baseline for the in-place DS4-to-Hebrus + repository moves described in + [`REBRAND_ROLLOUT_PLAN.md`](REBRAND_ROLLOUT_PLAN.md) + +This snapshot is preparatory evidence, not authorization to move a repository. +Repeat the API inventory immediately before every move because revisions, +cards, counters, and file inventories can change after this document is +committed. + +## Repository baseline + +| Current ID | Planned ID | Main revision | Downloads | Likes | Files | +| --- | --- | --- | ---: | ---: | ---: | +| `andreaborio/DeepSeek-V4-Flash-DS4-GGUF` | `andreaborio/DeepSeek-V4-Flash-Hebrus-GGUF` | `2ba412c5d8048f8adfb24446e776015133aed9e0` | 319 | 0 | 5 | +| `andreaborio/GLM-5.2-DS4-GGUF` | `andreaborio/GLM-5.2-Hebrus-GGUF` | `0182407316ad8a3cf6cb676dad9ae298af8e091a` | 98 | 0 | 19 | +| `andreaborio/Qwen3.6-35B-A3B-DS4-GGUF` | `andreaborio/Qwen3.6-35B-A3B-Hebrus-GGUF` | `7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02` | 343 | 0 | 7 | + +## Qualified runtime artifacts + +| Model | Immutable filename | Bytes | SHA-256 | +| --- | --- | ---: | --- | +| DeepSeek V4 Flash | `DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix-DS4-ExpertMajor-v2.gguf` | 86,720,114,272 | `8378080263eb9224f7228d72e2afa4ac3cf74a116023fdec2c596ff228a33e3f` | +| GLM-5.2 | `GLM-5.2-DS4-ExpertMajor-v2-Q2_K.gguf` | 262,147,193,504 | `7f5017e3076e706c78f2a5322b035a9e2f6519c65ff5b6be8b2d91aeff61505d` | +| Qwen3.6-35B-A3B | `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf` | 20,808,566,880 | `dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d` | + +The Qwen repository also contains historical/canonical comparison artifacts; +the GLM repository contains the archived multipart experimental tree. The move +must preserve every sibling and LFS/Xet object, not only the three qualified +files above. No GGUF filename, byte count, SHA-256, revision, manifest tensor +identifier, or historical directory is a branding rename target. + +## Collection and move checks + +The public baseline was collected from the official Hub model API with +`blobs=true`. Before each authenticated move, save the complete JSON response +and Git refs, then compare after the move: + +```sh +curl --fail --location \ + "https://huggingface.co/api/models/andreaborio/REPOSITORY?blobs=true" +``` + +Post-move verification must prove: + +1. the new ID resolves the exact recorded main revision; +2. the old page, API, Git, and `resolve` URLs redirect; +3. every sibling path, byte size, and LFS/Xet object identity is unchanged; +4. ranged download/resume and the pinned engine downloader still work; +5. Studio discovers the new ID and recognizes installations stored under the + old DS4 repository directory; +6. model cards retain DS4/antirez fork lineage and all material references; +7. download and like counters remain attached to the moved repository. diff --git a/docs/REBRAND_ROLLOUT_PLAN.md b/docs/REBRAND_ROLLOUT_PLAN.md new file mode 100644 index 000000000..809dfafe6 --- /dev/null +++ b/docs/REBRAND_ROLLOUT_PLAN.md @@ -0,0 +1,266 @@ +# Hebrus Rebrand Rollout Plan + +- Status: active integration; compatibility PRs open, public renames gated +- Last updated: 2026-07-22 +- Scope: Hebrus engine, Hebrus Studio, the Hebrus Studio website, and the + qualified Hugging Face model repositories +- Release posture: public development preview; final notarized macOS release pending + +This is the cross-repository source of truth for completing the Hebrus rebrand +without breaking runtime compatibility, installed application state, published +model artifacts, Git history, or upstream attribution. Detailed engine release +evidence remains in [`QA_BEFORE_RELEASES.md`](../QA_BEFORE_RELEASES.md); the +stable naming boundary remains in +[`BRAND_COMPATIBILITY.md`](contracts/BRAND_COMPATIBILITY.md). + +> [!IMPORTANT] +> Hebrus began as a fork of +> [`antirez/ds4`](https://github.com/antirez/ds4) and retains substantial code, +> architecture, utilities, and history from that project. Salvatore +> Sanfilippo/antirez must remain prominently credited in the engine, Studio, +> website, release notes, and future repository descriptions. `llama.cpp`, +> GGML, MLX, and other material references remain credited with links and +> scope. A repository rename must preserve this lineage rather than presenting +> Hebrus as a clean-room implementation. + +## Current snapshot + +| Surface | Canonical work | State | Immediate constraint | +| --- | --- | --- | --- | +| Engine | `andreaborio/ds4`, PR [#6](https://github.com/andreaborio/ds4/pull/6) | Rebased on current `origin/main`; local complete premerge green; macOS CI green | Fix the Linux-only zsh runner gate, then require all CI and model-backed release lanes before release | +| Studio | `andreaborio/dsbox`, PR [#11](https://github.com/andreaborio/dsbox/pull/11) | Rebased on current `main`; 409 tests, build, audit, and local packaged-app verification green; dual HF IDs implemented | Require the rerun macOS package check and all PR checks to be green before merge | +| Website | `andreaborio/hebrus-site`, `main` at `23507e2` | Published on GitHub Pages; clean install, zero-vulnerability audit, lint, static export, and Pages test green | Final logo, canonical post-rename links, and final signed download remain pending | +| Model Hub | Three consolidated `andreaborio/*-DS4-GGUF` repositories | Public pre-move snapshot recorded in [`HF_MODEL_RENAME_INVENTORY.md`](HF_MODEL_RENAME_INVENTORY.md); Studio bridge accepts future Hebrus IDs | Merge the compatibility bridge, repeat the inventory immediately before each authenticated move, then move one repository at a time | + +Release work must use the active engine rebrand branch, the active Studio bridge +branch, and the GitHub Pages site checkout identified in the table above. A +separate empty workspace checkout named `ds4` is unrelated and must not be used +for release work. The uncommitted modal-portal change in an older Studio +checkout must be reviewed and either ported deliberately to current Studio +`main` or discarded by its owner; it must not be accidentally included in the +rebrand integration. + +## Non-breaking identity matrix + +| May become canonical | Must remain compatible | Must not be renamed for branding | +| --- | --- | --- | +| `Hebrus`, `hebrus`, and the five `hebrus*` commands | Five `ds4*` command aliases through the complete 1.x line | `ds4.expert_major.v2`, wire values, GGUF metadata, disk-KV ABI, published filenames and hashes | +| `Hebrus Studio` visible product and Finder name | `com.dsbox.desktop`, `$HOME/.dsbox`, legacy Electron `userData`, browser keys, `DSBOX_*` configuration | Existing user models, conversations, settings, downloads, release artifacts, tags, and historical links | +| Future `andreaborio/hebrus` and `andreaborio/hebrus-studio` repository names | Old GitHub URLs through in-place rename redirects; consumers accept old and new engine identities | Git history, authorship, upstream remote, fork ancestry, licenses, acknowledgments, citations | +| `*-Hebrus-GGUF` as the canonical Hugging Face repository display identity | Historical `*-DS4-GGUF` repository IDs and already-installed repository directory names | GGUF filenames, bytes, immutable revisions, LFS/Xet objects, SHA-256 values, `ds4.expert_major.v2` metadata | +| Future `HEBRUS_*` aliases through one resolver | Existing `DS4_*` variables; conflicting old/new values fail closed | A textual replacement of environment variables, C symbols, paths, or serialized identifiers | + +## Phase 0 — Reconcile the three active repositories + +Goal: establish one reproducible head per product before changing another +public name. + +1. Fetch all remotes and record immutable starting SHAs for engine, Studio, and + website. +2. Rebase or reconstruct `codex/hebrus-phase-0` on current engine `main`. Keep + the three incoming Qwen Flash/SSD commits intact and resolve conflicts as + integration work, not as opportunities for extra kernel changes. +3. Reconstruct the Studio bridge on current Studio `main`, preserving the two + new visual-brand commits and the thirteen bridge/release-hardening commits. + Avoid a blind merge if it duplicates the brand-boundary inventory. +4. Review the older checkout's `createPortal(..., document.body)` modal change + against current Studio. Port it in a separate bug-fix commit only if the + current modal still has the stacking-context defect. +5. Mark the empty `Documents/ds4` checkout as non-canonical locally so builds, + scripts, and release notes cannot accidentally reference it. + +Exit gate: both integration branches are clean, based on current `main`, have +an explicit upstream tracking branch, and contain no unexplained duplicate or +dropped commits. + +## Phase 1 — Restore green CI before further renaming + +### Studio blockers and resolutions + +1. **Resolved:** the rebased lockfile reports zero vulnerabilities under + `npm audit --audit-level=high` from a clean install. +2. **Resolved locally:** the macOS package verifier false positive that classified the font path + `/dist/fonts/hebrus` as an embedded engine executable. The verifier must + distinguish executable payloads from font/assets by type and location, not + by basename alone. +3. **Resolved:** the persisted-artifact resume test is deterministic: the previous failure + compares `[null, null]` with `[null, undefined]`. Decide and document the + boundary representation, then assert that contract consistently rather than + weakening the compatibility check. +4. **Green locally:** typecheck, brand audit, the complete test suite, production build, + ad-hoc macOS packaging contract, and legacy upgrade/rollback exercise on the + same candidate SHA. + +### Engine gates + +The current brand audit is green with 29,877 legacy occurrences classified in +370 groups and no unclassified increase. After rebasing, rerun at minimum: + +- context and documentation-link audits; +- brand-boundary and Qwen release-contract checks; +- CPU/Linux and macOS arm64 builds; +- visible identity, capabilities, command-alias, install/uninstall, and server + alias parity tests; +- qualified Apple Silicon model-backed correctness and performance lanes for + every supported artifact affected by the incoming Metal changes. + +### Website gates + +Keep lint, static export tests, dependency audit, GitHub Pages deployment, HTTP +200 verification, canonical/OG metadata, and the exact GitHub Release download +URL green. The website must not host the DMG inside the Pages repository. + +Exit gate: required checks are green on the exact engine and Studio candidate +SHAs; the site deploy for its exact content SHA is green. + +## Phase 2 — Merge the bridge without renaming repositories + +1. Open focused integration PRs for engine and Studio. Keep inference changes, + visual polish, dependency fixes, and naming mechanics in reviewable commits. +2. Merge the engine capability contract and canonical `hebrus*` commands while + retaining `ds4*` aliases and invocation-aware identity. +3. Merge Hebrus Studio's `hebrus-server`-first discovery, `ds4-server` + fallback, dual `engine_id` acceptance, visible product identity, and + persisted DSBox compatibility identity. +4. Publish a fresh development candidate from the merged Studio `main`; verify + no-data-loss upgrade from an installed DSBox build and rollback to it. +5. Change ADR 0005 from Proposed to Accepted only after the merged bridge, + compatibility evidence, and namespace decision are all recorded. + +Exit gate: users can install Hebrus Studio and invoke Hebrus while old DS4 +commands, models, state, paths, and integrations continue to work. + +## Phase 3 — Freeze the public launch surface + +1. Select the final logo master. The website currently uses a temporary + text-based `H`; regenerate the favicon, application icons, and Open Graph + card together once the master is accepted. +2. Freeze the typography, colors, spacing, screenshot count, English copy, and + accessibility behavior across Studio documentation and the website. +3. Capture fresh screenshots from the exact release candidate, with no stale + DSBox visible copy except where a migration explanation requires it. +4. Synchronize README hero copy, repository descriptions, topics, social card, + release notes, install guide, support matrix, and changelog. +5. Verify every public surface carries the upstream DS4 origin and material + technical references without implying endorsement. + +Exit gate: visual assets and copy are versioned, consistent, accessible, and +traceable to the same candidate release. + +## Phase 4 — Administrative repository renames + +Perform in-place GitHub renames only; do not create history-less replacement +repositories and do not reuse the old names. + +1. Rename `andreaborio/ds4` to `andreaborio/hebrus`. +2. Verify clone, fetch, submodule/reference, issue, PR, release, and Actions + behavior through both the new canonical URL and GitHub's old-URL redirect. +3. Update active remotes, badges, repository topics/descriptions, security and + issue-template links, package metadata, CI references, Studio engine-source + admission, and website source links. Do not rewrite historical documents. +4. Rename `andreaborio/dsbox` to `andreaborio/hebrus-studio` only after Studio + consumes both old and new engine remote identities. +5. Repeat redirect and automation verification for Studio, then update the + website and release download URLs. Keep `hebrus-site` unchanged unless a + separate brand decision justifies moving the Pages base path. +6. Run repository-wide broken-link scans and fresh clone/build/install tests + from the new canonical URLs. + +Exit gate: new URLs are canonical, old URLs redirect, Actions and Pages are +green, releases and stars remain attached to the renamed repositories, and no +published artifact or historical attribution link was rewritten destructively. + +## Phase 5 — Rename the qualified Hugging Face repositories + +Hugging Face documents an in-place move operation that redirects the old URL +and preserves download counts and likes. Use that operation rather than copying +or re-uploading weights. The initial public move set is: + +| Current repository | Future canonical repository | +| --- | --- | +| `andreaborio/DeepSeek-V4-Flash-DS4-GGUF` | `andreaborio/DeepSeek-V4-Flash-Hebrus-GGUF` | +| `andreaborio/GLM-5.2-DS4-GGUF` | `andreaborio/GLM-5.2-Hebrus-GGUF` | +| `andreaborio/Qwen3.6-35B-A3B-DS4-GGUF` | `andreaborio/Qwen3.6-35B-A3B-Hebrus-GGUF` | + +Retired, experimental, lab, v1, and superseded v2 repositories are historical +records. Do not bulk-rename or revive them; classify each separately and keep +them hidden from the Studio catalog. + +1. Before any Hub move, update Studio catalog contracts so the new Hebrus ID is + canonical and the current DS4 ID is included in `previousRepositories`. + Installed paths using either repository name must continue to resolve to the + same model. +2. Update engine download manifests and tooling to accept both repository IDs + while pinning the same immutable revision, GGUF filename, byte size, and + SHA-256. Repository identity is transport metadata, not a model-format + change. +3. Add tests for API discovery, revision resolution, ranged download/resume, + local installed-path recognition, hidden retired repositories, and fallback + through the old redirected URL. +4. Record each source repository, destination repository, visibility, gated + status, main revision, refs, file inventory, file sizes, LFS/Xet object + identifiers, model-card metadata, downloads, and likes immediately before + the move. +5. Move one repository at a time using the authenticated Hub move operation. + Start with the least operationally critical model, validate it completely, + and stop the sequence if any invariant changes. +6. After each move, verify the old web, API, `resolve`, and Git URLs redirect; + the new URL resolves the same commit; range requests and Studio downloads + work; the exact GGUF hash is unchanged; the model card and attribution are + intact; and download counts/likes remain attached. +7. Update live model cards, Hebrus/Studio READMEs, catalog source labels, + website links, release manifests, and support tables to the new canonical + IDs. Preserve historical release notes and benchmark links unless they are + intended as live download instructions. +8. Do not rename files inside the repositories. A friendly Hebrus display name + may be added to the model card and manifest, but existing published GGUF + filenames and their checksums remain immutable. + +Exit gate: all three new Hub IDs are canonical, every old URL redirects, Studio +recognizes both old and new installed paths, exact revisions/files/hashes are +unchanged, and no model is duplicated or re-uploaded. + +## Phase 6 — Release and launch + +Two release lanes remain deliberately separate: + +- **Community/development preview:** ad-hoc signed, clearly labelled, includes + the Control-click installation path, checksums, provenance, SBOM, known + limitations, and no notarization claim. +- **Public macOS release:** Developer ID signed, notarized, stapled, verified on + a clean Mac under Gatekeeper, with the final DMG checksum and attestation. + +For either lane, cut tags only from the green immutable candidate SHAs, publish +engine and Studio notes together, update the website download atomically, and +run a post-publish smoke test covering download, install, first launch, runtime +discovery, one resident model, one SSD-streamed model, upgrade, and rollback. + +The launch announcement should lead with the differentiated product direction: +an open-source, Apple Metal-first inference engine with adaptive SSD streaming. +It must also state the DS4 fork origin prominently and link the acknowledgments, +compatibility guide, source, supported-artifact matrix, and reproducible +benchmark evidence. + +## Stop conditions + +Do not rename repositories, remove aliases, move application data, or describe +a build as a final release when any of the following is true: + +- either engine or Studio CI is red; +- a branch is not reconciled with current `main`; +- a model-backed Metal/SSD gate required by the diff is missing; +- upstream attribution, license inventory, checksums, provenance, or rollback + evidence is incomplete; +- the website points to an unavailable or differently qualified artifact; +- the macOS build is described as notarized without Developer ID, notarization, + stapling, and clean-machine Gatekeeper evidence. + +## Completion definition + +The rebrand is complete when the canonical repositories, commands, application, +website, release assets, documentation, and community surfaces say Hebrus or +Hebrus Studio; every intentionally retained DS4/DSBox identifier is classified +and tested as compatibility, serialized data, or history; old URLs and commands +still work within their promised window; CI and model-backed evidence are green; +and a new user can understand the product, its support boundary, its provenance, +and its installation path from the public landing page alone. diff --git a/docs/adr/0004-qwen-metal-hardware-memory-policy.md b/docs/adr/0004-qwen-metal-hardware-memory-policy.md index bf9e25dc6..5b1a310e3 100644 --- a/docs/adr/0004-qwen-metal-hardware-memory-policy.md +++ b/docs/adr/0004-qwen-metal-hardware-memory-policy.md @@ -44,10 +44,11 @@ AUTO selects resident only when both independent gates pass: An explicit resident request also requires both gates. Failure selects bounded SSD streaming for AUTO and rejects an explicit resident request. The current -Q4_K_S tensor payload alone exceeds 16 GiB, so that profile necessarily uses -SSD. A 24 GiB device is decided from its actual Metal working-set report rather -than an assumed ratio. The 32 GiB profile may use resident for shorter contexts -when current pressure allows it and falls back to SSD otherwise. +19.37 GiB routed tensor payload alone exceeds 16 GiB, so that profile +necessarily uses SSD. A 24 GiB device is decided from its actual Metal +working-set report rather than an assumed ratio. The 32 GiB profile may use +resident for shorter contexts when current pressure allows it and falls back +to SSD otherwise. For Qwen SSD streaming, pageable static weights share the larger ordinary headroom envelope but remain fully charged. While macOS reports normal pressure, @@ -62,7 +63,7 @@ plus the in-flight slot: 3,521 experts for this 40-layer, top-8 model. This keep the previously retained zero-swap tier and rejects the next warm-cache tier, which produced swap in validation. Because the cap is expressed in routing cycles while byte admission uses the artifact's exact per-expert size, it does -not treat Q4 storage as F32. +not treat compact MLX affine4/group-64 storage as F32. ## Consequences diff --git a/docs/adr/0005-hebrus-naming-and-compatibility-boundary.md b/docs/adr/0005-hebrus-naming-and-compatibility-boundary.md new file mode 100644 index 000000000..d6817b1ab --- /dev/null +++ b/docs/adr/0005-hebrus-naming-and-compatibility-boundary.md @@ -0,0 +1,78 @@ +# ADR 0005: Hebrus Naming And Compatibility Boundary + +- Status: Proposed +- Date: 2026-07-21 + +## Context + +This repository began as a fork of [`antirez/ds4`](https://github.com/antirez/ds4) +and still preserves that ancestry, attribution, and a path for generally useful +changes to return upstream. The fork now has a materially different product +focus: Apple Metal inference, mmap-backed execution, embedded ExpertMajor +stores, and SSD streaming across several model families. Continuing to present +the fork itself as DS4 obscures that boundary and makes application/runtime +ownership harder to explain. + +A textual rename is unsafe. Existing installations and automation depend on +`ds4` commands, `DS4_*` environment variables, repository URLs, C names, model +filenames, and application paths. Published GGUFs and disk-KV files also contain +identifiers whose bytes are part of a durable format. The pre-rename DSBox release +additionally probes exact source paths, C symbols, and diagnostic strings when +admitting an ExpertMajor runtime. + +## Decision + +After namespace and legal screening is complete, the public engine name will be +**Hebrus**. The existing GitHub fork will be renamed rather than copied, and the +README will continue to identify `antirez/ds4` prominently as the origin of the +fork and of substantial implementation work. Acknowledgments will also name +ggml, llama.cpp, MLX, and other material sources with precise links and scope. + +The migration separates public brand from compatibility identity: + +- canonical executable names become `hebrus`, `hebrus-server`, + `hebrus-agent`, `hebrus-bench`, and `hebrus-eval`; +- all current `ds4` executable names remain aliases to the same object graph + through at least the complete 1.x release line; +- new `HEBRUS_*` environment aliases may be introduced only through one + resolver; existing `DS4_*` variables remain accepted, and conflicting values + fail closed; +- the companion application must consume a versioned, model-free capability + document before internal source names move. Legacy source/binary probe markers + remain a temporary compatibility shim, not the new admission API; +- source, header, symbol, and compile-time macro renames occur later in + mechanical subsystem-sized changes with no inference or performance change. + +The following identifiers do not change as part of the brand migration: + +- the `ds4.expert_major.v2` tensor name and the retired + `ds4.expert_major.v1` rejection marker; +- ExpertMajor v2 magic, version, manifest layout, offsets, digest semantics, + storage wire values (`GGML = 0`, `MLX_AFFINE4 = 1`), and qualified Qwen group + size (`64`); +- published model filenames, byte counts, revisions, and SHA-256 values; +- disk-KV magic, version, and payload ABI; +- Git history, tags, authorship, copyright, fork ancestry, and historical issue, + PR, benchmark, and release links. + +Old model artifacts are not revived by this compatibility policy. In +particular, the retired Qwen GGML/Q4 ExpertMajor store remains rejected under +both command brands. Model-format lifecycle remains governed by ADR 0003. + +## Consequences + +- A Hebrus Studio bridge release must precede internal source renaming; its + visible rebrand may ship in that same bridge release once persisted identity + and rollback tests are green. +- Every remaining DS4 identifier must be classified as serialized, historical, + compatibility-owned, or erroneous; unclassified additions fail a repository + check. +- Canonical and legacy commands must have parity for options, exit status, + protocols, deterministic output, resolved runtime plan, and performance. +- Existing installations, checkouts, models, caches, and application data do + not need to be copied or rewritten for branding. +- The GitHub repository rename happens only after the compatibility application + recognizes both old and new remote identities. +- This ADR becomes Accepted only after the Hebrus identity is reserved, the + machine-readable capability contract is merged, the Hebrus Studio bridge is proven, + and the pre-rebrand release baseline is green. diff --git a/docs/architecture/CODEMAP.md b/docs/architecture/CODEMAP.md index 02f94f29d..0a41eb9fa 100644 --- a/docs/architecture/CODEMAP.md +++ b/docs/architecture/CODEMAP.md @@ -10,10 +10,21 @@ below that is relevant to the task. Historical benchmark notes are evidence, not architecture specifications. The production-backend boundary is recorded in [`ADR 0002`](../adr/0002-apple-metal-production-runtime.md). Qwen's RAM/working-set adaptation is isolated in -[`ADR 0004`](../adr/0004-qwen-metal-hardware-memory-policy.md). +[`ADR 0004`](../adr/0004-qwen-metal-hardware-memory-policy.md). The proposed +Hebrus public-name migration and its immutable compatibility boundary are +recorded in [`ADR 0005`](../adr/0005-hebrus-naming-and-compatibility-boundary.md). +For the runtime data and admission path at a glance, see the accessible +[`mmap → ExpertMajor → AUTO → Metal/SSD flow`](hebrus-runtime-flow.svg). ## Runtime Entry Points +The Makefile links one canonical `hebrus*` executable per role from these +entrypoints and publishes the corresponding `ds4*` name as a symlink to the +same file. It also owns the `DESTDIR`/`PREFIX`/`BINDIR` install boundary: +canonical executables are copied, compatibility aliases stay relative, and +uninstall names every removable path explicitly. There are no alias-specific +wrappers or object graphs. + | Path | Primary responsibility | | --- | --- | | `ds4_cli.c` | One-shot CLI, interactive transcript, and command-line orchestration | @@ -21,7 +32,7 @@ RAM/working-set adaptation is isolated in | `ds4_agent.c` | Stateful coding-agent TUI, tools, session commands, and interruption state | | `ds4_bench.c` | Context-frontier prefill/decode benchmark driver | | `ds4_eval.c` | Evaluation driver and extractor checks | -| `ds4_build.c` | Build identity reported by executables | +| `ds4_build.c` | Build identity and the versioned machine-readable capability contract reported by executables | These programs should use the public engine/session API in `ds4.h`. They should not acquire tensor, ExpertMajor, or Metal-kernel ownership. @@ -98,6 +109,13 @@ not validated. | Path | Primary responsibility | | --- | --- | | `tests/` | Model-free, model-backed, kernel, tokenizer, server, and build-isolation regressions | +| `tests/test_capabilities.py` | Exact schema and cross-executable checks for the model-free build/capability contract | +| `tests/test_command_aliases.py` | Canonical/legacy symlink layout, binary identity, and CLI-output parity checks | +| `tests/test_install.sh` | Temporary-root install/uninstall layout, path portability, capability, and explicit-removal checks | +| `tools/brand_boundary.json` + `tools/brand_boundary_audit.py` | Exact canonical, bridged, and permanently preserved identity contract plus explicit per-file legacy `ds4`/`DS4`/`DwarfStar` classification and monotonic count ceilings; `--check` rejects contract drift, new groups, and increases, while `--refresh` requires exact authorizations before widening a ceiling | +| `tests/test_brand_boundary_audit.py` | Fail-closed fixtures for new files and tokens, increases, reductions, deterministic refresh, and invalid manifests | +| `docs/contracts/qwen-release.json` + `tools/qwen_release_contract.py` | Canonical published/negative-only Qwen artifact identity and the model-free gate that parses its documentation, downloader, and test surfaces for drift | +| `tests/test_qwen_release_contract.py` | Fail-closed fixtures for prose, table, downloader, schema, status, and negative-only Qwen release-contract drift | | `tests/qwen/` | Qwen fixtures, provenance, reference collectors, and model-specific gates | | `tests/test-vectors/` | Official and local continuation vectors plus provenance | | `gguf-tools/` | Quantization, ExpertMajor conversion, imatrix, and quality-scoring tools | @@ -115,6 +133,7 @@ not be copied into new tests. | `CONTRIBUTING.md` | Contribution, regression, and upstream-coordination gates | | `QA_BEFORE_RELEASES.md` | Complete release checklist | | `docs/contracts/RUNTIME_SUPPORT.md` | Current supported runtime/model matrix | +| `docs/contracts/BRAND_COMPATIBILITY.md` | Canonical Hebrus names, compatibility aliases, and permanently stable identifiers | | `docs/adr/` | Accepted architectural decisions and their consequences | | `GOLD_METAL_SSD.md` | Metal/SSD planner details and performance gates; support authority remains `RUNTIME_SUPPORT.md` | | `FORK_NOTES.md` | Time-stamped fork/upstream boundary ledger | diff --git a/docs/architecture/hebrus-runtime-flow.svg b/docs/architecture/hebrus-runtime-flow.svg new file mode 100644 index 000000000..79ca25d0e --- /dev/null +++ b/docs/architecture/hebrus-runtime-flow.svg @@ -0,0 +1,121 @@ + + + Hebrus mmap, ExpertMajor, AUTO, Metal, and SSD runtime flow + + A mapped GGUF container passes its embedded routed-weight store through + ExpertMajor version 2 validation and indexing. The AUTO planner combines + model geometry, context memory, the Metal working-set recommendation, and + live host pressure. It admits either a fully resident Metal path or a Metal + path with a bounded routed-expert cache filled from SSD. Non-routed tensors + remain mapped in the SSD-streaming path. + + + + + + + + + + + + + + One mapped model, two admitted execution paths + GGUF stays the container; ExpertMajor validates routed weights; AUTO chooses residency from live constraints. + + + + + + + + + 1 + CONTAINER + mmap GGUF v3 + + • map the model once + • keep fixed and non-routed + tensors mapped + • expose one embedded + routed-weight store + + GGUF + mapped ranges + + + + 2 + DISK ABI + ExpertMajor v2 + + • validate manifest and + model geometry + • verify ranges and digest + • index routed-expert + records before inference + + ds4.expert_major.v2 + + + + 3 + ADMISSION + AUTO planner + + • model and context memory + • Metal recommended + working set + • live pressure and + required headroom + + one fail-closed plan + + + + 4A + EXECUTION + Metal resident + + • full admitted working set + • mapped tensors + Metal buffers + • no routed-expert cache fills + + + + + 4B + EXECUTION + Metal + SSD streaming + + • non-routed tensors stay mapped + • bounded routed-expert cache + • SSD fills overlap Metal work + + + + AUTO admits exactly one qualified plan or fails closed. The stable spelling above is a serialized disk identifier, not the product brand. + diff --git a/docs/contracts/BRAND_COMPATIBILITY.md b/docs/contracts/BRAND_COMPATIBILITY.md new file mode 100644 index 000000000..a5f449e3c --- /dev/null +++ b/docs/contracts/BRAND_COMPATIBILITY.md @@ -0,0 +1,122 @@ +# Hebrus Brand Compatibility Contract + +Status: bridge contract for the unreleased Hebrus naming transition. + +This document defines which public names are canonical, which historical names +remain accepted, and which identifiers must never be rewritten merely for +branding. Runtime and model support remain governed by +[`RUNTIME_SUPPORT.md`](RUNTIME_SUPPORT.md). + +## Canonical logo asset + +The public Hebrus mark is the maintainer-supplied +[`docs/media/hebrus-logo.png`](../media/hebrus-logo.png). Its source pixels stay +unchanged: the repository freezes the 1254 x 1254 RGBA PNG at SHA-256 +`4be8949c73bd52e7abef58396dcd57f636165a8bb6cd6d536a600bcbf880594c`. +Presentation surfaces may add a CSS `drop-shadow()` at render time; they must +not bake shadows, text, color changes, crops, or other edits into this master. + +## Command names + +Every command pair resolves to one real executable and one object graph. The +name used to invoke the executable controls its help/build identity, public +prompt identity, and `engine_id`; it does not select a different runtime. +Runtime diagnostic prefixes remain in the legacy namespace during this bridge +phase. + +| Canonical command | Compatibility command | Capability `engine_id` | +| --- | --- | --- | +| `hebrus` | `ds4` | `hebrus` / `ds4` | +| `hebrus-server` | `ds4-server` | `hebrus` / `ds4` | +| `hebrus-agent` | `ds4-agent` | `hebrus` / `ds4` | +| `hebrus-bench` | `ds4-bench` | `hebrus` / `ds4` | +| `hebrus-eval` | `ds4-eval` | `hebrus` / `ds4` | + +The compatibility commands remain supported through at least the complete 1.x +release line. Removing them would require usage evidence, a new accepted +decision record, release notes, and a separately tested migration. + +Options, defaults, streams, exit codes, model admission, HTTP protocols, +generated tokens, and runtime plans must remain equivalent across each pair. +Only the invoked command identity may differ. `make command-alias-test` checks +the active build profile; macOS `make premerge` also exercises the CPU aliases +through the build-isolation gate. + +## Structured runtime identity + +All ten command names expose `--capabilities=json` without loading a model. +Consumers must accept `engine_id` values `hebrus` and `ds4` during the bridge, +reject unknown schema versions, and validate the remaining fields exactly. +They must not infer support by scanning source filenames, C symbols, diagnostic +sentences, or binary strings. + +Changing `engine_id` does not change the schema-1 model-family list or the +ExpertMajor contract. In particular, both identities report: + +- `expert_major.version: 2`; +- `expert_major.tensor: "ds4.expert_major.v2"`; +- GGML storage wire value `0`; +- MLX affine4 storage wire value `1`, with Qwen group size `64`. + +## Permanently preserved identifiers + +The following are data or provenance, not product copy. They remain unchanged: + +- `ds4.expert_major.v2` and the retired `ds4.expert_major.v1` rejection marker; +- ExpertMajor magic, version, offsets, digest semantics, and numeric wire + values; +- disk-KV magic, version, and payload ABI; +- existing model filenames, immutable revisions, byte counts, and SHA-256 + values containing `DS4`; +- historical benchmark, release, issue, and pull-request links; +- Git history, authorship, copyright, and the fork relationship to + [`antirez/ds4`](https://github.com/antirez/ds4). + +A branded display label may say “Hebrus ExpertMajor v2”; serialized bytes and +identifiers do not change. + +## Environment variables and paths + +`DS4_*` environment variables remain the accepted runtime namespace in the +bridge release. `HEBRUS_*` aliases are deferred until a central resolver can +enforce one policy consistently. When aliases are introduced, conflicting old +and new values must fail closed; a textual mass replacement is not permitted. + +Existing checkouts, model paths, caches, benchmark records, and application +data are not copied or renamed. A repository redirect is transport +compatibility, not permission to reset an absolute local path. + +## Repository and application bridge + +Until the administrative rename occurs, the engine repository remains +`andreaborio/ds4`. Bridge consumers accept both `andreaborio/ds4` and the +planned `andreaborio/hebrus` identity over HTTPS or SSH. The existing fork is +renamed in place; no history-less replacement is created and the old repository +name must not be reused. + +The companion application's public name is **Hebrus Studio**. Its persisted +DSBox identity remains a compatibility boundary for the bridge release: +`com.dsbox.desktop`, `$HOME/.dsbox`, `DSBOX_*`, browser storage keys, and the +legacy Electron user-data directory do not move. The bridge application resolves +`hebrus-server` first, falls back to `ds4-server`, accepts both structured +engine identities, and permits source/string fallback only for a legacy +capability-less `ds4-server`. A capability-less `hebrus-server`, malformed JSON, +unknown schema, wrong backend/role, dirty or wrong revision, and contradictory +ExpertMajor fields fail closed. + +## Migration rules for maintainers + +1. Add compatibility before changing a public name. +2. Keep each rename tranche separate from inference, storage, kernel, and + performance changes. +3. Run `make brand-boundary-audit`; reductions pass, while every new or + increased legacy token group requires an exact reviewed classification. +4. Run the complete model-free gates for identity-only work and the canonical + model-backed release gates for any change that can affect code generation or + inference behavior. +5. Do not begin internal file/symbol migration until the Hebrus Studio bridge is + published and the Qwen MLX-affine baseline is qualified. + +The architectural rationale and the acceptance conditions for the rename are +recorded in +[`ADR 0005`](../adr/0005-hebrus-naming-and-compatibility-boundary.md). diff --git a/docs/contracts/RUNTIME_SUPPORT.md b/docs/contracts/RUNTIME_SUPPORT.md index c2139c9b9..75e17eb16 100644 --- a/docs/contracts/RUNTIME_SUPPORT.md +++ b/docs/contracts/RUNTIME_SUPPORT.md @@ -14,9 +14,9 @@ and Qwen's lower-memory extension in | Model family | Minimum unified memory | Qualified Metal modes | Release startup | | --- | ---: | --- | --- | -| DeepSeek V4 Flash | 64 GiB | AUTO resolving to resident or SSD; explicit resident and SSD according to the artifact gate | `./ds4 -m DEEPSEEK-DS4-ExpertMajor-v2.gguf` | -| Qwen3.6-35B-A3B | 16 GiB | AUTO resolving to resident or SSD; explicit resident and SSD according to the Qwen admission gates | `./ds4 -m QWEN-DS4-ExpertMajor-v2.gguf` | -| GLM 5.2 | 64 GiB | AUTO resolving to SSD streaming only; resident is rejected | `./ds4 -m GLM-DS4-ExpertMajor-v2.gguf --ctx 8192` | +| DeepSeek V4 Flash | 64 GiB | AUTO resolving to resident or SSD; explicit resident and SSD according to the artifact gate | `./hebrus -m DEEPSEEK-DS4-ExpertMajor-v2.gguf` | +| Qwen3.6-35B-A3B | 16 GiB | AUTO resolving to resident or SSD; explicit resident and SSD according to the Qwen admission gates | `./hebrus -m Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf` | +| GLM 5.2 | 64 GiB | AUTO resolving to SSD streaming only; resident is rejected | `./hebrus -m GLM-DS4-ExpertMajor-v2.gguf --ctx 8192` | All rows require Apple Metal and a validated embedded `ds4.expert_major.v2` store. Explicit residency options are controlled qualification tools; normal @@ -37,6 +37,18 @@ The lower-memory extension is Qwen-specific. Hosts below 64 GiB remain outside the DeepSeek and GLM production contract. Do not infer support for another family from Qwen's successful admission. +Runtime support and public artifact distribution are separate gates. The only +runnable Qwen store is the `published` +`Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf`, a +20,808,566,880-byte MLX affine4/group-64 artifact with SHA-256 +`dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d`. +`download_model.sh qwen-v2` pins immutable repository revision +`7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02`; its manifest requires runtime +commit `73a332fef82a0bcdd567d17e0de17aa004cad85d` or a compatible descendant. The +older `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-Q4_K_S.gguf` store is +`negative-only`, not a downloadable runtime fallback. The machine-readable +[Qwen release contract](qwen-release.json) is the canonical identity record. + ## Model Artifact Admission Production inference accepts the qualified embedded ExpertMajor v2 artifact for diff --git a/docs/contracts/qwen-release.json b/docs/contracts/qwen-release.json new file mode 100644 index 000000000..cd4ae3ad7 --- /dev/null +++ b/docs/contracts/qwen-release.json @@ -0,0 +1,22 @@ +{ + "schemaVersion": 1, + "modelFamily": "Qwen3.6-35B-A3B", + "downloadTarget": "qwen-v2", + "repository": "andreaborio/Qwen3.6-35B-A3B-DS4-GGUF", + "publishedArtifact": { + "status": "published", + "filename": "Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf", + "revision": "7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02", + "bytes": 20808566880, + "sha256": "dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d", + "runtimeCommit": "73a332fef82a0bcdd567d17e0de17aa004cad85d", + "storage": "mlx-affine4", + "groupSize": 64 + }, + "negativeArtifact": { + "status": "negative-only", + "filename": "Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-Q4_K_S.gguf", + "bytes": 20808566880, + "sha256": "d7c43a6388ec20e6fe5530850350f96fdb0ac37c5ce36d3e5f92b172c447f56b" + } +} diff --git a/docs/deepseek-expert-major-v2.md b/docs/deepseek-expert-major-v2.md index 3b4c7009f..23bbb8c04 100644 --- a/docs/deepseek-expert-major-v2.md +++ b/docs/deepseek-expert-major-v2.md @@ -1,4 +1,4 @@ -# DeepSeek DS4-native ExpertMajor v2 GGUF +# DeepSeek ExpertMajor v2 GGUF on Hebrus `ds4.expert_major.v2` is the self-describing routed-expert container used by DeepSeek V4, GLM 5.2, and Qwen3.6 in this fork. For DeepSeek it changes storage @@ -73,12 +73,13 @@ Native v2 activates automatically: ```sh make -j8 -./ds4 \ +./hebrus \ -m /absolute/path/to/DEEPSEEK-DS4-EXPERT-MAJOR-V2.gguf \ --ctx 32768 ``` -For the API, replace `./ds4` with `./ds4-server`. AUTO chooses resident mapping +For the API, replace `./hebrus` with `./hebrus-server`. The legacy command +names remain aliases to the same binaries. AUTO chooses resident mapping when the complete model safely fits or SSD streaming otherwise. The release command does not need an ExpertMajor variable, sidecar path, explicit backend, cache budget, preload policy, or residency flag. diff --git a/docs/expert-major-v2-roadmap.md b/docs/expert-major-v2-roadmap.md index 6d578597c..bf33c3cba 100644 --- a/docs/expert-major-v2-roadmap.md +++ b/docs/expert-major-v2-roadmap.md @@ -18,7 +18,7 @@ records: - record, component, layer, and payload offsets with explicit alignment; - source GGUF, payload, and manifest SHA-256 identities. -DS4 reconstructs canonical logical descriptors for graph binding and cache +Hebrus reconstructs canonical logical descriptors for graph binding and cache identity only. Resident mapping, prefill, decode, and SSD cache fills resolve physical bytes through the manifest. Unknown families or versions, canonical routed tensors beside a v2 store, incomplete inventories, unsupported types, @@ -30,9 +30,12 @@ The release runtime accepts ExpertMajor v2 artifacts only on local Apple Metal. The normal CLI is the same for all three families: ```sh -./ds4 -m /absolute/path/to/MODEL-DS4-ExpertMajor-v2.gguf --ctx 8192 +./hebrus -m /absolute/path/to/MODEL-DS4-ExpertMajor-v2.gguf --ctx 8192 ``` +`./ds4` remains a byte-identical compatibility alias; it is no longer the +primary command in new documentation. + AUTO chooses the family-qualified resident or SSD consumer. No ExpertMajor, sidecar, backend, cache, preload, or power flag is required. Canonical GGUFs remain inputs to `inspect`, `build`, and `verify`, but are not executable by @@ -49,14 +52,18 @@ layout. - **Implemented:** distinct `qwen35moe` family ID, 40-layer fail-closed geometry, generic v2 conversion, logical reconstruction, resident mapping, and SSD translation. -- **Qualified on M5 Pro 64 GiB:** resident output matches the retired v1 - control. The 2K v2/v1/v2 lane measured 318.96/29.54, 320.59/29.59, and - 318.83/29.54 prefill/decode t/s. -- **Correctness:** all three evidence files are byte-identical with SHA-256 - `399504c6ce3d4531ee0f2207702e96e2324c9b5c8dbf98adf47dfb9e64cae54d`; - no new swapout was observed. -- **Release artifact:** 20,808,566,880 bytes, SHA-256 - `d7c43a6388ec20e6fe5530850350f96fdb0ac37c5ce36d3e5f92b172c447f56b`. +- **Qualified and published:** MLX affine4/group-64 resident and SSD execution on + the hardware lanes recorded in the current benchmark index. +- **Correctness:** the final resident and SSD lane retains deterministic token + and logit comparisons with no new swapout in the recorded qualification. +- **Release artifact:** + `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf`, + 20,808,566,880 bytes, SHA-256 + `dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d`. + The immutable repository revision is + `7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02`; its manifest requires runtime + commit `73a332fef82a0bcdd567d17e0de17aa004cad85d` or a compatible descendant. + The former Q4_K_S artifact is retained only as a fail-closed negative case. ### DeepSeek V4 diff --git a/docs/glm52-expert-major-v2.md b/docs/glm52-expert-major-v2.md index 454ff9195..ce9fbaafe 100644 --- a/docs/glm52-expert-major-v2.md +++ b/docs/glm52-expert-major-v2.md @@ -1,6 +1,6 @@ -# GLM 5.2 DS4 ExpertMajor v2 +# GLM 5.2 ExpertMajor v2 on Hebrus -DS4 runs the 244.14 GiB GLM-5.2 Q2_K artifact on Apple Metal while streaming +Hebrus runs the 244.14 GiB GLM-5.2 Q2_K artifact on Apple Metal while streaming the routed experts from local storage. The qualified machine is an M5 Pro with 64 GiB unified memory. No claim is made for Macs below 64 GiB. @@ -17,7 +17,7 @@ revision may require a newly published GGUF. ```sh make -j8 -./ds4 \ +./hebrus \ -m /absolute/path/to/GLM-5.2-DS4-ExpertMajor-v2-Q2_K.gguf \ --ctx 8192 ``` @@ -25,7 +25,7 @@ make -j8 This starts the interactive CLI. A one-shot request only adds a prompt: ```sh -./ds4 \ +./hebrus \ -m /absolute/path/to/GLM-5.2-DS4-ExpertMajor-v2-Q2_K.gguf \ --ctx 8192 \ -p "Explain why locality matters in mixture-of-experts inference." @@ -34,7 +34,7 @@ This starts the interactive CLI. A one-shot request only adds a prompt: The local API uses the same contract: ```sh -./ds4-server \ +./hebrus-server \ -m /absolute/path/to/GLM-5.2-DS4-ExpertMajor-v2-Q2_K.gguf \ --ctx 8192 ``` diff --git a/docs/guides/MIGRATING_TO_HEBRUS.md b/docs/guides/MIGRATING_TO_HEBRUS.md new file mode 100644 index 000000000..48e1a064d --- /dev/null +++ b/docs/guides/MIGRATING_TO_HEBRUS.md @@ -0,0 +1,285 @@ +# Migrating From DS4 To Hebrus + +Status: bridge-release guide. The bridge is implemented in source but is not a +published release yet. Use the version, tag, repository URL, and checksums from +the eventual release announcement rather than guessing them from this guide. + +> [!IMPORTANT] +> Hebrus began as a fork of +> [`antirez/ds4`](https://github.com/antirez/ds4) and retains substantial +> implementation, architecture, utilities, and Git history from that project. +> [llama.cpp](https://github.com/ggml-org/llama.cpp), +> [GGML](https://github.com/ggml-org/ggml), and +> [MLX](https://github.com/ml-explore/mlx) are material engineering and +> validation references. The complete provenance boundary is in +> [`ACKNOWLEDGMENTS.md`](../../ACKNOWLEDGMENTS.md) and +> [`THIRD_PARTY_NOTICES.md`](../../THIRD_PARTY_NOTICES.md). + +This migration changes the public engine name without changing model bytes, +cache formats, runtime behavior, or existing automation all at once. The safe +path is to install the bridge first, prove both command identities, and only +then move human-facing scripts to the canonical Hebrus names. + +## What changes in the bridge + +| Surface | Canonical after installing the bridge | Compatibility behavior | +| --- | --- | --- | +| CLI | `hebrus` | `ds4` remains a symlink to the same CLI binary | +| HTTP server | `hebrus-server` | `ds4-server` remains a symlink to the same server binary | +| Agent | `hebrus-agent` | `ds4-agent` remains a symlink to the same agent binary | +| Benchmark | `hebrus-bench` | `ds4-bench` remains a symlink to the same benchmark binary | +| Evaluation | `hebrus-eval` | `ds4-eval` remains a symlink to the same evaluation binary | +| Structured identity | `engine_id: "hebrus"` | Legacy invocation reports `engine_id: "ds4"` | +| Companion app | Hebrus Studio is the selected public name | The bridge preserves its existing bundle ID, data root, and `DSBOX_*` configuration namespace | + +The invoked command name may change help headings, prompts, build labels, and +`engine_id`. Options, defaults, exit codes, protocols, generated tokens, model +admission, and resolved runtime plans must remain equivalent for every command +pair. Compatibility commands remain supported through at least the complete +1.x release line. + +## What does not change + +Do not rename, rewrite, repack, or copy any of the following as part of this +migration: + +- the embedded `ds4.expert_major.v2` tensor or its version, manifest, offsets, + digests, and storage wire values; +- the retired `ds4.expert_major.v1` rejection marker; +- published GGUF filenames, immutable revisions, byte counts, or SHA-256 + values that contain `DS4`; +- disk-KV magic, version, or payload ABI; +- current `DS4_*` engine environment variables; +- current Hebrus Studio `DSBOX_*` variables, `$HOME/.dsbox` data root, + `$HOME/Library/Application Support/DSBox` Electron `userData` directory, or + `com.dsbox.desktop` macOS bundle identifier; +- existing checkouts, model files, caches, conversations, downloads, Git + history, tags, authorship, or historical links. + +ExpertMajor is a disk ABI, not a product label. A display may say “Hebrus +ExpertMajor v2”; serialized identifiers must keep their historical bytes. + +## 1. Record the working installation + +Before changing a checkout or package, stop new inference work and record the +state needed to diagnose or roll back the upgrade: + +```sh +git rev-parse HEAD +git status --short +git remote -v + +ds4 --build-info +ds4 --capabilities=json +``` + +Also record the exact model path, byte count, checksum, and any `DS4_*` +variables used by the service. If Hebrus Studio manages the runtime, quit it +cleanly before replacing an application bundle; do not delete `$HOME/.dsbox`. + +## 2. Install the announced bridge revision + +Use the immutable release tag or commit from the release announcement. Until a +release is announced, the public repository remains +[`andreaborio/ds4`](https://github.com/andreaborio/ds4). + +For a user-local source installation: + +```sh +git fetch --tags origin +git switch --detach + +make -j +make model-free-test +make install PREFIX="$HOME/.local" +export PATH="$HOME/.local/bin:$PATH" +``` + +Replace the angle-bracket placeholder with the exact announced identity. Do +not paste the placeholder literally. Package maintainers should stage the same +layout without writing to the host: + +```sh +make install DESTDIR="$PWD/package-root" PREFIX=/usr/local +``` + +The staged `hebrus*` files are real executables. Each `ds4*` name is a relative +symlink to the corresponding canonical file, so a relocated package does not +capture a build-machine path. + +## 3. Verify both identities before changing automation + +Run the model-free identity checks first: + +```sh +hebrus --build-info +hebrus --capabilities=json +ds4 --build-info +ds4 --capabilities=json +``` + +Both capability documents must use schema version 1, the same non-empty build +SHA, backend, executable role, model-family list, and ExpertMajor contract. +Only `engine_id` changes with invocation. If building from source, the complete +repository checks are: + +```sh +make capabilities-test +make command-alias-test +make install-test +``` + +Stop if the build SHA, backend, role, model list, or ExpertMajor fields +contradict each other. Do not fall back from malformed Hebrus capability JSON +to source-string or binary-string probes. + +## 4. Move command consumers gradually + +New examples, packages, launch agents, and shell scripts should call the +canonical commands. Existing automation can continue using `ds4*` during the +bridge: + +```diff +--- before ++++ after +-exec ds4-server -m "$MODEL" --ctx 8192 ++exec hebrus-server -m "$MODEL" --ctx 8192 +``` + +Do not rename `DS4_*` variables to speculative `HEBRUS_*` variables. Those +aliases are deferred until one central resolver can detect conflicting old and +new values and fail closed. + +Integrations should discover the runtime by executing +`--capabilities=json`. During the bridge they must accept `engine_id` values +`hebrus` and `ds4`, require the expected role and backend, reject unknown schema +versions, and compare the complete ExpertMajor object. Do not infer support +from executable filenames alone. + +## 5. Migrate the companion application without moving its data + +Hebrus Studio is the selected public name for the companion application. Its +engine bridge should: + +1. resolve `hebrus-server` before `ds4-server`; +2. validate schema-1 capability JSON and accept both bridge `engine_id` values; +3. require the server role, expected backend, exact build identity, and exact + ExpertMajor contract; +4. use source/string fallback only for a legacy capability-less `ds4-server`; +5. reject a capability-less `hebrus-server`, malformed JSON, an unknown schema, + a contradictory contract, or the wrong build. + +The visible name does not authorize a storage migration. The bridge release +continues to use `com.dsbox.desktop`, `$HOME/.dsbox`, and the existing `DSBOX_*` +configuration names. Reusing those identities keeps installed models, +configuration, downloads, and local conversations in place. + +### Replace `DSBox.app` with `Hebrus Studio.app` + +The macOS upgrade is not an in-place Finder replacement. Because +`DSBox.app` and `Hebrus Studio.app` have different bundle filenames, dragging +the new application into `/Applications` does not replace `DSBox.app`; both +applications remain present until the old bundle is removed. +They nevertheless share the `com.dsbox.desktop` app identifier, legacy state, +and configured local control port (`4242` by default). Never run them at the +same time, including for a side-by-side comparison. + +Use this order after obtaining a verified Hebrus Studio package: + +1. Quit DSBox from its application menu and wait for its managed engine to + stop. Confirm neither DSBox nor its server remains active and that the + configured control port is no longer listening. +2. Confirm the existing state is present at both + `$HOME/Library/Application Support/DSBox` and `$HOME/.dsbox`. Back up any + irreplaceable configuration or conversation data according to the normal + host backup policy; do not rename either directory. +3. Copy `Hebrus Studio.app` into `/Applications`. Finder leaves + `/Applications/DSBox.app` beside it because the names differ; this is + expected and is not permission to launch both. +4. Launch only Hebrus Studio. The bridge must keep Electron `userData` at the + legacy `$HOME/Library/Application Support/DSBox` path and engine/application + state at `$HOME/.dsbox`; it must not create a parallel fresh profile merely + because the visible name changed. +5. Verify that existing settings, installed models, downloads, and local + conversations are visible, then start and stop one model to confirm the + expected control port and engine capability contract. +6. After verification, remove `/Applications/DSBox.app` or retain one old copy + only in an offline rollback archive outside `/Applications`. Do not leave a + routinely launchable old copy alongside Hebrus Studio. + +If either state directory changes, an empty profile appears, or the old process +still owns the control port, quit Hebrus Studio and stop the upgrade. Do not +merge two profile directories or launch DSBox to “repair” state while Hebrus +Studio is running. + +## 7. Handle the repository rename only after an announcement + +No repository rename is implied by this guide. Existing clones continue to use +their current remote. After maintainers announce and verify an administrative +rename, inspect the redirect and then update `origin` to the exact URL in that +announcement: + +```sh +git remote get-url origin +git fetch origin +``` + +Do not create a history-less replacement repository, reuse the old repository +name for another project, rewrite historical links, or change remotes to an +unreserved guessed namespace. + +## Rollback + +The engine compatibility bridge is designed to make rollback mechanical: + +1. stop the running engine or Hebrus Studio process and confirm its port has + been released; +2. restore the previously recorded executable package or checkout; +3. continue invoking `ds4*` with the unchanged model and data paths; +4. verify the restored `--build-info` and capability document; +5. preserve logs from the failed upgrade before retrying. + +Do not roll back by rewriting GGUF metadata, moving `$HOME/.dsbox`, or converting a +published model. If the new package installed into a private staging root, use +`make uninstall` with the same `DESTDIR` and `PREFIX`; it removes only the ten +explicit command paths. + +For an application rollback, keep the old app offline until it is needed, then: + +1. quit Hebrus Studio and its managed engine completely; +2. remove or archive `/Applications/Hebrus Studio.app` so only one application + can be launched; +3. restore the archived `DSBox.app` to `/Applications` without moving or + rewriting either legacy state directory; +4. launch only DSBox and verify its settings, model catalog, conversations, + control port, and engine build identity; +5. keep Hebrus Studio offline until the rollback issue is resolved. + +The bridge release does not require a state-schema migration, so both app +identities use the same legacy Electron `userData` and `$HOME/.dsbox` state. +If a later release explicitly introduces a schema migration, its own release +notes and backup procedure take precedence over this bridge rollback. + +## Completion checklist + +- [ ] The installed build matches the announced tag or commit. +- [ ] `hebrus*` and `ds4*` command pairs resolve to the same executable files. +- [ ] Canonical and compatibility capability documents differ only where the + invoked identity or role requires it. +- [ ] The exact qualified model artifact still passes size and SHA-256 checks. +- [ ] Existing `DS4_*`, `DSBOX_*`, `$HOME/.dsbox`, and serialized identifiers are + unchanged. +- [ ] Electron `userData` remains at + `$HOME/Library/Application Support/DSBox`; no second empty profile was + created for the visible name. +- [ ] DSBox and Hebrus Studio were never active simultaneously, and only one + application owns the configured control port. +- [ ] Hebrus Studio can admit the canonical server and fall back to the legacy + server without weakening validation. +- [ ] Rollback has been tested without copying or deleting user data. +- [ ] Remote URLs were changed only after an authoritative rename announcement. + +The normative compatibility rules are in +[`BRAND_COMPATIBILITY.md`](../contracts/BRAND_COMPATIBILITY.md); the supported +models and hardware boundary are in +[`RUNTIME_SUPPORT.md`](../contracts/RUNTIME_SUPPORT.md). diff --git a/docs/media/hebrus-logo.png b/docs/media/hebrus-logo.png new file mode 100644 index 000000000..9d889e6b3 Binary files /dev/null and b/docs/media/hebrus-logo.png differ diff --git a/docs/qwen-expert-major-store.md b/docs/qwen-expert-major-store.md index e741f4ea4..3c9b161e9 100644 --- a/docs/qwen-expert-major-store.md +++ b/docs/qwen-expert-major-store.md @@ -1,4 +1,4 @@ -# Qwen DS4-native ExpertMajor v2 GGUF +# Qwen ExpertMajor v2 GGUF on Hebrus Qwen3.6-35B-A3B uses the same self-describing `ds4.expert_major.v2` container as DeepSeek and GLM. The optimized Mac artifact stores every routed weight @@ -30,12 +30,13 @@ A valid Qwen v2 file activates automatically: ```sh make -j8 -./ds4 \ +./hebrus \ -m /absolute/path/to/Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf \ --ctx 8192 ``` -Use the same model with `./ds4-server` for the local API. Normal startup does +Use the same model with `./hebrus-server` for the local API. The legacy +`./ds4` and `./ds4-server` names remain aliases to the same build. Normal startup does not need an experimental guard, explicit Metal selection, a sidecar path, payload hashes, cache geometry, `--resident`, or `--ssd-streaming`. AUTO chooses resident mapping when the full working set fits its Metal and host-memory @@ -86,17 +87,38 @@ used for publication. | Item | Value | |---|---| -| Publication state | Published on Hugging Face; `dsbox.json` selects this exact byte identity | +| Publication state | `published` | +| Repository | `andreaborio/Qwen3.6-35B-A3B-DS4-GGUF` | | Artifact | `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf` | | Artifact bytes | 20,808,566,880 | | Artifact SHA-256 | `dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d` | +| Immutable revision | `7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02` | +| Minimum compatible runtime commit | `73a332fef82a0bcdd567d17e0de17aa004cad85d` | +| Storage | `mlx-affine4/group-64` | +| Negative fixture state | `negative-only` | +| Negative fixture | `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-Q4_K_S.gguf` | +| Negative fixture bytes | 20,808,566,880 | +| Negative fixture SHA-256 | `d7c43a6388ec20e6fe5530850350f96fdb0ac37c5ce36d3e5f92b172c447f56b` | | Canonical source bytes | 20,808,563,424 | | Canonical source SHA-256 | `c33efb67bde86c9ba1f9e79c2dc42627170963bef0e915ab9b91a55cfb6d0fcd` | | Storage marker | `MLX_AFFINE4`, group size 64 | +The machine-readable +[Qwen release contract](contracts/qwen-release.json) is canonical for the +repository, publication states, artifact identities, revision, and runtime +compatibility floor in this table. + The v2 output is 3,456 bytes larger than its canonical converter input. It does not contain a second copy of the routed weights. +These exact bytes and their matching manifest are published at immutable +repository revision `7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02`. +The manifest requires runtime commit +`73a332fef82a0bcdd567d17e0de17aa004cad85d` or a compatible descendant; +`download_model.sh qwen-v2` pins that revision and validates this relationship. +The older Q4_K_S object is incompatible with the current runtime contract and +is retained only for fail-closed testing. + ## Qualification On an M5 Pro with 64 GiB, final resident prefill measured 1,661.18 t/s at 2K, diff --git a/docs/releases/hebrus-bridge-launch-candidate.md b/docs/releases/hebrus-bridge-launch-candidate.md new file mode 100644 index 000000000..7686858f7 --- /dev/null +++ b/docs/releases/hebrus-bridge-launch-candidate.md @@ -0,0 +1,181 @@ +# Hebrus Bridge Release: Launch Candidate Notes + +- Status: local launch candidate; not published +- Version: to be assigned at release cut +- Release date: to be assigned at release cut +- Candidate commit: to be recorded after all release gates pass + +These notes separate behavior already implemented in source from external +release operations that still require an explicit decision, credentials, or a +published artifact. They are not a release announcement and do not promise a +repository URL, signed package, or availability date. + +> [!IMPORTANT] +> Hebrus began as a fork of +> [`antirez/ds4`](https://github.com/antirez/ds4) and retains substantial core +> implementation, architecture, utilities, and Git history from that project. +> [llama.cpp](https://github.com/ggml-org/llama.cpp), +> [GGML](https://github.com/ggml-org/ggml), and +> [MLX](https://github.com/ml-explore/mlx) remain material implementation and +> validation references. This lineage does not imply endorsement. See +> [`ACKNOWLEDGMENTS.md`](../../ACKNOWLEDGMENTS.md) and +> [`THIRD_PARTY_NOTICES.md`](../../THIRD_PARTY_NOTICES.md). + +## Candidate scope implemented in source + +### Canonical Hebrus command surface + +The bridge adds `hebrus`, `hebrus-server`, `hebrus-agent`, `hebrus-bench`, and +`hebrus-eval` as canonical commands. The corresponding `ds4*` names remain +relative symlinks to the same five binaries and preserve their invocation +identity. Options, protocols, model admission, generated tokens, and runtime +planning are shared; help text, prompts, build labels, and structured +`engine_id` reflect the command actually invoked. + +`make install` supports `PREFIX`, `BINDIR`, and `DESTDIR`. Its uninstall target +names only the ten public command paths and leaves unrelated files intact. + +### Versioned integration contract + +All ten commands expose deterministic `--capabilities=json` output without +loading a model. Schema 1 reports: + +- the invocation-aware `engine_id`; +- exact build Git SHA, backend, and executable role; +- the supported model-family identifiers; +- ExpertMajor version 2, the stable `ds4.expert_major.v2` tensor, and exact + GGML/MLX-affine storage wire values. + +Consumers can therefore validate a server without scanning source filenames, +C symbols, diagnostics, or binary strings. The bridge contract accepts +`engine_id` values `hebrus` and `ds4`; malformed documents, unknown schemas, +wrong roles or backends, and contradictory ExpertMajor fields fail closed. + +### Metal and SSD runtime focus + +The runtime remains deliberately narrow: qualified Apple Metal execution, +mmap-backed GGUF containers, checksummed embedded ExpertMajor v2 routed +weights, hardware-aware AUTO admission, and either resident Metal execution or +bounded SSD expert streaming. It is not a general GGUF runner. + +The current support authority is +[`RUNTIME_SUPPORT.md`](../contracts/RUNTIME_SUPPORT.md). Exact artifact names, +revisions, byte counts, hashes, and measured lanes remain in the model and +benchmark records; these launch-candidate notes do not replace them. + +### Companion bridge identity + +**Hebrus Studio** is the selected public name for the companion application. +The bridge resolves `hebrus-server` before `ds4-server` and validates the +structured capability contract while retaining a narrow legacy fallback for a +capability-less `ds4-server`. + +The visible name is separate from persisted compatibility identity. The bridge +release keeps the existing `com.dsbox.desktop` bundle identifier, `$HOME/.dsbox` +data root, legacy `$HOME/Library/Application Support/DSBox` Electron `userData` +directory, and `DSBOX_*` configuration namespace so installed models, +configuration, downloads, and local conversations are not moved. The desktop +application is delivered separately from the engine and is not embedded in +this repository's command package. + +`DSBox.app` and `Hebrus Studio.app` have different Finder names, so installing +the latter does not replace the former. They still share the app +identifier, state, and control port and must never run simultaneously. The +verified upgrade quits DSBox first, installs and validates Hebrus Studio, and +then removes the old bundle or retains it only in an offline rollback archive. + +### Canonical visual identity + +Both repositories use the same maintainer-supplied 1254 x 1254 RGBA Hebrus +master without pixel, crop, color, or encoding edits. Its SHA-256 is +`4be8949c73bd52e7abef58396dcd57f636165a8bb6cd6d536a600bcbf880594c`. +Hebrus Studio derives its macOS icon from that exact file during packaging and +adds a drop shadow only through CSS on web surfaces. The engine and application +release gates reject a modified master or a mismatched packaged copy. + +### Open-source project surface + +The candidate includes contribution, security, governance, conduct, +acknowledgment, third-party notice, changelog, citation, issue-template, and +release-checklist material. Linux and macOS hosted workflows cover the +model-free and compile boundaries appropriate to those runners. Hosted macOS +compilation is not evidence for qualified Metal kernels, model artifacts, +throughput, memory pressure, or SSD behavior. + +## Preserved compatibility + +This bridge intentionally does not rename: + +- `ds4.expert_major.v2`, the retired v1 rejection marker, or any ExpertMajor + on-disk bytes; +- published GGUF filenames, immutable revisions, sizes, or checksums; +- disk-KV magic, version, or payload ABI; +- existing `DS4_*` engine variables; +- Hebrus Studio's `DSBOX_*` variables, data root, or bundle identifier; +- source-level C identifiers, Git history, authorship, tags, or historical + benchmark and release links. + +Compatibility aliases remain supported through at least the complete 1.x +release line. Any future removal requires usage evidence, a new accepted +decision record, release notes, and its own migration tests. + +## Evidence required on the final candidate commit + +Before replacing the placeholders at the top of this document, the release +owner must attach evidence for the exact commit being published: + +- clean context, documentation-link, brand-boundary, model-free, build + isolation, command-alias, staged-install, and download-manifest gates; +- green SHA-pinned Linux and macOS hosted jobs for that commit; +- qualified Apple Silicon model-backed correctness and performance lanes for + every supported artifact, including exact output parity, resolved plan, + memory pressure, swap deltas, and SSD telemetry; +- exact source archive, engine package, model manifest, and companion-package + checksums; +- a clean-install and no-data-loss upgrade/rollback exercise for Hebrus Studio + that proves the legacy Electron `userData` and `$HOME/.dsbox` state are + reused, only one app owns the control port, and Finder's two bundle names do + not permit concurrent launch; +- an updated support matrix and benchmark evidence index that cite the same + immutable commit and artifacts. + +Follow [`QA_BEFORE_RELEASES.md`](../../QA_BEFORE_RELEASES.md); a passing narrow +smoke test cannot stand in for this matrix. + +## External and administrative gates still pending + +The following are not implied by source readiness and must remain described as +pending until independently verified: + +- namespace and legal screening, reservation, and acceptance of the naming ADR; +- administrative repository rename and publication of the compatibility app; +- final version/tag selection and remote push; +- GitHub private vulnerability reporting or another verified private intake; +- Apple Developer ID signing, notarization, stapling, and clean-machine + Gatekeeper verification for Hebrus Studio; +- public download URLs, release artifacts, checksums, screenshots, and launch + announcement; +- model-backed release evidence on the exact final commit and qualified + hardware. + +An ad-hoc-signed local application bundle proves package structure and seal +integrity only. It must not be described as Developer ID signed or notarized. + +## Upgrade and rollback + +Users and package maintainers should follow +[`Migrating From DS4 To Hebrus`](../guides/MIGRATING_TO_HEBRUS.md). The guide +keeps existing commands and persisted identities operational, validates both +capability identities before consumers switch, and provides a rollback that +does not rewrite models or user data. + +## Release-owner cut procedure + +1. Choose and record the immutable candidate commit. +2. Run the complete checklist and archive exact raw evidence. +3. Verify the release version appears consistently in packages and notes. +4. Replace only the version, date, and commit placeholders above. +5. Confirm every pending item is either completed with evidence or explicitly + disclosed as unavailable; do not silently omit a gate. +6. Publish only after the engine, Hebrus Studio bridge, migration guide, + checksums, provenance, and rollback instructions agree. diff --git a/download_model.sh b/download_model.sh index ce927b490..5e1247eb7 100755 --- a/download_model.sh +++ b/download_model.sh @@ -9,11 +9,13 @@ RUNTIME_GLM_REPO="andreaborio/GLM-5.2-DS4-GGUF" RUNTIME_GLM_FILE="GLM-5.2-DS4-ExpertMajor-v2-Q2_K.gguf" RUNTIME_GLM_BYTES=262147193504 RUNTIME_GLM_SHA256="7f5017e3076e706c78f2a5322b035a9e2f6519c65ff5b6be8b2d91aeff61505d" +RUNTIME_QWEN_STATUS="published" RUNTIME_QWEN_REPO="andreaborio/Qwen3.6-35B-A3B-DS4-GGUF" RUNTIME_QWEN_FILE="Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf" RUNTIME_QWEN_BYTES=20808566880 RUNTIME_QWEN_SHA256="dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d" RUNTIME_QWEN_REVISION="7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02" +RUNTIME_QWEN_MIN_RUNTIME_COMMIT="73a332fef82a0bcdd567d17e0de17aa004cad85d" RUNTIME_REVISION="ds4-v0.2.0" OFFLINE_DEEPSEEK_REPO="antirez/deepseek-v4-gguf" diff --git a/ds4.h b/ds4.h index 30172962b..0d280e107 100644 --- a/ds4.h +++ b/ds4.h @@ -234,11 +234,21 @@ uint64_t ds4_engine_hidden_f32_values(ds4_engine *e); int ds4_engine_model_id(ds4_engine *e); bool ds4_engine_is_glm_dsa(ds4_engine *e); const char *ds4_backend_name(ds4_backend backend); +typedef enum { + DS4_EXECUTABLE_ROLE_CLI, + DS4_EXECUTABLE_ROLE_SERVER, + DS4_EXECUTABLE_ROLE_AGENT, + DS4_EXECUTABLE_ROLE_BENCH, + DS4_EXECUTABLE_ROLE_EVAL, +} ds4_executable_role; const char *ds4_build_backend(void); const char *ds4_build_arch(void); const char *ds4_build_git_sha(void); -void ds4_build_info_print(FILE *fp); +void ds4_build_info_print(FILE *fp, const char *argv0); bool ds4_build_info_requested(int argc, char **argv); +bool ds4_capabilities_requested(int argc, char **argv); +void ds4_capabilities_print(FILE *fp, ds4_executable_role role, + const char *argv0); bool ds4_think_mode_enabled(ds4_think_mode mode); const char *ds4_think_mode_name(ds4_think_mode mode); const char *ds4_think_max_prefix(void); diff --git a/ds4_agent.c b/ds4_agent.c index fbfe08748..d7c6b976e 100644 --- a/ds4_agent.c +++ b/ds4_agent.c @@ -2,6 +2,7 @@ #include "ds4_help.h" #include "ds4_kvstore.h" #include "ds4_web.h" +#include "hebrus_identity.h" #include "linenoise.h" #include @@ -33,6 +34,8 @@ * after Enter is pressed while the model is still busy. */ int linenoiseEditInsert(struct linenoiseState *l, const char *c, size_t clen); +static const char *agent_invocation = "ds4-agent"; + static int set_nonblock(int fd, bool on, int *old_flags); static bool agent_parse_bool_default(const char *s, bool def); @@ -551,7 +554,7 @@ static agent_config parse_options(int argc, char **argv) { .mtp_margin = 3.0f, }, .gen = { - .system = "You are a helpful coding assistant running inside ds4-agent.", + .system = hebrus_agent_system_prompt_for(argv[0]), .n_predict = 50000, .ctx_size = 100000, .temperature = DS4_DEFAULT_TEMPERATURE, @@ -8903,7 +8906,7 @@ static void agent_progress_append(char *buf, size_t len, size_t *pos, static void build_prompt_text(const agent_status *st, char *buf, size_t len) { (void)st; - snprintf(buf, len, "ds4-agent> "); + hebrus_agent_format_prompt_for(agent_invocation, buf, len); } static void agent_progress_bar(int done, int total, double tps, @@ -9980,13 +9983,11 @@ static void agent_format_welcome_banner(const agent_config *cfg, char *buf, size_t len) { char ctx[32]; agent_format_ctx_size(cfg->gen.ctx_size, ctx, sizeof(ctx)); - if (stdout_is_tty()) { - snprintf(buf, len, - "\x1b[1;97mDwarf\x1b[1;94mStar\x1b[0m 🐋 Agent, context %s tokens\n\n", - ctx); - } else { - snprintf(buf, len, "DwarfStar Agent, context %s tokens\n\n", ctx); - } + hebrus_agent_format_welcome_banner_for(agent_invocation, + stdout_is_tty(), + ctx, + buf, + len); } static void editor_write_welcome_banner(agent_editor *editor, @@ -10822,8 +10823,14 @@ static int run_agent(ds4_engine *engine, agent_config *cfg) { #ifndef DS4_AGENT_TEST_NO_MAIN int main(int argc, char **argv) { + agent_invocation = argv[0]; + hebrus_help_set_invocation(argv[0]); if (ds4_build_info_requested(argc, argv)) { - ds4_build_info_print(stdout); + ds4_build_info_print(stdout, argv[0]); + return 0; + } + if (ds4_capabilities_requested(argc, argv)) { + ds4_capabilities_print(stdout, DS4_EXECUTABLE_ROLE_AGENT, argv[0]); return 0; } agent_config cfg = parse_options(argc, argv); diff --git a/ds4_bench.c b/ds4_bench.c index 1355e5e62..93f3f5330 100644 --- a/ds4_bench.c +++ b/ds4_bench.c @@ -1,5 +1,6 @@ #include "ds4.h" #include "ds4_help.h" +#include "hebrus_identity.h" #ifndef DS4_NO_GPU #include "ds4_gpu.h" #endif @@ -53,6 +54,8 @@ typedef struct { bool ssd_streaming_cold; } bench_config; +static const char *bench_invocation = "ds4-bench"; + static double bench_now_sec(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); @@ -384,7 +387,8 @@ static int write_frontier_logits_json( } const int argmax = ds4_session_argmax(session); - fprintf(fp, "{\n \"source\":\"ds4-bench\",\n \"model\":"); + fprintf(fp, "{\n \"source\":\"%s\",\n \"model\":", + hebrus_bench_command_for(bench_invocation)); json_write_string(fp, cfg->model_path); fprintf(fp, ",\n \"backend\":\"%s\",\n \"quality\":%s,\n" @@ -561,8 +565,14 @@ static void log_context_memory(ds4_backend backend, } int main(int argc, char **argv) { + bench_invocation = argv[0]; + hebrus_help_set_invocation(argv[0]); if (ds4_build_info_requested(argc, argv)) { - ds4_build_info_print(stdout); + ds4_build_info_print(stdout, argv[0]); + return 0; + } + if (ds4_capabilities_requested(argc, argv)) { + ds4_capabilities_print(stdout, DS4_EXECUTABLE_ROLE_BENCH, argv[0]); return 0; } bench_config cfg = parse_options(argc, argv); diff --git a/ds4_build.c b/ds4_build.c index 85f331564..61468099e 100644 --- a/ds4_build.c +++ b/ds4_build.c @@ -1,4 +1,6 @@ #include "ds4.h" +#include "ds4_expert_store.h" +#include "hebrus_identity.h" #include @@ -6,6 +8,16 @@ #define DS4_BUILD_GIT_SHA "unknown" #endif +#define DS4_BUILD_STATIC_ASSERT(name, condition) \ + typedef char ds4_build_static_assert_##name[(condition) ? 1 : -1] + +DS4_BUILD_STATIC_ASSERT(expert_store_v2_version, + DS4_EXPERT_STORE_V2_VERSION == 2); +DS4_BUILD_STATIC_ASSERT(expert_store_ggml_wire_value, + DS4_EXPERT_STORE_STORAGE_GGML == 0); +DS4_BUILD_STATIC_ASSERT(expert_store_mlx_affine4_wire_value, + DS4_EXPERT_STORE_STORAGE_MLX_AFFINE4 == 1); + const char *ds4_build_backend(void) { #ifdef DS4_NO_GPU return "cpu"; @@ -28,13 +40,14 @@ const char *ds4_build_git_sha(void) { return DS4_BUILD_GIT_SHA; } -void ds4_build_info_print(FILE *fp) { +void ds4_build_info_print(FILE *fp, const char *argv0) { if (!fp) fp = stdout; fprintf(fp, - "ds4 build\n" + "%s build\n" "git: %s\n" "backend: %s\n" "arch: %s\n", + hebrus_is_canonical_invocation(argv0) ? "hebrus" : "ds4", ds4_build_git_sha(), ds4_build_backend(), ds4_build_arch()); @@ -46,3 +59,95 @@ bool ds4_build_info_requested(int argc, char **argv) { } return false; } + +bool ds4_capabilities_requested(int argc, char **argv) { + for (int i = 1; i < argc; i++) { + if (argv[i] && strcmp(argv[i], "--capabilities=json") == 0) return true; + } + return false; +} + +static const char *ds4_executable_role_name(ds4_executable_role role) { + switch (role) { + case DS4_EXECUTABLE_ROLE_CLI: + return "cli"; + case DS4_EXECUTABLE_ROLE_SERVER: + return "server"; + case DS4_EXECUTABLE_ROLE_AGENT: + return "agent"; + case DS4_EXECUTABLE_ROLE_BENCH: + return "bench"; + case DS4_EXECUTABLE_ROLE_EVAL: + return "eval"; + } + return "unknown"; +} + +static void ds4_json_string_print(FILE *fp, const char *value) { + fputc('"', fp); + for (const unsigned char *p = (const unsigned char *)value; *p; p++) { + switch (*p) { + case '"': + fputs("\\\"", fp); + break; + case '\\': + fputs("\\\\", fp); + break; + case '\b': + fputs("\\b", fp); + break; + case '\f': + fputs("\\f", fp); + break; + case '\n': + fputs("\\n", fp); + break; + case '\r': + fputs("\\r", fp); + break; + case '\t': + fputs("\\t", fp); + break; + default: + if (*p < 0x20) { + fprintf(fp, "\\u%04x", (unsigned)*p); + } else { + fputc(*p, fp); + } + break; + } + } + fputc('"', fp); +} + +void ds4_capabilities_print(FILE *fp, ds4_executable_role role, + const char *argv0) { + if (!fp) fp = stdout; + fputs("{\n" + " \"schema_version\": 1,\n" + " \"engine_id\": ", fp); + ds4_json_string_print(fp, + hebrus_is_canonical_invocation(argv0) ? "hebrus" : "ds4"); + fputs(",\n \"build_git_sha\": ", fp); + ds4_json_string_print(fp, ds4_build_git_sha()); + fputs(",\n \"backend\": ", fp); + ds4_json_string_print(fp, ds4_build_backend()); + fputs(",\n \"executable_role\": ", fp); + ds4_json_string_print(fp, ds4_executable_role_name(role)); + fprintf(fp, + ",\n" + " \"model_families\": [\"deepseek4\", \"glm-dsa\", \"qwen35moe\"],\n" + " \"expert_major\": {\n" + " \"version\": %u,\n" + " \"tensor\": \"%s\",\n" + " \"storage_formats\": [\n" + " {\"id\": \"ggml\", \"wire_value\": %u, \"group_sizes\": []},\n" + " {\"id\": \"mlx-affine4\", \"wire_value\": %u, \"group_sizes\": [64]}\n" + " ]\n" + " }\n" + "}\n", + (unsigned)DS4_EXPERT_STORE_V2_VERSION, + DS4_EXPERT_STORE_V2_TENSOR, + (unsigned)DS4_EXPERT_STORE_STORAGE_GGML, + (unsigned)DS4_EXPERT_STORE_STORAGE_MLX_AFFINE4); +} diff --git a/ds4_cli.c b/ds4_cli.c index 49c2a42e7..f8d72d0f9 100644 --- a/ds4_cli.c +++ b/ds4_cli.c @@ -1,5 +1,6 @@ #include "ds4.h" #include "ds4_help.h" +#include "hebrus_identity.h" #include "linenoise.h" /* ds4 CLI. @@ -64,6 +65,7 @@ typedef struct { } cli_config; static volatile sig_atomic_t cli_interrupted; +static const char *cli_invocation = "ds4"; static void cli_sigint_handler(int sig) { (void)sig; @@ -1495,7 +1497,7 @@ static int run_repl(ds4_engine *engine, cli_config *cfg) { int rc = 0; for (;;) { errno = 0; - char *line = linenoise("ds4> "); + char *line = linenoise(hebrus_cli_prompt_for(cli_invocation)); if (!line) { if (errno == EAGAIN || cli_interrupt_requested()) { cli_interrupt_clear(); @@ -1891,8 +1893,14 @@ static cli_config parse_options(int argc, char **argv) { } int main(int argc, char **argv) { + cli_invocation = argv[0]; + hebrus_help_set_invocation(argv[0]); if (ds4_build_info_requested(argc, argv)) { - ds4_build_info_print(stdout); + ds4_build_info_print(stdout, argv[0]); + return 0; + } + if (ds4_capabilities_requested(argc, argv)) { + ds4_capabilities_print(stdout, DS4_EXECUTABLE_ROLE_CLI, argv[0]); return 0; } cli_config cfg = parse_options(argc, argv); diff --git a/ds4_eval.c b/ds4_eval.c index 19fedc61c..06ff104bc 100644 --- a/ds4_eval.c +++ b/ds4_eval.c @@ -1,5 +1,6 @@ #include "ds4.h" #include "ds4_help.h" +#include "hebrus_identity.h" /* ds4-eval: small built-in benchmark integration test. * @@ -1933,12 +1934,15 @@ static void tui_signal_restore(int sig) { raise(sig); } +static const char *eval_invocation = "ds4-eval"; + static void tui_draw_title(eval_ui *ui) { term_move(1, 1); tui_clear_left_line(ui, 1); char elapsed[32]; format_run_elapsed(elapsed, sizeof(elapsed), tui_run_clock_visible_sec(ui)); - fputs("ds4-eval (" ANSI_BOLD "p" ANSI_RESET ")ause (" ANSI_BOLD "q" ANSI_RESET ")uit", stdout); + printf("%s (" ANSI_BOLD "p" ANSI_RESET ")ause (" ANSI_BOLD "q" ANSI_RESET ")uit", + hebrus_eval_command_for(eval_invocation)); printf(" %s", elapsed); if (ui->paused) { fputs(" " ANSI_RED ANSI_BOLD "PAUSED" ANSI_RESET, stdout); @@ -4077,8 +4081,14 @@ static uint32_t eval_residency_context_hint(const eval_config *cfg) { } int main(int argc, char **argv) { + eval_invocation = argv[0]; + hebrus_help_set_invocation(argv[0]); if (ds4_build_info_requested(argc, argv)) { - ds4_build_info_print(stdout); + ds4_build_info_print(stdout, argv[0]); + return 0; + } + if (ds4_capabilities_requested(argc, argv)) { + ds4_capabilities_print(stdout, DS4_EXECUTABLE_ROLE_EVAL, argv[0]); return 0; } eval_config cfg = parse_options(argc, argv); diff --git a/ds4_gpu.h b/ds4_gpu.h index c4d9cef28..e0dbb29a1 100644 --- a/ds4_gpu.h +++ b/ds4_gpu.h @@ -1663,6 +1663,8 @@ int ds4_gpu_internal_qwen35_expert_group_test(void); /* Model-free source-translation/fail-closed regression used by * --metal-expert-pack. */ int ds4_gpu_internal_qwen35_expert_pack_test(void); +/* Model-free resident MLX-affine regression across the routed-MM threshold. */ +int ds4_gpu_internal_qwen35_affine_resident_short_test(void); /* Canonical-vs-embedded GLM Q2 regression for direct and grouped execution. */ int ds4_gpu_internal_expert_store_v2_kernel_test(void); diff --git a/ds4_help.c b/ds4_help.c index 1bc3c2887..9ad33af4f 100644 --- a/ds4_help.c +++ b/ds4_help.c @@ -1,4 +1,5 @@ #include "ds4_help.h" +#include "hebrus_identity.h" #include #include @@ -100,13 +101,19 @@ static bool topic_is(const char *topic, const char *name) { return topic && strcmp(topic, name) == 0; } +static bool help_hebrus_invocation; + +void hebrus_help_set_invocation(const char *argv0) { + help_hebrus_invocation = hebrus_is_canonical_invocation(argv0); +} + static const char *tool_name(ds4_help_tool tool) { switch (tool) { - case DS4_HELP_DS4: return "ds4"; - case DS4_HELP_SERVER: return "ds4-server"; - case DS4_HELP_AGENT: return "ds4-agent"; - case DS4_HELP_BENCH: return "ds4-bench"; - case DS4_HELP_EVAL: return "ds4-eval"; + case DS4_HELP_DS4: return help_hebrus_invocation ? "hebrus" : "ds4"; + case DS4_HELP_SERVER: return help_hebrus_invocation ? "hebrus-server" : "ds4-server"; + case DS4_HELP_AGENT: return help_hebrus_invocation ? "hebrus-agent" : "ds4-agent"; + case DS4_HELP_BENCH: return help_hebrus_invocation ? "hebrus-bench" : "ds4-bench"; + case DS4_HELP_EVAL: return help_hebrus_invocation ? "hebrus-eval" : "ds4-eval"; } return "ds4"; } @@ -144,28 +151,33 @@ bool ds4_help_reject_retired_distributed_option( return true; } -static const char *tool_usage(ds4_help_tool tool) { +static void print_tool_usage(FILE *fp, ds4_help_tool tool) { switch (tool) { case DS4_HELP_DS4: - return "Usage: ds4 [(-p PROMPT | --prompt-file FILE)] [options]"; + fprintf(fp, "Usage: %s [(-p PROMPT | --prompt-file FILE)] [options]", tool_name(tool)); + return; case DS4_HELP_SERVER: - return "Usage: ds4-server [options]"; + fprintf(fp, "Usage: %s [options]", tool_name(tool)); + return; case DS4_HELP_AGENT: - return "Usage: ds4-agent [options]"; + fprintf(fp, "Usage: %s [options]", tool_name(tool)); + return; case DS4_HELP_BENCH: - return "Usage: ds4-bench (--prompt-file FILE | --chat-prompt-file FILE) [options]"; + fprintf(fp, "Usage: %s (--prompt-file FILE | --chat-prompt-file FILE) [options]", tool_name(tool)); + return; case DS4_HELP_EVAL: - return "Usage: ds4-eval [options]"; + fprintf(fp, "Usage: %s [options]", tool_name(tool)); + return; } - return "Usage: ds4 [options]"; + fprintf(fp, "Usage: %s [options]", tool_name(tool)); } static const char *tool_summary(ds4_help_tool tool) { switch (tool) { case DS4_HELP_DS4: - return "Chat with a local DwarfStar model, run one-shot prompts, or inspect models."; + return "Chat with a local model, run one-shot prompts, or inspect models."; case DS4_HELP_SERVER: - return "Serve one loaded DwarfStar model through OpenAI, Responses, Anthropic, and completion-compatible HTTP APIs."; + return "Serve one loaded model through OpenAI, Responses, Anthropic, and completion-compatible HTTP APIs."; case DS4_HELP_AGENT: return "Run the native terminal coding agent with live tools, session save/restore, and a responsive prompt while the model works."; case DS4_HELP_BENCH: @@ -191,6 +203,7 @@ static void print_model_runtime(FILE *fp, const help_colors *c, opt(fp, c, "-t, --threads N", "CPU helper threads for host-side/reference work."); opt(fp, c, "--power N", "GPU duty-cycle target, 1..100. Default: 100"); opt(fp, c, "--build-info", "Print compiled backend, architecture, and git revision, then exit."); + opt(fp, c, "--capabilities=json", "Print the versioned machine-readable runtime contract, then exit."); opt(fp, c, "--ssd-streaming", "Force SSD-backed model streaming. Metal defaults to AUTO residency."); opt(fp, c, "--resident, --no-ssd-streaming", "Force full model residency instead of Metal AUTO selection."); opt(fp, c, "--ssd-streaming-cold", "SSD streaming: skip default popularity-based expert-cache preload."); @@ -462,45 +475,78 @@ static void print_more_info(FILE *fp, const help_colors *c, ds4_help_tool tool) fputc('\n', fp); } +static const char *example_command(char *buf, size_t size, ds4_help_tool tool, + const char *arguments) { + snprintf(buf, size, "./%s%s", tool_name(tool), arguments); + return buf; +} + +static const char *example_root_command(char *buf, size_t size, + const char *arguments) { + snprintf(buf, size, "./%s%s", + help_hebrus_invocation ? "hebrus" : "ds4", arguments); + return buf; +} + static void print_examples(FILE *fp, const help_colors *c, ds4_help_tool tool, const char *topic) { + char command[256]; title(fp, c, "Examples"); if (topic_is(topic, "distributed")) { para(fp, c, "There is no supported distributed startup command."); } else if (topic_is(topic, "runtime")) { if (tool == DS4_HELP_SERVER) { - opt(fp, c, "Metal API", "./ds4-server -m ds4flash.gguf --metal --ctx 100000"); - opt(fp, c, "quiet API", "./ds4-server --power 60 --host 127.0.0.1 --port 8000"); + opt(fp, c, "Metal API", example_command(command, sizeof(command), tool, + " -m ds4flash.gguf --metal --ctx 100000")); + opt(fp, c, "quiet API", example_command(command, sizeof(command), tool, + " --power 60 --host 127.0.0.1 --port 8000")); } else if (tool == DS4_HELP_AGENT) { - opt(fp, c, "agent", "./ds4-agent -m ds4flash.gguf --ctx 100000"); - opt(fp, c, "quiet agent", "./ds4-agent --power 50"); + opt(fp, c, "agent", example_command(command, sizeof(command), tool, + " -m ds4flash.gguf --ctx 100000")); + opt(fp, c, "quiet agent", example_command(command, sizeof(command), tool, + " --power 50")); } else if (tool == DS4_HELP_BENCH) { - opt(fp, c, "bench", "./ds4-bench --prompt-file long.txt --ctx-max 32768"); - opt(fp, c, "quiet bench", "./ds4-bench --prompt-file long.txt --power 70"); + opt(fp, c, "bench", example_command(command, sizeof(command), tool, + " --prompt-file long.txt --ctx-max 32768")); + opt(fp, c, "quiet bench", example_command(command, sizeof(command), tool, + " --prompt-file long.txt --power 70")); } else if (tool == DS4_HELP_EVAL) { - opt(fp, c, "eval", "./ds4-eval --questions 10 --ctx 100000"); - opt(fp, c, "CPU debug", "./ds4-eval --cpu --questions 1 --tokens 32"); + opt(fp, c, "eval", example_command(command, sizeof(command), tool, + " --questions 10 --ctx 100000")); + opt(fp, c, "CPU debug", example_command(command, sizeof(command), tool, + " --cpu --questions 1 --tokens 32")); } else { - opt(fp, c, "Metal", "./ds4 -m ds4flash.gguf --metal -c 100000"); - opt(fp, c, "quiet thermals", "./ds4 -p \"Summarize README\" --power 50"); + opt(fp, c, "Metal", example_command(command, sizeof(command), tool, + " -m ds4flash.gguf --metal -c 100000")); + opt(fp, c, "quiet thermals", example_command(command, sizeof(command), tool, + " -p \"Summarize README\" --power 50")); } } else if (topic_is(topic, "steering")) { - opt(fp, c, "steer FFN", "./ds4 -p \"Write tersely\" --dir-steering-file dir.bin --dir-steering-ffn 0.8"); + opt(fp, c, "steer FFN", example_root_command(command, sizeof(command), + " -p \"Write tersely\" --dir-steering-file dir.bin --dir-steering-ffn 0.8")); } else if (tool == DS4_HELP_SERVER || topic_is(topic, "api") || topic_is(topic, "kv-cache")) { - opt(fp, c, "local API", "./ds4-server --ctx 100000 --kv-disk-dir ~/.ds4/server-kv --kv-disk-space-mb 8192"); + opt(fp, c, "local API", example_command(command, sizeof(command), tool, + " --ctx 100000 --kv-disk-dir ~/.ds4/server-kv --kv-disk-space-mb 8192")); opt(fp, c, "curl", "curl http://127.0.0.1:8000/v1/models"); } else if (tool == DS4_HELP_AGENT || topic_is(topic, "sessions") || topic_is(topic, "tools")) { - opt(fp, c, "interactive", "./ds4-agent"); - opt(fp, c, "one shot", "./ds4-agent --non-interactive -p \"Create /tmp/hello.c\""); + opt(fp, c, "interactive", example_command(command, sizeof(command), tool, "")); + opt(fp, c, "one shot", example_command(command, sizeof(command), tool, + " --non-interactive -p \"Create /tmp/hello.c\"")); } else if (tool == DS4_HELP_BENCH || topic_is(topic, "benchmark")) { - opt(fp, c, "csv", "./ds4-bench --prompt-file long.txt --ctx-max 32768 --csv speed.csv"); - opt(fp, c, "prefill only", "./ds4-bench --prompt-file long.txt --gen-tokens 0"); + opt(fp, c, "csv", example_command(command, sizeof(command), tool, + " --prompt-file long.txt --ctx-max 32768 --csv speed.csv")); + opt(fp, c, "prefill only", example_command(command, sizeof(command), tool, + " --prompt-file long.txt --gen-tokens 0")); } else if (tool == DS4_HELP_EVAL || topic_is(topic, "evaluation")) { - opt(fp, c, "first 10", "./ds4-eval --questions 10 --trace eval.trace"); - opt(fp, c, "plain", "./ds4-eval --plain --nothink --tokens 512"); + opt(fp, c, "first 10", example_command(command, sizeof(command), tool, + " --questions 10 --trace eval.trace")); + opt(fp, c, "plain", example_command(command, sizeof(command), tool, + " --plain --nothink --tokens 512")); } else { - opt(fp, c, "chat", "./ds4"); - opt(fp, c, "one shot", "./ds4 -p \"Explain mmap in C\""); - opt(fp, c, "long prompt", "./ds4 --think-max --prompt-file prompt.txt --ctx 393216"); + opt(fp, c, "chat", example_command(command, sizeof(command), tool, "")); + opt(fp, c, "one shot", example_command(command, sizeof(command), tool, + " -p \"Explain mmap in C\"")); + opt(fp, c, "long prompt", example_command(command, sizeof(command), tool, + " --think-max --prompt-file prompt.txt --ctx 393216")); } fputc('\n', fp); } @@ -577,7 +623,8 @@ void ds4_help_print(FILE *fp, ds4_help_tool tool, const char *topic) { fprintf(fp, "%s%s%s\n", c.bright ? c.bright : "", tool_name(tool), c.off ? c.off : ""); fprintf(fp, "%s\n\n", tool_summary(tool)); - fprintf(fp, "%s\n\n", tool_usage(tool)); + print_tool_usage(fp, tool); + fputs("\n\n", fp); if (topic) print_topic(fp, &c, tool, topic); else { diff --git a/ds4_help.h b/ds4_help.h index a1a32628b..863ad9f69 100644 --- a/ds4_help.h +++ b/ds4_help.h @@ -12,6 +12,11 @@ typedef enum { DS4_HELP_EVAL, } ds4_help_tool; +/* Select the public command family from argv[0]. Canonical Hebrus binaries and + * legacy command symlinks share one object graph, so help text must use the name + * through which that graph was invoked. */ +void hebrus_help_set_invocation(const char *argv0); + /* Return true after printing the canonical error for a retired distributed * option. Both the exact spelling and --option=value forms are recognized. */ bool ds4_help_reject_retired_distributed_option( diff --git a/ds4_metal.m b/ds4_metal.m index 4bb329c8d..0304e8686 100644 --- a/ds4_metal.m +++ b/ds4_metal.m @@ -35522,6 +35522,341 @@ int ds4_gpu_internal_qwen35_expert_pack_test(void) { return ok; } +typedef struct { + uint8_t qs[32]; + uint16_t scale_bf16; + uint16_t bias_bf16; +} ds4_gpu_internal_mlx_affine4_block; + +static void ds4_gpu_internal_mlx_affine4_fill_row( + uint8_t *row, + uint8_t value) { + const uint8_t packed = (uint8_t)(value | (uint8_t)(value << 4u)); + for (uint32_t group = 0; group < 4u; group++) { + ds4_gpu_internal_mlx_affine4_block *block = + (ds4_gpu_internal_mlx_affine4_block *)( + row + (uint64_t)group * + sizeof(ds4_gpu_internal_mlx_affine4_block)); + memset(block->qs, packed, sizeof(block->qs)); + block->scale_bf16 = 0x3f80u; + block->bias_bf16 = 0u; + } +} + +static int ds4_gpu_internal_qwen35_affine_resident_case( + const void *model_map, + uint64_t model_size, + uint64_t gate_offset, + uint64_t up_offset, + uint64_t down_offset, + uint64_t gate_expert_bytes, + uint64_t down_expert_bytes, + uint32_t n_tokens, + bool expected_mid_f16) { + enum { + N_EXPERT = 256, + N_ROUTE = 8, + IN_DIM = 256, + MID_DIM = 256, + OUT_DIM = 4, + GGML_TYPE_Q4_K = 12, + }; + const uint64_t row_bytes = + 4u * sizeof(ds4_gpu_internal_mlx_affine4_block); + const uint64_t pair_rows = (uint64_t)n_tokens * N_ROUTE; + const uint64_t mid_values = pair_rows * MID_DIM; + const uint64_t out_values = (uint64_t)n_tokens * OUT_DIM; + const uint64_t input_values_count = (uint64_t)n_tokens * IN_DIM; + const uint64_t route_values = (uint64_t)n_tokens * N_ROUTE; + ds4_gpu_tensor *out = NULL; + ds4_gpu_tensor *gate = NULL; + ds4_gpu_tensor *up = NULL; + ds4_gpu_tensor *mid = NULL; + ds4_gpu_tensor *experts = NULL; + ds4_gpu_tensor *selected = NULL; + ds4_gpu_tensor *weights = NULL; + ds4_gpu_tensor *x = NULL; + float *out_host = NULL; + float *gate_host = NULL; + float *up_host = NULL; + float *input_host = NULL; + int32_t *selected_host = NULL; + float *weights_host = NULL; + int ok = 0; + const char *failure = "allocation"; + + if (n_tokens == 0) return 0; + out = ds4_gpu_tensor_alloc(out_values * sizeof(float)); + gate = ds4_gpu_tensor_alloc(mid_values * sizeof(float)); + up = ds4_gpu_tensor_alloc(mid_values * sizeof(float)); + mid = ds4_gpu_tensor_alloc(mid_values * sizeof(float)); + experts = ds4_gpu_tensor_alloc( + pair_rows * OUT_DIM * sizeof(float)); + selected = ds4_gpu_tensor_alloc(route_values * sizeof(int32_t)); + weights = ds4_gpu_tensor_alloc(route_values * sizeof(float)); + x = ds4_gpu_tensor_alloc(input_values_count * sizeof(float)); + out_host = malloc((size_t)out_values * sizeof(float)); + gate_host = malloc((size_t)mid_values * sizeof(float)); + up_host = malloc((size_t)mid_values * sizeof(float)); + input_host = malloc((size_t)input_values_count * sizeof(float)); + selected_host = malloc((size_t)route_values * sizeof(int32_t)); + weights_host = malloc((size_t)route_values * sizeof(float)); + if (!out || !gate || !up || !mid || !experts || !selected || !weights || + !x || !out_host || !gate_host || !up_host || !input_host || + !selected_host || !weights_host) { + goto cleanup; + } + + for (uint64_t index = 0; index < input_values_count; index++) { + input_host[index] = 1.0f / (float)IN_DIM; + } + for (uint32_t token = 0; token < n_tokens; token++) { + for (uint32_t route = 0; route < N_ROUTE; route++) { + const uint64_t index = (uint64_t)token * N_ROUTE + route; + selected_host[index] = (int32_t)((token + route) % 15u); + weights_host[index] = 1.0f / (float)N_ROUTE; + } + } + failure = "upload"; + if (!ds4_gpu_tensor_write(x, 0, input_host, + input_values_count * sizeof(float)) || + !ds4_gpu_tensor_write(selected, 0, selected_host, + route_values * sizeof(int32_t)) || + !ds4_gpu_tensor_write(weights, 0, weights_host, + route_values * sizeof(float))) { + goto cleanup; + } + + bool mid_is_f16 = !expected_mid_f16; + int expert_group_used = -1; + failure = "execution"; + if (!ds4_gpu_qwen35_routed_moe_batch_select_tensor( + out, gate, up, mid, experts, + model_map, model_size, + gate_offset, up_offset, down_offset, + GGML_TYPE_Q4_K, GGML_TYPE_Q4_K, + gate_expert_bytes, row_bytes, + down_expert_bytes, row_bytes, + IN_DIM, MID_DIM, OUT_DIM, + selected, weights, N_EXPERT, N_ROUTE, 0.0f, x, 0, + n_tokens, &mid_is_f16, 0, &expert_group_used)) { + goto cleanup; + } + if (mid_is_f16 != expected_mid_f16) { + failure = "mid precision"; + goto cleanup; + } + if (expert_group_used != 0) { + failure = "unexpected expert schedule"; + goto cleanup; + } + /* The batch-32 fused pair+SwiGLU kernel does not materialize gate/up. + * Below that threshold, checking both projections proves the MV path + * decoded the affine payload instead of treating its bytes as Q4_K. */ + const bool inspect_projections = n_tokens < 32u; + failure = "readback"; + if (!ds4_gpu_tensor_read(out, 0, out_host, + out_values * sizeof(float)) || + (inspect_projections && + (!ds4_gpu_tensor_read(gate, 0, gate_host, + mid_values * sizeof(float)) || + !ds4_gpu_tensor_read(up, 0, up_host, + mid_values * sizeof(float))))) { + goto cleanup; + } + + const float silu_one = 1.0f / (1.0f + expf(-1.0f)); + const float projection_tolerance = 1.0e-4f; + for (uint32_t token = 0; token < n_tokens; token++) { + float expected_out = 0.0f; + for (uint32_t route = 0; route < N_ROUTE; route++) { + const uint64_t pair = (uint64_t)token * N_ROUTE + route; + const float value = (float)(selected_host[pair] + 1); + expected_out += weights_host[pair] * value * value; + for (uint32_t column = 0; + inspect_projections && column < MID_DIM; + column++) { + const uint64_t index = pair * MID_DIM + column; + if (!isfinite(gate_host[index]) || + !isfinite(up_host[index]) || + fabsf(gate_host[index] - 1.0f) > projection_tolerance || + fabsf(up_host[index] - value) > projection_tolerance) { + fprintf(stderr, + "ds4: resident MLX-affine batch %u projection " + "mismatch token=%u route=%u column=%u " + "gate=%.9g up=%.9g expected_up=%.9g\n", + n_tokens, token, route, column, + gate_host[index], up_host[index], value); + failure = "affine projection"; + goto cleanup; + } + } + } + expected_out *= (float)MID_DIM * silu_one; + const float output_tolerance = fmaxf(0.5f, fabsf(expected_out) * 0.0025f); + for (uint32_t row = 0; row < OUT_DIM; row++) { + const float actual = out_host[(uint64_t)token * OUT_DIM + row]; + if (!isfinite(actual) || + fabsf(actual - expected_out) > output_tolerance) { + fprintf(stderr, + "ds4: resident MLX-affine batch %u output mismatch " + "token=%u row=%u actual=%.9g expected=%.9g " + "tolerance=%.9g\n", + n_tokens, token, row, actual, expected_out, + output_tolerance); + failure = "output"; + goto cleanup; + } + } + } + ok = 1; + +cleanup: + ds4_gpu_tensor_free(out); + ds4_gpu_tensor_free(gate); + ds4_gpu_tensor_free(up); + ds4_gpu_tensor_free(mid); + ds4_gpu_tensor_free(experts); + ds4_gpu_tensor_free(selected); + ds4_gpu_tensor_free(weights); + ds4_gpu_tensor_free(x); + free(out_host); + free(gate_host); + free(up_host); + free(input_host); + free(selected_host); + free(weights_host); + if (!ok) { + fprintf(stderr, + "ds4: resident MLX-affine synthetic batch %u failed at %s\n", + n_tokens, failure); + } + return ok; +} + +int ds4_gpu_internal_qwen35_affine_resident_short_test(void) { + enum { + N_EXPERT = 256, + MID_DIM = 256, + OUT_DIM = 4, + }; + const uint64_t row_bytes = + 4u * sizeof(ds4_gpu_internal_mlx_affine4_block); + const uint64_t gate_expert_bytes = (uint64_t)MID_DIM * row_bytes; + const uint64_t down_expert_bytes = (uint64_t)OUT_DIM * row_bytes; + const uint64_t record_bytes = + gate_expert_bytes * 2u + down_expert_bytes; + const uint64_t component_gate_bytes = + (uint64_t)N_EXPERT * gate_expert_bytes; + const uint64_t component_down_bytes = + (uint64_t)N_EXPERT * down_expert_bytes; + const uint64_t payload_bytes = (uint64_t)N_EXPERT * record_bytes; + const uint64_t gate_offset = 0; + const uint64_t up_offset = component_gate_bytes; + const uint64_t down_offset = component_gate_bytes * 2u; + const uint64_t model_size = down_offset + component_down_bytes; + char path[] = "/tmp/ds4-affine-resident-short.XXXXXX"; + int fd = -1; + void *model_map = MAP_FAILED; + int installed = 0; + int ok = 0; + + if (sizeof(ds4_gpu_internal_mlx_affine4_block) != 36u || + row_bytes != 144u || payload_bytes != model_size || + (!g_initialized && !ds4_gpu_init()) || + g_qwen35_expert_pack.active) { + return 0; + } + ds4_gpu_set_glm_model(false); + ds4_gpu_set_quality(false); + ds4_gpu_set_ssd_streaming(false); + ds4_gpu_set_model_fd(-1); + + fd = mkstemp(path); + if (fd < 0) goto cleanup; + (void)unlink(path); + if (ftruncate(fd, (off_t)model_size) != 0) goto cleanup; + model_map = mmap(NULL, (size_t)model_size, + PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (model_map == MAP_FAILED) goto cleanup; + + uint8_t *payload = model_map; + for (uint32_t expert = 0; expert < N_EXPERT; expert++) { + const uint8_t value = (uint8_t)(expert % 15u + 1u); + uint8_t *record = payload + (uint64_t)expert * record_bytes; + uint8_t *gate = record; + uint8_t *up = gate + gate_expert_bytes; + uint8_t *down = up + gate_expert_bytes; + for (uint32_t row = 0; row < MID_DIM; row++) { + ds4_gpu_internal_mlx_affine4_fill_row( + gate + (uint64_t)row * row_bytes, 1u); + ds4_gpu_internal_mlx_affine4_fill_row( + up + (uint64_t)row * row_bytes, value); + } + for (uint32_t row = 0; row < OUT_DIM; row++) { + ds4_gpu_internal_mlx_affine4_fill_row( + down + (uint64_t)row * row_bytes, value); + } + } + if (msync(model_map, (size_t)model_size, MS_SYNC) != 0) goto cleanup; + + const ds4_gpu_expert_store_layer_v2 layer = { + .layer = 0, + .data_offset = 0, + .data_size = payload_bytes, + .record_bytes = record_bytes, + .component_offset = { + 0, + gate_expert_bytes, + gate_expert_bytes * 2u, + }, + .component_bytes = { + gate_expert_bytes, + gate_expert_bytes, + down_expert_bytes, + }, + }; + if (!ds4_gpu_expert_store_v2_install( + fd, model_size, 1, N_EXPERT, + DS4_EXPERT_STORE_STORAGE_MLX_AFFINE4, 64, &layer)) { + goto cleanup; + } + installed = 1; + if (!ds4_gpu_expert_store_v2_bind_layer( + 0, model_size, gate_offset, up_offset, down_offset) || + !ds4_gpu_expert_store_v2_enable_resident() || + !ds4_gpu_internal_qwen35_affine_resident_case( + model_map, model_size, + gate_offset, up_offset, down_offset, + gate_expert_bytes, down_expert_bytes, + 1, false) || + !ds4_gpu_internal_qwen35_affine_resident_case( + model_map, model_size, + gate_offset, up_offset, down_offset, + gate_expert_bytes, down_expert_bytes, + 31, false) || + !ds4_gpu_internal_qwen35_affine_resident_case( + model_map, model_size, + gate_offset, up_offset, down_offset, + gate_expert_bytes, down_expert_bytes, + 32, true)) { + goto cleanup; + } + ok = 1; + +cleanup: + if (installed) ds4_gpu_expert_store_v2_clear(); + if (model_map != MAP_FAILED) { + (void)munmap(model_map, (size_t)model_size); + } + if (fd >= 0) close(fd); + if (!ok) { + fprintf(stderr, + "ds4: resident MLX-affine short-prompt regression failed\n"); + } + return ok; +} + typedef struct { uint8_t scales[16]; uint8_t qs[64]; diff --git a/ds4_server.c b/ds4_server.c index 1d532c060..c1421db5a 100644 --- a/ds4_server.c +++ b/ds4_server.c @@ -13351,8 +13351,13 @@ static server_config parse_options(int argc, char **argv) { #ifndef DS4_SERVER_TEST int main(int argc, char **argv) { + hebrus_help_set_invocation(argv[0]); if (ds4_build_info_requested(argc, argv)) { - ds4_build_info_print(stdout); + ds4_build_info_print(stdout, argv[0]); + return 0; + } + if (ds4_capabilities_requested(argc, argv)) { + ds4_capabilities_print(stdout, DS4_EXECUTABLE_ROLE_SERVER, argv[0]); return 0; } signal(SIGPIPE, SIG_IGN); diff --git a/hebrus_identity.h b/hebrus_identity.h new file mode 100644 index 000000000..f3e6293be --- /dev/null +++ b/hebrus_identity.h @@ -0,0 +1,74 @@ +#ifndef HEBRUS_IDENTITY_H +#define HEBRUS_IDENTITY_H + +#include +#include +#include +#include + +static inline const char *hebrus_invocation_name(const char *argv0) { + const char *name = argv0 ? strrchr(argv0, '/') : NULL; + return name ? name + 1 : argv0; +} + +static inline bool hebrus_is_canonical_invocation(const char *argv0) { + const char *name = hebrus_invocation_name(argv0); + return name && + (strcmp(name, "hebrus") == 0 || strcmp(name, "hebrus-server") == 0 || + strcmp(name, "hebrus-agent") == 0 || strcmp(name, "hebrus-bench") == 0 || + strcmp(name, "hebrus-eval") == 0); +} + +static inline const char *hebrus_cli_prompt_for(const char *argv0) { + return hebrus_is_canonical_invocation(argv0) ? "hebrus> " : "ds4> "; +} + +static inline const char *hebrus_agent_command_for(const char *argv0) { + return hebrus_is_canonical_invocation(argv0) ? "hebrus-agent" : "ds4-agent"; +} + +static inline const char *hebrus_agent_brand_for(const char *argv0) { + return hebrus_is_canonical_invocation(argv0) ? "Hebrus" : "DwarfStar"; +} + +static inline const char *hebrus_agent_system_prompt_for(const char *argv0) { + (void)argv0; + return "You are a helpful coding assistant running inside ds4-agent."; +} + +static inline void hebrus_agent_format_prompt_for(const char *argv0, + char *buf, + size_t len) { + if (len == 0) return; + snprintf(buf, len, "%s> ", hebrus_agent_command_for(argv0)); +} + +static inline void hebrus_agent_format_welcome_banner_for(const char *argv0, + bool tty, + const char *ctx, + char *buf, + size_t len) { + if (len == 0) return; + if (tty && hebrus_is_canonical_invocation(argv0)) { + snprintf(buf, len, + "\x1b[1;97mHebrus\x1b[0m 🐋 Agent, context %s tokens\n\n", + ctx); + } else if (tty) { + snprintf(buf, len, + "\x1b[1;97mDwarf\x1b[1;94mStar\x1b[0m 🐋 Agent, context %s tokens\n\n", + ctx); + } else { + snprintf(buf, len, "%s Agent, context %s tokens\n\n", + hebrus_agent_brand_for(argv0), ctx); + } +} + +static inline const char *hebrus_eval_command_for(const char *argv0) { + return hebrus_is_canonical_invocation(argv0) ? "hebrus-eval" : "ds4-eval"; +} + +static inline const char *hebrus_bench_command_for(const char *argv0) { + return hebrus_is_canonical_invocation(argv0) ? "hebrus-bench" : "ds4-bench"; +} + +#endif diff --git a/tests/ds4_test.c b/tests/ds4_test.c index 5a3c84dab..b70f73f14 100644 --- a/tests/ds4_test.c +++ b/tests/ds4_test.c @@ -3534,6 +3534,7 @@ static void test_metal_kernel_group(void) { static void test_metal_qwen35_expert_pack(void) { TEST_ASSERT(ds4_gpu_internal_qwen35_expert_pack_test() != 0); + TEST_ASSERT(ds4_gpu_internal_qwen35_affine_resident_short_test() != 0); TEST_ASSERT(ds4_gpu_internal_expert_store_v2_kernel_test() != 0); } diff --git a/tests/qwen/README.md b/tests/qwen/README.md index 622335133..94e1debfd 100644 --- a/tests/qwen/README.md +++ b/tests/qwen/README.md @@ -109,9 +109,11 @@ Use this exact release identity for every Qwen model-backed gate: | Field | Value | | --- | --- | -| File | `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-Q4_K_S.gguf` | +| Publication state | Published at immutable revision `7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02` | +| File | `Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf` | | Bytes | `20,808,566,880` | -| SHA-256 | `d7c43a6388ec20e6fe5530850350f96fdb0ac37c5ce36d3e5f92b172c447f56b` | +| SHA-256 | `dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d` | +| Routed storage | MLX affine4, group size 64 | | Hugging Face | [`andreaborio/Qwen3.6-35B-A3B-DS4-GGUF`](https://huggingface.co/andreaborio/Qwen3.6-35B-A3B-DS4-GGUF) | The v2 file contains one routed-weight copy inside its checksummed embedded @@ -127,7 +129,7 @@ The converter's full byte-identity verifier is the artifact gate: python3 gguf-tools/ds4-expert-major.py inspect CANONICAL-QWEN.gguf python3 gguf-tools/ds4-expert-major.py verify \ CANONICAL-QWEN.gguf \ - Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-Q4_K_S.gguf + Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf ``` Never use `--skip-verify` for a release artifact. @@ -138,7 +140,7 @@ Resolve `QWEN_V2` to the absolute path whose size and complete SHA-256 match the table above. Normal startup is flag-free AUTO: ```sh -QWEN_V2=/absolute/path/to/Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-Q4_K_S.gguf +QWEN_V2=/absolute/path/to/Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf ./ds4 -m "$QWEN_V2" --ctx 8192 \ -n 32 --temp 0 \ -p 'Scrivi solo una breve funzione Python che somma due interi.' @@ -179,7 +181,8 @@ SSD mode must allocate the first 321-expert slab within its admitted budget and must not introduce swap. Keep warm page-cache evidence separate from cold device-I/O evidence, and never bypass a failed admission to obtain a number. -The full publication baseline and interpretation are recorded in -[`2026-07-20-qwen-expert-major-v2.md`](../../docs/benchmarks/2026-07-20-qwen-expert-major-v2.md). -That dated record is evidence; `QA_BEFORE_RELEASES.md` remains authoritative for +The current affine qualification and interpretation are recorded in +[`2026-07-21-qwen-unified-affine-auto-ssd.md`](../../docs/benchmarks/2026-07-21-qwen-unified-affine-auto-ssd.md). +That dated record is qualification evidence; the immutable repository revision +is the publication manifest. `QA_BEFORE_RELEASES.md` remains authoritative for the complete release gate. diff --git a/tests/test_benchmark_env_guard.sh b/tests/test_benchmark_env_guard.sh index ba88cf6fa..d1a479924 100644 --- a/tests/test_benchmark_env_guard.sh +++ b/tests/test_benchmark_env_guard.sh @@ -4,6 +4,11 @@ set -eu ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) RUNNER=$ROOT/speed-bench/run_m5_dsflash_arm.sh +if ! command -v zsh >/dev/null 2>&1; then + echo "benchmark-env-guard: PASS (zsh-only M5 runner skipped on this host)" + exit 0 +fi + fail() { echo "benchmark-env-guard: FAIL: $*" >&2 exit 1 diff --git a/tests/test_brand_asset.py b/tests/test_brand_asset.py new file mode 100644 index 000000000..041b833fb --- /dev/null +++ b/tests/test_brand_asset.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Freeze the maintainer-supplied Hebrus logo as an unmodified brand asset.""" + +from __future__ import annotations + +import hashlib +import pathlib +import struct + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +ASSET = ROOT / "docs" / "media" / "hebrus-logo.png" +README = ROOT / "README.md" +EXPECTED_SHA256 = "4be8949c73bd52e7abef58396dcd57f636165a8bb6cd6d536a600bcbf880594c" +EXPECTED_SIZE = (1254, 1254) + + +def main() -> None: + payload = ASSET.read_bytes() + digest = hashlib.sha256(payload).hexdigest() + if digest != EXPECTED_SHA256: + raise AssertionError( + f"{ASSET.relative_to(ROOT)} was modified: expected {EXPECTED_SHA256}, got {digest}" + ) + + if payload[:16] != b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR": + raise AssertionError("Hebrus logo is not a canonical PNG with an IHDR first chunk") + width, height, bit_depth, color_type = struct.unpack(">IIBB", payload[16:26]) + if (width, height) != EXPECTED_SIZE: + raise AssertionError(f"unexpected logo dimensions: {width}x{height}") + if (bit_depth, color_type) != (8, 6): + raise AssertionError("Hebrus logo must retain its original 8-bit RGBA representation") + + if "docs/media/hebrus-logo.png" not in README.read_text(encoding="utf-8"): + raise AssertionError("README does not use the canonical Hebrus logo") + + print(f"brand-asset: PASS ({width}x{height}, RGBA, {digest})") + + +if __name__ == "__main__": + main() diff --git a/tests/test_brand_boundary_audit.py b/tests/test_brand_boundary_audit.py new file mode 100644 index 000000000..c92ca7318 --- /dev/null +++ b/tests/test_brand_boundary_audit.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Regression tests for the monotonic legacy-brand inventory.""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +AUDIT = REPOSITORY_ROOT / "tools" / "brand_boundary_audit.py" +CATEGORIES = ( + "serialized/permanent", + "compatibility", + "historical-attribution", + "migration-pending", +) +LOCATION_ORDER = {"path": 0, "content": 1} +TOKEN_ORDER = {"ds4": 0, "DS4": 1, "DwarfStar": 2} +IDENTITY_CONTRACT = { + "brand": "Hebrus", + "canonical_commands": [ + "hebrus", + "hebrus-server", + "hebrus-agent", + "hebrus-bench", + "hebrus-eval", + ], + "command_aliases": { + "ds4": "hebrus", + "ds4-server": "hebrus-server", + "ds4-agent": "hebrus-agent", + "ds4-bench": "hebrus-bench", + "ds4-eval": "hebrus-eval", + }, + "accepted_engine_ids": ["hebrus", "ds4"], + "environment_prefixes": { + "preserved": ["DS4_"], + "deferred": ["HEBRUS_"], + }, + "permanent_literals": ["ds4.expert_major.v1", "ds4.expert_major.v2"], + "repositories": { + "historical_origin": "antirez/ds4", + "pre_rename": "andreaborio/ds4", + }, +} + + +def manifest_document(entries: list[dict[str, object]]) -> dict[str, object]: + ordered = sorted( + entries, + key=lambda entry: ( + entry["path"], + LOCATION_ORDER[entry["location"]], + TOKEN_ORDER[entry["token"]], + ), + ) + return { + "schema_version": 1, + "identity_contract": IDENTITY_CONTRACT, + "scope": { + "tokens": ["ds4", "DS4", "DwarfStar"], + "locations": ["path", "content"], + "excluded_paths": ["tools/brand_boundary.json"], + }, + "category_definitions": { + category: f"Test definition for {category}." for category in CATEGORIES + }, + "refresh_policy": { + "reductions": "python3 tools/brand_boundary_audit.py --refresh", + "existing_increase": ( + "python3 tools/brand_boundary_audit.py --refresh " + "--accept-increase PATH:LOCATION:TOKEN" + ), + "new_group": ( + "python3 tools/brand_boundary_audit.py --refresh " + "--classify PATH:LOCATION:TOKEN=CLASSIFICATION" + ), + }, + "entries": ordered, + } + + +def entry( + path: str, + location: str, + token: str, + maximum: int, + classification: str = "migration-pending", +) -> dict[str, object]: + return { + "path": path, + "location": location, + "token": token, + "classification": classification, + "maximum": maximum, + } + + +class Fixture: + def __init__(self) -> None: + self.temporary = tempfile.TemporaryDirectory(prefix="brand-boundary-") + self.root = Path(self.temporary.name) + self.manifest = self.root / "tools" / "brand_boundary.json" + self.manifest.parent.mkdir() + subprocess.run(["git", "init", "-q"], cwd=self.root, check=True) + + def close(self) -> None: + self.temporary.cleanup() + + def write(self, relative: str, content: str) -> None: + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + def track(self, *relative: str) -> None: + subprocess.run(["git", "add", "--", *relative], cwd=self.root, check=True) + + def write_manifest(self, entries: list[dict[str, object]]) -> None: + self.manifest.write_text( + json.dumps(manifest_document(entries), indent=2) + "\n", + encoding="utf-8", + ) + + def run(self, *arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(AUDIT), + "--root", + str(self.root), + "--manifest", + "tools/brand_boundary.json", + *arguments, + ], + text=True, + capture_output=True, + check=False, + ) + + +class BrandBoundaryAuditTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture() + + def tearDown(self) -> None: + self.fixture.close() + + def seed_one_token(self) -> None: + self.fixture.write("engine.c", "const char *engine = \"ds4\";\n") + self.fixture.track("engine.c") + self.fixture.write_manifest([entry("engine.c", "content", "ds4", 1)]) + + def test_new_tracked_file_requires_exact_classification(self) -> None: + self.seed_one_token() + self.fixture.write("ds4_notes.txt", "brand bridge\n") + self.fixture.track("ds4_notes.txt") + + checked = self.fixture.run("--check") + self.assertEqual(checked.returncode, 1, checked.stderr) + self.assertIn( + "unclassified brand token group: ds4_notes.txt:path:ds4", + checked.stderr, + ) + + refused = self.fixture.run("--refresh") + self.assertEqual(refused.returncode, 1, refused.stderr) + + refreshed = self.fixture.run( + "--refresh", + "--classify", + "ds4_notes.txt:path:ds4=migration-pending", + ) + self.assertEqual(refreshed.returncode, 0, refreshed.stderr) + self.assertEqual(self.fixture.run("--check").returncode, 0) + + def test_new_token_in_existing_file_is_unclassified(self) -> None: + self.seed_one_token() + self.fixture.write("engine.c", "const char *engine = \"ds4 DS4\";\n") + + checked = self.fixture.run("--check") + self.assertEqual(checked.returncode, 1, checked.stderr) + self.assertIn( + "unclassified brand token group: engine.c:content:DS4", + checked.stderr, + ) + + def test_increase_fails_until_exactly_authorized(self) -> None: + self.seed_one_token() + self.fixture.write("engine.c", "const char *engine = \"ds4 ds4\";\n") + + checked = self.fixture.run("--check") + self.assertEqual(checked.returncode, 1, checked.stderr) + self.assertIn("engine.c:content:ds4 2 > 1", checked.stderr) + + original = self.fixture.manifest.read_bytes() + refused = self.fixture.run("--refresh") + self.assertEqual(refused.returncode, 1, refused.stderr) + self.assertEqual(self.fixture.manifest.read_bytes(), original) + + refreshed = self.fixture.run( + "--refresh", "--accept-increase", "engine.c:content:ds4" + ) + self.assertEqual(refreshed.returncode, 0, refreshed.stderr) + self.assertEqual(self.fixture.run("--check").returncode, 0) + document = json.loads(self.fixture.manifest.read_text(encoding="utf-8")) + self.assertEqual(document["entries"][0]["maximum"], 2) + + def test_reduction_passes_and_refresh_tightens_deterministically(self) -> None: + self.fixture.write("engine.c", "const char *engine = \"ds4\";\n") + self.fixture.track("engine.c") + self.fixture.write_manifest([entry("engine.c", "content", "ds4", 2)]) + + checked = self.fixture.run("--check") + self.assertEqual(checked.returncode, 0, checked.stderr) + self.assertIn("1 reductions", checked.stdout) + + refreshed = self.fixture.run("--refresh") + self.assertEqual(refreshed.returncode, 0, refreshed.stderr) + first = self.fixture.manifest.read_bytes() + self.assertEqual(self.fixture.run("--refresh").returncode, 0) + self.assertEqual(self.fixture.manifest.read_bytes(), first) + + self.fixture.write("engine.c", "const char *engine = \"ds4 ds4\";\n") + increased = self.fixture.run("--check") + self.assertEqual(increased.returncode, 1, increased.stderr) + self.assertIn("2 > 1", increased.stderr) + + def test_invalid_manifest_fails_closed(self) -> None: + self.seed_one_token() + document = json.loads(self.fixture.manifest.read_text(encoding="utf-8")) + document["entries"][0]["path"] = "**/*.c" + self.fixture.manifest.write_text( + json.dumps(document, indent=2) + "\n", encoding="utf-8" + ) + + checked = self.fixture.run("--check") + self.assertEqual(checked.returncode, 2, checked.stderr) + self.assertIn("must be an explicit normalized path", checked.stderr) + + def test_identity_contract_is_exact(self) -> None: + self.seed_one_token() + document = json.loads(self.fixture.manifest.read_text(encoding="utf-8")) + document["identity_contract"]["command_aliases"]["ds4-server"] = "other" + self.fixture.manifest.write_text( + json.dumps(document, indent=2) + "\n", encoding="utf-8" + ) + + checked = self.fixture.run("--check") + self.assertEqual(checked.returncode, 2, checked.stderr) + self.assertIn("identity_contract must exactly match", checked.stderr) + + def test_unused_refresh_authorization_fails_closed(self) -> None: + self.seed_one_token() + refreshed = self.fixture.run( + "--refresh", "--accept-increase", "engine.c:content:ds4" + ) + self.assertEqual(refreshed.returncode, 2, refreshed.stderr) + self.assertIn("does not match a current increase", refreshed.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_build_isolation.sh b/tests/test_build_isolation.sh index cdce7ac49..b23876bf7 100755 --- a/tests/test_build_isolation.sh +++ b/tests/test_build_isolation.sh @@ -5,7 +5,7 @@ ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) ARCH=$(uname -m) METAL_REL="build/metal-${ARCH}/bin" CPU_REL="build/cpu-${ARCH}/bin" -PROGRAMS="ds4 ds4-server ds4-bench ds4-eval ds4-agent" +PAIRS="hebrus:ds4 hebrus-server:ds4-server hebrus-bench:ds4-bench hebrus-eval:ds4-eval hebrus-agent:ds4-agent" MAKE_BIN=${MAKE:-make} fail() { @@ -43,16 +43,31 @@ cd "$ROOT" "$MAKE_BIN" metal root_state="" -for name in $PROGRAMS; do - [ -L "$name" ] || fail "$name is not a published Metal symlink" - [ "$(readlink "$name")" = "$METAL_REL/$name" ] || \ - fail "$name does not point at $METAL_REL/$name" - [ -x "$METAL_REL/$name" ] || fail "missing $METAL_REL/$name" - assert_metal_binary "$name" - assert_build_info "$name" metal - root_state="$root_state $name:$(stat -f '%i:%m' "$name")" +for pair in $PAIRS; do + canonical=${pair%%:*} + legacy=${pair#*:} + for name in "$canonical" "$legacy"; do + [ -L "$name" ] || fail "$name is not a published Metal symlink" + [ "$(readlink "$name")" = "$METAL_REL/$canonical" ] || \ + fail "$name does not point at $METAL_REL/$canonical" + assert_metal_binary "$name" + assert_build_info "$name" metal + root_state="$root_state $name:$(stat -f '%i:%m' "$name")" + done + [ -x "$METAL_REL/$canonical" ] || fail "missing $METAL_REL/$canonical" + [ ! -L "$METAL_REL/$canonical" ] || \ + fail "$METAL_REL/$canonical is not the canonical binary" + [ -L "$METAL_REL/$legacy" ] || fail "$METAL_REL/$legacy is not an alias" + [ "$(readlink "$METAL_REL/$legacy")" = "$canonical" ] || \ + fail "$METAL_REL/$legacy does not point at $canonical" done +python3 tests/test_capabilities.py --bin-dir "$METAL_REL" --backend metal +python3 tests/test_command_aliases.py --bin-dir "$METAL_REL" \ + --backend metal --layout profile +python3 tests/test_command_aliases.py --bin-dir . \ + --backend metal --layout published + [ -f "build/metal-${ARCH}/obj/ds4.o" ] || fail "missing Metal core object" [ -f "build/metal-${ARCH}/obj/ds4_metal.o" ] || fail "missing Metal backend object" @@ -62,23 +77,40 @@ done [ ! -e "build/cpu-${ARCH}/obj/ds4_metal.o" ] || fail "CPU profile contains a Metal object" after_cpu_state="" -for name in $PROGRAMS; do - [ -x "$CPU_REL/$name" ] || fail "missing $CPU_REL/$name" - assert_cpu_binary "$CPU_REL/$name" - assert_build_info "$CPU_REL/$name" cpu - assert_metal_binary "$name" - after_cpu_state="$after_cpu_state $name:$(stat -f '%i:%m' "$name")" +for pair in $PAIRS; do + canonical=${pair%%:*} + legacy=${pair#*:} + [ -x "$CPU_REL/$canonical" ] || fail "missing $CPU_REL/$canonical" + [ ! -L "$CPU_REL/$canonical" ] || \ + fail "$CPU_REL/$canonical is not the canonical binary" + [ -L "$CPU_REL/$legacy" ] || fail "$CPU_REL/$legacy is not an alias" + [ "$(readlink "$CPU_REL/$legacy")" = "$canonical" ] || \ + fail "$CPU_REL/$legacy does not point at $canonical" + for name in "$canonical" "$legacy"; do + assert_cpu_binary "$CPU_REL/$name" + assert_build_info "$CPU_REL/$name" cpu + assert_metal_binary "$name" + after_cpu_state="$after_cpu_state $name:$(stat -f '%i:%m' "$name")" + done done +python3 tests/test_capabilities.py --bin-dir "$CPU_REL" --backend cpu +python3 tests/test_command_aliases.py --bin-dir "$CPU_REL" \ + --backend cpu --layout profile + [ "$root_state" = "$after_cpu_state" ] || \ fail "make cpu modified one or more published Metal root links" "$MAKE_BIN" metal -for name in $PROGRAMS; do - [ "$(readlink "$name")" = "$METAL_REL/$name" ] || \ - fail "final $name does not point at the Metal profile" - assert_metal_binary "$name" +for pair in $PAIRS; do + canonical=${pair%%:*} + legacy=${pair#*:} + for name in "$canonical" "$legacy"; do + [ "$(readlink "$name")" = "$METAL_REL/$canonical" ] || \ + fail "final $name does not point at $METAL_REL/$canonical" + assert_metal_binary "$name" + done done echo "build-isolation: PASS (metal-${ARCH} -> cpu-${ARCH} -> metal-${ARCH})" diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py new file mode 100644 index 000000000..ae2f7277e --- /dev/null +++ b/tests/test_capabilities.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Validate the public, model-free engine capability contract.""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import subprocess +import sys + + +ROLES = { + "hebrus": "cli", + "hebrus-server": "server", + "hebrus-agent": "agent", + "hebrus-bench": "bench", + "hebrus-eval": "eval", + "ds4": "cli", + "ds4-server": "server", + "ds4-agent": "agent", + "ds4-bench": "bench", + "ds4-eval": "eval", +} + +ROOT_KEYS = { + "schema_version", + "engine_id", + "build_git_sha", + "backend", + "executable_role", + "model_families", + "expert_major", +} + + +def fail(message: str) -> None: + raise AssertionError(message) + + +def run(binary: pathlib.Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [str(binary), *args], + check=False, + capture_output=True, + text=True, + ) + + +def require_exact_int(value: object, expected: int, label: str) -> None: + if type(value) is not int or value != expected: + fail(f"{label}: expected integer {expected}, got {value!r}") + + +def validate(binary: pathlib.Path, backend: str, role: str) -> dict[str, object]: + result = run(binary, "--capabilities=json") + if result.returncode != 0: + fail(f"{binary}: --capabilities=json exited {result.returncode}: {result.stderr}") + if result.stderr: + fail(f"{binary}: --capabilities=json wrote to stderr: {result.stderr!r}") + try: + document = json.loads(result.stdout) + except json.JSONDecodeError as exc: + fail(f"{binary}: invalid capability JSON: {exc}") + + if type(document) is not dict: + fail(f"{binary}: capability document is not an object") + if set(document) != ROOT_KEYS: + fail(f"{binary}: unexpected root keys: {set(document) ^ ROOT_KEYS}") + require_exact_int(document["schema_version"], 1, f"{binary}: schema_version") + expected_engine_id = "hebrus" if binary.name.startswith("hebrus") else "ds4" + if document["engine_id"] != expected_engine_id or type(document["engine_id"]) is not str: + fail(f"{binary}: expected engine_id {expected_engine_id!r}") + if type(document["build_git_sha"]) is not str or not document["build_git_sha"]: + fail(f"{binary}: invalid build_git_sha") + if document["backend"] != backend or type(document["backend"]) is not str: + fail(f"{binary}: expected backend {backend!r}, got {document['backend']!r}") + if document["executable_role"] != role or type(document["executable_role"]) is not str: + fail(f"{binary}: expected role {role!r}, got {document['executable_role']!r}") + expected_families = ["deepseek4", "glm-dsa", "qwen35moe"] + if document["model_families"] != expected_families: + fail(f"{binary}: invalid model_families") + if any(type(item) is not str for item in document["model_families"]): + fail(f"{binary}: model_families entries must be strings") + + expert = document["expert_major"] + if type(expert) is not dict or set(expert) != {"version", "tensor", "storage_formats"}: + fail(f"{binary}: invalid expert_major object") + require_exact_int(expert["version"], 2, f"{binary}: expert_major.version") + if expert["tensor"] != "ds4.expert_major.v2" or type(expert["tensor"]) is not str: + fail(f"{binary}: invalid ExpertMajor tensor") + expected_storage = [ + {"id": "ggml", "wire_value": 0, "group_sizes": []}, + {"id": "mlx-affine4", "wire_value": 1, "group_sizes": [64]}, + ] + if expert["storage_formats"] != expected_storage: + fail(f"{binary}: invalid ExpertMajor storage formats") + for index, storage in enumerate(expert["storage_formats"]): + if type(storage) is not dict: + fail(f"{binary}: storage format {index} is not an object") + if type(storage["id"]) is not str: + fail(f"{binary}: storage format {index} id is not a string") + require_exact_int(storage["wire_value"], index, f"{binary}: storage wire_value") + if type(storage["group_sizes"]) is not list: + fail(f"{binary}: storage group_sizes is not an array") + if any(type(size) is not int for size in storage["group_sizes"]): + fail(f"{binary}: storage group_sizes must contain integers") + + build_info = run(binary, "--build-info") + if build_info.returncode != 0: + fail(f"{binary}: --build-info exited {build_info.returncode}") + git_lines = [line[9:] for line in build_info.stdout.splitlines() if line.startswith("git: ")] + if git_lines != [document["build_git_sha"]]: + fail(f"{binary}: capability SHA differs from --build-info") + + help_result = run(binary, "--help") + if help_result.returncode != 0 or "--capabilities=json" not in help_result.stdout: + fail(f"{binary}: help does not advertise the capability contract") + + invalid = run(binary, "--capabilities") + if invalid.returncode != 2: + fail(f"{binary}: bare --capabilities exited {invalid.returncode} instead of 2") + if invalid.stdout: + fail(f"{binary}: bare --capabilities unexpectedly wrote to stdout") + if "unknown option: --capabilities" not in invalid.stderr: + fail(f"{binary}: bare --capabilities did not take the invalid-option path") + + repeated = run(binary, "--capabilities=json") + if repeated.stdout != result.stdout or repeated.stderr != result.stderr: + fail(f"{binary}: capability output is not deterministic") + return document + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--bin-dir", required=True, type=pathlib.Path) + parser.add_argument("--backend", required=True, choices=("metal", "cpu")) + args = parser.parse_args() + + shared: dict[str, object] | None = None + for program, role in ROLES.items(): + binary = (args.bin_dir / program).absolute() + if not binary.is_file(): + fail(f"missing executable: {binary}") + document = validate(binary, args.backend, role) + common = { + key: value + for key, value in document.items() + if key not in {"engine_id", "executable_role"} + } + if shared is None: + shared = common + elif common != shared: + fail(f"{binary}: shared capability fields differ across executables") + + print(f"capabilities: PASS ({args.backend}, {len(ROLES)} executables)") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except AssertionError as exc: + print(f"capabilities: FAIL: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/tests/test_command_aliases.py b/tests/test_command_aliases.py new file mode 100644 index 000000000..41ea61b79 --- /dev/null +++ b/tests/test_command_aliases.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Prove canonical Hebrus commands and DS4 aliases are one executable surface.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import subprocess +import sys + + +PAIRS = ( + ("hebrus", "ds4", "cli"), + ("hebrus-server", "ds4-server", "server"), + ("hebrus-agent", "ds4-agent", "agent"), + ("hebrus-bench", "ds4-bench", "bench"), + ("hebrus-eval", "ds4-eval", "eval"), +) + +PARITY_CASES = ( + ("--capabilities=json",), + ("--build-info",), + ("--help",), + ("--help", "all"), + ("--capabilities",), + ("--role",), + ("--command-alias-invalid-test",), +) + + +def fail(message: str) -> None: + raise AssertionError(message) + + +def run(binary: pathlib.Path, args: tuple[str, ...]) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + [os.fsencode(binary), *(os.fsencode(arg) for arg in args)], + check=False, + capture_output=True, + ) + + +def sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fp: + for chunk in iter(lambda: fp.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def validate_layout(canonical: pathlib.Path, legacy: pathlib.Path, layout: str) -> None: + if not canonical.is_file() or not legacy.is_file(): + fail(f"missing command pair: {canonical}, {legacy}") + if not legacy.is_symlink(): + fail(f"legacy command is not a symlink: {legacy}") + if layout == "profile": + if canonical.is_symlink(): + fail(f"canonical profile command is unexpectedly a symlink: {canonical}") + if os.readlink(legacy) != canonical.name: + fail(f"{legacy} does not point directly to {canonical.name}") + elif not canonical.is_symlink(): + fail(f"published canonical command is not a symlink: {canonical}") + if not os.path.samefile(canonical, legacy): + fail(f"command pair does not resolve to one file: {canonical}, {legacy}") + if sha256(canonical) != sha256(legacy): + fail(f"command pair has different binary content: {canonical}, {legacy}") + + +def validate_pair( + canonical: pathlib.Path, + legacy: pathlib.Path, + role: str, + backend: str, +) -> None: + capabilities = run(canonical, ("--capabilities=json",)) + if capabilities.returncode != 0 or capabilities.stderr: + fail(f"{canonical}: capability invocation failed") + document = json.loads(capabilities.stdout) + if document.get("engine_id") != "hebrus": + fail(f"{canonical}: canonical engine_id is not hebrus") + legacy_document = json.loads(run(legacy, ("--capabilities=json",)).stdout) + if legacy_document.get("engine_id") != "ds4": + fail(f"{legacy}: compatibility engine_id is not ds4") + if document.get("backend") != backend: + fail(f"{canonical}: expected backend {backend}") + if document.get("executable_role") != role: + fail(f"{canonical}: expected executable role {role}") + + canonical_help_size = 0 + legacy_help_size = 0 + for args in PARITY_CASES: + canonical_result = run(canonical, args) + legacy_result = run(legacy, args) + canonical_name = os.fsencode(canonical.name) + legacy_name = os.fsencode(legacy.name) + + def normalize(data: bytes) -> bytes: + return ( + data.replace(canonical_name, legacy_name) + .replace(b'"engine_id": "hebrus"', b'"engine_id": "ds4"') + .replace(b"hebrus build", b"ds4 build") + ) + + canonical_record = ( + canonical_result.returncode, + normalize(canonical_result.stdout), + normalize(canonical_result.stderr), + ) + legacy_record = ( + legacy_result.returncode, + legacy_result.stdout, + legacy_result.stderr, + ) + if canonical_record != legacy_record: + fail(f"{canonical.name}/{legacy.name} differ for {' '.join(args)}") + + if args[0] == "--help": + if canonical_result.returncode != 0 or canonical_result.stderr: + fail(f"{canonical.name}: help invocation failed") + if legacy_result.returncode != 0 or legacy_result.stderr: + fail(f"{legacy.name}: help invocation failed") + if not canonical_result.stdout.startswith(canonical_name + b"\n"): + fail(f"{canonical.name}: help does not use canonical invocation name") + if not legacy_result.stdout.startswith(legacy_name + b"\n"): + fail(f"{legacy.name}: help does not preserve legacy invocation name") + if args == ("--help",): + canonical_help_size = len(canonical_result.stdout) + legacy_help_size = len(legacy_result.stdout) + elif ( + len(canonical_result.stdout) <= canonical_help_size + or len(legacy_result.stdout) <= legacy_help_size + ): + fail(f"{canonical.name}/{legacy.name}: --help all is not full help") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--bin-dir", required=True, type=pathlib.Path) + parser.add_argument("--backend", required=True, choices=("metal", "cpu")) + parser.add_argument("--layout", required=True, choices=("profile", "published")) + args = parser.parse_args() + + bin_dir = args.bin_dir.absolute() + resolved_targets: set[pathlib.Path] = set() + for canonical_name, legacy_name, role in PAIRS: + canonical = bin_dir / canonical_name + legacy = bin_dir / legacy_name + validate_layout(canonical, legacy, args.layout) + validate_pair(canonical, legacy, role, args.backend) + target = canonical.resolve() + if target in resolved_targets: + fail(f"roles unexpectedly share one executable: {target}") + resolved_targets.add(target) + + print(f"command-aliases: PASS ({args.backend}, {len(PAIRS)} pairs, {args.layout})") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (AssertionError, json.JSONDecodeError) as exc: + print(f"command-aliases: FAIL: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/tests/test_download_model.sh b/tests/test_download_model.sh index 68e504294..fcf2349bb 100755 --- a/tests/test_download_model.sh +++ b/tests/test_download_model.sh @@ -3,6 +3,7 @@ set -eu ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) SCRIPT="$ROOT/download_model.sh" +QWEN_CONTRACT="$ROOT/docs/contracts/qwen-release.json" TMP_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/ds4-download-model.XXXXXX") trap 'rm -rf "$TMP_ROOT"' EXIT HUP INT TERM HOME="$TMP_ROOT/home" @@ -10,6 +11,32 @@ export HOME unset HF_TOKEN mkdir -p "$HOME" +QWEN_EXPECTED="$TMP_ROOT/qwen-release-contract.sh" +python3 - "$QWEN_CONTRACT" >"$QWEN_EXPECTED" <<'PY' +import json +import shlex +import sys + +with open(sys.argv[1], encoding="utf-8") as handle: + contract = json.load(handle) +published = contract["publishedArtifact"] +negative = contract["negativeArtifact"] +values = { + "QWEN_STATUS": published["status"], + "QWEN_REPOSITORY": contract["repository"], + "QWEN_FILE": published["filename"], + "QWEN_BYTES": str(published["bytes"]), + "QWEN_SHA256": published["sha256"], + "QWEN_REVISION": published["revision"], + "QWEN_RUNTIME_COMMIT": published["runtimeCommit"], + "QWEN_NEGATIVE_FILE": negative["filename"], + "QWEN_NEGATIVE_SHA256": negative["sha256"], +} +for name, value in values.items(): + print(f"{name}={shlex.quote(value)}") +PY +. "$QWEN_EXPECTED" + fail() { echo "download-model: FAIL: $*" >&2 exit 1 @@ -62,9 +89,18 @@ assert_exact_line 'RUNTIME_DEEPSEEK_BYTES=86720114272' assert_exact_line 'RUNTIME_DEEPSEEK_SHA256="8378080263eb9224f7228d72e2afa4ac3cf74a116023fdec2c596ff228a33e3f"' assert_exact_line 'RUNTIME_GLM_BYTES=262147193504' assert_exact_line 'RUNTIME_GLM_SHA256="7f5017e3076e706c78f2a5322b035a9e2f6519c65ff5b6be8b2d91aeff61505d"' -assert_exact_line 'RUNTIME_QWEN_BYTES=20808566880' -assert_exact_line 'RUNTIME_QWEN_SHA256="dd17266185833a9f05531ce366fd7284ddca1ed64aa3dcf06e321e8c72c9ea3d"' -assert_exact_line 'RUNTIME_QWEN_REVISION="7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02"' +assert_exact_line "RUNTIME_QWEN_STATUS=\"$QWEN_STATUS\"" +assert_exact_line "RUNTIME_QWEN_REPO=\"$QWEN_REPOSITORY\"" +assert_exact_line "RUNTIME_QWEN_FILE=\"$QWEN_FILE\"" +assert_exact_line "RUNTIME_QWEN_BYTES=$QWEN_BYTES" +assert_exact_line "RUNTIME_QWEN_SHA256=\"$QWEN_SHA256\"" +assert_exact_line "RUNTIME_QWEN_REVISION=\"$QWEN_REVISION\"" +assert_exact_line "RUNTIME_QWEN_MIN_RUNTIME_COMMIT=\"$QWEN_RUNTIME_COMMIT\"" +[ "$QWEN_STATUS" = published ] || fail "canonical Qwen artifact is not published" +if grep -Fq -- "$QWEN_NEGATIVE_FILE" "$SCRIPT" || + grep -Fq -- "$QWEN_NEGATIVE_SHA256" "$SCRIPT"; then + fail "negative-only Qwen artifact is exposed by the downloader" +fi assert_case_assignment deepseek-v2 'MODEL_REPO=$RUNTIME_DEEPSEEK_REPO' assert_case_assignment deepseek-v2 'MODEL_FILE=$RUNTIME_DEEPSEEK_FILE' assert_case_assignment deepseek-v2 'MODEL_BYTES=$RUNTIME_DEEPSEEK_BYTES' @@ -188,9 +224,9 @@ run_runtime_success \ ds4-v0.2.0 run_runtime_success \ qwen-v2 \ - andreaborio/Qwen3.6-35B-A3B-DS4-GGUF \ - Qwen3.6-35B-A3B-DS4-ExpertMajor-v2-MLX-Affine4-G64.gguf \ - 7bf9c3f7f6136aeb2599d75ee61c0cc2f18e2b02 + "$QWEN_REPOSITORY" \ + "$QWEN_FILE" \ + "$QWEN_REVISION" CORRUPT_DIR="$TMP_ROOT/corrupt-existing" CORRUPT_FILE="$CORRUPT_DIR/GLM-5.2-DS4-ExpertMajor-v2-Q2_K.gguf" diff --git a/tests/test_install.sh b/tests/test_install.sh new file mode 100755 index 000000000..ca38ac6fa --- /dev/null +++ b/tests/test_install.sh @@ -0,0 +1,83 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +MAKE_BIN=${MAKE:-make} +BACKEND=${HEBRUS_INSTALL_BACKEND:?missing HEBRUS_INSTALL_BACKEND} +TMP_BASE=$(mktemp -d "${TMPDIR:-/tmp}/hebrus-install.XXXXXX") +STAGE="$TMP_BASE/stage root" +PREFIX_ONE=/portable-prefix +PREFIX_TWO=/unused-prefix +BINDIR_TWO=/custom/hebrus-bin +PROGRAMS="hebrus hebrus-server hebrus-bench hebrus-eval hebrus-agent ds4 ds4-server ds4-bench ds4-eval ds4-agent" +PAIRS="hebrus:ds4 hebrus-server:ds4-server hebrus-bench:ds4-bench hebrus-eval:ds4-eval hebrus-agent:ds4-agent" + +cleanup() { + rm -rf "$TMP_BASE" +} +trap cleanup 0 1 2 3 15 + +fail() { + echo "install-layout: FAIL: $*" >&2 + exit 1 +} + +assert_removed() { + bin_dir=$1 + for name in $PROGRAMS; do + path="$bin_dir/$name" + if [ -e "$path" ] || [ -L "$path" ]; then + fail "uninstall left $path" + fi + done +} + +assert_layout() { + bin_dir=$1 + count=$(find "$bin_dir" -mindepth 1 -maxdepth 1 -print | wc -l | tr -d ' ') + [ "$count" = 10 ] || fail "$bin_dir contains $count entries instead of 10" + for pair in $PAIRS; do + canonical=${pair%%:*} + legacy=${pair#*:} + [ -x "$bin_dir/$canonical" ] || fail "missing executable $canonical" + [ ! -L "$bin_dir/$canonical" ] || fail "$canonical is a symlink" + [ -L "$bin_dir/$legacy" ] || fail "$legacy is not a symlink" + [ "$(readlink "$bin_dir/$legacy")" = "$canonical" ] || \ + fail "$legacy is not a relative alias to $canonical" + done +} + +mkdir -p "$STAGE" +cd "$ROOT" + +"$MAKE_BIN" install DESTDIR="$STAGE" PREFIX="$PREFIX_ONE" +BIN_ONE="$STAGE$PREFIX_ONE/bin" +assert_layout "$BIN_ONE" +[ ! -e "$STAGE/usr/local/bin/hebrus" ] || fail "install ignored PREFIX" +python3 tests/test_capabilities.py --bin-dir "$BIN_ONE" --backend "$BACKEND" +python3 tests/test_command_aliases.py --bin-dir "$BIN_ONE" \ + --backend "$BACKEND" --layout profile + +for name in hebrus hebrus-server hebrus-bench hebrus-eval hebrus-agent; do + capabilities=$("$BIN_ONE/$name" --capabilities=json) + case "$capabilities" in + *"$ROOT"*|*"$STAGE"*) fail "$name exposes a checkout or install path" ;; + esac +done + +printf '%s\n' keep > "$BIN_ONE/keep-me" +"$MAKE_BIN" uninstall DESTDIR="$STAGE" PREFIX="$PREFIX_ONE" +assert_removed "$BIN_ONE" +[ "$(cat "$BIN_ONE/keep-me")" = keep ] || fail "uninstall removed an unrelated file" +"$MAKE_BIN" uninstall DESTDIR="$STAGE" PREFIX="$PREFIX_ONE" + +"$MAKE_BIN" install DESTDIR="$STAGE" PREFIX="$PREFIX_TWO" BINDIR="$BINDIR_TWO" +BIN_TWO="$STAGE$BINDIR_TWO" +assert_layout "$BIN_TWO" +[ ! -e "$STAGE$PREFIX_TWO/bin/hebrus" ] || fail "install ignored BINDIR" +python3 tests/test_command_aliases.py --bin-dir "$BIN_TWO" \ + --backend "$BACKEND" --layout profile +"$MAKE_BIN" uninstall DESTDIR="$STAGE" PREFIX="$PREFIX_TWO" BINDIR="$BINDIR_TWO" +assert_removed "$BIN_TWO" + +echo "install-layout: PASS ($BACKEND, DESTDIR/PREFIX/BINDIR, explicit uninstall)" diff --git a/tests/test_qwen_metadata.py b/tests/test_qwen_metadata.py index e53188cd4..51ae3ac28 100644 --- a/tests/test_qwen_metadata.py +++ b/tests/test_qwen_metadata.py @@ -400,6 +400,24 @@ def require(condition: bool, message: str, result: subprocess.CompletedProcess[s ) +def compiled_backend(binary: Path) -> str: + result = subprocess.run( + [str(binary), "--capabilities=json"], + text=True, + capture_output=True, + check=False, + ) + require(result.returncode == 0, "capability query failed", result) + try: + document = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise AssertionError(f"capability query returned invalid JSON: {exc}") from exc + backend = document.get("backend") + if backend not in {"cpu", "metal"}: + raise AssertionError(f"capability query returned invalid backend: {backend!r}") + return backend + + def check_frozen_reference() -> None: template = QWEN_CHAT_TEMPLATE_PATH.read_bytes() assert len(template) == 7764 @@ -536,6 +554,7 @@ def main() -> int: print(f"usage: {sys.argv[0]} /path/to/ds4", file=sys.stderr) return 2 binary = Path(sys.argv[1]).resolve() + backend = compiled_backend(binary) check_frozen_reference() with tempfile.TemporaryDirectory(prefix="ds4-qwen-metadata-") as tmp_name: @@ -755,12 +774,20 @@ def replace_tensor( v2_metal_output = v2_metal.stdout + v2_metal.stderr require(v2_metal.returncode != 0, "Qwen native v2 quality guard was not enforced", v2_metal) - require("does not support --quality yet" in v2_metal_output, - "Qwen native v2 did not reach Metal option validation", - v2_metal) - require("DS4_QWEN_EXPERIMENTAL_METAL" not in v2_metal_output, - "Qwen native v2 still requires the experimental Metal flag", - v2_metal) + if backend == "metal": + require("does not support --quality yet" in v2_metal_output, + "Qwen native v2 did not reach Metal option validation", + v2_metal) + require("DS4_QWEN_EXPERIMENTAL_METAL" not in v2_metal_output, + "Qwen native v2 still requires the experimental Metal flag", + v2_metal) + else: + require( + "metal backend requested but it is unavailable in this build" in + v2_metal_output, + "CPU build did not reject the unavailable Metal backend", + v2_metal, + ) v2_model.unlink() print("qwen metadata tests: OK") diff --git a/tests/test_qwen_release_contract.py b/tests/test_qwen_release_contract.py new file mode 100644 index 000000000..1c158fbb6 --- /dev/null +++ b/tests/test_qwen_release_contract.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Fail-closed fixtures for the machine-readable Qwen release contract.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +CHECKER = REPOSITORY_ROOT / "tools" / "qwen_release_contract.py" +MANIFEST = "docs/contracts/qwen-release.json" +SURFACES = ( + MANIFEST, + "README.md", + "CONTRIBUTING.md", + "QA_BEFORE_RELEASES.md", + "docs/contracts/RUNTIME_SUPPORT.md", + "docs/qwen-expert-major-store.md", + "download_model.sh", + "tests/test_download_model.sh", +) + + +class Fixture: + def __init__(self) -> None: + self.temporary = tempfile.TemporaryDirectory(prefix="qwen-release-contract-") + self.root = Path(self.temporary.name) + for relative in SURFACES: + destination = self.root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(REPOSITORY_ROOT / relative, destination) + + def close(self) -> None: + self.temporary.cleanup() + + def path(self, relative: str) -> Path: + return self.root / relative + + def contract(self) -> dict[str, object]: + return json.loads(self.path(MANIFEST).read_text(encoding="utf-8")) + + def replace_once(self, relative: str, old: str, new: str) -> None: + path = self.path(relative) + text = path.read_text(encoding="utf-8") + if text.count(old) != 1: + raise AssertionError( + f"fixture expected exactly one {old!r} in {relative}, " + f"found {text.count(old)}" + ) + path.write_text(text.replace(old, new, 1), encoding="utf-8") + + def run(self) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(CHECKER), + "--root", + str(self.root), + "--manifest", + MANIFEST, + ], + text=True, + capture_output=True, + check=False, + ) + + +class QwenReleaseContractTests(unittest.TestCase): + def setUp(self) -> None: + self.fixture = Fixture() + + def tearDown(self) -> None: + self.fixture.close() + + def assert_fails_with(self, expected: str) -> None: + result = self.fixture.run() + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(expected, result.stderr) + + def test_current_contract_and_surfaces_pass(self) -> None: + result = self.fixture.run() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("Qwen release contract: PASS", result.stdout) + + def test_every_human_surface_is_bound_to_the_manifest(self) -> None: + contract = self.fixture.contract() + published = contract["publishedArtifact"] + negative = contract["negativeArtifact"] + assert isinstance(published, dict) + assert isinstance(negative, dict) + cases = ( + ("README.md", published["filename"], "Qwen3.6-DRIFT.gguf"), + ("CONTRIBUTING.md", published["revision"], "a" * 40), + ( + "docs/contracts/RUNTIME_SUPPORT.md", + published["runtimeCommit"], + "b" * 40, + ), + ("QA_BEFORE_RELEASES.md", negative["sha256"], "c" * 64), + ( + "docs/qwen-expert-major-store.md", + contract["repository"], + "example.invalid/Qwen-release", + ), + ) + for relative, old, new in cases: + with self.subTest(relative=relative): + fixture = Fixture() + try: + fixture.replace_once(relative, str(old), new) + result = fixture.run() + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(relative, result.stderr) + finally: + fixture.close() + + def test_downloader_assignment_drift_fails(self) -> None: + contract = self.fixture.contract() + published = contract["publishedArtifact"] + assert isinstance(published, dict) + old = f"RUNTIME_QWEN_BYTES={published['bytes']}" + self.fixture.replace_once("download_model.sh", old, "RUNTIME_QWEN_BYTES=1") + self.assert_fails_with("download_model.sh: RUNTIME_QWEN_BYTES") + + def test_negative_artifact_cannot_become_downloadable(self) -> None: + contract = self.fixture.contract() + negative = contract["negativeArtifact"] + assert isinstance(negative, dict) + path = self.fixture.path("download_model.sh") + with path.open("a", encoding="utf-8") as handle: + handle.write(f"\n# accidental target: {negative['filename']}\n") + self.assert_fails_with("negative-only Qwen artifact must not be downloadable") + + def test_download_test_must_consume_instead_of_duplicate_contract(self) -> None: + contract = self.fixture.contract() + published = contract["publishedArtifact"] + assert isinstance(published, dict) + path = self.fixture.path("tests/test_download_model.sh") + with path.open("a", encoding="utf-8") as handle: + handle.write(f"\n# stale test fixture {published['sha256']}\n") + self.assert_fails_with("must consume the canonical contract") + + def test_manifest_statuses_fail_closed(self) -> None: + for mutation, expected in ( + (("publishedArtifact", "status", "candidate"), "must be published"), + (("negativeArtifact", "status", "retired"), "must be negative-only"), + ): + with self.subTest(mutation=mutation): + fixture = Fixture() + try: + document = fixture.contract() + group, field, value = mutation + artifact = document[group] + assert isinstance(artifact, dict) + artifact[field] = value + fixture.path(MANIFEST).write_text( + json.dumps(document, indent=2) + "\n", encoding="utf-8" + ) + result = fixture.run() + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(expected, result.stderr) + finally: + fixture.close() + + def test_manifest_schema_and_digest_fail_closed(self) -> None: + cases = ( + ("unknown-key", "contract has invalid keys"), + ("invalid-digest", "must be 64 lowercase hexadecimal characters"), + ) + for mutation, expected in cases: + with self.subTest(mutation=mutation): + fixture = Fixture() + try: + document = fixture.contract() + if mutation == "unknown-key": + document["notes"] = "not part of schema 1" + else: + published = document["publishedArtifact"] + assert isinstance(published, dict) + published["sha256"] = "A" * 64 + fixture.path(MANIFEST).write_text( + json.dumps(document, indent=2) + "\n", encoding="utf-8" + ) + result = fixture.run() + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn(expected, result.stderr) + finally: + fixture.close() + + def test_release_table_is_structural_not_an_unscoped_token_search(self) -> None: + self.fixture.replace_once( + "docs/qwen-expert-major-store.md", + "| Immutable revision |", + "| Unrelated note |", + ) + self.assert_fails_with("'Immutable revision' must be") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_retired_distributed_flags.sh b/tests/test_retired_distributed_flags.sh index 2835f841d..cb5d34583 100755 --- a/tests/test_retired_distributed_flags.sh +++ b/tests/test_retired_distributed_flags.sh @@ -3,7 +3,7 @@ set -eu ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) BIN_DIR=${DS4_BIN_DIR:-$ROOT} -PROGRAMS="ds4 ds4-server ds4-agent ds4-bench ds4-eval" +PROGRAMS="hebrus hebrus-server hebrus-agent hebrus-bench hebrus-eval ds4 ds4-server ds4-agent ds4-bench ds4-eval" OPTIONS="--role --layers --listen --coordinator --dist-prefill-chunk --dist-prefill-window --dist-activation-bits --dist-replay-check --debug" fail() { @@ -27,6 +27,10 @@ output_contains() { esac } +diagnostic_name() { + echo "$1" +} + for program in $PROGRAMS; do binary="$BIN_DIR/$program" [ -x "$binary" ] || fail "missing executable $binary" @@ -60,7 +64,7 @@ for program in $PROGRAMS; do done for option in $OPTIONS; do - expected="$program: distributed option $option was retired; distributed inference is not supported" + expected="$(diagnostic_name "$program"): distributed option $option was retired; distributed inference is not supported" run_and_capture "$binary" "$option" [ "$STATUS" -eq 2 ] || diff --git a/tests/test_server_alias_model.py b/tests/test_server_alias_model.py new file mode 100644 index 000000000..a19ecdc00 --- /dev/null +++ b/tests/test_server_alias_model.py @@ -0,0 +1,922 @@ +#!/usr/bin/env python3 +"""Model-backed Hebrus/DS4 server route, output, and shutdown parity gate.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import platform +import re +import signal +import socket +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Any, Callable, Mapping, Sequence + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +MODEL_CONTRACT = ROOT / "docs" / "contracts" / "qwen-release.json" +DEFAULT_LOCK_FILE = "/tmp/ds4.lock" +CANONICAL_SERVER_NAME = "hebrus-server" +LEGACY_SERVER_NAME = "ds4-server" +EXPECTED_COMPLETION_MARKER = "HEBRUS ALIAS PARITY OK" +EVIDENCE_MANIFEST_NAME = "server-alias-evidence-manifest.json" +VOLATILE_COMPLETION_KEYS = frozenset({"id", "created"}) +LOCAL_HTTP_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +@dataclass(frozen=True) +class PublishedArtifact: + filename: str + size: int + sha256: str + revision: str + runtime_commit: str + + +@dataclass(frozen=True) +class FileSignature: + device: int + inode: int + size: int + mtime_ns: int + + +def termination_signal_handler(signum: int, _frame: Any) -> None: + raise InterruptedError(f"received signal {signum}") + + +def install_termination_signal_handlers() -> None: + signal.signal(signal.SIGINT, termination_signal_handler) + signal.signal(signal.SIGTERM, termination_signal_handler) + if hasattr(signal, "SIGHUP"): + signal.signal(signal.SIGHUP, termination_signal_handler) + + +def sha256(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def file_signature(path: pathlib.Path) -> FileSignature: + stat = path.stat() + return FileSignature(stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns) + + +def load_published_artifact(path: pathlib.Path = MODEL_CONTRACT) -> PublishedArtifact: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise AssertionError(f"cannot read Qwen release contract {path}: {exc}") from exc + if ( + not isinstance(document, dict) + or type(document.get("schemaVersion")) is not int + or document.get("schemaVersion") != 1 + ): + raise AssertionError("Qwen release contract must use schemaVersion 1") + published = document.get("publishedArtifact") + if not isinstance(published, dict) or published.get("status") != "published": + raise AssertionError("Qwen release contract has no published artifact") + + required = ("filename", "bytes", "sha256", "revision", "runtimeCommit") + missing = [key for key in required if key not in published] + if missing: + raise AssertionError( + "Qwen published artifact is missing: " + ", ".join(missing) + ) + filename = published["filename"] + size = published["bytes"] + digest = published["sha256"] + revision = published["revision"] + runtime_commit = published["runtimeCommit"] + if not isinstance(filename, str) or not filename: + raise AssertionError("Qwen published filename is invalid") + if type(size) is not int or size <= 0: + raise AssertionError("Qwen published byte count is invalid") + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise AssertionError("Qwen published SHA-256 is invalid") + if not isinstance(revision, str) or re.fullmatch(r"[0-9a-f]{40}", revision) is None: + raise AssertionError("Qwen published revision is invalid") + if ( + not isinstance(runtime_commit, str) + or re.fullmatch(r"[0-9a-f]{40}", runtime_commit) is None + ): + raise AssertionError("Qwen compatible runtime commit is invalid") + return PublishedArtifact(filename, size, digest, revision, runtime_commit) + + +def verify_model( + requested_path: pathlib.Path, artifact: PublishedArtifact +) -> tuple[pathlib.Path, str, FileSignature]: + expanded = requested_path.expanduser() + if expanded.is_symlink(): + raise AssertionError("release model path must not be a symlink") + model = expanded.resolve(strict=True) + if not model.is_file(): + raise AssertionError(f"release model is not a regular file: {model}") + if model.name != artifact.filename: + raise AssertionError( + f"unexpected Qwen filename: {model.name!r}; expected {artifact.filename!r}" + ) + before = file_signature(model) + if before.size != artifact.size: + raise AssertionError(f"unexpected Qwen size: {before.size}") + digest = sha256(model) + after = file_signature(model) + if before != after: + raise AssertionError("Qwen artifact changed while its SHA-256 was computed") + if digest != artifact.sha256: + raise AssertionError(f"unexpected Qwen SHA-256: {digest}") + return model, digest, after + + +def normalize_completion(document: dict[str, Any]) -> dict[str, Any]: + """Drop only request-instance fields; preserve every nested/public id.""" + return { + key: value for key, value in document.items() if key not in VOLATILE_COMPLETION_KEYS + } + + +def validate_completion_content( + document: dict[str, Any], binary_name: str +) -> str: + choices = document.get("choices") + if not isinstance(choices, list) or len(choices) != 1: + raise AssertionError( + f"{binary_name}: completion must contain exactly one choice" + ) + choice = choices[0] + if not isinstance(choice, dict): + raise AssertionError(f"{binary_name}: completion choice is not an object") + message = choice.get("message") + if not isinstance(message, dict): + raise AssertionError(f"{binary_name}: completion choice has no message object") + content = message.get("content") + if not isinstance(content, str) or not content.strip(): + raise AssertionError(f"{binary_name}: completion content is empty or missing") + normalized = content.strip() + if normalized != EXPECTED_COMPLETION_MARKER: + raise AssertionError( + f"{binary_name}: completion content does not exactly match " + f"{EXPECTED_COMPLETION_MARKER!r}" + ) + return normalized + + +def command_output(argv: Sequence[str]) -> str | None: + try: + result = subprocess.run( + list(argv), + check=False, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + output = result.stdout.strip() + return output or None + + +def parse_battery_status(output: str | None) -> dict[str, Any]: + status: dict[str, Any] = { + "source": None, + "battery_percent": None, + "battery_state": None, + } + if not output: + return status + source_match = re.search(r"Now drawing from '([^']+)'", output) + if source_match is not None: + status["source"] = source_match.group(1) + percent_match = re.search(r"\b(\d{1,3})%;\s*([^;\n]+)", output) + if percent_match is not None: + percent = int(percent_match.group(1)) + if 0 <= percent <= 100: + status["battery_percent"] = percent + state = percent_match.group(2).strip() + status["battery_state"] = state or None + return status + + +def parse_powermode(output: str | None, power_source: object) -> int | None: + if not output: + return None + modes: dict[str, int] = {} + current_source: str | None = None + for line in output.splitlines(): + stripped = line.strip() + if stripped.endswith("Power:"): + current_source = stripped[:-1] + continue + match = re.fullmatch(r"powermode\s+(\d+)", stripped) + if match is not None: + modes[current_source or ""] = int(match.group(1)) + if isinstance(power_source, str) and power_source in modes: + return modes[power_source] + unique_modes = set(modes.values()) + if len(unique_modes) == 1: + return unique_modes.pop() + return None + + +def parse_positive_integer(value: str | None) -> int | None: + if value is None or re.fullmatch(r"[1-9][0-9]*", value) is None: + return None + return int(value) + + +def collect_host_metadata( + reader: Callable[[Sequence[str]], str | None] = command_output, + *, + system_name: str | None = None, + architecture: str | None = None, +) -> dict[str, Any]: + system = system_name if system_name is not None else platform.system() + machine = architecture if architecture is not None else platform.machine() + battery = parse_battery_status(reader(("pmset", "-g", "batt"))) + power_source = battery["source"] + battery["powermode"] = parse_powermode( + reader(("pmset", "-g", "custom")), power_source + ) + return { + "architecture": machine, + "hardware_model": reader(("sysctl", "-n", "hw.model")), + "chip": reader(("sysctl", "-n", "machdep.cpu.brand_string")), + "physical_memory_bytes": parse_positive_integer( + reader(("sysctl", "-n", "hw.memsize")) + ), + "os": { + "name": "macOS" if system == "Darwin" else system, + "version": reader(("sw_vers", "-productVersion")), + "build": reader(("sw_vers", "-buildVersion")), + }, + "power": battery, + } + + +def validate_qualified_host(host: Mapping[str, Any], expected_backend: str) -> None: + operating_system = host.get("os") + if expected_backend != "metal": + raise AssertionError("model-backed server parity requires the Metal backend") + if host.get("architecture") != "arm64": + raise AssertionError("model-backed server parity requires an arm64 host") + if not isinstance(operating_system, Mapping) or operating_system.get("name") != "macOS": + raise AssertionError("model-backed server parity requires macOS") + if not operating_system.get("version") or not operating_system.get("build"): + raise AssertionError("qualified host OS version/build metadata is unavailable") + memory = host.get("physical_memory_bytes") + if type(memory) is not int or memory <= 0: + raise AssertionError("qualified host physical memory metadata is unavailable") + + +def build_environment(source: Mapping[str, str]) -> tuple[dict[str, str], list[str]]: + environment: dict[str, str] = {} + removed: list[str] = [] + for key, value in source.items(): + if key == "DS4_LOCK_FILE": + environment[key] = value + elif key.startswith(("DS4_", "HEBRUS_")): + removed.append(key) + else: + environment[key] = value + if not environment.get("DS4_LOCK_FILE"): + environment["DS4_LOCK_FILE"] = DEFAULT_LOCK_FILE + return environment, sorted(removed) + + +def prepare_evidence_dir(requested: pathlib.Path | None) -> pathlib.Path: + if requested is None: + return pathlib.Path(tempfile.mkdtemp(prefix="hebrus-server-alias-")).resolve() + evidence_dir = requested.expanduser().resolve() + evidence_dir.mkdir(parents=True, exist_ok=False) + return evidence_dir + + +def write_report(path: pathlib.Path, report: dict[str, Any]) -> None: + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + temporary.replace(path) + + +def build_evidence_manifest(evidence_dir: pathlib.Path) -> dict[str, Any]: + artifacts: list[dict[str, Any]] = [] + for path in sorted(evidence_dir.iterdir(), key=lambda candidate: candidate.name): + if ( + not path.is_file() + or path.name == EVIDENCE_MANIFEST_NAME + or path.name.startswith(".") + ): + continue + artifacts.append( + { + "path": path.name, + "bytes": path.stat().st_size, + "sha256": sha256(path), + } + ) + core: dict[str, Any] = { + "schema_version": 1, + "artifacts": artifacts, + } + canonical = json.dumps( + core, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("utf-8") + return { + **core, + "bundle_sha256": hashlib.sha256(canonical).hexdigest(), + } + + +def write_evidence_manifest(evidence_dir: pathlib.Path) -> pathlib.Path: + manifest_path = evidence_dir / EVIDENCE_MANIFEST_NAME + write_report(manifest_path, build_evidence_manifest(evidence_dir)) + return manifest_path + + +def validate_requested_port(port: int) -> None: + if port != 0 and (port < 1024 or port > 65535): + raise AssertionError("--port must be 0 or an unprivileged TCP port") + + +def assert_port_available(port: int) -> None: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + probe.bind(("127.0.0.1", port)) + except OSError as exc: + raise AssertionError(f"TCP port {port} is already in use") from exc + + +def select_port(requested: int) -> int: + validate_requested_port(requested) + if requested: + assert_port_available(requested) + return requested + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", 0)) + selected = int(probe.getsockname()[1]) + if selected < 1024: + raise AssertionError(f"OS selected a privileged TCP port: {selected}") + return selected + + +def wait_for_port_closed(port: int, timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + while True: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + pass + except OSError: + return + if time.monotonic() >= deadline: + raise AssertionError(f"TCP port {port} still accepts connections after shutdown") + time.sleep(0.1) + + +def http_json( + url: str, + payload: dict[str, Any] | None = None, + *, + timeout: float, +) -> dict[str, Any]: + data = None + method = "GET" + headers: dict[str, str] = {} + if payload is not None: + data = json.dumps(payload, separators=(",", ":")).encode("utf-8") + method = "POST" + headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, data=data, headers=headers, method=method) + with LOCAL_HTTP_OPENER.open(request, timeout=timeout) as response: + if response.status != 200: + raise AssertionError(f"{method} {url} returned HTTP {response.status}") + document = json.loads(response.read()) + if not isinstance(document, dict): + raise AssertionError(f"{method} {url} did not return a JSON object") + return document + + +def build_sha_matches(actual: object, expected: str) -> bool: + if not isinstance(actual, str): + return False + if re.fullmatch(r"[0-9a-f]{12,40}", actual) is None: + return False + if re.fullmatch(r"[0-9a-f]{12,40}", expected) is None: + return False + return actual.startswith(expected) or expected.startswith(actual) + + +def repository_head(root: pathlib.Path = ROOT) -> str: + result = subprocess.run( + ["git", "rev-parse", "--verify", "HEAD"], + cwd=root, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + head = result.stdout.strip() + if result.returncode != 0 or re.fullmatch(r"[0-9a-f]{40}", head) is None: + raise AssertionError(f"cannot resolve candidate repository HEAD: {result.stderr.strip()}") + return head + + +def assert_clean_repository(root: pathlib.Path = ROOT) -> None: + result = subprocess.run( + ["git", "status", "--porcelain", "--untracked-files=normal"], + cwd=root, + check=False, + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode != 0: + raise AssertionError( + f"cannot inspect candidate repository state: {result.stderr.strip()}" + ) + if result.stdout: + raise AssertionError( + "candidate repository must be an exact clean committed tree" + ) + + +def validate_capability_document( + document: dict[str, Any], + *, + binary_name: str, + engine_id: str, + expected_backend: str, + expected_build_sha: str, +) -> None: + if type(document.get("schema_version")) is not int or document.get("schema_version") != 1: + raise AssertionError(f"{binary_name}: capability schema is not 1") + if document.get("engine_id") != engine_id: + raise AssertionError(f"{binary_name}: expected engine_id {engine_id!r}") + if document.get("executable_role") != "server": + raise AssertionError(f"{binary_name}: capability role is not server") + if document.get("backend") != expected_backend: + raise AssertionError( + f"{binary_name}: expected backend {expected_backend!r}, " + f"got {document.get('backend')!r}" + ) + if not build_sha_matches(document.get("build_git_sha"), expected_build_sha): + raise AssertionError( + f"{binary_name}: build {document.get('build_git_sha')!r} does not " + f"match candidate {expected_build_sha!r}" + ) + + +def read_capabilities( + binary: pathlib.Path, environment: Mapping[str, str] +) -> dict[str, Any]: + result = subprocess.run( + [str(binary), "--capabilities=json"], + check=False, + capture_output=True, + text=True, + env=dict(environment), + timeout=30, + ) + if result.returncode != 0 or result.stderr: + raise AssertionError( + f"{binary.name}: capability preflight failed with {result.returncode}: " + f"{result.stderr.strip()}" + ) + document = json.loads(result.stdout) + if not isinstance(document, dict): + raise AssertionError(f"{binary.name}: capability output is not a JSON object") + return document + + +def validate_capability_pair_documents( + canonical_document: dict[str, Any], + legacy_document: dict[str, Any], + *, + canonical_name: str, + legacy_name: str, + expected_backend: str, + expected_build_sha: str, +) -> None: + validate_capability_document( + canonical_document, + binary_name=canonical_name, + engine_id="hebrus", + expected_backend=expected_backend, + expected_build_sha=expected_build_sha, + ) + validate_capability_document( + legacy_document, + binary_name=legacy_name, + engine_id="ds4", + expected_backend=expected_backend, + expected_build_sha=expected_build_sha, + ) + canonical_common = { + key: value for key, value in canonical_document.items() if key != "engine_id" + } + legacy_common = { + key: value for key, value in legacy_document.items() if key != "engine_id" + } + if canonical_common != legacy_common: + raise AssertionError("server capability documents differ beyond engine_id") + + +def capability_preflight( + canonical: pathlib.Path, + legacy: pathlib.Path, + *, + expected_backend: str, + expected_build_sha: str, + environment: Mapping[str, str], +) -> dict[str, dict[str, Any]]: + canonical_document = read_capabilities(canonical, environment) + legacy_document = read_capabilities(legacy, environment) + validate_capability_pair_documents( + canonical_document, + legacy_document, + canonical_name=canonical.name, + legacy_name=legacy.name, + expected_backend=expected_backend, + expected_build_sha=expected_build_sha, + ) + return { + canonical.name: canonical_document, + legacy.name: legacy_document, + } + + +def wait_for_listener( + log_path: pathlib.Path, + process: subprocess.Popen[bytes], + port: int, + deadline: float, +) -> None: + marker = f"listening on http://127.0.0.1:{port}" + while time.monotonic() < deadline: + if process.poll() is not None: + raise AssertionError( + f"server exited during startup with code {process.returncode}" + ) + try: + log_text = log_path.read_text(encoding="utf-8", errors="replace") + except FileNotFoundError: + log_text = "" + if marker in log_text: + return + time.sleep(min(0.1, max(0.0, deadline - time.monotonic()))) + raise AssertionError(f"server startup timed out before listener ownership: {marker}") + + +def wait_for_models( + base_url: str, + process: subprocess.Popen[bytes], + log_path: pathlib.Path, + port: int, + timeout: float, +) -> dict[str, Any]: + deadline = time.monotonic() + timeout + wait_for_listener(log_path, process, port, deadline) + last_error = "server did not accept a request" + while time.monotonic() < deadline: + if process.poll() is not None: + raise AssertionError( + f"server exited during startup with code {process.returncode}" + ) + remaining = deadline - time.monotonic() + try: + return http_json( + f"{base_url}/v1/models", + timeout=max(0.05, min(2.0, remaining)), + ) + except (OSError, urllib.error.URLError, json.JSONDecodeError) as exc: + last_error = str(exc) + time.sleep(min(0.25, max(0.0, deadline - time.monotonic()))) + raise AssertionError(f"server startup timed out: {last_error}") + + +def terminate_process( + process: subprocess.Popen[bytes], binary_name: str, timeout: float +) -> int: + if process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.wait(timeout=timeout) + except subprocess.TimeoutExpired as exc: + process.kill() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired as kill_exc: + raise AssertionError( + f"{binary_name}: process survived SIGKILL" + ) from kill_exc + raise AssertionError(f"{binary_name}: graceful shutdown timed out") from exc + if process.returncode is None: + raise AssertionError(f"{binary_name}: process has no exit status") + return process.returncode + + +def run_server( + binary: pathlib.Path, + model: pathlib.Path, + port: int, + startup_timeout: float, + request_timeout: float, + shutdown_timeout: float, + evidence_dir: pathlib.Path, + environment: Mapping[str, str], +) -> dict[str, Any]: + assert_port_available(port) + log_path = evidence_dir / f"{binary.name}.log" + command = [ + str(binary), + "-m", str(model), + "--ctx", "8192", + "--tokens", "32", + "--threads", "8", + "--host", "127.0.0.1", + "--port", str(port), + ] + with log_path.open("wb") as log: + process = subprocess.Popen( + command, + cwd=ROOT, + env=dict(environment), + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + base_url = f"http://127.0.0.1:{port}" + models = wait_for_models( + base_url, process, log_path, port, startup_timeout + ) + entries = models.get("data") + if not isinstance(entries, list) or len(entries) != 1: + raise AssertionError(f"{binary.name}: unexpected model list: {models}") + entry = entries[0] + if not isinstance(entry, dict): + raise AssertionError(f"{binary.name}: model list entry is not an object") + model_id = entry.get("id") + if not isinstance(model_id, str) or not model_id: + raise AssertionError(f"{binary.name}: model list has no usable id") + model_document = http_json( + f"{base_url}/v1/models/{urllib.parse.quote(model_id, safe='')}", + timeout=request_timeout, + ) + completion = http_json( + f"{base_url}/v1/chat/completions", + { + "model": model_id, + "messages": [ + { + "role": "user", + "content": "Reply with exactly: HEBRUS ALIAS PARITY OK", + } + ], + "max_tokens": 16, + "temperature": 0, + "seed": 20260721, + "thinking": False, + "stream": False, + }, + timeout=request_timeout, + ) + completion_content = validate_completion_content( + completion, binary.name + ) + finally: + exit_code = terminate_process(process, binary.name, shutdown_timeout) + wait_for_port_closed(port) + if exit_code != 0: + raise AssertionError( + f"{binary.name}: shutdown returned {exit_code}; see {log_path}" + ) + + log_text = log_path.read_text(encoding="utf-8", errors="replace") + for marker in ( + f"listening on http://127.0.0.1:{port}", + "shutdown requested, draining requests", + ): + if marker not in log_text: + raise AssertionError(f"{binary.name}: missing log marker {marker!r}") + return { + "command": command, + "models": models, + "model": model_document, + "completion": normalize_completion(completion), + "completion_content": completion_content, + "log": str(log_path), + "exit_code": exit_code, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, type=pathlib.Path) + parser.add_argument("--bin-dir", default=ROOT, type=pathlib.Path) + parser.add_argument("--expected-backend", required=True, choices=("metal", "cpu")) + parser.add_argument("--expected-build-sha", required=True) + parser.add_argument( + "--port", + default=0, + type=int, + help="unprivileged port, or 0 to select an unused local port", + ) + parser.add_argument("--startup-timeout", default=180.0, type=float) + parser.add_argument("--request-timeout", default=300.0, type=float) + parser.add_argument("--shutdown-timeout", default=90.0, type=float) + parser.add_argument("--evidence-dir", type=pathlib.Path) + args = parser.parse_args() + for name in ("startup_timeout", "request_timeout", "shutdown_timeout"): + if getattr(args, name) <= 0: + parser.error(f"--{name.replace('_', '-')} must be positive") + if sys.platform == "darwin" and args.expected_backend != "metal": + parser.error("model-backed server parity on macOS requires the Metal backend") + return args + + +def main() -> int: + args = parse_args() + evidence_dir = prepare_evidence_dir(args.evidence_dir) + report_path = evidence_dir / "server-alias-parity.json" + report: dict[str, Any] = { + "schema_version": 3, + "status": "RUNNING", + "evidence_manifest": EVIDENCE_MANIFEST_NAME, + "results": {}, + } + write_report(report_path, report) + print(f"server-alias-model: evidence {report_path}", file=sys.stderr) + install_termination_signal_handlers() + + try: + host = collect_host_metadata() + report["host"] = host + write_report(report_path, report) + validate_qualified_host(host, args.expected_backend) + + bin_dir = args.bin_dir.expanduser().resolve(strict=True) + canonical = bin_dir / CANONICAL_SERVER_NAME + legacy = bin_dir / LEGACY_SERVER_NAME + if ( + not canonical.is_file() + or not legacy.is_file() + or not os.path.samefile(canonical, legacy) + ): + raise AssertionError( + f"{CANONICAL_SERVER_NAME} and {LEGACY_SERVER_NAME} " + "are not one executable surface" + ) + + candidate_head = repository_head() + assert_clean_repository() + if not build_sha_matches(candidate_head, args.expected_build_sha): + raise AssertionError( + f"candidate repository HEAD {candidate_head!r} does not match " + f"expected build {args.expected_build_sha!r}" + ) + environment, removed_environment = build_environment(os.environ) + binary_hash = sha256(canonical) + report.update( + { + "candidate": { + "expected_backend": args.expected_backend, + "expected_build_sha": args.expected_build_sha, + "repository_head": candidate_head, + "repository_clean": True, + "binary_sha256": binary_hash, + }, + "environment": { + "preserved_names": ["DS4_LOCK_FILE"], + "removed_names": removed_environment, + }, + "log_paths": { + canonical.name: str(evidence_dir / f"{canonical.name}.log"), + legacy.name: str(evidence_dir / f"{legacy.name}.log"), + }, + } + ) + write_report(report_path, report) + capabilities = capability_preflight( + canonical, + legacy, + expected_backend=args.expected_backend, + expected_build_sha=args.expected_build_sha, + environment=environment, + ) + if sha256(canonical) != binary_hash or not os.path.samefile(canonical, legacy): + raise AssertionError("server executable changed during capability preflight") + report["candidate"]["capabilities"] = capabilities + write_report(report_path, report) + + artifact = load_published_artifact() + model, model_hash, model_signature = verify_model(args.model, artifact) + port = select_port(args.port) + report.update( + { + "model": { + "contract": str(MODEL_CONTRACT), + "filename": artifact.filename, + "path": str(model), + "bytes": model_signature.size, + "sha256": model_hash, + "revision": artifact.revision, + "runtime_commit": artifact.runtime_commit, + }, + "port": port, + } + ) + write_report(report_path, report) + + results: dict[str, Any] = report["results"] + for binary in (canonical, legacy): + if sha256(canonical) != binary_hash or not os.path.samefile(canonical, legacy): + raise AssertionError("server executable changed during alias parity gate") + results[binary.name] = run_server( + binary, + model, + port, + args.startup_timeout, + args.request_timeout, + args.shutdown_timeout, + evidence_dir, + environment, + ) + if sha256(canonical) != binary_hash: + raise AssertionError("server executable changed during alias parity gate") + if file_signature(model) != model_signature: + raise AssertionError("Qwen artifact changed during alias parity gate") + write_report(report_path, report) + + for field in ( + "models", + "model", + "completion", + "completion_content", + "exit_code", + ): + if results[canonical.name][field] != results[legacy.name][field]: + raise AssertionError( + f"server aliases differ for {field}; see {report_path}" + ) + if repository_head() != candidate_head: + raise AssertionError("candidate repository HEAD changed during alias parity gate") + assert_clean_repository() + + report["status"] = "PASS" + write_report(report_path, report) + manifest_path = write_evidence_manifest(evidence_dir) + print( + f"server-alias-model: PASS " + f"({canonical.name}/{legacy.name}, {report_path}, {manifest_path})" + ) + return 0 + except BaseException as exc: + report["status"] = "FAIL" + detail = str(exc) or "interrupted" + report["error"] = f"{type(exc).__name__}: {detail}" + write_report(report_path, report) + try: + write_evidence_manifest(evidence_dir) + except Exception as manifest_exc: + print( + f"server-alias-model: failed to write evidence manifest: " + f"{manifest_exc}", + file=sys.stderr, + ) + print(f"server-alias-model: evidence retained at {report_path}", file=sys.stderr) + raise + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except KeyboardInterrupt as exc: + print(f"server-alias-model: FAIL: {exc or 'interrupted'}", file=sys.stderr) + raise SystemExit(130) + except ( + AssertionError, + OSError, + subprocess.SubprocessError, + urllib.error.URLError, + json.JSONDecodeError, + ) as exc: + print(f"server-alias-model: FAIL: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/tests/test_server_alias_model_unit.py b/tests/test_server_alias_model_unit.py new file mode 100644 index 000000000..99f85916a --- /dev/null +++ b/tests/test_server_alias_model_unit.py @@ -0,0 +1,536 @@ +#!/usr/bin/env python3 +"""Model-free regression tests for the server alias release gate helpers.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import pathlib +import shutil +import socket +import sys +import tempfile +import types +import unittest +import urllib.error +from unittest import mock + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "tests" / "test_server_alias_model.py" +SPEC = importlib.util.spec_from_file_location("server_alias_model", MODULE_PATH) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"cannot import {MODULE_PATH}") +SERVER_ALIAS = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = SERVER_ALIAS +SPEC.loader.exec_module(SERVER_ALIAS) + + +def capability(engine_id: str, backend: str = "metal") -> dict[str, object]: + return { + "schema_version": 1, + "engine_id": engine_id, + "build_git_sha": "123456789abc", + "backend": backend, + "executable_role": "server", + "model_families": ["qwen35moe"], + } + + +def completion(content: object) -> dict[str, object]: + return { + "id": "chatcmpl-volatile", + "created": 123, + "choices": [{"message": {"role": "assistant", "content": content}}], + } + + +class FakeProcess: + returncode = None + + def poll(self) -> None: + return None + + +class ServerAliasModelUnitTests(unittest.TestCase): + def test_completion_normalization_preserves_nested_and_model_ids(self) -> None: + document = { + "id": "chatcmpl-volatile", + "created": 123, + "model": "qwen3.6-35b-a3b", + "choices": [ + { + "message": { + "tool_calls": [{"id": "stable-tool-id"}], + } + } + ], + } + normalized = SERVER_ALIAS.normalize_completion(document) + self.assertNotIn("id", normalized) + self.assertNotIn("created", normalized) + self.assertEqual(normalized["model"], "qwen3.6-35b-a3b") + self.assertEqual( + normalized["choices"][0]["message"]["tool_calls"][0]["id"], + "stable-tool-id", + ) + + def test_completion_content_requires_exact_nonempty_marker(self) -> None: + valid = completion(f" {SERVER_ALIAS.EXPECTED_COMPLETION_MARKER}\n") + self.assertEqual( + SERVER_ALIAS.validate_completion_content(valid, "hebrus-server"), + SERVER_ALIAS.EXPECTED_COMPLETION_MARKER, + ) + + invalid_documents = ( + completion(""), + completion(" \n"), + completion(None), + completion("HEBRUS ALIAS PARITY"), + completion(f"prefix {SERVER_ALIAS.EXPECTED_COMPLETION_MARKER}"), + {"choices": []}, + {"choices": [{"message": {}}]}, + ) + for document in invalid_documents: + with self.subTest(document=document): + with self.assertRaises(AssertionError): + SERVER_ALIAS.validate_completion_content( + document, "hebrus-server" + ) + + def test_environment_preserves_lock_and_reports_only_removed_names(self) -> None: + environment, removed = SERVER_ALIAS.build_environment( + { + "PATH": "/usr/bin", + "DS4_LOCK_FILE": "/tmp/qualified.lock", + "DS4_TOKEN_TIMING": "1", + "HEBRUS_EXPERIMENT": "1", + } + ) + self.assertEqual(environment["DS4_LOCK_FILE"], "/tmp/qualified.lock") + self.assertNotIn("DS4_TOKEN_TIMING", environment) + self.assertNotIn("HEBRUS_EXPERIMENT", environment) + self.assertEqual(removed, ["DS4_TOKEN_TIMING", "HEBRUS_EXPERIMENT"]) + + def test_environment_sets_explicit_default_lock(self) -> None: + environment, removed = SERVER_ALIAS.build_environment({"PATH": "/usr/bin"}) + self.assertEqual(environment["DS4_LOCK_FILE"], SERVER_ALIAS.DEFAULT_LOCK_FILE) + self.assertEqual(removed, []) + + def test_host_power_parsers_select_the_active_source(self) -> None: + battery = SERVER_ALIAS.parse_battery_status( + "Now drawing from 'AC Power'\n" + " -InternalBattery-0\t100%; charged; 0:00 remaining present: true\n" + ) + self.assertEqual( + battery, + { + "source": "AC Power", + "battery_percent": 100, + "battery_state": "charged", + }, + ) + custom = ( + "Battery Power:\n powermode 1\n" + "AC Power:\n powermode 2\n" + ) + self.assertEqual( + SERVER_ALIAS.parse_powermode(custom, battery["source"]), 2 + ) + self.assertIsNone(SERVER_ALIAS.parse_powermode(custom, "UPS Power")) + self.assertEqual( + SERVER_ALIAS.parse_battery_status(None)["source"], None + ) + + def test_host_metadata_collection_omits_sensitive_network_identity(self) -> None: + values = { + ("pmset", "-g", "batt"): ( + "Now drawing from 'AC Power'\n" + " -InternalBattery-0 87%; charging; 0:10 remaining\n" + ), + ("pmset", "-g", "custom"): "AC Power:\n powermode 2\n", + ("sysctl", "-n", "hw.model"): "Mac17,1", + ("sysctl", "-n", "machdep.cpu.brand_string"): "Apple M5 Pro", + ("sysctl", "-n", "hw.memsize"): "68719476736", + ("sw_vers", "-productVersion"): "26.5.2", + ("sw_vers", "-buildVersion"): "25F90", + } + host = SERVER_ALIAS.collect_host_metadata( + lambda argv: values.get(tuple(argv)), + system_name="Darwin", + architecture="arm64", + ) + self.assertEqual(host["architecture"], "arm64") + self.assertEqual(host["physical_memory_bytes"], 68719476736) + self.assertEqual(host["os"]["name"], "macOS") + self.assertEqual(host["os"]["build"], "25F90") + self.assertEqual(host["power"]["powermode"], 2) + self.assertNotIn("hostname", host) + self.assertNotIn("address", json.dumps(host).lower()) + + def test_qualified_host_validation_fails_closed(self) -> None: + host = { + "architecture": "arm64", + "physical_memory_bytes": 68719476736, + "os": {"name": "macOS", "version": "26.5.2", "build": "25F90"}, + } + SERVER_ALIAS.validate_qualified_host(host, "metal") + with self.assertRaisesRegex(AssertionError, "Metal"): + SERVER_ALIAS.validate_qualified_host(host, "cpu") + with self.assertRaisesRegex(AssertionError, "arm64"): + SERVER_ALIAS.validate_qualified_host( + {**host, "architecture": "x86_64"}, "metal" + ) + with self.assertRaisesRegex(AssertionError, "macOS"): + SERVER_ALIAS.validate_qualified_host( + {**host, "os": {"name": "Linux", "version": "1", "build": "1"}}, + "metal", + ) + with self.assertRaisesRegex(AssertionError, "physical memory"): + SERVER_ALIAS.validate_qualified_host( + {**host, "physical_memory_bytes": None}, "metal" + ) + + def test_capability_validation_rejects_wrong_backend_or_dirty_build(self) -> None: + valid = capability("hebrus") + SERVER_ALIAS.validate_capability_document( + valid, + binary_name="hebrus-server", + engine_id="hebrus", + expected_backend="metal", + expected_build_sha="123456789abcdef0", + ) + + wrong_backend = dict(valid, backend="cpu") + with self.assertRaisesRegex(AssertionError, "expected backend"): + SERVER_ALIAS.validate_capability_document( + wrong_backend, + binary_name="hebrus-server", + engine_id="hebrus", + expected_backend="metal", + expected_build_sha="123456789abcdef0", + ) + + dirty = dict(valid, build_git_sha="123456789abc-dirty") + with self.assertRaisesRegex(AssertionError, "does not match candidate"): + SERVER_ALIAS.validate_capability_document( + dirty, + binary_name="hebrus-server", + engine_id="hebrus", + expected_backend="metal", + expected_build_sha="123456789abcdef0", + ) + + def test_build_identity_requires_a_clean_hex_commit_prefix(self) -> None: + self.assertTrue( + SERVER_ALIAS.build_sha_matches( + "123456789abc", "123456789abc" + ("d" * 28) + ) + ) + self.assertFalse( + SERVER_ALIAS.build_sha_matches("123456789abc-dirty", "123456789abc") + ) + self.assertFalse(SERVER_ALIAS.build_sha_matches("unknown", "123456789abc")) + + def test_repository_cleanliness_rejects_tracked_or_untracked_changes(self) -> None: + clean = SERVER_ALIAS.subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="" + ) + dirty = SERVER_ALIAS.subprocess.CompletedProcess( + args=[], returncode=0, stdout="?? release-notes.md\n", stderr="" + ) + with mock.patch.object(SERVER_ALIAS.subprocess, "run", return_value=clean): + SERVER_ALIAS.assert_clean_repository(pathlib.Path("/candidate")) + with mock.patch.object(SERVER_ALIAS.subprocess, "run", return_value=dirty): + with self.assertRaisesRegex(AssertionError, "exact clean committed tree"): + SERVER_ALIAS.assert_clean_repository(pathlib.Path("/candidate")) + + def test_capability_pair_rejects_drift_beyond_engine_identity(self) -> None: + canonical = capability("hebrus") + legacy = capability("ds4") + SERVER_ALIAS.validate_capability_pair_documents( + canonical, + legacy, + canonical_name="hebrus-server", + legacy_name="ds4-server", + expected_backend="metal", + expected_build_sha="123456789abcdef0", + ) + legacy["model_families"] = ["deepseek4"] + with self.assertRaisesRegex(AssertionError, "beyond engine_id"): + SERVER_ALIAS.validate_capability_pair_documents( + canonical, + legacy, + canonical_name="hebrus-server", + legacy_name="ds4-server", + expected_backend="metal", + expected_build_sha="123456789abcdef0", + ) + + def test_contract_is_loaded_from_manifest_and_rejects_invalid_digest(self) -> None: + with tempfile.TemporaryDirectory(prefix="alias-contract-") as directory: + path = pathlib.Path(directory) / "contract.json" + document = { + "schemaVersion": 1, + "publishedArtifact": { + "status": "published", + "filename": "qualified.gguf", + "bytes": 3, + "sha256": "a" * 64, + "revision": "b" * 40, + "runtimeCommit": "c" * 40, + }, + } + path.write_text(json.dumps(document), encoding="utf-8") + artifact = SERVER_ALIAS.load_published_artifact(path) + self.assertEqual(artifact.filename, "qualified.gguf") + self.assertEqual(artifact.size, 3) + + document["publishedArtifact"]["sha256"] = "not-a-digest" + path.write_text(json.dumps(document), encoding="utf-8") + with self.assertRaisesRegex(AssertionError, "SHA-256"): + SERVER_ALIAS.load_published_artifact(path) + + def test_model_verification_requires_manifest_filename_and_hash(self) -> None: + with tempfile.TemporaryDirectory(prefix="alias-model-") as directory: + model = pathlib.Path(directory) / "qualified.gguf" + model.write_bytes(b"abc") + artifact = SERVER_ALIAS.PublishedArtifact( + filename="qualified.gguf", + size=3, + sha256="ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + revision="b" * 40, + runtime_commit="c" * 40, + ) + resolved, digest, signature = SERVER_ALIAS.verify_model(model, artifact) + self.assertEqual(resolved, model.resolve()) + self.assertEqual(digest, artifact.sha256) + self.assertEqual(signature.size, 3) + + renamed = pathlib.Path(directory) / "renamed.gguf" + renamed.write_bytes(b"abc") + with self.assertRaisesRegex(AssertionError, "filename"): + SERVER_ALIAS.verify_model(renamed, artifact) + + def test_default_evidence_directory_is_persistent(self) -> None: + evidence = SERVER_ALIAS.prepare_evidence_dir(None) + try: + self.assertTrue(evidence.is_dir()) + marker = evidence / "marker" + marker.write_text("retained", encoding="utf-8") + self.assertEqual(marker.read_text(encoding="utf-8"), "retained") + finally: + shutil.rmtree(evidence) + + def test_evidence_manifest_is_deterministic_and_non_self_referential(self) -> None: + with tempfile.TemporaryDirectory(prefix="alias-evidence-") as directory: + evidence = pathlib.Path(directory) + (evidence / "server-alias-parity.json").write_text( + '{"status":"PASS"}\n', encoding="utf-8" + ) + (evidence / "hebrus-server.log").write_text( + "canonical log\n", encoding="utf-8" + ) + first = SERVER_ALIAS.build_evidence_manifest(evidence) + manifest_path = SERVER_ALIAS.write_evidence_manifest(evidence) + second = SERVER_ALIAS.build_evidence_manifest(evidence) + + self.assertEqual(first, second) + self.assertEqual( + [entry["path"] for entry in first["artifacts"]], + ["hebrus-server.log", "server-alias-parity.json"], + ) + self.assertNotIn( + SERVER_ALIAS.EVIDENCE_MANIFEST_NAME, + [entry["path"] for entry in first["artifacts"]], + ) + written = json.loads(manifest_path.read_text(encoding="utf-8")) + self.assertEqual(written, first) + core = { + "schema_version": written["schema_version"], + "artifacts": written["artifacts"], + } + canonical = json.dumps( + core, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("utf-8") + self.assertEqual( + written["bundle_sha256"], hashlib.sha256(canonical).hexdigest() + ) + + (evidence / "hebrus-server.log").write_text( + "changed log\n", encoding="utf-8" + ) + changed = SERVER_ALIAS.build_evidence_manifest(evidence) + self.assertNotEqual(changed["bundle_sha256"], first["bundle_sha256"]) + + def test_port_selection_and_collision_preflight(self) -> None: + selected = SERVER_ALIAS.select_port(0) + self.assertGreaterEqual(selected, 1024) + SERVER_ALIAS.assert_port_available(selected) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as held: + held.bind(("127.0.0.1", 0)) + occupied = int(held.getsockname()[1]) + with self.assertRaisesRegex(AssertionError, "already in use"): + SERVER_ALIAS.assert_port_available(occupied) + + def test_startup_http_probe_is_bounded_by_remaining_deadline(self) -> None: + with tempfile.TemporaryDirectory(prefix="alias-listener-") as directory: + log_path = pathlib.Path(directory) / "server.log" + log_path.write_text( + "ds4-server: listening on http://127.0.0.1:18081\n", + encoding="utf-8", + ) + with mock.patch.object( + SERVER_ALIAS, + "http_json", + side_effect=urllib.error.URLError("not ready"), + ) as request: + started = SERVER_ALIAS.time.monotonic() + with self.assertRaisesRegex(AssertionError, "startup timed out"): + SERVER_ALIAS.wait_for_models( + "http://127.0.0.1:18081", + FakeProcess(), + log_path, + 18081, + 0.05, + ) + elapsed = SERVER_ALIAS.time.monotonic() - started + self.assertLess(elapsed, 0.5) + self.assertTrue(request.called) + self.assertLessEqual(request.call_args.kwargs["timeout"], 0.05) + + def test_local_http_opener_disables_environment_proxies(self) -> None: + proxy_handlers = [ + handler + for handler in SERVER_ALIAS.LOCAL_HTTP_OPENER.handlers + if isinstance(handler, SERVER_ALIAS.urllib.request.ProxyHandler) + ] + self.assertFalse(any(handler.proxies for handler in proxy_handlers)) + + def test_graceful_process_cleanup_uses_sigterm(self) -> None: + process = mock.Mock() + process.poll.return_value = None + process.returncode = 0 + process.wait.return_value = 0 + self.assertEqual(SERVER_ALIAS.terminate_process(process, "server", 1.0), 0) + process.send_signal.assert_called_once_with(SERVER_ALIAS.signal.SIGTERM) + process.kill.assert_not_called() + + def test_cleanup_kills_after_graceful_timeout(self) -> None: + process = mock.Mock() + process.poll.return_value = None + process.returncode = -9 + process.wait.side_effect = [ + SERVER_ALIAS.subprocess.TimeoutExpired("server", 0.01), + -9, + ] + with self.assertRaisesRegex(AssertionError, "graceful shutdown timed out"): + SERVER_ALIAS.terminate_process(process, "server", 0.01) + process.kill.assert_called_once_with() + self.assertEqual(process.wait.call_count, 2) + + def test_run_server_cleans_up_after_keyboard_interrupt(self) -> None: + with tempfile.TemporaryDirectory(prefix="alias-interrupt-") as directory: + evidence = pathlib.Path(directory) + process = mock.Mock() + with ( + mock.patch.object(SERVER_ALIAS, "assert_port_available"), + mock.patch.object( + SERVER_ALIAS.subprocess, "Popen", return_value=process + ), + mock.patch.object( + SERVER_ALIAS, "wait_for_models", side_effect=KeyboardInterrupt() + ), + mock.patch.object( + SERVER_ALIAS, "terminate_process", return_value=0 + ) as terminate, + ): + with self.assertRaises(KeyboardInterrupt): + SERVER_ALIAS.run_server( + pathlib.Path("/candidate/hebrus-server"), + pathlib.Path("/models/qualified.gguf"), + 18081, + 1.0, + 1.0, + 1.0, + evidence, + {}, + ) + terminate.assert_called_once_with(process, "hebrus-server", 1.0) + + def test_main_records_fail_and_manifest_for_keyboard_interrupt(self) -> None: + with tempfile.TemporaryDirectory(prefix="alias-main-interrupt-") as directory: + root = pathlib.Path(directory) + bin_dir = root / "bin" + bin_dir.mkdir() + (bin_dir / SERVER_ALIAS.CANONICAL_SERVER_NAME).write_bytes(b"binary") + (bin_dir / SERVER_ALIAS.LEGACY_SERVER_NAME).write_bytes(b"binary") + evidence = root / "evidence" + args = types.SimpleNamespace( + evidence_dir=evidence, + bin_dir=bin_dir, + expected_backend="metal", + expected_build_sha="123456789abc", + model=root / "qualified.gguf", + port=0, + startup_timeout=1.0, + request_timeout=1.0, + shutdown_timeout=1.0, + ) + host = { + "architecture": "arm64", + "physical_memory_bytes": 68719476736, + "os": { + "name": "macOS", + "version": "26.5.2", + "build": "25F90", + }, + "power": {}, + } + with ( + mock.patch.object(SERVER_ALIAS, "parse_args", return_value=args), + mock.patch.object(SERVER_ALIAS, "install_termination_signal_handlers"), + mock.patch.object(SERVER_ALIAS, "collect_host_metadata", return_value=host), + mock.patch.object(SERVER_ALIAS.os.path, "samefile", return_value=True), + mock.patch.object( + SERVER_ALIAS, "repository_head", side_effect=KeyboardInterrupt() + ), + ): + with self.assertRaises(KeyboardInterrupt): + SERVER_ALIAS.main() + + report = json.loads( + (evidence / "server-alias-parity.json").read_text(encoding="utf-8") + ) + self.assertEqual(report["schema_version"], 3) + self.assertEqual(report["status"], "FAIL") + self.assertEqual(report["error"], "KeyboardInterrupt: interrupted") + self.assertEqual(report["host"], host) + manifest = json.loads( + (evidence / SERVER_ALIAS.EVIDENCE_MANIFEST_NAME).read_text( + encoding="utf-8" + ) + ) + self.assertEqual( + [entry["path"] for entry in manifest["artifacts"]], + ["server-alias-parity.json"], + ) + + def test_termination_signal_is_converted_to_cleanup_exception(self) -> None: + with self.assertRaisesRegex(InterruptedError, "received signal"): + SERVER_ALIAS.termination_signal_handler(SERVER_ALIAS.signal.SIGTERM, None) + + def test_interrupt_handlers_include_sigint(self) -> None: + with mock.patch.object(SERVER_ALIAS.signal, "signal") as register: + SERVER_ALIAS.install_termination_signal_handlers() + register.assert_any_call( + SERVER_ALIAS.signal.SIGINT, SERVER_ALIAS.termination_signal_handler + ) + register.assert_any_call( + SERVER_ALIAS.signal.SIGTERM, SERVER_ALIAS.termination_signal_handler + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_visible_identity.c b/tests/test_visible_identity.c new file mode 100644 index 000000000..549208622 --- /dev/null +++ b/tests/test_visible_identity.c @@ -0,0 +1,92 @@ +#include "hebrus_identity.h" + +#include +#include +#include +#include + +static int failures; + +static void expect_string(const char *label, + const char *actual, + const char *expected) { + if (strcmp(actual, expected) == 0) return; + fprintf(stderr, + "visible-identity: %s mismatch\n expected: %s\n actual: %s\n", + label, + expected, + actual); + failures++; +} + +static void expect_agent_prompt(const char *argv0, const char *expected) { + char buf[64]; + hebrus_agent_format_prompt_for(argv0, buf, sizeof(buf)); + expect_string(argv0, buf, expected); +} + +static void expect_agent_banner(const char *argv0, + bool tty, + const char *expected) { + char buf[192]; + hebrus_agent_format_welcome_banner_for(argv0, tty, "100K", buf, sizeof(buf)); + expect_string(argv0, buf, expected); +} + +int main(void) { + const char *canonical_system = + hebrus_agent_system_prompt_for("/tmp/hebrus-agent"); + const char *legacy_system = + hebrus_agent_system_prompt_for("/tmp/ds4-agent"); + const char *expected_system = + "You are a helpful coding assistant running inside ds4-agent."; + + expect_string("canonical CLI prompt", + hebrus_cli_prompt_for("/tmp/hebrus"), + "hebrus> "); + expect_string("legacy CLI prompt", + hebrus_cli_prompt_for("/tmp/ds4"), + "ds4> "); + + expect_agent_prompt("/tmp/hebrus-agent", "hebrus-agent> "); + expect_agent_prompt("/tmp/ds4-agent", "ds4-agent> "); + expect_agent_banner( + "/tmp/hebrus-agent", + true, + "\x1b[1;97mHebrus\x1b[0m 🐋 Agent, context 100K tokens\n\n"); + expect_agent_banner( + "/tmp/ds4-agent", + true, + "\x1b[1;97mDwarf\x1b[1;94mStar\x1b[0m 🐋 Agent, context 100K tokens\n\n"); + expect_agent_banner("/tmp/hebrus-agent", + false, + "Hebrus Agent, context 100K tokens\n\n"); + expect_agent_banner("/tmp/ds4-agent", + false, + "DwarfStar Agent, context 100K tokens\n\n"); + + expect_string("canonical eval title", + hebrus_eval_command_for("/tmp/hebrus-eval"), + "hebrus-eval"); + expect_string("legacy eval title", + hebrus_eval_command_for("/tmp/ds4-eval"), + "ds4-eval"); + expect_string("canonical benchmark source", + hebrus_bench_command_for("/tmp/hebrus-bench"), + "hebrus-bench"); + expect_string("legacy benchmark source", + hebrus_bench_command_for("/tmp/ds4-bench"), + "ds4-bench"); + + expect_string("canonical agent system prompt", canonical_system, expected_system); + expect_string("legacy agent system prompt", legacy_system, expected_system); + if (strlen(canonical_system) != strlen(legacy_system) || + memcmp(canonical_system, legacy_system, strlen(canonical_system)) != 0) { + fputs("visible-identity: canonical and legacy system prompts differ\n", stderr); + failures++; + } + + if (failures) return EXIT_FAILURE; + puts("visible-identity: PASS"); + return EXIT_SUCCESS; +} diff --git a/tools/brand_boundary.json b/tools/brand_boundary.json new file mode 100644 index 000000000..118539181 --- /dev/null +++ b/tools/brand_boundary.json @@ -0,0 +1,2699 @@ +{ + "schema_version": 1, + "identity_contract": { + "brand": "Hebrus", + "canonical_commands": [ + "hebrus", + "hebrus-server", + "hebrus-agent", + "hebrus-bench", + "hebrus-eval" + ], + "command_aliases": { + "ds4": "hebrus", + "ds4-server": "hebrus-server", + "ds4-agent": "hebrus-agent", + "ds4-bench": "hebrus-bench", + "ds4-eval": "hebrus-eval" + }, + "accepted_engine_ids": [ + "hebrus", + "ds4" + ], + "environment_prefixes": { + "preserved": [ + "DS4_" + ], + "deferred": [ + "HEBRUS_" + ] + }, + "permanent_literals": [ + "ds4.expert_major.v1", + "ds4.expert_major.v2" + ], + "repositories": { + "historical_origin": "antirez/ds4", + "pre_rename": "andreaborio/ds4" + } + }, + "scope": { + "tokens": [ + "ds4", + "DS4", + "DwarfStar" + ], + "locations": [ + "path", + "content" + ], + "excluded_paths": [ + "tools/brand_boundary.json" + ] + }, + "category_definitions": { + "serialized/permanent": "Identifiers embedded in versioned fixtures, wire schemas, on-disk data, or published artifact identities that must not be renamed by the brand migration.", + "compatibility": "Legacy commands, environment names, repository identities, probes, and bridge contracts retained for existing users and integrations.", + "historical-attribution": "Fork ancestry, authorship, third-party credit, release records, and benchmark evidence that must preserve their historical spelling.", + "migration-pending": "Active source, tests, tooling, or current documentation whose internal legacy spelling remains to be handled by a later mechanical migration." + }, + "refresh_policy": { + "reductions": "python3 tools/brand_boundary_audit.py --refresh", + "existing_increase": "python3 tools/brand_boundary_audit.py --refresh --accept-increase 'PATH:LOCATION:TOKEN'", + "new_group": "python3 tools/brand_boundary_audit.py --refresh --classify 'PATH:LOCATION:TOKEN=CLASSIFICATION'" + }, + "entries": [ + { + "path": ".github/ISSUE_TEMPLATE/bug_report.yml", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 3 + }, + { + "path": ".github/ISSUE_TEMPLATE/config.yml", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 2 + }, + { + "path": ".github/ISSUE_TEMPLATE/feature_request.yml", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 2 + }, + { + "path": ".github/ISSUE_TEMPLATE/feature_request.yml", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": ".github/ISSUE_TEMPLATE/performance_report.yml", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": ".github/ISSUE_TEMPLATE/performance_report.yml", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": ".github/pull_request_template.md", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": ".github/pull_request_template.md", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": ".gitignore", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 11 + }, + { + "path": "ACKNOWLEDGMENTS.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 3 + }, + { + "path": "CHANGELOG.md", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 6 + }, + { + "path": "CHANGELOG.md", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": "CITATION.cff", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 3 + }, + { + "path": "CONTRIBUTING.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 22 + }, + { + "path": "CONTRIBUTING.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "EXPERT_PRUNE.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "EXPERT_PRUNE.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 12 + }, + { + "path": "FORK_NOTES.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 20 + }, + { + "path": "FORK_NOTES.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 2 + }, + { + "path": "FORK_NOTES.md", + "location": "content", + "token": "DwarfStar", + "classification": "historical-attribution", + "maximum": 1 + }, + { + "path": "GOLD_METAL_SSD.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "GOLD_METAL_SSD.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 6 + }, + { + "path": "GOVERNANCE.md", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 2 + }, + { + "path": "LICENSE", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 1 + }, + { + "path": "MERGE_LOG.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 29 + }, + { + "path": "MERGE_LOG.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 5 + }, + { + "path": "Makefile", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 495 + }, + { + "path": "Makefile", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 43 + }, + { + "path": "ONEDGE_IMATRIX.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "ONEDGE_IMATRIX.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "QA_BEFORE_RELEASES.md", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 26 + }, + { + "path": "QA_BEFORE_RELEASES.md", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 10 + }, + { + "path": "README.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 12 + }, + { + "path": "README.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "SECURITY.md", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": "SSD_STREAMING_VERIFICATION.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "SSD_STREAMING_VERIFICATION.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 11 + }, + { + "path": "dir-steering/README.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 12 + }, + { + "path": "dir-steering/README.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "dir-steering/out/verbosity.json", + "location": "content", + "token": "ds4", + "classification": "serialized/permanent", + "maximum": 1 + }, + { + "path": "dir-steering/tools/build_direction.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 18 + }, + { + "path": "dir-steering/tools/build_direction.py", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "dir-steering/tools/run_sweep.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 6 + }, + { + "path": "docs/ENGINE_REFERENCE.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 48 + }, + { + "path": "docs/ENGINE_REFERENCE.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 43 + }, + { + "path": "docs/HF_MODEL_RENAME_INVENTORY.md", + "location": "content", + "token": "DS4", + "classification": "serialized/permanent", + "maximum": 9 + }, + { + "path": "docs/REBRAND_ROLLOUT_PLAN.md", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 12 + }, + { + "path": "docs/REBRAND_ROLLOUT_PLAN.md", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 11 + }, + { + "path": "docs/adr/0001-expert-major-v2-only.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "docs/adr/0005-hebrus-naming-and-compatibility-boundary.md", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 7 + }, + { + "path": "docs/adr/0005-hebrus-naming-and-compatibility-boundary.md", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 4 + }, + { + "path": "docs/architecture/CODEMAP.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 47 + }, + { + "path": "docs/architecture/CODEMAP.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "docs/architecture/CODEMAP.md", + "location": "content", + "token": "DwarfStar", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "docs/architecture/hebrus-runtime-flow.svg", + "location": "content", + "token": "ds4", + "classification": "serialized/permanent", + "maximum": 1 + }, + { + "path": "docs/benchmarks/2026-07-14-m5-pro.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 14 + }, + { + "path": "docs/benchmarks/2026-07-14-m5-pro.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 5 + }, + { + "path": "docs/benchmarks/2026-07-15-qwen-ds4-vs-llamacpp.md", + "location": "path", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 1 + }, + { + "path": "docs/benchmarks/2026-07-15-qwen-ds4-vs-llamacpp.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 7 + }, + { + "path": "docs/benchmarks/2026-07-15-qwen-ds4-vs-llamacpp.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 19 + }, + { + "path": "docs/benchmarks/2026-07-17-deepseek-native-expert-major.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 9 + }, + { + "path": "docs/benchmarks/2026-07-17-deepseek-native-expert-major.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 15 + }, + { + "path": "docs/benchmarks/2026-07-17-deepseek-qwen-transfer-audit.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 2 + }, + { + "path": "docs/benchmarks/2026-07-17-deepseek-qwen-transfer-audit.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 4 + }, + { + "path": "docs/benchmarks/2026-07-17-qwen-native-expert-major.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 1 + }, + { + "path": "docs/benchmarks/2026-07-20-agent-friendly-refactor-validation.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 6 + }, + { + "path": "docs/benchmarks/2026-07-20-agent-friendly-refactor-validation.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 8 + }, + { + "path": "docs/benchmarks/2026-07-20-glm52-expert-major-v2.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 2 + }, + { + "path": "docs/benchmarks/2026-07-20-glm52-expert-major-v2.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 7 + }, + { + "path": "docs/benchmarks/2026-07-20-long-context-metal-stack.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 8 + }, + { + "path": "docs/benchmarks/2026-07-20-qwen-expert-major-v2.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 2 + }, + { + "path": "docs/benchmarks/2026-07-20-qwen-expert-major-v2.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 4 + }, + { + "path": "docs/benchmarks/2026-07-21-qwen-f32-router-nax-prefill.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 6 + }, + { + "path": "docs/benchmarks/2026-07-21-qwen-mlx-affine-nax-prefill.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 4 + }, + { + "path": "docs/benchmarks/2026-07-21-qwen-pre-m5-exact-router-tile.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 6 + }, + { + "path": "docs/benchmarks/2026-07-21-qwen-pre-m5-exact-router-tile.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 3 + }, + { + "path": "docs/benchmarks/2026-07-21-qwen-q4-paired-mmid-prefill.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 1 + }, + { + "path": "docs/benchmarks/2026-07-21-qwen-q4-paired-mmid-prefill.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 1 + }, + { + "path": "docs/benchmarks/2026-07-21-qwen-split-k-hardware-policy.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 6 + }, + { + "path": "docs/benchmarks/2026-07-21-qwen-unified-affine-auto-ssd.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 2 + }, + { + "path": "docs/benchmarks/2026-07-21-qwen-unified-affine-auto-ssd.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 4 + }, + { + "path": "docs/benchmarks/2026-07-21-upstream-metal-transfer-audit.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 11 + }, + { + "path": "docs/benchmarks/2026-07-22-qwen-kv-pair-blit.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 8 + }, + { + "path": "docs/benchmarks/2026-07-22-qwen-kv-pair-blit.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 1 + }, + { + "path": "docs/benchmarks/2026-07-22-qwen-ssd-flash-prefill.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 2 + }, + { + "path": "docs/benchmarks/README.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 2 + }, + { + "path": "docs/contracts/BRAND_COMPATIBILITY.md", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 20 + }, + { + "path": "docs/contracts/BRAND_COMPATIBILITY.md", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 2 + }, + { + "path": "docs/contracts/RUNTIME_SUPPORT.md", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": "docs/contracts/RUNTIME_SUPPORT.md", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 5 + }, + { + "path": "docs/contracts/qwen-release.json", + "location": "content", + "token": "DS4", + "classification": "serialized/permanent", + "maximum": 3 + }, + { + "path": "docs/deepseek-expert-major-v2.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 4 + }, + { + "path": "docs/deepseek-expert-major-v2.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 6 + }, + { + "path": "docs/expert-major-v2-roadmap.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "docs/expert-major-v2-roadmap.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "docs/glm52-expert-major-v2.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "docs/glm52-expert-major-v2.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 11 + }, + { + "path": "docs/guides/MIGRATING_TO_HEBRUS.md", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 24 + }, + { + "path": "docs/guides/MIGRATING_TO_HEBRUS.md", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 6 + }, + { + "path": "docs/qwen-expert-major-store.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 8 + }, + { + "path": "docs/qwen-expert-major-store.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "docs/releases/hebrus-bridge-launch-candidate.md", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 8 + }, + { + "path": "docs/releases/hebrus-bridge-launch-candidate.md", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 2 + }, + { + "path": "docs/releases/v0.1.0.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 7 + }, + { + "path": "docs/releases/v0.1.0.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 3 + }, + { + "path": "docs/releases/v0.1.0.md", + "location": "content", + "token": "DwarfStar", + "classification": "historical-attribution", + "maximum": 1 + }, + { + "path": "docs/releases/v0.2.0.md", + "location": "content", + "token": "ds4", + "classification": "historical-attribution", + "maximum": 2 + }, + { + "path": "docs/releases/v0.2.0.md", + "location": "content", + "token": "DS4", + "classification": "historical-attribution", + "maximum": 4 + }, + { + "path": "docs/releases/v0.2.0.md", + "location": "content", + "token": "DwarfStar", + "classification": "historical-attribution", + "maximum": 1 + }, + { + "path": "download_model.sh", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 6 + }, + { + "path": "download_model.sh", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 10 + }, + { + "path": "ds4.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 5227 + }, + { + "path": "ds4.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 4470 + }, + { + "path": "ds4.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 320 + }, + { + "path": "ds4.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 40 + }, + { + "path": "ds4_agent.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_agent.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 353 + }, + { + "path": "ds4_agent.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 45 + }, + { + "path": "ds4_bench.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_bench.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 147 + }, + { + "path": "ds4_bench.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 21 + }, + { + "path": "ds4_build.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_build.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 26 + }, + { + "path": "ds4_build.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 20 + }, + { + "path": "ds4_cli.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_cli.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 403 + }, + { + "path": "ds4_cli.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 45 + }, + { + "path": "ds4_eval.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_eval.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 158 + }, + { + "path": "ds4_eval.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 22 + }, + { + "path": "ds4_expert_store.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_expert_store.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 31 + }, + { + "path": "ds4_expert_store.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 13 + }, + { + "path": "ds4_expert_store.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_expert_store.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 29 + }, + { + "path": "ds4_expert_store.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 21 + }, + { + "path": "ds4_gpu.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_gpu.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 726 + }, + { + "path": "ds4_gpu.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 13 + }, + { + "path": "ds4_help.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_help.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 32 + }, + { + "path": "ds4_help.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 81 + }, + { + "path": "ds4_help.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_help.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "ds4_help.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 8 + }, + { + "path": "ds4_kvstore.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_kvstore.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 233 + }, + { + "path": "ds4_kvstore.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 52 + }, + { + "path": "ds4_kvstore.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_kvstore.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 101 + }, + { + "path": "ds4_kvstore.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 20 + }, + { + "path": "ds4_metal.m", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_metal.m", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 6662 + }, + { + "path": "ds4_metal.m", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 1070 + }, + { + "path": "ds4_profile.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_profile.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 13 + }, + { + "path": "ds4_profile.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 10 + }, + { + "path": "ds4_profile.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_profile.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 8 + }, + { + "path": "ds4_profile.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 4 + }, + { + "path": "ds4_qwen.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_qwen.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 51 + }, + { + "path": "ds4_qwen.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_qwen.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 44 + }, + { + "path": "ds4_qwen.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "ds4_qwen_expert_group.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_qwen_expert_group.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 31 + }, + { + "path": "ds4_qwen_expert_group.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 21 + }, + { + "path": "ds4_qwen_expert_group.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_qwen_expert_group.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 26 + }, + { + "path": "ds4_qwen_expert_group.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 8 + }, + { + "path": "ds4_qwen_ref.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_qwen_ref.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 12 + }, + { + "path": "ds4_qwen_ref.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_qwen_ref.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 11 + }, + { + "path": "ds4_qwen_ref.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "ds4_qwen_unicode.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_qwen_unicode.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "ds4_qwen_unicode.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "ds4_qwen_unicode.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_qwen_unicode.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 6 + }, + { + "path": "ds4_qwen_unicode.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 6 + }, + { + "path": "ds4_qwen_unicode_data.inc", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_server.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_server.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 476 + }, + { + "path": "ds4_server.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 565 + }, + { + "path": "ds4_ssd.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_ssd.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 71 + }, + { + "path": "ds4_ssd.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 83 + }, + { + "path": "ds4_ssd.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_ssd.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 74 + }, + { + "path": "ds4_ssd.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 17 + }, + { + "path": "ds4_streaming_hotlist.inc", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_streaming_hotlist.inc", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "ds4_streaming_hotlist_glm52.inc", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_streaming_hotlist_glm52.inc", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 4 + }, + { + "path": "ds4_web.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_web.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 30 + }, + { + "path": "ds4_web.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 17 + }, + { + "path": "ds4_web.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "ds4_web.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 18 + }, + { + "path": "ds4_web.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "gguf-tools/Makefile", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 32 + }, + { + "path": "gguf-tools/Makefile", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "gguf-tools/README.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 14 + }, + { + "path": "gguf-tools/README.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 11 + }, + { + "path": "gguf-tools/deepseek4-quantize.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 47 + }, + { + "path": "gguf-tools/deepseek4-quantize.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 54 + }, + { + "path": "gguf-tools/ds4-expert-major.py", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "gguf-tools/ds4-expert-major.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "gguf-tools/ds4-expert-major.py", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "gguf-tools/imatrix/README.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "gguf-tools/imatrix/README.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 12 + }, + { + "path": "gguf-tools/imatrix/dataset/README.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "gguf-tools/imatrix/dataset/README.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "gguf-tools/imatrix/dataset/build_ds4_imatrix_dataset.py", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "gguf-tools/imatrix/dataset/build_ds4_imatrix_dataset.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 131 + }, + { + "path": "gguf-tools/imatrix/dataset/build_ds4_imatrix_dataset.py", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 8 + }, + { + "path": "gguf-tools/imatrix/dataset/build_ds4_imatrix_dataset.py", + "location": "content", + "token": "DwarfStar", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "gguf-tools/quality-testing/README.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "gguf-tools/quality-testing/score_official.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 28 + }, + { + "path": "gguf-tools/quality-testing/score_official.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "gguf-tools/quants.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 146 + }, + { + "path": "gguf-tools/quants.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 89 + }, + { + "path": "gguf-tools/quants.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 20 + }, + { + "path": "gguf-tools/quants.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 40 + }, + { + "path": "hebrus_identity.h", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 5 + }, + { + "path": "hebrus_identity.h", + "location": "content", + "token": "DwarfStar", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": "linenoise.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "metal/argsort.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 8 + }, + { + "path": "metal/argsort.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 10 + }, + { + "path": "metal/bin.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "metal/bin.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "metal/concat.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/concat.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/cpy.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/cpy.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/dense.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 29 + }, + { + "path": "metal/dense.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 13 + }, + { + "path": "metal/dsv4_hc.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 27 + }, + { + "path": "metal/dsv4_hc.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "metal/dsv4_kv.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 10 + }, + { + "path": "metal/dsv4_kv.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "metal/dsv4_misc.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 79 + }, + { + "path": "metal/dsv4_misc.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 15 + }, + { + "path": "metal/dsv4_rope.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/dsv4_rope.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/flash_attn.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 11 + }, + { + "path": "metal/flash_attn.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 6 + }, + { + "path": "metal/get_rows.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 4 + }, + { + "path": "metal/get_rows.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/glu.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "metal/glu.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "metal/moe.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 231 + }, + { + "path": "metal/moe.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 11 + }, + { + "path": "metal/norm.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "metal/norm.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "metal/qwen35.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 42 + }, + { + "path": "metal/qwen35.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/repeat.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/repeat.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/set_rows.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/set_rows.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/softmax.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "metal/softmax.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "metal/sum_rows.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/sum_rows.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "metal/unary.metal", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "metal/unary.metal", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 4 + }, + { + "path": "runtime/ds4_deepseek_cache_phase.inc", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "runtime/ds4_deepseek_cache_phase.inc", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 29 + }, + { + "path": "runtime/ds4_deepseek_cache_phase.inc", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "runtime/ds4_glm_graph.inc", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "runtime/ds4_glm_graph.inc", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 779 + }, + { + "path": "runtime/ds4_glm_graph.inc", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 774 + }, + { + "path": "runtime/ds4_metal_glm.inc", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "runtime/ds4_metal_glm.inc", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1104 + }, + { + "path": "runtime/ds4_metal_glm.inc", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 58 + }, + { + "path": "runtime/ds4_qwen_memory_policy.inc", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "runtime/ds4_qwen_memory_policy.inc", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 14 + }, + { + "path": "runtime/ds4_qwen_memory_policy.inc", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "speed-bench/README.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 4 + }, + { + "path": "speed-bench/README.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "speed-bench/build_long_context_prompt.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "speed-bench/plot_speed.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "speed-bench/run_m5_dsflash_arm.sh", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 11 + }, + { + "path": "speed-bench/run_m5_dsflash_arm.sh", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 59 + }, + { + "path": "tests/ds4_agent_test.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/ds4_agent_test.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 4 + }, + { + "path": "tests/ds4_test.c", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/ds4_test.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 810 + }, + { + "path": "tests/ds4_test.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 151 + }, + { + "path": "tests/gen_qwen_unicode.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "tests/generate_long_context_story_prompt.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/glm_long_context_smoke.sh", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 4 + }, + { + "path": "tests/glm_long_context_smoke.sh", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 10 + }, + { + "path": "tests/internal/ds4_agent_unit.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/internal/ds4_agent_unit.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/internal/ds4_agent_unit.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "tests/internal/ds4_agent_unit.inc", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/internal/ds4_agent_unit.inc", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/internal/ds4_qwen_cpu_test_hooks.h", + "location": "path", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/internal/ds4_qwen_cpu_test_hooks.h", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 7 + }, + { + "path": "tests/internal/ds4_qwen_cpu_test_hooks.h", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 4 + }, + { + "path": "tests/qwen/README.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 6 + }, + { + "path": "tests/qwen/README.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 6 + }, + { + "path": "tests/qwen/UNICODE_DATA_PROVENANCE.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/qwen/compare_logits.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 81 + }, + { + "path": "tests/qwen/compare_logits.py", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 25 + }, + { + "path": "tests/qwen/qwen_unicode_ucd_cache.txt", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/qwen/test_compare_logits.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 39 + }, + { + "path": "tests/qwen/test_qwen35_metal.m", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "tests/qwen/test_qwen35_metal.m", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "tests/test-vectors/README.md", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "tests/test-vectors/README.md", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "tests/test-vectors/fetch_official_vectors.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "tests/test-vectors/local-golden.vec", + "location": "content", + "token": "ds4", + "classification": "serialized/permanent", + "maximum": 1 + }, + { + "path": "tests/test-vectors/manifest.json", + "location": "content", + "token": "ds4", + "classification": "serialized/permanent", + "maximum": 1 + }, + { + "path": "tests/test-vectors/official.vec", + "location": "content", + "token": "ds4", + "classification": "serialized/permanent", + "maximum": 1 + }, + { + "path": "tests/test-vectors/official/long_code_audit.official.json", + "location": "content", + "token": "ds4", + "classification": "serialized/permanent", + "maximum": 1 + }, + { + "path": "tests/test-vectors/official/long_memory_archive.official.json", + "location": "content", + "token": "ds4", + "classification": "serialized/permanent", + "maximum": 1 + }, + { + "path": "tests/test-vectors/official/short_code_completion.official.json", + "location": "content", + "token": "ds4", + "classification": "serialized/permanent", + "maximum": 1 + }, + { + "path": "tests/test-vectors/official/short_italian_fact.official.json", + "location": "content", + "token": "ds4", + "classification": "serialized/permanent", + "maximum": 1 + }, + { + "path": "tests/test-vectors/official/short_reasoning_plain.official.json", + "location": "content", + "token": "ds4", + "classification": "serialized/permanent", + "maximum": 1 + }, + { + "path": "tests/test_benchmark_env_guard.sh", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "tests/test_brand_boundary_audit.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 31 + }, + { + "path": "tests/test_brand_boundary_audit.py", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "tests/test_brand_boundary_audit.py", + "location": "content", + "token": "DwarfStar", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "tests/test_build_isolation.sh", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "tests/test_capabilities.py", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 7 + }, + { + "path": "tests/test_command_aliases.py", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 9 + }, + { + "path": "tests/test_command_aliases.py", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": "tests/test_download_model.sh", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 5 + }, + { + "path": "tests/test_download_model.sh", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 12 + }, + { + "path": "tests/test_expert_major.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "tests/test_expert_major.py", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/test_expert_store.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 13 + }, + { + "path": "tests/test_expert_store.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "tests/test_install.sh", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 10 + }, + { + "path": "tests/test_metal_ssd_profile.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 16 + }, + { + "path": "tests/test_metal_ssd_profile.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 19 + }, + { + "path": "tests/test_q4k_dot.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 1 + }, + { + "path": "tests/test_q4k_dot.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "tests/test_q4k_top8.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 8 + }, + { + "path": "tests/test_qwen_attention_ref.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 27 + }, + { + "path": "tests/test_qwen_expert_group.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 63 + }, + { + "path": "tests/test_qwen_expert_group.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 44 + }, + { + "path": "tests/test_qwen_gdn_ref.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 31 + }, + { + "path": "tests/test_qwen_metadata.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 10 + }, + { + "path": "tests/test_qwen_metadata.py", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 7 + }, + { + "path": "tests/test_qwen_session.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 171 + }, + { + "path": "tests/test_qwen_session.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 62 + }, + { + "path": "tests/test_qwen_state.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 56 + }, + { + "path": "tests/test_qwen_tokenizer.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 252 + }, + { + "path": "tests/test_qwen_tokenizer.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 103 + }, + { + "path": "tests/test_qwen_unicode.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 29 + }, + { + "path": "tests/test_qwen_unicode.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 9 + }, + { + "path": "tests/test_retired_distributed_flags.sh", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 5 + }, + { + "path": "tests/test_retired_distributed_flags.sh", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": "tests/test_server_alias_model.py", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 3 + }, + { + "path": "tests/test_server_alias_model.py", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 6 + }, + { + "path": "tests/test_server_alias_model_unit.py", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 4 + }, + { + "path": "tests/test_server_alias_model_unit.py", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 6 + }, + { + "path": "tests/test_ssd_residency.c", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 226 + }, + { + "path": "tests/test_ssd_residency.c", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 53 + }, + { + "path": "tests/test_visible_identity.c", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 12 + }, + { + "path": "tests/test_visible_identity.c", + "location": "content", + "token": "DwarfStar", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": "tools/brand_boundary_audit.py", + "location": "content", + "token": "ds4", + "classification": "migration-pending", + "maximum": 12 + }, + { + "path": "tools/brand_boundary_audit.py", + "location": "content", + "token": "DS4", + "classification": "migration-pending", + "maximum": 3 + }, + { + "path": "tools/brand_boundary_audit.py", + "location": "content", + "token": "DwarfStar", + "classification": "migration-pending", + "maximum": 2 + }, + { + "path": "tools/context_audit.py", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 17 + }, + { + "path": "tools/context_audit.py", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 6 + }, + { + "path": "tools/context_budget.json", + "location": "content", + "token": "ds4", + "classification": "compatibility", + "maximum": 1 + }, + { + "path": "tools/context_budget.json", + "location": "content", + "token": "DS4", + "classification": "compatibility", + "maximum": 9 + } + ] +} diff --git a/tools/brand_boundary_audit.py b/tools/brand_boundary_audit.py new file mode 100644 index 000000000..06e5724cd --- /dev/null +++ b/tools/brand_boundary_audit.py @@ -0,0 +1,556 @@ +#!/usr/bin/env python3 +"""Enforce the explicit legacy-brand inventory during the Hebrus migration.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + + +DEFAULT_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_MANIFEST = "tools/brand_boundary.json" +SCHEMA_VERSION = 1 +TOKENS = ("ds4", "DS4", "DwarfStar") +LOCATIONS = ("path", "content") +CATEGORIES = ( + "serialized/permanent", + "compatibility", + "historical-attribution", + "migration-pending", +) +IDENTITY_CONTRACT = { + "brand": "Hebrus", + "canonical_commands": [ + "hebrus", + "hebrus-server", + "hebrus-agent", + "hebrus-bench", + "hebrus-eval", + ], + "command_aliases": { + "ds4": "hebrus", + "ds4-server": "hebrus-server", + "ds4-agent": "hebrus-agent", + "ds4-bench": "hebrus-bench", + "ds4-eval": "hebrus-eval", + }, + "accepted_engine_ids": ["hebrus", "ds4"], + "environment_prefixes": { + "preserved": ["DS4_"], + "deferred": ["HEBRUS_"], + }, + "permanent_literals": ["ds4.expert_major.v1", "ds4.expert_major.v2"], + "repositories": { + "historical_origin": "antirez/ds4", + "pre_rename": "andreaborio/ds4", + }, +} +TOP_LEVEL_KEYS = { + "schema_version", + "identity_contract", + "scope", + "category_definitions", + "refresh_policy", + "entries", +} +ENTRY_KEYS = {"path", "location", "token", "classification", "maximum"} +GLOB_CHARS = frozenset("*?[]{}") + + +class AuditError(Exception): + """A malformed manifest or unusable checkout.""" + + +@dataclass(frozen=True) +class Identity: + path: str + location: str + token: str + + +def identity_key(identity: Identity) -> tuple[str, int, int]: + return ( + identity.path, + LOCATIONS.index(identity.location), + TOKENS.index(identity.token), + ) + + +def identity_text(identity: Identity) -> str: + return f"{identity.path}:{identity.location}:{identity.token}" + + +def path_is_explicit(path: str) -> bool: + if not path or any(char in path for char in GLOB_CHARS): + return False + pure = PurePosixPath(path) + return not pure.is_absolute() and str(pure) == path and ".." not in pure.parts + + +def expected_exclusions(root: Path, manifest_path: Path) -> list[str]: + try: + relative = manifest_path.resolve().relative_to(root.resolve()) + except ValueError: + return [] + return [relative.as_posix()] + + +def require_exact_keys(value: dict[str, Any], expected: set[str], context: str) -> None: + actual = set(value) + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + details = [] + if missing: + details.append("missing " + ", ".join(missing)) + if extra: + details.append("unknown " + ", ".join(extra)) + raise AuditError(f"{context} has invalid keys ({'; '.join(details)})") + + +def load_manifest( + root: Path, manifest_path: Path +) -> tuple[dict[str, Any], dict[Identity, dict[str, Any]]]: + try: + document = json.loads(manifest_path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise AuditError(f"manifest does not exist: {manifest_path}") from exc + except (OSError, json.JSONDecodeError) as exc: + raise AuditError(f"cannot read manifest {manifest_path}: {exc}") from exc + + if not isinstance(document, dict): + raise AuditError("manifest root must be a JSON object") + require_exact_keys(document, TOP_LEVEL_KEYS, "manifest") + if ( + isinstance(document["schema_version"], bool) + or not isinstance(document["schema_version"], int) + or document["schema_version"] != SCHEMA_VERSION + ): + raise AuditError( + f"manifest schema_version must be {SCHEMA_VERSION}, " + f"got {document['schema_version']!r}" + ) + if document["identity_contract"] != IDENTITY_CONTRACT: + raise AuditError( + "identity_contract must exactly match the canonical, bridged, and " + "permanently preserved identity contract" + ) + + scope = document["scope"] + if not isinstance(scope, dict): + raise AuditError("manifest scope must be an object") + require_exact_keys(scope, {"tokens", "locations", "excluded_paths"}, "scope") + if scope["tokens"] != list(TOKENS): + raise AuditError(f"scope.tokens must be exactly {list(TOKENS)!r}") + if scope["locations"] != list(LOCATIONS): + raise AuditError(f"scope.locations must be exactly {list(LOCATIONS)!r}") + exclusions = expected_exclusions(root, manifest_path) + if scope["excluded_paths"] != exclusions: + raise AuditError( + "scope.excluded_paths may contain only the manifest itself: " + f"expected {exclusions!r}" + ) + + definitions = document["category_definitions"] + if not isinstance(definitions, dict) or set(definitions) != set(CATEGORIES): + raise AuditError( + "category_definitions must define exactly: " + ", ".join(CATEGORIES) + ) + if any(not isinstance(definitions[name], str) or not definitions[name].strip() + for name in CATEGORIES): + raise AuditError("every category definition must be a non-empty string") + + policy = document["refresh_policy"] + if not isinstance(policy, dict): + raise AuditError("refresh_policy must be an object") + require_exact_keys( + policy, + {"reductions", "existing_increase", "new_group"}, + "refresh_policy", + ) + if any(not isinstance(value, str) or not value.strip() for value in policy.values()): + raise AuditError("every refresh_policy command must be a non-empty string") + + raw_entries = document["entries"] + if not isinstance(raw_entries, list): + raise AuditError("manifest entries must be an array") + + entries: dict[Identity, dict[str, Any]] = {} + order: list[Identity] = [] + for index, entry in enumerate(raw_entries): + context = f"entries[{index}]" + if not isinstance(entry, dict): + raise AuditError(f"{context} must be an object") + require_exact_keys(entry, ENTRY_KEYS, context) + path = entry["path"] + location = entry["location"] + token = entry["token"] + classification = entry["classification"] + maximum = entry["maximum"] + if not isinstance(path, str) or not path_is_explicit(path): + raise AuditError(f"{context}.path must be an explicit normalized path") + if path in exclusions: + raise AuditError(f"{context}.path cannot be the excluded manifest") + if location not in LOCATIONS: + raise AuditError(f"{context}.location must be one of {LOCATIONS!r}") + if token not in TOKENS: + raise AuditError(f"{context}.token must be one of {TOKENS!r}") + if classification not in CATEGORIES: + raise AuditError( + f"{context}.classification must be one of {CATEGORIES!r}" + ) + if isinstance(maximum, bool) or not isinstance(maximum, int) or maximum < 1: + raise AuditError(f"{context}.maximum must be a positive integer") + identity = Identity(path, location, token) + if identity in entries: + raise AuditError(f"duplicate entry: {identity_text(identity)}") + entries[identity] = entry + order.append(identity) + + if order != sorted(order, key=identity_key): + raise AuditError("manifest entries must be in deterministic path/location/token order") + return document, entries + + +def tracked_paths(root: Path) -> list[str]: + try: + result = subprocess.run( + ["git", "ls-files", "-z"], + cwd=root, + check=True, + capture_output=True, + ) + except (OSError, subprocess.CalledProcessError) as exc: + raise AuditError(f"cannot enumerate tracked files under {root}: {exc}") from exc + return sorted( + item.decode("utf-8", "surrogateescape") + for item in result.stdout.split(b"\0") + if item + ) + + +def worktree_bytes(path: Path) -> bytes | None: + try: + if path.is_symlink(): + return os.readlink(path).encode("utf-8", "surrogateescape") + if not path.is_file(): + return None + return path.read_bytes() + except OSError as exc: + raise AuditError(f"cannot read tracked path {path}: {exc}") from exc + + +def count_tokens(blob: bytes) -> dict[str, int]: + counts = {} + for token in TOKENS: + count = blob.count(token.encode("ascii")) + if count: + counts[token] = count + return counts + + +def scan_tree(root: Path, excluded_paths: set[str]) -> dict[Identity, int]: + observations: dict[Identity, int] = {} + for relative in tracked_paths(root): + if relative in excluded_paths: + continue + path_counts = count_tokens(relative.encode("utf-8", "surrogateescape")) + for token, count in path_counts.items(): + observations[Identity(relative, "path", token)] = count + + content = worktree_bytes(root / relative) + if content is None: + continue + for token, count in count_tokens(content).items(): + observations[Identity(relative, "content", token)] = count + return observations + + +def parse_identity(value: str) -> Identity: + try: + path, location, token = value.rsplit(":", 2) + except ValueError as exc: + raise AuditError( + f"invalid identity {value!r}; expected PATH:LOCATION:TOKEN" + ) from exc + identity = Identity(path, location, token) + if not path_is_explicit(path): + raise AuditError(f"identity path must be explicit, not a glob: {path!r}") + if location not in LOCATIONS: + raise AuditError(f"identity location must be one of {LOCATIONS!r}") + if token not in TOKENS: + raise AuditError(f"identity token must be one of {TOKENS!r}") + return identity + + +def parse_classification(value: str) -> tuple[Identity, str]: + try: + identity_value, category = value.rsplit("=", 1) + except ValueError as exc: + raise AuditError( + f"invalid classification {value!r}; expected " + "PATH:LOCATION:TOKEN=CATEGORY" + ) from exc + if category not in CATEGORIES: + raise AuditError(f"classification must be one of {CATEGORIES!r}") + return parse_identity(identity_value), category + + +def render_violations( + unknown: list[Identity], + increases: list[tuple[Identity, int, int]], +) -> None: + print("brand boundary audit failed:", file=sys.stderr) + for identity in unknown: + print( + f"- unclassified brand token group: {identity_text(identity)}", + file=sys.stderr, + ) + for identity, actual, maximum in increases: + print( + f"- brand token count increased: {identity_text(identity)} " + f"{actual} > {maximum}", + file=sys.stderr, + ) + + +def inventory_delta( + observations: dict[Identity, int], + entries: dict[Identity, dict[str, Any]], +) -> tuple[list[Identity], list[tuple[Identity, int, int]], list[Identity]]: + unknown = sorted(set(observations) - set(entries), key=identity_key) + increases = sorted( + ( + (identity, actual, entries[identity]["maximum"]) + for identity, actual in observations.items() + if identity in entries and actual > entries[identity]["maximum"] + ), + key=lambda item: identity_key(item[0]), + ) + reductions = sorted( + ( + identity + for identity, entry in entries.items() + if observations.get(identity, 0) < entry["maximum"] + ), + key=identity_key, + ) + return unknown, increases, reductions + + +def summary( + observations: dict[Identity, int], + entries: dict[Identity, dict[str, Any]], + reductions: list[Identity], +) -> str: + totals = {category: 0 for category in CATEGORIES} + for identity, actual in observations.items(): + entry = entries.get(identity) + if entry: + totals[entry["classification"]] += actual + categories = ", ".join(f"{name}={totals[name]}" for name in CATEGORIES) + return ( + f"{sum(observations.values())} occurrences in {len(observations)} groups; " + f"{len(reductions)} reductions; {categories}" + ) + + +def check_inventory( + observations: dict[Identity, int], + entries: dict[Identity, dict[str, Any]], +) -> int: + unknown, increases, reductions = inventory_delta(observations, entries) + if unknown or increases: + render_violations(unknown, increases) + return 1 + print("brand boundary audit passed: " + summary(observations, entries, reductions)) + return 0 + + +def parse_refresh_authorizations( + raw_increases: list[str], raw_classifications: list[str] +) -> tuple[set[Identity], dict[Identity, str]]: + accepted: set[Identity] = set() + for value in raw_increases: + identity = parse_identity(value) + if identity in accepted: + raise AuditError(f"duplicate --accept-increase: {identity_text(identity)}") + accepted.add(identity) + + classified: dict[Identity, str] = {} + for value in raw_classifications: + identity, category = parse_classification(value) + if identity in classified: + raise AuditError(f"duplicate --classify: {identity_text(identity)}") + classified[identity] = category + return accepted, classified + + +def refresh_inventory( + manifest_path: Path, + document: dict[str, Any], + observations: dict[Identity, int], + entries: dict[Identity, dict[str, Any]], + raw_increases: list[str], + raw_classifications: list[str], +) -> int: + unknown, increases, reductions = inventory_delta(observations, entries) + accepted, classified = parse_refresh_authorizations( + raw_increases, raw_classifications + ) + increase_ids = {identity for identity, _, _ in increases} + unknown_ids = set(unknown) + + unused_increases = accepted - increase_ids + unused_classifications = set(classified) - unknown_ids + if unused_increases: + raise AuditError( + "--accept-increase does not match a current increase: " + + ", ".join(identity_text(item) for item in sorted(unused_increases, key=identity_key)) + ) + if unused_classifications: + raise AuditError( + "--classify may name only a current unclassified group: " + + ", ".join( + identity_text(item) + for item in sorted(unused_classifications, key=identity_key) + ) + ) + + missing_increases = increase_ids - accepted + missing_classifications = unknown_ids - set(classified) + if missing_increases or missing_classifications: + render_violations( + sorted(missing_classifications, key=identity_key), + [item for item in increases if item[0] in missing_increases], + ) + print( + "refresh refused: authorize every increase with --accept-increase " + "and every new group with --classify", + file=sys.stderr, + ) + return 1 + + refreshed_entries = [] + for identity in sorted(observations, key=identity_key): + category = ( + entries[identity]["classification"] + if identity in entries + else classified[identity] + ) + refreshed_entries.append( + { + "path": identity.path, + "location": identity.location, + "token": identity.token, + "classification": category, + "maximum": observations[identity], + } + ) + + refreshed = dict(document) + refreshed["entries"] = refreshed_entries + payload = json.dumps(refreshed, indent=2, ensure_ascii=True) + "\n" + temporary = manifest_path.with_name(manifest_path.name + ".tmp") + try: + temporary.write_text(payload, encoding="utf-8") + os.replace(temporary, manifest_path) + except OSError as exc: + try: + temporary.unlink(missing_ok=True) + except OSError: + pass + raise AuditError(f"cannot update manifest {manifest_path}: {exc}") from exc + + refreshed_map = { + Identity(entry["path"], entry["location"], entry["token"]): entry + for entry in refreshed_entries + } + print( + "brand boundary manifest refreshed: " + + summary(observations, refreshed_map, reductions) + ) + return 0 + + +def parser() -> argparse.ArgumentParser: + argument_parser = argparse.ArgumentParser( + description=( + "Check the exact tracked ds4/DS4/DwarfStar inventory. Plain refresh " + "only tightens the baseline; widening requires exact authorizations." + ) + ) + action = argument_parser.add_mutually_exclusive_group(required=True) + action.add_argument("--check", action="store_true", help="verify the inventory") + action.add_argument( + "--refresh", + action="store_true", + help="rewrite the manifest deterministically after explicit review", + ) + argument_parser.add_argument( + "--root", + help="repository root, defaulting to the checkout containing this tool", + ) + argument_parser.add_argument( + "--manifest", + help=f"manifest path relative to --root (default: {DEFAULT_MANIFEST})", + ) + argument_parser.add_argument( + "--accept-increase", + action="append", + default=[], + metavar="PATH:LOCATION:TOKEN", + help="authorize one existing group whose count increased; repeat as needed", + ) + argument_parser.add_argument( + "--classify", + action="append", + default=[], + metavar="PATH:LOCATION:TOKEN=CATEGORY", + help="classify one new exact group; repeat as needed", + ) + return argument_parser + + +def main(argv: list[str] | None = None) -> int: + arguments = parser().parse_args(argv) + if arguments.check and (arguments.accept_increase or arguments.classify): + print( + "brand boundary audit error: refresh authorizations require --refresh", + file=sys.stderr, + ) + return 2 + + root = Path(arguments.root).resolve() if arguments.root else DEFAULT_ROOT + manifest_path = Path(arguments.manifest) if arguments.manifest else Path(DEFAULT_MANIFEST) + if not manifest_path.is_absolute(): + manifest_path = root / manifest_path + manifest_path = manifest_path.resolve() + + try: + document, entries = load_manifest(root, manifest_path) + exclusions = set(document["scope"]["excluded_paths"]) + observations = scan_tree(root, exclusions) + if arguments.check: + return check_inventory(observations, entries) + return refresh_inventory( + manifest_path, + document, + observations, + entries, + arguments.accept_increase, + arguments.classify, + ) + except AuditError as exc: + print(f"brand boundary audit error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/context_audit.py b/tools/context_audit.py index eed713241..f51b0a40a 100644 --- a/tools/context_audit.py +++ b/tools/context_audit.py @@ -119,6 +119,7 @@ def source_metrics(files: list[Path]) -> tuple[dict[str, int], dict[str, list[st "largest_source_lines": largest_lines, } details = { + "direct_ds4_env_names": sorted(env_names), "implementation_includes": implementation_includes, "maybe_unused_markers": maybe_unused, "personal_absolute_path_hits": personal_paths, @@ -292,11 +293,55 @@ def main() -> int: issues.append(f"missing context budget: {args.budget}") else: budget = json.loads(args.budget.read_text(encoding="utf-8")) - for name, maximum in budget.items(): + for name, policy in budget.items(): actual = metrics.get(name) if actual is None: issues.append(f"unknown context budget metric: {name}") - elif actual > maximum: + continue + + maximum = policy + if isinstance(policy, dict): + maximum = policy.get("maximum") + baseline = policy.get("baseline") + reason = policy.get("reason") + classified = policy.get("accepted_additions") + if not isinstance(maximum, int): + issues.append(f"{name} context budget needs an integer maximum") + continue + if not isinstance(baseline, int): + issues.append(f"{name} classified context budget needs an integer baseline") + if not isinstance(reason, str) or not reason.strip(): + issues.append(f"{name} classified context budget needs a reason") + if not isinstance(classified, dict) or not classified: + issues.append(f"{name} classified context budget needs accepted additions") + else: + additions: list[str] = [] + for category, names in classified.items(): + if not isinstance(category, str) or not isinstance(names, list) or not all( + isinstance(item, str) for item in names + ): + issues.append(f"{name} has an invalid accepted-addition category") + continue + additions.extend(names) + if len(additions) != len(set(additions)): + issues.append(f"{name} accepted additions contain duplicates") + if isinstance(baseline, int) and baseline + len(set(additions)) != maximum: + issues.append( + f"{name} classified budget does not match baseline plus accepted additions" + ) + if name == "direct_ds4_env_names": + observed = set(details["direct_ds4_env_names"]) + missing = sorted(set(additions) - observed) + if missing: + issues.append( + f"{name} accepted additions are not direct source reads: " + + ", ".join(missing) + ) + elif not isinstance(policy, int): + issues.append(f"{name} context budget must be an integer or policy object") + continue + + if actual > maximum: issues.append(f"{name} increased: {actual} > budget {maximum}") print(json.dumps(metrics, indent=2, sort_keys=True)) diff --git a/tools/context_budget.json b/tools/context_budget.json index 98d912cef..c7ea3f0a5 100644 --- a/tools/context_budget.json +++ b/tools/context_budget.json @@ -1,7 +1,44 @@ { - "direct_ds4_env_names": 334, + "direct_ds4_env_names": { + "maximum": 333, + "baseline": 324, + "reason": "Accepted Qwen Metal benchmark, diagnostic-bisect, and profiling controls added with the affine runtime; normal release startup remains flag-free and every later direct name still exceeds this ceiling.", + "accepted_additions": { + "diagnostic_bisect": [ + "DS4_METAL_DISABLE_F32_NAX_PREFILL", + "DS4_METAL_DISABLE_MLX_AFFINE_NAX", + "DS4_METAL_DISABLE_MLX_AFFINE_STREAM_MM_ID", + "DS4_METAL_DISABLE_QWEN_GDN_PRENORMALIZE" + ], + "benchmark_canary": [ + "DS4_METAL_DISABLE_QWEN_GDN_PREEXP_DECAY", + "DS4_METAL_ENABLE_F32_NAX_PREFILL", + "DS4_METAL_ENABLE_QWEN_GDN_PREEXP_DECAY" + ], + "profiling": [ + "DS4_QWEN_METAL_PREFILL_DETAIL_PROFILE", + "DS4_QWEN_METAL_PREFILL_STAGE_PROFILE" + ] + } + }, "implementation_includes": 3, "maybe_unused_markers": 50, "personal_absolute_path_hits": 0, - "root_markdown_files": 10 + "root_markdown_files": { + "maximum": 16, + "baseline": 10, + "reason": "The open-source launch adds explicit provenance, third-party notices, community policy, security policy, governance, and release metadata; later root documentation still requires explicit classification.", + "accepted_additions": { + "oss_provenance": [ + "ACKNOWLEDGMENTS.md", + "THIRD_PARTY_NOTICES.md" + ], + "community_and_release_metadata": [ + "CHANGELOG.md", + "CODE_OF_CONDUCT.md", + "GOVERNANCE.md", + "SECURITY.md" + ] + } + } } diff --git a/tools/qwen_release_contract.py b/tools/qwen_release_contract.py new file mode 100644 index 000000000..84bcbbe3d --- /dev/null +++ b/tools/qwen_release_contract.py @@ -0,0 +1,563 @@ +#!/usr/bin/env python3 +"""Validate the Qwen release identity across its authoritative local surfaces.""" + +from __future__ import annotations + +import argparse +import json +import re +import shlex +import sys +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any + + +DEFAULT_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_MANIFEST = "docs/contracts/qwen-release.json" +SCHEMA_VERSION = 1 +HEX40_RE = re.compile(r"^[0-9a-f]{40}$") +HEX64_RE = re.compile(r"^[0-9a-f]{64}$") +HEADING_RE = re.compile(r"^(#{1,6})[ \t]+(.+?)[ \t]*$", re.MULTILINE) +LINK_RE = re.compile(r"(? None: + actual = set(value) + if actual == expected: + return + details: list[str] = [] + missing = sorted(expected - actual) + extra = sorted(actual - expected) + if missing: + details.append("missing " + ", ".join(missing)) + if extra: + details.append("unknown " + ", ".join(extra)) + raise ContractError(f"{label} has invalid keys ({'; '.join(details)})") + + +def require_string(value: Any, label: str) -> str: + if not isinstance(value, str) or not value: + raise ContractError(f"{label} must be a non-empty string") + return value + + +def require_positive_int(value: Any, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ContractError(f"{label} must be a positive integer") + return value + + +def require_filename(value: Any, label: str) -> str: + filename = require_string(value, label) + pure = PurePosixPath(filename) + if pure.name != filename or filename in {".", ".."}: + raise ContractError(f"{label} must be a bare filename") + return filename + + +def require_hex(value: Any, pattern: re.Pattern[str], label: str) -> str: + digest = require_string(value, label) + if pattern.fullmatch(digest) is None: + width = 40 if pattern is HEX40_RE else 64 + raise ContractError(f"{label} must be {width} lowercase hexadecimal characters") + return digest + + +def load_contract(path: Path) -> Contract: + try: + document = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise ContractError(f"contract does not exist: {path}") from exc + except (OSError, json.JSONDecodeError) as exc: + raise ContractError(f"cannot read contract {path}: {exc}") from exc + if not isinstance(document, dict): + raise ContractError("contract root must be a JSON object") + require_exact_keys( + document, + { + "schemaVersion", + "modelFamily", + "downloadTarget", + "repository", + "publishedArtifact", + "negativeArtifact", + }, + "contract", + ) + if document["schemaVersion"] != SCHEMA_VERSION or isinstance( + document["schemaVersion"], bool + ): + raise ContractError( + f"contract schemaVersion must be {SCHEMA_VERSION}, " + f"got {document['schemaVersion']!r}" + ) + + model_family = require_string(document["modelFamily"], "modelFamily") + if model_family != "Qwen3.6-35B-A3B": + raise ContractError("modelFamily must be Qwen3.6-35B-A3B") + download_target = require_string(document["downloadTarget"], "downloadTarget") + if download_target != "qwen-v2": + raise ContractError("downloadTarget must be qwen-v2") + repository = require_string(document["repository"], "repository") + if re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", repository) is None: + raise ContractError("repository must be an owner/name identifier") + + published_raw = document["publishedArtifact"] + negative_raw = document["negativeArtifact"] + if not isinstance(published_raw, dict) or not isinstance(negative_raw, dict): + raise ContractError("publishedArtifact and negativeArtifact must be objects") + require_exact_keys( + published_raw, + { + "status", + "filename", + "revision", + "bytes", + "sha256", + "runtimeCommit", + "storage", + "groupSize", + }, + "publishedArtifact", + ) + require_exact_keys( + negative_raw, + {"status", "filename", "bytes", "sha256"}, + "negativeArtifact", + ) + if published_raw["status"] != "published": + raise ContractError("publishedArtifact.status must be published") + if negative_raw["status"] != "negative-only": + raise ContractError("negativeArtifact.status must be negative-only") + if published_raw["storage"] != "mlx-affine4": + raise ContractError("publishedArtifact.storage must be mlx-affine4") + if published_raw["groupSize"] != 64 or isinstance( + published_raw["groupSize"], bool + ): + raise ContractError("publishedArtifact.groupSize must be 64") + + published = Artifact( + status="published", + filename=require_filename(published_raw["filename"], "publishedArtifact.filename"), + revision=require_hex( + published_raw["revision"], HEX40_RE, "publishedArtifact.revision" + ), + bytes=require_positive_int(published_raw["bytes"], "publishedArtifact.bytes"), + sha256=require_hex( + published_raw["sha256"], HEX64_RE, "publishedArtifact.sha256" + ), + runtime_commit=require_hex( + published_raw["runtimeCommit"], + HEX40_RE, + "publishedArtifact.runtimeCommit", + ), + storage="mlx-affine4", + group_size=64, + ) + negative = Artifact( + status="negative-only", + filename=require_filename(negative_raw["filename"], "negativeArtifact.filename"), + bytes=require_positive_int(negative_raw["bytes"], "negativeArtifact.bytes"), + sha256=require_hex( + negative_raw["sha256"], HEX64_RE, "negativeArtifact.sha256" + ), + ) + if published.filename == negative.filename: + raise ContractError("published and negative-only filenames must differ") + if published.sha256 == negative.sha256: + raise ContractError("published and negative-only SHA-256 values must differ") + return Contract(model_family, download_target, repository, published, negative) + + +def read_text(root: Path, relative: str) -> str: + path = root / relative + try: + return path.read_text(encoding="utf-8") + except FileNotFoundError as exc: + raise ContractError(f"required release surface does not exist: {relative}") from exc + except OSError as exc: + raise ContractError(f"cannot read release surface {relative}: {exc}") from exc + + +def markdown_section(root: Path, relative: str, heading: str) -> str: + text = read_text(root, relative) + matches = list(HEADING_RE.finditer(text)) + selected = [match for match in matches if match.group(2).strip() == heading] + if len(selected) != 1: + raise ContractError( + f"{relative}: expected one Markdown section {heading!r}, found {len(selected)}" + ) + match = selected[0] + level = len(match.group(1)) + end = len(text) + for candidate in matches: + if candidate.start() > match.start() and len(candidate.group(1)) <= level: + end = candidate.start() + break + return text[match.end() : end] + + +def strip_code(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value.startswith("`") and value.endswith("`"): + return value[1:-1] + return value + + +def parse_table(section: str, relative: str) -> list[dict[str, str]]: + lines = [line.strip() for line in section.splitlines()] + for index in range(len(lines) - 1): + header_line = lines[index] + separator_line = lines[index + 1] + if not header_line.startswith("|") or not separator_line.startswith("|"): + continue + separator = [cell.strip() for cell in separator_line.strip("|").split("|")] + if not separator or not all(re.fullmatch(r":?-{3,}:?", cell) for cell in separator): + continue + headers = [cell.strip() for cell in header_line.strip("|").split("|")] + if len(headers) != len(separator): + raise ContractError(f"{relative}: malformed Markdown table header") + rows: list[dict[str, str]] = [] + for line in lines[index + 2 :]: + if not line.startswith("|"): + break + cells = [cell.strip() for cell in line.strip("|").split("|")] + if len(cells) != len(headers): + raise ContractError(f"{relative}: malformed Markdown table row: {line}") + rows.append(dict(zip(headers, cells, strict=True))) + return rows + raise ContractError(f"{relative}: expected a Markdown table in the release section") + + +def require_tokens(label: str, text: str, tokens: list[str]) -> None: + missing = [token for token in tokens if token not in text] + if missing: + raise ContractError(f"{label}: missing contract value(s): {', '.join(missing)}") + + +def require_contract_link( + root: Path, relative: str, section: str, manifest_path: Path +) -> None: + source_parent = (root / relative).parent + destinations: list[Path] = [] + for match in LINK_RE.finditer(section): + raw = match.group(1).strip().split(maxsplit=1)[0] + if raw.startswith(("#", "http://", "https://", "mailto:")): + continue + destinations.append((source_parent / raw.split("#", 1)[0]).resolve()) + if manifest_path.resolve() not in destinations: + raise ContractError( + f"{relative}: release section must link to the canonical contract" + ) + + +def require_qwen_filenames( + label: str, text: str, expected: set[str] +) -> None: + actual = set(QWEN_FILENAME_RE.findall(text)) + if actual != expected: + raise ContractError( + f"{label}: Qwen artifact identities differ: " + f"expected {sorted(expected)!r}, got {sorted(actual)!r}" + ) + + +def check_prose_surface( + root: Path, + manifest_path: Path, + contract: Contract, + relative: str, + heading: str, +) -> None: + section = markdown_section(root, relative, heading) + published = contract.published + negative = contract.negative + require_contract_link(root, relative, section, manifest_path) + require_tokens( + f"{relative}#{heading}", + section, + [ + contract.model_family, + contract.download_target, + published.status, + published.filename, + f"{published.bytes:,}", + published.sha256, + published.revision or "", + published.runtime_commit or "", + negative.status, + negative.filename, + ], + ) + require_qwen_filenames( + f"{relative}#{heading}", section, {published.filename, negative.filename} + ) + + +def require_table_row( + rows: list[dict[str, str]], key_header: str, key: str, relative: str +) -> dict[str, str]: + matches = [row for row in rows if strip_code(row.get(key_header, "")) == key] + if len(matches) != 1: + raise ContractError(f"{relative}: expected one table row for {key!r}") + return matches[0] + + +def check_qa(root: Path, manifest_path: Path, contract: Contract) -> None: + relative = "QA_BEFORE_RELEASES.md" + section = markdown_section(root, relative, "Release Artifact Identity") + require_contract_link(root, relative, section, manifest_path) + rows = parse_table(section, relative) + published = require_table_row(rows, "Variable", "QWEN_V2", relative) + negative = require_table_row(rows, "Variable", "QWEN_RETIRED_Q4_NEGATIVE", relative) + published_text = published.get("Required identity", "") + negative_text = negative.get("Required identity", "") + p = contract.published + n = contract.negative + require_tokens( + f"{relative}:QWEN_V2", + published_text, + [ + p.status, + p.filename, + p.revision or "", + f"{p.bytes:,}", + p.sha256, + p.runtime_commit or "", + "MLX affine4/group-64", + ], + ) + require_tokens( + f"{relative}:QWEN_RETIRED_Q4_NEGATIVE", + negative_text, + [n.status, n.filename, f"{n.bytes:,}", n.sha256], + ) + + +def rows_by_item(rows: list[dict[str, str]], relative: str) -> dict[str, str]: + result: dict[str, str] = {} + for row in rows: + item = strip_code(row.get("Item", "")) + if not item or item in result: + raise ContractError(f"{relative}: duplicate or empty release identity item") + result[item] = strip_code(row.get("Value", "")) + return result + + +def require_exact_item(items: dict[str, str], key: str, value: str, relative: str) -> None: + actual = items.get(key) + if actual != value: + raise ContractError( + f"{relative}: {key!r} must be {value!r}, got {actual!r}" + ) + + +def check_qwen_store(root: Path, manifest_path: Path, contract: Contract) -> None: + relative = "docs/qwen-expert-major-store.md" + section = markdown_section(root, relative, "Release identity") + require_contract_link(root, relative, section, manifest_path) + items = rows_by_item(parse_table(section, relative), relative) + p = contract.published + n = contract.negative + require_exact_item(items, "Publication state", p.status, relative) + require_exact_item(items, "Repository", contract.repository, relative) + require_exact_item(items, "Artifact", p.filename, relative) + require_exact_item(items, "Artifact bytes", f"{p.bytes:,}", relative) + require_exact_item(items, "Artifact SHA-256", p.sha256, relative) + require_exact_item(items, "Immutable revision", p.revision or "", relative) + require_exact_item( + items, "Minimum compatible runtime commit", p.runtime_commit or "", relative + ) + require_exact_item(items, "Storage", f"{p.storage}/group-{p.group_size}", relative) + require_exact_item(items, "Negative fixture state", n.status, relative) + require_exact_item(items, "Negative fixture", n.filename, relative) + require_exact_item(items, "Negative fixture bytes", f"{n.bytes:,}", relative) + require_exact_item(items, "Negative fixture SHA-256", n.sha256, relative) + + +def parse_shell_assignments(text: str, relative: str) -> dict[str, str]: + assignments: dict[str, str] = {} + for line_number, line in enumerate(text.splitlines(), 1): + match = re.fullmatch(r"([A-Z][A-Z0-9_]*)=(.*)", line) + if match is None: + continue + name, raw_value = match.groups() + try: + values = shlex.split(raw_value, posix=True) + except ValueError as exc: + raise ContractError( + f"{relative}:{line_number}: invalid shell assignment: {exc}" + ) from exc + if len(values) != 1: + continue + if name in assignments: + raise ContractError(f"{relative}: duplicate top-level assignment {name}") + assignments[name] = values[0] + return assignments + + +def case_block(text: str, target: str, relative: str) -> str: + lines = text.splitlines() + starts = [index for index, line in enumerate(lines) if line == f" {target})"] + if len(starts) != 1: + raise ContractError(f"{relative}: expected one case arm for {target}") + start = starts[0] + for index in range(start + 1, len(lines)): + if lines[index] == " ;;": + return "\n".join(lines[start + 1 : index]) + raise ContractError(f"{relative}: unterminated case arm for {target}") + + +def check_downloader(root: Path, contract: Contract) -> None: + relative = "download_model.sh" + text = read_text(root, relative) + assignments = parse_shell_assignments(text, relative) + p = contract.published + expected = { + "RUNTIME_QWEN_STATUS": p.status, + "RUNTIME_QWEN_REPO": contract.repository, + "RUNTIME_QWEN_FILE": p.filename, + "RUNTIME_QWEN_BYTES": str(p.bytes), + "RUNTIME_QWEN_SHA256": p.sha256, + "RUNTIME_QWEN_REVISION": p.revision or "", + "RUNTIME_QWEN_MIN_RUNTIME_COMMIT": p.runtime_commit or "", + } + for name, value in expected.items(): + if assignments.get(name) != value: + raise ContractError( + f"{relative}: {name} must be {value!r}, got {assignments.get(name)!r}" + ) + block = case_block(text, contract.download_target, relative) + required_wiring = { + "MODEL_REPO=$RUNTIME_QWEN_REPO", + "MODEL_FILE=$RUNTIME_QWEN_FILE", + "MODEL_REVISION=$RUNTIME_QWEN_REVISION", + "MODEL_BYTES=$RUNTIME_QWEN_BYTES", + "MODEL_SHA256=$RUNTIME_QWEN_SHA256", + } + wiring = {line.strip() for line in block.splitlines() if line.strip()} + if not required_wiring.issubset(wiring): + missing = sorted(required_wiring - wiring) + raise ContractError( + f"{relative}: {contract.download_target} is missing wiring: {missing!r}" + ) + if contract.negative.filename in text or contract.negative.sha256 in text: + raise ContractError( + f"{relative}: negative-only Qwen artifact must not be downloadable" + ) + + +def check_download_test(root: Path, manifest_path: Path, contract: Contract) -> None: + relative = "tests/test_download_model.sh" + text = read_text(root, relative) + expected_manifest = manifest_path.resolve().relative_to(root.resolve()).as_posix() + require_tokens( + relative, + text, + [ + expected_manifest, + "RUNTIME_QWEN_STATUS", + "RUNTIME_QWEN_REPO", + "RUNTIME_QWEN_FILE", + "RUNTIME_QWEN_BYTES", + "RUNTIME_QWEN_SHA256", + "RUNTIME_QWEN_REVISION", + "RUNTIME_QWEN_MIN_RUNTIME_COMMIT", + "QWEN_NEGATIVE_FILE", + "QWEN_NEGATIVE_SHA256", + ], + ) + duplicated = [ + value + for value in ( + contract.repository, + contract.published.filename, + contract.published.sha256, + contract.published.revision or "", + contract.published.runtime_commit or "", + contract.negative.filename, + contract.negative.sha256, + ) + if value and value in text + ] + if duplicated: + raise ContractError( + f"{relative}: must consume the canonical contract, not duplicate values: " + + ", ".join(duplicated) + ) + + +def validate(root: Path, manifest_path: Path) -> Contract: + try: + manifest_path.resolve().relative_to(root.resolve()) + except ValueError as exc: + raise ContractError("manifest must be inside the repository root") from exc + contract = load_contract(manifest_path) + for relative, heading in ( + ("README.md", "Supported models"), + ("CONTRIBUTING.md", "Artifact publication boundary"), + ("docs/contracts/RUNTIME_SUPPORT.md", "Supported Matrix"), + ): + check_prose_surface(root, manifest_path, contract, relative, heading) + check_qa(root, manifest_path, contract) + check_qwen_store(root, manifest_path, contract) + check_downloader(root, contract) + check_download_test(root, manifest_path, contract) + return contract + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=DEFAULT_ROOT) + parser.add_argument("--manifest", default=DEFAULT_MANIFEST) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + root = args.root.resolve() + manifest_path = Path(args.manifest) + if not manifest_path.is_absolute(): + manifest_path = root / manifest_path + try: + contract = validate(root, manifest_path) + except ContractError as exc: + print(f"Qwen release contract: FAIL: {exc}", file=sys.stderr) + return 1 + print( + "Qwen release contract: PASS " + f"({contract.published.status} {contract.published.filename}; " + f"{contract.negative.status} {contract.negative.filename})" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())