refactor(tensor-pool): move behind a generic extension seam, opt-in and outside the 1.0 guarantees

.github/workflows/cargo-release.yml

@@ -95,7 +95,6 @@ jobs:
           publish_if_not_exists dora-tracing
           publish_if_not_exists dora-metrics
           publish_if_not_exists dora-hub-client
-          publish_if_not_exists dora-memory-pool
 
           # Publish rust API
           publish_if_not_exists dora-operator-api-types

.github/workflows/nightly.yml

@@ -115,65 +115,6 @@ jobs:
         if: steps.smoke.outputs.exit_code != '0'
         run: exit 1
 
-  # Memory-pool CPU transport smoke tests (dora-rs/dora#2168).
-  #
-  # These 6 tests (smoke_memory_pool_cpu2cpu + smoke_local_memory_pool_*) are
-  # `#[ignore]`-gated in tests/example-smoke.rs because they need torch + tqdm,
-  # which the smoke-suite job above does not install — so that job skips them
-  # and they ran in NO workflow (#2302 Finding 1; the gap #2264 Finding 1 first
-  # raised). They are CPU-only (sender_device/receiver_device: cpu), so no GPU
-  # is required: the per-node `build:` steps pip-install CPU torch from
-  # download.pytorch.org via uv. A dedicated job keeps the torch-download cost
-  # and any pytorch-index flakiness off the main smoke-suite, and makes the
-  # unsafe seqlock / pointer-arithmetic / bounds / free paths flagged by #2264
-  # execute at least once per nightly run.
-  memory-pool-smoke:
-    name: Memory-pool CPU smoke tests (#2168)
-    runs-on: ubuntu-latest
-    # 6 tests, --test-threads=1 (they share coordinator ports). Most of the
-    # wall clock is the one-time CPU-torch download/install in the per-node
-    # build steps; 45 min leaves margin on top of a cold cargo compile.
-    timeout-minutes: 45
-    steps:
-      - uses: actions/checkout@v6
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ env.RUST_VERSION }}
-      - uses: Swatinem/rust-cache@v2
-        with:
-          save-if: false
-      - uses: actions/setup-python@v6
-        with:
-          python-version: "3.12"
-      - uses: astral-sh/setup-uv@v8.1.0
-        with:
-          enable-cache: true
-      - name: Set up Python venv
-        run: |
-          uv venv --seed -p 3.12
-          echo "VIRTUAL_ENV=$PWD/.venv" >> "$GITHUB_ENV"
-          source .venv/bin/activate
-          uv pip install pyarrow
-          uv pip install -e apis/python/node
-      - name: Run memory-pool smoke tests
-        id: smoke
-        continue-on-error: true
-        # `--ignored memory_pool` runs only the ignored memory-pool tests; the
-        # per-node `build:` steps install torch/tqdm/numpy into the active venv.
-        run: |
-          cargo test -p dora-examples --test example-smoke -- \
-            --ignored --test-threads=1 memory_pool 2>&1 | tee memory-pool-smoke.log
-          echo "exit_code=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
-      - name: Upload smoke log
-        uses: actions/upload-artifact@v7
-        with:
-          name: memory-pool-smoke-output
-          path: memory-pool-smoke.log
-          retention-days: 14
-      - name: Fail the job if memory-pool smoke tests failed
-        if: steps.smoke.outputs.exit_code != '0'
-        run: exit 1
-
   # Hub e2e (tests/hub-smoke.rs). The `hub:` feature is unstable + non-core, so
   # it lives in nightly rather than the per-PR gate. Hermetic — a local git
   # fixture + DORA_HUB_ALLOW_LOCAL_SOURCES, no network — so it needs only Rust
@@ -2014,7 +1955,6 @@ jobs:
     needs:
       - build-cli
       - smoke-suite
-      - memory-pool-smoke
       - log-sinks
       - service-action
       - streaming
@@ -2078,10 +2018,6 @@ jobs:
               smoke-suite)
                 echo 'cargo test -p dora-examples --test example-smoke -- --test-threads=1'
                 ;;
-              memory-pool-smoke)
-                # Needs torch+tqdm; the per-node build steps install CPU torch.
-                echo 'cargo test -p dora-examples --test example-smoke -- --ignored --test-threads=1 memory_pool'
-                ;;
               log-sinks)
                 # Three dataflows in this job; run each individually on failure.
                 printf 'dora run examples/log-sink-file/dataflow.yml --uv --stop-after 15s\ndora run examples/log-sink-alert/dataflow.yml --uv --stop-after 15s\ndora run examples/log-sink-tcp/dataflow.yml --uv --stop-after 15s\n'
@@ -2163,7 +2099,6 @@ jobs:
             case "$1" in
               build-cli) echo '^Build dora CLI \(shared\)$' ;;
               smoke-suite) echo '^Smoke suite \(tests/example-smoke\.rs\)$' ;;
-              memory-pool-smoke) echo '^Memory-pool CPU smoke tests \(#2168\)$' ;;
               log-sinks) echo '^Log-sink examples \(Python sources\)$' ;;
               service-action) echo '^Service / Action patterns \(Rust-only\)$' ;;
               streaming) echo '^Streaming example \(Python\)$' ;;
@@ -2444,7 +2379,6 @@ jobs:
     needs:
       - build-cli
       - smoke-suite
-      - memory-pool-smoke
       - log-sinks
       - service-action
       - streaming

.github/workflows/release.yml

@@ -76,7 +76,6 @@ jobs:
             dora-tracing
             dora-metrics
             dora-hub-client
-            dora-memory-pool
             dora-operator-api-types
             dora-operator-api-macros
             dora-operator-api

CLAUDE.md

@@ -169,7 +169,7 @@ The deeper QA gates — `make qa-full`, `make qa-deep`, `make qa-nightly`, `make
 
 - `make qa-full` (qa-fast + full tests + coverage) — ~5-10 minutes. Run before a significant push if you want extra confidence; coverage is too slow for every push.
 - `make qa-deep` (qa-full + mutation testing + semver) — ~15 minutes. The **target** Tier 1 local gate. Today's CI PR gate only runs the fast subset (fmt/clippy/typos/audit/unwrap-budget + tests); `qa-deep` adds coverage, adversarial review, diff-scoped mutation, and semver — kept laptop-only because they're too slow for every PR (see `docs/plan-agentic-qa-strategy.md` §5). Alias: `make qa-tier1`, kept for back-compat.
-- `make qa-nightly` (qa-deep + proptest@1000 + miri + example-smoke + ci-nightly-jobs) — ~3-4 hours. **Full parity with `.github/workflows/nightly.yml`** (#1707, #1710, #1716). After the #1716 rebalance (plus cluster-record-replay from #2013 and memory-pool-smoke from #2302), nightly.yml has **27 test jobs** (`cli-tests` was split into `cli-tests` + `cli-tests-python` in #2742; re-counted in #2999, where the previous 23 counted neither `hub-smoke` nor the two `ros2-zenoh-*` jobs, and `multi-daemon-late-subscriber` is new). example-smoke covers the **4 example-backed** jobs (smoke-suite, log-sinks, service-action, streaming); `scripts/qa/ci-nightly-jobs.sh` drives **20** with platform-aware dispatch (record-replay, cluster-smoke, cluster-e2e [Linux, needs `openssh-server`], cluster-record-replay [Linux, needs `openssh-server`], topic-and-top, cpu-affinity [Linux], redb-backend, daemon-reconnect [Linux], state-reconstruction, multi-daemon-late-subscriber [Linux], test-cross-platform [macOS+Windows], examples, cli-tests **and cli-tests-python** — one local `cli-tests` invocation covers both halves, bench-example, cross-check, ros2-bridge [Linux, basic checks — no ROS distro], ros2-zenoh-humble, ros2-zenoh-kilted, msrv, kani-proofs [skipped if Kani not installed]). Of the rest, `memory-pool-smoke` (the torch-gated `#[ignore]` memory-pool example tests run with `--ignored`), is covered locally by `make qa-examples` / `scripts/smoke-all.sh`, **not** the qa-nightly example-smoke step (which skips `#[ignore]` tests), and `hub-smoke` has no local driver entry at all. A green local `qa-nightly` on platform X predicts a green CI nightly for platform X's jobs. **Requires both `uv` and Python 3.12** (both preflighted; the script fails fast with a specific install hint for whichever is missing — `curl -LsSf https://astral.sh/uv/install.sh \| sh` for uv, `uv python install 3.12` for the interpreter, matching the GHA `actions/setup-python` step at `.github/workflows/nightly.yml:56`). example-smoke creates a scratch venv at `target/qa-nightly-venv` and installs `-e apis/python/node` into it so Python nodes use the workspace bindings (not PyPI `dora-rs`, whose message format has drifted from the workspace — #1710). The CI-jobs script installs the CLI into a scratch dir (won't clobber `~/.cargo/bin/dora`) and bails if port 6013 is in use; cpu-affinity + daemon-reconnect skip on non-Linux. Skips miri if `cargo +nightly miri` isn't installed. **Does not** include full-repo mutation testing — that's split into `qa-mutation-audit` because it takes 10-18 hours on this workspace.
+- `make qa-nightly` (qa-deep + proptest@1000 + miri + example-smoke + ci-nightly-jobs) — ~3-4 hours. **Full parity with `.github/workflows/nightly.yml`** (#1707, #1710, #1716). After the #1716 rebalance (plus cluster-record-replay from #2013), nightly.yml has **26 test jobs** (`cli-tests` was split into `cli-tests` + `cli-tests-python` in #2742; re-counted in #2999, where the previous 23 counted neither `hub-smoke` nor the two `ros2-zenoh-*` jobs, and `multi-daemon-late-subscriber` is new). example-smoke covers the **4 example-backed** jobs (smoke-suite, log-sinks, service-action, streaming); `scripts/qa/ci-nightly-jobs.sh` drives **20** with platform-aware dispatch (record-replay, cluster-smoke, cluster-e2e [Linux, needs `openssh-server`], cluster-record-replay [Linux, needs `openssh-server`], topic-and-top, cpu-affinity [Linux], redb-backend, daemon-reconnect [Linux], state-reconstruction, multi-daemon-late-subscriber [Linux], test-cross-platform [macOS+Windows], examples, cli-tests **and cli-tests-python** — one local `cli-tests` invocation covers both halves, bench-example, cross-check, ros2-bridge [Linux, basic checks — no ROS distro], ros2-zenoh-humble, ros2-zenoh-kilted, msrv, kani-proofs [skipped if Kani not installed]). Of the rest, `hub-smoke` has no local driver entry at all. A green local `qa-nightly` on platform X predicts a green CI nightly for platform X's jobs. **Requires both `uv` and Python 3.12** (both preflighted; the script fails fast with a specific install hint for whichever is missing — `curl -LsSf https://astral.sh/uv/install.sh \| sh` for uv, `uv python install 3.12` for the interpreter, matching the GHA `actions/setup-python` step at `.github/workflows/nightly.yml:56`). example-smoke creates a scratch venv at `target/qa-nightly-venv` and installs `-e apis/python/node` into it so Python nodes use the workspace bindings (not PyPI `dora-rs`, whose message format has drifted from the workspace — #1710). The CI-jobs script installs the CLI into a scratch dir (won't clobber `~/.cargo/bin/dora`) and bails if port 6013 is in use; cpu-affinity + daemon-reconnect skip on non-Linux. Skips miri if `cargo +nightly miri` isn't installed. **Does not** include full-repo mutation testing — that's split into `qa-mutation-audit` because it takes 10-18 hours on this workspace.
 - `make qa-release-gate` (qa-deep + semver) — the automatable subset of Tier 3. The non-automatable parts (independent security audit, dogfood campaign, migration validation) are documented in `docs/plan-agentic-qa-strategy.md` §7 but not locally gateable.
 - `make qa-mutation-audit` — ~10-18 hours. Full-repo `cargo-mutants` across 6 critical crates. Deliberate test-quality audit; run before a release or when investigating a specific crate, not every nightly.
 - `make qa-examples` — ~15-20 min. Runs all **smoke-eligible** example dataflows end-to-end via `scripts/smoke-all.sh`. Skips examples that need CUDA, ROS2, webcam, multi-machine deploy, C/C++ toolchains, or interactive CLI (run `scripts/smoke-all.sh -h` to see the SKIP list). Orthogonal to the qa-fast/full/deep ladder: those targets `--exclude dora-examples` to keep per-commit / pre-push budgets tight. Run this when you want actual dataflows exercised (after touching node/operator APIs, CLI subcommands, or the descriptor surface). Pass flags via `ARGS`, e.g. `make qa-examples ARGS="--rust-only"` or `make qa-examples ARGS="-v"`.

Cargo.toml

@@ -67,7 +67,10 @@ members = [
     "libraries/extensions/download",
     "libraries/extensions/telemetry/*",
     "libraries/extensions/mavlink2-bridge",
-    "libraries/extensions/memory-pool",
+    # Opt-in tensor-pool transport. Outside the 1.0 compatibility
+    # guarantees — see libraries/extensions/tensor-pool/README.md.
+    "libraries/extensions/tensor-pool",
+    "libraries/extensions/tensor-pool/python",
     "libraries/extensions/ros2-bridge",
     "libraries/extensions/ros2-bridge/msg-gen",
     "libraries/extensions/ros2-bridge/python",
@@ -134,7 +137,8 @@ dora-ros2-bridge = { version = "1.0.0-rc.4", path = "libraries/extensions/ros2-b
 dora-ros2-bridge-msg-gen = { version = "1.0.0-rc.4", path = "libraries/extensions/ros2-bridge/msg-gen" }
 dora-ros2-bridge-python = { path = "libraries/extensions/ros2-bridge/python" }
 dora-mavlink2-bridge = { version = "1.0.0-rc.4", path = "libraries/extensions/mavlink2-bridge" }
-dora-memory-pool = { version = "1.0.0-rc.4", path = "libraries/extensions/memory-pool" }
+dora-tensor-pool = { version = "1.0.0-rc.4", path = "libraries/extensions/tensor-pool" }
+dora-tensor-pool-python = { version = "1.0.0-rc.4", path = "libraries/extensions/tensor-pool/python" }
 dora-message = { version = "1.0.0-rc.4", path = "libraries/message" }
 # Declared here so `dora-ros2-bridge` and `dora-ros2-bridge-arrow` cannot drift
 # apart on the ROS distro. `humble` selects the 24-byte `Gid` layout used by

Makefile

@@ -13,7 +13,7 @@
 #   make qa-nightly         ~3-4 hours  Full parity with .github/workflows/nightly.yml
 #                                        (qa-deep + proptest@1000 + miri + example-smoke
 #                                        + ci-nightly-jobs). After the #1716 rebalance,
-#                                        nightly.yml has 27 test jobs (re-counted
+#                                        nightly.yml has 26 test jobs (re-counted
 #                                        in #2999): example-smoke
 #                                        covers 4 (smoke-suite/log-sinks/service-action/
 #                                        streaming); scripts/qa/ci-nightly-jobs.sh drives
@@ -26,12 +26,8 @@
 #                                        test-cross-platform, examples, cli-tests,
 #                                        bench-example, cross-check, ros2-bridge [Linux+ROS2],
 #                                        ros2-zenoh-humble, ros2-zenoh-kilted,
-#                                        msrv, kani-proofs [skipped if Kani absent]). Of
-#                                        the rest, memory-pool-smoke (torch-gated #[ignore]
-#                                        memory-pool tests), is covered locally by
-#                                        `make qa-examples` / smoke-all.sh, not the
-#                                        qa-nightly example-smoke step (which skips
-#                                        #[ignore] tests). Green local run on platform X predicts
+#                                        msrv, kani-proofs [skipped if Kani absent]).
+#                                        Green local run on platform X predicts
 #                                        green CI nightly for platform X's jobs.
 #   make qa-release-gate                 Tier 3 automatable parts (deep + semver;
 #                                        audit/dogfood are human)

apis/python/node/Cargo.toml

@@ -14,6 +14,10 @@ publish = false  # PyO3 cdylib, shipped via PyPI as `dora-rs`, not crates.io
 
 [features]
 default = ["tracing", "metrics", "async"]
+# Opt-in tensor-pool transport. NOT covered by dora's 1.0 compatibility
+# guarantees — see libraries/extensions/tensor-pool/README.md. Off by default
+# so a standard wheel neither builds nor exposes it.
+tensor-pool = ["dep:dora-tensor-pool-python"]
 tracing = ["dora-node-api/tracing"]
 metrics = ["dora-node-api/metrics"]
 async = ["pyo3/experimental-async"]
@@ -35,7 +39,7 @@ pythonize = { workspace = true }
 futures = { workspace = true }
 dora-ros2-bridge-python = { workspace = true }
 dora-download = { workspace = true }
-shared_memory_extended = "0.13.0"
+dora-tensor-pool-python = { workspace = true, optional = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "time", "macros"] }
 tracing = { workspace = true }