From da53f1dad0138a3371f1bfc16ef996f31d1bbd00 Mon Sep 17 00:00:00 2001 From: yxrxy Date: Mon, 29 Jun 2026 18:53:54 +0800 Subject: [PATCH 01/14] feat: route mq through broker --- fluxon_py/_api_ext_chan/mpmc.py | 152 +- fluxon_py/_api_ext_chan/mpsc.py | 237 +- fluxon_py/kvclient/fluxon.py | 50 +- fluxon_py/runtime/__init__.py | 8 + fluxon_py/runtime/start_broker.py | 139 + .../test_api_chan_mpmc_base.py | 28 + .../test_mpmc_simple_bench.py | 91 +- .../test_api_chan_mpsc_base.py | 116 + fluxon_py/tests/test_lib.py | 7 +- .../tests/test_mq/test_example_ctrl_c_exit.py | 32 + fluxon_rs/Cargo.lock | 2 + fluxon_rs/fluxon_commu/src/facade/p2p.rs | 61 +- .../src/facade/transfer_engine.rs | 18 +- fluxon_rs/fluxon_kv/Cargo.toml | 1 + fluxon_rs/fluxon_kv/framework_init_steps.yaml | 10 +- fluxon_rs/fluxon_kv/src/kv_test.rs | 9 +- fluxon_rs/fluxon_kv/src/lib.rs | 223 ++ fluxon_rs/fluxon_kv/src/memholder/lifetime.rs | 8 +- fluxon_rs/fluxon_kv/src/profile.rs | 2 + fluxon_rs/fluxon_mq/Cargo.toml | 1 + fluxon_rs/fluxon_mq/src/broker.rs | 3006 +++++++++++++++++ fluxon_rs/fluxon_mq/src/consumer.rs | 1144 ++++++- fluxon_rs/fluxon_mq/src/create.rs | 2 + fluxon_rs/fluxon_mq/src/error.rs | 14 + fluxon_rs/fluxon_mq/src/keys.rs | 6 +- fluxon_rs/fluxon_mq/src/lib.rs | 2 + fluxon_rs/fluxon_mq/src/manager.rs | 5 + fluxon_rs/fluxon_mq/src/producer.rs | 250 +- fluxon_rs/fluxon_observability/src/types.rs | 2 + fluxon_rs/fluxon_pyo3/src/error.rs | 45 +- .../fluxon_pyo3/src/flatdict_zerocopy.rs | 202 +- fluxon_rs/fluxon_pyo3/src/lease_manager.rs | 31 +- fluxon_rs/fluxon_pyo3/src/lib.rs | 146 +- fluxon_rs/fluxon_pyo3/src/memholder.rs | 8 +- fluxon_rs/fluxon_pyo3/src/mpsc.rs | 1534 ++++++--- .../src/lease_manager/lease_handle.rs | 6 +- fluxon_test_stack/test_runner.py | 1 + setup_and_pack/nix/pack_fluxonkv_pylib.py | 13 +- setup_and_pack/pack_release.py | 22 +- ...est_pack_fluxonkv_pylib_bridge_prebuilt.py | 19 + .../test_pack_release_examples_layout.py | 25 + 41 files changed, 6848 insertions(+), 830 deletions(-) create mode 100644 fluxon_py/runtime/start_broker.py create mode 100644 fluxon_rs/fluxon_mq/src/broker.rs diff --git a/fluxon_py/_api_ext_chan/mpmc.py b/fluxon_py/_api_ext_chan/mpmc.py index 4ddbc1e..2b09443 100644 --- a/fluxon_py/_api_ext_chan/mpmc.py +++ b/fluxon_py/_api_ext_chan/mpmc.py @@ -96,18 +96,34 @@ LOCAL_MEMBER_ID_RANGE_SIZE = 32 MPMC_CREATE_LOCK_TTL_SECONDS = 10 MPMC_CREATE_LOCK_TIMEOUT_SECONDS = 10.0 +MPMC_CLEANUP_ETCD_TIMEOUT_SECONDS = 2.0 -def new_etcd_client(api: KvClient) -> Result[etcd3.Etcd3Client, ApiError]: +def _close_lease_handle(handle: Optional[object], label: str) -> None: + if handle is None: + return + try: + handle.close() # type: ignore[attr-defined] + except Exception as e: # noqa: BLE001 + logging.warning("failed to close lease handle %s: %s", label, e) + + +def new_etcd_client( + api: KvClient, *, timeout_seconds: Optional[float] = None +) -> Result[etcd3.Etcd3Client, ApiError]: """Create etcd client""" etcd_config: List[str] = api.get_etcd_config() first_address: str = etcd_config[0] host: str port_str: str host, port_str = first_address.split(":") - print(f"new_etcd_client: {host}:{port_str}") try: - client: etcd3.Etcd3Client = etcd3.client(host=host, port=int(port_str)) + kwargs: Dict[str, Any] = {} + if timeout_seconds is not None: + kwargs["timeout"] = float(timeout_seconds) + client: etcd3.Etcd3Client = etcd3.client( + host=host, port=int(port_str), **kwargs + ) return Result.new_ok(client) except Exception as e: return Result.new_error( @@ -136,8 +152,10 @@ def stable_revoke_lease(api: KvClient, lease_id: int) -> Result[OkNone, ApiError endpoint = endpoints[0] if endpoints else None errors: List[str] = [] - for attempt in range(3): - client_res = new_etcd_client(api) + for attempt in range(2): + client_res = new_etcd_client( + api, timeout_seconds=MPMC_CLEANUP_ETCD_TIMEOUT_SECONDS + ) if not client_res.is_ok(): err = client_res.unwrap_error() errors.append(str(err)) @@ -183,8 +201,10 @@ def stable_delete_ready_keys_for_member( member_id_str = str(member_id) errors: List[str] = [] - for attempt in range(3): - client_res = new_etcd_client(api) + for attempt in range(2): + client_res = new_etcd_client( + api, timeout_seconds=MPMC_CLEANUP_ETCD_TIMEOUT_SECONDS + ) if not client_res.is_ok(): err = client_res.unwrap_error() errors.append(str(err)) @@ -203,22 +223,7 @@ def stable_delete_ready_keys_for_member( for key in keys_to_delete: client.delete(key) - # Verify: keys should be gone immediately after delete on the same prefix. - remaining: List[bytes] = [] - for value, meta in client.get_prefix(prefix): - if value is None: - continue - if value.decode() != member_id_str: - continue - remaining.append(meta.key) - - if len(remaining) == 0: - return Result.new_ok(OK_NONE) - - errors.append( - f"attempt={attempt}: remaining ready keys after delete: {remaining!r}" - ) - time.sleep(0.1) + return Result.new_ok(OK_NONE) except Exception as e: # noqa: BLE001 errors.append(f"attempt={attempt}: {e}") time.sleep(0.1) @@ -1802,19 +1807,16 @@ def close(self) -> Result[OkNone, ApiError]: except Exception as e: # noqa: BLE001 logging.warning(f"MPMC channel {self.mpmc_id} stop_watching failed: {e}") - # Drop PyLease handles to stop keepalive; etcd leases with - # revoke_on_drop=False are intentionally not revoked. - # Setting to None drops the PyO3 handle immediately in CPython, - # which releases the underlying Rust RAII and unregisters from - # the keepalive actor. - if hasattr(self, "_lm_mpmc_member"): - self._lm_mpmc_member = None # type: ignore[assignment] - if hasattr(self, "_lm_mpmc_global"): - self._lm_mpmc_global = None # type: ignore[assignment] - if hasattr(self, "_lm_cluster_long"): - self._lm_cluster_long = None # type: ignore[assignment] - if hasattr(self, "_lm_kv_payload"): - self._lm_kv_payload = None # type: ignore[assignment] + # Close lease handles explicitly so keepalive entries are unregistered + # before the owning KvClient starts shutting down. + _close_lease_handle(self._lm_mpmc_member, "mpmc_member") + self._lm_mpmc_member = None + _close_lease_handle(self._lm_mpmc_global, "mpmc_global") + self._lm_mpmc_global = None + _close_lease_handle(self._lm_cluster_long, "mpmc_cluster_long") + self._lm_cluster_long = None + _close_lease_handle(self._lm_kv_payload, "mpmc_kv_payload") + self._lm_kv_payload = None # Return a minimal Ok result to satisfy the explicit Result API contract return Result.new_ok(OK_NONE) @@ -2025,6 +2027,12 @@ def _record_mpsc_producer(self, mpsc_producer: MPSCChanProducer): def put_data( self, value: Dict[str, Union[int, float, bool, str, bytes, DLPacked]] + ) -> Result[bool, ApiError]: + return self._put_data_impl(value) + + def _put_data_impl( + self, + value: Dict[str, Union[int, float, bool, str, bytes, DLPacked]], ) -> Result[bool, ApiError]: """Put data to the MPMC channel. @@ -2051,9 +2059,11 @@ def put_data( ) ) + capacity = int(self.chan_config["capacity"]) + assert capacity > 0, f"invalid MPMC channel capacity: {capacity}" + # Do not hold _op_lock while performing network-heavy operations (count_prefix/put_data). # Otherwise close() may block behind a long RPC and tests like MQ capacity+auto-clean can hang. - capacity = int(self.chan_config["capacity"]) # validated upfront while True: if self.shutdown_ctl.closed: return Result[bool, ApiError].new_error( @@ -2159,6 +2169,20 @@ def put_data( return Result[bool, ApiError].new_ok(True) err = put_result.unwrap_error() + if isinstance(err, MessageBufferFullError): + blocking_observed_unix_ms = int(time.time() * 1000) + try: + candidate.record_blocking_put_observed(blocking_observed_unix_ms) + except Exception as e: # noqa: BLE001 + logging.warning( + "MPMCChanProducer mpmc_id=%s failed to record broker backpressure on mpsc_id=%s producer_idx=%s: %s", + self.mpmc_id, + candidate.get_chan_id(), + candidate.get_producer_id(), + e, + ) + time.sleep(0.02) + continue logging.error( "MPMCChanProducer mpmc_id=%s failed to put data on mpsc_id=%s producer_idx=%s: %s", self.mpmc_id, @@ -2415,7 +2439,7 @@ def request_shutdown(self) -> None: self.shutdown_ctl.closed = True if self.mpsc_consumer is not None and hasattr(self.mpsc_consumer, "request_shutdown"): self.mpsc_consumer.request_shutdown() - + def get_chan_id(self) -> str: """ Get the channel id. @@ -2431,6 +2455,18 @@ def get_consumer_id(self) -> str: def get_data( self, batch_size: int = 1, try_time: Optional[int] = None, prefetch_num: int = 0 + ) -> Result[List[Dict[str, Union[int, float, bool, str, bytes, DLPacked]]], ApiError]: + del prefetch_num + return self._get_data_impl( + batch_size=batch_size, + try_time=try_time, + ) + + def _get_data_impl( + self, + *, + batch_size: int, + try_time: Optional[int], ) -> Result[List[Dict[str, Union[int, float, bool, str, bytes, DLPacked]]], ApiError]: """Get data from the bound MPSC channel. @@ -2463,22 +2499,7 @@ def get_data( # Get data from MPSC consumer (will automatically return producer info when MPSC acts as submodule) from .mpsc import ConsumedMessage - # # Map MPMC-level prefetch to per-MPSC prefetch: divide by active MPMC consumers, ceil, min divisor=1 - # try: - # active_consumers = self.mpmc_channel._get_active_consumer_count() - # except Exception as e: # noqa: BLE001 - # logging.warning( - # f"[Unreachable] Failed to get active consumer count: {e}" - # ) - # active_consumers = 0 - - # # ceil division without importing math: (a + b - 1) // b - # mapped_prefetch = 0 - # if prefetch_num > 0 and active_consumers > 0: - # mapped_prefetch = (prefetch_num + active_consumers - 1) // active_consumers - result = self.mpsc_consumer.get_data( - batch_size, try_time, prefetch_num=prefetch_num - ) + result = self.mpsc_consumer.get_data(batch_size, try_time=try_time) if not result.is_ok(): err = result.unwrap_error() if self.shutdown_ctl.closed: @@ -2548,6 +2569,18 @@ def close(self) -> Result[OkNone, ApiError]: f"MPMCChanConsumer {self.get_consumer_id()} before_close on underlying MPSC consumer failed: {e}" ) + # Close the underlying MPSC consumer first so local keepalive/prefetch + # tasks stop before lease revoke and ready-key cleanup. + try: + if self.mpsc_consumer is not None: + self.mpsc_consumer.release_local_handle().unwrap() + except Exception as e: # noqa: BLE001 + logging.warning( + f"MPMCChanConsumer {self.get_consumer_id()} failed to close underlying MPSC consumer: {e}" + ) + finally: + self.mpsc_consumer = None + # Delete ready keys for this consumer (best-effort). mpmc_id = self.mpmc_id assert mpmc_id is not None, "MPMC channel ID is None" @@ -2599,17 +2632,6 @@ def close(self) -> Result[OkNone, ApiError]: f"MPMCChanConsumer {self.get_consumer_id()} failed to revoke member lease: {e}" ) - # Close the underlying MPSC consumer and drop the handle. - try: - if self.mpsc_consumer is not None: - self.mpsc_consumer.release_local_handle().unwrap() - except Exception as e: # noqa: BLE001 - logging.warning( - f"MPMCChanConsumer {self.get_consumer_id()} failed to close underlying MPSC consumer: {e}" - ) - finally: - self.mpsc_consumer = None - # Optional sub-component cleanup. try: if hasattr(self, 'rate_limiter') and self.rate_limiter is not None: diff --git a/fluxon_py/_api_ext_chan/mpsc.py b/fluxon_py/_api_ext_chan/mpsc.py index 1eeac76..7905c4e 100644 --- a/fluxon_py/_api_ext_chan/mpsc.py +++ b/fluxon_py/_api_ext_chan/mpsc.py @@ -8,10 +8,9 @@ Old Python implementations (ChanManager, etcd watchers, prefetch queues) have been removed. -Currently this shim focuses on wiring up leases and identities. Data -path operations (`put_data`/`get_data`) are intentionally left as -placeholders and should be implemented in Rust and exposed via -`fluxon_pyo3` in follow-up work. +Broker-backed data-path operations are the default public contract. +The old direct MPSC data path is kept only behind private helpers for +short-lived internal checks. """ from __future__ import annotations @@ -55,6 +54,11 @@ logging = init_logger(__name__) +MPSC_PREFETCH_TARGET_MAX = 256 +MPSC_KVCLIENT_KEEPALIVE_RETRY_SLEEP_SECONDS = 0.05 +MPSC_KVCLIENT_KEEPALIVE_RETRIES = 3 +_LEASE_BACKEND_CALLBACK_LOCKS: Dict[str, threading.Lock] = {} +_LEASE_BACKEND_CALLBACK_LOCKS_GUARD = threading.Lock() # --------------------------------------------------------------------------- # Test-only GC close markers @@ -269,6 +273,11 @@ def _ensure_kvclient_lease_backend(api: KvClient, cluster: str) -> Any: message="KvClient must implement KvLeaseApi for MPSC payload lease", ) + with _LEASE_BACKEND_CALLBACK_LOCKS_GUARD: + callback_lock = _LEASE_BACKEND_CALLBACK_LOCKS.setdefault( + cluster, threading.Lock() + ) + def allocate_cb(ttl_seconds: int) -> int: """Bridge to KvLeaseApi.allocate_lease for the given TTL. @@ -279,7 +288,8 @@ def allocate_cb(ttl_seconds: int) -> int: Do NOT raise ApiError dataclasses here (they are not Exceptions) to avoid PyErr(TypeError: exceptions must derive from BaseException). """ - res = api.allocate_lease(int(ttl_seconds)) + with callback_lock: + res = api.allocate_lease(int(ttl_seconds)) if not res.is_ok(): # Raise a real Python Exception so PyO3 converts it to Err(...) raise RuntimeError( @@ -297,8 +307,21 @@ def keepalive_cb(lease_id: int) -> None: cause type conversion errors in PyO3. See logs: "exceptions must derive from BaseException" when raising non-Exception ApiError values. """ - # Keepalive must not alter TTL; do not pass custom_ttl - res = api.keepalive_lease(int(lease_id)) + # Keepalive must not alter TTL; do not pass custom_ttl. The PyO3 + # KvClient object uses mutable Rust borrows, so serialize callbacks + # from the lease actor to avoid re-entering the same client handle. + for attempt in range(MPSC_KVCLIENT_KEEPALIVE_RETRIES): + with callback_lock: + res = api.keepalive_lease(int(lease_id)) + if res.is_ok(): + _ = res.unwrap() + return None + err = res.unwrap_error() + if "Already mutably borrowed" in str(err) and attempt + 1 < MPSC_KVCLIENT_KEEPALIVE_RETRIES: + time.sleep(MPSC_KVCLIENT_KEEPALIVE_RETRY_SLEEP_SECONDS) + continue + break + if not res.is_ok(): err = res.unwrap_error() # When the client is shutting down, background keepalive calls can race with the @@ -311,9 +334,6 @@ def keepalive_cb(lease_id: int) -> None: raise RuntimeError( f"kvclient keepalive_lease failed for cluster={cluster}: {err}" ) - # Success: consume Ok(None) to satisfy strict Result policy - _ = res.unwrap() - # Success path: return None explicitly to map to Rust () return None # Inject kvclient allocate/keepalive callbacks while constructing LeaseBackendUid. @@ -403,6 +423,11 @@ def new_consumer( parent_mpmc_member_id_opt, ) + def delete_broker_channel(self, chan_id: str) -> list[str]: + if not isinstance(chan_id, str) or not chan_id.isdigit(): + raise ValueError(f"invalid broker channel id: {chan_id!r}") + return list(self._inner.delete_broker_channel(int(chan_id))) + def close(self) -> None: self._inner.close() @@ -503,11 +528,13 @@ def __init__( # through the Rust MPSC layer. self._payload_lease_id = self._handle.payload_lease_id() # type: ignore[attr-defined] + self._handle.init_broker() # type: ignore[attr-defined] + # Expose chan_id for legacy call sites that accessed the attribute. self.chan_id = self._chan_id logging.info( - "%s initialized via Rust MPSC: chan_id=%s, producer_idx=%s", + "%s initialized via Rust MPSC broker path: chan_id=%s, producer_idx=%s", self.dbg_tag(), self.get_chan_id(), self.get_producer_id(), @@ -543,6 +570,25 @@ def record_blocking_put_observed(self, unix_ms: int) -> None: def put_data( self, value: Dict[str, Union[int, float, bool, str, bytes, DLPacked]] + ) -> Result[bool, ApiError]: + return self._put_data_with_writer( + value, + self._handle.put_flat_dict_ptrs, # type: ignore[attr-defined] + ) + + def _put_data_legacy_for_internal_check( + self, value: Dict[str, Union[int, float, bool, str, bytes, DLPacked]] + ) -> Result[bool, ApiError]: + """Use the old direct MPSC write path for temporary internal checks only.""" + return self._put_data_with_writer( + value, + self._handle.put_flat_dict_ptrs_legacy_for_internal_check, # type: ignore[attr-defined] + ) + + def _put_data_with_writer( + self, + value: Dict[str, Union[int, float, bool, str, bytes, DLPacked]], + writer: Any, ) -> Result[bool, ApiError]: """Put data into the channel via Rust backend. @@ -576,7 +622,7 @@ def put_data( dlpack_capsules: List[object] = [] try: ptrs = _fluxon_kv.build_flat_dict_ptrs(value, keepalive, dlpack_capsules) - self._handle.put_flat_dict_ptrs(ptrs) # type: ignore[attr-defined] + writer(ptrs) except Exception as e: # pragma: no cover - thin shim if _is_close_during_put_error(e): self.shutdown_ctl.closed = True @@ -608,6 +654,10 @@ def put_data( # If Rust changes LeaseMgrError variants or mappings, update: # 1) The LeaseMgrError mapping in py_error_from_kv_error; # 2) The check here and its corresponding tests. + if e.__class__.__name__ == "MessageBufferFullError": + logging.debug("%s put_flat_dict_ptrs backpressured: %s", self.dbg_tag(), e) + return Result[bool, ApiError].new_error(e) # type: ignore[arg-type] + logging.error("%s put_flat_dict_ptrs failed: %s", self.dbg_tag(), e) if isinstance(e, PayloadLeaseNotFoundError): # Mark closed and best-effort notify Rust side to stop callbacks/holds. @@ -817,11 +867,12 @@ def __init__( else: self._handle.init_payload_callback(self._build_get_payload()) # type: ignore[attr-defined] self._handle.init_delete_callback(self._build_delete_callback()) # type: ignore[attr-defined] + self._handle.init_broker() # type: ignore[attr-defined] # Guard to make close idempotent without relying on None checks. self._closed_local: bool = False logging.info( - "%s initialized via Rust MPSC: chan_id=%s, consumer_idx=%s, payload_backend=%s", + "%s initialized via Rust MPSC broker path: chan_id=%s, consumer_idx=%s, payload_backend=%s", self._dbg_tag, self._chan_id, self._consumer_id, @@ -1080,38 +1131,144 @@ def get_data( List[Union[Dict[str, Union[int, float, bool, str, bytes, DLPacked]], ConsumedMessage]], ApiError, ]: - """Unified prefetch-first get API. + return self._get_data_broker( + batch_size=batch_size, + try_time=try_time, + prefetch_num=prefetch_num, + ) + + def _get_data_legacy_for_internal_check( + self, + batch_size: int = 1, + try_time: Optional[int] = None, + prefetch_num: int = 0, + ) -> Result[ + List[Union[Dict[str, Union[int, float, bool, str, bytes, DLPacked]], ConsumedMessage]], + ApiError, + ]: + """Use the old prefetch MPSC read path for temporary internal checks only.""" + return self._get_data_legacy_prefetch( + batch_size=batch_size, + try_time=try_time, + prefetch_num=prefetch_num, + ) + + def _get_data_broker( + self, + *, + batch_size: int, + try_time: Optional[int], + prefetch_num: int, + ) -> Result[ + List[Union[Dict[str, Union[int, float, bool, str, bytes, DLPacked]], ConsumedMessage]], + ApiError, + ]: + """Get data via the broker-backed public path.""" + timeout_ms = self._get_timeout_ms(try_time) + prefetch_target = min( + batch_size + max(prefetch_num, 0), + MPSC_PREFETCH_TARGET_MAX, + ) + try: + batch = self._handle.get_batch( # type: ignore[attr-defined] + batch_size, + prefetch_target, + timeout_ms, + ) + except Exception as e: + if self.shutdown_ctl.closed: + api_err: ApiError = ChannelClosedError( + message="Consumer is closed.", + channel_id=self._chan_id, + ) + elif isinstance(e, ApiError): + api_err = e + else: + api_err = MqGetDataUnknownError.from_exception( + e, channel_id=self._chan_id, consumer_id=self._consumer_id + ) + if isinstance(api_err, (MessageConsumptionNoNewMessageError, ChannelClosedError)): + logging.debug("%s get_batch finished without payload: %s", self.dbg_tag(), api_err) + else: + logging.error("%s get_batch failed: %s", self.dbg_tag(), api_err) + return Result[ + List[ + Union[Dict[str, Union[int, float, bool, str, bytes, DLPacked]], ConsumedMessage] + ], + ApiError, + ].new_error(api_err) + + if not batch: + return Result[ + List[ + Union[Dict[str, Union[int, float, bool, str, bytes, DLPacked]], ConsumedMessage] + ], + ApiError, + ].new_error( + MessageConsumptionNoNewMessageError("No message available") + ) + + return Result(batch) + + def _get_data_legacy_prefetch( + self, + *, + batch_size: int, + try_time: Optional[int], + prefetch_num: int, + ) -> Result[ + List[Union[Dict[str, Union[int, float, bool, str, bytes, DLPacked]], ConsumedMessage]], + ApiError, + ]: + """Get data through the old direct MPSC prefetch path.""" + timeout_ms = self._get_timeout_ms(try_time) + + return self._get_data_with_fetcher( + batch_size=batch_size, + fetch_one=lambda prefetch_target, _timeout_ms: self._handle.get_one_legacy_for_internal_check( # type: ignore[attr-defined] + prefetch_target, + timeout_ms, + ), + prefetch_target=min( + batch_size + max(prefetch_num, 0), + MPSC_PREFETCH_TARGET_MAX, + ), + timeout_ms=timeout_ms, + ) + + def _get_timeout_ms(self, try_time: Optional[int]) -> Optional[int]: + if try_time is None: + return None + t_sec = try_time if try_time > 0 else 1 + timeout_ms = int(t_sec * 1000) + assert timeout_ms > 0 + return timeout_ms + + def _get_data_with_fetcher( + self, + *, + batch_size: int, + fetch_one: Any, + prefetch_target: int = 0, + timeout_ms: Optional[int] = None, + ) -> Result[ + List[Union[Dict[str, Union[int, float, bool, str, bytes, DLPacked]], ConsumedMessage]], + ApiError, + ]: + """Common get loop used by broker and internal legacy checks. Semantics: - If it returns Ok([...]), each element is from a successful get_one call. - - If any get_one in this batch raises an error, the entire batch fails and - returns Err(ApiError) immediately (no "partial success" Ok list). - - The window size is mapped to `batch_size + prefetch_num`, so the underlying - Rust actor maintains a local prefetch queue of that size. + - NoNewMessage/ChannelClosed only fail the call when the batch is still empty. + Already-consumed items must be returned to avoid losing partial progress. + - Payload/decode/unknown errors still fail immediately. """ - prefetch_target = batch_size + max(prefetch_num, 0) - - # Inline minimal fetch loop with explicit prefetch_target to keep - # ChannelConsumer.try_get_data signature aligned while still - # honoring the calculated window size here. results: List[Union[Dict[str, Union[int, float, bool, str, bytes, DLPacked]], ConsumedMessage]] = [] - # try_time is seconds in Python; Rust get_one expects milliseconds. - timeout_ms: Optional[int] - if try_time is None: - timeout_ms = None - else: - # Compatibility: try_time must not be 0; if callers pass 0, treat it as 1 second. - t_sec = try_time if try_time > 0 else 1 - timeout_ms = int(t_sec * 1000) - assert timeout_ms > 0 - + for _ in range(batch_size): try: - # Pass timeout_ms (converted from try_time seconds) to Rust. - obj = self._handle.get_one(prefetch_target, timeout_ms) # type: ignore[attr-defined] + obj = fetch_one(prefetch_target, timeout_ms) except Exception as e: - logging.error("%s get_one failed: %s", self.dbg_tag(), e) # Rust is expected to raise an extension-layer ApiError. To avoid carrying # arbitrary Exception types in Result, wrap non-ApiError into # MqGetDataUnknownError to keep the error taxonomy narrow. @@ -1126,6 +1283,12 @@ def get_data( api_err = MqGetDataUnknownError.from_exception( e, channel_id=self._chan_id, consumer_id=self._consumer_id ) + if isinstance(api_err, (MessageConsumptionNoNewMessageError, ChannelClosedError)): + logging.debug("%s get_one finished without payload: %s", self.dbg_tag(), api_err) + if results: + return Result(results) + else: + logging.error("%s get_one failed: %s", self.dbg_tag(), api_err) return Result[ List[ Union[Dict[str, Union[int, float, bool, str, bytes, DLPacked]], ConsumedMessage] diff --git a/fluxon_py/kvclient/fluxon.py b/fluxon_py/kvclient/fluxon.py index 1325e3d..6a4dacc 100644 --- a/fluxon_py/kvclient/fluxon.py +++ b/fluxon_py/kvclient/fluxon.py @@ -299,6 +299,9 @@ def __init__(self, config: FluxonKvClientConfig): self._client: Optional[fluxon_pyo3.KvClient] = None self._config = config self._init_error: Optional[ApiError] = None + self._client_op_lock = threading.RLock() + self._closing = False + self._closed = False cluster_name = config.fluxonkv_spec_cluster_name self._blocking_put_outer_total_log_window = _BlockingPutOuterTotalLogWindow( f"FluxonKVCacheStore[{cluster_name}]" @@ -776,20 +779,31 @@ def instance_key(self) -> Result[str, ApiError]: def close(self) -> Result[OkNone, ApiError]: """Close and tear down the store.""" try: - # Backend returns a Result; MUST be explicitly consumed to avoid - # leaking an unconsumed Result that triggers __del__ assertion. - res = self._client.close() - if not res.is_ok(): - # Propagate backend error (already an ApiError) - return Result.new_error(res.unwrap_error()) - # Consume Ok(None-like) to satisfy strict consumption policy - _ = res.unwrap() - unregister_store_from_cleanup(self) - # English note: - # After a successful close, clear the backend handle to prevent any further calls and - # allow deterministic resource release without relying on Python GC timing. - self._client = None + with self._client_op_lock: + if self._closed: + return Result.new_ok(OkNone()) + self._closing = True + if self._client is None: + self._closed = True + unregister_store_from_cleanup(self) + return Result.new_ok(OkNone()) + # Backend returns a Result; MUST be explicitly consumed to avoid + # leaking an unconsumed Result that triggers __del__ assertion. + res = self._client.close() + if not res.is_ok(): + # Propagate backend error (already an ApiError) + return Result.new_error(res.unwrap_error()) + # Consume Ok(None-like) to satisfy strict consumption policy + _ = res.unwrap() + unregister_store_from_cleanup(self) + # English note: + # After a successful close, clear the backend handle to prevent any further calls and + # allow deterministic resource release without relying on Python GC timing. + self._client = None + self._closed = True return Result.new_ok(OkNone()) + except KeyboardInterrupt as e: + return Result.new_error(GeneralError(f"Store close interrupted: {str(e)}")) except Exception as e: return Result.new_error(GeneralError(f"Failed to close client: {str(e)}")) @@ -892,7 +906,10 @@ def metrics_snapshot(self) -> MetricSnapshot: # --- Fluxon-kv lease helpers (synchronous) --- def allocate_lease(self, ttl_seconds: int) -> Result[int, ApiError]: try: - inner = self._client.allocate_lease(ttl_seconds) + with self._client_op_lock: + if self._closing or self._closed or self._client is None: + return Result.new_error(GeneralError("allocate_lease called after store close started")) + inner = self._client.allocate_lease(ttl_seconds) if not inner.is_ok(): return Result.new_error(inner.unwrap_error()) lease_id = inner.unwrap() @@ -903,7 +920,10 @@ def allocate_lease(self, ttl_seconds: int) -> Result[int, ApiError]: def keepalive_lease(self, lease_id: int) -> Result[OkNone, ApiError]: try: - inner = self._client.keepalive_lease(lease_id, "kvclient") + with self._client_op_lock: + if self._closing or self._closed or self._client is None: + return Result.new_ok(OkNone()) + inner = self._client.keepalive_lease(lease_id, "kvclient") if not inner.is_ok(): return Result.new_error(inner.unwrap_error()) # Success returns a None-like sentinel from PyO3; normalize to OkNone diff --git a/fluxon_py/runtime/__init__.py b/fluxon_py/runtime/__init__.py index 692b741..fda3b65 100644 --- a/fluxon_py/runtime/__init__.py +++ b/fluxon_py/runtime/__init__.py @@ -8,6 +8,10 @@ "run_kv_master_service_blocking", "start_kv_master_process", "start_kv_master_process_with_config_b64", + "run_broker_blocking", + "run_broker_service_blocking", + "start_broker_process", + "start_broker_process_with_config_b64", "run_owner_kvclient_blocking", "run_owner_kvclient_service_blocking", "start_owner_kvclient_process", @@ -37,6 +41,10 @@ "run_kv_master_service_blocking": ("start_master", "run_kv_master_service_blocking"), "start_kv_master_process": ("start_master", "start_kv_master_process"), "start_kv_master_process_with_config_b64": ("start_master", "start_kv_master_process_with_config_b64"), + "run_broker_blocking": ("start_broker", "run_kv_broker_blocking"), + "run_broker_service_blocking": ("start_broker", "run_kv_broker_service_blocking"), + "start_broker_process": ("start_broker", "start_kv_broker_process"), + "start_broker_process_with_config_b64": ("start_broker", "start_kv_broker_process_with_config_b64"), "run_owner_kvclient_blocking": ("start_owner_kvclient", "run_owner_kvclient_blocking"), "run_owner_kvclient_service_blocking": ("start_owner_kvclient", "run_owner_kvclient_service_blocking"), "start_owner_kvclient_process": ("start_owner_kvclient", "start_owner_kvclient_process"), diff --git a/fluxon_py/runtime/start_broker.py b/fluxon_py/runtime/start_broker.py new file mode 100644 index 0000000..dd7a70e --- /dev/null +++ b/fluxon_py/runtime/start_broker.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import subprocess +from pathlib import Path +import yaml + +from fluxon_py.tool import import_fluxon_pyo3_local + +from .process_runner import ( + bind_current_process_parent_death_sigterm, + build_runtime_singleton_spec, + RuntimeConfigInput, + decode_runtime_config_b64, + encode_runtime_config_b64, + resolve_runtime_config_path, + run_singleton_process, + start_python_module_process, + start_python_module_process_with_config_b64, +) + + +BROKER_MODULE_NAME = "fluxon_py.runtime.start_broker" +STOP_EXISTING_BROKER_TIMEOUT_SECONDS = 30 +BROKER_RUNTIME_CONFIG_FILENAME = "kv_broker.runtime.yaml" + + +def run_kv_broker_blocking( + *, + workdir: Path, + config: RuntimeConfigInput | None = None, + config_path: Path | None = None, +) -> None: + resolved_workdir = workdir.resolve() + resolved_config = resolve_runtime_config_path( + workdir=resolved_workdir, + runtime_config_filename=BROKER_RUNTIME_CONFIG_FILENAME, + config=config, + config_path=config_path, + ) + singleton_spec = build_runtime_singleton_spec( + module_name=BROKER_MODULE_NAME, + entrypoint_path=Path(__file__), + workdir=workdir, + ) + run_singleton_process( + config_path=resolved_config, + singleton_spec=singleton_spec, + stop_timeout_seconds=STOP_EXISTING_BROKER_TIMEOUT_SECONDS, + start_fn=lambda: run_kv_broker_service_blocking( + config_path=resolved_config, + workdir=resolved_workdir, + ), + ) + + +def run_kv_broker_service_blocking(*, config_path: Path, workdir: Path) -> None: + fluxon_pyo3 = import_fluxon_pyo3_local() + result = fluxon_pyo3.run_broker_blocking(str(config_path)) + if not result.is_ok(): + raise RuntimeError(f"run_broker_blocking failed: {result.unwrap_error()}") + + _ = result.unwrap() + + +def run_kv_broker_service_blocking_from_yaml_text(*, config_yaml: str) -> None: + config = yaml.safe_load(config_yaml) + if not isinstance(config, dict): + raise TypeError(f"broker config must decode to dict, got {type(config).__name__}") + fluxon_pyo3 = import_fluxon_pyo3_local() + result = fluxon_pyo3.run_broker_blocking(config) + if not result.is_ok(): + raise RuntimeError(f"run_broker_blocking failed: {result.unwrap_error()}") + + _ = result.unwrap() + + +def start_kv_broker_process( + *, + workdir: Path | None = None, + config: RuntimeConfigInput | None = None, + config_path: Path | None = None, + log_path: Path | None = None, +) -> subprocess.Popen[bytes]: + if config_path is None and isinstance(config, dict) and workdir is None: + return start_kv_broker_process_with_config_b64(config=config, log_path=log_path) + if workdir is None: + raise ValueError("workdir is required when config is not a dict and config_path is not provided") + resolved_workdir = workdir.resolve() + resolved_config = resolve_runtime_config_path( + workdir=resolved_workdir, + runtime_config_filename=BROKER_RUNTIME_CONFIG_FILENAME, + config=config, + config_path=config_path, + ) + return start_python_module_process( + module_name=BROKER_MODULE_NAME, + config_path=resolved_config, + workdir=resolved_workdir, + extra_cli_args=(), + log_path=log_path, + ) + + +def start_kv_broker_process_with_config_b64( + *, + config: dict, + log_path: Path | None = None, +) -> subprocess.Popen[bytes]: + return start_python_module_process_with_config_b64( + module_name=BROKER_MODULE_NAME, + config_b64=encode_runtime_config_b64(config), + extra_cli_args=(), + log_path=log_path, + ) + + +def main() -> None: + bind_current_process_parent_death_sigterm() + parser = argparse.ArgumentParser(description="Start Fluxon KV broker (blocking)") + parser.add_argument("-c", "--config", type=Path, required=False, help="Path to broker YAML config") + parser.add_argument("-w", "--workdir", type=Path, required=False, help="Working directory") + parser.add_argument("--config-b64", required=False, help="Base64-encoded YAML config") + args = parser.parse_args() + if args.config_b64 is not None: + # Keep the same config transport contract as other runtime entrypoints. + run_kv_broker_service_blocking_from_yaml_text( + config_yaml=decode_runtime_config_b64(args.config_b64) + ) + return + if args.config is None or args.workdir is None: + raise ValueError("--config and --workdir are required when --config-b64 is not used") + run_kv_broker_blocking(config=args.config, workdir=args.workdir) + + +if __name__ == "__main__": + main() diff --git a/fluxon_py/tests/test_api_chan_mpmc/test_api_chan_mpmc_base.py b/fluxon_py/tests/test_api_chan_mpmc/test_api_chan_mpmc_base.py index f992c2d..0dee4d5 100644 --- a/fluxon_py/tests/test_api_chan_mpmc/test_api_chan_mpmc_base.py +++ b/fluxon_py/tests/test_api_chan_mpmc/test_api_chan_mpmc_base.py @@ -45,6 +45,7 @@ def _find_project_root(start: Path) -> Path: sys.path.insert(0, str(PROJECT_ROOT)) from typing import Dict, List, Optional, Tuple +from types import SimpleNamespace import etcd3 @@ -1399,6 +1400,33 @@ def test_mpmc_dynamic_suite() -> None: run_with_argmatrix(_test_mpmc_dynamic_suite_once) +def test_mpmc_get_data_prefetch_is_per_consumer_not_divided() -> None: + calls: List[Tuple[int, Optional[int], int]] = [] + + class _DummyInnerConsumer: + def get_data( + self, + batch_size: int, + try_time: Optional[int] = None, + prefetch_num: int = 0, + ) -> Result[List[Dict[str, object]], ApiError]: + calls.append((batch_size, try_time, prefetch_num)) + return Result.new_ok([]) + + consumer = object.__new__(MPMCChanConsumer) + consumer.shutdown_ctl = mpsc.MqShutdownCtl() + consumer.mpmc_id = "123" + consumer.mpmc_channel = SimpleNamespace( + _get_active_consumer_count=lambda: 8, + ) + consumer.mpsc_consumer = _DummyInnerConsumer() + + res = consumer.get_data(batch_size=40, try_time=2, prefetch_num=40) + + assert res.is_ok() + assert calls == [(40, 2, 40)] + + if __name__ == "__main__": diff --git a/fluxon_py/tests/test_api_chan_mpmc/test_mpmc_simple_bench.py b/fluxon_py/tests/test_api_chan_mpmc/test_mpmc_simple_bench.py index a29c46f..903ba7f 100644 --- a/fluxon_py/tests/test_api_chan_mpmc/test_mpmc_simple_bench.py +++ b/fluxon_py/tests/test_api_chan_mpmc/test_mpmc_simple_bench.py @@ -52,10 +52,12 @@ def _find_project_root(start: Path) -> Path: from fluxon_py import FluxonKvClientConfig, new_store # noqa: E402 from fluxon_py.api_error import ( # noqa: E402 ChannelClosedError, + KeyNotFoundError, MessageConsumptionNoNewMessageError, ProducerClosedError, ) from fluxon_py.api_ext_chan import ChanType # noqa: E402 +from fluxon_py._api_ext_chan.mpsc import MpscContext # noqa: E402 from fluxon_py.kvclient import KvClientType # noqa: E402 from fluxon_py.kvclient.nonzerocopy_encode import DLPackBytesView # noqa: E402 from fluxon_py.logging import init_logger # noqa: E402 @@ -382,17 +384,23 @@ def _run_one_case( ) _put_etcd_key(stop_key, b"1") time.sleep(SUMMARY_STOP_GRACE_SECONDS) - _signal_live_processes(worker_processes, signum=signal.SIGINT) try: _wait_for_processes_exit(worker_processes, timeout_seconds=WORKER_EXIT_TIMEOUT_SECONDS) except RuntimeError as err: + _signal_live_processes(worker_processes, signum=signal.SIGINT) logging.warning("[bench] worker shutdown timeout bench_id=%s error=%s", bench_id, err) + raise else: - _warn_if_worker_exited_nonzero(worker_processes, bench_id=bench_id) + _raise_if_worker_exited_nonzero(worker_processes, bench_id=bench_id) finally: _terminate_processes(worker_processes) _delete_etcd_key(stop_key) _clear_etcd_prefix(f"{SUMMARY_KEY_PREFIX}{bench_id}/") + if bootstrap_store is not None and bootstrap_producer is not None: + _best_effort_delete_case_broker_channels( + store=bootstrap_store, + mpmc_id=str(bootstrap_producer.get_chan_id()), + ) if bootstrap_producer is not None: _best_effort_close(bootstrap_producer, role="bootstrap_producer") _best_effort_close(bootstrap_store, role="bootstrap_store") @@ -985,18 +993,18 @@ def _index_summaries_by_consumer_id(summaries: list[dict[str, Any]]) -> dict[str return indexed -def _warn_if_worker_exited_nonzero(processes: list[subprocess.Popen[str]], *, bench_id: str) -> None: +def _raise_if_worker_exited_nonzero(processes: list[subprocess.Popen[str]], *, bench_id: str) -> None: + failures: list[str] = [] for proc in processes: return_code = proc.poll() if return_code is None: continue if return_code != 0: - logging.warning( - "[bench] worker exited non-zero during teardown bench_id=%s pid=%s code=%s", - bench_id, - proc.pid, - return_code, - ) + failures.append(f"pid={proc.pid} code={return_code}") + if failures: + raise RuntimeError( + f"worker exited non-zero during teardown bench_id={bench_id}: {', '.join(failures)}" + ) def _maybe_write_consumer_summary( @@ -1215,6 +1223,71 @@ def _clear_etcd_prefix(prefix: str) -> None: etcd_client.delete(meta.key) +def _best_effort_delete_case_broker_channels(*, store: Any, mpmc_id: str) -> None: + if not isinstance(mpmc_id, str) or not mpmc_id.isdigit(): + logging.warning("[bench] skip broker cleanup for invalid mpmc_id=%r", mpmc_id) + return + + channels_key = f"/mpmc_channels/{mpmc_id}/mpsc_channels" + try: + with etcd3.client(ETCD_HOST, ETCD_PORT) as etcd_client: + raw, _ = etcd_client.get(channels_key) + if raw is None: + return + loaded = json.loads(raw.decode("utf-8")) + if not isinstance(loaded, list): + raise TypeError(f"{channels_key} must contain a list, got {type(loaded).__name__}") + + ctx = MpscContext(store) + payload_key_count = 0 + payload_delete_ok = 0 + payload_delete_failed = 0 + try: + for chan_id in loaded: + if not isinstance(chan_id, str) or not chan_id.isdigit(): + raise ValueError(f"invalid sub-MPSC channel id in {channels_key}: {chan_id!r}") + payload_keys = ctx.delete_broker_channel(chan_id) + payload_key_count += len(payload_keys) + for payload_key in payload_keys: + res = store.remove(payload_key) + if res.is_ok(): + _ = res.unwrap() + payload_delete_ok += 1 + continue + err = res.unwrap_error() + if isinstance(err, KeyNotFoundError): + payload_delete_ok += 1 + continue + payload_delete_failed += 1 + logging.warning( + "[bench] broker payload cleanup failed key=%s err=%s", + payload_key, + err, + ) + finally: + ctx.close() + logging.info( + "[bench] deleted broker channels for mpmc_id=%s count=%s payload_keys=%s payload_delete_ok=%s payload_delete_failed=%s", + mpmc_id, + len(loaded), + payload_key_count, + payload_delete_ok, + payload_delete_failed, + ) + print( + "BENCH_BROKER_CLEANUP " + f"mpmc_id={mpmc_id} channels={len(loaded)} payload_keys={payload_key_count} " + f"payload_delete_ok={payload_delete_ok} payload_delete_failed={payload_delete_failed}", + flush=True, + ) + except Exception as err: # noqa: BLE001 + logging.warning( + "[bench] broker channel cleanup failed for mpmc_id=%s: %s", + mpmc_id, + err, + ) + + def _best_effort_close(obj: Any, *, role: str) -> None: close_res = obj.close() if close_res.is_ok(): diff --git a/fluxon_py/tests/test_api_chan_mpsc/test_api_chan_mpsc_base.py b/fluxon_py/tests/test_api_chan_mpsc/test_api_chan_mpsc_base.py index 884c748..f40a046 100644 --- a/fluxon_py/tests/test_api_chan_mpsc/test_api_chan_mpsc_base.py +++ b/fluxon_py/tests/test_api_chan_mpsc/test_api_chan_mpsc_base.py @@ -30,6 +30,8 @@ ChanKeyNotFoundError, ChanMessageConsumptionError, ChanMessageProduceError, + ChannelClosedError, + MessageConsumptionNoNewMessageError, ConsumerRegistrationError, ProducerRegistrationError, ) @@ -54,6 +56,7 @@ from fluxon_py._api_ext_chan.mpsc import ( # noqa: E402 _new_produce_offset_of_all_producer_key, ) +from fluxon_py._api_ext_chan import mpsc # noqa: E402 from fluxon_py.logging import init_logger # noqa: E402 from fluxon_py.tests.test_lib import ( # noqa: E402 KV_SVC_IP, @@ -1601,6 +1604,119 @@ def test_mpsc_channel_suite() -> None: run_with_argmatrix(_test_mpsc_channel_suite_once) +def test_mpsc_get_data_clamps_prefetch_target() -> None: + consumer = object.__new__(MPSCChanConsumer) + consumer.shutdown_ctl = mpsc.MqShutdownCtl() + consumer._chan_id = "1" + consumer._consumer_id = "2" + consumer._dbg_tag = "[MPSCChanConsumer chan_id=1 consumer_idx=2]" + consumer._closed_local = True + + observed_targets: List[int] = [] + + class _DummyHandle: + def get_one_legacy_for_internal_check( + self, + prefetch_target: int, + timeout_ms: Optional[int], + ) -> Dict[str, bytes]: + observed_targets.append(prefetch_target) + return {"payload": b"x"} + + consumer._handle = _DummyHandle() + + res = consumer._get_data_legacy_for_internal_check( + batch_size=40, + try_time=1, + prefetch_num=400, + ) + + assert res.is_ok() + assert observed_targets + assert all(target == mpsc.MPSC_PREFETCH_TARGET_MAX for target in observed_targets) + + +def test_mpsc_get_data_returns_partial_batch_on_no_message() -> None: + consumer = object.__new__(MPSCChanConsumer) + consumer.shutdown_ctl = mpsc.MqShutdownCtl() + consumer._chan_id = "1" + consumer._consumer_id = "2" + consumer._dbg_tag = "[MPSCChanConsumer chan_id=1 consumer_idx=2]" + consumer._closed_local = True + + class _DummyHandle: + def get_batch( + self, + batch_size: int, + prefetch_target: int, + timeout_ms: Optional[int], + ) -> List[Dict[str, bytes]]: + del batch_size, prefetch_target, timeout_ms + return [{"payload": b"x"}] + + consumer._handle = _DummyHandle() + + res = consumer.get_data(batch_size=8, try_time=1, prefetch_num=0) + + assert res.is_ok() + assert res.unwrap() == [{"payload": b"x"}] + + +def test_mpsc_get_data_returns_partial_batch_on_channel_closed() -> None: + consumer = object.__new__(MPSCChanConsumer) + consumer.shutdown_ctl = mpsc.MqShutdownCtl() + consumer._chan_id = "1" + consumer._consumer_id = "2" + consumer._dbg_tag = "[MPSCChanConsumer chan_id=1 consumer_idx=2]" + consumer._closed_local = True + + class _DummyHandle: + def get_batch( + self, + batch_size: int, + prefetch_target: int, + timeout_ms: Optional[int], + ) -> List[Dict[str, bytes]]: + del batch_size, prefetch_target, timeout_ms + return [{"payload": b"x"}] + + consumer._handle = _DummyHandle() + + res = consumer.get_data(batch_size=8, try_time=1, prefetch_num=0) + + assert res.is_ok() + assert res.unwrap() == [{"payload": b"x"}] + + +def test_mpsc_get_data_broker_passes_prefetch_target_to_batch() -> None: + consumer = object.__new__(MPSCChanConsumer) + consumer.shutdown_ctl = mpsc.MqShutdownCtl() + consumer._chan_id = "1" + consumer._consumer_id = "2" + consumer._dbg_tag = "[MPSCChanConsumer chan_id=1 consumer_idx=2]" + consumer._closed_local = True + + observed: List[int] = [] + + class _DummyHandle: + def get_batch( + self, + batch_size: int, + prefetch_target: int, + timeout_ms: Optional[int], + ) -> List[Dict[str, bytes]]: + del batch_size, timeout_ms + observed.append(prefetch_target) + return [{"payload": b"x"}] + + consumer._handle = _DummyHandle() + + res = consumer.get_data(batch_size=40, try_time=1, prefetch_num=400) + + assert res.is_ok() + assert observed == [mpsc.MPSC_PREFETCH_TARGET_MAX] + + def test_new_or_bind_unique_key_namespace_collision() -> None: setup_test_environment(logging) env = create_channel_env() diff --git a/fluxon_py/tests/test_lib.py b/fluxon_py/tests/test_lib.py index 9be7003..41e4557 100644 --- a/fluxon_py/tests/test_lib.py +++ b/fluxon_py/tests/test_lib.py @@ -173,10 +173,11 @@ def setup_test_environment(logger: Logger, print_config: bool = True): # except RuntimeError as e: # print(f"Failed to set start method to spawn: {e}, current start method: {multiprocessing.get_start_method()}") - loglevel_str="DEBUG" + loglevel_str = os.environ.get("FLUXON_LOG") or os.environ.get("LOG_LEVEL") or "DEBUG" + loglevel_str = str(loglevel_str).upper() os.environ["LOG_LEVEL"] = loglevel_str os.environ["FLUXON_LOG"] = loglevel_str - LOGGING_LEVEL= logging.DEBUG + LOGGING_LEVEL = getattr(logging, loglevel_str, logging.DEBUG) update_log_level(loglevel_str) print("=================================================") @@ -190,7 +191,7 @@ def emit(self, record): self.flush() # Flush immediately for every log record handler = FlushStreamHandler(sys.stdout) - handler.setLevel(logging.DEBUG) + handler.setLevel(LOGGING_LEVEL) formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) diff --git a/fluxon_py/tests/test_mq/test_example_ctrl_c_exit.py b/fluxon_py/tests/test_mq/test_example_ctrl_c_exit.py index c1b3193..1de07b3 100644 --- a/fluxon_py/tests/test_mq/test_example_ctrl_c_exit.py +++ b/fluxon_py/tests/test_mq/test_example_ctrl_c_exit.py @@ -51,6 +51,7 @@ def _find_project_root(start: Path) -> Path: CHAN_CONFIG_TEST = {"capacity": 10, "ttl_seconds": 90, "weight": 1} MASTER_SCRIPT = [sys.executable, "-m", "fluxon_py.runtime.start_master"] +BROKER_SCRIPT = [sys.executable, "-m", "fluxon_py.runtime.start_broker"] KVCLIENT_SCRIPT = [sys.executable, "-m", "fluxon_py.runtime.start_owner_kvclient"] ETCD_BIN = PROJECT_ROOT / "fluxon_release" / "ext_images" / "etcd" / "etcd" GREPTIME_BIN = PROJECT_ROOT / "fluxon_release" / "ext_images" / "greptime" / "greptime" @@ -463,6 +464,7 @@ def _build_example_config( share_mem_path: str, greptime_http_port: int, master_port: int, + broker_port: int, ) -> dict[str, Any]: capacity = max(128, int(CHAN_CONFIG_TEST["capacity"])) ttl_seconds = max(90, int(CHAN_CONFIG_TEST["ttl_seconds"])) @@ -475,6 +477,15 @@ def _build_example_config( "log_dir": str((Path(share_mem_path).parent / "log" / "master").resolve()), "monitoring": _monitoring_block(greptime_http_port=greptime_http_port), }, + "broker": { + "instance_key": f"example_ctrlc_broker_{unique_suffix}", + "fluxonkv_spec": { + "cluster_name": cluster_name, + "shared_memory_path": shared_memory_path, + "shared_file_path": str((Path(shared_memory_path).parent / "sharefile").resolve()), + "p2p_listen_port": broker_port, + }, + }, "kvclient": { "instance_key": f"example_ctrlc_owner_{unique_suffix}", "contribute_to_cluster_pool_size": {"dram": 1073741824, "vram": {}}, @@ -589,6 +600,7 @@ def _start_local_stack(*, temp_root: Path, config_path: Path) -> list[tuple[subp cluster_name = f"example_ctrlc_cluster_{unique_suffix}" share_mem_path = str((temp_root / "sharemem").resolve()) master_port = _pick_free_port() + broker_port = _pick_free_port() config = _build_example_config( unique_suffix=unique_suffix, cluster_name=cluster_name, @@ -596,14 +608,17 @@ def _start_local_stack(*, temp_root: Path, config_path: Path) -> list[tuple[subp share_mem_path=share_mem_path, greptime_http_port=greptime_http_port, master_port=master_port, + broker_port=broker_port, ) config_path.write_text( yaml.safe_dump(config, sort_keys=False), encoding="utf-8", ) master_config_path = temp_root / "master.yaml" + broker_config_path = temp_root / "broker.yaml" kvclient_config_path = temp_root / "kvclient.yaml" _write_runtime_subconfig(path=master_config_path, config=config, key="master") + _write_runtime_subconfig(path=broker_config_path, config=config, key="broker") _write_runtime_subconfig(path=kvclient_config_path, config=config, key="kvclient") master_proc = _spawn_logged( @@ -643,8 +658,25 @@ def _start_local_stack(*, temp_root: Path, config_path: Path) -> list[tuple[subp proc=kvclient_proc, log_path=kvclient_log, ) + + broker_log = temp_root / "log" / "broker.log" + broker_proc = _spawn_logged( + cmd=[ + *BROKER_SCRIPT, + "-c", + str(broker_config_path), + "-w", + str((temp_root / "broker_work").resolve()), + ], + workdir=PROJECT_ROOT, + log_path=broker_log, + env=env, + ) + time.sleep(2.0) + _require_process_running(broker_proc, label="broker", log_path=broker_log) return [ (kvclient_proc, kvclient_log), + (broker_proc, broker_log), (master_proc, master_log), (etcd_proc, etcd_log), (greptime_proc, greptime_log), diff --git a/fluxon_rs/Cargo.lock b/fluxon_rs/Cargo.lock index a4b0ecd..964cd8c 100644 --- a/fluxon_rs/Cargo.lock +++ b/fluxon_rs/Cargo.lock @@ -1230,6 +1230,7 @@ dependencies = [ "fluxon_commu", "fluxon_framework", "fluxon_framework_compiled", + "fluxon_mq", "fluxon_observability", "fluxon_util", "futures", @@ -1275,6 +1276,7 @@ version = "0.2.1" dependencies = [ "anyhow", "async-trait", + "bitcode", "downcast-rs", "etcd-client", "fluxon_commu", diff --git a/fluxon_rs/fluxon_commu/src/facade/p2p.rs b/fluxon_rs/fluxon_commu/src/facade/p2p.rs index 8bcc169..79114f1 100644 --- a/fluxon_rs/fluxon_commu/src/facade/p2p.rs +++ b/fluxon_rs/fluxon_commu/src/facade/p2p.rs @@ -93,6 +93,19 @@ pub mod __hidden { self.view.upgrade() } + pub fn try_with_cluster_manager( + &self, + f: impl FnOnce(&crate::cluster_manager::ClusterManager) -> R, + ) -> Option { + let arc_view = self.view.upgrade()?; + unsafe { + let ptr = + std::ptr::NonNull::new(Arc::as_ptr(&arc_view) as *const _ as *mut _).unwrap(); + let view_ref: &dyn P2pModuleViewTrait = ptr.as_ref(); + Some(f(view_ref.cluster_manager())) + } + } + pub fn resource_registry(&self) -> &ResourceRegistry { let arc_view = self.view.upgrade().expect( "view of module P2pModule has been dropped when accessing resource registry", @@ -489,11 +502,6 @@ impl P2pModule { return true; } let view = self.module_view(); - let cm = view.cluster_manager(); - let self_info = cm.get_self_info(); - if self_info.node_role() != crate::NodeRole::External { - return false; - } let snapshot = self.cached_tier_snapshot(); let Some(peer_gen) = snapshot.peer_gen(logical_peer) else { return false; @@ -501,24 +509,31 @@ impl P2pModule { if !snapshot.is_send_ready_intra_effective(&peer_gen) { return false; } - let Some(owner_id) = self_info - .metadata - .get(crate::META_KEY_SHARED_STORAGE_NODE_ID) - else { - return false; - }; - if logical_peer.as_ref() == owner_id.as_str() { - return false; - } - let Some(handle) = cm.ipc_bandwidth_attributor_handle() else { - return false; - }; - match direction { - "tx" => handle.record_rx_bytes(bytes), - "rx" => handle.record_tx_bytes(bytes), - _ => return false, - } - true + view.try_with_cluster_manager(|cm| { + let self_info = cm.get_self_info(); + if self_info.node_role() != crate::NodeRole::External { + return false; + } + let Some(owner_id) = self_info + .metadata + .get(crate::META_KEY_SHARED_STORAGE_NODE_ID) + else { + return false; + }; + if logical_peer.as_ref() == owner_id.as_str() { + return false; + } + let Some(handle) = cm.ipc_bandwidth_attributor_handle() else { + return false; + }; + match direction { + "tx" => handle.record_rx_bytes(bytes), + "rx" => handle.record_tx_bytes(bytes), + _ => return false, + } + true + }) + .unwrap_or(false) } } diff --git a/fluxon_rs/fluxon_commu/src/facade/transfer_engine.rs b/fluxon_rs/fluxon_commu/src/facade/transfer_engine.rs index 878e5c6..e5353a5 100644 --- a/fluxon_rs/fluxon_commu/src/facade/transfer_engine.rs +++ b/fluxon_rs/fluxon_commu/src/facade/transfer_engine.rs @@ -74,12 +74,10 @@ impl ClosedLocalSegmentLeaseRegistry { where G: Send + Sync + 'static, { - let boxed = self - .guards - .lock() - .await - .remove(&handle) - .ok_or_else(|| format!("closed sdk local segment lease handle {handle} not found"))?; + let boxed = + self.guards.lock().await.remove(&handle).ok_or_else(|| { + format!("closed sdk local segment lease handle {handle} not found") + })?; boxed.downcast::().map(|guard| *guard).map_err(|_| { format!( "closed sdk local segment lease handle {handle} has unexpected runtime guard type" @@ -461,7 +459,7 @@ impl ClientTransferEngineCore { len, seg_guard, ) - .await + .await } } @@ -482,7 +480,11 @@ impl ClientTransferEngineCore { let initial_local_segment_guard = match seg_guard { Some(guard) => Some(guard), None if runtime.supports_local_segment_transfer() => { - let local_addr = if peer_src_or_target { target_addr } else { src_addr }; + let local_addr = if peer_src_or_target { + target_addr + } else { + src_addr + }; match runtime.ensure_local_segment_guard(local_addr, None).await { Ok(guard) => Some(guard), Err(_) => None, diff --git a/fluxon_rs/fluxon_kv/Cargo.toml b/fluxon_rs/fluxon_kv/Cargo.toml index 22ff136..fe7c669 100644 --- a/fluxon_rs/fluxon_kv/Cargo.toml +++ b/fluxon_rs/fluxon_kv/Cargo.toml @@ -95,6 +95,7 @@ limit_thirdparty = { path = "../limit_thirdparty" } fluxon_cli = { path = "../fluxon_cli" } fluxon_util = { path = "../fluxon_util" } fluxon_observability = { path = "../fluxon_observability" } +fluxon_mq = { path = "../fluxon_mq" } [build-dependencies] tonic-build = { workspace = true } fluxon_util = { path = "../fluxon_util" } diff --git a/fluxon_rs/fluxon_kv/framework_init_steps.yaml b/fluxon_rs/fluxon_kv/framework_init_steps.yaml index 923ae30..c90cd28 100644 --- a/fluxon_rs/fluxon_kv/framework_init_steps.yaml +++ b/fluxon_rs/fluxon_kv/framework_init_steps.yaml @@ -4,6 +4,8 @@ title: fluxon_kv init variants: - id: master tags: [master] + - id: broker + tags: [broker, external] - id: owner tags: [owner] - id: external @@ -20,8 +22,8 @@ variants: # - A step depends on a resource by declaring `deps: ["res:"]`. resources: - id: cluster_member_watch_ready - tags: [master, owner, external] - publish_tags: [master, owner, external] + tags: [master, broker, owner, external] + publish_tags: [master, broker, owner, external] published_by: ClusterManager.step.1.init2 doc: | - ClusterManager: member watch is established and continuous observation is available @@ -56,8 +58,8 @@ resources: # `Framework.step.0.attach_views`. module_tags: - ClusterManager: [master, owner, external] - P2pModule: [master, owner, external] + ClusterManager: [master, broker, owner, external] + P2pModule: [master, broker, owner, external] MasterSegManager: [master] MasterKvRouter: [master] MetricReporter: [master, owner, external] diff --git a/fluxon_rs/fluxon_kv/src/kv_test.rs b/fluxon_rs/fluxon_kv/src/kv_test.rs index 5f0a9e2..eb9e6c9 100644 --- a/fluxon_rs/fluxon_kv/src/kv_test.rs +++ b/fluxon_rs/fluxon_kv/src/kv_test.rs @@ -1381,7 +1381,10 @@ async fn key_meta_cache_check( } } - tracing::info!("🔍 Starting PUT and GET in parallel: {}", parallel_unique_key); + tracing::info!( + "🔍 Starting PUT and GET in parallel: {}", + parallel_unique_key + ); for i in 0..10 { let (put_client, other_client) = if i % 2 == 0 { (client, client2) @@ -1420,7 +1423,9 @@ async fn key_meta_cache_check( } assert!( - put_client.client_kv_api().has_cached_key(parallel_unique_key), + put_client + .client_kv_api() + .has_cached_key(parallel_unique_key), "put client should have immediate local cache metadata for key {} after put time {}", parallel_unique_key, i diff --git a/fluxon_rs/fluxon_kv/src/lib.rs b/fluxon_rs/fluxon_kv/src/lib.rs index edaa386..ac2b61b 100644 --- a/fluxon_rs/fluxon_kv/src/lib.rs +++ b/fluxon_rs/fluxon_kv/src/lib.rs @@ -86,6 +86,10 @@ use external_client_api::{ExternalClientApi, ExternalClientApiNewArg}; use fluxon_commu::TransferBackendActivationMode; use fluxon_framework::LogicalModule; use fluxon_framework::{AnyResult, define_framework}; +use fluxon_mq::{ + FLUXON_MQ_COMPONENT_BROKER_METADATA_VALUE, FLUXON_MQ_COMPONENT_METADATA_KEY, + register_broker_service, +}; use master_kv_router::{MasterKvRouter, MasterKvRouterNewArg}; use master_seg_manager::MasterSegManager; use metric_reporter::{ @@ -194,6 +198,11 @@ pub(crate) struct MasterRunTestOverrides { pub transfer_backend_activation_mode: Option, } +#[derive(Clone, Debug)] +pub(crate) struct BrokerRunTestOverrides { + pub rdma_control_init: ClusterManagerRdmaControlInit, +} + /// Result of a unified `get` that carries the role-specific holder types. #[derive(Clone)] pub enum KvGetResult { @@ -460,6 +469,12 @@ enum Commands { #[arg(short = 'f', long = "config")] config: Option, }, + /// Run as broker node + Broker { + /// Configuration file path + #[arg(short = 'f', long = "config")] + config: Option, + }, /// Run as client node Client { /// Configuration file path @@ -1336,6 +1351,15 @@ pub async fn entry() -> Result<()> { .await .map_err(|e| anyhow::anyhow!("{}", e))?; } + Commands::Broker { config } => { + let config_arg = config.map_or(ConfigArg::None, ConfigArg::File); + let (framework, _) = run_broker(config_arg).await?; + framework.wait_shutdown_signal().await; + framework + .shutdown() + .await + .map_err(|e| anyhow::anyhow!("{}", e))?; + } Commands::Client { config } => { let config_arg = config.map_or(ConfigArg::None, ConfigArg::File); let (framework, _) = run_client(config_arg).await?; @@ -1548,6 +1572,205 @@ pub async fn run_master( run_master_impl(config_arg, None).await } +async fn run_broker_impl( + config_arg: ConfigArg, + test_overrides: Option, +) -> Result<(Arc, ClientConfig)> { + #[cfg(unix)] + segfault_handler::install_sigsegv_classifier(); + + println!("Starting cache backend in BROKER mode"); + + let build_version = fluxon_util::git_version_build_record::get_current_git_commitid().unwrap(); + let source_sha256 = fluxon_util::build_info::SOURCE_SHA256; + println!("Build version (git commit): {}", build_version); + println!("Build version (source-sha256): {}", source_sha256); + + let config = load_client_config(config_arg) + .await + .map_err(|e| anyhow::anyhow!("Failed to load broker config: {}", e))?; + + let dram = config.contribute_to_cluster_pool_size.dram; + let vram_is_zero = config + .contribute_to_cluster_pool_size + .vram + .values() + .all(|&v| v == 0); + if dram != 0 || !vram_is_zero { + anyhow::bail!( + "broker config must be a zero-contribution external-client config; instance_key={}", + config.instance_key + ); + } + if matches!( + config.test_spec_config.side_transfer_role, + Some(SideTransferRole::Worker) + ) { + anyhow::bail!( + "broker config must not set test_spec_config.side_transfer_role=worker; instance_key={}", + config.instance_key + ); + } + + unsafe { + std::env::set_var( + "FLUXON_ENABLE_ICEORYX_LOGS", + if config.test_spec_config.enable_iceoryx_logs { + "1" + } else { + "0" + }, + ); + } + + let kv_logs_dir = config + .large_file_paths + .kv_logs_dir(&config.cluster_name) + .map_err(|e| anyhow::anyhow!("invalid large_file_paths for broker kv logs: {}", e))?; + let observability_disabled = config.test_spec_config.disable_observability; + let greptime_tracing_rx = if observability_disabled { + fluxon_util::init_log(&kv_logs_dir, &config.instance_key); + None + } else { + let (greptime_tracing_layer, greptime_tracing_rx) = + fluxon_observability::greptime_otlp_tracing::new_tracing_layer( + crate::config::DEFAULT_OTLP_LOG_MAX_QUEUE_LINES, + ); + fluxon_util::init_log_with_extra_layer( + &kv_logs_dir, + &config.instance_key, + greptime_tracing_layer, + ); + Some(greptime_tracing_rx) + }; + info!("Broker config: {:?}", config); + info!("Build version (git commit): {}", build_version); + info!("Build version (source-sha256): {}", source_sha256); + + let config = bootstrap_zero_contribution_client_config(config).await?; + + let mut metadata = HashMap::from([ + ("external_client".to_string(), "true".to_string()), + ( + FLUXON_MQ_COMPONENT_METADATA_KEY.to_string(), + FLUXON_MQ_COMPONENT_BROKER_METADATA_VALUE.to_string(), + ), + ("version".to_string(), build_version.clone()), + ]); + merge_startup_member_metadata(&mut metadata, HashMap::new())?; + + let rdma_control_init = test_overrides + .as_ref() + .map(|overrides| overrides.rdma_control_init.clone()) + .or_else(|| test_spec_config_rdma_control_init(Some(&config.test_spec_config))) + .unwrap_or_else(|| cluster_manager_rdma_control_init_from_config(&config)); + + let init_args = InitArgsBroker { + cluster_manager_arg: ClusterManagerNewArg { + etcd_endpoints: config.fluxonkv_spec.etcd_addresses.clone(), + cluster_name: config.cluster_name.clone(), + instance_name: Some(config.instance_key.clone()), + port: None, + metadata, + local_ipc_root: cluster_manager_local_ipc_root( + &config.share_mem_path, + &config.test_spec_config, + ), + rdma_control_init, + sub_cluster: config.fluxonkv_spec.sub_cluster.clone(), + network: None, + }, + p2p_arg: P2pModuleNewArg::new( + config.fluxonkv_spec.p2p_listen_port, + tcp_thread_transport_tuning_from_test_spec_config(&config.test_spec_config), + config.test_spec_config.disable_crossowner_ipc, + config.test_spec_config.iceoryx_external_busy_poll, + ) + .with_iceoryx_owner_client_busy_poll(config.test_spec_config.iceoryx_owner_client_busy_poll) + .with_user_rpc_sync_handler_thread_count( + config.test_spec_config.user_rpc_sync_handler_thread_count, + ), + metric_reporter_arg: MetricReporterNewArg { + test_spec_config: config.test_spec_config.clone(), + }, + external_client_api_arg: ExternalClientApiNewArg { + share_mem_path: config.share_mem_path.clone(), + large_file_paths: config.large_file_paths.clone(), + expected_cluster_name: config.cluster_name.clone(), + expected_protocol_version: build_version.clone(), + enable_side_transfer: config.test_spec_config.enable_side_transfer, + short_circuit_put_payload_path: config.test_spec_config.short_circuit_put_payload_path, + }, + }; + + let framework = Framework::new(format!( + "fluxon_kv.broker:{}:{}", + config.cluster_name, config.instance_key + )); + info!("Initializing broker framework..."); + + init_framework_broker(&framework, init_args) + .await + .map_err(|e| anyhow::anyhow!("Failed to initialize broker framework: {:#}", e))?; + register_broker_service(framework.p2p_view().clone(), 4096); + + let framework = Arc::new(framework); + + if !observability_disabled { + let otlp_cluster_name = config.cluster_name.clone(); + let otlp_member_id = config.instance_key.clone(); + let cm_view = framework.cluster_manager_view().clone(); + let p2p_view = framework.p2p_view().clone(); + let spawner = cm_view.clone(); + let _ = spawner.spawn("wait_master_otlp_log_api_broker", async move { + let outcome = wait_master_observe_broadcast( + &cm_view, + std::time::Duration::from_secs(60), + std::time::Duration::from_secs(10), + ) + .await; + let Some(cfg) = outcome.otlp_log_api() else { + warn!( + "Broker OTLP log exporter disabled: master metadata does not carry otlp_log_api" + ); + return; + }; + + start_greptime_otlp_tracing_exporter_kv( + cm_view, + p2p_view, + Some(cfg), + greptime_tracing_rx, + &otlp_cluster_name, + fluxon_observability::types::FluxonMemberRole::Broker, + &otlp_member_id, + ); + }); + } + + let shutdown_waiter = framework.cluster_manager_view().register_shutdown_waiter(); + let kv_profiles_dir = config + .large_file_paths + .kv_profiles_dir(&config.cluster_name) + .map_err(|e| anyhow::anyhow!("invalid large_file_paths for broker kv profiles: {}", e))?; + profile::spawn_pprof_flamegraph_on_timeout_or_shutdown( + config.pprof_duration_seconds, + kv_profiles_dir, + config.cluster_name.clone(), + profile::PprofRole::Broker, + config.instance_key.clone(), + shutdown_waiter, + ); + + Ok((framework, config)) +} + +pub async fn run_broker( + config_arg: ConfigArg, +) -> Result<(Arc, ClientConfig)> { + run_broker_impl(config_arg, None).await +} + #[cfg(feature = "test_bins")] pub(crate) async fn run_master_with_test_overrides( config_arg: ConfigArg, diff --git a/fluxon_rs/fluxon_kv/src/memholder/lifetime.rs b/fluxon_rs/fluxon_kv/src/memholder/lifetime.rs index ad23b4d..1301a98 100755 --- a/fluxon_rs/fluxon_kv/src/memholder/lifetime.rs +++ b/fluxon_rs/fluxon_kv/src/memholder/lifetime.rs @@ -448,8 +448,8 @@ impl MemholderManagerTrait for MasterOwnerMemMgr { const DELETE_SUBMIT_QUEUE_CAPACITY: usize = 1000; const DELETE_TARGET_QUEUE_CAPACITY: usize = 1000; - const DELETE_MERGE_WINDOW_MILLIS: u64 = 1000; - const DELETE_RETRY_INTERVAL_MILLIS: u64 = 1000; + const DELETE_MERGE_WINDOW_MILLIS: u64 = 10; + const DELETE_RETRY_INTERVAL_MILLIS: u64 = 200; #[inline] fn inner_map(&self) -> &DashMap { @@ -737,8 +737,8 @@ impl MemholderManagerTrait for OwnerExternalMemMgr { const DELETE_SUBMIT_QUEUE_CAPACITY: usize = 1000; const DELETE_TARGET_QUEUE_CAPACITY: usize = 1000; - const DELETE_MERGE_WINDOW_MILLIS: u64 = 1000; - const DELETE_RETRY_INTERVAL_MILLIS: u64 = 1000; + const DELETE_MERGE_WINDOW_MILLIS: u64 = 10; + const DELETE_RETRY_INTERVAL_MILLIS: u64 = 200; #[inline] fn inner_map(&self) -> &DashMap { diff --git a/fluxon_rs/fluxon_kv/src/profile.rs b/fluxon_rs/fluxon_kv/src/profile.rs index c2f40d7..2d04374 100755 --- a/fluxon_rs/fluxon_kv/src/profile.rs +++ b/fluxon_rs/fluxon_kv/src/profile.rs @@ -7,6 +7,7 @@ use tracing::{info, warn}; #[derive(Debug, Clone, Copy)] pub(crate) enum PprofRole { Master, + Broker, Client, } @@ -14,6 +15,7 @@ impl PprofRole { fn as_str(self) -> &'static str { match self { PprofRole::Master => "master", + PprofRole::Broker => "broker", PprofRole::Client => "client", } } diff --git a/fluxon_rs/fluxon_mq/Cargo.toml b/fluxon_rs/fluxon_mq/Cargo.toml index 4f10f44..15f6329 100644 --- a/fluxon_rs/fluxon_mq/Cargo.toml +++ b/fluxon_rs/fluxon_mq/Cargo.toml @@ -17,6 +17,7 @@ parking_lot = { workspace = true } paste = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +bitcode = { workspace = true } etcd-client = { workspace = true } fluxon_util = { path = "../fluxon_util" } fluxon_observability = { path = "../fluxon_observability" } diff --git a/fluxon_rs/fluxon_mq/src/broker.rs b/fluxon_rs/fluxon_mq/src/broker.rs new file mode 100644 index 0000000..89e2e9b --- /dev/null +++ b/fluxon_rs/fluxon_mq/src/broker.rs @@ -0,0 +1,3006 @@ +use std::collections::{HashMap, HashSet, VecDeque}; +use std::env; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use bitcode::{Decode, Encode}; +use fluxon_commu::cluster_manager::ClusterManagerView; +use fluxon_commu::p2p::rpc::{MsgPack, MsgPackSerializePart, RPCCaller, RPCHandler, RPCReq}; +use fluxon_commu::p2p::P2pModuleView; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot}; + +use crate::keys::{self, MqCategory}; +use crate::manager::PRODUCE_OFFSET_BEGIN; + +const BROKER_RPC_REQ_MSG_ID: u32 = 8101; +const BROKER_RPC_RESP_MSG_ID: u32 = 8102; +pub const FLUXON_MQ_COMPONENT_METADATA_KEY: &str = "fluxon_mq_component"; +pub const FLUXON_MQ_COMPONENT_BROKER_METADATA_VALUE: &str = "broker"; +const BROKER_PAYLOAD_BYTES_CAP_ENV: &str = "FLUXON_MQ_BROKER_PAYLOAD_BYTES_CAP"; +const BROKER_PAYLOAD_BYTES_CAP_PERCENT_ENV: &str = "FLUXON_MQ_BROKER_PAYLOAD_BYTES_CAP_PERCENT"; +const BROKER_CLEANUP_RELEASE_DELAY_MS_ENV: &str = "FLUXON_MQ_BROKER_CLEANUP_RELEASE_DELAY_MS"; +const OWNER_POOL_DRAM_BYTES_ENV: &str = "FLUXON_OWNER_POOL_DRAM_BYTES"; +const DEFAULT_BROKER_PAYLOAD_BYTES_CAP: u64 = 64 * 1024 * 1024 * 1024; +const DEFAULT_BROKER_PAYLOAD_BYTES_CAP_PERCENT: u64 = 60; +const DEFAULT_BROKER_CLEANUP_RELEASE_DELAY_MS: u64 = 0; +const BROKER_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(15); + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)] +pub struct BrokerChannelConfig { + pub channel_id: i64, + pub capacity: i64, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, Encode, Decode)] +pub struct BrokerReserveRequest { + pub channel_id: i64, + pub producer_id: String, + pub category: MqCategory, + pub payload_bytes: u64, + pub now_ms: i64, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, Encode, Decode)] +pub struct BrokerFetchRequest { + pub channel_id: i64, + pub consumer_id: String, + pub now_ms: i64, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)] +pub struct BrokerEnvelope { + pub channel_id: i64, + pub producer_id: String, + pub msg_id: i64, + pub reservation_id: u64, + pub payload_key: String, + pub payload_bytes: u64, + pub reserved_at_ms: i64, + pub published_at_ms: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)] +pub struct BrokerReservation { + pub envelope: BrokerEnvelope, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)] +pub struct BrokerFetchedMessage { + pub envelope: BrokerEnvelope, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)] +pub struct BrokerFetchBatch { + pub messages: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)] +pub struct BrokerCommitOutcome { + pub first_commit: bool, + pub cleanup: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)] +pub struct BrokerCommitBatchOutcome { + pub first_commit_count: usize, + pub cleanup: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum BrokerLogRecord { + UpsertChannel { + config: BrokerChannelConfig, + }, + DeleteChannel { + channel_id: i64, + }, + Reserve { + channel_id: i64, + producer_id: String, + msg_id: i64, + reservation_id: u64, + payload_key: String, + payload_bytes: u64, + reserved_at_ms: i64, + }, + Publish { + channel_id: i64, + reservation_id: u64, + published_at_ms: i64, + }, + Abort { + channel_id: i64, + reservation_id: u64, + }, + Fetch { + channel_id: i64, + reservation_id: u64, + consumer_id: String, + fetched_at_ms: i64, + }, + Commit { + channel_id: i64, + reservation_id: u64, + committed_at_ms: i64, + }, + CleanupAck { + channel_id: i64, + reservation_id: u64, + }, + RequeueInflight { + channel_id: i64, + reservation_id: u64, + }, +} + +impl BrokerLogRecord { + fn reservation_id(&self) -> Option { + match self { + BrokerLogRecord::Reserve { reservation_id, .. } + | BrokerLogRecord::Publish { reservation_id, .. } + | BrokerLogRecord::Abort { reservation_id, .. } + | BrokerLogRecord::Fetch { reservation_id, .. } + | BrokerLogRecord::Commit { reservation_id, .. } + | BrokerLogRecord::CleanupAck { reservation_id, .. } + | BrokerLogRecord::RequeueInflight { reservation_id, .. } => Some(*reservation_id), + BrokerLogRecord::UpsertChannel { .. } | BrokerLogRecord::DeleteChannel { .. } => None, + } + } +} + +#[derive(Debug, Error, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode)] +pub enum BrokerError { + #[error("broker channel not found: channel_id={0}")] + ChannelNotFound(i64), + + #[error( + "broker channel capacity must be positive: channel_id={channel_id} capacity={capacity}" + )] + InvalidCapacity { channel_id: i64, capacity: i64 }, + + #[error( + "broker channel is full: channel_id={channel_id} capacity={capacity} used_slots={used_slots}" + )] + ChannelFull { + channel_id: i64, + capacity: i64, + used_slots: i64, + }, + + #[error( + "broker payload byte budget is full: requested_bytes={requested_bytes} capacity_bytes={capacity_bytes} used_bytes={used_bytes}" + )] + PayloadBytesFull { + requested_bytes: u64, + capacity_bytes: u64, + used_bytes: u64, + }, + + #[error( + "broker payload is larger than byte budget: requested_bytes={requested_bytes} capacity_bytes={capacity_bytes}" + )] + PayloadTooLarge { + requested_bytes: u64, + capacity_bytes: u64, + }, + + #[error( + "broker reservation not found: channel_id={channel_id} reservation_id={reservation_id}" + )] + ReservationNotFound { + channel_id: i64, + reservation_id: u64, + }, + + #[error( + "broker delivery not in-flight: channel_id={channel_id} reservation_id={reservation_id}" + )] + DeliveryNotFound { + channel_id: i64, + reservation_id: u64, + }, + + #[error("invalid broker log record: {0}")] + InvalidRecord(String), + + #[error("broker master unavailable: {0}")] + BrokerUnavailable(String), + + #[error("broker rpc error: {0}")] + Rpc(String), + + #[error("broker actor closed")] + ActorClosed, +} + +#[derive(Debug, Default)] +pub struct LocalBroker { + state: BrokerState, + log: Vec, +} + +#[derive(Debug)] +struct BrokerState { + channels: HashMap, + payload_byte_capacity: u64, + used_payload_bytes: u64, +} + +impl Default for BrokerState { + fn default() -> Self { + Self { + channels: HashMap::new(), + payload_byte_capacity: default_payload_byte_capacity(), + used_payload_bytes: 0, + } + } +} + +#[derive(Debug)] +struct ChannelState { + config: BrokerChannelConfig, + next_reservation_id: u64, + next_msg_by_producer: HashMap, + pending: HashMap, + visible: VecDeque, + inflight: HashMap, + inflight_order: VecDeque, + cleanup: VecDeque, + cleanup_inflight: HashMap, + committed: HashSet, + used_slots: i64, + reserve_waiters: VecDeque, + fetch_waiters: VecDeque, +} + +impl ChannelState { + fn new(config: BrokerChannelConfig) -> Self { + Self { + config, + next_reservation_id: 1, + next_msg_by_producer: HashMap::new(), + pending: HashMap::new(), + visible: VecDeque::new(), + inflight: HashMap::new(), + inflight_order: VecDeque::new(), + cleanup: VecDeque::new(), + cleanup_inflight: HashMap::new(), + committed: HashSet::new(), + used_slots: 0, + reserve_waiters: VecDeque::new(), + fetch_waiters: VecDeque::new(), + } + } +} + +#[derive(Debug)] +struct ReserveWaiter { + req: BrokerReserveRequest, + reply: oneshot::Sender>, +} + +#[derive(Debug)] +struct FetchWaiter { + req: BrokerFetchRequest, + reply: oneshot::Sender, BrokerError>>, +} + +impl LocalBroker { + pub fn new() -> Self { + Self::default() + } + + #[cfg(test)] + fn with_payload_byte_capacity(payload_byte_capacity: u64) -> Self { + Self { + state: BrokerState { + channels: HashMap::new(), + payload_byte_capacity: payload_byte_capacity.max(1), + used_payload_bytes: 0, + }, + log: Vec::new(), + } + } + + pub fn from_log(records: &[BrokerLogRecord]) -> Result { + let mut broker = Self::new(); + for record in records { + broker.apply_log_record(record.clone())?; + broker.log.push(record.clone()); + } + Ok(broker) + } + + pub fn log_records(&self) -> &[BrokerLogRecord] { + &self.log + } + + pub fn upsert_channel(&mut self, config: BrokerChannelConfig) -> Result<(), BrokerError> { + let record = self.upsert_channel_record(config)?; + self.apply_and_record(record) + } + + pub fn delete_channel(&mut self, channel_id: i64) -> Result, BrokerError> { + let payload_keys = self.delete_channel_state(channel_id); + self.log.push(BrokerLogRecord::DeleteChannel { channel_id }); + Ok(payload_keys) + } + + pub fn reserve(&mut self, req: BrokerReserveRequest) -> Result { + let channel_id = req.channel_id; + let record = self.reserve_record(req)?; + let reservation_id = record.reservation_id().ok_or_else(|| { + BrokerError::InvalidRecord("reserve record missing reservation_id".to_string()) + })?; + self.apply_and_record(record)?; + let envelope = self + .channel(channel_id)? + .pending + .get(&reservation_id) + .cloned() + .ok_or(BrokerError::ReservationNotFound { + channel_id, + reservation_id, + })?; + Ok(BrokerReservation { envelope }) + } + + pub fn publish( + &mut self, + channel_id: i64, + reservation_id: u64, + now_ms: i64, + ) -> Result { + let record = self.publish_record(channel_id, reservation_id, now_ms)?; + self.apply_and_record(record)?; + self.channel(channel_id)? + .visible + .iter() + .find(|env| env.reservation_id == reservation_id) + .cloned() + .ok_or(BrokerError::ReservationNotFound { + channel_id, + reservation_id, + }) + } + + pub fn abort(&mut self, channel_id: i64, reservation_id: u64) -> Result<(), BrokerError> { + let record = self.abort_record(channel_id, reservation_id)?; + self.apply_and_record(record) + } + + pub fn fetch_next( + &mut self, + req: BrokerFetchRequest, + ) -> Result, BrokerError> { + let channel_id = req.channel_id; + let Some(record) = self.fetch_record(req)? else { + return Ok(None); + }; + let reservation_id = record.reservation_id().ok_or_else(|| { + BrokerError::InvalidRecord("fetch record missing reservation_id".to_string()) + })?; + self.apply_and_record(record)?; + let envelope = self + .channel(channel_id)? + .inflight + .get(&reservation_id) + .cloned() + .ok_or(BrokerError::DeliveryNotFound { + channel_id, + reservation_id, + })?; + Ok(Some(BrokerFetchedMessage { envelope })) + } + + pub fn fetch_batch_available( + &mut self, + req: BrokerFetchRequest, + max_items: usize, + ) -> Result { + let mut messages = Vec::new(); + for _ in 0..max_items { + let Some(message) = self.fetch_next(req.clone())? else { + break; + }; + messages.push(message); + } + Ok(BrokerFetchBatch { messages }) + } + + pub fn commit( + &mut self, + channel_id: i64, + reservation_id: u64, + now_ms: i64, + ) -> Result { + let Some(record) = self.commit_record(channel_id, reservation_id, now_ms)? else { + return Ok(BrokerCommitOutcome { + first_commit: false, + cleanup: None, + }); + }; + self.apply_and_record(record)?; + let cleanup = self.channel(channel_id)?.cleanup.back().cloned(); + Ok(BrokerCommitOutcome { + first_commit: true, + cleanup, + }) + } + + pub fn commit_batch( + &mut self, + channel_id: i64, + reservation_ids: Vec, + now_ms: i64, + ) -> Result { + let mut cleanup = Vec::new(); + let mut first_commit_count = 0usize; + for reservation_id in reservation_ids { + let outcome = self.commit(channel_id, reservation_id, now_ms)?; + if outcome.first_commit { + first_commit_count += 1; + if let Some(envelope) = outcome.cleanup { + cleanup.push(envelope); + } + } + } + Ok(BrokerCommitBatchOutcome { + first_commit_count, + cleanup, + }) + } + + pub fn requeue_inflight( + &mut self, + channel_id: i64, + reservation_id: u64, + ) -> Result<(), BrokerError> { + let record = self.requeue_inflight_record(channel_id, reservation_id)?; + self.apply_and_record(record) + } + + pub fn requeue_all_inflight(&mut self, channel_id: i64) -> Result<(), BrokerError> { + let reservation_ids: Vec = self + .channel(channel_id)? + .inflight_order + .iter() + .rev() + .copied() + .collect(); + for reservation_id in reservation_ids { + self.requeue_inflight(channel_id, reservation_id)?; + } + Ok(()) + } + + pub fn take_cleanup_batch( + &mut self, + channel_id: i64, + max_items: usize, + ) -> Result, BrokerError> { + let channel = self.channel_mut(channel_id)?; + let mut batch = Vec::new(); + for _ in 0..max_items { + let Some(envelope) = channel.cleanup.pop_front() else { + break; + }; + channel + .cleanup_inflight + .insert(envelope.reservation_id, envelope.clone()); + batch.push(envelope); + } + Ok(batch) + } + + pub fn cleanup_ack(&mut self, channel_id: i64, reservation_id: u64) -> Result<(), BrokerError> { + let record = self.cleanup_ack_record(channel_id, reservation_id)?; + let _ = self.apply_cleanup_ack_and_record(record, true)?; + Ok(()) + } + + pub fn cleanup_ack_for_delayed_release( + &mut self, + channel_id: i64, + reservation_id: u64, + ) -> Result { + let record = self.cleanup_ack_record(channel_id, reservation_id)?; + self.apply_cleanup_ack_and_record(record, false) + } + + pub fn cleanup_nack( + &mut self, + channel_id: i64, + reservation_id: u64, + ) -> Result<(), BrokerError> { + let channel = self.channel_mut(channel_id)?; + if let Some(envelope) = channel.cleanup_inflight.remove(&reservation_id) { + channel.cleanup.push_front(envelope); + } + Ok(()) + } + + pub fn apply_log_record(&mut self, record: BrokerLogRecord) -> Result<(), BrokerError> { + match record { + BrokerLogRecord::UpsertChannel { config } => { + validate_capacity(&config)?; + match self.state.channels.get_mut(&config.channel_id) { + Some(channel) => { + if config.capacity < channel.used_slots { + return Err(BrokerError::InvalidRecord(format!( + "channel_id={} capacity={} below used_slots={}", + config.channel_id, config.capacity, channel.used_slots + ))); + } + channel.config = config; + } + None => { + self.state + .channels + .insert(config.channel_id, ChannelState::new(config)); + } + } + } + BrokerLogRecord::DeleteChannel { channel_id } => { + let _ = self.delete_channel_state(channel_id); + } + BrokerLogRecord::Reserve { + channel_id, + producer_id, + msg_id, + reservation_id, + payload_key, + payload_bytes, + reserved_at_ms, + } => { + let channel = self.channel(channel_id)?; + if channel.pending.contains_key(&reservation_id) + || channel + .visible + .iter() + .any(|env| env.reservation_id == reservation_id) + || channel.inflight.contains_key(&reservation_id) + || channel.committed.contains(&reservation_id) + { + return Err(BrokerError::InvalidRecord(format!( + "duplicate reservation_id={} for channel_id={}", + reservation_id, channel_id + ))); + } + if payload_bytes > self.state.payload_byte_capacity { + return Err(BrokerError::PayloadTooLarge { + requested_bytes: payload_bytes, + capacity_bytes: self.state.payload_byte_capacity, + }); + } + if self.state.used_payload_bytes.saturating_add(payload_bytes) + > self.state.payload_byte_capacity + { + return Err(BrokerError::PayloadBytesFull { + requested_bytes: payload_bytes, + capacity_bytes: self.state.payload_byte_capacity, + used_bytes: self.state.used_payload_bytes, + }); + } + + let channel = self.channel_mut(channel_id)?; + channel.next_reservation_id = channel.next_reservation_id.max(reservation_id + 1); + let next_msg = channel + .next_msg_by_producer + .entry(producer_id.clone()) + .or_insert(PRODUCE_OFFSET_BEGIN + 1); + *next_msg = (*next_msg).max(msg_id + 1); + + let envelope = BrokerEnvelope { + channel_id, + producer_id, + msg_id, + reservation_id, + payload_key, + payload_bytes, + reserved_at_ms, + published_at_ms: None, + }; + channel.pending.insert(reservation_id, envelope); + channel.used_slots += 1; + self.state.used_payload_bytes += payload_bytes; + } + BrokerLogRecord::Publish { + channel_id, + reservation_id, + published_at_ms, + } => { + let channel = self.channel_mut(channel_id)?; + let mut envelope = channel.pending.remove(&reservation_id).ok_or( + BrokerError::ReservationNotFound { + channel_id, + reservation_id, + }, + )?; + envelope.published_at_ms = Some(published_at_ms); + channel.visible.push_back(envelope); + } + BrokerLogRecord::Abort { + channel_id, + reservation_id, + } => { + let channel = self.channel_mut(channel_id)?; + let envelope = channel.pending.remove(&reservation_id).ok_or( + BrokerError::ReservationNotFound { + channel_id, + reservation_id, + }, + )?; + channel.used_slots -= 1; + self.release_payload_bytes(envelope.payload_bytes); + } + BrokerLogRecord::Fetch { + channel_id, + reservation_id, + consumer_id: _, + fetched_at_ms: _, + } => { + let channel = self.channel_mut(channel_id)?; + let Some(front) = channel.visible.front() else { + return Err(BrokerError::ReservationNotFound { + channel_id, + reservation_id, + }); + }; + if front.reservation_id != reservation_id { + return Err(BrokerError::InvalidRecord(format!( + "fetch order mismatch for channel_id={}: expected_reservation_id={} actual_reservation_id={}", + channel_id, front.reservation_id, reservation_id + ))); + } + let envelope = channel + .visible + .pop_front() + .expect("visible front checked above"); + channel.inflight.insert(reservation_id, envelope); + channel.inflight_order.push_back(reservation_id); + } + BrokerLogRecord::Commit { + channel_id, + reservation_id, + committed_at_ms: _, + } => { + let channel = self.channel_mut(channel_id)?; + if channel.committed.contains(&reservation_id) { + return Ok(()); + } + let envelope = channel.inflight.remove(&reservation_id).ok_or( + BrokerError::DeliveryNotFound { + channel_id, + reservation_id, + }, + )?; + remove_from_deque(&mut channel.inflight_order, reservation_id); + channel.committed.insert(reservation_id); + channel.cleanup.push_back(envelope); + channel.used_slots -= 1; + } + BrokerLogRecord::CleanupAck { + channel_id, + reservation_id, + } => { + self.apply_cleanup_ack(channel_id, reservation_id, true)?; + } + BrokerLogRecord::RequeueInflight { + channel_id, + reservation_id, + } => { + let channel = self.channel_mut(channel_id)?; + let envelope = channel.inflight.remove(&reservation_id).ok_or( + BrokerError::DeliveryNotFound { + channel_id, + reservation_id, + }, + )?; + remove_from_deque(&mut channel.inflight_order, reservation_id); + channel.visible.push_front(envelope); + } + } + Ok(()) + } + + fn release_payload_bytes(&mut self, payload_bytes: u64) { + self.state.used_payload_bytes = self.state.used_payload_bytes.saturating_sub(payload_bytes); + } + + fn delete_channel_state(&mut self, channel_id: i64) -> Vec { + let Some(mut channel) = self.state.channels.remove(&channel_id) else { + return Vec::new(); + }; + + let mut payload_bytes = 0u64; + let mut payload_keys = Vec::new(); + collect_deleted_payloads( + channel.pending.drain().map(|(_, envelope)| envelope), + &mut payload_keys, + &mut payload_bytes, + ); + collect_deleted_payloads( + channel.visible.drain(..), + &mut payload_keys, + &mut payload_bytes, + ); + collect_deleted_payloads( + channel.inflight.drain().map(|(_, envelope)| envelope), + &mut payload_keys, + &mut payload_bytes, + ); + collect_deleted_payloads( + channel.cleanup.drain(..), + &mut payload_keys, + &mut payload_bytes, + ); + collect_deleted_payloads( + channel + .cleanup_inflight + .drain() + .map(|(_, envelope)| envelope), + &mut payload_keys, + &mut payload_bytes, + ); + + while let Some(waiter) = channel.reserve_waiters.pop_front() { + let _ = waiter + .reply + .send(Err(BrokerError::ChannelNotFound(channel_id))); + } + while let Some(waiter) = channel.fetch_waiters.pop_front() { + let _ = waiter + .reply + .send(Err(BrokerError::ChannelNotFound(channel_id))); + } + + self.release_payload_bytes(payload_bytes); + payload_keys + } + + fn apply_cleanup_ack( + &mut self, + channel_id: i64, + reservation_id: u64, + release_payload_now: bool, + ) -> Result { + let channel = self.channel_mut(channel_id)?; + let envelope = if let Some(envelope) = channel.cleanup_inflight.remove(&reservation_id) { + envelope + } else if let Some(pos) = channel + .cleanup + .iter() + .position(|env| env.reservation_id == reservation_id) + { + channel + .cleanup + .remove(pos) + .expect("cleanup envelope position checked above") + } else { + return Err(BrokerError::ReservationNotFound { + channel_id, + reservation_id, + }); + }; + channel.committed.remove(&reservation_id); + let payload_bytes = envelope.payload_bytes; + if release_payload_now { + self.release_payload_bytes(payload_bytes); + } + Ok(payload_bytes) + } + + fn apply_and_record(&mut self, record: BrokerLogRecord) -> Result<(), BrokerError> { + self.apply_log_record(record.clone())?; + self.log.push(record); + Ok(()) + } + + fn apply_cleanup_ack_and_record( + &mut self, + record: BrokerLogRecord, + release_payload_now: bool, + ) -> Result { + let BrokerLogRecord::CleanupAck { + channel_id, + reservation_id, + } = record.clone() + else { + return Err(BrokerError::InvalidRecord( + "apply_cleanup_ack_and_record requires CleanupAck".to_string(), + )); + }; + let payload_bytes = + self.apply_cleanup_ack(channel_id, reservation_id, release_payload_now)?; + self.log.push(record); + Ok(payload_bytes) + } + + fn upsert_channel_record( + &self, + config: BrokerChannelConfig, + ) -> Result { + validate_capacity(&config)?; + if let Some(channel) = self.state.channels.get(&config.channel_id) { + if config.capacity < channel.used_slots { + return Err(BrokerError::InvalidRecord(format!( + "channel_id={} capacity={} below used_slots={}", + config.channel_id, config.capacity, channel.used_slots + ))); + } + } + Ok(BrokerLogRecord::UpsertChannel { config }) + } + + fn reserve_record(&self, req: BrokerReserveRequest) -> Result { + let channel = self.channel(req.channel_id)?; + if broker_category_enforces_capacity(req.category) + && channel.used_slots >= channel.config.capacity + { + return Err(BrokerError::ChannelFull { + channel_id: req.channel_id, + capacity: channel.config.capacity, + used_slots: channel.used_slots, + }); + } + + let msg_id = channel + .next_msg_by_producer + .get(&req.producer_id) + .copied() + .unwrap_or(PRODUCE_OFFSET_BEGIN + 1); + let reservation_id = channel.next_reservation_id; + let payload_key = keys::backend_message_key_with_category( + req.channel_id, + &req.producer_id, + msg_id, + &req.category, + ); + let payload_bytes = req.payload_bytes.max(1); + if payload_bytes > self.state.payload_byte_capacity { + return Err(BrokerError::PayloadTooLarge { + requested_bytes: payload_bytes, + capacity_bytes: self.state.payload_byte_capacity, + }); + } + if self.state.used_payload_bytes.saturating_add(payload_bytes) + > self.state.payload_byte_capacity + { + return Err(BrokerError::PayloadBytesFull { + requested_bytes: payload_bytes, + capacity_bytes: self.state.payload_byte_capacity, + used_bytes: self.state.used_payload_bytes, + }); + } + Ok(BrokerLogRecord::Reserve { + channel_id: req.channel_id, + producer_id: req.producer_id, + msg_id, + reservation_id, + payload_key, + payload_bytes, + reserved_at_ms: req.now_ms, + }) + } + + fn publish_record( + &self, + channel_id: i64, + reservation_id: u64, + now_ms: i64, + ) -> Result { + let channel = self.channel(channel_id)?; + if !channel.pending.contains_key(&reservation_id) { + return Err(BrokerError::ReservationNotFound { + channel_id, + reservation_id, + }); + } + Ok(BrokerLogRecord::Publish { + channel_id, + reservation_id, + published_at_ms: now_ms, + }) + } + + fn abort_record( + &self, + channel_id: i64, + reservation_id: u64, + ) -> Result { + let channel = self.channel(channel_id)?; + if !channel.pending.contains_key(&reservation_id) { + return Err(BrokerError::ReservationNotFound { + channel_id, + reservation_id, + }); + } + Ok(BrokerLogRecord::Abort { + channel_id, + reservation_id, + }) + } + + fn fetch_record( + &self, + req: BrokerFetchRequest, + ) -> Result, BrokerError> { + let reservation_id = match self.channel(req.channel_id)?.visible.front() { + Some(env) => env.reservation_id, + None => return Ok(None), + }; + Ok(Some(BrokerLogRecord::Fetch { + channel_id: req.channel_id, + reservation_id, + consumer_id: req.consumer_id, + fetched_at_ms: req.now_ms, + })) + } + + fn commit_record( + &self, + channel_id: i64, + reservation_id: u64, + now_ms: i64, + ) -> Result, BrokerError> { + let channel = self.channel(channel_id)?; + if channel.committed.contains(&reservation_id) { + return Ok(None); + } + if !channel.inflight.contains_key(&reservation_id) { + return Err(BrokerError::DeliveryNotFound { + channel_id, + reservation_id, + }); + } + Ok(Some(BrokerLogRecord::Commit { + channel_id, + reservation_id, + committed_at_ms: now_ms, + })) + } + + fn cleanup_ack_record( + &self, + channel_id: i64, + reservation_id: u64, + ) -> Result { + let channel = self.channel(channel_id)?; + let exists = channel.cleanup_inflight.contains_key(&reservation_id) + || channel + .cleanup + .iter() + .any(|env| env.reservation_id == reservation_id); + if !exists { + return Err(BrokerError::ReservationNotFound { + channel_id, + reservation_id, + }); + } + Ok(BrokerLogRecord::CleanupAck { + channel_id, + reservation_id, + }) + } + + fn requeue_inflight_record( + &self, + channel_id: i64, + reservation_id: u64, + ) -> Result { + let channel = self.channel(channel_id)?; + if !channel.inflight.contains_key(&reservation_id) { + return Err(BrokerError::DeliveryNotFound { + channel_id, + reservation_id, + }); + } + Ok(BrokerLogRecord::RequeueInflight { + channel_id, + reservation_id, + }) + } + + fn channel(&self, channel_id: i64) -> Result<&ChannelState, BrokerError> { + self.state + .channels + .get(&channel_id) + .ok_or(BrokerError::ChannelNotFound(channel_id)) + } + + fn channel_mut(&mut self, channel_id: i64) -> Result<&mut ChannelState, BrokerError> { + self.state + .channels + .get_mut(&channel_id) + .ok_or(BrokerError::ChannelNotFound(channel_id)) + } +} + +fn drain_reserve_waiters(broker: &mut LocalBroker) { + loop { + let channel_ids: Vec = broker.state.channels.keys().copied().collect(); + let mut progressed = false; + for channel_id in channel_ids { + progressed |= drain_reserve_waiters_for_channel(broker, channel_id); + } + if !progressed { + return; + } + } +} + +fn drain_reserve_waiters_for_channel(broker: &mut LocalBroker, channel_id: i64) -> bool { + let mut progressed = false; + loop { + let waiter = match broker.channel_mut(channel_id) { + Ok(channel) => channel.reserve_waiters.pop_front(), + Err(_) => return progressed, + }; + let Some(waiter) = waiter else { + return progressed; + }; + + match broker.reserve(waiter.req.clone()) { + Ok(reservation) => { + if let Err(Ok(reservation)) = waiter.reply.send(Ok(reservation)) { + let _ = broker.abort(channel_id, reservation.envelope.reservation_id); + } + progressed = true; + } + Err(BrokerError::ChannelFull { .. }) | Err(BrokerError::PayloadBytesFull { .. }) => { + if let Ok(channel) = broker.channel_mut(channel_id) { + channel.reserve_waiters.push_front(waiter); + } + return progressed; + } + Err(err) => { + let _ = waiter.reply.send(Err(err)); + progressed = true; + } + } + } +} + +fn drain_fetch_waiters_for_channel(broker: &mut LocalBroker, channel_id: i64) { + loop { + let waiter = match broker.channel_mut(channel_id) { + Ok(channel) => channel.fetch_waiters.pop_front(), + Err(_) => return, + }; + let Some(waiter) = waiter else { + return; + }; + + match broker.fetch_next(waiter.req.clone()) { + Ok(Some(fetched)) => { + if let Err(Ok(Some(fetched))) = waiter.reply.send(Ok(Some(fetched))) { + let _ = broker.requeue_inflight( + fetched.envelope.channel_id, + fetched.envelope.reservation_id, + ); + } + } + Ok(None) => { + if let Ok(channel) = broker.channel_mut(channel_id) { + channel.fetch_waiters.push_front(waiter); + } + return; + } + Err(err) => { + let _ = waiter.reply.send(Err(err)); + } + } + } +} + +fn fail_all_waiters_with_actor_closed(broker: &mut LocalBroker) { + for channel in broker.state.channels.values_mut() { + while let Some(waiter) = channel.reserve_waiters.pop_front() { + let _ = waiter.reply.send(Err(BrokerError::ActorClosed)); + } + while let Some(waiter) = channel.fetch_waiters.pop_front() { + let _ = waiter.reply.send(Err(BrokerError::ActorClosed)); + } + } +} + +fn collect_deleted_payloads( + envelopes: impl Iterator, + payload_keys: &mut Vec, + payload_bytes: &mut u64, +) { + for envelope in envelopes { + *payload_bytes = payload_bytes.saturating_add(envelope.payload_bytes); + payload_keys.push(envelope.payload_key); + } +} + +enum BrokerCommand { + UpsertChannel { + config: BrokerChannelConfig, + reply: oneshot::Sender>, + }, + DeleteChannel { + channel_id: i64, + reply: oneshot::Sender, BrokerError>>, + }, + Reserve { + req: BrokerReserveRequest, + reply: oneshot::Sender>, + }, + Publish { + channel_id: i64, + reservation_id: u64, + now_ms: i64, + reply: oneshot::Sender>, + }, + Abort { + channel_id: i64, + reservation_id: u64, + reply: oneshot::Sender>, + }, + FetchNext { + req: BrokerFetchRequest, + reply: oneshot::Sender, BrokerError>>, + }, + FetchBatchAvailable { + req: BrokerFetchRequest, + max_items: usize, + reply: oneshot::Sender>, + }, + Commit { + channel_id: i64, + reservation_id: u64, + now_ms: i64, + reply: oneshot::Sender>, + }, + CommitBatch { + channel_id: i64, + reservation_ids: Vec, + now_ms: i64, + reply: oneshot::Sender>, + }, + RequeueInflight { + channel_id: i64, + reservation_id: u64, + reply: oneshot::Sender>, + }, + RequeueAllInflight { + channel_id: i64, + reply: oneshot::Sender>, + }, + TakeCleanupBatch { + channel_id: i64, + max_items: usize, + reply: oneshot::Sender, BrokerError>>, + }, + CleanupAck { + channel_id: i64, + reservation_id: u64, + reply: oneshot::Sender>, + }, + CleanupNack { + channel_id: i64, + reservation_id: u64, + reply: oneshot::Sender>, + }, + ReleasePayloadBytes { + payload_bytes: u64, + }, + Shutdown { + reply: oneshot::Sender>, + }, +} + +#[derive(Clone, Debug)] +struct LocalBrokerHandle { + tx: mpsc::Sender, +} + +impl LocalBrokerHandle { + fn spawn_actor(broker: LocalBroker, queue_capacity: usize) -> Self { + Self::spawn_actor_with_cleanup_release_delay( + broker, + queue_capacity, + default_cleanup_release_delay(), + ) + } + + fn spawn_actor_with_cleanup_release_delay( + broker: LocalBroker, + queue_capacity: usize, + cleanup_release_delay: Duration, + ) -> Self { + let (tx, mut rx) = mpsc::channel(queue_capacity.max(1)); + let tx_for_actor = tx.clone(); + tokio::spawn(async move { + let mut broker = broker; + while let Some(cmd) = rx.recv().await { + match cmd { + BrokerCommand::UpsertChannel { config, reply } => { + let channel_id = config.channel_id; + let result = broker.upsert_channel(config); + if result.is_ok() { + let _ = channel_id; + drain_reserve_waiters(&mut broker); + } + let _ = reply.send(result); + } + BrokerCommand::DeleteChannel { channel_id, reply } => { + let result = broker.delete_channel(channel_id); + if result.is_ok() { + drain_reserve_waiters(&mut broker); + } + let _ = reply.send(result); + } + BrokerCommand::Reserve { req, reply } => { + let req_clone = req.clone(); + match broker.reserve(req_clone) { + Ok(reservation) => { + let _ = reply.send(Ok(reservation)); + } + Err(err) => { + let _ = reply.send(Err(err)); + } + } + } + BrokerCommand::Publish { + channel_id, + reservation_id, + now_ms, + reply, + } => { + let result = broker.publish(channel_id, reservation_id, now_ms); + if result.is_ok() { + drain_fetch_waiters_for_channel(&mut broker, channel_id); + } + let _ = reply.send(result); + } + BrokerCommand::Abort { + channel_id, + reservation_id, + reply, + } => { + let result = broker.abort(channel_id, reservation_id); + if result.is_ok() { + drain_reserve_waiters(&mut broker); + } + let _ = reply.send(result); + } + BrokerCommand::FetchNext { req, reply } => { + let req_clone = req.clone(); + match broker.fetch_next(req_clone) { + Ok(Some(message)) => { + let _ = reply.send(Ok(Some(message))); + } + Ok(None) => match broker.channel_mut(req.channel_id) { + Ok(channel) => { + channel.fetch_waiters.push_back(FetchWaiter { req, reply }) + } + Err(err) => { + let _ = reply.send(Err(err)); + } + }, + Err(err) => { + let _ = reply.send(Err(err)); + } + } + } + BrokerCommand::FetchBatchAvailable { + req, + max_items, + reply, + } => { + let _ = reply.send(broker.fetch_batch_available(req, max_items)); + } + BrokerCommand::Commit { + channel_id, + reservation_id, + now_ms, + reply, + } => { + let result = broker.commit(channel_id, reservation_id, now_ms); + if result.is_ok() { + drain_reserve_waiters(&mut broker); + } + let _ = reply.send(result); + } + BrokerCommand::CommitBatch { + channel_id, + reservation_ids, + now_ms, + reply, + } => { + let result = broker.commit_batch(channel_id, reservation_ids, now_ms); + if result.is_ok() { + drain_reserve_waiters(&mut broker); + } + let _ = reply.send(result); + } + BrokerCommand::RequeueInflight { + channel_id, + reservation_id, + reply, + } => { + let result = broker.requeue_inflight(channel_id, reservation_id); + if result.is_ok() { + drain_fetch_waiters_for_channel(&mut broker, channel_id); + } + let _ = reply.send(result); + } + BrokerCommand::RequeueAllInflight { channel_id, reply } => { + let result = broker.requeue_all_inflight(channel_id); + if result.is_ok() { + drain_fetch_waiters_for_channel(&mut broker, channel_id); + } + let _ = reply.send(result); + } + BrokerCommand::TakeCleanupBatch { + channel_id, + max_items, + reply, + } => { + let _ = reply.send(broker.take_cleanup_batch(channel_id, max_items)); + } + BrokerCommand::CleanupAck { + channel_id, + reservation_id, + reply, + } => { + let result = + broker.cleanup_ack_for_delayed_release(channel_id, reservation_id); + match result { + Ok(payload_bytes) if cleanup_release_delay.is_zero() => { + broker.release_payload_bytes(payload_bytes); + drain_reserve_waiters(&mut broker); + let _ = reply.send(Ok(())); + } + Ok(payload_bytes) => { + let tx_release = tx_for_actor.clone(); + tokio::spawn(async move { + tokio::time::sleep(cleanup_release_delay).await; + let _ = tx_release + .send(BrokerCommand::ReleasePayloadBytes { payload_bytes }) + .await; + }); + let _ = reply.send(Ok(())); + } + Err(err) => { + let _ = reply.send(Err(err)); + } + } + } + BrokerCommand::ReleasePayloadBytes { payload_bytes } => { + broker.release_payload_bytes(payload_bytes); + if payload_bytes > 0 { + drain_reserve_waiters(&mut broker); + } + } + BrokerCommand::CleanupNack { + channel_id, + reservation_id, + reply, + } => { + let _ = reply.send(broker.cleanup_nack(channel_id, reservation_id)); + } + BrokerCommand::Shutdown { reply } => { + fail_all_waiters_with_actor_closed(&mut broker); + let _ = reply.send(Ok(())); + break; + } + } + } + }); + Self { tx } + } + + async fn upsert_channel(&self, config: BrokerChannelConfig) -> Result<(), BrokerError> { + self.request(|reply| BrokerCommand::UpsertChannel { config, reply }) + .await + } + + async fn delete_channel(&self, channel_id: i64) -> Result, BrokerError> { + self.request(|reply| BrokerCommand::DeleteChannel { channel_id, reply }) + .await + } + + async fn reserve(&self, req: BrokerReserveRequest) -> Result { + self.request(|reply| BrokerCommand::Reserve { req, reply }) + .await + } + + async fn publish( + &self, + channel_id: i64, + reservation_id: u64, + now_ms: i64, + ) -> Result { + self.request(|reply| BrokerCommand::Publish { + channel_id, + reservation_id, + now_ms, + reply, + }) + .await + } + + async fn abort(&self, channel_id: i64, reservation_id: u64) -> Result<(), BrokerError> { + self.request(|reply| BrokerCommand::Abort { + channel_id, + reservation_id, + reply, + }) + .await + } + + async fn fetch_next( + &self, + req: BrokerFetchRequest, + ) -> Result, BrokerError> { + self.request(|reply| BrokerCommand::FetchNext { req, reply }) + .await + } + + async fn fetch_batch_available( + &self, + req: BrokerFetchRequest, + max_items: usize, + ) -> Result { + self.request(|reply| BrokerCommand::FetchBatchAvailable { + req, + max_items, + reply, + }) + .await + } + + async fn commit( + &self, + channel_id: i64, + reservation_id: u64, + now_ms: i64, + ) -> Result { + self.request(|reply| BrokerCommand::Commit { + channel_id, + reservation_id, + now_ms, + reply, + }) + .await + } + + async fn commit_batch( + &self, + channel_id: i64, + reservation_ids: Vec, + now_ms: i64, + ) -> Result { + self.request(|reply| BrokerCommand::CommitBatch { + channel_id, + reservation_ids, + now_ms, + reply, + }) + .await + } + + async fn requeue_inflight( + &self, + channel_id: i64, + reservation_id: u64, + ) -> Result<(), BrokerError> { + self.request(|reply| BrokerCommand::RequeueInflight { + channel_id, + reservation_id, + reply, + }) + .await + } + + async fn requeue_all_inflight(&self, channel_id: i64) -> Result<(), BrokerError> { + self.request(|reply| BrokerCommand::RequeueAllInflight { channel_id, reply }) + .await + } + + async fn take_cleanup_batch( + &self, + channel_id: i64, + max_items: usize, + ) -> Result, BrokerError> { + self.request(|reply| BrokerCommand::TakeCleanupBatch { + channel_id, + max_items, + reply, + }) + .await + } + + async fn cleanup_ack(&self, channel_id: i64, reservation_id: u64) -> Result<(), BrokerError> { + self.request(|reply| BrokerCommand::CleanupAck { + channel_id, + reservation_id, + reply, + }) + .await + } + + async fn cleanup_nack(&self, channel_id: i64, reservation_id: u64) -> Result<(), BrokerError> { + self.request(|reply| BrokerCommand::CleanupNack { + channel_id, + reservation_id, + reply, + }) + .await + } + + async fn shutdown(&self) -> Result<(), BrokerError> { + self.request(|reply| BrokerCommand::Shutdown { reply }) + .await + } + + async fn request( + &self, + make_cmd: impl FnOnce(oneshot::Sender>) -> BrokerCommand, + ) -> Result { + let (reply_tx, reply_rx) = oneshot::channel(); + self.tx + .send(make_cmd(reply_tx)) + .await + .map_err(|_| BrokerError::ActorClosed)?; + reply_rx.await.map_err(|_| BrokerError::ActorClosed)? + } +} + +#[derive(Debug, Clone, Default, Encode, Decode)] +enum BrokerRpcOperation { + #[default] + Noop, + UpsertChannel { + config: BrokerChannelConfig, + }, + DeleteChannel { + channel_id: i64, + }, + Reserve { + req: BrokerReserveRequest, + }, + Publish { + channel_id: i64, + reservation_id: u64, + now_ms: i64, + }, + Abort { + channel_id: i64, + reservation_id: u64, + }, + FetchNext { + req: BrokerFetchRequest, + }, + FetchBatchAvailable { + req: BrokerFetchRequest, + max_items: usize, + }, + Commit { + channel_id: i64, + reservation_id: u64, + now_ms: i64, + }, + CommitBatch { + channel_id: i64, + reservation_ids: Vec, + now_ms: i64, + }, + RequeueInflight { + channel_id: i64, + reservation_id: u64, + }, + RequeueAllInflight { + channel_id: i64, + }, + TakeCleanupBatch { + channel_id: i64, + max_items: usize, + }, + CleanupAck { + channel_id: i64, + reservation_id: u64, + }, + CleanupNack { + channel_id: i64, + reservation_id: u64, + }, +} + +#[derive(Debug, Clone, Default, Encode, Decode)] +struct BrokerRpcRequest { + op: BrokerRpcOperation, +} + +impl MsgPackSerializePart for BrokerRpcRequest { + fn msg_id(&self) -> u32 { + BROKER_RPC_REQ_MSG_ID + } +} + +impl RPCReq for BrokerRpcRequest { + type Resp = BrokerRpcResponse; +} + +#[derive(Debug, Clone, Encode, Decode)] +enum BrokerRpcReply { + Unit(Result<(), BrokerError>), + PayloadKeys(Result, BrokerError>), + Reservation(Result), + Envelope(Result), + Fetch(Result, BrokerError>), + FetchBatch(Result), + Commit(Result), + CommitBatch(Result), + CleanupBatch(Result, BrokerError>), +} + +impl Default for BrokerRpcReply { + fn default() -> Self { + Self::Unit(Ok(())) + } +} + +#[derive(Debug, Clone, Default, Encode, Decode)] +struct BrokerRpcResponse { + reply: BrokerRpcReply, +} + +impl MsgPackSerializePart for BrokerRpcResponse { + fn msg_id(&self) -> u32 { + BROKER_RPC_RESP_MSG_ID + } +} + +async fn execute_rpc_request( + broker: &LocalBrokerHandle, + request: BrokerRpcRequest, +) -> BrokerRpcResponse { + let reply = match request.op { + BrokerRpcOperation::Noop => BrokerRpcReply::Unit(Err(BrokerError::Rpc( + "broker noop request is invalid".to_string(), + ))), + BrokerRpcOperation::UpsertChannel { config } => { + BrokerRpcReply::Unit(broker.upsert_channel(config).await) + } + BrokerRpcOperation::DeleteChannel { channel_id } => { + BrokerRpcReply::PayloadKeys(broker.delete_channel(channel_id).await) + } + BrokerRpcOperation::Reserve { req } => { + BrokerRpcReply::Reservation(broker.reserve(req).await) + } + BrokerRpcOperation::Publish { + channel_id, + reservation_id, + now_ms, + } => BrokerRpcReply::Envelope(broker.publish(channel_id, reservation_id, now_ms).await), + BrokerRpcOperation::Abort { + channel_id, + reservation_id, + } => BrokerRpcReply::Unit(broker.abort(channel_id, reservation_id).await), + BrokerRpcOperation::FetchNext { req } => { + BrokerRpcReply::Fetch(broker.fetch_next(req).await) + } + BrokerRpcOperation::FetchBatchAvailable { req, max_items } => { + BrokerRpcReply::FetchBatch(broker.fetch_batch_available(req, max_items).await) + } + BrokerRpcOperation::Commit { + channel_id, + reservation_id, + now_ms, + } => BrokerRpcReply::Commit(broker.commit(channel_id, reservation_id, now_ms).await), + BrokerRpcOperation::CommitBatch { + channel_id, + reservation_ids, + now_ms, + } => BrokerRpcReply::CommitBatch( + broker + .commit_batch(channel_id, reservation_ids, now_ms) + .await, + ), + BrokerRpcOperation::RequeueInflight { + channel_id, + reservation_id, + } => BrokerRpcReply::Unit(broker.requeue_inflight(channel_id, reservation_id).await), + BrokerRpcOperation::RequeueAllInflight { channel_id } => { + BrokerRpcReply::Unit(broker.requeue_all_inflight(channel_id).await) + } + BrokerRpcOperation::TakeCleanupBatch { + channel_id, + max_items, + } => BrokerRpcReply::CleanupBatch(broker.take_cleanup_batch(channel_id, max_items).await), + BrokerRpcOperation::CleanupAck { + channel_id, + reservation_id, + } => BrokerRpcReply::Unit(broker.cleanup_ack(channel_id, reservation_id).await), + BrokerRpcOperation::CleanupNack { + channel_id, + reservation_id, + } => BrokerRpcReply::Unit(broker.cleanup_nack(channel_id, reservation_id).await), + }; + BrokerRpcResponse { reply } +} + +pub fn register_broker_service(p2p_view: P2pModuleView, queue_capacity: usize) { + let broker = LocalBrokerHandle::spawn_actor(LocalBroker::new(), queue_capacity); + let handler_view = p2p_view.clone(); + RPCHandler::::new().regist(p2p_view.p2p_module(), move |resp, msg| { + let broker = broker.clone(); + let handler_view = handler_view.clone(); + let _ = handler_view.spawn("fluxon_mq.broker.rpc", async move { + let response = execute_rpc_request(&broker, msg.serialize_part).await; + let _ = resp + .send_resp(MsgPack { + serialize_part: response, + raw_bytes: Vec::new(), + }) + .await; + }); + Ok(()) + }); +} + +#[derive(Clone)] +struct RemoteBrokerHandle { + cluster_manager_view: ClusterManagerView, + p2p_view: P2pModuleView, +} + +#[derive(Clone)] +enum BrokerHandleInner { + Local(LocalBrokerHandle), + Remote(RemoteBrokerHandle), +} + +pub struct BrokerHandle { + inner: BrokerHandleInner, +} + +impl Clone for BrokerHandle { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl std::fmt::Debug for BrokerHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.inner { + BrokerHandleInner::Local(_) => f + .debug_struct("BrokerHandle") + .field("kind", &"local") + .finish(), + BrokerHandleInner::Remote(_) => f + .debug_struct("BrokerHandle") + .field("kind", &"remote") + .finish(), + } + } +} + +impl BrokerHandle { + pub fn new_distributed( + cluster_manager_view: ClusterManagerView, + p2p_view: P2pModuleView, + ) -> Self { + Self { + inner: BrokerHandleInner::Remote(RemoteBrokerHandle { + cluster_manager_view, + p2p_view, + }), + } + } + + #[cfg(test)] + pub fn new_local_for_test(queue_capacity: usize) -> Self { + Self { + inner: BrokerHandleInner::Local( + LocalBrokerHandle::spawn_actor_with_cleanup_release_delay( + LocalBroker::new(), + queue_capacity, + Duration::ZERO, + ), + ), + } + } + + #[cfg(test)] + pub fn new_local_with_payload_byte_capacity_for_test( + payload_byte_capacity: u64, + queue_capacity: usize, + ) -> Self { + Self { + inner: BrokerHandleInner::Local( + LocalBrokerHandle::spawn_actor_with_cleanup_release_delay( + LocalBroker::with_payload_byte_capacity(payload_byte_capacity), + queue_capacity, + Duration::ZERO, + ), + ), + } + } + + pub async fn upsert_channel(&self, config: BrokerChannelConfig) -> Result<(), BrokerError> { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::UpsertChannel { config }, + }) + .await? + .reply + { + BrokerRpcReply::Unit(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for upsert_channel: {:?}", + other + ))), + } + } + + pub async fn delete_channel(&self, channel_id: i64) -> Result, BrokerError> { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::DeleteChannel { channel_id }, + }) + .await? + .reply + { + BrokerRpcReply::PayloadKeys(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for delete_channel: {:?}", + other + ))), + } + } + + pub async fn reserve( + &self, + req: BrokerReserveRequest, + ) -> Result { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::Reserve { req }, + }) + .await? + .reply + { + BrokerRpcReply::Reservation(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for reserve: {:?}", + other + ))), + } + } + + pub async fn publish( + &self, + channel_id: i64, + reservation_id: u64, + now_ms: i64, + ) -> Result { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::Publish { + channel_id, + reservation_id, + now_ms, + }, + }) + .await? + .reply + { + BrokerRpcReply::Envelope(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for publish: {:?}", + other + ))), + } + } + + pub async fn abort(&self, channel_id: i64, reservation_id: u64) -> Result<(), BrokerError> { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::Abort { + channel_id, + reservation_id, + }, + }) + .await? + .reply + { + BrokerRpcReply::Unit(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for abort: {:?}", + other + ))), + } + } + + pub async fn fetch_next( + &self, + req: BrokerFetchRequest, + ) -> Result, BrokerError> { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::FetchNext { req }, + }) + .await? + .reply + { + BrokerRpcReply::Fetch(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for fetch_next: {:?}", + other + ))), + } + } + + pub async fn fetch_batch_available( + &self, + req: BrokerFetchRequest, + max_items: usize, + ) -> Result { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::FetchBatchAvailable { req, max_items }, + }) + .await? + .reply + { + BrokerRpcReply::FetchBatch(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for fetch_batch_available: {:?}", + other + ))), + } + } + + pub async fn commit( + &self, + channel_id: i64, + reservation_id: u64, + now_ms: i64, + ) -> Result { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::Commit { + channel_id, + reservation_id, + now_ms, + }, + }) + .await? + .reply + { + BrokerRpcReply::Commit(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for commit: {:?}", + other + ))), + } + } + + pub async fn commit_batch( + &self, + channel_id: i64, + reservation_ids: Vec, + now_ms: i64, + ) -> Result { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::CommitBatch { + channel_id, + reservation_ids, + now_ms, + }, + }) + .await? + .reply + { + BrokerRpcReply::CommitBatch(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for commit_batch: {:?}", + other + ))), + } + } + + pub async fn requeue_inflight( + &self, + channel_id: i64, + reservation_id: u64, + ) -> Result<(), BrokerError> { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::RequeueInflight { + channel_id, + reservation_id, + }, + }) + .await? + .reply + { + BrokerRpcReply::Unit(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for requeue_inflight: {:?}", + other + ))), + } + } + + pub async fn requeue_all_inflight(&self, channel_id: i64) -> Result<(), BrokerError> { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::RequeueAllInflight { channel_id }, + }) + .await? + .reply + { + BrokerRpcReply::Unit(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for requeue_all_inflight: {:?}", + other + ))), + } + } + + pub async fn take_cleanup_batch( + &self, + channel_id: i64, + max_items: usize, + ) -> Result, BrokerError> { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::TakeCleanupBatch { + channel_id, + max_items, + }, + }) + .await? + .reply + { + BrokerRpcReply::CleanupBatch(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for take_cleanup_batch: {:?}", + other + ))), + } + } + + pub async fn cleanup_ack( + &self, + channel_id: i64, + reservation_id: u64, + ) -> Result<(), BrokerError> { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::CleanupAck { + channel_id, + reservation_id, + }, + }) + .await? + .reply + { + BrokerRpcReply::Unit(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for cleanup_ack: {:?}", + other + ))), + } + } + + pub async fn cleanup_nack( + &self, + channel_id: i64, + reservation_id: u64, + ) -> Result<(), BrokerError> { + match self + .request(BrokerRpcRequest { + op: BrokerRpcOperation::CleanupNack { + channel_id, + reservation_id, + }, + }) + .await? + .reply + { + BrokerRpcReply::Unit(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for cleanup_nack: {:?}", + other + ))), + } + } + + pub async fn shutdown(&self) -> Result<(), BrokerError> { + match &self.inner { + BrokerHandleInner::Local(local) => local.shutdown().await, + BrokerHandleInner::Remote(_) => Err(BrokerError::Rpc( + "shutdown is unsupported for distributed broker handles".to_string(), + )), + } + } + + async fn request(&self, request: BrokerRpcRequest) -> Result { + match &self.inner { + BrokerHandleInner::Local(local) => Ok(execute_rpc_request(local, request).await), + BrokerHandleInner::Remote(remote) => remote.request(request).await, + } + } +} + +impl RemoteBrokerHandle { + async fn request(&self, request: BrokerRpcRequest) -> Result { + let broker_node = + find_or_wait_broker_node(self.cluster_manager_view.cluster_manager()).await?; + let response = RPCCaller::::new() + .call( + self.p2p_view.p2p_module(), + broker_node.into(), + MsgPack { + serialize_part: request, + raw_bytes: Vec::new(), + }, + None, + 6, + ) + .await + .map_err(|e| BrokerError::Rpc(format!("broker rpc call failed: {}", e)))?; + Ok(response.serialize_part) + } +} + +async fn find_or_wait_broker_node( + cluster_manager: &fluxon_commu::ClusterManager, +) -> Result { + let members = cluster_manager.get_members(); + let broker_nodes: Vec<_> = members + .iter() + .filter(|member| is_broker_member(member)) + .collect(); + if broker_nodes.len() == 1 { + return Ok(broker_nodes[0].id.to_string()); + } + if broker_nodes.len() > 1 { + return Err(BrokerError::BrokerUnavailable(format!( + "multiple brokers found: {:?}", + broker_nodes + .into_iter() + .map(|member| member.id.to_string()) + .collect::>() + ))); + } + + let mut rx = cluster_manager.listen(); + tokio::time::timeout(BROKER_DISCOVERY_TIMEOUT, async move { + while let Ok(event) = rx.recv().await { + match event { + fluxon_commu::ClusterEvent::MemberJoined(member) + | fluxon_commu::ClusterEvent::MemberUpdated(member) + if is_broker_member(&member) => + { + return Ok(member.id.to_string()); + } + _ => {} + } + } + Err(BrokerError::BrokerUnavailable( + "broker node not found from cluster manager".to_string(), + )) + }) + .await + .unwrap_or_else(|_| { + Err(BrokerError::BrokerUnavailable(format!( + "timed out waiting {}s for broker node registration; start fluxon_py.runtime.start_broker first", + BROKER_DISCOVERY_TIMEOUT.as_secs() + ))) + }) +} + +fn is_broker_member(member: &fluxon_commu::ClusterMember) -> bool { + member + .metadata + .get(FLUXON_MQ_COMPONENT_METADATA_KEY) + .is_some_and(|value| value == FLUXON_MQ_COMPONENT_BROKER_METADATA_VALUE) +} + +fn broker_category_enforces_capacity(category: MqCategory) -> bool { + matches!(category, MqCategory::MpmcSub { .. }) +} + +pub fn now_unix_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock is before UNIX_EPOCH") + .as_millis() as i64 +} + +fn validate_capacity(config: &BrokerChannelConfig) -> Result<(), BrokerError> { + if config.capacity <= 0 { + return Err(BrokerError::InvalidCapacity { + channel_id: config.channel_id, + capacity: config.capacity, + }); + } + Ok(()) +} + +fn default_payload_byte_capacity() -> u64 { + if let Ok(raw) = env::var(BROKER_PAYLOAD_BYTES_CAP_ENV) { + if let Ok(value) = raw.trim().parse::() { + if value > 0 { + return value; + } + } + } + + if let Ok(raw) = env::var(OWNER_POOL_DRAM_BYTES_ENV) { + if let Ok(value) = raw.trim().parse::() { + if value > 0 { + let percent = payload_byte_capacity_percent(); + return ((value as u128) * (percent as u128) / 100).max(1) as u64; + } + } + } + + DEFAULT_BROKER_PAYLOAD_BYTES_CAP +} + +fn payload_byte_capacity_percent() -> u64 { + env::var(BROKER_PAYLOAD_BYTES_CAP_PERCENT_ENV) + .ok() + .and_then(|raw| raw.trim().parse::().ok()) + .filter(|value| (1..=100).contains(value)) + .unwrap_or(DEFAULT_BROKER_PAYLOAD_BYTES_CAP_PERCENT) +} + +fn default_cleanup_release_delay() -> Duration { + Duration::from_millis( + env::var(BROKER_CLEANUP_RELEASE_DELAY_MS_ENV) + .ok() + .and_then(|raw| raw.trim().parse::().ok()) + .unwrap_or(DEFAULT_BROKER_CLEANUP_RELEASE_DELAY_MS), + ) +} + +fn remove_from_deque(queue: &mut VecDeque, reservation_id: u64) { + if let Some(pos) = queue.iter().position(|id| *id == reservation_id) { + queue.remove(pos); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn reserve_req(channel_id: i64, producer_id: &str, now_ms: i64) -> BrokerReserveRequest { + reserve_req_with_category(channel_id, producer_id, MqCategory::Mpsc, 1, now_ms) + } + + fn reserve_req_with_category( + channel_id: i64, + producer_id: &str, + category: MqCategory, + payload_bytes: u64, + now_ms: i64, + ) -> BrokerReserveRequest { + BrokerReserveRequest { + channel_id, + producer_id: producer_id.to_string(), + category, + payload_bytes, + now_ms, + } + } + + fn reserve_req_bytes( + channel_id: i64, + producer_id: &str, + payload_bytes: u64, + now_ms: i64, + ) -> BrokerReserveRequest { + BrokerReserveRequest { + channel_id, + producer_id: producer_id.to_string(), + category: MqCategory::Mpsc, + payload_bytes, + now_ms, + } + } + + fn fetch_req(channel_id: i64, consumer_id: &str, now_ms: i64) -> BrokerFetchRequest { + BrokerFetchRequest { + channel_id, + consumer_id: consumer_id.to_string(), + now_ms, + } + } + + #[test] + fn reserve_publish_fetch_commit_frees_capacity_for_mpmc_sub() { + let mut broker = LocalBroker::new(); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 7, + capacity: 2, + }) + .unwrap(); + + let first = broker + .reserve(reserve_req_with_category( + 7, + "p0", + MqCategory::MpmcSub { parent_mpmc_id: 70 }, + 1, + 10, + )) + .unwrap(); + let second = broker + .reserve(reserve_req_with_category( + 7, + "p0", + MqCategory::MpmcSub { parent_mpmc_id: 70 }, + 1, + 11, + )) + .unwrap(); + assert_eq!(first.envelope.msg_id, 0); + assert_eq!(second.envelope.msg_id, 1); + assert_eq!( + broker + .reserve(reserve_req_with_category( + 7, + "p0", + MqCategory::MpmcSub { parent_mpmc_id: 70 }, + 1, + 12, + )) + .unwrap_err(), + BrokerError::ChannelFull { + channel_id: 7, + capacity: 2, + used_slots: 2, + } + ); + + broker + .publish(7, first.envelope.reservation_id, 20) + .unwrap(); + let fetched = broker.fetch_next(fetch_req(7, "c0", 30)).unwrap().unwrap(); + assert_eq!( + fetched.envelope.reservation_id, + first.envelope.reservation_id + ); + + let committed = broker + .commit(7, fetched.envelope.reservation_id, 40) + .unwrap(); + assert!(committed.first_commit); + assert_eq!( + committed + .cleanup + .as_ref() + .map(|env| env.payload_key.as_str()), + Some( + keys::backend_message_key_with_category( + 7, + "p0", + 0, + &MqCategory::MpmcSub { parent_mpmc_id: 70 }, + ) + .as_str() + ) + ); + + let third = broker + .reserve(reserve_req_with_category( + 7, + "p0", + MqCategory::MpmcSub { parent_mpmc_id: 70 }, + 1, + 50, + )) + .unwrap(); + assert_eq!(third.envelope.msg_id, 2); + } + + #[test] + fn abort_releases_pending_slot_for_mpmc_sub() { + let mut broker = LocalBroker::new(); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 8, + capacity: 1, + }) + .unwrap(); + + let reservation = broker + .reserve(reserve_req_with_category( + 8, + "p0", + MqCategory::MpmcSub { parent_mpmc_id: 80 }, + 1, + 10, + )) + .unwrap(); + assert!(matches!( + broker.reserve(reserve_req_with_category( + 8, + "p0", + MqCategory::MpmcSub { parent_mpmc_id: 80 }, + 1, + 11, + )), + Err(BrokerError::ChannelFull { .. }) + )); + + broker + .abort(8, reservation.envelope.reservation_id) + .unwrap(); + let next = broker + .reserve(reserve_req_with_category( + 8, + "p0", + MqCategory::MpmcSub { parent_mpmc_id: 80 }, + 1, + 12, + )) + .unwrap(); + assert_eq!(next.envelope.msg_id, 1); + } + + #[test] + fn replay_log_restores_visible_and_inflight_state() { + let mut broker = LocalBroker::new(); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 9, + capacity: 4, + }) + .unwrap(); + let first = broker.reserve(reserve_req(9, "p0", 10)).unwrap(); + let second = broker.reserve(reserve_req(9, "p1", 11)).unwrap(); + broker + .publish(9, first.envelope.reservation_id, 20) + .unwrap(); + broker + .publish(9, second.envelope.reservation_id, 21) + .unwrap(); + let fetched = broker.fetch_next(fetch_req(9, "c0", 30)).unwrap().unwrap(); + assert_eq!(fetched.envelope.producer_id, "p0"); + + let log = broker.log_records().to_vec(); + let mut restored = LocalBroker::from_log(&log).unwrap(); + restored.requeue_all_inflight(9).unwrap(); + + let redelivered = restored + .fetch_next(fetch_req(9, "c0", 40)) + .unwrap() + .unwrap(); + assert_eq!( + redelivered.envelope.reservation_id, + first.envelope.reservation_id + ); + } + + #[test] + fn requeue_all_inflight_preserves_fetch_order() { + let mut broker = LocalBroker::new(); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 10, + capacity: 4, + }) + .unwrap(); + let first = broker.reserve(reserve_req(10, "p0", 10)).unwrap(); + let second = broker.reserve(reserve_req(10, "p0", 11)).unwrap(); + broker + .publish(10, first.envelope.reservation_id, 20) + .unwrap(); + broker + .publish(10, second.envelope.reservation_id, 21) + .unwrap(); + + let _ = broker.fetch_next(fetch_req(10, "c0", 30)).unwrap().unwrap(); + let _ = broker.fetch_next(fetch_req(10, "c0", 31)).unwrap().unwrap(); + broker.requeue_all_inflight(10).unwrap(); + + let redelivered_first = broker.fetch_next(fetch_req(10, "c0", 40)).unwrap().unwrap(); + let redelivered_second = broker.fetch_next(fetch_req(10, "c0", 41)).unwrap().unwrap(); + assert_eq!( + redelivered_first.envelope.reservation_id, + first.envelope.reservation_id + ); + assert_eq!( + redelivered_second.envelope.reservation_id, + second.envelope.reservation_id + ); + } + + #[test] + fn batch_fetch_and_commit_preserves_order_and_frees_capacity() { + let mut broker = LocalBroker::new(); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 11, + capacity: 3, + }) + .unwrap(); + + let first = broker.reserve(reserve_req(11, "p0", 10)).unwrap(); + let second = broker.reserve(reserve_req(11, "p0", 11)).unwrap(); + let third = broker.reserve(reserve_req(11, "p1", 12)).unwrap(); + for reservation in [&first, &second, &third] { + broker + .publish(11, reservation.envelope.reservation_id, 20) + .unwrap(); + } + + let batch = broker + .fetch_batch_available(fetch_req(11, "c0", 30), 2) + .unwrap(); + assert_eq!(batch.messages.len(), 2); + assert_eq!(batch.messages[0].envelope.msg_id, 0); + assert_eq!(batch.messages[1].envelope.msg_id, 1); + + let outcome = broker + .commit_batch( + 11, + batch + .messages + .iter() + .map(|message| message.envelope.reservation_id) + .collect(), + 40, + ) + .unwrap(); + assert_eq!(outcome.first_commit_count, 2); + assert_eq!(outcome.cleanup.len(), 2); + + let next = broker.reserve(reserve_req(11, "p0", 50)).unwrap(); + assert_eq!(next.envelope.msg_id, 2); + } + + #[test] + fn payload_byte_budget_is_global_and_released_on_cleanup_ack_or_abort() { + let mut broker = LocalBroker::with_payload_byte_capacity(10); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 21, + capacity: 8, + }) + .unwrap(); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 22, + capacity: 8, + }) + .unwrap(); + + let first = broker.reserve(reserve_req_bytes(21, "p0", 6, 10)).unwrap(); + assert_eq!(first.envelope.payload_bytes, 6); + assert!(matches!( + broker.reserve(reserve_req_bytes(22, "p1", 5, 11)), + Err(BrokerError::PayloadBytesFull { .. }) + )); + + broker + .publish(21, first.envelope.reservation_id, 20) + .unwrap(); + let fetched = broker.fetch_next(fetch_req(21, "c0", 30)).unwrap().unwrap(); + broker + .commit(21, fetched.envelope.reservation_id, 40) + .unwrap(); + assert!(matches!( + broker.reserve(reserve_req_bytes(22, "p1", 5, 41)), + Err(BrokerError::PayloadBytesFull { .. }) + )); + broker + .cleanup_ack(21, fetched.envelope.reservation_id) + .unwrap(); + let second = broker.reserve(reserve_req_bytes(22, "p1", 5, 50)).unwrap(); + broker.abort(22, second.envelope.reservation_id).unwrap(); + let third = broker.reserve(reserve_req_bytes(22, "p1", 10, 60)).unwrap(); + assert_eq!(third.envelope.payload_bytes, 10); + } + + #[test] + fn mpsc_reserve_does_not_gate_on_channel_capacity() { + let mut broker = LocalBroker::new(); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 201, + capacity: 1, + }) + .unwrap(); + + let first = broker.reserve(reserve_req(201, "p0", 10)).unwrap(); + let second = broker.reserve(reserve_req(201, "p0", 11)).unwrap(); + + assert_eq!(first.envelope.msg_id, 0); + assert_eq!(second.envelope.msg_id, 1); + } + + #[test] + fn mpmc_sub_reserve_still_gates_on_channel_capacity() { + let mut broker = LocalBroker::new(); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 202, + capacity: 1, + }) + .unwrap(); + + let _ = broker + .reserve(reserve_req_with_category( + 202, + "p0", + MqCategory::MpmcSub { parent_mpmc_id: 9 }, + 1, + 10, + )) + .unwrap(); + + assert!(matches!( + broker.reserve(reserve_req_with_category( + 202, + "p0", + MqCategory::MpmcSub { parent_mpmc_id: 9 }, + 1, + 11, + )), + Err(BrokerError::ChannelFull { .. }) + )); + } + + #[test] + fn cleanup_ack_releases_payload_after_cleanup_batch_take() { + let mut broker = LocalBroker::with_payload_byte_capacity(10); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 23, + capacity: 8, + }) + .unwrap(); + + let first = broker.reserve(reserve_req_bytes(23, "p0", 6, 10)).unwrap(); + broker + .publish(23, first.envelope.reservation_id, 20) + .unwrap(); + let fetched = broker.fetch_next(fetch_req(23, "c0", 30)).unwrap().unwrap(); + broker + .commit(23, fetched.envelope.reservation_id, 40) + .unwrap(); + assert_eq!(broker.take_cleanup_batch(23, 8).unwrap().len(), 1); + assert!(matches!( + broker.reserve(reserve_req_bytes(23, "p1", 5, 41)), + Err(BrokerError::PayloadBytesFull { .. }) + )); + + broker + .cleanup_ack(23, fetched.envelope.reservation_id) + .unwrap(); + let second = broker.reserve(reserve_req_bytes(23, "p1", 5, 50)).unwrap(); + assert_eq!(second.envelope.payload_bytes, 5); + } + + #[test] + fn delete_channel_releases_payload_budget_for_all_queues() { + let mut broker = LocalBroker::with_payload_byte_capacity(100); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 31, + capacity: 16, + }) + .unwrap(); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 32, + capacity: 16, + }) + .unwrap(); + + let pending = broker.reserve(reserve_req_bytes(31, "p0", 10, 10)).unwrap(); + + let inflight = broker.reserve(reserve_req_bytes(31, "p0", 12, 12)).unwrap(); + broker + .publish(31, inflight.envelope.reservation_id, 21) + .unwrap(); + let _ = broker.fetch_next(fetch_req(31, "c0", 30)).unwrap().unwrap(); + + let cleanup_inflight = broker.reserve(reserve_req_bytes(31, "p0", 13, 13)).unwrap(); + broker + .publish(31, cleanup_inflight.envelope.reservation_id, 22) + .unwrap(); + let fetched = broker.fetch_next(fetch_req(31, "c0", 31)).unwrap().unwrap(); + broker + .commit(31, fetched.envelope.reservation_id, 40) + .unwrap(); + assert_eq!(broker.take_cleanup_batch(31, 1).unwrap().len(), 1); + + let cleanup = broker.reserve(reserve_req_bytes(31, "p0", 14, 14)).unwrap(); + broker + .publish(31, cleanup.envelope.reservation_id, 23) + .unwrap(); + let fetched = broker.fetch_next(fetch_req(31, "c0", 32)).unwrap().unwrap(); + broker + .commit(31, fetched.envelope.reservation_id, 41) + .unwrap(); + + let visible = broker.reserve(reserve_req_bytes(31, "p0", 11, 15)).unwrap(); + broker + .publish(31, visible.envelope.reservation_id, 24) + .unwrap(); + + assert_eq!(broker.state.used_payload_bytes, 60); + assert!(matches!( + broker.reserve(reserve_req_bytes(32, "p1", 41, 50)), + Err(BrokerError::PayloadBytesFull { .. }) + )); + + let mut payload_keys = broker.delete_channel(31).unwrap(); + payload_keys.sort(); + let mut expected_payload_keys = vec![ + pending.envelope.payload_key, + inflight.envelope.payload_key, + cleanup_inflight.envelope.payload_key, + cleanup.envelope.payload_key, + visible.envelope.payload_key, + ]; + expected_payload_keys.sort(); + assert_eq!(payload_keys, expected_payload_keys); + assert_eq!(broker.state.used_payload_bytes, 0); + assert_eq!(broker.delete_channel(31), Ok(Vec::new())); + assert_eq!( + broker.fetch_next(fetch_req(31, "c0", 60)).unwrap_err(), + BrokerError::ChannelNotFound(31) + ); + + let next = broker + .reserve(reserve_req_bytes(32, "p1", 100, 70)) + .unwrap(); + assert_eq!(next.envelope.payload_bytes, 100); + } + + #[tokio::test] + async fn broker_handle_roundtrip_uses_local_actor() { + let handle = BrokerHandle::new_local_for_test(32); + handle + .upsert_channel(BrokerChannelConfig { + channel_id: 12, + capacity: 2, + }) + .await + .unwrap(); + let reserved = handle.reserve(reserve_req(12, "p0", 10)).await.unwrap(); + handle + .publish(12, reserved.envelope.reservation_id, 20) + .await + .unwrap(); + let fetched = handle + .fetch_next(fetch_req(12, "c0", 30)) + .await + .unwrap() + .unwrap(); + assert_eq!(fetched.envelope.msg_id, 0); + handle + .commit(12, fetched.envelope.reservation_id, 40) + .await + .unwrap(); + assert_eq!(handle.take_cleanup_batch(12, 8).await.unwrap().len(), 1); + handle + .cleanup_ack(12, fetched.envelope.reservation_id) + .await + .unwrap(); + handle.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn broker_handle_delete_channel_releases_payload_budget() { + let handle = BrokerHandle::new_local_with_payload_byte_capacity_for_test(10, 8); + handle + .upsert_channel(BrokerChannelConfig { + channel_id: 24, + capacity: 4, + }) + .await + .unwrap(); + + let first = handle + .reserve(reserve_req_bytes(24, "p0", 6, 10)) + .await + .unwrap(); + assert!(matches!( + handle.reserve(reserve_req_bytes(24, "p1", 5, 11)).await, + Err(BrokerError::PayloadBytesFull { .. }) + )); + + assert_eq!( + handle.delete_channel(24).await.unwrap(), + vec![first.envelope.payload_key] + ); + assert_eq!( + handle.delete_channel(24).await.unwrap(), + Vec::::new() + ); + handle + .upsert_channel(BrokerChannelConfig { + channel_id: 25, + capacity: 4, + }) + .await + .unwrap(); + let next = handle + .reserve(reserve_req_bytes(25, "p1", 10, 20)) + .await + .unwrap(); + assert_eq!(next.envelope.payload_bytes, 10); + + handle.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn broker_handle_returns_actor_closed_after_shutdown() { + let handle = BrokerHandle::new_local_for_test(8); + handle + .upsert_channel(BrokerChannelConfig { + channel_id: 13, + capacity: 1, + }) + .await + .unwrap(); + handle.shutdown().await.unwrap(); + assert_eq!( + handle.reserve(reserve_req(13, "p0", 10)).await.unwrap_err(), + BrokerError::ActorClosed + ); + } + + #[tokio::test] + async fn broker_handle_returns_channel_full_without_waiting_for_mpmc_sub() { + let handle = BrokerHandle::new_local_for_test(8); + handle + .upsert_channel(BrokerChannelConfig { + channel_id: 14, + capacity: 1, + }) + .await + .unwrap(); + + let first = handle + .reserve(reserve_req_with_category( + 14, + "p0", + MqCategory::MpmcSub { + parent_mpmc_id: 140, + }, + 1, + 10, + )) + .await + .unwrap(); + assert!(matches!( + handle + .reserve(reserve_req_with_category( + 14, + "p0", + MqCategory::MpmcSub { + parent_mpmc_id: 140 + }, + 1, + 11, + )) + .await, + Err(BrokerError::ChannelFull { .. }) + )); + + handle + .abort(14, first.envelope.reservation_id) + .await + .unwrap(); + let second = handle + .reserve(reserve_req_with_category( + 14, + "p0", + MqCategory::MpmcSub { + parent_mpmc_id: 140, + }, + 1, + 12, + )) + .await + .unwrap(); + assert_eq!(second.envelope.msg_id, 1); + + handle.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn broker_handle_returns_payload_bytes_full_without_waiting() { + let handle = BrokerHandle::new_local_with_payload_byte_capacity_for_test(10, 8); + handle + .upsert_channel(BrokerChannelConfig { + channel_id: 16, + capacity: 8, + }) + .await + .unwrap(); + + let first = handle + .reserve(reserve_req_bytes(16, "p0", 6, 10)) + .await + .unwrap(); + assert!(matches!( + handle.reserve(reserve_req_bytes(16, "p1", 5, 11)).await, + Err(BrokerError::PayloadBytesFull { .. }) + )); + + handle + .publish(16, first.envelope.reservation_id, 20) + .await + .unwrap(); + let fetched = handle + .fetch_next(fetch_req(16, "c0", 30)) + .await + .unwrap() + .unwrap(); + handle + .commit(16, fetched.envelope.reservation_id, 40) + .await + .unwrap(); + + handle + .cleanup_ack(16, fetched.envelope.reservation_id) + .await + .unwrap(); + + let second = handle + .reserve(reserve_req_bytes(16, "p1", 5, 50)) + .await + .unwrap(); + assert_eq!(second.envelope.producer_id, "p1"); + assert_eq!(second.envelope.payload_bytes, 5); + + handle.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn broker_handle_waits_for_message_then_resumes() { + use std::time::Duration; + use tokio::time::sleep; + + let handle = BrokerHandle::new_local_for_test(8); + handle + .upsert_channel(BrokerChannelConfig { + channel_id: 15, + capacity: 2, + }) + .await + .unwrap(); + + let waiter_handle = handle.clone(); + let pending = + tokio::spawn(async move { waiter_handle.fetch_next(fetch_req(15, "c0", 10)).await }); + + sleep(Duration::from_millis(50)).await; + assert!(!pending.is_finished()); + + let reservation = handle.reserve(reserve_req(15, "p0", 11)).await.unwrap(); + handle + .publish(15, reservation.envelope.reservation_id, 12) + .await + .unwrap(); + + let fetched = pending.await.unwrap().unwrap().unwrap(); + assert_eq!(fetched.envelope.msg_id, 0); + + handle.shutdown().await.unwrap(); + } +} diff --git a/fluxon_rs/fluxon_mq/src/consumer.rs b/fluxon_rs/fluxon_mq/src/consumer.rs index c5e5fa4..b1f262b 100644 --- a/fluxon_rs/fluxon_mq/src/consumer.rs +++ b/fluxon_rs/fluxon_mq/src/consumer.rs @@ -47,6 +47,7 @@ use crate::nonblocking_monitor::{ }; use crate::shutdown::ShutdownCtl; use crate::LifecycleView; +use crate::{BrokerEnvelope, BrokerFetchRequest, BrokerFetchedMessage, BrokerHandle}; use tracing::{debug, info, warn}; const NO_MESSAGE_WARN_INTERVAL: Duration = Duration::from_secs(30); @@ -64,6 +65,9 @@ const PREFETCH_HANDLE_AWAIT_WARN_INTERVAL: Duration = Duration::from_secs(2); const COMMIT_PROGRESS_RETENTION: usize = 1024; const STALE_PRODUCER_PROBE_TOMB_TTL: Duration = Duration::from_secs(10); const READY_TRACE_HISTORY_PER_PRODUCER: usize = 64; +const PREFETCH_REFILL_BURST_MAX: usize = 128; +const PREFETCH_NO_MESSAGE_RETRY_EMPTY_SLEEP: Duration = Duration::from_millis(1); +const PREFETCH_NO_MESSAGE_RETRY_PARTIAL_SLEEP: Duration = Duration::from_millis(5); static NEXT_CONSUMER_INSTANCE_ID: AtomicUsize = AtomicUsize::new(1); fn map_prefix_scan_error(err: EtcdPrefixScanError) -> MpscError { @@ -96,6 +100,21 @@ fn merge_offset_cache_monotonic(current: &mut HashMap, fetched: Has } } +fn prefetch_refill_launch_budget(target: usize, current: usize) -> usize { + target + .saturating_sub(current) + .min(PREFETCH_REFILL_BURST_MAX) + .max(1) +} + +fn prefetch_no_message_retry_sleep(current: usize) -> Duration { + if current == 0 { + PREFETCH_NO_MESSAGE_RETRY_EMPTY_SLEEP + } else { + PREFETCH_NO_MESSAGE_RETRY_PARTIAL_SLEEP + } +} + fn prefetch_job_stage_name(stage: u8) -> &'static str { match stage { 0 => "init", @@ -296,9 +315,7 @@ impl CommitSequencer { let mut current_blocker_begin_at = wait_begin; loop { if shutdown.is_closed() { - return Err(MpscError::Internal( - "consumer closed during consume-offset commit wait".to_string(), - )); + return Err(MpscError::Closed); } let observed_next_seq = self.next_seq.load(Ordering::SeqCst); if observed_next_seq == seq { @@ -366,9 +383,7 @@ impl CommitSequencer { ); } _ = shutdown.wait_closed() => { - return Err(MpscError::Internal( - "consumer closed during consume-offset commit wait".to_string(), - )); + return Err(MpscError::Closed); } } } @@ -759,9 +774,16 @@ struct ReadyPathLatencySample { } /// Application-level payload (type-erased) to avoid coupling with upper layers. -pub trait MqPayload: Downcast + Send {} +pub trait MqPayload: Downcast + Send { + fn attach_cleanup(&mut self, cleanup: PayloadCleanup) -> Result<(), PayloadCleanup> { + Err(cleanup) + } +} impl_downcast!(MqPayload); +pub type PayloadCleanupFuture = Pin + Send + 'static>>; +pub type PayloadCleanup = Box PayloadCleanupFuture + Send + 'static>; + /// Callback result: deliver a payload or indicate retry/non-retry. pub enum PayloadResult { Ok(Box), @@ -813,10 +835,12 @@ pub struct MpscConsumer { /// /// 队列元素是一次完整 get 操作的 JoinHandle；consumer /// 只需 pop 并等待其完成即可，保证按提交顺序消费。 - inflight_rx: mpsc::Receiver, + inflight_queue: Arc>>, inflight_consume_notify: Arc, /// 控制通道，仅用于下发回调设置等控制类命令。 cmd_tx: mpsc::Sender, + /// Local mirror of payload callback for non-prefetch direct paths. + payload_cb: Option, /// delete callback invoked after successful consume-offset commit. delete_cb: Option, /// Shared shutdown controller used by higher layers to signal @@ -1242,10 +1266,13 @@ impl MpscConsumer { } async fn recv_next_inflight_handle_with_idle_warn(&mut self) -> Option { - match self.inflight_rx.try_recv() { - Ok(handle) => return Some(handle), - Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => return None, - Err(tokio::sync::mpsc::error::TryRecvError::Empty) => {} + if let Some(handle) = self + .inflight_queue + .lock() + .expect("inflight queue mutex poisoned") + .pop_front() + { + return Some(handle); } let idle_warn_sleep = tokio::time::sleep(NO_MESSAGE_WARN_INTERVAL); @@ -1255,10 +1282,19 @@ impl MpscConsumer { if self.shutdown.is_closed() { return None; } + let queue_notify = self.inflight_consume_notify.notified(); + tokio::pin!(queue_notify); tokio::select! { biased; - handle_opt = self.inflight_rx.recv() => { - return handle_opt; + _ = &mut queue_notify => { + if let Some(handle) = self + .inflight_queue + .lock() + .expect("inflight queue mutex poisoned") + .pop_front() + { + return Some(handle); + } } _ = &mut idle_warn_sleep => { let parent_mpmc_id = match self.category { @@ -1399,7 +1435,7 @@ impl MpscConsumer { let global_lease_id = chan_mgr.global_lease.id() as i64; let ( cmd_tx, - inflight_rx, + inflight_queue, target_inflight, inflight_queue_size, inflight_consume_notify, @@ -1438,9 +1474,10 @@ impl MpscConsumer { chan_mgr, target_inflight, inflight_queue_size, - inflight_rx, + inflight_queue, cmd_tx, inflight_consume_notify, + payload_cb: None, delete_cb: None, shutdown, category, @@ -1480,6 +1517,10 @@ impl MpscConsumer { &self.consumer_idx } + pub fn channel_capacity(&self) -> i64 { + self.chan_mgr.capacity() + } + pub fn lease_manager(&self) -> &LeaseManager { &self.lease_manager } @@ -1570,6 +1611,7 @@ impl MpscConsumer { /// This method is synchronous and only pushes a control command to the /// internal actor via `try_send`. pub fn set_payload_callback(&mut self, cb: PayloadCallback) { + self.payload_cb = Some(cb.clone()); let _ = self.cmd_tx.try_send(ConsumerCmd::SetCallback(cb)); } @@ -1619,8 +1661,7 @@ impl MpscConsumer { } else { self.recv_next_inflight_handle_with_idle_warn().await }; - let inflight_item = - handle_opt.ok_or_else(|| MpscError::Internal("prefetch actor closed".to_string()))?; + let inflight_item = handle_opt.ok_or(MpscError::Closed)?; debug!( "[MpscConsumer get_with_payload] instance_id={} chan_id={} seq={} producer_id={} consume_offset={} inflight_queue_size_after_pop={}", self.instance_id, @@ -1893,6 +1934,48 @@ impl MpscConsumer { .await } + pub async fn get_with_payload_via_broker( + &mut self, + broker: &BrokerHandle, + ) -> Result { + let cb = self + .payload_cb + .as_ref() + .ok_or_else(|| MpscError::Internal("payload callback not set".to_string()))? + .clone(); + get_payload_via_broker( + broker, + self.chan_id, + self.consumer_idx.clone(), + cb, + self.delete_cb.clone(), + self.shutdown.clone(), + ) + .await + } + + pub async fn get_batch_with_payload_via_broker( + &mut self, + broker: &BrokerHandle, + batch_size: usize, + ) -> Result, MpscError> { + let cb = self + .payload_cb + .as_ref() + .ok_or_else(|| MpscError::Internal("payload callback not set".to_string()))? + .clone(); + get_payload_batch_via_broker( + broker, + self.chan_id, + self.consumer_idx.clone(), + batch_size, + cb, + self.delete_cb.clone(), + self.shutdown.clone(), + ) + .await + } + /// Runs the KV payload fetch stage with retry semantics. /// Consume-offset commit is handled by the prefetch job. async fn run_single_get( @@ -1909,9 +1992,7 @@ impl MpscConsumer { let mut payload_obj: Option> = None; loop { if shutdown.is_closed() { - return Err(MpscError::Internal( - "consumer closed during get_with_payload".to_string(), - )); + return Err(MpscError::Closed); } let msg_key = keys::backend_message_key_with_category( chan_id, @@ -1978,10 +2059,7 @@ impl MpscConsumer { loop { if shutdown.is_closed() { - return Err(MpscError::Internal(format!( - "consumer closed during consume-offset commit: seq={} producer_id={} consume_offset={}", - seq, producer_id, consume_offset - ))); + return Err(MpscError::Closed); } attempts += 1; @@ -1996,10 +2074,7 @@ impl MpscConsumer { let put_res = tokio::select! { biased; _ = shutdown.wait_closed() => { - return Err(MpscError::Internal(format!( - "consumer closed during consume-offset commit: seq={} producer_id={} consume_offset={}", - seq, producer_id, consume_offset - ))); + return Err(MpscError::Closed); } res = tokio::time::timeout( COMMIT_OFFSET_PUT_TIMEOUT, @@ -2073,10 +2148,7 @@ impl MpscConsumer { tokio::select! { biased; _ = shutdown.wait_closed() => { - return Err(MpscError::Internal(format!( - "consumer closed during consume-offset retry sleep: seq={} producer_id={} consume_offset={}", - seq, producer_id, consume_offset - ))); + return Err(MpscError::Closed); } _ = sleep(COMMIT_OFFSET_RETRY_SLEEP) => {} } @@ -2176,6 +2248,586 @@ impl MpscConsumer { } } +async fn get_payload_via_broker( + broker: &BrokerHandle, + chan_id: i64, + consumer_id: String, + cb: PayloadCallback, + delete_cb: Option, + shutdown: ShutdownCtl, +) -> Result { + let fetched = broker + .fetch_next(BrokerFetchRequest { + channel_id: chan_id, + consumer_id: consumer_id.clone(), + now_ms: now_ms(), + }) + .await + .map_err(|e| { + MpscError::Internal(format!( + "broker fetch failed: chan_id={} consumer_id={} err={}", + chan_id, consumer_id, e + )) + })? + .ok_or(MpscError::NoMessage)?; + let envelope = fetched.envelope; + let reservation_id = envelope.reservation_id; + let producer_id = envelope.producer_id.clone(); + let payload_key = envelope.payload_key.clone(); + let mut requeue_guard = + BrokerInflightRequeueGuard::new(broker.clone(), chan_id, vec![reservation_id]); + let mut payload = match run_payload_callback( + chan_id, + cb, + producer_id.clone(), + payload_key, + shutdown.clone(), + ) + .await + { + Ok((payload, _kv_get_latency_ns)) => payload, + Err(err) => { + requeue_guard.requeue_now().await; + return Err(err); + } + }; + + let commit_outcome = match broker.commit(chan_id, reservation_id, now_ms()).await { + Ok(outcome) => outcome, + Err(err) => { + requeue_guard.requeue_now().await; + return Err(MpscError::Internal(format!( + "broker commit failed: chan_id={} consumer_id={} reservation_id={} err={}", + chan_id, consumer_id, reservation_id, err + ))); + } + }; + requeue_guard.mark_completed(reservation_id); + if !commit_outcome.first_commit { + return Err(MpscError::Internal(format!( + "broker commit returned duplicate first_commit=false: chan_id={} consumer_id={} reservation_id={}", + chan_id, consumer_id, reservation_id + ))); + } + + if let Some(envelope) = commit_outcome.cleanup { + attach_or_run_broker_cleanup( + payload.as_mut(), + broker.clone(), + chan_id, + delete_cb.clone(), + shutdown.clone(), + envelope, + ) + .await?; + } + + Ok(ConsumedPayload { + producer_id, + payload, + nonblocking_hit: true, + }) +} + +struct BrokerBatchPayload { + producer_id: String, + payload: Box, +} + +struct BrokerInflightRequeueGuard { + broker: BrokerHandle, + chan_id: i64, + reservation_ids: Vec, +} + +impl BrokerInflightRequeueGuard { + fn new(broker: BrokerHandle, chan_id: i64, reservation_ids: Vec) -> Self { + Self { + broker, + chan_id, + reservation_ids, + } + } + + fn extend(&mut self, reservation_ids: I) + where + I: IntoIterator, + { + self.reservation_ids.extend(reservation_ids); + } + + fn mark_completed(&mut self, reservation_id: u64) { + if let Some(pos) = self + .reservation_ids + .iter() + .position(|current| *current == reservation_id) + { + self.reservation_ids.remove(pos); + } + } + + async fn requeue_now(&mut self) { + let reservation_ids = std::mem::take(&mut self.reservation_ids); + requeue_pending_broker_inflight(&self.broker, self.chan_id, reservation_ids).await; + } +} + +impl Drop for BrokerInflightRequeueGuard { + fn drop(&mut self) { + let reservation_ids = std::mem::take(&mut self.reservation_ids); + if reservation_ids.is_empty() { + return; + } + let broker = self.broker.clone(); + let chan_id = self.chan_id; + tokio::spawn(async move { + requeue_pending_broker_inflight(&broker, chan_id, reservation_ids).await; + }); + } +} + +async fn get_payload_batch_via_broker( + broker: &BrokerHandle, + chan_id: i64, + consumer_id: String, + batch_size: usize, + cb: PayloadCallback, + delete_cb: Option, + shutdown: ShutdownCtl, +) -> Result, MpscError> { + if batch_size == 0 { + return Ok(Vec::new()); + } + + let first = broker + .fetch_next(BrokerFetchRequest { + channel_id: chan_id, + consumer_id: consumer_id.clone(), + now_ms: now_ms(), + }) + .await + .map_err(|e| { + MpscError::Internal(format!( + "broker fetch failed: chan_id={} consumer_id={} err={}", + chan_id, consumer_id, e + )) + })? + .ok_or(MpscError::NoMessage)?; + + let mut fetched = Vec::with_capacity(batch_size); + let mut requeue_guard = BrokerInflightRequeueGuard::new( + broker.clone(), + chan_id, + vec![first.envelope.reservation_id], + ); + fetched.push(first); + + let remaining = batch_size.saturating_sub(1); + if remaining > 0 { + let mut more = match broker + .fetch_batch_available( + BrokerFetchRequest { + channel_id: chan_id, + consumer_id: consumer_id.clone(), + now_ms: now_ms(), + }, + remaining, + ) + .await + { + Ok(batch) => { + requeue_guard.extend( + batch + .messages + .iter() + .map(|message| message.envelope.reservation_id), + ); + batch.messages + } + Err(err) => { + requeue_guard.requeue_now().await; + return Err(MpscError::Internal(format!( + "broker batch fetch failed: chan_id={} consumer_id={} err={}", + chan_id, consumer_id, err + ))); + } + }; + fetched.append(&mut more); + } + + match load_broker_payloads_commit_on_ready( + broker, + chan_id, + &consumer_id, + fetched, + cb, + delete_cb, + shutdown.clone(), + requeue_guard, + ) + .await + { + Ok(payloads) => Ok(payloads + .into_iter() + .map(|item| ConsumedPayload { + producer_id: item.producer_id, + payload: item.payload, + nonblocking_hit: true, + }) + .collect()), + Err(err) => Err(err), + } +} + +async fn load_broker_payloads_commit_on_ready( + broker: &BrokerHandle, + chan_id: i64, + consumer_id: &str, + fetched: Vec, + cb: PayloadCallback, + delete_cb: Option, + shutdown: ShutdownCtl, + mut requeue_guard: BrokerInflightRequeueGuard, +) -> Result, MpscError> { + let reservation_ids: Vec = fetched + .iter() + .map(|message| message.envelope.reservation_id) + .collect(); + let mut join_set = JoinSet::new(); + + for message in fetched { + let envelope = message.envelope; + let reservation_id = envelope.reservation_id; + let producer_id = envelope.producer_id.clone(); + let payload_key = envelope.payload_key.clone(); + let cb = cb.clone(); + let shutdown = shutdown.clone(); + join_set.spawn(async move { + let result = + run_payload_callback(chan_id, cb, producer_id.clone(), payload_key, shutdown) + .await + .map(|(payload, _kv_get_latency_ns)| BrokerBatchPayload { + producer_id, + payload, + }); + (reservation_id, result) + }); + } + + let mut payload_results: HashMap> = + HashMap::with_capacity(reservation_ids.len()); + let mut batch_load_failure: Option = None; + while let Some(join_res) = join_set.join_next().await { + match join_res { + Ok((reservation_id, Ok(payload))) => { + payload_results.insert(reservation_id, Ok(payload)); + } + Ok((reservation_id, Err(err))) => { + payload_results.insert(reservation_id, Err(err)); + join_set.abort_all(); + break; + } + Err(err) => { + join_set.abort_all(); + batch_load_failure = Some(MpscError::JoinError(err)); + break; + } + } + } + + let mut committed_payloads = Vec::with_capacity(reservation_ids.len()); + let mut remaining_reservation_ids = Vec::new(); + let mut stop_error = batch_load_failure; + let mut stop_after_current = stop_error.is_some(); + + for reservation_id in reservation_ids { + if stop_after_current { + remaining_reservation_ids.push(reservation_id); + continue; + } + + let Some(payload_result) = payload_results.remove(&reservation_id) else { + stop_error = Some(MpscError::Internal(format!( + "broker batch payload load canceled before ordered commit: chan_id={} consumer_id={} reservation_id={}", + chan_id, consumer_id, reservation_id + ))); + stop_after_current = true; + remaining_reservation_ids.push(reservation_id); + continue; + }; + + let mut payload = match payload_result { + Ok(payload) => payload, + Err(err) => { + stop_error = Some(err); + stop_after_current = true; + remaining_reservation_ids.push(reservation_id); + continue; + } + }; + + let commit_outcome = match broker.commit(chan_id, reservation_id, now_ms()).await { + Ok(outcome) => outcome, + Err(err) => { + stop_error = Some(MpscError::Internal(format!( + "broker commit failed during batch consume: chan_id={} consumer_id={} reservation_id={} err={}", + chan_id, consumer_id, reservation_id, err + ))); + stop_after_current = true; + remaining_reservation_ids.push(reservation_id); + continue; + } + }; + requeue_guard.mark_completed(reservation_id); + if !commit_outcome.first_commit { + stop_error = Some(MpscError::Internal(format!( + "broker commit returned duplicate during batch consume: chan_id={} consumer_id={} reservation_id={}", + chan_id, consumer_id, reservation_id + ))); + stop_after_current = true; + remaining_reservation_ids.push(reservation_id); + continue; + } + if let Some(envelope) = commit_outcome.cleanup { + if let Err(err) = attach_or_run_broker_cleanup( + payload.payload.as_mut(), + broker.clone(), + chan_id, + delete_cb.clone(), + shutdown.clone(), + envelope, + ) + .await + { + warn!( + "broker cleanup failed during batch consume: chan_id={} consumer_id={} reservation_id={} err={}", + chan_id, consumer_id, reservation_id, err + ); + committed_payloads.push(payload); + stop_error = Some(err); + stop_after_current = true; + continue; + } + } + + committed_payloads.push(payload); + } + + if !remaining_reservation_ids.is_empty() { + requeue_guard.requeue_now().await; + } + + if !committed_payloads.is_empty() { + return Ok(committed_payloads); + } + + Err(stop_error.unwrap_or_else(|| { + MpscError::Internal(format!( + "broker batch consume stopped without committed payloads: chan_id={} consumer_id={}", + chan_id, consumer_id + )) + })) +} + +async fn run_payload_callback( + chan_id: i64, + cb: PayloadCallback, + producer_id: String, + payload_key: String, + shutdown: ShutdownCtl, +) -> Result<(Box, u128), MpscError> { + use tokio::time::sleep; + + let kv_get_begin = Instant::now(); + loop { + if shutdown.is_closed() { + return Err(MpscError::Closed); + } + let f = cb.clone(); + let producer_for_closure = producer_id.clone(); + let key_for_closure = payload_key.clone(); + let res = (f)(producer_for_closure, key_for_closure).await; + + match res { + PayloadResult::Ok(payload) => { + return Ok((payload, kv_get_begin.elapsed().as_nanos())); + } + PayloadResult::Retryable(msg) => { + warn!( + "[MpscConsumer chan_id={}] get payload retryable: {}", + chan_id, msg + ); + sleep(Duration::from_millis(50)).await; + } + PayloadResult::NonRetryable(msg) => { + return Err(MpscError::GetPayloadNonRetryable { message: msg }); + } + } + } +} + +async fn run_delete_callback( + chan_id: i64, + delete_cb: &DeleteCallback, + payload_key: String, + shutdown: &ShutdownCtl, +) -> Result<(), MpscError> { + use tokio::time::sleep; + + loop { + if shutdown.is_closed() { + return Ok(()); + } + let f = delete_cb.clone(); + let key_clone = payload_key.clone(); + let delete_begin = Instant::now(); + let delete_fut = (f)(key_clone.clone()); + tokio::pin!(delete_fut); + let res = loop { + tokio::select! { + biased; + _ = shutdown.wait_closed() => { + debug!( + "[MpscConsumer chan_id={}] stop delete callback on shutdown: key={}", + chan_id, + key_clone, + ); + break DeleteResult::Ok; + } + res = &mut delete_fut => { + break res; + } + _ = sleep(DELETE_CALLBACK_WARN_INTERVAL) => { + warn!( + "[MpscConsumer chan_id={}] delete callback still pending: key={} waited_ms={}", + chan_id, + key_clone, + delete_begin.elapsed().as_millis(), + ); + } + } + }; + match res { + DeleteResult::Ok => return Ok(()), + DeleteResult::Retryable(msg) => { + warn!( + "[MpscConsumer chan_id={}] delete payload retryable: {}", + chan_id, msg + ); + sleep(Duration::from_millis(50)).await; + } + DeleteResult::NonRetryable(msg) => { + return Err(MpscError::DeletePayloadNonRetryable { message: msg }); + } + } + } +} + +async fn cleanup_broker_envelope( + broker: &BrokerHandle, + chan_id: i64, + delete_cb: Option<&DeleteCallback>, + shutdown: &ShutdownCtl, + envelope: BrokerEnvelope, +) -> Result<(), MpscError> { + let reservation_id = envelope.reservation_id; + if let Some(delete_cb) = delete_cb { + run_delete_callback(chan_id, delete_cb, envelope.payload_key, shutdown).await?; + } + broker + .cleanup_ack(chan_id, reservation_id) + .await + .map_err(|e| { + MpscError::Internal(format!( + "broker cleanup ack failed: chan_id={} reservation_id={} err={}", + chan_id, reservation_id, e + )) + })?; + Ok(()) +} + +async fn attach_or_run_broker_cleanup( + payload: &mut dyn MqPayload, + broker: BrokerHandle, + chan_id: i64, + delete_cb: Option, + shutdown: ShutdownCtl, + envelope: BrokerEnvelope, +) -> Result<(), MpscError> { + let cleanup_envelope = envelope.clone(); + let deferred_broker = broker.clone(); + let deferred_delete_cb = delete_cb.clone(); + let deferred_shutdown = shutdown.clone(); + let cleanup = Box::new(move || { + Box::pin(async move { + if let Some(delete_cb) = deferred_delete_cb.as_ref() { + if let Err(err) = run_delete_callback( + chan_id, + delete_cb, + cleanup_envelope.payload_key.clone(), + &deferred_shutdown, + ) + .await + { + warn!( + "deferred broker payload delete failed: chan_id={} reservation_id={} err={}", + chan_id, cleanup_envelope.reservation_id, err + ); + let _ = deferred_broker + .cleanup_nack(chan_id, cleanup_envelope.reservation_id) + .await; + return; + } + } + if let Err(err) = deferred_broker + .cleanup_ack(chan_id, cleanup_envelope.reservation_id) + .await + { + warn!( + "deferred broker cleanup ack failed: chan_id={} reservation_id={} err={}", + chan_id, cleanup_envelope.reservation_id, err + ); + } + }) as PayloadCleanupFuture + }); + match payload.attach_cleanup(cleanup) { + Ok(()) => Ok(()), + Err(_) => { + cleanup_broker_envelope(&broker, chan_id, delete_cb.as_ref(), &shutdown, envelope).await + } + } +} + +async fn requeue_pending_broker_inflight( + broker: &BrokerHandle, + chan_id: i64, + reservation_ids: Vec, +) { + for reservation_id in reservation_ids.into_iter().rev() { + requeue_broker_inflight_best_effort(broker, chan_id, reservation_id).await; + } +} + +async fn requeue_broker_inflight_best_effort( + broker: &BrokerHandle, + chan_id: i64, + reservation_id: u64, +) { + if let Err(err) = broker.requeue_inflight(chan_id, reservation_id).await { + warn!( + "best-effort broker requeue failed: chan_id={} reservation_id={} err={}", + chan_id, reservation_id, err + ); + } +} + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock is before UNIX_EPOCH") + .as_millis() as i64 +} + /// MPSC consumer actor，持有 selector、offset、lease 等完整状态。 /// 仅在 mpsc 模块内部可见，对上层 crate 透明。 pub struct ConsumedPayload { @@ -2454,9 +3106,10 @@ struct ConsumerActor { producer_selector: ProducerSelectorForConsumer, /// payload 回调，由上层通过 ConsumerCmd::SetCallback 设置. payload_cb: Option, - /// 每个 producer 当前已预取但尚未持久化消费的“下一条 offset” - /// 提示，用于避免在 etcd consume offset 尚未更新时重复预取 - /// 同一条消息。 + /// 每个 producer 的本地 reservation cursor（下一条待预取 offset）。 + /// + /// 这个 cursor 可能领先于 etcd consume offset，因为 actor 会在 + /// consume-offset 持久化之前先连续发起多条 prefetch。 prefetch_offset_map: HashMap, /// 本地缓存的 produce offset（来自 etcd），仅在无消息或 /// 初始化时 refresh；平时 select_next_message 只读该缓存。 @@ -2479,7 +3132,7 @@ struct ConsumerActor { /// 向 consumer 暴露的预取队列 sender。 /// /// 队列元素为一次完整 get 操作的 JoinHandle。 - inflight_tx: mpsc::Sender, + inflight_queue: Arc>>, /// inflight consume notify inflight_consume_notify: Arc, /// 共享的预取窗口目标。 @@ -2597,10 +3250,12 @@ impl ConsumerActor { } fn cached_next_hint(&self, producer_id: &str) -> i64 { + let committed_next = self.cached_consume_offset(producer_id); self.prefetch_offset_map .get(producer_id) .copied() - .unwrap_or_else(|| self.cached_consume_offset(producer_id)) + .map(|hint| hint.max(committed_next)) + .unwrap_or(committed_next) } fn cached_produce_offset(&self, producer_id: &str) -> i64 { @@ -2616,6 +3271,12 @@ impl ConsumerActor { || self.prefetch_offset_map.contains_key(producer_id) } + fn producer_has_prefetch_room(&self, producer_id: &str) -> bool { + let visible_tail = self.cached_produce_offset(producer_id); + let next_hint = self.cached_next_hint(producer_id); + next_hint <= visible_tail + } + fn refresh_ready_state_from_local(&mut self, producer_id: &str) -> bool { let ready_before = self.ready_producers.contains(producer_id); let stale_before = self.stale_no_room_producers.contains(producer_id); @@ -2626,8 +3287,7 @@ impl ConsumerActor { return ready_before || stale_before; } - let has_room = - self.cached_produce_offset(producer_id) >= self.cached_next_hint(producer_id); + let has_room = self.producer_has_prefetch_room(producer_id); if has_room { self.ready_producers.insert(producer_id.to_string()); self.stale_no_room_producers.remove(producer_id); @@ -2878,7 +3538,7 @@ impl ConsumerActor { global_lease_id: i64, ) -> ( mpsc::Sender, - mpsc::Receiver, + Arc>>, Arc, Arc, Arc, @@ -2889,7 +3549,7 @@ impl ConsumerActor { let (cmd_tx, cmd_rx) = mpsc::channel(8); let (meta_tx, meta_rx) = mpsc::channel(8); let (produce_offset_tx, produce_offset_rx) = mpsc::channel(128); - let (inflight_tx, inflight_rx) = mpsc::channel(32); + let inflight_queue = Arc::new(Mutex::new(VecDeque::new())); let target_inflight = Arc::new(AtomicUsize::new(0)); let inflight_queue_size = Arc::new(AtomicUsize::new(0)); let inflight_consume_notify = Arc::new(Notify::new()); @@ -2911,7 +3571,7 @@ impl ConsumerActor { ready_producers: HashSet::new(), ready_trace_history: HashMap::new(), stale_no_room_producers: HashSet::new(), - inflight_tx, + inflight_queue: inflight_queue.clone(), inflight_consume_notify: inflight_consume_notify.clone(), target_inflight: target_inflight.clone(), inflight_queue_size: inflight_queue_size.clone(), @@ -2960,7 +3620,7 @@ impl ConsumerActor { ( cmd_tx, - inflight_rx, + inflight_queue, target_inflight, inflight_queue_size, inflight_consume_notify, @@ -3118,7 +3778,7 @@ impl ConsumerActor { } // Do not poll `prefetch_tick()` as a `tokio::select!` branch. If the - // branch is canceled while `inflight_tx.send(...)` is pending, the + // branch is canceled while queueing a new inflight item is pending, the // oneshot receiver inside `InflightItem` is dropped after the // prefetch job has already started, which strands commit ordering. self.drain_pending_actor_inputs(&mut rx, &mut meta_rx, &mut produce_offset_rx); @@ -3163,22 +3823,35 @@ impl ConsumerActor { return; } - for _ in 0..1 { + let initial_queue_size = self.inflight_queue_size.load(Ordering::SeqCst); + let burst_limit = prefetch_refill_launch_budget(target, initial_queue_size); + let mut launched = 0usize; + loop { let current = self.inflight_queue_size.load(Ordering::SeqCst); if current >= target { - self.wait_actor_inputs_or_inflight_consume(rx, meta_rx, produce_offset_rx) - .await; + if launched == 0 { + self.wait_actor_inputs_or_inflight_consume(rx, meta_rx, produce_offset_rx) + .await; + } + return; + } + if launched >= burst_limit { return; } match self.try_prefetch_one().await { Ok(()) => { + launched += 1; self.prefetch_no_message_next_warn_at = tokio::time::Instant::now() + NO_MESSAGE_WARN_INTERVAL; self.maybe_log_select_next_message_stats(false); } Err(MpscError::NoMessage) => { self.select_next_message_stats.record_no_message_backoff(); + if launched > 0 { + self.maybe_log_select_next_message_stats(false); + return; + } let now = tokio::time::Instant::now(); if now >= self.prefetch_no_message_next_warn_at { let parent_mpmc_id = match self.category { @@ -3195,7 +3868,13 @@ impl ConsumerActor { self.prefetch_no_message_next_warn_at = now + NO_MESSAGE_WARN_INTERVAL; } self.maybe_log_select_next_message_stats(false); - self.wait_actor_inputs(rx, meta_rx, produce_offset_rx).await; + self.wait_actor_inputs_or_timeout( + rx, + meta_rx, + produce_offset_rx, + prefetch_no_message_retry_sleep(current), + ) + .await; return; } Err(other) => { @@ -3213,6 +3892,7 @@ impl ConsumerActor { Duration::from_millis(100), ) .await; + return; } } } @@ -3223,7 +3903,7 @@ impl ConsumerActor { /// 返回 `MpscError::NoMessage`。 async fn try_prefetch_one(&mut self) -> Result<(), MpscError> { if self.shutdown.is_closed() { - return Err(MpscError::Internal("consumer closed".to_string())); + return Err(MpscError::Closed); } let cb = self .payload_cb @@ -3305,16 +3985,17 @@ impl ConsumerActor { queue_size_after_inc, self.target_inflight.load(Ordering::SeqCst), ); - self.inflight_tx - .send(InflightItem { + self.inflight_queue + .lock() + .expect("inflight queue mutex poisoned") + .push_back(InflightItem { seq, producer_id: producer_id_for_queue, consume_offset, ready_path_trace, rx, - }) - .await - .map_err(|_| MpscError::Internal("prefetch queue closed".to_string()))?; + }); + self.inflight_consume_notify.notify_one(); debug!( "[MpscConsumer enqueue] instance_id={} chan_id={} seq={} queue_send_completed queue_size_now={}", self.instance_id, @@ -3445,31 +4126,66 @@ impl ConsumerActor { return Err(MpscError::NoMessage); } - self.producer_selector.moveon_round_robin(); - let producer_id = self - .producer_selector - .current_producer_idx() - .ok_or(MpscError::NoMessage)? - .to_string(); + let ready_count = self.ready_producers.len(); + for _ in 0..ready_count { + self.producer_selector.moveon_round_robin(); + let producer_id = self + .producer_selector + .current_producer_idx() + .ok_or(MpscError::NoMessage)? + .to_string(); - let prod_off = self.cached_produce_offset(&producer_id); - let next_hint = self.cached_next_hint(&producer_id); + let next_hint = self.cached_next_hint(&producer_id); - if prod_off < next_hint { + if !self.producer_has_prefetch_room(&producer_id) { + if self.refresh_ready_state_from_local(&producer_id) { + self.rebuild_ready_selector(); + } + continue; + } + + let actual_offset = next_hint; + self.prefetch_offset_map + .insert(producer_id.clone(), actual_offset + 1); if self.refresh_ready_state_from_local(&producer_id) { self.rebuild_ready_selector(); } - return Err(MpscError::NoMessage); + + return Ok((producer_id, actual_offset)); } - let actual_offset = next_hint; - self.prefetch_offset_map - .insert(producer_id.clone(), actual_offset + 1); - if self.refresh_ready_state_from_local(&producer_id) { - self.rebuild_ready_selector(); + if !self.stale_no_room_producers.is_empty() { + self.probe_stale_no_room_producers_timed(trace).await?; + if !self.ready_producers.is_empty() { + let retry_ready_count = self.ready_producers.len(); + for _ in 0..retry_ready_count { + self.producer_selector.moveon_round_robin(); + let producer_id = self + .producer_selector + .current_producer_idx() + .ok_or(MpscError::NoMessage)? + .to_string(); + + let next_hint = self.cached_next_hint(&producer_id); + if !self.producer_has_prefetch_room(&producer_id) { + if self.refresh_ready_state_from_local(&producer_id) { + self.rebuild_ready_selector(); + } + continue; + } + + let actual_offset = next_hint; + self.prefetch_offset_map + .insert(producer_id.clone(), actual_offset + 1); + if self.refresh_ready_state_from_local(&producer_id) { + self.rebuild_ready_selector(); + } + return Ok((producer_id, actual_offset)); + } + } } - Ok((producer_id, actual_offset)) + Err(MpscError::NoMessage) } async fn refresh_offsets_from_etcd_timed( @@ -3623,8 +4339,22 @@ impl ConsumerActor { #[cfg(test)] mod tests { - use super::{merge_monotonic_offset, merge_offset_cache_monotonic}; + use super::{ + get_payload_batch_via_broker, get_payload_via_broker, merge_monotonic_offset, + merge_offset_cache_monotonic, MqPayload, PayloadCallback, PayloadResult, + }; + use crate::{ + keys::MqCategory, BrokerChannelConfig, BrokerFetchRequest, BrokerHandle, + BrokerReserveRequest, + }; use std::collections::HashMap; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Notify; + + struct TestPayload; + + impl MqPayload for TestPayload {} #[test] fn merge_monotonic_offset_keeps_cached_when_probe_missing() { @@ -3654,6 +4384,276 @@ mod tests { assert_eq!(current.get("producer_b"), Some(&41)); assert_eq!(current.get("producer_c"), Some(&7)); } + + #[test] + fn visible_tail_does_not_allow_prefetch_past_last_published_offset() { + let visible_tail = 0; + let next_visible = 0; + let next_not_yet_published = 1; + + assert!(next_visible <= visible_tail); + assert!(next_not_yet_published > visible_tail); + } + + async fn fetch_next_for_test( + broker: &BrokerHandle, + channel_id: i64, + consumer_id: &str, + now_ms: i64, + ) -> crate::BrokerFetchedMessage { + tokio::time::timeout( + Duration::from_secs(1), + broker.fetch_next(BrokerFetchRequest { + channel_id, + consumer_id: consumer_id.to_string(), + now_ms, + }), + ) + .await + .expect("timed out waiting for broker redelivery") + .unwrap() + .unwrap() + } + + #[tokio::test] + async fn broker_single_consume_timeout_requeues_reserved_message() { + let broker = BrokerHandle::new_local_for_test(32); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 72, + capacity: 2, + }) + .await + .unwrap(); + + let reserved = broker + .reserve(BrokerReserveRequest { + channel_id: 72, + producer_id: "p0".to_string(), + category: MqCategory::Mpsc, + payload_bytes: 1, + now_ms: 10, + }) + .await + .unwrap(); + broker + .publish(72, reserved.envelope.reservation_id, 20) + .await + .unwrap(); + + let callback_started = Arc::new(Notify::new()); + let cb_started_for_callback = callback_started.clone(); + let cb: PayloadCallback = Arc::new(move |_producer_id: String, _key: String| { + let cb_started_for_callback = cb_started_for_callback.clone(); + Box::pin(async move { + cb_started_for_callback.notify_one(); + tokio::time::sleep(Duration::from_millis(50)).await; + PayloadResult::Ok(Box::new(TestPayload)) + }) + }); + + let mut consume = Box::pin(get_payload_via_broker( + &broker, + 72, + "c0".to_string(), + cb, + None, + crate::ShutdownCtl::new(), + )); + tokio::select! { + _ = callback_started.notified() => {} + result = &mut consume => panic!("consume completed before timeout setup: {:?}", result.err()), + } + assert!(tokio::time::timeout(Duration::from_millis(5), &mut consume) + .await + .is_err()); + drop(consume); + + let redelivered = fetch_next_for_test(&broker, 72, "c1", 30).await; + assert_eq!( + redelivered.envelope.reservation_id, + reserved.envelope.reservation_id + ); + } + + #[tokio::test] + async fn broker_batch_consume_timeout_requeues_reserved_messages_in_order() { + let broker = BrokerHandle::new_local_for_test(32); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 73, + capacity: 2, + }) + .await + .unwrap(); + + let first = broker + .reserve(BrokerReserveRequest { + channel_id: 73, + producer_id: "p0".to_string(), + category: MqCategory::Mpsc, + payload_bytes: 1, + now_ms: 10, + }) + .await + .unwrap(); + let second = broker + .reserve(BrokerReserveRequest { + channel_id: 73, + producer_id: "p0".to_string(), + category: MqCategory::Mpsc, + payload_bytes: 1, + now_ms: 11, + }) + .await + .unwrap(); + broker + .publish(73, first.envelope.reservation_id, 20) + .await + .unwrap(); + broker + .publish(73, second.envelope.reservation_id, 21) + .await + .unwrap(); + + let callback_started = Arc::new(Notify::new()); + let cb_started_for_callback = callback_started.clone(); + let cb: PayloadCallback = Arc::new(move |_producer_id: String, _key: String| { + let cb_started_for_callback = cb_started_for_callback.clone(); + Box::pin(async move { + cb_started_for_callback.notify_one(); + tokio::time::sleep(Duration::from_millis(50)).await; + PayloadResult::Ok(Box::new(TestPayload)) + }) + }); + + let mut consume = Box::pin(get_payload_batch_via_broker( + &broker, + 73, + "c0".to_string(), + 2, + cb, + None, + crate::ShutdownCtl::new(), + )); + tokio::select! { + _ = callback_started.notified() => {} + result = &mut consume => panic!("batch consume completed before timeout setup: {:?}", result.err()), + } + assert!(tokio::time::timeout(Duration::from_millis(5), &mut consume) + .await + .is_err()); + drop(consume); + + let redelivered_first = fetch_next_for_test(&broker, 73, "c1", 30).await; + let redelivered_second = fetch_next_for_test(&broker, 73, "c1", 31).await; + assert_eq!( + redelivered_first.envelope.reservation_id, + first.envelope.reservation_id + ); + assert_eq!( + redelivered_second.envelope.reservation_id, + second.envelope.reservation_id + ); + } + + #[tokio::test] + async fn broker_batch_consume_requeues_without_out_of_order_commit() { + let broker = BrokerHandle::new_local_for_test(32); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 71, + capacity: 2, + }) + .await + .unwrap(); + + let first = broker + .reserve(BrokerReserveRequest { + channel_id: 71, + producer_id: "p0".to_string(), + category: MqCategory::Mpsc, + payload_bytes: 1, + now_ms: 10, + }) + .await + .unwrap(); + let second = broker + .reserve(BrokerReserveRequest { + channel_id: 71, + producer_id: "p0".to_string(), + category: MqCategory::Mpsc, + payload_bytes: 1, + now_ms: 11, + }) + .await + .unwrap(); + broker + .publish(71, first.envelope.reservation_id, 20) + .await + .unwrap(); + broker + .publish(71, second.envelope.reservation_id, 21) + .await + .unwrap(); + + let first_key = first.envelope.payload_key.clone(); + let cb: PayloadCallback = Arc::new(move |_producer_id: String, key: String| { + let first_key = first_key.clone(); + Box::pin(async move { + if key == first_key { + tokio::time::sleep(Duration::from_millis(50)).await; + PayloadResult::NonRetryable("first payload failed".to_string()) + } else { + PayloadResult::Ok(Box::new(TestPayload)) + } + }) + }); + + let err = get_payload_batch_via_broker( + &broker, + 71, + "c0".to_string(), + 2, + cb, + None, + crate::ShutdownCtl::new(), + ) + .await + .err() + .expect("batch consume should fail when the first payload callback fails"); + assert!(matches!( + err, + crate::MpscError::GetPayloadNonRetryable { .. } + )); + + let redelivered_first = broker + .fetch_next(crate::BrokerFetchRequest { + channel_id: 71, + consumer_id: "c1".to_string(), + now_ms: 30, + }) + .await + .unwrap() + .unwrap(); + let redelivered_second = broker + .fetch_next(crate::BrokerFetchRequest { + channel_id: 71, + consumer_id: "c1".to_string(), + now_ms: 31, + }) + .await + .unwrap() + .unwrap(); + assert_eq!( + redelivered_first.envelope.reservation_id, + first.envelope.reservation_id + ); + assert_eq!( + redelivered_second.envelope.reservation_id, + second.envelope.reservation_id + ); + } } /// Producer selector for consumer-side weighted round robin. diff --git a/fluxon_rs/fluxon_mq/src/create.rs b/fluxon_rs/fluxon_mq/src/create.rs index 4fbb753..79da7ce 100644 --- a/fluxon_rs/fluxon_mq/src/create.rs +++ b/fluxon_rs/fluxon_mq/src/create.rs @@ -311,6 +311,7 @@ pub async fn create_mpsc_channel( global_lease: global_lease_handle, global_long_lease: global_long_lease_handle, payload_lease: payload_lease_handle, + capacity: cfg.capacity, etcd_client, }) } @@ -534,6 +535,7 @@ impl ChanManager { global_lease, global_long_lease, payload_lease, + capacity: meta.capacity, etcd_client: client, }) } diff --git a/fluxon_rs/fluxon_mq/src/error.rs b/fluxon_rs/fluxon_mq/src/error.rs index b4f1171..9d25d39 100755 --- a/fluxon_rs/fluxon_mq/src/error.rs +++ b/fluxon_rs/fluxon_mq/src/error.rs @@ -12,12 +12,24 @@ pub enum MpscError { #[error("no new message available")] NoMessage, + #[error("consumer is closed")] + Closed, + #[error("etcd error: {0}")] Etcd(#[from] etcd_client::Error), #[error("spawn blocking task failed: {0}")] JoinError(#[from] tokio::task::JoinError), + #[error( + "message buffer full: channel_id={channel_id} capacity={capacity} used_slots={used_slots}" + )] + MessageBufferFull { + channel_id: i64, + capacity: i64, + used_slots: i64, + }, + #[error("put payload returned non-retryable error (code=2)")] PutPayloadNonRetryable, @@ -61,10 +73,12 @@ impl MpscError { match self { // 可重试类 MpscError::NoMessage => 1000, + MpscError::Closed => 1001, // etcd / 系统 MpscError::Etcd(_) => 2000, MpscError::JoinError(_) => 2001, + MpscError::MessageBufferFull { .. } => 2002, // put payload MpscError::PutPayloadNonRetryable => 3000, diff --git a/fluxon_rs/fluxon_mq/src/keys.rs b/fluxon_rs/fluxon_mq/src/keys.rs index 1d55754..e2c8a4e 100644 --- a/fluxon_rs/fluxon_mq/src/keys.rs +++ b/fluxon_rs/fluxon_mq/src/keys.rs @@ -1,9 +1,13 @@ use std::fmt::Write as _; +use bitcode::{Decode, Encode}; +use serde::{Deserialize, Serialize}; + /// MQ category for key generation. -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, Encode, Decode)] pub enum MqCategory { /// Standalone MPSC usage + #[default] Mpsc, /// MPSC acts as a submodule under an MPMC producer; carries parent mpmc id only. /// The producer member id is the same as `producer_idx` passed alongside and diff --git a/fluxon_rs/fluxon_mq/src/lib.rs b/fluxon_rs/fluxon_mq/src/lib.rs index 3dded48..70b2024 100644 --- a/fluxon_rs/fluxon_mq/src/lib.rs +++ b/fluxon_rs/fluxon_mq/src/lib.rs @@ -1,3 +1,4 @@ +pub mod broker; pub mod consumer; pub mod create; pub mod error; @@ -10,6 +11,7 @@ pub mod nonblocking_monitor; pub mod producer; pub mod shutdown; +pub use crate::broker::*; pub use crate::consumer::DeleteResult; pub use crate::consumer::MpscConsumer; pub use crate::create::{create_mpsc_channel, ChanCreateConfig}; diff --git a/fluxon_rs/fluxon_mq/src/manager.rs b/fluxon_rs/fluxon_mq/src/manager.rs index b6d581a..fb5ffdb 100644 --- a/fluxon_rs/fluxon_mq/src/manager.rs +++ b/fluxon_rs/fluxon_mq/src/manager.rs @@ -206,6 +206,7 @@ pub struct ChanManager { /// 决定好 payload lease id，并通过 LeaseManager 注册 /// 对应的 kvclient keepalive；此处始终持有一个有效句柄。 pub payload_lease: GeneralLease, + pub(crate) capacity: i64, pub(crate) etcd_client: etcd::Client, } @@ -227,4 +228,8 @@ impl ChanManager { pub fn member_lease_id(&self) -> i64 { self.member_lease.id() as i64 } + + pub fn capacity(&self) -> i64 { + self.capacity + } } diff --git a/fluxon_rs/fluxon_mq/src/producer.rs b/fluxon_rs/fluxon_mq/src/producer.rs index fb4e4ea..72082c2 100644 --- a/fluxon_rs/fluxon_mq/src/producer.rs +++ b/fluxon_rs/fluxon_mq/src/producer.rs @@ -28,10 +28,15 @@ use crate::nonblocking_monitor::{ }; use crate::shutdown::ShutdownCtl; use crate::LifecycleView; +use crate::{BrokerError, BrokerHandle, BrokerReserveRequest}; use tokio::sync::watch; use tracing::warn; const PRODUCE_OFFSET_ETCD_SLOW_WARN_THRESHOLD: Duration = Duration::from_secs(1); +const BROKER_BACKPRESSURE_INITIAL_SLEEP_MS: u64 = 2; +const BROKER_BACKPRESSURE_MAX_SLEEP_MS: u64 = 50; +const BROKER_BACKPRESSURE_JITTER_MS: u64 = 7; +const BROKER_BACKPRESSURE_WARN_INTERVAL: Duration = Duration::from_secs(5); #[derive(Debug, Clone, Serialize, Deserialize)] struct ProducerMemberMeta { @@ -266,6 +271,10 @@ impl MpscProducer { self.chan_mgr.payload_lease.id() as i64 } + pub fn channel_capacity(&self) -> i64 { + self.chan_mgr.capacity() + } + /// Shared shutdown controller for this producer instance. pub fn shutdown_ctl(&self) -> ShutdownCtl { self.shutdown.clone() @@ -420,9 +429,7 @@ impl MpscProducer { let put_payload = Arc::new(put_payload); loop { if self.shutdown.is_closed() { - return Err(MpscError::Internal( - "producer closed during put_with_payload".to_string(), - )); + return Err(MpscError::Closed); } let key_clone = msg_key.clone(); let f = put_payload.clone(); @@ -479,6 +486,243 @@ impl MpscProducer { } Ok(()) } + + /// Broker-backed put path. + /// + /// This keeps the existing payload callback contract but moves + /// message id allocation and publish visibility into the broker. + /// The current etcd-backed `put_with_payload` remains untouched + /// until call sites are switched to this path. + pub async fn put_with_payload_via_broker( + &mut self, + broker: &BrokerHandle, + payload_bytes: u64, + put_payload: F, + ) -> Result<(), MpscError> + where + F: Fn(String, i64, Option) -> i32 + Send + Sync + 'static, + { + let preferred_sub_cluster_for_call = self.preferred_sub_cluster_for_put()?; + let published_msg_id = put_payload_via_broker( + broker, + self.chan_id, + &self.producer_idx, + self.category, + payload_bytes, + self.shutdown.clone(), + preferred_sub_cluster_for_call, + put_payload, + ) + .await?; + self.next_msg_id = self.next_msg_id.max(published_msg_id + 1); + Ok(()) + } +} + +async fn put_payload_via_broker( + broker: &BrokerHandle, + chan_id: i64, + producer_idx: &str, + category: MqCategory, + payload_bytes: u64, + shutdown: ShutdownCtl, + preferred_sub_cluster_for_call: Option, + put_payload: F, +) -> Result +where + F: Fn(String, i64, Option) -> i32 + Send + Sync + 'static, +{ + use limit_thirdparty::tokio::task; + use tokio::time::sleep; + + let put_payload = Arc::new(put_payload); + let reserve_wait_begin = Instant::now(); + let mut reserve_retry_attempt: u32 = 0; + let mut payload_retry_attempt: u32 = 0; + let mut next_reserve_warn_at = Instant::now() + BROKER_BACKPRESSURE_WARN_INTERVAL; + let mut next_payload_warn_at = Instant::now() + BROKER_BACKPRESSURE_WARN_INTERVAL; + + loop { + if shutdown.is_closed() { + return Err(MpscError::Closed); + } + + let reservation = match broker + .reserve(BrokerReserveRequest { + channel_id: chan_id, + producer_id: producer_idx.to_string(), + category, + payload_bytes, + now_ms: broker_now_ms(), + }) + .await + { + Ok(reservation) => { + reserve_retry_attempt = 0; + reservation + } + Err(BrokerError::ChannelFull { + channel_id, + capacity, + used_slots, + }) => { + let now = Instant::now(); + if now >= next_reserve_warn_at { + warn!( + "broker reserve backpressured: chan_id={} producer_idx={} capacity={} used_slots={} waited_ms={}", + channel_id, + producer_idx, + capacity, + used_slots, + reserve_wait_begin.elapsed().as_millis(), + ); + next_reserve_warn_at = now + BROKER_BACKPRESSURE_WARN_INTERVAL; + } + let sleep_for = + broker_backpressure_sleep_duration(producer_idx, reserve_retry_attempt); + reserve_retry_attempt = reserve_retry_attempt.saturating_add(1); + sleep(sleep_for).await; + continue; + } + Err(BrokerError::PayloadBytesFull { + capacity_bytes, + used_bytes, + requested_bytes, + }) => { + let now = Instant::now(); + if now >= next_reserve_warn_at { + warn!( + "broker payload budget backpressured: chan_id={} producer_idx={} requested_bytes={} capacity_bytes={} used_bytes={} waited_ms={}", + chan_id, + producer_idx, + requested_bytes, + capacity_bytes, + used_bytes, + reserve_wait_begin.elapsed().as_millis(), + ); + next_reserve_warn_at = now + BROKER_BACKPRESSURE_WARN_INTERVAL; + } + let sleep_for = + broker_backpressure_sleep_duration(producer_idx, reserve_retry_attempt); + reserve_retry_attempt = reserve_retry_attempt.saturating_add(1); + sleep(sleep_for).await; + continue; + } + Err(BrokerError::PayloadTooLarge { + requested_bytes, + capacity_bytes, + }) => { + return Err(MpscError::Internal(format!( + "broker payload too large: chan_id={} producer_idx={} requested_bytes={} capacity_bytes={}", + chan_id, producer_idx, requested_bytes, capacity_bytes + ))); + } + Err(other) => { + return Err(MpscError::Internal(format!( + "broker reserve failed: chan_id={} producer_idx={} err={}", + chan_id, producer_idx, other + ))); + } + }; + let reservation_id = reservation.envelope.reservation_id; + let msg_id = reservation.envelope.msg_id; + let msg_key = reservation.envelope.payload_key.clone(); + + let key_clone = msg_key.clone(); + let f = put_payload.clone(); + let hint = preferred_sub_cluster_for_call.clone(); + let code = task::spawn_blocking(move || (f)(key_clone, msg_id, hint)) + .await + .map_err(|e| { + abort_on_payload_failure_async(broker.clone(), chan_id, reservation_id); + MpscError::JoinError(e) + })?; + + match code { + 0 => { + broker + .publish(chan_id, reservation_id, broker_now_ms()) + .await + .map_err(|e| { + MpscError::Internal(format!( + "broker publish failed after payload write: chan_id={} producer_idx={} reservation_id={} msg_id={} err={}", + chan_id, producer_idx, reservation_id, msg_id, e + )) + })?; + return Ok(msg_id); + } + 1 => { + abort_broker_reservation_best_effort(broker, chan_id, reservation_id).await; + let now = Instant::now(); + if now >= next_payload_warn_at { + warn!( + "broker payload write backpressured by owner pool: chan_id={} producer_idx={} waited_ms={}", + chan_id, + producer_idx, + reserve_wait_begin.elapsed().as_millis(), + ); + next_payload_warn_at = now + BROKER_BACKPRESSURE_WARN_INTERVAL; + } + let sleep_for = + broker_backpressure_sleep_duration(producer_idx, payload_retry_attempt); + payload_retry_attempt = payload_retry_attempt.saturating_add(1); + sleep(sleep_for).await; + continue; + } + 2 => { + abort_broker_reservation_best_effort(broker, chan_id, reservation_id).await; + return Err(MpscError::PutPayloadNonRetryable); + } + other => { + abort_broker_reservation_best_effort(broker, chan_id, reservation_id).await; + return Err(MpscError::PutPayloadUnknownCode { code: other }); + } + } + } +} + +fn broker_backpressure_sleep_duration(producer_idx: &str, retry_attempt: u32) -> Duration { + let shift = retry_attempt.min(6); + let base_ms = BROKER_BACKPRESSURE_INITIAL_SLEEP_MS + .saturating_mul(1_u64 << shift) + .min(BROKER_BACKPRESSURE_MAX_SLEEP_MS); + let jitter_ms = if BROKER_BACKPRESSURE_JITTER_MS == 0 { + 0 + } else { + producer_idx + .bytes() + .fold(retry_attempt as u64, |acc, byte| { + acc.wrapping_mul(31).wrapping_add(byte as u64) + }) + % (BROKER_BACKPRESSURE_JITTER_MS + 1) + }; + Duration::from_millis((base_ms + jitter_ms).min(BROKER_BACKPRESSURE_MAX_SLEEP_MS)) +} + +async fn abort_broker_reservation_best_effort( + broker: &BrokerHandle, + chan_id: i64, + reservation_id: u64, +) { + if let Err(err) = broker.abort(chan_id, reservation_id).await { + warn!( + "best-effort broker abort failed: chan_id={} reservation_id={} err={}", + chan_id, reservation_id, err + ); + } +} + +fn abort_on_payload_failure_async(broker: BrokerHandle, chan_id: i64, reservation_id: u64) { + tokio::spawn(async move { + abort_broker_reservation_best_effort(&broker, chan_id, reservation_id).await; + }); +} + +fn broker_now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock is before UNIX_EPOCH") + .as_millis() as i64 } fn spawn_consumer_meta_watch( diff --git a/fluxon_rs/fluxon_observability/src/types.rs b/fluxon_rs/fluxon_observability/src/types.rs index 446c43d..42db8aa 100644 --- a/fluxon_rs/fluxon_observability/src/types.rs +++ b/fluxon_rs/fluxon_observability/src/types.rs @@ -20,6 +20,7 @@ impl FluxonMemberKind { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FluxonMemberRole { Master, + Broker, OwnerClient, ExternalClient, SideTransferWorker, @@ -30,6 +31,7 @@ impl FluxonMemberRole { pub fn as_str(self) -> &'static str { match self { FluxonMemberRole::Master => "master", + FluxonMemberRole::Broker => "broker", FluxonMemberRole::OwnerClient => "owner_client", FluxonMemberRole::ExternalClient => "external_client", FluxonMemberRole::SideTransferWorker => "side_transfer_worker", diff --git a/fluxon_rs/fluxon_pyo3/src/error.rs b/fluxon_rs/fluxon_pyo3/src/error.rs index 97ab680..6f59b8c 100644 --- a/fluxon_rs/fluxon_pyo3/src/error.rs +++ b/fluxon_rs/fluxon_pyo3/src/error.rs @@ -1,6 +1,6 @@ -use pyo3::PyErr; use pyo3::prelude::*; use pyo3::types::PyDict; +use pyo3::PyErr; use fluxon_mq::MpscError as CoreMpscError; // Re-export the core MPSC error type for callers who want to depend on a single error hub. @@ -51,6 +51,26 @@ pub(crate) fn pyerr_message_consumption_no_new_message( }) } +pub(crate) fn pyerr_channel_closed(py: Python<'_>, message: &str, channel_id: i64) -> PyErr { + build_ext_error(py, "ChannelClosedError", message, |kw| { + kw.set_item("channel_id", channel_id).unwrap(); + }) +} + +pub(crate) fn pyerr_producer_closed( + py: Python<'_>, + message: &str, + channel_id: i64, + producer_idx: Option<&str>, +) -> PyErr { + build_ext_error(py, "ProducerClosedError", message, |kw| { + kw.set_item("channel_id", channel_id).unwrap(); + if let Some(p) = producer_idx { + kw.set_item("producer_idx", p).unwrap(); + } + }) +} + pub(crate) fn pyerr_message_consumption( py: Python<'_>, message: &str, @@ -87,6 +107,18 @@ pub(crate) fn pyerr_chan_message_produce( }) } +pub(crate) fn pyerr_message_buffer_full( + py: Python<'_>, + message: &str, + channel_id: i64, + buffer_size: i64, +) -> PyErr { + build_ext_error(py, "MessageBufferFullError", message, |kw| { + kw.set_item("channel_id", channel_id).unwrap(); + kw.set_item("buffer_size", buffer_size).unwrap(); + }) +} + // System/bridge category constructors (distinct helpers for clarity) pub(crate) fn pyerr_etcd(py: Python<'_>, message: &str, component: &str) -> PyErr { build_ext_error(py, "EtcdError", message, |kw| { @@ -264,10 +296,13 @@ pub(crate) fn new_store_closed_error(py: Python<'_>, message: &str) -> PyObject pub(crate) fn new_result_success(py: Python<'_>, value: PyObject) -> PyObject { let api_error_module = py.import_bound("fluxon_py.api_error").unwrap(); let result_class = api_error_module.getattr("Result").unwrap(); - result_class - .call_method1("new_ok", (value,)) - .unwrap() - .into() + match result_class.call_method1("new_ok", (value,)) { + Ok(obj) => obj.into(), + Err(err) => { + let message = format!("Failed to build Result.new_ok: {}", err); + new_result_error(py, new_general_error(py, &message)) + } + } } pub(crate) fn new_result_error(py: Python<'_>, error: PyObject) -> PyObject { diff --git a/fluxon_rs/fluxon_pyo3/src/flatdict_zerocopy.rs b/fluxon_rs/fluxon_pyo3/src/flatdict_zerocopy.rs index 335f36e..c80e775 100644 --- a/fluxon_rs/fluxon_pyo3/src/flatdict_zerocopy.rs +++ b/fluxon_rs/fluxon_pyo3/src/flatdict_zerocopy.rs @@ -1,6 +1,7 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::os::raw::c_void; -use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use fluxon_kv::memholder::kvclient_encode::{ BorrowedFlatKvValueRange, FLAT_KV_TYPE_BOOL, FLAT_KV_TYPE_BYTES, FLAT_KV_TYPE_FLOAT64, @@ -30,25 +31,90 @@ const DLPACK_USED_CAPSULE_NAME_CSTR: &[u8] = b"used_dltensor\0"; #[derive(Clone)] pub(crate) enum FlatDictDataOwner { - OwnedBytes(Arc<[u8]>), - UserMemHolder(Arc), - ExternalMemHolder(Arc), + OwnedBytes(Arc), + UserMemHolder(Arc), + ExternalMemHolder(Arc), } impl FlatDictDataOwner { pub(crate) fn from_owned_bytes(bytes: Vec) -> Self { - Self::OwnedBytes(Arc::<[u8]>::from(bytes)) + Self::OwnedBytes(Arc::new(FlatDictOwnedBytes { + bytes: Arc::<[u8]>::from(bytes), + })) + } + + pub(crate) fn from_user_memholder(holder: Arc) -> Self { + Self::UserMemHolder(Arc::new(FlatDictUserMemHolder { holder })) + } + + pub(crate) fn from_external_memholder(holder: Arc) -> Self { + Self::ExternalMemHolder(Arc::new(FlatDictExternalMemHolder { holder })) } fn bytes(&self) -> &[u8] { match self { - Self::OwnedBytes(bytes) => bytes.as_ref(), - Self::UserMemHolder(holder) => holder.bytes(), - Self::ExternalMemHolder(holder) => holder.bytes(), + Self::OwnedBytes(owner) => owner.bytes.as_ref(), + Self::UserMemHolder(owner) => owner.holder.bytes(), + Self::ExternalMemHolder(owner) => owner.holder.bytes(), } } } +pub(crate) type FlatDictCleanup = Box; + +#[derive(Clone)] +pub(crate) struct FlatDictSharedCleanup { + state: Arc, +} + +struct FlatDictSharedCleanupState { + remaining_views: AtomicUsize, + cleanup: Mutex>, +} + +impl FlatDictSharedCleanup { + fn new(view_count: usize, cleanup: FlatDictCleanup) -> Self { + assert!( + view_count > 0, + "flatdict cleanup requires at least one view" + ); + Self { + state: Arc::new(FlatDictSharedCleanupState { + remaining_views: AtomicUsize::new(view_count), + cleanup: Mutex::new(Some(cleanup)), + }), + } + } + + fn release(self) { + let previous = self.state.remaining_views.fetch_sub(1, Ordering::AcqRel); + assert!(previous > 0, "flatdict cleanup released too many times"); + if previous == 1 { + let cleanup = self.state.cleanup.lock().unwrap().take(); + run_flatdict_cleanup(cleanup); + } + } +} + +pub(crate) struct FlatDictOwnedBytes { + bytes: Arc<[u8]>, +} + +pub(crate) struct FlatDictUserMemHolder { + holder: Arc, +} + +pub(crate) struct FlatDictExternalMemHolder { + holder: Arc, +} + +fn run_flatdict_cleanup(cleanup: Option) { + let Some(cleanup) = cleanup else { + return; + }; + cleanup(); +} + pub(crate) struct FlatDictEncodePlan { ptrs: Vec<(u8, usize, u32, u64, u32, Option)>, key_storage: Vec>, @@ -254,6 +320,7 @@ pub(crate) struct FlatDictDLPackView { bits: u8, lanes: u16, shape: Arc<[i64]>, + cleanup: Mutex>, } #[pymethods] @@ -300,6 +367,32 @@ impl FlatDictDLPackView { bits, lanes, shape, + cleanup: Mutex::new(None), + } + } + + pub(crate) fn attach_cleanup( + &self, + cleanup: FlatDictSharedCleanup, + ) -> Result<(), FlatDictSharedCleanup> { + let mut guard = self.cleanup.lock().unwrap(); + if guard.is_some() { + return Err(cleanup); + } + *guard = Some(cleanup); + Ok(()) + } + + fn has_cleanup(&self) -> bool { + self.cleanup.lock().unwrap().is_some() + } +} + +impl Drop for FlatDictDLPackView { + fn drop(&mut self) { + let cleanup = self.cleanup.get_mut().unwrap().take(); + if let Some(cleanup) = cleanup { + cleanup.release(); } } } @@ -1002,3 +1095,94 @@ pub(crate) fn decode_flat_dict_to_wrapped_py_object( } Ok(dict.into()) } + +pub(crate) fn attach_cleanup_to_flatdict_pyobject( + py: Python<'_>, + obj: &PyObject, + make_cleanup: F, +) -> bool +where + F: FnOnce() -> FlatDictCleanup, +{ + let any = obj.bind(py); + let Ok(dict) = any.downcast::() else { + return false; + }; + let mut seen_views = HashSet::::new(); + let mut view_count = 0usize; + for value in dict.values() { + if let Ok(view) = value.extract::>() { + let view_ptr = (&*view as *const FlatDictDLPackView) as usize; + if !seen_views.insert(view_ptr) { + continue; + } + if view.has_cleanup() { + return false; + } + view_count += 1; + } + } + if view_count == 0 { + return false; + } + + let cleanup = FlatDictSharedCleanup::new(view_count, make_cleanup()); + let mut seen_views = HashSet::::new(); + for value in dict.values() { + if let Ok(view) = value.extract::>() { + let view_ptr = (&*view as *const FlatDictDLPackView) as usize; + if !seen_views.insert(view_ptr) { + continue; + } + assert!( + view.attach_cleanup(cleanup.clone()).is_ok(), + "flatdict cleanup attach must succeed after pre-check" + ); + } + } + true +} + +#[cfg(test)] +mod tests { + use super::*; + use pyo3::types::PyDict; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[test] + fn cleanup_waits_for_all_dlpack_views() { + Python::with_gil(|py| { + let owner = FlatDictDataOwner::from_owned_bytes(vec![0u8; 16]); + let shape = Arc::<[i64]>::from(vec![8_i64]); + let view1 = Py::new( + py, + FlatDictDLPackView::new(owner.clone(), 0, 8, 1, 8, 1, shape.clone()), + ) + .unwrap(); + let view2 = Py::new(py, FlatDictDLPackView::new(owner, 8, 8, 1, 8, 1, shape)).unwrap(); + + let dict = PyDict::new_bound(py); + dict.set_item("a", view1.bind(py)).unwrap(); + dict.set_item("b", view2.bind(py)).unwrap(); + + let hits = Arc::new(AtomicUsize::new(0)); + let hits_for_cleanup = hits.clone(); + let dict_obj = dict.into_any().into_py(py); + assert!(attach_cleanup_to_flatdict_pyobject(py, &dict_obj, || { + let hits_for_cleanup = hits_for_cleanup.clone(); + Box::new(move || { + hits_for_cleanup.fetch_add(1, Ordering::SeqCst); + }) + })); + + drop(dict_obj); + assert_eq!(hits.load(Ordering::SeqCst), 0); + + drop(view1); + assert_eq!(hits.load(Ordering::SeqCst), 0); + + drop(view2); + assert_eq!(hits.load(Ordering::SeqCst), 1); + }); + } +} diff --git a/fluxon_rs/fluxon_pyo3/src/lease_manager.rs b/fluxon_rs/fluxon_pyo3/src/lease_manager.rs index 99b8a79..52645e9 100755 --- a/fluxon_rs/fluxon_pyo3/src/lease_manager.rs +++ b/fluxon_rs/fluxon_pyo3/src/lease_manager.rs @@ -139,26 +139,39 @@ pub struct LeaseManagerHandle { // 仅作为 fluxon_mq::lease_manager::Lease 的包装，避免在 fluxon_pyo3 中重复实现 RAII 逻辑。 #[pyclass] pub struct PyGeneralLease { - lease: fluxon_mq::lease_manager::GeneralLease, + lease: Option, } #[pymethods] impl PyGeneralLease { #[getter] - fn id(&self) -> u64 { - self.lease.id() + fn id(&self) -> PyResult { + Ok(self + .lease + .as_ref() + .ok_or_else(|| { + PyErr::new::("lease handle is closed") + })? + .id()) } fn __repr__(&self) -> String { - match self.lease.kind() { + let Some(lease) = self.lease.as_ref() else { + return "".to_string(); + }; + match lease.kind() { fluxon_util::lease_manager::LeaseType::Etcd => { - format!("", self.id()) + format!("", lease.id()) } fluxon_util::lease_manager::LeaseType::KvClient => { - format!("", self.id()) + format!("", lease.id()) } } } + + fn close(&mut self) { + self.lease = None; + } } #[pymethods] @@ -220,7 +233,7 @@ impl LeaseManagerHandle { "end allocate_etcd_lease: id={}, elapsed_ms={}", lease.id(), t0.elapsed().as_millis() ); - Ok(PyGeneralLease { lease }) + Ok(PyGeneralLease { lease: Some(lease) }) } /// Register existing etcd lease id for keepalive and wrap the core Lease. @@ -272,7 +285,7 @@ impl LeaseManagerHandle { "end register_etcd_lease: id={}, elapsed_ms={}", lease.id(), t0.elapsed().as_millis() ); - Ok(PyGeneralLease { lease }) + Ok(PyGeneralLease { lease: Some(lease) }) } /// Register a kvclient lease via constructed backend uid carrying callbacks. @@ -330,7 +343,7 @@ impl LeaseManagerHandle { "end register_kvclient_lease_via_backend: id={}, elapsed_ms={}", lease.id(), t0.elapsed().as_millis() ); - Ok(PyGeneralLease { lease }) + Ok(PyGeneralLease { lease: Some(lease) }) } /// Debug-only: dump current active lease entries from the keepalive actor. diff --git a/fluxon_rs/fluxon_pyo3/src/lib.rs b/fluxon_rs/fluxon_pyo3/src/lib.rs index a73591f..31d3c59 100644 --- a/fluxon_rs/fluxon_pyo3/src/lib.rs +++ b/fluxon_rs/fluxon_pyo3/src/lib.rs @@ -31,7 +31,7 @@ use fluxon_kv::user_api::FluxonUserApi; use fluxon_kv::{ ConfigArg, Framework, KvClientTrait, KvGetResult, config::{ClientConfig, MasterConfig}, - run_client, run_master, + run_broker, run_client, run_master, }; use fluxon_ops; use fluxon_proxy; @@ -2623,6 +2623,31 @@ fn python_config_to_master_config( } } +fn python_config_to_client_config( + py: Python, + py_config: &Bound<'_, PyDict>, +) -> ApiResult { + let config: serde_yaml::Value = match pyany_to_serde_value(py, &py_config.to_object(py)) { + Ok(val) => val, + Err(e) => return ApiResult::new_error(new_invalid_argument_error(py, &e.to_string())), + }; + + let yaml_str = match serde_yaml::to_string(&config) { + Ok(s) => s, + Err(e) => return ApiResult::new_error(new_invalid_argument_error(py, &e.to_string())), + }; + + let config: ClientConfigYaml = match ClientConfigYaml::from_str(&yaml_str) { + Ok(config) => config, + Err(e) => return ApiResult::new_error(new_invalid_argument_error(py, &e.to_string())), + }; + + match config.verify() { + Ok(config) => ApiResult::new_success(config.into()), + Err(e) => ApiResult::new_error(new_invalid_argument_error(py, &e.to_string())), + } +} + fn pyany_to_serde_value(py: Python, obj: &PyObject) -> PyResult { if obj.is_none(py) { Ok(Value::Null) @@ -4138,6 +4163,124 @@ fn run_master_blocking(config: Option<&Bound<'_, PyAny>>, py: Python) -> PyObjec run_master_inner(config, py).into_py_object(py) } +/// Run broker with automatic lifecycle management +/// This function creates a broker, runs it until Ctrl+C, then shuts down +#[pyfunction] +#[pyo3(signature = (config=None))] +fn run_broker_blocking(config: Option<&Bound<'_, PyAny>>, py: Python) -> PyObject { + fn run_broker_inner(config: Option<&Bound<'_, PyAny>>, py: Python) -> ApiResult { + println!("🛠️ Broker init configuration: {:?}", config); + + let runtime = match Runtime::new() { + Ok(rt) => rt, + Err(e) => { + return ApiResult::new_error(new_general_error( + py, + &format!("Failed to create runtime: {}", e), + )); + } + }; + + let config_arg = match config { + None => ConfigArg::None, + Some(py_obj) => { + if py_obj.is_instance_of::() { + let path_str: String = match py_obj.extract() { + Ok(path) => path, + Err(_) => { + return ApiResult::new_error(new_invalid_argument_error( + py, + "Invalid configuration file path", + )); + } + }; + ConfigArg::File(PathBuf::from(path_str)) + } else if py_obj.is_instance_of::() { + let py_dict = match py_obj.downcast::() { + Ok(dict) => dict, + Err(_) => { + return ApiResult::new_error(new_invalid_argument_error( + py, + "Invalid configuration dictionary", + )); + } + }; + match python_config_to_client_config(py, py_dict) { + ApiResult::Success(client_config) => ConfigArg::Config(client_config), + ApiResult::Error(error) => return ApiResult::new_error(error), + } + } else { + return ApiResult::new_error(new_invalid_argument_error( + py, + "Config parameter must be None, string (file path), or dict (config object)", + )); + } + } + }; + + println!("🚀 Starting KV Broker..."); + + let (framework, final_config) = match py.allow_threads(|| { + runtime.run_async_from_sync(async move { fluxon_kv::run_broker(config_arg).await }) + }) { + Ok(Ok((fw, cfg))) => (fw, cfg), + Ok(Err(e)) => { + return ApiResult::new_error(new_backend_init_failed_error( + py, + &format!("Failed to initialize KV broker: {}", e), + Some("unified"), + )); + } + Err(e) => { + return ApiResult::new_error(new_backend_init_failed_error( + py, + &format!("Runtime bridge failed: {}", e), + Some("unified"), + )); + } + }; + + println!("✅ KV Broker started successfully"); + println!("📊 Instance: {}", final_config.instance_key); + println!("🏷️ Cluster: {}", final_config.cluster_name); + match final_config.fluxonkv_spec.p2p_listen_port { + Some(port) => println!("🔌 Port: {}", port), + None => println!("🔌 Port: auto"), + } + println!("🚀 Broker is running... Press Ctrl+C to stop"); + + let shutdown_result = py.allow_threads(|| { + runtime.block_on(async move { + if let Err(e) = tokio::signal::ctrl_c().await { + eprintln!("Failed to listen for shutdown signal: {}", e); + } + match framework.shutdown().await { + Ok(_) => { + println!("✅ Broker shut down successfully"); + Ok(()) + } + Err(e) => { + eprintln!("⚠️ Warning during shutdown: {}", e); + Err(e) + } + } + }) + }); + + let out = match shutdown_result { + Ok(_) => ApiResult::new_success(new_none_success_instance(py)), + Err(e) => ApiResult::new_error(new_general_error( + py, + &format!("Error during shutdown: {}", e), + )), + }; + runtime.shutdown_background(); + out + } + + run_broker_inner(config, py).into_py_object(py) +} + /// Python module definition #[pymodule] #[pyo3(name = "fluxon_pyo3")] @@ -4158,6 +4301,7 @@ fn fluxon_pyo3(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_function(wrap_pyfunction!(run_master_blocking, m)?)?; + m.add_function(wrap_pyfunction!(run_broker_blocking, m)?)?; m.add_function(wrap_pyfunction!(monitor_render_cli, m)?)?; m.add_function(wrap_pyfunction!(monitor_render_web, m)?)?; m.add_function(wrap_pyfunction!(fluxon_ops_controller_blocking, m)?)?; diff --git a/fluxon_rs/fluxon_pyo3/src/memholder.rs b/fluxon_rs/fluxon_pyo3/src/memholder.rs index b2750e6..7b66aec 100755 --- a/fluxon_rs/fluxon_pyo3/src/memholder.rs +++ b/fluxon_rs/fluxon_pyo3/src/memholder.rs @@ -34,9 +34,11 @@ impl MemHolder { let data_owner = match &holder.holder { MemHolderInner::Seg(seg_holder) => { - FlatDictDataOwner::UserMemHolder(seg_holder.clone()) + FlatDictDataOwner::from_user_memholder(seg_holder.clone()) + } + MemHolderInner::Owned(bytes) => { + FlatDictDataOwner::from_owned_bytes(bytes.as_ref().to_vec()) } - MemHolderInner::Owned(bytes) => FlatDictDataOwner::OwnedBytes(bytes.clone()), }; match decode_flat_dict_to_wrapped_py_object(py, data_owner) { Ok(obj) => { @@ -126,7 +128,7 @@ impl ExternalMemHolder { return ApiResult::new_success(cached.clone_ref(py).into_py(py)); } - let data_owner = FlatDictDataOwner::ExternalMemHolder(holder.holder.clone()); + let data_owner = FlatDictDataOwner::from_external_memholder(holder.holder.clone()); match decode_flat_dict_to_wrapped_py_object(py, data_owner) { Ok(obj) => { *holder.access_cache.write() = Some(obj.clone_ref(py)); diff --git a/fluxon_rs/fluxon_pyo3/src/mpsc.rs b/fluxon_rs/fluxon_pyo3/src/mpsc.rs index 22d0d07..1b1759d 100644 --- a/fluxon_rs/fluxon_pyo3/src/mpsc.rs +++ b/fluxon_rs/fluxon_pyo3/src/mpsc.rs @@ -1,31 +1,33 @@ use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant}; use crossbeam_channel as cbchan; -use fluxon_mq::DeleteResult as CoreDeleteResult; use fluxon_mq::consumer::{ ConsumedPayload as CoreConsumedPayload, MqPayload as CoreMqPayload, PayloadResult as CorePayloadResult, }; +use fluxon_mq::DeleteResult as CoreDeleteResult; use fluxon_mq::{ - ChanManager, MpscConsumer as CoreMpscConsumer, MpscError as CoreMpscError, - MpscProducer as CoreMpscProducer, ShutdownCtl, - create::{ChanCreateConfig, create_mpsc_channel}, + create::{create_mpsc_channel, ChanCreateConfig}, + BrokerChannelConfig, BrokerHandle, ChanManager, MpscConsumer as CoreMpscConsumer, + MpscError as CoreMpscError, MpscProducer as CoreMpscProducer, ShutdownCtl, }; -use pyo3::Py; -use pyo3::PyErr; use pyo3::prelude::*; use pyo3::types::{PyAny, PyBytes, PyString}; +use pyo3::Py; +use pyo3::PyErr; use tokio::runtime::Handle; use tokio::runtime::Runtime; // (no local payload buffering) -use crate::flatdict_zerocopy::{FlatDictDataOwner, decode_flat_dict_to_wrapped_py_object}; +use crate::flatdict_zerocopy::{ + attach_cleanup_to_flatdict_pyobject, decode_flat_dict_to_wrapped_py_object, FlatDictDataOwner, +}; use crate::lease_manager::PyLeaseBackendUid; use fluxon_kv::{Framework as KvFramework, KvClientTrait}; use fluxon_mq::lease_manager::LeaseBackendUid; -use fluxon_util::lease_manager::{GLOBAL_LM, LeaseManager}; +use fluxon_util::lease_manager::{LeaseManager, GLOBAL_LM}; use fluxon_util::run_async_from_sync::SyncAsyncBridge; use tracing::{debug, warn}; @@ -33,9 +35,35 @@ use tracing::{debug, warn}; // that implements the core MqPayload trait so we can downcast later. struct PyPayload { inner: PyObject, + cleanup_runtime: Handle, } -impl CoreMqPayload for PyPayload {} +impl CoreMqPayload for PyPayload { + fn attach_cleanup( + &mut self, + cleanup: fluxon_mq::consumer::PayloadCleanup, + ) -> Result<(), fluxon_mq::consumer::PayloadCleanup> { + let runtime = self.cleanup_runtime.clone(); + let mut cleanup = Some(cleanup); + let attached = Python::with_gil(|py| { + attach_cleanup_to_flatdict_pyobject(py, &self.inner, || { + let cleanup = cleanup + .take() + .expect("cleanup must be present when DLPack attach is selected"); + Box::new(move || { + runtime.spawn(async move { + cleanup().await; + }); + }) + }) + }); + if attached { + Ok(()) + } else { + Err(cleanup.expect("cleanup must remain present when attach fails")) + } + } +} // Shared runtime for PyO3 helpers that are not lifecycle-governed by a KV Framework. // MQ producer/consumer operations should prefer the KV client's runtime/framework to @@ -60,6 +88,59 @@ pub(crate) fn get_global_runtime() -> Arc { .clone() } +fn connect_distributed_broker(kv_framework: &Arc) -> BrokerHandle { + BrokerHandle::new_distributed( + kv_framework.cluster_manager_view().clone(), + kv_framework.p2p_view().clone(), + ) +} + +fn init_broker_for_channel( + runtime: &Handle, + broker: &BrokerHandle, + channel_id: i64, + capacity: i64, +) -> PyResult<()> { + use pyo3::exceptions::PyRuntimeError; + + runtime + .run_async_from_sync(async move { + broker + .upsert_channel(BrokerChannelConfig { + channel_id, + capacity, + }) + .await + }) + .map_err(|e| PyRuntimeError::new_err(format!("broker runtime bridge failed: {}", e)))? + .map_err(|e| { + PyRuntimeError::new_err(format!( + "failed to upsert broker channel config: chan_id={} capacity={} err={}", + channel_id, capacity, e + )) + })?; + Ok(()) +} + +fn delete_broker_channel( + runtime: &Handle, + broker: &BrokerHandle, + channel_id: i64, +) -> PyResult> { + use pyo3::exceptions::PyRuntimeError; + + let payload_keys = runtime + .run_async_from_sync(async move { broker.delete_channel(channel_id).await }) + .map_err(|e| PyRuntimeError::new_err(format!("broker runtime bridge failed: {}", e)))? + .map_err(|e| { + PyRuntimeError::new_err(format!( + "failed to delete broker channel: chan_id={} err={}", + channel_id, e + )) + })?; + Ok(payload_keys) +} + fn get_consumed_message_class(py: Python<'_>) -> PyResult> { if let Some(c) = CONSUMED_MESSAGE_CLASS.get() { return Ok(c.clone_ref(py)); @@ -126,6 +207,120 @@ fn finalize_payload_result( result } +fn map_producer_result( + py: Python<'_>, + result: Result<(), CoreMpscError>, + producer: &CoreMpscProducer, +) -> PyResult<()> { + use crate::error::CoreMpscErrorReExport as CoreErr; + + match result { + Ok(()) => Ok(()), + Err(e) => match e { + CoreErr::MessageBufferFull { + channel_id, + capacity, + .. + } => Err(crate::error::pyerr_message_buffer_full( + py, + &e.to_string(), + channel_id, + capacity, + )), + CoreErr::PutPayloadNonRetryable | CoreErr::PutPayloadUnknownCode { .. } => { + Err(crate::error::pyerr_chan_message_produce( + py, + &e.to_string(), + producer.chan_id(), + Some(&producer.producer_idx().to_string()), + None, + )) + } + CoreErr::Etcd(_) => Err(crate::error::pyerr_etcd(py, &e.to_string(), "mpsc_rust")), + CoreErr::JoinError(_) => Err(crate::error::pyerr_join_error( + py, + &e.to_string(), + "mpsc_rust", + )), + CoreErr::Closed => Err(crate::error::pyerr_producer_closed( + py, + &e.to_string(), + producer.chan_id(), + Some(producer.producer_idx()), + )), + CoreErr::Internal(_) | CoreErr::NoMessage => Err(crate::error::pyerr_internal( + py, + &e.to_string(), + "mpsc_rust", + )), + _ => Err(crate::error::pyerr_internal( + py, + &e.to_string(), + "mpsc_rust", + )), + }, + } +} + +fn map_consumer_error(py: Python<'_>, err: CoreMpscError, chan_id: i64) -> PyErr { + use crate::error::CoreMpscErrorReExport as CoreErr; + let message = err.to_string(); + + match err { + CoreErr::NoMessage => crate::error::pyerr_message_consumption_no_new_message( + py, &message, chan_id, None, None, + ), + CoreErr::MessageBufferFull { capacity, .. } => { + crate::error::pyerr_message_buffer_full(py, &message, chan_id, capacity) + } + CoreErr::GetPayloadNonRetryable { .. } + | CoreErr::GetPayloadUnknownCode { .. } + | CoreErr::ConsumeOffsetUpdate { .. } + | CoreErr::DeletePayloadNonRetryable { .. } + | CoreErr::DeletePayloadUnknownCode { .. } => { + crate::error::pyerr_message_consumption(py, &message, chan_id, None, None) + } + CoreErr::PutPayloadNonRetryable | CoreErr::PutPayloadUnknownCode { .. } => { + crate::error::pyerr_chan_message_produce(py, &message, chan_id, None, None) + } + CoreErr::Etcd(_) => crate::error::pyerr_etcd(py, &message, "mpsc_rust"), + CoreErr::JoinError(_) => crate::error::pyerr_join_error(py, &message, "mpsc_rust"), + CoreErr::Internal(_) | CoreErr::Closed => { + crate::error::pyerr_internal(py, &message, "mpsc_rust") + } + } +} + +fn consumed_payload_to_pyobject( + py: Python<'_>, + consumed: CoreConsumedPayload, +) -> PyResult<(PyObject, u64)> { + use pyo3::exceptions::PyRuntimeError; + + let CoreConsumedPayload { payload, .. } = consumed; + let pyobj = match payload.downcast::() { + Ok(v) => v.inner, + Err(_) => { + return Err(PyRuntimeError::new_err( + "payload type mismatch: expected PyPayload", + )); + } + }; + + let payload_len: u64 = { + let any = pyobj.bind(py); + if any.is_instance_of::() { + let b = any + .downcast::() + .expect("PyBytes downcast failed after is_instance_of"); + b.as_bytes().len() as u64 + } else { + 0 + } + }; + Ok((pyobj, payload_len)) +} + // (LeaseManagerHandle and PyLease moved to lease_manager.rs) /// Shared MPSC context bound to a specific etcd endpoint set. @@ -307,6 +502,7 @@ impl MpscContext { Ok(MpscProducerHandle { inner: Some(producer), + broker: None, shutdown, kv_framework: self.kv_framework.clone(), kv_runtime: self.kv_runtime.clone(), @@ -449,6 +645,7 @@ impl MpscContext { Ok(MpscConsumerHandle { inner: Some(consumer), + broker: None, shutdown, parent_mpmc_id_opt, kv_framework: self.kv_framework.clone(), @@ -493,6 +690,11 @@ impl MpscContext { ))), } } + + fn delete_broker_channel(&self, chan_id: i64) -> PyResult> { + let broker = connect_distributed_broker(&self.kv_framework); + delete_broker_channel(&self.kv_runtime, &broker, chan_id) + } } /// PyO3 handle for MPSC producer. Currently this focuses on @@ -501,6 +703,7 @@ impl MpscContext { #[pyclass] pub struct MpscProducerHandle { pub(crate) inner: Option, + broker: Option, shutdown: ShutdownCtl, kv_framework: Arc, kv_runtime: Handle, @@ -509,50 +712,18 @@ pub struct MpscProducerHandle { put_profile_window_bytes: u64, } -#[pymethods] -impl MpscProducerHandle { - fn chan_id(&self) -> i64 { - self.inner - .as_ref() - .expect("MpscProducerHandle inner not initialized or already taken by an in-flight put") - .chan_id() - } - - fn producer_idx(&self) -> String { - self.inner - .as_ref() - .expect("MpscProducerHandle inner not initialized or already taken by an in-flight put") - .producer_idx() - .to_string() - } - - fn payload_lease_id(&self) -> i64 { - self.inner - .as_ref() - .expect("MpscProducerHandle inner not initialized or already taken by an in-flight put") - .payload_lease_id() - } +enum ProducerPutMode { + Etcd, + Broker { broker: BrokerHandle }, +} - /// Put a message payload into the underlying KV backend by passing raw ptr tuples. - /// - /// This avoids calling back into Python for kvclient.put and lets the KV backend - /// encode/copy directly into segment memory. - /// - /// `ptrs` is a list of `(type_id, dict_key_ptr, dict_key_len, val_u64, val_len, extra)`: - /// - `dict_key_ptr/dict_key_len`: UTF-8 bytes of the dict field key. - /// - For scalar types (bool/int64/float64), `val_u64` stores raw bits and `val_len` is fixed. - /// - For bytes-like types (string/bytes), `val_u64` stores a pointer and `val_len` is the byte length. - /// - /// Safety/lifetime contract: - /// - This is async on the Rust side; the caller must keep the memory behind pointers - /// alive and immutable until this method returns. - #[pyo3(signature = (ptrs))] - fn put_flat_dict_ptrs( +impl MpscProducerHandle { + fn put_flat_dict_ptrs_impl( &mut self, ptrs: Vec<(u8, u64, u32, u64, u32, Option)>, + mode: ProducerPutMode, ) -> PyResult<()> { use pyo3::exceptions::PyRuntimeError; - use std::sync::{Arc, Mutex}; if self.shutdown.is_closed() { return Err(PyRuntimeError::new_err("MpscProducerHandle is closed")); @@ -598,56 +769,79 @@ impl MpscProducerHandle { let (tx, rx) = cbchan::bounded::<(Result<(), CoreMpscError>, CoreMpscProducer)>(1); runtime.spawn(async move { - let mut guard = ProducerGuard::new(inner, tx); + let mut guard = ProducerGuard::new(inner, tx, ShutdownCtl::new()); let payload_lease_id = guard.inner_mut().payload_lease_id() as u64; - - let res = guard - .inner_mut() - .put_with_payload(move |key: String, _msg_id: i64, preferred_sub_cluster| { - let mut o = fluxon_kv::client_kv_api::PutOptionalArgs::new(); - o.0.push(fluxon_kv::client_kv_api::PutOptionalArg::LeaseId( - payload_lease_id, + let put_payload: Arc< + dyn Fn(String, i64, Option) -> i32 + Send + Sync + 'static, + > = Arc::new(move |key: String, _msg_id: i64, preferred_sub_cluster| { + let mut o = fluxon_kv::client_kv_api::PutOptionalArgs::new(); + o.0.push(fluxon_kv::client_kv_api::PutOptionalArg::LeaseId( + payload_lease_id, + )); + if let Some(sc) = preferred_sub_cluster { + o.0.push(fluxon_kv::client_kv_api::PutOptionalArg::PreferredSubCluster( + sc, )); - if let Some(sc) = preferred_sub_cluster { - o.0.push(fluxon_kv::client_kv_api::PutOptionalArg::PreferredSubCluster( - sc, - )); - } + } - let ptrs_for_call: Vec<(u8, usize, u32, u64, u32, Option)> = - (*ptrs_arc).clone(); - let kv_framework_for_call = kv_framework.clone(); - let kv_runtime_for_call = kv_runtime.clone(); - let put_res = kv_runtime_for_call.run_async_from_sync(async move { - unsafe { kv_framework_for_call.kv_put_ptrs(&key, ptrs_for_call, o).await } - }); + let ptrs_for_call: Vec<(u8, usize, u32, u64, u32, Option)> = + (*ptrs_arc).clone(); + let kv_framework_for_call = kv_framework.clone(); + let kv_runtime_for_call = kv_runtime.clone(); + let put_res = kv_runtime_for_call.run_async_from_sync(async move { + unsafe { kv_framework_for_call.kv_put_ptrs(&key, ptrs_for_call, o).await } + }); - match put_res { - Ok(Ok(())) => 0, - Ok(Err(e)) => { - if matches!( - &e, - fluxon_kv::rpcresp_kvresult_convert::msg_and_error::KvError::Api( - fluxon_kv::rpcresp_kvresult_convert::msg_and_error::ApiError::NoSpace { .. } - ) - ) { - 1 - } else { - if let Ok(mut g) = err_for_closure.lock() { - *g = Some(e.to_string()); - } - 2 - } - } - Err(e) => { + match put_res { + Ok(Ok(())) => 0, + Ok(Err(e)) => { + if matches!( + &e, + fluxon_kv::rpcresp_kvresult_convert::msg_and_error::KvError::Api( + fluxon_kv::rpcresp_kvresult_convert::msg_and_error::ApiError::NoSpace { .. } + ) + ) { + 1 + } else { if let Ok(mut g) = err_for_closure.lock() { - *g = Some(format!("runtime bridge failed: {}", e)); + *g = Some(e.to_string()); } 2 } } - }) - .await; + Err(e) => { + if let Ok(mut g) = err_for_closure.lock() { + *g = Some(format!("runtime bridge failed: {}", e)); + } + 2 + } + } + }); + + let res = match mode { + ProducerPutMode::Etcd => { + let put_payload = put_payload.clone(); + guard + .inner_mut() + .put_with_payload(move |key, msg_id, preferred_sub_cluster| { + (put_payload)(key, msg_id, preferred_sub_cluster) + }) + .await + } + ProducerPutMode::Broker { broker } => { + let put_payload = put_payload.clone(); + guard + .inner_mut() + .put_with_payload_via_broker( + &broker, + payload_len, + move |key, msg_id, preferred_sub_cluster| { + (put_payload)(key, msg_id, preferred_sub_cluster) + }, + ) + .await + } + }; guard.finish(res); }); @@ -658,8 +852,15 @@ impl MpscProducerHandle { Ok(v) => break v, Err(cbchan::RecvTimeoutError::Timeout) => {} Err(cbchan::RecvTimeoutError::Disconnected) => { + self.shutdown.close(); return ( - Err(PyRuntimeError::new_err("put_flat_dict_ptrs task cancelled")), + Err(crate::error::pyerr_chan_message_produce( + py, + "producer is closed", + self.chan_id(), + Some(&self.producer_idx().to_string()), + None, + )), None, ); } @@ -685,42 +886,10 @@ impl MpscProducerHandle { } } - let mapped = match result { - Ok(()) => Ok(()), - Err(e) => { - use crate::error::CoreMpscErrorReExport as CoreErr; - match e { - CoreErr::PutPayloadNonRetryable | CoreErr::PutPayloadUnknownCode { .. } => { - Err(crate::error::pyerr_chan_message_produce( - py, - &e.to_string(), - producer_back.chan_id(), - Some(&producer_back.producer_idx().to_string()), - None, - )) - } - CoreErr::Etcd(_) => { - Err(crate::error::pyerr_etcd(py, &e.to_string(), "mpsc_rust")) - } - CoreErr::JoinError(_) => Err(crate::error::pyerr_join_error( - py, - &e.to_string(), - "mpsc_rust", - )), - CoreErr::Internal(_) => Err(crate::error::pyerr_internal( - py, - &e.to_string(), - "mpsc_rust", - )), - _ => Err(crate::error::pyerr_internal( - py, - &e.to_string(), - "mpsc_rust", - )), - } - } - }; - (mapped, Some(producer_back)) + ( + map_producer_result(py, result, &producer_back), + Some(producer_back), + ) }); if let Some(back) = maybe_back { @@ -743,6 +912,71 @@ impl MpscProducerHandle { mapped } +} + +#[pymethods] +impl MpscProducerHandle { + fn chan_id(&self) -> i64 { + self.inner + .as_ref() + .expect("MpscProducerHandle inner not initialized or already taken by an in-flight put") + .chan_id() + } + + fn producer_idx(&self) -> String { + self.inner + .as_ref() + .expect("MpscProducerHandle inner not initialized or already taken by an in-flight put") + .producer_idx() + .to_string() + } + + fn payload_lease_id(&self) -> i64 { + self.inner + .as_ref() + .expect("MpscProducerHandle inner not initialized or already taken by an in-flight put") + .payload_lease_id() + } + + #[pyo3(signature = (ptrs))] + fn put_flat_dict_ptrs( + &mut self, + ptrs: Vec<(u8, u64, u32, u64, u32, Option)>, + ) -> PyResult<()> { + use pyo3::exceptions::PyRuntimeError; + + let broker = self + .broker + .clone() + .ok_or_else(|| PyRuntimeError::new_err("broker is not initialized"))?; + self.put_flat_dict_ptrs_impl(ptrs, ProducerPutMode::Broker { broker }) + } + + #[pyo3(signature = (ptrs))] + fn put_flat_dict_ptrs_legacy_for_internal_check( + &mut self, + ptrs: Vec<(u8, u64, u32, u64, u32, Option)>, + ) -> PyResult<()> { + self.put_flat_dict_ptrs_impl(ptrs, ProducerPutMode::Etcd) + } + + fn init_broker(&mut self) -> PyResult<()> { + use pyo3::exceptions::PyRuntimeError; + + let inner = self + .inner + .as_ref() + .ok_or_else(|| PyRuntimeError::new_err("MpscProducerHandle inner not initialized"))?; + let broker = connect_distributed_broker(&self.kv_framework); + init_broker_for_channel( + &self.kv_runtime, + &broker, + inner.chan_id(), + inner.channel_capacity(), + )?; + self.broker = Some(broker); + Ok(()) + } // Removed: the legacy `put_with_payload(callback)` API was intentionally deleted to // force a single supported data path (put_flat_dict_ptrs) and avoid Python callbacks @@ -797,6 +1031,7 @@ impl PyShutdownCtl { #[pyclass] pub struct MpscConsumerHandle { pub(crate) inner: Option, + broker: Option, shutdown: ShutdownCtl, /// Optional parent MPMC id when this MPSC acts as a submodule of a MPMC channel. /// Only used for diagnostics (rate-limited retry logging) and not for behavior. @@ -829,60 +1064,438 @@ pub struct MpscConsumerHandle { get_one_profile_last_timeout_ms: Option, } -#[pymethods] -impl MpscConsumerHandle { - fn chan_id(&self) -> i64 { - self.inner - .as_ref() - .expect("MpscConsumerHandle inner not initialized") - .chan_id() - } - - fn consumer_idx(&self) -> String { - self.inner - .as_ref() - .expect("MpscConsumerHandle inner not initialized") - .consumer_idx() - .to_string() - } +enum ConsumerGetMode { + Prefetch { + prefetch_target: usize, + timeout_ms: Option, + maybe_sync_sub_cluster: Option>, + }, + Broker { + broker: BrokerHandle, + timeout_ms: Option, + }, +} - /// Initialize the global payload callback for this consumer. - /// - /// 回调在 consumer 生命周期内复用；后续 `get_one` / - /// `get_with_payload` 调用都不会再传入回调参数。 - #[pyo3(signature = (callback))] - fn init_payload_callback(&mut self, callback: PyObject) -> PyResult<()> { +impl MpscConsumerHandle { + fn get_one_impl( + &mut self, + py: Python<'_>, + mode: ConsumerGetMode, + profile_prefetch_target: usize, + profile_timeout_ms: Option, + ) -> PyResult { use pyo3::exceptions::PyRuntimeError; - use std::sync::Arc; - - let cb: Arc = Arc::new(callback); + use std::time::Duration; + let get_one_begin = std::time::Instant::now(); + let chan_id_for_profile = self.chan_id(); + let consumer_idx_for_profile = self.consumer_idx(); + self.get_one_profile_last_prefetch_target = profile_prefetch_target; + self.get_one_profile_last_timeout_ms = profile_timeout_ms; + if self.shutdown.is_closed() { + return Err(crate::error::pyerr_channel_closed( + py, + "consumer is closed", + self.chan_id(), + )); + } - // Capture identifiers for rate-limited retry logging (diagnostic only). - let mpsc_id_for_log = self.chan_id(); - let parent_mpmc_id_opt = self.parent_mpmc_id_opt; + let runtime = self.kv_runtime.clone(); - // Rate limit helper lives in fluxon_util::limitrate + let inner = self + .inner + .take() + .ok_or_else(|| PyRuntimeError::new_err("MpscConsumerHandle is already in use"))?; - let bridge_cb: fluxon_mq::consumer::PayloadCallback = Arc::new( - move |producer_id: String, key: String| { - let cb_for_call = cb.clone(); - Box::pin(async move { - let producer_id_for_call = producer_id.clone(); - let key_for_call = key.clone(); + let (tx, rx) = + cbchan::bounded::<(Result, CoreMpscConsumer)>(1); - let join = limit_thirdparty::tokio::task::spawn_blocking(move || { - // Run the Python callback via a global Python executor. - // This avoids blocking the Tokio scheduler thread. - let (pid_obj, key_obj) = Python::with_gil(|py| { - ( - PyString::new_bound(py, &producer_id_for_call) - .unbind() - .into(), - PyString::new_bound(py, &key_for_call).unbind().into(), + runtime.spawn(async move { + let mut guard = ConsumerGuard::new(inner, tx, ShutdownCtl::new()); + let (chan_id_for_log, consumer_idx_for_log) = { + let inner_ref = guard.inner_mut(); + (inner_ref.chan_id(), inner_ref.consumer_idx().to_string()) + }; + let res = match mode { + ConsumerGetMode::Prefetch { + prefetch_target, + timeout_ms, + maybe_sync_sub_cluster, + } => { + if let Some(sc) = maybe_sync_sub_cluster { + if let Err(e) = guard.inner_mut().sync_kvclient_sub_cluster(sc.clone()).await + { + warn!( + "[MpscConsumer chan_id={} consumer_idx={}] failed to sync kvclient_sub_cluster={:?}: {}; continuing consumption", + chan_id_for_log, consumer_idx_for_log, sc, e + ); + } + } + if let Some(ms) = timeout_ms { + guard + .inner_mut() + .get_with_payload_retry_wait_timeout( + prefetch_target, + Duration::from_millis(ms as u64), ) - }); - - match fluxon_util::pyo3::run_longtime_py_function( + .await + } else { + guard + .inner_mut() + .get_with_payload_retry(prefetch_target) + .await + } + } + ConsumerGetMode::Broker { broker, timeout_ms } => { + let fut = guard.inner_mut().get_with_payload_via_broker(&broker); + if let Some(ms) = timeout_ms { + match tokio::time::timeout(Duration::from_millis(ms as u64), fut).await { + Ok(result) => result, + Err(_) => Err(CoreMpscError::NoMessage), + } + } else { + fut.await + } + } + }; + match &res { + Ok(payload) => { + debug!( + "[MpscConsumerHandle chan_id={} consumer_idx={}] async get finished: producer_id={} nonblocking_hit={}", + chan_id_for_log, + consumer_idx_for_log, + payload.producer_id, + payload.nonblocking_hit, + ); + } + Err(err) => { + debug!( + "[MpscConsumerHandle chan_id={} consumer_idx={}] async get finished with error: {:?}", + chan_id_for_log, + consumer_idx_for_log, + err, + ); + } + } + guard.finish(res); + }); + + let mut wait_rx_ns: u64 = 0; + let mut wait_rx_max_ns: u64 = 0; + let mut signal_ns: u64 = 0; + let mut signal_max_ns: u64 = 0; + let mut recv_timeouts: u64 = 0; + let mut recv_calls: u64 = 0; + let wait_begin = Instant::now(); + let mut next_pending_warn_at = wait_begin + GET_ONE_PENDING_WARN_INTERVAL; + + let (result, consumer_back) = loop { + recv_calls += 1; + let recv_begin = Instant::now(); + let recv_res = py.allow_threads(|| rx.recv_timeout(Duration::from_millis(50))); + let recv_elapsed_ns = recv_begin.elapsed().as_nanos() as u64; + wait_rx_ns += recv_elapsed_ns; + if recv_elapsed_ns > wait_rx_max_ns { + wait_rx_max_ns = recv_elapsed_ns; + } + + match recv_res { + Ok(v) => break v, + Err(cbchan::RecvTimeoutError::Timeout) => { + recv_timeouts += 1; + let now = Instant::now(); + if now >= next_pending_warn_at { + warn!( + "[MpscConsumerHandle chan_id={} consumer_idx={}] get_one still pending: elapsed_ms={} recv_calls={} recv_timeouts={} prefetch_target={} timeout_ms={:?}", + chan_id_for_profile, + consumer_idx_for_profile, + wait_begin.elapsed().as_millis(), + recv_calls, + recv_timeouts, + profile_prefetch_target, + profile_timeout_ms, + ); + next_pending_warn_at = now + GET_ONE_PENDING_WARN_INTERVAL; + } + } + Err(cbchan::RecvTimeoutError::Disconnected) => { + return Err(PyRuntimeError::new_err("get_one task cancelled")); + } + } + + let signal_begin = Instant::now(); + let signal_res = py.check_signals(); + let signal_elapsed_ns = signal_begin.elapsed().as_nanos() as u64; + signal_ns += signal_elapsed_ns; + if signal_elapsed_ns > signal_max_ns { + signal_max_ns = signal_elapsed_ns; + } + + if let Err(e) = signal_res { + self.shutdown.close(); + return Err(e); + } + }; + + let post_begin = Instant::now(); + self.inner = Some(consumer_back); + + let consumed = match result { + Ok(v) => v, + Err(e) => return Err(map_consumer_error(py, e, self.chan_id())), + }; + let (pyobj, payload_len) = consumed_payload_to_pyobject(py, consumed)?; + + let get_one_total = get_one_begin.elapsed(); + let total_ns = get_one_total.as_nanos() as u64; + let post_ns = post_begin.elapsed().as_nanos() as u64; + + self.get_one_profile_cnt += 1; + self.get_one_profile_window_bytes += payload_len; + self.get_one_profile_total_sum_ns += total_ns; + if total_ns > self.get_one_profile_total_max_ns { + self.get_one_profile_total_max_ns = total_ns; + } + self.get_one_profile_wait_rx_sum_ns += wait_rx_ns; + if wait_rx_max_ns > self.get_one_profile_wait_rx_max_ns { + self.get_one_profile_wait_rx_max_ns = wait_rx_max_ns; + } + self.get_one_profile_signal_sum_ns += signal_ns; + if signal_max_ns > self.get_one_profile_signal_max_ns { + self.get_one_profile_signal_max_ns = signal_max_ns; + } + self.get_one_profile_post_sum_ns += post_ns; + if post_ns > self.get_one_profile_post_max_ns { + self.get_one_profile_post_max_ns = post_ns; + } + self.get_one_profile_recv_timeouts += recv_timeouts; + self.get_one_profile_recv_calls += recv_calls; + + let now = Instant::now(); + if now >= self.get_one_profile_next_log_at && self.get_one_profile_cnt > 0 { + let cnt = self.get_one_profile_cnt; + let avg_total_ms = + (self.get_one_profile_total_sum_ns as f64) / (cnt as f64) / 1_000_000.0; + let avg_wait_rx_ms = + (self.get_one_profile_wait_rx_sum_ns as f64) / (cnt as f64) / 1_000_000.0; + let avg_signal_ms = + (self.get_one_profile_signal_sum_ns as f64) / (cnt as f64) / 1_000_000.0; + let avg_post_ms = + (self.get_one_profile_post_sum_ns as f64) / (cnt as f64) / 1_000_000.0; + let max_total_ms = (self.get_one_profile_total_max_ns as f64) / 1_000_000.0; + let max_wait_rx_ms = (self.get_one_profile_wait_rx_max_ns as f64) / 1_000_000.0; + let max_signal_ms = (self.get_one_profile_signal_max_ns as f64) / 1_000_000.0; + let max_post_ms = (self.get_one_profile_post_max_ns as f64) / 1_000_000.0; + + tracing::info!( + "[MpscConsumerHandle chan_id={} consumer_idx={}] get_one breakdown: avg_total_ms={:.3} max_total_ms={:.3} avg_wait_rx_ms={:.3} max_wait_rx_ms={:.3} avg_signal_ms={:.3} max_signal_ms={:.3} avg_post_ms={:.3} max_post_ms={:.3} cnt={} recv_calls={} recv_timeouts={} last_prefetch_target={} last_timeout_ms={:?}", + chan_id_for_profile, + consumer_idx_for_profile, + avg_total_ms, + max_total_ms, + avg_wait_rx_ms, + max_wait_rx_ms, + avg_signal_ms, + max_signal_ms, + avg_post_ms, + max_post_ms, + cnt, + self.get_one_profile_recv_calls, + self.get_one_profile_recv_timeouts, + self.get_one_profile_last_prefetch_target, + self.get_one_profile_last_timeout_ms, + ); + + self.inner + .as_ref() + .expect("MpscConsumerHandle inner not initialized") + .observe_get_one_breakdown_window_ms( + avg_total_ms, + max_total_ms, + avg_wait_rx_ms, + max_wait_rx_ms, + avg_signal_ms, + max_signal_ms, + avg_post_ms, + max_post_ms, + cnt, + self.get_one_profile_recv_timeouts, + self.get_one_profile_window_bytes, + ); + + self.get_one_profile_next_log_at = now + Duration::from_secs(30); + self.get_one_profile_cnt = 0; + self.get_one_profile_total_sum_ns = 0; + self.get_one_profile_total_max_ns = 0; + self.get_one_profile_wait_rx_sum_ns = 0; + self.get_one_profile_wait_rx_max_ns = 0; + self.get_one_profile_signal_sum_ns = 0; + self.get_one_profile_signal_max_ns = 0; + self.get_one_profile_post_sum_ns = 0; + self.get_one_profile_post_max_ns = 0; + self.get_one_profile_recv_timeouts = 0; + self.get_one_profile_recv_calls = 0; + self.get_one_profile_window_bytes = 0; + } + Ok(pyobj) + } + + fn get_batch_via_broker_impl( + &mut self, + py: Python<'_>, + broker: BrokerHandle, + batch_size: usize, + timeout_ms: Option, + ) -> PyResult> { + use pyo3::exceptions::PyRuntimeError; + + if self.shutdown.is_closed() { + return Err(crate::error::pyerr_channel_closed( + py, + "consumer is closed", + self.chan_id(), + )); + } + + let chan_id_for_log = self.chan_id(); + let consumer_idx_for_log = self.consumer_idx(); + let runtime = self.kv_runtime.clone(); + let inner = self + .inner + .take() + .ok_or_else(|| PyRuntimeError::new_err("MpscConsumerHandle is already in use"))?; + let (tx, rx) = cbchan::bounded::<( + Result, CoreMpscError>, + CoreMpscConsumer, + )>(1); + + runtime.spawn(async move { + let mut guard = BatchConsumerGuard::new(inner, tx, ShutdownCtl::new()); + let res = { + let fut = guard + .inner_mut() + .get_batch_with_payload_via_broker(&broker, batch_size); + if let Some(ms) = timeout_ms { + match tokio::time::timeout(Duration::from_millis(ms as u64), fut).await { + Ok(result) => result, + Err(_) => Err(CoreMpscError::NoMessage), + } + } else { + fut.await + } + }; + guard.finish(res); + }); + + let wait_begin = Instant::now(); + let mut next_pending_warn_at = wait_begin + GET_ONE_PENDING_WARN_INTERVAL; + let mut recv_calls: u64 = 0; + let mut recv_timeouts: u64 = 0; + + let (result, consumer_back) = loop { + recv_calls += 1; + let recv_res = py.allow_threads(|| rx.recv_timeout(Duration::from_millis(50))); + match recv_res { + Ok(v) => break v, + Err(cbchan::RecvTimeoutError::Timeout) => { + recv_timeouts += 1; + let now = Instant::now(); + if now >= next_pending_warn_at { + warn!( + "[MpscConsumerHandle chan_id={} consumer_idx={}] get_batch still pending: elapsed_ms={} recv_calls={} recv_timeouts={} batch_size={} timeout_ms={:?}", + chan_id_for_log, + consumer_idx_for_log, + wait_begin.elapsed().as_millis(), + recv_calls, + recv_timeouts, + batch_size, + timeout_ms, + ); + next_pending_warn_at = now + GET_ONE_PENDING_WARN_INTERVAL; + } + } + Err(cbchan::RecvTimeoutError::Disconnected) => { + return Err(PyRuntimeError::new_err("get_batch task cancelled")); + } + } + + if let Err(e) = py.check_signals() { + self.shutdown.close(); + return Err(e); + } + }; + + self.inner = Some(consumer_back); + let payloads = match result { + Ok(v) => v, + Err(e) => return Err(map_consumer_error(py, e, chan_id_for_log)), + }; + + let mut objects = Vec::with_capacity(payloads.len()); + for consumed in payloads { + let (pyobj, payload_len) = consumed_payload_to_pyobject(py, consumed)?; + self.get_one_profile_window_bytes += payload_len; + objects.push(pyobj); + } + Ok(objects) + } +} + +#[pymethods] +impl MpscConsumerHandle { + fn chan_id(&self) -> i64 { + self.inner + .as_ref() + .expect("MpscConsumerHandle inner not initialized") + .chan_id() + } + + fn consumer_idx(&self) -> String { + self.inner + .as_ref() + .expect("MpscConsumerHandle inner not initialized") + .consumer_idx() + .to_string() + } + + /// Initialize the global payload callback for this consumer. + /// + /// 回调在 consumer 生命周期内复用；后续 `get_one` / + /// `get_with_payload` 调用都不会再传入回调参数。 + #[pyo3(signature = (callback))] + fn init_payload_callback(&mut self, callback: PyObject) -> PyResult<()> { + use pyo3::exceptions::PyRuntimeError; + use std::sync::Arc; + + let cb: Arc = Arc::new(callback); + let kv_runtime = self.kv_runtime.clone(); + + // Capture identifiers for rate-limited retry logging (diagnostic only). + let mpsc_id_for_log = self.chan_id(); + let parent_mpmc_id_opt = self.parent_mpmc_id_opt; + + // Rate limit helper lives in fluxon_util::limitrate + + let bridge_cb: fluxon_mq::consumer::PayloadCallback = Arc::new( + move |producer_id: String, key: String| { + let cb_for_call = cb.clone(); + let kv_runtime_for_call = kv_runtime.clone(); + Box::pin(async move { + let producer_id_for_call = producer_id.clone(); + let key_for_call = key.clone(); + + let join = limit_thirdparty::tokio::task::spawn_blocking(move || { + // Run the Python callback via a global Python executor. + // This avoids blocking the Tokio scheduler thread. + let (pid_obj, key_obj) = Python::with_gil(|py| { + ( + PyString::new_bound(py, &producer_id_for_call) + .unbind() + .into(), + PyString::new_bound(py, &key_for_call).unbind().into(), + ) + }); + + match fluxon_util::pyo3::run_longtime_py_function( cb_for_call.as_ref(), vec![pid_obj, key_obj], None, @@ -921,6 +1534,7 @@ impl MpscConsumerHandle { } else { CorePayloadResult::Ok(Box::new(PyPayload { inner: obj.clone_ref(py), + cleanup_runtime: kv_runtime_for_call.clone(), })) } }) @@ -1129,8 +1743,10 @@ impl MpscConsumerHandle { let py_wrap_begin = Instant::now(); stage.store(3, Ordering::Relaxed); let payload_owner = match &holder { - KvHolder::Owner(h) => FlatDictDataOwner::UserMemHolder(h.clone()), - KvHolder::External(h) => FlatDictDataOwner::ExternalMemHolder(h.clone()), + KvHolder::Owner(h) => FlatDictDataOwner::from_user_memholder(h.clone()), + KvHolder::External(h) => { + FlatDictDataOwner::from_external_memholder(h.clone()) + } }; let pyobj_res: Result = Python::with_gil(|py| { stage_for_py.store(4, Ordering::Relaxed); @@ -1159,376 +1775,149 @@ impl MpscConsumerHandle { match pyobj_res { Ok(obj) => finalize_payload_result( - CorePayloadResult::Ok(Box::new(PyPayload { inner: obj })), - &stage, - &done, - payload_begin, - &producer_id, - &key, - kv_get_ns, - decode_ns, - py_wrap_ns, - ), - Err(msg) => finalize_payload_result( - CorePayloadResult::NonRetryable(msg), + CorePayloadResult::Ok(Box::new(PyPayload { + inner: obj, + cleanup_runtime: kv_runtime_for_call.clone(), + })), &stage, &done, payload_begin, - &producer_id, - &key, - kv_get_ns, - decode_ns, - py_wrap_ns, - ), - } - }) - }, - ); - - match self.inner.as_mut() { - Some(inner) => { - inner.set_payload_callback(bridge_cb); - Ok(()) - } - None => Err(PyRuntimeError::new_err( - "MpscConsumerHandle inner not initialized", - )), - } - } - - /// New get API that relies on the previously initialized - /// payload callback and returns the Python payload object. - /// - /// `prefetch_target` 用于驱动 Rust 侧预取窗口大小，通常 - /// 由 Python `get_data(batch_size, prefetch_num)` 计算得出。 - /// - /// `timeout_ms` is an optional timeout (milliseconds) for waiting on an - /// available inflight slot. If it fires, the call returns `NoMessage`. - /// - /// Important: once a message is reserved (i.e. an inflight JoinHandle is - /// popped), the call will await it to completion to avoid dropping in-flight - /// fetches and stranding offsets. - #[pyo3(signature = (prefetch_target, timeout_ms=None))] - fn get_one( - &mut self, - py: Python<'_>, - prefetch_target: usize, - timeout_ms: Option, - ) -> PyResult { - use pyo3::exceptions::PyRuntimeError; - use std::time::Duration; - let get_one_begin = std::time::Instant::now(); - let chan_id_for_profile = self.chan_id(); - let consumer_idx_for_profile = self.consumer_idx(); - self.get_one_profile_last_prefetch_target = prefetch_target; - self.get_one_profile_last_timeout_ms = timeout_ms; - if self.shutdown.is_closed() { - return Err(PyRuntimeError::new_err("MpscConsumerHandle is closed")); - } - - let maybe_sync_sub_cluster = { - let now = Instant::now(); - if now >= self.next_sub_cluster_sync_at { - self.next_sub_cluster_sync_at = now + SUB_CLUSTER_SYNC_INTERVAL; - Some( - self.kv_framework - .cluster_manager_view() - .cluster_manager() - .get_self_info() - .sub_cluster - .clone(), - ) - } else { - None - } - }; - - let runtime = self.kv_runtime.clone(); - - let inner = self - .inner - .take() - .ok_or_else(|| PyRuntimeError::new_err("MpscConsumerHandle is already in use"))?; - - let (tx, rx) = - cbchan::bounded::<(Result, CoreMpscConsumer)>(1); - - runtime.spawn(async move { - let mut guard = ConsumerGuard::new(inner, tx); - let (chan_id_for_log, consumer_idx_for_log) = { - let inner_ref = guard.inner_mut(); - (inner_ref.chan_id(), inner_ref.consumer_idx().to_string()) - }; - if let Some(sc) = maybe_sync_sub_cluster { - if let Err(e) = guard.inner_mut().sync_kvclient_sub_cluster(sc.clone()).await { - warn!( - "[MpscConsumer chan_id={} consumer_idx={}] failed to sync kvclient_sub_cluster={:?}: {}; continuing consumption", - chan_id_for_log, consumer_idx_for_log, sc, e - ); - } - } - let res = if let Some(ms) = timeout_ms { - guard - .inner_mut() - .get_with_payload_retry_wait_timeout(prefetch_target, Duration::from_millis(ms as u64)) - .await - } else { - guard.inner_mut().get_with_payload_retry(prefetch_target).await - }; - match &res { - Ok(payload) => { - debug!( - "[MpscConsumerHandle chan_id={} consumer_idx={}] async get finished: producer_id={} nonblocking_hit={}", - chan_id_for_log, - consumer_idx_for_log, - payload.producer_id, - payload.nonblocking_hit, - ); - } - Err(err) => { - debug!( - "[MpscConsumerHandle chan_id={} consumer_idx={}] async get finished with error: {:?}", - chan_id_for_log, - consumer_idx_for_log, - err, - ); - } - } - guard.finish(res); - }); - - let mut wait_rx_ns: u64 = 0; - let mut wait_rx_max_ns: u64 = 0; - let mut signal_ns: u64 = 0; - let mut signal_max_ns: u64 = 0; - let mut recv_timeouts: u64 = 0; - let mut recv_calls: u64 = 0; - let wait_begin = Instant::now(); - let mut next_pending_warn_at = wait_begin + GET_ONE_PENDING_WARN_INTERVAL; - - let (result, consumer_back) = loop { - recv_calls += 1; - let recv_begin = Instant::now(); - let recv_res = py.allow_threads(|| rx.recv_timeout(Duration::from_millis(50))); - let recv_elapsed_ns = recv_begin.elapsed().as_nanos() as u64; - wait_rx_ns += recv_elapsed_ns; - if recv_elapsed_ns > wait_rx_max_ns { - wait_rx_max_ns = recv_elapsed_ns; - } - - match recv_res { - Ok(v) => break v, - Err(cbchan::RecvTimeoutError::Timeout) => { - recv_timeouts += 1; - let now = Instant::now(); - if now >= next_pending_warn_at { - warn!( - "[MpscConsumerHandle chan_id={} consumer_idx={}] get_one still pending: elapsed_ms={} recv_calls={} recv_timeouts={} prefetch_target={} timeout_ms={:?}", - chan_id_for_profile, - consumer_idx_for_profile, - wait_begin.elapsed().as_millis(), - recv_calls, - recv_timeouts, - prefetch_target, - timeout_ms, - ); - next_pending_warn_at = now + GET_ONE_PENDING_WARN_INTERVAL; + &producer_id, + &key, + kv_get_ns, + decode_ns, + py_wrap_ns, + ), + Err(msg) => finalize_payload_result( + CorePayloadResult::NonRetryable(msg), + &stage, + &done, + payload_begin, + &producer_id, + &key, + kv_get_ns, + decode_ns, + py_wrap_ns, + ), } - } - Err(cbchan::RecvTimeoutError::Disconnected) => { - return Err(PyRuntimeError::new_err("get_one task cancelled")); - } - } + }) + }, + ); - let signal_begin = Instant::now(); - let signal_res = py.check_signals(); - let signal_elapsed_ns = signal_begin.elapsed().as_nanos() as u64; - signal_ns += signal_elapsed_ns; - if signal_elapsed_ns > signal_max_ns { - signal_max_ns = signal_elapsed_ns; + match self.inner.as_mut() { + Some(inner) => { + inner.set_payload_callback(bridge_cb); + Ok(()) } + None => Err(PyRuntimeError::new_err( + "MpscConsumerHandle inner not initialized", + )), + } + } - if let Err(e) = signal_res { - self.shutdown.close(); - return Err(e); - } - }; + /// New get API that relies on the previously initialized + /// payload callback and returns the Python payload object. + /// + /// `prefetch_target` 用于驱动 Rust 侧预取窗口大小，通常 + /// 由 Python `get_data(batch_size, prefetch_num)` 计算得出。 + /// + /// `timeout_ms` is an optional timeout (milliseconds) for waiting on an + /// available inflight slot. If it fires, the call returns `NoMessage`. + /// + /// Important: once a message is reserved (i.e. an inflight JoinHandle is + /// popped), the call will await it to completion to avoid dropping in-flight + /// fetches and stranding offsets. + #[pyo3(signature = (prefetch_target, timeout_ms=None))] + fn get_one( + &mut self, + py: Python<'_>, + prefetch_target: usize, + timeout_ms: Option, + ) -> PyResult { + use pyo3::exceptions::PyRuntimeError; - let post_begin = Instant::now(); - self.inner = Some(consumer_back); + let _ = prefetch_target; + let broker = self + .broker + .clone() + .ok_or_else(|| PyRuntimeError::new_err("broker is not initialized"))?; + self.get_one_impl( + py, + ConsumerGetMode::Broker { broker, timeout_ms }, + 0, + timeout_ms, + ) + } - let consumed = match result { - Ok(v) => v, - Err(e) => { - use crate::error::CoreMpscErrorReExport as CoreErr; - return Err(match e { - CoreErr::NoMessage => crate::error::pyerr_message_consumption_no_new_message( - py, - &e.to_string(), - self.chan_id(), - None, - None, - ), - CoreErr::GetPayloadNonRetryable { .. } - | CoreErr::GetPayloadUnknownCode { .. } - | CoreErr::ConsumeOffsetUpdate { .. } - | CoreErr::DeletePayloadNonRetryable { .. } - | CoreErr::DeletePayloadUnknownCode { .. } => { - crate::error::pyerr_message_consumption( - py, - &e.to_string(), - self.chan_id(), - None, - None, - ) - } - CoreErr::PutPayloadNonRetryable | CoreErr::PutPayloadUnknownCode { .. } => { - crate::error::pyerr_chan_message_produce( - py, - &e.to_string(), - self.chan_id(), - None, - None, - ) - } - CoreErr::Etcd(_) => crate::error::pyerr_etcd(py, &e.to_string(), "mpsc_rust"), - CoreErr::JoinError(_) => { - crate::error::pyerr_join_error(py, &e.to_string(), "mpsc_rust") - } - CoreErr::Internal(_) => { - crate::error::pyerr_internal(py, &e.to_string(), "mpsc_rust") - } - }); - } - }; - // Downcast to PyPayload and extract the PyObject - let CoreConsumedPayload { payload, .. } = consumed; - let pyobj = match payload.downcast::() { - Ok(v) => v.inner, - Err(_) => { - return Err(PyRuntimeError::new_err( - "payload type mismatch: expected PyPayload", - )); - } - }; + #[pyo3(signature = (batch_size, prefetch_target, timeout_ms=None))] + fn get_batch( + &mut self, + py: Python<'_>, + batch_size: usize, + prefetch_target: usize, + timeout_ms: Option, + ) -> PyResult> { + use pyo3::exceptions::PyRuntimeError; + + let _ = prefetch_target; + let broker = self + .broker + .clone() + .ok_or_else(|| PyRuntimeError::new_err("broker is not initialized"))?; + self.get_batch_via_broker_impl(py, broker, batch_size, timeout_ms) + } - // English note: - // - MQ payload is expected to be bytes in the common path. - // - If payload is not a `bytes` object, we skip size accounting to avoid guessing. - let payload_len: u64 = { - let any = pyobj.bind(py); - if any.is_instance_of::() { - let b = any - .downcast::() - .expect("PyBytes downcast failed after is_instance_of"); - b.as_bytes().len() as u64 + #[pyo3(signature = (prefetch_target, timeout_ms=None))] + fn get_one_legacy_for_internal_check( + &mut self, + py: Python<'_>, + prefetch_target: usize, + timeout_ms: Option, + ) -> PyResult { + let maybe_sync_sub_cluster = { + let now = Instant::now(); + if now >= self.next_sub_cluster_sync_at { + self.next_sub_cluster_sync_at = now + SUB_CLUSTER_SYNC_INTERVAL; + Some( + self.kv_framework + .cluster_manager_view() + .cluster_manager() + .get_self_info() + .sub_cluster + .clone(), + ) } else { - 0 + None } }; + self.get_one_impl( + py, + ConsumerGetMode::Prefetch { + prefetch_target, + timeout_ms, + maybe_sync_sub_cluster, + }, + prefetch_target, + timeout_ms, + ) + } - let get_one_total = get_one_begin.elapsed(); - let total_ns = get_one_total.as_nanos() as u64; - let post_ns = post_begin.elapsed().as_nanos() as u64; - - self.get_one_profile_cnt += 1; - self.get_one_profile_window_bytes += payload_len; - self.get_one_profile_total_sum_ns += total_ns; - if total_ns > self.get_one_profile_total_max_ns { - self.get_one_profile_total_max_ns = total_ns; - } - self.get_one_profile_wait_rx_sum_ns += wait_rx_ns; - if wait_rx_max_ns > self.get_one_profile_wait_rx_max_ns { - self.get_one_profile_wait_rx_max_ns = wait_rx_max_ns; - } - self.get_one_profile_signal_sum_ns += signal_ns; - if signal_max_ns > self.get_one_profile_signal_max_ns { - self.get_one_profile_signal_max_ns = signal_max_ns; - } - self.get_one_profile_post_sum_ns += post_ns; - if post_ns > self.get_one_profile_post_max_ns { - self.get_one_profile_post_max_ns = post_ns; - } - self.get_one_profile_recv_timeouts += recv_timeouts; - self.get_one_profile_recv_calls += recv_calls; - - let now = Instant::now(); - if now >= self.get_one_profile_next_log_at && self.get_one_profile_cnt > 0 { - let cnt = self.get_one_profile_cnt; - let avg_total_ms = - (self.get_one_profile_total_sum_ns as f64) / (cnt as f64) / 1_000_000.0; - let avg_wait_rx_ms = - (self.get_one_profile_wait_rx_sum_ns as f64) / (cnt as f64) / 1_000_000.0; - let avg_signal_ms = - (self.get_one_profile_signal_sum_ns as f64) / (cnt as f64) / 1_000_000.0; - let avg_post_ms = - (self.get_one_profile_post_sum_ns as f64) / (cnt as f64) / 1_000_000.0; - let max_total_ms = (self.get_one_profile_total_max_ns as f64) / 1_000_000.0; - let max_wait_rx_ms = (self.get_one_profile_wait_rx_max_ns as f64) / 1_000_000.0; - let max_signal_ms = (self.get_one_profile_signal_max_ns as f64) / 1_000_000.0; - let max_post_ms = (self.get_one_profile_post_max_ns as f64) / 1_000_000.0; - - tracing::info!( - "[MpscConsumerHandle chan_id={} consumer_idx={}] get_one breakdown: \ -avg_total_ms={:.3} max_total_ms={:.3} \ -avg_wait_rx_ms={:.3} max_wait_rx_ms={:.3} \ -avg_signal_ms={:.3} max_signal_ms={:.3} \ -avg_post_ms={:.3} max_post_ms={:.3} \ -cnt={} recv_calls={} recv_timeouts={} last_prefetch_target={} last_timeout_ms={:?}", - chan_id_for_profile, - consumer_idx_for_profile, - avg_total_ms, - max_total_ms, - avg_wait_rx_ms, - max_wait_rx_ms, - avg_signal_ms, - max_signal_ms, - avg_post_ms, - max_post_ms, - cnt, - self.get_one_profile_recv_calls, - self.get_one_profile_recv_timeouts, - self.get_one_profile_last_prefetch_target, - self.get_one_profile_last_timeout_ms, - ); - - self.inner - .as_ref() - .expect("MpscConsumerHandle inner not initialized") - .observe_get_one_breakdown_window_ms( - avg_total_ms, - max_total_ms, - avg_wait_rx_ms, - max_wait_rx_ms, - avg_signal_ms, - max_signal_ms, - avg_post_ms, - max_post_ms, - cnt, - self.get_one_profile_recv_timeouts, - self.get_one_profile_window_bytes, - ); + fn init_broker(&mut self) -> PyResult<()> { + use pyo3::exceptions::PyRuntimeError; - self.get_one_profile_next_log_at = now + Duration::from_secs(30); - self.get_one_profile_cnt = 0; - self.get_one_profile_total_sum_ns = 0; - self.get_one_profile_total_max_ns = 0; - self.get_one_profile_wait_rx_sum_ns = 0; - self.get_one_profile_wait_rx_max_ns = 0; - self.get_one_profile_signal_sum_ns = 0; - self.get_one_profile_signal_max_ns = 0; - self.get_one_profile_post_sum_ns = 0; - self.get_one_profile_post_max_ns = 0; - self.get_one_profile_recv_timeouts = 0; - self.get_one_profile_recv_calls = 0; - self.get_one_profile_window_bytes = 0; - } - // println!( - // "[MpscConsumer chan_id={}] get_one total duration: {:?}", - // self.chan_id(), - // get_one_total - // ); - Ok(pyobj) + let inner = self + .inner + .as_ref() + .ok_or_else(|| PyRuntimeError::new_err("MpscConsumerHandle inner not initialized"))?; + let broker = connect_distributed_broker(&self.kv_framework); + init_broker_for_channel( + &self.kv_runtime, + &broker, + inner.chan_id(), + inner.channel_capacity(), + )?; + self.broker = Some(broker); + Ok(()) } /// Initialize a delete callback which will be invoked by Rust after @@ -1793,16 +2182,19 @@ cnt={} recv_calls={} recv_timeouts={} last_prefetch_target={} last_timeout_ms={: struct ProducerGuard { inner: Option, tx: Option, CoreMpscProducer)>>, + shutdown: ShutdownCtl, } impl ProducerGuard { fn new( inner: CoreMpscProducer, tx: cbchan::Sender<(Result<(), CoreMpscError>, CoreMpscProducer)>, + shutdown: ShutdownCtl, ) -> Self { Self { inner: Some(inner), tx: Some(tx), + shutdown, } } @@ -1822,12 +2214,12 @@ impl ProducerGuard { impl Drop for ProducerGuard { fn drop(&mut self) { if let (Some(inner), Some(tx)) = (self.inner.take(), self.tx.take()) { - let _ = tx.send(( - Err(CoreMpscError::Internal( - "producer guard dropped unexpectedly".to_string(), - )), - inner, - )); + let err = if self.shutdown.is_closed() { + CoreMpscError::Closed + } else { + CoreMpscError::Internal("producer guard dropped unexpectedly".to_string()) + }; + let _ = tx.send((Err(err), inner)); } } } @@ -1837,16 +2229,19 @@ impl Drop for ProducerGuard { struct ConsumerGuard { inner: Option, tx: Option, CoreMpscConsumer)>>, + shutdown: ShutdownCtl, } impl ConsumerGuard { fn new( inner: CoreMpscConsumer, tx: cbchan::Sender<(Result, CoreMpscConsumer)>, + shutdown: ShutdownCtl, ) -> Self { Self { inner: Some(inner), tx: Some(tx), + shutdown, } } @@ -1885,12 +2280,65 @@ impl ConsumerGuard { impl Drop for ConsumerGuard { fn drop(&mut self) { if let (Some(inner), Some(tx)) = (self.inner.take(), self.tx.take()) { - let _ = tx.send(( - Err(CoreMpscError::Internal( - "consumer guard dropped unexpectedly".to_string(), - )), - inner, - )); + let err = if self.shutdown.is_closed() { + CoreMpscError::Closed + } else { + CoreMpscError::Internal("consumer guard dropped unexpectedly".to_string()) + }; + let _ = tx.send((Err(err), inner)); + } + } +} + +struct BatchConsumerGuard { + inner: Option, + tx: Option< + cbchan::Sender<( + Result, CoreMpscError>, + CoreMpscConsumer, + )>, + >, + shutdown: ShutdownCtl, +} + +impl BatchConsumerGuard { + fn new( + inner: CoreMpscConsumer, + tx: cbchan::Sender<( + Result, CoreMpscError>, + CoreMpscConsumer, + )>, + shutdown: ShutdownCtl, + ) -> Self { + Self { + inner: Some(inner), + tx: Some(tx), + shutdown, + } + } + + fn inner_mut(&mut self) -> &mut CoreMpscConsumer { + self.inner + .as_mut() + .expect("BatchConsumerGuard inner already taken") + } + + fn finish(mut self, res: Result, CoreMpscError>) { + if let (Some(inner), Some(tx)) = (self.inner.take(), self.tx.take()) { + let _ = tx.send((res, inner)); + } + } +} + +impl Drop for BatchConsumerGuard { + fn drop(&mut self) { + if let (Some(inner), Some(tx)) = (self.inner.take(), self.tx.take()) { + let err = if self.shutdown.is_closed() { + CoreMpscError::Closed + } else { + CoreMpscError::Internal("batch consumer guard dropped unexpectedly".to_string()) + }; + let _ = tx.send((Err(err), inner)); } } } diff --git a/fluxon_rs/fluxon_util/src/lease_manager/lease_handle.rs b/fluxon_rs/fluxon_util/src/lease_manager/lease_handle.rs index 1c24cf2..af70d23 100755 --- a/fluxon_rs/fluxon_util/src/lease_manager/lease_handle.rs +++ b/fluxon_rs/fluxon_util/src/lease_manager/lease_handle.rs @@ -70,20 +70,16 @@ impl GeneralLease { impl Drop for GeneralLease { fn drop(&mut self) { - // Instrument drop of the high-level lease handle so we can correlate - // who released the last user-visible handle. let lease_id = self.id(); let kind_str = match self.kind() { LeaseType::Etcd => "Etcd", LeaseType::KvClient => "KvClient", }; let label = super::lifecycle::get_register_by(lease_id); - let bt = std::backtrace::Backtrace::force_capture(); - tracing::info!( + tracing::debug!( lease_id, kind = kind_str, label = %label.clone().unwrap_or_else(|| "".to_string()), - backtrace = %format!("{:?}", bt), "GeneralLease drop: releasing user-visible lease handle", ); // AutoCleanMapEntry drop happens after this method returns; the map diff --git a/fluxon_test_stack/test_runner.py b/fluxon_test_stack/test_runner.py index 1a5ca7f..e7c37c3 100644 --- a/fluxon_test_stack/test_runner.py +++ b/fluxon_test_stack/test_runner.py @@ -158,6 +158,7 @@ ) CI_CLUSTER_RUNTIME_REMOTE_STAGE_VERIFY_RELPATHS = ( "src/fluxon_py/runtime/start_master.py", + "src/fluxon_py/runtime/start_broker.py", "src/fluxon_py/runtime/start_owner_kvclient.py", ) CI_RUNNER_REMOTE_STAGE_INCLUDE_RELPATHS = ( diff --git a/setup_and_pack/nix/pack_fluxonkv_pylib.py b/setup_and_pack/nix/pack_fluxonkv_pylib.py index e12f8fe..69ee96d 100644 --- a/setup_and_pack/nix/pack_fluxonkv_pylib.py +++ b/setup_and_pack/nix/pack_fluxonkv_pylib.py @@ -587,6 +587,8 @@ def main() -> int: target_cache_key = _sha256_json_bytes(raw=target_cache_key_descriptor) target_cache_dir = layout.target_caches_root_dir / target_cache_key target_cache_manifest_path = target_cache_dir / TARGET_CACHE_MANIFEST_FILE_NAME + _ensure_host_path_writable(layout.target_caches_root_dir) + _ensure_host_path_writable(target_cache_dir) _maybe_promote_compatible_target_cache_dir( target_caches_root=layout.target_caches_root_dir, target_cache_dir=target_cache_dir, @@ -1348,7 +1350,6 @@ def _build_docker_argv( if prepare_build_scenario is not None: container_lines.extend( [ - "export FLUXON_PREPARE_BUILD_SKIP_EXISTING_VENDOR_RUNTIME=1", f"python3 /workspace/setup_and_pack/pub_prepare_build.py --scenario {shlex.quote(prepare_build_scenario)}", "if [ -x \"$CARGO_TARGET_DIR/cxxpacked/bin/protoc\" ]; then", " export PROTOC=\"$CARGO_TARGET_DIR/cxxpacked/bin/protoc\"", @@ -3044,6 +3045,16 @@ def _sudo_remove_tree(path: Path) -> None: ) +def _ensure_host_path_writable(path: Path) -> None: + if not path.exists(): + return + host_sudo = host_sudo_prefix() + subprocess.run( + host_sudo + ["chmod", "777", str(path)], + check=True, + ) + + def _replace_workspace_entry(*, link_path: Path, target_path: str) -> None: normalized_target_path = Path(os.path.abspath(target_path)) normalized_link_path = Path(os.path.abspath(link_path)) diff --git a/setup_and_pack/pack_release.py b/setup_and_pack/pack_release.py index 4d99163..3ae4195 100644 --- a/setup_and_pack/pack_release.py +++ b/setup_and_pack/pack_release.py @@ -138,7 +138,7 @@ def main() -> int: with_tikv_runtime=args.with_tikv_runtime == "true", ) - with script_utils.stage("Seeding profile cache compatibility entries"): + with script_utils.stage("Normalizing profile cache compatibility entries"): _seed_profile_cache_compat_entries(release_dir=release_dir) wheel = _find_single(release_dir, "fluxon-*.whl", "release wheel") @@ -320,18 +320,24 @@ def _seed_invariant_release_runtime( def _seed_profile_cache_compat_entries(*, release_dir: Path) -> None: profiles_dir = release_dir / "profiles" - profiles_dir.mkdir(parents=True, exist_ok=True) + if not profiles_dir.exists(): + return + if not profiles_dir.is_dir(): + raise RuntimeError(f"release profiles path exists but is not a directory: {profiles_dir}") + # Old releases seeded profiles/ -> .. as a compatibility + # alias. That layout is not safe for recursive dispatch because it makes scp walk the + # release root through profiles/ forever. Keep real materialized profile releases, but + # remove the legacy recursive alias when it is present. profile_link = profiles_dir / _FIXED_TRANSPORT_PROFILE_ID expected_target = Path("..") - if profile_link.is_symlink(): - if Path(os.readlink(profile_link)) == expected_target: - return + if profile_link.is_symlink() and Path(os.readlink(profile_link)) == expected_target: profile_link.unlink() - elif profile_link.exists(): - raise RuntimeError(f"profile cache compatibility path already exists and is not a symlink: {profile_link}") - profile_link.symlink_to(expected_target) + try: + next(profiles_dir.iterdir()) + except StopIteration: + profiles_dir.rmdir() def _remove_release_wheels( diff --git a/setup_and_pack/tests/test_pack_fluxonkv_pylib_bridge_prebuilt.py b/setup_and_pack/tests/test_pack_fluxonkv_pylib_bridge_prebuilt.py index bae0e86..796c890 100644 --- a/setup_and_pack/tests/test_pack_fluxonkv_pylib_bridge_prebuilt.py +++ b/setup_and_pack/tests/test_pack_fluxonkv_pylib_bridge_prebuilt.py @@ -291,6 +291,25 @@ def test_build_docker_argv_invokes_container_finalize_script_file(self) -> None: self.assertIn("pack_release_in_container.py", container_cmd) self.assertIn("--wheel-finalize-steps-json", container_cmd) self.assertNotIn("python3 - <<'PY'", container_cmd) + self.assertNotIn("FLUXON_PREPARE_BUILD_SKIP_EXISTING_VENDOR_RUNTIME=1", container_cmd) + + def test_ensure_host_path_writable_chmods_existing_path(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + with mock.patch.object( + _PACKMOD, + "host_sudo_prefix", + return_value=["sudo", "-n"], + ), mock.patch.object( + _PACKMOD.subprocess, + "run", + ) as run_mock: + _PACKMOD._ensure_host_path_writable(path) + + run_mock.assert_called_once_with( + ["sudo", "-n", "chmod", "777", str(path)], + check=True, + ) if __name__ == "__main__": diff --git a/setup_and_pack/tests/test_pack_release_examples_layout.py b/setup_and_pack/tests/test_pack_release_examples_layout.py index 7e04311..052ff4b 100644 --- a/setup_and_pack/tests/test_pack_release_examples_layout.py +++ b/setup_and_pack/tests/test_pack_release_examples_layout.py @@ -104,6 +104,31 @@ def test_main_syncs_external_source_repos_before_packing(self) -> None: check_call.assert_any_call([sys.executable, str(sync_script)], cwd=str(repo_root)) run_pack_steps.assert_called_once() + def test_seed_profile_cache_compat_entries_removes_legacy_recursive_symlink(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + release_dir = Path(tmpdir) / "fluxon_release" + profiles_dir = release_dir / "profiles" + profiles_dir.mkdir(parents=True) + (profiles_dir / _PACK_RELEASE._FIXED_TRANSPORT_PROFILE_ID).symlink_to("..") + + _PACK_RELEASE._seed_profile_cache_compat_entries(release_dir=release_dir) + + self.assertFalse((profiles_dir / _PACK_RELEASE._FIXED_TRANSPORT_PROFILE_ID).exists()) + self.assertFalse(profiles_dir.exists()) + + def test_seed_profile_cache_compat_entries_preserves_materialized_profile_release(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + release_dir = Path(tmpdir) / "fluxon_release" + profile_dir = release_dir / "profiles" / _PACK_RELEASE._FIXED_TRANSPORT_PROFILE_ID + profile_dir.mkdir(parents=True) + (profile_dir / "install.py").write_text("print('install')\n", encoding="utf-8") + (profile_dir / "fluxon_release.sha256").write_text("", encoding="utf-8") + + _PACK_RELEASE._seed_profile_cache_compat_entries(release_dir=release_dir) + + self.assertTrue(profile_dir.is_dir()) + self.assertTrue((profile_dir / "install.py").is_file()) + def test_resolve_examples_dir_prefers_app_examples(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_root = Path(tmpdir) From 697bbf88c96c9f03b9d6fa9806f011ada55a2c30 Mon Sep 17 00:00:00 2001 From: yxrxy Date: Mon, 29 Jun 2026 19:09:26 +0800 Subject: [PATCH 02/14] chore: restore setup and pack from main --- setup_and_pack/nix/pack_fluxonkv_pylib.py | 13 +--------- setup_and_pack/pack_release.py | 22 ++++++---------- ...est_pack_fluxonkv_pylib_bridge_prebuilt.py | 19 -------------- .../test_pack_release_examples_layout.py | 25 ------------------- 4 files changed, 9 insertions(+), 70 deletions(-) diff --git a/setup_and_pack/nix/pack_fluxonkv_pylib.py b/setup_and_pack/nix/pack_fluxonkv_pylib.py index 69ee96d..e12f8fe 100644 --- a/setup_and_pack/nix/pack_fluxonkv_pylib.py +++ b/setup_and_pack/nix/pack_fluxonkv_pylib.py @@ -587,8 +587,6 @@ def main() -> int: target_cache_key = _sha256_json_bytes(raw=target_cache_key_descriptor) target_cache_dir = layout.target_caches_root_dir / target_cache_key target_cache_manifest_path = target_cache_dir / TARGET_CACHE_MANIFEST_FILE_NAME - _ensure_host_path_writable(layout.target_caches_root_dir) - _ensure_host_path_writable(target_cache_dir) _maybe_promote_compatible_target_cache_dir( target_caches_root=layout.target_caches_root_dir, target_cache_dir=target_cache_dir, @@ -1350,6 +1348,7 @@ def _build_docker_argv( if prepare_build_scenario is not None: container_lines.extend( [ + "export FLUXON_PREPARE_BUILD_SKIP_EXISTING_VENDOR_RUNTIME=1", f"python3 /workspace/setup_and_pack/pub_prepare_build.py --scenario {shlex.quote(prepare_build_scenario)}", "if [ -x \"$CARGO_TARGET_DIR/cxxpacked/bin/protoc\" ]; then", " export PROTOC=\"$CARGO_TARGET_DIR/cxxpacked/bin/protoc\"", @@ -3045,16 +3044,6 @@ def _sudo_remove_tree(path: Path) -> None: ) -def _ensure_host_path_writable(path: Path) -> None: - if not path.exists(): - return - host_sudo = host_sudo_prefix() - subprocess.run( - host_sudo + ["chmod", "777", str(path)], - check=True, - ) - - def _replace_workspace_entry(*, link_path: Path, target_path: str) -> None: normalized_target_path = Path(os.path.abspath(target_path)) normalized_link_path = Path(os.path.abspath(link_path)) diff --git a/setup_and_pack/pack_release.py b/setup_and_pack/pack_release.py index 3ae4195..4d99163 100644 --- a/setup_and_pack/pack_release.py +++ b/setup_and_pack/pack_release.py @@ -138,7 +138,7 @@ def main() -> int: with_tikv_runtime=args.with_tikv_runtime == "true", ) - with script_utils.stage("Normalizing profile cache compatibility entries"): + with script_utils.stage("Seeding profile cache compatibility entries"): _seed_profile_cache_compat_entries(release_dir=release_dir) wheel = _find_single(release_dir, "fluxon-*.whl", "release wheel") @@ -320,24 +320,18 @@ def _seed_invariant_release_runtime( def _seed_profile_cache_compat_entries(*, release_dir: Path) -> None: profiles_dir = release_dir / "profiles" - if not profiles_dir.exists(): - return - if not profiles_dir.is_dir(): - raise RuntimeError(f"release profiles path exists but is not a directory: {profiles_dir}") + profiles_dir.mkdir(parents=True, exist_ok=True) - # Old releases seeded profiles/ -> .. as a compatibility - # alias. That layout is not safe for recursive dispatch because it makes scp walk the - # release root through profiles/ forever. Keep real materialized profile releases, but - # remove the legacy recursive alias when it is present. profile_link = profiles_dir / _FIXED_TRANSPORT_PROFILE_ID expected_target = Path("..") - if profile_link.is_symlink() and Path(os.readlink(profile_link)) == expected_target: + if profile_link.is_symlink(): + if Path(os.readlink(profile_link)) == expected_target: + return profile_link.unlink() + elif profile_link.exists(): + raise RuntimeError(f"profile cache compatibility path already exists and is not a symlink: {profile_link}") - try: - next(profiles_dir.iterdir()) - except StopIteration: - profiles_dir.rmdir() + profile_link.symlink_to(expected_target) def _remove_release_wheels( diff --git a/setup_and_pack/tests/test_pack_fluxonkv_pylib_bridge_prebuilt.py b/setup_and_pack/tests/test_pack_fluxonkv_pylib_bridge_prebuilt.py index 796c890..bae0e86 100644 --- a/setup_and_pack/tests/test_pack_fluxonkv_pylib_bridge_prebuilt.py +++ b/setup_and_pack/tests/test_pack_fluxonkv_pylib_bridge_prebuilt.py @@ -291,25 +291,6 @@ def test_build_docker_argv_invokes_container_finalize_script_file(self) -> None: self.assertIn("pack_release_in_container.py", container_cmd) self.assertIn("--wheel-finalize-steps-json", container_cmd) self.assertNotIn("python3 - <<'PY'", container_cmd) - self.assertNotIn("FLUXON_PREPARE_BUILD_SKIP_EXISTING_VENDOR_RUNTIME=1", container_cmd) - - def test_ensure_host_path_writable_chmods_existing_path(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - path = Path(tmpdir) - with mock.patch.object( - _PACKMOD, - "host_sudo_prefix", - return_value=["sudo", "-n"], - ), mock.patch.object( - _PACKMOD.subprocess, - "run", - ) as run_mock: - _PACKMOD._ensure_host_path_writable(path) - - run_mock.assert_called_once_with( - ["sudo", "-n", "chmod", "777", str(path)], - check=True, - ) if __name__ == "__main__": diff --git a/setup_and_pack/tests/test_pack_release_examples_layout.py b/setup_and_pack/tests/test_pack_release_examples_layout.py index 052ff4b..7e04311 100644 --- a/setup_and_pack/tests/test_pack_release_examples_layout.py +++ b/setup_and_pack/tests/test_pack_release_examples_layout.py @@ -104,31 +104,6 @@ def test_main_syncs_external_source_repos_before_packing(self) -> None: check_call.assert_any_call([sys.executable, str(sync_script)], cwd=str(repo_root)) run_pack_steps.assert_called_once() - def test_seed_profile_cache_compat_entries_removes_legacy_recursive_symlink(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - release_dir = Path(tmpdir) / "fluxon_release" - profiles_dir = release_dir / "profiles" - profiles_dir.mkdir(parents=True) - (profiles_dir / _PACK_RELEASE._FIXED_TRANSPORT_PROFILE_ID).symlink_to("..") - - _PACK_RELEASE._seed_profile_cache_compat_entries(release_dir=release_dir) - - self.assertFalse((profiles_dir / _PACK_RELEASE._FIXED_TRANSPORT_PROFILE_ID).exists()) - self.assertFalse(profiles_dir.exists()) - - def test_seed_profile_cache_compat_entries_preserves_materialized_profile_release(self) -> None: - with tempfile.TemporaryDirectory() as tmpdir: - release_dir = Path(tmpdir) / "fluxon_release" - profile_dir = release_dir / "profiles" / _PACK_RELEASE._FIXED_TRANSPORT_PROFILE_ID - profile_dir.mkdir(parents=True) - (profile_dir / "install.py").write_text("print('install')\n", encoding="utf-8") - (profile_dir / "fluxon_release.sha256").write_text("", encoding="utf-8") - - _PACK_RELEASE._seed_profile_cache_compat_entries(release_dir=release_dir) - - self.assertTrue(profile_dir.is_dir()) - self.assertTrue((profile_dir / "install.py").is_file()) - def test_resolve_examples_dir_prefers_app_examples(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_root = Path(tmpdir) From a1399a92f9b24ac53b2826eef005af50e32887ae Mon Sep 17 00:00:00 2001 From: yxrxy Date: Wed, 1 Jul 2026 11:32:39 +0800 Subject: [PATCH 03/14] feat: harden mq broker path and docs --- README.md | 2 +- README_CN.md | 2 +- ...66\351\235\242\351\207\215\346\236\204.md" | 81 ++++ ...71\350\261\241\347\274\223\345\255\230.md" | 153 +++++++ ...04\345\222\214\346\246\202\345\277\265.md" | 2 +- .../User - 1 - Architecture and Concepts.md | 2 +- fluxon_rs/fluxon_kv/src/lib.rs | 4 +- fluxon_rs/fluxon_mq/src/broker.rs | 400 ++++++++++++++---- fluxon_rs/fluxon_mq/src/consumer.rs | 20 +- fluxon_test_stack/test_runner.py | 108 ++++- .../fluxon_architecture.png | Bin .../fluxon_architecture_overview.png | Bin pics/mq_bench.svg | 186 ++++++++ 13 files changed, 854 insertions(+), 106 deletions(-) create mode 100644 "fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" create mode 100644 "fluxon_doc_cn/blog/blog_3_FluxonFS S1\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" rename "pics/fluxon\346\236\266\346\236\204\345\233\27620260423.png" => pics/fluxon_architecture.png (100%) rename "pics/\346\236\266\346\236\204\345\205\250\346\231\257\345\233\276.png" => pics/fluxon_architecture_overview.png (100%) create mode 100644 pics/mq_bench.svg diff --git a/README.md b/README.md index 0c640fd..a67de33 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Fluxon is designed around these problems. It separates data-plane resources, obj - **MQ (Elastic message queue)**: Decouples system dependencies and supports elastic message transport across heterogeneous resource pools - **FS (`S3`-compatible file, object, and cache acceleration system)**: Unifies multi-form storage so one system can cache key-value, file, and object data, while supporting remote access, `S3` forwarding, and large-scale cross-cluster migration for AI data and model files -![](./pics/fluxon架构图20260423.png) +![](./pics/fluxon_architecture.png) diff --git a/README_CN.md b/README_CN.md index 3f978e7..1ef4c20 100644 --- a/README_CN.md +++ b/README_CN.md @@ -21,7 +21,7 @@ Fluxon 的设计正是围绕这些问题展开。它将数据面资源、对象 - **MQ（弹性消息队列）**：解耦系统依赖，支撑异构资源池之间的弹性消息传输 - **FS（兼容 `S3` 的文件、对象与缓存加速系统）**：统一键值、文件、对象三类缓存能力，并支持 AI 数据与模型文件的远端访问、`S3` 转发和跨集群大规模迁移 -![](./pics/fluxon架构图20260423.png) +![](./pics/fluxon_architecture.png)
diff --git "a/fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" "b/fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" new file mode 100644 index 0000000..6b74ecb --- /dev/null +++ "b/fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" @@ -0,0 +1,81 @@ +# FluxonMQ：一次 AI 大 Payload 消息队列的控制面重构 + +AI 训练和推理系统里的消息队列，处理的已经不再是几 KB 的业务事件。在 VAE 解耦训练、数据处理流水线、多模态中间态传递和跨资源池任务交接里，producer 传出去的往往是几十 MB 甚至更大的张量 Payload。consumer 可能动态加入、退出、扩缩容，也可能分布在不同机器、资源池或子集群。FluxonMQ 服务的就是这类场景：让 producer 和 consumer 通过消息语义解耦，同时让大 Payload 继续利用 Fluxon KV owner 的共享内存和跨节点传输路径。 + +在这个设计里，MQ 层负责消息状态，KV 数据面负责 Payload。其中消息状态覆盖消息可见性、in-flight 归属、提交确认、失败重投和清理确认。Payload 保存在 KV owner 管理的内存和传输路径中，consumer 拿到消息后通过 Payload key 读取数据。这种分工让 MQ 可以承载大对象交接。 + +早期 FluxonMQ 使用 etcd 推进消息状态。producer 写入 Payload 后，把消息可见状态写到 etcd；consumer 从 etcd 扫描和抢占消息，读取 Payload 后再写回消费进度。这条路径结构清晰，也复用了 etcd 的一致性和租约能力。问题出现在高并发热路径上：每条消息周围的 ready、claim、inflight、offset、commit 都会形成控制面读写。Payload 传输还在 KV owner 中进行，但消息能否被及时发现、抢占和提交，开始受 etcd 状态推进速度限制。 + +这次 broker 优化针对的就是这段控制面热路径。etcd 仍然负责成员发现、租约、broker 发现和 channel 长期元数据；broker 接管每条消息的排队、抢占、提交、失败放回和清理确认；KV owner 继续负责 Payload 存储和传输。这个拆分把低频集群元数据和高频队列状态分开，让消息推进从外部 KV 存储操作转为 broker 内存状态更新。 + +![](../../pics/fluxon_mq.png) + +## 基础链路：Payload 在 KV，状态在队列 + +早期链路的关键是把 Payload 和消息状态分离。producer 先把大对象写进 KV owner，再把指向 Payload 的消息状态写入 etcd。consumer 从 etcd 扫描可消费消息，完成抢占后拿到 Payload key，再从 KV owner 读取实际数据，处理完成后把消费进度写回 etcd。etcd 只保存消息状态和进度，避免承担大对象存储压力。 + +随着 producer 和 consumer 数量增加，队列状态推进会成为更明显的成本。consumer 为了保持吞吐，会提高 batch size 和 prefetch 深度。prefetch 可以提前发起查找和抢占，但它并没有减少 etcd 上的控制面操作，只是把这些操作前移。高并发下，本地 inflight 能否填深，取决于 etcd 能否持续快速完成可见消息查找、抢占和提交推进。 + +broker 链路把这些状态推进移到 broker 内部。producer 写入前先向 broker 申请 reservation。reservation 是一次写入尝试的占位，broker 返回 `reservation_id` 和 `msg_id`，并记录这条消息预计占用的 Payload bytes。Payload 写入 KV owner 成功后，producer 调用 `publish`，消息进入可消费队列。Payload 写失败时，producer 调用 `abort`，broker 释放占位和字节预算。这个顺序保证了 consumer 只能看到已经写入成功的 Payload。 + +consumer 通过 `fetch` 获取消息。broker 将消息从可消费队列移动到 in-flight，并返回 Payload key。in-flight 表示消息已经被某个 consumer 拿走，但还没有确认消费完成。consumer 读取 Payload 并完成处理后调用 `commit`，这一步成功后，broker 才认为这条消息已经完成消费。后续 Payload 删除或释放完成后，consumer 再发送 `cleanup ack`，broker 释放对应的清理状态。consumer 失败、超时或被取消时，未 commit 的消息会重新放回可消费队列，等待后续投递。 + +这个流程把每条消息的状态推进留在 broker 内存中。`fetch`、`commit`、`requeue` 和 `cleanup ack` 都通过 P2P RPC 调用 broker，broker 更新本地状态后返回结果。etcd 从消息热路径中退出，只处理成员、租约和发现这类低频职责。 + +## broker 的进程边界 + +broker 作为独立进程运行，长期维护 MQ 队列状态。它的生命周期独立于 producer、consumer 和 KV owner。master 继续负责集群控制、租约和 owner 管理，broker 负责高频消息排队。把 broker 放在独立进程里，可以避免 MQ 热路径占用 master，并减少 master 故障和 MQ 队列状态之间的耦合。 + +当前实现中，broker 底层通信身份复用 external client，没有新增 closed runtime 角色。MQ 业务身份通过 member metadata 中的 `fluxon_mq_component=broker` 标记。broker 不注册 segment，不贡献共享内存，也不拥有 Payload。producer 和 consumer 通过 broker discovery 找到 broker，再用 P2P RPC 调用 broker。 + +这个边界保留了 Fluxon 现有通信层结构。broker 不会被 master 当作 KV owner 等待 segment 注册，P2P relay 和 external client 接入规则也可以继续复用。MQ 增加了一个控制面进程，但没有扩展一套新的底层角色体系。 + +## 实现结构 + +Rust 侧的 broker 状态位于 `fluxon_rs/fluxon_mq/src/broker.rs`。它维护 `pending`、`visible`、`inflight`、`cleanup` 和 `cleanup_inflight` 等队列。`pending` 保存已 reserve 但尚未 publish 的消息，`visible` 保存可被 consumer 获取的消息，`inflight` 保存已 fetch 但尚未 commit 的消息，`cleanup` 和 `cleanup_inflight` 保存已提交但仍等待 Payload 清理确认的消息。broker 保存消息信封、Payload key、容量计数和字节预算，不保存 Payload bytes。 + +producer 热路径位于 `fluxon_rs/fluxon_mq/src/producer.rs`。新的写入流程是 `reserve`、写 KV Payload、`publish`。当 broker 满或 Payload byte budget 满时，producer 在 Rust 热路径内退避重试，避免把可恢复的背压错误抛到 Python 外层，再由 Python 固定 sleep 后同步重试。这个调整减少了高并发下的 RPC 冲击，也让 producer 的等待逻辑更贴近真实队列状态。 + +consumer 热路径位于 `fluxon_rs/fluxon_mq/src/consumer.rs` 和 `fluxon_rs/fluxon_pyo3/src/mpsc.rs`。consumer 从 broker `fetch` 消息，读取 Payload，随后 `commit` 并执行 cleanup。Python 层主要负责 API 包装、bench 编排和 teardown；消息推进已经迁移到 Rust 和 broker 路径。 + +MPMC bench 的清理逻辑位于 `fluxon_py/tests/test_api_chan_mpmc/test_mpmc_simple_bench.py`。teardown 时会删除本轮 MPMC 子 MPSC channel，并继续删除 broker 返回的 Payload keys。这样可以同时释放 broker byte budget 和 KV owner 中的实际 Payload，避免连续 case 后 owner pool 被旧数据占住。 + +## 性能结果 + +测试环境为单机，owner pool 为 `100GB`，channel capacity 为 `4096`，低日志运行，Payload 为 DLPack 数据。对比对象是 etcd 队列推进和 broker 队列推进，两边使用相同的 producer、consumer、batch、prefetch 和 Payload 参数。 + +![](../../pics/mq_bench.svg) + +| case | P/C | batch/prefetch | Payload | etcd MB/s | broker MB/s | 变化 | +| --- | ---: | ---: | --- | ---: | ---: | ---: | +| 01 | 16/8 | 40/40 | 4.8MB | 7660.80 | 8010.24 | +4.6% | +| 02 | 16/12 | 40/40 | 4.8MB | 7372.80 | 9496.80 | +28.8% | +| 03 | 24/8 | 40/40 | 4.8MB | 7046.40 | 7350.24 | +4.3% | +| 04 | 16/8 | 40/120 | 4.8MB | 6931.20 | 9791.52 | +41.3% | +| 05 | 16/4 | 40/40 | 4.8MB | 7756.80 | 8294.40 | +6.9% | +| 06 | 16/2 | 40/40 | 4.8MB | 6201.60 | 5875.20 | -5.3% | +| 07 | 16/4 | 48/48 | 4.8MB | 7925.76 | 8155.68 | +2.9% | +| 08 | 16/4 | 64/64 | 4.8MB | 7802.88 | 8382.24 | +7.4% | +| 09 | 16/4 | 48/48 | 8MB | 12441.60 | 14153.60 | +13.8% | +| 10 | 16/4 | 48/48 | 12MB | 17625.60 | 18356.40 | +4.1% | +| 11 | 16/4 | 48/48 | 16MB | 22041.60 | 26102.40 | +18.4% | +| 12 | 16/4 | 48/48 | 20MB | 26016.00 | 18222.00 | -30.0% | +| 13 | 16/4 | 48/48 | 24MB | 29030.40 | 46552.80 | +60.4% | +| 14 | 16/4 | 48/48 | 32MB | 34252.80 | 56624.00 | +65.3% | +| 15 | 24/4 | 48/48 | 32MB | 42393.60 | 44067.20 | +3.9% | +| 16 | 32/4 | 48/48 | 32MB | 35328.00 | 42198.40 | +19.4% | +| 17 | 24/2 | 48/48 | 32MB | 17817.60 | 36969.60 | +107.5% | +| 18 | 24/4 | 48/48 | 40MB | 51264.00 | 63656.00 | +24.2% | +| 19 | 24/4 | 48/48 | 48MB | 54835.20 | 54451.20 | -0.7% | +| 20 | 24/4 | 48/48 | 56MB | 57792.00 | 85254.40 | +47.5% | +| 21 | 24/4 | 48/48 | 64MB | 48844.80 | 89952.00 | +84.2% | + +小 Payload 下，broker 的收益取决于并发组织。`16p/12c b40/pf40 4.8MB` 从 `7372.80 MB/s` 提升到 `9496.80 MB/s`，提升 `28.8%`；`16p/8c b40/pf120 4.8MB` 从 `6931.20 MB/s` 提升到 `9791.52 MB/s`，提升 `41.3%`。这些点的共同特征是 consumer 或 prefetch 对控制面推进的需求更强，broker 能让本地 inflight 更稳定地填起来。 + +大 Payload 下，控制面阻塞减少后，数据面更容易持续跑满。`24MB` 从 `29030.40 MB/s` 提升到 `46552.80 MB/s`，`32MB` 从 `34252.80 MB/s` 提升到 `56624.00 MB/s`，`56MB` 从 `57792.00 MB/s` 提升到 `85254.40 MB/s`，`64MB` 从 `48844.80 MB/s` 提升到 `89952.00 MB/s`。纯 etcd 路径的最佳点是 `24p/4c b48/pf48 dlpack 56MB`，稳态吞吐 `57792.00 MB/s`；broker 路径的最佳点是 `24p/4c b48/pf48 dlpack 64MB`，稳态吞吐 `89952.00 MB/s`。 + +## 结尾 + +FluxonMQ broker 优化把每条消息的高频状态推进从 etcd 迁到 broker，etcd 保留成员、租约、发现和长期元数据职责，KV owner 继续承载大 Payload 数据面。这个调整让 MQ 控制面更贴近消息运行时状态，也让 Payload 传输继续复用 Fluxon 的共享内存和跨节点数据路径。 + +在单机 `100GB` owner pool 测试中，etcd 路径最高 `57.79GB/s`，broker 路径最高 `89.95GB/s`。更重要的是，队列推进已经从外部 KV 存储读写变成内存状态机更新，为后续多 broker 分片、批量 RPC、跨节点 MQ 和更细粒度容量治理提供了更清晰的演进基础。 diff --git "a/fluxon_doc_cn/blog/blog_3_FluxonFS S1\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" "b/fluxon_doc_cn/blog/blog_3_FluxonFS S1\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" new file mode 100644 index 0000000..aa842a1 --- /dev/null +++ "b/fluxon_doc_cn/blog/blog_3_FluxonFS S1\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" @@ -0,0 +1,153 @@ +# FluxonFS S1：面向 AI 训练访问模式的文件对象缓存 + +FluxonFS 是 Fluxon 面向文件对象访问提供的缓存加速层。它服务的对象包括训练样本、模型文件、checkpoint、高分辨率视频、轨迹数据和远端 export。上层仍然使用文件语义访问数据；下层则复用 Fluxon KV 的共享内存、跨节点传输、容量治理和可观测性能力，让文件对象进入统一的数据面加速底座。 + +## AI 训练为什么需要文件对象缓存 + +AI 训练链路中的文件访问已经超出单一数据集读取。训练任务启动前需要发现目录、扫描样本、读取模型文件；训练过程中会持续加载样本、写入日志和阶段性产物；训练恢复和容灾依赖 checkpoint 的稳定保存与读取。随着数据集规模、模型体积和训练节点数量增长，远端访问、本机缓存、跨节点复用和大文件传输会同时出现在一条训练链路里。 + +常见文件访问可以分成几类： + +| 访问类型 | 典型对象 | 对数据面的要求 | +| --- | --- | --- | +| 训练前加载 | 样本文件、索引文件、配置文件 | 小文件和中等对象重复读取稳定，冷读和热读差距可控 | +| 训练中写入 | 日志、中间结果、阶段性产物 | 写入、关闭和提交路径开销可控 | +| checkpoint | 模型权重、优化器状态、训练快照 | 大块连续写入和整文件读取稳定 | +| 随机读取 | 被打散访问的小对象或切片数据 | 热态随机访问延迟和吞吐稳定 | +| 元数据扫描 | 目录遍历、文件发现、状态查询 | 高并发 `list/stat` 能力稳定 | +| 远端 export | 跨节点或远端目录 | 本机缓存和跨节点传输协同工作 | + +如果这些路径分别依赖远端对象存储、本机临时缓存、独立文件系统和额外同步脚本，训练系统会在多个组件之间反复搬运、落盘和重新索引同一批数据。FluxonFS 的定位是把这些文件对象接入 Fluxon 已有的数据面，让文件访问、KV 缓存和跨节点传输共享一套底层资源。 + +## FluxonFS 的架构位置 + +FluxonFS 建立在 Fluxon KV 服务平面之上。KV 平面提供 `etcd`、`greptime`、`master` 和 `owner`，其中 `owner` 贡献共享内存池并承载跨节点传输。FS 在这条链路上增加 `fs_master` 和 `fs_agent`：`fs_master` 承载 FS 控制面、panel 和 export 快照分发；`fs_agent` 注册 export，并对外提供远端目录访问。用户进程通过 `FluxonFsPatcher` 挂载远端目录后，继续使用 `open()`、`read()`、`write()` 和 `close()` 访问文件。 + +这条架构有两个关键点。 + +第一，FS 角色本身不重新建立一套大对象数据面。文件内容会被切成 `KeyValue` 片段，进入 Fluxon KV 的缓存、传输和容量治理路径。这样，文件对象和 KV 对象可以复用同机共享内存、跨节点 P2P 传输以及统一的观测链路。 + +第二，用户进程仍然以文件语义接入。业务代码面对的是远端 export 和普通文件读写接口，不需要直接感知底层对象切片、owner 放置或跨节点传输路径。这个分层让 FluxonFS 可以同时服务“像文件一样使用”和“像数据面对象一样治理”两个目标。 + +## 测试覆盖的访问模式 + +测试选择了训练链路中最常见的六类文件访问行为： + +| 场景 | 关注点 | 参数 | +| --- | --- | --- | +| `read_baseline` | 训练前样本读取和整文件拉取 | `4KiB x 2000`、`256KiB x 400`、`4MiB x 40`，`iterations=3`；单大文件为 `1GiB x 1`，`iterations=2`，`worker_threads=1` | +| `write_commit_baseline` | 训练产物写入并提交 | `4KiB x 2000`、`256KiB x 400`、`4MiB x 40`，`chunk_size=256KiB`，`iterations=3` | +| `ml_dataloader` | loader 连续读取同一批样本 | `32KiB x 2000`，`epochs=3` | +| `checkpoint_save` | 模型快照保存 | `128MiB x 8`，`chunk_size=256KiB`，`iterations=2` | +| `random_access` | 小对象随机访问 | `4KiB`，`working_set=1000`，`access=500`，`iterations=3` | +| `metadata_scan` | 目录遍历、文件发现和状态查询 | `4KiB x 10000`，`iterations=3` | + +这里的 `cold` 表示这批数据第一次被读取，系统里还没有这批数据的缓存；`warm` 表示同一批数据已经读过，再读一次。`local` 表示访问本机节点上的数据，`remote` 表示跨节点访问数据。`epoch` 只出现在训练加载场景里，表示 loader 把同一批样本完整读过一轮。 + +吞吐用 `MB/s = total_bytes / elapsed_seconds / 1048576` 计算；元数据扫描用 `ops/s = total_ops / elapsed_seconds` 计算。训练加载按 `file_size x file_count x epochs` 计算有效读取量，checkpoint 保存按 `file_size x file_count x iterations` 计算写入量。 + +测试运行在双机环境里。压测机硬件为 `AMD Ryzen Threadripper PRO 7995WX`，`96 cores / 192 threads`，`502 GiB` 内存，`Ubuntu 24.04.1`，内核 `6.17.0-35-generic`。本次 FS 测试使用 `16` 个 worker，总 inflight 为 `64`。小文件场景统计 `30s`，checkpoint 和单大文件场景统计 `120s`。 + +## 读路径 + +训练开始前，loader 往往会反复读取大量样本文件。样本可能是 `4KiB` 级的小对象，也可能是 `256KiB` 到 `4MiB` 的中大对象。本次顺序读测试把本机数据和远端数据分开看。 + +| 测试项 | FluxonFS | Alluxio | 差异 | +| --- | ---: | ---: | ---: | +| `4KiB cold local` | 45.5 MB/s | 38.2 MB/s | +19.1% | +| `4KiB warm local` | 61.4 MB/s | 22.0 MB/s | +179.1% | +| `256KiB cold local` | 2148.1 MB/s | 1741.6 MB/s | +23.3% | +| `256KiB warm local` | 3286.3 MB/s | 1366.2 MB/s | +140.5% | +| `4MiB cold local` | 2096.3 MB/s | 6367.1 MB/s | -67.1% | +| `4MiB warm local` | 1676.0 MB/s | 6154.1 MB/s | -72.8% | +| `4KiB cold remote` | 44.0 MB/s | 35.4 MB/s | +24.3% | +| `4KiB warm remote` | 60.1 MB/s | 24.3 MB/s | +147.3% | +| `256KiB cold remote` | 2027.3 MB/s | 1321.3 MB/s | +53.4% | +| `256KiB warm remote` | 3802.1 MB/s | 1328.9 MB/s | +186.1% | +| `4MiB cold remote` | 53.7 MB/s | 14.6 MB/s | +267.8% | +| `4MiB warm remote` | 107.9 MB/s | 6336.0 MB/s | -98.3% | + +FluxonFS 在 `4KiB` 和 `256KiB` 上更稳定，热读优势尤其明显。这类对象更接近样本文件、切片数据和中等粒度训练输入。`4MiB` 顺序读则暴露出边界：本机 cold/warm 两个点 Alluxio 更高，远端 warm 点差距也明显高于同组其它结果。 + +这说明 FluxonFS 适合高频样本和中等对象的重复读取，但 `4MiB` 级连续读还需要继续优化。尤其是 `4MiB warm remote` 这个点，FluxonFS 为 `107.9 MB/s`，Alluxio 为 `6336.0 MB/s`，差距明显大于同组其它结果，后续复测应优先确认。 + +## 写路径：本机提交写优势明显，远端大块写仍需优化 + +训练过程中会持续写入中间结果、日志和阶段性产物。`write_commit_baseline` 看的是文件写入并完成提交后的吞吐。 + +| 测试项 | FluxonFS | Alluxio | 差异 | +| --- | ---: | ---: | ---: | +| `4KiB local` | 21.2 MB/s | 5.7 MB/s | +271.9% | +| `256KiB local` | 1920.7 MB/s | 247.4 MB/s | +676.4% | +| `4MiB local` | 11634.0 MB/s | 4515.2 MB/s | +157.7% | +| `4KiB remote` | 12.9 MB/s | 5.3 MB/s | +143.4% | +| `256KiB remote` | 51.6 MB/s | 142.6 MB/s | -63.8% | +| `4MiB remote` | 78.7 MB/s | 254.6 MB/s | -69.1% | + +本机写入是 FluxonFS 的优势场景，`256KiB` 和 `4MiB` 都明显高于 Alluxio。远端写入里，FluxonFS 仍然在 `4KiB` 上占优，但 `256KiB` 和 `4MiB` 落后。这个结果表明，跨节点大块写入需要进一步优化。 + +## 训练加载：整体接近持平 + +`ml_dataloader` 场景让 loader 连续读取同一批样本 `3` 轮，更接近训练时持续取样的状态。 + +| 测试项 | FluxonFS | Alluxio | 差异 | +| --- | ---: | ---: | ---: | +| `local` | 65.7 MB/s | 59.0 MB/s | +11.4% | +| `remote` | 65.7 MB/s | 69.3 MB/s | -5.2% | + +FluxonFS 和 Alluxio 在这组 loader 测试里接近持平。本机数据 FluxonFS 略高，远端数据 Alluxio 略高，整体差距不大。 + +## Checkpoint：本机快照保存优势明显 + +checkpoint 保存对应模型快照写入，重点看大块连续写能否跑出足够高的吞吐。 + +| 测试项 | FluxonFS | Alluxio | 差异 | +| --- | ---: | ---: | ---: | +| `local` | 32165.9 MB/s | 3407.0 MB/s | +844.1% | +| `remote` | 95.6 MB/s | 60.5 MB/s | +58.0% | + +本机 checkpoint 写入中，FluxonFS 高出 Alluxio 一个数量级。远端写入也保持领先，但优势没有本机明显。这个差异也提醒我们：引用 checkpoint 结果时需要同时写清 `local` 和 `remote`，不能只拿本机峰值代表全部写入路径。 + +## 随机访问：热态小对象优势更明显 + +随机访问模拟小对象被打散读取的情况。这个场景重点看数据读过一次之后，再次访问是否还能保持稳定。 + +| 测试项 | FluxonFS | Alluxio | 差异 | +| --- | ---: | ---: | ---: | +| `cold local` | 37.1 MB/s | 36.6 MB/s | +1.4% | +| `warm local` | 41.0 MB/s | 23.5 MB/s | +74.5% | +| `cold remote` | 37.4 MB/s | 36.3 MB/s | +3.0% | +| `warm remote` | 37.7 MB/s | 27.1 MB/s | +39.1% | + +第一次读取时两者接近。再次读取时 FluxonFS 领先更明显，说明小对象重复访问的路径更稳定。这类结果更贴近高频样本、小文件切片和训练过程中的重复读取。 + +## 元数据扫描：首次目录遍历是短板 + +元数据扫描看的是目录遍历、文件发现和状态查询能力，单位是 `ops/s`，不代表文件内容吞吐。 + +| 测试项 | FluxonFS | Alluxio | 差异 | +| --- | ---: | ---: | ---: | +| `cold local` | 21479.3 ops/s | 62437.3 ops/s | -65.6% | +| `warm local` | 25344.3 ops/s | 23572.9 ops/s | +7.5% | +| `cold remote` | 16660.8 ops/s | 60392.6 ops/s | -72.4% | +| `warm remote` | 20570.5 ops/s | 23618.1 ops/s | -12.9% | + +第一次扫描时 Alluxio 明显更快；再次扫描后差距缩小，本机 warm 场景里 FluxonFS 略高。目录扫描不是 FluxonFS 当前最强的场景。对任务启动前需要大量列目录、探测文件状态的工作流，这个边界需要单独纳入评估。 + +## 需要单独复测的异常点 + +这组数据里有几个点和整体趋势不一致，后续优化和复测应优先覆盖。 + +| 位置 | 现象 | 可能影响 | +| --- | --- | --- | +| `read_baseline`，`4MiB warm remote` | FluxonFS 为 `107.9 MB/s`，Alluxio 为 `6336.0 MB/s`，差距远大于同场景其它点 | 会强烈影响远端大块热读判断，需要单独复测 | +| `read_baseline`，`4MiB cold remote` | FluxonFS 为 `53.7 MB/s`，Alluxio 为 `14.6 MB/s`，两者都明显低于 `256KiB remote` | `4MiB` 远端冷读没有随文件变大提升，可能受链路或测试状态影响 | +| `write_commit_baseline`，`remote` | `4MiB remote` 为 `78.7 MB/s` vs `254.6 MB/s`，远低于本机 `4MiB` 写入 | 跨节点写入和本机写入差距过大，是写路径里最需要解释的点 | +| `checkpoint_save`，`local vs remote` | FluxonFS local 为 `32165.9 MB/s`，remote 为 `95.6 MB/s` | checkpoint 本机写入优势明显，但跨节点落差极大，不能只引用 local 结果 | +| `Plain` 顺序读 | `4MiB x 40` warm local 为 `53164.5 MB/s`，明显高于文件系统对比值 | `Plain` 更像本机缓存或内存上限，不适合作为真实用户侧对比结论 | + +这些点不改变整体结论，但会影响边界解释。性能结论不应只引用优势点，也需要保留异常和限制。对文件对象缓存这类数据面系统来说，异常点通常更能指导下一轮工程优化。 + +## 结尾 + +FluxonFS 这轮测试给出的判断较为明确：在小文件重复读取、本机提交写、checkpoint 保存、随机访问热读和单大文件读取上，FluxonFS 已经表现出文件对象缓存加速层的价值。 diff --git "a/fluxon_doc_cn/user_doc/\347\224\250\346\210\267 - 1 - \346\236\266\346\236\204\345\222\214\346\246\202\345\277\265.md" "b/fluxon_doc_cn/user_doc/\347\224\250\346\210\267 - 1 - \346\236\266\346\236\204\345\222\214\346\246\202\345\277\265.md" index fc1dd8b..47313f0 100644 --- "a/fluxon_doc_cn/user_doc/\347\224\250\346\210\267 - 1 - \346\236\266\346\236\204\345\222\214\346\246\202\345\277\265.md" +++ "b/fluxon_doc_cn/user_doc/\347\224\250\346\210\267 - 1 - \346\236\266\346\236\204\345\222\214\346\246\202\345\277\265.md" @@ -8,7 +8,7 @@ ### 系统全景架构 -![](../../pics/fluxon架构图20260423.png) +![](../../pics/fluxon_architecture.png) 组件视角的全景图，用来定位各组件的职责和依赖关系。 diff --git a/fluxon_doc_en/user_doc/User - 1 - Architecture and Concepts.md b/fluxon_doc_en/user_doc/User - 1 - Architecture and Concepts.md index f0a6417..2a8fd0a 100644 --- a/fluxon_doc_en/user_doc/User - 1 - Architecture and Concepts.md +++ b/fluxon_doc_en/user_doc/User - 1 - Architecture and Concepts.md @@ -8,7 +8,7 @@ This page explains the core concepts and config fields that appear throughout th ### System Overview -![](../../pics/架构全景图.png) +![](../../pics/fluxon_architecture_overview.png) - Control plane / metadata: `etcd + Master` for members, leases, routing, and connection-state metadata - Data plane: `shared memory + transfer engine` for same-host reuse and cross-node data transfer diff --git a/fluxon_rs/fluxon_kv/src/lib.rs b/fluxon_rs/fluxon_kv/src/lib.rs index ac2b61b..4b8ca79 100644 --- a/fluxon_rs/fluxon_kv/src/lib.rs +++ b/fluxon_rs/fluxon_kv/src/lib.rs @@ -1623,6 +1623,8 @@ async fn run_broker_impl( ); } + let config = bootstrap_zero_contribution_client_config(config).await?; + let kv_logs_dir = config .large_file_paths .kv_logs_dir(&config.cluster_name) @@ -1647,8 +1649,6 @@ async fn run_broker_impl( info!("Build version (git commit): {}", build_version); info!("Build version (source-sha256): {}", source_sha256); - let config = bootstrap_zero_contribution_client_config(config).await?; - let mut metadata = HashMap::from([ ("external_client".to_string(), "true".to_string()), ( diff --git a/fluxon_rs/fluxon_mq/src/broker.rs b/fluxon_rs/fluxon_mq/src/broker.rs index 89e2e9b..39685db 100644 --- a/fluxon_rs/fluxon_mq/src/broker.rs +++ b/fluxon_rs/fluxon_mq/src/broker.rs @@ -1,5 +1,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::env; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, OnceLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use bitcode::{Decode, Encode}; @@ -8,7 +10,7 @@ use fluxon_commu::p2p::rpc::{MsgPack, MsgPackSerializePart, RPCCaller, RPCHandle use fluxon_commu::p2p::P2pModuleView; use serde::{Deserialize, Serialize}; use thiserror::Error; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::{mpsc, oneshot, Mutex}; use crate::keys::{self, MqCategory}; use crate::manager::PRODUCE_OFFSET_BEGIN; @@ -25,6 +27,10 @@ const DEFAULT_BROKER_PAYLOAD_BYTES_CAP: u64 = 64 * 1024 * 1024 * 1024; const DEFAULT_BROKER_PAYLOAD_BYTES_CAP_PERCENT: u64 = 60; const DEFAULT_BROKER_CLEANUP_RELEASE_DELAY_MS: u64 = 0; const BROKER_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(15); +const BROKER_RPC_RESPONSE_CACHE_LIMIT: usize = 65536; + +static BROKER_RPC_REQUEST_SEQ: AtomicU64 = AtomicU64::new(1); +static BROKER_RPC_REQUEST_PREFIX: OnceLock = OnceLock::new(); #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)] pub struct BrokerChannelConfig { @@ -461,6 +467,34 @@ impl LocalBroker { self.apply_and_record(record) } + pub fn requeue_inflight_batch( + &mut self, + channel_id: i64, + reservation_ids: Vec, + ) -> Result<(), BrokerError> { + let channel = self.channel(channel_id)?; + let mut seen = HashSet::new(); + for reservation_id in &reservation_ids { + if !seen.insert(*reservation_id) { + return Err(BrokerError::InvalidRecord(format!( + "duplicate requeue reservation_id={} for channel_id={}", + reservation_id, channel_id + ))); + } + if !channel.inflight.contains_key(reservation_id) { + return Err(BrokerError::DeliveryNotFound { + channel_id, + reservation_id: *reservation_id, + }); + } + } + + for reservation_id in reservation_ids.into_iter().rev() { + self.requeue_inflight(channel_id, reservation_id)?; + } + Ok(()) + } + pub fn requeue_all_inflight(&mut self, channel_id: i64) -> Result<(), BrokerError> { let reservation_ids: Vec = self .channel(channel_id)? @@ -1167,6 +1201,11 @@ enum BrokerCommand { reservation_id: u64, reply: oneshot::Sender>, }, + RequeueInflightBatch { + channel_id: i64, + reservation_ids: Vec, + reply: oneshot::Sender>, + }, RequeueAllInflight { channel_id: i64, reply: oneshot::Sender>, @@ -1330,6 +1369,17 @@ impl LocalBrokerHandle { } let _ = reply.send(result); } + BrokerCommand::RequeueInflightBatch { + channel_id, + reservation_ids, + reply, + } => { + let result = broker.requeue_inflight_batch(channel_id, reservation_ids); + if result.is_ok() { + drain_fetch_waiters_for_channel(&mut broker, channel_id); + } + let _ = reply.send(result); + } BrokerCommand::RequeueAllInflight { channel_id, reply } => { let result = broker.requeue_all_inflight(channel_id); if result.is_ok() { @@ -1499,6 +1549,19 @@ impl LocalBrokerHandle { .await } + async fn requeue_inflight_batch( + &self, + channel_id: i64, + reservation_ids: Vec, + ) -> Result<(), BrokerError> { + self.request(|reply| BrokerCommand::RequeueInflightBatch { + channel_id, + reservation_ids, + reply, + }) + .await + } + async fn requeue_all_inflight(&self, channel_id: i64) -> Result<(), BrokerError> { self.request(|reply| BrokerCommand::RequeueAllInflight { channel_id, reply }) .await @@ -1596,6 +1659,10 @@ enum BrokerRpcOperation { channel_id: i64, reservation_id: u64, }, + RequeueInflightBatch { + channel_id: i64, + reservation_ids: Vec, + }, RequeueAllInflight { channel_id: i64, }, @@ -1615,9 +1682,19 @@ enum BrokerRpcOperation { #[derive(Debug, Clone, Default, Encode, Decode)] struct BrokerRpcRequest { + request_id: String, op: BrokerRpcOperation, } +impl BrokerRpcRequest { + fn new(op: BrokerRpcOperation) -> Self { + Self { + request_id: String::new(), + op, + } + } +} + impl MsgPackSerializePart for BrokerRpcRequest { fn msg_id(&self) -> u32 { BROKER_RPC_REQ_MSG_ID @@ -1652,6 +1729,13 @@ struct BrokerRpcResponse { reply: BrokerRpcReply, } +#[derive(Default)] +struct BrokerRpcResponseCache { + completed: HashMap, + completed_order: VecDeque, + in_flight: HashMap>>, +} + impl MsgPackSerializePart for BrokerRpcResponse { fn msg_id(&self) -> u32 { BROKER_RPC_RESP_MSG_ID @@ -1661,6 +1745,7 @@ impl MsgPackSerializePart for BrokerRpcResponse { async fn execute_rpc_request( broker: &LocalBrokerHandle, request: BrokerRpcRequest, + allow_wait: bool, ) -> BrokerRpcResponse { let reply = match request.op { BrokerRpcOperation::Noop => BrokerRpcReply::Unit(Err(BrokerError::Rpc( @@ -1684,9 +1769,15 @@ async fn execute_rpc_request( channel_id, reservation_id, } => BrokerRpcReply::Unit(broker.abort(channel_id, reservation_id).await), - BrokerRpcOperation::FetchNext { req } => { + BrokerRpcOperation::FetchNext { req } if allow_wait => { BrokerRpcReply::Fetch(broker.fetch_next(req).await) } + BrokerRpcOperation::FetchNext { req } => BrokerRpcReply::Fetch( + broker + .fetch_batch_available(req, 1) + .await + .map(|batch| batch.messages.into_iter().next()), + ), BrokerRpcOperation::FetchBatchAvailable { req, max_items } => { BrokerRpcReply::FetchBatch(broker.fetch_batch_available(req, max_items).await) } @@ -1708,6 +1799,14 @@ async fn execute_rpc_request( channel_id, reservation_id, } => BrokerRpcReply::Unit(broker.requeue_inflight(channel_id, reservation_id).await), + BrokerRpcOperation::RequeueInflightBatch { + channel_id, + reservation_ids, + } => BrokerRpcReply::Unit( + broker + .requeue_inflight_batch(channel_id, reservation_ids) + .await, + ), BrokerRpcOperation::RequeueAllInflight { channel_id } => { BrokerRpcReply::Unit(broker.requeue_all_inflight(channel_id).await) } @@ -1727,14 +1826,70 @@ async fn execute_rpc_request( BrokerRpcResponse { reply } } +async fn execute_rpc_request_with_cache( + broker: &LocalBrokerHandle, + response_cache: &Arc>, + request: BrokerRpcRequest, + allow_wait: bool, +) -> BrokerRpcResponse { + let request_id = request.request_id.clone(); + if request_id.is_empty() { + return execute_rpc_request(broker, request, allow_wait).await; + } + + let wait_for_existing = { + let mut cache = response_cache.lock().await; + if let Some(response) = cache.completed.get(&request_id) { + return response.clone(); + } + if let Some(waiters) = cache.in_flight.get_mut(&request_id) { + let (tx, rx) = oneshot::channel(); + waiters.push(tx); + Some(rx) + } else { + cache.in_flight.insert(request_id.clone(), Vec::new()); + None + } + }; + + if let Some(rx) = wait_for_existing { + return rx.await.unwrap_or(BrokerRpcResponse { + reply: BrokerRpcReply::Unit(Err(BrokerError::ActorClosed)), + }); + } + + let response = execute_rpc_request(broker, request, allow_wait).await; + let waiters = { + let mut cache = response_cache.lock().await; + let waiters = cache.in_flight.remove(&request_id).unwrap_or_default(); + cache.completed.insert(request_id.clone(), response.clone()); + cache.completed_order.push_back(request_id); + while cache.completed_order.len() > BROKER_RPC_RESPONSE_CACHE_LIMIT { + if let Some(old_request_id) = cache.completed_order.pop_front() { + cache.completed.remove(&old_request_id); + } + } + waiters + }; + + for waiter in waiters { + let _ = waiter.send(response.clone()); + } + response +} + pub fn register_broker_service(p2p_view: P2pModuleView, queue_capacity: usize) { let broker = LocalBrokerHandle::spawn_actor(LocalBroker::new(), queue_capacity); + let response_cache = Arc::new(Mutex::new(BrokerRpcResponseCache::default())); let handler_view = p2p_view.clone(); RPCHandler::::new().regist(p2p_view.p2p_module(), move |resp, msg| { let broker = broker.clone(); + let response_cache = response_cache.clone(); let handler_view = handler_view.clone(); let _ = handler_view.spawn("fluxon_mq.broker.rpc", async move { - let response = execute_rpc_request(&broker, msg.serialize_part).await; + let response = + execute_rpc_request_with_cache(&broker, &response_cache, msg.serialize_part, false) + .await; let _ = resp .send_resp(MsgPack { serialize_part: response, @@ -1829,9 +1984,9 @@ impl BrokerHandle { pub async fn upsert_channel(&self, config: BrokerChannelConfig) -> Result<(), BrokerError> { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::UpsertChannel { config }, - }) + .request(BrokerRpcRequest::new(BrokerRpcOperation::UpsertChannel { + config, + })) .await? .reply { @@ -1845,9 +2000,9 @@ impl BrokerHandle { pub async fn delete_channel(&self, channel_id: i64) -> Result, BrokerError> { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::DeleteChannel { channel_id }, - }) + .request(BrokerRpcRequest::new(BrokerRpcOperation::DeleteChannel { + channel_id, + })) .await? .reply { @@ -1864,9 +2019,7 @@ impl BrokerHandle { req: BrokerReserveRequest, ) -> Result { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::Reserve { req }, - }) + .request(BrokerRpcRequest::new(BrokerRpcOperation::Reserve { req })) .await? .reply { @@ -1885,13 +2038,11 @@ impl BrokerHandle { now_ms: i64, ) -> Result { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::Publish { - channel_id, - reservation_id, - now_ms, - }, - }) + .request(BrokerRpcRequest::new(BrokerRpcOperation::Publish { + channel_id, + reservation_id, + now_ms, + })) .await? .reply { @@ -1905,12 +2056,10 @@ impl BrokerHandle { pub async fn abort(&self, channel_id: i64, reservation_id: u64) -> Result<(), BrokerError> { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::Abort { - channel_id, - reservation_id, - }, - }) + .request(BrokerRpcRequest::new(BrokerRpcOperation::Abort { + channel_id, + reservation_id, + })) .await? .reply { @@ -1927,9 +2076,7 @@ impl BrokerHandle { req: BrokerFetchRequest, ) -> Result, BrokerError> { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::FetchNext { req }, - }) + .request(BrokerRpcRequest::new(BrokerRpcOperation::FetchNext { req })) .await? .reply { @@ -1947,9 +2094,9 @@ impl BrokerHandle { max_items: usize, ) -> Result { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::FetchBatchAvailable { req, max_items }, - }) + .request(BrokerRpcRequest::new( + BrokerRpcOperation::FetchBatchAvailable { req, max_items }, + )) .await? .reply { @@ -1968,13 +2115,11 @@ impl BrokerHandle { now_ms: i64, ) -> Result { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::Commit { - channel_id, - reservation_id, - now_ms, - }, - }) + .request(BrokerRpcRequest::new(BrokerRpcOperation::Commit { + channel_id, + reservation_id, + now_ms, + })) .await? .reply { @@ -1993,13 +2138,11 @@ impl BrokerHandle { now_ms: i64, ) -> Result { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::CommitBatch { - channel_id, - reservation_ids, - now_ms, - }, - }) + .request(BrokerRpcRequest::new(BrokerRpcOperation::CommitBatch { + channel_id, + reservation_ids, + now_ms, + })) .await? .reply { @@ -2017,18 +2160,39 @@ impl BrokerHandle { reservation_id: u64, ) -> Result<(), BrokerError> { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::RequeueInflight { + .request(BrokerRpcRequest::new(BrokerRpcOperation::RequeueInflight { + channel_id, + reservation_id, + })) + .await? + .reply + { + BrokerRpcReply::Unit(result) => result, + other => Err(BrokerError::Rpc(format!( + "unexpected response for requeue_inflight: {:?}", + other + ))), + } + } + + pub async fn requeue_inflight_batch( + &self, + channel_id: i64, + reservation_ids: Vec, + ) -> Result<(), BrokerError> { + match self + .request(BrokerRpcRequest::new( + BrokerRpcOperation::RequeueInflightBatch { channel_id, - reservation_id, + reservation_ids, }, - }) + )) .await? .reply { BrokerRpcReply::Unit(result) => result, other => Err(BrokerError::Rpc(format!( - "unexpected response for requeue_inflight: {:?}", + "unexpected response for requeue_inflight_batch: {:?}", other ))), } @@ -2036,9 +2200,9 @@ impl BrokerHandle { pub async fn requeue_all_inflight(&self, channel_id: i64) -> Result<(), BrokerError> { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::RequeueAllInflight { channel_id }, - }) + .request(BrokerRpcRequest::new( + BrokerRpcOperation::RequeueAllInflight { channel_id }, + )) .await? .reply { @@ -2056,12 +2220,12 @@ impl BrokerHandle { max_items: usize, ) -> Result, BrokerError> { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::TakeCleanupBatch { + .request(BrokerRpcRequest::new( + BrokerRpcOperation::TakeCleanupBatch { channel_id, max_items, }, - }) + )) .await? .reply { @@ -2079,12 +2243,10 @@ impl BrokerHandle { reservation_id: u64, ) -> Result<(), BrokerError> { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::CleanupAck { - channel_id, - reservation_id, - }, - }) + .request(BrokerRpcRequest::new(BrokerRpcOperation::CleanupAck { + channel_id, + reservation_id, + })) .await? .reply { @@ -2102,12 +2264,10 @@ impl BrokerHandle { reservation_id: u64, ) -> Result<(), BrokerError> { match self - .request(BrokerRpcRequest { - op: BrokerRpcOperation::CleanupNack { - channel_id, - reservation_id, - }, - }) + .request(BrokerRpcRequest::new(BrokerRpcOperation::CleanupNack { + channel_id, + reservation_id, + })) .await? .reply { @@ -2130,14 +2290,20 @@ impl BrokerHandle { async fn request(&self, request: BrokerRpcRequest) -> Result { match &self.inner { - BrokerHandleInner::Local(local) => Ok(execute_rpc_request(local, request).await), + BrokerHandleInner::Local(local) => Ok(execute_rpc_request(local, request, true).await), BrokerHandleInner::Remote(remote) => remote.request(request).await, } } } impl RemoteBrokerHandle { - async fn request(&self, request: BrokerRpcRequest) -> Result { + async fn request( + &self, + mut request: BrokerRpcRequest, + ) -> Result { + if request.request_id.is_empty() { + request.request_id = next_broker_rpc_request_id(); + } let broker_node = find_or_wait_broker_node(self.cluster_manager_view.cluster_manager()).await?; let response = RPCCaller::::new() @@ -2160,6 +2326,7 @@ impl RemoteBrokerHandle { async fn find_or_wait_broker_node( cluster_manager: &fluxon_commu::ClusterManager, ) -> Result { + let mut rx = cluster_manager.listen(); let members = cluster_manager.get_members(); let broker_nodes: Vec<_> = members .iter() @@ -2178,7 +2345,6 @@ async fn find_or_wait_broker_node( ))); } - let mut rx = cluster_manager.listen(); tokio::time::timeout(BROKER_DISCOVERY_TIMEOUT, async move { while let Ok(event) = rx.recv().await { match event { @@ -2204,6 +2370,18 @@ async fn find_or_wait_broker_node( }) } +fn next_broker_rpc_request_id() -> String { + let prefix = BROKER_RPC_REQUEST_PREFIX.get_or_init(|| { + let started_ns = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock is before UNIX_EPOCH") + .as_nanos(); + format!("{}-{}", std::process::id(), started_ns) + }); + let seq = BROKER_RPC_REQUEST_SEQ.fetch_add(1, Ordering::Relaxed); + format!("{}-{}", prefix, seq) +} + fn is_broker_member(member: &fluxon_commu::ClusterMember) -> bool { member .metadata @@ -2323,6 +2501,86 @@ mod tests { } } + #[tokio::test] + async fn rpc_request_cache_deduplicates_retried_reserve() { + let broker = LocalBrokerHandle::spawn_actor_with_cleanup_release_delay( + LocalBroker::new(), + 8, + Duration::ZERO, + ); + let cache = Arc::new(Mutex::new(BrokerRpcResponseCache::default())); + let upsert = BrokerRpcRequest::new(BrokerRpcOperation::UpsertChannel { + config: BrokerChannelConfig { + channel_id: 41, + capacity: 2, + }, + }); + let _ = execute_rpc_request_with_cache(&broker, &cache, upsert, false).await; + + let reserve = BrokerRpcRequest { + request_id: "reserve-retry-1".to_string(), + op: BrokerRpcOperation::Reserve { + req: reserve_req(41, "p0", 10), + }, + }; + let first = execute_rpc_request_with_cache(&broker, &cache, reserve.clone(), false).await; + let second = execute_rpc_request_with_cache(&broker, &cache, reserve, false).await; + let first_reservation = match first.reply { + BrokerRpcReply::Reservation(Ok(reservation)) => reservation, + other => panic!("unexpected first reserve response: {:?}", other), + }; + let second_reservation = match second.reply { + BrokerRpcReply::Reservation(Ok(reservation)) => reservation, + other => panic!("unexpected second reserve response: {:?}", other), + }; + assert_eq!( + first_reservation.envelope.reservation_id, + second_reservation.envelope.reservation_id + ); + + let next = broker.reserve(reserve_req(41, "p0", 11)).await.unwrap(); + assert_eq!(next.envelope.reservation_id, 2); + broker.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn rpc_fetch_next_without_wait_returns_none() { + let broker = LocalBrokerHandle::spawn_actor_with_cleanup_release_delay( + LocalBroker::new(), + 8, + Duration::ZERO, + ); + broker + .upsert_channel(BrokerChannelConfig { + channel_id: 42, + capacity: 2, + }) + .await + .unwrap(); + let cache = Arc::new(Mutex::new(BrokerRpcResponseCache::default())); + let response = tokio::time::timeout( + Duration::from_millis(50), + execute_rpc_request_with_cache( + &broker, + &cache, + BrokerRpcRequest { + request_id: "fetch-empty-1".to_string(), + op: BrokerRpcOperation::FetchNext { + req: fetch_req(42, "c0", 10), + }, + }, + false, + ), + ) + .await + .expect("remote-style fetch must not wait"); + match response.reply { + BrokerRpcReply::Fetch(Ok(None)) => {} + other => panic!("unexpected fetch response: {:?}", other), + } + broker.shutdown().await.unwrap(); + } + #[test] fn reserve_publish_fetch_commit_frees_capacity_for_mpmc_sub() { let mut broker = LocalBroker::new(); diff --git a/fluxon_rs/fluxon_mq/src/consumer.rs b/fluxon_rs/fluxon_mq/src/consumer.rs index b1f262b..8da2bb5 100644 --- a/fluxon_rs/fluxon_mq/src/consumer.rs +++ b/fluxon_rs/fluxon_mq/src/consumer.rs @@ -2803,20 +2803,16 @@ async fn requeue_pending_broker_inflight( chan_id: i64, reservation_ids: Vec, ) { - for reservation_id in reservation_ids.into_iter().rev() { - requeue_broker_inflight_best_effort(broker, chan_id, reservation_id).await; + if reservation_ids.is_empty() { + return; } -} - -async fn requeue_broker_inflight_best_effort( - broker: &BrokerHandle, - chan_id: i64, - reservation_id: u64, -) { - if let Err(err) = broker.requeue_inflight(chan_id, reservation_id).await { + if let Err(err) = broker + .requeue_inflight_batch(chan_id, reservation_ids) + .await + { warn!( - "best-effort broker requeue failed: chan_id={} reservation_id={} err={}", - chan_id, reservation_id, err + "best-effort broker batch requeue failed: chan_id={} err={}", + chan_id, err ); } } diff --git a/fluxon_test_stack/test_runner.py b/fluxon_test_stack/test_runner.py index e7c37c3..de759cb 100644 --- a/fluxon_test_stack/test_runner.py +++ b/fluxon_test_stack/test_runner.py @@ -147,9 +147,9 @@ RUNTIME_LAYER_CASE, ) CI_BASE_RUNTIME_SERVICE_IDS = ("etcd", "greptime") -CI_CLUSTER_MEMBER_INSTANCE_IDS = ("master", "owner_0") -CI_CLUSTER_RUNTIME_INSTANCE_IDS = ("master", "owner_0") -CI_CASE_RUNTIME_INSTANCE_IDS = ("master", "owner_0", "ci_runner") +CI_CLUSTER_MEMBER_INSTANCE_IDS = ("master", "owner_0", "broker") +CI_CLUSTER_RUNTIME_INSTANCE_IDS = ("master", "owner_0", "broker") +CI_CASE_RUNTIME_INSTANCE_IDS = ("master", "owner_0", "broker", "ci_runner") CI_CLUSTER_RUNTIME_REMOTE_STAGE_INCLUDE_RELPATHS = ( "configs", "src/fluxon_py/runtime", @@ -3031,6 +3031,8 @@ def _ci_cluster_runtime_stage(resolved_case: Dict[str, Any]) -> _RemoteRunDirSta verify_relpaths.append("configs/ci_owner_0.yaml") if _ci_has_instance(resolved_case, instance_id="master"): verify_relpaths.append("configs/ci_master.yaml") + if _ci_has_instance(resolved_case, instance_id="broker"): + verify_relpaths.append("configs/ci_broker.yaml") return _RemoteRunDirStage( archive_prefix="fluxon_ci_cluster_runtime_run_dir__", stage_prefix="fluxon_ci_cluster_runtime_stage_", @@ -3047,6 +3049,8 @@ def _ci_runner_runtime_stage(resolved_case: Dict[str, Any]) -> _RemoteRunDirStag verify_relpaths.append("configs/ci_owner_0.yaml") if _ci_has_instance(resolved_case, instance_id="master"): verify_relpaths.append("configs/ci_master.yaml") + if _ci_has_instance(resolved_case, instance_id="broker"): + verify_relpaths.append("configs/ci_broker.yaml") include_relpaths = list(CI_RUNNER_REMOTE_STAGE_INCLUDE_RELPATHS) if _ci_runtime_contract_id(resolved_case) == CI_RUNTIME_CONTRACT_CLUSTER_KV_OWNER: for relpath in ("fluxon_release", "test_rsc"): @@ -3141,15 +3145,34 @@ def _compile_case_plan(resolved_case: Dict[str, Any]) -> _CasePlan: prepare_instance_ids = _ci_cluster_runtime_instance_ids(resolved_case) prepare_phases: Tuple[_RuntimePhase, ...] = () if prepare_instance_ids: - prepare_phases = ( - _RuntimePhase( - phase_id="cluster_runtime", - layer=RUNTIME_LAYER_CASE, - instance_ids=prepare_instance_ids, - write_ctx="CI", - stage_run_dir=_ci_cluster_runtime_stage(resolved_case), - ), + prepare_phase_list: List[_RuntimePhase] = [] + broker_prepare_ids = tuple( + instance_id for instance_id in prepare_instance_ids if instance_id == "broker" + ) + cluster_prepare_ids = tuple( + instance_id for instance_id in prepare_instance_ids if instance_id != "broker" ) + if cluster_prepare_ids: + prepare_phase_list.append( + _RuntimePhase( + phase_id="cluster_runtime", + layer=RUNTIME_LAYER_CASE, + instance_ids=cluster_prepare_ids, + write_ctx="CI", + stage_run_dir=_ci_cluster_runtime_stage(resolved_case), + ) + ) + if broker_prepare_ids: + prepare_phase_list.append( + _RuntimePhase( + phase_id="broker_runtime", + layer=RUNTIME_LAYER_CASE, + instance_ids=broker_prepare_ids, + write_ctx="CI", + stage_run_dir=_ci_cluster_runtime_stage(resolved_case), + ) + ) + prepare_phases = tuple(prepare_phase_list) return _CasePlan( case_family=case_family, prepare_phases=prepare_phases, @@ -3694,6 +3717,9 @@ def _wait_ci_instance_ready(resolved_case: Dict[str, Any], *, instance_id: str) timeout_s=180, ) return + if instance_id == "broker": + _wait_instance_running(resolved_case, instance_id=instance_id, timeout_s=60) + return if instance_id == "ci_runner": _wait_instance_running(resolved_case, instance_id=instance_id, timeout_s=30) return @@ -5962,7 +5988,7 @@ def _validate_profile_ci_runtime_block(runtime: Dict[str, Any], ctx: str, target f"{tpl_ctx}.deployer", ) target = _require_str(deployer.get("target"), f"{tpl_ctx}.deployer.target") - if instance_id in ("owner_0", "ci_runner"): + if instance_id in ("owner_0", "broker", "ci_runner"): if target != "__TARGET__": raise ValueError(f"{tpl_ctx}.deployer.target must be '__TARGET__'") elif target not in target_ip_map: @@ -8922,7 +8948,7 @@ def _ci_materialized_target_for_instance(*, topology: Any, targets: Dict[str, An primary = _require_str(targets.get("primary"), f"{ctx}.targets.primary") if instance_id == "master": return primary - if instance_id in ("owner_0", "ci_runner"): + if instance_id in ("owner_0", "broker", "ci_runner"): if machine_count == 1: return primary if machine_count == 2: @@ -8930,6 +8956,27 @@ def _ci_materialized_target_for_instance(*, topology: Any, targets: Dict[str, An raise ValueError(f"{ctx} unsupported CI instance id for placement: {instance_id}") +def _default_ci_broker_runtime_template() -> Dict[str, Any]: + return { + "lifecycle": "service", + "k8s_ref": "deployment/broker", + "deployer": { + "target": "__TARGET__", + "command": ["/bin/bash", "-lc"], + "args": [ + """ +set -euo pipefail +cd __RUN_DIR__/src +mkdir -p __RUN_DIR__/services/broker +exec __RUN_DIR__/venv/bin/python3 -m fluxon_py.runtime.start_broker \\ + -c __RUN_DIR__/configs/ci_broker.yaml \\ + -w __RUN_DIR__/services/broker +""".strip() + ], + }, + } + + def _compile_ci_case(resolved_case: Dict[str, Any]) -> None: scale = _require_dict(resolved_case.get("scale"), "resolved_case.scale") topology = scale.get("topology") @@ -8940,13 +8987,24 @@ def _compile_ci_case(resolved_case: Dict[str, Any]) -> None: profile_ci = _require_dict(profile.get("ci"), "resolved_case.profile.ci") runtime_templates = _require_dict(profile_ci.get("runtime"), "resolved_case.profile.ci.runtime") deploy = _require_dict(resolved_case.get("deploy"), "resolved_case.deploy") - case_runtime_templates = _require_dict( - runtime_templates.get(RUNTIME_LAYER_CASE), - f"resolved_case.profile.ci.runtime.{RUNTIME_LAYER_CASE}", + case_runtime_templates = copy.deepcopy( + _require_dict( + runtime_templates.get(RUNTIME_LAYER_CASE), + f"resolved_case.profile.ci.runtime.{RUNTIME_LAYER_CASE}", + ) ) + if ( + "master" in case_runtime_templates + and "owner_0" in case_runtime_templates + and "ci_runner" in case_runtime_templates + and "broker" not in case_runtime_templates + ): + case_runtime_templates["broker"] = _default_ci_broker_runtime_template() ordered_instance_ids = [ - instance_id for instance_id in CI_CASE_RUNTIME_INSTANCE_IDS if instance_id in case_runtime_templates + instance_id + for instance_id in CI_CASE_RUNTIME_INSTANCE_IDS + if instance_id in case_runtime_templates ] if not ordered_instance_ids: raise ValueError("resolved_case.profile.ci.runtime.case_runtime must be non-empty") @@ -13970,6 +14028,7 @@ def _write_ci_master_owner_configs( owner_dram_bytes: int, ) -> tuple[Path, Path]: owner_work_root = run_dir / "services" / "owner_0" + broker_work_root = run_dir / "services" / "broker" master_cfg = { "etcd_endpoints": ["__ETCD__"], "cluster_name": cluster_name, @@ -14005,6 +14064,15 @@ def _write_ci_master_owner_configs( }, } + broker_cfg = { + "instance_key": "ci_broker", + "contribute_to_cluster_pool_size": {"dram": 0, "vram": {}}, + "fluxonkv_spec": { + "cluster_name": cluster_name, + "share_mem_path": share_mem_path, + }, + } + etcd_ip = _ci_base_runtime_service_target_ip(resolved_case, service_id="etcd") etcd_port = _ci_base_runtime_service_port(resolved_case, service_id="etcd") greptime_ip = _ci_base_runtime_service_target_ip(resolved_case, service_id="greptime") @@ -14028,8 +14096,12 @@ def _write_ci_master_owner_configs( cfg_dir.mkdir(parents=True, exist_ok=True) master_path = cfg_dir / "ci_master.yaml" owner_path = cfg_dir / "ci_owner_0.yaml" + broker_path = cfg_dir / "ci_broker.yaml" _write_yaml_file(master_path, master_cfg) _write_yaml_file(owner_path, owner_cfg) + if _ci_has_instance(resolved_case, instance_id="broker"): + broker_work_root.mkdir(parents=True, exist_ok=True) + _write_yaml_file(broker_path, broker_cfg) return master_path, owner_path @@ -16256,6 +16328,8 @@ def _ui_ops_logs_base_url(controller_url: str) -> str: def _ui_test_stack_member_role_for_instance_id(instance_id: str) -> str: if instance_id == "master": return "master" + if instance_id == "broker": + return "broker" return "owner_client" diff --git "a/pics/fluxon\346\236\266\346\236\204\345\233\27620260423.png" b/pics/fluxon_architecture.png similarity index 100% rename from "pics/fluxon\346\236\266\346\236\204\345\233\27620260423.png" rename to pics/fluxon_architecture.png diff --git "a/pics/\346\236\266\346\236\204\345\205\250\346\231\257\345\233\276.png" b/pics/fluxon_architecture_overview.png similarity index 100% rename from "pics/\346\236\266\346\236\204\345\205\250\346\231\257\345\233\276.png" rename to pics/fluxon_architecture_overview.png diff --git a/pics/mq_bench.svg b/pics/mq_bench.svg new file mode 100644 index 0000000..f938bf5 --- /dev/null +++ b/pics/mq_bench.svg @@ -0,0 +1,186 @@ + + + + +MQ Benchmark Comparison +Concurrency Sweep +Throughput (MB/s) +Payload Sweep +Throughput (MB/s) + +0 + +11,600 + +23,200 + +34,800 + +46,400 + +58,000 + + + +16p/2c + +16p/4c + +16p/8c + +16p/12c + +24p/2c + +24p/4c + +24p/8c + +32p/4c +Producer / Consumer Concurrency + +0 + +18,000 + +36,000 + +54,000 + +72,000 + +90,000 + + + +4.8 + +8 + +12 + +16 + +20 + +24 + +32 + +40 + +48 + +56 + +64 +Payload Size (MB) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +etcd 4.8MB b1/pf0 + + +broker 4.8MB b1/pf0 + + +etcd 32MB b48/pf48 + + +broker 32MB b48/pf48 + + +etcd 24p/4c + + +broker 24p/4c + + +etcd 16p/4c + + +broker 16p/4c + \ No newline at end of file From 5374a3aee2ce965e7114c8974c888e6041b7a04b Mon Sep 17 00:00:00 2001 From: yxrxy Date: Wed, 1 Jul 2026 11:35:30 +0800 Subject: [PATCH 04/14] fix: doc --- ...44\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename "fluxon_doc_cn/blog/blog_3_FluxonFS S1\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" => "fluxon_doc_cn/blog/blog_3_FluxonFS\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" (100%) diff --git "a/fluxon_doc_cn/blog/blog_3_FluxonFS S1\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" "b/fluxon_doc_cn/blog/blog_3_FluxonFS\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" similarity index 100% rename from "fluxon_doc_cn/blog/blog_3_FluxonFS S1\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" rename to "fluxon_doc_cn/blog/blog_3_FluxonFS\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" From ee159c4769d1bb9e342a3d7815422fb755cd00bf Mon Sep 17 00:00:00 2001 From: yxrxy Date: Wed, 1 Jul 2026 12:01:13 +0800 Subject: [PATCH 05/14] add: doc --- README.md | 4 ++++ README_CN.md | 3 +++ ...7\271\350\261\241\347\274\223\345\255\230.md" | 16 +--------------- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a67de33..4faecbb 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,10 @@ Fluxon is designed around these problems. It separates data-plane resources, obj ![](./pics/fluxon_architecture.png) +## Acknowledgements + +Fluxon learns from and builds on ideas and components from projects including `pplx-gardon`, `iceoryx`, `Alluxio`, `Mooncake`, and `Moka`: local IPC and shared-memory paths, large-object data-plane design, cache governance, and AI-oriented data movement. + ## 🧭 Contents diff --git a/README_CN.md b/README_CN.md index 1ef4c20..76d5690 100644 --- a/README_CN.md +++ b/README_CN.md @@ -23,6 +23,9 @@ Fluxon 的设计正是围绕这些问题展开。它将数据面资源、对象 ![](./pics/fluxon_architecture.png) +## 致谢 + +Fluxon 在设计和实现中学习并参考了 `pplx-gardon`、`iceoryx`、`Alluxio`、`Mooncake`、`Moka` 等项目，包括本机 IPC 与共享内存路径、大对象数据面、缓存治理和 AI 数据流转等方向。
diff --git "a/fluxon_doc_cn/blog/blog_3_FluxonFS\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" "b/fluxon_doc_cn/blog/blog_3_FluxonFS\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" index aa842a1..96abd4e 100644 --- "a/fluxon_doc_cn/blog/blog_3_FluxonFS\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" +++ "b/fluxon_doc_cn/blog/blog_3_FluxonFS\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" @@ -132,21 +132,7 @@ checkpoint 保存对应模型快照写入，重点看大块连续写能否跑出 | `cold remote` | 16660.8 ops/s | 60392.6 ops/s | -72.4% | | `warm remote` | 20570.5 ops/s | 23618.1 ops/s | -12.9% | -第一次扫描时 Alluxio 明显更快；再次扫描后差距缩小，本机 warm 场景里 FluxonFS 略高。目录扫描不是 FluxonFS 当前最强的场景。对任务启动前需要大量列目录、探测文件状态的工作流，这个边界需要单独纳入评估。 - -## 需要单独复测的异常点 - -这组数据里有几个点和整体趋势不一致，后续优化和复测应优先覆盖。 - -| 位置 | 现象 | 可能影响 | -| --- | --- | --- | -| `read_baseline`，`4MiB warm remote` | FluxonFS 为 `107.9 MB/s`，Alluxio 为 `6336.0 MB/s`，差距远大于同场景其它点 | 会强烈影响远端大块热读判断，需要单独复测 | -| `read_baseline`，`4MiB cold remote` | FluxonFS 为 `53.7 MB/s`，Alluxio 为 `14.6 MB/s`，两者都明显低于 `256KiB remote` | `4MiB` 远端冷读没有随文件变大提升，可能受链路或测试状态影响 | -| `write_commit_baseline`，`remote` | `4MiB remote` 为 `78.7 MB/s` vs `254.6 MB/s`，远低于本机 `4MiB` 写入 | 跨节点写入和本机写入差距过大，是写路径里最需要解释的点 | -| `checkpoint_save`，`local vs remote` | FluxonFS local 为 `32165.9 MB/s`，remote 为 `95.6 MB/s` | checkpoint 本机写入优势明显，但跨节点落差极大，不能只引用 local 结果 | -| `Plain` 顺序读 | `4MiB x 40` warm local 为 `53164.5 MB/s`，明显高于文件系统对比值 | `Plain` 更像本机缓存或内存上限，不适合作为真实用户侧对比结论 | - -这些点不改变整体结论，但会影响边界解释。性能结论不应只引用优势点，也需要保留异常和限制。对文件对象缓存这类数据面系统来说，异常点通常更能指导下一轮工程优化。 +第一次扫描时 Alluxio 明显更快；再次扫描后差距缩小，本机 warm 场景里 FluxonFS 略高。 ## 结尾 From 9914aee26dfad1d8e27b5a806eb3b5f6152167cc Mon Sep 17 00:00:00 2001 From: yxrxy Date: Wed, 1 Jul 2026 12:57:03 +0800 Subject: [PATCH 06/14] add: doc --- README.md | 15 +++++++++++---- README_CN.md | 15 +++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4faecbb..6c4d829 100644 --- a/README.md +++ b/README.md @@ -34,10 +34,6 @@ Fluxon is designed around these problems. It separates data-plane resources, obj ![](./pics/fluxon_architecture.png) -## Acknowledgements - -Fluxon learns from and builds on ideas and components from projects including `pplx-gardon`, `iceoryx`, `Alluxio`, `Mooncake`, and `Moka`: local IPC and shared-memory paths, large-object data-plane design, cache governance, and AI-oriented data movement. - ## 🧭 Contents @@ -50,6 +46,7 @@ Fluxon learns from and builds on ideas and components from projects including `p - [Repository Structure](#repository-structure) - [Contributing](#contributing) - [Contributors](#contributors) +- [Acknowledgements](#acknowledgements) - [License](#license) - [Stargazers over time](#stargazers-over-time) @@ -319,6 +316,16 @@ Some earlier contribution records are no longer fully reflected in the current c - `RuileLu`: `KV Lease` support - `Summage`: Initial KV architecture optimization +## Acknowledgements + +- [Eclipse iceoryx2](https://github.com/eclipse-iceoryx/iceoryx2): used by the same-node IPC path for local transport between Fluxon processes. +- [Moka](https://github.com/moka-rs/moka): forked for the cache controller, with additional dynamic-capacity support for global memory governance. +- [Mooncake](https://github.com/kvcache-ai/Mooncake): kept as a `KV Cache` backend wrapper and used as a reference point for large-object KV design and benchmarks. +- [Alluxio](https://github.com/Alluxio/alluxio): referenced in FS design and evaluation for file/object caching and data locality. +- [pplx-garden](https://github.com/perplexityai/pplx-garden): referenced for broader AI data-plane design and high-performance AI infrastructure ideas. + +We thank these projects and their communities for making this work available as open source. + ## 📄 License diff --git a/README_CN.md b/README_CN.md index 76d5690..fca424e 100644 --- a/README_CN.md +++ b/README_CN.md @@ -23,10 +23,6 @@ Fluxon 的设计正是围绕这些问题展开。它将数据面资源、对象 ![](./pics/fluxon_architecture.png) -## 致谢 - -Fluxon 在设计和实现中学习并参考了 `pplx-gardon`、`iceoryx`、`Alluxio`、`Mooncake`、`Moka` 等项目，包括本机 IPC 与共享内存路径、大对象数据面、缓存治理和 AI 数据流转等方向。 -
[![Linux Only](https://img.shields.io/badge/Linux-Only-2ea44f)](#运行要求) @@ -51,6 +47,7 @@ Fluxon 在设计和实现中学习并参考了 `pplx-gardon`、`iceoryx`、`Allu - [项目结构](#项目结构) - [贡献](#贡献) - [Contributors](#contributors) +- [致谢](#致谢) - [许可证](#许可证) - [Star 增长趋势](#star-增长趋势) @@ -320,6 +317,16 @@ ui - `RuileLu`: `KV Lease` 功能支持 - `Summage`: 初始 KV 架构设计优化 +## 致谢 + +- [Eclipse iceoryx2](https://github.com/eclipse-iceoryx/iceoryx2)：用于本机 IPC 路径，支撑同机 Fluxon 进程之间的本地传输。 +- [Moka](https://github.com/moka-rs/moka)：作为缓存控制器的 fork 基础，并扩展动态容量调整能力，用于全局内存治理。 +- [Mooncake](https://github.com/kvcache-ai/Mooncake)：保留为 `KV Cache` backend wrapper，也是大对象 KV 设计和基准测试的重要参考。 +- [Alluxio](https://github.com/Alluxio/alluxio)：用于 FS 设计和评估参考，尤其是文件/对象缓存和数据本地性相关能力。 +- [pplx-garden](https://github.com/perplexityai/pplx-garden)：用于更完整的 AI 数据面设计和高性能 AI 基础设施思路参考。 + +感谢这些项目和社区持续开放高质量的开源工作。 + ## 📄 许可证 From 5e2b6cb7e409218ccc2926fd517aaeed0afe0b7e Mon Sep 17 00:00:00 2001 From: yxrxy Date: Wed, 1 Jul 2026 13:37:18 +0800 Subject: [PATCH 07/14] add: doc --- README.md | 2 +- README_CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6c4d829..37cb21c 100644 --- a/README.md +++ b/README.md @@ -324,7 +324,7 @@ Some earlier contribution records are no longer fully reflected in the current c - [Alluxio](https://github.com/Alluxio/alluxio): referenced in FS design and evaluation for file/object caching and data locality. - [pplx-garden](https://github.com/perplexityai/pplx-garden): referenced for broader AI data-plane design and high-performance AI infrastructure ideas. -We thank these projects and their communities for making this work available as open source. +We thank the maintainers and communities for their open-source work. diff --git a/README_CN.md b/README_CN.md index fca424e..8f58a93 100644 --- a/README_CN.md +++ b/README_CN.md @@ -325,7 +325,7 @@ ui - [Alluxio](https://github.com/Alluxio/alluxio)：用于 FS 设计和评估参考，尤其是文件/对象缓存和数据本地性相关能力。 - [pplx-garden](https://github.com/perplexityai/pplx-garden)：用于更完整的 AI 数据面设计和高性能 AI 基础设施思路参考。 -感谢这些项目和社区持续开放高质量的开源工作。 +感谢维护者和社区的开源工作。 From c035be2c1b1d0a4435f15f36cd3f4ba6a08d8658 Mon Sep 17 00:00:00 2001 From: yxrxy Date: Wed, 1 Jul 2026 14:24:03 +0800 Subject: [PATCH 08/14] docs: split independent docs from broker branch --- README.md | 11 -- README_CN.md | 11 -- ...71\350\261\241\347\274\223\345\255\230.md" | 139 ------------------ 3 files changed, 161 deletions(-) delete mode 100644 "fluxon_doc_cn/blog/blog_3_FluxonFS\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" diff --git a/README.md b/README.md index 37cb21c..a67de33 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,6 @@ Fluxon is designed around these problems. It separates data-plane resources, obj - [Repository Structure](#repository-structure) - [Contributing](#contributing) - [Contributors](#contributors) -- [Acknowledgements](#acknowledgements) - [License](#license) - [Stargazers over time](#stargazers-over-time) @@ -316,16 +315,6 @@ Some earlier contribution records are no longer fully reflected in the current c - `RuileLu`: `KV Lease` support - `Summage`: Initial KV architecture optimization -## Acknowledgements - -- [Eclipse iceoryx2](https://github.com/eclipse-iceoryx/iceoryx2): used by the same-node IPC path for local transport between Fluxon processes. -- [Moka](https://github.com/moka-rs/moka): forked for the cache controller, with additional dynamic-capacity support for global memory governance. -- [Mooncake](https://github.com/kvcache-ai/Mooncake): kept as a `KV Cache` backend wrapper and used as a reference point for large-object KV design and benchmarks. -- [Alluxio](https://github.com/Alluxio/alluxio): referenced in FS design and evaluation for file/object caching and data locality. -- [pplx-garden](https://github.com/perplexityai/pplx-garden): referenced for broader AI data-plane design and high-performance AI infrastructure ideas. - -We thank the maintainers and communities for their open-source work. - ## 📄 License diff --git a/README_CN.md b/README_CN.md index 8f58a93..a138d86 100644 --- a/README_CN.md +++ b/README_CN.md @@ -47,7 +47,6 @@ Fluxon 的设计正是围绕这些问题展开。它将数据面资源、对象 - [项目结构](#项目结构) - [贡献](#贡献) - [Contributors](#contributors) -- [致谢](#致谢) - [许可证](#许可证) - [Star 增长趋势](#star-增长趋势) @@ -317,16 +316,6 @@ ui - `RuileLu`: `KV Lease` 功能支持 - `Summage`: 初始 KV 架构设计优化 -## 致谢 - -- [Eclipse iceoryx2](https://github.com/eclipse-iceoryx/iceoryx2)：用于本机 IPC 路径，支撑同机 Fluxon 进程之间的本地传输。 -- [Moka](https://github.com/moka-rs/moka)：作为缓存控制器的 fork 基础，并扩展动态容量调整能力，用于全局内存治理。 -- [Mooncake](https://github.com/kvcache-ai/Mooncake)：保留为 `KV Cache` backend wrapper，也是大对象 KV 设计和基准测试的重要参考。 -- [Alluxio](https://github.com/Alluxio/alluxio)：用于 FS 设计和评估参考，尤其是文件/对象缓存和数据本地性相关能力。 -- [pplx-garden](https://github.com/perplexityai/pplx-garden)：用于更完整的 AI 数据面设计和高性能 AI 基础设施思路参考。 - -感谢维护者和社区的开源工作。 - ## 📄 许可证 diff --git "a/fluxon_doc_cn/blog/blog_3_FluxonFS\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" "b/fluxon_doc_cn/blog/blog_3_FluxonFS\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" deleted file mode 100644 index 96abd4e..0000000 --- "a/fluxon_doc_cn/blog/blog_3_FluxonFS\357\274\232\351\235\242\345\220\221 AI \350\256\255\347\273\203\350\256\277\351\227\256\346\250\241\345\274\217\347\232\204\346\226\207\344\273\266\345\257\271\350\261\241\347\274\223\345\255\230.md" +++ /dev/null @@ -1,139 +0,0 @@ -# FluxonFS S1：面向 AI 训练访问模式的文件对象缓存 - -FluxonFS 是 Fluxon 面向文件对象访问提供的缓存加速层。它服务的对象包括训练样本、模型文件、checkpoint、高分辨率视频、轨迹数据和远端 export。上层仍然使用文件语义访问数据；下层则复用 Fluxon KV 的共享内存、跨节点传输、容量治理和可观测性能力，让文件对象进入统一的数据面加速底座。 - -## AI 训练为什么需要文件对象缓存 - -AI 训练链路中的文件访问已经超出单一数据集读取。训练任务启动前需要发现目录、扫描样本、读取模型文件；训练过程中会持续加载样本、写入日志和阶段性产物；训练恢复和容灾依赖 checkpoint 的稳定保存与读取。随着数据集规模、模型体积和训练节点数量增长，远端访问、本机缓存、跨节点复用和大文件传输会同时出现在一条训练链路里。 - -常见文件访问可以分成几类： - -| 访问类型 | 典型对象 | 对数据面的要求 | -| --- | --- | --- | -| 训练前加载 | 样本文件、索引文件、配置文件 | 小文件和中等对象重复读取稳定，冷读和热读差距可控 | -| 训练中写入 | 日志、中间结果、阶段性产物 | 写入、关闭和提交路径开销可控 | -| checkpoint | 模型权重、优化器状态、训练快照 | 大块连续写入和整文件读取稳定 | -| 随机读取 | 被打散访问的小对象或切片数据 | 热态随机访问延迟和吞吐稳定 | -| 元数据扫描 | 目录遍历、文件发现、状态查询 | 高并发 `list/stat` 能力稳定 | -| 远端 export | 跨节点或远端目录 | 本机缓存和跨节点传输协同工作 | - -如果这些路径分别依赖远端对象存储、本机临时缓存、独立文件系统和额外同步脚本，训练系统会在多个组件之间反复搬运、落盘和重新索引同一批数据。FluxonFS 的定位是把这些文件对象接入 Fluxon 已有的数据面，让文件访问、KV 缓存和跨节点传输共享一套底层资源。 - -## FluxonFS 的架构位置 - -FluxonFS 建立在 Fluxon KV 服务平面之上。KV 平面提供 `etcd`、`greptime`、`master` 和 `owner`，其中 `owner` 贡献共享内存池并承载跨节点传输。FS 在这条链路上增加 `fs_master` 和 `fs_agent`：`fs_master` 承载 FS 控制面、panel 和 export 快照分发；`fs_agent` 注册 export，并对外提供远端目录访问。用户进程通过 `FluxonFsPatcher` 挂载远端目录后，继续使用 `open()`、`read()`、`write()` 和 `close()` 访问文件。 - -这条架构有两个关键点。 - -第一，FS 角色本身不重新建立一套大对象数据面。文件内容会被切成 `KeyValue` 片段，进入 Fluxon KV 的缓存、传输和容量治理路径。这样，文件对象和 KV 对象可以复用同机共享内存、跨节点 P2P 传输以及统一的观测链路。 - -第二，用户进程仍然以文件语义接入。业务代码面对的是远端 export 和普通文件读写接口，不需要直接感知底层对象切片、owner 放置或跨节点传输路径。这个分层让 FluxonFS 可以同时服务“像文件一样使用”和“像数据面对象一样治理”两个目标。 - -## 测试覆盖的访问模式 - -测试选择了训练链路中最常见的六类文件访问行为： - -| 场景 | 关注点 | 参数 | -| --- | --- | --- | -| `read_baseline` | 训练前样本读取和整文件拉取 | `4KiB x 2000`、`256KiB x 400`、`4MiB x 40`，`iterations=3`；单大文件为 `1GiB x 1`，`iterations=2`，`worker_threads=1` | -| `write_commit_baseline` | 训练产物写入并提交 | `4KiB x 2000`、`256KiB x 400`、`4MiB x 40`，`chunk_size=256KiB`，`iterations=3` | -| `ml_dataloader` | loader 连续读取同一批样本 | `32KiB x 2000`，`epochs=3` | -| `checkpoint_save` | 模型快照保存 | `128MiB x 8`，`chunk_size=256KiB`，`iterations=2` | -| `random_access` | 小对象随机访问 | `4KiB`，`working_set=1000`，`access=500`，`iterations=3` | -| `metadata_scan` | 目录遍历、文件发现和状态查询 | `4KiB x 10000`，`iterations=3` | - -这里的 `cold` 表示这批数据第一次被读取，系统里还没有这批数据的缓存；`warm` 表示同一批数据已经读过，再读一次。`local` 表示访问本机节点上的数据，`remote` 表示跨节点访问数据。`epoch` 只出现在训练加载场景里，表示 loader 把同一批样本完整读过一轮。 - -吞吐用 `MB/s = total_bytes / elapsed_seconds / 1048576` 计算；元数据扫描用 `ops/s = total_ops / elapsed_seconds` 计算。训练加载按 `file_size x file_count x epochs` 计算有效读取量，checkpoint 保存按 `file_size x file_count x iterations` 计算写入量。 - -测试运行在双机环境里。压测机硬件为 `AMD Ryzen Threadripper PRO 7995WX`，`96 cores / 192 threads`，`502 GiB` 内存，`Ubuntu 24.04.1`，内核 `6.17.0-35-generic`。本次 FS 测试使用 `16` 个 worker，总 inflight 为 `64`。小文件场景统计 `30s`，checkpoint 和单大文件场景统计 `120s`。 - -## 读路径 - -训练开始前，loader 往往会反复读取大量样本文件。样本可能是 `4KiB` 级的小对象，也可能是 `256KiB` 到 `4MiB` 的中大对象。本次顺序读测试把本机数据和远端数据分开看。 - -| 测试项 | FluxonFS | Alluxio | 差异 | -| --- | ---: | ---: | ---: | -| `4KiB cold local` | 45.5 MB/s | 38.2 MB/s | +19.1% | -| `4KiB warm local` | 61.4 MB/s | 22.0 MB/s | +179.1% | -| `256KiB cold local` | 2148.1 MB/s | 1741.6 MB/s | +23.3% | -| `256KiB warm local` | 3286.3 MB/s | 1366.2 MB/s | +140.5% | -| `4MiB cold local` | 2096.3 MB/s | 6367.1 MB/s | -67.1% | -| `4MiB warm local` | 1676.0 MB/s | 6154.1 MB/s | -72.8% | -| `4KiB cold remote` | 44.0 MB/s | 35.4 MB/s | +24.3% | -| `4KiB warm remote` | 60.1 MB/s | 24.3 MB/s | +147.3% | -| `256KiB cold remote` | 2027.3 MB/s | 1321.3 MB/s | +53.4% | -| `256KiB warm remote` | 3802.1 MB/s | 1328.9 MB/s | +186.1% | -| `4MiB cold remote` | 53.7 MB/s | 14.6 MB/s | +267.8% | -| `4MiB warm remote` | 107.9 MB/s | 6336.0 MB/s | -98.3% | - -FluxonFS 在 `4KiB` 和 `256KiB` 上更稳定，热读优势尤其明显。这类对象更接近样本文件、切片数据和中等粒度训练输入。`4MiB` 顺序读则暴露出边界：本机 cold/warm 两个点 Alluxio 更高，远端 warm 点差距也明显高于同组其它结果。 - -这说明 FluxonFS 适合高频样本和中等对象的重复读取，但 `4MiB` 级连续读还需要继续优化。尤其是 `4MiB warm remote` 这个点，FluxonFS 为 `107.9 MB/s`，Alluxio 为 `6336.0 MB/s`，差距明显大于同组其它结果，后续复测应优先确认。 - -## 写路径：本机提交写优势明显，远端大块写仍需优化 - -训练过程中会持续写入中间结果、日志和阶段性产物。`write_commit_baseline` 看的是文件写入并完成提交后的吞吐。 - -| 测试项 | FluxonFS | Alluxio | 差异 | -| --- | ---: | ---: | ---: | -| `4KiB local` | 21.2 MB/s | 5.7 MB/s | +271.9% | -| `256KiB local` | 1920.7 MB/s | 247.4 MB/s | +676.4% | -| `4MiB local` | 11634.0 MB/s | 4515.2 MB/s | +157.7% | -| `4KiB remote` | 12.9 MB/s | 5.3 MB/s | +143.4% | -| `256KiB remote` | 51.6 MB/s | 142.6 MB/s | -63.8% | -| `4MiB remote` | 78.7 MB/s | 254.6 MB/s | -69.1% | - -本机写入是 FluxonFS 的优势场景，`256KiB` 和 `4MiB` 都明显高于 Alluxio。远端写入里，FluxonFS 仍然在 `4KiB` 上占优，但 `256KiB` 和 `4MiB` 落后。这个结果表明，跨节点大块写入需要进一步优化。 - -## 训练加载：整体接近持平 - -`ml_dataloader` 场景让 loader 连续读取同一批样本 `3` 轮，更接近训练时持续取样的状态。 - -| 测试项 | FluxonFS | Alluxio | 差异 | -| --- | ---: | ---: | ---: | -| `local` | 65.7 MB/s | 59.0 MB/s | +11.4% | -| `remote` | 65.7 MB/s | 69.3 MB/s | -5.2% | - -FluxonFS 和 Alluxio 在这组 loader 测试里接近持平。本机数据 FluxonFS 略高，远端数据 Alluxio 略高，整体差距不大。 - -## Checkpoint：本机快照保存优势明显 - -checkpoint 保存对应模型快照写入，重点看大块连续写能否跑出足够高的吞吐。 - -| 测试项 | FluxonFS | Alluxio | 差异 | -| --- | ---: | ---: | ---: | -| `local` | 32165.9 MB/s | 3407.0 MB/s | +844.1% | -| `remote` | 95.6 MB/s | 60.5 MB/s | +58.0% | - -本机 checkpoint 写入中，FluxonFS 高出 Alluxio 一个数量级。远端写入也保持领先，但优势没有本机明显。这个差异也提醒我们：引用 checkpoint 结果时需要同时写清 `local` 和 `remote`，不能只拿本机峰值代表全部写入路径。 - -## 随机访问：热态小对象优势更明显 - -随机访问模拟小对象被打散读取的情况。这个场景重点看数据读过一次之后，再次访问是否还能保持稳定。 - -| 测试项 | FluxonFS | Alluxio | 差异 | -| --- | ---: | ---: | ---: | -| `cold local` | 37.1 MB/s | 36.6 MB/s | +1.4% | -| `warm local` | 41.0 MB/s | 23.5 MB/s | +74.5% | -| `cold remote` | 37.4 MB/s | 36.3 MB/s | +3.0% | -| `warm remote` | 37.7 MB/s | 27.1 MB/s | +39.1% | - -第一次读取时两者接近。再次读取时 FluxonFS 领先更明显，说明小对象重复访问的路径更稳定。这类结果更贴近高频样本、小文件切片和训练过程中的重复读取。 - -## 元数据扫描：首次目录遍历是短板 - -元数据扫描看的是目录遍历、文件发现和状态查询能力，单位是 `ops/s`，不代表文件内容吞吐。 - -| 测试项 | FluxonFS | Alluxio | 差异 | -| --- | ---: | ---: | ---: | -| `cold local` | 21479.3 ops/s | 62437.3 ops/s | -65.6% | -| `warm local` | 25344.3 ops/s | 23572.9 ops/s | +7.5% | -| `cold remote` | 16660.8 ops/s | 60392.6 ops/s | -72.4% | -| `warm remote` | 20570.5 ops/s | 23618.1 ops/s | -12.9% | - -第一次扫描时 Alluxio 明显更快；再次扫描后差距缩小，本机 warm 场景里 FluxonFS 略高。 - -## 结尾 - -FluxonFS 这轮测试给出的判断较为明确：在小文件重复读取、本机提交写、checkpoint 保存、随机访问热读和单大文件读取上，FluxonFS 已经表现出文件对象缓存加速层的价值。 From 1ee5d7b04c5d768792ec0e09ff36cea95c4da4be Mon Sep 17 00:00:00 2001 From: yxrxy Date: Wed, 1 Jul 2026 14:28:26 +0800 Subject: [PATCH 09/14] add: doc --- ...66\351\235\242\351\207\215\346\236\204.md" | 46 ++++++++++++++++-- pics/blog2_mq_broker_state.png | Bin 0 -> 43734 bytes pics/blog2_mq_payload_flow.png | Bin 0 -> 50774 bytes 3 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 pics/blog2_mq_broker_state.png create mode 100644 pics/blog2_mq_payload_flow.png diff --git "a/fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" "b/fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" index 6b74ecb..8f5c5c3 100644 --- "a/fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" +++ "b/fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" @@ -32,13 +32,51 @@ broker 作为独立进程运行，长期维护 MQ 队列状态。它的生命周 ## 实现结构 -Rust 侧的 broker 状态位于 `fluxon_rs/fluxon_mq/src/broker.rs`。它维护 `pending`、`visible`、`inflight`、`cleanup` 和 `cleanup_inflight` 等队列。`pending` 保存已 reserve 但尚未 publish 的消息，`visible` 保存可被 consumer 获取的消息，`inflight` 保存已 fetch 但尚未 commit 的消息，`cleanup` 和 `cleanup_inflight` 保存已提交但仍等待 Payload 清理确认的消息。broker 保存消息信封、Payload key、容量计数和字节预算，不保存 Payload bytes。 +Rust 侧的 broker 状态位于 `fluxon_rs/fluxon_mq/src/broker.rs`。这部分实现沿用 KV 设计里的角色边界：`master` 维护集群控制面和路由，`owner` 承载共享内存、对象副本和跨节点传输，producer、consumer 和 broker 都以 `external_client` 身份接入，不贡献 owner 容量。这个边界在 [KV 设计 1 - 概览与分层](../design/kv_1_概览与分层.md) 里有完整说明。 -producer 热路径位于 `fluxon_rs/fluxon_mq/src/producer.rs`。新的写入流程是 `reserve`、写 KV Payload、`publish`。当 broker 满或 Payload byte budget 满时，producer 在 Rust 热路径内退避重试，避免把可恢复的背压错误抛到 Python 外层，再由 Python 固定 sleep 后同步重试。这个调整减少了高并发下的 RPC 冲击，也让 producer 的等待逻辑更贴近真实队列状态。 +broker 保存的是消息控制面状态和 Payload 引用。Payload bytes 仍然由 KV owner 管理，broker 只记录 `payload_key`、`payload_bytes`、消息信封和队列位置。 -consumer 热路径位于 `fluxon_rs/fluxon_mq/src/consumer.rs` 和 `fluxon_rs/fluxon_pyo3/src/mpsc.rs`。consumer 从 broker `fetch` 消息，读取 Payload，随后 `commit` 并执行 cleanup。Python 层主要负责 API 包装、bench 编排和 teardown；消息推进已经迁移到 Rust 和 broker 路径。 +| 结构 | 关键字段 | 含义 | +| --- | --- | --- | +| `BrokerState` | `channels` | 按 `channel_id` 保存每个 channel 的队列状态 | +| `BrokerState` | `payload_byte_capacity` | broker 维度的 Payload byte budget 上限 | +| `BrokerState` | `used_payload_bytes` | 当前所有未释放消息占用的 Payload byte budget | +| `ChannelState` | `config` | `BrokerChannelConfig`，包含 `channel_id` 和 `capacity` | +| `ChannelState` | `next_reservation_id` | channel 内递增的 reservation 编号 | +| `ChannelState` | `next_msg_by_producer` | 每个 `producer_id` 的下一个 `msg_id` | +| `ChannelState` | `pending` | 已 `reserve`、尚未 `publish` 的消息 | +| `ChannelState` | `visible` | 已写入 Payload 且可被 consumer `fetch` 的消息 | +| `ChannelState` | `inflight` / `inflight_order` | 已被 consumer 取走、尚未 `commit` 的消息及其顺序 | +| `ChannelState` | `cleanup` | 已 `commit`、等待 Payload 清理的消息 | +| `ChannelState` | `cleanup_inflight` | 已分配给清理任务、等待 `cleanup_ack` 的消息 | +| `ChannelState` | `committed` | 已提交的 `reservation_id` 集合，用于处理重复提交 | +| `ChannelState` | `used_slots` | channel 当前占用的消息槽位 | +| `ChannelState` | `reserve_waiters` / `fetch_waiters` | 因容量或可见消息不足而等待的请求 | -MPMC bench 的清理逻辑位于 `fluxon_py/tests/test_api_chan_mpmc/test_mpmc_simple_bench.py`。teardown 时会删除本轮 MPMC 子 MPSC channel，并继续删除 broker 返回的 Payload keys。这样可以同时释放 broker byte budget 和 KV owner 中的实际 Payload，避免连续 case 后 owner pool 被旧数据占住。 +broker RPC 和内部状态机使用的主要消息结构如下： + +| 结构 | 字段 | 用途 | +| --- | --- | --- | +| `BrokerReserveRequest` | `channel_id`、`producer_id`、`category`、`payload_bytes`、`now_ms` | producer 申请消息占位和 Payload byte budget | +| `BrokerFetchRequest` | `channel_id`、`consumer_id`、`now_ms` | consumer 请求下一条可见消息 | +| `BrokerEnvelope` | `channel_id`、`producer_id`、`msg_id`、`reservation_id` | 标识一条消息和一次写入 reservation | +| `BrokerEnvelope` | `payload_key`、`payload_bytes` | 指向 KV owner 中的 Payload，并计入 broker byte budget | +| `BrokerEnvelope` | `reserved_at_ms`、`published_at_ms` | 记录消息进入 broker 状态机的时间 | +| `BrokerCommitOutcome` | `first_commit`、`cleanup` | 告诉 consumer 本次提交是否首次生效，以及是否产生清理任务 | + +状态流转可以简化为下面这条链路： + +![](../../pics/blog2_mq_broker_state.png) + +producer 热路径位于 `fluxon_rs/fluxon_mq/src/producer.rs`。broker 路径的写入顺序是 `reserve -> KV put -> publish`。`reserve` 成功后，broker 已经生成 `payload_key` 并扣减 `payload_bytes`；producer 随后把实际 Payload 写入 KV owner。只有 KV 写入成功后，`publish` 才会把消息从 `pending` 推到 `visible`，因此 consumer 只能 fetch 到已经完成 Payload 写入的消息。如果 KV 写入失败，producer 会调用 `abort` 释放 reservation 和 byte budget。 + +当 channel 满或 `payload_byte_capacity` 不足时，producer 在 Rust 热路径里按 `BrokerError::ChannelFull` 或 `BrokerError::PayloadBytesFull` 做退避重试。这个重试发生在 broker reserve 阶段，等待条件直接来自 `used_slots` 和 `used_payload_bytes`，比 Python 外层固定 sleep 更贴近真实队列状态。 + +consumer 热路径位于 `fluxon_rs/fluxon_mq/src/consumer.rs` 和 `fluxon_rs/fluxon_pyo3/src/mpsc.rs`。consumer 先通过 broker `fetch` 取得 `BrokerEnvelope`，再用其中的 `payload_key` 从 KV owner 读取 Payload。业务处理和 commit 完成后，consumer 执行 Payload delete，并通过 cleanup 路径释放 broker 的 byte budget。Python 层主要负责 API 包装、bench 编排和 teardown；消息推进、背压等待和 cleanup 状态已经收敛到 Rust broker 路径。 + +![](../../pics/blog2_mq_payload_flow.png) + +MPMC bench 的清理逻辑位于 `fluxon_py/tests/test_api_chan_mpmc/test_mpmc_simple_bench.py`。teardown 时会删除本轮 MPMC 子 MPSC channel，并继续删除 broker 返回的 Payload keys。这里需要同时处理两类资源：broker 侧的 `used_payload_bytes` 和 KV owner 侧的真实 Payload。前者靠 `cleanup_ack`、`abort` 或 `delete_channel` 释放；后者靠对 `payload_key` 执行 KV delete 释放。两边都释放后，连续 case 才不会被上一轮残留数据占住 owner pool 或 broker byte budget。 ## 性能结果 diff --git a/pics/blog2_mq_broker_state.png b/pics/blog2_mq_broker_state.png new file mode 100644 index 0000000000000000000000000000000000000000..c7e7d98effaf1f687e9295b53a0a7be65fc5625a GIT binary patch literal 43734 zcmZ^KWmp~E*5rW@AV3HtxCaRC?iSo#4-nklJ-9<~clY3K!QI{6-6b&1d++^Zo|%6< zhwj~5*Q!-jJ6Kj)4+23ffMukRAwh4v*9a{De*r0Rifu<->Ug`l|3WuVK$qs)a!f1hi0X5Jf5dF9L9j+JNF@??3lTp5Wz== zqF?`^qoboyXU-3TVm4O(SyGgh1##AfZTHtk=S=tB743y!B(`M(1O(DF0q~3cRyga+E^9lc zH63E0-BDTnf7-LgDJUqII=J=>_zJ7L*&S@RH!1}5fbA3m>=BAid%D;0A+?7ijfRQ} z;h(0wwVij9jM^R#OLlj6(`B^mMwqG4j(|yQiPejNfVD^J>(D-4tUYSITYY+YIui-U znwgmB+_61tdpd=|cSnLi!Xk+9UQ1#$8s&X`7@MD0h2kUg-U5F5@adBX$PbUL3fNfB zklpmadcObHdX95Kdj<~e5P}3gb<>}7qJ_SPKyuAgg8+d%A&T|Zoj!m0f(*)n*~GX{ z%g-l);@kH&eE;@#A<(Lrw6rkLY7p=k7B=?VW67v*yCahW+7l+C6qNM=cnuf$s2_>3 zGU&gy3TOmw-AP_v9_ioh1_VI=cf0@YRVm05hKBc(z?Zj9SwFZSB?B{!WZk@iB9!$t zIxTHwVd22Gaa=YiBn0klZ1>zjdUR0IX&k_ecn9=T*Du@d4ku+L|IpVbh9p4hDCc_H zyt26O+P|Qo0lSSGn9H+u>GzNX*L>CgV^V;AwtqoI_5bIIAEM923Dk;T2*t!T{YR74 z!4xBzG4T{iMZM!coTe}aFdL(@A$g`4$#&=tW>B&8T31j<#5dqC(So6Z{qEvOe2XP5 zI%qTZgAW93qS!*iwXLU1qvF}Cza`CzOL;77DtLLmdJQnl3{vG=&FWhe32G(KsE9Gb zYFG%Vxx@_|+HQ%#3-;*h=psN8&;i{tS@BC5GgQs%_|)i)gNM2wp0pf ztb=P5Fw5eKDp8@BMpJHc-JGgI4b&%0Wre@SC%SOZi<5vo2FRFfPjski-0E~ zXH*7}QY^){Bm^siQbx9DA$d78PLcq(C46FboDzN&qbsRZDf+6=To<)xK{VE>ls+`= z@yAIvrH;I%LwjgS4OKE^+oa*{uoeO2NT$QEK4g$L#3lv;CKhGfkr{z3V)Z~sgO4K1 zhFkE!j@u z)uJR#!a)kgN=zSOYG1mFXi4R0r(~bW@$cDWF+*{Z3NuKkls-K<<*ADAj&%f}E_m0z zAB9LzX?Q1YqK#>>ODnQ#H;((WeXHgZN(2NeDKM%ftnQfdX&%= zNGh2~XNt0FPlVC1norZxkwP1J6o!T^r($Mg0LK;MASkl6aM>f&2zHHBoAhEtgFr$8 zjust3A`VWDp-<12(;cE{O1N&-LBCE>QFSF_b%VXnz{p?>-O0xu2Q<>8fsLwPsPql) zp2vyw0|Nsa1rZCPJ|`a*L-Wr@^jZgub0r;>h4X2*)wmFKx@j*S%-3XNyLJf@ts%Z5 zk!3m|1iVX2h#9g|jw`-4OzAQ#DgrA|sc9MhmP0XE$kL)#DE!!l&V&k0$(_`fIn`)) z$v~#3mQ*)8J8M{0LPH)eis~wHmT+>tO+QsUhf&HF?lD6bm6$~NW43r`ni(xHXCPN) zfcSG`FG86f@I4ObE}zknn1tWgO{RId(;CXx!z(d@Qprm z79t?nGf{&6JA)g(Kj#z^6NCLZ{>}#Bc{J;L;9G3|AHh5sm4J!g0im3se#xVgar|GQ z9c*wkTej5d=EVCBhuc+=A`ZxVxIi8qG#p93S+*OqSZTUBX8b}b&Gj_@gKm3g$H36= zKaVglFyu?6C~Il8na)tGF$R9ZzvXtm8ez@wdU-IbXl3E$#fLyTe#@?RbL}t8R8;1B zqiJ5xTlix<55tUOz#p#|%%;=H%w|ZSub@+C$Y-C$&}=e}w9&1LV@FO)%nS|(aWWJE z9A6{|U`Rd7jdmL9>gV{xq<4yzge{ z8xR~gQ`hi}v85udKg!5Hrn1>Us`NE@gayF(3b9$>&aE-A9Gzakg4%&G(9GPH-6r1m z2gl^(=$Vjpq!Dwbq%4lSI-pT)h4~0z-opgxS&Q*3n1{AHUT(kYB~z}sZUm>$|1%FkAFu_j0>5pv~5I=eKimrGaau24i22CmZ!o%usqhw z^bNXEWE(VbEd<{_^YU1l&9|35Zsmn${I*zba8yUYl9#ioN8w>+aoj|bLHlRRmxGCs z6Id)3uSL&3O&OvuGgDJ|9qbOilgJ-2%=wC%QD@DM1xY~059HIuCvp>$rWM*DF;VZ(63B58$1tMkBbx* z?s(b+&FUQNxmR-3Cf4>(0`E#amKj}-TSF6!@60urS#LW(!f+yiJU;^IstQfrQ_xV z5i-aV5%_EoVv5V&qEZO8$mX8m)k0y8&Ynh`h%==!`j||F(Usv)0=1~gv-4kwwKg_5 zp-Z?>^x+=bZ)2*7DFIk_tm`U>XVhy8snN})){BBHr_xkzisNn-4R;euDW4SajJJ53 zO05lTKU>|xE%?MD!a=OR0ztl9RirOc!z-idX8DVtc$@ppS6AM=_ucL({kL~P1&+tItvfaZ~vw4AM zgCkNfOrUM1Eb1$fyJ+!@Xb1|y#P%hw!MHRG=F2o-;@f$#1iu zZ2tZL4vLTO-=)a^w=e}Nzz#x@NVrcT{09gEIepLvZtuLIfg?d>J6yOGA&n9SF0=Ic zX(70dzWxarfr{sAG=~e^d!jjdR>}{fND!WHAoKVU`ye7?>4Y~;urqI~yC`5x!>WlJ z?Jc5y7Fdr?idP_}G z_QjLiwWuL^*8gw;yeV(>UiBCcB^A}N*-yC*GMcXpTsS@$_vip({0(4(_Se5>V_a8~ z($a`TDIxKe>sP#R0V=A z4iI$rS1z{tv|SIr1$KGAN{NdLcpEO(nAmf)W@dsCZef4&0X5usXqna+&(m=#yPYh- z(?uVCXh_J*-!m^F&@&zrWrxcP@R`vJ-VEY_G)^aIS2VpC5-ItTnQFrk0iawfjNng* zkB?8#_Dsqq1EXlWoz#*Aty?85R2zBSE*RyAL;?_)5-2SzU9NV$Uam5L0S#N!uLSrY zzi@ipZv?)*0pw$-pV~f#LWN7!Mlz0$j;gAv4h{~#4e0~c*4@rp>wk1Ymdu2J&Lw1V z18Gu5ZYL#Zg#-oHPX+g~W-bk2y>_tkVLo?5*O4aUaMkOYptL70Yy zCY{590(vG1!N@c=+W0BhH; z&oG~JvmGPDb3OiDO-&6J^a`4>DJ(24ZMj=@+Z#^08lr*~4EM`{PrT1RLFezrOJQvd3=Bp_Mx6I%9Og6w z`K;opH5_06-o3uuwTpnjO0u{xWwdSggruY}ci&!a>)ZL!?(|lg9Nzwak^c^Iz1ZiF zjjc?x`L+Pq7)RQ#Sorvn9p9XRDc>7OAw_}(jw&#m&iwp*Xx5+t$)s%gB7#ub6$#qb zYha1bR$H+ltmF6`a1^Zz=2cKcL_~Z!(n2SVuJ){O*!l?a+y^IVl7Y4{5Q2Y)!^U2A{ zc<$$&z~U~G2TF9q1c~`Y?alox&lujWAH=q`vnH({DQKgnkL>u3bkLFdbch`{@DzXcqV z%gamR&mSO(%$6H$8YC@EO!z?vq(NJtpLb`xcguG8p7&qDlpdgnWaftjp*Xa|Y!%TM z@l5T;^g4v~icJ$)<@i7d3ke-){vhpoJFexf4 z_jY#^Snt4c(25X$wxzwhuLWTE?2fJGAw4lmPQGq^XuYU%>JX8+t4r20U9Jc{r=+P% za8!T%0GzX=W797YRRrjv_nlTHtkdl5_d+C1y7y0|zeO$ZLMqV4fxy&$yT|+La=AUg zkc8m<_4^*&Ea74q8ZCW0{rp6zwo*doWcnxR~kl4fPi z^5&n*)rewaW9|3Hv&3RZ9s+C zk0NKh&V)rTf@IS)v!gE$6l3_6@>!;rG-2B*H?j#UohF@`Op+KiqD{)Cr>no^+47$W zmE{Q6sv8(m%cH=5Xs<>lY(N!f=(veah+W z6>izGyxJYfO77u-GGJ9^wHEq@x>)v2H$o|gig6ygbR7Iq$pH1EWR|Krnf^j}vaM8x z#=L%+t3#3Z;ICMV6?J7|0<39qB^9T5!jVNdqPS=k;)Gx}G^wP;xu4-7A-Z%JntL)! z2nLr7L<1x7{mKK~2mOWe&d_>>Wz&Udd4i0V8cIc8Di6iQWXv2K3Ux}_-qjcdCI{mF z>4Q|Os%k~i`7y5kV2ewdulxj)|7Ng+4qA=qP0AI?ZbW6NhPH0P)Eo~}6`RUV!RJdWOB2|6~yZq)$-mK@q(stvcQFhPf=#Yb1pq@L?%3~cPst%llK z=J+%MDtU`;@8=5j$7KZ_SZJ$68~zhCP6#r(sE;>eH^s3Ds6!;@dI0xO$K_eyvcXv*>56&|QHWJ*k1G?ZM*OJ=o=z~{qhfpXVkHZwDIDPgI}A@$JenVc;rHq! zBcxRF+X|C!w&|C=zt3Mm`g5Y9b))81yQN|9>sdF3l_%^YAjS}!IDS^|dP9<)+cPTK z+#ZkDO%6v$=U=mP#(F>SJBRSdEZ%rA8>ut-dO8wgrJWO#&$vyFKSW0}q?c#QYm|?> z3?42KGnhmal;48cA#R$b(i0Q!(`<(FRd|Ts%llKMrSuCnENSL`|C`M6M*mN`0aIG&m1^XXo`^7XFzW}{w%kaj+tzkG zGqehWz^V5(?zQ|lPoa@KkFZ7;0Hc7!7HXR|1Pe}=5BhN%t<1LW`n>JP$$fByJ$xdj z%aOLvxS0s{R;59Tk}}*vl=2r=`m;+Rlwf=Fa(=E5yJKh9EyW-==dDkCVE>6Jp7i|( zERppCvpZE1AMd4au?9b*OzffI6KbGX3q(!hDHO%2MdZeFd?oi_8Q7H?PJZYu*rhgD1W<=HQ)HeL2Pif z5JXh80VwhY*KqdV9qp@>56Q-F~lDd`jpsE~RH>hdoVgK^Rn3&c7M0sVmM^B~;p{PQbPw zMzFcaM$xFPl3d&R@x>=5_-eke$TG74?!hDD*}HudpzsAW^(vWXOs8FORw2QgUl7m& zPnixb+GGZG=oJg?joP)jto~RqVjr4Pv%v0bK-##+wO$TV0&E=s7x5v{_4Ic?>^pp$ z6hWh{_-%U?8VZ#@V#T5DhyS&;lz`OAM(w+}r!)o4Vbga)j&Cxlr_-h2=b+r~pA4ZP zLeD&q@uOb`av}ApG#VQ+`Lf5^6~4hW?oV#9$*Gc!>@2?g4Y@yXfToZjMh^AIkx0tD z_nC@YC|XFOYOZvz70@HOP_2@8;mOfz1uGKBDYLUbFsR|(T}I;OTwR8IJBY1REaG-z z!Rs7FIX8x_ob+#bS%nFYOQPiqn28oat0=X-L?3Y9i&_7bK)uPy6y1{QlMSblFPbQM zFRtrD)Gz%F%O1|9XadfKOLE<{PelQM(_rv&*C*aE$Ynb4Sz?tU8v21MAfS|fW$STr zW@cp-5fr4uKtO>7fD=WDgi5UV{fl!>jBMK~y<(rr$Ql5H=jZ1M`gM^2*!$&Sx<~=` zCndL*b|BI&M&eJUfS5;t>QGaZ^|Gk-;nVIQ~?dgDl)3^4|%HEi&#+PM`oo$a zLZlxk5`tLufSS(|%4hNZVhhMhDo`Cez7nwEepYP{yIYaG$|W!WQ8EEGx8=e+WGl@jv zwE{Ia6707Ig$;YegO!}-`Rl4>a1tJ(6{7e!C!SWG7OWpe3%#G2e!>tct_S&C6-)>4 zEu9*htzJNK+>|DS%rXLyt%ZdJ>`w{knG5T$VwZqaxm0V04hi8?vp-`R-l^|^T&l0- znyyNi`NkNa~i*u*L>k?m0`9A>Mz55LeLt#YVP)S8wll_4_6r}%%EQQL9 zh|$nsouyI`)@N8z-;~oYF_-sr6oIQ@aXt| zxBy#Qpkm8iSr%oOvh%}CU-JIlbBW^Jv(tFzu~y$^omex2=9n%|avR(2qo`c7DB37G zDPP7ovDa@PkH=qE_hNZDPp_1YEgCaxYC#=GK|VNiKD{7NIS;0tQ%draZ|ll@lk05d z;`j1h{u_fN(R_~!7DuK1d}&=aJW4)X!AL%mo}}Oy+QToS7E8Il8Iq_;uw|f{I+5U| zfV?h35E0n3WYzBRbW%@*#I~P0jnX7t@Qd9~fZvcKqi^@}5#P6Fj&&VT0IuXH9cwtu zX9qLA+;`TTi0uJ~Bm2v>wr6l6gBPaNG?@(RAAKS4z`_zIW1~Bbrd8!Uc|`4~4;#BX zVym0lq>H3Q>wR`*T#ce=(Rs$|xP4``8nrQ_rVr>t-@Rz0?$u>xGAj z|A9ch3T`cvM2Xooyg0XivknGkW_7kqV}k|XJLt!i-)Dc>V`BzPhvbmMNd_8`sJ#`Z z*&IE77iWBz=qN(kCL5-Y^=xDbP6UwQn2fvK!C%tZZncM8xysN`lsW%UWkDoi~4tds)SN)e6tX>a$Bs zQh`QT1_qXlO~Fbxar>;*{O`Z7sRf_&cVHXLp}W-XqAdIFw)d^4pL<|-&amVRS6dFi zEC!F;T_@TcK@$ZLNxUuvRrx26jh~hW!y`r36uo}mk;M!dq*i&!<;k~Qud-ewC4os8 zBaD~JmsUlsqt$XEJzC9t9v@Y?XoC$`YE7ofds&#iG8*+l z?X_z~l_1>OP)RTE>%G<=tw<@wCzkSVFLxuHuedY}eL+Uf);O_d{q>c_>*W~;K+Hq%!tn)G-yoH(YCqIde)XZjOia3ik)>glAbJ zcnym)X7)KtNh(S@RclLaYdu#_@+T+t*rQqFpe-#iuphZZYDxF!N`UEc^KT% z;_ldiuYo-1^v;_NPaI$$y_m5^9&e81g1l20Z{D2^+BJfy(BQo#K&(cP-}L|%2_WBi zAv0lcXxO5VL2qv{ICFk4nOSu`DuexrXUwE-WVClMRe%io>jV6c)9Eyo+l>Wsols6r z4i^^}_NOO5&Zi`RB^ya*hKI05UakOm)^|Qrd;!Ceyf3h%!oFG?hYhW*xDeyCz(KbG zh#Rx=MrePEF*T!1E0GAC2S8Q4Ij$Zfr2BP#ZjDk59K_ze3@;v!M|+fFf!l_^65aX? zuK+vLsSf$qclBbtW99-dmH2@6KcBGbo60*20L2=`r3{_vibIi5w9k&||>X z{d}b!?9VoS9BuN@oI}NMt<7)g42=r)37J*5y%?l&8Hkt9ejW;3Vc@ZBgC;JWxkCUdK;ZA7C>7`T{R=*lvpo2tYz+Y6J6*0R8S? zT?iW5`Sb0Q`*24O4O_$}P%xlp#KDLV`lbU=ydUV-_Xh#XO#Jun-#^`+a6WAn%H_~B zNYUY~1C%H~$ljN&jgXp}np8TK4+IvZLVIJrtlIwq1mtM14Y2BIPOC0JTn@x@g@qXZ zxOpG4&K40G`W*z`2DI(ICudfRMZfQuE+u+ve-GPVU!F9qT5DTd|5|xHVFMcq7{?_5 zp#c9F9>h2{P6IMBvfJ%(rxG1axDAvS;O_#8vBIU3bbxJK9)t%I9Q$~pHBgz>J|+YE z>sw3|1{tOV|7c_1A`^MqIU50=ajhoNGT?i4EV|xkmeWr@n z-y`jdc7WDb90kkub1GgoC_gg zKuAak_Gc&`j&@XC9ZQhjcsmT9GpL^2U9%HV_8Y+wz((u7K-F>n3VQU{Km)i<(o2Bw z{fD4!^^VD{=m&guA5}OI#c@fcUrlo&IF}1#K6bX9$_2$R-*n;wLzTG`8+i)cCA{w5 zKff?nKW2u4?1YRcZ6E|tbhFc0?+PE3PmswN{g6tzLYh(Hm+Zorf?wKjkr)QN_Jjdq zDNnGJwg~3C(C_-AI&;Y2&I-M$5E=4Y`w%r2ttbECH^1UqJ+|K-+(8YaDkRMbOE>Q&e;SG)K)NT0fT;jh41lt*P~nXG zB_0;;5MSP-xKf3uaqTv9+$$hBBlY?2IX0UxVL^6gPkHRBXo%RGum6?4U|_{R;3Mb& z4nJPE%`Y3=v)oAde(ik>*Y4mg&!!<{Fvt8yW@l#=JpK+HVg79dtmv*q;Z;og-C+|p zL5g+Yz11Rxg6y0eWDp49eb82Cmrd9cDl7;T0NH)9zQKn0DbSZ20`&dert|rm$MHKX zu!dkjY53104vvm>H8lhv5C)v}^2K@@j|T_f-t$HpFVtjXHY>lietJLV#AKvZyODSLSjW$~6aDJ8wi(2G(auqLv)8gI@0e$zX-zM_HXwv!rWBP?PZYUZsAc+7BZ zV*&yK$v@}nwzpaXtMJ7SY#7ULiJn;T1x+FiWh4&C+)d4oyAyc|kdo-UIu*zVKFGoT zcdp^)VLWwcEolbamL72#CJDdo*?JTg#o#XD0uYT8&3aVpFlK# z?Dz29v^=*yo3{jz8607Z;W9IB66GCB`jLt~ljWL;u!+-%RFFCAow0IF5^IH8YCJQu zxwz}3ZzSzRvc`nV2pny3Yo%+?Buq@cba?Alr_UW?ofel2R~l+|?ln1}^H-f|HaU7R zLUsZQ_)iJsIJz7H>dPNDs{yCImTp_C;&!>~<(Oi7D-ta(OU0z|I2g_6J$GKqwKgZz z(=0u34??UU6RQQ<1S|G#O4a7aq6eFBEdT6fch9bdt@B;f(AOH)s`GG7-m_xWl-Y*c z;DWi7tRfu1mkH58+_3a(DAIJfv2`LL826)_6N})gJe}UA%gK73ux}%^Nw1GTql?&= zDY%d|I>l10L>!lw8ZxfN1DUp}hlv)S7LlPjnr7u#QHPyu-0!HNM0{|^aFNN zo01yo>FtVYZCC4Ab^XLz>gw*>{!@g}?@cb+a3Vt_q(i^g5sh$fHf;SGuqR`1@l?Mn zZ^u{#YlV$WTf=&tqCqUPEi)*Z(N_F8?#~TP2ngV}$2}k-y?Oem6N|pFk_gVl2wYx*qy#J$gOIy+ z4Le>Q+tL$j&KgP|3>-Cxj&GqE>73!MU+Tl)ch8pH$SfBhYwooILc%iARa&Ue$ThY) zPtID~bdQSt$y-hqxC1k-7whC#cpYjM7CEw=_1svvt+gVthqN0W2Oey=DUK5TVnv~R z&;avk0!_Tr+_K-{NmJw!phz3`Ptx#=Hk8>sX)Sjra&z-Ur*T>6*hrfB8n+%;o83OJ zi{sMcnA1q!UZ*WBMKIL}R(;uY>$V|wlV`xj4HFmrVn3Es`(W)h+^q`bgAO>G8W8AP zd@u(B?JHUr61iAY<|No0aaL~^Cm&=g7TnK&mo?hB?F3x#0T3(wDp3@Hc^ngJW$fHvS z_JD%+oq!C-%;xoIB;$-iQqVj9K;tq2e}EI;+fMKu_WyEp{o~30xBqJsz}V zUdja;)%gvJ@>5vIPbf0yE7CgPm&+V#U|WqBr3?c3wh znw%6|r8nqsnijaY(*VIB9FB#Gcltm|HGD-JBPAGtC?i&2_h1j@^X6yXQ>!z_4DE5< zP2$>S<+)$?tu>q5gaK4!Ky0_?_}hzZ)e>h%`MR4zD(y#whzCr7eNjgbm z)wYaf+k+uaH)*SN((>Zs<3T~jn>2WGd>j@Mk_?!M&b(fq0NIg~3kR_r)*N`<`Awr| zNjU*rWFjv{leuC#I+{))9iIq^h-@w=jhVnCF~p5tuP@&N5jWU`-%^&ZtWl+^wH(#w z;B=Q=%5r!ah89vYg60sT!!21tIudf-YJ3_A9^<6v~rM=MNNsEhu+hpZ9oSp#WJ9Da7$)F)l6b@4Q|l_rrD!84}oO#lf;3 z`|g|j8cvByMK34b*C=0vZhBaDwxQdTrMm^Aj1}jta5C%j2lr>%Q}id#XFx_?93TGy zDA}-RNPPr;Z=9Ij?r`VHoKlH$79XQtZx~=nU-xh_%7_xrb8W>_d4z>}KOYAp&;y`1 zgGyDN=ku<_L=2Rl)=wBWg%KVoa6N#(0UH_FX33)dU)!?tYN zSQ^1Eqnu~cGc#Y2`hh3umVWDWk!HC2e0d1Kw&deK31?k(26}%n$_)hxgeq8Rngj{r z&y@@2E!2?$UArSGua^n!s%6ShkbpBCR^aLGtQ|mmZ>~kR(?%Ho=ubTNKYBt?5d+_O zgP=r|fBv|-U9w6U*_Gk_i{hV4tx>n^xMac1%nS>K1vnsN)oM+ZL`3?AXxqaClmrE# zR?7mNGzpQ6-v(i9Wi>W3lE&@ksH5{0&H`Y3<%{eOCi7sSP=Iwqc$3&U&swg49s@c$ zG$btGt~4|-h>MLSKw<;l?^vqTgIVQTYjwNx`|t&6|IH=P*Vp%^l*2;70xitHTts@U z6KGW%euP9to zuWscVB?|S3hKzi%XjUOc{2A&Q510|rH%EvR>k8D}Q45N*kW>nbRG$3T^|f^-BqV1F z3JU8seIuistX;=upj3+R8RZ7hKj5GQt`=~l0m}J-i_M_~#0)(w9i?(EP_!r?%@Q@?Zq`{ouwqO-oc0uFmBs}&xk z@1GxZ(&=0atE&-!NrVmBdjlXhj`sH(01Y3x0TL*{fI^VJgBCcNpxifb;P zmp3V%DbzFw2hmDK^zVK^k5Mj`k9bYWE6H!;QgXJ-YMESG94JUFZZ3lxJ7<{@4dl=b zL&Z&-wQ4umgYBxXuCDOoc-lRmzE;io7nm~_>vAO;1z*A&Q21KK=c;`hrB*zs{U-E~ zK6h?jn-7nT&~PEE`rEaBYNbIeddot2{jN+)f%rCTQ%&CC4x4wbbiZ+R)Gg9k3@|lZ zwOR!-x$Ks+>`R1RU3p0TI<4Q*v=hpE%wJtY)KYq!%r0l9{UafJq~W9GDWZZ2Oq%vm zy;WcH85AT092eBaiXkhZj@(7IJ;}>)Lb>98(eP$?TpWc7N0CbZ42g`pEK93=R;Bf~ zJPk^vo+L+2|A_<4X6^JS{r`)g9Rewln8*+9sJn&amR8wrMRWGev;?%H;GoMi<=a1) za>r?v7MCtlj2Wrhhs?AI2|;CJ-JlA)re|l{9wmdD#5D8i(=xoF>N8txXDkrZFA0^D zY8e7eiezJru@jTBcS`gp!4YNRMWs~;`FOoYepQE*#r^idl0GQ}d#$)?EGw!_RSEUJ z?V0Dr@>u%|RpIFyP<~=tYAVD{_fGaRK&It&c)UKC(IS&=Bo}s!VxkSr5bZ72QB~%F~8}ZCj&{sSRN-8nj7!H{(oKOK!&}QV*Op1&vEpCDZM!C$i_SlM-> zts#55|1)abhkG2BG^Wz1ScI2uzRl&qgPW7_MD#XXI3r%X&{@6DVa6@c&4JB+Gu*bo z-2q-#;2K-H^X3N>DlSanaW+VktY0%*T{3jm28IzNzm^WfIM*oC7m;rk`*LOH(24o= zkeJlPM{VT{$GW;<(P-Le7i)zZoAEZ(<9pXbLv0)P>V~{*a9%B@Klt^grie0y^OfTb z`7WpP=}p?eeNu$E`=c|9ZLthi(~3VMfDf_#_>3!a1AcA`@mXC(9918ED2X_SV%s5D z*~=GQ)O7wR{Iqr9@OEwMc&(k6octg^#ZrLZcl1BkQB=UPh}dGk4}x@UG5UXW;;Ob4 zNamQwyEZpdMsWKtEmWaINi!mTmeEW4VcQ_7Pom1@c~{3_QY|OR#V+WYFA&|2UXZRF z)P*i&j~4kI0~wbV^68jRFx*JQ9_I|R&z>dHKg_UtrJgIjSF{zkl7Y(tjkF2O=8=XH z(Ote7TQ!3{Lc>~+A(%gLE;@R8O?-k@&^$+5ptTm^`$KEqCDYEDFXf0jf~&l~_WOcO zJ@!V_--|)Vx9fL%8L!VqZ=ORE`~RrOWfrz<$)+XZA>ZWvR|PhrS-yAJ%$1;0wD!#E z7cVO6oYDx`TnrN)J1F&%EbFiw%~0swZJ60>Nz;s}DHTtqSRkh4<=aYC>boR$+C8JC z_2@JWKblti5~i5e?qDg*Z3lG^gCVMwHS^0W*M;Z~lo*1yXJjJg7Z+V=$U6syhxA#D z#VJjb&o7N7l#36^(q%hv72T0nLKT+utEZKV2E&=~LU(JHo)!=G|FZnXDlxS{F=13H zns!*#gK|rteA%y03G{PVc@-rHJD^3H%^S%{y<2f(OE^yz+eZU#sRmWjyJaZaD_4;tV>a?vTJaFJpRTljLSqG6|HFG zJBltqizYD`z(EpyBP4{RX*?hWuw4dOLTJb?oZb7sctnwRXlkk7odlNwR|v8q=fyK# zxUgOG3hX_^KCbWT-}6q!Q0d3o67Fyqm+MJyU4dwRq(e~!~wS4 zJ8FmWmm4?t=P=^J2$)Z-E9H&5SSWBbH-sN@TiZJMWuy<qL}75OaGV?+Ky-(7j)weqQ{1=S0Tn&Y4$%P zqh`vWr~cxT86J$;e5JubAgz^8a5^<2ks*tM|F-ur#`%T7K3!FlTDUcd-EP-u%_9MD zfOiD3IUFh~DxzRH#^%W@>yD|2GwwsJ&VLR@n!~TKaZv(pzda8D9^yAWg^*5uadB}} zbJPMOY8Kk;E^qGu%&(OW+PoKdi)#!9gR#J3B#jv2MFw_&L_@WdidmT0qV`%GtE#wi+#C%_I+e8lS;5gi>)SiV{65B@?*I2O)X zW004;C`_vRAOC~=sXi@f-(9O9P@ZH9n3dM6whj@W*?;;kUjPbfNY#9ym_|#(@|2Hy zQazE2+=BRHFCp`U!wkouTnRPh+>CmfGc*pK7I7sKEw#Y}8oG*IccUM_USWN6Hv2)_ zm!rRzlZoHoz1;zSb7Q|5A?c7(vJ~@~vhC$6C6mYOj8Q0+Y7udQl*D`TT?x6hDZ0iA zemzDYh?7?{$bwxim8TZZuiP!2I2?kHcB?7IC1FlST^*2B zF6v?WaBrb+>7MXM-8UP6^ayyHQsV$CI<$+Lg_;orCwfFOhMUUiN+Y|)r~T5olOhF8c$`ry9up=zvIiuu(w zZ1;)fIu$CCEa7f5pL?sKJ!Z=-rLe1%b+Mtbs)efD$r^x{)@?0&IN9k&GYYHtfCBWL z+<(kR)1^a36Ck`84TqfpcM9@8>WbZFQ|s#HxdG-(E^X8R<%qOQ`ow7ZhlAf3*r}~x zO}s#RK|$w+Gl5sAv@V^dQW6Q3-e|NFtc7F>_{(%085dxny#|$RV8^o+jnp>jLvW|` z!eO{!!qYQA{^O6?qa~n9xsDc}X;Fm^99|Bi0_<$NysGN___+0UKQa`J@ZRXa2*IKF z&bPS7WGoM%q{Saum6%j43Eo&v#vjpA_8jkuO+$vVqHf`7{6hc(*=8dB3NO72`vUnk zVqS})Sz%F;e1bB@i9K5A{@(Vz7Lq-hv_o}UQ{8}JQJH)kwh0&7h)KQMpZI`K7ZKmA z^4aC-oZq);O4@mUhAgX-Zob*>STd;n>VhBn9-XBI@w~`5#zpiwnCInQ4~j;Rc(-31 z1Hdu0rnTXsqewKg1L*5%mh3}$xAG1}-5rtx!=u{7n@pcWKKrW*lfyUlUdQ($%SG*p z%W_okW$Bwy)HX$%!f>-QYSWAko$F`sZ9B(nsg%XWsv|S{XV652{vXQTGA^pN@B0Nr zK^g?4y9A`AOG3I4q?Jaxy9ERUX#}LZySp0%l#ZdhJKe|V75Bcb``MrUoUcx0WM8JJ5)%*gqqM7sQ+A|S_+O7wQ zlS3%MBK^P2s_n`*`;65-9-L4_(uRp1=deI;Sg+qJ0?85}I6%fGsLkdE82+G1^t(u> z5f@cxV4@LaC;3tr6s`0tXDIEg93&0c65?O+gLZFgBecLGn{Wbz97-1Kqf5D%=Wt%2 zItK3}bib`Y6Kw|S$cWnIok8f8?-=3l06PN_#eA~J71-B6nA3tUX@mOo`_)r{0RiL? zZ>SyzJ;BtAHH|9UjqDJ%L3qci)*Zy<5U;3#ZYB9}J?hQ$uB>g_fx_k*EJAsKR11HH z1AZ?HKzVqlcwWrJ#o@v;A?O;c_I~~P*~dWZ{`sAiFk}WOm_Q+`v04b3OEZ zx9`Wf8q1H>4WRqu>ji`<1E2@ez*K+YYc@vdA#TxA0HniOa_7ku!le~4hs(lotj+Ct#t5TMuz@Oh0$LT z5{1MC0^1}N?tBo?@z{q+TuWS?@_{8Vs5@XwLwizk32f_=TGrxZFkn#x;V|eyNtV9^r64smwTvG} zpt2>K!p{R*RlaSYpYMljmC|xHE+h2&&Yq5Fulhr0AP7MMy_piD41ClPS`7|nR#r?_ zP#(^6I?3gI7=kbw+)zwJLm-b|E$i-pozDY!wsU);gV}2HC9hjcpnY>~{74t|zL`kH zK(qmmqk^n}pb^Lk+yk9<@7#-Ge}A95h>wp?N%$JtrMWu#(Dc|x7y>?B@~pJ9_rD6> z*4dU@&eb66g7cXvo}ug@{08bcAs^1T;Qtnd4iQ zo^gIr2MhtQk6!}5v-f!^NKt?z64!5t2u}>Y3_AxP*rBfLYIJ{dUdM5PZe^VgCM;H~Anvh(yF|6+~)F z3JbNWKjOf(BLgFbh^QUN@2fpw=I0d6KvE7)!U4e&4gyq((~_?QG-902-?{AvJPC9x zd&8^iwfD5JNN?Bn?P<$?7?>IKgx{cA%W8{I{WW!n<^#6QGeqimd+P7Z?dMXY>6dTH z#W=!PeVsPW@Exk8|c*V&ter)uM-*rp{?1o!vxHpu%v;9D#lA4f_2G&)lWMqgXzyD49Qu#f*Dc+UBs zW;<)Ygu?YPIyxnYZ-0e>zu);kSc?C_g@Pzo9q2`~oredwCn6A(?WZ}#Lc;SuXvowO z)XFMchK)B_ZWEWITk%YfEq$xm>7CgQ9Yl8~gTj7b@uY_lrdmrZC3TN37!;o3gNuk% z|3hVy&nU@W_0;UJ7Cd^%>|&rWi~oc$44yRc%bM_SRgFkUIdWnf%?qP%a)!L<9;51- zueOU?+JvROM*xI^_`+t}P%?9?bhSn?4lbvin6e`GO@>RNog+>TV}Z-mxVYc^T;>jw z+e^4J6(dx4=>!cU_j~+`L!5`8i!58n4P# zlNlcP5}iii{#ZdsZDjBp?>&oZAN1t$8I(v1CuPu^s@^OSxY{oHEtue86RO=(%i}aV z5U9hvi2vMqGqiE!XVVQ?X01|^bBT+5`OA}%fv+`|?opT5yCDNTBv0pz7w_UOFiPLw zGgueN@oASRs*!-(E4`pz4=GnZBzm?~FctBY-4R$)zGvaZaPLW-+Q_EvR`{Nx!@`xm z4{4_?j%P_3uA2D4Z?;^KyFsfUn^5!qKH6-H+ws7No^5}$aFBF3NyvVqqnezs4rLYN8Ift@zLo2p%!#fpg+@WcgEYjgvsD zad7-(xzU_js~FF*la2Zq4iCIm%HZKt0f8Lb(ch1=G#{Fcqw` z+)Nowk3!Q2--0iBS2K1Rx=~WLw@t2(t%-5(uJh}k3e5g!4wX%es9^Oo$>nb*Z z_fwq)-ZKVXjOXpewgc!JO_F0^qj~PK?F)GA-dk`K@oFX|X;ISBxT2?-3Kb=%mYRB> z-0!1POQg5k-OV4bhi@Rn%ZrgkfDL5Z=Z}l6A{@+?#|3}t`j_zxy1x|&t-UlmAeDIV zsDPB^gNY+#zptsWi`1QZ!XO|NzAmGuqti#YR~;bC0`M*GKXvB_sMg=7OT)Ka{{z$Y zfAaVLei_?*tzaC;KPz!ipgc|+8DU@TnE>|}uuuI#9sbPq>h>8O43GE0=WtXBs7d|k zN5E&ISKGB?65fbSVuCmR4C@=^h#)O(05g(s+H2mjuz7WEN~Y5@o>wV1OhgzDoOcBN zE#gNuDYqe54%S2;$}%Z3JA2tobLGi@V{;z&;UXE?J!@UCksv z>wYfb)N=lnR;>_ZmwdqG>0cl-@TvZ7nfn~vhh6h054rYrGdY&wwZBG;zgQRH- zpnyQb5vt&nfS)gtl#~>PMmO<%B7Kr@pza{}(_akEKdx+*a6XY3BqV~+)tT@T^fL7n zg#UQm&gek25-g$qP2KtXshmL{VKa=xLI#fD8#rwM9RanGf4|bRnIdlc4HUThjs;;q zviIVsbWj)^%#(2wBEDpK_?uU@D_zhiq}orPEeGC*MUiw=Z&zTQf^;JnFwom1+Zs>2 zpoI{S8Fl^x6#DbLjHz{i*7r9T(9zdbF2_7L9J_6~MNcV5Z(H-uh(ZbG$Uk-?Pa`^2 zN7XDH1LpnoV?{?<4f;@I8S_UJ(h_L$Lmm2-&){=fI~z~lG2()E2aDaCRo%YhH_bz8 zW?j}1N$d-AI8X_USDzNt{YacFaS#$U;&?T!GgcIHrd>`IG|KP(>6zK9yg`QZ>M$zk zlAnHfbxc4(rG}u|E_%_zu|5;Opa1pi^mh@GE(FNR^)FNsCDii6aw`K4yiHp=MN5a~ zBAX=Nsr7QYZye&|=PfWEdDIlua?KP=?RYFK3DLS{_%Hd-676R?*ry-BiB8 zMhURiu_>!T;xLcY(rEWl19f7X2ZUqCm$ z7eP{`B|q?Xo6SEl_i0g`G}R;HcCojHd^GSU9u1G@rPcc$-!AA)_CJ*rUE>xp#x>p%pg1J#dR(o#JrtM4W$L)-?57@~ z8Z~i_HmsRl(P1j?61KJpLcH&@P>dY+^4<={zU7P)X4Im7hWrinqj*C5q|RuAERInu zr+m0r16}log<`e0u9{KYp)&EMlPcTY1p@=cCmuQCx$lZEIGvxuDXDpi<0LRe3Pe(B z=4!^s8VqY>{))dG(syW&h?3=eo%qwPelmsPMPboA-Cz!(jC#RGqUTJOa{tk`o#)ak z$d}HaR#nK_lrds6^FqpJelhkGxPz%eD_4c}K|DZg$hBJw6 ztC5X6N;acg91}aq{Y``6ZH4R`2kGS=`r*eKUud=dOEg+f;?mVCaSO{K7uQzyRgdst z%aitG=J}`^V)u;Fdsyw90COr!jWGS}$)6)UG=vdvHB38gVVw{gDVvbhpo#p#@Yc=` z61+ATka7Z4ARFnQftuzd2>7p_NQ6i-jhy`aGW4CoHd#!=q$+kAR`70^YLPa)7N%{roI7g9B;g{dCJuvUJodKz~+KCJd> z)3Zd%s&Hbfc#lYYiV#a0$$VFRif|~rsHkNMF_~65uTtr9PVzPCc>!-I<=4TB7TMJe zrjwV}Z=BE+N=s_Fk?QhFX163qBG(NeOXON~xW6Wg7sFs+DD_Q~aAJPtsqOACI>paj z!iRa)%*hyJ+cgqAhmQ^wZGZ2mNM%$*X-xC=*V}^?b!82Xux@$f6z(3vhkVAH?4kPJ z2sHEJ1j8JYlZH1`)BGf%<4Hw^U%8z)CN(fOx!UkR)p2PW`Z|HRLO2e|0L&GB>G(Bu_+b{EO~xIsp^ zbm54B&5HUaCZD{|+I zhC2k)PF>)QuV>6(5ZCq3pE?p$ZdwVC64CuF7V%(o`kH*HJ&DbBql%$l2Yr=fwpeMS z^TqgW4=E{p1$CwTm+D(bRL?4q9_}LOIOnD67f_8t?FZcC2cseI>hFs5@i-EZ!t4-I zR2-yBWSRQ8TMLD4PO<6ZNYJaKG@751tKCo}6!}c8Y9TT+7H=k!@kh*wpVt~Zs}0A_ zeiN6MC)>7Jww?7MEGw%wg-Vk-_oYgfKd*J{voUgSEB2keJuy*Gqx`t@Ghg)8D@F=9 z`FvDP(M!T?i}M@a5N2Ft&&G7R+FWeDOEbav@{~~lrs>aHro68+mj!bsPNkEHbpT-Q)J2qef}Vpq7qqqRfWq+|lxJFw}*;Pfb> zrt8W7-JA}0t}$Iy`6|QQZvd%{jH=X*>EGgc8CRQ_UCC~jh0lW(ls%bN+*a*~U#rSw znA+NP)>YKBuHlN6R6mlIpEsSQXe$$M@5actoYO~AIC?9};gy=FfL6I^y1FrNQsEKi zq#Jes8RJD3y;m;r!98XVj|G(=R8|3Q}KnJ>5wx(T*2oYMsjzh)UV?;YpS#;u@uLh{+wi7e`fgPz}3j8ChKkFn=TZsV>Ga&$Rt zw&BSPyg(!~_E*c_R{0J6(yTmn@fJPBpnoqVGiypvtLlRKK<85jl2>T0eNcdKTPd9H@btc`Cseb-K@ z1P(0M98t<;OU}EBd3)KX;u=nQ>Vw!H{SP0@1++^mY_N`a+h^4msJ1AlpEDK@e@Vi1 z9I+46VRmvWi7+Ts&Oc^bCYN3tSm188=U@KH#TqfFX~L1n7@ndb^FxCuXRxUB;*RrU zq3Ze@g0JHCv-|hOyne~_5UemwBeU2Uc1lNa;$Txm7#0f9>TxY){?6};rIRwY*Ds|M zbZJ&|m)Wq0Bec6Lao8Mu<#j8>=_&niaOn{J@S7Wb)4pZqu1D+8y3~>SX{xrS&9m_D zHg4l*LqBa_-U_)CS`4=CX!gjBGSIvE2r&^Uz0|XsBGn%rNH~}i#}(w|ZL+<*SO9n+ zIhT#1FIOa%5y*jJna`{fZJ$k8{j~IIzVpAIJj$}@pK!5qNxW#-Zw#0J_KMkGz>SE@ z{)u&!QLXXBuJ!g#V(;-mtjpI=XeKnrEB(v`OA9CM-o?xLJjkS&R5d^E?FSo|Y!19R zoTKbZ51Iku2;#4>6CN(-e)JW6Atj}c>|4{28PVB&_)F`&nypqI{L$b3XHp+lwrqI0 z9NCIL+zN4!ZVCR_CX}ZhaXT|Re%wnO^mR3drhcdJ_wfS&BB#xnziu7WYdAy4AF>&* zC~QD8S{qs!m!LH?T*O{7zCkQ)LBG)GbiEYRXkwuhB%<9ibsKzsqad_k=|D3)ur}V= zNGnqYG66M&$8%Rx4{>7?C*A0>CM2AuHFvv;C&k)@?GV{SuF*E&ZL33rj@rm0bo=7-m(c;IUcD$1{y~JI%ggF)zJ=Q-LWs>ktO7+SJT5CD z0tzVdimdL_x#PJ5pl%7b+Nji5S371L@?-p{^=Vl=R`kTM!L6;TqN1V67!*sCrmnb= z70>_Vy=aVc7VeL(J%$RlCb+n7GEs9*Ma$Su#p|~hbw;wJ-oDG?>?{v@jT9aW`<@Ci ztTjzA<t*;#Nx2vuB4s4yRu@ctaZ&$%w%` z`g&HmI(;no-L?yJ2wkWBKx?T+ItHH6ze54l+nAAR$VHF*JG93u0L$On*~?qfFl@fpL+nVHy1D+2RyI$Fxe;o&}9JG0LziK)yGwi%Dt zPV~uD?}%;0XB6SUX4abstl11x8aoiQ$TXO6?37BMW05yM$2yK_9wXA7$`<-;`sB5N zEltmjSH^@{$^?C5PsUii_EP=#6qz85v7$-?S7uy@gD6oD7PP^1JFwgU`}(V0Jt!P& zJM70iFjKAEZtt0Yevh#rp(d8fg}-tq3KQHWK!Ksn^Y)ks`(tfZc%C6#clD#$kE(151#u_ZlAq4BC9}-AmjcANH@G!lBQd8JG6tk;_`R3Q|`8r26`@Jfn0jD;wlhQm9GiNYZ0gFw*A_ zMa*XgzfL@2fAyV*wdbpMn@~$4?VV|4L6MJ4#2@%KWSf#LirtP{{Ba9mm?B!Vu1-3u zTZv2CU-UOq-5O9Ogex;>2h3FC-zw!_k7sxYj+KdKb8=#8zorS=vprxido{^3tx+F3 zRrv5tj4-uYwUD`2Lw4p&S|b4da*lc9O`yA)FI{pqZ+%ge)tu)9ML{h(Z%4TJlu)jozSNfM0kVjH$^d>7?Q%fdW zI^=|nKcG~Rk#lO$@hweyjQJ}KI(e^@h_8B*Ly`33_C?r_0ObF18n0jw?(i_gshVo5 z2pcWAVlFvlRo>nuW=8R_9`U)+tH~&$_JGTiOd{iG9NWkFI?h2NvP`Km&hHqVLPmxm z=}$m8JY&*595?$9h5L*i2IjfnP91_Dw{u|OAt-?Kz#&*iLlqf&QO{OS^(b5k30u^We`dKeNa$uO6Z8#0 zVG_o6{*ie7h>l;I{bM-4?Pf6)zE^z7iYvq(MB`g4Ygv`dj8;$o4b~RWrF^%z>p3=q z&olmoD7aJE{D+$AR*ccSay>@)GWtt^^WaQE1%k`xRr`6S(Eb&S3V~l3>s6XkM7b?Y z%+a(%Sqx{IFW04+>d3;Hr&P|g3tPGP3uL<`QM!;`|DOSBRMXwVrMNIN?hmabt4-RsKP{ z89c8skD|J{6~+Htpm55_ttaaDpRoCdWzY?xoqtRau{>%{%V&AF-OvBTkQD|3r%0SC zj#)G(Hj~#NR$!lLD(~iH8ws%>m)$fx;n944K;fLr+TNn%vKmx6MW@cW9v3I=;(CJh zAGU1DSb#WNkaE2gd3Trr0Ry!toyr<=nhGpT9fo%0%6J6L+w%FCbj6TSoeI#^Cl>Ph zJ{)L4EG(>7sh{U@y&exbw&zoNpe7Alu<9EjXRPAipO#;7nq@!2*9Oh0F%dtQ^EmEc z1EmS0_Yv7J!SxQY{OGY+w+MGFJ`4-snIZ^}zAG7HmVkj_83N3=e@~;hu~7R2Y}Oy8 z6S)d1*pC@L3YUeH0UPS+Ga53On@JNQc08Mza#&~`n>QI0ejE{fe82NL?re=^{#ybB zSH6E~iGY9!LkW90ki>0E@#pmbb+C!y@MyX7e}6nkalWSlF9Dc!;PiU}YI;|XI6q=r z==#V&#}HgxIQ07$SbX8^^Z z=XJfFG74#l8(iPH9s(BanVSGy8dSX$W#;6(06DI@)o?z#kR3-K&?>oGXZHS+=KJ%A z&J`#g3lNb&H+$Hao%eSl&L}8a1fmjxKupQnnjyR}7myt3o^N7ViaTB4ko1< z$8C?$J%CSJ=X?xwcuE*I;NQNl``o<(nnWZ*UPS5xzpr|QY*r~47z)eFBgqgGWCLNi zq0Ut2R={)JwqTEyuBVw}SKx!b9f4oA;Rfj{E;xt>bajz=cIF_MN{Wj+3DRu0I9IUA z@_Nwf34{XrQ)Z$T4uauJZVC$hgZX+u3KfCTh6e{{IbiTZlL9*J$=?DP6e~^+;|3_@ zZDm?jremP_($U$eUZNcgv-L<@1o2aYULtP`OVuoTbt)O(SXupH4j15kFI4eqSI_wc zA~4uGGEEx9@=4Qs+q7CmYl4*pH}jPUqiHb;m*{xiVRE*}4} zGEN4W5#e)Gtj_iyR@qNW#a{=VCyh-lVcmA;7XK~rWb92|fbO+o1i@h1{ZAV>Umv(( zB#eykYQ?ELV-6(c^5e+Nj&xU|_rO&CyyK8+CWKjaImzoka_5%y=+A@y{U!K^K>a^P zviz(5=hITfd7#m{W7|Gol=ni#;Lye9q2RKH^nll8FXF=1X>v6Qof0Mxl)PY?&2Bvg zSItfz7`b|fbjS)l45~fUM0Rz%n}B;K z2U>GlXz<{m^I?X$!`M{4sd@y?MDr(S9@#IpM{$G>=BoEcpQZxb z_B+q%=;C2_HAbE4dg>3$71a&uhNrzGF+#c18~cJ1%TGyX>T4>DN@VR#mugPThS!md z)|X-D$cS^kI-;^01}4suF1>3LOw+Ej%0=~k4-2jgz&I{12bM~5IH#7DgoU<<#Tqrr zd{gUPvwwM|(Jra9Oh_u7i)n%fC!59fw8~=7sJ7N|e|wm@`kK+`=t%T)p_obI*~RHe z0pp>w#|pUeNqym)!_0m1yQVDP-Se4M4LkOzXUwEnS*6&K*XsLg9`OkU!SR@adblpg z1)DQyD|2>Llkyy0M}FI`yxuFJUCCXF`VvmWJw3nd=~#LfTyHV2^^MKwjKBL})v{)F zf%<5AZM*5Jq&LzaQ&D2~7rV;+GPd zwm(o+$!eVe%3G5|VX45ip|=vgHCANH<~%BQinz&0V|{04B6@Y~!~Wg%t{Vq0MJA#{ zy$tTFi+MQlZm~CxL_F|jL>%Chu2VFeKf~|es%{SKd2H5ndNN(}yLkY2v16~Qa#Tz_ zJdAopIYqPCPpGCBTVEOLNBYO)bV|$pXji}Bl;A2*F+07tYt-7u4xg(mUl{Xh%Dmr` zu_WHVN?KLsr6(f;Lsfi2yRwG*Qb=6i?>7wi3(VH=Iigpk8mo>StIjSU7R;+h}lSaoDRfy zXit4bbwYm{oAZwoM7NP3yAm-%-_Sd3X6JjA${)Jj4rAjuhi0|=KTGr~x%U@01x1`= zOLZfh9>%O??4F^uw0sn=zE(V49xdd;=Km(-z2HR{Vn|EdOTwdAGPh(i-DDZ6-`!rf zZOfghHG7+6FN?(tn^4GgC^Wz5G<~$ld+nRjxP$ofUA;|rpk-JP;f3meW7wXSvBFmg zGD+w=U+l_yd3D>aRlsGPU*zHWOXj8NJXJ~>r29mo^6~L=)lAljP%Z+O*j~WU5lQo8eD^)N1{TgxmjMyU%>irxuwucE{nRyO#u~9oaUVjL< z+t$eL#WV+QG}E^ab3S}OTbP{IV)i69Yv1AyX-&QEIyQN3t;^{jCR~QdXHR2#xX3;K z@5Gic%uT_@^r+lHQh+U;aMixv3FO(FNz?K5fY~FiG(P$7vk!5OV|vjTh(!M{>kagd z;RC;!?bc3_V|0N^^M9Ihe{c1GdE~-D@KTUOJ{S%L1r=Q2!EStb*rt3&7za0yqN7tW z0tp#8^z$dnC74I8EEpL*U1AZv^K_E+v-6QkN84yDip2gHuTsPnX%*7R#Chi}>duVQ zd=C}CBfBTQS*-m2=zr?Pq<3)h%99g%wYEhOnIcB-A+2=waE9O^QCRH^dH_i5#MymI7;TFYXa%%_-gCtN}MlhQjw z+43G#n{7~@!P|l=*~rMq*47pPoY-U0%1aW&pUs)~_H>hB_<-|NSjnJ+)<#0_XzgTu z5G3I$^k?+fOg02;R*{)iqp4gkzzuoi(>arFo0r1qE_8YQm-d2tKHNOhgZ>@+hadB~RaKx27tG_>*_8-xg7Yq#MW@fIS-y69kBO`N4eCUh3 z2Ef(LVB!`a-WHaYN&ugp=n-TF=b~5W0n=Fr0C^OB`i6(QlaYveuULaRS@BT7Le?K$ zP?+pJltQu?>L-Mb2ww+Ks?frCGiauQWtk!b+A+2Doj;X7%FfP);-5j}=(F&905jsh zlR6?+g@>C8C+4$Jx(4WLJrF>pOQWKq0+V~%)rnK9J+AFYzFtD;H#ReK4JMhS`Md&CVnDC-er40T zZutplZU8tkl)r`u53;owA!;SsJeIRnAP9#2n2k4CS6O*{R|edc>d;k`l**kCb>ZRQ z!EfYwtP75dXgk{BBoupIP*{kG2oGXZRr7#F!<_{h>>(ol%K3LIq0G>t`QHjKISA9X z%^gD5aOrE{?SXD?b#*o36+!n+g5w8H@8e<5Yf)_`f=(JQ+WIy)7+3y81nnT9yf8(6RsDzw% zApSQzFdzc)NaC`7m`Hv2d3qw+MhX&ZkSh1f15mVi&4^3+BMGdt8Q+z`zm$ z9PF!gaYNa-0FClB0~`S2*@uCc7uVOBDJ*tpO)s@?B9+Ij3`$Tsu)*88_L_^Q&; ziHV7U9Y(s(V_PCB8(c6Sk0^luu9^c3yaUbiUZJTc(fwln;-CP!=MgQPjT^cSh(@n{ ztwF*R*3+K^Oub7xIUSYjY62tu{^kONpagwH?F#VyUro!UL-H*I!3X%SpaZpXgLzS^ z%fy*<4^nv)e`+F-vDsPa+zHG&Sy1NZv0eE@S~Dw(+GX2E{*OSxSQ);rhBnGnH#qQ} z3!O5gM$>W8AIEVwHwIDBIIxIbnV(dK(!W$6xB46w)~#nu_x>+laxu^ksmm6lopcc7 z4OBjJ!rCSRfG5YD&FAN!r$l+9!@hS^V9yJ}wr??d+x@G!-icm4TDPlzLlyoa=+_EL zLunJ}S}da?b8~O*_&r+i6YEW1(9eI8YkR#yG;GQCOB!}Tt?;e6)6XQ^H|*RVzLJbs zHp~`9$~B4Ou_3&1V&tRCSpP|r^bvyCYQzi!`r`SylR5{j4jn!AQ!M6hPk$LI#^$5C zAg(Krr95a#ia%}s{Ia=59Q}J$nU*}`V(3Xsh z0aGv`>Pu`?3Par5Q3AFuNB&5NVl#(Al4h3j8WqJqXT?#x%Lg;?0-6q7mH(qsMW~bR zJH@$k8vofF9S29#pt-xUA)1}vz}*|uB~!Y-VUFtPrv27i?ci`u_ITrxi6~ z6xMBp>U-`oX#U#D(i`#gC)1My87m*d>5^Es_**HOI9j+Q&nk}=*rlz1?0h6l7BF6O z9n;P($t&REa$D3aAX8@?Sa!TW<{6r;P!2OeQA6^2{ly!7b2N~{e4VXYh0J^N_b`b+ z9G+_LYYMh)&RE81%w*-da_LItwyea0m=L*hF}D^~n&=H^sRU~{fN=V1^j$hf+V>Ux z@7*zSr7^Mi!4}s&1I5y;Q)=YurU*L{QPl;p)Ne;(zUwE*eh){X9*li5=eo+x?^>G6 z7RsfjR*YDRl6<7^jb}h)N^V}>d;?9qm2$k<&Hj;cjJpGsU1CAl+5D@I zmPSBd&SzFqL!DP})Icqea}>*w)Qst*+;yaPEY4ls{jYomzL&{=w9&2zWeX$Biy715 zOJQJ}TU2xMPO3xcnw?b)Z#Zs#nh~FOhQ79 z#x3HYsM4)tXIa;lbZMNA3nr3-X%-N? z3`PZfS_~=19R21Jrg-M}myMsSK1Kgfht_R8bn<6b(ylz!;<4={K9P$l^bZe98@lyi z27l?n(*Cw#TYvx7!bKy&k0RmIVS+h*XiP~F`!`|@4FFl*p7ih=zF?Aj!)6o(NXc0d zo(~&-oms^-Wv4N0*7tOhM&gBGF`;5(DbyLK-Q6uuqx0m|we+lM4_mB`a(&6PXUTf_L53x8N@ zs#Kz@`6164arz#&ZxR3H$xEG{di?U4gzjvu7raT^$<(PvrP27iMN|)+RjRwPXFPTZ zKN|&Npk1Pnth|yEZ1F^T?ChQ`B3D~^3ST(lF54qJj;#R()SNdam&YyR z)6ehlpI<+G`CD?A`goc9XW*e57sX0j)OLlfGIlYxr|o}vxiDNbH3hs)nGS>MkIVtYPoi7piK1@JdE%xE9Ha#-0? zB05;*??sAQYugMEvma@B?KkpaEdR}DOXve3TME8oE9}br7eoM;U*Gxo?BJ~fYQJPp z>yxJ74>g$I?<&28nwr-6M;{%M;{b6xzogNMZ-5k~$+Cd)7iGHY z3x&;!?U;O3`ocr6NV_>_b0^5?8>CFWk1uMY?HUOOa5rPlzkgvSy&Yg52zV6Iv&CLw zE4_lA)Tl%`Ja*=hxZIXn{tK;YuHFxLf8EMBe6J$uW34 z?&HP^SI&4sW$KC*SzU@~(}F0eU(H{>bgnf$O*>tLAsBZHqd<5!cmIF|N^MM(8W|aE zu&JFn(Sfg`ux{%=n8IV?G}(}vAmpS{h}^}R1lBrw+bFK}?O zgbO0F?T;tJvMcf=r|!d#JluY3*}U4~f5d9q6;ZNThzLk_ynKB37t>@-qyz6iAYoJZ zTF<3YxyKp9XHc8SoPI7C4^c){c+o@nvrd}Iv?}KNdWP(quR>OS-P^Q7O{o&`m4~Bg zw4?ww|0jDPAKPve6&K?ITE>b)540VWI$lMSOa_kv_i5XTk zu}y6q9lE^01JvA$Z$4nOYAjlngn7hPQ5Wna^wJNyVo2&`I#bb^7Uyr+tvbH{j-%V@ zp#OPkIU#_(&9SQ1F=GpLAI4ufhg8iSd+_+rz3Vb)bh9vw0BrqVm$g1gWFS>A@od+v zKSPI7Yoa=pHIV_sa043$>9gkB1VOy;+8}N;k~(ZLCiCR6&q7r2NrS?mTlyrl7|hJf zFL=Qev~k^cM&vs(`~vHg_=e3f?2+@jot^7QfpYAkh5a<+t61|8>Oq8Qb@?*((x zNX;U`6%CM{x-X|T%{_WkGgv9EYg92abb}=KvKz%PK&gJEiQ|<@ z6)H{|$7>0xUbtkAOsljh0{H(FE>*2;daiL7#>8i}{i9(XwnVGNc(@-*3)wiAe_2iz zzU@gdX&;N2P%KfzAG&(}u3Bos$Hh-|BW;mq`C6@#&@n|S0d+??EG(Kxjz@qHPF7dm z38`N61@kY0j~@$uX5+l!d+6YwqIx~VXd4+}rX2rP15Pj^T?3PUOFNxxCT%W@g$ujT z@c-17JLKa+ilk$V`T3IhrMi)gI?~A+HDfafsJ*DBFh&y6p?Hp7_V}|i?SGo6!SLIA zJO~;uWJ<^K;|mF;+}MN!^k_y-Em-Vtbvyh7P5Oo~265|E38Zv+GHk7$g?`E^2tVOu z01}78M<=F2-R8SUwl$su7gr@T=d_h?AsdL|_TL+RVTGA2scujtPz_eLesEqQejjKAn`cN6jx>N7-On@3)P_FqJyF7{220JMT_NF_onVDe7Y z;^kbH=J#B&vHqDA4__5a@dZLwl)H1EFVxD(bN0fnz{7eUaSu0|n*LL>L`|k!%q}oZ zm`cjlbT5QWid7?rkxr3rCQ-V$B#Yqi)$mY!=qcM)-sUCTLnhq->)_AqS$9r;Cpj`w z7osP%%tTfHj`s^eoEk-MW9a9E;&EEoKYnoSzhe-b(#O;3KMy|+5BYFkrO6oA)zu}_ zC@g4{&&V`hZdeej@!%l&Nc#|UIava6opMg$;rv(Z>iOMuN+HJ7kO2@gKO6iI5sK5Z zh|VdtqT1zNxhNF_1%XBd~pLty&8;lVAI)QNOo)0XKyCa6rqI_?~ z{S7}H(vBQv{CLLjU^j%v0k`%yCG#6ifO>Fu=Sy11)OkHSLL6cP+w>?np1ULTFm*Tt zh&xaUEEG+wzC-`J8s4oEuE5}jkQ>F<)X#9D78ZM8%&U!Ya-%nO^6=`dJ^KtZq_iC1 zr8wZ>woALwUmxPZ$AY~fsel`{ugoMEt6f`ocX>AsjWGBP0O~OCRpg@D9c{0c;kn$; zNx^I~ESJ=)*fVPXr&em8#WDbh8SioYGTr-X&3%A-{teYLAP7K?YkzXd@NkwZ)C{V{ zh-b)$?Q`|^w1(xs|57wRlh4A!1x`Og=U3FzYXw*?_?y7isyIdsKc6M=2!3ttDga=G z#Ou0kXM>Jdw=r^sJ1Tkt(1Re%#X#$!gm3}Psj$HxcVO@26Cq5HFmg13dxcHrL% zXG9XvyZ|K`s&`~WOwG^Vbv;vP0u1869nQOr(NrO`@vQURsZ0zfz-adK^D8Kz|J&kx z${h@qz(9K~BVx^lTdn9_IS}UH??7(_5eGo$Ru8={8!;@BNO&E#0Aj!kz<59@6Q~=X z_`w`G?ikUXG{2i0H|^)vOD)8IT9+a+^711<`ML+=)gA} zHv!HgsH`=DsSeO`=^HnzNmueEYF;oP+6Qh%=m8A4nf#{~aQqcCSzFI1wLmKqaOvn) zPOrIh!M{HRl~3)vonBIJqAk=gfZgLwmY>fNCbNMS|G@l;i@h1j_(9MQcW;*g#s(0X z7MNS_{NN@47BMv{>g1=20y%~vux`=4O^+oAe1!vRw=NHtT0vnlMPX%$@qr5ieazC# zEP)G5S7V0OGyhm;v-?i)aN4%EtRqV)3&Bddb(LS>kREh3%HFpFIOn*g`i_jUR33&1o( z0MZYNk_M_G)F>uty*mk=3C+j@myl&|TZ&b*iZRpg0LgbELlZ4UAG^XS4kaS{f*S3o zD)1=*0Mza6VXL*ZH7z3Op|4u-d|9n4F5U-+`fQZj3kbE&`?E%WV2VK1CP%Fe?hPaC z)i2M;$gpwN2e3SW(TUDrFpCa$K2F?g1Ng3Gn#F&by05ABz#BzGL0Zh}Zgl(8d479x z`gMQlz}mh#`J=gQCPSbQQf?Ekz?Zp}qq;J;UDj+h0eiJ3@%1z+o47o@uv-WPE_82JicrSoQ?%QjYi}vdK-Mt65s3+c-ff>GY z_u;|?0~{r^?3G$dYKiu?xU+QlA?{|@9tsg&673_#@OOZlipWVEmK493bkfKBXyrMR z``gQb3OJG*?dz^HT27C}K)A_eOF~Wy^X1m7o>`ox@&$!AJtubR{E5l9{!j8Pe@7vR zGoU>=a(hZ|lSYWhM2s3+80Rt1rc+}(=U4UaQ026eR5P;iO_@t))PPovP+}mQQt$C( z92%#P_3|DE=SQn8FW$KL=1)OSikq{`)3YgArX*RwBpUeG@Sgo&2y-e(lN(M(8;r`8`ICz)KeV^%Dkvp%a8EiKI-crH%t z))|=9tMU9^ETZF|obQ$?AZ%uSr7oC)!{Aohxc}o{wyb;TkNRyh^F^)3v-Yffm5vB&Z-CXpfa@=*|OiOb_u{9^B#b0IdlTM0qY8(x8vEbA}Oazf$RmNO`ijU_Z>L6z9 z|Jq)5sp%zP_iT?2n4qaxbv;F{nBXNq342tdm*4kkMy6sSN?$+97`M+p+7iLZ%j>o8 z9_nb{8vnIQ<;2Qrbj~{NB9?bvdOPR)pR40_*TU<*oDZ3UNG3G2$Ht`*FD19r+|V%i z3}!ZBk&%o?zNa?6w-T9a_m|MbgbrdQtHYo>QAV&7vV)vbvQC(s&gdv~DYR1K6Z6=V z)py?R=w3gwpQ)>5F&fx%4O!@&v*>NKx3xHN(Dj&F?JHNx%XEYcDq`)e#blKjg?^x= z?W!T>;cB<1a7V$##}Q4dDWU0y(Il`WJ~n&tRA=GamDWb?Ib()Fl#A^`SYqF_ z61VbRmka%%)62tIA<<2YFP=7tjSo!i=i;iehN=>C>5CMs^lWw*Tl9 z!oGYVCbVCKOLcz=EA$vd6VtFdpHWFz@s*vvpW|qE-CwO4t!@bX)QFVRo$5(XZ)d@X z8GrI~2j`9?z|c@u8je@BnB5_(*pp3xWh`2EzxQjcQ`$QR*V%kDh3duZ%zKyK8Rp=N zn(&LhZ&~RwJ$ZQASt`ZE3$RPR$QLy-^o72^eRf=qC@3ht$6&i}tQxhOR~xuqAu z&0RBU5;{rEs{`%A>2aSY2e~c;l*AH(OZ8Ll1yQokeu4J9pPZpxd*NZy5*@B0!X!h= zh^L#5k&KA%U9rD4<2SFdF>9!oZpf|)yz~WD>j!i9Oykm$nBgw>i)pBt-cxG{==!Z0 zcP!okvNcYx6D^ZXZxZi&Z&GZf4|Y!cuJ#p136fb0->Yjc`{B19JLEO$CP&?Gd8lfz zSmPcBzpT*;;bKl|e?2dB1y{pH)#V%KpPXfPQr8z#3idl&I0m^k zOIC4yINvNarcGy1JnvDs#G(FS@PT$h5_&d)P}a5sFI*e?zb1aW5Hg3&>8cdH5|>YH zCmnzzLgBg^C**JXFgE$*xgM++xt)Id&U17q4_IN;yZ_XFK6y}AJ5|6Iqj$)+@Ci_{ zK!w-whGX97ISQlk?9INTX1$9b$v0P6F-wr z_7O#q5RtgeXJw7RqJ$qAAZ!9dJ}#W3BI$beMJa*$sBuRN`?0r%bw!{5w6hG{J{0>I&7lQNM zu?sK(oZQtR5$2&x0vpT>2@VeS=87-j~d3EK& zJ&6j#LItM0AX=#ayOSBLt4i>o&lnmCeD&8bFgD!~;aUd2BpeX5@*#({^$?%7a56V% zfP?X2crqNqzy@m5q1}Ekc#;5kM8~~=cZLk+YuIoTL5FnX^#Q8kvYKw1_mQKgxJ`d_ zM1&1soWQpUL+pyuyfJ#!LwZ+%6%6~@4CM*Dtw*-a?G>QVh{B-2TTA|lU3z4L0(k@Q zYhl6IpZLMY2+KYXU@%TTl(qs^+#q{e*xW2YPmlB`%ggWz?A5?7IUKF-h-nMJQQ)9+ zh~VmIuh!%BC}3_tOaWl7p(8TD-+Blx8M1*VaXQ;l;zor(Lw^tq6xP(7ZTCk2b9;b4 z2})moek-}ngdEpBI5;-%Uu*?+E(0n$G&4$sj>;6@XfjFyFHCAf^p65Jm`oHI8~av8 z#UEx%)ZU&wY52{b!9k`&AQ}ONBq91NMo?|r)q_3rJ$MlWF3iJV8?w8*iwX1m8aUZe zQBfFw2P7pVB!JC6D+p5pnh{7fK0@+eg$Fa>0rFMkHz z;7Pi;xb%^vp~k0@vVrkFE!?lyrOco=uVDCK^B zaPxFxFb-fe3{^}IL>YWR%fW1R_0p+SZd*8Cl7G&)#Mxbz=^^v5bpn2{%ECRd=_$hL#zgdF&(0NUp)#ph#w3P z?(2jGHkOx{>8c_opA?3QKq>$Sp;Lq+GEcPt3I+0`OL|?by1Tny0vh9rs&rUXC?0p#&wzx@0VVZxP<6`+P$PqwRO15=VM*j=c7na_>ZA@jdAro?PB`fxC zyZ3S@HY;D1qVPGLOt*H0G)+F)$?1Q%y?q}_l+{&#B@qe?rv4oeJ#-NNFUlX}c^8id zjct1$P3-^E*m=ie-S&MP3E6vPRg!timXT~CdynL@NA@O#RCe}^6j?eWzW%L{8aU9>z=lxcHb8VNJP@qpTr@Fct5c!j#&pYp= z4@gzqzOV3#Qc>e_#jrtAvSIpLgs*URMxhaJonHqcP;rjttMyms;-Ns=g!pbvy5%V8 zl8(Jla3)5MW>4L*Vz*no9Z49}r~6lVA^1zVRnb4s0xaHAH?X-=cg;(7Lb*!gvmY1|?jmPrFb zzxx?UZGSq47nX}cF@!VyXS?T~WSP3hP}2V;c2D2s8rI9#b1uF%u$M2GE-Vv1l$vDU z-LPSfqaE{`@QSpiwl>Q94q_?~F-Sag?<&Efu>Qq0l+)07r;bChuEndj75f$yV}IMF z{ZnQNbqby8rFHuTIy!sy)!hMJ*i)hRWszoiFXhw66a7;4BK6NdGz=`50%^nlN##!H z;8004L5z;;vfCslU$hzXvRb=6toUwB_AO_HL4EDeEgKc^L-O%Zl9M>*`_}%KfwXi< zsN(<+jUy3{MT7efCI6eFAykGF6B4hol)Q~;k~8@EI#;MjB=sDvsG4Z3J-!5lp#nQp zOUiqjdOOYQ?pAT8w`Ei+VVWV6E0ayXzzK!`4w7kO@M(sKla@+3gSQgCrdh32pRT3N zpOe<_sNqPWn=hK;c@v^4q=w-*v^*{Hptjbm;n9neXe->NZB-X*d#J?Al~r zEXOq$Vbe`(8BNaMBaNIn6d4FQ&mw(QOpGUfjX{}RkWr1eYD5Z=y6Va6W9zS~_ za%WQ^1xrVOSnd~gxn<9ZBC>Tk09D$_p5*b4gQ&Hmk7{>^DB^~WVnA&2_l=5f7X?ke zB>uycrZqD?Gcy~Q$3FIaJjrdAFIj=xT`FvA6A-NYljZeK>BDI3YCSnPYjQ&Y=B=c- z7-ZiS6%|@v+-5b!IzC&@60w7mn3s;NPicMZ65|ivXE6TsdO*3B<;9|F>RO$M1C-Pe`Jc}}sFk~{XqF5D%@3m2L1Xc?@0n(}9i8kN4HU{{{@#o1e{Fh~5Sb!4uY z`Tgel$kJ={9qi$p1I~YrE~9$7OndjFUS?%KG$&!SNzSI^wMcB2q!TdDcR$>{=F8vM zV&cS+zLq`}M(NBFr?_^!E|u07ER4i zVJbMIrclA@>zTcjZ-S^Kc#I>!}C=7hBRYoElwc z9PG>rMkm$SPscH1S7N!8b3FTA9v)^ANRzj>*e`V^`_)WiWEpaJOU=pGWBLyZqIdO@ zHkD$8SL(tl8AzG@^I#KPtSt986LMsM(jWH;LbuPIAXIjbmurp1PW;sSJZBI`qf6>zK_D;7mM-CRN&~boD28;=L|RHTGNGpYYa&htR|VIn>W#vQ zlp57|>Bbd7_dbWj+H6KYJwiQQWpyG@Ibmw}nyl3^-(VT|as z-p``#GW8$)_ZHLacx|*PH{IQ#^REK-vp5G5KdWzz68+yYl$mt4{^;SPEE!y7BExn# zQ|;{TuA6&uQH$VZC6d?rdSG>EeUAR57GH@Hs`XW6j=TgunWZ8)yaPuimfpudwmY8K zV<;%ERa3!?T)oJQq$#59H9IB_sBCIX0&S5KCNr~N)b^Ed9jYXu^uBJFKpkO9y;5v6 zU}9HMt1Y-|d-dsBar-;^OB0ood5WgRqI<1?-@swqMyZ%Vjr(dxRI;Z*{{$BU+00IeKZN@)x)*Gmy5{M8gR+^8fx^J$K_TnoW5PyUh#;aGCsTm1;Y_lJYy)^h=!%*p)e*e}& z|C{B`vSn9Li6S0C$~(RPmZ2VyT%0lQVV^fOy~W}e8WP1@pz8Ymc~Z+Cg<5NGSImD5 z+UW4yE>RcFG`7es%14G0llIRHA(@(8h8Vi5IVxq%i_6^;UUL-`jN9#cROo(TzU^N$ zqNS^wk9c=|++D2J4wVYyL~9)-eN@vEszKA~pXCwH)6#kFjTCnciO|-OZf7Q|vU~LW zYIh*ERn*q_y9=+ME1=DbNYz+bTHMXj&*!cmTYhis$URD}mXgZgjQm!!u2q)8HH0@q zltCDloZW3@mX##DD4fEhmn{-jU^=C;jay_s{K0_(v?By!ks+ z#JcAuK^DImBK6AktZh3=LsiEriD&$E7>9eQ^4xx2bhnm$wqr+nt-d*ajeltp&%3lW)JdkXu5N%-#)jHP->UsJbBQ> z(Nln4S)ROV!ns&Aigr?K4Ibb3uj%`2ym`}lvh)`uYa)HogRODa*IP7iG>$wRxRnzx zB;+pq>e{JGb3Zeq`8k1KzxP#n8Do|+5yQJnJ`duSl>gDUr@}`sk9Ns?b0yL2Op%JA z<6|-(i=3ZC+i$Y3+?@)9txgku&JZ(mDk9-#db`yVD)qa|!lb{0+f!iFy+4x`y@Hng zop;^cOZ-KqMEc14Q`QVJFV@xViQ8ERtAq1IaguFY<&LBMdfGo%r;2y^CVbK;__rcn z1wNE4{jo^<$G&}ud#aE*Z-aD`nC;Q(>UP`IgfHZG#pcp0D>(K7Uj&vlo;&f6ct+6w z&IfsNLf{YM6HKTZ<+<|e-ea-u_?hxG8%mmlLh5FU1}po(z>ht)M7a|W|5X?CH=KS% zu=Imje#`H%)WMK05gp>B$2GWR0b1#fkR^=*_h&BF)^%Z#G~x_yNll5TL_M=SxFAdy zb)Am(Jqp_GxaNjxEqV5qkD_dHZv8-{ElZ?$iNVlkw-l;#tE>0QjW}_i+r|VPkQS`0 zBLA?w%=E!0|E00Ot8GzdOUpTk5o@{ablEDhs${Nwio0vk9JuQjVQ4gO{O6Rn(%i}o zC5JMG>dwGh1Hd})^6@FkvOk~q6Q4WyIyN@8Fhu4yHhb6q9!&MJ+F1hE0qCsEhRgzT zCaiob=!szvp}!5zBm-XBr@ci1mzrH_05=;%j!(c1d)juGC$3A5;Do&V3Z5b~GoS!) z?g2!!EMsVB2sT;-QrloP})T`X(!O!g^OHPw50T|)YBl!{ia<%*Fk)A4IKIdB4-FeJK) zpa<|t2Kg6dD{nh)@Q+yfnTL%&kMz8T3aOVkl+H5VeHXc0s`^e;BCtUxN-)(`udHJw zgGY4jo!)4SJkx)O-m49Xmh>IWVy$`4MZ0HqUT!?%*84D9#e2Z);o|3y zi;HXAYB&x)n#0FcGT4h4e9u`T{ zQ%Ib*iZ=D|YG|)@KbO?KLmpeJu)$HuH=amdrcGEIU5sm)cMF)##r?j>hUdQKD65E4Id^dgx;gb>y6*GK%AR<4k?nJnJZTNu^M+qPnpi;Cmw`rO_xRvI|P zg)aCMKT;EROwQN0m5RJ;uC07|hB-5h%UG&*SW8=f!rG~LxLBBkx(D+ctSs7SwN(h- zzn|FBuV<>S!yC`NKP@kAj~tz|zdkgak@+%_DG#VJH*iAErFI~1-E^D%1ZQN~>FEs2 z%rVmf^rl!?gjA{uvU(?VtyVB$vm~{Bebi(| z&{%xSx7(d6VIgkVupWK(^f&)RT4>Ov`f2->?#xsQng0fpQQ7lDN|3TLGfyJ}k2f`C zPC2AJ4U~euvix{*EJ8tXX6_Bm;^|VN*6xL2-^FJiz}vP#_XxUT{rZ$a!vbg7nZgbO@6B1B z2|Fc6dc2-GKF4=8(7#*(E5F>3)3BYqU3Pw>X69@*jLcfl*ua36(Y=Yf{fKG@4rUj4 z-}KuPL<4SwSPN&J)3D{VK0DotrsvSaN@qABEtvq)GZBU}SKM>1oQ7Cr=D59(Y5BMlFRG zwVdIkRp@~-JDl>(fws}qR`64-ZP=~k+CwD@sgo^ar=hJ)K`4zWiGgJN7vyII>4|YM zc_C7BP$c$3dCIOk?Gj~#)|YS9_f7Tnhe2;hffbDBvlLz`b23M^w6p|kjZ6`z6oV4Q zOE2^CK0Axb)1E`)4hul040B)1fS^a=GUQZP2z7uTPQTeV`4D|dfYk^%@Y-)1KFPPk z4-e+F8o*1aMScT#pe|qDn${rr_(-s4%h+O@{DV(+lqZ|)U0m97v*P1JaAu&1RO>Lv zaq>p#sNy!383LRKQ4r(6GwcMSs7amQ4y+8KaYo?53|{;akFoGz&mO`QJE!4jo~Uk= z2px`V&xxs)HNvN_{C6Y;`B(TkSV6f#&=-;cXnY7|XG}~CZ0DML<{u$8WD6lqk-7*x z?9aK;6Le>+ofxn@tfEdJ&!UVW7vKQ?1xehX6&n)+c=SM5)DrFD!;!bfVRTy2NLi!+t*MzMzEc1!e5gJtUa?1?o#p_cK9Yt95e((MjecmB9nC%6Lao4oH>PGE zwhw;|q^s5dMh>pYxwpoZU!Q&2i;RxOP#O6BrtO7GO{%_wVCDAh+j3aA9O9~s!~-N^ zlaCIQF^ip#J{z5VeT_*`TocaKz%-w+e-F<`0~Gh7ww!k{&AZy#0JwzDTiIU1jDSZF zJmleWI|FL`bE3_#uBZntdN7`gi%alaINe(V9>ydLnt+#gJyaaQ%NrdV+s0>w3t($; zJbA3td=JzKeAozN_IR%J4%-XYleR7FY&XK;ndl*FuvaQz;hukHb^S9vG2W)htSv${0U0p$-ZE@rqNcbp& z0{akU?$U+x^cV~=+vg@T{C1ENWg7t0$3hG zdoVnJ$5J)5t98l4UGv;wXdmeb+!5ekCBr9WdT+#wS?3K54kjihQc+RGA3A!XP$8GM-c3x`KigIy!GeMbBnZEm#W6(5$;Gbs?Gk$b&ha{J^u; z-`N+)$Si8TxQ4pezQTP#5sDrF6O+qT^!}hGH5e*?ZMZ&^1g0*tq>owP{bn`(*9F2v6_}}|B zr?CT<-L-CIBhRoexLi|j$iC`#3lizHy%0-P5GUgMmEnVcK=zqD%wRyRYhTIS8WXD+ zMLS2pf?49T(Zg?NIW#f?i0raCUa%A+>($6V7iz!W&S5Hb+Yy3^4AMR1VpCUYS+=y~ zMKbg-OM8t-?BH}*LfJ=w&+cenXM1%$A^X&5j+eF@L6!0CaN+2Jx!BhB@-~ARv(Q+4gV)V~Ax}b9CQ0m2$g?0FVebN; zN@+0Xnp@S@ysTLt!@&Gw%xsU{&B$mmZCfv-K(~K-`fMo`e>-gEbki4ZRLK67W&j8V zny}3NrT8Bx(2^V+^Sz1@jhP*=%Uo;BY|5&1`(si;v;OU1ec=3KB0GU;l4n>1cOhzp z%y1Spm6gSIAw+|K;jD%AENJIyh|hYV4)>Bca@bKs50WZy6-zKW0YBJ=az$ncuSt|6 zIj>w9JUl_4&eU8KOG#JvKHB*{TvAzCNr5E`4NJ2JkShdxg8lvb^+)-Ig`6ik8XBZH zY~P@G+z8QQOzJf5=I_-{A?F(GT#$25V8$6ZUFQ#o7=lOSe=$8`|C2QRk5W5eU9h%^ zVEp>uS=jD>Bcd_Z?0*4o{y+bq84bi5GbI-c>zU!{{y?@M_?Aeo**;Dp>KV=gBUR{Cg9?d-f06?asD60to z5IqC{Zf@VXNhq=9`RV`wJOn7ozSZ_l-&-R0r#%Uf#%tVLq~1}v`It%ko*4V@+5|&G zYsdBP8_E)5`IKMT9jngkAMIBnZSqalKkPY0y$k!5WKbJ*wLk^kr`ms8$YzoAR2 zBX9XOy!pL7ab&dVyV&R3@6rX9CGhK5v3{@g+3=%2C8wPi9FF3lM3 zPEIHKhR`m+@^7jC$cr8(|GnmXWB#|jMS!5YgzTT2w^#|;=X(#6ZAysH6rYvat{LDL z;x#K=qNK}um_2u1vHfb?CF%G<;5q$GwjiuT7!Hl*YY*tIa;^7@B!33Ucj$Fp@!!krBQdNDMS~&D_5r-AX^LeslQrq{)mN zCW*lx-%9k0^`>FU~sm5k_Aoj`a0O|{?HfY;*m%!zwiN|mq#OH#4!Vk^h-N*$pv1$GN>#?X< ztui(YfhxpK;)#jGqDVHtct_|})h=(U?XQ@XeqV)v>+?xk(zD|NB^&Xj;*BYjZXrl? zt)S4R$)Nu>Go$}`Ps3dRQ^nQCWDz}P9<{p{5!)tEZYVb#QSm_3XM%H z_;q_Y#S&lR^9P27f6#FJ@Hy}iu%&>;-d;JqFYuqu394}p4t~jW)bcyq)nOUKRPn9! z%NyDO*{T7%ZSKQNd^*Eo;Ni4jGz>xmX`NibR(aVwj6EjCNxRXE%^i@ZON4bi$qst! zW7({#!Q4U^J%IM-j$4H6{f(OxgzOFR9o4_3PoMMMAY}jBZ>mB&1Y0eagyh%KW3O9| zQqx*v2bVvcydh~o=;&dG_WOJ$C#eISFJ%_MeX02X)rc)%QJI!}RT;1p$I<|rlXOv-*KV+q!)oIh zh%V?aN1K7Bmt!hh&wwWEaE}2jW5CB=1fnxT<8_rgC&a?kuQgxY|G7W_XiyN!SEZx3 zRDA-8xE;~c0J)5BDgSfv8p~ok4ys(y%lht*9!olmBq<>Eg0sA=vA&$$U2&9XunF55 zhH^^IHsDZC%~12;ZpnTG9!*=!i10iafB_-M^8Jcfd(c@A8;oiW&lxzbK6nrrJ3amT z3rH4^tP!eD(;J*JG?fe_HKUzO`O^b@v6Xlc6jd9lS;KVVXd5ax<<;aG|mgWpMtL=k~d^S^S878BkA-0jj*dIp^qao^^h8M59td-djRH1!}T(ibn6@X4b$Hk8*@q=iGR zj)|tclJzp2(!M;n(E-v#V@VAJL*S^vKdo;5T0fFDFWY_NK(B4JI8_wA88`(b!w>(@YNU{H68$A2CeHqf|U~#aSd~Io(+hWfCkL-Ht3@^dH$S-1~EC3=N zW!Fu8K#=ERM0nLSlw-2|b=OCv$Rgt=WKG)33;zD(R{+?f3UT%6D5Q|_l*juhZMR#_iCwYfIasmD}ZdX0v4s#Vr8~=oaRjW=MKmV7(1K9&spnP7~0J!{JbjFU=J3 z>(QOO7g0smngs@I1MtJO^)<5F*S)D$BPxmT6S4(T6O;ZKeoRQ=pMb5T>zx5uT2f#R z)7hw`h_=r5xn?Nd!uJ!LUK+KKauwtU!D2HPC0j2)Aet4RsgTxVWLUec-=QB?lJn8$ zcp-#Ww~!x$N}_3;shadfm-P&g1DC_<=k*%D`ej4JXp?af43A5zFl+Sd8T+gaFxf?>&3+Gnts;jBg| z0}hX^wjE+*<81R95os`jQMs7%S6pdN31zhz%(k5zx>;Y<(HqsrvyiD!MN2zigAvBq z%re+X zWeQ7&NujPj9bYMk$Wxa3u&4MJy$xVL%99NRd-Q}D{2uY!ImJ zb+YY?m$B!`y$0vA{{4LE&=N{slcq&+bfqO5s_EbY3RA$ekh;L+35nZF** zgnsSGYD;v{>EGaLxWZ1v(r(w8xD63PmD9bsFzVZ)_By28T;=n}qh1%j zXY&&=F#n;GNUB7bR@hW$fK?`6jHDIXCuxN35?dVRzcthV_wO5u3>l!Ra78@74`W9n zEie*3-r%GDPDc>Tv_faFrU?YQn!-eGxQ!h|8nyc%P*3R*rf=JGv3uUzJp)T*^&!5W z9bL`vt1=Rpc6J3wd0&dtNXY=`v}sNex)ojK-tMzS_D&}RN51QWpPGsE{otHwZtz}6 zv=QVDj{{+q$RWY3p(O*{D!d=emfY*F^4gTI?OJ2xdrLU^Q9YN{yP&3vx-f<*v#(}q zpn%jiJmo;EO!ObA^RCuiP?br}^>j0SlV!aJ!BAwtkKQOWWAH!e!AK483~a$#-A9(8 zDSI$zlBqxv$uzNwlw+=0L5Yu^nE>c)3dgCoBeAvQA0{;!Y4RG$?N6HO51bfQ z-hoLvOnvduq0m~fl9sw?KpPLZ;hshX?i$(^&5I1`4l$3%FI^1UPfelxP)!>G3jkc??{_(&rrQg zU_mI`Ld~BY*zu-bi{T<-E^tUBHOWlV{<_;&>MG**)d-2%89J*G88X=7A8rkYQDK?+ z=z*SRLA&fvc2*EL%kTfzvBkM=3&A!hSei>UF)8!DSXbLayJuW?BzuNlfz*{ju` z3};#_9*XP3`fm%jc}d!g8MJDcDv`emupe;eELb;J=GJ%X*12R@!gob3F#4XCY(!Fh zzdEZV-Hd;0e=r)cw~Pv~LS4zUKDsd>IyI zPviG#U-0p!w8xmB7biOubY_%Prpn~!g9s2pz<>J6TGP|BJ5l>62%2@a zDec8*?4b&YS#k$I0*y9`NBFSCaO^dXJ%#0k@@U*By@Do%T>w zw?HX@lobr{lsVkH3+s>eP3B5w%jAJ6p-A9qr{wr4dRAKdVhLYdkYx>S=POb>P4m*V z*a9YtwA#AHNq=U;9gejJoPg$I*^o=jIx~UHt@%E0k5Fk}HKvLnGIK*1!|fLm>bZD-$m>D`tP?b`gE9XSDwZrTdA3qOkHJ&6v3XkdX?PA6=$2G?O!=?r^UJtD>} z*Wo-dQmiJP-Emcq_ipYUL9s6zVt^>}_Dab%^ij_YgJA(ZT)Mh4o-#iax;e~vy^xU6 z;+okE@>tnASPtX|y|oV`9zgq?Wz%{gF6jgAs!X*Klk$jwF7~%$WDXMw?dc)aRqD-b zWhnIGe4TseYKTO0J)L2b^l4_F$q0&P7Zz{4c-Y0&Lj<{)0w?i``ss>{v=6!sZRjD6 zMzGw700g~>M6YQ+C2yNgcyqu2g@zPn^~=*4qX%tFI`Q))oB!qlbVw<+i~R`G!}p^b zs%M6~%>i+`AA^2AV^|LB<&PN_XE94}`NfYM%Bxr<{knU7el8sqkdldqTSTYxY+H5d z=**2Njl``i#RQ^**z$UJH1H~kg8qIg2uS+@qs!4}Y+4LVbvF2tj2xOEAT6QS z_^bZH_FCYAg&lRN)SY7q*ZQa8JyLOf1Df=MtsVO-+i^I$Z7Z0t8CJA_v1H`9Ow@II zu~7ZF6>GA6R<|H}ewWAUGPr*x^u%H_cp;2OMH;^Xttj?AWWr(b2c#MT$vtdxcSZYs)hkp(^eJ1t1S?Tw%t0c9oQ2>!MWB_&BHJP=U$eyX*H ziyNr$0%+0!oh}wgL1*Ib0VUJ*-( zz6%Wk@_)S>SPcQX1o)49aikhFvNJJGfFCvbp2RV8%NyFYtju@zF|fDYAhhB(y1B(~ zYYwyE49o8fn2TA7; zh@XnWBBibK&0coz9Af-Om!Wf#Iv&2!u*H2igU?YaFs@o*z~dHVGnNP1bf9URZ@ z|0i(3Yrq827`XC!fqrN@gWzc)MSQizA(Q8DNEYriQKCXQ@Z54MX)`)$5xW{Oe9)#sZjkv8y59TnfJnsTxPJph zpV)$GeJfhK#WF+>9$IpXB;X@P+-f2*_{`-QHr1zpyai><8E`FObZWebYE)RPatUWQ zWrj0Q!}d~{6Ff`GrTj;lfS@J`{yo@nA{5wkx+wvgV{CF9h+MDk=^CKjKV4|pYr4?) zlM=NW;2}$5ksPQfYw5==-voDCYd~z5h}vg<&so8ya2?VnI85{OZ^a(f3~0g}F%$SH*P}^J)bEe)WNQ!N>QDpL;N5^`y$oUGH$6*1D`xF`Z)cA1 zzP^YVd7H%D-raW4@7dsgSirKnZ^Y{eb40QnQ;95U)_D>K1RTnS@+5{Nplzq>P!AAa z6qYb5l~(8UF|m#nso+7sxC{(1fp`eE3p8syqlM)I;eLqR@m6Tk;KCtx^++P~+3U?r za?`QL*dZOG_G;NuBbdu&UBlx0lXWo4_aFg@ml56S2qIb+^sPKjXulfIcR#qXrnR0x zh49DjgQZSVQa+4))#7Xkc<+ZMij?xYsYQR8c@TkM!b;H@NBb-v);s%b6xiU?-M4;B ztoCHc9A#q5>sKQAtTO}8rbbGe&%QJf?rbz@z&@df2lMqHKRUF%uO*LmGp~jq1M&8N zeqz9;k&8b6)jGCWK)@2o0_Z@75y-cToJ4bXo%Mnc6&D0Qv((!MfMr36Bmm$?hziQk%#?XyeYaAWRH^DYHPZtzp1rehC`1VDFf~l!wy;-!r;t& za-j%+bnv3j7J)`g=UoihdccK#ocsda4qm3;#~nAh|F-{S@_V_@^HjdC+7c^`v+G)C`xmt6-j7rj>_s;6*_r4A|>O<^ToIerS!oVht1BcEb zXEBmMy#RCg%|yD6m=z@-g@CNUM?5N`{E~xWuJi3V6<6p5LOMvhEI47*tWfU8pyPZ( zkw&dzVJQL#S2!U~I=Jc5oCgwJ+s>+oc)Qq{9cF;1ke6wOK5O*W2*0bRI#x87%dPIJ zdfFy7u+QnH^9Pq0uCv}uaBV{&$imn0+V655OldkCxSWo>1Osuzf&JE{2UX<5V*8eP z*sY$7KmS~z{Lrch#_qT084U?o%6k(ReFtBBwoJV|pF%dyt$am^g6!q*G_+x2HAdQ8 zLSg|OKYcf=+D2GU>;18uHwCn8E|=>wXWD(SPpQ)3bG*s#G@N;)uJ`b)*9o_xsVM`9 z{{NtD|0itlka-W@yoN3A1FQXkP_qnmLi)R%H+fBCN!;qw1$n_yQJuAJXZUjFJ;?ac zr-}jEk$A05LDR8*GF}IYF81w2z)w{O7Y$ufJbN(A0rC{Em_z$WV)3SmB zX-cn9<A<>b?X!Zw_Dmppfpa=l=H@FSrhld$)d{o(;cc?T6ZPuJlKg2-`j!>IZswyj>SWc ze5eeRc$1Qz4o|N;y6YBVxT>G6m;ThdLvN0pRqFMBQp+$w_tIlgSU%$Re7Ajqz@0DI zijB-@bkeQ215`)E31IK(hHoVHm`p`VNM67PUYa|-FE5%0?+&wqWOznO&aEmXlQl(& zfX}STCS%&ge$|Jx3d%+=@D?!NfLIv?2}P2@>rmr*8wZz4jslsc9$pO48GO~``QZ% z$K912C1QhzF~?nY4|T$82MsW2ik2s$dwWO?pl)E|pE!T$I0~zJRi0b8gZj>b6d@V7 z8_gP^*u`HcHl6X4E=q{|1|YfZ{YP=7`yXD0Pl7RDjY*3AWcX~ca%++%&?bGpMYEUZ zctusp8L-@oSU+Q9@f4nMgYRTPUq`Ds7!d1EH9O7YhX1LQO>nQSv({<=ZET%caGH!> zTj;szSytjAi_RmON>17vQS(=iWky$7vQZzudY=zBZ(Kf@faBEVIq%b-PZ{$lPL=z;7|Se|XMJ?9F!DB9sdlLV zvxL?-fS2oY)40Xz{}cZJmb`c0y39GvV(dXf@ejS|NpL1ZnxLSkCE!Y7>{+pP9fgUqL= zDYIwxz)!kvh#x5VKJW8q#5}Tg+xrupsqhZkq-`r&tS0J*Q{z#~nOi*tdppJZ&u}#9 zzqEG^+Xf%YSEwoHVQ6h@XG2!&kHFl3j=hi$kMU>0Jsi{9knO@axnt$qIQNVVN0S%v zYMW0&trLtv++y#wqlCc{WiG!{tV~^->{5cCIrAocQ2wd=y)xavW7a!SXiiotYj8B~ zXJ*nfNuYHDnQP9#a82#_vcCQFv3AuDj><0vBPyIj>dH}2Sfa?i91$Awrdy%We`-rR zbq3shnZ9aS89shU({hnxYRF>|Ws~;i-q8<3@rvj3sCTF^*$(3zGmg10JktE}8SeHg0dC9RU={CIc&CU~d(TLo#aC0^Xu zbnQi22YX!3c(&1$2jdX815;a>z6`LzzgAa~?YOo3?TG9wO%=R!lVrW$mz`lPV3fLa zm;@(3OULWy$;4KpAl?#)U~N*l{Ve*)c$K$!MdgFs>2r&T^J-S9s%3!pJ|E(ef+aEr6|;={!GV5 zU-nuLC{fhov4>x_2@qF+Ok2=Wv-JHOR(asn!GB)>udkEhG%e+^p7gP90!>{`qG#`B?f9AzU35se{aIDHQS@kPeOB+dwV z`B;lPMfsTe-SOkT5|Vgxqn5z6Gr1Q%R!)11Q?GM3p9JKS2Am9%uiphFTYc-%w&nY1 zcWy6T_9}sLkM1&(c7U3d`uI)B_^10?CPQ}0KIQVXIlwao1M}PmQN~{wLiI_(@Wx|- zDZ!NQ7#XvzFzsF^(U?t2wdulN7y~8iVooEI_{|YayD)KsXa@rQ@r?=D)#4Ax#i`#Y$URZ4T-Z>#;3y8ovC z*O%<|6*S*CMf}?cXB3<6hpgCaYd}rhw%TrudC3!Uk~N|dQ=1R>@{=6JMXUrK{5m;# zsVqJY_k}oAhqRZZ1B*x^-*l(*|4_BEJ`tTxf5sVoey8uQE^n5yQiK%f85)=*w|#z@ zbJ#6?|1FP|vWkT^N5T(%_o$Co1GQt#rSeLrHjx)(iai3QQ-5f0rM*>RJ0TYBaeBdo z#JwUB4`D}g?IBf-uiqfsOyz3=oxU(YmHRK-_Q&W%`f)G&wynw)2ezJ;dstt{joQD7 zvdfQjpH+m2zPLNkJM`^Ue(@Z9{A3KMUtv{ipc;SDq2|UI{dM>O+klKlo=~U5DyxKq z%vy@@IoKpKwD#27XQOS-BJYFRwaQzapV`4Ze1tB&_q3q6drGhr`#b2gmz_t-uC;r2 zSsw;>CRbRH{3w>C=cCt>Yg1lTP$u`_)*^4h@IhrgrE~y}i*EdpnyMGCcl+v5stT9p z_nd|(hOxbS#9yySVPgoI+1TnWka5anh4!s z*|pq$ST$>PXB($Y-JG*7Qz2X2Q5B4qib~LCOd0aKmwN6lHlA+nUV41fi@;cDy`PM` zWkry{Tsp6j@Z&pk!u65zP`fqX27{!z?_uf1u%USJTQn`m#bgKPyVEn3^4(UFf@+QA z3dwYqWRx|NNUzP%Z}tX}Bc|f?;O#kTxWjKyqjN$nj7xNd_wb9KH7lLNe!@nps*@UC zlPcmT4-Jn{%{4O>V0lmYFE+fz`dm{`53&b_IMSfC64x@HZMbA26g3jv>A> zzJQN@$NiHV-}1uqd9#g7{l~Qrh%E4*^u4(gjp=P+jpP=I2wR4QOD7A^0J!1>pMZ5e zxy!B(4hlB=bOZb#Cv+}ubMb!9J+CAKl$xRddTDXzp(X8gU}*}P z@Jggsbc*H>PTK$e^*CO#h0{55>uW;J=euHn#w3G>e zTn;KSnpfvvK|ZMC_ev&dm0h7TkpdNicLA>HgkNd0mu9)mr}a_UrGU^>?#~=e6rVZ7 z0HQ3ggAm}=L74PU>l6TB(4Sx(aaC?Top)Su^*-5|!Y+ZcUMk%Jn6q}%uo7x-dt6_l z+pl+^75F_{X%*IZM%=bS`%yxL|J52J8vr2i@aDv${VN;VMe7n-ocC%9}f? z3LY!1kyJqZ9YPhiPGYK7-06acC2FxrKKBr6hTA#X7*Il*xDlFH@Yek1XRVx)lAYG$ zVOqun0KkTq;A|3QDQ`XclLmrOf4B>nI=NOz+I}?-tcdQ*_kt`=kpAKni7U$W$0B{_sd-*ju4U0aMk%8JHWI1?OV!RzMsJx7`~P9Uc2lY#}o>a(oR#5eS?Hm8y` zCuL4G;uF|&QlNO4EZfSIIlZZKVcJgMF$)E$++#5V7H*Ky+@v;JW!@~~+$eSYldd}t zKt@_Tz`nNs01fvChQ{5$0N1PBYJbMw_36l_lL63aAs-bou zl<=iUvtzC55@PP3ztcTip)5Uk5cQ@OYiVW%n*%!B?ZKDmmIq{xRm_ z4%;^}HL|+bMyd+JXDuui51oWr(`w2Kw^BtP8h&(eNm?|Kip`Uuy=r^xxJsN^w0>u8 z6bn*2)MW*3klt%3?ce%zFFo+W6~tO$m%eQ0;8@MElsWw!%C)_&P%dQAeC&7|$VyLk z%QCJcW_oU&C+2jN0h3lhH<2OsZ94lkzI6iaepmVE2l`$OT?$`jY=tOcMA%gOho1a@ zfHftWPBPD#YMmnbM0`TD`I5QS3dxk>I+vc31NxZV4|wtqdOH!&Tj+il5_ zWD<$bp&YIV z{Ny>Rf^wipuu|51;!FHVW}+a|PN8i&D~@wW|LJ9S=gUGgqoHH$?e*M!pvq2ly`o@q zafn8U49j(Y_9R3!1KcXjx zi7H!6P$zvh*liD7pgMph{z46H7}Aj;d2n6+o{c(&+WszJ`RjAOA-eCKt4|z7b2hd5 zbA0lHb?@oJrdjystkgg1>-L}Y#u3R)*||5gILnL050dgy@kl+&9opbm4*w3b340ZO z;uVxT!UO#f=WCJU!;!B~Psyye8`&enJOBGoSDiYMW z6I%u{QFhtHkEfsCNl#=Pk9K1zNmt@%?RLp&A@wR8>R{L?@qwx_#`be3%ZrrV6nV+T zxnk$T0#Ez-B$2Xi-rnZNh5N6a%0upet|dF6AGx%gdF~ETFzFSCUl+MaruR1u+hh#e z-Di{E>9Xo-)B;zh+wZZ~MCl#NC76Eie$JvPxcS7rpKge)YdhhozygVA72IOduTsLk zj8iK)FKn$z03&R}9ON>->{u+MeqoYL@ADb}aP=p=#N8|D`nIp3WE&eM3)4%D-b;!^ zmPt=@-2=Y;da<KG9LrjAI^?7!7^*6O!_^md&rDi{?O& zxJ@aLa&lXmV+N+N^mBB`tf%St2dvd%tDi_jjbtJVJ4@K_t<1VLOpy?2aWIVH`pQG# z$+`O@{KhBSpOM!k5z4uiqR%3nxe#SL)?6n{Q{rbR*=BPQ|!a1rVdfL3bf(qH&`2mo~4eQTa-fNAZ}K5lJ-w~VpT zklR!Ig#bmA6Dywkv@fnmI&19=WQtlpTgysGxj-^m3x?W?YZEmGeN8oQTLihnSJ1m9 z9}`q7=S1wcvu(E~({n0*G3Ed2d{zRDX>jQr{}Y~<5c$4@Kko+xx4)~9Cj%rSj3mZ< zHx%Fwp%?;H8`7kSzZ@{k=Mba$sO__EX!k{I9$!*m=i8E*Aeitsg;Up@Sa0sm9eE2kSDB*2%X zFPA-<_In$xJfqk9HSWCo2%x6=P6{SwDj1o6q4}26?MuqA#@M`4s~HU^@h$n0dnMwi zJz-jrhrlv&P8&Z8eo%!NeCDHgfUS*t_>sRkXC{Mzgtls*Wx)8Wb|N#S+i7hdaJ-eR zDW>j1;OzcC&=lE3b?2uORcD`#v%?(3n-uu_oie?If9wtY)F|7$w5-B+gDA5FGChto z+C{=^S@nT5!%w$WQ^O}fGA~{|cg#U1A71uWRg2Tiuw9YW< z&o72(S6kNZO8HJJN~6@T9z_BM>3c1!>}xjwpVxx#P2B(We+J$e-^RSdW;(0<+ZGt2 z>;1aiq$-Zey%dIwE!B)7>FzX|aArKRaOT0^CAGl(ddXI85ucg#ZEVxStD!--;66!k z&&uP8l~G(kCnfZsx^e@SiY5^9_gtBanHKx9$hg;&kcul;&tYp?H27G@+_ot~uAxliSjS?Xm()TtxlKyqAd2<6br zCe1DDw8zxK*=Z=JxS@52`8XjC>L>?Rg?%RUu_f82oaXFAG8cQOwVG6#IbV}%*?vQl zckmduIK4kR-M3U+ln`rLZ~>oOEn}+2S##Zf$eYSJ4yt<*mE-8T<=W2;1tt#L;{KE#nisZA9;ES*%+3_PrCym#A18E8 zcpDF*Euv%a=bThy+15z~JfMeoQS6pYL18^*7Q_n@f31A8@4l8BpR*mOt-HPkD-Q?J z2aQ}2;eSTbtu={r=bS`?e*9zCs_*w@Hy7z|3Vvzns+xq&RcjzD^g-?9xL~d9WK08z z7P0eU7%AU(C8ym3hJ62DQ{P^QaTrH+n`DvIiO37~nmb~eht7)%+a$oh!|3>?D8ZwU zpIz}|oSFI#T&xQqXEnZod2u?eL=C$+XBj_HW=1zI=n!uf!sL*GU8Bbv02tOxj!UB} zSO=&-BJA}ZgbQLm7Vu3%@*hWHMB2~Q{^jzgQ0=GZqDf%zUJjDJS|R`Ix*q1gqndy_ zZD`N`fb5|%o0L{CG>hyeoD2YnrakAQAG*F8x%NjBlrEhs3v#-Da{>8o+MHTCo+Fu( zEnqZk`3tGk#aPU^tru$X9Myt0^_HmC*oJ}ilT)h|E z>8h{A8lO$&_g#&9z_3hr{^yWQSXdbGTRxhoYVPa97|81?RvJ}l{9z2^bS)SSPcrpI zva4+oev3`&FubZjE|SuO5T1it4|}ii=<8^qG2re3s6F7j+TSMC&6r9|dz^110re}* z5wHMEi$y6C+kTBR0=0SmEEwQz*^5)*6)Q!aZ)HJ_heR|=kur=Iqg^4S18Q68?!#=- z=SuUKE>hrqGO~m;lH0fKe@QD(+?F~B(g@hm3KQB#v`Z%qWatx=CwC{|1^tQqBvS|e zlx_0ij*bp}ilu^)-yr`VKaL5({r4|IstU)DYxH$KFa>k6T)CCyy&5ioYFs1qEwA*| zW&5??#ShAy?bCP-DWwJxr`h)|hAmz^0g8pk+ou(QC(uJR(+Y4YM~dqB{>ko449NF% z&K7xE@4V0}g6o#P>i(#Pz@IksiQqOx+Orp~Pt3068m^I9_~JzmLVt~ijl4d-zZAGr zT)2HIjbD>aOoN@2fyTlJPENrk{7D5wtzo8&_IxCE$#cd8^N}+lXJMg%;52r2cHUrZ zel;z7oiRI!KuAx^UUG=kH8f0w)p9kfjVLxC=LXa;c^D>oz*50l-h#*A2CKOC#g7MR zE)!5%ge>rfR37$YX*m3=8%3L?r5eU-Y=3%s`cMs%Ik8VK;Wg^)+dZHrl@%kEXU3tK z`js9b+wO{C!d?6wOP8kbK(_BfI4tB^WQoCOTh}EA!I$?lvri;oO9GKi4mlq%ixw&~ zs3};)j1XouW(62{NUEt^>#?wQ_%jXFw3SJiKSpfAERPA3B{4D4P3!r?f5;gjSD!Q^ z;yf59u+}u1=8`--eCddr-2fR)FQEe*GtTA?ihU`!9m_q3>>7Vw5hu-)} z1YiT8vP%ZONWPQIgbxAKZ(J2 zJNu~i5ur3)vN9AYYV5O-;5btraL{@C?>U)twg1WK8A6@^HA(l2fy_{0JhD~%;-8V3 zy(lG4vA6$+{uW1({U5L~rL-&=BP6UqqV8?J=I2mh^jhA-=Sx7G+urT0Gd@-^VMc6o z>!E)Svp_#y55d$|Fw5w4D$C#JuLJ-TKDs@}}c z`iJU?i$VOznjlA&u0}ia6OSW7+Jt%{HRU3ABi;VGYWN9H!Fq69)5o%ol5=k;TkoW+ zjHXmYr7qYgZQxI=h4?%*YH0mw6a}4{Gc;sU9MpwE%<6aVY%oFGe+XU^NsHgIeolE` z_9V$UvBLSN{+nPZ{j^N{?;6>D*#H=dZpHhpxwcx7hkfmXjl->sz$}&baT&Dl`dj5i z-Gdv-#<&$;mr?5_1qS6SYr6khC-vF>9rKmK!9*>@InzqlrRd|mfR*=Pem2MZAV;S| z@t^PA-#a3Yrf8642^MAWuV*pPUX<&9Xk>L_!}HtDpA4cr3T6D*5w)WvkcwZa-a&A$ z7pl*?-;1wCy{zAVVdJaty53_UJLv#K<&WyJ^{qLQyL{k}i33u|McfXE6_gC_oSxnE zoN`rWrHpv#R;`S#akkdd$}q2Ol=YI~UNV0gbqlbp`YzJ6%t%}|*{jtDa+X!)tTK@2$ppkv`fg2=``HAIj3h=jCzVS( zd2jqEAjrIT$Ju@AM-YmvGku9F3hPm}Y0>H0&N(Y5!}}rAoZBXzC|ym^ef`-+eD?6m zg@-%gpnoY?npM+tv#Owa!FG1r3xZ(c)$g;Bp;IjKaBzP9MV0;}n*A$L8d7!jE5i{+ z_-n9IxxJQ#VD0n^^4}D^8u0?uY8W}jx%Hd$o^-2jT zTq$g!JA!ML0)4mnr`_ko*dttQGIu#TLiPo=vC%`5k(X#HEqE(j{{`0yJJ@Y z-k%#J2Hi9@R3j#{vVHmuCFkdOQ?s1ZX)Cbi?$b0!$X4O1+d2sX93kLYWQu;L`iMtu z{~MiQ`I}35oipzor5vOZxKs7SG4M0Pq~BKYVrrEV_cdS0S?a}Z5~n6{s`FA#R3%$g z8r~ROS57$S!^@CY5>9(>zQ{ie$WHndDNG^2ANNjN={Q~{#?mQDAEfIp zcwcQVTVHW&k|r#WR*p~FpSO@%(u z9Qwoz9Jw#h+|Nf6x}GrpxX_jF$wEE!XKP7P*~_nmEa|S?u~4db`=N1JqE1D66`ntw zmSwzKqy^2B!ODGn*|!s5Mec*!qR<#k^6~#fbD;kse9cq0lvdUo-bQ37Qth!yI@R8d zKP@ENXik%!FqtnmwE~x)jHU2sa(WRnhdcSaoynQwz8dmA_xJ?bg$u+L8;vdW$S$5;rwI0gGSht+Z-UHRl`ngivyWP9=|#7d==DB?4YSsp9t@_$nk1V-`Z+4w}S z%ew%h+yo`(L_Ome>}Ya6D4T6*&#P&1!EFHWVb-H6-twSO0oZX|bYk16Ze zzl}%)hdzMiacOF1eSguT`nv4EBUI^MI8Vx2Z;yX$|hwv zWnA>G`m(r8S%0$1TN@)Oh+2TKR z&Tl|cO~jh#nVD6~tcCs?Hw!Qp8+L=2>2n{qgkI3aC1I?m zPMjH3I7hcjp)EqH#%d=*dsdQGAtZ&b=6rMYw_s-cNr@g&xn^8y-wi#YYEaX$m;Prg z+h&TLs`bwE-U`0bRYK=9iVb0=$0ZBz#DdNFCnsC>m1~zKRNJ+g=El}X5!FSMFT?NH z5N)mjmhY$iH+*^%Z7sDn8T32Vt2lcAOb6ngiJ5y|2H%DXZLXPWXx>jiESd)4g|D9< z(nP|zw1%UlOkt_4ZZg2jqrR2hiB--(Ui<9gLz2X$_ zyM0SC*PE_yF%}yl0M*<7MzFDKDRIHbJ6div0D{wikN^Pl#{)^g>#>+X0660wilzT&?Wu!Tm0^vYNkjU``j^N7@Jxza=?!Xo@rh*1hr^V9Ro= zfcdc$CU#fXSOH+Jj6}!zfkEy9a2AoJ6AXrcKWRO3o15qD1k1ULOG^GYF}?)YGX6cu zo_uRaexqpJt;q#;ZgcSiklyn*Za06NWa2ePfLdWix88^TJu4kQ+xTg%X=_#JrunVF zXMbxbssPqjg1P>lnEu~s%@lp4EIzyapdjIJq?jmnQ{Jq)Xw#A*iEV;fIRD>VKxWo@ zk!oet7@|Ve?)_~GxQ&^aDU0nlf^hzTePh?*^R zDtVF<{(0agej$QyN+13-qA7X}ptH08{Sz7Ou6JTOwQ_lb1aI=bQ0owFR9t-bbR%P$bl2l+yYW$i7(3n$>5pgn(cD)6-s670S{B zJF1UIK{w*xg8g1VpE>cOFDS?O<`tx-ca+56F@1;*c@dsoI)2~!uh677FQZPJCh~GT zSK+WUE~@8gTAxjxZA`hnM#?QQ7K5u;ubL-hw=AgTEWQ1-(i+swr~Ip11QG|izHZ!7 z(25*zcp348{NjbZf#v=BUf~L8XkPG8XY1S#6*BzB*5+%RgC)Cu-vq_h<-8<$Dvs9V z<)fWFb(Q?UC1+E%_gj`%t(eL*PHUFzVrXGu_uI#vuFc+O5B|HoEipQ$KQ9}K)*EAG zrB)h!oui1O9^y_t+3tL-Gu%I&s5a)6HcxfXlU(p#DA_XOBHJx6sEfzVyM?f@L`=F5G&pv7%wjWFR^BECCS2{shkd^9F_V2xxFCM z#*_FdLCunnVl~^9V@*vKUJC%W<~@)28gbp!$YPHbKzHmraY~J?r@F4bZR&3aSSCuA zSnhGFy~r6T$dC6``zEBHF;FU>SU?d=J6oV2G@m7V9LJ|;x3?4N>lpP_DAoBU$8VYV z()96(r)<+k-!B3P@=6bn*86RiS6`M`N3-2nYr@&%8GI{a{#wLd@O&9-j_YRnZknz1 z@Y~P$ky&4Y3@b|D4T8HX{|3dB6jgLMGaa#UE(xQdgc-V5bbUqDr^6rjOLk}f6%oKsv9HP45uT_D zV466&`^XlsVq=w@yzj3L34)jw8V*%4c_@B%p0?AVvdn0j!Mq8Cyp(23lIt zu1p-L%w%MZ{cNC;6n6heMv5r#)*Bgqv8B3S!l2nt^T`;Z{O{kLV|osK{%X!@i@I3d zCfjOH5=RLmU`SDy+{@^=ne?qe&*M=Sjc3=!_71OQakIpgHZkMA_B6w@W3=17i%*%I ze>^`>7hri-S;aQN3fIvM-$fbJyA_lkdRxNinN!KjjI6m9J;0dBhu0iyk=4|?0!AYA zgux?0kL|&Vk!;`66`bp4j(XX-Ok4%VK-%9)@{zeg|5q8fcq2d`03*ip1bz{^h}!mN zWn-f7DDtf0Pp_inPKX-c+!METC88$@!(8#p%5oWR>~oe=>t0fx8r=M-f3qecK^A+2 zK!9)-%d<63ey^;}`%y8SM9Dn^>z!SCrq8OzUy_G()Vn*HhUUwOdCJ;`vG?4F8jp2^ zMV?=Ia>H*gbvSmUYf>igeay96=Bh(Y$mJEB@Ss=A9}BMAV7|6E#UzfYs4JAOJlPUA zxTRZOR)c+wlW#%z@hz3(n!V$Y;`*kY1JuH*(h5ddi)VW+glP&3A-JGS!-UE-B;#-% zG)$7a%#S~Bq=$OAZ#vO+mzBTBxe~^)p&$~o#a_FjEr}*@ujP5P2Yc8HKQ0LjVo#H; z)*Jl0BU2_g&oyhG-|>#4E%vn4>{-*TISIJ=+2LBJ%s=Gs;E9l#quE#>+#&XMy+%C}YsEu<&oZfohzZ{UOb*n!~BdDi<82mxMPh zVnddL8-`uh-(5pWczp#86B5w~T^#k`_`e=BKO3(*(-dX|6$%>=Fy1=be4{%}x_TeU zwtulRuGs71eEM{e3xN3TLLLmDHZvvy1(3$U3jBh_mJf(u6f`w+%qWs7o+>LpV<#de zCRWYU*taB)X8)_c6D-O|nN0vxm%D-h5bX2qW_-lQySj&N|?GC$5xz~>=jP;yC?UTb{j076X(sZ=7yQ~s;1F(lVE(VA$&h3k z^be1a5RM4_lEKsrcy!L~H0>G{d6!nTU$J=4{K;ctZAaD}7pb0w8yIO4u9dZr`iDiR zkbM1$>JS_z&~l~@RXo&ItO4h-5)^}v0)WMa;VSjZ;y2-#oF`O|dw1S=JYFDuzt3>L z2dH8GQFNHGv>-avbJ7zi>2gbx^YZ#2K(eG^ny!#qXU1RZYP|IQu8r`j4ii~et(TEc zo#uVi24lk_p0(rP&3H-SuCV~1u6nR!&<@~)9;##&a}H23&lY0(Dy9!4QcnP zni*q8KMIj7f%vEKs~Q8dGR8Ra^T`JH5)~~-I6do#3Uvw&gShOxvV@-G9-p&gR}km< z-kAGlm4A@$%Kz9ohcnRr_ufsT}VrjtBr!fG7tYoe*Cr%t6Cz4 z0Y=aGPi=OnFD*xs6kZn{>E3Ud36Nt)=}BSESbkA)Sm0suoqPLm%QGVC=4g|ZG*{^6 zr?)X2&$EttD_F$jTgW>|I4d|u&tDsk?(ksviX09fSXtwecVMviKC^WZa7uu#(>2msG zynic)iwR@mhsnU7E1EH+$L?2)Vupm|WbOV~<2M&1w4B7&s~p-j9L}PRe<3SCXZFzW zgPkm6Qf2&7*Zx$n;4Qisq#0M}s?n$5vHB~Sxzk8r8dLSbhlaO5D)18oVaX7^ z;f_$g8)w8>z=SMyVEnMlpzq9$BjPKba%pTxGunLP7-9EEgyF~|ev%k^Yfs_sVOLj; z{=O#4`8oKUSg|Q#jQt+R`qW23t!OSyn6G&Vq075)#M0_Ax)yKP(A6BX63yuHaiC|` zTwx7dkzfX_ACfKoQ+;Iaz)!D6A32DNKE$4yHTz`y`c93N{oP(5upNB}_R+O1IDB5> zUu1m64HMQem8v}NnQUSXA2;>9LC2=E(RgvM@5`Pm{*tmr%^BNn{qyKP`cjxkvVY~b%;xv_juBH38Ghn$pH%7e0INUff^ExY zET>FgCvb{urV90g1;-dgrtKezVE{Rp33A&XRJ{QFTFvQB2edD+GJ(( z@&{NukZ^gOp9V5knod=_EnvJ~DqsXX`F(&PqJTKE1naHZ%sh=!sqnO&#y2`ngDQuh1Dnh(V_s6cjSF^v9?#2}8TvJ=Zj+j~AG{)OB^e zT1j0_=KyUBbfbVd%fJMu(MeW5{o=We&6YG%O^Nj+FESFd6IkH%RMbD)>OXnX+MUSf z^`nveHquMlS71Lb#^~A2%gcjvT7m1JcojtL87Hq5)af8fewGVaP+(heFABcaqcxeM z(~0=@zp((1_u!~dJaWO#Eo@V>j|Y<;)^flAl;;gle=y%nN0@2o#|wE0{{4~h2LZQl zyr~78GN?jmFW-Pgvv`#UxL=iAH%f>3$wDWsnY^xX0}tY(H`f7RsUGchGCqBNJX|+b zXxQp?zT_1iA)=^CdE39)0Gwl0#{zW7XyA}#%Nxpfumpcp0i~@fQ{CwfAo9fwad>!o z1P&)sLj{kUaO6%$lg`xHi8u(*fe2UX9=KU%`#zghx~GjdL)Sg#zy+rhVl(N6o_!KI zawAjIWl*T}MD-F4@+>73gL}rHSw;{n4({{~=uVqC%`Mdug?rf4I7le_8)rn(sWPg4 z`e2T%tPGS2(=SdoX61q6!ToH17^ov`N;M#{0^*9$w!a)U(o4nb&{5||GoJUt%1&ZK z`9~3mVxVK#i75H9m(JwrFPO&WxbRxEe5KA2t{-b}|BZX|TObppzF1vbGgHziF}?K{ z61O(Eu`}qV=H|yh;ygYzCEFpkReQqd>5EDYZoBrZAqvt==MNy6J}pibxz}MD*3j!+2|M3LA|BIFL|MEK%u5wH-ISex532dLvcR@;J z*`Y`PE4`0pFUQ( zN*}okxz6$RwT`5II2M8kyejpQPt%k_EJkp#c#b4Bl!1(S?+&~DdcfJNZTHl-B738F zeT#{(Sh<8J#!RWpp-vSRBmN9a+B#cy58!+_Z)HsK96&XmK%g8jO)fL<%u=h;^*nT# z_fJ$Pu%{I{alA+>KWab!a4b?yPvimZt%$A#GmG2%n2|EM0z}hgfw818E3063$_2&3 z+~g?++w{$oSG6b4i?bURP0ex^4KfgHApi|It;u$Nn%SJ*5a zftPpe>ds?P8W~~qrj&D8AV+dK{;e^~S9RI~>;`$>==VZI*P2*w2n5d5R9E~3;aOpdv*3mxd9{8ArK3&$vL_`x5^FDoieR?^+k5xXYfbR*n4mN=@j;RiA zG*4IIX4<@IuKx7|Pq0TKam(k5xtn#63^e?KV9Aa01I%+Tv2pHm17;yz{n-bsJt@!>T~VV_S@lv z4fF%4c%5D;>BVT6XWgp(L;p=Boi0RexTVNPn%b;Mda*BU{p+!u>`$4w-lP#ynN7` zErUW8Ba`DCBN51+d$-b(C|Gdy^N>>Xh!!TDO|IRsL1F(IM`K*`;Jyll340t&G|W#a z{g*BBc$Fr-MOn9l7^ROm*iQ>t1dD~i!gg?n#02}QY}lPI!zgnsUZg#@f4LEpBK5CU z!3q@YQU%uJcFHu^1Q_`2a}y>x>-`z|1fqZ9W`-TeCl}w_$CtEE@bOZ~?vY^5N;vp6 zPXKIi`L>1Y)as3vR2_JHUyoV=dE1V2W>lNWjrO$Eg}31$htyPW7R2PLau%;G>0uaG zMVSDR^c${ND`QW^50f258vI`dS*Q!A7)d;96@N0`u;bzW%@RWKhf*XPRk5T`uO$RC zQ=uU--%%2acA*;bSZwH{LC1_YK}XY6Zqe!@aYt|5Ue`}V&q}CYt}+vG3s!RQgPj9G za};0S#h?YBI}0U&q*2GsbiBMFarZ@P8RA0p#vr}}7%ckFYMd`?9R_j=&?$ylIp1Gb z3f2)~dqI%RnPzHu#jwI$MKQ8XT$;}1AKK*Pvg2!p`mM4(pE^H>nLeqkY}@Q?D(qk{ zWc5~9kZy@NAHy$*kfnJ zn+zoAFntpbnZdw0ROp}83u1RX3mA$GkkoaEB`Mw|xP;E?w%<|l92vScj-$n4RV!9T z-&{tDr0Duu800H#dKe8~{LdQPv7Ke4@?O1T9(j3& z4k9w3V>u3|wz7Z33?|uM(r5SZ>l4Vj^D^sk=5VeI?@&Qqpp>pUZqN!jdi}D~ov3I! zyio>9QaOB3W}~}lG{#99zS405K0Q_Q=Qh~$XPQa+Bfstp}Ln^~U z1>Mx3#b+E+tYHkRR)^2WQ+c|xg%Hw7v>05C8hNKi@}ifzVNZ5?o3eOxc9Xu-#yiEw zSIOE;6v7qk(pqq;c}Ww$u@zT{#!sE}TSc|9zGoHvY>v9UlKc7w*h(U2EMc|2A9@f3 z83;2`Q&XFnYbKv=XspF6?~F@Uma8$wD=HJRe=m&2-`|s&I!7}7mgs-&ocoKq z>Oa}jhSb|wSOTUvd9Y2LQTA`a;iv31B#p-jyLdY7OfUAs>hji7IY|19IeInFVNO@J zC1jX~$Ab**UKMDL>s>hYqQIjZ*?NU ze`Qij-tI^y@To~p&D(tYhAFz-pFfWFr%SE3Mf+Z3Y*QJg&Bi>=&A;tvh_W5D%*z^K z&mh-oE1SyXw#lwzGrtiYTfoMxZC6Y^6xB~1Vl(u<;Q5ZlBlGy)u+R|$3R|=5cR*#; zXsR?;?Y28r^L*N47wB=B8K8obu-IvBNP<#0wI9@{9GR~{{VWY(b*TLN1B$@Erdz1y z0W_J*jrK|@7o1#NII&YxBpxo)Yg_z>hsxyKXn(n66ftUazmvn}CcFH`PB>TS(osXh zV;G_ZKUxr?En}w+s&l2r;7LE4-Yl_PlVKY~o38kgcMrK0s6>#uUqaldwzf7?_4hAw zFLojzdF%^NezzB1zmu^dcQMn>rF)Tm@*OMxHK8c@-{}kF)+^9A##-c8ECTUt+tMzC z6+#lrenP0b_<3sLON+JJ!8RTn7P^70bD+f55C*-CSu*gjB`14BXZmAC3RIX-emD5* zf0VgIdKr-_C4cx$KBc&ics*jkklXTZ|N0J9;QR<}+y{$o{fRz9p3z&+WzdKx8%l&_ z!{fJhIi4HRutY`>w)KTDPHs=F-LJv znd85akWo=mkU9DzBqSUi^-I=$eYMm){k&G%Aks9cZD*UHLuXfuQ5kmZx6g2cXE$J0 zQ@q|?e>>lPls#>nH(}T^{7naPX5yJ2L_|wGXUIGxx7ep9ZNNl}Se?L@TIBc=M_g?c zTxM4xP52h&ihNDlmz`4B6rTLk@Qp70(iZl}m>pJ@s zG~4xLE9^XOoA{&x*v{%f97FTZ_h^$cDeTHiD@>;zWTYm}PRZ!>aX0i03>hg_3IN@r zYHyZb`+WhqCdZ}W5dPcF!dUlv`rS1lA6?&b#%n>@*Q07V|5iQ*>WtkxY@mNa9n^|% zZtuVDt5(^-+x3k$;ojvdE}R`HyMVz-=lPiCw}s7DYXm=f%}1b6t;wRz3Dv+bLbO@N z)=u%^Z}Hs4l7lOhh5B8djzs^sH})8OkR?wV!ga zseW-=f?#?YUVj_^)+c4@N#a*LRZK)%D;izRuV&AE5Z=w_y}kr?2uuTQnD6-`W_jNI zcS_6^eD@1KPSd)eRk7XZ(h)ih)$PKm-6hw2aTx=`T5wpG_uwra5#1hg1U<%oCvr6L z*TcuC5algMN{xkK+H!6;;*+>h;yiz9sugf)i}QOgq2%!9hJ&?6EfaR9T8&jc5v%PF zPsoNAXNUq%Y@>2$Wiz``OaS#RZFFgz6KWa?C63B^^^4AN9SH894mDnrC|NoV*($+j z&FUm6lHC%|_eo1m%ewX2zojcFP^LN?P$>SLSStDbJkPez$)zpzD7 zv~u84VhPeKU#YT%*vB1g!@S~m3I+<#;$yD*b>01$!;zT_)es? zsVb|qr7WCmH;B_D?_N^I0={rIEP70#z_{;T?_MUz=&Q>YzJM!AkIeM=l$LIPEMUJd}V;jLV84|0tt%iT{Ni?ZV@3y z;bclY5elw;iSPZL#s3yGnm(}V6`?)Tkg3KROPkY&y?T_E;BnicNe>S|)Zn7=efFhI zTn~9Y#utZK5LY{IPm;=IA>#NdJD%80eu+B-9m8y^7k|gdRl#r`>pe zS85I8w`X0l2<+D2W^c)1W&ovUdfe79yg&OQ%k7 zxM$QjN)0Jh1yM)3wnxsb>u9_~e7*IA)eMV1k5*FS=z%h!LP)q7K}6$LF^xy!Y+8l( zd^Sx5R!L}M%N1J!2pN>2;z*M0qb19!4|*QHV8v;D_kDk6Pr&HZ>&|hD>=i_izpjx( zDYxX|S>a#JrnkDJQn>P%;(^Hssv+Y>@7A=W?c0P%vNQD&GX4T?vw948ZwHZRH7eD- zjZW>i^Sp(XC=hg*vp6Q^Zrpq0%yT703W%4(xJIt}G(3zyI5@MrfWMAZx%DR8EgsdZ zL8edKFI%EbwCrO+S3u^SDQaf>{6{AJd$d&soM8m-N)TY|D7?r!XR%Ur@>(&`>>p|Nu*pSfBzw<@1*#Rbj9 zo>#5?6yH`+8D5?D!qvdS*eF~D4ZkLWtB?KjE%sujsc3dcvRvXc#%&z7QB&0&d4D|R z#^yL7wvgfUd&C-G$`#6l31hu23(rMM>Ud^71r&t{$`LX6Q?vA29mA#UR=d4l2NvLa zf}g;X4+-`lCioH8zrq$!retm~H~(o0V^;+sUXC}{fB_s+5+p%<9*F>+BZexZfLG%ILls*6O2Y%v1Z(^1M{M zs%XzP*+k0|YxmO#2ILCACT!HNI|2amBI#n+iwu}skZCYLDTk!zOhiKBaelM}vBCg4 zcI${{5D^i%DvJv?7Nq)I4wG(@_z35%sT#XEaSf|uz|g9yM_(rqfuKmM9#D)YVYi(9pS+w9ptta0kK=Iio&)ZdN6|6NX9cev(tm$6FTVMQW70MXb zJ>QAxfO&1>dp#x>1m(5k32<#)pUf!GS;$z+uL=>ID6=A-JO>UUmrBpIyz*#xG=ru% zIdz0&}wD3xab zuFaYdT{2y6qu_%ld|2_hDS_ z{5b5ZV!4OgzJ#CGzN(iP{P_5I*;!5wYf`7e{#iv=A&Nctn11aL~WuNB$z# zdfIg^cp(G)g2D_}e%bOBwmCSN+x)Fcm_1@E`&$t*iSbl@Ba>r?hf&2@>wDPysh;gylOJW~5w(NVYj?KI)U38WQsyH1tIas_sut+f%x!aR zGgi#KxlG82O;n3cf0JJLQ8X|fY&z-DT-QlxAgSfcQGYN91#~>UAFt9}`}qbO!|RUV zaO=wJ@@7((Aa$p#;TGA;2tSv95=+n<@jo=D=8+`J=%#rH`bQk10q?it3(|O!f}2im zb4QIKI~=m^1CdxFnMU`!oC5KyHlKgYsR+ZA>}us_?9mq?A}zzUaxOi*gXL8w6h{lV zn>k0&pL~F#He`qDfm)KqlR_@z<-@wu)0wK6cXMRoD~Ad>8(LYB<#blrm(XIv03XyvTfH3hUuJZk#= z80i!`F`r!vM@GoH9{BCW`Im83Cdg*yj84;#7S5+_cpcHy{RnQ|2&ATZ7Jd+R19ASB z2KmkpPqOl<%yImf0Ru1YPdqhrqoZTzz4KspR1*<{%iCx*hGI-qqzBv< z_`^d{&n5h>geie!CL$?1z>jf*JCpE+`)B3;o@hXrH*SfvOuCznqi>>^$3%1dj=w1H zM25_Lf!h4EwzwebM>V?y)KtBR-jXbuG5Lc1OmW$KJXZB@ z%dYdPp(RnzKUKg*i<>vLzK?K2mlIMEh$obG3rqP+Mw#i{CgFTfXl9^(pUS{rMrR>6 zc!MlVXr8;c7|1Z(az>t7JXmZg3i$<$_VnkI(seZ}RoX^6!Io|~S$pk;k<~@)p^i1i zwNc>Eyye768DUvK5qZR~a0i_(wYXt(|~+WTI+DRLjXI-?QSbeUt} zW{cCf^8%GRv49k4Je^7d_uZ;zMwAG;xC^`WxWl+&8BmS;Sy)LLBfC}HYQj<;yf*CM zQzb@6=Nl7o+GjZc0RT#BdlgQ**;t-PNS7WSO8@(k1!~Agk)qt>=&MMI=y!mQCQ`?+|CpOu!aVoR5aOsZtiIbGmiaX7g8CryS2L5UTJ?z{ddBYN7TZTAhg?Sm zl(wA^V8)ijioMV7Zg`w2zQC6Cs?U=1E^q-TeF5!Kr^mmH>v=yCsDLaU;&+J&tb)%a zC;$n`0sWM}3l6YUf=>P=nM>~aA*eXQtN z2iA-4R{%oxC;$G0oyc=B&IFt=E))bc>bvAtFid>o&EFqTUp`(d@B+#>G&Er6tyh8+ z0_w}JZZdvwaH!A<`3L2$BA5huAAWz?gGAX~{!lC<G0fhsc6@*Vs>Db#VDT8@Cbr;L2 zp=Bi{pS2i&oyd&ib4}vYW?-m>NcByZy41LpV3Y^tE`MVITvOb(Ge|tw0ryVQC0Wua zbW$u7*lh4a?mKdQBw+@wo> z^KqvRAe`j7UT6Ci4ofe(JF z&(g+ZRZI(oh8q!?SUI6DVTT;Pwqu{4;9fGZ_2&ljQbvuEw}9?tyhL74A#eSX;9&(Y z-EWN+FHO2kr}ucQNb7><3UNnJ1vsvPAbb@(9bdb+rY)HX`d8S&W6}b4-Ir`DK*BL( zB6I@w;Y;eho)84N8$kcL>o~{{@4687J{CtGE15lgJ%=$IP7Q0Gc+S$D+eXUTH23krFS z@v)?{^);$pqS;Q2sWNdWUyDrlV;fZwvB3&zjwz+>V)7T@%>ieyk4k}3NI?|p%1|My zLY+&-@nu85`Odok;ZuyL?((g&01S)-vfMw-8JR08Q4{pQK;34irl!w4dG+- z3ZK~+v!pWkKC2g%i+vMFMy0;B*?qf#k;vYE=d}(CoMguGn;+8ytPdmPW7rg%PucO- z%vtx)Uir7V|F^hdZ_T~fVvtz*K7UY)o?uh;4-8QZ8F4v&QHH!7q||6OYGMdt7@J*b|{?(*GN?V($(-?ZQmd zDvhT-e5$?rIGrN4*GI|$y)_c8-F|Q)tU&TNrnX;Gnp|CNF;w1GPAQ!f<|!Fa<1PEEz5cO@F!4cvr3BwCsgsGMiPK zY4#o?cQ?I8oMW!8?l+Mfv1C0N-LS6ZW-8@DHoPakQiM68Vd4IkVebs6->bO7>%UeY zCrTnR*C7t|(U262A2l@^Q?oFwL3tdOZ{n(0%A2&gWBdCYNgFe$GViox49jrv z$&L)j3}v-x`7;s8$gL`|qp3Vf(CouyEL1Z^(37Y;!Lv`=wQYUD){RuZ*Dy4|#S=S1 zWTHI>d?S46CH1569vErkh-Le{N`^=_08`nb=6u-CS6}eotQjz%H%<>boMuCP{4`~a zNV^=R=v9^EJdZ<7N2GOK*Wj`a{!#*_AP%M|nyJ`VBZq6EY;T=+e|&paID}a2zyDsj zyOMaEjl;TplG_Z+tzm(6i8D7lM{_|M;cb6Mg{aU&TUrt3D^6XSpsyDo%3knZS6<3srj|j!pfbDWeoqR#secP!q9Wy21hA2uGsBlWW<$+kVP^WH$u?G z;Zxnm+f*^#=3g4h>qu|Waj(A7qIc3t+65_BRTP9iA_>&w&OG}3&6f08W7?ea>1N0P zHbjaQ58`wY;T;*Re)yjsNc$iDPQ~b$U-6#mXgXh* zsSEQOH8ojwWk}=-y&OK@br(>fzj)8MNWyIkNU+(m7{Q&Dis8IRMQE`GJ^W>dqfJxM zTRLKT#=r_WF|>QEx^)Q%BeNm?6Oa~*oTao1L~9gC#ki0&bSAC$;~o^n0%Z?c!ASf)1?-}JznR#neW}rK~mCHz39v3Afm{$F9pQf z6nJh}U$^l5-vGh94w89L(hU%Vf7$Sod!2|pZ;23iY#T_}Gg>RTi z+Fc)e#6K_!>po+jhqn^Tc`;~melb`#9w%m^QinQa?@6nCl@WVzW?l- z_kpd(0{Sbx2e(7&FLe`0o$f^P`+5#uCMu>Y0f&}5vHf8ZQY=5yK(ghe7~`n$`s^Pc zU8BM|c(ZVD6x~%8+ZQRmvK^Ko-7(_vDKd5YNI?x@{9$9S$d0E<{io}KiRS! zN`a5R(%X5y>;MwW{~j~nWmRTO$}{e^*5BQ4T~;X4b}2& zj)u`r)K@)Bv5Eg)8<=l{?&-f-8(8@stbCx6;Mdj}AKeK<`iBJzTKjDA6a(X2(~_J9 z;M;#m*|I|y)V3_8G`(e^+-fL+A)jcTiu+-;GyvSYRT8?0$7iXKqAu&V-zfRKEjJ`g z7TWD(yDCcrEjCETG!>LF9SmBZzXSbv&8#o+wS)(t?+2gA<8fF-$Hc7t{^jP2y1Lrh z^XZGTX#ja!A-7^dk6*|*jEhOP+STrU*oSce0RhMH1^X!YGCQ{Cj~{cPhz&J*@p7Bm zqeXiJNCj?kHS&uJ!K{FwDyISA&O zQnNmWTu(ot-6v#judVqKyX|o>(DwBC$uh?s(JEm{v|N3j%OYcgYB(Dv^Oe?>Kuvt* z!}wm1H)g-3-yW|UxBW5#%>zyZNzh9M(Y-_0~Ea*ywiD`5p8*29B?QE-)fElKiz zP0{ib07-M!2rXnN=coj8^zliS3msPU1lt?~)LFLtGH1)?7{J*QhhN4RyUXN{>P5g8dH?v3Q9%NDvPMBa0LU>p~T;OE;jxD9Bk-HyfI zedtV>ptV-P4Q>s|=ZCdj62~{r&%8+M(NxKpa-k4p2;v=Nx1xVI>AseC6Q7a36!$mA zy8?PgT_DC!mcK3wBBTJ3v3?0b)2ZY>Qi{I~JrzjB2vV1c$9e%X09Xcp{#WLX7bnJt zLTSo6(^51DH>bT7LEU05>a2Nm`Sf$642cVuyWgCT!o}=wW9rOtgJ_gOo}F>WCIrd4 zP}b&O(A#Fr7Y&k>@6pIX!OB39d?v~9I|Qp6U0KT6CG%O-Cyq^tP8y;%NrI{te?e0_ zj!}nM=KDAaL6?$b{bonXnNUl*PsW7}RYO*6{U#~idRadjf6i>VOjPdHV0c#4|E{fX zp0drASM_f6prCH5GLJ@%x__yAc*vu`+pGU-y`<+9Y8@?u+QHc@yn^7JXa!!;A0Ao8Dg06WZ)-v{wrC?#9dST~!*N zx4MNwwW5jBM~})ww{Yh^H#7hzLX=KJzrTP*lueO^bZllTN)3mpH%Q_taM-3!^%6-h zSkS)2TA$mcMJ+07gV&JerF(yiGcAN58g(HEwlWRpcJjxK3B6tdHzk{am;ND zQPzNmRQGF|25_Nq2DUt}RBsM^C@Sk&HFf+rKWfwwu{nXa6&?2;;AZ61$F|v2ke8s31d5=?y!@$IQs02?? z|4$O!QJ~x1caPV1t3tHTv!Q4>=x3u;bv4~i)my-L)ulDXeoL%xKq}8q@iM2XDVX0# z#-K)fX3rx6G?HOMzG{yhM@t;T#qo|kcrSB_@dd!RTBq*s4sshc?&*!~D^7Qz#HosV zyei%Fyq5eO@s{_KHE9Ju4EyQowzaM^?e?u8w9C;ICNhE;Dzn z{yM<_khoLTS{~7HQ=s6NK9Cd{)Z%Cj5)@(UeAgap=K8KJtJNMgq9Qr9!$N>vWnjSV z)QOBE3B7FUjC?DQyxMivudes_!iu0=Rx>JPuCfp3=a3PFc4^3Ic{#*`ND&(1A@{;L z)q8OM9?@Gh&j5MQa)x#9h0O`4COr(jN@2CLLsWmg9@5anJ!5AAesh0Id9U%2v#A5q zkmA*zJ>z|s3F$1zetwkYMn{~AAEOkNQ3BX5m<*1iLf56Go%J43sBnXtxO{(uv5n)S z!6mdV6-4!JPF%K}MQ z5@z|q;_-I-_pQTB?}5pW6=>#yq+6<8rGX*kWKD6o$ZeFxZk}T-qj17RA4JKQhIX9# zZK%lQZiI4ZQ|7Dew)>TdQ7y^JgXsh!usHeF1dAU&{lvwgmSIfh&!&cw=hF31&T%Up z&ya-^jSy}F|^<*6sSI3?%ME8*Y{ZW|(Q<~=+MALR^ZSLUI~qRzR#YiFDC@8XF&D-`9gm;Qgs83@xc zMTdUM7RZ*iDmo7utON?Y_BZJmXS(mPy@?MkFQ;w}xhtshVgAp#5F)^IHDlRyBgFk4 zzOujcGp%0t6O8nS%ez;t)3>wGuY?a9h z&Sn@H%tltMyw~U?te3oedZi0J5({!dI+=54_MRlRI$Skz@~ga7-v%kNWNv~U55)T1 z0MKakB{Ui&o(@fFIP!P_^-;fcq-@3~2OI1U&p?kiR^;Mn?rC z1DLm96Dr^~Q}3P6n9C8j35j5KzG_nD>W;X?yvZ{EBE2~C)OLICgG}NSH8HkD@tMk4zLCL3-;XlwrghU40p{&TOJ%v?7SC$dYVHfouE!@_ zFMm$yGxGHLg0769;D5})i{`Y^-d6)_Eh=(hPzps!?uh9KPe`ezaM}83P3lJ@LDN zQ$PCTAJH>rgJUB@M!H5eE7vM5vN1=MquhPMUX9!eeo`H+b)POmp-VEtfv{p|csHlj zXMxN-0k)6lP*Y%7R3d6O)bf;ns_(AR#qM9Vz%HuVZI%DOe#|Nv_R0I^`E;tj92I*mTt7K zFX?BWFiBmc^nuT-7H+9`-BY*UNfYoy{HCpMabM=( zJYF#OWbq{DeK=*l7+I|qpo(sj!-3#=0!=QsPi}|$4rd(?2PP|?gD`%uxD)&E>dOGI z9oym=aBq&_(U<}85m6&cdJWJb8m9G;{g4v)c;tFwjT(tg=P=^H#1&V+`q>Kc(p&(wKgEk=I1|CdZHoM9HcGS z*voR4>92Ygh*YC6j9oPsS7cXC>PX2F?<+ABj9wtEC9BFNZztpd>On_8itWzte&^YX zsfQQSUoR_NE%! zyhX68Irfg9*YQ%~7%1qLJ9`VM+Z^IXQ$I_99^2|HTsYF(h+Uc6UMEG2R9O+rpOL?6ib=RZ|)Zg=h1 zcq@f>0SS7G`_0oUu668bW)b4l(QJegkNcMvacLvkLNRp!xJ$qBDb_#pPAxkR7`5a0 z5jYa$KXL5CLL6;_9oQPjMd5cz&7O4Y-68Gdio{ z0_R({(5ZY(je0VM=~tijB=qmtmmE<*XD_K!Z2hOK`U^F#Le;6oA}ebvYncHxtBy_% zEglw@pRn!Of=e5$R*Dp92HOD$%fLt)+i_{~qrHhW6@u$XK~LF|7hf(CYKJ(!`eTPn zAUyg!9DytHn8v>U`u`P2G!U$VwLz+cf)XG|Qt8h9N?4PXWkr$7?iw`eVa#s^nL z0jU?T*m~rb#WakcTdz|RwDchT2Wl99=dA&Fg*h0+e`AEEnh~|y zfdurrAKwB2%^C-VoS4)D=Z7vMpr%!q>n=r2bv`~mPk82i$WQOWWC5*st7q4^_y|O% zGoQIMyg~t5mjQ?xWvidH?e&7Z8Bl^bXo4XDo89QMn7%zlr;ocmLUmf^kP*GLT^4LB z#}M!UA&z=_Gk`EHRHNVD-v=gg8nA|=UsBsbV8;Z0X#04-hnwoMvL$8g2c19%*H&ny z@nH;b;{0TFz-Vq}Y;J1$;d&6@SU4bw-xYIn^RG&uOXhFt{yjif5+QoPa@Ov-0^3g@0AgH=HKLMhHA%~u3Fcc@2=!X?D zQZWo92R=K3qP_p-+Sz0fB>?oS2ABocq$9A4zY*4-taPfoINK_>92Nd$s81j)xwf~5 zkUQMp7XZoj0V%y!_O}ljHsgnIi2piFx(Z7Nr*!oJZL17FvJ1rlUH zrBTT-+}8^P^>fygY_Sk|n#w*DlBAP?5(bDtsQrsUZu*YR>AIvvO5QonS{$N0L10e@ z=(bn&Yd~=uKoobS=V=iLgZEO+)vY@RuA$WI>;)R=x+wMmKuj!&@Fz=7o@^?xQq;l7 z7AFIqVs^;&!!`rW0;szdtxHvfpoPKY7ez2b0qXni0fXh|+#S5Tv-2}Xu z2Q^!sTLE5o@-rpq zDUFcAs+@7a`~&T!6OXQ`*tBq*^k0*-Xl6`X$@VKla(^1vY1m1)X)m!~kt~IxDn15s z7RQfAMoY%cvsw6RtkMJ)s$6m5k3};OTUr45vOUgH;O`}q%}IESjV|^j>is{Na*m0H zE0KS?hbmVsxv?dmN!BWx9X{{uPtIef=;Po2|G=QPbPHkML``v5&_5cZwh5Gwvx5lq z)w#Vt`19ZoP7gwbw)Rf6UWKp!VMa3MVb$laeHtNc#MRV~q0%QYv-cR>VI)~0n2#4S2&r-i*>BG zx|h^#5wZ87JudPK-#Yaf(fXdu6F9okf+b}yjL3tbRGil<_A*(>-NqwN+Nh{4fpf)G zE#fXVEltvoFZ589AfS#BvV&mCY6xBC^^CE4mXheSheg$Ng^O%hA)C)^zfw?9|3}qj z73iwYH!@C{CvpJT_t5t~%$*}E1Xo0Mq|E#kp+QRH9BD$@HI`Ie_l5I(wq(<8I!C;h z8@n1Z-+tu*l)-H*j+T4)C{XPmewGS~=_|0d=2tuh*)|`C4u^bqcJi&(thK7G(-I3i zZdkyX$!c`n>q@G1BHY!K)WnAT(tE6weFSXOX=?Jt|0^lkUsC=xA*?`KC#w0>umm+wU~b1nESo2>;7Gn(bPrm z(fPyuj_aVdcrwp4p%o|w-M~JT>cGjS>!IXQx@^=qFqdM2b}wPVI3M4qilSix%kK3? zfhbwMl4D1$rP(Mz#Y^<(^iOUOaW{PA`pD&HQRmLy{SX6SzEji{s^}k&TM*>`V=8f4yglrgBWW(3{TFq1KQR zC}yQ&z*OU&Vo5ASnS5smJ7N&lXZ8S^>B(={$P2#{NkEc599TGxRBuY@u$?gJ z|D)+q%OlN5rxwyt$Q4;f1#Cjy$(;GZR3Pbtxn2u_ZmXkrj7ZKeBCC&8!La?k~ z94wEN)8u~UTeA6>jPU;8wSw0r+C31@lO#%DhNDi)3b2kM;pY&q=EVh=zK ztS&2lgr*koq#M*9OnyMwgg^&P1Q#D&-5}|mwR({}0&0bC58~urf*8e49&-y1(-k=^wRXd~QEaVK}_doP!CC6>uvFh92^NYl$ zGH*`D6gH20*UL!Wdx%L0*feiV-PZVJ-Qt0>N;-e%%S%J;gTyG3$0jF|nL4DrWqvDA z4VCh;!T8VJNC9wF{zo2QXnRO1DjgVa!J?oU`_EX-QP3zM91>IB`RiUt z=^aT7$c?K|k>!_GQ3H!*m&gb^uFg~;t2)8op=8Xr`qd;8SU zL!{KlOn}5#Bz;P$ob<11|r3rlGSup3Xk>@QeN;*28qH5N1UWebL zU|~Lv*BAl1g)T$I(RNet+LIMVc1>8H8yi9|KlZ%A3_NH z;?^4NW!>MdPI6R+1WAouZ;sj#Zl3fBw|oElG^sF}`!iwuo^OH#HVmNNgX zqGG*d`47raH}eb2l%LTZP&-tv=RI}F2IYy8z07-$;#istZabJF{;H`AHw(jJHVqiG z`F>bueGTgvbMeDZ)+zZgY}-?5?i5)0Lgk4z4|59BfFnC{@Q1>c?Q{nMfF7L>I*D8^ zJ?acxOLk_rl@xJn>iDi(wIB@zC-~TsIe}_;<093Pq=NO@O>Ou(du_>@G-%?Y*cnmQ z;NB1bY=^J@v45h!o#1bIA*p(MT+1(k#F`N5a=-U6h9kSSWxgx5hU$fF99rBTW+OJX z@A6$|6O9A{2jl+u6tg+rpO(iYN)Gjtd7>%HJ0hbRS`rgC<-h#A)>*s`XeC4GUSM!O z6JdGaT&xv^rCNKRoz_7ZR@BXV(LoeTDQR{GFo3wyGhG3$yOG<32#56PyXdVZI&maDjlOs(KqheSX z9UuLikdYUnKidCICh%Fxr(<|kr5ik+nsZ9j_S5Ja0gc)BS#j8VrC>-E0oEdAg{4Iy zf9q?aNj+Q(@KjvvhtM|Q2k_fi0Y*FGe)G5I)23_*qyqJgYlKhnullq@$ew9bT2one z69P=lvV1%d_vEPLl)TOOGEgRi7*`4j5mnThXxqfkk_Zl0-UqaPa1PCfPnt$GU+rIM zD@_=1teat)Nb2_QU_{;<6Bp}s{^ZO@4lyRzb-sQY4!9W6^CfA$PIKt}7>!SR=pu)* zvPRm!t-D6yrcCt0A<#rg!h)Y^x6tgvWFd^kPXiyQk8x$X0Bll_Q829WKH_~|(6y!FU*%Jr#I<~CUR;>RGjI(Hyz$uqovHf zE8t-N0#Nw=LPirDIHQ9wAdn{wnn}>rN)^e&oHq4&2apGs{!xFWZrhD?~-dVsH_NGR!EeI#BP$YpbLLS#k$`;)pP!lAQ?&N z&mL$=zBv1oQM8P^m_YUFCdFg3S|GLmnT){ru7Z@<8lfILvYYJNIUM0it$}j~$AV&> zn-e?)EVwGgmRZJqP(1y7h^XW2i&PF?DCj2(VS*W7QZC~re;A@#^B`f~3xuceGgpou zFo82a59}d1H1`r%rlQ%&=D?81-8E3G0mPOZ8sI#@XK1Q)*`wge!N&IMPlwJ711Jm! z5lA9EVVRWs^L2JWv9^|WxVuZm*0k@U9^=?_0f|5#GTeL;pCJ5uu0sOTP z;Td@FNs|Sj;Rh7qoBhNt3)!SF>4#%;Nxp{%r0!dFTlvD)SP#cQAGIq#z0nhphc~2< z-Ono0;7O7-RtvzO=&d~<5CUC36@H%dR<4_YOYGuTq|qX^kw1=pG^_*Q!*8f&8@(}Q zYW;`jx^DYbz_)7iat6K-^Zoa1C|(DqTRf<7pTm_npil*&!*$2QhC^YXQOW#YpTqc+ z0Bmh|M4l2y5sp?-5tmHdo{i#ys;7C!Ea>rdEw z7fr+9sr1i@z8bP|C9zoY|46Nqy!u8+f@M#8YKv`R39GIxl(comW?_8GD*P|L_2_ou{UqT@@H z5d&td0F(14GgGNZjDzCux<+FwXQiuXXYFo2EHorvjH1g@whMmyzZNx}!3&2Rqvh$b zCD6gG5%ght)D)+J{s!rb50qp6jV>)6#PJ{V6(wARF^A#pMm8RV`|A{haq`B&Q7PJh zHn`%`Mj2{(EJ&QkX1}fxqVY9f`&>LjNIMing(puaElj)%D#uJu{hb4(jiA!ugHu8W zV8*hBQJgm}i~0@gjU(VgOr(nHYAGg3*2*VX7V?Zw-MYs>gjlq7IdVcH!y3V_o>6|} zkz+Rc8|>OS;OJMyl64!i)3|2;X){(yH{l;f;Sm14qG%AA|5tR!xoPGUEzNt}%>x8u z8~{mf_T@gS`~_gVg&TNc?fw{ej+YBclchHw#mTZ_5nHmvDasfHX%(>&VApM@lR>!c zWOQ;8{LZ5OMd$PH`HK?MQuAn7Irh;`>!yfMp^UAt8vjy|=iax{}7vBY|$*U@U zuv~wHppo1p01S-9-^1dhml9+6lQmZf`lw~}lt|(|QAJxP7PYDI$P|`zqbcdWB6C(0 zmk9rM!eh-BlSva~m|{=eUJ{J*S9R)VD%jZbGCEa)x&F@8e#@vEkd{I|R}+7aBWzeK znknt zQ!bAgU#*IZZPv89FP|eC1Euu{R-8V1aOz5J^KZ;-Waztp0%1}Ii_cvvx%Ba-0;HHm z7d*kA;!}XI3#tU?v(_iPS)n1xY=9k@Uk1al;H}d-Uq)|`zMeupIZ4P9f8^U>P{n}>VVST8^!1&I)gA=eKq4sF_k3+<-sK9vog1m9^2`Ljr1?p9YIrsdynFfS@`ynStF#%Bi~q% z#Hp#JzTlVpAnQBfwrmk~_Iiyp^t~b7Xc!O8Th^-ibeVHQ3^Ku$J;S&*4&*33&4%X6 zK`_utg(0Wte0)Y>dj1Gg`=>xEU;qWiGd`Jxa0aW;l0&EBc*bg2usikiQhl zG&1=4j)Rpji;6D(bYkkl9*7*zzAL3{yrj{=;~Iu-phy0WOR3$GBxCfUl6pQ`o~ zz^qOZqO-a%N!nkfzDpGDAR!zP=0mjWX|E_i{@J_n0rvHHC;ZHjqv|U?^RZ&KlBmG| zH!^dYD&J+xW$TgBEA4mOLTT1MrY2(%*0a{#Aw=NvK=_D*qBns>^(JuE&FmsC`X7NepA0^mx2{mTjA(f}_ZWE-{6Z_;E*1!!o7zT!Z+25Fk$agi}5LB;rvJ9aC8KNC5;OD6)ps^$b-sF6$(q257jQlFC`gc?vF4qp$_{^8-qqH)^XTtH=77 zU5Tc*t~0`iJ%DFubr*hkz{j=(V9;xOub^q=U-85|w}{oB*c%TC3w^A{wt~9c6gcVH zJaEGDPT2XoxvCBBbK~?ADcdcExoZF7f7(B*qA$4=N;1ws$Gft|+Q!X2r~; zu3`P7zlcl-l0k3VQ(S^LwWBS}e;ufFE=Uxl6i&`+uY5=E$yY2__Otm6veaIUXJjkg zLL+4&y%MNmGP=GALFqfLQypzOPd*Xa(&D-rKD(pEi3s>yBYe*5cKiDd%i?nK&Sl|i zb=_h+;h$OW9EF7ITS2%ae%q>(3(|;9N|D0UZxEdGA@jr1`Fj4jnLG2T}}86 zw!FldcRi`mz1xCB!Ei;6nx?#tngr=x{yazRKFj3ECt_NJZmKvI?bTvVe*QXuT%M;? z5AbImB9z+0Rbx~wfM3pMW@PmF_LbWi(@jtLo+XcZg)yd_JSmUGG(cNaZ9b0aqV!6> zjqWi5q{v0Mpic7(2yGilgHDxr0m;qi=;$qg&7TnuEzvVxsX(PDG08kJK0e;-*HITG z%0q9bB@ZWid;9&_D)XvKU_>`|#FQEV;T*1|GkK@oU2jQ9NN{1J#wN?k+gq94CIEpe znoILIgi@>Chs;Senood70F&QY*znIm zXomJ30C%Cn@8X4{&x7z8;r=y1Q09W`@EiX6@$kA92nh$44tOLY z{E=cLp|pV7OAw5M@X7wi6dJ-0j_9bU3jH?!ogK4>2w$;)4Oxm23jBqISple-e?Cv- zYGx=}CR>dO=&kO+DTjTbLtBEL*S`T~ygcv?ka#^1e~e!JYQsiBMwSvte&uoupTq}@ z5UHM*@K>gd2j-u%+4=chfU>`2>I%s38|wlvGV|n#^2!zjz$GMY`85!?|J~ zege$^$A)hCJOis@ibFP^jb9K8chRs5QFzLxTs-kw6wIf)R&(KGDm=APn~V{XqYh1#s+)@LU7SO zA?L0aJN0()?wg&cy^7gw15Jwj7LXud8k8t2Ww*UHKR*vW83P9H@zFYR*O-x8Ll1M3rLBKo4}+5oXYI84(a8If_BYELNj5nckJ01ys24$8f=MHsGMXnX zPMyaTg9E>yp6a^oCF@@#R+ZJ~$zD~iwweUC;u<%+kwJ*Oy-Az?^)`-|K}C)xN4i?Z zn)AbazsAmJ$mGFSQ@?EkuA}*24r4h8?=GV)OG`iAXVxVxNsPi*ZmJ`pvP?RqA`ettm#PM*4jU=^}&GrJi)HCwI+fCdqdEuLDb zb2w(Se%`cz#4E2g%gG({ zUgbSn%dmPI04Bo3G(s=S`h+3SbaJy|`px7^Uld!9S9?=Gi0&A-9WENSq z@}bpnSY%cN`G{IEHdlvGk%?!3-P?V_pQ3mf&5|N_1%v(8ofb856xO7 zlHKx)bkr9VhfF9_9VcTu$xmYjZgxP|30+wH;d~GGe3eiu9HqnPM>R>o4)b4GTaOIq zwXcdc=9Es0U&>W2oFe@A6Z1^KCcq6s&^G=(w?H!Uft+10? zk3*~RI5aV00xsLGb1X{T?k1WhoaowYIo2nqNUJS5b3)3j82EQBU-GJHXZG;E*3H=5 zUb-nPZrY-3I9KI4IXtO$6F!-`Ub~46vJI8b%{zeoYFP0p7K=qgToC`D2%*&K^wsVr z3Y5{s6W+ldx&hmNb5hIsAoIIRt9xdqO}S}h-eeR{D(jIZBQ}MD3XVwwr?nU<-@)N` z9;u56o6wXrU+JP3_9FJ%g_&{nKd0wY1Xhg@&$JV@Q<{xD%nDd?r>cA3EsPIM6^^@l zq>#!=xlClNK+fW?u3??Wr+N zJWk|+P`f8nD5HqHZS~v3*9?M2d5Fj+wFmKp2MkWACqhFvkSBOtrL}|_-5G^y0)mW9 zzBD+=W!SC1 z@a>SeoZEFG?(M_GpK!nusOj}1Zzz_#85Bts%&OF6$AipdI`!k@hF1^4YsF2k7md^2V(&FE{PD+Ld^^Z9)C0I?_b9 zR-2Zlj=_m$(qGA?aWp;=?M04+=hajExRZvtZHId?^_mt3ikBf(cJr>|PhpMPU5+E@ z5s+xJNn+e7@)MXcb4VEhz4|xEtSZUg?r{j#XjniQkBW$Vn7Q)|=!AM$=mnP17<%n{ z{1e-+J)S3w6vnbd$87h1JpLl`M0Q5Z-ue?KIaV_NNDf71l|9OqB#CN?k@Pp{Yt8;W z*l(X)4er}B|APx+2bS5T#o^3S^5TiWo#@V6osBo)Lt9{F*F;U%7xbqZ!gmJq12n-Q zp;a9R@?CoETd!8>XXjx15Z--{+k#7tt`QI9bRh_y-AKB=@)~>ERy44dTiK7HP=WEY zz~;4R)?d{zHV%#$LTtWKNH=?KB#paaykLjOx?G|Oc{`h|T!@vi`}iRYS35?>^F}3N z4Re4l7&{ecus5l!PA}b-B*8igMWnDbErz>#vf38e_PGpw-(*Y*HOEr>BPREFwm#)k z?1MQS9hr_#SpWDr%(*JK;Vf(fcFfU{=yT%1)INM^o~2@6y> zwDXJIT)UisJ%|IG5SjkGZJ+Lv5aT-PQ9U7SRR6B!ZtJUwl6qb);fqWLPEwNxY8F^gpfr;kShK*%U z-Om#oiNlaqZ<3?d@Ij1{gCoW#uuJVd%;y9#txa4+(_5y*$e@f*g>1Q)ox|aybm5fj zRhwcOSWP)6hOR`3srA_tAsAxC&_?`^1RJI$!H1tS5ru?p5+sxRpp7Z ztEwh48kJj(f@Dt_P+RoNKkhmRxTec8i#GCYm)2z29Asp1H5lQH`dQJQ8u9lM(rnC6yc7F1Gdq-jI`Q2`>q%6 z6P`w0Y4u_&YD~b)2_ch4WTs*IR#9TSNO3jEhlU`*Y0kDUB@W6kX1d=)eXQ0>r0ah6 zQ6{(jq-zxM5vZAkwy9^+F*lg3POS6xpgH!F$?pYpKO|oXiM@}Kj*Hai3rl@iW^qd< zn;`CfhLaMB6d?LD{c+|5Nwk(88!j0U%j6I+fQRL|!evvbZ(^tq=?i-sX?b-<*~p29 zS{@Z#RoNG*#t>!A?ilglR|b6ZUj(1+M!~jmBr6W(Op5yLzGpaV)}8V=9R}`1wA9lq z1Sm13x3GAeiTaPPu_@Db=or9d%HJw_rcCuJv$3k;^;_U^(Ye1Kj4 zL@wTKY=0Ft7Dw(N+coyEh@L5F(+0{k z@EJc(^ApbEGbAl4f-Oo;F4+MHX*dQlRb@Rx>f)S}jl}_em$8U*X1U%7Zzq#ZB+I;` zOP~sldtZdBoq%1Jo6hUpm(Y4#2Rh-54H363g=PkwGep*Ij4~zgXvn#sHeDMH?ZP!) zp+$ir@riRu+WeN!qPKH|N|b&L%q~t;46~Yw`$U#xdrpnDTcLYS=l(`9A9u$Q zTiG~>qrMEJ8G0IUKr&@?anD_^uEjG|^2c%Och^xjcE*D#K(oPpL4CK^B7=izw7LO~ zVq9hylwba_CyZDkc4?gJXl^0J*qGFxNJ+5cAT3PC|cEgu;9SG^ADJ$pHD{cmOEjVgI zz5LVoXs)f5ZGlbtLsHa+c)x(?Ry8N(zOI$w zj=>RwhBM`diV)Qgyk%S(TzM%v?xkJ)KWM$$2FaC8m8a-ja=PEqrVpKr;qQIQG*MFS z9AY1?d+8D}T6_NOxouV+U29KeOwiW~I(2llmZrLjbCsG47L`bJeoJ|qHd4qe^h>6Y z%N2SXa9}d`M$O5#`>?++RFR5ed^h-J7DU~Rn5~z)Z_T{MH@sZ$Sc(R-FK#R;5d6+Y z4AD49|y@=|rF;944qAQx$&@mu&{65NG$V}n( zAuXZ6qWyQlU(c$n-aEu7uR!t_$5t>$TylQvIV}_;hQT_$ff<2EZtz~f6LK|H9S20lOCE6Ed&plUsNY83tv~qHVjQ(n*UVfGE>Nu0a&2Fy9lhP45 za&4>Jf=P9qC|H6i%(lSl<O3KL)w>=!T0dLlEhM-;hvDGq_a@<0s6^#$g(bcqR|6I-Xo5kYrX4b8 zS|T%@>S8nz)gaXC!Sf+PX=%GOXn%><6a%^Kd5czFt@5CDmQlMUl!S*%!{JeHZ>NTt zQ}9CCw;?;ozRJF>xa^_*&|rV)Vgu~gqq3ajpy{+ z^*T}5^Taymr)kA#2~Ji8Ff2J&$feo3J=6siZ0zCZTS^w_E{vNp$S+hBpILuM9W$7w zT`KRRFQ`Z;hH|3AMBOs)z4I9n_Xhst8V5M0b!yb&&NT9sRF-DZ znQCR~^r4DMTQHLXf3MW-msHYApcEl|Ai%Z4xqJH1?>lH#x4xNY4)1*weIfa&{ zh51k%2<9)aLyrMJ&T8xDy2w2rOt_mej(qf03*mJ>&kmo{+wFJ>rJ>t z)ayeZxGE%5v=|_e5iUJLLHMfAkA(24`_!SS8OYy32yVsrZ?yUT>cDiH>LsSofU~hj zT2QlkdSfzT_^BSgW;~Jc34Y!*S?p!%=x@}IhT|d?^2S1-NTb2lmDV@+>n{g3IuxL_ zBu-T>8ObRag+e}&5*B&&=9wIeduLIPCcQ=kzSx_QJqtjm9T9-(IAM^wKTaEkECw&*$xjxp_oA4nbF?jN$2abiPsxP`domRW|4ZPt0?Qr*h* z7W~N~-yj}1Geu6THJZEPavnKJM2q>0T&}hxot&U^3T45rSFgvZBM82j6faXR#v~-n z3<)=B-HLp)j|?>O2BFZJ8z+XiyCz&c#JpU Date: Wed, 1 Jul 2026 15:40:57 +0800 Subject: [PATCH 10/14] async: delete payload --- ...66\351\235\242\351\207\215\346\236\204.md" | 99 +- .../src/lib.rs | 53 +- .../src/agent_service/transfer_agent.rs | 1662 ++++++++++------- fluxon_rs/fluxon_fs/src/cache_controller.rs | 2 +- fluxon_rs/fluxon_fs_s3_gateway/src/lib.rs | 5 +- .../fluxon_kv/src/client_seg_pool/mod.rs | 13 +- fluxon_rs/fluxon_kv/src/config.rs | 18 +- .../fluxon_kv/src/external_client_api/mod.rs | 3 +- fluxon_rs/fluxon_kv/src/kv_test.rs | 9 +- fluxon_rs/fluxon_kv/src/lib.rs | 4 +- .../lease_manager_test.rs | 12 +- fluxon_rs/fluxon_mq/src/consumer.rs | 175 +- fluxon_rs/fluxon_ops/build.rs | 20 +- fluxon_rs/fluxon_ops/src/lib.rs | 433 +++-- fluxon_rs/fluxon_pyo3/src/error.rs | 2 +- fluxon_rs/fluxon_pyo3/src/mpsc.rs | 12 +- fluxon_rs/fluxon_util/src/dev_config.rs | 2 +- fluxon_rs/fluxon_util/src/lib.rs | 14 +- fluxon_rs/fluxon_util/src/log.rs | 46 +- fluxon_rs/fluxon_util/tests/log_mgmt.rs | 12 +- pics/blog2_mq_payload_flow.png | Bin 50774 -> 55188 bytes 21 files changed, 1434 insertions(+), 1162 deletions(-) diff --git "a/fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" "b/fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" index 8f5c5c3..2960414 100644 --- "a/fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" +++ "b/fluxon_doc_cn/blog/blog_2_\344\270\200\346\254\241 AI \345\244\247 Payload \346\266\210\346\201\257\351\230\237\345\210\227\347\232\204\346\216\247\345\210\266\351\235\242\351\207\215\346\236\204.md" @@ -18,9 +18,9 @@ AI 训练和推理系统里的消息队列，处理的已经不再是几 KB 的 broker 链路把这些状态推进移到 broker 内部。producer 写入前先向 broker 申请 reservation。reservation 是一次写入尝试的占位，broker 返回 `reservation_id` 和 `msg_id`，并记录这条消息预计占用的 Payload bytes。Payload 写入 KV owner 成功后，producer 调用 `publish`，消息进入可消费队列。Payload 写失败时，producer 调用 `abort`，broker 释放占位和字节预算。这个顺序保证了 consumer 只能看到已经写入成功的 Payload。 -consumer 通过 `fetch` 获取消息。broker 将消息从可消费队列移动到 in-flight，并返回 Payload key。in-flight 表示消息已经被某个 consumer 拿走，但还没有确认消费完成。consumer 读取 Payload 并完成处理后调用 `commit`，这一步成功后，broker 才认为这条消息已经完成消费。后续 Payload 删除或释放完成后，consumer 再发送 `cleanup ack`，broker 释放对应的清理状态。consumer 失败、超时或被取消时，未 commit 的消息会重新放回可消费队列，等待后续投递。 +consumer 通过 `fetch` 获取消息。broker 将消息从可消费队列移动到 in-flight，并返回 Payload key。in-flight 表示消息已经被某个 consumer 拿走，但还没有确认消费完成。consumer 读取 Payload 并完成处理后调用 `commit`，这一步成功后，broker 才认为这条消息已经完成消费。consumer 返回 Payload 后，Rust 后台任务异步删除 KV Payload；删除完成后再由内部 cleanup 路径释放 broker 的 Payload byte budget。consumer 失败、超时或被取消时，未 commit 的消息会重新放回可消费队列，等待后续投递。 -这个流程把每条消息的状态推进留在 broker 内存中。`fetch`、`commit`、`requeue` 和 `cleanup ack` 都通过 P2P RPC 调用 broker，broker 更新本地状态后返回结果。etcd 从消息热路径中退出，只处理成员、租约和发现这类低频职责。 +这个流程把每条消息的状态推进留在 broker 内存中。`fetch`、`commit` 和 `requeue` 通过 P2P RPC 调用 broker，broker 更新本地状态后返回结果；cleanup 只作为 Rust 内部清理路径继续维护容量统计。etcd 从消息热路径中退出，只处理成员、租约和发现这类低频职责。 ## broker 的进程边界 @@ -36,33 +36,78 @@ Rust 侧的 broker 状态位于 `fluxon_rs/fluxon_mq/src/broker.rs`。这部分 broker 保存的是消息控制面状态和 Payload 引用。Payload bytes 仍然由 KV owner 管理，broker 只记录 `payload_key`、`payload_bytes`、消息信封和队列位置。 -| 结构 | 关键字段 | 含义 | -| --- | --- | --- | -| `BrokerState` | `channels` | 按 `channel_id` 保存每个 channel 的队列状态 | -| `BrokerState` | `payload_byte_capacity` | broker 维度的 Payload byte budget 上限 | -| `BrokerState` | `used_payload_bytes` | 当前所有未释放消息占用的 Payload byte budget | -| `ChannelState` | `config` | `BrokerChannelConfig`，包含 `channel_id` 和 `capacity` | -| `ChannelState` | `next_reservation_id` | channel 内递增的 reservation 编号 | -| `ChannelState` | `next_msg_by_producer` | 每个 `producer_id` 的下一个 `msg_id` | -| `ChannelState` | `pending` | 已 `reserve`、尚未 `publish` 的消息 | -| `ChannelState` | `visible` | 已写入 Payload 且可被 consumer `fetch` 的消息 | -| `ChannelState` | `inflight` / `inflight_order` | 已被 consumer 取走、尚未 `commit` 的消息及其顺序 | -| `ChannelState` | `cleanup` | 已 `commit`、等待 Payload 清理的消息 | -| `ChannelState` | `cleanup_inflight` | 已分配给清理任务、等待 `cleanup_ack` 的消息 | -| `ChannelState` | `committed` | 已提交的 `reservation_id` 集合，用于处理重复提交 | -| `ChannelState` | `used_slots` | channel 当前占用的消息槽位 | -| `ChannelState` | `reserve_waiters` / `fetch_waiters` | 因容量或可见消息不足而等待的请求 | +```rust +pub struct LocalBroker { + state: BrokerState, // broker 内存状态 + log: Vec, // 状态变更记录，用于回放和恢复边界 +} + +struct BrokerState { + channels: HashMap, // 按 channel_id 保存队列状态 + payload_byte_capacity: u64, // broker 维度的 Payload byte budget 上限 + used_payload_bytes: u64, // 当前未释放消息占用的 Payload byte budget +} + +struct ChannelState { + config: BrokerChannelConfig, // channel_id 和 capacity + next_reservation_id: u64, // channel 内递增的 reservation 编号 + next_msg_by_producer: HashMap, // 每个 producer_id 的下一个 msg_id + pending: HashMap, // 已 reserve、尚未 publish 的消息 + visible: VecDeque, // 已写入 Payload、可被 consumer fetch 的消息 + inflight: HashMap, // 已被 consumer 取走、尚未 commit 的消息 + inflight_order: VecDeque, // inflight 消息顺序 + cleanup: VecDeque, // 已 commit、等待 Payload 清理的消息 + cleanup_inflight: HashMap, // 已分配给清理任务、等待 cleanup_ack 的消息 + committed: HashSet, // 已提交的 reservation_id，用于处理重复提交 + used_slots: i64, // channel 当前占用的消息槽位 + reserve_waiters: VecDeque, // 因容量不足等待 reserve 的请求 + fetch_waiters: VecDeque, // 因可见消息不足等待 fetch 的请求 +} +``` broker RPC 和内部状态机使用的主要消息结构如下： -| 结构 | 字段 | 用途 | -| --- | --- | --- | -| `BrokerReserveRequest` | `channel_id`、`producer_id`、`category`、`payload_bytes`、`now_ms` | producer 申请消息占位和 Payload byte budget | -| `BrokerFetchRequest` | `channel_id`、`consumer_id`、`now_ms` | consumer 请求下一条可见消息 | -| `BrokerEnvelope` | `channel_id`、`producer_id`、`msg_id`、`reservation_id` | 标识一条消息和一次写入 reservation | -| `BrokerEnvelope` | `payload_key`、`payload_bytes` | 指向 KV owner 中的 Payload，并计入 broker byte budget | -| `BrokerEnvelope` | `reserved_at_ms`、`published_at_ms` | 记录消息进入 broker 状态机的时间 | -| `BrokerCommitOutcome` | `first_commit`、`cleanup` | 告诉 consumer 本次提交是否首次生效，以及是否产生清理任务 | +```rust +pub struct BrokerChannelConfig { + pub channel_id: i64, // channel 标识 + pub capacity: i64, // channel 消息槽位上限 +} + +pub struct BrokerReserveRequest { + pub channel_id: i64, // 目标 channel + pub producer_id: String, // producer 标识 + pub category: MqCategory, // MPSC 或 MPMC 子队列类型 + pub payload_bytes: u64, // 本条消息预计占用的 Payload bytes + pub now_ms: i64, // reserve 时间 +} + +pub struct BrokerFetchRequest { + pub channel_id: i64, // 目标 channel + pub consumer_id: String, // consumer 标识 + pub now_ms: i64, // fetch 时间 +} + +pub struct BrokerEnvelope { + pub channel_id: i64, // channel 标识 + pub producer_id: String, // producer 标识 + pub msg_id: i64, // producer 内递增消息编号 + pub reservation_id: u64, // 本次写入 reservation 编号 + pub payload_key: String, // KV owner 中的 Payload key + pub payload_bytes: u64, // Payload byte budget 计数 + pub reserved_at_ms: i64, // reserve 时间 + pub published_at_ms: Option, // publish 时间，未 publish 时为空 +} + +pub struct BrokerCommitOutcome { + pub first_commit: bool, // 本次 commit 是否首次生效 + pub cleanup: Option, // 首次 commit 后生成的清理任务 +} + +pub struct BrokerCommitBatchOutcome { + pub first_commit_count: usize, // batch 中首次 commit 成功的数量 + pub cleanup: Vec, // batch 生成的清理任务 +} +``` 状态流转可以简化为下面这条链路： @@ -72,7 +117,7 @@ producer 热路径位于 `fluxon_rs/fluxon_mq/src/producer.rs`。broker 路径 当 channel 满或 `payload_byte_capacity` 不足时，producer 在 Rust 热路径里按 `BrokerError::ChannelFull` 或 `BrokerError::PayloadBytesFull` 做退避重试。这个重试发生在 broker reserve 阶段，等待条件直接来自 `used_slots` 和 `used_payload_bytes`，比 Python 外层固定 sleep 更贴近真实队列状态。 -consumer 热路径位于 `fluxon_rs/fluxon_mq/src/consumer.rs` 和 `fluxon_rs/fluxon_pyo3/src/mpsc.rs`。consumer 先通过 broker `fetch` 取得 `BrokerEnvelope`，再用其中的 `payload_key` 从 KV owner 读取 Payload。业务处理和 commit 完成后，consumer 执行 Payload delete，并通过 cleanup 路径释放 broker 的 byte budget。Python 层主要负责 API 包装、bench 编排和 teardown；消息推进、背压等待和 cleanup 状态已经收敛到 Rust broker 路径。 +consumer 热路径位于 `fluxon_rs/fluxon_mq/src/consumer.rs` 和 `fluxon_rs/fluxon_pyo3/src/mpsc.rs`。consumer 先通过 broker `fetch` 取得 `BrokerEnvelope`，再用其中的 `payload_key` 从 KV owner 读取 Payload。`commit` 成功后，Payload 立即返回给上层；KV delete 和 broker byte budget 释放由 Rust 后台清理任务继续推进。Python 层主要负责 API 包装、bench 编排和 teardown；消息推进、背压等待和 cleanup 状态已经收敛到 Rust broker 路径。 ![](../../pics/blog2_mq_payload_flow.png) diff --git a/fluxon_rs/fluxon_commu_closed_sdk_consumer/src/lib.rs b/fluxon_rs/fluxon_commu_closed_sdk_consumer/src/lib.rs index 6fab54e..caad34b 100644 --- a/fluxon_rs/fluxon_commu_closed_sdk_consumer/src/lib.rs +++ b/fluxon_rs/fluxon_commu_closed_sdk_consumer/src/lib.rs @@ -11,9 +11,9 @@ use fluxon_commu_contract::{ ClosedRuntimeCallRawObservedOutputView, ClosedRuntimeClusterEventStreamItem, ClosedRuntimeClusterManagerCall, ClosedRuntimeClusterManagerResponse, ClosedRuntimeClusterRdmaResolvedConfigStreamItem, ClosedRuntimeDesiredTransferPeer, - ClosedRuntimeDispatchRequestView, - ClosedRuntimeDispatchResponse, ClosedRuntimeDispatchTransportPolicy, ClosedRuntimeError, - ClosedRuntimeHandle, ClosedRuntimeHostCallbackHandle, ClosedRuntimeP2pCall, + ClosedRuntimeDispatchRequestView, ClosedRuntimeDispatchResponse, + ClosedRuntimeDispatchTransportPolicy, ClosedRuntimeError, ClosedRuntimeHandle, + ClosedRuntimeHostCallbackHandle, ClosedRuntimeP2pCall, ClosedRuntimeP2pCallRawObservedRequestView, ClosedRuntimeP2pResponse, ClosedRuntimeP2pSendResponseRawRequestView, ClosedRuntimePeerGen, ClosedRuntimeRawSlice, ClosedRuntimeRequest, ClosedRuntimeResponse, ClosedRuntimeTransferEngineCall, @@ -491,11 +491,15 @@ impl WireBodyPartsOwner { let (raw_lengths, raw_payload) = match raw_bytes.len() { 0 => (WireBodyRawLengths::Empty, WireBodyRawPayload::Empty), 1 => { - let part = raw_bytes.into_iter().next().expect("single raw part missing"); - let len = - u32::try_from(part.len()).map_err(|_| ClosedSdkConsumerError::RuntimeDecode { + let part = raw_bytes + .into_iter() + .next() + .expect("single raw part missing"); + let len = u32::try_from(part.len()).map_err(|_| { + ClosedSdkConsumerError::RuntimeDecode { detail: format!("wire raw part too large for u32 length: {}", part.len()), - })?; + } + })?; ( WireBodyRawLengths::Single([len]), WireBodyRawPayload::Single(part), @@ -849,8 +853,7 @@ fn decode_call_raw_observed_output_view( return Err(ClosedSdkConsumerError::RuntimeDecode { detail: format!( "closed SDK call_raw_observed serialize_part overflow: serialize_len={} full_len={}", - message_view.body.serialize_part.len, - message_view.body.full_body.len, + message_view.body.serialize_part.len, message_view.body.full_body.len, ), }); } @@ -860,21 +863,19 @@ fn decode_call_raw_observed_output_view( .ok_or_else(|| ClosedSdkConsumerError::RuntimeDecode { detail: "closed SDK call_raw_observed raw_bytes length overflow".to_string(), })?; - let expected_full_len = - message_view - .body - .serialize_part - .len - .checked_add(raw_total) - .ok_or_else(|| ClosedSdkConsumerError::RuntimeDecode { - detail: "closed SDK call_raw_observed body length overflow".to_string(), - })?; + let expected_full_len = message_view + .body + .serialize_part + .len + .checked_add(raw_total) + .ok_or_else(|| ClosedSdkConsumerError::RuntimeDecode { + detail: "closed SDK call_raw_observed body length overflow".to_string(), + })?; if expected_full_len != message_view.body.full_body.len { return Err(ClosedSdkConsumerError::RuntimeDecode { detail: format!( "closed SDK call_raw_observed body length mismatch: expected={} full_len={}", - expected_full_len, - message_view.body.full_body.len, + expected_full_len, message_view.body.full_body.len, ), }); } @@ -923,9 +924,7 @@ fn decode_call_raw_observed_output_view( frame_recv_done_ts_us: message_view.local_observe.frame_recv_done_ts_us, dispatch_enqueued_ts_us: message_view.local_observe.dispatch_enqueued_ts_us, dispatch_started_ts_us: message_view.local_observe.dispatch_started_ts_us, - complete_pending_call_ts_us: message_view - .local_observe - .complete_pending_call_ts_us, + complete_pending_call_ts_us: message_view.local_observe.complete_pending_call_ts_us, }, }, observe: fluxon_commu_contract::ClosedRuntimeRpcCallTransportObserveTrace { @@ -1550,8 +1549,8 @@ async fn invoke_completion_async_with_keepalive( ) -> i32, ) -> Result<(i32, Bytes), ClosedSdkConsumerError> { let (sender, receiver) = tokio::sync::oneshot::channel::<(i32, Bytes)>(); - let user_data = Box::into_raw(Box::new(RuntimeCompletionState { sender, keepalive })) - .cast::(); + let user_data = + Box::into_raw(Box::new(RuntimeCompletionState { sender, keepalive })).cast::(); let submit_status = submit(user_data, Some(runtime_completion_callback)); if submit_status != 0 { unsafe { @@ -2082,7 +2081,9 @@ pub async fn p2p_call_raw_observed( ) .await?; match status_code { - FLUXON_COMMU_CLOSED_RUNTIME_RESULT_OK => decode_call_raw_observed_output_view(payload.as_ref()), + FLUXON_COMMU_CLOSED_RUNTIME_RESULT_OK => { + decode_call_raw_observed_output_view(payload.as_ref()) + } FLUXON_COMMU_CLOSED_RUNTIME_RESULT_ERR => { let error = bitcode::decode::(payload.as_ref()).map_err( |decode_error| ClosedSdkConsumerError::RuntimeDecode { diff --git a/fluxon_rs/fluxon_fs/src/agent_service/transfer_agent.rs b/fluxon_rs/fluxon_fs/src/agent_service/transfer_agent.rs index 1738ade..ca54a71 100644 --- a/fluxon_rs/fluxon_fs/src/agent_service/transfer_agent.rs +++ b/fluxon_rs/fluxon_fs/src/agent_service/transfer_agent.rs @@ -9,28 +9,23 @@ use std::time::{Duration, Instant}; use fluxon_fs_core::config::{ FS_AGENT_TRANSFER_STREAM_CLOSE_RPC_PATH, FS_AGENT_TRANSFER_STREAM_NEXT_RPC_PATH, - FS_AGENT_TRANSFER_STREAM_OPEN_RPC_PATH, - FS_MASTER_TRANSFER_SCHEDULER_HEARTBEAT_RPC_PATH, FS_MASTER_TRANSFER_SCHEDULER_RESULT_RPC_PATH, - FluxonFsTransferBatchCollectInfoWire, FluxonFsTransferBatchKind, - FluxonFsTransferCollectInfoKind, FluxonFsTransferDispositionWire, - FluxonFsTransferFailedFileReasonKindWire, - FluxonFsTransferReadStreamCloseWire, FluxonFsTransferReadStreamNextResultWire, - FluxonFsTransferReadStreamNextWire, FluxonFsTransferReadStreamOpenResultWire, - FluxonFsTransferReadStreamOpenWire, - FluxonFsTransferSkipEntryKind, FluxonFsTransferSkipEntryWire, - FluxonFsTransferManifestEntryWire, FluxonFsTransferManifestWire, - FluxonFsTransferScanMode, - FluxonFsTransferScanEventAckWire, FluxonFsTransferScanEventKindWire, - FluxonFsTransferScanEventWire, FluxonFsTransferScanLaunchResultWire, + FS_AGENT_TRANSFER_STREAM_OPEN_RPC_PATH, FS_MASTER_TRANSFER_SCHEDULER_HEARTBEAT_RPC_PATH, + FS_MASTER_TRANSFER_SCHEDULER_RESULT_RPC_PATH, FluxonFsTransferBatchCollectInfoWire, + FluxonFsTransferBatchKind, FluxonFsTransferCollectInfoKind, FluxonFsTransferDispositionWire, + FluxonFsTransferFailedFileReasonKindWire, FluxonFsTransferManifestEntryWire, + FluxonFsTransferManifestWire, FluxonFsTransferReadStreamCloseWire, + FluxonFsTransferReadStreamNextResultWire, FluxonFsTransferReadStreamNextWire, + FluxonFsTransferReadStreamOpenResultWire, FluxonFsTransferReadStreamOpenWire, FluxonFsTransferScanAssignmentWire, FluxonFsTransferScanBatchWire, - FluxonFsTransferScanChildUnitWire, FluxonFsTransferScanFrontier, + FluxonFsTransferScanChildUnitWire, FluxonFsTransferScanEventAckWire, + FluxonFsTransferScanEventKindWire, FluxonFsTransferScanEventWire, FluxonFsTransferScanFrontier, FluxonFsTransferScanFrontierDirEntry, FluxonFsTransferScanFrontierEntry, - FluxonFsTransferScanResultWire, - FluxonFsTransferSymlinkNoticeEntryWire, FluxonFsTransferWorkerCollectInfoResultWire, - FluxonFsTransferWorkerAssignmentWire, FluxonFsTransferWorkerFileResultWire, - FluxonFsTransferWorkerFailedFileResultWire, - FluxonFsTransferWorkerHeartbeatResultWire, FluxonFsTransferWorkerHeartbeatTelemetryWire, - FluxonFsTransferWorkerHeartbeatWire, + FluxonFsTransferScanLaunchResultWire, FluxonFsTransferScanMode, FluxonFsTransferScanResultWire, + FluxonFsTransferSkipEntryKind, FluxonFsTransferSkipEntryWire, + FluxonFsTransferSymlinkNoticeEntryWire, FluxonFsTransferWorkerAssignmentWire, + FluxonFsTransferWorkerCollectInfoResultWire, FluxonFsTransferWorkerFailedFileResultWire, + FluxonFsTransferWorkerFileResultWire, FluxonFsTransferWorkerHeartbeatResultWire, + FluxonFsTransferWorkerHeartbeatTelemetryWire, FluxonFsTransferWorkerHeartbeatWire, FluxonFsTransferWorkerLaunchResultWire, FluxonFsTransferWorkerResultAckWire, FluxonFsTransferWorkerResultWire, FluxonFsTransferWorkerStopReasonWire, transfer_collect_info_output_relpath, @@ -39,8 +34,8 @@ use fluxon_fs_core::retry::{ BackoffConfig, DEFAULT_WARN_INTERVAL_SECS, WarnConfig, next_backoff, should_warn, }; use fluxon_kv::rpcresp_kvresult_convert::msg_and_error::{ApiError, KvError}; -use fluxon_kv::user_api::flat_dict::{FlatDict, FlatValue}; use fluxon_kv::user_api::FluxonUserApi; +use fluxon_kv::user_api::flat_dict::{FlatDict, FlatValue}; use parking_lot::{Condvar, Mutex}; use super::{ @@ -202,16 +197,13 @@ fn transfer_scan_session_state() -> &'static Mutex { TRANSFER_SCAN_SESSION_STATE.get_or_init(|| Mutex::new(TransferScanSessionState::default())) } -fn cleanup_expired_transfer_scan_sessions( - state: &mut TransferScanSessionState, - now_unix_ms: i64, -) { - state - .root_dir_listing_sessions - .retain(|_, session| session.lease_expire_unix_ms <= 0 || session.lease_expire_unix_ms > now_unix_ms); - state - .subtree_streaming_sessions - .retain(|_, session| session.lease_expire_unix_ms <= 0 || session.lease_expire_unix_ms > now_unix_ms); +fn cleanup_expired_transfer_scan_sessions(state: &mut TransferScanSessionState, now_unix_ms: i64) { + state.root_dir_listing_sessions.retain(|_, session| { + session.lease_expire_unix_ms <= 0 || session.lease_expire_unix_ms > now_unix_ms + }); + state.subtree_streaming_sessions.retain(|_, session| { + session.lease_expire_unix_ms <= 0 || session.lease_expire_unix_ms > now_unix_ms + }); } fn same_root_continuation_scan_unit( @@ -301,10 +293,7 @@ fn flush_pending_root_direct_files_batch( return Ok(None); } let batch = build_direct_files_only_batch_from_entries_with_batch_id( - direct_files_only_batch_id_for_partition( - assignment, - session.next_direct_files_batch_index, - ), + direct_files_only_batch_id_for_partition(assignment, session.next_direct_files_batch_index), assignment, assignment.root_relpath.clone(), std::mem::take(&mut session.pending_direct_files), @@ -313,7 +302,8 @@ fn flush_pending_root_direct_files_batch( )?; session.pending_direct_bytes = 0; session.next_direct_files_batch_index = session.next_direct_files_batch_index.saturating_add(1); - session.emitted_direct_files_batch_count = session.emitted_direct_files_batch_count.saturating_add(1); + session.emitted_direct_files_batch_count = + session.emitted_direct_files_batch_count.saturating_add(1); Ok(Some(batch)) } @@ -414,7 +404,8 @@ fn open_transfer_root_dir_listing_session( root_dir_abs: &str, assignment: &FluxonFsTransferScanAssignmentWire, ) -> Result, FlatDict> { - let dir_abs = safe_join_root(root_dir_abs, assignment.root_relpath.as_str()).map_err(resp_err_kverr)?; + let dir_abs = + safe_join_root(root_dir_abs, assignment.root_relpath.as_str()).map_err(resp_err_kverr)?; let read_dir = match retry_after_target_path_chmod( dir_abs.as_path(), "root_read_dir", @@ -458,7 +449,10 @@ fn take_transfer_root_dir_listing_session( let now_unix_ms = chrono::Utc::now().timestamp_millis(); let mut state = transfer_scan_session_state().lock(); cleanup_expired_transfer_scan_sessions(&mut state, now_unix_ms); - if let Some(mut session) = state.root_dir_listing_sessions.remove(assignment.scan_unit_id.as_str()) { + if let Some(mut session) = state + .root_dir_listing_sessions + .remove(assignment.scan_unit_id.as_str()) + { if session.job_id == assignment.job_id && session.scan_epoch == assignment.scan_epoch && session.root_relpath == assignment.root_relpath @@ -507,7 +501,8 @@ fn open_transfer_subtree_streaming_session( if is_relpath_skipped(&assignment.skip_entries, assignment.root_relpath.as_str()) { return Ok(None); } - let dir_abs = safe_join_root(root_dir_abs, assignment.root_relpath.as_str()).map_err(resp_err_kverr)?; + let dir_abs = + safe_join_root(root_dir_abs, assignment.root_relpath.as_str()).map_err(resp_err_kverr)?; let root_md = retry_after_target_path_chmod( Path::new(root_dir_abs), "subtree_stream_root_symlink_metadata", @@ -790,7 +785,8 @@ fn collect_transfer_root_dir_listing_slice( assignment: &FluxonFsTransferScanAssignmentWire, deadline: Option, ) -> Result { - let Some(mut session) = take_transfer_root_dir_listing_session(root_dir_abs, assignment)? else { + let Some(mut session) = take_transfer_root_dir_listing_session(root_dir_abs, assignment)? + else { return Ok(TransferRootDirListingOutcome::Finished( build_finished_empty_transfer_scan_result(assignment), )); @@ -848,7 +844,8 @@ fn collect_transfer_root_dir_listing_slice( }; scanned_entries = scanned_entries.saturating_add(1); let name = ent.file_name().to_string_lossy().to_string(); - let child_relpath = normalize_child_relpath(assignment.root_relpath.as_str(), name.as_str()); + let child_relpath = + normalize_child_relpath(assignment.root_relpath.as_str(), name.as_str()); if is_relpath_skipped(&assignment.skip_entries, child_relpath.as_str()) { continue; } @@ -899,10 +896,12 @@ fn collect_transfer_root_dir_listing_slice( let size = md.len().min(i64::MAX as u64) as i64; session.root_visible_entries = true; session.root_total_bytes = session.root_total_bytes.saturating_add(size); - session.pending_direct_files.push(FluxonFsTransferScanFrontierEntry { - relpath: child_relpath, - size, - }); + session + .pending_direct_files + .push(FluxonFsTransferScanFrontierEntry { + relpath: child_relpath, + size, + }); session.pending_direct_bytes = session.pending_direct_bytes.saturating_add(size); if should_flush_direct_batch( assignment.batch_ready_bytes, @@ -910,7 +909,9 @@ fn collect_transfer_root_dir_listing_slice( session.pending_direct_files.len(), session.pending_direct_empty_dirs.len(), ) { - if let Some(batch) = flush_pending_root_direct_files_batch(assignment, &mut session)? { + if let Some(batch) = + flush_pending_root_direct_files_batch(assignment, &mut session)? + { direct_files_only_batches.push(batch); } } @@ -933,14 +934,18 @@ fn collect_transfer_root_dir_listing_slice( session.pending_direct_files.len(), session.pending_direct_empty_dirs.len(), ) { - if let Some(batch) = flush_pending_root_direct_files_batch(assignment, &mut session)? { + if let Some(batch) = + flush_pending_root_direct_files_batch(assignment, &mut session)? + { direct_files_only_batches.push(batch); } } } else { - session.direct_dirs.push(FluxonFsTransferScanFrontierDirEntry { - relpath: child_relpath, - }); + session + .direct_dirs + .push(FluxonFsTransferScanFrontierDirEntry { + relpath: child_relpath, + }); } } } @@ -1244,12 +1249,14 @@ impl TransferWorkerProgressWindow { fn record_written_bytes_and_maybe_ramp(&self, bytes: i64, now_unix_ms: i64) { let normalized = bytes.max(0); self.window_bytes.fetch_add(normalized, Ordering::SeqCst); - self.total_written_bytes.fetch_add(normalized, Ordering::SeqCst); + self.total_written_bytes + .fetch_add(normalized, Ordering::SeqCst); self.maybe_ramp(now_unix_ms); } fn record_materialized_empty_dir(&self) { - self.total_materialized_empty_dirs.fetch_add(1, Ordering::SeqCst); + self.total_materialized_empty_dirs + .fetch_add(1, Ordering::SeqCst); } fn total_materialized_empty_dirs(&self) -> i64 { @@ -1298,8 +1305,9 @@ impl TransferWorkerProgressWindow { } if previous_goodput > 0 { let delta = current_goodput.saturating_sub(previous_goodput); - let improvement_percent = - delta.saturating_mul(100).saturating_div(previous_goodput.max(1)); + let improvement_percent = delta + .saturating_mul(100) + .saturating_div(previous_goodput.max(1)); if improvement_percent < self.policy.min_improvement_percent { return; } @@ -1335,10 +1343,8 @@ impl TransferWorkerProgressWindow { .saturating_mul(1000) .saturating_div(window_elapsed_ms.max(1)) }; - self.peak_sample_goodput_bytes_per_sec.fetch_max( - window_goodput_bytes_per_sec.max(0), - Ordering::SeqCst, - ); + self.peak_sample_goodput_bytes_per_sec + .fetch_max(window_goodput_bytes_per_sec.max(0), Ordering::SeqCst); Some(TransferWorkerThroughputSample { window_started_unix_ms, window_elapsed_ms, @@ -1448,7 +1454,8 @@ impl TransferReadStreamActorOwned { data: Vec::new(), }); } - self.fill_prefetch_queue().map_err(|err| self.cache_terminal_error(err))?; + self.fill_prefetch_queue() + .map_err(|err| self.cache_terminal_error(err))?; let to_take = std::cmp::min(length as usize, (self.file_size - next_offset) as usize); let buf = self .take_prefetched_bytes(to_take) @@ -1457,7 +1464,8 @@ impl TransferReadStreamActorOwned { self.replay_offset = next_offset; self.replay_data = buf.clone(); self.next_offset = next_offset.saturating_add(buf.len() as i64); - self.fill_prefetch_queue().map_err(|err| self.cache_terminal_error(err))?; + self.fill_prefetch_queue() + .map_err(|err| self.cache_terminal_error(err))?; Ok(FluxonFsTransferReadStreamNextResultWire { stream_missing: false, data: buf, @@ -1598,7 +1606,11 @@ impl TransferReadStreamActorHandle { struct TransferWorkerCoordinator where - ReadChunkFn: Fn(&FluxonFsTransferManifestEntryWire, i64, i64) -> Result, TransferWorkerExecutionError>, + ReadChunkFn: Fn( + &FluxonFsTransferManifestEntryWire, + i64, + i64, + ) -> Result, TransferWorkerExecutionError>, CheckpointFn: Fn() -> Result<(), TransferWorkerExecutionError>, { log_context: TransferWorkerLogContext, @@ -1611,7 +1623,11 @@ where impl TransferWorkerCoordinator where - ReadChunkFn: Fn(&FluxonFsTransferManifestEntryWire, i64, i64) -> Result, TransferWorkerExecutionError>, + ReadChunkFn: Fn( + &FluxonFsTransferManifestEntryWire, + i64, + i64, + ) -> Result, TransferWorkerExecutionError>, CheckpointFn: Fn() -> Result<(), TransferWorkerExecutionError>, { fn new( @@ -1693,7 +1709,8 @@ where } fn progress_snapshot(&self) -> TransferWorkerProgressSnapshot { - self.progress.snapshot(chrono::Utc::now().timestamp_millis()) + self.progress + .snapshot(chrono::Utc::now().timestamp_millis()) } fn stop(&self) { @@ -1737,7 +1754,8 @@ impl TransferReadStreamRegistryHandle { }); } } - let full_path = safe_join_root(root_dir_abs, open.relpath.as_str()).map_err(resp_err_kverr)?; + let full_path = + safe_join_root(root_dir_abs, open.relpath.as_str()).map_err(resp_err_kverr)?; let file = open_file_with_target_path_chmod_retry(&full_path, "open_stream")?; let md = file.metadata().map_err(resp_err_io)?; let file_size = md.len().min(i64::MAX as u64) as i64; @@ -1764,12 +1782,15 @@ impl TransferReadStreamRegistryHandle { }); } state.streams.insert(stream_id.clone(), entry); - state.dedup_by_worker_file.insert(dedup_key, stream_id.clone()); + state + .dedup_by_worker_file + .insert(dedup_key, stream_id.clone()); drop(state); if let Err(resp) = TransferReadStreamActorHandle::start(stream_id.as_str(), actor) { let mut state = self.state.lock(); state.streams.remove(stream_id.as_str()); - state.dedup_by_worker_file + state + .dedup_by_worker_file .retain(|_, existing_stream_id| existing_stream_id != &stream_id); return Err(resp); } @@ -1811,7 +1832,9 @@ impl TransferReadStreamRegistryHandle { let Some(entry) = state.streams.remove(stream_id) else { return; }; - state.dedup_by_worker_file.retain(|_, existing_stream_id| existing_stream_id != stream_id); + state + .dedup_by_worker_file + .retain(|_, existing_stream_id| existing_stream_id != stream_id); entry.close(); } } @@ -1894,20 +1917,22 @@ fn decode_transfer_stream_open_result_payload( return Err(TransferWorkerRpcFailure::Fatal(resp.clone())); } Ok(FluxonFsTransferReadStreamOpenResultWire { - stream_id: require_str(resp, "stream_id").map_err(resp_err_kverr).map_err( - |err| { + stream_id: require_str(resp, "stream_id") + .map_err(resp_err_kverr) + .map_err(|err| { invalid_transfer_rpc_response(format!( "transfer read stream open response missing stream_id: err={}", transfer_rpc_response_err_text(&err) )) - }, - )?, - size: require_i64(resp, "size").map_err(resp_err_kverr).map_err(|err| { - invalid_transfer_rpc_response(format!( - "transfer read stream open response missing size: err={}", - transfer_rpc_response_err_text(&err) - )) - })?, + })?, + size: require_i64(resp, "size") + .map_err(resp_err_kverr) + .map_err(|err| { + invalid_transfer_rpc_response(format!( + "transfer read stream open response missing size: err={}", + transfer_rpc_response_err_text(&err) + )) + })?, }) } @@ -1980,11 +2005,15 @@ fn is_relpath_skipped(skip_entries: &[FluxonFsTransferSkipEntryWire], relpath: & } fn file_name_from_relpath(relpath: &str) -> Result<&str, FlatDict> { - relpath.rsplit('/').next().filter(|v| !v.is_empty()).ok_or_else(|| { - resp_err_kverr(KvError::Api(ApiError::InvalidArgument { - detail: format!("relpath must contain file name: {}", relpath), - })) - }) + relpath + .rsplit('/') + .next() + .filter(|v| !v.is_empty()) + .ok_or_else(|| { + resp_err_kverr(KvError::Api(ApiError::InvalidArgument { + detail: format!("relpath must contain file name: {}", relpath), + })) + }) } fn transfer_staging_dir_for_file(staging_prefix: &str, relpath: &str) -> String { @@ -2106,8 +2135,12 @@ where match attempt() { Ok(value) => Ok(value), Err(initial_err) if initial_err.kind() == ErrorKind::PermissionDenied => { - let repair_dir = - repair_permission_denied_dir_for_retry(repair_anchor, op, target_path, &initial_err)?; + let repair_dir = repair_permission_denied_dir_for_retry( + repair_anchor, + op, + target_path, + &initial_err, + )?; attempt().map_err(|retry_err| { resp_err_kverr(KvError::Api(ApiError::Unknown { detail: format!( @@ -2334,10 +2367,11 @@ fn collect_transfer_tree_with_deadline( continue; } }; - out.symlink_notices.push(FluxonFsTransferSymlinkNoticeEntryWire { - relpath: child_rel, - link_target: link_target.to_string_lossy().to_string(), - }); + out.symlink_notices + .push(FluxonFsTransferSymlinkNoticeEntryWire { + relpath: child_rel, + link_target: link_target.to_string_lossy().to_string(), + }); continue; } if md.is_dir() { @@ -2361,7 +2395,8 @@ fn collect_transfer_tree_with_deadline( } out.files.sort_by(|a, b| a.relpath.cmp(&b.relpath)); out.empty_dirs.sort(); - out.symlink_notices.sort_by(|a, b| a.relpath.cmp(&b.relpath)); + out.symlink_notices + .sort_by(|a, b| a.relpath.cmp(&b.relpath)); Ok(out) } @@ -2574,10 +2609,8 @@ fn build_transfer_scan_events_for_result( event_seq_no_start: i64, result: FluxonFsTransferScanResultWire, ) -> (Vec, bool, i64) { - let (child_scan_units, continue_locally) = split_same_root_continuation_from_child_scan_units( - assignment, - result.child_scan_units, - ); + let (child_scan_units, continue_locally) = + split_same_root_continuation_from_child_scan_units(assignment, result.child_scan_units); if continue_locally { let event = build_transfer_scan_event( assignment, @@ -2588,11 +2621,7 @@ fn build_transfer_scan_events_for_result( result.full_dir_batches, String::new(), ); - return ( - vec![event], - true, - event_seq_no_start.saturating_add(1), - ); + return (vec![event], true, event_seq_no_start.saturating_add(1)); } let mut next_event_seq_no = event_seq_no_start; let mut events = Vec::new(); @@ -2620,11 +2649,7 @@ fn build_transfer_scan_events_for_result( Vec::new(), String::new(), )); - ( - events, - false, - next_event_seq_no.saturating_add(1), - ) + (events, false, next_event_seq_no.saturating_add(1)) } fn send_transfer_scan_event_once( @@ -2637,10 +2662,7 @@ fn send_transfer_scan_event_once( } let event_json = serde_json::to_string(event) .map_err(|e| format!("serialize transfer scan event failed: {}", e))?; - let payload = FlatDict::from([( - "scan_event_json".to_string(), - FlatValue::String(event_json), - )]); + let payload = FlatDict::from([("scan_event_json".to_string(), FlatValue::String(event_json))]); let resp = api .rpc_client() .call( @@ -2793,30 +2815,28 @@ fn run_transfer_scan_background_task( } let mut next_event_seq_no = 1_i64; loop { - let result = match build_transfer_scan_result_for_root_dir_abs( - root_dir_abs.as_str(), - &assignment, - ) { - Ok(v) => v, - Err(resp) => { - let failed = build_transfer_scan_event( - &assignment, - next_event_seq_no, - FluxonFsTransferScanEventKindWire::Failed, - Vec::new(), - Vec::new(), - Vec::new(), - transfer_rpc_response_err_text(&resp), - ); - let _ = send_transfer_scan_event_with_retry( - api.as_ref(), - master_id.as_str(), - &mut assignment, - &failed, - ); - break; - } - }; + let result = + match build_transfer_scan_result_for_root_dir_abs(root_dir_abs.as_str(), &assignment) { + Ok(v) => v, + Err(resp) => { + let failed = build_transfer_scan_event( + &assignment, + next_event_seq_no, + FluxonFsTransferScanEventKindWire::Failed, + Vec::new(), + Vec::new(), + Vec::new(), + transfer_rpc_response_err_text(&resp), + ); + let _ = send_transfer_scan_event_with_retry( + api.as_ref(), + master_id.as_str(), + &mut assignment, + &failed, + ); + break; + } + }; let (events, continue_locally, next_seq_no_after_events) = build_transfer_scan_events_for_result(&assignment, next_event_seq_no, result); next_event_seq_no = next_seq_no_after_events; @@ -2922,17 +2942,14 @@ impl TransferScanRegistryHandle { let assignment2 = assignment.clone(); let thread_name = format!("fluxon_fs_transfer_scan_{}", assignment.scan_task_id); match thread::Builder::new().name(thread_name).spawn(move || { - run_transfer_scan_background_task( - registry, - api2, - master_id2, - exports2, - assignment2, - ); + run_transfer_scan_background_task(registry, api2, master_id2, exports2, assignment2); }) { Ok(_) => Ok(FluxonFsTransferScanLaunchResultWire::started()), Err(err) => { - self.state.lock().tasks.remove(assignment.scan_task_id.as_str()); + self.state + .lock() + .tasks + .remove(assignment.scan_task_id.as_str()); Err(resp_err_kverr(KvError::Api(ApiError::Unknown { detail: format!( "spawn transfer scan thread failed: scan_task_id={} err={}", @@ -3006,22 +3023,16 @@ impl TransferWorkerRegistryHandle { let master_id2 = master_id.to_string(); let exports2 = exports.clone(); let assignment2 = assignment.clone(); - let thread_name = format!( - "fluxon_fs_transfer_worker_{}", - assignment.worker_task_id - ); + let thread_name = format!("fluxon_fs_transfer_worker_{}", assignment.worker_task_id); match thread::Builder::new().name(thread_name).spawn(move || { - run_transfer_worker_background_task( - registry, - api2, - master_id2, - exports2, - assignment2, - ); + run_transfer_worker_background_task(registry, api2, master_id2, exports2, assignment2); }) { Ok(_) => Ok(FluxonFsTransferWorkerLaunchResultWire::started()), Err(err) => { - self.state.lock().tasks.remove(assignment.worker_task_id.as_str()); + self.state + .lock() + .tasks + .remove(assignment.worker_task_id.as_str()); Err(resp_err_kverr(KvError::Api(ApiError::Unknown { detail: format!( "spawn transfer worker thread failed: worker_task_id={} err={}", @@ -3298,10 +3309,7 @@ fn send_transfer_worker_result_once( detail: format!("serialize transfer worker result failed: {}", e), }))) })?; - let payload = FlatDict::from([( - "result_json".to_string(), - FlatValue::String(result_json), - )]); + let payload = FlatDict::from([("result_json".to_string(), FlatValue::String(result_json))]); let resp = api .rpc_client() .call( @@ -3396,7 +3404,10 @@ fn open_transfer_read_stream_via_rpc_once( "relpath".to_string(), FlatValue::String(file.relpath.clone()), ), - ("initial_offset".to_string(), FlatValue::Int64(initial_offset)), + ( + "initial_offset".to_string(), + FlatValue::Int64(initial_offset), + ), ]); let resp = api .rpc_client() @@ -3518,25 +3529,25 @@ impl TransferWorkerRemoteControl { loop { self.before_heartbeat_retry_attempt()?; let current_materialized_empty_dirs = self.progress.total_materialized_empty_dirs(); - match self - .heartbeat - .ensure_continue( - force, - current_materialized_empty_dirs, - |heartbeat_unix_ms, _heartbeat_detail| { - let progress_snapshot = - self.progress.snapshot(chrono::Utc::now().timestamp_millis()); - let telemetry = - Some(transfer_worker_telemetry_from_progress_snapshot(&progress_snapshot)); - send_transfer_worker_heartbeat_once( - self.api.as_ref(), - self.master_id.as_str(), - &self.assignment, - heartbeat_unix_ms, - telemetry, - ) - }, - ) { + match self.heartbeat.ensure_continue( + force, + current_materialized_empty_dirs, + |heartbeat_unix_ms, _heartbeat_detail| { + let progress_snapshot = self + .progress + .snapshot(chrono::Utc::now().timestamp_millis()); + let telemetry = Some(transfer_worker_telemetry_from_progress_snapshot( + &progress_snapshot, + )); + send_transfer_worker_heartbeat_once( + self.api.as_ref(), + self.master_id.as_str(), + &self.assignment, + heartbeat_unix_ms, + telemetry, + ) + }, + ) { Ok(()) => return Ok(()), Err(TransferWorkerHeartbeatGateError::Terminal(err)) => return Err(err), Err(TransferWorkerHeartbeatGateError::Retryable { @@ -3631,10 +3642,7 @@ impl TransferWorkerRemoteControl { ) } - fn close_stream_with_retry( - &self, - stream_id: &str, - ) -> Result<(), TransferWorkerExecutionError> { + fn close_stream_with_retry(&self, stream_id: &str) -> Result<(), TransferWorkerExecutionError> { let api = self.api.clone(); let assignment = self.assignment.clone(); let op_detail = format!( @@ -3744,9 +3752,9 @@ impl TransferWorkerRemoteControl { if ack.accepted { return Ok(()); } - Err(TransferWorkerExecutionError::Stop(stop_reason_or_superseded( - ack.stop_reason, - ))) + Err(TransferWorkerExecutionError::Stop( + stop_reason_or_superseded(ack.stop_reason), + )) } } @@ -3825,10 +3833,12 @@ impl TransferWorkerHeartbeatGate { mut heartbeat_op: HeartbeatOp, ) -> Result<(), TransferWorkerHeartbeatGateError> where - HeartbeatOp: FnMut( - i64, - &'static str, - ) -> Result, + HeartbeatOp: + FnMut( + i64, + &'static str, + ) + -> Result, { loop { let (heartbeat_unix_ms, heartbeat_detail) = { @@ -3875,15 +3885,13 @@ impl TransferWorkerHeartbeatGate { state.heartbeat_inflight = false; let result = match heartbeat_result { Ok(heartbeat_result) if heartbeat_result.continue_running => { - state.last_heartbeat_completed_unix_ms = - chrono::Utc::now().timestamp_millis(); + state.last_heartbeat_completed_unix_ms = chrono::Utc::now().timestamp_millis(); state.last_heartbeat_materialized_empty_dirs = current_materialized_empty_dirs; state.granted_lease_expire_unix_ms = heartbeat_result.lease_expire_unix_ms; Ok(()) } Ok(heartbeat_result) => { - state.last_heartbeat_completed_unix_ms = - chrono::Utc::now().timestamp_millis(); + state.last_heartbeat_completed_unix_ms = chrono::Utc::now().timestamp_millis(); state.last_heartbeat_materialized_empty_dirs = current_materialized_empty_dirs; let reason = stop_reason_or_superseded(heartbeat_result.stop_reason); state.terminal_state = @@ -3933,7 +3941,8 @@ fn run_transfer_worker_background_task( )); let dedup_expire_unix_ms = match control.ensure_continue(true) { Ok(()) => { - let dst_export_root = match exports.export_root_dir_abs(assignment.dst_export.as_str()) { + let dst_export_root = match exports.export_root_dir_abs(assignment.dst_export.as_str()) + { Ok(v) => v, Err(err) => { tracing::warn!( @@ -3996,7 +4005,9 @@ fn run_transfer_worker_background_task( }, { let control = control.clone(); - move |file, read_offset, length| control.read_chunk_with_retry(file, read_offset, length) + move |file, read_offset, length| { + control.read_chunk_with_retry(file, read_offset, length) + } }, ) { Ok(result) => { @@ -4004,34 +4015,38 @@ fn run_transfer_worker_background_task( if let Err(resp) = cleanup_transfer_worker_attempt_artifacts(&dst_root, &assignment) { - log_transfer_worker_cleanup_failure("before_result_submit", &assignment, &resp); - } - match control.submit_result_with_retry(&result) { - Ok(()) => control.dedup_expire_unix_ms(), - Err(TransferWorkerExecutionError::Stop(reason)) => { - tracing::info!( - "transfer worker result submission stopped: job_id={} batch_id={} worker_id={} worker_task_id={} reason={:?}", - assignment.job_id, - assignment.batch_id, - assignment.worker_id, - assignment.worker_task_id, - reason + log_transfer_worker_cleanup_failure( + "before_result_submit", + &assignment, + &resp, ); - control.dedup_expire_unix_ms() } - Err(TransferWorkerExecutionError::Fatal(resp)) => { - tracing::warn!( - "transfer worker result submission failed: job_id={} batch_id={} worker_id={} worker_task_id={} resp={:?}", - assignment.job_id, - assignment.batch_id, - assignment.worker_id, - assignment.worker_task_id, - resp - ); - control.dedup_expire_unix_ms() + match control.submit_result_with_retry(&result) { + Ok(()) => control.dedup_expire_unix_ms(), + Err(TransferWorkerExecutionError::Stop(reason)) => { + tracing::info!( + "transfer worker result submission stopped: job_id={} batch_id={} worker_id={} worker_task_id={} reason={:?}", + assignment.job_id, + assignment.batch_id, + assignment.worker_id, + assignment.worker_task_id, + reason + ); + control.dedup_expire_unix_ms() + } + Err(TransferWorkerExecutionError::Fatal(resp)) => { + tracing::warn!( + "transfer worker result submission failed: job_id={} batch_id={} worker_id={} worker_task_id={} resp={:?}", + assignment.job_id, + assignment.batch_id, + assignment.worker_id, + assignment.worker_task_id, + resp + ); + control.dedup_expire_unix_ms() + } } } - } Err(TransferWorkerExecutionError::Stop(reason)) => { control.close_all_streams(); if let Err(resp) = @@ -4054,10 +4069,13 @@ fn run_transfer_worker_background_task( if let Err(cleanup_resp) = cleanup_transfer_worker_attempt_artifacts(&dst_root, &assignment) { - log_transfer_worker_cleanup_failure("after_fatal", &assignment, &cleanup_resp); + log_transfer_worker_cleanup_failure( + "after_fatal", + &assignment, + &cleanup_resp, + ); } - if let Some((fatal_kind, fatal_message)) = - classify_transfer_worker_fatal(&resp) + if let Some((fatal_kind, fatal_message)) = classify_transfer_worker_fatal(&resp) { match report_transfer_worker_fatal_once( control.api.as_ref(), @@ -4107,15 +4125,25 @@ fn run_transfer_worker_background_task( } } Err(TransferWorkerExecutionError::Stop(reason)) => { - let dst_export_root = exports.export_root_dir_abs(assignment.dst_export.as_str()).ok(); + let dst_export_root = exports + .export_root_dir_abs(assignment.dst_export.as_str()) + .ok(); let dst_root = dst_export_root.and_then(|dst_export_root| { - safe_join_root(dst_export_root.as_str(), assignment.dst_root_relpath.as_str()) - .ok() - .map(PathBuf::from) + safe_join_root( + dst_export_root.as_str(), + assignment.dst_root_relpath.as_str(), + ) + .ok() + .map(PathBuf::from) }); if let Some(dst_root) = dst_root { - if let Err(resp) = cleanup_transfer_worker_attempt_artifacts(&dst_root, &assignment) { - log_transfer_worker_cleanup_failure("before_execution_stop", &assignment, &resp); + if let Err(resp) = cleanup_transfer_worker_attempt_artifacts(&dst_root, &assignment) + { + log_transfer_worker_cleanup_failure( + "before_execution_stop", + &assignment, + &resp, + ); } } tracing::info!( @@ -4129,11 +4157,16 @@ fn run_transfer_worker_background_task( control.dedup_expire_unix_ms() } Err(TransferWorkerExecutionError::Fatal(resp)) => { - let dst_export_root = exports.export_root_dir_abs(assignment.dst_export.as_str()).ok(); + let dst_export_root = exports + .export_root_dir_abs(assignment.dst_export.as_str()) + .ok(); let dst_root = dst_export_root.and_then(|dst_export_root| { - safe_join_root(dst_export_root.as_str(), assignment.dst_root_relpath.as_str()) - .ok() - .map(PathBuf::from) + safe_join_root( + dst_export_root.as_str(), + assignment.dst_root_relpath.as_str(), + ) + .ok() + .map(PathBuf::from) }); if let Some(dst_root) = dst_root { if let Err(cleanup_resp) = @@ -4509,7 +4542,9 @@ fn plan_transfer_subtree_batches( total_bytes: 0, root_is_empty: true, mergeable_empty_dir_count: 1, - mergeable_empty_dir_estimated_bytes: estimate_empty_dir_manifest_entry_bytes(root_relpath), + mergeable_empty_dir_estimated_bytes: estimate_empty_dir_manifest_entry_bytes( + root_relpath, + ), direct_files_only_batches: Vec::new(), full_dir_batches: Vec::new(), child_scan_units: Vec::new(), @@ -4540,8 +4575,7 @@ fn plan_transfer_subtree_batches( let child_empty_dir_count = child_plan.mergeable_empty_dir_count; let child_empty_dir_estimated_bytes = child_plan.mergeable_empty_dir_estimated_bytes; - if mergeable_empty_dir_count - .saturating_add(child_empty_dir_count) + if mergeable_empty_dir_count.saturating_add(child_empty_dir_count) > TRANSFER_MERGEABLE_EMPTY_DIR_BUDGET || mergeable_empty_dir_estimated_bytes .saturating_add(child_empty_dir_estimated_bytes) @@ -4716,7 +4750,11 @@ fn build_root_direct_files_only_batch_from_entries( } fn sort_transfer_scan_batches(batches: &mut [FluxonFsTransferScanBatchWire]) { - batches.sort_by(|a, b| a.root_relpath.cmp(&b.root_relpath).then(a.batch_id.cmp(&b.batch_id))); + batches.sort_by(|a, b| { + a.root_relpath + .cmp(&b.root_relpath) + .then(a.batch_id.cmp(&b.batch_id)) + }); } fn build_full_dir_batch_for_mergeable_subtree( @@ -4750,14 +4788,12 @@ fn build_transfer_scan_result_for_subtree_streaming_root_dir_abs( root_dir_abs: &str, assignment: &FluxonFsTransferScanAssignmentWire, ) -> Result { - let Some(mut session) = take_transfer_subtree_streaming_session(root_dir_abs, assignment)? else { + let Some(mut session) = take_transfer_subtree_streaming_session(root_dir_abs, assignment)? + else { return Ok(build_finished_empty_subtree_stream_result(assignment)); }; loop { - if session - .dir_stack - .is_empty() - { + if session.dir_stack.is_empty() { let mut full_dir_batches = Vec::new(); if let Some(batch) = flush_pending_subtree_stream_batch(assignment, &mut session)? { full_dir_batches.push(batch); @@ -4776,7 +4812,9 @@ fn build_transfer_scan_result_for_subtree_streaming_root_dir_abs( finished: true, }); } - if TransferScanDeadline::from_assignment(assignment).is_some_and(|deadline| deadline.reached()) { + if TransferScanDeadline::from_assignment(assignment) + .is_some_and(|deadline| deadline.reached()) + { let mut full_dir_batches = Vec::new(); if let Some(batch) = flush_pending_subtree_stream_batch(assignment, &mut session)? { full_dir_batches.push(batch); @@ -4808,7 +4846,10 @@ fn build_transfer_scan_result_for_subtree_streaming_root_dir_abs( if should_flush_subtree_stream_batch( assignment.batch_ready_bytes, session.pending_bytes, - session.pending_files.len().saturating_add(session.pending_symlink_notices.len()), + session + .pending_files + .len() + .saturating_add(session.pending_symlink_notices.len()), session.pending_empty_dirs.len(), ) { let batch = flush_pending_subtree_stream_batch(assignment, &mut session)?.unwrap(); @@ -4865,22 +4906,30 @@ fn build_transfer_scan_result_for_subtree_streaming_root_dir_abs( }); } else if md.is_dir() { frame.saw_visible_child = true; - session.dir_stack.push(open_transfer_subtree_streaming_dir_frame( - child_path, - child_relpath, - )?); + session + .dir_stack + .push(open_transfer_subtree_streaming_dir_frame( + child_path, + child_relpath, + )?); } else if md.is_file() { frame.saw_visible_child = true; let size = md.len().min(i64::MAX as u64) as i64; session.pending_bytes = session.pending_bytes.saturating_add(size); session .pending_files - .push(FluxonFsTransferScanFrontierEntry { relpath: child_relpath, size }); + .push(FluxonFsTransferScanFrontierEntry { + relpath: child_relpath, + size, + }); } if should_flush_subtree_stream_batch( assignment.batch_ready_bytes, session.pending_bytes, - session.pending_files.len().saturating_add(session.pending_symlink_notices.len()), + session + .pending_files + .len() + .saturating_add(session.pending_symlink_notices.len()), session.pending_empty_dirs.len(), ) { let batch = flush_pending_subtree_stream_batch(assignment, &mut session)?.unwrap(); @@ -4928,11 +4977,12 @@ pub(crate) fn build_transfer_scan_result_for_root_dir_abs( ); } let deadline = TransferScanDeadline::from_assignment(assignment); - let root_listing = match collect_transfer_root_dir_listing_slice(root_dir_abs, assignment, deadline)? { - TransferRootDirListingOutcome::Complete(v) => v, - TransferRootDirListingOutcome::Finished(result) => return Ok(result), - TransferRootDirListingOutcome::Partial(result) => return Ok(result), - }; + let root_listing = + match collect_transfer_root_dir_listing_slice(root_dir_abs, assignment, deadline)? { + TransferRootDirListingOutcome::Complete(v) => v, + TransferRootDirListingOutcome::Finished(result) => return Ok(result), + TransferRootDirListingOutcome::Partial(result) => return Ok(result), + }; let mut direct_files = root_listing.direct_files; let mut direct_symlink_notices = root_listing.direct_symlink_notices; let mut direct_empty_dirs = root_listing.direct_empty_dirs; @@ -4976,7 +5026,10 @@ pub(crate) fn build_transfer_scan_result_for_root_dir_abs( if (!direct_files.is_empty() || !direct_symlink_notices.is_empty() || !direct_empty_dirs.is_empty()) - && !direct_files_only_disposition_covers_root(assignment, assignment.root_relpath.as_str()) + && !direct_files_only_disposition_covers_root( + assignment, + assignment.root_relpath.as_str(), + ) { let mut next_partition_index = root_listing.emitted_direct_files_batch_count; direct_files_only_batches.extend(build_partitioned_root_direct_files_only_batches( @@ -4987,17 +5040,15 @@ pub(crate) fn build_transfer_scan_result_for_root_dir_abs( direct_empty_dirs.clone(), )?); } - child_scan_units.extend( - direct_dirs[delegated_child_scan_unit_count..] - .iter() - .map(|entry| { - new_child_scan_unit( - entry.relpath.clone(), - assignment.generation + 1, - delegated_child_scan_mode(), - ) - }), - ); + child_scan_units.extend(direct_dirs[delegated_child_scan_unit_count..].iter().map( + |entry| { + new_child_scan_unit( + entry.relpath.clone(), + assignment.generation + 1, + delegated_child_scan_mode(), + ) + }, + )); child_scan_units.sort_by(|a, b| a.root_relpath.cmp(&b.root_relpath)); sort_transfer_scan_batches(&mut direct_files_only_batches); return Ok(FluxonFsTransferScanResultWire { @@ -5027,7 +5078,8 @@ pub(crate) fn build_transfer_scan_result_for_root_dir_abs( let mut root_partitioned = root_listing.emitted_direct_files_batch_count > 0 || direct_files_only_disposition_covers_root(assignment, assignment.root_relpath.as_str()); let mut mergeable_empty_dir_count = direct_empty_dirs.len(); - let mut mergeable_empty_dir_estimated_bytes = estimate_empty_dir_manifest_bytes(&direct_empty_dirs); + let mut mergeable_empty_dir_estimated_bytes = + estimate_empty_dir_manifest_bytes(&direct_empty_dirs); for child_relpath in direct_dirs.iter().map(|entry| entry.relpath.clone()) { let child_plan = plan_transfer_subtree_batches( root_dir_abs, @@ -5043,8 +5095,7 @@ pub(crate) fn build_transfer_scan_result_for_root_dir_abs( let child_empty_dir_count = child_plan.mergeable_empty_dir_count; let child_empty_dir_estimated_bytes = child_plan.mergeable_empty_dir_estimated_bytes; - if mergeable_empty_dir_count - .saturating_add(child_empty_dir_count) + if mergeable_empty_dir_count.saturating_add(child_empty_dir_count) > TRANSFER_MERGEABLE_EMPTY_DIR_BUDGET || mergeable_empty_dir_estimated_bytes .saturating_add(child_empty_dir_estimated_bytes) @@ -5083,7 +5134,10 @@ pub(crate) fn build_transfer_scan_result_for_root_dir_abs( if (!direct_files.is_empty() || !direct_symlink_notices.is_empty() || !mergeable_empty_child_relpaths.is_empty()) - && !direct_files_only_disposition_covers_root(assignment, assignment.root_relpath.as_str()) + && !direct_files_only_disposition_covers_root( + assignment, + assignment.root_relpath.as_str(), + ) { let mut next_partition_index = root_listing.emitted_direct_files_batch_count; direct_empty_dirs.extend(mergeable_empty_child_relpaths); @@ -5212,10 +5266,11 @@ fn handle_transfer_scan_assignment( assignment.generation, assignment.known_dispositions.len(), ); - let result = match build_transfer_scan_result_for_root_dir_abs(root_dir_abs.as_str(), &assignment) { - Ok(v) => v, - Err(resp) => return resp, - }; + let result = + match build_transfer_scan_result_for_root_dir_abs(root_dir_abs.as_str(), &assignment) { + Ok(v) => v, + Err(resp) => return resp, + }; encode_transfer_scan_result(&result, "transfer scan result") } @@ -5228,8 +5283,11 @@ fn prepare_transfer_file_streaming( coordinator: &TransferWorkerCoordinator, ) -> Result where - ReadChunkFn: - Fn(&FluxonFsTransferManifestEntryWire, i64, i64) -> Result, TransferWorkerExecutionError>, + ReadChunkFn: Fn( + &FluxonFsTransferManifestEntryWire, + i64, + i64, + ) -> Result, TransferWorkerExecutionError>, CheckpointFn: Fn() -> Result<(), TransferWorkerExecutionError>, { let staging_relpath = transfer_staging_file_relpath(staging_prefix, file.relpath.as_str()) @@ -5239,9 +5297,12 @@ where .map_err(TransferWorkerExecutionError::fatal)?; ensure_transfer_parent_dirs(dst_root, final_relpath.as_str()) .map_err(TransferWorkerExecutionError::fatal)?; - let staging_abs = safe_join_root(dst_root.to_string_lossy().as_ref(), staging_relpath.as_str()) - .map_err(resp_err_kverr) - .map_err(TransferWorkerExecutionError::fatal)?; + let staging_abs = safe_join_root( + dst_root.to_string_lossy().as_ref(), + staging_relpath.as_str(), + ) + .map_err(resp_err_kverr) + .map_err(TransferWorkerExecutionError::fatal)?; let mut dst_file = open_create_file_with_parent_dir_chmod_retry(&staging_abs) .map_err(TransferWorkerExecutionError::fatal)?; dst_file @@ -5254,14 +5315,14 @@ where let remaining = file.size.saturating_sub(copied); let chunk = coordinator.read_chunk(file, copied, remaining.min(CHUNK_BYTES as i64))?; if chunk.is_empty() { - return Err(TransferWorkerExecutionError::fatal(resp_err_kverr(KvError::Api( - ApiError::InvalidArgument { + return Err(TransferWorkerExecutionError::fatal(resp_err_kverr( + KvError::Api(ApiError::InvalidArgument { detail: format!( "transfer worker source ended before expected size: relpath={} expected={} copied={}", file.relpath, file.size, copied ), - }, - )))); + }), + ))); } dst_file .write_all(&chunk) @@ -5271,13 +5332,14 @@ where coordinator.record_written_bytes(chunk.len() as i64); } if copied != file.size { - return Err(TransferWorkerExecutionError::fatal(resp_err_kverr(KvError::Api( - ApiError::InvalidArgument { - detail: format!( - "transfer worker size mismatch before staging completion: relpath={} expected={} actual={}", - file.relpath, file.size, copied - ), - })))); + return Err(TransferWorkerExecutionError::fatal(resp_err_kverr( + KvError::Api(ApiError::InvalidArgument { + detail: format!( + "transfer worker size mismatch before staging completion: relpath={} expected={} actual={}", + file.relpath, file.size, copied + ), + }), + ))); } // The staged file is still invisible at this point, so one more checkpoint // keeps supersession able to stop the worker before any later visible @@ -5302,48 +5364,57 @@ fn execute_transfer_single_file( coordinator: &TransferWorkerCoordinator, ) -> Result where - ReadChunkFn: - Fn(&FluxonFsTransferManifestEntryWire, i64, i64) -> Result, TransferWorkerExecutionError>, + ReadChunkFn: Fn( + &FluxonFsTransferManifestEntryWire, + i64, + i64, + ) -> Result, TransferWorkerExecutionError>, CheckpointFn: Fn() -> Result<(), TransferWorkerExecutionError>, { coordinator.checkpoint_continue()?; - let prepared = match prepare_transfer_file_streaming(dst_root, staging_prefix, file, coordinator) { - Ok(v) => v, - Err(TransferWorkerExecutionError::Fatal(resp)) => { - if let Some(failed) = classify_transfer_failed_file(file, &resp) { - let staging_relpath = - transfer_staging_file_relpath(staging_prefix, file.relpath.as_str()) - .map_err(TransferWorkerExecutionError::fatal)?; - let staging_abs = safe_join_root( - dst_root.to_string_lossy().as_ref(), - staging_relpath.as_str(), - ) - .map_err(resp_err_kverr) - .map_err(TransferWorkerExecutionError::fatal)?; - match fs::remove_file(&staging_abs) { - Ok(()) => {} - Err(err) if err.kind() == ErrorKind::NotFound => {} - Err(err) => return Err(TransferWorkerExecutionError::fatal(resp_err_io(err))), + let prepared = + match prepare_transfer_file_streaming(dst_root, staging_prefix, file, coordinator) { + Ok(v) => v, + Err(TransferWorkerExecutionError::Fatal(resp)) => { + if let Some(failed) = classify_transfer_failed_file(file, &resp) { + let staging_relpath = + transfer_staging_file_relpath(staging_prefix, file.relpath.as_str()) + .map_err(TransferWorkerExecutionError::fatal)?; + let staging_abs = safe_join_root( + dst_root.to_string_lossy().as_ref(), + staging_relpath.as_str(), + ) + .map_err(resp_err_kverr) + .map_err(TransferWorkerExecutionError::fatal)?; + match fs::remove_file(&staging_abs) { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => { + return Err(TransferWorkerExecutionError::fatal(resp_err_io(err))); + } + } + return Ok(TransferWorkerLaneOutcome::Failed( + TransferWorkerLaneFailedFileResult { result: failed }, + )); } - return Ok(TransferWorkerLaneOutcome::Failed( - TransferWorkerLaneFailedFileResult { result: failed }, - )); + return Err(TransferWorkerExecutionError::Fatal(resp)); } - return Err(TransferWorkerExecutionError::Fatal(resp)); - } - Err(err) => return Err(err), - }; + Err(err) => return Err(err), + }; coordinator.checkpoint_continue()?; - let result = promote_prepared_transfer_file(dst_root, PreparedTransferFile { - staging_relpath: prepared.staging_relpath.clone(), - final_relpath: prepared.final_relpath.clone(), - visible_size: prepared.visible_size, - }) + let result = promote_prepared_transfer_file( + dst_root, + PreparedTransferFile { + staging_relpath: prepared.staging_relpath.clone(), + final_relpath: prepared.final_relpath.clone(), + visible_size: prepared.visible_size, + }, + ) .map_err(TransferWorkerExecutionError::fatal); match result { - Ok(result) => Ok(TransferWorkerLaneOutcome::Visible(TransferWorkerLaneFileResult { - result, - })), + Ok(result) => Ok(TransferWorkerLaneOutcome::Visible( + TransferWorkerLaneFileResult { result }, + )), Err(TransferWorkerExecutionError::Fatal(resp)) => { if let Some(failed) = classify_transfer_failed_file(file, &resp) { let staging_abs = safe_join_root( @@ -5374,8 +5445,11 @@ fn execute_transfer_empty_dir( coordinator: &TransferWorkerCoordinator, ) -> Result where - ReadChunkFn: - Fn(&FluxonFsTransferManifestEntryWire, i64, i64) -> Result, TransferWorkerExecutionError>, + ReadChunkFn: Fn( + &FluxonFsTransferManifestEntryWire, + i64, + i64, + ) -> Result, TransferWorkerExecutionError>, CheckpointFn: Fn() -> Result<(), TransferWorkerExecutionError>, { coordinator.checkpoint_continue()?; @@ -5393,11 +5467,14 @@ fn execute_transfer_worker_assignment_with_policy( read_chunk: ReadChunkFn, ) -> Result where - ReadChunkFn: - Fn(&FluxonFsTransferManifestEntryWire, i64, i64) -> Result, TransferWorkerExecutionError> - + Send - + Sync - + 'static, + ReadChunkFn: Fn( + &FluxonFsTransferManifestEntryWire, + i64, + i64, + ) -> Result, TransferWorkerExecutionError> + + Send + + Sync + + 'static, CheckpointFn: Fn() -> Result<(), TransferWorkerExecutionError> + Send + Sync + 'static, { let policy = policy.normalized(); @@ -5424,11 +5501,14 @@ fn execute_transfer_worker_assignment_with_policy_and_progress Result where - ReadChunkFn: - Fn(&FluxonFsTransferManifestEntryWire, i64, i64) -> Result, TransferWorkerExecutionError> - + Send - + Sync - + 'static, + ReadChunkFn: Fn( + &FluxonFsTransferManifestEntryWire, + i64, + i64, + ) -> Result, TransferWorkerExecutionError> + + Send + + Sync + + 'static, CheckpointFn: Fn() -> Result<(), TransferWorkerExecutionError> + Send + Sync + 'static, { create_dir_all_with_parent_dir_chmod_retry(dst_root) @@ -5436,9 +5516,11 @@ where let manifest = FluxonFsTransferManifestWire::decode_from_blob(assignment.manifest_blob.as_slice()) .map_err(|e| { - TransferWorkerExecutionError::fatal(resp_err_kverr(KvError::Api(ApiError::InvalidArgument { - detail: format!("decode transfer worker manifest failed: {}", e), - }))) + TransferWorkerExecutionError::fatal(resp_err_kverr(KvError::Api( + ApiError::InvalidArgument { + detail: format!("decode transfer worker manifest failed: {}", e), + }, + ))) })?; if transfer_manifest_is_empty_dirs_only_batch(&manifest, assignment.collect_infos.as_slice()) { // Empty-dir-only batches never generate byte-based ramp-up signals, so @@ -5664,10 +5746,16 @@ fn promote_prepared_transfer_file( dst_root: &PathBuf, file: PreparedTransferFile, ) -> Result { - let staging_abs = safe_join_root(dst_root.to_string_lossy().as_ref(), file.staging_relpath.as_str()) - .map_err(resp_err_kverr)?; - let final_abs = safe_join_root(dst_root.to_string_lossy().as_ref(), file.final_relpath.as_str()) - .map_err(resp_err_kverr)?; + let staging_abs = safe_join_root( + dst_root.to_string_lossy().as_ref(), + file.staging_relpath.as_str(), + ) + .map_err(resp_err_kverr)?; + let final_abs = safe_join_root( + dst_root.to_string_lossy().as_ref(), + file.final_relpath.as_str(), + ) + .map_err(resp_err_kverr)?; rename_with_dst_parent_dir_chmod_retry(&staging_abs, &final_abs)?; Ok(FluxonFsTransferWorkerFileResultWire { relpath: file.final_relpath.clone(), @@ -5694,15 +5782,15 @@ fn prepare_transfer_collect_info_materialization( ), })) })?; - let staging_relpath = transfer_collect_info_staging_relpath( - batch_id, - worker_task_id, - collect_info.collect_kind, - )?; + let staging_relpath = + transfer_collect_info_staging_relpath(batch_id, worker_task_id, collect_info.collect_kind)?; ensure_transfer_parent_dirs(dst_root, staging_relpath.as_str())?; ensure_transfer_parent_dirs(dst_root, output_relpath.as_str())?; - let staging_abs = safe_join_root(dst_root.to_string_lossy().as_ref(), staging_relpath.as_str()) - .map_err(resp_err_kverr)?; + let staging_abs = safe_join_root( + dst_root.to_string_lossy().as_ref(), + staging_relpath.as_str(), + ) + .map_err(resp_err_kverr)?; let mut dst_file = open_create_file_with_parent_dir_chmod_retry(&staging_abs)?; dst_file.set_len(0).map_err(resp_err_io)?; dst_file @@ -5723,14 +5811,15 @@ fn transfer_collect_info_staging_relpath( worker_task_id: &str, collect_kind: FluxonFsTransferCollectInfoKind, ) -> Result { - let output_relpath = transfer_collect_info_output_relpath(batch_id, collect_kind).map_err(|detail| { - resp_err_kverr(KvError::Api(ApiError::InvalidArgument { - detail: format!( - "build transfer collect info output relpath failed: batch_id={} err={}", - batch_id, detail - ), - })) - })?; + let output_relpath = + transfer_collect_info_output_relpath(batch_id, collect_kind).map_err(|detail| { + resp_err_kverr(KvError::Api(ApiError::InvalidArgument { + detail: format!( + "build transfer collect info output relpath failed: batch_id={} err={}", + batch_id, detail + ), + })) + })?; Ok(format!("{}.{}.fluxon.part", output_relpath, worker_task_id)) } @@ -5750,9 +5839,12 @@ fn prune_empty_parent_dirs(mut current: PathBuf, root: &PathBuf) -> Result<(), F Ok(()) } -fn cleanup_attempt_staging_prefix(dst_root: &PathBuf, staging_prefix: &str) -> Result<(), FlatDict> { - let staging_abs = - safe_join_root(dst_root.to_string_lossy().as_ref(), staging_prefix).map_err(resp_err_kverr)?; +fn cleanup_attempt_staging_prefix( + dst_root: &PathBuf, + staging_prefix: &str, +) -> Result<(), FlatDict> { + let staging_abs = safe_join_root(dst_root.to_string_lossy().as_ref(), staging_prefix) + .map_err(resp_err_kverr)?; match fs::remove_dir_all(&staging_abs) { Ok(()) => {} Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), @@ -5865,7 +5957,8 @@ pub(crate) fn read_transfer_chunk_from_root_dir_abs( return Ok(Vec::new()); } let to_read = std::cmp::min(length, size - offset) as usize; - f.seek(SeekFrom::Start(offset as u64)).map_err(resp_err_io)?; + f.seek(SeekFrom::Start(offset as u64)) + .map_err(resp_err_io)?; let mut buf = vec![0u8; to_read]; f.read_exact(&mut buf).map_err(resp_err_io)?; Ok(buf) @@ -5881,11 +5974,14 @@ pub(crate) fn execute_transfer_worker_assignment( read_chunk: ReadChunkFn, ) -> Result where - ReadChunkFn: - Fn(&FluxonFsTransferManifestEntryWire, i64, i64) -> Result, TransferWorkerExecutionError> - + Send - + Sync - + 'static, + ReadChunkFn: Fn( + &FluxonFsTransferManifestEntryWire, + i64, + i64, + ) -> Result, TransferWorkerExecutionError> + + Send + + Sync + + 'static, CheckpointFn: Fn() -> Result<(), TransferWorkerExecutionError> + Send + Sync + 'static, { execute_transfer_worker_assignment_with_policy( @@ -5943,12 +6039,19 @@ pub(super) fn handle_transfer_read(exports: &AgentExportsHandle, payload: FlatDi Ok(v) => v, Err(e) => return resp_err_kverr(e), }; - let buf = - match read_transfer_chunk_from_root_dir_abs(root_dir_abs.as_str(), relpath.as_str(), offset, length) { - Ok(v) => v, - Err(resp) => return resp, - }; - resp_ok(BTreeMap::from([("data".to_string(), FlatValue::Bytes(buf))])) + let buf = match read_transfer_chunk_from_root_dir_abs( + root_dir_abs.as_str(), + relpath.as_str(), + offset, + length, + ) { + Ok(v) => v, + Err(resp) => return resp, + }; + resp_ok(BTreeMap::from([( + "data".to_string(), + FlatValue::Bytes(buf), + )])) } pub(super) fn handle_transfer_stream_open( @@ -6010,7 +6113,9 @@ pub(super) fn handle_transfer_worker( Err(resp) => return resp, }; match registry.launch_task(api, master_id, exports, assignment) { - Ok(result) => encode_transfer_worker_launch_result(&result, "transfer worker launch result"), + Ok(result) => { + encode_transfer_worker_launch_result(&result, "transfer worker launch result") + } Err(resp) => resp, } } @@ -6086,15 +6191,15 @@ mod tests { .collect() } - fn assert_all_child_scan_units_are_subtree_streaming( - result: &FluxonFsTransferScanResultWire, - ) { - assert!(result - .child_scan_units - .iter() - .all(|child| child.scan_mode == FluxonFsTransferScanMode::SubtreeStreaming)); - } - + fn assert_all_child_scan_units_are_subtree_streaming(result: &FluxonFsTransferScanResultWire) { + assert!( + result + .child_scan_units + .iter() + .all(|child| child.scan_mode == FluxonFsTransferScanMode::SubtreeStreaming) + ); + } + fn ok_bool(resp: &FlatDict) -> bool { matches!(resp.get("ok"), Some(FlatValue::Bool(true))) } @@ -6106,10 +6211,7 @@ mod tests { panic!("unexpected open result fatal decode error: {:?}", other) } Err(TransferWorkerRpcFailure::Retryable { detail }) => { - panic!( - "unexpected open result retryable decode error: {}", - detail - ) + panic!("unexpected open result retryable decode error: {}", detail) } } } @@ -6121,10 +6223,7 @@ mod tests { panic!("unexpected next result fatal decode error: {:?}", other) } Err(TransferWorkerRpcFailure::Retryable { detail }) => { - panic!( - "unexpected next result retryable decode error: {}", - detail - ) + panic!("unexpected next result retryable decode error: {}", detail) } } } @@ -6140,10 +6239,7 @@ mod tests { .collect() } - fn test_worker_assignment( - relpath: &str, - size: i64, - ) -> FluxonFsTransferWorkerAssignmentWire { + fn test_worker_assignment(relpath: &str, size: i64) -> FluxonFsTransferWorkerAssignmentWire { FluxonFsTransferWorkerAssignmentWire { job_id: "job".to_string(), batch_id: "batch".to_string(), @@ -6158,12 +6254,13 @@ mod tests { root_relpath: ".".to_string(), staging_prefix: ".fluxon.stage/job/batch".to_string(), lease_expire_unix_ms: 0, - manifest_blob: build_transfer_manifest_blob(vec![ - FluxonFsTransferScanFrontierEntry { + manifest_blob: build_transfer_manifest_blob( + vec![FluxonFsTransferScanFrontierEntry { relpath: relpath.to_string(), size, - }, - ], Vec::new()) + }], + Vec::new(), + ) .unwrap(), collect_infos: Vec::new(), } @@ -6183,7 +6280,11 @@ mod tests { read_chunk: ReadChunkFn, ) -> TransferWorkerCoordinator where - ReadChunkFn: Fn(&FluxonFsTransferManifestEntryWire, i64, i64) -> Result, TransferWorkerExecutionError>, + ReadChunkFn: Fn( + &FluxonFsTransferManifestEntryWire, + i64, + i64, + ) -> Result, TransferWorkerExecutionError>, CheckpointFn: Fn() -> Result<(), TransferWorkerExecutionError>, { let policy = Arc::new(TransferWorkerLanePolicy::production_default()); @@ -6217,16 +6318,19 @@ mod tests { #[test] fn build_transfer_manifest_blob_round_trips_entries() { - let blob = build_transfer_manifest_blob(vec![ - FluxonFsTransferScanFrontierEntry { - relpath: "a".to_string(), - size: 1, - }, - FluxonFsTransferScanFrontierEntry { - relpath: "b/c".to_string(), - size: 2, - }, - ], vec!["empty".to_string()]) + let blob = build_transfer_manifest_blob( + vec![ + FluxonFsTransferScanFrontierEntry { + relpath: "a".to_string(), + size: 1, + }, + FluxonFsTransferScanFrontierEntry { + relpath: "b/c".to_string(), + size: 2, + }, + ], + vec!["empty".to_string()], + ) .unwrap(); let manifest = FluxonFsTransferManifestWire::decode_from_blob(&blob).unwrap(); assert_eq!(manifest.entry_count, 2); @@ -6250,11 +6354,12 @@ mod tests { #[test] fn materialize_transfer_collect_info_writes_task_scoped_staging_then_output_file() { let root = TempDir::new().unwrap(); - let collect_infos = build_symlink_collect_infos(vec![FluxonFsTransferSymlinkNoticeEntryWire { - relpath: "root/link-file.bin".to_string(), - link_target: "target/file.bin".to_string(), - }]) - .unwrap(); + let collect_infos = + build_symlink_collect_infos(vec![FluxonFsTransferSymlinkNoticeEntryWire { + relpath: "root/link-file.bin".to_string(), + link_target: "target/file.bin".to_string(), + }]) + .unwrap(); let prepared = prepare_transfer_collect_info_materialization( &root.path().to_path_buf(), "batch-1", @@ -6383,7 +6488,10 @@ mod tests { &exports, FlatDict::from([ ("export".to_string(), FlatValue::String("src".to_string())), - ("relpath".to_string(), FlatValue::String("f.bin".to_string())), + ( + "relpath".to_string(), + FlatValue::String("f.bin".to_string()), + ), ("offset".to_string(), FlatValue::Int64(2)), ("length".to_string(), FlatValue::Int64(3)), ]), @@ -6398,7 +6506,10 @@ mod tests { &exports, FlatDict::from([ ("export".to_string(), FlatValue::String("src".to_string())), - ("relpath".to_string(), FlatValue::String("f.bin".to_string())), + ( + "relpath".to_string(), + FlatValue::String("f.bin".to_string()), + ), ("offset".to_string(), FlatValue::Int64(6)), ("length".to_string(), FlatValue::Int64(1)), ]), @@ -6423,7 +6534,10 @@ mod tests { &exports, FlatDict::from([ ("export".to_string(), FlatValue::String("src".to_string())), - ("relpath".to_string(), FlatValue::String("f.bin".to_string())), + ( + "relpath".to_string(), + FlatValue::String("f.bin".to_string()), + ), ("offset".to_string(), FlatValue::Int64(1)), ("length".to_string(), FlatValue::Int64(3)), ]), @@ -6446,9 +6560,15 @@ mod tests { &exports, FlatDict::from([ ("export".to_string(), FlatValue::String("src".to_string())), - ("relpath".to_string(), FlatValue::String("f.bin".to_string())), + ( + "relpath".to_string(), + FlatValue::String("f.bin".to_string()), + ), ("offset".to_string(), FlatValue::Int64(0)), - ("length".to_string(), FlatValue::Int64(CHUNK_BYTES as i64 + 1)), + ( + "length".to_string(), + FlatValue::Int64(CHUNK_BYTES as i64 + 1), + ), ]), ); assert!(matches!(resp.get("ok"), Some(FlatValue::Bool(false)))); @@ -6470,7 +6590,10 @@ mod tests { FlatValue::String("task-0".to_string()), ), ("export".to_string(), FlatValue::String("src".to_string())), - ("relpath".to_string(), FlatValue::String("f.bin".to_string())), + ( + "relpath".to_string(), + FlatValue::String("f.bin".to_string()), + ), ("initial_offset".to_string(), FlatValue::Int64(0)), ]), ); @@ -6551,7 +6674,10 @@ mod tests { FlatValue::String("task-1".to_string()), ), ("export".to_string(), FlatValue::String("src".to_string())), - ("relpath".to_string(), FlatValue::String("f.bin".to_string())), + ( + "relpath".to_string(), + FlatValue::String("f.bin".to_string()), + ), ("initial_offset".to_string(), FlatValue::Int64(0)), ]), ); @@ -6584,7 +6710,10 @@ mod tests { ("length".to_string(), FlatValue::Int64(2)), ]), ); - assert!(matches!(invalid_resp.get("ok"), Some(FlatValue::Bool(false)))); + assert!(matches!( + invalid_resp.get("ok"), + Some(FlatValue::Bool(false)) + )); } #[test] @@ -6603,7 +6732,10 @@ mod tests { FlatValue::String("task-2".to_string()), ), ("export".to_string(), FlatValue::String("src".to_string())), - ("relpath".to_string(), FlatValue::String("f.bin".to_string())), + ( + "relpath".to_string(), + FlatValue::String("f.bin".to_string()), + ), ("initial_offset".to_string(), FlatValue::Int64(3)), ]), ); @@ -6647,7 +6779,10 @@ mod tests { FlatValue::String("task-3".to_string()), ), ("export".to_string(), FlatValue::String("src".to_string())), - ("relpath".to_string(), FlatValue::String("f.bin".to_string())), + ( + "relpath".to_string(), + FlatValue::String("f.bin".to_string()), + ), ("initial_offset".to_string(), FlatValue::Int64(3)), ]), ); @@ -6718,7 +6853,10 @@ mod tests { let result = decode_result_json(&resp); assert!(result.finished); assert!(result.direct_files_only_batches.is_empty()); - assert_eq!(child_scan_unit_roots(&result), vec!["root/child".to_string()]); + assert_eq!( + child_scan_unit_roots(&result), + vec!["root/child".to_string()] + ); assert_all_child_scan_units_are_subtree_streaming(&result); assert!(result.full_dir_batches.is_empty()); } @@ -6781,7 +6919,8 @@ mod tests { } #[test] - fn handle_transfer_scan_assignment_groups_empty_children_into_direct_batch_without_direct_files() { + fn handle_transfer_scan_assignment_groups_empty_children_into_direct_batch_without_direct_files() + { let root = TempDir::new().unwrap(); write_file(&root, "root/big/data.bin", b"12345"); fs::create_dir_all(root.path().join("root/empty-a")).unwrap(); @@ -6814,9 +6953,10 @@ mod tests { assert_eq!(child_scan_unit_roots(&result), vec!["root/big".to_string()]); assert_all_child_scan_units_are_subtree_streaming(&result); assert!(result.full_dir_batches.is_empty()); - let direct_manifest = - FluxonFsTransferManifestWire::decode_from_blob(&result.direct_files_only_batches[0].manifest_blob) - .unwrap(); + let direct_manifest = FluxonFsTransferManifestWire::decode_from_blob( + &result.direct_files_only_batches[0].manifest_blob, + ) + .unwrap(); assert!(direct_manifest.entries.is_empty()); assert_eq!( direct_manifest.empty_dir_relpaths, @@ -6854,16 +6994,17 @@ mod tests { assert_eq!(result.direct_files_only_batches.len(), 1); assert_eq!(result.full_dir_batches.len(), 0); assert_eq!(result.child_scan_units.len(), 1); - assert_eq!(result.child_scan_units[0].root_relpath, "root/huge".to_string()); + assert_eq!( + result.child_scan_units[0].root_relpath, + "root/huge".to_string() + ); let manifest = FluxonFsTransferManifestWire::decode_from_blob( &result.direct_files_only_batches[0].manifest_blob, ) .unwrap(); assert!(manifest.entries.is_empty()); assert!(!manifest.empty_dir_relpaths.is_empty()); - assert!( - manifest.empty_dir_relpaths.len() <= TRANSFER_MERGEABLE_EMPTY_DIR_BUDGET - ); + assert!(manifest.empty_dir_relpaths.len() <= TRANSFER_MERGEABLE_EMPTY_DIR_BUDGET); assert!( estimate_empty_dir_manifest_bytes(&manifest.empty_dir_relpaths) <= TRANSFER_MERGEABLE_EMPTY_DIR_ESTIMATED_BYTES_BUDGET @@ -6879,10 +7020,10 @@ mod tests { )) + 1; for idx in 0..child_count { - fs::create_dir_all(root.path().join(format!( - "root/branch-{idx:05}/{}", - "x".repeat(200) - ))) + fs::create_dir_all( + root.path() + .join(format!("root/branch-{idx:05}/{}", "x".repeat(200))), + ) .unwrap(); } let result = build_transfer_scan_result_for_root_dir_abs( @@ -6908,10 +7049,12 @@ mod tests { assert!(!result.finished); assert!(result.direct_files_only_batches.is_empty()); assert!(!result.child_scan_units.is_empty()); - assert!(result - .child_scan_units - .iter() - .any(|child| child.scan_mode == FluxonFsTransferScanMode::FullTree)); + assert!( + result + .child_scan_units + .iter() + .any(|child| child.scan_mode == FluxonFsTransferScanMode::FullTree) + ); assert!(result.child_scan_units.iter().all(|child| { child.scan_mode == FluxonFsTransferScanMode::FullTree || child.scan_mode == FluxonFsTransferScanMode::SubtreeStreaming @@ -6963,10 +7106,16 @@ mod tests { assert!(!continue_locally); assert_eq!(next_event_seq_no, 9); assert_eq!(events.len(), 2); - assert_eq!(events[0].event_kind, FluxonFsTransferScanEventKindWire::Append); + assert_eq!( + events[0].event_kind, + FluxonFsTransferScanEventKindWire::Append + ); assert_eq!(events[0].event_seq_no, 7); assert_eq!(events[0].full_dir_batches.len(), 1); - assert_eq!(events[1].event_kind, FluxonFsTransferScanEventKindWire::Finished); + assert_eq!( + events[1].event_kind, + FluxonFsTransferScanEventKindWire::Finished + ); assert_eq!(events[1].event_seq_no, 8); assert!(events[1].direct_files_only_batches.is_empty()); assert!(events[1].child_scan_units.is_empty()); @@ -6997,17 +7146,21 @@ mod tests { skip_entries: Vec::new(), }; - let first = build_transfer_scan_result_for_root_dir_abs( - root.path().to_str().unwrap(), - &assignment, - ) - .unwrap(); + let first = + build_transfer_scan_result_for_root_dir_abs(root.path().to_str().unwrap(), &assignment) + .unwrap(); assert!(!first.finished); assert!(!first.direct_files_only_batches.is_empty()); assert!(first.full_dir_batches.is_empty()); assert_eq!(first.child_scan_units.len(), 1); - assert_eq!(first.child_scan_units[0].scan_unit_id, assignment.scan_unit_id); - assert_eq!(first.child_scan_units[0].root_relpath, assignment.root_relpath); + assert_eq!( + first.child_scan_units[0].scan_unit_id, + assignment.scan_unit_id + ); + assert_eq!( + first.child_scan_units[0].root_relpath, + assignment.root_relpath + ); assert_eq!(first.child_scan_units[0].generation, assignment.generation); let first_entry_count = first .direct_files_only_batches @@ -7019,7 +7172,10 @@ mod tests { .len() }) .sum::(); - assert_eq!(first_entry_count, TRANSFER_SCAN_ROOT_LISTING_SLICE_ENTRY_LIMIT); + assert_eq!( + first_entry_count, + TRANSFER_SCAN_ROOT_LISTING_SLICE_ENTRY_LIMIT + ); let second_assignment = FluxonFsTransferScanAssignmentWire { scan_task_id: "task-2".to_string(), @@ -7086,7 +7242,8 @@ mod tests { } #[test] - fn build_transfer_scan_result_root_direct_fanout_only_emits_child_scan_units_without_recursing() { + fn build_transfer_scan_result_root_direct_fanout_only_emits_child_scan_units_without_recursing() + { let root = TempDir::new().unwrap(); write_file(&root, "root/direct.bin", b"abc"); write_file(&root, "root/child/payload.bin", b"xyz"); @@ -7114,14 +7271,18 @@ mod tests { assert_eq!(result.direct_files_only_batches.len(), 1); assert_eq!(result.child_scan_units.len(), 1); assert!(result.full_dir_batches.is_empty()); - assert_eq!(result.child_scan_units[0].root_relpath, "root/child".to_string()); + assert_eq!( + result.child_scan_units[0].root_relpath, + "root/child".to_string() + ); assert_eq!( result.child_scan_units[0].scan_mode, FluxonFsTransferScanMode::FullTree ); - let manifest = - FluxonFsTransferManifestWire::decode_from_blob(&result.direct_files_only_batches[0].manifest_blob) - .unwrap(); + let manifest = FluxonFsTransferManifestWire::decode_from_blob( + &result.direct_files_only_batches[0].manifest_blob, + ) + .unwrap(); assert_eq!( manifest.entries, vec![FluxonFsTransferManifestEntryWire { @@ -7133,7 +7294,8 @@ mod tests { } #[test] - fn build_transfer_scan_result_directory_direct_fanout_only_emits_child_scan_units_without_recursing() { + fn build_transfer_scan_result_directory_direct_fanout_only_emits_child_scan_units_without_recursing() + { let root = TempDir::new().unwrap(); write_file(&root, "root/child/direct.bin", b"abc"); write_file(&root, "root/child/grand/payload.bin", b"xyz"); @@ -7161,14 +7323,18 @@ mod tests { assert_eq!(result.direct_files_only_batches.len(), 1); assert_eq!(result.child_scan_units.len(), 1); assert!(result.full_dir_batches.is_empty()); - assert_eq!(result.child_scan_units[0].root_relpath, "root/child/grand".to_string()); + assert_eq!( + result.child_scan_units[0].root_relpath, + "root/child/grand".to_string() + ); assert_eq!( result.child_scan_units[0].scan_mode, FluxonFsTransferScanMode::FullTree ); - let manifest = - FluxonFsTransferManifestWire::decode_from_blob(&result.direct_files_only_batches[0].manifest_blob) - .unwrap(); + let manifest = FluxonFsTransferManifestWire::decode_from_blob( + &result.direct_files_only_batches[0].manifest_blob, + ) + .unwrap(); assert_eq!( manifest.entries, vec![FluxonFsTransferManifestEntryWire { @@ -7206,10 +7372,14 @@ mod tests { .unwrap(); assert!(result.finished); assert_eq!(result.direct_files_only_batches.len(), 1); - assert_eq!(result.direct_files_only_batches[0].root_relpath, "root/child"); - let manifest = - FluxonFsTransferManifestWire::decode_from_blob(&result.direct_files_only_batches[0].manifest_blob) - .unwrap(); + assert_eq!( + result.direct_files_only_batches[0].root_relpath, + "root/child" + ); + let manifest = FluxonFsTransferManifestWire::decode_from_blob( + &result.direct_files_only_batches[0].manifest_blob, + ) + .unwrap(); assert_eq!( manifest.entries, vec![FluxonFsTransferManifestEntryWire { @@ -7218,7 +7388,10 @@ mod tests { }] ); assert!(manifest.empty_dir_relpaths.is_empty()); - assert_eq!(child_scan_unit_roots(&result), vec!["root/child/grand".to_string()]); + assert_eq!( + child_scan_unit_roots(&result), + vec!["root/child/grand".to_string()] + ); assert_all_child_scan_units_are_subtree_streaming(&result); assert!(result.full_dir_batches.is_empty()); } @@ -7253,10 +7426,14 @@ mod tests { assert!(result.finished); assert_eq!(result.direct_files_only_batches.len(), 1); assert_eq!(result.child_scan_units.len(), 1); - assert_eq!(result.child_scan_units[0].root_relpath, "root/child".to_string()); - let manifest = - FluxonFsTransferManifestWire::decode_from_blob(&result.direct_files_only_batches[0].manifest_blob) - .unwrap(); + assert_eq!( + result.child_scan_units[0].root_relpath, + "root/child".to_string() + ); + let manifest = FluxonFsTransferManifestWire::decode_from_blob( + &result.direct_files_only_batches[0].manifest_blob, + ) + .unwrap(); assert_eq!( manifest.entries, vec![FluxonFsTransferManifestEntryWire { @@ -7330,9 +7507,10 @@ mod tests { assert_eq!(result.direct_files_only_batches.len(), 1); assert!(result.child_scan_units.is_empty()); assert!(result.full_dir_batches.is_empty()); - let manifest = - FluxonFsTransferManifestWire::decode_from_blob(&result.direct_files_only_batches[0].manifest_blob) - .unwrap(); + let manifest = FluxonFsTransferManifestWire::decode_from_blob( + &result.direct_files_only_batches[0].manifest_blob, + ) + .unwrap(); assert_eq!( manifest.entries, vec![FluxonFsTransferManifestEntryWire { @@ -7407,7 +7585,10 @@ mod tests { assert!(result.finished); assert_eq!(result.direct_files_only_batches.len(), 1); assert_eq!(result.child_scan_units.len(), 1); - assert_eq!(result.child_scan_units[0].root_relpath, "root/child-b".to_string()); + assert_eq!( + result.child_scan_units[0].root_relpath, + "root/child-b".to_string() + ); assert_eq!( result.child_scan_units[0].scan_mode, FluxonFsTransferScanMode::FullTree @@ -7447,9 +7628,10 @@ mod tests { assert_eq!(result.direct_files_only_batches.len(), 1); assert!(result.child_scan_units.is_empty()); assert!(result.full_dir_batches.is_empty()); - let manifest = - FluxonFsTransferManifestWire::decode_from_blob(&result.direct_files_only_batches[0].manifest_blob) - .unwrap(); + let manifest = FluxonFsTransferManifestWire::decode_from_blob( + &result.direct_files_only_batches[0].manifest_blob, + ) + .unwrap(); assert_eq!( manifest.entries, vec![FluxonFsTransferManifestEntryWire { @@ -7490,9 +7672,10 @@ mod tests { assert!(result.direct_files_only_batches.is_empty()); assert!(result.child_scan_units.is_empty()); assert_eq!(result.full_dir_batches.len(), 1); - let manifest = - FluxonFsTransferManifestWire::decode_from_blob(&result.full_dir_batches[0].manifest_blob) - .unwrap(); + let manifest = FluxonFsTransferManifestWire::decode_from_blob( + &result.full_dir_batches[0].manifest_blob, + ) + .unwrap(); assert!(manifest.entries.is_empty()); assert_eq!(manifest.empty_dir_relpaths, vec!["root".to_string()]); } @@ -7533,7 +7716,8 @@ mod tests { } #[test] - fn handle_transfer_scan_assignment_does_not_reaggregate_root_when_descendant_batch_is_durable() { + fn handle_transfer_scan_assignment_does_not_reaggregate_root_when_descendant_batch_is_durable() + { let root = TempDir::new().unwrap(); write_file(&root, "root/direct.bin", b"abc"); write_file(&root, "root/big/data.bin", b"12345"); @@ -7579,21 +7763,29 @@ mod tests { size: 3, }] ); - assert!(result - .full_dir_batches - .iter() - .all(|batch| batch.root_relpath != "root")); - assert!(result - .full_dir_batches - .iter() - .all(|batch| batch.root_relpath != "root/big")); - assert_eq!(child_scan_unit_roots(&result), vec!["root/small".to_string()]); + assert!( + result + .full_dir_batches + .iter() + .all(|batch| batch.root_relpath != "root") + ); + assert!( + result + .full_dir_batches + .iter() + .all(|batch| batch.root_relpath != "root/big") + ); + assert_eq!( + child_scan_unit_roots(&result), + vec!["root/small".to_string()] + ); assert_all_child_scan_units_are_subtree_streaming(&result); assert!(result.full_dir_batches.is_empty()); } #[test] - fn handle_transfer_scan_assignment_honors_cross_generation_descendant_full_dir_during_restart() { + fn handle_transfer_scan_assignment_honors_cross_generation_descendant_full_dir_during_restart() + { let root = TempDir::new().unwrap(); write_file(&root, "root/direct.bin", b"abc"); write_file(&root, "root/big/data.bin", b"12345"); @@ -7639,21 +7831,29 @@ mod tests { size: 3, }] ); - assert!(result - .full_dir_batches - .iter() - .all(|batch| batch.root_relpath != "root")); - assert!(result - .full_dir_batches - .iter() - .all(|batch| batch.root_relpath != "root/big")); - assert_eq!(child_scan_unit_roots(&result), vec!["root/small".to_string()]); + assert!( + result + .full_dir_batches + .iter() + .all(|batch| batch.root_relpath != "root") + ); + assert!( + result + .full_dir_batches + .iter() + .all(|batch| batch.root_relpath != "root/big") + ); + assert_eq!( + child_scan_unit_roots(&result), + vec!["root/small".to_string()] + ); assert_all_child_scan_units_are_subtree_streaming(&result); assert!(result.full_dir_batches.is_empty()); } #[test] - fn handle_transfer_scan_assignment_replays_descendant_current_layer_when_only_partial_descendant_direct_files_batch_is_durable() { + fn handle_transfer_scan_assignment_replays_descendant_current_layer_when_only_partial_descendant_direct_files_batch_is_durable() + { let root = TempDir::new().unwrap(); write_file(&root, "root/child/a.bin", b"ab"); write_file(&root, "root/child/b.bin", b"cd"); @@ -7688,10 +7888,14 @@ mod tests { assert!(result.child_scan_units.is_empty()); assert!(result.full_dir_batches.is_empty()); assert_eq!(result.direct_files_only_batches.len(), 1); - assert_eq!(result.direct_files_only_batches[0].root_relpath, "root/child"); - let manifest = - FluxonFsTransferManifestWire::decode_from_blob(&result.direct_files_only_batches[0].manifest_blob) - .unwrap(); + assert_eq!( + result.direct_files_only_batches[0].root_relpath, + "root/child" + ); + let manifest = FluxonFsTransferManifestWire::decode_from_blob( + &result.direct_files_only_batches[0].manifest_blob, + ) + .unwrap(); assert_eq!( manifest.entries, vec![ @@ -7737,7 +7941,10 @@ mod tests { assert!(result.finished); assert!(result.direct_files_only_batches.is_empty()); - assert_eq!(child_scan_unit_roots(&result), vec!["root/parent".to_string()]); + assert_eq!( + child_scan_unit_roots(&result), + vec!["root/parent".to_string()] + ); assert_all_child_scan_units_are_subtree_streaming(&result); assert!(result.full_dir_batches.is_empty()); } @@ -7772,9 +7979,10 @@ mod tests { assert!(result.finished); assert_eq!(result.direct_files_only_batches.len(), 1); - let manifest = - FluxonFsTransferManifestWire::decode_from_blob(&result.direct_files_only_batches[0].manifest_blob) - .unwrap(); + let manifest = FluxonFsTransferManifestWire::decode_from_blob( + &result.direct_files_only_batches[0].manifest_blob, + ) + .unwrap(); assert_eq!( manifest.entries, vec![FluxonFsTransferManifestEntryWire { @@ -7782,7 +7990,10 @@ mod tests { size: 10, }] ); - assert_eq!(child_scan_unit_roots(&result), vec!["root/child".to_string()]); + assert_eq!( + child_scan_unit_roots(&result), + vec!["root/child".to_string()] + ); assert_all_child_scan_units_are_subtree_streaming(&result); assert!(result.full_dir_batches.is_empty()); } @@ -7820,7 +8031,10 @@ mod tests { assert!(ok_bool(&resp)); assert!(result.finished); - assert_eq!(child_scan_unit_roots(&result), vec!["root/blocked".to_string()]); + assert_eq!( + child_scan_unit_roots(&result), + vec!["root/blocked".to_string()] + ); assert_all_child_scan_units_are_subtree_streaming(&result); assert!(result.full_dir_batches.is_empty()); } @@ -7870,7 +8084,11 @@ mod tests { ); assert_eq!( child_scan_unit_roots(&result), - vec!["root/a".to_string(), "root/b".to_string(), "root/c".to_string()] + vec![ + "root/a".to_string(), + "root/b".to_string(), + "root/c".to_string() + ] ); assert_all_child_scan_units_are_subtree_streaming(&result); assert!(result.full_dir_batches.is_empty()); @@ -7905,7 +8123,10 @@ mod tests { ); let result = decode_result_json(&resp); assert_eq!(result.full_dir_batches.len(), 1); - assert_eq!(result.full_dir_batches[0].batch_kind, FluxonFsTransferBatchKind::SubtreeSlice); + assert_eq!( + result.full_dir_batches[0].batch_kind, + FluxonFsTransferBatchKind::SubtreeSlice + ); assert_eq!(result.full_dir_batches[0].collect_infos.len(), 1); assert_eq!( decode_symlink_notice_collect_blob( @@ -7957,11 +8178,15 @@ mod tests { assert!(result.direct_files_only_batches.is_empty()); assert!(result.child_scan_units.is_empty()); assert_eq!(result.full_dir_batches.len(), 1); - assert_eq!(result.full_dir_batches[0].batch_kind, FluxonFsTransferBatchKind::SubtreeSlice); + assert_eq!( + result.full_dir_batches[0].batch_kind, + FluxonFsTransferBatchKind::SubtreeSlice + ); assert_eq!(result.full_dir_batches[0].root_relpath, "root".to_string()); - let manifest = - FluxonFsTransferManifestWire::decode_from_blob(&result.full_dir_batches[0].manifest_blob) - .unwrap(); + let manifest = FluxonFsTransferManifestWire::decode_from_blob( + &result.full_dir_batches[0].manifest_blob, + ) + .unwrap(); assert_eq!( manifest.entries, vec![ @@ -7982,7 +8207,9 @@ mod tests { FluxonFsTransferCollectInfoKind::SymlinkNotice ); let mut notices = decode_symlink_notice_collect_blob( - direct_files_only_batch.collect_infos[0].collect_blob.as_slice() + direct_files_only_batch.collect_infos[0] + .collect_blob + .as_slice(), ); notices.sort_by(|a, b| a.relpath.cmp(&b.relpath)); assert_eq!( @@ -8004,16 +8231,17 @@ mod tests { fn prepare_transfer_file_from_chunks_promotes_staged_file_to_final_path() { let root = TempDir::new().unwrap(); let dst_root = root.path().to_path_buf(); - let coordinator = test_transfer_coordinator( - || Ok(()), - { - let chunks = Arc::new(Mutex::new(vec![b"ab".to_vec(), b"cde".to_vec(), Vec::new()])); - move |_file, _read_offset, _length| { - let mut chunks = chunks.lock(); - Ok(chunks.remove(0)) - } - }, - ); + let coordinator = test_transfer_coordinator(|| Ok(()), { + let chunks = Arc::new(Mutex::new(vec![ + b"ab".to_vec(), + b"cde".to_vec(), + Vec::new(), + ])); + move |_file, _read_offset, _length| { + let mut chunks = chunks.lock(); + Ok(chunks.remove(0)) + } + }); let prepared = prepare_transfer_file_streaming( &dst_root, ".fluxon.stage/job/batch", @@ -8038,32 +8266,31 @@ mod tests { fs::read(root.path().join("dir/file.bin")).unwrap(), b"abcde".to_vec() ); - assert!(!root - .path() - .join(".fluxon.stage/job/batch/dir/file.bin/file.bin.fluxon.part") - .exists()); + assert!( + !root + .path() + .join(".fluxon.stage/job/batch/dir/file.bin/file.bin.fluxon.part") + .exists() + ); } #[test] fn prepare_transfer_file_from_chunks_truncates_existing_staging_file() { let root = TempDir::new().unwrap(); let dst_root = root.path().to_path_buf(); - let stale_staging = - root.path() - .join(".fluxon.stage/job/batch/dir/file.bin/file.bin.fluxon.part"); + let stale_staging = root + .path() + .join(".fluxon.stage/job/batch/dir/file.bin/file.bin.fluxon.part"); fs::create_dir_all(stale_staging.parent().unwrap()).unwrap(); fs::write(&stale_staging, b"stale-data").unwrap(); - let coordinator = test_transfer_coordinator( - || Ok(()), - { - let chunks = Arc::new(Mutex::new(vec![b"xy".to_vec(), Vec::new()])); - move |_file, _read_offset, _length| { - let mut chunks = chunks.lock(); - Ok(chunks.remove(0)) - } - }, - ); + let coordinator = test_transfer_coordinator(|| Ok(()), { + let chunks = Arc::new(Mutex::new(vec![b"xy".to_vec(), Vec::new()])); + move |_file, _read_offset, _length| { + let mut chunks = chunks.lock(); + Ok(chunks.remove(0)) + } + }); let prepared = prepare_transfer_file_streaming( &dst_root, ".fluxon.stage/job/batch", @@ -8076,23 +8303,23 @@ mod tests { .unwrap(); promote_prepared_transfer_file(&dst_root, prepared).unwrap(); - assert_eq!(fs::read(root.path().join("dir/file.bin")).unwrap(), b"xy".to_vec()); + assert_eq!( + fs::read(root.path().join("dir/file.bin")).unwrap(), + b"xy".to_vec() + ); } #[test] fn prepare_transfer_file_from_chunks_rejects_size_mismatch_and_keeps_staging_file() { let root = TempDir::new().unwrap(); let dst_root = root.path().to_path_buf(); - let coordinator = test_transfer_coordinator( - || Ok(()), - { - let chunks = Arc::new(Mutex::new(vec![b"xy".to_vec(), Vec::new()])); - move |_file, _read_offset, _length| { - let mut chunks = chunks.lock(); - Ok(chunks.remove(0)) - } - }, - ); + let coordinator = test_transfer_coordinator(|| Ok(()), { + let chunks = Arc::new(Mutex::new(vec![b"xy".to_vec(), Vec::new()])); + move |_file, _read_offset, _length| { + let mut chunks = chunks.lock(); + Ok(chunks.remove(0)) + } + }); let err = prepare_transfer_file_streaming( &dst_root, ".fluxon.stage/job/batch", @@ -8275,15 +8502,10 @@ mod tests { let file_bytes = b"hello".to_vec(); let assignment = test_worker_assignment("dir/file.bin", file_bytes.len() as i64); - let result = execute_transfer_worker_assignment( - &assignment, - &dst_root, - || Ok(()), - { - let file_bytes = file_bytes.clone(); - move |_file, _read_offset, _length| Ok(file_bytes.clone()) - }, - ) + let result = execute_transfer_worker_assignment(&assignment, &dst_root, || Ok(()), { + let file_bytes = file_bytes.clone(); + move |_file, _read_offset, _length| Ok(file_bytes.clone()) + }) .unwrap(); assert_eq!(result.file_results.len(), 1); @@ -8303,7 +8525,10 @@ mod tests { create_dir_all_with_parent_dir_chmod_retry(&target).unwrap(); assert!(target.is_dir()); - assert_eq!(fs::metadata(&locked_parent).unwrap().permissions().mode() & 0o777, 0o777); + assert_eq!( + fs::metadata(&locked_parent).unwrap().permissions().mode() & 0o777, + 0o777 + ); } #[cfg(unix)] @@ -8318,21 +8543,19 @@ mod tests { let file_bytes = b"hello".to_vec(); let assignment = test_worker_assignment("dir/file.bin", file_bytes.len() as i64); - let result = execute_transfer_worker_assignment( - &assignment, - &dst_root, - || Ok(()), - { - let file_bytes = file_bytes.clone(); - move |_file, _read_offset, _length| Ok(file_bytes.clone()) - }, - ) + let result = execute_transfer_worker_assignment(&assignment, &dst_root, || Ok(()), { + let file_bytes = file_bytes.clone(); + move |_file, _read_offset, _length| Ok(file_bytes.clone()) + }) .unwrap(); assert_eq!(result.file_results.len(), 1); assert!(dst_root.is_dir()); assert_eq!(fs::read(dst_root.join("dir/file.bin")).unwrap(), file_bytes); - assert_eq!(fs::metadata(&locked_parent).unwrap().permissions().mode() & 0o777, 0o777); + assert_eq!( + fs::metadata(&locked_parent).unwrap().permissions().mode() & 0o777, + 0o777 + ); } #[test] @@ -8349,47 +8572,48 @@ mod tests { let assignment = assignment.clone(); let heartbeat_attempts = heartbeat_attempts.clone(); move || { - retry_transfer_worker_rpc_with_backoff( - &assignment, - "checkpoint", - "test-checkpoint", - BackoffConfig { - initial_secs: 0, - max_secs: 0, - }, - WarnConfig { - warn_interval_secs: 0, - }, - || { - let attempt = - heartbeat_attempts.fetch_add(1, Ordering::SeqCst) + 1; - if attempt < 3 { - return Err(TransferWorkerRpcFailure::Retryable { - detail: format!( - "transient heartbeat failure attempt={}", - attempt - ), - }); - } - Ok(()) - }, - ) - .map_err(TransferWorkerExecutionError::fatal) - } + retry_transfer_worker_rpc_with_backoff( + &assignment, + "checkpoint", + "test-checkpoint", + BackoffConfig { + initial_secs: 0, + max_secs: 0, + }, + WarnConfig { + warn_interval_secs: 0, + }, + || { + let attempt = heartbeat_attempts.fetch_add(1, Ordering::SeqCst) + 1; + if attempt < 3 { + return Err(TransferWorkerRpcFailure::Retryable { + detail: format!( + "transient heartbeat failure attempt={}", + attempt + ), + }); + } + Ok(()) + }, + ) + .map_err(TransferWorkerExecutionError::fatal) + } }, { let file_bytes = file_bytes.clone(); move |file, read_offset, _length| { - if file.relpath != "dir/file.bin" { - return Err(TransferWorkerExecutionError::fatal(resp_err_kverr(KvError::Api(ApiError::InvalidArgument { - detail: format!("unexpected file relpath: {}", file.relpath), - })))); - } - if read_offset == 0 { - return Ok(file_bytes.clone()); + if file.relpath != "dir/file.bin" { + return Err(TransferWorkerExecutionError::fatal(resp_err_kverr( + KvError::Api(ApiError::InvalidArgument { + detail: format!("unexpected file relpath: {}", file.relpath), + }), + ))); + } + if read_offset == 0 { + return Ok(file_bytes.clone()); + } + Ok(Vec::new()) } - Ok(Vec::new()) - } }, ) .unwrap(); @@ -8409,24 +8633,20 @@ mod tests { let file_bytes = b"payload".to_vec(); let assignment = test_worker_assignment("dir/file.bin", file_bytes.len() as i64); let read_attempts = Arc::new(AtomicUsize::new(0)); - let result = execute_transfer_worker_assignment( - &assignment, - &dst_root, - || Ok(()), - { - let assignment = assignment.clone(); - let file_bytes = file_bytes.clone(); - let read_attempts = read_attempts.clone(); - move |file, read_offset, _length| { + let result = execute_transfer_worker_assignment(&assignment, &dst_root, || Ok(()), { + let assignment = assignment.clone(); + let file_bytes = file_bytes.clone(); + let read_attempts = read_attempts.clone(); + move |file, read_offset, _length| { if file.relpath != "dir/file.bin" { - return Err(TransferWorkerExecutionError::fatal(resp_err_kverr(KvError::Api(ApiError::InvalidArgument { - detail: format!("unexpected file relpath: {}", file.relpath), - })))); + return Err(TransferWorkerExecutionError::fatal(resp_err_kverr( + KvError::Api(ApiError::InvalidArgument { + detail: format!("unexpected file relpath: {}", file.relpath), + }), + ))); } - let op_detail = format!( - "test-read relpath={} offset={}", - file.relpath, read_offset - ); + let op_detail = + format!("test-read relpath={} offset={}", file.relpath, read_offset); retry_transfer_worker_rpc_with_backoff( &assignment, "read_chunk", @@ -8443,10 +8663,7 @@ mod tests { let attempt = read_attempts.fetch_add(1, Ordering::SeqCst) + 1; if attempt < 3 { return Err(TransferWorkerRpcFailure::Retryable { - detail: format!( - "transient read failure attempt={}", - attempt - ), + detail: format!("transient read failure attempt={}", attempt), }); } return Ok(file_bytes.clone()); @@ -8456,8 +8673,7 @@ mod tests { ) .map_err(TransferWorkerExecutionError::fatal) } - }, - ) + }) .unwrap(); assert_eq!(read_attempts.load(Ordering::SeqCst), 3); @@ -8481,23 +8697,23 @@ mod tests { { let checkpoint_calls = checkpoint_calls.clone(); move || { - let calls = checkpoint_calls.fetch_add(1, Ordering::SeqCst) + 1; - if calls >= 4 { - return Err(TransferWorkerExecutionError::Stop( - FluxonFsTransferWorkerStopReasonWire::Superseded, - )); + let calls = checkpoint_calls.fetch_add(1, Ordering::SeqCst) + 1; + if calls >= 4 { + return Err(TransferWorkerExecutionError::Stop( + FluxonFsTransferWorkerStopReasonWire::Superseded, + )); + } + Ok(()) } - Ok(()) - } }, { let file_bytes = file_bytes.clone(); move |_file, read_offset, _length| { - if read_offset == 0 { - return Ok(file_bytes.clone()); + if read_offset == 0 { + return Ok(file_bytes.clone()); + } + Ok(Vec::new()) } - Ok(Vec::new()) - } }, ); assert!(matches!( @@ -8567,12 +8783,7 @@ mod tests { break; } if max_in_flight - .compare_exchange( - observed, - current, - Ordering::SeqCst, - Ordering::SeqCst, - ) + .compare_exchange(observed, current, Ordering::SeqCst, Ordering::SeqCst) .is_ok() { break; @@ -8588,8 +8799,14 @@ mod tests { assert_eq!(result.file_results.len(), 2); assert!(max_in_flight.load(Ordering::SeqCst) >= 2); - assert_eq!(fs::read(root.path().join("dir/a.bin")).unwrap(), b"xxx".to_vec()); - assert_eq!(fs::read(root.path().join("dir/b.bin")).unwrap(), b"xxx".to_vec()); + assert_eq!( + fs::read(root.path().join("dir/a.bin")).unwrap(), + b"xxx".to_vec() + ); + assert_eq!( + fs::read(root.path().join("dir/b.bin")).unwrap(), + b"xxx".to_vec() + ); } #[test] @@ -8597,29 +8814,25 @@ mod tests { let root = TempDir::new().unwrap(); let dst_root = root.path().to_path_buf(); let file_bytes = b"hello".to_vec(); - let collect_infos = build_symlink_collect_infos(vec![FluxonFsTransferSymlinkNoticeEntryWire { - relpath: "dir/link.bin".to_string(), - link_target: "dir/file.bin".to_string(), - }]) - .unwrap(); + let collect_infos = + build_symlink_collect_infos(vec![FluxonFsTransferSymlinkNoticeEntryWire { + relpath: "dir/link.bin".to_string(), + link_target: "dir/file.bin".to_string(), + }]) + .unwrap(); let assignment = FluxonFsTransferWorkerAssignmentWire { collect_infos: collect_infos.clone(), ..test_worker_assignment("dir/file.bin", file_bytes.len() as i64) }; - let result = execute_transfer_worker_assignment( - &assignment, - &dst_root, - || Ok(()), - { - let file_bytes = file_bytes.clone(); - move |_file, read_offset, _length| { - if read_offset == 0 { - return Ok(file_bytes.clone()); - } - Ok(Vec::new()) + let result = execute_transfer_worker_assignment(&assignment, &dst_root, || Ok(()), { + let file_bytes = file_bytes.clone(); + move |_file, read_offset, _length| { + if read_offset == 0 { + return Ok(file_bytes.clone()); } - }, - ) + Ok(Vec::new()) + } + }) .unwrap(); assert_eq!(result.file_results.len(), 1); @@ -8633,7 +8846,11 @@ mod tests { "fluxon_collect_info/batches/batch/symlinks.jsonl" ); assert_eq!( - fs::read(root.path().join("fluxon_collect_info/batches/batch/symlinks.jsonl")).unwrap(), + fs::read( + root.path() + .join("fluxon_collect_info/batches/batch/symlinks.jsonl") + ) + .unwrap(), collect_infos[0].collect_blob ); } @@ -8643,16 +8860,19 @@ mod tests { let root = TempDir::new().unwrap(); let dst_root = root.path().to_path_buf(); let assignment = FluxonFsTransferWorkerAssignmentWire { - manifest_blob: build_transfer_manifest_blob(vec![ - FluxonFsTransferScanFrontierEntry { - relpath: "dir/good.bin".to_string(), - size: 5, - }, - FluxonFsTransferScanFrontierEntry { - relpath: "dir/bad.bin".to_string(), - size: 5, - }, - ], Vec::new()) + manifest_blob: build_transfer_manifest_blob( + vec![ + FluxonFsTransferScanFrontierEntry { + relpath: "dir/good.bin".to_string(), + size: 5, + }, + FluxonFsTransferScanFrontierEntry { + relpath: "dir/bad.bin".to_string(), + size: 5, + }, + ], + Vec::new(), + ) .unwrap(), ..test_worker_assignment("dir/good.bin", 5) }; @@ -8816,20 +9036,16 @@ mod tests { .unwrap(); let progress_heartbeat_count = Arc::new(AtomicUsize::new(0)); - gate.ensure_continue( - false, - TRANSFER_WORKER_HEARTBEAT_EMPTY_DIR_PROGRESS_COUNT, - { - let progress_heartbeat_count = progress_heartbeat_count.clone(); - move |_heartbeat_unix_ms, heartbeat_detail| { - assert_eq!(heartbeat_detail, "empty_dir_progress"); - progress_heartbeat_count.fetch_add(1, Ordering::SeqCst); - Ok(FluxonFsTransferWorkerHeartbeatResultWire::continue_running( - chrono::Utc::now().timestamp_millis() + 60_000, - )) - } - }, - ) + gate.ensure_continue(false, TRANSFER_WORKER_HEARTBEAT_EMPTY_DIR_PROGRESS_COUNT, { + let progress_heartbeat_count = progress_heartbeat_count.clone(); + move |_heartbeat_unix_ms, heartbeat_detail| { + assert_eq!(heartbeat_detail, "empty_dir_progress"); + progress_heartbeat_count.fetch_add(1, Ordering::SeqCst); + Ok(FluxonFsTransferWorkerHeartbeatResultWire::continue_running( + chrono::Utc::now().timestamp_millis() + 60_000, + )) + } + }) .unwrap(); gate.ensure_continue( @@ -8927,20 +9143,15 @@ mod tests { let dst_root = root.path().to_path_buf(); let file_bytes = b"hello".to_vec(); let assignment = test_worker_assignment("dir/file.bin", file_bytes.len() as i64); - let result = execute_transfer_worker_assignment( - &assignment, - &dst_root, - || Ok(()), - { - let file_bytes = file_bytes.clone(); - move |_file, read_offset, _length| { - if read_offset == 0 { - return Ok(file_bytes.clone()); - } - Ok(Vec::new()) + let result = execute_transfer_worker_assignment(&assignment, &dst_root, || Ok(()), { + let file_bytes = file_bytes.clone(); + move |_file, read_offset, _length| { + if read_offset == 0 { + return Ok(file_bytes.clone()); } - }, - ) + Ok(Vec::new()) + } + }) .unwrap(); assert_eq!(result.file_results.len(), 1); @@ -8948,7 +9159,10 @@ mod tests { cleanup_transfer_worker_attempt_artifacts(&dst_root, &assignment).unwrap(); - assert_eq!(fs::read(root.path().join("dir/file.bin")).unwrap(), file_bytes); + assert_eq!( + fs::read(root.path().join("dir/file.bin")).unwrap(), + file_bytes + ); assert!(!root.path().join(".fluxon.stage").exists()); } @@ -8957,11 +9171,12 @@ mod tests { let root = TempDir::new().unwrap(); let dst_root = root.path().to_path_buf(); let file_bytes = b"hello".to_vec(); - let collect_infos = build_symlink_collect_infos(vec![FluxonFsTransferSymlinkNoticeEntryWire { - relpath: "root/link-file.bin".to_string(), - link_target: "target/file.bin".to_string(), - }]) - .unwrap(); + let collect_infos = + build_symlink_collect_infos(vec![FluxonFsTransferSymlinkNoticeEntryWire { + relpath: "root/link-file.bin".to_string(), + link_target: "target/file.bin".to_string(), + }]) + .unwrap(); let assignment = FluxonFsTransferWorkerAssignmentWire { collect_infos: collect_infos.clone(), ..test_worker_assignment("dir/file.bin", file_bytes.len() as i64) @@ -8976,9 +9191,11 @@ mod tests { let result = execute_transfer_worker_assignment( &assignment, &dst_root, - || Err(TransferWorkerExecutionError::Stop( - FluxonFsTransferWorkerStopReasonWire::Superseded, - )), + || { + Err(TransferWorkerExecutionError::Stop( + FluxonFsTransferWorkerStopReasonWire::Superseded, + )) + }, { let file_bytes = file_bytes.clone(); move |_file, read_offset, _length| { @@ -8996,11 +9213,20 @@ mod tests { FluxonFsTransferWorkerStopReasonWire::Superseded )) )); - assert!(root.path().join(prepared_collect.staging_relpath.as_str()).exists()); + assert!( + root.path() + .join(prepared_collect.staging_relpath.as_str()) + .exists() + ); cleanup_transfer_worker_attempt_artifacts(&dst_root, &assignment).unwrap(); assert!(!root.path().join(".fluxon.stage").exists()); - assert!(!root.path().join(prepared_collect.staging_relpath.as_str()).exists()); + assert!( + !root + .path() + .join(prepared_collect.staging_relpath.as_str()) + .exists() + ); } } diff --git a/fluxon_rs/fluxon_fs/src/cache_controller.rs b/fluxon_rs/fluxon_fs/src/cache_controller.rs index 8a0845c..13ce5a8 100644 --- a/fluxon_rs/fluxon_fs/src/cache_controller.rs +++ b/fluxon_rs/fluxon_fs/src/cache_controller.rs @@ -429,8 +429,8 @@ fn now_ms() -> i64 { #[cfg(test)] mod tests { use super::*; - use std::sync::mpsc; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; + use std::sync::mpsc; use std::sync::{Condvar, Mutex}; use tokio::time::{Duration, sleep}; diff --git a/fluxon_rs/fluxon_fs_s3_gateway/src/lib.rs b/fluxon_rs/fluxon_fs_s3_gateway/src/lib.rs index 827bb23..0866432 100644 --- a/fluxon_rs/fluxon_fs_s3_gateway/src/lib.rs +++ b/fluxon_rs/fluxon_fs_s3_gateway/src/lib.rs @@ -5344,10 +5344,9 @@ mod tests { }; use crate::transfer::encode_transfer_manifest_blob_with_empty_dirs; use fluxon_fs_core::config::{ - FS_CACHE_DEFAULT_WRITE_SESSION_TARGET_INFLIGHT_BYTES_V1, - FS_EXPORT_DEFAULT_INLINE_BYTES_MAX_BYTES_V1, - FS_EXPORT_DEFAULT_METADATA_CACHE_TTL_MS_V1, FLUXON_FS_LOCAL_TRANSFER_CHECK_DST_EXPORT, FLUXON_FS_LOCAL_TRANSFER_CHECK_SRC_EXPORT, + FS_CACHE_DEFAULT_WRITE_SESSION_TARGET_INFLIGHT_BYTES_V1, + FS_EXPORT_DEFAULT_INLINE_BYTES_MAX_BYTES_V1, FS_EXPORT_DEFAULT_METADATA_CACHE_TTL_MS_V1, FluxonFsAccessModel, FluxonFsAccessUser, FluxonFsExport, FluxonFsExportRoutingMode, FluxonFsGlobalConfig, FluxonFsLocalTransferCheckJobSpecWire, FluxonFsRequestIdentity, FluxonFsS3GatewayConfig, FluxonFsS3KvMissPolicy, FluxonFsS3PermissionAccount, diff --git a/fluxon_rs/fluxon_kv/src/client_seg_pool/mod.rs b/fluxon_rs/fluxon_kv/src/client_seg_pool/mod.rs index 1aa6954..8c7cc78 100644 --- a/fluxon_rs/fluxon_kv/src/client_seg_pool/mod.rs +++ b/fluxon_rs/fluxon_kv/src/client_seg_pool/mod.rs @@ -237,10 +237,7 @@ impl ClientSegPool { std::path::Path::new(share_mem_path).join(SIDE_TRANSFER_PEERS_DIRNAME) } - pub fn side_transfer_peer_file_path( - share_mem_path: &str, - side_id: &str, - ) -> std::path::PathBuf { + pub fn side_transfer_peer_file_path(share_mem_path: &str, side_id: &str) -> std::path::PathBuf { Self::side_transfer_peers_dir(share_mem_path).join(format!("{side_id}.json")) } @@ -399,17 +396,13 @@ impl ClientSegPool { crate::rpcresp_kvresult_convert::msg_and_error::SharedMemError::MappingFailed { path: String::new(), len: map_len as u64, - detail: "share_mem_path is empty; explicit configuration required" - .to_string(), + detail: "share_mem_path is empty; explicit configuration required".to_string(), }, )); } let base_path = &share_mem_path; - tracing::info!( - "Using share_mem_path: {} for memory-mapped file", - base_path - ); + tracing::info!("Using share_mem_path: {} for memory-mapped file", base_path); std::fs::create_dir_all(base_path).map_err(|e| { KvError::SharedMem( crate::rpcresp_kvresult_convert::msg_and_error::SharedMemError::MappingFailed { diff --git a/fluxon_rs/fluxon_kv/src/config.rs b/fluxon_rs/fluxon_kv/src/config.rs index f9c7691..1577651 100644 --- a/fluxon_rs/fluxon_kv/src/config.rs +++ b/fluxon_rs/fluxon_kv/src/config.rs @@ -733,7 +733,7 @@ pub struct ClientConfig { pub pprof_duration_seconds: Option, pub redis_compat_listen_addr: Option, pub fluxonkv_spec: FluxonKvSpec, - pub share_mem_path: String, // Mandatory shared bundle path + pub share_mem_path: String, // Mandatory shared bundle path pub large_file_paths: LargeFilePaths, // Mandatory large-file roots for logs and caches pub test_spec_config: TestSpecConfig, } @@ -1170,13 +1170,15 @@ impl ClientConfigYaml { } else { let Some(large_file_paths_yaml) = self.fluxonkv_spec.large_file_paths.as_ref() else { return Err(ConfigError::InvalidClientConfig { - detail: "fluxonkv_spec.large_file_paths is required for owner mode" - .to_string(), + detail: "fluxonkv_spec.large_file_paths is required for owner mode".to_string(), } .into_kverror()); }; LargeFilePaths { - paths: verify_non_empty_root_path_list(&large_file_paths_yaml.0, "large_file_paths")?, + paths: verify_non_empty_root_path_list( + &large_file_paths_yaml.0, + "large_file_paths", + )?, } }; @@ -1647,7 +1649,9 @@ fluxonkv_spec: .unwrap(); let err = cfg.verify().unwrap_err(); let text = format!("{err}"); - assert!(text.contains("fluxonkv_spec.large_file_paths is forbidden in zero-contribution mode")); + assert!( + text.contains("fluxonkv_spec.large_file_paths is forbidden in zero-contribution mode") + ); } #[test] @@ -1667,7 +1671,9 @@ fluxonkv_spec: let logs_dir = large_file_paths.kv_logs_dir("test_cluster").unwrap(); assert_eq!( logs_dir, - first_root.join("child").join("test_cluster_cluster_kv_logs") + first_root + .join("child") + .join("test_cluster_cluster_kv_logs") ); assert!(logs_dir.exists()); diff --git a/fluxon_rs/fluxon_kv/src/external_client_api/mod.rs b/fluxon_rs/fluxon_kv/src/external_client_api/mod.rs index 9cb291f..b7715dd 100644 --- a/fluxon_rs/fluxon_kv/src/external_client_api/mod.rs +++ b/fluxon_rs/fluxon_kv/src/external_client_api/mod.rs @@ -865,8 +865,7 @@ impl ExternalInner { return Ok(false); } - self.finish_owner_recover(&share_mem_path, payload) - .await?; + self.finish_owner_recover(&share_mem_path, payload).await?; Ok(true) } diff --git a/fluxon_rs/fluxon_kv/src/kv_test.rs b/fluxon_rs/fluxon_kv/src/kv_test.rs index eb9e6c9..910aac8 100644 --- a/fluxon_rs/fluxon_kv/src/kv_test.rs +++ b/fluxon_rs/fluxon_kv/src/kv_test.rs @@ -11,8 +11,9 @@ use crate::cluster_manager::ClusterManagerRdmaControlInit; use crate::config::{ - ClientConfig, ContributeToClusterPoolSize, FluxonKvSpec, LargeFilePaths, MasterConfig, MonitoringConfig, - ProtocolConfig, ProtocolType, TestSpecConfig, TestSpecTransportMode, TransferEngineType, + ClientConfig, ContributeToClusterPoolSize, FluxonKvSpec, LargeFilePaths, MasterConfig, + MonitoringConfig, ProtocolConfig, ProtocolType, TestSpecConfig, TestSpecTransportMode, + TransferEngineType, }; use crate::run_master_with_test_overrides; use crate::{ClientRunTestOverrides, MasterRunTestOverrides, run_client_with_test_overrides}; @@ -802,7 +803,6 @@ impl KvTestRoundOptions { kv_test_run_scope() ) } - } #[derive(Clone, Debug)] @@ -842,8 +842,7 @@ fn default_client_large_file_paths( instance_key: &str, contribute_to_cluster_pool_size: &ContributeToClusterPoolSize, ) -> LargeFilePaths { - if contribute_to_cluster_pool_size.dram == 0 - && contribute_to_cluster_pool_size.vram.is_empty() + if contribute_to_cluster_pool_size.dram == 0 && contribute_to_cluster_pool_size.vram.is_empty() { return LargeFilePaths { paths: Vec::new() }; } diff --git a/fluxon_rs/fluxon_kv/src/lib.rs b/fluxon_rs/fluxon_kv/src/lib.rs index 4b8ca79..a7fd905 100644 --- a/fluxon_rs/fluxon_kv/src/lib.rs +++ b/fluxon_rs/fluxon_kv/src/lib.rs @@ -2959,8 +2959,8 @@ mod tests { large_file_paths: crate::config::LargeFilePaths { paths: vec![owner_large_root.to_string_lossy().into_owned()], }, - protocol_version: - fluxon_util::git_version_build_record::get_current_git_commitid().unwrap(), + protocol_version: fluxon_util::git_version_build_record::get_current_git_commitid() + .unwrap(), write_ts: Some(chrono::Utc::now().timestamp_micros()), }; let shared_meta_json = serde_json::to_string(&shared_meta).unwrap(); diff --git a/fluxon_rs/fluxon_kv/src/master_lease_manager/lease_manager_test.rs b/fluxon_rs/fluxon_kv/src/master_lease_manager/lease_manager_test.rs index 5c20cc1..5d344c9 100755 --- a/fluxon_rs/fluxon_kv/src/master_lease_manager/lease_manager_test.rs +++ b/fluxon_rs/fluxon_kv/src/master_lease_manager/lease_manager_test.rs @@ -22,7 +22,8 @@ async fn test1_lease_expire_removes_keys() { unsafe { std::env::set_var("FLUXON_LOG", "debug"); } - let (master_fw, client_fw) = start_master_and_client("lease_master_t1", "lease_client_t1").await; + let (master_fw, client_fw) = + start_master_and_client("lease_master_t1", "lease_client_t1").await; let client_view = client_fw.client_kv_api_view(); wait_master_ready(&client_view).await; @@ -82,7 +83,8 @@ async fn test2_rebind_to_new_lease_preserves_until_new_expire() { unsafe { std::env::set_var("FLUXON_LOG", "debug"); } - let (master_fw, client_fw) = start_master_and_client("lease_master_t2", "lease_client_t2").await; + let (master_fw, client_fw) = + start_master_and_client("lease_master_t2", "lease_client_t2").await; let client_view = client_fw.client_kv_api_view(); wait_master_ready(&client_view).await; @@ -161,7 +163,8 @@ async fn test3_keepalive() { unsafe { std::env::set_var("FLUXON_LOG", "debug"); } - let (master_fw, client_fw) = start_master_and_client("lease_master_t3", "lease_client_t3").await; + let (master_fw, client_fw) = + start_master_and_client("lease_master_t3", "lease_client_t3").await; let client_view = client_fw.client_kv_api_view(); wait_master_ready(&client_view).await; @@ -236,7 +239,8 @@ async fn test4_delete_under_lease_then_get_fails() { unsafe { std::env::set_var("FLUXON_LOG", "debug"); } - let (master_fw, client_fw) = start_master_and_client("lease_master_t4", "lease_client_t4").await; + let (master_fw, client_fw) = + start_master_and_client("lease_master_t4", "lease_client_t4").await; let client_view = client_fw.client_kv_api_view(); wait_master_ready(&client_view).await; diff --git a/fluxon_rs/fluxon_mq/src/consumer.rs b/fluxon_rs/fluxon_mq/src/consumer.rs index 8da2bb5..91e4c6f 100644 --- a/fluxon_rs/fluxon_mq/src/consumer.rs +++ b/fluxon_rs/fluxon_mq/src/consumer.rs @@ -55,6 +55,10 @@ const PREFETCH_LATENCY_LOG_INTERVAL: Duration = NO_MESSAGE_WARN_INTERVAL; const PREFETCH_LATENCY_WINDOW_SIZE: usize = 16; const NONBLOCKING_QUEUE_WAIT_THRESHOLD: Duration = Duration::from_millis(500); const DELETE_CALLBACK_WARN_INTERVAL: Duration = Duration::from_secs(1); +const BROKER_CLEANUP_DELETE_RETRY_INITIAL_SLEEP: Duration = Duration::from_millis(50); +const BROKER_CLEANUP_DELETE_RETRY_MAX_SLEEP: Duration = Duration::from_secs(5); +const BROKER_CLEANUP_ACK_RETRY_INITIAL_SLEEP: Duration = Duration::from_millis(50); +const BROKER_CLEANUP_ACK_RETRY_MAX_SLEEP: Duration = Duration::from_secs(5); const COMMIT_WAIT_WARN_INTERVAL: Duration = Duration::from_secs(10); const COMMIT_WAIT_BREAKDOWN_SUMMARY_THRESHOLD: Duration = Duration::from_millis(50); const COMMIT_OFFSET_PUT_TIMEOUT: Duration = Duration::from_secs(10); @@ -2276,7 +2280,7 @@ async fn get_payload_via_broker( let payload_key = envelope.payload_key.clone(); let mut requeue_guard = BrokerInflightRequeueGuard::new(broker.clone(), chan_id, vec![reservation_id]); - let mut payload = match run_payload_callback( + let payload = match run_payload_callback( chan_id, cb, producer_id.clone(), @@ -2311,15 +2315,7 @@ async fn get_payload_via_broker( } if let Some(envelope) = commit_outcome.cleanup { - attach_or_run_broker_cleanup( - payload.as_mut(), - broker.clone(), - chan_id, - delete_cb.clone(), - shutdown.clone(), - envelope, - ) - .await?; + spawn_broker_cleanup(broker.clone(), chan_id, delete_cb.clone(), envelope); } Ok(ConsumedPayload { @@ -2556,7 +2552,7 @@ async fn load_broker_payloads_commit_on_ready( continue; }; - let mut payload = match payload_result { + let payload = match payload_result { Ok(payload) => payload, Err(err) => { stop_error = Some(err); @@ -2589,25 +2585,7 @@ async fn load_broker_payloads_commit_on_ready( continue; } if let Some(envelope) = commit_outcome.cleanup { - if let Err(err) = attach_or_run_broker_cleanup( - payload.payload.as_mut(), - broker.clone(), - chan_id, - delete_cb.clone(), - shutdown.clone(), - envelope, - ) - .await - { - warn!( - "broker cleanup failed during batch consume: chan_id={} consumer_id={} reservation_id={} err={}", - chan_id, consumer_id, reservation_id, err - ); - committed_payloads.push(payload); - stop_error = Some(err); - stop_after_current = true; - continue; - } + spawn_broker_cleanup(broker.clone(), chan_id, delete_cb.clone(), envelope); } committed_payloads.push(payload); @@ -2666,18 +2644,15 @@ async fn run_payload_callback( } } -async fn run_delete_callback( +async fn run_delete_callback_until_success( chan_id: i64, delete_cb: &DeleteCallback, payload_key: String, - shutdown: &ShutdownCtl, -) -> Result<(), MpscError> { +) { use tokio::time::sleep; + let mut retry_sleep = BROKER_CLEANUP_DELETE_RETRY_INITIAL_SLEEP; loop { - if shutdown.is_closed() { - return Ok(()); - } let f = delete_cb.clone(); let key_clone = payload_key.clone(); let delete_begin = Instant::now(); @@ -2685,21 +2660,12 @@ async fn run_delete_callback( tokio::pin!(delete_fut); let res = loop { tokio::select! { - biased; - _ = shutdown.wait_closed() => { - debug!( - "[MpscConsumer chan_id={}] stop delete callback on shutdown: key={}", - chan_id, - key_clone, - ); - break DeleteResult::Ok; - } res = &mut delete_fut => { break res; } _ = sleep(DELETE_CALLBACK_WARN_INTERVAL) => { warn!( - "[MpscConsumer chan_id={}] delete callback still pending: key={} waited_ms={}", + "[MpscConsumer chan_id={}] async broker delete callback still pending: key={} waited_ms={}", chan_id, key_clone, delete_begin.elapsed().as_millis(), @@ -2708,94 +2674,73 @@ async fn run_delete_callback( } }; match res { - DeleteResult::Ok => return Ok(()), + DeleteResult::Ok => return, DeleteResult::Retryable(msg) => { warn!( - "[MpscConsumer chan_id={}] delete payload retryable: {}", - chan_id, msg + "[MpscConsumer chan_id={}] async broker delete payload retryable; retry_after_ms={}: {}", + chan_id, + retry_sleep.as_millis(), + msg ); - sleep(Duration::from_millis(50)).await; } DeleteResult::NonRetryable(msg) => { - return Err(MpscError::DeletePayloadNonRetryable { message: msg }); + warn!( + "[MpscConsumer chan_id={}] async broker delete payload non-retryable; keep retrying to preserve broker byte budget; retry_after_ms={}: {}", + chan_id, + retry_sleep.as_millis(), + msg + ); } } + sleep(retry_sleep).await; + retry_sleep = retry_sleep + .saturating_mul(2) + .min(BROKER_CLEANUP_DELETE_RETRY_MAX_SLEEP); } } -async fn cleanup_broker_envelope( - broker: &BrokerHandle, +async fn run_broker_cleanup_ack_until_success( + broker: BrokerHandle, chan_id: i64, - delete_cb: Option<&DeleteCallback>, - shutdown: &ShutdownCtl, - envelope: BrokerEnvelope, -) -> Result<(), MpscError> { - let reservation_id = envelope.reservation_id; - if let Some(delete_cb) = delete_cb { - run_delete_callback(chan_id, delete_cb, envelope.payload_key, shutdown).await?; + reservation_id: u64, +) { + use tokio::time::sleep; + + let mut retry_sleep = BROKER_CLEANUP_ACK_RETRY_INITIAL_SLEEP; + loop { + match broker.cleanup_ack(chan_id, reservation_id).await { + Ok(()) => return, + Err(err) => { + warn!( + "async broker cleanup ack failed; retry_after_ms={}: chan_id={} reservation_id={} err={}", + retry_sleep.as_millis(), + chan_id, + reservation_id, + err + ); + } + } + sleep(retry_sleep).await; + retry_sleep = retry_sleep + .saturating_mul(2) + .min(BROKER_CLEANUP_ACK_RETRY_MAX_SLEEP); } - broker - .cleanup_ack(chan_id, reservation_id) - .await - .map_err(|e| { - MpscError::Internal(format!( - "broker cleanup ack failed: chan_id={} reservation_id={} err={}", - chan_id, reservation_id, e - )) - })?; - Ok(()) } -async fn attach_or_run_broker_cleanup( - payload: &mut dyn MqPayload, +fn spawn_broker_cleanup( broker: BrokerHandle, chan_id: i64, delete_cb: Option, - shutdown: ShutdownCtl, envelope: BrokerEnvelope, -) -> Result<(), MpscError> { - let cleanup_envelope = envelope.clone(); - let deferred_broker = broker.clone(); - let deferred_delete_cb = delete_cb.clone(); - let deferred_shutdown = shutdown.clone(); - let cleanup = Box::new(move || { - Box::pin(async move { - if let Some(delete_cb) = deferred_delete_cb.as_ref() { - if let Err(err) = run_delete_callback( - chan_id, - delete_cb, - cleanup_envelope.payload_key.clone(), - &deferred_shutdown, - ) - .await - { - warn!( - "deferred broker payload delete failed: chan_id={} reservation_id={} err={}", - chan_id, cleanup_envelope.reservation_id, err - ); - let _ = deferred_broker - .cleanup_nack(chan_id, cleanup_envelope.reservation_id) - .await; - return; - } - } - if let Err(err) = deferred_broker - .cleanup_ack(chan_id, cleanup_envelope.reservation_id) - .await - { - warn!( - "deferred broker cleanup ack failed: chan_id={} reservation_id={} err={}", - chan_id, cleanup_envelope.reservation_id, err - ); - } - }) as PayloadCleanupFuture - }); - match payload.attach_cleanup(cleanup) { - Ok(()) => Ok(()), - Err(_) => { - cleanup_broker_envelope(&broker, chan_id, delete_cb.as_ref(), &shutdown, envelope).await +) { + tokio::spawn(async move { + let reservation_id = envelope.reservation_id; + if let Some(delete_cb) = delete_cb.as_ref() { + run_delete_callback_until_success(chan_id, delete_cb, envelope.payload_key.clone()) + .await; } - } + run_broker_cleanup_ack_until_success(broker, chan_id, reservation_id).await; + }); } async fn requeue_pending_broker_inflight( diff --git a/fluxon_rs/fluxon_ops/build.rs b/fluxon_rs/fluxon_ops/build.rs index 585fbfc..51e95c4 100644 --- a/fluxon_rs/fluxon_ops/build.rs +++ b/fluxon_rs/fluxon_ops/build.rs @@ -59,9 +59,17 @@ print( } fn render_log_shard_helper(repo_root: &Path) -> String { - let helper_path = repo_root.join("deployment").join("utils").join("log_shard.py"); - fs::read_to_string(&helper_path) - .unwrap_or_else(|e| panic!("read log shard helper failed: {} ({})", helper_path.display(), e)) + let helper_path = repo_root + .join("deployment") + .join("utils") + .join("log_shard.py"); + fs::read_to_string(&helper_path).unwrap_or_else(|e| { + panic!( + "read log shard helper failed: {} ({})", + helper_path.display(), + e + ) + }) } fn main() { @@ -87,6 +95,10 @@ fn main() { ); println!( "cargo:rerun-if-changed={}", - repo_root.join("deployment").join("utils").join("log_shard.py").display() + repo_root + .join("deployment") + .join("utils") + .join("log_shard.py") + .display() ); } diff --git a/fluxon_rs/fluxon_ops/src/lib.rs b/fluxon_rs/fluxon_ops/src/lib.rs index 29d9434..3adb053 100644 --- a/fluxon_rs/fluxon_ops/src/lib.rs +++ b/fluxon_rs/fluxon_ops/src/lib.rs @@ -80,7 +80,8 @@ const DELETE_APPLY_NO_WAIT_DELAY_SECONDS: u64 = 30; const EMBEDDED_SELECTION_SUPERVISOR_SOURCE: &str = include_str!(concat!(env!("OUT_DIR"), "/selection_supervisor.py")); -const EMBEDDED_LOG_SHARD_HELPER_SOURCE: &str = include_str!(concat!(env!("OUT_DIR"), "/log_shard.py")); +const EMBEDDED_LOG_SHARD_HELPER_SOURCE: &str = + include_str!(concat!(env!("OUT_DIR"), "/log_shard.py")); // Ops controller uses Fluxon user-RPC to talk to ops agents. // Keep the timeout as a fixed constant to avoid config surface area. @@ -351,7 +352,10 @@ fn workload_log_latest_shard_identity(logical_path: &Path) -> anyhow::Result anyhow::Result Ok(resolved) } -fn ensure_embedded_selection_supervisor_runtime(workdir: &Path) -> anyhow::Result<(PathBuf, PathBuf)> { +fn ensure_embedded_selection_supervisor_runtime( + workdir: &Path, +) -> anyhow::Result<(PathBuf, PathBuf)> { let runtime_dir = workdir.join(OPS_SELECTION_SUPERVISOR_DIR_NAME); std::fs::create_dir_all(&runtime_dir).with_context(|| { format!( @@ -1657,10 +1663,11 @@ fn selection_owner_supervisor( scope_key: Option<&str>, exclude_pid: Option, ) -> anyhow::Result> { - let owners: Vec = live_selection_supervisors(snapshot, Some(label), scope_key)? - .into_iter() - .filter(|supervisor| exclude_pid != Some(supervisor.pid())) - .collect(); + let owners: Vec = + live_selection_supervisors(snapshot, Some(label), scope_key)? + .into_iter() + .filter(|supervisor| exclude_pid != Some(supervisor.pid())) + .collect(); if owners.is_empty() { return Ok(None); } @@ -2068,7 +2075,16 @@ fn wait_for_selection_attached( argv: &[String], cwd: Option<&str>, ) -> anyhow::Result { - wait_for_selection_attached_for_scope(kind, name, authority, None, apply_id, owner_ts_ms, argv, cwd) + wait_for_selection_attached_for_scope( + kind, + name, + authority, + None, + apply_id, + owner_ts_ms, + argv, + cwd, + ) } fn wait_for_selection_attached_without_present_for_scope( @@ -2803,10 +2819,9 @@ impl SupervisorBackedWorkloads { fn list_workloads(&self) -> anyhow::Result> { let mut out: Vec = Vec::new(); let snapshot = selection_supervisor_proc_snapshot()?; - for status in observe_all_selection_statuses_for_snapshot( - &snapshot, - Some(self.scope_key.as_str()), - )? { + for status in + observe_all_selection_statuses_for_snapshot(&snapshot, Some(self.scope_key.as_str()))? + { let kind = status.kind.with_context(|| { format!( "selection supervisor list item missing kind: label={}", @@ -3159,7 +3174,10 @@ impl UserRpcHandler for ReadWorkloadLogChunkHandler { None => { let Some(path) = resolve_readable_log_path(&logical_path) else { let resp = make_err_resp( - format!("log file is not available yet: logical_path={}", logical_path.display()), + format!( + "log file is not available yet: logical_path={}", + logical_path.display() + ), None, ); return Ok(serde_json::to_vec(&resp).unwrap()); @@ -3186,138 +3204,11 @@ impl UserRpcHandler for ReadWorkloadLogChunkHandler { }; let file_size = meta.len(); - let (start, end, start_cursor, end_cursor, effective_path, effective_file_size) = - match req.direction { - LogReadDirection::Forward => { - if let Some(cursor) = req.cursor.as_ref() { - if cursor.offset > file_size { - let resp = make_err_resp( - format!( - "cursor out of range: shard={} cursor={} file_size={}", - cursor.shard, cursor.offset, file_size - ), - Some(file_size), - ); - return Ok(serde_json::to_vec(&resp).unwrap()); - } - let mut effective_path = path.clone(); - let mut effective_shard = shard.clone(); - let mut effective_file_size = file_size; - let mut start = cursor.offset; - if cursor.offset == file_size { - if let Ok(Some(next_shard)) = - workload_log_next_shard(&logical_path, &cursor.shard) - { - let next_path = match workload_log_path_for_shard(&logical_path, &next_shard) { - Ok(v) => v, - Err(e) => { - let resp = make_err_resp(format!("{}", e), Some(file_size)); - return Ok(serde_json::to_vec(&resp).unwrap()); - } - }; - match std::fs::metadata(&next_path) { - Ok(next_meta) => { - effective_file_size = next_meta.len(); - effective_path = next_path; - effective_shard = next_shard; - start = 0; - } - Err(e) => { - let resp = make_err_resp( - format!( - "stat next log shard failed: path={} err={}", - next_path.display(), - e - ), - Some(file_size), - ); - return Ok(serde_json::to_vec(&resp).unwrap()); - } - } - } else if let Ok(Some(latest_shard)) = - workload_log_latest_shard_identity(&logical_path) - { - if latest_shard != cursor.shard { - let latest_path = - match workload_log_path_for_shard(&logical_path, &latest_shard) { - Ok(v) => v, - Err(e) => { - let resp = make_err_resp(format!("{}", e), Some(file_size)); - return Ok(serde_json::to_vec(&resp).unwrap()); - } - }; - match std::fs::metadata(&latest_path) { - Ok(latest_meta) => { - effective_file_size = latest_meta.len(); - effective_path = latest_path; - effective_shard = latest_shard; - start = 0; - } - Err(e) => { - let resp = make_err_resp( - format!( - "stat latest log shard failed: path={} err={}", - latest_path.display(), - e - ), - Some(file_size), - ); - return Ok(serde_json::to_vec(&resp).unwrap()); - } - } - } - } - } - let end = match max_bytes { - Some(max_bytes) => { - std::cmp::min(effective_file_size, start.saturating_add(max_bytes)) - } - None => effective_file_size, - }; - ( - start, - end, - Some(WorkloadLogCursor { - shard: effective_shard.clone(), - offset: start, - }), - Some(WorkloadLogCursor { - shard: effective_shard.clone(), - offset: end, - }), - effective_path, - effective_file_size, - ) - } else { - let end = file_size; - let start = match max_bytes { - Some(max_bytes) => end.saturating_sub(max_bytes), - None => 0, - }; - ( - start, - end, - Some(WorkloadLogCursor { - shard: shard.clone(), - offset: start, - }), - Some(WorkloadLogCursor { - shard: shard.clone(), - offset: end, - }), - path.clone(), - file_size, - ) - } - } - LogReadDirection::Backward => { - let Some(cursor) = req.cursor.as_ref() else { - let resp = make_err_resp( - "cursor is required for Backward reads".to_string(), - Some(file_size), - ); - return Ok(serde_json::to_vec(&resp).unwrap()); - }; + let (start, end, start_cursor, end_cursor, effective_path, effective_file_size) = match req + .direction + { + LogReadDirection::Forward => { + if let Some(cursor) = req.cursor.as_ref() { if cursor.offset > file_size { let resp = make_err_resp( format!( @@ -3331,30 +3222,31 @@ impl UserRpcHandler for ReadWorkloadLogChunkHandler { let mut effective_path = path.clone(); let mut effective_shard = shard.clone(); let mut effective_file_size = file_size; - let mut end = cursor.offset; - if cursor.offset == 0 { - if let Ok(Some(prev_shard)) = - workload_log_previous_shard(&logical_path, &cursor.shard) + let mut start = cursor.offset; + if cursor.offset == file_size { + if let Ok(Some(next_shard)) = + workload_log_next_shard(&logical_path, &cursor.shard) { - let prev_path = match workload_log_path_for_shard(&logical_path, &prev_shard) { - Ok(v) => v, - Err(e) => { - let resp = make_err_resp(format!("{}", e), Some(file_size)); - return Ok(serde_json::to_vec(&resp).unwrap()); - } - }; - match std::fs::metadata(&prev_path) { - Ok(prev_meta) => { - effective_file_size = prev_meta.len(); - effective_path = prev_path; - effective_shard = prev_shard; - end = effective_file_size; + let next_path = + match workload_log_path_for_shard(&logical_path, &next_shard) { + Ok(v) => v, + Err(e) => { + let resp = make_err_resp(format!("{}", e), Some(file_size)); + return Ok(serde_json::to_vec(&resp).unwrap()); + } + }; + match std::fs::metadata(&next_path) { + Ok(next_meta) => { + effective_file_size = next_meta.len(); + effective_path = next_path; + effective_shard = next_shard; + start = 0; } Err(e) => { let resp = make_err_resp( format!( - "stat previous log shard failed: path={} err={}", - prev_path.display(), + "stat next log shard failed: path={} err={}", + next_path.display(), e ), Some(file_size), @@ -3362,11 +3254,47 @@ impl UserRpcHandler for ReadWorkloadLogChunkHandler { return Ok(serde_json::to_vec(&resp).unwrap()); } } + } else if let Ok(Some(latest_shard)) = + workload_log_latest_shard_identity(&logical_path) + { + if latest_shard != cursor.shard { + let latest_path = + match workload_log_path_for_shard(&logical_path, &latest_shard) + { + Ok(v) => v, + Err(e) => { + let resp = + make_err_resp(format!("{}", e), Some(file_size)); + return Ok(serde_json::to_vec(&resp).unwrap()); + } + }; + match std::fs::metadata(&latest_path) { + Ok(latest_meta) => { + effective_file_size = latest_meta.len(); + effective_path = latest_path; + effective_shard = latest_shard; + start = 0; + } + Err(e) => { + let resp = make_err_resp( + format!( + "stat latest log shard failed: path={} err={}", + latest_path.display(), + e + ), + Some(file_size), + ); + return Ok(serde_json::to_vec(&resp).unwrap()); + } + } + } } } - let start = match max_bytes { - Some(max_bytes) => end.saturating_sub(max_bytes), - None => 0, + let end = match max_bytes { + Some(max_bytes) => { + std::cmp::min(effective_file_size, start.saturating_add(max_bytes)) + } + None => effective_file_size, }; ( start, @@ -3382,8 +3310,103 @@ impl UserRpcHandler for ReadWorkloadLogChunkHandler { effective_path, effective_file_size, ) + } else { + let end = file_size; + let start = match max_bytes { + Some(max_bytes) => end.saturating_sub(max_bytes), + None => 0, + }; + ( + start, + end, + Some(WorkloadLogCursor { + shard: shard.clone(), + offset: start, + }), + Some(WorkloadLogCursor { + shard: shard.clone(), + offset: end, + }), + path.clone(), + file_size, + ) } - }; + } + LogReadDirection::Backward => { + let Some(cursor) = req.cursor.as_ref() else { + let resp = make_err_resp( + "cursor is required for Backward reads".to_string(), + Some(file_size), + ); + return Ok(serde_json::to_vec(&resp).unwrap()); + }; + if cursor.offset > file_size { + let resp = make_err_resp( + format!( + "cursor out of range: shard={} cursor={} file_size={}", + cursor.shard, cursor.offset, file_size + ), + Some(file_size), + ); + return Ok(serde_json::to_vec(&resp).unwrap()); + } + let mut effective_path = path.clone(); + let mut effective_shard = shard.clone(); + let mut effective_file_size = file_size; + let mut end = cursor.offset; + if cursor.offset == 0 { + if let Ok(Some(prev_shard)) = + workload_log_previous_shard(&logical_path, &cursor.shard) + { + let prev_path = + match workload_log_path_for_shard(&logical_path, &prev_shard) { + Ok(v) => v, + Err(e) => { + let resp = make_err_resp(format!("{}", e), Some(file_size)); + return Ok(serde_json::to_vec(&resp).unwrap()); + } + }; + match std::fs::metadata(&prev_path) { + Ok(prev_meta) => { + effective_file_size = prev_meta.len(); + effective_path = prev_path; + effective_shard = prev_shard; + end = effective_file_size; + } + Err(e) => { + let resp = make_err_resp( + format!( + "stat previous log shard failed: path={} err={}", + prev_path.display(), + e + ), + Some(file_size), + ); + return Ok(serde_json::to_vec(&resp).unwrap()); + } + } + } + } + let start = match max_bytes { + Some(max_bytes) => end.saturating_sub(max_bytes), + None => 0, + }; + ( + start, + end, + Some(WorkloadLogCursor { + shard: effective_shard.clone(), + offset: start, + }), + Some(WorkloadLogCursor { + shard: effective_shard.clone(), + offset: end, + }), + effective_path, + effective_file_size, + ) + } + }; if end < start { let resp = make_err_resp( @@ -3416,7 +3439,11 @@ impl UserRpcHandler for ReadWorkloadLogChunkHandler { Ok(v) => v, Err(e) => { let resp = make_err_resp( - format!("open log failed: path={} err={}", effective_path.display(), e), + format!( + "open log failed: path={} err={}", + effective_path.display(), + e + ), Some(effective_file_size), ); return Ok(serde_json::to_vec(&resp).unwrap()); @@ -3425,7 +3452,11 @@ impl UserRpcHandler for ReadWorkloadLogChunkHandler { if let Err(e) = std::io::Seek::seek(&mut f, std::io::SeekFrom::Start(start)) { let resp = make_err_resp( - format!("seek log failed: path={} err={}", effective_path.display(), e), + format!( + "seek log failed: path={} err={}", + effective_path.display(), + e + ), Some(effective_file_size), ); return Ok(serde_json::to_vec(&resp).unwrap()); @@ -3434,7 +3465,11 @@ impl UserRpcHandler for ReadWorkloadLogChunkHandler { let mut buf: Vec = vec![0; len]; if let Err(e) = std::io::Read::read_exact(&mut f, &mut buf) { let resp = make_err_resp( - format!("read log failed: path={} err={}", effective_path.display(), e), + format!( + "read log failed: path={} err={}", + effective_path.display(), + e + ), Some(effective_file_size), ); return Ok(serde_json::to_vec(&resp).unwrap()); @@ -4067,8 +4102,7 @@ fn desired_workload_matches_running( &desired.name, &desired.authority, Some(workloads.scope_key.as_str()), - ) - else { + ) else { return false; }; desired_workload_status_matches_goal(&status, desired) @@ -14337,14 +14371,14 @@ mod tests { assert_eq!(scoped_b.len(), 1); assert_eq!(scoped_b[0].pid(), 22); - let listed_a = observe_all_selection_statuses_for_snapshot(&snapshot, Some("/tmp/scope-a")) - .unwrap(); + let listed_a = + observe_all_selection_statuses_for_snapshot(&snapshot, Some("/tmp/scope-a")).unwrap(); assert_eq!(listed_a.len(), 1); assert_eq!(listed_a[0].label, "DaemonSet/target"); assert_eq!(listed_a[0].pid, Some(11)); - let listed_b = observe_all_selection_statuses_for_snapshot(&snapshot, Some("/tmp/scope-b")) - .unwrap(); + let listed_b = + observe_all_selection_statuses_for_snapshot(&snapshot, Some("/tmp/scope-b")).unwrap(); assert_eq!(listed_b.len(), 1); assert_eq!(listed_b[0].label, "DaemonSet/target"); assert_eq!(listed_b[0].pid, Some(22)); @@ -14548,8 +14582,8 @@ mod tests { zombie_infos: Vec::new(), }; - let listed = observe_apply_runtime_statuses_for_snapshot("apply-1", &snapshot, None) - .unwrap(); + let listed = + observe_apply_runtime_statuses_for_snapshot("apply-1", &snapshot, None).unwrap(); assert_eq!(listed.len(), 1); assert_eq!(listed[0].name.as_deref(), Some("target-present")); assert!(listed[0].present); @@ -14774,12 +14808,8 @@ mod tests { None )); - let delete_old = workloads.delete_generation( - WorkloadKind::Deployment, - &name, - &name, - Some("apply-1"), - ); + let delete_old = + workloads.delete_generation(WorkloadKind::Deployment, &name, &name, Some("apply-1")); if !delete_old.ok { let err = delete_old.err.as_deref().unwrap_or_default(); assert!( @@ -14813,8 +14843,7 @@ mod tests { delete_current.ok, "unguarded delete should bind and retire the current visible generation: {delete_current:?}" ); - wait_for_selection_absent(WorkloadKind::Deployment, &name, &name, Some("apply-2")) - .unwrap(); + wait_for_selection_absent(WorkloadKind::Deployment, &name, &name, Some("apply-2")).unwrap(); } #[test] @@ -14826,9 +14855,12 @@ mod tests { python_exe.display() ); let workdir = tempfile::tempdir().unwrap(); - let runtime = - SelectionSupervisorRuntime::materialize(workdir.path(), workdir.path(), python_exe.as_path()) - .unwrap(); + let runtime = SelectionSupervisorRuntime::materialize( + workdir.path(), + workdir.path(), + python_exe.as_path(), + ) + .unwrap(); assert!(runtime.script_path.exists()); assert!( runtime @@ -14849,9 +14881,12 @@ mod tests { python_exe.display() ); let workdir = tempfile::tempdir().unwrap(); - let runtime = - SelectionSupervisorRuntime::materialize(workdir.path(), workdir.path(), python_exe.as_path()) - .unwrap(); + let runtime = SelectionSupervisorRuntime::materialize( + workdir.path(), + workdir.path(), + python_exe.as_path(), + ) + .unwrap(); let log_path = workdir.path().join("startup.log"); let command = vec![ python_exe.display().to_string(), @@ -14876,7 +14911,9 @@ mod tests { "--".to_string(), "/bin/true".to_string(), ]; - let pid = runtime.spawn_detached_command(&log_path, command.as_slice()).unwrap(); + let pid = runtime + .spawn_detached_command(&log_path, command.as_slice()) + .unwrap(); let deadline = Instant::now() + Duration::from_secs(10); let expected = "owner-ts-ms must be positive"; let mut saw_expected = false; @@ -15164,7 +15201,9 @@ mod tests { }), max_bytes: Some(65536), }; - let raw = handler.handle("n1".into(), &serde_json::to_vec(&req).unwrap()).unwrap(); + let raw = handler + .handle("n1".into(), &serde_json::to_vec(&req).unwrap()) + .unwrap(); let resp: ReadWorkloadLogResp = serde_json::from_slice(&raw).unwrap(); assert!(resp.ok, "{resp:?}"); assert_eq!(resp.text.as_deref(), Some("new\n")); @@ -15209,7 +15248,9 @@ mod tests { }), max_bytes: Some(65536), }; - let raw = handler.handle("n1".into(), &serde_json::to_vec(&req).unwrap()).unwrap(); + let raw = handler + .handle("n1".into(), &serde_json::to_vec(&req).unwrap()) + .unwrap(); let resp: ReadWorkloadLogResp = serde_json::from_slice(&raw).unwrap(); assert!(resp.ok, "{resp:?}"); assert_eq!(resp.text.as_deref(), Some("old\n")); diff --git a/fluxon_rs/fluxon_pyo3/src/error.rs b/fluxon_rs/fluxon_pyo3/src/error.rs index 6f59b8c..e153ebc 100644 --- a/fluxon_rs/fluxon_pyo3/src/error.rs +++ b/fluxon_rs/fluxon_pyo3/src/error.rs @@ -1,6 +1,6 @@ +use pyo3::PyErr; use pyo3::prelude::*; use pyo3::types::PyDict; -use pyo3::PyErr; use fluxon_mq::MpscError as CoreMpscError; // Re-export the core MPSC error type for callers who want to depend on a single error hub. diff --git a/fluxon_rs/fluxon_pyo3/src/mpsc.rs b/fluxon_rs/fluxon_pyo3/src/mpsc.rs index 1b1759d..c0820ae 100644 --- a/fluxon_rs/fluxon_pyo3/src/mpsc.rs +++ b/fluxon_rs/fluxon_pyo3/src/mpsc.rs @@ -3,31 +3,31 @@ use std::sync::{Arc, Mutex, OnceLock}; use std::time::{Duration, Instant}; use crossbeam_channel as cbchan; +use fluxon_mq::DeleteResult as CoreDeleteResult; use fluxon_mq::consumer::{ ConsumedPayload as CoreConsumedPayload, MqPayload as CoreMqPayload, PayloadResult as CorePayloadResult, }; -use fluxon_mq::DeleteResult as CoreDeleteResult; use fluxon_mq::{ - create::{create_mpsc_channel, ChanCreateConfig}, BrokerChannelConfig, BrokerHandle, ChanManager, MpscConsumer as CoreMpscConsumer, MpscError as CoreMpscError, MpscProducer as CoreMpscProducer, ShutdownCtl, + create::{ChanCreateConfig, create_mpsc_channel}, }; -use pyo3::prelude::*; -use pyo3::types::{PyAny, PyBytes, PyString}; use pyo3::Py; use pyo3::PyErr; +use pyo3::prelude::*; +use pyo3::types::{PyAny, PyBytes, PyString}; use tokio::runtime::Handle; use tokio::runtime::Runtime; // (no local payload buffering) use crate::flatdict_zerocopy::{ - attach_cleanup_to_flatdict_pyobject, decode_flat_dict_to_wrapped_py_object, FlatDictDataOwner, + FlatDictDataOwner, attach_cleanup_to_flatdict_pyobject, decode_flat_dict_to_wrapped_py_object, }; use crate::lease_manager::PyLeaseBackendUid; use fluxon_kv::{Framework as KvFramework, KvClientTrait}; use fluxon_mq::lease_manager::LeaseBackendUid; -use fluxon_util::lease_manager::{LeaseManager, GLOBAL_LM}; +use fluxon_util::lease_manager::{GLOBAL_LM, LeaseManager}; use fluxon_util::run_async_from_sync::SyncAsyncBridge; use tracing::{debug, warn}; diff --git a/fluxon_rs/fluxon_util/src/dev_config.rs b/fluxon_rs/fluxon_util/src/dev_config.rs index c860910..d3f5953 100644 --- a/fluxon_rs/fluxon_util/src/dev_config.rs +++ b/fluxon_rs/fluxon_util/src/dev_config.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result, anyhow}; use serde_yaml::Value; use std::fs; use std::path::{Path, PathBuf}; diff --git a/fluxon_rs/fluxon_util/src/lib.rs b/fluxon_rs/fluxon_util/src/lib.rs index a85aed0..cde6dc7 100644 --- a/fluxon_rs/fluxon_util/src/lib.rs +++ b/fluxon_rs/fluxon_util/src/lib.rs @@ -37,10 +37,9 @@ pub mod limitrate; pub mod pyo3; // Re-export for stable public API: existing call sites can keep using `fluxon_util::init_log`. pub use log::{ - current_daily_sharded_log_path, current_log_file_path, daily_sharded_log_path, - display_runtime_log_path, init_log, init_log_test, init_log_with_extra_layer, - latest_existing_daily_sharded_log_path, resolve_readable_log_path, - DEFAULT_DAILY_LOG_RETENTION_DAYS, + DEFAULT_DAILY_LOG_RETENTION_DAYS, current_daily_sharded_log_path, current_log_file_path, + daily_sharded_log_path, display_runtime_log_path, init_log, init_log_test, + init_log_with_extra_layer, latest_existing_daily_sharded_log_path, resolve_readable_log_path, }; #[cfg(test)] mod test_util_test; @@ -251,7 +250,12 @@ mod tests { ); assert_logged_text( &active_log_path, - &["debug message", "info message", "warning message", "error message"], + &[ + "debug message", + "info message", + "warning message", + "error message", + ], ); } } diff --git a/fluxon_rs/fluxon_util/src/log.rs b/fluxon_rs/fluxon_util/src/log.rs index fc6066f..4db4ae4 100644 --- a/fluxon_rs/fluxon_util/src/log.rs +++ b/fluxon_rs/fluxon_util/src/log.rs @@ -61,9 +61,7 @@ fn read_test_log_shard_window_config() -> anyhow::Result 0"); @@ -85,7 +83,9 @@ fn read_test_log_shard_window_config() -> anyhow::Result) -> anyhow::Result { +fn resolve_shard_date_from_datetime( + now: chrono::DateTime, +) -> anyhow::Result { let Some(config) = read_test_log_shard_window_config()? else { return Ok(now.date_naive()); }; @@ -99,8 +99,8 @@ fn resolve_shard_date_from_datetime(now: chrono::DateTime) -> anyho ); } let bucket_index = delta_seconds / config.window_seconds; - let base_date = chrono::NaiveDate::from_ymd_opt(2026, 1, 1) - .expect("valid hard-coded synthetic base date"); + let base_date = + chrono::NaiveDate::from_ymd_opt(2026, 1, 1).expect("valid hard-coded synthetic base date"); Ok(base_date + chrono::Days::new(bucket_index as u64)) } @@ -108,10 +108,7 @@ fn current_shard_date() -> anyhow::Result { resolve_shard_date_from_datetime(chrono::Utc::now()) } -fn cleanup_old_daily_sharded_logs( - base_path: &Path, - retention_days: usize, -) -> anyhow::Result<()> { +fn cleanup_old_daily_sharded_logs(base_path: &Path, retention_days: usize) -> anyhow::Result<()> { let parent = match base_path.parent() { Some(parent) => parent, None => return Ok(()), @@ -124,7 +121,8 @@ fn cleanup_old_daily_sharded_logs( return Ok(()); }; fs::create_dir_all(parent)?; - let keep_since = current_shard_date()? - chrono::Days::new(retention_days.saturating_sub(1) as u64); + let keep_since = + current_shard_date()? - chrono::Days::new(retention_days.saturating_sub(1) as u64); let prefix = format!("{stem}."); for entry in std::fs::read_dir(parent)? { let entry = entry?; @@ -180,10 +178,7 @@ impl DailyShardedFileWriter { current_daily_sharded_log_path(&self.base_path) } - fn rotate_if_needed( - &self, - state: &mut DailyShardedFileWriterState, - ) -> io::Result<()> { + fn rotate_if_needed(&self, state: &mut DailyShardedFileWriterState) -> io::Result<()> { let next_path = self .current_path() .map_err(|err| io::Error::new(io::ErrorKind::Other, err.to_string()))?; @@ -314,20 +309,19 @@ pub fn daily_sharded_log_path( base_path: &Path, date: chrono::NaiveDate, ) -> anyhow::Result { - let file_name = base_path.file_name().and_then(|v| v.to_str()).ok_or_else(|| { - anyhow::anyhow!( - "log path must end with a valid utf-8 filename: {}", - base_path.display() - ) - })?; + let file_name = base_path + .file_name() + .and_then(|v| v.to_str()) + .ok_or_else(|| { + anyhow::anyhow!( + "log path must end with a valid utf-8 filename: {}", + base_path.display() + ) + })?; let stem = file_name .strip_suffix(".log") .ok_or_else(|| anyhow::anyhow!("log path must end with .log: {}", base_path.display()))?; - Ok(base_path.with_file_name(format!( - "{}.{}.log", - stem, - date.format("%Y-%m-%d") - ))) + Ok(base_path.with_file_name(format!("{}.{}.log", stem, date.format("%Y-%m-%d")))) } pub fn current_daily_sharded_log_path(base_path: &Path) -> anyhow::Result { diff --git a/fluxon_rs/fluxon_util/tests/log_mgmt.rs b/fluxon_rs/fluxon_util/tests/log_mgmt.rs index 431c5fc..f459337 100644 --- a/fluxon_rs/fluxon_util/tests/log_mgmt.rs +++ b/fluxon_rs/fluxon_util/tests/log_mgmt.rs @@ -59,7 +59,10 @@ fn kv_log_shards_roll_and_cleanup_with_test_window() { .expect("unix epoch") .as_secs() as i64; let _window_guard = EnvVarGuard::set(TEST_LOG_SHARD_WINDOW_SECONDS_ENV, "10"); - let _anchor_guard = EnvVarGuard::set(TEST_LOG_SHARD_ANCHOR_UNIX_SECONDS_ENV, (now - 2).to_string()); + let _anchor_guard = EnvVarGuard::set( + TEST_LOG_SHARD_ANCHOR_UNIX_SECONDS_ENV, + (now - 2).to_string(), + ); fluxon_util::init_log(log_path, instance_key); tracing::info!(target: "fluxon_util", "[kv-log-mgmt][phase=before] ts={}", now); @@ -115,7 +118,8 @@ fn resolve_readable_log_path_ignores_plain_base_log_when_daily_shards_exist() { let shard_path = temp_dir.path().join("startup.2026-06-21.log"); fs::write(&shard_path, "shard\n").expect("write shard log"); - let resolved = fluxon_util::resolve_readable_log_path(&base_path).expect("resolve readable log path"); + let resolved = + fluxon_util::resolve_readable_log_path(&base_path).expect("resolve readable log path"); assert_eq!(resolved, shard_path); } @@ -128,7 +132,7 @@ fn latest_existing_daily_sharded_log_path_skips_invalid_candidates() { fs::write(&invalid_shard_path, "invalid\n").expect("write invalid shard"); fs::write(&valid_shard_path, "valid\n").expect("write valid shard"); - let resolved = - fluxon_util::latest_existing_daily_sharded_log_path(&base_path).expect("resolve latest shard"); + let resolved = fluxon_util::latest_existing_daily_sharded_log_path(&base_path) + .expect("resolve latest shard"); assert_eq!(resolved, valid_shard_path); } diff --git a/pics/blog2_mq_payload_flow.png b/pics/blog2_mq_payload_flow.png index 5060d1a8438a6583abf3e9d61ad65d061abb45ff..49fd5efc1ffb62105437eb9c55bb89adad88b4c7 100644 GIT binary patch literal 55188 zcmcG$bx@mayDv&hTiRkR6t@>I?rwdHdy88L#VHQKC8ZQ7UZ8k@;>C-5fZ|SY4-UbC z1PgZ3_dDy`XPsGV&)ze8|H0fdd6MCI?n`q0pXhP!9Njc_&;(TuBsL`sEQaplW@#(qc0_>lcq=l1wf_RO7eKU4fRFzY zb6;4ZP3}+stZL7Ko?(_Bc#l{x%P0O9DSuuacw_eG?q9E)xP`8C`miEygOT7B$qhit zqh%9TL{wYj5nd*fWUxDyRM_^=FV#hgr)r}Il1IC$==QtdM;gB$<4U(@f=7B~s=qcL zNj`jc5@v&AIs_GRS&@3k8dQ{`@7jS=jz%#o|Nb^n0#2^GxT-(K%gl?w9VtoykLZr) zbPsA=2zdXxyekLp7m|q{;G3*#sBEsebC3m^oq0r#TG*&KrFo^bg@qU7V>`Fh4>Vq4 zZ;vp#xGj4d-ot4^(xD$a`XB9vY(Fu*xm^jH;j|vT#|la~Eif8l(M8lTFNPE%>s8W{ z?oc^_@>3ydk0W|n_uB2d)wFKik>jUikm>qPVn78q|0o$Xbx=Yb@&da8dYQC$I@mYl z(gaVcsrynR2p|P^nL0GDijzk4WLz*)^H?$}_v)HC_V)l85jL&oY!F4Qt+o*9b;$ z4NGh<{XLWv{9z?Jczdm`4k3Vq_@IpoJw8U6I?1%J^sb{)MVybyF8b|2zeGVtOA}4K zV#wBDS>#BHb_89;pHN!;#rpuWd=F}WgjwD_c}R>|-eBT|S^m#o6kzAYKb-$HV3{Uz zGHO-g^juB=C#D3)ZKbF}Umtv1w7&Qe-O{pme9(;3U*lutCtC{wTT}Xdsa8bbdmK+) zMEl(0SwoEjjwKPsnuI)*i??H;qkyy3$n8{as8SQ++*#`GCXPo4{rr2(1<7l#fV<*R zul|i!l>VDVidpj#(a6XN-YtRsL{@*y|8t)?8twV}YBjW^{O;yPW%G0f(ld1nqoumLRX}FA`h>pWYZ7!do4x#oNi;3T_kc9UfSP z|6zGy((>7x-DtAFgqH23h+eAldzG2f>c27xp)HK)rQVBGvT)++`y%8ev>J(icMUjC zoPn>+wynq6D1vSA57njP8+SsD%Z-%Gn)cxy@L^yBa#QGn34Hzb&c?Bx$A32+Xd88y zIaL9#K9Lf&DBNIAsyGiQyW7P^6kZTecAUCDpd8f~lDy*=J8~eq&A8ar)yM3^cH5aP z+sgP+U^A*xSF3h%_RJrVA(PUm8UJtrNO3xA5V$h8qrZckAfZcgx|=4hszd~BH$Uub zRmKPOXhvge22?kp&)c{411<~vdhbl78ZO?Wn=hw8!@CPekwE%fs^CHwR0Go^!Nv|J=hVGT*>(V8Z!;ie}c!N)f;Z25Um+tPG zrfC=OlMY-By>7SOVr&W8;h0I1On9Y*%jQ6&!8*_N1Pp0?0Kffv0brlRW7Oa-O zA7g(Q$ONCJ9E-YKt52bZy{C>YM-N5j@6;yAX6bKaMH_nhm&w|U^ro-iw#2tyB*@Y||J#Z$X3k;apb}HPyZ{HvjP0hSYr`$WiH3EO zb&;~`Wt7x6f9We(c+fSJc1cLds{rYDK5U<>d3Tu}1EZr19`(FzYrIao_R9)9X9bpc z+sYmlVMpp+f1YXaxet)oXpMzaO$C6jLyV>C@5-mn(~;qF&X_%UwwF*T1y13kC0us7 zAB`WqD}}B-GfJjps@Q+(G+SwV8@mZVTegP>AiZdlpSAaY;Au0YR+G*Y3N$CMu1Ahl zG;N*8D|7~JXUz&Vph@knRyi!X2kf1IMKaCy@P;wjnZ>hW6rqgYUH|zWFgoy*9^L6R z#No3EwOy~z0!*BvCNyQJ?%H_Z^{dN(u)uSgzQwjx>8m6^s9)zvhw`#*erEMPKt{=_ zLQ3+h@^bv(wEwC7+9?f9zdk?VQoThS(plzyJBx6c!PJn-7nMoHK7jJ)ldVv0CR;uW zUbaafyj+ZW7jVtP=M$_}8xS7jbz|Ag!}&9}B0%7ptIrkw3{YH{ zhsL`p_jp;~+2{2Y>a_L3q;_us2)vA*c)Uz^A$?no1BLS6j+M72AivTqwq3MO-5;&n)Q33-ZpQg# zRl%>T4>aq;M`b4ESq(1M|D>FYnfMWkaUs+6wJE@tmV>*s#|!^9f4jX}!ZS{&J}Gd3 zrx`shM0PN^6_~6JtvcBfs;M=f5~|e)`c4w4E})%&tv3w4W$@eM*fu})?&?@1rOA5p z+$?Bw7%Ft_C22LJ_zSnZY4hH+)nO<$s$;322B-XF$?fbdn&UdHxxC1GzTqL)_uUy_ z#%&Ce!#ERoTJKe6YT$HxELE*q0o8l8lSx)?QtNWGDdJJJAG$Nqzf%?yNm}0GJ3Js? z0d1nwCUA*oCoOOKpx&*mN#2%doi(q~m99_F_73NwtvKQFAZfI5daEs)g!jr-TrcA#YRVfrc~3#_=5AVi9mL0kwqt&BZqTc3%}c`4KL{J*hz&v z(x~+-^o%7SH!dP_deOJlVt>l*M2M$p%O2PU>Ydr6Y%77&wgfmFuN(N5zt98PtMTe` z0zB-x@AL^FtBV@?@&?BZM0+%3D)_UxWgCn$~{!tw(dc=PVL{g+CgaE)Jbp*G`YqXQ$of{gTGU~k0tIdC9%I1Qr;m#B74Zp*_jt3if*WPLwi5S+LQ1*ML za=qGD_8C`#$%y6pOf(~~l7xgLxEUOH z&)95AYvNst%=D`O1bzqTj!pOp;WrQJnYOwiiiJu-A|fL7%1QgudFa~xT*QZdwdWy6 zrBPrRe*cM!UVjO0!n%Mm)S#k#_wSH=o4!jnU?r&Is9Lwuj4k@zbuG4FGbd%BkRE5t z&(BgS+rig_)XSc*Dyv9#Tj%S{1<#4KamTI5d$$F5Kd-hM7Dtl?+to{v%DaLamGfTU zrIMW)otu--?p13Ax$~vfpHR^ck4R@V>=;bb%?KFtS@U$p-5_k=pl7W>9H(Toz8byX zsUa;Pk^3Shz~jl>xzXt0TUXSSasf->(NmU(v^?Cs~9g}ccedx9_ z-0`>DWul}KJb)xp;bX@>13KaM6YSR%q@kg%Q#ZtNHjr^_q0vKR9SbQ?AcLR z&OmkR;V!z}e5EpV+sCm8zD!JnJ&Qo2=i6Yn!9lnk-l4evWMmhWlWD_VcH#Nq=%F|2 zQ)j*PRfBu!#2}XhI(X0$F6Cpydr@mL9iEuI^Aat&E($xy?PO*O6TIKI@h7oZ41! zl1udm;)Z6iCS@)@_smOpzJBugVg-~^DH1?Pa(8(%`c}I4W$pa9=9NC2`s(XwdIiDw z3`705yXVc|%PCRsmCX)DU~(|JB)D^dTe9Wgt~J#Lm_53dZ$IM<&gE_C{XHgdFvF7p zGQs^~WqunAmH^of-^#S;{ul(#9WN;om~4u^5E>Zi{dM>KoN9*rwmj=LrabGu<3PWv zVUMWb0oMmp@bK@#X4>$AxxEUC+bNgTVj)i>eWjSzQxLNJ!0^0SC^?2lBJMU9*?{dF zAEHX88?_hk@;1DPuznBKdz+Krv*#JNbTKx1x0+M#oNT?r&f=5HmG1v5p+cf>_go63*(7kVDF`41N>k8WDZ z5bA%c?)jZKeuBL~w=A?p`S<$o02rWKncF4-r|1zA+twoTpC zu8xoFK=3X=!|YadXJ-4g30Z*@g3r4NWrS?owsPr=8jH;@ay4OeH^HMw?+t{0^cg#; zsqf3Vi61pE>T?t6$3$zBB$qe(TtUNV;g0@z45yc}&B-by z$yEx7fKB%6gNxDcby$u(B>(P2<5x25vhIdR3lUUfgKz!^7ob6L%84@$ zQ#5^9UddBz5O}5{b@eW-)YJ#|TeZTd=^{I<3Y2-6)9>ZYAVauKApYLtdbcC&rssA( zpJV3SIbe0`I+N0*!4fmrRdpHeBQY}_RVMGr^zTG{uEOr6kOaU=LFk_X^8lJO?#zQ5 z;Dz7|7Uj!^)APc8_|1G;Bx!`#RE7UE4*@dnK%8LXXyUuEWd}RYr_C;YZ+g^u}squ6EF@(mz&m|Y_*jEQGH&Aqc1+3 z4cso=xlL{7@cNQuTB@M*-L2t5u5K`7?_xZt!Z$v*Gh_ znIckC0Q`LTG2s$-zv1$yk6nlxVz9Yi*V7kf{92TBz>m!Iz`(cW7T|G% z33PmK5SSINggAU;3yWz`?=N2vM&RvS|Krd$5t+EPdZtlE2$r zW&io?T>ekE>z*Pz5=5MBZNEaeLHehf4^gFm%{ zW>1X|THKet{C83L)gq>BhrZ_by(P&+DJ-LRIY%%}S5p zNci1zmY@e%F%N^Dt(59l4tR~WnxB`)f=!y7$=(uUwKK&u`x0WgYjzJT$hOvpVK^x@ z0+RoVio*V%@l^ji7e|;2VPn^_+e3`J8KI$1HQ2rnr7PZdlzbqS==Huk!qZ6iVcrnr zFL&MkfyY<26_V3Q!*=Gkj^f{HbmARh-zT&>+Roqj5r13C-2ey19@h*hjXaFa8{3x3fbD z8PD6!50~L>w_AGxO<&W&uHQ1iBOzpbmIdM$i|(Y{Vi&-M#LflCTW=jELIZNSz~$=ewtU-_l$n}v$jJVNLhZEBJ2Yw#I!KzK-5 zN=KUs-*#5@{&>V@-PNh}P?@C7U%bQqNE&MNvWTfmU=mq{u^N_{k>hop|MBK&MF~K` z%+uGUldg>60%4r|A|+#ohpUo{XURL+s`0@8VJ}_?5W{c%|M|@Sojm!^Oz8jGCkqTq z$7ZyUvDP)pl%Gw3qPYp$O5r$?K@I$d3Cnb6KA-!4H(03*A&)hocgh@0=~8E~J8iF) zLgd#PP~RSIel78AO-kK1#?L70v=sR(KnisiU!d5YO}Chq5Z$!OVNM zZA)DXntb}be8#HNk9vB7Z*dx6Kr3nm8+&`sToQ`N>X^eI+HM%kS1q7=fA6UZBpy#F zOpCJD_>n1>Ms47Lxy38Pe*4Aw(7%>CR^83*=I0rTE8p1{x_+?c(gJ8_Sn_FDcx%6= zuWspffJHWW3r3?YltON}uqm~yAytN9d{hJlz5+s{sa_GA zqv|005xxUxy~e?DcK+Hj%s5hBJHk{+{mHxISrzzwW>hzbB>1#j8bNXK{jdSFA@l{cV*L#N&}2(otGk#%h&; zXU{%8CnLj{inHkasKI@l;U#6Xd6!}s#oo{L1bW}E3PEL3{nrv}8LLmi*h>^?WPuH} z3EsbB9g#<_{VsmRKh)i%d!=KwLpl<~7`R%wUQL)dE@^r?gNiF%G-cL*lFq4V)`4nJ z6uO+_x?}`}%sD70CuM_&G?fN6v$E#&;z!E9V~7Mgehqam2m6G)!QEGyYVU_FBg)OT zEl?gabar63<&WW`%;OQ2%Jk{`+vSa`Joim{CiFrzkC)!Mt{J9XnipK@zQ|DHPUU!k zDO&s>L>^zu5*LLK9djb5fwKo}{)K(fiISRdqoM`5-r!kR{O#iGUq zI$^ss|0KRVZ$du3uwcNPD6c&!CYl@90##o(t!=d`YWhM{ed??7^B!|9r}TcN*^);J z2}0{{`*}ZW3pT9|=?UZ(VdsaCRt~82_Qp7rSbLqj$M-5vP;9#_ebkM&J2N2CrcaJH zd|x$wsD4hF>`~k(yKJ<%5{-`AA)*jUp6P)QJ8JT)%6YNTq{(@au-a_+OJ__t#%Jg$ zik58f*JXUKV@+h%rA_WMuWC#e7f*vEWRCKUuCqxnXyAc*I!(huszr;bAq79>M|O$k z?jEVrq72mrNR>@`qW0M=Ag7eg$AShBC!9EeV(H2-%uY=`Qs$(In&3Lo*1Z!gX->*~ z8Yx>`wuamIc@D9*DaT$hv97Wu+Aqe!{xJc<%)i>Lrf)f1^NV?VQrowa8mm1ujIA-D z$IS%6pilj7Qc6N(vB9@fNj6^_rgj+pL74R5Becxp(}?+_w2vIR9-75xbwXu~_I&5H zsNc%uF;q~aRBC(2xUdN~eC5K37^7z8(c+KXm3IP!bo2DUgu)LtpG^ByBnqy%;s@f& z3=`tmpXa~FCyXQqe^eM|BgPHM33q@B#@3iG9f8Jo0()s+%F-;Km*o_1RfkK=I7rm| zejaHAia!xlxA`)#TCUJ5aHVY7pb|mK%iE|jUMF*l zSwO;sor4soW^G5i(=D+kEQkSiqImv;>ePZ!ag;w%o92N4+C%_PDy&$TNvDw$P$N<} z*QZ)5NZr42mBB3JF7s*6l1W%l+9p-5_q?5^HY=j?DhV@>w50zr;pRn*7+^n<Dbnc%AoJUpIUhV{(ynIXI`3W$ zJvO&1F2v)epD`vXcI|Am0qPkn5jjq!Kkel+R(ks5TQk0iZjk(u$zR$hX9$R5Iy%Z;D`uj#)?o1@eQc;tudo7zFQr?%qZr*ZO z*WS)NqEK%`bo;WXPZ{VTI1&ry^SZQIRwTf}(!f{iSEmwpU)$iX%*fZQCqw6O!JwZp zbHjCK4^FFL!W8|Oy-@WKed?)H_&82vjhc^zK-ZIbVQvL4;Fc|RFEkj z8^lJ{tLkyRVIE~VV`Fe6pP z@+)uP$8cl%Ic0IDAqec;JIjegKV_I8$THrrd$z{;5>y3AW%cP!p&8#$>uE@gblT%| zG#>L{rg$#sJov4DWSdVsm63gG^C7E&-Q#!q=(PZTd=cLw)&Bkh#yC8n(D~2kx4B82 zOUss$A8t!KKcwb`(JxyIZIP>LHxyk?YUigx1g-CPKtRbd*s0=n%y3N)Q}1tyaCdL% zk;^ux7Je3=sUl+FkV3nlX=qe_mA?l8mYJSxh#lIcZazBDL%X^@VOk`DRR2e)g}RWm zOlb&5%us+#c=Ke6%jEp8H?^7Hc^;cSvJ z7uMP%J^l|aV4jjxM6{2OPdSJszqWRnm|wxH%szW&-{L8S;M-{xK=j#^t5J5m91ah$ zkvj~%4ofjBd#P>_lfGHTvEkXKKVqL$i?0qUw%G?P7O;c0p+XyNTzOl%P-}}>Wn5?T zvV{%jmiKG=v(tLvNf6gHxma?$mRxb`iefi02WqZ^C+|=5w$w-CAHYb<+82Qw08^B zOl~ZW=7bvPw^eA7CA}plQV4gUOHDdF`c;<%b@z_1mq0-3S7>zn`!&nqoX97d0ULMB zNn1QsKEX4|gkb@Snas<=m^NJxJFQZCL>7A`ij}gyg^%$okpGdi;nzL5Bcms49+CZc z`Gdz>TX~byYjs0)s)u;6D?xQIkk-AHgu5~l{3oG_5;B1fZEKE zl@VNoftFX2^yxD!-g?Z(r}=e-Ci8`Byu^(BZT3fujg0IZ*aDCfAMkk4#-HOiZ5Mu# z9`8ikEFz>Bm=D_=wdW-z`$eT}5w6hlUoA76`3Ko{N0LJ|x%Dgq4t~jX5({{s;ujaMvvn_KwA=6)l zF$!3-E}{Z-?r%$a8D(yH9%8XtVps-i7WB&P1zi8tW_B2@?^-ykQ|p;tQo~cMc8B10 zkWL*T;EXLf8ms-8a^ZxhtPTH-qvd}M$Ne`1M1dX#l38ZF0EAyj-8H<`_?$%G z)0$l)?3wy6aQ9Fwnnfb1zegC~Z#Q<@PxEM}KwO~5Hef>92;;aZc7N9X>iL|3ApKm) zql9ka7@t1n)vh--tYW9LcA})1ca&f-FHwl4OF)gSl>l4M3{0;xZ(bxMK_$**dcJ_A zqBHO8LVQ2&s0BP>YTAxD*!lae#1Ngc>o1qiCz% zf?%f{ahW}De$Q2P+gn6e){j|~-G^?zqsy<=e*F6vOXay!+wmisIqZPYohl140gI9M zU>o;S4Zhhe_niQk`T?IIW7NuX!Kv`>@O8T#+wwIG_e3p%DIZcim}Tl@jYpVN%VI6t zclRd}>q|6y4F>jk74MY190$X_yf7e3I7=)+i6d`pfijBEWA)J6W~{MLYSKcj^d*6{k=1yLSEo{3YB z%AlMq-x{u0Z=JAmQ|a!;jtv-fnYQm{>HaoQTlvd;HOCF?U_rv)Uz|wU&brX@JyFrB zdz$LjS3p%3*j6*nOs?#iN3S{TlN8w;HFD`Kc&Rxys4EiJ~-AVL3Zb6-p$7RyF@DNhQF3~bI4?-wKn5Y9Nb2msbx3h zIOE=fjLz)NRg>O&+Q_i_KuD9OgG}m;5-povJteQ!S$`#JPBb%LlNXl)63dvT=5Fp! z;foAwCBU_D)e#d_f6r=K9(e`Qs%$XP^WC&^)QbMG9YsX=LF{j$;hOl4)B_d(*N!$# z_&mJO0Mhf?)~@eBP(TrwI6Vl_Bfd@6{gqOdA67%&wdZ3bVqw5FL#%pE2GUNfm}7Or zKA|=D5PO^?q+4dpdb~f$TJzJoijqdSZ130S2`*ZX01Giz(Sx4jRBBN7+cy5uHG*D= zbmk6ABDuo+pT*c+Yaqj^ytsFQjXc^l`V#Dl=Xqn=4(<}sRa7=!%Kr2m_WND5sC7|+ zyq~ar%J+bpI{7Tie*mGPAdEgDlT!PBq(<0PvhUA6KGk-s#u;?Y-?MRj^ZsIg(Pe!X zpAkP%!%7LSrWPfaTjmuysS-8_N0F5L8&F*R0TkoyU9O#t?HdO6k&=)kdnIvQ(B!(;(VskHck}DWm>;NHV5S@d+MsRt&Q9aYBg-==xyD8(5pynYTO(l@MtwN@KC{ z2dg`V6HlIiGAv_v{+5Xm$Q=xS_vJ{V2-MzGQ0%8wR$pc(C1`eSq?;v5i)o>JPt~d; zT^%jixk&BRCz8w(?M%{l>y+LCu-xtiifdZjy3|TxkenqxpLt=G+{G31VKyUJozLS% zUi%=rbn-BeW-ph@Hd=`mq0WP6yZM{Sz7$Ap*rrt276T28quuTM6!|eOG^<_tJL=ur=OMEO@V}(%@?g}_P3}v8c_S$f%U%jr@2G!MUJ0mrr zvd=2J9ap`pJscg4`y46ir#IFgV0~H-VQZWV%Xb$xVX3QzHNt?W5eDa-!WppP*?j*1XtL;$kPg7;~^96P$~2Rhgj89jHtKpfQr35$t%Nz!In)J6`K6@nbhDapLg@g&WbI?wJ+kp*1LT+MEKF1E zBaNys{8}hf-O2xR!(*A7e48s_r++L$+fFF5lX+u(^~=_`o(+q<}dS_LW}(czkl;s{ud;=n&8jiS{(;mDN+z(dO6Rd0>iJ>Q%)#j zqG8k6-T+1W2$Is46(jvl7NgrU=~$5z_sOzqcNVs>4XuSeDcVuTy<8QxXEYs3Y9F22 z-w&vEF*PkYXy(rBl{u>$OKDm0*N9zBkQapMp+@9WToqV59=HAsrHCl>NTo3 zQ%b{1?cxOlNVKP#p77Bd)Ex2P*wmGvtSY$ssO9b_AD3=lo||PilWy%vgt-2uTS{N}7$OQF*smgVo*JiN zJ@1W~P`MUS_=Ruj=f=r3qVJ6!_Am&+iC@H~$r)s;FtCP@FlDhWE$>F#;W>NYq{#^_ zo6w1^L0>BzJu>!Jy#FF)lfdN0V9V934;T7NxW>nU^~Ge&xJ^SPb9R(WT+5F0GZm}c z%_4F_toFI*m~p!uGqDy0V=9jT(}!saHZ<@GHexg-Y43OS+c4ua){4Vtwnhv2TrELL zU0sj6o2x(8+Yy$hYMapt`9TlX{gP#lf(S7#aV*jFzj)O*VTyFOtAt=t%EZvt(-8@t zR@C?)Pg{%2>hDAj{c&c@q?-dSH>xeJs&Z?vgXPF40|T#YCrI>7Tst0Ru!Iz!e zS>U@Gk%NVLWTL)lYL4CUS~nefEKDI?`XnxPjC~2=FyHKjEUAF{$|Wp0Rs8%(?$mVj z8<(efze;%rdaG0OgubvN=u5mkMAZ&ZQoS{o|Pyuom{TR67qz99~)!}kX%Ha zX5ITc5VwNVmr#SiJGb2e%x!8q_@WVois?^qa9D-U&d*~UbCS!8(zS{vyI6dmD5MjG zcXbs&p-@;sJv`9Qd;)y84c1sfO&dE$JNoVx6~>ex&SrozIpI$htJIxaj(cn;cmj_j z!`1IkW$p#$<$&14>6zYZ;g#yw7f6{LM0*gnMf%-!+r6JTae&LddQDcix`gLG`2uPy z-I*39*DG0vXuI2MV-d!0>kCWI^grpxyus^`+A+t*!(5v`71Csfl0zV&@|bL+@>g*p zvXEiENQd8S;mBP64F=!4yvW|6P%WHzgr)IRnptFF^-pd{GkclyGVAyBs+xMQhLo3c z8|`N&NF!S>XfdYjPu0&JPVil&?bBka?q6pF#=@jgUf>I_nm$w?W31%)ti``lry&d> z%4t}YS6*ITSQv&$oc6g#|EC1e|4!&i5qO?znN>=MRt2iI%A~>i!nM{OX#Q3Hjra!# zUt2HuyJPJ?qLrq}-vcbj+ox*0`md@gZDS?~wzJlKHW|B0rfl?s@BQCVFI5tNUo( zgnOBCx1U8ex})KyphX~Q)86Ev;`4s%SdlU6pBFxhPMUv-n*_r)my>*BYkX@ribTXG zt&E7oC+WG8YrI>z5FN0W-={tOBx#)kR9k=;TF)Xb-r*RQfoZu10Us-G;_PlSKo$Ay z2H8Bj7fo!jGK+;fJ$zTc88<7(pV=N}@C*H7F+LN|>a!g86U-GP#+`0vHuA_>VkQCP zN;N2!SoN?S%X_s+HM!C+Q5^Z=LEqF=m{AYLNqrAZN8pjO!i?#nHd%_8NfIsl^Z~U^ z0{}V|c4rOAe5%8?dgi2MimOtfT zAN7B7aO+vT?|Y1J7<+|lS-ymp9bZm=`u4n* zHGHSau1Jn8X~#`mx2B9l-AbH^v-8FJ*RwkVKS5{b-NieD^e`p{fFXk@vvjgjviU!X z3)A+=siLs9uSh);XHbkmslYycdDnr?`B|2 zOIr-#E=+@JqWM{^>-XF4i|8-w@B#{=cA3HcT}(+)L-!jX%2U5ta=# zCeI$W)DCn^3>M0R!lTj#+k`gfyZu>44MSwBT@$VS5|7D7*c%@K^aQ8>%aPyJXJ!Mc z1~WZqF!QztC>IK)FR+bq{+D7!fmRNcTyv&SdBZZ`cR?@Kwq7(RY=*~FWRb6AKXWgsJ8#NzJQ_nH@PO7 zgCVYy{Wxs1VbH$rXZfEelLmHOTqndDh*fv~A9*p78xIHhy6vnkNB!Q`PJ8G6y(Mqk z%c;rrtYBQB1z_}$LgXN;6ePa=AH@_v#%(>E;2pO<|GT~!RwWy$HfJ(J3Z+MW$i9;obdZh?AJ zA|Yayzs#=HxwOnR)zxur)I|Zp zy~LDW$JP8yTM;t~=x;zYuYWy=5+ogo=(d&#I4i(ya%EJ1%QyE}LH9)5nM|$cRyjN~s(`Tf@&5n=YCmCiZ{qf1 zlMlPuEdAosR%$e+WtnM`k~ApmF&jdp4e#tS|BDXGx01dj_5)`-zp>y_}?iOjKa z?c&(M2G%4`zQMm+44-l3Ur(rgR$sBh5(+is-0Ic2#0-Muk62Q!43?YTI$iiVH&+8l z_WX0gtD6@6tB!j-8)i*qVy@pB{o0UAwyL7k=jVZR@DJ{69ibt&v!px&19;9hu` zy{nhL=v#Q~TF1S1GJCRM#Mq<%6!!UR2A5vMcmvfp(J&r^-it?0?y4fwtlrKUmZ_y- zj#shMDolDnuErTJB3Q#_vFI5YL2B`|(AO&^(KwOjbV^97Ue>W#%T{KN?&~G`E-|J! zj;cB>)A7>)08H+l?-9$0(K=sYdUS`QQi>F}DhttW9#xe!jQLxRA!Ty%&U=0M5+_@> zuh*H4ll`#1P$}_n!kO8knqkZ?1cJm2MI?q&VzEC7IR2Lqg+hTf zWLjjfmga9nN}&x%YCRkAWCM`JXPugjG>n&S2I)G9Bqx|UFlQ^0iTUU8pwM> zr^{U34e5YAqKDPmcikPcNy$zvgH?;3(m@uX&xwbmN>@+@y5f;urEoUKG2B$|f8mw) z!((XT1@p8W+D35EPj_AJ-taVai_-AEF(s- zsO$W`7~b)n8so+LBOqKMG{Hukw4LB>oE6Sbe~u9*{Lv6zzdG@jB$PaQll|%b2|wiE z&lqV&bc#8&=zU}+omCiwPz9pL?=S9e!937Qh6kd?s|CnhG!iI9dxZeM#7fcQK~h+P z-A6PN@PZ82FjB((MAjWDA?wDnH-^S*(G`b2ivi2#k4#V%uv4tr=AjHq46H-^9zyfTwYWSTA)cfjY zs^plh(Xi8!zYNfPj=>=(@ywlBQJ6+u{qoPDNVP)@8oC*>krrTKe*kIy*W+*}?;SQH z6?7hwhyL^6wvLX>ST0Zh?@0_ApO)#$E*>R>xZ);yw-bNm$ouASKB;(p7H-_1ahAEY z{JZN(eto0#iv$n1&>c4*zxHdIWarK%^f4YXcM5!@yy>FZrHoMmU$ z49o%WJ3bkUPChH+o$@loi1JGOL_VUqyI(eGggiVt?QIcX+ge$LT67bP%{;_ljK=oJ z0@v{ApZ`)WIMt1fyMnx5ldG4=hmr-TKd6tMz({I-g7o%ZTt4d<>l<5r@L-;UvO#nr zzEn>O3;EEPfVa_G-woes@fejPZ=&xxYr5^_D&GQsK{MB$&h|L7_wD(7{8mg%M^*}7qO%VEoq4)(ND|U749u7 z3qb@4=_~i+22&R4Bl7m7AcALPdoWI_z9W~%QGr>GfkuO zNIZe6>Gu}fv3SrivgKEdP!}7{>G!>?ve{qEAyn!~V*hq=sjz7e(|=60?c2#o&s0`U zo?7*=^DJH4v^1i}!s5ly86D(35ONF?ZjEl=aEbKK&PsA(>6KWljUP6D380P3ekL!G zld($9z-k^ImE%s}Tn*$`VHf1_ZsAT3Umx;JC?^G_{ay8Vgxw{qj$c(s11-XptVUd@ zm{Uuy=Q1T8A!xC~IcSUU*3_D|onjr^ujKoel;Gx4L!k37DFJH<*Cx>N>@@X8XW=1MdsHx+7-05P`g zp|0y5ewO8P`OeE<9s1h)%7N+Mz5t_n_S}V>K(B94b)jz(N`C(I^K@!(JqwMiWzUEZ zZb{aP&F`8_Rol)W72jrgQVMy06&~^yCMOJ}JdW^4c>K2Y$8J<^JO!=RW}nZGqNwap zvhjO+#liT_0CM*@M*Hd^C1C!<$1PN;tC1Myw14&Ehs06(oF5q}iVSoj9hirCNI$4ece>|3(hC4H(Pwm zH5CX@4y80+SSM)fHZQZAC!%&2h~xH(LG&o8Ra%O7+mORFPvetf#=@)SV^~YLN?ogU zEys_4v1D=;V|Qzna}bQ3wx{m|#jkgar^TC4OY>9W_ChedC?$ZG8Aow!!;-=Hue@*J z81h(l#b9~JwsD*~K3=(hH~Y-(nYf4oh&iM0#Rt#;;@euLfqQ@|f*8%x!E#jNCTM8> zqn0d&W-LN6^rd*w)?Y;iS+42d%2A=l<`n9xVY}EFJr^WnK3rjqRq-V)eTaXoI7r^t zD0$-?#XxJgl3EM(o(gf8PB9PLANvfbrU?>ZbK(o@3l`juE{SvHV3JVoMR5Lb^*@$g zVlNjR@qN*`P|BI1|NU${b9^_Ok(2b)h%F40L7bnT6G!zPeg5Gg>}wnfg2H3d_^GNG%e1A!YT8Sq>aD z>tQVMJ|ro(fA1>~$x2s~nd9r73kBQou&1@dwbZ>+HhecY^%4}hI?N|PH0cn7A-Gyx?95zURJ}Vf6Z=r7<`|Wa2~|D z&+_o*&`xY)gHxWsNwrSdfz^Ebg0L@2F*r6C-lb>hP9;tvxEjgYAT&A>l6)IJAetscpB>9`Osa7$0Jw1&v^%zdr{xs7pa+L4tOL|hc zyw-G+-;od;QxF80g0ON8Gk0=ps!ZB7QxrDBrlw6h%_ zbrQB+A2vV2GSk5Pa>gfZhF8yhBr}oV`|ILjBFqjDV7NMsXMhFQR_lD@&5LXux;+_H{DY#$hYk5Y@gFD8cA;#I5YqN0La{5jb>=rI689CUor_` z4M{&$Q`Ggo-Gu)?yuEc?Ronh9jEW*5C?H5H2uMpesB|OJUDDktf&$V4(j~R%?pC^C z(cPWWg2X!}y3g6?+;i{!oX`9I-u>TJ7pyhboO3+msqgOTy&Lb>f3y-)jAhubLs@qDryI&N){aNr8{IM_e$IT%g=X!$V2+_;{AI*Jv zp?dRu72<3YSc?CdYKgY2-|)wtj%eRSz&MDoAtg*2_Ar5v;g1IcMjWyG={42fd&=kQ z!1vL%qJ9&>>J6)Uk}Mun2B109#Pq=cSTAcJJeq z2r+CZlEr@95O~@Tm&Hq}{yPlly7(+~nV@r3S=-9|BIQBR2X-Qj~ z7>T44j;E&X_*}plI_w0;8XJ2|#D`QJW;Tm0O)b=@Vj34)i;i^zf#VKv0CB{jwL6y- z(BT7DL5@apeju!?NGcCGS;v&8p7n9stL=`PQuKmKZxzt&>{`q3(8lB|E7BYaO@!Ar zkj>y(SoODB*GU)GhoOlKluqvSmdEO9i4$}Ga`0RxbFADNiPctnp=*Q|@Ib4Q83%G` zC(ADUXpRw6Hys>UA1IHFmo#M?6~Vz^U$-G05a2O~8Ppzrx z5yG@1^cJ|M>M;tF?Zw*)bac#J)XfX6zX?7A4)FgNhKrZfiGMx9|HHCfYvOI0EHQ#I zL0KM#RGNjQW@NF(S-#AZQq`)m2kY?`;aidOQbgwuE~J&UdY18g1Ej0q#i4!NIu`<^ zydAU;rqi=H!`Jc`mb#3(=_1 z(Q5Ove(TdjDxO6J3PJwpO^lMpCT9BYBnMf_!w59xR63rEIPh4zCoGSJ9yup+cGiiz zyd1`BOMapNxq9tRdJAH0*$7hh}(M$_CkHj>+CLl23Y4JZgkQ1CV8Q zaLnxOKfJm*p;7Jg$lSe(+m>0!o8nAEV-v)YPU z+K_=w%`jZIzJOh$-!D>v0viO^oP70rPo4(4=KBFJ6EY>`MWILf_><3)OcDqFk(&gX z?$mbf0f}PzsMVUHZHpHhp)}8h=S3)|P&%XstQQIH*pUWFod+&-gbV~j+f$R`c>~MR_8%$ibGF+1sMj{a_a#1JC=TLFc8q>Ak(p7z z!-ou>LtuM&b_>M%qN(gONCrPdUV&0ii2hcRD^pW{;ZX;w|oBb@-w;0&Y4{*-r!X)UcN(HfnUf$;jCQy@9 zE!4iDVW_RGZPsIab$Kyvb{u1v2rQ7W8>ZvMZ=sFOqW*M+Ccn}P0U@3VvKLmnFYV{> zUyUXubFM2Z=F{TUe56R5M`GdR{k2unp}!Br%k$+Zt(`rm%LRr@=w&x6*9j1$D^Pp2 zoC1xFO&n@23NAMah~0jCPfR3^>N;v7x%>f(vp72{@Q^}4fDr(8q1y#l!w!zy@Ux|j z*tL7x!Q&`Lt8O*X73{aAufPISEkcAu-@D@y%0wqpOh1quS^oewT-(kTT&?qjUAI+F zB;ub*SEnR55uQtdy;nP5rOxH3Y*h1CD#^tsz+PYh3up8PzCAyg+Ov%ufJ9>FOm8D- zqZcdF?lxTU0jL4=XN2cSya(OP4mAnTE1+*#{Q&SL7l#@Ab^Dz($Ozw9UV0-VtbF-^ zgz@fTQ%kE@zr&FKmkRVtWp!@8XEp;X3)mTbuG+%^39a)5TrH1_Bae$wj|6^?>)(<2 z{KmJK(7sj2?TMqMaG9N%I#=*D04Q;flhY1=ryiiB%d^e(BtY3fJ1`Q2LTHY9>KI@i zSN&KG@3zZEjighS(*Xto*56Ppuud1*6%|nXj_)3TmZ3XRro6?Z8{x`FEw z77|6LCJfB_fC@$6$oqLaisjw9CqJRnM`~=8riU;=(!C7+i*@kphV0#~&?ZVVmjJD# zUa0+j0TUpTi6>xv@T^8P95o!L?R$E9ejRVSo=iJ3(1O!5^FxSN>J0?i|2ZZ9sbAmf zqH`j+gUq0JOEEk;dUAXW&}7WL|7qnH+^+w1a&qn|_Sq{wWBZ7L60M{(g-scXUu7NT zgG^5fD$*@=<7P-ue|pt*e%!IEDw!FaQhS{78Sqr@$%koV^w&AaJzwM8UEa>KlHmO2 zb7Bv5yb7zO%!qnfpghMAAfW*3ATq*Sv+=2X_1eS$!O_H+n5r!#48urVJy__=+A1e} zOi(2Z9)%BKZvGfIn5u-`Gu`{Br&IB^n62JKBY|@~xZRG88$6M|A;Tl|D_Ik}LPvxL z3K<`FYQ#}@i!61h6cf0Ey1oX}W=Str%wn^u=(C_59C+a2DkHwUmG1+g#WG3S zAz~?*f`hm6qGfhCZHrwyiCk{u3u#69Y0^{UzS^m(dd{-Omo}V2!yHBVrF29dzhG9p z(*|9C`2ysqFnd)ehnDQ4VO*-Jrdhki2JGIkif8Zj24j-f#B=!Ex1%=FNS z*$T8B@34j7pqfiyJW|syZXP}!3lCvcVbWk!tx36^Wi&H0RCQ6rh9M?cU0iyz@xD}O zKz=sQHtK{3zEh==GqCVu&a}=+?2izHNM>kC?F9zng@^civl2 z(ReWsZlaGq?-2DWKyBlid`kB~f=YiNRnO>Ir?FgD2rU20!!y!tv6?k9mmS%rwG%Sy^ zEBji)rX^C_r1h3DCO4&hQ#=`4OL>N3d_9fWHPcWJk5V)xX2(HaO-hU$bwo`AJ@`%o zbx5;*xNGp+dNB{zs~tzDjA#8jw=>k1Lc(|e14BQ-!)j;dg*`S<1aHY%FEy~`zO8rP zA*va-xR`C+NG14*4QXWiGp^MiY`@66Ubi+X6Q+EFlVRR2Y1t`!gh6zZ{j0Gh@$%Xl zo?QXB?BkV&j6M$sJ%Q_}l1F_YQR$5}eks+P)yCL_Vv7u00@qk#e%3~!{HD37stC^W z<1)Sc+-lf^U>)&M;}QL-Bz^Td&P0NG8|HL-@th*N%3{K*O+`bAHx?};Ohs@?BD$j`pYxy_*de4h;QS3)D}tb4J3OW1(eA@p@`q^5 zwIr}G&Q6okE7B|;JH%E-Q&;RD-qMd&gd(M$H{8SEHVU5Qk%>m7N4=j_*R4R`T73BK zhqpfmUX00kbg#+!i*Rzyx4TP84+^(+C@K=~L@sk0aJ=iCCa9WQgQN&);vP=)&7aL<-+bYXXV%x{4|dr|SIN<*EYeg8$e^Fpi;=K~bO7Hyub zxj;47Smvc@#t58%GMKSMp-DF#r)u@N4~cefYH}6_Pcdl_#uog{@G#yQkp;m-yy2B( z1dfHBU#Pf|Zf?Ar-Uz8n>^i@<Jb<@*7!~MvS3a_74ykX^HG2aeP67Et&_K%%qzu!ws-c1w9=GOe%rL} zGB1)DCNO!+U8BbiaPO|xhlzJ_5vfoo5@)481pd+j9s}tR17i3Pi40zaL3(OW&^A>N zu8@Tj@^96cb~=xBK642o1_vGmi4`LA^f1_=hk=;dSQLdJllXY5K1gn$wU!chJT39x zV{EH$p2ui~V=L&>isWT|C-g;uZbj|ez&<%Kv2n1LrVP?yIEmy!IUl;rj$M|^gI|Pe z%?v5macbq37N8D~jk)zT^99B^EvP8ulv_2+xjWj{vpOwqxP=zYOkm!p3D^%MkXJS3 zS!rG2=Ea%Ow|7>Lw0b(^ytmrUIB)hBUq?5AO_%@To}=Vs5$aW~$#@0ZUm+{mYD(3< z6MLbzttxbZ4vn0ENtP{q*FzEUqqnA_5)o?^b;MikR)vr}Yadr!xu{)=bVA`RVXM5H z(2BCN8hN4DQgVa54!vp4&7zA$!;G#95nVcqr76pM7-8GvTiXia#7@wN| zHdQV9JPr4(+`LP|Ge-UN!y5=+8TeBsx$QG2ouYAe!P~Jnetc&B^l*wU!Gg)Rh|Mx{ z(3M9mI{PM(0%ETTgw9rggAssJbAWZh|2}7iU|tLhVT8N_9X1MWKp)ACZ)gy&X4pEG z$xFs^5{ro4x)jmVR*>C|mA;)w4XmI}H;^Mm7ZFaTy)yD7WoQA*d)3>d|UeANX z%_mm15KD(v39ve_9L_+kpCGH1VryGacMLFdok&z_Bg+BruXYk8l&OtjiM!p>r?^N- zWDE~Dq<=1mBXHK)Gna3zhycK&qJ3;6wW;$jeV@0_n?u8rtOl zSsVQSDr5zI&VR~oaZq{d+a7H^P#8`;(>8=z(N037-+iU_qBjed*~5zpNWWNY7J@y{ z!UvgjJlr#Pqk~;1L|!E!$henkCFM%(fW&#@(q#_~ z)7Z^5YPAB@>q_E|=woS+&Sqo_rF121#=S9kDb}GYQR*y|U&4wFN*27ZrtGE4^qRcG z(Ab3AtIJ^~^duO!Q;o~O;kkfnt~}H(lP^vA6v9==`{6qZ629klL^_WtX|gRs&j}hx z)7TI(d;4(;tg9FfZ*NXktpvYU*!g?#t4;rg#(F?$Ipsq=H+0Ld-xueM%we2wix~ZL zmsl;#1qw?&DNYR8c+L+!JKZ&rNoB)Qh(ImQ##QOB4MIt?EP@^7%Nnlvo)Y=}zgzrwIvX3z42n#Q7d8f`R8;5q>NL>7=FKlM=c z2hd8`v1V}HRO5~l%5$`pXvAFm)WX$7jEP#7fi8vludEC-OhY#Z43QZv_4B{gX|YtZ+WhQ5H!DsE&3rkuiNn!zQ%;Q&s)cXtP|>IECTw6%@}C#s=2zIue~9aBo%+5$imO`@)&Q z5mtX#+wHXD=iuPzBqqZVOaJ*5HKufNnHupV@jYc*xTqYXAAMY;_)_UW{$N&$d0wTH zf2ib^XKT8sYuJfnGk4QHbOlARs#WsA;RzwP?>^LT=Gzaxs^$o>4TjV4lAFL}iU8VL z&J8Pz7VD&BUPQJ>osBGBzw0zr>hWge)Hwz<`m>xsvC@tf@+{mL3ZDOMfg!yWD;!JtLPhq-z z3G%e`QNy2{*|LQDZL8!6Ec0xBv_>0G2fnsFtz~SKV7TD#>gDBWWLHq3FreX=%hT-% zrp~dB_Zoal?z(@g`rqm(ruF}BZyfQ5WeGDC`Mv@@5V)Y4FdrRp5!eU`u7w4ji+Q-};+j?$FMtSr4>Y_BC;+6EvH?sYVT2>i&CqFDX zO0$mR>qE{R*|a(8@f0>$7B59EOmXH{bXu`Ta8`vL@)3`fd4Y?4*3%oI7P1@q`tYYG z&+$Tyx423vW7UjwHzZ*P(h~kF>F*<`D2CkFtXE^reHB%8UI&mJXji8^*pbCRYlA00 zV)T5APN&c56I`XtBK)&vD#0-W*%%LReiPkW`ms%_t07Gl*%_RvNcLH5`$|Cibbjg; z(>S>=@lMY(H@A1swtDg4o z<{?5E8pW{AkfrqbX;b`t%s{BNJ(@nZgTmQuL)wEhL5qf1vs>P#do5jQ^p0JtYPAb9 zi_vXbjqkGZD8Gnk+kW>fY~v-r9a#GK1@jGUjPxF!@Sz%~;%bkVlVjHpb1C0v`#(By zf|vY-QsduBVh(i6`CC@ZJ4SU7rzphD068;7h# zB}t^-I<4k2VH57e-10*3^mb1xjeM;~nk+bwV&c`7$q?kmXHgs~IdUr~shX>n%w7Ad za{Pw8Rj|)oweHhWPKtrt4oL^%Pb)Sy-vyaEIt{YE2mY$n!G!q4XyMrS9hx7FYuE~Y zfFXyGOE`9{*-+2)aS_@He|nCV@UmnCu~wZFTgueVrsJht5uYBhw315`T~W8#)C;|* z6q>>I>3#xC29-vy+p@`ZjTfbf>iCIKT15C+`@T^8a0zRb5>oXj7GdM`Z7W!I0qAW_ zP=QncF_ayTY^q{+d=!J2_{ke%9sdoZC&GP(3sfrX!OXw~NTVf>?V`^%kz~CVQ>#g< z5H5)xB^nm`Ea80#N&S`kK&a%vt%RRgq7*R-fsDuGFdi9Nz4F}qy!R{y$~Z>~q}~y8 zytFA^7k%2KJZP zY5aK0d&4!uZkkvold*yprLaa6)*D&_B>wXDh#OvI$zNv)k~J`s%c~;EPWNi4r<+Qh zyS*N6mOF@s2m4b{P~uMHI^P_TYw3J?JNC=Sh>J}7+O+EEE9Obj{;vW?`-5zjxCsRV zME9PsCme24(pqXfsrXfAqgj$`Zj|Evctjree61JOexk>JKF{4mrtkOo^OxKWh4jf> z<=BUV$!D4IlwgM$=~U@X%+8lOnjubP4zLJ`4l4I)q3a3jc;cttCp|e<%L2TJ@4|>x zO{OV47K1Pw8q3SYOPAV*lzZomt;plp+l(WO7o4A4c-wH&X)w-^6K69RpGqc{1k2Q0 zvBj~oun20`o}i<JV`DBN%e62W@jH$bB;A3h zMTn`fXZu7i0Osafymv-+9Ys&{Yl9lAae#Xk|}>5u+0p*oe0pm0{~^L&qF zvZ$LLKqfsk&FVe+E}O|y#9%z=Fgm9nEf^9mZSyw7^ip~{JqE!v(c9tSsMEJ80TPT$Xv6E!(QUi>6G671CGvoQnrfNHh<@@*Ylvph{ zI*mopasghg*>bd}vA!~o8(4`A(pj}~8;qDark@`CSa9mY_?(;_89?E9uIRA1Z3@Yb z@3bJOgazCxfYYm5U(ZOQxaN-$5LRFw=WQOfAa8`_hv*sZ;|f8z6&T1CaVGUO0LVy0 zM3nSvY852*?*uowZ&9w@`mX)?o0Gt9xW?6KYx>9mT~rSxSlYy21s4-Ft{$gTi&g5q z^H%-j2?+`A?(V2Kv~$k&4GsJ_0o#BE)*U?P<3eX0_bahG*ZRByZSZDfBvKC4NDz}DPhIEJE@rsGCu<`DbbAAt!$v# z+2g!jgpX;W)M;zi_G@jmyoD5q+kZwPY#ymkxbm_d_^U?D5w4n{wmL zkvs@-`@~(Icw9N^4u+9S2*(}oOuM!`R3d~QPTBTdY#H(sn4eX6{Hh>}J>H&VFE<8K z`(XsL&|pa)A9wlsui2~10*{N$A}N-^T-7p@Qez-TV8ubAK~@}n^&06}@S{m=KS9vJ zz|H~sZol2AK8NH2nu1zV_2LbT2E5X|6`WWw;i4NN;s?#=w%HSZX~b$O`d*0_IAmvQ zIxj~YGP-}3&RCCFSB}*LynI%iV!k@(lwg4*RvZcx=7M5UV-zlL@h3say?Y=ZUy7fm z-O27OztCVC|K%Q>!i6_(><|b7LFOsA)9kZuwr1fix_FaM{TG@kCgz8va+C^)Cl7Hw z_Hf^@!`<(pp#_>(FNI0?aJxBsc)T1v*^Q!LgXhh-Z05~u`DymHvB6ExCoLOz%Z*R>m-XWE>(4gC;y~R*!aQu=od~{U$0ZU6)K$hJ1(SlU4u}d(Ql)k*MeBW+TYaKg z*@AI=hshYBduw7as*P1(p(r!8Ah$aIK~*I-2Y_b=`dq*)=RCMl-FduwuQ@5 zsK{MOEH@w{{tu4V0B1{CIZ-y(jns>?W}M%ih(9Nr#-Dj!GaVINDJPP}GV~3=yr+#u zhLu;Ksj-ZFvELw|sr4+68pO6pl!A+yh-@X2*esIiPso{X16AT2_(KuCh?)3_g_m3D zek}>VRNJMb%m$5uSYdVI+8usDo;w;nX($W4A!YI<>z|9>7dbUDVsJWC$8T+r-jnwA z1FvrbTFxA8TDsbXny5f&l|QqM;Og0n?3G;$$qT)TvbOESrfQf7)NOey*nm%?@UGba?>3OffhpjpyysR{qjn0Ik6>xmm7q+ve#R` z;+Sz|Ui6Xi@6;Fkx`k$nX}Wt{V?H%cAJ%iEVTYFJ>EIz+gmQ~h<+Xjl(yVe8wU+Wihwzbj;e;yc?Z?B;GZK+W1YEezi zqJ52Dwt~~p{s6?R-&1kEFS&0Rv1P^LEUZmEec!_GJqwS;=&3 zE4B?TE0hx=m-&AsseYzmS)swPL2>UGe4OYQV!p99cVO<7H_ho1m%D#PMCfyb!D@DP z%yOuX)6sj*;kUZ1>w{+~xqV#=M}NCH2?j6`2>B5y-j_UB%`6X>(@iNNHC`g`O2%${ zo*TL8NB#@7I;(SZ3rk9meTisCwZow36MaZ~Cg)LJ=&}h#)?MO|qT=5@bTG;q6;Y)D zhPq!b_dY}oY+&WRzAh{X5B?0D7jUwaHSbrZFv;?46d3{1j$`dXVXn}=Gts#b#zNla zQ_scDV_*-$0sCUFxNwl4;Mvm6q*Sps^8Hjg*v3D*)}%Xb$GVL)#TB6`b$o$MCpyzMMZBvNQcfy?^ep<<85;2pAyz$O! zT@AEj`$d+WvW6qmZ?0#{o7z*W@tOVC)IP$45jGY-iZJ|fNh@~N%)C54j2A@v=S(jUviP^GoBHtY85~%-=u?hfUqL)iAr5o7q zZ?YSpFlW_vm@J1z0T7Fyoc0zmOn;qmYQ0aT>qL!iw z9(A4C;9o)v6JV^gVf{d(tv#9bXwa=aJy->{ILHzPr2Uf_kKObz_g}~NuEZ`t{DB&$ z%R_7cXu>rn!`UH`>R_*xh@2}?MX=%m{Li=b$lgocCVtCg4Eidt9ZCqk#zyp9tSo9t zyN!y|Z-3{`oz{`~j)RZb>9ona@#6uo3c1j z0`LZSEkLkXk_Pe-a6=ZwD}ZTSL9Y{O=v|<<1K%OS^9*&jy_VV2UPof5InS%pq${9W z4?;oYmDu4NB`w$OGX67o2zXO0FTtvFu3;tpYz7zrWa44SkhZ<@IG|I$N`*`#!CL!Yi6gcT-g_ubhU;qKQCk*l^KwpzIr_+zQ(T2+-a*0T{*$g5A zHwi}*iuWrXV#>jJfGy4z09-oTEV$YPDem)A(7yHC%g`H$tL3%_LBC)kZcVUq>@!YH z+YaI3KaM9YPYw^w#*3H1%Nlm;f8j7_g0mG218^x2rBF*sW-|*v{ZP$^yvJeNBLb6uc=23rOeu96YIiGsVxs!a^A4bEW^t zz{;vHB}Lc&9<)9ezfN+pCuV0GTu*jzHJv;!#iCHXtPX%*0PwMy)Hgt{0CW`$ua1=S z#Z2!{ygN;BoWaw`+~mr`fFngjL;!+s?iR^cWWSX(Pft&H1Qg&0Edh#3N(EXCiF}&n zW_aMvYk`R0QdO(Vu!iIEhbz3}nQ4zgccMD-GCSHgE|c{Pzxm|IGVB&LIeP{YO&xzxm5(@6t>CVlD2&D9JG` zk`0>AAQyF~?g;)aA^!cV;&Izj^WTA+rJrl!mi@S&62`3Nu)$>vys{!)M|LeGs#J1D zF-FhmhK3dXVuHt&T|)xfS!^K^_*-{!*B(w%&BXTJnc6E)0fnSjT0^_`Z-_lAW#E2c zXl2a<++;yL2^XqQrXzGUSRI0m)<2IfV%*tq*K}mnV9k0&06cVrKNOYnBB$!LX;Y2e z=`>}m(g@SLAlNo}va^QprM_@>Gpa)cT&BPK0vI2eMvYLXrS|Ymyi_P-zb}Phsr)qu zgxd=W>RFSqyO0RHM+$4dYjTlF1tYl=FKWAe=b#IdVk3^p+5e=<6FGRd`#!j_fx6Of zh9!o2PpyV~(!}3AOU~Qd;1iJPtDh6k;@_7`KGy$EI(!;W6K`)$rx}+h)fxW=drS_= zMbm)iM3B3NyEcu|EG_QeX;52`BY;t=i9fq$G#82wd`=3wc=M8T$FPH1xYQuZl?}ge zVDc4X&+y2F{)un~KI%z7n$lW8DW$WANGW!^)>pY-EO+(!@kt}KDf6L}v^9H|z)a#7wH0#;4 zSS1l8y%HZ;xwCWaq!g{HbM#dvJqV zXht3)Q}%xx_)QFAtYSHX%sKq7Qb*wK%W&7Gylp?bci)sdN58aY5luW$gAAzrDGM)t zv!Hzu4lhC|qLi;9Y<;ep@bpTBOfJnhe9`kLKZ_Yx3@ zP>bs;i>~O05mT>S`jJWpR_ur+Qzwh@duSNtCinEqyy>3CZIY!=-=(L-m3$hWYZi%b z#K+09*P_Z1Mic44J$0+Cq(%=)ypK%kq$YT8s2vV)sAvoew{ptTN?VHtmX*_KucP>+ z$c^nj4{oi`58_*B&2J`VB#w55oQMd_w-bDd*YLPF(>J~CcxJS!*+2b3Qy=jGnqsnH zV86%0b&pyy-J2>q)%y`|k{>|@-?|O;AGR*2WMb_Lkp_0((mIPwG(q zcp2sE5~MWig0hjMtxzdxnm3{`w5^UD6_@>I6>Og`r z=;JwX;+cWsw_iiOO1LV!4GJpAd1da4YVxs&wV@~`Mtbs83_D^fY2-zeaPTTjl)Ds+ zs*&_6ID^cbDTm6}$(r~`(qvoP#SP9}*}=>mF!CR_mL!fOO8NHoj?NWa&p1XLo??`k|!Cjdk3rgk4qN z2~O~XGx!Ok6}r|76N`{TR& zoJn_$<+-`CXn+oCdv9K|3ZQ?pCRM5fgf|K^GFAbyqj+ATBfiu8wGjj)D7}T#fict< z5G^708Y0?WRgnCPAl(l({*w_wFJ*KpWc!$_ggcd*PqN_yC7GzM zOwY>dGeY-k=nAr$0-n>$4N(1aD@aNYcc39(vo7{qw}W@jWkP#HN$3Q!*C z8o?-o-BBLaKREX>Jr_SRe>_VyOQiiB z9*c@LsIKLH*9tgjf(vx;5$urR-wNnMbRERq=tJpmC%s}D?-T1?UJWrz`uFwWR0L`1!Obx1;x>lim#XK)4y(-P~;$o ziB{-A<}H<*w!^@x4^TJ!bW5yHkUgo0grtpSWf^a1FUF&LZufd#y^81> z5Q^gH5!TF3ucB0&$W|uo_a|h>^ia+ZqcgcBq#-7B?WK}KiZBi0g~wuY%Qfo8i2Hnt zhpZ3?wt?1zjv}&5K^nr8L#kxiIe8ULttRWT?g5Oht+6>H1I^endF|HH7(Eg#^AUKk zFg+G?eJ?3U$m(3eO14r&Yi&Z82D(1GVIaDZlo&m_WjZdBKO#Ad6BSaIt#JDHZ9K7Z zaU$2mwQ6Nx^TG_|6j4IK-j0~7^QRo0pagL+jCcHI&&%J!5d~e8VUHrS;S|P9_~&8W zqV|2zVkZH6TUV-zfAYLX2O9GuV9x4pAbVg0JFS=?HRqQY0SZX@CZV zyG8#l!!WA>xWP8NUi5<^r)#nJBU!SvX-S)2-mIOPeT|A$b-b6jc0TZc1A6|WEh3j* zI32D#QC&T}!KIE2gW9>IMTe+EqBUV2KEF%>$K5y#+Wx+R+`{Y2{m9t}^~mLJw>u<= zy)VH@NZj-{_!&!mgPr92(DlP(OCBWuofgj!cg;V4j`~UnS!BMAcRVdS+%b8A&~_6V zv;#zey*3ice~1E>?we{B6#(B~%-R!aIaw7$KMCjpz!6UEZ!_zDGHo{~b#>l%rTz^2 zZhR&N;)jwsIw&jPrJ)b!!bkUkJ>K62vG7iyXB;(z1EVEG)>>N z@5_I#Q>_oYGT=zY;GC#c>txLq=MLyo;kXnysCB0ZhbqqAu)Jl85g~E6^M~=e9!r{? zL2sx?O0B};Vz}TPYq|06OdaSyYcYbzcnY?Q$%czbqe0s~-jkA~i>*(vbcrb7^|q=s z29W|#1m=D`UL*x5_!B_EYlWD*tfxmQ=Bctyl!D`*pA-h)&$%Hf$f56b8l zv*irW*(j(UizA-y^SHD<%UJZkI%s6BHfOEqw<87}58%{|F!z)2k?hq;%OQD`I@7sQ z>?3g0hb;L0ZD81x2^VkvdS1m0{-hoFb;0Im;F-w@81P@~x8GH+LoID%c1OcXA#8M7 z^=`U@R=#@IEO?O=jbiSjNkD^BvO#8jI||cEV<#;U7QlTQl&*uFVy=kgVL(ff6!i3c zAay88`G)}qw(i->d zKDj*7x`I*)5_4(Legru6BIQ7E)(1yP@ZXpJiJ{cp)~aA9@EO2&A4-xuHuc3ZY`i>^ z)$)v|@&?TK$a+IKs0cG^kd|(o*CR}an#@R|B4)la+~=;^xdC15D_meCg~J`eU{GI^ z@1#g0ghYpRFWtMkni`8!0Q5JPiohPUfieXyH@3+5k8r*C_eJvTL&ZhM#CV~|(yh?L zvzL|*%hdcHrH#J#;I-pBcZ3AB%)!7YR+~g)Fu zm+R<#tu`}t|Affqw->4Gf-yRlYl&99uR%3Z2Jgiyi#hwOyKGCnv@5 z2=c)6zj?YJF7dp&!<29}h2Lc))Vh#siXxAy%2dgxT3Q@4#F=#ZlR(CLGqp#7rEbjz zBIqfGD2#cNkByk}ifNkhT>^V`Q&@+>6$XP|Zkw9QU5%NWk1v!wNM{(BNR&4!ohO`m z@uBT?AXGD*D%aZq+8|7z#Gys;n`n9>J&|wu zyh(@(VT9gXP!s$wGh1Fnnfd=MLHOzkpuS^rsxnHrH!9LG#v&OEWaq$m7aWwSoE z<=WlrzGshJnw^m1asA$CqQQgp1i*;Y$_D@IUQCCQU#)8GJ=Bk5r{yy91Hry3gZ0b9 zxD+rwCV+;-eA?h?>%t3<8C8)!Aw$_OHPr)eWS-!JL4|0dqZXhK%~{_+xI}zJmx9U^ zaZV7rBjR6!WsQ_~R!0x$Sh3hxGqNdm4O1bl7D5eTrnqh;aDM;jh$v%N@apYDS_9Md zWTE>`yajmnU$WFfzUJ5~i{JT+3t*Xw&O2|XeKb7+oISdJv0~I-ZDg^5M6M>17u