diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c5da3fd..5748469b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,17 @@ jobs: - uses: ./.github/actions/setup - run: cargo test --features integration + # Runs the whole lib unit suite with `grpc` off, the proof that the abstraction builds and runs + # without gRPC. `lfs: true` because some planner unit tests load parquet testdata. + unit-test-no-grpc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - uses: ./.github/actions/setup + - run: cargo test --no-default-features --lib + tpch-correctness-test: runs-on: ubuntu-latest strategy: @@ -50,7 +61,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test --features tpch --test tpch_correctness_test + - run: cargo test -p datafusion-distributed-benchmarks --features tpch --test tpch_correctness_test env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -59,7 +70,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test --features tpch --test tpch_plans_test + - run: cargo test -p datafusion-distributed-benchmarks --features tpch --test tpch_plans_test tpcds-correctness-test: runs-on: ubuntu-latest @@ -73,9 +84,9 @@ jobs: - uses: ./.github/actions/setup - uses: actions/cache@v4 with: - path: testdata/tpcds/main.zip + path: benchmarks/testdata/tpcds/main.zip key: "main.zip" - - run: cargo test --features tpcds --test tpcds_correctness_test shard${{ matrix.shard }} + - run: cargo test -p datafusion-distributed-benchmarks --features tpcds --test tpcds_correctness_test shard${{ matrix.shard }} env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -86,9 +97,9 @@ jobs: - uses: ./.github/actions/setup - uses: actions/cache@v4 with: - path: testdata/tpcds/main.zip + path: benchmarks/testdata/tpcds/main.zip key: "main.zip" - - run: cargo test --features tpcds --test tpcds_plans_test + - run: cargo test -p datafusion-distributed-benchmarks --features tpcds --test tpcds_plans_test clickbench-correctness-test: runs-on: ubuntu-latest @@ -101,9 +112,9 @@ jobs: - uses: ./.github/actions/setup - uses: actions/cache@v4 with: - path: testdata/clickbench/ + path: benchmarks/testdata/clickbench/ key: "data" - - run: cargo test --features clickbench --test clickbench_correctness_test + - run: cargo test -p datafusion-distributed-benchmarks --features clickbench --test clickbench_correctness_test env: ADAPTIVE: ${{ matrix.planning_mode == 'adaptive' }} @@ -114,9 +125,9 @@ jobs: - uses: ./.github/actions/setup - uses: actions/cache@v4 with: - path: testdata/clickbench/ + path: benchmarks/testdata/clickbench/ key: "data" - - run: cargo test --features clickbench --test clickbench_plans_test + - run: cargo test -p datafusion-distributed-benchmarks --features clickbench --test clickbench_plans_test format-check: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index dbe6096c..12a4e62b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2199,7 +2199,6 @@ dependencies = [ "crossbeam-queue", "dashmap", "datafusion", - "datafusion-distributed-benchmarks", "datafusion-proto", "delegate", "futures", @@ -2249,12 +2248,14 @@ dependencies = [ "object_store", "openssl", "parquet", + "pretty_assertions", "reqwest", "serde", "serde_json", "sketches-ddsketch", "structopt", "sysinfo", + "test-case", "tokio", "tonic", "tpchgen", diff --git a/Cargo.toml b/Cargo.toml index 79366ed3..1bb4874d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ sysinfo = { version = "0.30", optional = true } sketches-ddsketch = { version = "0.3", features = ["use_serde"] } num-traits = "0.2" bincode = "1" -tonic-prost = "0.14.2" +tonic-prost = { version = "0.14.2", optional = true } # grpc-specific features arrow-flight = { version = "58", optional = true } @@ -68,6 +68,7 @@ grpc = [ "arrow-select", "arrow-ipc", "tonic", + "tonic-prost", "tower" ] @@ -75,14 +76,9 @@ integration = ["insta", "parquet", "arrow", "hyper-util", "grpc"] system-metrics = ["sysinfo"] -tpch = ["integration"] -tpcds = ["integration"] -clickbench = ["integration"] -slow-tests = [] sysinfo = ["dep:sysinfo"] [dev-dependencies] -datafusion-distributed-benchmarks = { path = "benchmarks" } structopt = "0.3" insta = { version = "1.46.0", features = ["filters"] } parquet = "58" @@ -92,6 +88,43 @@ hyper-util = "0.1.16" pretty_assertions = "1.4" test-case = "3.3.1" +# Every example drives the gRPC transport, so a no-grpc build skips them. +[[example]] +name = "custom_distributed_partial_reduction_tree" +required-features = ["integration"] + +[[example]] +name = "custom_execution_plan" +required-features = ["integration"] + +[[example]] +name = "custom_worker_url_routing" +required-features = ["integration"] + +[[example]] +name = "work_unit_feed" +required-features = ["integration"] + +[[example]] +name = "in_memory_cluster" +required-features = ["grpc"] + +[[example]] +name = "localhost_run" +required-features = ["grpc"] + +[[example]] +name = "localhost_versioned_run" +required-features = ["grpc"] + +[[example]] +name = "localhost_worker" +required-features = ["grpc"] + +[[example]] +name = "localhost_versioned_worker" +required-features = ["grpc"] + [workspace.lints.clippy] disallowed_types = "deny" disallowed-methods = "deny" diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 531a8aba..58736c66 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -34,9 +34,20 @@ openssl = { version = "0.10", features = ["vendored"] } # Keep this. Necessary f mimalloc = "0.1" sketches-ddsketch = "0.3" +[features] +# Gates for the dataset test suites under `tests/`. They live here rather than in the library +# because, as a dev-dependency of the library, this crate re-enables `grpc` on every test build +# through feature unification, which made the no-grpc config untestable. +tpch = [] +tpcds = [] +clickbench = [] +slow-tests = [] + [dev-dependencies] criterion = "0.5" sysinfo = "0.30" +pretty_assertions = "1.4" +test-case = "3.3.1" [build-dependencies] built = { version = "0.8", features = ["git2", "chrono"] } diff --git a/benchmarks/cdk/bin/worker.rs b/benchmarks/cdk/bin/worker.rs index 96a399c8..cdeb64a3 100644 --- a/benchmarks/cdk/bin/worker.rs +++ b/benchmarks/cdk/bin/worker.rs @@ -1,3 +1,4 @@ +#![allow(clippy::disallowed_types)] use async_trait::async_trait; use aws_config::BehaviorVersion; use aws_sdk_ec2::Client as Ec2Client; diff --git a/tests/clickbench_correctness_test.rs b/benchmarks/tests/clickbench_correctness_test.rs similarity index 99% rename from tests/clickbench_correctness_test.rs rename to benchmarks/tests/clickbench_correctness_test.rs index acf3daa8..25fb38f8 100644 --- a/tests/clickbench_correctness_test.rs +++ b/benchmarks/tests/clickbench_correctness_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "clickbench", test))] +#[cfg(all(feature = "clickbench", test))] mod tests { use datafusion::arrow::array::RecordBatch; use datafusion::common::plan_err; diff --git a/tests/clickbench_plans_test.rs b/benchmarks/tests/clickbench_plans_test.rs similarity index 99% rename from tests/clickbench_plans_test.rs rename to benchmarks/tests/clickbench_plans_test.rs index b9c32bb1..7291d4f7 100644 --- a/tests/clickbench_plans_test.rs +++ b/benchmarks/tests/clickbench_plans_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "clickbench", test))] +#[cfg(all(feature = "clickbench", test))] mod tests { use datafusion::error::Result; use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; diff --git a/tests/stateful_data_cleanup.rs b/benchmarks/tests/stateful_data_cleanup.rs similarity index 98% rename from tests/stateful_data_cleanup.rs rename to benchmarks/tests/stateful_data_cleanup.rs index fae892d2..f977c61a 100644 --- a/tests/stateful_data_cleanup.rs +++ b/benchmarks/tests/stateful_data_cleanup.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "tpch", test))] +#[cfg(all(feature = "tpch", test))] mod tests { use datafusion::common::instant::Instant; use datafusion::error::Result; diff --git a/tests/tpcds_correctness_test.rs b/benchmarks/tests/tpcds_correctness_test.rs similarity index 99% rename from tests/tpcds_correctness_test.rs rename to benchmarks/tests/tpcds_correctness_test.rs index e4baeba9..8d5d8722 100644 --- a/tests/tpcds_correctness_test.rs +++ b/benchmarks/tests/tpcds_correctness_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "tpcds", test))] +#[cfg(all(feature = "tpcds", test))] mod tests { use datafusion::arrow::array::RecordBatch; use datafusion::common::plan_err; diff --git a/tests/tpcds_plans_test.rs b/benchmarks/tests/tpcds_plans_test.rs similarity index 99% rename from tests/tpcds_plans_test.rs rename to benchmarks/tests/tpcds_plans_test.rs index 412c50a2..b6363ad4 100644 --- a/tests/tpcds_plans_test.rs +++ b/benchmarks/tests/tpcds_plans_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "tpcds", test))] +#[cfg(all(feature = "tpcds", test))] mod tests { use datafusion::error::Result; use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; diff --git a/tests/tpch_correctness_test.rs b/benchmarks/tests/tpch_correctness_test.rs similarity index 99% rename from tests/tpch_correctness_test.rs rename to benchmarks/tests/tpch_correctness_test.rs index 3ae089dc..7d6ab2ce 100644 --- a/tests/tpch_correctness_test.rs +++ b/benchmarks/tests/tpch_correctness_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "tpch", test))] +#[cfg(all(feature = "tpch", test))] mod tests { use datafusion::physical_plan::execute_stream; use datafusion::prelude::SessionContext; diff --git a/tests/tpch_plans_test.rs b/benchmarks/tests/tpch_plans_test.rs similarity index 99% rename from tests/tpch_plans_test.rs rename to benchmarks/tests/tpch_plans_test.rs index 016e21ea..f625cfec 100644 --- a/tests/tpch_plans_test.rs +++ b/benchmarks/tests/tpch_plans_test.rs @@ -1,4 +1,4 @@ -#[cfg(all(feature = "integration", feature = "tpch", test))] +#[cfg(all(feature = "tpch", test))] mod tests { use datafusion_distributed::test_utils::in_memory_channel_resolver::start_in_memory_context; use datafusion_distributed::{ diff --git a/console/src/main.rs b/console/src/main.rs index 731b504b..ab1171d7 100644 --- a/console/src/main.rs +++ b/console/src/main.rs @@ -63,10 +63,10 @@ async fn run_app( terminal.draw(|frame| ui::render(frame, app))?; // Check for keyboard input (16ms timeout ~ 60fps responsiveness) - if event::poll(Duration::from_millis(16))? { - if let Event::Key(key) = event::read()? { - input::handle_key_event(app, key); - } + if event::poll(Duration::from_millis(16))? + && let Event::Key(key) = event::read()? + { + input::handle_key_event(app, key); } if app.should_quit { diff --git a/console/src/worker.rs b/console/src/worker.rs index 4a23c5b9..44c06260 100644 --- a/console/src/worker.rs +++ b/console/src/worker.rs @@ -176,32 +176,32 @@ impl WorkerConn { // Detect completed tasks: tasks that were running but disappeared for old_task in &self.tasks { - if old_task.status == TaskStatus::Running as i32 { - if let Some(sk) = &old_task.task_key { - let key = (sk.query_id.clone(), sk.stage_id, sk.task_number); - if !new_task_keys.contains(&key) { - // Task disappeared — assume completed - let observed_duration = self - .task_first_seen - .get(&key) - .map(|first| first.elapsed()) - .unwrap_or_default(); - - self.completed_tasks.push_front(CompletedTaskRecord { - query_id: sk.query_id.clone(), - stage_id: sk.stage_id, - task_number: sk.task_number, - observed_duration, - }); - - // Maintain bounded size - while self.completed_tasks.len() > MAX_COMPLETED_TASKS { - self.completed_tasks.pop_back(); - } - - // Remove from first_seen tracking - self.task_first_seen.remove(&key); + if old_task.status == TaskStatus::Running as i32 + && let Some(sk) = &old_task.task_key + { + let key = (sk.query_id.clone(), sk.stage_id, sk.task_number); + if !new_task_keys.contains(&key) { + // Task disappeared — assume completed + let observed_duration = self + .task_first_seen + .get(&key) + .map(|first| first.elapsed()) + .unwrap_or_default(); + + self.completed_tasks.push_front(CompletedTaskRecord { + query_id: sk.query_id.clone(), + stage_id: sk.stage_id, + task_number: sk.task_number, + observed_duration, + }); + + // Maintain bounded size + while self.completed_tasks.len() > MAX_COMPLETED_TASKS { + self.completed_tasks.pop_back(); } + + // Remove from first_seen tracking + self.task_first_seen.remove(&key); } } } diff --git a/src/coordinator/metrics_store.rs b/src/coordinator/metrics_store.rs index 8ccc1f7c..ed55db2c 100644 --- a/src/coordinator/metrics_store.rs +++ b/src/coordinator/metrics_store.rs @@ -27,7 +27,7 @@ impl MetricsStore { self.rx.borrow().get(key).cloned() } - #[cfg(test)] + #[cfg(all(test, feature = "grpc"))] pub(crate) fn from_entries(entries: impl IntoIterator) -> Self { let map: HashMap<_, _> = entries.into_iter().collect(); let (tx, rx) = watch::channel(map); diff --git a/src/execution_plans/metrics.rs b/src/execution_plans/metrics.rs index 0fefdf3e..de97ed04 100644 --- a/src/execution_plans/metrics.rs +++ b/src/execution_plans/metrics.rs @@ -21,7 +21,7 @@ impl MetricsWrapperExec { Self { inner, metrics } } - #[cfg(test)] + #[cfg(all(test, feature = "grpc"))] pub(crate) fn inner(&self) -> &Arc { &self.inner } diff --git a/src/execution_plans/mod.rs b/src/execution_plans/mod.rs index ecdcbc35..97904ef1 100644 --- a/src/execution_plans/mod.rs +++ b/src/execution_plans/mod.rs @@ -8,7 +8,9 @@ mod network_coalesce; mod network_shuffle; mod sampler; -#[cfg(any(test, feature = "integration"))] +// The benchmark fixtures spin up real gRPC workers (`tonic` servers, Flight channels), so they +// only exist when that transport is compiled in. +#[cfg(all(feature = "grpc", any(test, feature = "integration")))] pub mod benchmarks; pub use broadcast::BroadcastExec; diff --git a/src/lib.rs b/src/lib.rs index b132caf5..f7becd63 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,14 +46,19 @@ pub mod test_utils; #[cfg(feature = "grpc")] pub use protocol::grpc; +/// The worker-protocol prost message types, independent of any transport. A non-gRPC transport +/// reaches for these to speak the same wire shape the gRPC path serializes. Unstable: this +/// mirrors `worker.proto`, which is regenerated freely. +pub use protocol::generated::worker as proto; + pub use codec::DistributedCodec; pub use worker_resolver::{WorkerResolver, get_distributed_worker_resolver}; pub use protocol::{ ChannelResolver, CoordinatorToWorkerMsg, ExecuteTaskRequest, GetWorkerInfoRequest, - GetWorkerInfoResponse, LoadInfo, ProducerHeadSpec, SetPlanRequest, TaskKey, TaskMetrics, - WorkUnitBatch, WorkUnitFeedDeclaration, WorkUnitMsg, WorkerChannel, WorkerToCoordinatorMsg, - get_distributed_channel_resolver, + GetWorkerInfoResponse, InProcessChannelResolver, LoadInfo, ProducerHeadSpec, SetPlanRequest, + TaskKey, TaskMetrics, WorkUnitBatch, WorkUnitFeedDeclaration, WorkUnitMsg, WorkerChannel, + WorkerToCoordinatorMsg, get_distributed_channel_resolver, }; pub use stage::{ DistributedTaskContext, Stage, display_plan_ascii, display_plan_graphviz, explain_analyze, @@ -66,7 +71,7 @@ pub use worker::{ Worker, WorkerQueryContext, WorkerSessionBuilder, }; -#[cfg(any(feature = "integration", test))] +#[cfg(all(feature = "grpc", any(feature = "integration", test)))] pub use execution_plans::benchmarks::{ LocalRepartitionBench, LocalRepartitionFixture, LocalRepartitionMode, ShuffleBench, ShuffleFixture, TransportBench, TransportBenchMode, TransportFixture, diff --git a/src/metrics/task_metrics_collector.rs b/src/metrics/task_metrics_collector.rs index ba6e6855..f0647b95 100644 --- a/src/metrics/task_metrics_collector.rs +++ b/src/metrics/task_metrics_collector.rs @@ -19,7 +19,8 @@ pub fn collect_plan_metrics(plan: &Arc) -> Result, +} +/// Nested message and enum types in `CoordinatorToWorkerMsg`. +pub mod coordinator_to_worker_msg { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Inner { + /// Sends a subplan to a worker so that a future ExecuteTask call can actually execute it. + /// The plan is identified by a TaskKey. + #[prost(message, tag = "1")] + SetPlanRequest(super::SetPlanRequest), + /// A batch of messages from a work unit feed belonging to different partitions from one node from the plan set in + /// set_plan_request. A work unit feed is a per-partition stream of information that tells the node what should + /// be executed within a partition, for example, a stream of file addresses that should be read. + #[prost(message, tag = "2")] + WorkUnitBatch(super::WorkUnitBatch), + /// Signals an EOS for WorkUnits. After this message is received, no more WorkUnits will be sent. + #[prost(bool, tag = "3")] + WorkUnitEos(bool), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct WorkerToCoordinatorMsg { + #[prost(oneof = "worker_to_coordinator_msg::Inner", tags = "1, 2, 3")] + pub inner: ::core::option::Option, +} +/// Nested message and enum types in `WorkerToCoordinatorMsg`. +pub mod worker_to_coordinator_msg { + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Inner { + /// Sends the metrics collected during task execution back to the coordinator. + /// This is sent after all partitions of a task have finished (or been dropped), + /// ensuring metrics are never lost due to early stream termination. + /// metrics\[i\] is the set of metrics for plan node i in pre-order traversal order. + #[prost(message, tag = "1")] + TaskMetrics(super::TaskMetrics), + /// Load information reported by a task. This information is used for dynamically + /// sizing the number of workers involved in a query. + #[prost(message, tag = "2")] + LoadInfo(super::LoadInfo), + #[prost(bool, tag = "3")] + LoadInfoEos(bool), + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TaskMetrics { + /// Metrics for a single task's plan nodes in pre-order traversal order. + /// The TaskKey is implicit — it is determined by the SetPlanRequest that + /// opened this coordinator channel connection. + #[prost(message, repeated, tag = "1")] + pub pre_order_plan_metrics: ::prost::alloc::vec::Vec, + /// Metrics related to the execution of a task within a stage. This metrics, instead of being + /// associated to a specific node, they are global to the task, like the time at which the plan + /// was fed by the coordinator to the worker. + #[prost(message, optional, tag = "2")] + pub task_metrics: ::core::option::Option, +} +/// Load information reported for a specific partition with information about this +/// amount of data flowing through the plan. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LoadInfo { + /// The partition index to which this message belongs to. + #[prost(uint64, tag = "1")] + pub partition: u64, + /// The amount of rows ready to be returned. + #[prost(uint64, tag = "2")] + pub rows_ready: u64, + /// The amount of bytes ready to be returned per column. + #[prost(uint64, repeated, tag = "4")] + pub per_column_bytes_ready: ::prost::alloc::vec::Vec, + /// Approximate ratio of NDV for each column. + #[prost(float, repeated, tag = "6")] + pub per_column_ndv_percentage: ::prost::alloc::vec::Vec, + /// Approximate ratio of null count for each column. + #[prost(float, repeated, tag = "7")] + pub per_column_null_percentage: ::prost::alloc::vec::Vec, + /// The amount of rows that were pulled from leaf nodes while this partition was sampling data. + #[prost(uint64, tag = "8")] + pub rows_pulled_from_leaf: u64, + /// Whether the sampled partition stream reached end-of-stream (i.e. the partition finished + /// producing all of its output) by the time this LoadInfo was captured. + #[prost(bool, tag = "9")] + pub reached_eos: bool, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetWorkerInfoRequest {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetWorkerInfoResponse { + #[prost(string, tag = "1")] + pub version: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct SetPlanRequest { + /// The unique identifier of the task to which the subplan belongs to. + #[prost(message, optional, tag = "1")] + pub task_key: ::core::option::Option, + /// The amount of tasks that share the same subplan. Necessary for building the DistributedTaskContext during execution. + #[prost(uint64, tag = "2")] + pub task_count: u64, + /// The serialized subplan the worker is expected to execute on an ExecuteTask gRPC call. + #[prost(bytes = "vec", tag = "3")] + pub plan_proto: ::prost::alloc::vec::Vec, + /// Information about all the work unit feeds that will be streamed from coordinator to worker. + /// This information is needed here because at the moment of setting the plan, all the appropriate + /// channels for the incoming work unit feeds need to be constructed. + /// + /// If no WorkUnitFeedExec nodes are present in the plan, this should be empty. + #[prost(message, repeated, tag = "4")] + pub work_unit_feed_declarations: + ::prost::alloc::vec::Vec, + /// The worker URL to which this message will go. The receiving worker will use this information to identify + /// itself, and avoid further gRPC calls in case it needs to call itself for executing remote tasks. + #[prost(string, tag = "5")] + pub target_worker_url: ::prost::alloc::string::String, + /// Unix nanos when the query started as reported by the coordinator. Used for collecting temporal metrics + /// relative to when the query was fired in the coordinator. + #[prost(uint64, tag = "6")] + pub query_start_time_ns: u64, +} +/// Nested message and enum types in `SetPlanRequest`. +pub mod set_plan_request { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] + pub struct WorkUnitFeedDeclaration { + /// Unique identifier of the node to which work unit feeds are expected to be streamed. + #[prost(bytes = "vec", tag = "1")] + pub id: ::prost::alloc::vec::Vec, + /// The amount of partitions expected to be streamed. + #[prost(uint64, tag = "2")] + pub partitions: u64, + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct WorkUnitBatch { + /// A batch of WorkUnits. + #[prost(message, repeated, tag = "1")] + pub batch: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct WorkUnit { + /// Identifier of the node to which this work unit feed belongs to. + #[prost(bytes = "vec", tag = "1")] + pub id: ::prost::alloc::vec::Vec, + /// The partition index within the node to which the work unit feed belongs to. + #[prost(uint64, tag = "2")] + pub partition: u64, + /// Arbitrary user-defined data (e.g., a file address) necessary during execution. + #[prost(bytes = "vec", tag = "3")] + pub body: ::prost::alloc::vec::Vec, + /// Unix timestamp in nanoseconds at which this message was created in the coordinator. + #[prost(uint64, tag = "4")] + pub created_timestamp_unix_nanos: u64, + /// Unix timestamp in nanoseconds at which this message was sent by the coordinator. + #[prost(uint64, tag = "5")] + pub sent_timestamp_unix_nanos: u64, + /// Unix timestamp in nanoseconds at which this message was received by a worker. + #[prost(uint64, tag = "6")] + pub received_timestamp_unix_nanos: u64, + /// Unix timestamp in nanoseconds at which this message started being processed. + #[prost(uint64, tag = "7")] + pub processed_timestamp_unix_nanos: u64, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExecuteTaskRequest { + /// The unique identifier of the task that is going to get executed. + #[prost(message, optional, tag = "1")] + pub task_key: ::core::option::Option, + /// The start of the partition range of the specified task that is going to be executed. + #[prost(uint64, tag = "2")] + pub target_partition_start: u64, + /// The end of the partition range of the specified task that is going to be executed. + #[prost(uint64, tag = "3")] + pub target_partition_end: u64, + /// The head node the requested task should have. Depending on the network boundary executing + /// the task, the head node should be prepared differently, for example: + /// + /// * A RepartitionExecHead implies a RepartitionExec at the head of the task. + /// * A BroadcastExecHead implies a BroadcastExec at the head of the task. + /// * A NoneHead does not need any specific head. + #[prost(oneof = "execute_task_request::ProducerHead", tags = "6, 7, 8")] + pub producer_head: ::core::option::Option, +} +/// Nested message and enum types in `ExecuteTaskRequest`. +pub mod execute_task_request { + /// The head node the requested task should have. Depending on the network boundary executing + /// the task, the head node should be prepared differently, for example: + /// + /// * A RepartitionExecHead implies a RepartitionExec at the head of the task. + /// * A BroadcastExecHead implies a BroadcastExec at the head of the task. + /// * A NoneHead does not need any specific head. + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum ProducerHead { + /// The boundary executing the task is a NetworkCoalesceExec. + #[prost(message, tag = "6")] + None(super::NoneHead), + /// The boundary executing the task is a NetworkBroadcastExec. + #[prost(message, tag = "7")] + Broadcast(super::BroadcastExecHead), + /// The boundary executing the task is a NetworkShuffleExec. + #[prost(message, tag = "8")] + Repartition(super::RepartitionExecHead), + } +} +/// Head needed by a NetworkCoalesceExec. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct NoneHead {} +/// Head needed by a NetworkBroadcastExec. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct BroadcastExecHead { + /// The amount of output partitions + #[prost(uint64, tag = "1")] + pub output_partitions: u64, +} +/// Head needed by a NetworkShuffleExec. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RepartitionExecHead { + /// `Partitioning` message from datafusion.proto + #[prost(bytes = "vec", tag = "1")] + pub partitioning: ::prost::alloc::vec::Vec, +} +/// A key that uniquely identifies a task in a query. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct TaskKey { + /// Our query id. + #[prost(bytes = "vec", tag = "1")] + pub query_id: ::prost::alloc::vec::Vec, + /// Our stage id. + #[prost(uint64, tag = "2")] + pub stage_id: u64, + /// The task number within the stage. + #[prost(uint64, tag = "3")] + pub task_number: u64, +} +/// FlightAppMetadata represents all types of app_metadata which we use in the distributed execution. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct FlightAppMetadata { + #[prost(uint64, tag = "1")] + pub partition: u64, + /// Unix timestamp in nanoseconds at which this message was created. + #[prost(uint64, tag = "2")] + pub created_timestamp_unix_nanos: u64, +} +/// A Label mirrors datafusion::physical_plan::metrics::Label. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Label { + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub value: ::prost::alloc::string::String, +} +/// A Metric is a protobuf mirror of datafusion::physical_plan::metrics::Metric. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Metric { + #[prost(message, repeated, tag = "1")] + pub labels: ::prost::alloc::vec::Vec