From ffbbfdd83c0214a606c07512eb6f2cf962996587 Mon Sep 17 00:00:00 2001 From: Nemo Yu Date: Tue, 16 Jun 2026 11:39:19 -0400 Subject: [PATCH 1/5] struct scalar conversation in vortex-datafusion Signed-off-by: Nemo Yu --- vortex-datafusion/Cargo.toml | 1 + vortex-datafusion/src/convert/scalars.rs | 113 ++++++++++++++++++++++- vortex-datafusion/src/convert/schema.rs | 15 ++- 3 files changed, 126 insertions(+), 3 deletions(-) diff --git a/vortex-datafusion/Cargo.toml b/vortex-datafusion/Cargo.toml index 2215bd4db8e..c2e1673c635 100644 --- a/vortex-datafusion/Cargo.toml +++ b/vortex-datafusion/Cargo.toml @@ -34,6 +34,7 @@ object_store = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "fs"] } tokio-stream = { workspace = true } tracing = { workspace = true, features = ["std", "attributes"] } +arrow-array = { workspace = true } vortex = { workspace = true, features = ["object_store", "tokio", "files"] } vortex-utils = { workspace = true, features = ["dashmap"] } diff --git a/vortex-datafusion/src/convert/scalars.rs b/vortex-datafusion/src/convert/scalars.rs index 95c088742b3..f2b49f3c7a7 100644 --- a/vortex-datafusion/src/convert/scalars.rs +++ b/vortex-datafusion/src/convert/scalars.rs @@ -1,6 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::sync::Arc; + +use arrow_array::Array; +use arrow_array::StructArray; +use arrow_schema::Field; +use arrow_schema::Fields; use datafusion_common::ScalarValue; use vortex::buffer::ByteBuffer; use vortex::dtype::DType; @@ -14,6 +20,7 @@ use vortex::dtype::i256; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_bail; +use vortex::error::vortex_err; use vortex::extension::datetime::AnyTemporal; use vortex::extension::datetime::TemporalMetadata; use vortex::extension::datetime::TimeUnit; @@ -113,7 +120,42 @@ impl TryToDataFusion for Scalar { ), DType::List(..) => todo!("list scalar conversion"), DType::FixedSizeList(..) => todo!("fixed-size list scalar conversion"), - DType::Struct(..) => todo!("struct scalar conversion"), + DType::Struct(struct_fields, _) => { + let scalar = self.as_struct(); + let (fields, arrays): (Vec, Vec<_>) = struct_fields + .names() + .iter() + .zip(struct_fields.fields()) + .enumerate() + .map(|(idx, (name, field_dtype))| { + let nullable = field_dtype.is_nullable(); + let child = if scalar.is_null() { + Scalar::null(field_dtype) + } else { + scalar + .field_by_idx(idx) + .ok_or_else(|| vortex_err!("missing struct field {name}"))? + }; + let array = child + .try_to_df()? + .to_array() + .map_err(|e| vortex_err!("failed to build struct field array: {e}"))?; + let field = Field::new(name.as_ref(), array.data_type().clone(), nullable); + Ok((field, array)) + }) + .collect::>>()? + .into_iter() + .unzip(); + + let fields = Fields::from(fields); + let struct_array = if scalar.is_null() { + StructArray::new_null(fields, 1) + } else { + StructArray::try_new(fields, arrays, None) + .map_err(|e| vortex_err!("failed to build struct scalar array: {e}"))? + }; + ScalarValue::Struct(Arc::new(struct_array)) + } DType::Union(..) => todo!("union scalar conversion"), DType::Variant(_) => vortex_bail!("Variant scalars aren't supported with DF"), DType::Extension(ext) => { @@ -288,6 +330,28 @@ impl FromDataFusion for Scalar { } } ScalarValue::Dictionary(_, v) => Scalar::from_df(v.as_ref()), + ScalarValue::Struct(array) => { + let nullable = array.is_null(0); + let dtype = DType::from_arrow((array.data_type(), nullable.into())); + if nullable { + Scalar::null(dtype) + } else { + let children = array + .columns() + .iter() + .map(|column| { + Scalar::from_df( + &ScalarValue::try_from_array(&**column, 0).unwrap_or_else(|e| { + unimplemented!( + "Can't convert struct field to a Vortex scalar: {e}" + ) + }), + ) + }) + .collect::>(); + Scalar::struct_(dtype, children) + } + } _ => unimplemented!("Can't convert {value:?} value to a Vortex scalar"), } } @@ -301,8 +365,10 @@ mod tests { use vortex::buffer::ByteBuffer; use vortex::dtype::DType; use vortex::dtype::DecimalDType; + use vortex::dtype::FieldNames; use vortex::dtype::Nullability; use vortex::dtype::PType; + use vortex::dtype::StructFields; use vortex::dtype::i256; use vortex::scalar::DecimalValue; use vortex::scalar::Scalar; @@ -691,4 +757,49 @@ mod tests { .into(); assert_eq!(result_bytes, vec![1u8, 2, 3, 4, 5]); } + + #[test] + fn struct_scalar_round_trips() -> VortexResult<()> { + let dtype = DType::Struct( + StructFields::new( + FieldNames::from(["x", "y"]), + vec![ + DType::Primitive(PType::F64, Nullability::NonNullable), + DType::Primitive(PType::F64, Nullability::NonNullable), + ], + ), + Nullability::NonNullable, + ); + let original = Scalar::struct_( + dtype, + vec![Scalar::from(-111.7610f64), Scalar::from(34.8697f64)], + ); + + let df = original.try_to_df()?; + assert!(matches!(df, ScalarValue::Struct(_))); + + // Back through `from_df` and out again yields the identical DataFusion struct value. + let back = Scalar::from_df(&df); + assert_eq!(back.try_to_df()?, df); + Ok(()) + } + + #[test] + fn null_struct_scalar_round_trips() -> VortexResult<()> { + let dtype = DType::Struct( + StructFields::new( + FieldNames::from(["x", "y"]), + vec![ + DType::Primitive(PType::F64, Nullability::Nullable), + DType::Primitive(PType::F64, Nullability::Nullable), + ], + ), + Nullability::Nullable, + ); + + let df = Scalar::null(dtype).try_to_df()?; + assert!(matches!(df, ScalarValue::Struct(_))); + assert!(Scalar::from_df(&df).is_null()); + Ok(()) + } } diff --git a/vortex-datafusion/src/convert/schema.rs b/vortex-datafusion/src/convert/schema.rs index 26d9d86ce18..172bff84791 100644 --- a/vortex-datafusion/src/convert/schema.rs +++ b/vortex-datafusion/src/convert/schema.rs @@ -8,6 +8,7 @@ use datafusion_common::Result as DFResult; use datafusion_common::exec_datafusion_err; use vortex::array::arrow::ArrowSession; use vortex::dtype::DType; +use vortex::dtype::StructFields; /// Calculate the physical Arrow schema for a Vortex file given its DType and the expected logical schema. /// @@ -85,9 +86,9 @@ fn calculate_physical_field_type( // RunEndEncoded loses its encoding DataType::RunEndEncoded(..) => logical_type.clone(), - // For struct types, recursively check each field + // For struct types, recursively check each field. DataType::Struct(logical_fields) => { - if let DType::Struct(struct_dtype, _) = dtype { + if let Some(struct_dtype) = struct_fields(dtype) { let physical_fields: Vec = struct_dtype .names() .iter() @@ -204,6 +205,16 @@ fn calculate_physical_field_type( }) } +/// The struct fields of `dtype` if it is a struct, or of an extension type whose storage is +/// (eventually) a struct -- e.g. the native geo Point over `Struct`. +fn struct_fields(dtype: &DType) -> Option<&StructFields> { + match dtype { + DType::Struct(fields, _) => Some(fields), + DType::Extension(ext) => struct_fields(ext.storage_dtype()), + _ => None, + } +} + #[cfg(test)] mod tests { use std::sync::Arc; From a16cfc5f34537116236e0c08eef85598226e839a Mon Sep 17 00:00:00 2001 From: Nemo Yu Date: Tue, 16 Jun 2026 13:20:03 -0400 Subject: [PATCH 2/5] address review comments Signed-off-by: Nemo Yu --- Cargo.lock | 1 + vortex-datafusion/src/convert/scalars.rs | 123 ++++++++++++----------- vortex-datafusion/src/convert/schema.rs | 18 ++-- 3 files changed, 72 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57be369d7c3..c73c9c121eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9607,6 +9607,7 @@ name = "vortex-datafusion" version = "0.1.0" dependencies = [ "anyhow", + "arrow-array", "arrow-schema", "async-trait", "datafusion 54.0.0", diff --git a/vortex-datafusion/src/convert/scalars.rs b/vortex-datafusion/src/convert/scalars.rs index f2b49f3c7a7..06a2d0be6b9 100644 --- a/vortex-datafusion/src/convert/scalars.rs +++ b/vortex-datafusion/src/convert/scalars.rs @@ -21,6 +21,7 @@ use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_bail; use vortex::error::vortex_err; +use vortex::error::vortex_panic; use vortex::extension::datetime::AnyTemporal; use vortex::extension::datetime::TemporalMetadata; use vortex::extension::datetime::TimeUnit; @@ -120,42 +121,7 @@ impl TryToDataFusion for Scalar { ), DType::List(..) => todo!("list scalar conversion"), DType::FixedSizeList(..) => todo!("fixed-size list scalar conversion"), - DType::Struct(struct_fields, _) => { - let scalar = self.as_struct(); - let (fields, arrays): (Vec, Vec<_>) = struct_fields - .names() - .iter() - .zip(struct_fields.fields()) - .enumerate() - .map(|(idx, (name, field_dtype))| { - let nullable = field_dtype.is_nullable(); - let child = if scalar.is_null() { - Scalar::null(field_dtype) - } else { - scalar - .field_by_idx(idx) - .ok_or_else(|| vortex_err!("missing struct field {name}"))? - }; - let array = child - .try_to_df()? - .to_array() - .map_err(|e| vortex_err!("failed to build struct field array: {e}"))?; - let field = Field::new(name.as_ref(), array.data_type().clone(), nullable); - Ok((field, array)) - }) - .collect::>>()? - .into_iter() - .unzip(); - - let fields = Fields::from(fields); - let struct_array = if scalar.is_null() { - StructArray::new_null(fields, 1) - } else { - StructArray::try_new(fields, arrays, None) - .map_err(|e| vortex_err!("failed to build struct scalar array: {e}"))? - }; - ScalarValue::Struct(Arc::new(struct_array)) - } + DType::Struct(..) => struct_to_df(self)?, DType::Union(..) => todo!("union scalar conversion"), DType::Variant(_) => vortex_bail!("Variant scalars aren't supported with DF"), DType::Extension(ext) => { @@ -330,33 +296,74 @@ impl FromDataFusion for Scalar { } } ScalarValue::Dictionary(_, v) => Scalar::from_df(v.as_ref()), - ScalarValue::Struct(array) => { - let nullable = array.is_null(0); - let dtype = DType::from_arrow((array.data_type(), nullable.into())); - if nullable { - Scalar::null(dtype) - } else { - let children = array - .columns() - .iter() - .map(|column| { - Scalar::from_df( - &ScalarValue::try_from_array(&**column, 0).unwrap_or_else(|e| { - unimplemented!( - "Can't convert struct field to a Vortex scalar: {e}" - ) - }), - ) - }) - .collect::>(); - Scalar::struct_(dtype, children) - } - } + ScalarValue::Struct(array) => struct_from_df(array), _ => unimplemented!("Can't convert {value:?} value to a Vortex scalar"), } } } +/// Converts a Vortex struct scalar to a DataFusion `ScalarValue::Struct`. +fn struct_to_df(scalar: &Scalar) -> VortexResult { + let scalar = scalar.as_struct(); + let struct_fields = scalar.struct_fields(); + let (fields, arrays): (Vec, Vec<_>) = struct_fields + .names() + .iter() + .zip(struct_fields.fields()) + .enumerate() + .map(|(idx, (name, field_dtype))| { + let nullable = field_dtype.is_nullable(); + let child = if scalar.is_null() { + Scalar::null(field_dtype) + } else { + scalar + .field_by_idx(idx) + .ok_or_else(|| vortex_err!("missing struct field {name}"))? + }; + let array = child + .try_to_df()? + .to_array() + .map_err(|e| vortex_err!("failed to build struct field array: {e}"))?; + Ok(( + Field::new(name.as_ref(), array.data_type().clone(), nullable), + array, + )) + }) + .collect::>>()? + .into_iter() + .unzip(); + + let fields = Fields::from(fields); + let struct_array = if scalar.is_null() { + StructArray::new_null(fields, 1) + } else { + StructArray::try_new(fields, arrays, None) + .map_err(|e| vortex_err!("failed to build struct scalar array: {e}"))? + }; + Ok(ScalarValue::Struct(Arc::new(struct_array))) +} + +/// Converts a DataFusion `ScalarValue::Struct` (a one-row struct array) to a Vortex struct scalar. +fn struct_from_df(array: &StructArray) -> Scalar { + let dtype = DType::from_arrow((array.data_type(), Nullability::Nullable)); + if array.is_null(0) { + Scalar::null(dtype) + } else { + let children = array + .columns() + .iter() + .map(|column| { + Scalar::from_df( + &ScalarValue::try_from_array(column.as_ref(), 0).unwrap_or_else(|e| { + vortex_panic!("cannot convert struct field to a Vortex scalar: {e}") + }), + ) + }) + .collect::>(); + Scalar::struct_(dtype, children) + } +} + #[cfg(test)] mod tests { use datafusion_common::ScalarValue; diff --git a/vortex-datafusion/src/convert/schema.rs b/vortex-datafusion/src/convert/schema.rs index 172bff84791..ecfaa72d01e 100644 --- a/vortex-datafusion/src/convert/schema.rs +++ b/vortex-datafusion/src/convert/schema.rs @@ -8,7 +8,6 @@ use datafusion_common::Result as DFResult; use datafusion_common::exec_datafusion_err; use vortex::array::arrow::ArrowSession; use vortex::dtype::DType; -use vortex::dtype::StructFields; /// Calculate the physical Arrow schema for a Vortex file given its DType and the expected logical schema. /// @@ -88,7 +87,12 @@ fn calculate_physical_field_type( // For struct types, recursively check each field. DataType::Struct(logical_fields) => { - if let Some(struct_dtype) = struct_fields(dtype) { + // Walk through any extension layers to reach the underlying struct fields. + let mut inner = dtype; + while let DType::Extension(ext) = inner { + inner = ext.storage_dtype(); + } + if let DType::Struct(struct_dtype, _) = inner { let physical_fields: Vec = struct_dtype .names() .iter() @@ -205,16 +209,6 @@ fn calculate_physical_field_type( }) } -/// The struct fields of `dtype` if it is a struct, or of an extension type whose storage is -/// (eventually) a struct -- e.g. the native geo Point over `Struct`. -fn struct_fields(dtype: &DType) -> Option<&StructFields> { - match dtype { - DType::Struct(fields, _) => Some(fields), - DType::Extension(ext) => struct_fields(ext.storage_dtype()), - _ => None, - } -} - #[cfg(test)] mod tests { use std::sync::Arc; From f52a2c4b16b07944349e892d6f31524173125f73 Mon Sep 17 00:00:00 2001 From: Nemo Yu Date: Tue, 16 Jun 2026 13:32:24 -0400 Subject: [PATCH 3/5] fix toml order Signed-off-by: Nemo Yu --- vortex-datafusion/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vortex-datafusion/Cargo.toml b/vortex-datafusion/Cargo.toml index c2e1673c635..4aaefec35ff 100644 --- a/vortex-datafusion/Cargo.toml +++ b/vortex-datafusion/Cargo.toml @@ -14,6 +14,7 @@ rust-version = { workspace = true } version = { workspace = true } [dependencies] +arrow-array = { workspace = true } arrow-schema = { workspace = true } async-trait = { workspace = true } datafusion-catalog = { workspace = true } @@ -34,7 +35,6 @@ object_store = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "fs"] } tokio-stream = { workspace = true } tracing = { workspace = true, features = ["std", "attributes"] } -arrow-array = { workspace = true } vortex = { workspace = true, features = ["object_store", "tokio", "files"] } vortex-utils = { workspace = true, features = ["dashmap"] } From f5d17745321ef6385a42c2e305f59d3b65867743 Mon Sep 17 00:00:00 2001 From: Nemo Yu Date: Tue, 16 Jun 2026 14:52:06 -0400 Subject: [PATCH 4/5] geo function pushdown in datafusion Signed-off-by: Nemo Yu --- vortex-datafusion/Cargo.toml | 2 ++ vortex-datafusion/src/convert/exprs.rs | 42 ++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/vortex-datafusion/Cargo.toml b/vortex-datafusion/Cargo.toml index 4aaefec35ff..9ff5dc6fce1 100644 --- a/vortex-datafusion/Cargo.toml +++ b/vortex-datafusion/Cargo.toml @@ -30,12 +30,14 @@ datafusion-physical-expr-common = { workspace = true } datafusion-physical-plan = { workspace = true } datafusion-pruning = { workspace = true } futures = { workspace = true } +geodatafusion = { workspace = true } itertools = { workspace = true } object_store = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "fs"] } tokio-stream = { workspace = true } tracing = { workspace = true, features = ["std", "attributes"] } vortex = { workspace = true, features = ["object_store", "tokio", "files"] } +vortex-geo = { workspace = true } vortex-utils = { workspace = true, features = ["dashmap"] } [dev-dependencies] diff --git a/vortex-datafusion/src/convert/exprs.rs b/vortex-datafusion/src/convert/exprs.rs index b6d741fc5ec..8905490b648 100644 --- a/vortex-datafusion/src/convert/exprs.rs +++ b/vortex-datafusion/src/convert/exprs.rs @@ -18,6 +18,7 @@ use datafusion_physical_expr::projection::ProjectionExprs; use datafusion_physical_expr::utils::collect_columns; use datafusion_physical_expr_common::physical_expr::is_dynamic_physical_expr; use datafusion_physical_plan::expressions as df_expr; +use geodatafusion::udf::geo::measurement::Distance; use itertools::Itertools; use vortex::dtype::DType; use vortex::dtype::Nullability; @@ -35,11 +36,13 @@ use vortex::expr::not; use vortex::expr::pack; use vortex::expr::root; use vortex::scalar::Scalar; +use vortex::scalar_fn::EmptyOptions; use vortex::scalar_fn::ScalarFnVTableExt; use vortex::scalar_fn::fns::binary::Binary; use vortex::scalar_fn::fns::like::Like; use vortex::scalar_fn::fns::like::LikeOptions; use vortex::scalar_fn::fns::operators::Operator; +use vortex_geo::scalar_fn::distance::GeoDistance; use crate::convert::FromDataFusion; @@ -140,12 +143,37 @@ impl DefaultExpressionConvertor { return Ok(result); } + // Geospatial UDFs map to Vortex scalar functions pushed into the scan. + if let Some(expr) = self.try_convert_geo_scalar_function(scalar_fn)? { + return Ok(expr); + } + Err(exec_datafusion_err!( "Unsupported ScalarFunctionExpr: {}", scalar_fn.name() )) } + /// Converts a geospatial UDF to the matching Vortex scalar function, if `scalar_fn` is one. + fn try_convert_geo_scalar_function( + &self, + scalar_fn: &ScalarFunctionExpr, + ) -> DFResult> { + // geodatafusion's `st_distance` -> Vortex `GeoDistance`, the only geo pushdown Q1 needs. + // Other geo functions so far run above the scan. + if ScalarFunctionExpr::try_downcast_func::(scalar_fn).is_some() { + let [a, b] = scalar_fn.args() else { + return Err(exec_datafusion_err!("st_distance expects 2 arguments")); + }; + return Ok(Some(GeoDistance.new_expr( + EmptyOptions, + [self.convert(a.as_ref())?, self.convert(b.as_ref())?], + ))); + } + + Ok(None) + } + /// Attempts to convert a DataFusion CaseExpr to a Vortex expression. fn try_convert_case_expr(&self, case_expr: &df_expr::CaseExpr) -> DFResult { // DataFusion CaseExpr has: @@ -525,10 +553,20 @@ fn supported_data_types(dt: &DataType) -> bool { is_supported } -/// Checks if a scalar function can be pushed down. -/// Currently only GetFieldFunc is supported. +/// Checks if a scalar function can be pushed down: nested field access, or a geospatial +/// function that maps to a Vortex scalar function. fn can_scalar_fn_be_pushed_down(scalar_fn: &ScalarFunctionExpr) -> bool { ScalarFunctionExpr::try_downcast_func::(scalar_fn).is_some() + || is_pushable_geo_scalar_function(scalar_fn) +} + +/// Whether `scalar_fn` is a geospatial UDF that maps to a Vortex scalar function +/// and whose operands are all convertible.. +fn is_pushable_geo_scalar_function(scalar_fn: &ScalarFunctionExpr) -> bool { + // `st_distance` is pushable only when its operands convert (a Point column or folded coord + // literal); a WKB operand like `ST_GeomFromWKB(col)` falls back to geodatafusion instead. + ScalarFunctionExpr::try_downcast_func::(scalar_fn).is_some() + && scalar_fn.args().iter().all(is_convertible_expr) } // TODO(adam): Replace with `DataType::is_decimal` once its released. From d842e32dd3223fb6bba37484389d43867a491de2 Mon Sep 17 00:00:00 2001 From: Nemo Yu Date: Tue, 16 Jun 2026 14:54:21 -0400 Subject: [PATCH 5/5] update temp dependencies Signed-off-by: Nemo Yu --- Cargo.lock | 190 +++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 + 2 files changed, 192 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index c73c9c121eb..ff85db24a7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3441,6 +3441,16 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "earcutr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79127ed59a85d7687c409e9978547cffb7dc79675355ed22da6b66fd5f6ead01" +dependencies = [ + "itertools 0.11.0", + "num-traits", +] + [[package]] name = "educe" version = "0.6.0" @@ -3703,6 +3713,12 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "float_next_after" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" + [[package]] name = "fnv" version = "1.0.7" @@ -3920,6 +3936,24 @@ dependencies = [ "version_check", ] +[[package]] +name = "geo" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc1a1678e54befc9b4bcab6cd43b8e7f834ae8ea121118b0fd8c42747675b4a" +dependencies = [ + "earcutr", + "float_next_after", + "geo-types", + "geographiclib-rs", + "i_overlay", + "log", + "num-traits", + "robust", + "rstar", + "spade", +] + [[package]] name = "geo-traits" version = "0.3.0" @@ -3937,6 +3971,8 @@ checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5" dependencies = [ "approx", "num-traits", + "rayon", + "rstar", "serde", ] @@ -3966,6 +4002,20 @@ dependencies = [ "wkt", ] +[[package]] +name = "geoarrow-expr-geo" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e4a62ac19c86827c6ec81ea584594b3ee96db5a8119b9774d3466c6b373c434" +dependencies = [ + "arrow-array", + "arrow-buffer", + "geo", + "geo-traits", + "geoarrow-array", + "geoarrow-schema", +] + [[package]] name = "geoarrow-schema" version = "0.8.0" @@ -3979,6 +4029,44 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "geodatafusion" +version = "0.4.0" +source = "git+https://github.com/HarukiMoriarty/geodatafusion?rev=3d50d7e549df720707133852848edd1ecff89265#3d50d7e549df720707133852848edd1ecff89265" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-schema", + "datafusion 54.0.0", + "geo", + "geo-traits", + "geoarrow-array", + "geoarrow-expr-geo", + "geoarrow-schema", + "geohash", + "thiserror 1.0.69", + "wkt", +] + +[[package]] +name = "geographiclib-rs" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5a7f08910fd98737a6eda7568e7c5e645093e073328eeef49758cfe8b0489c7" +dependencies = [ + "libm", +] + +[[package]] +name = "geohash" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f58890382f70caccc5fa388981f7ac80c913795042afce9f3e065695d8f7464" +dependencies = [ + "geo-types", + "libm", +] + [[package]] name = "get_dir" version = "0.5.0" @@ -4101,6 +4189,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -4151,6 +4248,16 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.5.0" @@ -4312,6 +4419,49 @@ dependencies = [ "serde", ] +[[package]] +name = "i_float" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010025c2c532c8d82e42d0b8bb5184afa449fa6f06c709ea9adcb16c49ae405b" +dependencies = [ + "libm", +] + +[[package]] +name = "i_key_sort" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9190f86706ca38ac8add223b2aed8b1330002b5cdbbce28fb58b10914d38fc27" + +[[package]] +name = "i_overlay" +version = "4.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413183068e6e0289e18d7d0a1f661b81546e6918d5453a44570b9ab30cbed1b3" +dependencies = [ + "i_float", + "i_key_sort", + "i_shape", + "i_tree", + "rayon", +] + +[[package]] +name = "i_shape" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ea154b742f7d43dae2897fcd5ead86bc7b5eefcedd305a7ebf9f69d44d61082" +dependencies = [ + "i_float", +] + +[[package]] +name = "i_tree" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e6d558e6d4c7b82bc51d9c771e7a927862a161a7d87bf2b0541450e0e20915" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -4609,6 +4759,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -7558,6 +7717,23 @@ dependencies = [ "byteorder", ] +[[package]] +name = "robust" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839" + +[[package]] +name = "rstar" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421400d13ccfd26dfa5858199c30a5d76f9c54e0dba7575273025b43c5175dbb" +dependencies = [ + "heapless", + "num-traits", + "smallvec", +] + [[package]] name = "rstest" version = "0.26.1" @@ -8216,6 +8392,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spade" +version = "2.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9699399fd9349b00b184f5635b074f9ec93afffef30c853f8c875b32c0f8c7fa" +dependencies = [ + "hashbrown 0.16.1", + "num-traits", + "robust", + "smallvec", +] + [[package]] name = "sqllogictest" version = "0.29.1" @@ -9624,6 +9812,7 @@ dependencies = [ "datafusion-physical-plan 54.0.0", "datafusion-pruning 54.0.0", "futures", + "geodatafusion", "insta", "itertools 0.14.0", "object_store 0.13.2", @@ -9634,6 +9823,7 @@ dependencies = [ "tracing", "url", "vortex", + "vortex-geo", "vortex-utils", ] diff --git a/Cargo.toml b/Cargo.toml index a587ef1db16..8c4e9e01b1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,6 +160,8 @@ fuzzy-matcher = "0.3" geo-traits = "0.3.0" geo-types = "0.7.19" geoarrow = "0.8.0" +# Temporary fork bumped to DataFusion 54 until the upstream PR lands; pinned to an exact rev. +geodatafusion = { git = "https://github.com/HarukiMoriarty/geodatafusion", rev = "3d50d7e549df720707133852848edd1ecff89265" } get_dir = "0.5.0" glob = "0.3.2" goldenfile = "1"