Skip to content

Commit 64976e9

Browse files
alambalamb
committed
Update code for API changes
1 parent 8eb9cb7 commit 64976e9

11 files changed

Lines changed: 69 additions & 71 deletions

File tree

‎datafusion-cli/src/main.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -613,9 +613,9 @@ mod tests {
613613
+-----------------------------------+-----------------+---------------------+------+------------------+
614614
| filename | file_size_bytes | metadata_size_bytes | hits | extra |
615615
+-----------------------------------+-----------------+---------------------+------+------------------+
616-
| alltypes_plain.parquet | 1851 | 8882 | 2 | page_index=false |
617-
| alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true |
618-
| lz4_raw_compressed_larger.parquet | 380836 | 1339 | 2 | page_index=false |
616+
| alltypes_plain.parquet | 1851 | 8794 | 2 | page_index=false |
617+
| alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true |
618+
| lz4_raw_compressed_larger.parquet | 380836 | 1331 | 2 | page_index=false |
619619
+-----------------------------------+-----------------+---------------------+------+------------------+
620620
");
621621

@@ -644,9 +644,9 @@ mod tests {
644644
+-----------------------------------+-----------------+---------------------+------+------------------+
645645
| filename | file_size_bytes | metadata_size_bytes | hits | extra |
646646
+-----------------------------------+-----------------+---------------------+------+------------------+
647-
| alltypes_plain.parquet | 1851 | 8882 | 5 | page_index=false |
648-
| alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true |
649-
| lz4_raw_compressed_larger.parquet | 380836 | 1339 | 3 | page_index=false |
647+
| alltypes_plain.parquet | 1851 | 8794 | 5 | page_index=false |
648+
| alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true |
649+
| lz4_raw_compressed_larger.parquet | 380836 | 1331 | 3 | page_index=false |
650650
+-----------------------------------+-----------------+---------------------+------+------------------+
651651
");
652652

‎datafusion/common/src/file_options/mod.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -114,14 +114,14 @@ mod tests {
114114
properties
115115
.bloom_filter_properties(&ColumnPath::from(""))
116116
.expect("expected bloom properties!")
117-
.fpp,
117+
.fpp(),
118118
0.123
119119
);
120120
assert_eq!(
121121
properties
122122
.bloom_filter_properties(&ColumnPath::from(""))
123123
.expect("expected bloom properties!")
124-
.ndv,
124+
.ndv(),
125125
123
126126
);
127127

@@ -242,31 +242,31 @@ mod tests {
242242
properties
243243
.bloom_filter_properties(&col1)
244244
.expect("expected bloom properties!")
245-
.fpp,
245+
.fpp(),
246246
0.123
247247
);
248248

249249
assert_eq!(
250250
properties
251251
.bloom_filter_properties(&col2_nested)
252252
.expect("expected bloom properties!")
253-
.fpp,
253+
.fpp(),
254254
0.456
255255
);
256256

257257
assert_eq!(
258258
properties
259259
.bloom_filter_properties(&col1)
260260
.expect("expected bloom properties!")
261-
.ndv,
261+
.ndv(),
262262
123
263263
);
264264

265265
assert_eq!(
266266
properties
267267
.bloom_filter_properties(&col2_nested)
268268
.expect("expected bloom properties!")
269-
.ndv,
269+
.ndv(),
270270
456
271271
);
272272

‎datafusion/common/src/file_options/parquet_writer.rs‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,8 @@ impl TryFrom<&TableParquetOptions> for WriterPropertiesBuilder {
157157
}
158158

159159
if let Some(bloom_filter_ndv) = options.bloom_filter_ndv {
160-
builder =
161-
builder.set_column_bloom_filter_ndv(path.clone(), bloom_filter_ndv);
160+
builder = builder
161+
.set_column_bloom_filter_max_ndv(path.clone(), bloom_filter_ndv);
162162
}
163163
}
164164

@@ -273,7 +273,7 @@ impl ParquetOptions {
273273
builder = builder.set_bloom_filter_fpp(*bloom_filter_fpp);
274274
};
275275
if let Some(bloom_filter_ndv) = bloom_filter_ndv {
276-
builder = builder.set_bloom_filter_ndv(*bloom_filter_ndv);
276+
builder = builder.set_bloom_filter_max_ndv(*bloom_filter_ndv);
277277
};
278278
if let Some(dictionary_enabled) = dictionary_enabled {
279279
builder = builder.set_dictionary_enabled(*dictionary_enabled);
@@ -534,8 +534,8 @@ mod tests {
534534
}
535535
.into(),
536536
),
537-
bloom_filter_fpp: bloom_filter_default_props.map(|p| p.fpp),
538-
bloom_filter_ndv: bloom_filter_default_props.map(|p| p.ndv),
537+
bloom_filter_fpp: bloom_filter_default_props.map(|p| p.fpp()),
538+
bloom_filter_ndv: bloom_filter_default_props.map(|p| p.ndv()),
539539
}
540540
}
541541

@@ -830,10 +830,12 @@ mod tests {
830830
);
831831
assert_eq!(
832832
default_writer_props.bloom_filter_properties(&"default".into()),
833-
Some(&BloomFilterProperties {
834-
fpp: 0.42,
835-
ndv: DEFAULT_BLOOM_FILTER_NDV
836-
}),
833+
Some(
834+
&BloomFilterProperties::builder()
835+
.with_fpp(0.42)
836+
.with_max_ndv(DEFAULT_BLOOM_FILTER_NDV)
837+
.build()
838+
),
837839
"should have only the fpp set, and the ndv at default",
838840
);
839841
}
@@ -937,7 +939,7 @@ mod tests {
937939
// the WriterProperties::default, with only ndv set
938940
let default_writer_props = WriterProperties::builder()
939941
.set_bloom_filter_enabled(true)
940-
.set_bloom_filter_ndv(42)
942+
.set_bloom_filter_max_ndv(42)
941943
.build();
942944

943945
assert_eq!(
@@ -947,10 +949,12 @@ mod tests {
947949
);
948950
assert_eq!(
949951
default_writer_props.bloom_filter_properties(&"default".into()),
950-
Some(&BloomFilterProperties {
951-
fpp: DEFAULT_BLOOM_FILTER_FPP,
952-
ndv: 42
953-
}),
952+
Some(
953+
&BloomFilterProperties::builder()
954+
.with_fpp(DEFAULT_BLOOM_FILTER_FPP)
955+
.with_max_ndv(42)
956+
.build()
957+
),
954958
"should have only the ndv set, and the fpp at default",
955959
);
956960
}

‎datafusion/core/tests/datasource/object_store_access.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -904,7 +904,7 @@ async fn query_single_parquet_file_with_single_predicate() {
904904
RequestCountingObjectStore()
905905
Total Requests: 2
906906
- GET (opts) path=parquet_table.parquet head=true
907-
- GET (ranges) path=parquet_table.parquet ranges=1064-1481,1481-1594,1594-2011,2011-2124
907+
- GET (ranges) path=parquet_table.parquet ranges=1064-1594,1594-2124
908908
"
909909
);
910910
}
@@ -928,8 +928,8 @@ async fn query_single_parquet_file_multi_row_groups_multiple_predicates() {
928928
RequestCountingObjectStore()
929929
Total Requests: 3
930930
- GET (opts) path=parquet_table.parquet head=true
931-
- GET (ranges) path=parquet_table.parquet ranges=4-421,421-534,534-951,951-1064
932-
- GET (ranges) path=parquet_table.parquet ranges=1064-1481,1481-1594,1594-2011,2011-2124
931+
- GET (ranges) path=parquet_table.parquet ranges=4-534,534-1064
932+
- GET (ranges) path=parquet_table.parquet ranges=1064-1594,1594-2124
933933
"
934934
);
935935
}

‎datafusion/core/tests/extension_types/pretty_printing.rs‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,16 @@ async fn create_test_table() -> Result<DataFrame> {
4040
// define data.
4141
let batch = RecordBatch::try_new(
4242
schema,
43-
vec![Arc::new(FixedSizeBinaryArray::from(vec![
44-
&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
45-
&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 5, 6],
46-
]))],
43+
vec![Arc::new(
44+
FixedSizeBinaryArray::try_from_iter(
45+
vec![
46+
&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
47+
&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 5, 6],
48+
]
49+
.into_iter(),
50+
)
51+
.unwrap(),
52+
)],
4753
)?;
4854

4955
let state = SessionStateBuilder::default()

‎datafusion/core/tests/parquet/mod.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -726,11 +726,11 @@ fn make_bytearray_batch(
726726
let name: StringArray = std::iter::repeat_n(Some(name), num_rows).collect();
727727
let service_string: StringArray = string_values.iter().map(Some).collect();
728728
let service_binary: BinaryArray = binary_values.iter().map(Some).collect();
729-
let service_fixedsize: FixedSizeBinaryArray = fixedsize_values
730-
.iter()
731-
.map(|value| Some(value.as_slice()))
732-
.collect::<Vec<_>>()
733-
.into();
729+
let service_fixedsize = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
730+
fixedsize_values.iter().map(|value| Some(value.as_slice())),
731+
3,
732+
)
733+
.unwrap();
734734
let service_large_binary: LargeBinaryArray =
735735
large_binary_values.iter().map(Some).collect();
736736

‎datafusion/datasource-parquet/src/row_group_filter.rs‎

Lines changed: 5 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -955,10 +955,7 @@ mod tests {
955955
let schema =
956956
Arc::new(Schema::new(vec![Field::new("c1", Decimal128(9, 2), false)]));
957957
let field = PrimitiveTypeField::new("c1", PhysicalType::INT32)
958-
.with_logical_type(LogicalType::Decimal {
959-
scale: 2,
960-
precision: 9,
961-
})
958+
.with_logical_type(LogicalType::decimal(2, 9))
962959
.with_scale(2)
963960
.with_precision(9);
964961
let schema_descr = get_test_schema_descr(vec![field]);
@@ -1023,10 +1020,7 @@ mod tests {
10231020
Arc::new(Schema::new(vec![Field::new("c1", Decimal128(9, 0), false)]));
10241021

10251022
let field = PrimitiveTypeField::new("c1", PhysicalType::INT32)
1026-
.with_logical_type(LogicalType::Decimal {
1027-
scale: 0,
1028-
precision: 9,
1029-
})
1023+
.with_logical_type(LogicalType::decimal(0, 9))
10301024
.with_scale(0)
10311025
.with_precision(9);
10321026
let schema_descr = get_test_schema_descr(vec![field]);
@@ -1118,10 +1112,7 @@ mod tests {
11181112
false,
11191113
)]));
11201114
let field = PrimitiveTypeField::new("c1", PhysicalType::INT64)
1121-
.with_logical_type(LogicalType::Decimal {
1122-
scale: 2,
1123-
precision: 18,
1124-
})
1115+
.with_logical_type(LogicalType::decimal(2, 18))
11251116
.with_scale(2)
11261117
.with_precision(18);
11271118
let schema_descr = get_test_schema_descr(vec![field]);
@@ -1176,10 +1167,7 @@ mod tests {
11761167
false,
11771168
)]));
11781169
let field = PrimitiveTypeField::new("c1", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1179-
.with_logical_type(LogicalType::Decimal {
1180-
scale: 2,
1181-
precision: 18,
1182-
})
1170+
.with_logical_type(LogicalType::decimal(2, 18))
11831171
.with_scale(2)
11841172
.with_precision(18)
11851173
.with_byte_len(16);
@@ -1254,10 +1242,7 @@ mod tests {
12541242
false,
12551243
)]));
12561244
let field = PrimitiveTypeField::new("c1", PhysicalType::BYTE_ARRAY)
1257-
.with_logical_type(LogicalType::Decimal {
1258-
scale: 2,
1259-
precision: 18,
1260-
})
1245+
.with_logical_type(LogicalType::decimal(2, 18))
12611246
.with_scale(2)
12621247
.with_precision(18)
12631248
.with_byte_len(16);

‎datafusion/physical-expr-common/src/utils.rs‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -614,11 +614,9 @@ mod tests {
614614

615615
#[test]
616616
fn scatter_fixed_size_binary_test() -> Result<()> {
617-
let truthy = Arc::new(FixedSizeBinaryArray::from(vec![
618-
&[1u8, 2][..],
619-
&[3, 4][..],
620-
&[5, 6][..],
621-
]));
617+
let truthy = Arc::new(FixedSizeBinaryArray::try_from_iter(
618+
vec![&[1u8, 2][..], &[3, 4][..], &[5, 6][..]].into_iter(),
619+
)?);
622620
let mask = BooleanArray::from(vec![true, false, true, false, true]);
623621

624622
let result = scatter(&mask, truthy.as_ref())?;

‎datafusion/physical-expr/benches/in_list_strategy.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -993,7 +993,7 @@ fn bench_fixed_size_binary_inner(
993993
.collect();
994994

995995
let refs: Vec<&[u8]> = values.iter().map(|v| v.as_slice()).collect();
996-
let array = FixedSizeBinaryArray::from(refs);
996+
let array = FixedSizeBinaryArray::try_from_iter(refs.into_iter()).unwrap();
997997

998998
let schema = Schema::new(vec![Field::new("a", array.data_type().clone(), true)]);
999999
let exprs: Vec<_> = haystack

‎datafusion/physical-plan/src/joins/sort_merge_join/tests.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ fn build_fixed_size_binary_table(
175175
let batch = RecordBatch::try_new(
176176
Arc::new(schema),
177177
vec![
178-
Arc::new(FixedSizeBinaryArray::from(a.1.clone())),
178+
Arc::new(FixedSizeBinaryArray::try_from_iter(a.1.iter().copied()).unwrap()),
179179
Arc::new(Int32Array::from(b.1.clone())),
180180
Arc::new(Int32Array::from(c.1.clone())),
181181
],

0 commit comments

Comments
 (0)