Skip to content

Commit e616241

Browse files
alambalamb
committed
Update code for API changes
1 parent b4cb213 commit e616241

11 files changed

Lines changed: 69 additions & 71 deletions

File tree

‎datafusion-cli/src/main.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -613,9 +613,9 @@ mod tests {
613613
+-----------------------------------+-----------------+---------------------+------+------------------+
614614
| filename | file_size_bytes | metadata_size_bytes | hits | extra |
615615
+-----------------------------------+-----------------+---------------------+------+------------------+
616-
| alltypes_plain.parquet | 1851 | 8882 | 2 | page_index=false |
617-
| alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true |
618-
| lz4_raw_compressed_larger.parquet | 380836 | 1339 | 2 | page_index=false |
616+
| alltypes_plain.parquet | 1851 | 8794 | 2 | page_index=false |
617+
| alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true |
618+
| lz4_raw_compressed_larger.parquet | 380836 | 1331 | 2 | page_index=false |
619619
+-----------------------------------+-----------------+---------------------+------+------------------+
620620
");
621621

@@ -644,9 +644,9 @@ mod tests {
644644
+-----------------------------------+-----------------+---------------------+------+------------------+
645645
| filename | file_size_bytes | metadata_size_bytes | hits | extra |
646646
+-----------------------------------+-----------------+---------------------+------+------------------+
647-
| alltypes_plain.parquet | 1851 | 8882 | 5 | page_index=false |
648-
| alltypes_tiny_pages.parquet | 454233 | 269074 | 2 | page_index=true |
649-
| lz4_raw_compressed_larger.parquet | 380836 | 1339 | 3 | page_index=false |
647+
| alltypes_plain.parquet | 1851 | 8794 | 5 | page_index=false |
648+
| alltypes_tiny_pages.parquet | 454233 | 268970 | 2 | page_index=true |
649+
| lz4_raw_compressed_larger.parquet | 380836 | 1331 | 3 | page_index=false |
650650
+-----------------------------------+-----------------+---------------------+------+------------------+
651651
");
652652

‎datafusion/common/src/file_options/mod.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -114,14 +114,14 @@ mod tests {
114114
properties
115115
.bloom_filter_properties(&ColumnPath::from(""))
116116
.expect("expected bloom properties!")
117-
.fpp,
117+
.fpp(),
118118
0.123
119119
);
120120
assert_eq!(
121121
properties
122122
.bloom_filter_properties(&ColumnPath::from(""))
123123
.expect("expected bloom properties!")
124-
.ndv,
124+
.ndv(),
125125
123
126126
);
127127

@@ -242,31 +242,31 @@ mod tests {
242242
properties
243243
.bloom_filter_properties(&col1)
244244
.expect("expected bloom properties!")
245-
.fpp,
245+
.fpp(),
246246
0.123
247247
);
248248

249249
assert_eq!(
250250
properties
251251
.bloom_filter_properties(&col2_nested)
252252
.expect("expected bloom properties!")
253-
.fpp,
253+
.fpp(),
254254
0.456
255255
);
256256

257257
assert_eq!(
258258
properties
259259
.bloom_filter_properties(&col1)
260260
.expect("expected bloom properties!")
261-
.ndv,
261+
.ndv(),
262262
123
263263
);
264264

265265
assert_eq!(
266266
properties
267267
.bloom_filter_properties(&col2_nested)
268268
.expect("expected bloom properties!")
269-
.ndv,
269+
.ndv(),
270270
456
271271
);
272272

‎datafusion/common/src/file_options/parquet_writer.rs‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,8 @@ impl TryFrom<&TableParquetOptions> for WriterPropertiesBuilder {
157157
}
158158

159159
if let Some(bloom_filter_ndv) = options.bloom_filter_ndv {
160-
builder =
161-
builder.set_column_bloom_filter_ndv(path.clone(), bloom_filter_ndv);
160+
builder = builder
161+
.set_column_bloom_filter_max_ndv(path.clone(), bloom_filter_ndv);
162162
}
163163
}
164164

@@ -271,7 +271,7 @@ impl ParquetOptions {
271271
builder = builder.set_bloom_filter_fpp(*bloom_filter_fpp);
272272
};
273273
if let Some(bloom_filter_ndv) = bloom_filter_ndv {
274-
builder = builder.set_bloom_filter_ndv(*bloom_filter_ndv);
274+
builder = builder.set_bloom_filter_max_ndv(*bloom_filter_ndv);
275275
};
276276
if let Some(dictionary_enabled) = dictionary_enabled {
277277
builder = builder.set_dictionary_enabled(*dictionary_enabled);
@@ -530,8 +530,8 @@ mod tests {
530530
}
531531
.into(),
532532
),
533-
bloom_filter_fpp: bloom_filter_default_props.map(|p| p.fpp),
534-
bloom_filter_ndv: bloom_filter_default_props.map(|p| p.ndv),
533+
bloom_filter_fpp: bloom_filter_default_props.map(|p| p.fpp()),
534+
bloom_filter_ndv: bloom_filter_default_props.map(|p| p.ndv()),
535535
}
536536
}
537537

@@ -823,10 +823,12 @@ mod tests {
823823
);
824824
assert_eq!(
825825
default_writer_props.bloom_filter_properties(&"default".into()),
826-
Some(&BloomFilterProperties {
827-
fpp: 0.42,
828-
ndv: DEFAULT_BLOOM_FILTER_NDV
829-
}),
826+
Some(
827+
&BloomFilterProperties::builder()
828+
.with_fpp(0.42)
829+
.with_max_ndv(DEFAULT_BLOOM_FILTER_NDV)
830+
.build()
831+
),
830832
"should have only the fpp set, and the ndv at default",
831833
);
832834
}
@@ -910,7 +912,7 @@ mod tests {
910912
// the WriterProperties::default, with only ndv set
911913
let default_writer_props = WriterProperties::builder()
912914
.set_bloom_filter_enabled(true)
913-
.set_bloom_filter_ndv(42)
915+
.set_bloom_filter_max_ndv(42)
914916
.build();
915917

916918
assert_eq!(
@@ -920,10 +922,12 @@ mod tests {
920922
);
921923
assert_eq!(
922924
default_writer_props.bloom_filter_properties(&"default".into()),
923-
Some(&BloomFilterProperties {
924-
fpp: DEFAULT_BLOOM_FILTER_FPP,
925-
ndv: 42
926-
}),
925+
Some(
926+
&BloomFilterProperties::builder()
927+
.with_fpp(DEFAULT_BLOOM_FILTER_FPP)
928+
.with_max_ndv(42)
929+
.build()
930+
),
927931
"should have only the ndv set, and the fpp at default",
928932
);
929933
}

‎datafusion/core/tests/datasource/object_store_access.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -904,7 +904,7 @@ async fn query_single_parquet_file_with_single_predicate() {
904904
RequestCountingObjectStore()
905905
Total Requests: 2
906906
- GET (opts) path=parquet_table.parquet head=true
907-
- GET (ranges) path=parquet_table.parquet ranges=1064-1481,1481-1594,1594-2011,2011-2124
907+
- GET (ranges) path=parquet_table.parquet ranges=1064-1594,1594-2124
908908
"
909909
);
910910
}
@@ -928,8 +928,8 @@ async fn query_single_parquet_file_multi_row_groups_multiple_predicates() {
928928
RequestCountingObjectStore()
929929
Total Requests: 3
930930
- GET (opts) path=parquet_table.parquet head=true
931-
- GET (ranges) path=parquet_table.parquet ranges=4-421,421-534,534-951,951-1064
932-
- GET (ranges) path=parquet_table.parquet ranges=1064-1481,1481-1594,1594-2011,2011-2124
931+
- GET (ranges) path=parquet_table.parquet ranges=4-534,534-1064
932+
- GET (ranges) path=parquet_table.parquet ranges=1064-1594,1594-2124
933933
"
934934
);
935935
}

‎datafusion/core/tests/extension_types/pretty_printing.rs‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,16 @@ async fn create_test_table() -> Result<DataFrame> {
4040
// define data.
4141
let batch = RecordBatch::try_new(
4242
schema,
43-
vec![Arc::new(FixedSizeBinaryArray::from(vec![
44-
&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
45-
&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 5, 6],
46-
]))],
43+
vec![Arc::new(
44+
FixedSizeBinaryArray::try_from_iter(
45+
vec![
46+
&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
47+
&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 5, 6],
48+
]
49+
.into_iter(),
50+
)
51+
.unwrap(),
52+
)],
4753
)?;
4854

4955
let state = SessionStateBuilder::default()

‎datafusion/core/tests/parquet/mod.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -726,11 +726,11 @@ fn make_bytearray_batch(
726726
let name: StringArray = std::iter::repeat_n(Some(name), num_rows).collect();
727727
let service_string: StringArray = string_values.iter().map(Some).collect();
728728
let service_binary: BinaryArray = binary_values.iter().map(Some).collect();
729-
let service_fixedsize: FixedSizeBinaryArray = fixedsize_values
730-
.iter()
731-
.map(|value| Some(value.as_slice()))
732-
.collect::<Vec<_>>()
733-
.into();
729+
let service_fixedsize = FixedSizeBinaryArray::try_from_sparse_iter_with_size(
730+
fixedsize_values.iter().map(|value| Some(value.as_slice())),
731+
3,
732+
)
733+
.unwrap();
734734
let service_large_binary: LargeBinaryArray =
735735
large_binary_values.iter().map(Some).collect();
736736

‎datafusion/datasource-parquet/src/row_group_filter.rs‎

Lines changed: 5 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -955,10 +955,7 @@ mod tests {
955955
let schema =
956956
Arc::new(Schema::new(vec![Field::new("c1", Decimal128(9, 2), false)]));
957957
let field = PrimitiveTypeField::new("c1", PhysicalType::INT32)
958-
.with_logical_type(LogicalType::Decimal {
959-
scale: 2,
960-
precision: 9,
961-
})
958+
.with_logical_type(LogicalType::decimal(2, 9))
962959
.with_scale(2)
963960
.with_precision(9);
964961
let schema_descr = get_test_schema_descr(vec![field]);
@@ -1023,10 +1020,7 @@ mod tests {
10231020
Arc::new(Schema::new(vec![Field::new("c1", Decimal128(9, 0), false)]));
10241021

10251022
let field = PrimitiveTypeField::new("c1", PhysicalType::INT32)
1026-
.with_logical_type(LogicalType::Decimal {
1027-
scale: 0,
1028-
precision: 9,
1029-
})
1023+
.with_logical_type(LogicalType::decimal(0, 9))
10301024
.with_scale(0)
10311025
.with_precision(9);
10321026
let schema_descr = get_test_schema_descr(vec![field]);
@@ -1118,10 +1112,7 @@ mod tests {
11181112
false,
11191113
)]));
11201114
let field = PrimitiveTypeField::new("c1", PhysicalType::INT64)
1121-
.with_logical_type(LogicalType::Decimal {
1122-
scale: 2,
1123-
precision: 18,
1124-
})
1115+
.with_logical_type(LogicalType::decimal(2, 18))
11251116
.with_scale(2)
11261117
.with_precision(18);
11271118
let schema_descr = get_test_schema_descr(vec![field]);
@@ -1176,10 +1167,7 @@ mod tests {
11761167
false,
11771168
)]));
11781169
let field = PrimitiveTypeField::new("c1", PhysicalType::FIXED_LEN_BYTE_ARRAY)
1179-
.with_logical_type(LogicalType::Decimal {
1180-
scale: 2,
1181-
precision: 18,
1182-
})
1170+
.with_logical_type(LogicalType::decimal(2, 18))
11831171
.with_scale(2)
11841172
.with_precision(18)
11851173
.with_byte_len(16);
@@ -1254,10 +1242,7 @@ mod tests {
12541242
false,
12551243
)]));
12561244
let field = PrimitiveTypeField::new("c1", PhysicalType::BYTE_ARRAY)
1257-
.with_logical_type(LogicalType::Decimal {
1258-
scale: 2,
1259-
precision: 18,
1260-
})
1245+
.with_logical_type(LogicalType::decimal(2, 18))
12611246
.with_scale(2)
12621247
.with_precision(18)
12631248
.with_byte_len(16);

‎datafusion/physical-expr-common/src/utils.rs‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -614,11 +614,9 @@ mod tests {
614614

615615
#[test]
616616
fn scatter_fixed_size_binary_test() -> Result<()> {
617-
let truthy = Arc::new(FixedSizeBinaryArray::from(vec![
618-
&[1u8, 2][..],
619-
&[3, 4][..],
620-
&[5, 6][..],
621-
]));
617+
let truthy = Arc::new(FixedSizeBinaryArray::try_from_iter(
618+
vec![&[1u8, 2][..], &[3, 4][..], &[5, 6][..]].into_iter(),
619+
)?);
622620
let mask = BooleanArray::from(vec![true, false, true, false, true]);
623621

624622
let result = scatter(&mask, truthy.as_ref())?;

‎datafusion/physical-expr/benches/in_list_strategy.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -993,7 +993,7 @@ fn bench_fixed_size_binary_inner(
993993
.collect();
994994

995995
let refs: Vec<&[u8]> = values.iter().map(|v| v.as_slice()).collect();
996-
let array = FixedSizeBinaryArray::from(refs);
996+
let array = FixedSizeBinaryArray::try_from_iter(refs.into_iter()).unwrap();
997997

998998
let schema = Schema::new(vec![Field::new("a", array.data_type().clone(), true)]);
999999
let exprs: Vec<_> = haystack

‎datafusion/physical-plan/src/joins/sort_merge_join/tests.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ fn build_fixed_size_binary_table(
175175
let batch = RecordBatch::try_new(
176176
Arc::new(schema),
177177
vec![
178-
Arc::new(FixedSizeBinaryArray::from(a.1.clone())),
178+
Arc::new(FixedSizeBinaryArray::try_from_iter(a.1.iter().copied()).unwrap()),
179179
Arc::new(Int32Array::from(b.1.clone())),
180180
Arc::new(Int32Array::from(c.1.clone())),
181181
],

0 commit comments

Comments
 (0)