Skip to content

Commit eea5a87

Browse files
committed
test(physical-plan): use a (Float16, Int32) schema in the GroupColumn unit test
Per kosiew's review: a single-column Float16 test doesn't exercise the all-or-nothing multi-column gate this PR is actually about. Switching to (Float16, Int32) does — group count goes from 4 (single column: 1.0, ±0.0, NaN, null) to 5, since (0.0, 4) now splits off from (±0.0, 3) on the Int32 key even though the Float16 values are equal. Also trimmed the slt/test comments to match the Duration and Interval siblings.
1 parent b93c342 commit eea5a87

3 files changed

Lines changed: 43 additions & 39 deletions

File tree

datafusion/physical-plan/benches/multi_group_by.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -456,8 +456,8 @@ fn make_f16_schema() -> SchemaRef {
456456
///
457457
/// `f16` has only ~63.5k finite values, so `num_distinct_groups` must stay well
458458
/// under that (see `bench_float16`). Distinct keys are the low finite `f16` bit
459-
/// patterns (NaN/inf skipped); the `Int32` column is keyed identically so the
460-
/// combined cardinality equals `num_distinct_groups`.
459+
/// patterns, skipping NaN and inf. The `Int32` column is keyed identically so
460+
/// the combined cardinality equals `num_distinct_groups`.
461461
fn generate_f16_batches(
462462
num_distinct_groups: usize,
463463
num_rows: usize,

datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs

Lines changed: 38 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1264,7 +1264,7 @@ mod tests {
12641264
use std::{collections::HashMap, sync::Arc};
12651265

12661266
use arrow::array::{
1267-
Array, ArrayRef, Float16Array, Int64Array, RecordBatch, StringArray,
1267+
Array, ArrayRef, Float16Array, Int32Array, Int64Array, RecordBatch, StringArray,
12681268
StringViewArray,
12691269
};
12701270
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
@@ -1290,7 +1290,7 @@ mod tests {
12901290
///
12911291
/// This test fuzzes a representative cross-section of types and asserts
12921292
/// both directions of the biconditional. When a new specialization is
1293-
/// added (`FixedSizeList`, `Struct`, `Decimal256`, ...) it should be added
1293+
/// added (`Float16`, `FixedSizeList`, `Struct`, ...) it should be added
12941294
/// to the supported_cases vector; when a type is intentionally rejected
12951295
/// it should be added to unsupported_cases.
12961296
#[test]
@@ -1359,57 +1359,69 @@ mod tests {
13591359
}
13601360
}
13611361

1362-
/// End-to-end coverage for `Float16` group keys. Beyond routing through the
1363-
/// column-wise `GroupValuesColumn` fast path, this exercises the float
1364-
/// canonicalization the builder relies on (shared with Float32 / Float64 via
1365-
/// `HashValue`): `-0.0` and `+0.0` collapse into one group stored as `+0.0`,
1366-
/// and equal `NaN`s collapse into a single group.
1362+
// `(Float16, Int32)` keys: ±0.0 collapse (stored as +0.0), NaNs collapse, and
1363+
// the Int32 key keeps `(0.0, 4)` distinct from `(±0.0, 3)`.
13671364
#[test]
13681365
fn test_group_values_column_float16() {
13691366
use half::f16;
13701367

1371-
let schema =
1372-
Arc::new(Schema::new(vec![Field::new("f", DataType::Float16, true)]));
1368+
let schema = Arc::new(Schema::new(vec![
1369+
Field::new("f", DataType::Float16, true),
1370+
Field::new("i", DataType::Int32, true),
1371+
]));
13731372
assert!(supported_schema(&schema));
13741373
let mut group_values =
13751374
GroupValuesColumn::<false>::try_new(Arc::clone(&schema)).unwrap();
13761375

1377-
// Rows: 1.0, -0.0, +0.0, NaN, 1.0, NaN, null, null.
1378-
// First-seen groups: 1.0 -> 0, -0.0/+0.0 -> 1, NaN -> 2, null -> 3.
1379-
let input: ArrayRef = Arc::new(Float16Array::from(vec![
1376+
let f: ArrayRef = Arc::new(Float16Array::from(vec![
13801377
Some(f16::from_f32(1.0)),
13811378
Some(f16::from_f32(-0.0)),
13821379
Some(f16::from_f32(0.0)),
1380+
Some(f16::from_f32(0.0)),
13831381
Some(f16::NAN),
1384-
Some(f16::from_f32(1.0)),
13851382
Some(f16::NAN),
13861383
None,
13871384
None,
13881385
]));
1386+
let i: ArrayRef = Arc::new(Int32Array::from(vec![
1387+
Some(3),
1388+
Some(3),
1389+
Some(3),
1390+
Some(4),
1391+
Some(3),
1392+
Some(3),
1393+
Some(3),
1394+
Some(3),
1395+
]));
13891396
let mut groups = Vec::new();
1390-
group_values.intern(&[input], &mut groups).unwrap();
1391-
assert_eq!(groups, vec![0, 1, 1, 2, 0, 2, 3, 3]);
1397+
group_values.intern(&[f, i], &mut groups).unwrap();
1398+
assert_eq!(groups, vec![0, 1, 1, 2, 3, 3, 4, 4]);
13921399

13931400
let emitted = group_values.emit(EmitTo::All).unwrap();
1394-
assert_eq!(emitted.len(), 1);
1401+
assert_eq!(emitted.len(), 2);
13951402
assert_eq!(emitted[0].data_type(), &DataType::Float16);
1396-
let actual = emitted[0]
1403+
let keys = emitted[0]
13971404
.as_any()
13981405
.downcast_ref::<Float16Array>()
13991406
.expect("emitted column should be a Float16Array");
1400-
assert_eq!(actual.len(), 4);
1401-
assert_eq!(actual.value(0), f16::from_f32(1.0));
1407+
assert_eq!(keys.len(), 5);
1408+
assert_eq!(keys.value(0), f16::from_f32(1.0));
14021409
// The ±0.0 group is stored canonically as +0.0 (not -0.0).
1403-
assert_eq!(actual.value(1).to_bits(), f16::from_f32(0.0).to_bits());
1404-
assert!(actual.value(2).is_nan());
1405-
assert!(actual.is_null(3));
1410+
assert_eq!(keys.value(1).to_bits(), f16::from_f32(0.0).to_bits());
1411+
assert_eq!(keys.value(2).to_bits(), f16::from_f32(0.0).to_bits());
1412+
assert!(keys.value(3).is_nan());
1413+
assert!(keys.is_null(4));
1414+
let ids = emitted[1]
1415+
.as_any()
1416+
.downcast_ref::<Int32Array>()
1417+
.expect("emitted column should be an Int32Array");
1418+
assert_eq!(ids.values().to_vec(), vec![3, 3, 4, 3, 3]);
14061419
}
14071420

14081421
#[test]
14091422
fn supported_schema_rejects_mix_of_supported_and_unsupported() {
1410-
// One permanently-unsupported column flips the whole schema to the
1411-
// GroupValuesRows fallback. Use an invalid Arrow unit combo
1412-
// (Time64(Second)) so this stays stable as new primitive builders land.
1423+
// One unsupported column flips the whole schema to the GroupValuesRows
1424+
// fallback. Time64(Second) stays invalid as new primitive builders land.
14131425
let schema = Schema::new(vec![
14141426
Field::new("a", DataType::Int32, true),
14151427
Field::new("b", DataType::Utf8, true),
@@ -1527,7 +1539,7 @@ mod tests {
15271539
// `emit(EmitTo::First(4))` calls can `take_n` without panicking.
15281540
// The hashmap entries below reference group indices 0..=11, so the
15291541
// single column builder needs at least 12 rows to back them.
1530-
let seed: ArrayRef = Arc::new(arrow::array::Int32Array::from(vec![0_i32; 12]));
1542+
let seed: ArrayRef = Arc::new(Int32Array::from(vec![0_i32; 12]));
15311543
for row in 0..12 {
15321544
group_values.group_values[0]
15331545
.append_val(&seed, row)

datafusion/sqllogictest/test_files/aggregate.slt

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2268,12 +2268,7 @@ statement ok
22682268
DROP TABLE approx_distinct_duration_test;
22692269

22702270

2271-
# GROUP BY over Float16 group keys. In a multi-column GROUP BY a Float16 key
2272-
# routes through the GroupValuesColumn column-wise fast path (added here); a
2273-
# single-column GROUP BY uses the pre-existing single-column primitive path.
2274-
# Verify both, including that -0.0 / +0.0 fold into one group via the float
2275-
# canonicalization shared with Float32 / Float64. The count-of-groups form
2276-
# avoids depending on Float16's textual display.
2271+
# GROUP BY with Float16 group keys (single- and multi-column).
22772272
statement ok
22782273
CREATE TABLE float16_group_test AS VALUES
22792274
(arrow_cast(1.5, 'Float16'), 1),
@@ -2282,16 +2277,13 @@ CREATE TABLE float16_group_test AS VALUES
22822277
(arrow_cast(-0.0, 'Float16'), 3),
22832278
(arrow_cast(0.0, 'Float16'), 3);
22842279

2285-
# Single-column GROUP BY (single-column primitive path): five rows collapse to
2286-
# three groups: 1.5, 2.5, and the merged -0.0 / +0.0.
2280+
# Single Float16 group key ({1.5, 2.5, ±0.0}) via the GroupValuesPrimitive path.
22872281
query I
22882282
SELECT count(*) FROM (SELECT column1 FROM float16_group_test GROUP BY column1);
22892283
----
22902284
3
22912285

2292-
# Multi-column GROUP BY: a Float16 key alongside a primitive key, exercising the
2293-
# GroupValuesColumn path this PR adds. column2 is aligned 1:1 with the Float16
2294-
# value (the -0.0 / +0.0 rows share id 3), so the result is still three groups.
2286+
# Multi-column GROUP BY: a primitive key and a Float16 key on the same path.
22952287
query I
22962288
SELECT count(*) FROM (SELECT column1, column2 FROM float16_group_test GROUP BY column1, column2);
22972289
----

0 commit comments

Comments
 (0)