From 2f90e1564165f77dfb3dbe1037ca19d29ad47f99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Mar 2026 16:37:20 +0100 Subject: [PATCH 01/15] GH-46600: [C++][CI] Add job with ARROW_LARGE_MEMORY_TESTS enabled --- .github/workflows/cpp_extra.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 8ebf3b9dfbfe..a06a574ba5b6 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -107,6 +107,11 @@ jobs: - image: alpine-linux-cpp runs-on: ubuntu-latest title: AMD64 Alpine Linux + - image: ubuntu-cpp + run-options: >- + -e ARROW_LARGE_MEMORY_TESTS=ON + runs-on: "runs-on=${{ github.run_id }}/family=x8i.xlarge/spot=capacity-optimized" + title: AMD64 Ubuntu Large Memory Tests - image: conda-cpp run-options: >- -e ARROW_USE_MESON=ON From c972717c4853ac2db1ce754b363a9a6004641bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Mar 2026 16:43:45 +0100 Subject: [PATCH 02/15] Try removing /spot=capacity-optimized to see if quota issues are solved --- .github/workflows/cpp_extra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index a06a574ba5b6..1274779c3354 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -110,7 +110,7 @@ jobs: - image: ubuntu-cpp run-options: >- -e ARROW_LARGE_MEMORY_TESTS=ON - runs-on: "runs-on=${{ github.run_id }}/family=x8i.xlarge/spot=capacity-optimized" + runs-on: "runs-on=${{ github.run_id }}/family=x8i.xlarge" title: AMD64 Ubuntu Large Memory Tests - image: conda-cpp run-options: >- From 78a6b222bd6e97bab6796fd5e2b0755e879dfe65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 10 Mar 2026 16:48:40 +0100 Subject: [PATCH 03/15] Force spot=false --- .github/workflows/cpp_extra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 1274779c3354..818711ecad60 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -110,7 +110,7 @@ jobs: - image: ubuntu-cpp run-options: >- -e ARROW_LARGE_MEMORY_TESTS=ON - runs-on: "runs-on=${{ github.run_id }}/family=x8i.xlarge" + runs-on: "runs-on=${{ github.run_id }}/family=x8i.xlarge/spot=false" title: AMD64 Ubuntu Large Memory Tests - image: conda-cpp run-options: >- From e8c640f91b47d93d6371e55c2c29af04d61da356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 11 Mar 2026 10:46:31 +0100 Subject: [PATCH 04/15] Try with 2xlarge to see if tests finish without timing out --- .github/workflows/cpp_extra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 818711ecad60..bfcf4bd850b9 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -110,7 +110,7 @@ jobs: - image: ubuntu-cpp run-options: >- -e ARROW_LARGE_MEMORY_TESTS=ON - runs-on: "runs-on=${{ github.run_id }}/family=x8i.xlarge/spot=false" + runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/spot=capacity-optimized" title: AMD64 Ubuntu Large Memory Tests - image: conda-cpp run-options: >- From e316fb56bb10232a1e4a5fb70d06546cb614981d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 11 Mar 2026 15:08:58 +0100 Subject: [PATCH 05/15] Try setting build type to Release and ctest timeout to 15 minutes --- .github/workflows/cpp_extra.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index bfcf4bd850b9..e851e1493935 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -109,6 +109,8 @@ jobs: title: AMD64 Alpine Linux - image: ubuntu-cpp run-options: >- + -e ARROW_BUILD_TYPE=Release \ + -e ARROW_CTEST_TIMEOUT=900 \ -e ARROW_LARGE_MEMORY_TESTS=ON runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/spot=capacity-optimized" title: AMD64 Ubuntu Large Memory Tests From 12b83e03fc0b4befcd4ff89a1b6e2489057f4eff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 11 Mar 2026 15:14:28 +0100 Subject: [PATCH 06/15] Try single line just to validate --- .github/workflows/cpp_extra.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index e851e1493935..590c6cfe5611 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -109,9 +109,7 @@ jobs: title: AMD64 Alpine Linux - image: ubuntu-cpp run-options: >- - -e ARROW_BUILD_TYPE=Release \ - -e ARROW_CTEST_TIMEOUT=900 \ - -e ARROW_LARGE_MEMORY_TESTS=ON + -e ARROW_BUILD_TYPE=Release -e ARROW_CTEST_TIMEOUT=900 -e ARROW_LARGE_MEMORY_TESTS=ON runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/spot=capacity-optimized" title: AMD64 Ubuntu Large Memory Tests - image: conda-cpp From 581f50636a13d4a36d12820c5b229ae58f2a0834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 11 Mar 2026 15:38:05 +0100 Subject: [PATCH 07/15] That worked but what about removing the backslash --- .github/workflows/cpp_extra.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 590c6cfe5611..d43c9f259057 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -109,7 +109,9 @@ jobs: title: AMD64 Alpine Linux - image: ubuntu-cpp run-options: >- - -e ARROW_BUILD_TYPE=Release -e ARROW_CTEST_TIMEOUT=900 -e ARROW_LARGE_MEMORY_TESTS=ON + -e ARROW_BUILD_TYPE=Release + -e ARROW_CTEST_TIMEOUT=900 + -e ARROW_LARGE_MEMORY_TESTS=ON runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/spot=capacity-optimized" title: AMD64 Ubuntu Large Memory Tests - image: conda-cpp From b0e1b36431abd5a273a4adc0a797eb3f88b74e51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 11 Mar 2026 15:51:07 +0100 Subject: [PATCH 08/15] Increase timeout even more --- .github/workflows/cpp_extra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index d43c9f259057..920c0fcdf7f9 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -110,7 +110,7 @@ jobs: - image: ubuntu-cpp run-options: >- -e ARROW_BUILD_TYPE=Release - -e ARROW_CTEST_TIMEOUT=900 + -e ARROW_CTEST_TIMEOUT=1200 -e ARROW_LARGE_MEMORY_TESTS=ON runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/spot=capacity-optimized" title: AMD64 Ubuntu Large Memory Tests From 530cc0da4b6ddb1be705f6613515b8b8882fe23d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 11 Mar 2026 16:37:59 +0100 Subject: [PATCH 09/15] Add an absurdely high timeout --- .github/workflows/cpp_extra.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 920c0fcdf7f9..89fd20f6e121 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -108,9 +108,11 @@ jobs: runs-on: ubuntu-latest title: AMD64 Alpine Linux - image: ubuntu-cpp + # Use Release to speed up large memory tests that take a long time and + # increase timeout because some tests take more than 20 minutes. run-options: >- -e ARROW_BUILD_TYPE=Release - -e ARROW_CTEST_TIMEOUT=1200 + -e ARROW_CTEST_TIMEOUT=2400 -e ARROW_LARGE_MEMORY_TESTS=ON runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/spot=capacity-optimized" title: AMD64 Ubuntu Large Memory Tests From 935939365522cfaf48da22ad1cbaa070abf08e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 11 Mar 2026 17:03:27 +0100 Subject: [PATCH 10/15] Use ARROW_C_FLAGS_DEBUG and ARROW_CXX_FLAGS_DEBUG -O1 and BUILD_WARNING_LEVEL=PRODUCTION --- .github/workflows/cpp_extra.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 89fd20f6e121..0977fa99fb00 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -108,12 +108,12 @@ jobs: runs-on: ubuntu-latest title: AMD64 Alpine Linux - image: ubuntu-cpp - # Use Release to speed up large memory tests that take a long time and - # increase timeout because some tests take more than 20 minutes. run-options: >- - -e ARROW_BUILD_TYPE=Release - -e ARROW_CTEST_TIMEOUT=2400 + -e ARROW_CTEST_TIMEOUT=900 + -e ARROW_C_FLAGS_DEBUG="-O1" + -e ARROW_CXX_FLAGS_DEBUG="-O1" -e ARROW_LARGE_MEMORY_TESTS=ON + -e BUILD_WARNING_LEVEL=PRODUCTION runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/spot=capacity-optimized" title: AMD64 Ubuntu Large Memory Tests - image: conda-cpp From b927560748938aa0cfd758e77585c11010802cb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 12 Mar 2026 09:38:45 +0100 Subject: [PATCH 11/15] Increase volume size to avoid out of disk space --- .github/workflows/cpp_extra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 0977fa99fb00..040a75cc76b0 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -114,7 +114,7 @@ jobs: -e ARROW_CXX_FLAGS_DEBUG="-O1" -e ARROW_LARGE_MEMORY_TESTS=ON -e BUILD_WARNING_LEVEL=PRODUCTION - runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/spot=capacity-optimized" + runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/volume=80gb/spot=capacity-optimized" title: AMD64 Ubuntu Large Memory Tests - image: conda-cpp run-options: >- From 4960ed2a7af37f52b8f579bd508f46c4894668eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 12 Mar 2026 11:51:49 +0100 Subject: [PATCH 12/15] Increase timeout for parquet-arrow-reader-writer-test --- .github/workflows/cpp_extra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 040a75cc76b0..584431f9c1d7 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -109,7 +109,7 @@ jobs: title: AMD64 Alpine Linux - image: ubuntu-cpp run-options: >- - -e ARROW_CTEST_TIMEOUT=900 + -e ARROW_CTEST_TIMEOUT=2400 -e ARROW_C_FLAGS_DEBUG="-O1" -e ARROW_CXX_FLAGS_DEBUG="-O1" -e ARROW_LARGE_MEMORY_TESTS=ON From 0bdc786d0925b04d203f0290b7a769f4e26d0cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 12 Mar 2026 12:42:09 +0100 Subject: [PATCH 13/15] Reduce timeout it took 1522 seconds and add ARROW_GANDIVA=OFF --- .github/workflows/cpp_extra.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 584431f9c1d7..46b66f359e6d 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -109,9 +109,10 @@ jobs: title: AMD64 Alpine Linux - image: ubuntu-cpp run-options: >- - -e ARROW_CTEST_TIMEOUT=2400 + -e ARROW_CTEST_TIMEOUT=1800 -e ARROW_C_FLAGS_DEBUG="-O1" -e ARROW_CXX_FLAGS_DEBUG="-O1" + -e ARROW_GANDIVA=OFF -e ARROW_LARGE_MEMORY_TESTS=ON -e BUILD_WARNING_LEVEL=PRODUCTION runs-on: "runs-on=${{ github.run_id }}/family=x8i.2xlarge/volume=80gb/spot=capacity-optimized" From 70f036a1c8ec0dd662bec0ba811b8a9f5ca78b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 12 Mar 2026 14:01:23 +0100 Subject: [PATCH 14/15] Failed with 1800 --- .github/workflows/cpp_extra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 46b66f359e6d..f13b01041a89 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -109,7 +109,7 @@ jobs: title: AMD64 Alpine Linux - image: ubuntu-cpp run-options: >- - -e ARROW_CTEST_TIMEOUT=1800 + -e ARROW_CTEST_TIMEOUT=2000 -e ARROW_C_FLAGS_DEBUG="-O1" -e ARROW_CXX_FLAGS_DEBUG="-O1" -e ARROW_GANDIVA=OFF From bf0f9ab810ab58d235347008c6e530bd25c06ddc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Mon, 23 Mar 2026 11:59:28 +0100 Subject: [PATCH 15/15] GH-49502: [C++][Parquet] For those large memory tests we want huge pages --- cpp/src/parquet/column_writer_test.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/src/parquet/column_writer_test.cc b/cpp/src/parquet/column_writer_test.cc index 157e73ffec43..a45394917203 100644 --- a/cpp/src/parquet/column_writer_test.cc +++ b/cpp/src/parquet/column_writer_test.cc @@ -1053,8 +1053,10 @@ TEST(TestColumnWriter, LARGE_MEMORY_TEST(WriteLargeDictEncodedPage)) { { PrimitiveNode::Make("item", Repetition::REQUIRED, Type::INT32), })); - auto properties = - WriterProperties::Builder().data_pagesize(1024 * 1024 * 1024)->build(); + auto properties = WriterProperties::Builder() + .data_pagesize(1024 * 1024 * 1024) + ->max_rows_per_page(std::numeric_limits::max()) + ->build(); auto file_writer = ParquetFileWriter::Open(sink, schema, properties); auto rg_writer = file_writer->AppendRowGroup(); @@ -1124,8 +1126,10 @@ TEST(TestColumnWriter, LARGE_MEMORY_TEST(ThrowsOnDictIndicesTooLarge)) { { PrimitiveNode::Make("item", Repetition::REQUIRED, Type::INT32), })); - auto properties = - WriterProperties::Builder().data_pagesize(4 * 1024LL * 1024 * 1024)->build(); + auto properties = WriterProperties::Builder() + .data_pagesize(4 * 1024LL * 1024 * 1024) + ->max_rows_per_page(std::numeric_limits::max()) + ->build(); auto file_writer = ParquetFileWriter::Open(sink, schema, properties); auto rg_writer = file_writer->AppendRowGroup();