From d3cdeaebb2a8004e3f28d00ace2cce1f6cb9de2a Mon Sep 17 00:00:00 2001
From: Arham Chopra <arham.chopra@cubistsystematic.com>
Date: Wed, 10 Jun 2026 15:43:43 -0400
Subject: [PATCH 01/11] Refactor parquet output to a RecordBatch sink
 architecture

ParquetWriter builds RecordBatches and hands them to a pluggable
RecordBatchSink (onStart/onBatch/onFileChange/onStop). Removes the old C++
file-writer hierarchy and unifies output conversion via visitCspValueType.

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>
---
 cpp/csp/adapters/arrow/ArrowTypeVisitor.h     |  57 ++-
 .../parquet/ArrowBackedArrayBuilder.cpp       | 158 ++++++++
 .../parquet/ArrowBackedArrayBuilder.h         |  86 +++++
 .../parquet/ArrowIPCFileWriterWrapper.cpp     |  49 ---
 .../parquet/ArrowIPCFileWriterWrapper.h       |  44 ---
 .../parquet/ArrowSingleColumnArrayBuilder.h   | 272 +-------------
 cpp/csp/adapters/parquet/CMakeLists.txt       |  14 +-
 .../DialectGenericListWriterInterface.h       |  55 ---
 .../parquet/FileNameGeneratorReplicator.h     |  95 +++++
 .../adapters/parquet/FileWriterWrapper.cpp    |  49 ---
 cpp/csp/adapters/parquet/FileWriterWrapper.h  |  54 ---
 .../parquet/FileWriterWrapperContainer.cpp    | 116 ------
 .../parquet/FileWriterWrapperContainer.h      |  74 ----
 .../parquet/ParquetDictBasketOutputWriter.cpp |  68 ++--
 .../parquet/ParquetDictBasketOutputWriter.h   |   7 +-
 .../parquet/ParquetFileWriterWrapper.cpp      |  57 ---
 .../parquet/ParquetFileWriterWrapper.h        |  43 ---
 .../adapters/parquet/ParquetOutputAdapter.cpp | 346 +++---------------
 .../adapters/parquet/ParquetOutputAdapter.h   |  78 ++--
 .../parquet/ParquetOutputAdapterManager.cpp   |  39 +-
 .../parquet/ParquetOutputAdapterManager.h     |  29 +-
 cpp/csp/adapters/parquet/ParquetWriter.cpp    | 127 +++----
 cpp/csp/adapters/parquet/ParquetWriter.h      |  33 +-
 cpp/csp/adapters/parquet/RecordBatchSink.h    |  31 ++
 .../adapters/parquet/RecordBatchWithFlag.h    |  21 ++
 .../python/adapters/parquetadapterimpl.cpp    | 261 ++++++-------
 csp/adapters/_parquet_rb_writer.py            | 197 ++++++++++
 csp/adapters/output_adapters/parquet.py       |  33 ++
 28 files changed, 1013 insertions(+), 1480 deletions(-)
 create mode 100644 cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
 create mode 100644 cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
 delete mode 100644 cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp
 delete mode 100644 cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.h
 delete mode 100644 cpp/csp/adapters/parquet/DialectGenericListWriterInterface.h
 create mode 100644 cpp/csp/adapters/parquet/FileNameGeneratorReplicator.h
 delete mode 100644 cpp/csp/adapters/parquet/FileWriterWrapper.cpp
 delete mode 100644 cpp/csp/adapters/parquet/FileWriterWrapper.h
 delete mode 100644 cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp
 delete mode 100644 cpp/csp/adapters/parquet/FileWriterWrapperContainer.h
 delete mode 100644 cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp
 delete mode 100644 cpp/csp/adapters/parquet/ParquetFileWriterWrapper.h
 create mode 100644 cpp/csp/adapters/parquet/RecordBatchSink.h
 create mode 100644 cpp/csp/adapters/parquet/RecordBatchWithFlag.h
 create mode 100644 csp/adapters/_parquet_rb_writer.py

diff --git a/cpp/csp/adapters/arrow/ArrowTypeVisitor.h b/cpp/csp/adapters/arrow/ArrowTypeVisitor.h
index 67e74abb7..eabfb6d92 100644
--- a/cpp/csp/adapters/arrow/ArrowTypeVisitor.h
+++ b/cpp/csp/adapters/arrow/ArrowTypeVisitor.h
@@ -1,9 +1,13 @@
 // Arrow type visitor: maps arrow::Type::type to the corresponding C++ value type.
+// Eliminates repeated switch statements on arrow types across the codebase.
 //
 // Usage:
 //   visitArrowValueType( typeId,
-//       [&]( auto tag ) { using T = typename decltype(tag)::type; ... },
-//       [&]() { /* unsupported */ } );
+//       [&]( auto tag ) -> ReturnType {
+//           using T = typename decltype( tag )::type;
+//           return doSomething<T>( ... );
+//       },
+//       [&]() -> ReturnType { /* unsupported type fallback */ } );
 
 #ifndef _IN_CSP_ADAPTERS_ARROW_ArrowTypeVisitor_H
 #define _IN_CSP_ADAPTERS_ARROW_ArrowTypeVisitor_H
@@ -20,12 +24,14 @@ namespace csp::adapters::arrow
 template<typename T>
 struct TypeTag { using type = T; };
 
-// Invokes fn(TypeTag<CppType>{}) for the C++ type corresponding to the arrow type.
+// Invokes fn( TypeTag<CppType>{} ) for the C++ value type corresponding to
+// the given arrow type.  Calls onDefault() for unrecognised arrow types.
 template<typename Fn, typename DefaultFn>
 decltype(auto) visitArrowValueType( ::arrow::Type::type typeId, Fn && fn, DefaultFn && onDefault )
 {
     switch( typeId )
     {
+        // --- Numeric ---
         case ::arrow::Type::BOOL:   return fn( TypeTag<bool>{} );
         case ::arrow::Type::INT8:   return fn( TypeTag<int8_t>{} );
         case ::arrow::Type::INT16:  return fn( TypeTag<int16_t>{} );
@@ -39,6 +45,7 @@ decltype(auto) visitArrowValueType( ::arrow::Type::type typeId, Fn && fn, Defaul
         case ::arrow::Type::FLOAT:
         case ::arrow::Type::DOUBLE: return fn( TypeTag<double>{} );
 
+        // --- String / Binary ---
         case ::arrow::Type::STRING:
         case ::arrow::Type::LARGE_STRING:
         case ::arrow::Type::BINARY:
@@ -47,6 +54,7 @@ decltype(auto) visitArrowValueType( ::arrow::Type::type typeId, Fn && fn, Defaul
         case ::arrow::Type::DICTIONARY:
             return fn( TypeTag<std::string>{} );
 
+        // --- Temporal ---
         case ::arrow::Type::TIMESTAMP: return fn( TypeTag<DateTime>{} );
         case ::arrow::Type::DURATION:  return fn( TypeTag<TimeDelta>{} );
         case ::arrow::Type::DATE32:
@@ -54,6 +62,7 @@ decltype(auto) visitArrowValueType( ::arrow::Type::type typeId, Fn && fn, Defaul
         case ::arrow::Type::TIME32:
         case ::arrow::Type::TIME64:    return fn( TypeTag<Time>{} );
 
+        // --- List / Struct ---
         case ::arrow::Type::LIST:
         case ::arrow::Type::LARGE_LIST:
             return fn( TypeTag<DialectGenericType>{} );
@@ -65,7 +74,7 @@ decltype(auto) visitArrowValueType( ::arrow::Type::type typeId, Fn && fn, Defaul
     }
 }
 
-// Overload that throws TypeError for unrecognised types.
+// Overload that throws TypeError for unrecognised arrow types.
 template<typename Fn>
 decltype(auto) visitArrowValueType( ::arrow::Type::type typeId, Fn && fn )
 {
@@ -76,6 +85,44 @@ decltype(auto) visitArrowValueType( ::arrow::Type::type typeId, Fn && fn )
         } );
 }
 
+// Maps CspType::Type → C++ value type via TypeTag.
+// Calls fn(TypeTag<CppType>{}) for the matching type.
+template<typename Fn, typename DefaultFn>
+decltype(auto) visitCspValueType( CspType::Type cspType, Fn && fn, DefaultFn && onDefault )
+{
+    switch( cspType )
+    {
+        case CspType::Type::BOOL:      return fn( TypeTag<bool>{} );
+        case CspType::Type::INT8:      return fn( TypeTag<int8_t>{} );
+        case CspType::Type::UINT8:     return fn( TypeTag<uint8_t>{} );
+        case CspType::Type::INT16:     return fn( TypeTag<int16_t>{} );
+        case CspType::Type::UINT16:    return fn( TypeTag<uint16_t>{} );
+        case CspType::Type::INT32:     return fn( TypeTag<int32_t>{} );
+        case CspType::Type::UINT32:    return fn( TypeTag<uint32_t>{} );
+        case CspType::Type::INT64:     return fn( TypeTag<int64_t>{} );
+        case CspType::Type::UINT64:    return fn( TypeTag<uint64_t>{} );
+        case CspType::Type::DOUBLE:    return fn( TypeTag<double>{} );
+        case CspType::Type::DATETIME:  return fn( TypeTag<DateTime>{} );
+        case CspType::Type::TIMEDELTA: return fn( TypeTag<TimeDelta>{} );
+        case CspType::Type::DATE:      return fn( TypeTag<Date>{} );
+        case CspType::Type::TIME:      return fn( TypeTag<Time>{} );
+        case CspType::Type::STRING:    return fn( TypeTag<std::string>{} );
+        case CspType::Type::ENUM:      return fn( TypeTag<CspEnum>{} );
+        default:                       return onDefault();
+    }
+}
+
+// Convenience overload that throws on unhandled types.
+template<typename Fn>
+decltype(auto) visitCspValueType( CspType::Type cspType, Fn && fn )
+{
+    return visitCspValueType( cspType, std::forward<Fn>( fn ),
+        [cspType]() -> decltype( fn( TypeTag<bool>{} ) )
+        {
+            CSP_THROW( TypeError, "Unhandled CspType in visitor: " << cspType );
+        } );
+}
+
 } // namespace csp::adapters::arrow
 
-#endif // _IN_CSP_ADAPTERS_ARROW_ArrowTypeVisitor_H
+#endif
diff --git a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
new file mode 100644
index 000000000..c09114002
--- /dev/null
+++ b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
@@ -0,0 +1,158 @@
+#include <csp/adapters/parquet/ArrowBackedArrayBuilder.h>
+#include <csp/adapters/arrow/ArrowFieldWriter.h>
+#include <csp/adapters/arrow/ArrowTypeVisitor.h>
+
+namespace csp::adapters::parquet
+{
+
+static StructFieldPtr createStructFieldFromCspType( const std::string & name, CspTypePtr cspType )
+{
+    if( cspType -> type() == CspType::Type::STRING )
+        return std::make_shared<StringStructField>( CspType::STRING(), name, false );
+    if( cspType -> type() == CspType::Type::ENUM )
+        return std::make_shared<CspEnumStructField>( cspType, name, false );
+
+    return csp::adapters::arrow::visitCspValueType( cspType -> type(),
+        [&]( auto tag ) -> StructFieldPtr
+        {
+            using T = typename decltype( tag )::type;
+            if constexpr( std::is_same_v<T, std::string> || std::is_same_v<T, CspEnum> )
+                return {};  // unreachable; STRING/ENUM handled above
+            else
+                return std::make_shared<NativeStructField<T>>( name, false );
+        },
+        [&]() -> StructFieldPtr
+        {
+            CSP_THROW( TypeError, "Unsupported CSP type for struct field: " << cspType -> type() );
+        } );
+}
+
+ScratchFieldInfo createScratchField( const std::string & name, CspTypePtr cspType )
+{
+    auto field = createStructFieldFromCspType( name, cspType );
+
+    auto meta = std::make_shared<StructMeta>(
+        "__scratch_" + name,
+        StructMeta::Fields{ field },
+        false
+    );
+
+    return ScratchFieldInfo{ meta, meta -> field( name ) };
+}
+
+ArrowBackedArrayBuilder::ArrowBackedArrayBuilder(
+    const std::string & columnName, std::uint32_t chunkSize,
+    CspTypePtr cspType, bool isBytes )
+    : ArrowSingleColumnArrayBuilder( columnName, chunkSize )
+    , m_isScratchMode( true )
+{
+    // For bytes fields, we need to create a STRING type scratch field
+    // but the FieldWriter should produce binary output.
+    // The createScratchField always creates STRING type fields for STRING cspType.
+    auto scratch = createScratchField( columnName, cspType );
+    m_scratchMeta = scratch.meta;
+    m_field       = scratch.field;
+    m_scratch     = m_scratchMeta -> create();
+
+    // If isBytes, wrap the STRING field so FieldWriter produces binary
+    if( isBytes )
+    {
+        auto bytesField = std::make_shared<StringStructField>(
+            std::make_shared<CspStringType>( true ), columnName, false );
+        auto bytesMeta = std::make_shared<StructMeta>(
+            "__scratch_bytes_" + columnName,
+            StructMeta::Fields{ bytesField },
+            false
+        );
+        m_scratchMeta = bytesMeta;
+        m_field       = bytesMeta -> field( columnName );
+        m_scratch     = m_scratchMeta -> create();
+    }
+
+    init( columnName, m_field );
+}
+
+ArrowBackedArrayBuilder::ArrowBackedArrayBuilder(
+    const std::string & columnName, std::uint32_t chunkSize,
+    const StructFieldPtr & structField )
+    : ArrowSingleColumnArrayBuilder( columnName, chunkSize )
+    , m_field( structField )
+    , m_isScratchMode( false )
+{
+    init( columnName, structField );
+}
+
+ArrowBackedArrayBuilder::~ArrowBackedArrayBuilder() = default;
+
+void ArrowBackedArrayBuilder::init( const std::string & columnName, const StructFieldPtr & field )
+{
+    auto created = csp::adapters::arrow::createFieldWriter( columnName, field );
+    m_writer = std::move( created.writer );
+}
+
+std::shared_ptr<::arrow::DataType> ArrowBackedArrayBuilder::getDataType()
+{
+    return m_writer -> dataTypes()[0];
+}
+
+std::shared_ptr<::arrow::ArrayBuilder> ArrowBackedArrayBuilder::getBuilder()
+{
+    return m_writer -> builder();
+}
+
+int64_t ArrowBackedArrayBuilder::length() const
+{
+    return m_writer -> builder() -> length();
+}
+
+void ArrowBackedArrayBuilder::handleRowFinished()
+{
+    if( m_isScratchMode )
+    {
+        // Scratch mode: FieldWriter reads from scratch struct.
+        // writeNext checks isSet internally and appends value or null.
+        m_writer -> writeNext( m_scratch.get() );
+
+        // Clear the isSet bit so next cycle starts with null
+        uint8_t * mask = reinterpret_cast<uint8_t *>( m_scratch.get() ) + m_field -> maskOffset();
+        *mask &= ~m_field -> maskBitMask();
+    }
+    else
+    {
+        // External mode: read from externally-provided struct
+        if( m_hasExternalValue )
+        {
+            m_writer -> writeNext( m_externalStruct );
+            m_hasExternalValue = false;
+            m_externalStruct = nullptr;
+        }
+        else
+        {
+            m_writer -> writeNull();
+        }
+    }
+}
+
+std::shared_ptr<::arrow::Array> ArrowBackedArrayBuilder::buildArray()
+{
+    auto arrays = m_writer -> finish();
+    CSP_TRUE_OR_THROW_RUNTIME( arrays.size() == 1,
+        "ArrowBackedArrayBuilder expected 1 array from FieldWriter, got " << arrays.size() );
+    return arrays[0];
+}
+
+std::shared_ptr<ArrowBackedArrayBuilder> createArrowBackedArrayBuilder(
+    const std::string & columnName, std::uint32_t chunkSize,
+    CspTypePtr cspType, bool isBytes )
+{
+    return std::make_shared<ArrowBackedArrayBuilder>( columnName, chunkSize, cspType, isBytes );
+}
+
+std::shared_ptr<ArrowBackedArrayBuilder> createArrowBackedArrayBuilderForField(
+    const std::string & columnName, std::uint32_t chunkSize,
+    const StructFieldPtr & structField )
+{
+    return std::make_shared<ArrowBackedArrayBuilder>( columnName, chunkSize, structField );
+}
+
+} // namespace csp::adapters::parquet
diff --git a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
new file mode 100644
index 000000000..a8e91ed31
--- /dev/null
+++ b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
@@ -0,0 +1,86 @@
+#ifndef _IN_CSP_ADAPTERS_PARQUET_ArrowBackedArrayBuilder_H
+#define _IN_CSP_ADAPTERS_PARQUET_ArrowBackedArrayBuilder_H
+
+#include <csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h>
+
+// Forward declare to avoid namespace conflicts (see ArrowBackedColumnAdapter.h)
+namespace csp::adapters::arrow { class FieldWriter; }
+
+namespace csp::adapters::parquet
+{
+
+// Factory: create a scratch StructMeta + field for a given CSP type.
+struct ScratchFieldInfo
+{
+    std::shared_ptr<StructMeta> meta;
+    StructFieldPtr field;
+};
+
+ScratchFieldInfo createScratchField( const std::string & name, CspTypePtr cspType );
+
+// Wraps an ArrowFieldWriter inside the ArrowSingleColumnArrayBuilder interface.
+// Two modes:
+//   Scratch mode: owns a single-field struct, caller writes value, handleRowFinished appends.
+//   External mode: caller provides a Struct*, handleRowFinished reads field and appends.
+class ArrowBackedArrayBuilder : public ArrowSingleColumnArrayBuilder
+{
+public:
+    // Scratch mode: creates FieldWriter + scratch struct from a CspType.
+    // Used by SingleColumnParquetOutputHandler.
+    ArrowBackedArrayBuilder( const std::string & columnName, std::uint32_t chunkSize,
+                             CspTypePtr cspType, bool isBytes = false );
+
+    // External mode: creates FieldWriter from an existing StructFieldPtr.
+    // Used by StructParquetOutputHandler (reads from source struct directly).
+    ArrowBackedArrayBuilder( const std::string & columnName, std::uint32_t chunkSize,
+                             const StructFieldPtr & structField );
+
+    ~ArrowBackedArrayBuilder() override;
+
+    // --- ArrowSingleColumnArrayBuilder interface ---
+    std::shared_ptr<::arrow::DataType> getDataType() override;
+    std::shared_ptr<::arrow::ArrayBuilder> getBuilder() override;
+    int64_t length() const override;
+    void handleRowFinished() override;
+    std::shared_ptr<::arrow::Array> buildArray() override;
+
+    // --- Scratch mode API ---
+    // Get the scratch struct for value setting
+    Struct * scratch() { return m_scratch.get(); }
+    const StructFieldPtr & scratchField() const { return m_field; }
+
+    // --- External mode API ---
+    // Set the source struct for this row (marks value available)
+    void setStruct( const Struct * s )
+    {
+        m_externalStruct    = s;
+        m_hasExternalValue  = true;
+    }
+
+private:
+    void init( const std::string & columnName, const StructFieldPtr & field );
+
+    std::unique_ptr<csp::adapters::arrow::FieldWriter> m_writer;
+    std::shared_ptr<StructMeta>                        m_scratchMeta; // non-null in scratch mode
+    StructPtr                                          m_scratch;     // non-null in scratch mode
+    StructFieldPtr                                     m_field;
+
+    // External mode state
+    const Struct * m_externalStruct   = nullptr;
+    bool           m_hasExternalValue = false;
+    bool           m_isScratchMode    = false;
+};
+
+// Factory: create ArrowBackedArrayBuilder for a given CspType (scratch mode).
+std::shared_ptr<ArrowBackedArrayBuilder> createArrowBackedArrayBuilder(
+    const std::string & columnName, std::uint32_t chunkSize,
+    CspTypePtr cspType, bool isBytes = false );
+
+// Factory: create ArrowBackedArrayBuilder for a struct field (external mode).
+std::shared_ptr<ArrowBackedArrayBuilder> createArrowBackedArrayBuilderForField(
+    const std::string & columnName, std::uint32_t chunkSize,
+    const StructFieldPtr & structField );
+
+} // namespace csp::adapters::parquet
+
+#endif
diff --git a/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp b/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp
deleted file mode 100644
index daf23337e..000000000
--- a/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp
+++ /dev/null
@@ -1,49 +0,0 @@
-#include <csp/adapters/parquet/ArrowIPCFileWriterWrapper.h>
-#include <csp/adapters/parquet/ParquetStatusUtils.h>
-#include <arrow/io/file.h>
-#include <arrow/table.h>
-#include <arrow/ipc/writer.h>
-#include <parquet/api/io.h>
-
-namespace csp::adapters::parquet
-{
-void ArrowIPCFileWriterWrapper::openImpl( const std::string &fileName, const std::string &compression )
-{
-    CSP_TRUE_OR_THROW_RUNTIME( m_outputStream == nullptr, "Trying to open parquet file while previous was not closed" );
-
-    PARQUET_ASSIGN_OR_THROW(
-            m_outputStream,
-            arrow::io::FileOutputStream::Open( fileName.c_str()));
-
-    arrow::ipc::IpcWriteOptions writeOptions;
-    writeOptions.codec  = resolveCompressionCodec( compression );
-
-    STATUS_OK_OR_THROW_RUNTIME(
-            ::arrow::ipc::MakeStreamWriter( m_outputStream.get(), getSchema(), writeOptions ).Value(&m_fileWriter),
-            "Failed to open arrow file writer" );
-}
-
-void ArrowIPCFileWriterWrapper::close()
-{
-    if( m_outputStream )
-    {
-        // Let's move them first, if there are any exceptions, we still want the pointer to be null
-        std::shared_ptr<::arrow::io::FileOutputStream> outputStream{ std::move( m_outputStream ) };
-        std::shared_ptr<::arrow::ipc::RecordBatchWriter>  fileWriter{ std::move( m_fileWriter ) };
-        // Should be done by move constructor but let's be safe:
-        m_outputStream = nullptr;
-        m_fileWriter   = nullptr;
-
-        if( fileWriter )
-            STATUS_OK_OR_THROW_RUNTIME( fileWriter->Close(), "Failed to close arrow file writer" );
-        if( outputStream )
-            STATUS_OK_OR_THROW_RUNTIME( outputStream->Close(), "Failed to close arrow output stream" );
-    }
-}
-
-void ArrowIPCFileWriterWrapper::writeTable( const std::shared_ptr<::arrow::Table> &table )
-{
-    STATUS_OK_OR_THROW_RUNTIME( m_fileWriter->WriteTable( *table, table->num_rows()), "Failed to write to arrow file" );
-}
-
-}
diff --git a/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.h b/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.h
deleted file mode 100644
index 30d5e340d..000000000
--- a/cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.h
+++ /dev/null
@@ -1,44 +0,0 @@
-#ifndef _IN_CSP_ADAPTERS_PARQUET_ArrowIPCFileWriterWrapper_H
-#define _IN_CSP_ADAPTERS_PARQUET_ArrowIPCFileWriterWrapper_H
-
-#include <csp/adapters/parquet/FileWriterWrapper.h>
-#include <string>
-
-namespace arrow::io
-{
-class FileOutputStream;
-}
-
-namespace arrow::ipc
-{
-class RecordBatchWriter;
-}
-
-namespace csp::adapters::parquet
-{
-
-class ArrowIPCFileWriterWrapper : public FileWriterWrapper
-{
-public:
-    ArrowIPCFileWriterWrapper( std::shared_ptr<::arrow::Schema> schema )
-            : FileWriterWrapper( schema )
-    {
-    }
-
-    ~ArrowIPCFileWriterWrapper() { close(); }
-
-    void close() override;
-    void writeTable( const std::shared_ptr<::arrow::Table> &table ) override;
-
-protected:
-    void openImpl( const std::string &fileName, const std::string &compression ) override;
-
-private:
-    std::shared_ptr<::arrow::io::FileOutputStream> m_outputStream;
-    std::shared_ptr<::arrow::ipc::RecordBatchWriter> m_fileWriter;
-};
-
-}
-
-
-#endif
\ No newline at end of file
diff --git a/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h b/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
index af338d23a..16999bedc 100644
--- a/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
+++ b/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
@@ -1,13 +1,13 @@
 #ifndef _IN_CSP_ADAPTERS_PARQUET_ArrowSingleColumnArrayBuilder_H
 #define _IN_CSP_ADAPTERS_PARQUET_ArrowSingleColumnArrayBuilder_H
 
+#include <csp/adapters/arrow/ArrowFieldWriter.h>
 #include <csp/adapters/parquet/ParquetStatusUtils.h>
 #include <csp/core/Exception.h>
 #include <csp/core/Time.h>
 #include <csp/engine/Struct.h>
 #include <arrow/builder.h>
 #include <string>
-#include <csp/adapters/parquet/DialectGenericListWriterInterface.h>
 
 namespace csp::adapters::parquet
 {
@@ -15,7 +15,7 @@ class ArrowSingleColumnArrayBuilder
 {
 public:
     ArrowSingleColumnArrayBuilder( std::string columnName, std::uint32_t chunkSize )
-            : m_columnName( columnName ), m_chunkSize( chunkSize )
+            : m_columnName( std::move( columnName ) ), m_chunkSize( chunkSize )
     {
     }
 
@@ -53,7 +53,7 @@ class StructColumnArrayBuilder : public ArrowSingleColumnArrayBuilder
                               const std::shared_ptr<::arrow::DataType> &type,
                               const std::vector<ColumnBuilderPtr> &childArrayBuilders,
                               FieldValueSetter fieldValueSetter )
-            : ArrowSingleColumnArrayBuilder( columnName, chunkSize ),
+            : ArrowSingleColumnArrayBuilder( std::move( columnName ), chunkSize ),
               m_childArrayBuilders( childArrayBuilders ),
               m_builderPtr( std::make_shared<::arrow::StructBuilder>( type, ::arrow::default_memory_pool(),
                                                                       getArrowChildArrayBuilders( childArrayBuilders ) ) ),
@@ -137,11 +137,11 @@ class ListColumnArrayBuilder : public ArrowSingleColumnArrayBuilder
 
     ListColumnArrayBuilder( std::string columnName, std::uint32_t chunkSize,
                             const std::shared_ptr<::arrow::ArrayBuilder>& valueBuilder,
-                            const DialectGenericListWriterInterface::Ptr& listWriterInterface)
-            : ArrowSingleColumnArrayBuilder( columnName, chunkSize ),
+                            csp::adapters::arrow::ListItemsWriter writeItemsFn )
+            : ArrowSingleColumnArrayBuilder( std::move( columnName ), chunkSize ),
               m_valueBuilder( valueBuilder ),
               m_builderPtr( std::make_shared<::arrow::ListBuilder>( ::arrow::default_memory_pool(), m_valueBuilder ) ),
-              m_listWriterInterface(listWriterInterface)
+              m_writeItemsFn( std::move( writeItemsFn ) )
     {
 
     }
@@ -167,7 +167,7 @@ class ListColumnArrayBuilder : public ArrowSingleColumnArrayBuilder
         if( m_value.has_value() )
         {
             CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr -> Append().ok(), "Failed to append list" );
-            m_listWriterInterface->writeItems(m_value.value());
+            m_writeItemsFn( m_value.value() );
             m_value.reset();
         }
         else
@@ -190,263 +190,13 @@ class ListColumnArrayBuilder : public ArrowSingleColumnArrayBuilder
     }
 
 private:
-    static std::shared_ptr<::arrow::ArrayBuilder> createValueArrayBuilderForType( const std::shared_ptr<::arrow::DataType> &childType)
-    {
-        switch(childType->id())
-        {
-            case ::arrow::Type::INT64:
-                return std::make_shared<::arrow::Int64Builder>();
-            case ::arrow::Type::DOUBLE:
-                return std::make_shared<::arrow::DoubleBuilder>();
-            case ::arrow::Type::BOOL:
-                return std::make_shared<::arrow::BooleanBuilder>();
-            default:
-                CSP_THROW(TypeError, "Trying to create arrow list array builder for unsupported element type " << childType->name());
-        }
-    }
-
-    static std::vector<std::shared_ptr<::arrow::ArrayBuilder>> getArrowChildArrayBuilders(
-            const std::vector<ColumnBuilderPtr> &childArrayBuilders )
-    {
-        std::vector<std::shared_ptr<::arrow::ArrayBuilder>> res;
-
-        for( auto &columnBuilder : childArrayBuilders )
-        {
-            res.push_back( columnBuilder -> getBuilder() );
-        }
-        return res;
-    }
-
-private:
-    std::shared_ptr<::arrow::ArrayBuilder>   m_valueBuilder;
-    std::shared_ptr<::arrow::ListBuilder>  m_builderPtr;
-    DialectGenericListWriterInterface::Ptr m_listWriterInterface;
-    std::optional<DialectGenericType>      m_value;
-};
-
-
-template< typename ValueType, typename ArrowBuilderType >
-class BaseTypedArrayBuilder : public ArrowSingleColumnArrayBuilder
-{
-public:
-    using ValueTypeT = ValueType;
-    using ArrowBuilderTypeT = ArrowBuilderType;
-
-    BaseTypedArrayBuilder( std::string columnName, std::uint32_t chunkSize )
-            : ArrowSingleColumnArrayBuilder( columnName, chunkSize ),
-              m_builderPtr( std::make_shared<ArrowBuilderType>() )
-    {
-        CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr -> Reserve( getChunkSize() ).ok(), "Failed to reserve arrow array size" );
-    }
-
-    std::shared_ptr<::arrow::ArrayBuilder> getBuilder() override
-    {
-        return m_builderPtr;
-    }
-
-    std::shared_ptr<::arrow::DataType> getDataType() override
-    {
-        return m_builderPtr -> type();
-    }
-
-    virtual int64_t length() const override
-    {
-        return m_builderPtr -> length();
-    }
-
-    void handleRowFinished() override
-    {
-        if( m_value )
-        {
-            pushValueToArray();
-        }
-        else
-        {
-            STATUS_OK_OR_THROW_RUNTIME( m_builderPtr -> AppendNull(), "Failed to append null to arrow array" );
-        }
-        m_value = nullptr;
-    }
-
-    std::shared_ptr<::arrow::Array> buildArray() override
-    {
-        std::shared_ptr<::arrow::Array> array;
-        CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr -> Finish( &array ).ok(), "Failed to create arrow array" );
-        return array;
-    }
-
-    void setValue( const ValueType &value )
-    {
-        m_value = &value;
-    }
-
-protected:
-    template< typename ...BuilderArgs >
-    BaseTypedArrayBuilder( std::string columnName, std::uint32_t chunkSize, BuilderArgs ...args )
-            : ArrowSingleColumnArrayBuilder( columnName, chunkSize ),
-              m_builderPtr( std::make_shared<ArrowBuilderType>( args... ) )
-    {
-
-    }
-
-    virtual void pushValueToArray() = 0;
-
-protected:
-    std::shared_ptr<ArrowBuilderType> m_builderPtr;
-    const ValueType                   *m_value = nullptr;
-};
-
-template< typename ValueType, typename ArrowBuilderType >
-class PrimitiveTypedArrayBuilder : public BaseTypedArrayBuilder<ValueType, ArrowBuilderType>
-{
-public:
-    using BaseTypedArrayBuilder<ValueType, ArrowBuilderType>::BaseTypedArrayBuilder;
-
-protected:
-    void pushValueToArray()
-    {
-        [[maybe_unused]] auto status = this -> m_builderPtr -> Append( *this -> m_value );        
-    }
-};
-
-class StringArrayBuilder : public BaseTypedArrayBuilder<std::string, ::arrow::StringBuilder>
-{
-public:
-    using BaseTypedArrayBuilder<std::string, ::arrow::StringBuilder>::BaseTypedArrayBuilder;
-
-protected:
-    void pushValueToArray()
-    {
-        const std::string &value = *m_value;
-        STATUS_OK_OR_THROW_RUNTIME( this -> m_builderPtr -> Append( value.c_str(), value.length() ),
-                                    "Failed to append value to arrow array" );
-    }
-};
-
-class BytesArrayBuilder : public BaseTypedArrayBuilder<std::string, ::arrow::BinaryBuilder>
-{
-public:
-    using BaseTypedArrayBuilder<std::string, ::arrow::BinaryBuilder>::BaseTypedArrayBuilder;
-
-protected:
-    void pushValueToArray()
-    {
-        const std::string &value = *m_value;
-        STATUS_OK_OR_THROW_RUNTIME( this -> m_builderPtr -> Append( value.c_str(), value.length() ),
-                                    "Failed to append value to arrow array" );
-    }
-};
-
-class [[maybe_unused]] CStringArrayBuilder : public BaseTypedArrayBuilder<const char *, ::arrow::StringBuilder>
-{
-public:
-    using BaseTypedArrayBuilder<const char *, ::arrow::StringBuilder>::BaseTypedArrayBuilder;
-
-    [[maybe_unused]] void setValueCopyPtr( const char *value )
-    {
-        m_ptrCopy = value;
-        setValue( m_ptrCopy );
-    }
-
-protected:
-    void pushValueToArray()
-    {
-        const char *value = *m_value;
-        STATUS_OK_OR_THROW_RUNTIME( this -> m_builderPtr -> Append( value ), "Failed to append value to arrow array" );
-    }
-
-private:
-    const char *m_ptrCopy;
+    std::shared_ptr<::arrow::ArrayBuilder>                m_valueBuilder;
+    std::shared_ptr<::arrow::ListBuilder>                 m_builderPtr;
+    csp::adapters::arrow::ListItemsWriter                 m_writeItemsFn;
+    std::optional<DialectGenericType>                      m_value;
 };
 
 
-class DatetimeArrayBuilder : public BaseTypedArrayBuilder<csp::DateTime, ::arrow::TimestampBuilder>
-{
-public:
-    DatetimeArrayBuilder( std::string columnName, std::uint32_t chunkSize )
-            : BaseTypedArrayBuilder<csp::DateTime, ::arrow::TimestampBuilder>( columnName,
-                                                                             chunkSize,
-                                                                             std::make_shared<::arrow::TimestampType>( ::arrow::TimeUnit::NANO,
-                                                                                                                     "UTC" ),
-                                                                             ::arrow::default_memory_pool() )
-    {
-    }
-
-protected:
-    void pushValueToArray()
-    {
-        STATUS_OK_OR_THROW_RUNTIME( this -> m_builderPtr -> Append( m_value -> asNanoseconds() ),
-                                    "Failed to append timestamp value to arrow array" );
-    }
-};
-
-class DateArrayBuilder : public BaseTypedArrayBuilder<csp::Date, ::arrow::Date32Builder>
-{
-public:
-    DateArrayBuilder( std::string columnName, std::uint32_t chunkSize )
-            : BaseTypedArrayBuilder<csp::Date, ::arrow::Date32Builder>( columnName,
-                                                                      chunkSize )
-    {
-    }
-
-protected:
-    void pushValueToArray()
-    {
-        auto timedelta = *m_value - getOrigin();
-        STATUS_OK_OR_THROW_RUNTIME( this -> m_builderPtr -> Append( timedelta.days() ), "Failed to append date value to arrow array" );
-    }
-
-private:
-    static csp::Date &getOrigin()
-    {
-        static csp::Date origin{ 1970, 1, 1 };
-        return origin;
-    }
-};
-
-class TimeArrayBuilder : public BaseTypedArrayBuilder<csp::Time, ::arrow::Time64Builder>
-{
-public:
-    TimeArrayBuilder( std::string columnName, std::uint32_t chunkSize )
-        : BaseTypedArrayBuilder<csp::Time, ::arrow::Time64Builder>( columnName, chunkSize,
-                                                                  std::make_shared<::arrow::Time64Type>( ::arrow::TimeUnit::NANO ), ::arrow::default_memory_pool() )
-    {
-    }
-
-protected:
-    void pushValueToArray()
-    {
-        STATUS_OK_OR_THROW_RUNTIME( this -> m_builderPtr -> Append( m_value -> asNanoseconds() ), "Failed to append time value to arrow array" );
-    }
-};
-
-class TimedeltaArrayBuilder : public BaseTypedArrayBuilder<csp::TimeDelta, ::arrow::DurationBuilder>
-{
-public:
-    TimedeltaArrayBuilder( std::string columnName, std::uint32_t chunkSize )
-            : BaseTypedArrayBuilder<csp::TimeDelta, ::arrow::DurationBuilder>(
-            columnName, chunkSize, std::make_shared<::arrow::DurationType>( ::arrow::TimeUnit::NANO ),
-            ::arrow::default_memory_pool() )
-    {
-    }
-
-protected:
-    void pushValueToArray()
-    {
-        STATUS_OK_OR_THROW_RUNTIME( this -> m_builderPtr -> Append( m_value -> asNanoseconds() ),
-                                    "Failed to append date value to arrow array" );
-    }
-};
-
-using BoolArrayBuilder = PrimitiveTypedArrayBuilder<bool, ::arrow::BooleanBuilder>;
-using Int8ArrayBuilder = PrimitiveTypedArrayBuilder<std::int8_t, ::arrow::Int8Builder>;
-using Int16ArrayBuilder = PrimitiveTypedArrayBuilder<std::int16_t, ::arrow::Int16Builder>;
-using Int32ArrayBuilder = PrimitiveTypedArrayBuilder<std::int32_t, ::arrow::Int32Builder>;
-using Int64ArrayBuilder = PrimitiveTypedArrayBuilder<std::int64_t, ::arrow::Int64Builder>;
-using UInt8ArrayBuilder = PrimitiveTypedArrayBuilder<std::uint8_t, ::arrow::UInt8Builder>;
-using UInt16ArrayBuilder = PrimitiveTypedArrayBuilder<std::uint16_t, ::arrow::UInt16Builder>;
-using UInt32ArrayBuilder = PrimitiveTypedArrayBuilder<std::uint32_t, ::arrow::UInt32Builder>;
-using UInt64ArrayBuilder = PrimitiveTypedArrayBuilder<std::uint64_t, ::arrow::UInt64Builder>;
-using DoubleArrayBuilder = PrimitiveTypedArrayBuilder<double, ::arrow::DoubleBuilder>;
 }
 
 #endif
diff --git a/cpp/csp/adapters/parquet/CMakeLists.txt b/cpp/csp/adapters/parquet/CMakeLists.txt
index ae3f639a2..ab452829d 100644
--- a/cpp/csp/adapters/parquet/CMakeLists.txt
+++ b/cpp/csp/adapters/parquet/CMakeLists.txt
@@ -1,10 +1,7 @@
 set(PARQUET_HEADER_FILES
-        ArrowIPCFileWriterWrapper.h
+        ArrowBackedArrayBuilder.h
         ArrowSingleColumnArrayBuilder.h
-        DialectGenericListWriterInterface.h
-        FileWriterWrapper.h
-        ParquetFileWriterWrapper.h
-        FileWriterWrapperContainer.h
+        FileNameGeneratorReplicator.h
         ParquetDictBasketOutputWriter.h
         ParquetInputAdapterManager.h
         ParquetOutputAdapter.h
@@ -12,14 +9,13 @@ set(PARQUET_HEADER_FILES
         ParquetOutputFilenameAdapter.h
         ParquetStatusUtils.h
         ParquetWriter.h
+        RecordBatchSink.h
         RecordBatchStreamSource.h
+        RecordBatchWithFlag.h
 )
 
 set(PARQUET_SOURCE_FILES
-        ArrowIPCFileWriterWrapper.cpp
-        FileWriterWrapper.cpp
-        ParquetFileWriterWrapper.cpp
-        FileWriterWrapperContainer.cpp
+        ArrowBackedArrayBuilder.cpp
         ParquetDictBasketOutputWriter.cpp
         ParquetInputAdapterManager.cpp
         ParquetOutputAdapter.cpp
diff --git a/cpp/csp/adapters/parquet/DialectGenericListWriterInterface.h b/cpp/csp/adapters/parquet/DialectGenericListWriterInterface.h
deleted file mode 100644
index e9795d15f..000000000
--- a/cpp/csp/adapters/parquet/DialectGenericListWriterInterface.h
+++ /dev/null
@@ -1,55 +0,0 @@
-#ifndef _IN_CSP_ADAPTERS_PARQUET_DialectGenericListWriterInterface_H
-#define _IN_CSP_ADAPTERS_PARQUET_DialectGenericListWriterInterface_H
-
-#include <memory>
-#include <functional>
-#include <csp/core/Exception.h>
-#include <csp/engine/DialectGenericType.h>
-#include <arrow/array/builder_base.h>
-
-namespace csp::adapters::parquet
-{
-
-class DialectGenericListWriterInterface
-{
-public:
-    using ArrayBuilderPtr = std::shared_ptr<::arrow::ArrayBuilder>;
-    using Ptr = std::shared_ptr<DialectGenericListWriterInterface>;
-
-    virtual void writeItems( const csp::DialectGenericType &listObject ) = 0;
-    virtual ~DialectGenericListWriterInterface() = default;
-};
-
-template< typename T >
-class TypedDialectGenericListWriterInterface : public DialectGenericListWriterInterface
-{
-public:
-    using WriteFuncT = std::function<void( const T & )>;
-
-    TypedDialectGenericListWriterInterface()
-    {
-        m_writeFunc = []( const T & )
-        {
-            CSP_THROW( RuntimeException, "Write function for TypedDialectGenericListWriterInterface is not set" );
-        };
-    }
-
-    void setWriteFunction( const WriteFuncT &func )
-    {
-        m_writeFunc = func;
-    }
-
-
-    void writeValue( const T &value )
-    {
-        m_writeFunc( value );
-    }
-
-private:
-    std::function<void( const T & )> m_writeFunc;
-};
-
-
-}
-
-#endif
\ No newline at end of file
diff --git a/cpp/csp/adapters/parquet/FileNameGeneratorReplicator.h b/cpp/csp/adapters/parquet/FileNameGeneratorReplicator.h
new file mode 100644
index 000000000..fdbb53cf2
--- /dev/null
+++ b/cpp/csp/adapters/parquet/FileNameGeneratorReplicator.h
@@ -0,0 +1,95 @@
+#ifndef _IN_CSP_ADAPTERS_PARQUET_FileNameGeneratorReplicator_H
+#define _IN_CSP_ADAPTERS_PARQUET_FileNameGeneratorReplicator_H
+
+#include <csp/core/Generator.h>
+#include <csp/core/Time.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace csp::adapters::parquet
+{
+
+/**
+ * Consumes a string generator (producing folder/file names) and caches the results so they can be
+ * replayed by multiple child generators.  Each replica can optionally append a suffix.
+ */
+class FileNameGeneratorReplicator
+{
+public:
+    using Ptr = std::shared_ptr<FileNameGeneratorReplicator>;
+    using GeneratorPtr = csp::Generator<std::string, csp::DateTime, csp::DateTime>::Ptr;
+
+    FileNameGeneratorReplicator( GeneratorPtr source )
+            : m_generatorPtr( source )
+    {
+    }
+
+    void init( csp::DateTime start, csp::DateTime end )
+    {
+        m_generatorPtr -> init( start, end );
+    }
+
+    const std::vector<std::string> &getFileNames(){ return m_fileNames; }
+
+    void consumeNextGeneratorFile()
+    {
+        std::string nextFile;
+        if( m_generatorPtr -> next( nextFile ) )
+        {
+            m_fileNames.push_back( std::move( nextFile ) );
+        }
+    }
+
+    GeneratorPtr getGeneratorReplica( const std::string &suffix = "" )
+    {
+        return std::make_shared<ChildGenerator>( *this, suffix );
+    }
+
+private:
+    class ChildGenerator : public csp::Generator<std::string, csp::DateTime, csp::DateTime>
+    {
+    public:
+        ChildGenerator( FileNameGeneratorReplicator &owner, const std::string &suffix )
+                : m_owner( owner ), m_suffix( suffix ), m_nextIndex( 0 )
+        {
+        }
+
+        void init( csp::DateTime, csp::DateTime ){}
+
+        bool next( std::string &value )
+        {
+            if( m_nextIndex < 0 )
+            {
+                return false;
+            }
+            const std::vector<std::string> &folders = m_owner.getFileNames();
+            if( m_nextIndex >= static_cast<int>(folders.size()) )
+            {
+                m_owner.consumeNextGeneratorFile();
+            }
+            if( m_nextIndex >= static_cast<int>(folders.size() ) )
+            {
+                m_nextIndex = -1;
+                return false;
+            }
+
+            value = folders[ m_nextIndex++ ] + m_suffix;
+            return true;
+        }
+
+    private:
+        FileNameGeneratorReplicator &m_owner;
+        const std::string           m_suffix;
+        int                         m_nextIndex = 0;
+    };
+
+private:
+    GeneratorPtr             m_generatorPtr;
+    std::vector<std::string> m_fileNames;
+};
+
+} // namespace csp::adapters::parquet
+
+#endif
diff --git a/cpp/csp/adapters/parquet/FileWriterWrapper.cpp b/cpp/csp/adapters/parquet/FileWriterWrapper.cpp
deleted file mode 100644
index 02e846311..000000000
--- a/cpp/csp/adapters/parquet/FileWriterWrapper.cpp
+++ /dev/null
@@ -1,49 +0,0 @@
-#include <csp/adapters/parquet/FileWriterWrapper.h>
-#include <csp/adapters/parquet/ParquetStatusUtils.h>
-#include <csp/core/Exception.h>
-#include <csp/core/FileUtils.h>
-#include <string>
-#include <filesystem>
-
-namespace csp::adapters::parquet
-{
-std::unordered_map<std::string, ::arrow::Compression::type> FileWriterWrapper::m_compressionNameMapping{
-        { "",          ::arrow::Compression::UNCOMPRESSED },
-        { "snappy",    ::arrow::Compression::SNAPPY },
-        { "gzip",      ::arrow::Compression::GZIP },
-        { "brotli",    ::arrow::Compression::BROTLI },
-        { "zstd",      ::arrow::Compression::ZSTD },
-        { "lz4",       ::arrow::Compression::LZ4 },
-        { "lz4_frame", ::arrow::Compression::LZ4_FRAME },
-        { "lzo",       ::arrow::Compression::LZO },
-        { "bz2",       ::arrow::Compression::BZ2 }
-};
-
-void FileWriterWrapper::open( const std::string &fileName, const std::string &compression, bool allowOverwrite )
-{
-    if( !allowOverwrite )
-    {
-        CSP_TRUE_OR_THROW_RUNTIME( !utils::fileExists( fileName ),
-                                   "Trying to overwrite existing file " << fileName << " while allow_overwrite is false" );
-    }
-
-    utils::mkdir( utils::dirname( fileName ) );
-    openImpl( fileName, compression );
-}
-
-::arrow::Compression::type FileWriterWrapper::resolveCompression( const std::string &compression )
-{
-    auto it = m_compressionNameMapping.find( compression );
-    CSP_TRUE_OR_THROW_RUNTIME( it != m_compressionNameMapping.end(), "Unable to resolve compression: " << compression );
-    return it->second;
-}
-
-std::unique_ptr<arrow::util::Codec> FileWriterWrapper::resolveCompressionCodec( const std::string &compression )
-{
-    auto compressionType = resolveCompression( compression );
-    auto res = arrow::util::Codec::Create( compressionType );
-    STATUS_OK_OR_THROW_RUNTIME(res.status(), "Failed to create arrow codec for " << compressionType );
-    return std::move(res.ValueUnsafe());
-}
-
-}
diff --git a/cpp/csp/adapters/parquet/FileWriterWrapper.h b/cpp/csp/adapters/parquet/FileWriterWrapper.h
deleted file mode 100644
index 07f0bcdef..000000000
--- a/cpp/csp/adapters/parquet/FileWriterWrapper.h
+++ /dev/null
@@ -1,54 +0,0 @@
-#ifndef _IN_CSP_ADAPTERS_PARQUET_FileWriterWrapper_H
-#define _IN_CSP_ADAPTERS_PARQUET_FileWriterWrapper_H
-
-#include <unordered_map>
-#include <string>
-#include <memory>
-#include <arrow/util/compression.h>
-
-namespace arrow
-{
-class Schema;
-
-class Table;
-}
-
-namespace csp::adapters::parquet
-{
-
-class FileWriterWrapper
-{
-public:
-    FileWriterWrapper( std::shared_ptr<::arrow::Schema> schema )
-            : m_schema( schema )
-    {
-    }
-
-    FileWriterWrapper( const FileWriterWrapper &other ) = delete;
-    FileWriterWrapper &operator=( const FileWriterWrapper &other ) = delete;
-
-    virtual ~FileWriterWrapper() {}
-
-    void open( const std::string &fileName, const std::string &compression, bool allowOverwrite = false );
-    virtual void close() = 0;
-    virtual void writeTable( const std::shared_ptr<::arrow::Table> &table ) = 0;
-
-    const std::shared_ptr<::arrow::Schema> &getSchema() const { return m_schema; }
-
-protected:
-    virtual void openImpl( const std::string &fileName, const std::string &compression ) = 0;
-
-
-    static ::arrow::Compression::type resolveCompression( const std::string &compression );
-    static std::unique_ptr<::arrow::util::Codec> resolveCompressionCodec( const std::string &compression );
-
-private:
-    std::shared_ptr<::arrow::Schema>                                   m_schema;
-    static std::unordered_map<std::string, ::arrow::Compression::type> m_compressionNameMapping;
-
-};
-
-}
-
-
-#endif
diff --git a/cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp b/cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp
deleted file mode 100644
index 9fe5b6801..000000000
--- a/cpp/csp/adapters/parquet/FileWriterWrapperContainer.cpp
+++ /dev/null
@@ -1,116 +0,0 @@
-#include <csp/adapters/parquet/FileWriterWrapperContainer.h>
-#include <csp/adapters/parquet/ParquetWriter.h>
-#include <csp/adapters/parquet/ArrowIPCFileWriterWrapper.h>
-#include <csp/adapters/parquet/ParquetFileWriterWrapper.h>
-#include <csp/adapters/parquet/ParquetOutputAdapter.h>
-#include <csp/adapters/parquet/ParquetStatusUtils.h>
-#include <arrow/array.h>
-#include <arrow/builder.h>
-#include <arrow/io/file.h>
-#include <arrow/table.h>
-#include <parquet/arrow/writer.h>
-#include <csp/core/Exception.h>
-
-namespace csp::adapters::parquet
-{
-
-FileWriterWrapperContainer::WriterPtr FileWriterWrapperContainer::createSingleFileWrapper( const std::shared_ptr<arrow::Schema> &schema,
-                                                                                           bool isWriteArrowBinary )
-{
-    if( isWriteArrowBinary )
-    {
-        return std::make_unique<ArrowIPCFileWriterWrapper>( schema );
-    }
-    else
-    {
-        return std::make_unique<ParquetFileWriterWrapper>( schema );
-    }
-}
-
-SingleFileWriterWrapperContainer::SingleFileWriterWrapperContainer( std::shared_ptr<arrow::Schema> schema, bool isWriteArrowBinary )
-        : m_fileWriterWrapper( createSingleFileWrapper( schema, isWriteArrowBinary ) )
-{
-}
-
-void SingleFileWriterWrapperContainer::SingleFileWriterWrapperContainer::open( const std::string &fileName, const std::string &compression,
-                                                                               bool allowOverwrite )
-{
-    m_fileWriterWrapper -> open( fileName, compression, allowOverwrite );
-    setOpen(true);
-}
-
-void SingleFileWriterWrapperContainer::close()
-{
-    m_fileWriterWrapper -> close();
-    setOpen(false);
-}
-
-void SingleFileWriterWrapperContainer::writeData( const std::vector<std::shared_ptr<ArrowSingleColumnArrayBuilder>> &columnBuilders )
-{
-    std::vector<std::shared_ptr<arrow::Array>> columns;
-    columns.reserve( columnBuilders.size() );
-    for( auto &&columnBuilder:columnBuilders )
-    {
-        columns.push_back( columnBuilder -> buildArray() );
-    }
-
-    auto table = arrow::Table::Make( m_fileWriterWrapper -> getSchema(), columns );
-
-    m_fileWriterWrapper -> writeTable( table );
-}
-
-
-MultipleFileWriterWrapperContainer::MultipleFileWriterWrapperContainer( std::shared_ptr<arrow::Schema> schema, bool isWriteArrowBinary )
-{
-    auto &fields = schema -> fields();
-    m_fileWriterWrappers.reserve( fields.size() );
-    for( auto &&field:fields )
-    {
-        std::vector<std::shared_ptr<arrow::Field>> curFields;
-        std::string                                fileExtension = isWriteArrowBinary ? ".arrow" : ".parquet";
-        std::string                                fileName      = field -> name() + fileExtension;
-
-        m_fileWriterWrappers.push_back( { fileName,
-                                          createSingleFileWrapper( arrow::schema( { field } ), isWriteArrowBinary ) } );
-    }
-}
-
-void MultipleFileWriterWrapperContainer::open( const std::string &fileName, const std::string &compression, bool allowOverwrite )
-{
-    for( auto &&record : m_fileWriterWrappers )
-    {
-        record.m_fileWriterWrapper -> open( fileName + '/' + record.m_fileName,
-                                              compression,
-                                              allowOverwrite );
-
-    }
-    setOpen(true);
-}
-
-void MultipleFileWriterWrapperContainer::close()
-{
-    for( auto &&record : m_fileWriterWrappers )
-    {
-        record.m_fileWriterWrapper -> close();
-    }
-    setOpen(false);
-}
-
-void MultipleFileWriterWrapperContainer::writeData( const std::vector<std::shared_ptr<ArrowSingleColumnArrayBuilder>> &columnBuilders )
-{
-    std::vector<std::shared_ptr<arrow::Array>> columns;
-    columns.reserve( 1 );
-    CSP_TRUE_OR_THROW_RUNTIME( columnBuilders.size() == m_fileWriterWrappers.size(),
-                               "Internal error - column builders and file wrappers are expected to have same size" );
-
-    for( unsigned i = 0; i < columnBuilders.size(); ++i )
-    {
-        columns.clear();
-        columns.push_back( columnBuilders[ i ] -> buildArray() );
-        auto &fileWriterWrapper = m_fileWriterWrappers[ i ].m_fileWriterWrapper;
-        auto table              = arrow::Table::Make( fileWriterWrapper -> getSchema(), columns );
-        fileWriterWrapper -> writeTable( table );
-    }
-}
-
-}
\ No newline at end of file
diff --git a/cpp/csp/adapters/parquet/FileWriterWrapperContainer.h b/cpp/csp/adapters/parquet/FileWriterWrapperContainer.h
deleted file mode 100644
index 183d92cad..000000000
--- a/cpp/csp/adapters/parquet/FileWriterWrapperContainer.h
+++ /dev/null
@@ -1,74 +0,0 @@
-#ifndef _IN_CSP_ADAPTERS_PARQUET_FileWriterWrapperContainer_H
-#define _IN_CSP_ADAPTERS_PARQUET_FileWriterWrapperContainer_H
-
-#include <string>
-#include <vector>
-#include <memory>
-#include <csp/adapters/parquet/FileWriterWrapper.h>
-
-namespace arrow
-{
-class Schema;
-}
-
-namespace csp::adapters::parquet
-{
-class ArrowSingleColumnArrayBuilder;
-
-class FileWriterWrapperContainer
-{
-public:
-    FileWriterWrapperContainer() = default;
-    FileWriterWrapperContainer( FileWriterWrapperContainer & ) = delete;
-    FileWriterWrapperContainer &operator=( const FileWriterWrapperContainer & ) = delete;
-    virtual ~FileWriterWrapperContainer() = default;
-
-    virtual void open( const std::string &fileName, const std::string &compression, bool allowOverwrite = false ) = 0;
-
-    bool isOpen() const{ return m_isOpen; }
-
-    virtual void close() = 0;
-    virtual void writeData( const std::vector<std::shared_ptr<ArrowSingleColumnArrayBuilder>> &columnBuilders ) = 0;
-protected:
-    void setOpen( bool open ){ m_isOpen = open; }
-
-protected:
-    using WriterPtr = std::unique_ptr<FileWriterWrapper>;
-
-    static WriterPtr createSingleFileWrapper( const std::shared_ptr<::arrow::Schema> &schema, bool isWriteArrowBinary );
-private:
-    bool m_isOpen = false;
-};
-
-class SingleFileWriterWrapperContainer final : public FileWriterWrapperContainer
-{
-public:
-    SingleFileWriterWrapperContainer( std::shared_ptr<::arrow::Schema> schema, bool isWriteArrowBinary );
-    virtual void open( const std::string &fileName, const std::string &compression, bool allowOverwrite = false ) override;
-    virtual void close() override;
-    virtual void writeData( const std::vector<std::shared_ptr<ArrowSingleColumnArrayBuilder>> &columnBuilders ) override;
-private:
-    std::unique_ptr<FileWriterWrapper> m_fileWriterWrapper;
-};
-
-class MultipleFileWriterWrapperContainer : public FileWriterWrapperContainer
-{
-public:
-    MultipleFileWriterWrapperContainer( std::shared_ptr<::arrow::Schema> schema, bool isWriteArrowBinary );
-    virtual void open( const std::string &fileName, const std::string &compression, bool allowOverwrite = false ) override;
-    virtual void close() override;
-    virtual void writeData( const std::vector<std::shared_ptr<ArrowSingleColumnArrayBuilder>> &columnBuilders ) override;
-private:
-    struct SingleFileWrapperInfo
-    {
-        std::string                        m_fileName;
-        std::unique_ptr<FileWriterWrapper> m_fileWriterWrapper;
-    };
-
-private:
-    std::vector<SingleFileWrapperInfo> m_fileWriterWrappers;
-};
-
-}
-
-#endif
diff --git a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
index 325dd2497..908b09a6e 100644
--- a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
@@ -1,6 +1,6 @@
 #include <csp/adapters/parquet/ParquetDictBasketOutputWriter.h>
 #include <csp/adapters/parquet/ParquetOutputAdapter.h>
-#include <parquet/arrow/writer.h>
+#include <arrow/record_batch.h>
 
 namespace csp::adapters::parquet
 {
@@ -17,28 +17,27 @@ ParquetDictBasketOutputWriter::ParquetDictBasketOutputWriter(
 void ParquetDictBasketOutputWriter::start()
 {
     ParquetWriter::start();
-    m_indexFileWriterContainer = std::make_unique<MultipleFileWriterWrapperContainer>(
-            arrow::schema( { arrow::field( m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getColumnName(),
-                                           m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getDataType() ) } ),
-            m_adapterMgr.isWriteArrowBinary() );
-    if( !m_adapterMgr.getFileName().empty() )
-    {
-        m_indexFileWriterContainer -> open( m_adapterMgr.getFileName(),
-                                            m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() );
-
-    }
+    m_indexSchema = ::arrow::schema( { ::arrow::field(
+        m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getColumnName(),
+        m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getDataType() ) } );
+    if( m_indexSink.onStart )
+        m_indexSink.onStart( m_indexSchema );
+    auto & fileName = m_adapterMgr.getFileName();
+    if( !fileName.empty() && m_indexSink.onFileChange )
+        m_indexSink.onFileChange( fileName );
 }
 
 void ParquetDictBasketOutputWriter::stop()
 {
-    auto &&indexColumnArrayBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 );
-    if( indexColumnArrayBuilder -> length() > 0 )
+    auto && indexBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 );
+    if( indexBuilder -> length() > 0 && m_indexSink.onBatch )
     {
-        CSP_TRUE_OR_THROW_RUNTIME(isFileOpen(), "On stop ParquetDictBasketOutputWriter has data to write but no open file");
-        m_indexFileWriterContainer -> writeData( { indexColumnArrayBuilder } );
+        auto arr = indexBuilder -> buildArray();
+        auto rb  = ::arrow::RecordBatch::Make( m_indexSchema, arr -> length(), { arr } );
+        m_indexSink.onBatch( rb );
     }
-    m_indexFileWriterContainer -> close();
-    m_indexFileWriterContainer = nullptr;
+    if( m_indexSink.onStop )
+        m_indexSink.onStop();
 
     ParquetWriter::stop();
 }
@@ -46,7 +45,7 @@ void ParquetDictBasketOutputWriter::stop()
 void ParquetDictBasketOutputWriter::writeValue( const std::string &valueKey, const TimeSeriesProvider *ts )
 {
     m_adapterMgr.scheduleEndCycle();
-    m_symbolOutputAdapter -> writeValue<std::string, StringArrayBuilder>( valueKey );
+    m_symbolOutputAdapter -> writeValue<std::string>( valueKey );
     ParquetWriter::onEndCycle();
     ++m_nextCycleIndex;
 }
@@ -56,12 +55,14 @@ void ParquetDictBasketOutputWriter::onEndCycle()
 {
     if(isFileOpen())
     {
-        m_cycleIndexOutputAdapter -> writeValue<std::uint16_t, UInt16ArrayBuilder>( m_nextCycleIndex );
-        auto &&indexColumnArrayBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 );
-        indexColumnArrayBuilder -> handleRowFinished();
-        if( indexColumnArrayBuilder -> length() >= getChunkSize() )
+        m_cycleIndexOutputAdapter -> writeValue<std::uint16_t>( m_nextCycleIndex );
+        auto && indexBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 );
+        indexBuilder -> handleRowFinished();
+        if( indexBuilder -> length() >= getChunkSize() && m_indexSink.onBatch )
         {
-            m_indexFileWriterContainer -> writeData( { indexColumnArrayBuilder } );
+            auto arr = indexBuilder -> buildArray();
+            auto rb  = ::arrow::RecordBatch::Make( m_indexSchema, arr -> length(), { arr } );
+            m_indexSink.onBatch( rb );
         }
         m_nextCycleIndex = 0;
     }
@@ -74,21 +75,16 @@ void ParquetDictBasketOutputWriter::onEndCycle()
 void ParquetDictBasketOutputWriter::onFileNameChange( const std::string &fileName )
 {
     ParquetWriter::onFileNameChange( fileName );
-    if( m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> length() > 0 )
-    {
-        CSP_TRUE_OR_THROW_RUNTIME( m_indexFileWriterContainer -> isOpen(), "Trying to write basket index data to closed file" );
-        m_indexFileWriterContainer -> writeData( { m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) } );
-    }
-    if (m_indexFileWriterContainer->isOpen())
-    {
-        m_indexFileWriterContainer->close();
-    }
-    if(!fileName.empty())
+    // Flush any pending index data
+    auto && indexBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 );
+    if( indexBuilder -> length() > 0 && m_indexSink.onBatch )
     {
-        m_indexFileWriterContainer
-                -> open( fileName, m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() );
+        auto arr = indexBuilder -> buildArray();
+        auto rb  = ::arrow::RecordBatch::Make( m_indexSchema, arr -> length(), { arr } );
+        m_indexSink.onBatch( rb );
     }
-
+    if( m_indexSink.onFileChange )
+        m_indexSink.onFileChange( fileName );
 }
 
 SingleColumnParquetOutputHandler *ParquetDictBasketOutputWriter::createScalarOutputHandler( CspTypePtr type, const std::string &name )
diff --git a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h
index 5b23dbee2..7362edb17 100644
--- a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h
+++ b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h
@@ -1,10 +1,8 @@
 #ifndef _IN_CSP_ADAPTERS_PARQUET_ParquetDictBasketOutputHandler_H
 #define _IN_CSP_ADAPTERS_PARQUET_ParquetDictBasketOutputHandler_H
 
-#include <csp/adapters/parquet/FileWriterWrapperContainer.h>
 #include <csp/adapters/parquet/ParquetWriter.h>
 #include <csp/engine/CspType.h>
-#include <parquet/arrow/writer.h>
 #include <string>
 
 namespace csp::adapters::parquet
@@ -18,6 +16,8 @@ class ParquetDictBasketOutputWriter : public ParquetWriter
     void start() override;
     void stop() override;
 
+    void setIndexSink( RecordBatchSink sink ) { m_indexSink = std::move( sink ); }
+
     virtual void writeValue( const std::string &valueKey, const TimeSeriesProvider *ts );
 
     void onEndCycle() override final;
@@ -31,7 +31,8 @@ class ParquetDictBasketOutputWriter : public ParquetWriter
     SingleColumnParquetOutputHandler                    *m_cycleIndexOutputAdapter;
     std::uint16_t                                       m_nextCycleIndex;
     std::vector<std::unique_ptr<ParquetOutputHandler>>  m_allHandlers;
-    std::unique_ptr<MultipleFileWriterWrapperContainer> m_indexFileWriterContainer;
+    RecordBatchSink                                     m_indexSink;
+    std::shared_ptr<::arrow::Schema>                    m_indexSchema;
 };
 
 class ParquetScalarDictBasketOutputWriter final : public ParquetDictBasketOutputWriter
diff --git a/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp b/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp
deleted file mode 100644
index 633ac4975..000000000
--- a/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.cpp
+++ /dev/null
@@ -1,57 +0,0 @@
-#include <csp/adapters/parquet/ParquetFileWriterWrapper.h>
-#include <csp/adapters/parquet/ParquetStatusUtils.h>
-#include <arrow/io/file.h>
-#include <arrow/table.h>
-#include <arrow/util/config.h>
-#include <parquet/arrow/writer.h>
-
-namespace csp::adapters::parquet
-{
-void ParquetFileWriterWrapper::openImpl( const std::string &fileName, const std::string &compression )
-{
-    CSP_TRUE_OR_THROW_RUNTIME( m_outputStream == nullptr, "Trying to open parquet file while previous was not closed" );
-
-    PARQUET_ASSIGN_OR_THROW(
-            m_outputStream,
-            arrow::io::FileOutputStream::Open( fileName.c_str()));
-
-    ::parquet::WriterProperties::Builder builder;
-    builder.compression( resolveCompression( compression ));
-#if ARROW_VERSION_MAJOR >= 20
-    builder.version(::parquet::ParquetVersion::PARQUET_2_6 );
-#else
-    builder.version(::parquet::ParquetVersion::PARQUET_2_0 );
-#endif
-    ::parquet::ArrowWriterProperties::Builder arrowBuilder;
-    arrowBuilder.store_schema();
-
-    auto res = ::parquet::arrow::FileWriter::Open( *getSchema(), arrow::default_memory_pool(), m_outputStream, builder.build(), arrowBuilder.build() );
-    STATUS_OK_OR_THROW_RUNTIME( res.status(), "Failed to open parquet file writer" );
-    m_fileWriter = res.MoveValueUnsafe();
-}
-
-void ParquetFileWriterWrapper::close()
-{
-    if( m_outputStream )
-    {
-        // Let's move them first, if there are any exceptions, we still want the pointer to be null
-        std::shared_ptr<::arrow::io::FileOutputStream> outputStream{ std::move( m_outputStream ) };
-        std::unique_ptr<::parquet::arrow::FileWriter>  fileWriter{ std::move( m_fileWriter ) };
-        // Should be done by move constructor but let's be safe:
-        m_outputStream = nullptr;
-        m_fileWriter   = nullptr;
-
-        if(fileWriter)
-            STATUS_OK_OR_THROW_RUNTIME( fileWriter->Close(), "Failed to close parquet file writer" );
-        if(outputStream)
-            STATUS_OK_OR_THROW_RUNTIME( outputStream->Close(), "Failed to close parquet output stream" );
-    }
-}
-
-void ParquetFileWriterWrapper::writeTable( const std::shared_ptr<::arrow::Table> &table )
-{
-    CSP_TRUE_OR_THROW_RUNTIME(m_fileWriter, "File writer is null!!!");
-    STATUS_OK_OR_THROW_RUNTIME( m_fileWriter->WriteTable( *table, table->num_rows()), "Failed to write to parquet file" );
-}
-
-}
diff --git a/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.h b/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.h
deleted file mode 100644
index b73f53456..000000000
--- a/cpp/csp/adapters/parquet/ParquetFileWriterWrapper.h
+++ /dev/null
@@ -1,43 +0,0 @@
-#ifndef _IN_CSP_ADAPTERS_PARQUET_ParquetFileWriterWrapper_H
-#define _IN_CSP_ADAPTERS_PARQUET_ParquetFileWriterWrapper_H
-
-#include <csp/adapters/parquet/FileWriterWrapper.h>
-#include <string>
-
-namespace arrow::io
-{
-class FileOutputStream;
-}
-namespace parquet::arrow
-{
-class FileWriter;
-}
-
-namespace csp::adapters::parquet
-{
-
-class ParquetFileWriterWrapper : public FileWriterWrapper
-{
-public:
-    ParquetFileWriterWrapper( std::shared_ptr<::arrow::Schema> schema )
-            : FileWriterWrapper( schema )
-    {
-    }
-
-    ~ParquetFileWriterWrapper() { close(); }
-
-    void close() override;
-    void writeTable( const std::shared_ptr<::arrow::Table> &table ) override;
-
-protected:
-    void openImpl( const std::string &fileName, const std::string &compression ) override;
-
-private:
-    std::shared_ptr<::arrow::io::FileOutputStream> m_outputStream;
-    std::unique_ptr<::parquet::arrow::FileWriter>  m_fileWriter;
-};
-
-}
-
-
-#endif
\ No newline at end of file
diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp
index f73092254..0df2efadc 100644
--- a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp
@@ -1,5 +1,7 @@
 #include <csp/adapters/parquet/ParquetOutputAdapter.h>
+#include <csp/adapters/parquet/ArrowBackedArrayBuilder.h>
 #include <csp/adapters/parquet/ParquetWriter.h>
+#include <csp/adapters/arrow/ArrowTypeVisitor.h>
 
 using namespace csp;
 using namespace csp::adapters::parquet;
@@ -17,115 +19,57 @@ SingleColumnParquetOutputHandler::SingleColumnParquetOutputHandler( Engine *engi
                                                                     std::string columnName )
         : ParquetOutputHandler( parquetWriter, type )
 {
-    switch( m_type -> type() )
+    bool isBytes = false;
+    CspTypePtr effectiveType = type;
+
+    if( type -> type() == CspType::TypeTraits::STRING )
     {
-        case CspType::TypeTraits::BOOL:
-            createColumnBuilder<BoolArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::INT8:
-            createColumnBuilder<Int8ArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::UINT8:
-            createColumnBuilder<UInt8ArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::INT16:
-            createColumnBuilder<Int16ArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::UINT16:
-            createColumnBuilder<UInt16ArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::INT32:
-            createColumnBuilder<Int32ArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::UINT32:
-            createColumnBuilder<UInt32ArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::INT64:
-            createColumnBuilder<Int64ArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::UINT64:
-            createColumnBuilder<UInt64ArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::DOUBLE:
-            createColumnBuilder<DoubleArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::DATETIME:
-            createColumnBuilder<DatetimeArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::TIMEDELTA:
-            createColumnBuilder<TimedeltaArrayBuilder>( columnName );
-            break;
-        case CspType::TypeTraits::DATE:
-            createColumnBuilder<DateArrayBuilder>( columnName );
-            break;
-       case CspType::TypeTraits::TIME:
-           createColumnBuilder<TimeArrayBuilder>( columnName );
-           break;
+        isBytes = static_cast<const CspStringType &>( *type ).isBytes();
+    }
 
-        case CspType::TypeTraits::STRING:
-        {
-            const CspStringType &strType = static_cast<const CspStringType &>(*type);
-            if( strType.isBytes() )
-            {
-                createColumnBuilder<BytesArrayBuilder>( columnName );
-            }
-            else
-            {
-                createColumnBuilder<StringArrayBuilder>( columnName );
-            }
-            break;
-        }
-        case CspType::TypeTraits::ENUM:
+    auto arrowBuilder = createArrowBackedArrayBuilder( columnName, getChunkSize(), effectiveType, isBytes );
+    auto * scratch    = arrowBuilder -> scratch();
+    auto   field      = arrowBuilder -> scratchField();
+    m_columnArrayBuilder = arrowBuilder;
+
+    csp::adapters::arrow::visitCspValueType( type -> type(),
+        [&]( auto tag )
         {
-            auto enumMetaPtr = std::static_pointer_cast<const CspEnumType>( type ) -> meta();
-            createEnumColumnBuilder( columnName, enumMetaPtr );
-            break;
-        }
-        default:
+            using T = typename decltype( tag )::type;
+            createValueHandler<T>( scratch, field );
+        },
+        [&]()
         {
             CSP_THROW( TypeError, "Writing of " << m_type -> type().asString() << " to parquet is not supported" );
-        }
-    }
+        } );
 }
 
-
-template< typename ColumnBuilder >
-inline void SingleColumnParquetOutputHandler::createColumnBuilder( const std::string &columnName )
+void SingleColumnParquetOutputAdapter::executeImpl()
 {
-    m_columnArrayBuilder = std::make_unique<ColumnBuilder>( columnName, getChunkSize() );
-    m_valueHandler       = std::make_unique<ValueHandler>(
-            [ this ]( const TimeSeriesProvider *input )
-            {
-                static_cast<ColumnBuilder *>(this -> m_columnArrayBuilder.get())
-                        -> setValue( input -> lastValueTyped<typename ColumnBuilder::ValueTypeT>() );
-            } );
+    ( *m_valueHandler )( input() );
+    m_parquetWriter.scheduleEndCycleEvent();
 }
 
-void SingleColumnParquetOutputHandler::createEnumColumnBuilder( const std::string &columnName, CspEnumMeta::Ptr enumMetaPtr )
+// writeValue: used by dict basket writers to set values directly
+template< typename T, typename Ignored >
+void SingleColumnParquetOutputHandler::writeValue( const T & value )
 {
-    m_columnArrayBuilder = std::make_unique<StringArrayBuilder>( columnName, getChunkSize() );
-    m_valueHandler       = std::make_unique<ValueHandler>(
-            [ this ]( const TimeSeriesProvider *input )
-            {
-                static_cast<StringArrayBuilder *>(this -> m_columnArrayBuilder.get())
-                        -> setValue( input -> lastValueTyped<CspEnum>().name() );
-            } );
+    auto * builder = static_cast<ArrowBackedArrayBuilder *>( m_columnArrayBuilder.get() );
+    builder -> scratchField() -> setValue<T>( builder -> scratch(), value );
 }
 
-void SingleColumnParquetOutputAdapter::executeImpl()
-{
-    ( *m_valueHandler )( input() );
-    m_parquetWriter.scheduleEndCycleEvent();
-}
+// Explicit instantiations for types used by ParquetDictBasketOutputWriter
+template void SingleColumnParquetOutputHandler::writeValue<std::string, void>( const std::string & );
+template void SingleColumnParquetOutputHandler::writeValue<std::uint16_t, void>( const std::uint16_t & );
 
 ListColumnParquetOutputHandler::ListColumnParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &elemType,
-                                                                const std::string &columnName,
-                                                                DialectGenericListWriterInterface::Ptr &listWriterInterface )
-        : ParquetOutputHandler( parquetWriter, CspType::DIALECT_GENERIC() ),
-          m_columnArrayBuilder(
-                  std::make_shared<ListColumnArrayBuilder>( columnName, getChunkSize(), createValueBuilder( elemType, listWriterInterface ),
-                                                            listWriterInterface ) )
+                                                                const std::string &columnName )
+        : ParquetOutputHandler( parquetWriter, CspType::DIALECT_GENERIC() )
 {
+    auto [ valueBuilder, writeItemsFn ] = csp::adapters::arrow::createListFieldWriter( elemType );
+    m_columnArrayBuilder = std::make_shared<ListColumnArrayBuilder>(
+        columnName, getChunkSize(), std::move( valueBuilder ), std::move( writeItemsFn ) );
+
     m_valueHandler = std::make_unique<ValueHandler>(
             [ this ]( const TimeSeriesProvider *input )
             {
@@ -134,46 +78,6 @@ ListColumnParquetOutputHandler::ListColumnParquetOutputHandler( Engine *engine,
             } );
 }
 
-namespace
-{
-template< typename A, typename V = typename A::value_type >
-inline std::shared_ptr<::arrow::ArrayBuilder> makeArrayAndAttachToWriter( DialectGenericListWriterInterface::Ptr &listWriterInterface )
-{
-    auto&& typedWriter = std::dynamic_pointer_cast<TypedDialectGenericListWriterInterface<V>>( listWriterInterface );
-    auto& listWriterInterfaceRef = *listWriterInterface;
-    CSP_TRUE_OR_THROW( typedWriter != nullptr, TypeError,
-                       "Expected " << typeid( TypedDialectGenericListWriterInterface<V> ).name() << " " << " got " <<
-                                   typeid( listWriterInterfaceRef ).name() );
-
-    auto res = std::make_shared<A>();
-    typedWriter -> setWriteFunction(
-            [ res ]( const V &value ){ STATUS_OK_OR_THROW_RUNTIME( res -> Append( value ), "Failed to append value to list array" ); } );
-    return res;
-}
-
-}
-
-std::shared_ptr<::arrow::ArrayBuilder> ListColumnParquetOutputHandler::createValueBuilder( const CspTypePtr &elemType,
-                                                                                           DialectGenericListWriterInterface::Ptr &listWriterInterface )
-{
-    switch( elemType -> type() )
-    {
-        case CspType::TypeTraits::BOOL:
-            return makeArrayAndAttachToWriter<arrow::BooleanBuilder>(listWriterInterface);
-        case CspType::TypeTraits::INT64:
-            return makeArrayAndAttachToWriter<arrow::Int64Builder>(listWriterInterface);
-        case CspType::TypeTraits::DOUBLE:
-            return makeArrayAndAttachToWriter<arrow::DoubleBuilder>(listWriterInterface);
-        case CspType::TypeTraits::STRING:
-            return makeArrayAndAttachToWriter<arrow::StringBuilder, std::string>(listWriterInterface);
-        default:
-        {
-            CSP_THROW( TypeError,
-                       "Writing of list with elements of type " << elemType -> type().asString() << " to parquet is not supported" );
-        }
-    }
-}
-
 void ListColumnParquetOutputAdapter::executeImpl()
 {
     ( *m_valueHandler )( input() );
@@ -188,177 +92,31 @@ StructParquetOutputHandler::StructParquetOutputHandler( Engine *engine, ParquetW
 
     for( auto it = fieldMap -> begin(); it != fieldMap -> end(); ++it )
     {
-        createColumnBuilder( structMetaPtr, it.value<std::string>(), it.key(), nullptr );
-    }
-}
+        auto structFieldName = it.key();
+        auto columnName      = it.value<std::string>();
+        auto structField     = structMetaPtr -> field( structFieldName );
 
-void StructParquetOutputHandler::writeValueFromTs( const TimeSeriesProvider *input )
-{
-    const Struct *structData = input -> lastValueTyped<StructPtr>().get();
-
-    for( auto &&valueHandler: m_valueHandlers )
-    {
-        valueHandler( structData );
-    }
-    m_parquetWriter.scheduleEndCycleEvent();
-}
-
-inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createColumnBuilder(
-        const StructMeta *structMeta,
-        const std::string &columnName, const std::string &structFieldName,
-        const std::string *path )
-{
-    auto fieldPtr = structMeta -> field( structFieldName ).get();
-    switch( fieldPtr -> type() -> type() )
-    {
-        case CspType::TypeTraits::BOOL:
-            return createColumnBuilder<BoolArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::INT8:
-            return createColumnBuilder<Int8ArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::UINT8:
-            return createColumnBuilder<UInt8ArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::INT16:
-            return createColumnBuilder<Int16ArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::UINT16:
-            return createColumnBuilder<UInt16ArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::INT32:
-            return createColumnBuilder<Int32ArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::UINT32:
-            return createColumnBuilder<UInt32ArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::INT64:
-            return createColumnBuilder<Int64ArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::UINT64:
-            return createColumnBuilder<UInt64ArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::DOUBLE:
-            return createColumnBuilder<DoubleArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::DATETIME:
-            return createColumnBuilder<DatetimeArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::TIMEDELTA:
-            return createColumnBuilder<TimedeltaArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::DATE:
-            return createColumnBuilder<DateArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::TIME:
-            return createColumnBuilder<TimeArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::STRING:
-            return createColumnBuilder<StringArrayBuilder>( fieldPtr, columnName, path );
-        case CspType::TypeTraits::ENUM:
-            return createEnumColumnBuilder( fieldPtr, columnName, path );
-        case CspType::TypeTraits::STRUCT:
-            return createStructColumnBuilder( fieldPtr, columnName, path );
-        default:
-            CSP_THROW( TypeError, "Writing of column " << columnName << " of type " << fieldPtr -> type() -> type().asString()
-                                                       << " to parquet is not supported" );
-    }
-}
-
-template< typename ColumnBuilder >
-inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createColumnBuilder(
-        const StructField *field,
-        const std::string &columnName,
-        const std::string *path )
-{
-    std::shared_ptr<ColumnBuilder> columnBuilderPtr = std::make_shared<ColumnBuilder>( resolveFullColumnName( path, columnName ),
-                                                                                       getChunkSize() );
+        if( !structField )
+            CSP_THROW( ValueError, "Struct field '" << structFieldName << "' not found" );
 
-    auto columnBuilderRawPtr = columnBuilderPtr.get();
-    using T = typename ColumnBuilder::ValueTypeT;
+        if( structField -> type() -> type() == CspType::Type::DIALECT_GENERIC )
+            continue;
 
-    ValueHandler res = [ field, columnBuilderRawPtr ]( const Struct *s )
-    {
-        if( field -> isSet( s ) )
-        {
-            columnBuilderRawPtr -> setValue( field -> value<T>( s ) );
-        }
-    };
-    // We need to collect on the top level value handlers and array builders. Value handlers and builders of nested structs are stored
-    // in the struct field handlers. Path is non null in this case.
-    if( !path )
-    {
-        m_valueHandlers.push_back( res );
-        m_columnArrayBuilders.push_back( columnBuilderPtr );
+        auto builder = createArrowBackedArrayBuilderForField( columnName, getChunkSize(), structField );
+        m_arrowBuilders.push_back( builder.get() );
+        m_columnArrayBuilders.push_back( std::move( builder ) );
     }
-    return { columnBuilderPtr, res };
 }
 
-inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createEnumColumnBuilder(
-        const StructField *field,
-        const std::string &columnName,
-        const std::string *path )
-{
-    auto columnBuilderPtr{ std::make_shared<StringArrayBuilder>( resolveFullColumnName( path, columnName ), getChunkSize() ) };
-    auto columnBuilderRawPtr{ columnBuilderPtr.get() };
-    auto enumMetaPtr = std::static_pointer_cast<const CspEnumType>( field -> type() ) -> meta();
-
-    ValueHandler res = [ field, columnBuilderRawPtr ]( const Struct *s )
-    {
-        if( field -> isSet( s ) )
-        {
-            columnBuilderRawPtr -> setValue( field -> value<CspEnum>( s ).name() );
-        }
-    };
-    if( path == nullptr )
-    {
-        m_valueHandlers.push_back( res );
-        m_columnArrayBuilders.push_back( columnBuilderPtr );
-    }
-    return { columnBuilderPtr, res };
-}
-
-inline StructParquetOutputHandler::ColumnBuilderResultType StructParquetOutputHandler::createStructColumnBuilder(
-        const StructField *structField,
-        const std::string &columnName,
-        const std::string *path )
+void StructParquetOutputHandler::writeValueFromTs( const TimeSeriesProvider *input )
 {
-    std::vector<std::shared_ptr<::arrow::Field>>            fields;
-    std::vector<StructColumnArrayBuilder::ColumnBuilderPtr> childArrayBuilders;
-    std::vector<ValueHandler>                               childFieldSetters;
-
-    auto structFieldMetaPtr = std::static_pointer_cast<const CspStructType>( structField -> type() ) -> meta().get();
-
-    for( auto &subField:structFieldMetaPtr -> fields() )
-    {
-        std::string fieldSubPath = resolveFullColumnName( path, subField -> fieldname() );
-
-        auto childBuilderRes = createColumnBuilder( structFieldMetaPtr,
-                                                    subField -> fieldname(),
-                                                    subField -> fieldname(),
-                                                    &fieldSubPath );
-        childArrayBuilders.push_back( childBuilderRes.m_columnBuilder );
-        childFieldSetters.push_back( childBuilderRes.m_valueHandler );
-        fields.push_back(
-                std::make_shared<::arrow::Field>( subField -> fieldname(), childBuilderRes.m_columnBuilder -> getDataType() ) );
-    }
-
-    auto subFieldValueSetter = [ childFieldSetters ]( const Struct *s )
-    {
-        for( auto &childFieldSetter:childFieldSetters )
-        {
-            childFieldSetter( s );
-        }
-    };
-
-    auto columnBuilderPtr{ std::make_shared<StructColumnArrayBuilder>(
-            resolveFullColumnName( path, columnName ), getChunkSize(),
-            std::make_shared<::arrow::StructType>( fields ),
-            childArrayBuilders,
-            std::move( subFieldValueSetter )
-    ) };
-
-    auto columnBuilderRawPtr = columnBuilderPtr.get();
+    const Struct *structData = input -> lastValueTyped<StructPtr>().get();
 
-    ValueHandler valueHandler = [ structField, columnBuilderRawPtr ]( const Struct *s )
-    {
-        if( structField -> isSet( s ) )
-        {
-            columnBuilderRawPtr -> setValue( structField -> value<StructPtr>( s ).get() );
-        }
-    };
-    if( path == nullptr )
+    for( auto * builder : m_arrowBuilders )
     {
-        m_valueHandlers.push_back( valueHandler );
-        m_columnArrayBuilders.push_back( columnBuilderPtr );
+        builder -> setStruct( structData );
     }
-    return { columnBuilderPtr, valueHandler };
+    m_parquetWriter.scheduleEndCycleEvent();
 }
 
 void StructParquetOutputAdapter::executeImpl()
diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapter.h b/cpp/csp/adapters/parquet/ParquetOutputAdapter.h
index 18fdbee68..8c6e26e4f 100644
--- a/cpp/csp/adapters/parquet/ParquetOutputAdapter.h
+++ b/cpp/csp/adapters/parquet/ParquetOutputAdapter.h
@@ -5,7 +5,8 @@
 #include <csp/engine/Dictionary.h>
 #include <csp/engine/OutputAdapter.h>
 #include <string>
-#include <csp/adapters/parquet/DialectGenericListWriterInterface.h>
+
+namespace csp::adapters::parquet { class ArrowBackedArrayBuilder; }
 
 namespace arrow
 {
@@ -54,21 +55,25 @@ class SingleColumnParquetOutputHandler : public ParquetOutputHandler
 
     std::shared_ptr<ArrowSingleColumnArrayBuilder> getColumnArrayBuilder( unsigned index ) override{ return m_columnArrayBuilder; };
 
-    template< typename T, typename ColumnBuilderType >
-    void writeValue( const T &value )
-    {
-        static_cast<ColumnBuilderType *>(this -> m_columnArrayBuilder.get()) -> setValue( value );
-    }
-
     void writeValueFromTs( const TimeSeriesProvider *input ) override final
     {
         ( *m_valueHandler )( input );
     }
 
+    // Write a value directly to the underlying scratch struct (used by dict basket writers)
+    template< typename T, typename Ignored = void >
+    void writeValue( const T & value );
+
 private:
-    template< typename ColumnBuilder >
-    void createColumnBuilder( const std::string &columnName );
-    void createEnumColumnBuilder( const std::string &columnName, CspEnumMeta::Ptr enumMetaPtr );
+    template< typename CspValueType >
+    void createValueHandler( Struct * scratch, const StructFieldPtr & field )
+    {
+        m_valueHandler = std::make_unique<ValueHandler>(
+            [ scratch, field ]( const TimeSeriesProvider * input )
+            {
+                field -> setValue<CspValueType>( scratch, input -> lastValueTyped<CspValueType>() );
+            } );
+    }
 
 protected:
     using ValueHandler = std::function<void( const TimeSeriesProvider * )>;
@@ -80,8 +85,7 @@ class SingleColumnParquetOutputHandler : public ParquetOutputHandler
 class ListColumnParquetOutputHandler : public ParquetOutputHandler
 {
 public:
-    ListColumnParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &elemType, const std::string &columnName,
-                                    DialectGenericListWriterInterface::Ptr& listWriterInterface  );
+    ListColumnParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &elemType, const std::string &columnName );
 
     uint32_t getNumColumns() override{ return 1; }
 
@@ -98,11 +102,6 @@ class ListColumnParquetOutputHandler : public ParquetOutputHandler
         ( *m_valueHandler )( input );
     }
 
-private:
-    std::shared_ptr<::arrow::ArrayBuilder> createValueBuilder( const CspTypePtr &elemType,
-                                                             DialectGenericListWriterInterface::Ptr &listWriterInterface );
-
-
 protected :
     using ValueHandler = std::function<void( const TimeSeriesProvider * )>;
 
@@ -126,9 +125,8 @@ class SingleColumnParquetOutputAdapter final : public OutputAdapter, public Sing
 class ListColumnParquetOutputAdapter : public OutputAdapter, public ListColumnParquetOutputHandler
 {
 public:
-    ListColumnParquetOutputAdapter( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, std::string columnName,
-                                    DialectGenericListWriterInterface::Ptr listWriterInterface )
-            : OutputAdapter( engine ), ListColumnParquetOutputHandler( engine, parquetWriter, type, columnName, listWriterInterface )
+    ListColumnParquetOutputAdapter( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, std::string columnName )
+            : OutputAdapter( engine ), ListColumnParquetOutputHandler( engine, parquetWriter, type, columnName )
     {
     }
 
@@ -138,12 +136,14 @@ class ListColumnParquetOutputAdapter : public OutputAdapter, public ListColumnPa
 };
 
 
+class ArrowBackedArrayBuilder;
+
 class StructParquetOutputHandler : public ParquetOutputHandler
 {
 public:
     StructParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type, DictionaryPtr fieldMap );
 
-    uint32_t getNumColumns() override{ return m_valueHandlers.size(); }
+    uint32_t getNumColumns() override{ return m_columnArrayBuilders.size(); }
 
     std::shared_ptr<ArrowSingleColumnArrayBuilder> getColumnArrayBuilder( unsigned index ) override
     {
@@ -152,41 +152,9 @@ class StructParquetOutputHandler : public ParquetOutputHandler
 
     void writeValueFromTs( const TimeSeriesProvider *input ) override final;
 
-private:
-    using ValueHandler = std::function<void( const Struct * )>;
-    struct ColumnBuilderResultType
-    {
-        std::shared_ptr<ArrowSingleColumnArrayBuilder> m_columnBuilder;
-        ValueHandler                                   m_valueHandler;
-    };
-
-
-    ColumnBuilderResultType createColumnBuilder( const StructMeta *structMeta,
-                                                 const std::string &columnName,
-                                                 const std::string &structFieldName,
-                                                 const std::string *path );
-
-    template< typename ColumnBuilder >
-    ColumnBuilderResultType createColumnBuilder( const StructField *field,
-                                                 const std::string &columnName,
-                                                 const std::string *path );
-
-    ColumnBuilderResultType createEnumColumnBuilder( const StructField *field,
-                                                     const std::string &columnName,
-                                                     const std::string *path );
-
-    ColumnBuilderResultType createStructColumnBuilder( const StructField *structField,
-                                                       const std::string &columnName,
-                                                       const std::string *path );
-
-    inline std::string resolveFullColumnName( const std::string *path, const std::string &columnName )
-    {
-        return path == nullptr ? columnName : *path + "." + columnName;
-    }
-
 protected:
-    std::vector<ValueHandler>                                   m_valueHandlers;
-    std::vector<std::shared_ptr<ArrowSingleColumnArrayBuilder>> m_columnArrayBuilders;
+    std::vector<ArrowBackedArrayBuilder *>                          m_arrowBuilders;
+    std::vector<std::shared_ptr<ArrowSingleColumnArrayBuilder>>     m_columnArrayBuilders;
 };
 
 class StructParquetOutputAdapter final : public OutputAdapter, public StructParquetOutputHandler
diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp
index e662d98d1..3b48561d6 100644
--- a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp
+++ b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp
@@ -9,16 +9,12 @@ namespace csp::adapters::parquet
 {
 
 
-ParquetOutputAdapterManager::ParquetOutputAdapterManager( csp::Engine *engine, const Dictionary &properties, FileVisitorCallback fileVisitor ) :
-    AdapterManager( engine ), m_fileVisitor( fileVisitor ), m_outputFilenameAdapter( nullptr )
+ParquetOutputAdapterManager::ParquetOutputAdapterManager( csp::Engine *engine, const Dictionary &properties ) :
+    AdapterManager( engine ), m_outputFilenameAdapter( nullptr )
 {
     m_fileName            = properties.get<std::string>( "file_name" );
     m_timestampColumnName = properties.get<std::string>( "timestamp_column_name" );
-    m_allowOverwrite      = properties.get<bool>( "allow_overwrite" );
     m_batchSize           = properties.get<std::int64_t>( "batch_size" );
-    m_compression         = properties.get<std::string>( "compression" );
-    m_writeArrowBinary    = properties.get<bool>( "write_arrow_binary" );
-    m_splitColumnsToFiles = properties.get<bool>( "split_columns_to_files" );
     m_parquetWriter       = std::make_unique<ParquetWriter>( this, properties );
 }
 
@@ -31,13 +27,15 @@ void ParquetOutputAdapterManager::start( DateTime starttime, DateTime endtime )
     m_parquetWriter -> start();
     for( auto &&writer:m_dictBasketWriters )
     {
+        // Only overwrite per-basket index sink if a shared one was explicitly set
+        if( m_indexSink.onBatch )
+            writer -> setIndexSink( m_indexSink );
         writer -> start();
     }
 }
 
 void ParquetOutputAdapterManager::stop()
 {
-    bool visitFile = m_fileVisitor && m_parquetWriter -> isFileOpen();
     m_parquetWriter -> stop();
     m_parquetWriter = nullptr;
 
@@ -46,9 +44,6 @@ void ParquetOutputAdapterManager::stop()
         writer -> stop();
     }
     m_dictBasketWriters.clear();
-
-    if( visitFile )
-        m_fileVisitor( m_fileName );
 }
 
 DateTime ParquetOutputAdapterManager::processNextSimTimeSlice( DateTime time )
@@ -68,11 +63,10 @@ OutputAdapter *ParquetOutputAdapterManager::getOutputAdapter( CspTypePtr &type,
     }
 }
 
-OutputAdapter *ParquetOutputAdapterManager::getListOutputAdapter( CspTypePtr &elemType, const Dictionary &properties,
-                                                                  const DialectGenericListWriterInterface::Ptr& listWriterInterface)
+OutputAdapter *ParquetOutputAdapterManager::getListOutputAdapter( CspTypePtr &elemType, const Dictionary &properties )
 {
     auto columnName = properties.get<std::string>( "column_name" );
-    return m_parquetWriter -> getListOutputAdapter( elemType, columnName, listWriterInterface );
+    return m_parquetWriter -> getListOutputAdapter( elemType, columnName );
 }
 
 
@@ -92,8 +86,17 @@ ParquetOutputAdapterManager::createDictOutputBasketWriter( const char *columnNam
         m_dictBasketWriters.push_back( std::make_unique<ParquetScalarDictBasketOutputWriter>( this, columnName, cspTypePtr ) );
     }
 
+    // Provide data sink via factory if available
+    auto * writer = m_dictBasketWriters.back().get();
+    if( m_sinkFactory )
+    {
+        writer -> setSink( m_sinkFactory( columnName ) );
+        std::string indexName = std::string( columnName ) + "__csp_index";
+        writer -> setIndexSink( m_sinkFactory( indexName ) );
+    }
+
     m_dictBasketWriterIndexByName[ columnName ] = m_dictBasketWriters.size() - 1;
-    return m_dictBasketWriters.back().get();
+    return writer;
 }
 
 OutputAdapter *ParquetOutputAdapterManager::createOutputFileNameAdapter()
@@ -110,9 +113,6 @@ void ParquetOutputAdapterManager::changeFileName( const std::string &filename )
         m_parquetWriter -> onFileNameChange( filename );
     }
 
-    if( m_fileVisitor )
-        m_fileVisitor( m_fileName );
-
     m_fileName = filename;
 }
 
@@ -127,6 +127,11 @@ void ParquetOutputAdapterManager::scheduleEndCycle()
     }
 }
 
+void ParquetOutputAdapterManager::setSink( RecordBatchSink sink )
+{
+    m_parquetWriter -> setSink( std::move( sink ) );
+}
+
 OutputAdapter *ParquetOutputAdapterManager::getScalarOutputAdapter( CspTypePtr &type, const Dictionary &properties )
 {
     auto columnName = properties.get<std::string>( "column_name" );
diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h
index e1d6f601f..b6d2bf538 100644
--- a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h
+++ b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h
@@ -1,14 +1,13 @@
 #ifndef _IN_CSP_ADAPTERS_PARQUET_ParquetOutputAdapterManager_H
 #define _IN_CSP_ADAPTERS_PARQUET_ParquetOutputAdapterManager_H
 
+#include <csp/adapters/parquet/RecordBatchSink.h>
 #include <csp/adapters/utils/StructAdapterInfo.h>
 #include <csp/core/Generator.h>
 #include <csp/engine/AdapterManager.h>
 #include <csp/engine/Dictionary.h>
-#include <set>
 #include <string>
 #include <unordered_map>
-#include <csp/adapters/parquet/DialectGenericListWriterInterface.h>
 
 
 namespace csp::adapters::parquet
@@ -23,9 +22,9 @@ class ParquetDictBasketOutputWriter;
 class ParquetOutputAdapterManager final : public csp::AdapterManager
 {
 public:
-    using FileVisitorCallback = std::function<void(const std::string &)>;
+    using SinkFactory = std::function<RecordBatchSink( const std::string & )>;
 
-    ParquetOutputAdapterManager( csp::Engine *engine, const Dictionary &properties, FileVisitorCallback fileVisitor );
+    ParquetOutputAdapterManager( csp::Engine *engine, const Dictionary &properties );
     ~ParquetOutputAdapterManager();
 
     const char *name() const override{ return "ParquetOutputAdapterManager"; }
@@ -34,16 +33,8 @@ class ParquetOutputAdapterManager final : public csp::AdapterManager
 
     const std::string &getTimestampColumnName() const{ return m_timestampColumnName; }
 
-    bool isAllowOverwrite() const{ return m_allowOverwrite; }
-
     uint32_t getBatchSize() const{ return m_batchSize; }
 
-    std::string getCompression() const{ return m_compression; }
-
-    bool isWriteArrowBinary() const{ return m_writeArrowBinary; }
-
-    bool isSplitColumnsToFiles() const{ return m_splitColumnsToFiles; }
-
     //start the writer, open file if necessary
     void start( DateTime starttime, DateTime endtime ) override;
 
@@ -53,8 +44,7 @@ class ParquetOutputAdapterManager final : public csp::AdapterManager
     DateTime processNextSimTimeSlice( DateTime time ) override;
 
     OutputAdapter *getOutputAdapter( CspTypePtr &type, const Dictionary &properties );
-    OutputAdapter *getListOutputAdapter( CspTypePtr &elemType, const Dictionary &properties,
-                                         const DialectGenericListWriterInterface::Ptr& listWriterInterface );
+    OutputAdapter *getListOutputAdapter( CspTypePtr &elemType, const Dictionary &properties );
     ParquetDictBasketOutputWriter *createDictOutputBasketWriter( const char *columnName, const CspTypePtr &cspTypePtr);
     OutputAdapter *createOutputFileNameAdapter();
 
@@ -62,22 +52,23 @@ class ParquetOutputAdapterManager final : public csp::AdapterManager
 
     void scheduleEndCycle();
 
+    void setSink( RecordBatchSink sink );
+    void setIndexSink( RecordBatchSink sink ) { m_indexSink = std::move( sink ); }
+    void setSinkFactory( SinkFactory factory ) { m_sinkFactory = std::move( factory ); }
+
 private:
     OutputAdapter *getScalarOutputAdapter( CspTypePtr &type, const Dictionary &properties );
     OutputAdapter *getStructOutputAdapter( CspTypePtr &type, const Dictionary &properties );
 
     std::string                                                 m_fileName;
     std::string                                                 m_timestampColumnName;
-    bool                                                        m_allowOverwrite;
     uint32_t                                                    m_batchSize;
-    std::string                                                 m_compression;
-    bool                                                        m_writeArrowBinary;
-    bool                                                        m_splitColumnsToFiles;
     std::unique_ptr<ParquetWriter>                              m_parquetWriter;
     std::unordered_map<std::string, int>                        m_dictBasketWriterIndexByName;
     std::vector<std::unique_ptr<ParquetDictBasketOutputWriter>> m_dictBasketWriters;
-    FileVisitorCallback                                         m_fileVisitor;
     ParquetOutputFilenameAdapter                                *m_outputFilenameAdapter;
+    RecordBatchSink                                             m_indexSink;
+    SinkFactory                                                 m_sinkFactory;
 };
 
 }
diff --git a/cpp/csp/adapters/parquet/ParquetWriter.cpp b/cpp/csp/adapters/parquet/ParquetWriter.cpp
index 3b7eecd6f..75eb4e8bd 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetWriter.cpp
@@ -1,14 +1,10 @@
 #include <csp/adapters/parquet/ParquetWriter.h>
-#include <csp/adapters/parquet/ArrowIPCFileWriterWrapper.h>
-#include <csp/adapters/parquet/FileWriterWrapperContainer.h>
+#include <csp/adapters/parquet/ArrowBackedArrayBuilder.h>
 #include <csp/adapters/parquet/ParquetOutputAdapter.h>
 #include <csp/adapters/parquet/ParquetStatusUtils.h>
-#include <csp/adapters/parquet/FileWriterWrapperContainer.h>
 #include <arrow/array.h>
 #include <arrow/builder.h>
-#include <arrow/io/file.h>
 #include <arrow/util/key_value_metadata.h>
-#include <parquet/arrow/writer.h>
 
 namespace csp::adapters::parquet
 {
@@ -22,7 +18,7 @@ ParquetWriter::ParquetWriter( ParquetOutputAdapterManager *mgr, const Dictionary
     if( properties.exists( "file_metadata" ) )
     {
         auto file_meta = properties.get<DictionaryPtr>( "file_metadata" );
-        m_fileMetaData = std::make_shared<arrow::KeyValueMetadata>();
+        m_fileMetaData = std::make_shared<::arrow::KeyValueMetadata>();
         for( auto it = file_meta -> begin(); it != file_meta -> end(); ++it )
         {
             const std::string * value = std::get_if<std::string>( &it.getUntypedValue() );
@@ -41,7 +37,7 @@ ParquetWriter::ParquetWriter( ParquetOutputAdapterManager *mgr, const Dictionary
             if( !cmeta )
                 CSP_THROW( TypeError, "parquet column metadata expects dictionary entry per column, got unrecognized type for column '" << colIt.key() << "'" );
 
-            auto kv_metadata = std::make_shared<arrow::KeyValueMetadata>();
+            auto kv_metadata = std::make_shared<::arrow::KeyValueMetadata>();
 
             for( auto it = (*cmeta) -> begin(); it != (*cmeta) -> end(); ++it )
             {
@@ -58,10 +54,6 @@ ParquetWriter::ParquetWriter( ParquetOutputAdapterManager *mgr, const Dictionary
 
 ParquetWriter::~ParquetWriter()
 {
-    if( m_curChunkSize > 0 )
-    {
-        writeCurChunkToFile();
-    }
 }
 
 SingleColumnParquetOutputAdapter *ParquetWriter::getScalarOutputAdapter( CspTypePtr &type, const std::string &columnName )
@@ -101,22 +93,20 @@ StructParquetOutputHandler *ParquetWriter::getStructOutputHandler( CspTypePtr &t
     return adapter;
 }
 
-ListColumnParquetOutputHandler *ParquetWriter::getListOutputHandler( CspTypePtr &elemType, const std::string &columnName,
-                                                                     const DialectGenericListWriterInterface::Ptr &listWriterInterface )
+ListColumnParquetOutputHandler *ParquetWriter::getListOutputHandler( CspTypePtr &elemType, const std::string &columnName )
 {
     CSP_TRUE_OR_THROW_RUNTIME( m_publishedColumnNames.emplace( columnName ).second,
                                "Trying to publish column " << columnName << " more than once" );
     //create and register adapter
-    auto adapter = createListOutputHandler( elemType, columnName,  listWriterInterface);
+    auto adapter = createListOutputHandler( elemType, columnName );
     m_adapters.emplace_back( adapter );
     return adapter;
 }
 
 ListColumnParquetOutputAdapter *ParquetWriter::getListOutputAdapter(
-        CspTypePtr &elemType, const std::string &columnName,
-        const DialectGenericListWriterInterface::Ptr &listWriterInterface )
+        CspTypePtr &elemType, const std::string &columnName )
 {
-    auto res = static_cast<ListColumnParquetOutputAdapter *>(getListOutputHandler( elemType, columnName, listWriterInterface ));
+    auto res = static_cast<ListColumnParquetOutputAdapter *>(getListOutputHandler( elemType, columnName ));
     return res;
 }
 
@@ -127,12 +117,15 @@ PushInputAdapter *ParquetWriter::getStatusAdapter()
 
 void ParquetWriter::start()
 {
-    std::vector<std::shared_ptr<arrow::Field>> arrowFields;
+    std::vector<std::shared_ptr<::arrow::Field>> arrowFields;
     if( !m_writeTimestampColumn.has_value() && !m_adapterMgr.getTimestampColumnName().empty() )
     {
         m_writeTimestampColumn = true;
-        m_columnBuilders.push_back( std::make_shared<DatetimeArrayBuilder>( m_adapterMgr.getTimestampColumnName(), getChunkSize() ) );
-        std::shared_ptr<arrow::KeyValueMetadata> colMetaData;
+        auto tsBuilder = createArrowBackedArrayBuilder(
+            m_adapterMgr.getTimestampColumnName(), getChunkSize(), CspType::DATETIME() );
+        m_timestampBuilder = tsBuilder.get();
+        m_columnBuilders.push_back( std::move( tsBuilder ) );
+        std::shared_ptr<::arrow::KeyValueMetadata> colMetaData;
         auto colMetaIt = m_columnMetaData.find( m_adapterMgr.getTimestampColumnName() );
         if( colMetaIt != m_columnMetaData.end() )
         {
@@ -141,7 +134,7 @@ void ParquetWriter::start()
         }
 
         arrowFields.push_back(
-            arrow::field( m_adapterMgr.getTimestampColumnName(), m_columnBuilders.back() -> getDataType(), colMetaData ) );
+            ::arrow::field( m_adapterMgr.getTimestampColumnName(), m_columnBuilders.back() -> getDataType(), colMetaData ) );
     }
     else
     {
@@ -153,14 +146,14 @@ void ParquetWriter::start()
         {
             m_columnBuilders.push_back( adapter -> getColumnArrayBuilder( i ) );
 
-            std::shared_ptr<arrow::KeyValueMetadata> colMetaData;
+            std::shared_ptr<::arrow::KeyValueMetadata> colMetaData;
             auto colMetaIt = m_columnMetaData.find( m_columnBuilders.back() -> getColumnName() );
             if( colMetaIt != m_columnMetaData.end() )
             {
                 colMetaData = colMetaIt -> second;
                 m_columnMetaData.erase( colMetaIt );
             }
-            arrowFields.push_back( arrow::field( m_columnBuilders.back() -> getColumnName(), 
+            arrowFields.push_back( ::arrow::field( m_columnBuilders.back() -> getColumnName(), 
                                                  m_columnBuilders.back() -> getDataType(), 
                                                  colMetaData ) );
         }
@@ -168,34 +161,34 @@ void ParquetWriter::start()
 
     if( !m_columnMetaData.empty() )
         CSP_THROW( ValueError, "parquet column metadata has unmapped column: '" << m_columnMetaData.begin() -> first << "'" );
-    initFileWriterContainer( arrow::schema( arrowFields, m_fileMetaData ) );
+
+    m_schema = ::arrow::schema( arrowFields, m_fileMetaData );
+    if( m_sink.onStart )
+        m_sink.onStart( m_schema );
+    auto & fileName = m_adapterMgr.getFileName();
+    m_fileOpen = !fileName.empty();
+    if( m_fileOpen && m_sink.onFileChange )
+        m_sink.onFileChange( fileName );
 }
 
 void ParquetWriter::stop()
 {
-    if( m_fileWriterWrapperContainer )
-    {
-        if( m_curChunkSize > 0 )
-        {
-            writeCurChunkToFile();
-        }
-        m_fileWriterWrapperContainer -> close();
-        m_fileWriterWrapperContainer = nullptr;
-    }
+    if( m_curChunkSize > 0 )
+        flushBatch();
+    if( m_sink.onStop )
+        m_sink.onStop();
+    m_fileOpen = false;
 }
 
 void ParquetWriter::onEndCycle()
 {
     if( isFileOpen() ) [[likely]]
     {
-        // This must be defined outside of the "if" below, the datetime object must live till the end of cycle since
-        // We pass all values by reference
         DateTime now;
         if( m_writeTimestampColumn.value() )
         {
-            // Set the timestamp value it's always the first
             now = m_adapterMgr.rootEngine() -> now();
-            static_cast<DatetimeArrayBuilder *>(m_columnBuilders[ 0 ].get()) -> setValue( now );
+            m_timestampBuilder -> scratchField() -> setValue<DateTime>( m_timestampBuilder -> scratch(), now );
         }
         for( auto &&columnBuilder:m_columnBuilders )
         {
@@ -203,26 +196,17 @@ void ParquetWriter::onEndCycle()
         }
         if( ++m_curChunkSize >= getChunkSize() )
         {
-            writeCurChunkToFile();
+            flushBatch();
         }
     }
 }
 
-bool ParquetWriter::isFileOpen() const
-{
-    return m_fileWriterWrapperContainer != nullptr && m_fileWriterWrapperContainer -> isOpen();
-}
-
 void ParquetWriter::onFileNameChange( const std::string &fileName )
 {
-    CSP_TRUE_OR_THROW_RUNTIME( m_fileWriterWrapperContainer, "Trying to set file name when file writer already closed" );
-    writeCurChunkToFile();
-    m_fileWriterWrapperContainer -> close();
-    if( !fileName.empty() )
-    {
-        m_fileWriterWrapperContainer
-                -> open( fileName, m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() );
-    }
+    flushBatch();
+    if( m_sink.onFileChange )
+        m_sink.onFileChange( fileName );
+    m_fileOpen = !fileName.empty();
 }
 
 SingleColumnParquetOutputHandler *ParquetWriter::createScalarOutputHandler( CspTypePtr type, const std::string &name )
@@ -230,10 +214,9 @@ SingleColumnParquetOutputHandler *ParquetWriter::createScalarOutputHandler( CspT
     return m_engine -> createOwnedObject<SingleColumnParquetOutputAdapter>( *this, type, name );
 }
 
-ListColumnParquetOutputHandler *ParquetWriter::createListOutputHandler( CspTypePtr &elemType, const std::string &columnName,
-                                                                        DialectGenericListWriterInterface::Ptr listWriterInterface )
+ListColumnParquetOutputHandler *ParquetWriter::createListOutputHandler( CspTypePtr &elemType, const std::string &columnName )
 {
-    return m_engine -> createOwnedObject<ListColumnParquetOutputAdapter>( *this, elemType, columnName, listWriterInterface );
+    return m_engine -> createOwnedObject<ListColumnParquetOutputAdapter>( *this, elemType, columnName );
 }
 
 
@@ -243,39 +226,23 @@ StructParquetOutputHandler *ParquetWriter::createStructOutputHandler( CspTypePtr
     return m_engine -> createOwnedObject<StructParquetOutputAdapter>( *this, type, fieldMap );
 }
 
-void ParquetWriter::initFileWriterContainer( std::shared_ptr<arrow::Schema> schema )
+std::shared_ptr<::arrow::RecordBatch> ParquetWriter::buildRecordBatch()
 {
-    if( m_adapterMgr.isSplitColumnsToFiles() )
-    {
-        m_fileWriterWrapperContainer = std::make_unique<MultipleFileWriterWrapperContainer>( schema,
-                                                                                             m_adapterMgr.isWriteArrowBinary() );
-    }
-    else
-    {
-        m_fileWriterWrapperContainer = std::make_unique<SingleFileWriterWrapperContainer>( schema,
-                                                                                           m_adapterMgr.isWriteArrowBinary() );
-    }
-    if( !m_adapterMgr.getFileName().empty() )
-    {
-        m_fileWriterWrapperContainer -> open( m_adapterMgr.getFileName(),
-                                              m_adapterMgr.getCompression(), m_adapterMgr.isAllowOverwrite() );
-    }
+    std::vector<std::shared_ptr<::arrow::Array>> columns;
+    columns.reserve( m_columnBuilders.size() );
+    for( auto && builder : m_columnBuilders )
+        columns.push_back( builder -> buildArray() );
+    return ::arrow::RecordBatch::Make( m_schema, m_curChunkSize, std::move( columns ) );
 }
 
-void ParquetWriter::writeCurChunkToFile()
+void ParquetWriter::flushBatch()
 {
     if( m_curChunkSize > 0 )
     {
         if( !isFileOpen() ) [[unlikely]]
-        {
-            if(m_curChunkSize != 0 )
-            {
-                CSP_THROW( csp::RuntimeException, "Trying to write to parquet/arrow file, when no file name was provided" );
-
-            }
-            return;
-        }
-        m_fileWriterWrapperContainer -> writeData( m_columnBuilders );
+            CSP_THROW( csp::RuntimeException, "Trying to write to parquet/arrow file, when no file name was provided" );
+        if( m_sink.onBatch )
+            m_sink.onBatch( buildRecordBatch() );
         m_curChunkSize = 0;
     }
 }
diff --git a/cpp/csp/adapters/parquet/ParquetWriter.h b/cpp/csp/adapters/parquet/ParquetWriter.h
index a244f2940..3d779bd71 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.h
+++ b/cpp/csp/adapters/parquet/ParquetWriter.h
@@ -1,18 +1,20 @@
 #ifndef _IN_CSP_ADAPTERS_PARQUET_ParquetWriter_H
 #define _IN_CSP_ADAPTERS_PARQUET_ParquetWriter_H
 
-#include <csp/adapters/parquet/DialectGenericListWriterInterface.h>
 #include <csp/adapters/parquet/ParquetOutputAdapterManager.h>
+#include <csp/adapters/parquet/RecordBatchSink.h>
 #include <csp/engine/Dictionary.h>
 #include <csp/engine/Engine.h>
+#include <arrow/record_batch.h>
 #include <string>
 #include <optional>
 #include <unordered_map>
 
-namespace arrow{ class KeyValueMetadata; }
+namespace arrow{ class KeyValueMetadata; class Schema; }
 
 namespace csp::adapters::parquet
 {
+class ArrowBackedArrayBuilder;
 class ArrowSingleColumnArrayBuilder;
 
 class ParquetOutputHandler;
@@ -23,10 +25,6 @@ class StructParquetOutputAdapter;
 class ListColumnParquetOutputHandler;
 class ListColumnParquetOutputAdapter;
 
-class FileWriterWrapper;
-
-class FileWriterWrapperContainer;
-
 
 class ParquetWriter : public EndCycleListener
 {
@@ -36,15 +34,15 @@ class ParquetWriter : public EndCycleListener
 
     ~ParquetWriter();
 
+    void setSink( RecordBatchSink sink ) { m_sink = std::move( sink ); }
+
     SingleColumnParquetOutputAdapter *getScalarOutputAdapter( CspTypePtr &type, const std::string &columnName );
     StructParquetOutputAdapter *getStructOutputAdapter( CspTypePtr &type, csp::DictionaryPtr fieldMap );
-    ListColumnParquetOutputAdapter *getListOutputAdapter( CspTypePtr &elemType, const std::string &columnName,
-                                                          const DialectGenericListWriterInterface::Ptr &listWriterInterface );
+    ListColumnParquetOutputAdapter *getListOutputAdapter( CspTypePtr &elemType, const std::string &columnName );
 
     SingleColumnParquetOutputHandler *getScalarOutputHandler( CspTypePtr &type, const std::string &columnName );
     StructParquetOutputHandler *getStructOutputHandler( CspTypePtr &type, csp::DictionaryPtr fieldMap );
-    ListColumnParquetOutputHandler *getListOutputHandler( CspTypePtr &elemType, const std::string &columnName,
-                                                          const DialectGenericListWriterInterface::Ptr &listWriterInterface );
+    ListColumnParquetOutputHandler *getListOutputHandler( CspTypePtr &elemType, const std::string &columnName );
 
     PushInputAdapter *getStatusAdapter();
 
@@ -60,31 +58,32 @@ class ParquetWriter : public EndCycleListener
         m_adapterMgr.scheduleEndCycle();
     }
 
-    bool isFileOpen() const;
+    bool isFileOpen() const { return m_fileOpen; }
 
     virtual void onFileNameChange(const std::string& fileName);
 protected:
     virtual SingleColumnParquetOutputHandler *createScalarOutputHandler( CspTypePtr type, const std::string& name );
-    virtual ListColumnParquetOutputHandler *createListOutputHandler( CspTypePtr &elemType, const std::string &columnName,
-                                                                     DialectGenericListWriterInterface::Ptr listWriterInterface );
+    virtual ListColumnParquetOutputHandler *createListOutputHandler( CspTypePtr &elemType, const std::string &columnName );
     virtual StructParquetOutputHandler *createStructOutputHandler( CspTypePtr type, const DictionaryPtr &fieldMap );
 
 private:
-    void initFileWriterContainer( std::shared_ptr<::arrow::Schema> schema );
-    void writeCurChunkToFile();
+    void flushBatch();
+    std::shared_ptr<::arrow::RecordBatch> buildRecordBatch();
 protected:
     using Adapters = std::vector<ParquetOutputHandler *>;
     using PublishedColumnNames = std::unordered_set<std::string>;
 
     ParquetOutputAdapterManager &m_adapterMgr;
     Engine                      *m_engine;
+    ArrowBackedArrayBuilder     *m_timestampBuilder = nullptr;
 private:
     Adapters             m_adapters;
     PublishedColumnNames m_publishedColumnNames;
 
+    RecordBatchSink                                             m_sink;
+    bool                                                        m_fileOpen = false;
 
-    std::unique_ptr<FileWriterWrapperContainer> m_fileWriterWrapperContainer;
-
+    std::shared_ptr<::arrow::Schema>                            m_schema;
     std::vector<std::shared_ptr<ArrowSingleColumnArrayBuilder>> m_columnBuilders;
     std::uint32_t                                               m_curChunkSize;
     std::optional<bool>                                         m_writeTimestampColumn;
diff --git a/cpp/csp/adapters/parquet/RecordBatchSink.h b/cpp/csp/adapters/parquet/RecordBatchSink.h
new file mode 100644
index 000000000..c7b0036e7
--- /dev/null
+++ b/cpp/csp/adapters/parquet/RecordBatchSink.h
@@ -0,0 +1,31 @@
+#ifndef _IN_CSP_ADAPTERS_PARQUET_RecordBatchSink_H
+#define _IN_CSP_ADAPTERS_PARQUET_RecordBatchSink_H
+
+#include <arrow/record_batch.h>
+#include <functional>
+#include <memory>
+#include <string>
+
+namespace arrow { class Schema; }
+
+namespace csp::adapters::parquet
+{
+
+// Callback interface for receiving RecordBatches from ParquetWriter.
+// Python side implements this via a sink object.
+struct RecordBatchSink
+{
+    using SchemaCallback     = std::function<void( const std::shared_ptr<::arrow::Schema> & )>;
+    using BatchCallback      = std::function<void( const std::shared_ptr<::arrow::RecordBatch> & )>;
+    using FileChangeCallback = std::function<void( const std::string & )>;
+    using StopCallback       = std::function<void()>;
+
+    SchemaCallback     onStart;       // called with schema at start()
+    BatchCallback      onBatch;       // called with each full RecordBatch
+    FileChangeCallback onFileChange;  // called when file rotation requested
+    StopCallback       onStop;        // called at stop()
+};
+
+} // namespace csp::adapters::parquet
+
+#endif
diff --git a/cpp/csp/adapters/parquet/RecordBatchWithFlag.h b/cpp/csp/adapters/parquet/RecordBatchWithFlag.h
new file mode 100644
index 000000000..7f0f688b7
--- /dev/null
+++ b/cpp/csp/adapters/parquet/RecordBatchWithFlag.h
@@ -0,0 +1,21 @@
+#ifndef _IN_CSP_ADAPTERS_PARQUET_RecordBatchWithFlag_H
+#define _IN_CSP_ADAPTERS_PARQUET_RecordBatchWithFlag_H
+
+#include <arrow/record_batch.h>
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+namespace csp::adapters::parquet
+{
+
+struct RecordBatchWithFlag
+{
+    std::shared_ptr<::arrow::RecordBatch> batch;
+    std::unordered_map<std::string, std::shared_ptr<::arrow::RecordBatch>> basketBatches;
+    bool                                  schemaChanged;
+};
+
+} // namespace csp::adapters::parquet
+
+#endif
diff --git a/cpp/csp/python/adapters/parquetadapterimpl.cpp b/cpp/csp/python/adapters/parquetadapterimpl.cpp
index 979762233..bf1bfcb95 100644
--- a/cpp/csp/python/adapters/parquetadapterimpl.cpp
+++ b/cpp/csp/python/adapters/parquetadapterimpl.cpp
@@ -3,6 +3,8 @@
 #include <csp/adapters/parquet/ParquetOutputAdapterManager.h>
 #include <csp/adapters/parquet/ParquetDictBasketOutputWriter.h>
 #include <csp/adapters/parquet/ParquetStatusUtils.h>
+#include <csp/adapters/parquet/ParquetWriter.h>
+#include <csp/adapters/parquet/RecordBatchSink.h>
 #include <csp/engine/PushInputAdapter.h>
 #include <csp/python/Conversions.h>
 #include <csp/python/Exception.h>
@@ -582,137 +584,6 @@ create_parquet_input_adapter( csp::AdapterManager *manager, PyEngine *pyengine,
     }
 }
 
-template< typename CspCType >
-class NumpyArrayWriterImpl : public TypedDialectGenericListWriterInterface<CspCType>
-{
-public:
-    NumpyArrayWriterImpl( PyArray_Descr *expectedArrayDesc )
-            : m_expectedArrayDesc( expectedArrayDesc )
-    {
-    }
-
-    void writeItems( const csp::DialectGenericType &listObject ) override
-    {
-        PyObject *object = csp::python::toPythonBorrowed( listObject );
-        if( !PyArray_Check( object ) )
-            CSP_THROW( csp::TypeError, "While writing to parquet expected numpy array type, got " << Py_TYPE( object ) -> tp_name );
-
-        PyArrayObject *arrayObject = ( PyArrayObject * ) ( object );
-        char npy_type = PyArray_DESCR( arrayObject ) -> type;
-        if( PyArray_DESCR( arrayObject ) -> kind != m_expectedArrayDesc -> kind )
-            CSP_THROW( csp::TypeError,
-                       "Expected array of type " << csp::python::PyObjectPtr::own( PyObject_Repr( ( PyObject * ) m_expectedArrayDesc ) )
-                                                 << " got "
-                                                 << csp::python::PyObjectPtr::own( PyObject_Repr( ( PyObject * ) PyArray_DESCR( arrayObject ) ) ) );
-
-        auto ndim = PyArray_NDIM( arrayObject );
-        CSP_TRUE_OR_THROW_RUNTIME( ndim == 1, "While writing to parquet expected numpy array with 1 dimension" << " got " << ndim );
-        switch( npy_type )
-        {
-            case NPY_BYTELTR:      writeValues<char>( arrayObject );            break;
-            case NPY_UBYTELTR:     writeValues<unsigned char>( arrayObject );   break;
-            case NPY_SHORTLTR:     writeValues<short>( arrayObject );           break;
-            case NPY_USHORTLTR:    writeValues<unsigned short>( arrayObject );  break;
-            case NPY_INTLTR:       writeValues<int>( arrayObject );             break;
-            case NPY_UINTLTR:      writeValues<unsigned int>( arrayObject );    break;
-            case NPY_LONGLTR:      writeValues<long>( arrayObject );            break;
-            case NPY_ULONGLTR:     writeValues<unsigned long>( arrayObject );   break;
-            case NPY_LONGLONGLTR:  writeValues<long long>( arrayObject );       break;
-            case NPY_ULONGLONGLTR: writeValues<unsigned long long>( arrayObject ); break;
-            case NPY_FLOATLTR:  writeValues<float>( arrayObject );  break;
-            case NPY_DOUBLELTR: writeValues<double>( arrayObject ); break;
-            default:
-                writeValues<CspCType>( arrayObject );
-        }
-    }
-private:
-    template<typename NumpyCType>
-    void writeValues( PyArrayObject * arrayObject )
-    {
-        auto arraySize = PyArray_Size( ( PyObject * ) arrayObject );
-        if( PyArray_ISCARRAY_RO(arrayObject) )
-        {
-            NumpyCType* data = reinterpret_cast<NumpyCType*>( PyArray_DATA( arrayObject ) );
-            for (decltype(arraySize) i = 0; i < arraySize; ++i)
-                this->writeValue(static_cast<CspCType>(data[i]));
-        }
-        else
-        {
-            for (decltype(arraySize) i = 0; i < arraySize; ++i)
-                this->writeValue(static_cast<CspCType>(*reinterpret_cast<NumpyCType*>(PyArray_GETPTR1(arrayObject, i))));
-        }
-    }
-
-    PyArray_Descr *m_expectedArrayDesc;
-};
-
-class NumpyUnicodeArrayWriter : public TypedDialectGenericListWriterInterface<std::string>
-{
-public:
-    void writeItems( const csp::DialectGenericType &listObject ) override
-    {
-        PyObject *object = csp::python::toPythonBorrowed( listObject );
-        if( !PyArray_Check( object ) )
-            CSP_THROW( csp::TypeError, "While writing to parquet expected numpy array type, got " << Py_TYPE( object ) -> tp_name );
-
-        PyArrayObject *arrayObject = ( PyArrayObject * ) ( object );
-        if( PyArray_DESCR( arrayObject ) -> type_num != NPY_UNICODE )
-            CSP_THROW( csp::TypeError, "Expected unicode array, got " <<
-                       csp::python::PyObjectPtr::own( PyObject_Repr( ( PyObject * ) PyArray_DESCR( arrayObject ) ) ) );
-
-        auto elementSize = PyDataType_ELSIZE( PyArray_DESCR( arrayObject ) );
-        auto ndim        = PyArray_NDIM( arrayObject );
-        CSP_TRUE_OR_THROW_RUNTIME( ndim == 1, "While writing to parquet expected numpy array with 1 dimension" << " got " << ndim );
-        std::wstring_convert<std::codecvt_utf8<char32_t>,char32_t> converter;
-
-        auto arraySize = PyArray_Size( object );
-        if( PyArray_ISCARRAY_RO( arrayObject ) )
-        {
-            auto data = reinterpret_cast<char *>(PyArray_DATA( arrayObject ));
-            for( decltype( arraySize ) i = 0; i < arraySize; ++i )
-            {
-                std::string value = converter.to_bytes( reinterpret_cast<char32_t*>(data + elementSize * i),
-                                                        reinterpret_cast<char32_t*>(data + elementSize * ( i + 1 )) );
-                this -> writeValue( value );
-            }
-        }
-        else
-        {
-            for( decltype( arraySize ) i = 0; i < arraySize; ++i )
-            {
-                char        *elementPtr = reinterpret_cast<char *>(PyArray_GETPTR1( arrayObject, i ));
-                std::string value       = converter.to_bytes( reinterpret_cast<char32_t*>(elementPtr),
-                                                              reinterpret_cast<char32_t*>(elementPtr + elementSize ) );
-                this -> writeValue( value );
-            }
-        }
-    }
-};
-
-static inline DialectGenericListWriterInterface::Ptr create_numpy_array_writer_impl( const csp::CspTypePtr &type )
-{
-    try
-    {
-        return csp::PartialSwitchCspType<csp::CspType::Type::DOUBLE, csp::CspType::Type::INT64,
-                csp::CspType::Type::BOOL, csp::CspType::Type::STRING>::invoke(
-                type.get(),
-                []( auto tag ) -> DialectGenericListWriterInterface::Ptr
-                {
-                    using CValueType = typename decltype( tag )::type;
-                    auto numpy_dtype = PyArray_DescrFromType( csp::python::NPY_TYPE<CValueType>::value );
-                    if constexpr (std::is_same_v<CValueType,std::string>)
-                        return std::make_shared<NumpyUnicodeArrayWriter>();
-                    else
-                        return std::make_shared<NumpyArrayWriterImpl<CValueType>>(numpy_dtype);
-                }
-        );
-    }
-    catch( csp::TypeError &e )
-    {
-        CSP_THROW( csp::TypeError, "Unsupported array value type when writing to parquet:" << type -> type().asString() );
-    }
-}
-
 static OutputAdapter *create_parquet_output_adapter( csp::AdapterManager *manager, PyEngine *pyengine, PyObject *args )
 {
     PyObject *pyProperties;
@@ -732,7 +603,7 @@ static OutputAdapter *create_parquet_output_adapter( csp::AdapterManager *manage
     if( propertiesDict.get( "is_array", false ) )
     {
         auto &&valueType = pyTypeAsCspType( toPythonBorrowed( propertiesDict.get<DialectGenericType>( "array_value_type" ) ) );
-        return parquetManager -> getListOutputAdapter( valueType, propertiesDict, create_numpy_array_writer_impl( valueType ) );
+        return parquetManager -> getListOutputAdapter( valueType, propertiesDict );
     }
     else
     {
@@ -828,21 +699,125 @@ static PyObject *create_parquet_input_adapter_manager_native( PyObject *args )
     CSP_RETURN_NULL;
 }
 
+// --- Helpers for exporting Arrow data to Python via C Data Interface ---
+
+static void releaseArrowSchemaCapsule( PyObject * capsule )
+{
+    auto * schema = reinterpret_cast<ArrowSchema*>( PyCapsule_GetPointer( capsule, "arrow_schema" ) );
+    if( schema && schema -> release )
+        schema -> release( schema );
+    free( schema );
+}
+
+static void releaseArrowArrayCapsule( PyObject * capsule )
+{
+    auto * array = reinterpret_cast<ArrowArray*>( PyCapsule_GetPointer( capsule, "arrow_array" ) );
+    if( array && array -> release )
+        array -> release( array );
+    free( array );
+}
+
+// Export an Arrow Schema as a single capsule
+static PyObjectPtr exportSchema( const std::shared_ptr<::arrow::Schema> & schema )
+{
+    auto * c_schema = static_cast<ArrowSchema*>( malloc( sizeof( ArrowSchema ) ) );
+    auto st = ::arrow::ExportSchema( *schema, c_schema );
+    if( !st.ok() )
+    {
+        free( c_schema );
+        CSP_THROW( ValueError, "Failed to export Schema: " << st.ToString() );
+    }
+    return PyObjectPtr::own( PyCapsule_New( c_schema, "arrow_schema", releaseArrowSchemaCapsule ) );
+}
+
+// Build a RecordBatchSink that delegates to a Python object with
+// on_start(schema_capsule), on_batch(schema_capsule, array_capsule), on_file_change(path), on_stop() methods.
+static RecordBatchSink createSinkFromPython( PyObjectPtr pySink )
+{
+    RecordBatchSink sink;
+
+    sink.onStart = [pySink]( const std::shared_ptr<::arrow::Schema> & schema )
+    {
+        auto capsule = exportSchema( schema );
+        auto rv = PyObjectPtr::own( PyObject_CallMethod( pySink.get(), "on_start", "O", capsule.get() ) );
+        if( !rv.get() )
+            CSP_THROW( PythonPassthrough, "" );
+    };
+
+    sink.onBatch = [pySink]( const std::shared_ptr<::arrow::RecordBatch> & rb )
+    {
+        auto * c_schema = static_cast<ArrowSchema*>( malloc( sizeof( ArrowSchema ) ) );
+        auto * c_array  = static_cast<ArrowArray*>( malloc( sizeof( ArrowArray ) ) );
+        auto st = ::arrow::ExportRecordBatch( *rb, c_array, c_schema );
+        if( !st.ok() )
+        {
+            free( c_array );
+            free( c_schema );
+            CSP_THROW( ValueError, "Failed to export RecordBatch: " << st.ToString() );
+        }
+        auto pySchemaCapsule = PyObjectPtr::own( PyCapsule_New( c_schema, "arrow_schema", releaseArrowSchemaCapsule ) );
+        auto pyArrayCapsule  = PyObjectPtr::own( PyCapsule_New( c_array, "arrow_array", releaseArrowArrayCapsule ) );
+        auto rv = PyObjectPtr::own( PyObject_CallMethod( pySink.get(), "on_batch", "OO",
+                                                          pySchemaCapsule.get(), pyArrayCapsule.get() ) );
+        if( !rv.get() )
+            CSP_THROW( PythonPassthrough, "" );
+    };
+
+    sink.onFileChange = [pySink]( const std::string & path )
+    {
+        auto pyPath = PyObjectPtr::own( PyUnicode_FromStringAndSize( path.c_str(), path.size() ) );
+        auto rv = PyObjectPtr::own( PyObject_CallMethod( pySink.get(), "on_file_change", "O", pyPath.get() ) );
+        if( !rv.get() )
+            CSP_THROW( PythonPassthrough, "" );
+    };
+
+    sink.onStop = [pySink]()
+    {
+        auto rv = PyObjectPtr::own( PyObject_CallMethod( pySink.get(), "on_stop", nullptr ) );
+        if( !rv.get() )
+            CSP_THROW( PythonPassthrough, "" );
+    };
+
+    return sink;
+}
+
 csp::AdapterManager *create_parquet_output_adapter_manager( PyEngine *engine, const Dictionary &properties )
 {
-    ParquetOutputAdapterManager::FileVisitorCallback fileVisitor;
-    DialectGenericType pyFilenameVisitorDG;
-    if( properties.tryGet( "file_visitor", pyFilenameVisitorDG ) )
+    // Extract Python sink object from properties
+    RecordBatchSink mainSink;
+    RecordBatchSink indexSink;
+    DialectGenericType pySinkDG, pyIndexSinkDG;
+    if( properties.tryGet( "rb_sink", pySinkDG ) )
     {
-        PyObjectPtr pyFilenameVisitor = PyObjectPtr::own( toPython( pyFilenameVisitorDG ) );
-        fileVisitor = [pyFilenameVisitor]( const std::string & filename )
-            {
-                PyObjectPtr rv =  PyObjectPtr::own( PyObject_CallFunction( pyFilenameVisitor.get(), "O", PyObjectPtr::own( toPython( filename ) ).get() ) );
-                if( !rv.get() )
-                    CSP_THROW( PythonPassthrough, "" );
-            };
+        auto pySink = PyObjectPtr::own( toPython( pySinkDG ) );
+        mainSink = createSinkFromPython( std::move( pySink ) );
     }
-    return engine -> engine() -> createOwnedObject<ParquetOutputAdapterManager>( properties, fileVisitor );
+    if( properties.tryGet( "rb_index_sink", pyIndexSinkDG ) )
+    {
+        auto pyIndexSink = PyObjectPtr::own( toPython( pyIndexSinkDG ) );
+        indexSink = createSinkFromPython( std::move( pyIndexSink ) );
+    }
+
+    auto * mgr = engine -> engine() -> createOwnedObject<ParquetOutputAdapterManager>( properties );
+    mgr -> setSink( std::move( mainSink ) );
+    mgr -> setIndexSink( std::move( indexSink ) );
+
+    // Sink factory for dict basket writers: calls Python to create new sinks on demand
+    DialectGenericType pySinkFactoryDG;
+    if( properties.tryGet( "rb_sink_factory", pySinkFactoryDG ) )
+    {
+        auto pySinkFactory = PyObjectPtr::own( toPython( pySinkFactoryDG ) );
+        mgr -> setSinkFactory( [pySinkFactory]( const std::string & name ) -> RecordBatchSink
+        {
+            auto pyName = PyObjectPtr::own( PyUnicode_FromStringAndSize( name.c_str(), name.size() ) );
+            auto pySink = PyObjectPtr::own( PyObject_CallFunctionObjArgs( pySinkFactory.get(), pyName.get(), nullptr ) );
+            if( !pySink.get() )
+                CSP_THROW( PythonPassthrough, "" );
+            return createSinkFromPython( std::move( pySink ) );
+        } );
+    }
+
+    return mgr;
 }
 
 
diff --git a/csp/adapters/_parquet_rb_writer.py b/csp/adapters/_parquet_rb_writer.py
new file mode 100644
index 000000000..8df211a33
--- /dev/null
+++ b/csp/adapters/_parquet_rb_writer.py
@@ -0,0 +1,197 @@
+"""RecordBatch sink for parquet/arrow output.
+
+Receives RecordBatches from C++ (via the Arrow C Data Interface)
+and writes them using a pluggable writer backend.
+"""
+
+import os
+
+import pyarrow as pa
+import pyarrow.ipc as ipc
+import pyarrow.parquet as pq
+
+
+# ── Writer helpers ────────────────────────────────────────────────────
+
+
+class _IPCWriter:
+    """Wraps an IPC stream writer + underlying file sink."""
+
+    __slots__ = ("_writer", "_sink")
+
+    def __init__(self, writer, sink):
+        self._writer = writer
+        self._sink = sink
+
+    def write_batch(self, rb):
+        self._writer.write_batch(rb)
+
+    def close(self):
+        self._writer.close()
+        self._sink.close()
+
+
+class _SplitColumnsWriter:
+    """Dispatches each column of a RecordBatch to its own writer."""
+
+    __slots__ = ("_writers", "_schema")
+
+    def __init__(self, writers, schema):
+        self._writers = writers
+        self._schema = schema
+
+    def write_batch(self, rb):
+        for i, field in enumerate(self._schema):
+            col_rb = pa.RecordBatch.from_arrays([rb.column(i)], schema=pa.schema([field]))
+            self._writers[field.name].write_batch(col_rb)
+
+    def close(self):
+        for w in self._writers.values():
+            w.close()
+
+
+def _ensure_parent(path, allow_overwrite):
+    if not allow_overwrite and os.path.exists(path):
+        raise FileExistsError(f"File already exists: {path}")
+    parent = os.path.dirname(path)
+    if parent:
+        os.makedirs(parent, exist_ok=True)
+
+
+def _apply_metadata(schema, file_metadata, column_metadata):
+    """Attach file/column-level metadata to a pyarrow schema."""
+    if file_metadata:
+        existing = schema.metadata or {}
+        merged = {**existing, **{k.encode(): v.encode() for k, v in file_metadata.items()}}
+        schema = schema.with_metadata(merged)
+    if column_metadata:
+        fields = []
+        for field in schema:
+            if field.name in column_metadata:
+                meta = {k.encode(): v.encode() for k, v in column_metadata[field.name].items()}
+                existing = field.metadata or {}
+                fields.append(field.with_metadata({**existing, **meta}))
+            else:
+                fields.append(field)
+        schema = pa.schema(fields, metadata=schema.metadata)
+    return schema
+
+
+# ── Writer factories ──────────────────────────────────────────────────
+
+
+def _parquet_writer_factory(compression, allow_overwrite, file_metadata=None, column_metadata=None):
+    compression = compression or "none"
+
+    def factory(path, schema):
+        schema = _apply_metadata(schema, file_metadata, column_metadata)
+        _ensure_parent(path, allow_overwrite)
+        return pq.ParquetWriter(path, schema, compression=compression)
+
+    return factory
+
+
+def _arrow_ipc_writer_factory(compression, allow_overwrite):
+    def factory(path, schema):
+        _ensure_parent(path, allow_overwrite)
+        options = ipc.IpcWriteOptions()
+        if compression and compression != "none":
+            options = ipc.IpcWriteOptions(compression=pa.Codec(compression))
+        sink = pa.OSFile(path, "wb")
+        return _IPCWriter(ipc.new_stream(sink, schema, options=options), sink)
+
+    return factory
+
+
+def _split_columns_writer_factory(compression, allow_overwrite, write_arrow_binary):
+    compression = compression or "none"
+    ext = ".arrow" if write_arrow_binary else ".parquet"
+
+    def factory(dir_name, schema):
+        if dir_name:
+            os.makedirs(dir_name, exist_ok=True)
+        writers = {}
+        for field in schema:
+            col_schema = pa.schema([field])
+            file_path = os.path.join(dir_name, field.name + ext)
+            if not allow_overwrite and os.path.exists(file_path):
+                raise FileExistsError(f"File already exists: {file_path}")
+            if write_arrow_binary:
+                sink = pa.OSFile(file_path, "wb")
+                writers[field.name] = _IPCWriter(ipc.new_stream(sink, col_schema), sink)
+            else:
+                writers[field.name] = pq.ParquetWriter(file_path, col_schema, compression=compression)
+        return _SplitColumnsWriter(writers, schema)
+
+    return factory
+
+
+# ── Unified sink ──────────────────────────────────────────────────────
+
+
+class RecordBatchSink:
+    """Receives RecordBatches from C++ and writes them via a pluggable backend.
+
+    writer_factory(path, schema) must return an object with write_batch() and close().
+    """
+
+    def __init__(self, writer_factory, file_visitor=None):
+        self._writer_factory = writer_factory
+        self._file_visitor = file_visitor
+        self._schema = None
+        self._writer = None
+        self._current_file = None
+
+    def on_start(self, schema_capsule):
+        self._schema = pa.Schema._import_from_c_capsule(schema_capsule)
+
+    def on_batch(self, schema_capsule, array_capsule):
+        rb = pa.RecordBatch._import_from_c_capsule(schema_capsule, array_capsule)
+        if self._writer is not None:
+            self._writer.write_batch(rb)
+
+    def on_file_change(self, new_path):
+        if self._writer is not None:
+            self._writer.close()
+            if self._file_visitor:
+                self._file_visitor(self._current_file)
+        if new_path:
+            self._writer = self._writer_factory(new_path, self._schema)
+            self._current_file = new_path
+        else:
+            self._writer = None
+
+    def on_stop(self):
+        if self._writer is not None:
+            self._writer.close()
+            if self._file_visitor:
+                self._file_visitor(self._current_file)
+            self._writer = None
+
+
+# ── Public factory functions ──────────────────────────────────────────
+
+
+def create_sink(file_name, compression, allow_overwrite, write_arrow_binary,
+                split_columns_to_files, file_visitor=None,
+                file_metadata=None, column_metadata=None):
+    """Factory function to create the appropriate sink based on config."""
+    if split_columns_to_files:
+        wf = _split_columns_writer_factory(compression, allow_overwrite, write_arrow_binary)
+    elif write_arrow_binary:
+        wf = _arrow_ipc_writer_factory(compression, allow_overwrite)
+    else:
+        wf = _parquet_writer_factory(compression, allow_overwrite, file_metadata, column_metadata)
+    return RecordBatchSink(wf, file_visitor)
+
+
+def create_sink_factory(compression, allow_overwrite, write_arrow_binary, file_visitor=None):
+    """Returns a callable(name) -> sink for dict basket writers.
+
+    Dict basket writers always use split_columns_to_files=True for their data,
+    and each basket gets its own sink identified by name.
+    """
+    def factory(name):
+        wf = _split_columns_writer_factory(compression, allow_overwrite, write_arrow_binary)
+        return RecordBatchSink(wf, file_visitor)
+    return factory
diff --git a/csp/adapters/output_adapters/parquet.py b/csp/adapters/output_adapters/parquet.py
index c987e85c7..b7167a365 100644
--- a/csp/adapters/output_adapters/parquet.py
+++ b/csp/adapters/output_adapters/parquet.py
@@ -224,6 +224,39 @@ def _create(self, engine, memo):
         if self._parquet_output_adapter_manager:
             return self._parquet_output_adapter_manager
 
+        from csp.adapters._parquet_rb_writer import create_sink, create_sink_factory
+
+        # Create main sink for the primary writer
+        file_name = self._properties.get("file_name", "")
+        compression = self._properties.get("compression", "snappy")
+        allow_overwrite = self._properties.get("allow_overwrite", False)
+        write_arrow_binary = self._properties.get("write_arrow_binary", False)
+        split_columns = self._properties.get("split_columns_to_files", False)
+        file_visitor = self._properties.get("file_visitor")
+        file_metadata = self._properties.get("file_metadata")
+        column_metadata = self._properties.get("column_metadata")
+
+        sink = create_sink(
+            file_name=file_name,
+            compression=compression,
+            allow_overwrite=allow_overwrite,
+            write_arrow_binary=write_arrow_binary,
+            split_columns_to_files=split_columns,
+            file_visitor=file_visitor,
+            file_metadata=file_metadata,
+            column_metadata=column_metadata,
+        )
+        self._properties["rb_sink"] = sink
+
+        # Sink factory for dict basket writers
+        sink_factory = create_sink_factory(
+            compression=compression,
+            allow_overwrite=allow_overwrite,
+            write_arrow_binary=write_arrow_binary,
+            file_visitor=file_visitor,
+        )
+        self._properties["rb_sink_factory"] = sink_factory
+
         self._parquet_output_adapter_manager = _parquetadapterimpl._parquet_output_adapter_manager(
             engine, self._properties
         )

From 31cbb239d292303035d3e06a13d7e6dbcccb8912 Mon Sep 17 00:00:00 2001
From: Arham Chopra <arham.chopra@cubistsystematic.com>
Date: Thu, 11 Jun 2026 12:17:24 -0400
Subject: [PATCH 02/11] Clean up parquet output: drop dead code, fix split
 metadata

Remove dead StructColumnArrayBuilder and parquet_dict_basket_output_adapter,
de-virtualize scheduleEndCycleEvent, propagate file metadata to per-column
files in split mode, and expand output tests.

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>
---
 .../parquet/ArrowBackedArrayBuilder.h         |    2 +-
 .../parquet/ArrowSingleColumnArrayBuilder.h   |   87 --
 cpp/csp/adapters/parquet/ParquetWriter.h      |    2 +-
 .../python/adapters/parquetadapterimpl.cpp    |   26 -
 csp/adapters/_parquet_rb_writer.py            |   11 +-
 csp/adapters/output_adapters/parquet.py       |    3 +-
 csp/tests/adapters/test_parquet_output.py     | 1244 +++++++++++++++++
 7 files changed, 1255 insertions(+), 120 deletions(-)
 create mode 100644 csp/tests/adapters/test_parquet_output.py

diff --git a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
index a8e91ed31..b0c970da8 100644
--- a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
+++ b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
@@ -3,7 +3,7 @@
 
 #include <csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h>
 
-// Forward declare to avoid namespace conflicts (see ArrowBackedColumnAdapter.h)
+// Forward declare FieldWriter (defined in csp/adapters/arrow/ArrowFieldWriter.h) to keep this header light.
 namespace csp::adapters::arrow { class FieldWriter; }
 
 namespace csp::adapters::parquet
diff --git a/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h b/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
index 16999bedc..00e564a8f 100644
--- a/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
+++ b/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
@@ -43,93 +43,6 @@ class ArrowSingleColumnArrayBuilder
     const std::uint32_t m_chunkSize;
 };
 
-class StructColumnArrayBuilder : public ArrowSingleColumnArrayBuilder
-{
-public:
-    using ColumnBuilderPtr = std::shared_ptr<ArrowSingleColumnArrayBuilder>;
-    using FieldValueSetter = std::function<void( const Struct * )>;
-
-    StructColumnArrayBuilder( std::string columnName, std::uint32_t chunkSize,
-                              const std::shared_ptr<::arrow::DataType> &type,
-                              const std::vector<ColumnBuilderPtr> &childArrayBuilders,
-                              FieldValueSetter fieldValueSetter )
-            : ArrowSingleColumnArrayBuilder( std::move( columnName ), chunkSize ),
-              m_childArrayBuilders( childArrayBuilders ),
-              m_builderPtr( std::make_shared<::arrow::StructBuilder>( type, ::arrow::default_memory_pool(),
-                                                                      getArrowChildArrayBuilders( childArrayBuilders ) ) ),
-              m_fieldValueSetter( fieldValueSetter ),
-              m_hasValue( false )
-    {
-    }
-
-    virtual std::shared_ptr<::arrow::DataType> getDataType() override
-    {
-        return m_builderPtr -> type();
-    }
-
-    virtual std::shared_ptr<::arrow::ArrayBuilder> getBuilder() override
-    {
-        return m_builderPtr;
-    }
-
-    virtual int64_t length() const override
-    {
-        return m_builderPtr -> length();
-    }
-
-    // Called for each row of the output parquet row, if no value was provided a null should be appended
-    virtual void handleRowFinished() override
-    {
-        if( m_hasValue )
-        {
-            m_hasValue = false;
-            for( auto &childBuilder : m_childArrayBuilders )
-            {
-                childBuilder -> handleRowFinished();
-            }
-            CSP_TRUE_OR_THROW_RUNTIME( m_builderPtr -> Append().ok(), "Failed to append struct" );
-        }
-        else
-        {
-            STATUS_OK_OR_THROW_RUNTIME( m_builderPtr -> AppendNull(), "Failed to create arrow array" );
-        }
-    }
-
-    // Release the array from the currently built values
-    virtual std::shared_ptr<::arrow::Array> buildArray() override
-    {
-        std::shared_ptr<::arrow::Array> array;
-        STATUS_OK_OR_THROW_RUNTIME( m_builderPtr -> Finish( &array ), "Failed to create arrow array" );
-        return array;
-    }
-
-    void setValue( const Struct *value )
-    {
-        m_hasValue = true;
-        m_fieldValueSetter( value );
-    }
-
-private:
-    static std::vector<std::shared_ptr<::arrow::ArrayBuilder>> getArrowChildArrayBuilders(
-            const std::vector<ColumnBuilderPtr> &childArrayBuilders )
-    {
-        std::vector<std::shared_ptr<::arrow::ArrayBuilder>> res;
-
-        for( auto &columnBuilder : childArrayBuilders )
-        {
-            res.push_back( columnBuilder -> getBuilder() );
-        }
-        return res;
-    }
-
-private:
-    std::vector<ColumnBuilderPtr>           m_childArrayBuilders;
-    std::shared_ptr<::arrow::StructBuilder> m_builderPtr;
-    FieldValueSetter                        m_fieldValueSetter;
-    bool                                    m_hasValue;
-};
-
-
 class ListColumnArrayBuilder : public ArrowSingleColumnArrayBuilder
 {
 public:
diff --git a/cpp/csp/adapters/parquet/ParquetWriter.h b/cpp/csp/adapters/parquet/ParquetWriter.h
index 3d779bd71..30d03b9a9 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.h
+++ b/cpp/csp/adapters/parquet/ParquetWriter.h
@@ -53,7 +53,7 @@ class ParquetWriter : public EndCycleListener
 
     std::uint32_t getChunkSize() const{ return m_adapterMgr.getBatchSize(); }
 
-    virtual void scheduleEndCycleEvent()
+    void scheduleEndCycleEvent()
     {
         m_adapterMgr.scheduleEndCycle();
     }
diff --git a/cpp/csp/python/adapters/parquetadapterimpl.cpp b/cpp/csp/python/adapters/parquetadapterimpl.cpp
index bf1bfcb95..5b87db8b2 100644
--- a/cpp/csp/python/adapters/parquetadapterimpl.cpp
+++ b/cpp/csp/python/adapters/parquetadapterimpl.cpp
@@ -611,30 +611,6 @@ static OutputAdapter *create_parquet_output_adapter( csp::AdapterManager *manage
     }
 }
 
-static OutputAdapter *create_parquet_dict_basket_output_adapter( csp::AdapterManager *manager, PyEngine *pyengine, PyObject *args )
-{
-    PyObject *pyProperties;
-    PyObject *pyType;
-    auto     *parquetManager = dynamic_cast<ParquetOutputAdapterManager *>( manager );
-    if( !parquetManager )
-        CSP_THROW( TypeError, "Expected ParquetOutputAdapterManager" );
-    if( !PyArg_ParseTuple( args, "O!O!",
-                           &PyTuple_Type, &pyType,
-                           &PyDict_Type, &pyProperties ) )
-        CSP_THROW( PythonPassthrough, "" );
-    PyObject *keyType;
-    PyObject *valueType;
-    if( !PyArg_ParseTuple( pyType, "O!O!",
-                           &PyType_Type, &keyType,
-                           &PyType_Type, &valueType ) )
-        CSP_THROW( PythonPassthrough, "Invalid basket key/value tuple" );
-
-    auto cspKeyType   = pyTypeAsCspType( keyType );
-    auto cspValueType = pyTypeAsCspType( valueType );
-
-    CSP_THROW( NotImplemented, "Output basket is not implement yet" );
-}
-
 static OutputAdapter *parquet_output_filename_adapter( csp::AdapterManager *manager, PyEngine *pyengine, PyObject *args )
 {
     auto *parquetManager = dynamic_cast<ParquetOutputAdapterManager *>( manager );
@@ -830,8 +806,6 @@ REGISTER_INPUT_ADAPTER( _parquet_input_adapter, create_parquet_input_adapter );
 
 REGISTER_OUTPUT_ADAPTER( _parquet_output_adapter, create_parquet_output_adapter );
 
-REGISTER_OUTPUT_ADAPTER( _parquet_dict_basket_output_adapter, create_parquet_dict_basket_output_adapter );
-
 REGISTER_OUTPUT_ADAPTER( _parquet_output_filename_adapter, parquet_output_filename_adapter );
 static PyModuleDef _parquetadapterimpl_module = {
         PyModuleDef_HEAD_INIT,
diff --git a/csp/adapters/_parquet_rb_writer.py b/csp/adapters/_parquet_rb_writer.py
index 8df211a33..56e980ebf 100644
--- a/csp/adapters/_parquet_rb_writer.py
+++ b/csp/adapters/_parquet_rb_writer.py
@@ -103,16 +103,21 @@ def factory(path, schema):
     return factory
 
 
-def _split_columns_writer_factory(compression, allow_overwrite, write_arrow_binary):
+def _split_columns_writer_factory(compression, allow_overwrite, write_arrow_binary, file_metadata=None):
     compression = compression or "none"
     ext = ".arrow" if write_arrow_binary else ".parquet"
 
     def factory(dir_name, schema):
         if dir_name:
             os.makedirs(dir_name, exist_ok=True)
+        # Carry file-level metadata from the parent schema (or explicit param) to each per-column file
+        meta = schema.metadata
+        if file_metadata:
+            existing = meta or {}
+            meta = {**existing, **{k.encode(): v.encode() for k, v in file_metadata.items()}}
         writers = {}
         for field in schema:
-            col_schema = pa.schema([field])
+            col_schema = pa.schema([field], metadata=meta)
             file_path = os.path.join(dir_name, field.name + ext)
             if not allow_overwrite and os.path.exists(file_path):
                 raise FileExistsError(f"File already exists: {file_path}")
@@ -177,7 +182,7 @@ def create_sink(file_name, compression, allow_overwrite, write_arrow_binary,
                 file_metadata=None, column_metadata=None):
     """Factory function to create the appropriate sink based on config."""
     if split_columns_to_files:
-        wf = _split_columns_writer_factory(compression, allow_overwrite, write_arrow_binary)
+        wf = _split_columns_writer_factory(compression, allow_overwrite, write_arrow_binary, file_metadata)
     elif write_arrow_binary:
         wf = _arrow_ipc_writer_factory(compression, allow_overwrite)
     else:
diff --git a/csp/adapters/output_adapters/parquet.py b/csp/adapters/output_adapters/parquet.py
index b7167a365..24acf0b5f 100644
--- a/csp/adapters/output_adapters/parquet.py
+++ b/csp/adapters/output_adapters/parquet.py
@@ -248,12 +248,11 @@ def _create(self, engine, memo):
         )
         self._properties["rb_sink"] = sink
 
-        # Sink factory for dict basket writers
+        # Sink factory for dict basket writers (no file_visitor — only main sink visits)
         sink_factory = create_sink_factory(
             compression=compression,
             allow_overwrite=allow_overwrite,
             write_arrow_binary=write_arrow_binary,
-            file_visitor=file_visitor,
         )
         self._properties["rb_sink_factory"] = sink_factory
 
diff --git a/csp/tests/adapters/test_parquet_output.py b/csp/tests/adapters/test_parquet_output.py
new file mode 100644
index 000000000..18ee01dd3
--- /dev/null
+++ b/csp/tests/adapters/test_parquet_output.py
@@ -0,0 +1,1244 @@
+"""Comprehensive tests for the parquet output adapter.
+
+Tests all output scenarios: scalar types, structs, file rotation, split columns,
+Arrow IPC, dict baskets, numpy arrays, batch sizes, compression, metadata, etc.
+"""
+
+import os
+import tempfile
+import unittest
+from datetime import date, datetime, time, timedelta
+
+import numpy
+import pandas
+import pyarrow
+import pyarrow.ipc
+import pyarrow.parquet
+import pytz
+
+import csp
+from csp.adapters.output_adapters.parquet import ParquetOutputConfig
+from csp.adapters.parquet import ParquetReader, ParquetWriter
+
+
+START = datetime(2022, 1, 1, tzinfo=pytz.utc)
+
+
+class TestOutputScalarTypes(unittest.TestCase):
+    """Test writing all scalar types to parquet and reading them back."""
+
+    def _write_and_read_scalar(self, col_type, values, column_name="value"):
+        """Helper: write a scalar column and read it back via pandas."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+            curve_data = [(timedelta(seconds=i + 1), v) for i, v in enumerate(values)]
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish(column_name, csp.curve(col_type, curve_data))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=len(values) + 2))
+            df = pandas.read_parquet(fname)
+            return df
+
+    def test_int(self):
+        df = self._write_and_read_scalar(int, [1, 2, 3, -100, 0, 2**60])
+        self.assertEqual(df["value"].tolist(), [1, 2, 3, -100, 0, 2**60])
+
+    def test_float(self):
+        df = self._write_and_read_scalar(float, [1.1, 2.2, 0.0, -3.14, float("inf")])
+        self.assertAlmostEqual(df["value"].iloc[0], 1.1)
+        self.assertAlmostEqual(df["value"].iloc[3], -3.14)
+        self.assertEqual(df["value"].iloc[4], float("inf"))
+
+    def test_bool(self):
+        df = self._write_and_read_scalar(bool, [True, False, True, True, False])
+        self.assertEqual(df["value"].tolist(), [True, False, True, True, False])
+
+    def test_string(self):
+        df = self._write_and_read_scalar(str, ["hello", "world", "", "with spaces", "unicode: 日本語"])
+        self.assertEqual(df["value"].tolist(), ["hello", "world", "", "with spaces", "unicode: 日本語"])
+
+    def test_datetime(self):
+        dts = [datetime(2022, 1, i + 1, tzinfo=pytz.utc) for i in range(5)]
+        df = self._write_and_read_scalar(datetime, dts)
+        for i, dt in enumerate(dts):
+            self.assertEqual(df["value"].iloc[i].to_pydatetime(), dt)
+
+    def test_timedelta(self):
+        tds = [timedelta(seconds=1), timedelta(minutes=5), timedelta(days=1), timedelta(microseconds=500)]
+        df = self._write_and_read_scalar(timedelta, tds)
+        for i, td in enumerate(tds):
+            self.assertEqual(df["value"].iloc[i], td)
+
+    def test_date(self):
+        dates = [date(2022, 1, 1), date(2022, 6, 15), date(2023, 12, 31)]
+        df = self._write_and_read_scalar(date, dates)
+        for i, d in enumerate(dates):
+            self.assertEqual(df["value"].iloc[i], d)
+
+    def test_time(self):
+        times = [time(9, 30, 0), time(12, 0, 0), time(23, 59, 59)]
+        # Use csp reader for proper round-trip (pandas reads time64 differently)
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+            curve_data = [(timedelta(seconds=i + 1), t) for i, t in enumerate(times)]
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish("value", csp.curve(time, curve_data))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(fname, time_column="csp_timestamp")
+                csp.add_graph_output("value", reader.subscribe_all(time, "value"))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=10))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=10))
+            read_times = [v[1] for v in res["value"]]
+            self.assertEqual(read_times, times)
+
+    def test_enum(self):
+        class Color(csp.Enum):
+            RED = 1
+            GREEN = 2
+            BLUE = 3
+
+        values = [Color.RED, Color.GREEN, Color.BLUE, Color.RED]
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+            curve_data = [(timedelta(seconds=i + 1), v) for i, v in enumerate(values)]
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish("color", csp.curve(Color, curve_data))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(fname, time_column="csp_timestamp")
+                csp.add_graph_output("color", reader.subscribe_all(Color, "color"))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=10))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=10))
+            self.assertEqual([v[1] for v in res["color"]], values)
+
+    def test_bytes(self):
+        values = [b"hello", b"\x00\x01\x02", b"", b"binary data"]
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+            curve_data = [(timedelta(seconds=i + 1), v) for i, v in enumerate(values)]
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish("data", csp.curve(bytes, curve_data))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(fname, time_column="csp_timestamp")
+                csp.add_graph_output("data", reader.subscribe_all(bytes, "data"))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=10))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=10))
+            self.assertEqual([v[1] for v in res["data"]], values)
+
+    def test_multiple_columns_different_types(self):
+        """Write multiple columns of different types simultaneously."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish("i", csp.curve(int, [(timedelta(seconds=1), 42), (timedelta(seconds=2), 99)]))
+                writer.publish("f", csp.curve(float, [(timedelta(seconds=1), 3.14), (timedelta(seconds=2), 2.71)]))
+                writer.publish("s", csp.curve(str, [(timedelta(seconds=1), "a"), (timedelta(seconds=2), "b")]))
+                writer.publish("b", csp.curve(bool, [(timedelta(seconds=1), True), (timedelta(seconds=2), False)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            df = pandas.read_parquet(fname)
+            self.assertEqual(df["i"].tolist(), [42, 99])
+            self.assertEqual(df["f"].tolist(), [3.14, 2.71])
+            self.assertEqual(df["s"].tolist(), ["a", "b"])
+            self.assertEqual(df["b"].tolist(), [True, False])
+
+    def test_no_timestamp_column(self):
+        """Write without a timestamp column."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, None, config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(5)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=10))
+            df = pandas.read_parquet(fname)
+            self.assertNotIn("csp_timestamp", df.columns)
+            self.assertEqual(df["x"].tolist(), list(range(5)))
+
+
+class TestOutputStruct(unittest.TestCase):
+    """Test struct publishing."""
+
+    def test_basic_struct(self):
+        class Trade(csp.Struct):
+            price: float
+            size: int
+            exchange: str
+
+        values = [
+            Trade(price=100.5, size=10, exchange="NYSE"),
+            Trade(price=101.0, size=20, exchange="ARCA"),
+            Trade(price=99.5, size=15, exchange="NYSE"),
+        ]
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish_struct(csp.curve(Trade, [(timedelta(seconds=i + 1), v) for i, v in enumerate(values)]))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(fname, time_column="csp_timestamp")
+                csp.add_graph_output("data", reader.subscribe_all(Trade))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=10))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=10))
+            read_values = [v[1] for v in res["data"]]
+            self.assertEqual(read_values, values)
+
+    def test_struct_with_field_map(self):
+        class Trade(csp.Struct):
+            price: float
+            size: int
+
+        values = [Trade(price=100.0, size=10), Trade(price=200.0, size=20)]
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish_struct(
+                    csp.curve(Trade, [(timedelta(seconds=i + 1), v) for i, v in enumerate(values)]),
+                    field_map={"price": "trade_price", "size": "trade_size"},
+                )
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            df = pandas.read_parquet(fname)
+            self.assertIn("trade_price", df.columns)
+            self.assertIn("trade_size", df.columns)
+            self.assertNotIn("price", df.columns)
+            self.assertEqual(df["trade_price"].tolist(), [100.0, 200.0])
+            self.assertEqual(df["trade_size"].tolist(), [10, 20])
+
+    def test_wide_struct(self):
+        class Wide(csp.Struct):
+            f0: float
+            f1: float
+            f2: float
+            f3: float
+            f4: float
+            f5: int
+            f6: int
+            f7: str
+            f8: bool
+            f9: float
+
+        v = Wide(f0=1.0, f1=2.0, f2=3.0, f3=4.0, f4=5.0, f5=6, f6=7, f7="eight", f8=True, f9=10.0)
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish_struct(csp.const(v))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=2))
+            df = pandas.read_parquet(fname)
+            self.assertEqual(len(df), 1)
+            self.assertEqual(df["f0"].iloc[0], 1.0)
+            self.assertEqual(df["f7"].iloc[0], "eight")
+            self.assertEqual(df["f8"].iloc[0], True)
+
+
+class TestOutputFileRotation(unittest.TestCase):
+    """Test file rotation via filename_provider."""
+
+    def test_basic_rotation(self):
+        with tempfile.TemporaryDirectory() as d:
+            @csp.graph
+            def g():
+                data = csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(10)])
+                rotate = csp.curve(str, [
+                    (timedelta(seconds=5), os.path.join(d, "part1.parquet")),
+                ])
+                writer = ParquetWriter(
+                    os.path.join(d, "part0.parquet"), "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    filename_provider=rotate,
+                )
+                writer.publish("value", data)
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=15))
+            df0 = pandas.read_parquet(os.path.join(d, "part0.parquet"))
+            df1 = pandas.read_parquet(os.path.join(d, "part1.parquet"))
+            self.assertEqual(df0["value"].tolist(), [0, 1, 2, 3])
+            self.assertEqual(df1["value"].tolist(), [4, 5, 6, 7, 8, 9])
+
+    def test_multiple_rotations(self):
+        with tempfile.TemporaryDirectory() as d:
+            @csp.graph
+            def g():
+                data = csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(9)])
+                rotate = csp.curve(str, [
+                    (timedelta(seconds=4), os.path.join(d, "p1.parquet")),
+                    (timedelta(seconds=7), os.path.join(d, "p2.parquet")),
+                ])
+                writer = ParquetWriter(
+                    os.path.join(d, "p0.parquet"), "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    filename_provider=rotate,
+                )
+                writer.publish("value", data)
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=15))
+            df0 = pandas.read_parquet(os.path.join(d, "p0.parquet"))
+            df1 = pandas.read_parquet(os.path.join(d, "p1.parquet"))
+            df2 = pandas.read_parquet(os.path.join(d, "p2.parquet"))
+            self.assertEqual(df0["value"].tolist(), [0, 1, 2])
+            self.assertEqual(df1["value"].tolist(), [3, 4, 5])
+            self.assertEqual(df2["value"].tolist(), [6, 7, 8])
+
+    def test_file_visitor_called(self):
+        visited = []
+        with tempfile.TemporaryDirectory() as d:
+            @csp.graph
+            def g():
+                data = csp.curve(int, [(timedelta(seconds=1), 1), (timedelta(seconds=3), 2)])
+                rotate = csp.curve(str, [(timedelta(seconds=2), os.path.join(d, "p1.parquet"))])
+                writer = ParquetWriter(
+                    os.path.join(d, "p0.parquet"), "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    filename_provider=rotate,
+                    file_visitor=lambda f: visited.append(f),
+                )
+                writer.publish("value", data)
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            # file_visitor called for p0 when rotated, and p1 on stop
+            self.assertEqual(len(visited), 2)
+            self.assertTrue(visited[0].endswith("p0.parquet"))
+            self.assertTrue(visited[1].endswith("p1.parquet"))
+
+    def test_rotation_no_initial_file(self):
+        """filename_provider only, no initial file_name."""
+        with tempfile.TemporaryDirectory() as d:
+            @csp.graph
+            def g():
+                data = csp.curve(int, [(timedelta(seconds=2), 1), (timedelta(seconds=3), 2)])
+                rotate = csp.curve(str, [(timedelta(seconds=1), os.path.join(d, "output.parquet"))])
+                writer = ParquetWriter(
+                    None, "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    filename_provider=rotate,
+                )
+                writer.publish("value", data)
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            df = pandas.read_parquet(os.path.join(d, "output.parquet"))
+            self.assertEqual(df["value"].tolist(), [1, 2])
+
+
+class TestOutputArrowIPC(unittest.TestCase):
+    """Test Arrow IPC (binary arrow) output format."""
+
+    def test_basic_ipc(self):
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.arrow")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=True, write_arrow_binary=True, compression="")
+                writer = ParquetWriter(fname, "csp_timestamp", config=config)
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=i + 1), i * 10) for i in range(5)]))
+                writer.publish("y", csp.curve(float, [(timedelta(seconds=i + 1), i * 1.5) for i in range(5)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=10))
+            reader = pyarrow.ipc.open_stream(pyarrow.memory_map(fname))
+            table = reader.read_all()
+            self.assertEqual(table.column("x").to_pylist(), [0, 10, 20, 30, 40])
+            self.assertEqual(table.column("y").to_pylist(), [0.0, 1.5, 3.0, 4.5, 6.0])
+
+    def test_ipc_with_compression(self):
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.arrow")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=True, write_arrow_binary=True, compression="zstd")
+                writer = ParquetWriter(fname, "csp_timestamp", config=config)
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 42)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            reader = pyarrow.ipc.open_stream(pyarrow.memory_map(fname))
+            table = reader.read_all()
+            self.assertEqual(table.column("x").to_pylist(), [42])
+
+    def test_ipc_round_trip_via_csp(self):
+        """Write IPC and read back via csp ParquetReader."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.arrow")
+            values = [10, 20, 30]
+
+            @csp.graph
+            def g_write():
+                config = ParquetOutputConfig(allow_overwrite=True, write_arrow_binary=True, compression="")
+                writer = ParquetWriter(fname, "csp_timestamp", config=config)
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=i + 1), v) for i, v in enumerate(values)]))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(fname, time_column="csp_timestamp", binary_arrow=True)
+                csp.add_graph_output("x", reader.subscribe_all(int, "x"))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=10))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=10))
+            self.assertEqual([v[1] for v in res["x"]], values)
+
+
+class TestOutputSplitColumns(unittest.TestCase):
+    """Test split_columns_to_files output."""
+
+    def test_split_scalars(self):
+        with tempfile.TemporaryDirectory() as d:
+            outdir = os.path.join(d, "split_out")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(
+                    outdir, "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                )
+                writer.publish("a", csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(5)]))
+                writer.publish("b", csp.curve(float, [(timedelta(seconds=i + 1), i * 2.0) for i in range(5)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=10))
+            self.assertTrue(os.path.isdir(outdir))
+            self.assertTrue(os.path.isfile(os.path.join(outdir, "a.parquet")))
+            self.assertTrue(os.path.isfile(os.path.join(outdir, "b.parquet")))
+            self.assertTrue(os.path.isfile(os.path.join(outdir, "csp_timestamp.parquet")))
+
+            # Read back and verify
+            df_a = pandas.read_parquet(os.path.join(outdir, "a.parquet"))
+            df_b = pandas.read_parquet(os.path.join(outdir, "b.parquet"))
+            self.assertEqual(df_a["a"].tolist(), list(range(5)))
+            self.assertEqual(df_b["b"].tolist(), [0.0, 2.0, 4.0, 6.0, 8.0])
+
+    def test_split_round_trip_via_csp(self):
+        """Write split columns and read back via csp ParquetReader."""
+        with tempfile.TemporaryDirectory() as d:
+            outdir = os.path.join(d, "split_data")
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(
+                    outdir, "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                )
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=i + 1), i * 10) for i in range(3)]))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(outdir, time_column="csp_timestamp", split_columns_to_files=True)
+                csp.add_graph_output("x", reader.subscribe_all(int, "x"))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=10))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=10))
+            self.assertEqual([v[1] for v in res["x"]], [0, 10, 20])
+
+    def test_split_struct(self):
+        class S(csp.Struct):
+            a: int
+            b: float
+
+        with tempfile.TemporaryDirectory() as d:
+            outdir = os.path.join(d, "split_struct")
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(
+                    outdir, "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                )
+                writer.publish_struct(csp.curve(S, [
+                    (timedelta(seconds=1), S(a=1, b=1.1)),
+                    (timedelta(seconds=2), S(a=2, b=2.2)),
+                ]))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(outdir, time_column="csp_timestamp", split_columns_to_files=True)
+                csp.add_graph_output("data", reader.subscribe_all(S))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=5))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=5))
+            vals = [v[1] for v in res["data"]]
+            self.assertEqual(vals[0], S(a=1, b=1.1))
+            self.assertEqual(vals[1], S(a=2, b=2.2))
+
+    def test_split_ipc_format(self):
+        """split_columns_to_files with write_arrow_binary=True produces .arrow files."""
+        with tempfile.TemporaryDirectory() as d:
+            outdir = os.path.join(d, "split_ipc")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=True, write_arrow_binary=True, compression="")
+                writer = ParquetWriter(outdir, "csp_timestamp", config=config, split_columns_to_files=True)
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 42)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            self.assertTrue(os.path.isfile(os.path.join(outdir, "x.arrow")))
+            self.assertTrue(os.path.isfile(os.path.join(outdir, "csp_timestamp.arrow")))
+
+
+class TestOutputDictBasket(unittest.TestCase):
+    """Test dict basket output."""
+
+    def test_basic_dict_basket(self):
+        with tempfile.TemporaryDirectory() as d:
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(
+                    os.path.join(d, "basket"),
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                )
+                basket = {
+                    "AAPL": csp.curve(float, [(timedelta(seconds=1), 150.0), (timedelta(seconds=2), 151.0)]),
+                    "IBM": csp.curve(float, [(timedelta(seconds=1), 130.0), (timedelta(seconds=3), 131.0)]),
+                }
+                writer.publish_dict_basket("price", basket, str, float)
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(
+                    os.path.join(d, "basket"),
+                    time_column="csp_timestamp",
+                    split_columns_to_files=True,
+                )
+                basket = reader.subscribe_dict_basket(float, "price", ["AAPL", "IBM"])
+                csp.add_graph_output("AAPL", basket["AAPL"])
+                csp.add_graph_output("IBM", basket["IBM"])
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=5))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=5))
+            aapl = [v[1] for v in res["AAPL"]]
+            ibm = [v[1] for v in res["IBM"]]
+            self.assertEqual(aapl, [150.0, 151.0])
+            self.assertEqual(ibm, [130.0, 131.0])
+
+    def test_dict_basket_many_symbols(self):
+        n_symbols = 20
+        symbols = [f"SYM{i}" for i in range(n_symbols)]
+
+        with tempfile.TemporaryDirectory() as d:
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(
+                    os.path.join(d, "basket"),
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                )
+                basket = {
+                    sym: csp.curve(float, [(timedelta(seconds=1), float(i))])
+                    for i, sym in enumerate(symbols)
+                }
+                writer.publish_dict_basket("val", basket, str, float)
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(
+                    os.path.join(d, "basket"),
+                    time_column="csp_timestamp",
+                    split_columns_to_files=True,
+                )
+                basket = reader.subscribe_dict_basket(float, "val", symbols)
+                for sym in symbols:
+                    csp.add_graph_output(sym, basket[sym])
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=5))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=5))
+            for i, sym in enumerate(symbols):
+                self.assertEqual([v[1] for v in res[sym]], [float(i)])
+
+
+class TestOutputNumpyArrays(unittest.TestCase):
+    """Test numpy array column output."""
+
+    def test_1d_float_array(self):
+        arrays = [numpy.array([1.0, 2.0, 3.0]), numpy.array([4.0, 5.0, 6.0])]
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish("arr", csp.curve(
+                    csp.typing.Numpy1DArray[float],
+                    [(timedelta(seconds=i + 1), a) for i, a in enumerate(arrays)],
+                ))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(fname, time_column="csp_timestamp")
+                csp.add_graph_output("arr", reader.subscribe_all(csp.typing.Numpy1DArray[float], "arr"))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=5))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=5))
+            for i, (_, arr) in enumerate(res["arr"]):
+                numpy.testing.assert_array_equal(arr, arrays[i])
+
+    def test_1d_int_array(self):
+        arrays = [numpy.array([10, 20, 30], dtype="int64"), numpy.array([40, 50], dtype="int64")]
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish("arr", csp.curve(
+                    csp.typing.Numpy1DArray[int],
+                    [(timedelta(seconds=i + 1), a) for i, a in enumerate(arrays)],
+                ))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(fname, time_column="csp_timestamp")
+                csp.add_graph_output("arr", reader.subscribe_all(csp.typing.Numpy1DArray[int], "arr"))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=5))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=5))
+            for i, (_, arr) in enumerate(res["arr"]):
+                numpy.testing.assert_array_equal(arr, arrays[i])
+
+    def test_2d_ndarray(self):
+        class NDStruct(csp.Struct):
+            arr: csp.typing.NumpyNDArray[float]
+
+        arr_2d = numpy.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish_struct(csp.const(NDStruct(arr=arr_2d)))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(fname, time_column="csp_timestamp")
+                csp.add_graph_output("data", reader.subscribe_all(NDStruct))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=2))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=2))
+            result_arr = res["data"][0][1].arr
+            self.assertEqual(result_arr.shape, arr_2d.shape)
+            numpy.testing.assert_array_equal(result_arr, arr_2d)
+
+    def test_3d_ndarray(self):
+        class NDStruct(csp.Struct):
+            arr: csp.typing.NumpyNDArray[float]
+
+        arr_3d = numpy.arange(24.0).reshape((2, 3, 4))
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish_struct(csp.const(NDStruct(arr=arr_3d)))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(fname, time_column="csp_timestamp")
+                csp.add_graph_output("data", reader.subscribe_all(NDStruct))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=2))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=2))
+            result_arr = res["data"][0][1].arr
+            self.assertEqual(result_arr.shape, arr_3d.shape)
+            numpy.testing.assert_array_equal(result_arr, arr_3d)
+
+
+class TestOutputCompression(unittest.TestCase):
+    """Test various compression codecs — verify codec is actually applied."""
+
+    def _write_and_check_codec(self, compression, expected_codec, n_rows=100):
+        """Write with compression and verify the codec in file metadata."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=True, compression=compression)
+                writer = ParquetWriter(fname, "csp_timestamp", config=config)
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(n_rows)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=n_rows + 5))
+            pf = pyarrow.parquet.ParquetFile(fname)
+            # Check actual codec in row group metadata
+            actual_codec = pf.metadata.row_group(0).column(0).compression
+            self.assertEqual(actual_codec, expected_codec)
+            # Verify data integrity
+            df = pandas.read_parquet(fname)
+            self.assertEqual(df["x"].tolist(), list(range(n_rows)))
+
+    def test_snappy(self):
+        self._write_and_check_codec("snappy", "SNAPPY")
+
+    def test_gzip(self):
+        self._write_and_check_codec("gzip", "GZIP")
+
+    def test_zstd(self):
+        self._write_and_check_codec("zstd", "ZSTD")
+
+    def test_none(self):
+        self._write_and_check_codec("", "UNCOMPRESSED")
+
+
+class TestOutputBatchSize(unittest.TestCase):
+    """Test batch_size controls row group flushing."""
+
+    def _write_and_check_row_groups(self, batch_size, n_rows):
+        """Write and verify row group count matches batch_size."""
+        import math
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=True, batch_size=batch_size)
+                writer = ParquetWriter(fname, "csp_timestamp", config=config)
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(n_rows)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=n_rows + 5))
+            pf = pyarrow.parquet.ParquetFile(fname)
+            expected_row_groups = math.ceil(n_rows / batch_size)
+            self.assertEqual(pf.metadata.num_row_groups, expected_row_groups)
+            # Verify data integrity
+            df = pandas.read_parquet(fname)
+            self.assertEqual(df["x"].tolist(), list(range(n_rows)))
+
+    def test_tiny_batch(self):
+        self._write_and_check_row_groups(1, 10)
+
+    def test_small_batch(self):
+        self._write_and_check_row_groups(4, 20)
+
+    def test_batch_larger_than_data(self):
+        self._write_and_check_row_groups(1000, 10)
+
+    def test_batch_exactly_divides_data(self):
+        self._write_and_check_row_groups(5, 15)
+
+    def test_partial_final_batch(self):
+        """n_rows not evenly divisible → final partial batch flushed at stop()."""
+        self._write_and_check_row_groups(7, 20)  # 20/7 = 2 full + 1 partial = 3 row groups
+
+
+class TestOutputMetadata(unittest.TestCase):
+    """Test file and column metadata."""
+
+    def test_file_metadata(self):
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(
+                    fname, "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    file_metadata={"created_by": "test", "version": "1.0"},
+                )
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 1)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            pf = pyarrow.parquet.ParquetFile(fname)
+            metadata = pf.schema_arrow.metadata
+            self.assertEqual(metadata[b"created_by"], b"test")
+            self.assertEqual(metadata[b"version"], b"1.0")
+
+    def test_column_metadata(self):
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(
+                    fname, "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    column_metadata={"x": {"units": "meters", "source": "sensor_1"}},
+                )
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 1)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            pf = pyarrow.parquet.ParquetFile(fname)
+            x_field = pf.schema_arrow.field("x")
+            self.assertEqual(x_field.metadata[b"units"], b"meters")
+            self.assertEqual(x_field.metadata[b"source"], b"sensor_1")
+
+    def test_file_metadata_in_split_mode(self):
+        """file_metadata is preserved in split-column mode (regression test for metadata propagation)."""
+        with tempfile.TemporaryDirectory() as d:
+            outdir = os.path.join(d, "split_meta")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(
+                    outdir, "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                    file_metadata={"author": "test_suite", "version": "2.0"},
+                    column_metadata={"x": {"units": "kg"}},
+                )
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 42)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+
+            # Each per-column file should carry file-level metadata
+            x_file = os.path.join(outdir, "x.parquet")
+            ts_file = os.path.join(outdir, "csp_timestamp.parquet")
+            self.assertTrue(os.path.isfile(x_file))
+            self.assertTrue(os.path.isfile(ts_file))
+
+            pf_x = pyarrow.parquet.ParquetFile(x_file)
+            pf_ts = pyarrow.parquet.ParquetFile(ts_file)
+
+            # File-level metadata on both files
+            self.assertEqual(pf_x.schema_arrow.metadata[b"author"], b"test_suite")
+            self.assertEqual(pf_x.schema_arrow.metadata[b"version"], b"2.0")
+            self.assertEqual(pf_ts.schema_arrow.metadata[b"author"], b"test_suite")
+            self.assertEqual(pf_ts.schema_arrow.metadata[b"version"], b"2.0")
+
+            # Column-level metadata preserved on x
+            x_field = pf_x.schema_arrow.field("x")
+            self.assertEqual(x_field.metadata[b"units"], b"kg")
+
+    def test_file_metadata_in_split_ipc_mode(self):
+        """file_metadata preserved in split-column + IPC mode."""
+        with tempfile.TemporaryDirectory() as d:
+            outdir = os.path.join(d, "split_ipc_meta")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=True, write_arrow_binary=True, compression="")
+                writer = ParquetWriter(
+                    outdir, "csp_timestamp", config=config,
+                    split_columns_to_files=True,
+                    file_metadata={"source": "ipc_test"},
+                )
+                writer.publish("val", csp.curve(int, [(timedelta(seconds=1), 99)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+
+            val_file = os.path.join(outdir, "val.arrow")
+            self.assertTrue(os.path.isfile(val_file))
+            reader = pyarrow.ipc.open_stream(pyarrow.memory_map(val_file))
+            self.assertEqual(reader.schema.metadata[b"source"], b"ipc_test")
+
+
+class TestOutputAllowOverwrite(unittest.TestCase):
+    """Test allow_overwrite behavior."""
+
+    def test_overwrite_true(self):
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+            # Write first file
+            with open(fname, "w") as f:
+                f.write("dummy")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=True)
+                writer = ParquetWriter(fname, "csp_timestamp", config=config)
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 42)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            df = pandas.read_parquet(fname)
+            self.assertEqual(df["x"].tolist(), [42])
+
+    def test_overwrite_false_raises(self):
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+            with open(fname, "w") as f:
+                f.write("dummy")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=False)
+                writer = ParquetWriter(fname, "csp_timestamp", config=config)
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 42)]))
+
+            with self.assertRaises(FileExistsError):
+                csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+
+
+class TestOutputEdgeCases(unittest.TestCase):
+    """Edge cases and special scenarios."""
+
+    def test_multiple_ticks_same_timestamp(self):
+        """Multiple values at the same engine time produce multiple rows."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                # csp.curve with same timedelta produces ticks at same engine time
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 10), (timedelta(seconds=1), 20), (timedelta(seconds=1), 30)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            df = pandas.read_parquet(fname)
+            self.assertEqual(df["x"].tolist(), [10, 20, 30])
+
+    def test_empty_output_no_ticks(self):
+        """No data ticked — file should still be created with schema."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                # Curve with data beyond endtime → no ticks
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=100), 1)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            df = pandas.read_parquet(fname)
+            self.assertEqual(len(df), 0)
+            self.assertIn("x", df.columns)
+
+    def test_large_number_of_columns(self):
+        """Write 100 columns."""
+        n_cols = 100
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                for c in range(n_cols):
+                    writer.publish(f"col_{c}", csp.curve(int, [(timedelta(seconds=1), c)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            df = pandas.read_parquet(fname)
+            self.assertEqual(len(df.columns), n_cols + 1)  # +1 for timestamp
+            for c in range(n_cols):
+                self.assertEqual(df[f"col_{c}"].iloc[0], c)
+
+    def test_creates_parent_directories(self):
+        """Writer creates parent directories if they don't exist."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "sub", "dir", "deep", "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 1)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            self.assertTrue(os.path.isfile(fname))
+
+    def test_duplicate_column_name_raises(self):
+        """Publishing same column name twice raises."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 1)]))
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 2)]))
+
+            with self.assertRaises(KeyError):
+                csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+
+    def test_very_long_strings(self):
+        """Write very long string values."""
+        long_str = "x" * 100_000
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish("s", csp.curve(str, [(timedelta(seconds=1), long_str)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            df = pandas.read_parquet(fname)
+            self.assertEqual(df["s"].iloc[0], long_str)
+
+    def test_write_read_round_trip_all_types(self):
+        """Full round-trip test with all supported types in one file."""
+
+        class MyEnum(csp.Enum):
+            X = 1
+            Y = 2
+
+        class FullStruct(csp.Struct):
+            i: int
+            f: float
+            b: bool
+            s: str
+            dt: datetime
+            td: timedelta
+            d: date
+            t: time
+            e: MyEnum
+
+        v = FullStruct(
+            i=42, f=3.14, b=True, s="hello",
+            dt=datetime(2022, 6, 15, 12, 0, tzinfo=pytz.utc),
+            td=timedelta(seconds=123),
+            d=date(2022, 6, 15),
+            t=time(12, 30, 45),
+            e=MyEnum.X,
+        )
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish_struct(csp.const(v))
+
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(fname, time_column="csp_timestamp")
+                csp.add_graph_output("data", reader.subscribe_all(FullStruct))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=2))
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=2))
+            read_v = res["data"][0][1]
+            self.assertEqual(read_v, v)
+
+    def test_float_nan_round_trip(self):
+        """NaN values round-trip correctly."""
+        import math
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish("x", csp.curve(float, [
+                    (timedelta(seconds=1), 1.0),
+                    (timedelta(seconds=2), float("nan")),
+                    (timedelta(seconds=3), 3.0),
+                ]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            df = pandas.read_parquet(fname)
+            self.assertEqual(df["x"].iloc[0], 1.0)
+            self.assertTrue(math.isnan(df["x"].iloc[1]))
+            self.assertEqual(df["x"].iloc[2], 3.0)
+
+    def test_struct_with_unset_fields_produces_nulls(self):
+        """Struct fields that are never set produce null cells."""
+        class Partial(csp.Struct):
+            a: int
+            b: float
+            c: str
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g_write():
+                # Only set 'a', leave 'b' and 'c' unset
+                v = Partial(a=42)
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish_struct(csp.const(v))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=2))
+            df = pandas.read_parquet(fname)
+            self.assertEqual(df["a"].iloc[0], 42)
+            self.assertTrue(pandas.isna(df["b"].iloc[0]))
+            self.assertTrue(pandas.isna(df["c"].iloc[0]))
+
+    def test_multi_batch_with_rotation(self):
+        """Rows spanning multiple batches + file rotation land in correct files."""
+        with tempfile.TemporaryDirectory() as d:
+            @csp.graph
+            def g():
+                # 20 rows, batch_size=4, rotate at row 10
+                data = csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(20)])
+                rotate = csp.curve(str, [(timedelta(seconds=11), os.path.join(d, "p1.parquet"))])
+                config = ParquetOutputConfig(allow_overwrite=True, batch_size=4)
+                writer = ParquetWriter(
+                    os.path.join(d, "p0.parquet"), "csp_timestamp",
+                    config=config, filename_provider=rotate,
+                )
+                writer.publish("value", data)
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=25))
+            df0 = pandas.read_parquet(os.path.join(d, "p0.parquet"))
+            df1 = pandas.read_parquet(os.path.join(d, "p1.parquet"))
+            # First 10 rows in p0, next 10 in p1
+            self.assertEqual(df0["value"].tolist(), list(range(10)))
+            self.assertEqual(df1["value"].tolist(), list(range(10, 20)))
+            # Verify batch_size=4 → correct row group counts
+            pf0 = pyarrow.parquet.ParquetFile(os.path.join(d, "p0.parquet"))
+            pf1 = pyarrow.parquet.ParquetFile(os.path.join(d, "p1.parquet"))
+            import math
+            self.assertEqual(pf0.metadata.num_row_groups, math.ceil(10 / 4))  # 3
+            self.assertEqual(pf1.metadata.num_row_groups, math.ceil(10 / 4))  # 3
+
+    def test_file_visitor_exact_contract(self):
+        """file_visitor called once per closed file, in order, including final at stop()."""
+        visited = []
+        with tempfile.TemporaryDirectory() as d:
+            @csp.graph
+            def g():
+                data = csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(9)])
+                rotate = csp.curve(str, [
+                    (timedelta(seconds=4), os.path.join(d, "p1.parquet")),
+                    (timedelta(seconds=7), os.path.join(d, "p2.parquet")),
+                ])
+                writer = ParquetWriter(
+                    os.path.join(d, "p0.parquet"), "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    filename_provider=rotate,
+                    file_visitor=lambda f: visited.append(f),
+                )
+                writer.publish("value", data)
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=15))
+            # Expect 3 visits: p0 (closed at rotation 1), p1 (closed at rotation 2), p2 (closed at stop)
+            self.assertEqual(len(visited), 3)
+            self.assertTrue(visited[0].endswith("p0.parquet"))
+            self.assertTrue(visited[1].endswith("p1.parquet"))
+            self.assertTrue(visited[2].endswith("p2.parquet"))
+            # Each visited file should be readable
+            for f in visited:
+                pandas.read_parquet(f)
+
+    def test_metadata_in_ipc_mode(self):
+        """file_metadata and column_metadata work in Arrow IPC mode."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.arrow")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=True, write_arrow_binary=True, compression="")
+                writer = ParquetWriter(
+                    fname, "csp_timestamp", config=config,
+                    file_metadata={"author": "test"},
+                    column_metadata={"x": {"units": "kg"}},
+                )
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 1)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+            reader = pyarrow.ipc.open_stream(pyarrow.memory_map(fname))
+            schema = reader.schema
+            self.assertEqual(schema.metadata[b"author"], b"test")
+            x_field = schema.field("x")
+            self.assertEqual(x_field.metadata[b"units"], b"kg")
+
+
+class TestOutputSinkDirect(unittest.TestCase):
+    """Direct unit tests for the Python RecordBatchSink without the csp engine."""
+
+    def test_sink_parquet_lifecycle(self):
+        """Test on_start → on_file_change → on_batch → on_stop lifecycle."""
+        from csp.adapters._parquet_rb_writer import RecordBatchSink, _parquet_writer_factory
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "test.parquet")
+            wf = _parquet_writer_factory("snappy", allow_overwrite=True)
+            sink = RecordBatchSink(wf)
+
+            schema = pyarrow.schema([pyarrow.field("x", pyarrow.int64())])
+            # Simulate C++ calling the sink
+            sink.on_start(schema.__arrow_c_schema__())
+            sink.on_file_change(fname)
+            # Write a batch
+            rb = pyarrow.record_batch([pyarrow.array([1, 2, 3])], schema=schema)
+            schema_capsule, array_capsule = rb.__arrow_c_array__()
+            sink.on_batch(schema_capsule, array_capsule)
+            sink.on_stop()
+
+            # Verify file was written
+            df = pandas.read_parquet(fname)
+            self.assertEqual(df["x"].tolist(), [1, 2, 3])
+
+    def test_sink_ipc_lifecycle(self):
+        """Test IPC sink lifecycle."""
+        from csp.adapters._parquet_rb_writer import RecordBatchSink, _arrow_ipc_writer_factory
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "test.arrow")
+            wf = _arrow_ipc_writer_factory("", allow_overwrite=True)
+            sink = RecordBatchSink(wf)
+
+            schema = pyarrow.schema([pyarrow.field("val", pyarrow.float64())])
+            sink.on_start(schema.__arrow_c_schema__())
+            sink.on_file_change(fname)
+            rb = pyarrow.record_batch([pyarrow.array([1.1, 2.2])], schema=schema)
+            schema_capsule, array_capsule = rb.__arrow_c_array__()
+            sink.on_batch(schema_capsule, array_capsule)
+            sink.on_stop()
+
+            reader = pyarrow.ipc.open_stream(pyarrow.memory_map(fname))
+            table = reader.read_all()
+            self.assertEqual(table.column("val").to_pylist(), [1.1, 2.2])
+
+    def test_sink_file_visitor_called_on_rotation(self):
+        """file_visitor called when rotating and on stop."""
+        from csp.adapters._parquet_rb_writer import RecordBatchSink, _parquet_writer_factory
+
+        visited = []
+        with tempfile.TemporaryDirectory() as d:
+            wf = _parquet_writer_factory("snappy", allow_overwrite=True)
+            sink = RecordBatchSink(wf, file_visitor=lambda f: visited.append(f))
+
+            schema = pyarrow.schema([pyarrow.field("x", pyarrow.int64())])
+            sink.on_start(schema.__arrow_c_schema__())
+
+            f1 = os.path.join(d, "f1.parquet")
+            f2 = os.path.join(d, "f2.parquet")
+            sink.on_file_change(f1)
+            rb = pyarrow.record_batch([pyarrow.array([1])], schema=schema)
+            s, a = rb.__arrow_c_array__()
+            sink.on_batch(s, a)
+            sink.on_file_change(f2)  # closes f1, visits f1
+            rb2 = pyarrow.record_batch([pyarrow.array([2])], schema=schema)
+            s2, a2 = rb2.__arrow_c_array__()
+            sink.on_batch(s2, a2)
+            sink.on_stop()  # closes f2, visits f2
+
+            self.assertEqual(visited, [f1, f2])
+
+
+if __name__ == "__main__":
+    unittest.main()

From eab7d985e5e6828fdbaef70662e266f30ac9626e Mon Sep 17 00:00:00 2001
From: Arham Chopra <arham.chopra@cubistsystematic.com>
Date: Fri, 12 Jun 2026 14:29:12 -0400
Subject: [PATCH 03/11] Move parquet output file I/O into C++
 (RecordBatchFileSink)

Closure-based C++ sink writes parquet/IPC/split-column files directly,
removing the per-batch C++<->Python hop. Adds FileExistsError.

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>
---
 cpp/csp/adapters/parquet/CMakeLists.txt       |   2 +
 .../adapters/parquet/RecordBatchFileSink.cpp  | 227 ++++++++++++++++++
 .../adapters/parquet/RecordBatchFileSink.h    |  29 +++
 cpp/csp/core/Exception.h                      |   1 +
 cpp/csp/python/Exception.h                    |   1 +
 .../python/adapters/parquetadapterimpl.cpp    | 138 +++--------
 6 files changed, 290 insertions(+), 108 deletions(-)
 create mode 100644 cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
 create mode 100644 cpp/csp/adapters/parquet/RecordBatchFileSink.h

diff --git a/cpp/csp/adapters/parquet/CMakeLists.txt b/cpp/csp/adapters/parquet/CMakeLists.txt
index ab452829d..a6312bb92 100644
--- a/cpp/csp/adapters/parquet/CMakeLists.txt
+++ b/cpp/csp/adapters/parquet/CMakeLists.txt
@@ -9,6 +9,7 @@ set(PARQUET_HEADER_FILES
         ParquetOutputFilenameAdapter.h
         ParquetStatusUtils.h
         ParquetWriter.h
+        RecordBatchFileSink.h
         RecordBatchSink.h
         RecordBatchStreamSource.h
         RecordBatchWithFlag.h
@@ -22,6 +23,7 @@ set(PARQUET_SOURCE_FILES
         ParquetOutputAdapterManager.cpp
         ParquetOutputFilenameAdapter.cpp
         ParquetWriter.cpp
+        RecordBatchFileSink.cpp
         ${PARQUET_HEADER_FILES}
         )
 
diff --git a/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp b/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
new file mode 100644
index 000000000..353481f74
--- /dev/null
+++ b/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
@@ -0,0 +1,227 @@
+#include <csp/adapters/parquet/RecordBatchFileSink.h>
+#include <csp/adapters/parquet/ParquetStatusUtils.h>
+#include <csp/core/FileUtils.h>
+
+#include <arrow/io/file.h>
+#include <arrow/ipc/writer.h>
+#include <arrow/record_batch.h>
+#include <arrow/table.h>
+#include <arrow/type.h>
+#include <arrow/util/compression.h>
+#include <arrow/util/config.h>
+#include <parquet/arrow/writer.h>
+
+#include <memory>
+#include <optional>
+#include <unordered_map>
+#include <vector>
+
+namespace csp::adapters::parquet
+{
+namespace
+{
+
+// A single open output file: write one batch, then close. Type-erased over format
+// so the rest of the machinery never branches on parquet-vs-ipc-vs-split.
+struct FileWriter
+{
+    std::function<void( const std::shared_ptr<::arrow::RecordBatch> & )> writeBatch;
+    std::function<void()>                                                close;
+};
+
+// (path, schema) -> open FileWriter.  The factory IS the format/layout choice.
+using FileWriterFactory =
+    std::function<FileWriter( const std::string &, const std::shared_ptr<::arrow::Schema> & )>;
+
+::arrow::Compression::type resolveCompression( const std::string & name )
+{
+    static const std::unordered_map<std::string, ::arrow::Compression::type> mapping{
+        { "",          ::arrow::Compression::UNCOMPRESSED },
+        { "none",      ::arrow::Compression::UNCOMPRESSED },
+        { "snappy",    ::arrow::Compression::SNAPPY },
+        { "gzip",      ::arrow::Compression::GZIP },
+        { "brotli",    ::arrow::Compression::BROTLI },
+        { "zstd",      ::arrow::Compression::ZSTD },
+        { "lz4",       ::arrow::Compression::LZ4 },
+        { "lz4_frame", ::arrow::Compression::LZ4_FRAME },
+    };
+    auto it = mapping.find( name );
+    CSP_TRUE_OR_THROW_RUNTIME( it != mapping.end(), "Unable to resolve compression: " << name );
+    return it -> second;
+}
+
+// Enforce overwrite policy, create parent dirs, and open the output stream.
+std::shared_ptr<::arrow::io::OutputStream> openOutputStream( const std::string & path, bool allowOverwrite )
+{
+    if( !allowOverwrite && utils::fileExists( path ) )
+        CSP_THROW( FileExistsError, "Trying to overwrite existing file " << path << " while allow_overwrite is false" );
+    utils::mkdir( utils::dirname( path ) );
+
+    auto result = ::arrow::io::FileOutputStream::Open( path );
+    STATUS_OK_OR_THROW_RUNTIME( result.status(), "Failed to open output stream " << path );
+    return result.MoveValueUnsafe();
+}
+
+FileWriter makeParquetFileWriter( const std::string & path, const std::shared_ptr<::arrow::Schema> & schema,
+                                  const std::string & compression, bool allowOverwrite )
+{
+    auto stream = openOutputStream( path, allowOverwrite );
+
+    ::parquet::WriterProperties::Builder props;
+    props.compression( resolveCompression( compression ) );
+#if ARROW_VERSION_MAJOR >= 20
+    props.version( ::parquet::ParquetVersion::PARQUET_2_6 );
+#else
+    props.version( ::parquet::ParquetVersion::PARQUET_2_0 );
+#endif
+    ::parquet::ArrowWriterProperties::Builder arrowProps;
+    arrowProps.store_schema(); // preserve arrow (file/column) schema metadata in the parquet file
+
+    auto result = ::parquet::arrow::FileWriter::Open(
+        *schema, ::arrow::default_memory_pool(), stream, props.build(), arrowProps.build() );
+    STATUS_OK_OR_THROW_RUNTIME( result.status(), "Failed to open parquet writer for " << path );
+    std::shared_ptr<::parquet::arrow::FileWriter> writer = result.MoveValueUnsafe();
+
+    return FileWriter{
+        [writer]( const std::shared_ptr<::arrow::RecordBatch> & rb )
+        {
+            // Write each flushed batch as its own row group (chunk_size = num_rows),
+            // matching the batch_size -> row-group contract.
+            auto table = ::arrow::Table::FromRecordBatches( rb -> schema(), { rb } );
+            STATUS_OK_OR_THROW_RUNTIME( table.status(), "Failed to assemble table for parquet write" );
+            STATUS_OK_OR_THROW_RUNTIME( writer -> WriteTable( *table.ValueUnsafe(), rb -> num_rows() ),
+                                        "Failed to write parquet record batch" );
+        },
+        [writer, stream]()
+        {
+            STATUS_OK_OR_THROW_RUNTIME( writer -> Close(), "Failed to close parquet writer" );
+            STATUS_OK_OR_THROW_RUNTIME( stream -> Close(), "Failed to close parquet output stream" );
+        } };
+}
+
+FileWriter makeIpcFileWriter( const std::string & path, const std::shared_ptr<::arrow::Schema> & schema,
+                              const std::string & compression, bool allowOverwrite )
+{
+    auto stream = openOutputStream( path, allowOverwrite );
+
+    auto options         = ::arrow::ipc::IpcWriteOptions::Defaults();
+    auto compressionType = resolveCompression( compression );
+    if( compressionType != ::arrow::Compression::UNCOMPRESSED )
+    {
+        auto codec = ::arrow::util::Codec::Create( compressionType );
+        STATUS_OK_OR_THROW_RUNTIME( codec.status(), "Failed to create codec for " << compression );
+        options.codec = codec.MoveValueUnsafe();
+    }
+
+    auto result = ::arrow::ipc::MakeStreamWriter( stream, schema, options );
+    STATUS_OK_OR_THROW_RUNTIME( result.status(), "Failed to open arrow IPC writer for " << path );
+    std::shared_ptr<::arrow::ipc::RecordBatchWriter> writer = result.MoveValueUnsafe();
+
+    return FileWriter{
+        [writer]( const std::shared_ptr<::arrow::RecordBatch> & rb )
+        {
+            STATUS_OK_OR_THROW_RUNTIME( writer -> WriteRecordBatch( *rb ), "Failed to write arrow record batch" );
+        },
+        [writer, stream]()
+        {
+            STATUS_OK_OR_THROW_RUNTIME( writer -> Close(), "Failed to close arrow IPC writer" );
+            STATUS_OK_OR_THROW_RUNTIME( stream -> Close(), "Failed to close arrow output stream" );
+        } };
+}
+
+// A split-columns writer is itself a FileWriter that fans out to one sub-writer per
+// column.  Each column file carries the parent file-level metadata.
+FileWriter makeSplitWriter( const std::string & dir, const std::shared_ptr<::arrow::Schema> & schema,
+                            const FileWriterFactory & perColumn, const std::string & extension )
+{
+    utils::mkdir( dir );
+
+    auto subWriters = std::make_shared<std::vector<FileWriter>>();
+    auto colSchemas = std::make_shared<std::vector<std::shared_ptr<::arrow::Schema>>>();
+    subWriters -> reserve( schema -> num_fields() );
+    colSchemas -> reserve( schema -> num_fields() );
+
+    for( int i = 0; i < schema -> num_fields(); ++i )
+    {
+        auto colSchema = ::arrow::schema( { schema -> field( i ) }, schema -> metadata() );
+        colSchemas -> push_back( colSchema );
+        subWriters -> push_back( perColumn( dir + "/" + schema -> field( i ) -> name() + extension, colSchema ) );
+    }
+
+    return FileWriter{
+        [subWriters, colSchemas]( const std::shared_ptr<::arrow::RecordBatch> & rb )
+        {
+            for( int i = 0; i < rb -> num_columns(); ++i )
+            {
+                auto colBatch = ::arrow::RecordBatch::Make( ( *colSchemas )[i], rb -> num_rows(), { rb -> column( i ) } );
+                ( *subWriters )[i].writeBatch( colBatch );
+            }
+        },
+        [subWriters]()
+        {
+            for( auto & w : *subWriters )
+                w.close();
+        } };
+}
+
+} // namespace
+
+RecordBatchSink makeFileSink( bool writeArrowBinary, bool splitColumns,
+                              const std::string & compression, bool allowOverwrite,
+                              std::function<void( const std::string & )> fileVisitor )
+{
+    const std::string extension = writeArrowBinary ? ".arrow" : ".parquet";
+
+    // Per-file factory: format choice is the only difference.
+    FileWriterFactory perFile =
+        [writeArrowBinary, compression, allowOverwrite]( const std::string & path, const std::shared_ptr<::arrow::Schema> & schema )
+        {
+            return writeArrowBinary ? makeIpcFileWriter( path, schema, compression, allowOverwrite )
+                                    : makeParquetFileWriter( path, schema, compression, allowOverwrite );
+        };
+
+    // Layout choice: single file vs. one file per column.
+    FileWriterFactory factory = perFile;
+    if( splitColumns )
+        factory = [perFile, extension]( const std::string & dir, const std::shared_ptr<::arrow::Schema> & schema )
+        {
+            return makeSplitWriter( dir, schema, perFile, extension );
+        };
+
+    // State shared across the sink callbacks.
+    auto schemaHolder = std::make_shared<std::shared_ptr<::arrow::Schema>>();
+    auto current      = std::make_shared<std::optional<FileWriter>>();
+    auto currentPath  = std::make_shared<std::string>();
+
+    auto closeCurrent = [current, currentPath, fileVisitor]()
+    {
+        if( current -> has_value() )
+        {
+            ( *current ) -> close();
+            if( fileVisitor )
+                fileVisitor( *currentPath );
+            current -> reset();
+        }
+    };
+
+    RecordBatchSink sink;
+    sink.onStart      = [schemaHolder]( const std::shared_ptr<::arrow::Schema> & schema ) { *schemaHolder = schema; };
+    sink.onBatch      = [current]( const std::shared_ptr<::arrow::RecordBatch> & rb )
+    {
+        if( current -> has_value() )
+            ( *current ) -> writeBatch( rb );
+    };
+    sink.onFileChange = [factory, schemaHolder, current, currentPath, closeCurrent]( const std::string & path )
+    {
+        closeCurrent();
+        if( !path.empty() )
+        {
+            *current     = factory( path, *schemaHolder );
+            *currentPath = path;
+        }
+    };
+    sink.onStop = closeCurrent;
+    return sink;
+}
+
+} // namespace csp::adapters::parquet
diff --git a/cpp/csp/adapters/parquet/RecordBatchFileSink.h b/cpp/csp/adapters/parquet/RecordBatchFileSink.h
new file mode 100644
index 000000000..9a6ff753f
--- /dev/null
+++ b/cpp/csp/adapters/parquet/RecordBatchFileSink.h
@@ -0,0 +1,29 @@
+#ifndef _IN_CSP_ADAPTERS_PARQUET_RecordBatchFileSink_H
+#define _IN_CSP_ADAPTERS_PARQUET_RecordBatchFileSink_H
+
+#include <csp/adapters/parquet/RecordBatchSink.h>
+#include <functional>
+#include <string>
+
+namespace csp::adapters::parquet
+{
+
+// Builds a RecordBatchSink that writes RecordBatches to files entirely in C++
+// (no Python / no Arrow C Data Interface hop on the hot path).
+//
+// One sink == one logical output target:
+//   - splitColumns == false : a single file (.parquet or .arrow) with all columns.
+//   - splitColumns == true  : a directory; each column is written to its own file.
+//
+// File rotation is just onFileChange (close current, open next). compression /
+// allowOverwrite / fileVisitor behave as on the Python sink. fileVisitor, if set,
+// is invoked with the path of each file after it is closed.
+RecordBatchSink makeFileSink( bool writeArrowBinary,
+                              bool splitColumns,
+                              const std::string & compression,
+                              bool allowOverwrite,
+                              std::function<void( const std::string & )> fileVisitor = {} );
+
+} // namespace csp::adapters::parquet
+
+#endif // _IN_CSP_ADAPTERS_PARQUET_RecordBatchFileSink_H
diff --git a/cpp/csp/core/Exception.h b/cpp/csp/core/Exception.h
index 53a1e00d6..9c442af9a 100644
--- a/cpp/csp/core/Exception.h
+++ b/cpp/csp/core/Exception.h
@@ -75,6 +75,7 @@ CSP_DECLARE_EXCEPTION( IOError,            RuntimeException )
 CSP_DECLARE_EXCEPTION( OSError,            RuntimeException )
 CSP_DECLARE_EXCEPTION( OutOfMemoryError,   RuntimeException )
 CSP_DECLARE_EXCEPTION( FileNotFoundError,  IOError )
+CSP_DECLARE_EXCEPTION( FileExistsError,    OSError )
 
 template<typename T>
 [[noreturn]] NO_INLINE void throw_exc(T&& e);
diff --git a/cpp/csp/python/Exception.h b/cpp/csp/python/Exception.h
index 5c7c2e80a..fe415ff8f 100644
--- a/cpp/csp/python/Exception.h
+++ b/cpp/csp/python/Exception.h
@@ -73,6 +73,7 @@ inline bool& capture_cpp_exception_trace_flag()
     CSP_CATCH_HELPER( ::csp::OverflowError,     PyExc_OverflowError,     RETURN_STMT )     \
     CSP_CATCH_HELPER( ::csp::DivideByZero,      PyExc_ZeroDivisionError, RETURN_STMT )     \
     CSP_CATCH_HELPER( ::csp::RecursionError,    PyExc_RecursionError,    RETURN_STMT )     \
+    CSP_CATCH_HELPER( ::csp::FileExistsError,   PyExc_FileExistsError,   RETURN_STMT )     \
     CSP_CATCH_HELPER( ::csp::OSError,           PyExc_OSError,           RETURN_STMT )     \
     CSP_CATCH_HELPER( ::csp::OutOfMemoryError,  PyExc_MemoryError,       RETURN_STMT )     \
     CSP_CATCH_HELPER( ::csp::FileNotFoundError, PyExc_FileNotFoundError, RETURN_STMT )     \
diff --git a/cpp/csp/python/adapters/parquetadapterimpl.cpp b/cpp/csp/python/adapters/parquetadapterimpl.cpp
index 5b87db8b2..39a275acd 100644
--- a/cpp/csp/python/adapters/parquetadapterimpl.cpp
+++ b/cpp/csp/python/adapters/parquetadapterimpl.cpp
@@ -4,6 +4,7 @@
 #include <csp/adapters/parquet/ParquetDictBasketOutputWriter.h>
 #include <csp/adapters/parquet/ParquetStatusUtils.h>
 #include <csp/adapters/parquet/ParquetWriter.h>
+#include <csp/adapters/parquet/RecordBatchFileSink.h>
 #include <csp/adapters/parquet/RecordBatchSink.h>
 #include <csp/engine/PushInputAdapter.h>
 #include <csp/python/Conversions.h>
@@ -675,123 +676,44 @@ static PyObject *create_parquet_input_adapter_manager_native( PyObject *args )
     CSP_RETURN_NULL;
 }
 
-// --- Helpers for exporting Arrow data to Python via C Data Interface ---
+// --- Output adapter manager: writes RecordBatches to files entirely in C++ ---
 
-static void releaseArrowSchemaCapsule( PyObject * capsule )
-{
-    auto * schema = reinterpret_cast<ArrowSchema*>( PyCapsule_GetPointer( capsule, "arrow_schema" ) );
-    if( schema && schema -> release )
-        schema -> release( schema );
-    free( schema );
-}
-
-static void releaseArrowArrayCapsule( PyObject * capsule )
-{
-    auto * array = reinterpret_cast<ArrowArray*>( PyCapsule_GetPointer( capsule, "arrow_array" ) );
-    if( array && array -> release )
-        array -> release( array );
-    free( array );
-}
-
-// Export an Arrow Schema as a single capsule
-static PyObjectPtr exportSchema( const std::shared_ptr<::arrow::Schema> & schema )
-{
-    auto * c_schema = static_cast<ArrowSchema*>( malloc( sizeof( ArrowSchema ) ) );
-    auto st = ::arrow::ExportSchema( *schema, c_schema );
-    if( !st.ok() )
-    {
-        free( c_schema );
-        CSP_THROW( ValueError, "Failed to export Schema: " << st.ToString() );
-    }
-    return PyObjectPtr::own( PyCapsule_New( c_schema, "arrow_schema", releaseArrowSchemaCapsule ) );
-}
-
-// Build a RecordBatchSink that delegates to a Python object with
-// on_start(schema_capsule), on_batch(schema_capsule, array_capsule), on_file_change(path), on_stop() methods.
-static RecordBatchSink createSinkFromPython( PyObjectPtr pySink )
+csp::AdapterManager *create_parquet_output_adapter_manager( PyEngine *engine, const Dictionary &properties )
 {
-    RecordBatchSink sink;
-
-    sink.onStart = [pySink]( const std::shared_ptr<::arrow::Schema> & schema )
-    {
-        auto capsule = exportSchema( schema );
-        auto rv = PyObjectPtr::own( PyObject_CallMethod( pySink.get(), "on_start", "O", capsule.get() ) );
-        if( !rv.get() )
-            CSP_THROW( PythonPassthrough, "" );
-    };
-
-    sink.onBatch = [pySink]( const std::shared_ptr<::arrow::RecordBatch> & rb )
-    {
-        auto * c_schema = static_cast<ArrowSchema*>( malloc( sizeof( ArrowSchema ) ) );
-        auto * c_array  = static_cast<ArrowArray*>( malloc( sizeof( ArrowArray ) ) );
-        auto st = ::arrow::ExportRecordBatch( *rb, c_array, c_schema );
-        if( !st.ok() )
+    // Output file I/O is done entirely in C++ via makeFileSink (no per-batch Python hop).
+    bool        writeArrowBinary = properties.get<bool>( "write_arrow_binary" );
+    bool        splitColumns     = properties.get<bool>( "split_columns_to_files" );
+    bool        allowOverwrite   = properties.get<bool>( "allow_overwrite" );
+    std::string compression      = properties.get<std::string>( "compression" );
+
+    // Optional Python file_visitor: wrapped as a plain C++ callback, invoked (with the GIL
+    // held during the run) after each file is closed.  Only the main sink visits files.
+    std::function<void( const std::string & )> fileVisitor;
+    DialectGenericType visitorDG;
+    if( properties.tryGet( "file_visitor", visitorDG ) )
+    {
+        auto pyVisitor = PyObjectPtr::own( toPython( visitorDG ) );
+        if( pyVisitor.get() && pyVisitor.get() != Py_None )
         {
-            free( c_array );
-            free( c_schema );
-            CSP_THROW( ValueError, "Failed to export RecordBatch: " << st.ToString() );
+            fileVisitor = [pyVisitor]( const std::string & path )
+            {
+                auto pyPath = PyObjectPtr::own( PyUnicode_FromStringAndSize( path.c_str(), path.size() ) );
+                auto rv = PyObjectPtr::own( PyObject_CallFunctionObjArgs( pyVisitor.get(), pyPath.get(), nullptr ) );
+                if( !rv.get() )
+                    CSP_THROW( PythonPassthrough, "" );
+            };
         }
-        auto pySchemaCapsule = PyObjectPtr::own( PyCapsule_New( c_schema, "arrow_schema", releaseArrowSchemaCapsule ) );
-        auto pyArrayCapsule  = PyObjectPtr::own( PyCapsule_New( c_array, "arrow_array", releaseArrowArrayCapsule ) );
-        auto rv = PyObjectPtr::own( PyObject_CallMethod( pySink.get(), "on_batch", "OO",
-                                                          pySchemaCapsule.get(), pyArrayCapsule.get() ) );
-        if( !rv.get() )
-            CSP_THROW( PythonPassthrough, "" );
-    };
-
-    sink.onFileChange = [pySink]( const std::string & path )
-    {
-        auto pyPath = PyObjectPtr::own( PyUnicode_FromStringAndSize( path.c_str(), path.size() ) );
-        auto rv = PyObjectPtr::own( PyObject_CallMethod( pySink.get(), "on_file_change", "O", pyPath.get() ) );
-        if( !rv.get() )
-            CSP_THROW( PythonPassthrough, "" );
-    };
-
-    sink.onStop = [pySink]()
-    {
-        auto rv = PyObjectPtr::own( PyObject_CallMethod( pySink.get(), "on_stop", nullptr ) );
-        if( !rv.get() )
-            CSP_THROW( PythonPassthrough, "" );
-    };
-
-    return sink;
-}
-
-csp::AdapterManager *create_parquet_output_adapter_manager( PyEngine *engine, const Dictionary &properties )
-{
-    // Extract Python sink object from properties
-    RecordBatchSink mainSink;
-    RecordBatchSink indexSink;
-    DialectGenericType pySinkDG, pyIndexSinkDG;
-    if( properties.tryGet( "rb_sink", pySinkDG ) )
-    {
-        auto pySink = PyObjectPtr::own( toPython( pySinkDG ) );
-        mainSink = createSinkFromPython( std::move( pySink ) );
-    }
-    if( properties.tryGet( "rb_index_sink", pyIndexSinkDG ) )
-    {
-        auto pyIndexSink = PyObjectPtr::own( toPython( pyIndexSinkDG ) );
-        indexSink = createSinkFromPython( std::move( pyIndexSink ) );
     }
 
     auto * mgr = engine -> engine() -> createOwnedObject<ParquetOutputAdapterManager>( properties );
-    mgr -> setSink( std::move( mainSink ) );
-    mgr -> setIndexSink( std::move( indexSink ) );
+    mgr -> setSink( makeFileSink( writeArrowBinary, splitColumns, compression, allowOverwrite, fileVisitor ) );
 
-    // Sink factory for dict basket writers: calls Python to create new sinks on demand
-    DialectGenericType pySinkFactoryDG;
-    if( properties.tryGet( "rb_sink_factory", pySinkFactoryDG ) )
+    // Dict-basket writers always write split-column files (one per value/symbol/index column),
+    // without a file visitor.
+    mgr -> setSinkFactory( [writeArrowBinary, compression, allowOverwrite]( const std::string & ) -> RecordBatchSink
     {
-        auto pySinkFactory = PyObjectPtr::own( toPython( pySinkFactoryDG ) );
-        mgr -> setSinkFactory( [pySinkFactory]( const std::string & name ) -> RecordBatchSink
-        {
-            auto pyName = PyObjectPtr::own( PyUnicode_FromStringAndSize( name.c_str(), name.size() ) );
-            auto pySink = PyObjectPtr::own( PyObject_CallFunctionObjArgs( pySinkFactory.get(), pyName.get(), nullptr ) );
-            if( !pySink.get() )
-                CSP_THROW( PythonPassthrough, "" );
-            return createSinkFromPython( std::move( pySink ) );
-        } );
-    }
+        return makeFileSink( writeArrowBinary, /*splitColumns=*/true, compression, allowOverwrite, {} );
+    } );
 
     return mgr;
 }

From 789d0e693e5adff9cdcea38f0543c1b9592ed8ac Mon Sep 17 00:00:00 2001
From: Arham Chopra <arham.chopra@cubistsystematic.com>
Date: Fri, 12 Jun 2026 16:52:12 -0400
Subject: [PATCH 04/11] Parquet output: fix sink bugs, remove orphaned Python
 sink

From the multi-model review:
- delete the now-dead Python sink (_parquet_rb_writer.py, rb_sink wiring,
  TestOutputSinkDirect) since file I/O is all C++ now
- RecordBatchFileSink: guard mkdir(""), fix close-path exception safety,
  resolve compression via Arrow's Codec API (case-insensitive)
- honor an explicit writeTimestampColumn
- fail fast on single-file + dict basket
- add regression tests

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>
---
 cpp/csp/adapters/parquet/ParquetWriter.cpp    |   9 +-
 .../adapters/parquet/RecordBatchFileSink.cpp  |  66 ++--
 csp/adapters/_parquet_rb_writer.py            | 202 ----------
 csp/adapters/output_adapters/parquet.py       |  37 +-
 csp/tests/adapters/test_parquet_output.py     | 350 +++++++++++-------
 5 files changed, 283 insertions(+), 381 deletions(-)
 delete mode 100644 csp/adapters/_parquet_rb_writer.py

diff --git a/cpp/csp/adapters/parquet/ParquetWriter.cpp b/cpp/csp/adapters/parquet/ParquetWriter.cpp
index 75eb4e8bd..6d988e3fd 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetWriter.cpp
@@ -118,8 +118,15 @@ PushInputAdapter *ParquetWriter::getStatusAdapter()
 void ParquetWriter::start()
 {
     std::vector<std::shared_ptr<::arrow::Field>> arrowFields;
-    if( !m_writeTimestampColumn.has_value() && !m_adapterMgr.getTimestampColumnName().empty() )
+    // Honor an explicitly-set writeTimestampColumn; otherwise auto-detect from whether
+    // a timestamp column name was configured.
+    bool wantTimestamp = m_writeTimestampColumn.has_value()
+                             ? m_writeTimestampColumn.value()
+                             : !m_adapterMgr.getTimestampColumnName().empty();
+    if( wantTimestamp )
     {
+        CSP_TRUE_OR_THROW_RUNTIME( !m_adapterMgr.getTimestampColumnName().empty(),
+            "writeTimestampColumn is true but no timestamp_column_name was provided" );
         m_writeTimestampColumn = true;
         auto tsBuilder = createArrowBackedArrayBuilder(
             m_adapterMgr.getTimestampColumnName(), getChunkSize(), CspType::DATETIME() );
diff --git a/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp b/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
index 353481f74..d3c934e4f 100644
--- a/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
+++ b/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
@@ -11,6 +11,9 @@
 #include <arrow/util/config.h>
 #include <parquet/arrow/writer.h>
 
+#include <algorithm>
+#include <cctype>
+#include <exception>
 #include <memory>
 #include <optional>
 #include <unordered_map>
@@ -35,19 +38,20 @@ using FileWriterFactory =
 
 ::arrow::Compression::type resolveCompression( const std::string & name )
 {
-    static const std::unordered_map<std::string, ::arrow::Compression::type> mapping{
-        { "",          ::arrow::Compression::UNCOMPRESSED },
-        { "none",      ::arrow::Compression::UNCOMPRESSED },
-        { "snappy",    ::arrow::Compression::SNAPPY },
-        { "gzip",      ::arrow::Compression::GZIP },
-        { "brotli",    ::arrow::Compression::BROTLI },
-        { "zstd",      ::arrow::Compression::ZSTD },
-        { "lz4",       ::arrow::Compression::LZ4 },
-        { "lz4_frame", ::arrow::Compression::LZ4_FRAME },
-    };
-    auto it = mapping.find( name );
-    CSP_TRUE_OR_THROW_RUNTIME( it != mapping.end(), "Unable to resolve compression: " << name );
-    return it -> second;
+    // csp treats "" / "none" as no compression; Arrow expects lower-case codec names.
+    std::string lower( name );
+    std::transform( lower.begin(), lower.end(), lower.begin(),
+                    []( unsigned char c ) { return static_cast<char>( std::tolower( c ) ); } );
+    if( lower.empty() || lower == "none" )
+        return ::arrow::Compression::UNCOMPRESSED;
+
+    // Arrow is the source of truth for which compression names exist...
+    auto compressionType = ::arrow::util::Codec::GetCompressionType( lower );
+    CSP_TRUE_OR_THROW_RUNTIME( compressionType.ok(), "Unsupported compression '" << name << "'" );
+    // ...and whether this Arrow build was actually compiled with support for it.
+    CSP_TRUE_OR_THROW_RUNTIME( ::arrow::util::Codec::IsAvailable( compressionType.ValueUnsafe() ),
+        "Compression '" << name << "' is not available in this Arrow build" );
+    return compressionType.ValueUnsafe();
 }
 
 // Enforce overwrite policy, create parent dirs, and open the output stream.
@@ -55,7 +59,11 @@ std::shared_ptr<::arrow::io::OutputStream> openOutputStream( const std::string &
 {
     if( !allowOverwrite && utils::fileExists( path ) )
         CSP_THROW( FileExistsError, "Trying to overwrite existing file " << path << " while allow_overwrite is false" );
-    utils::mkdir( utils::dirname( path ) );
+    // Only create the parent directory when the path has one; a bare relative
+    // filename (e.g. "out.parquet") has an empty dirname, and mkdir("") fails.
+    auto parentDir = utils::dirname( path );
+    if( !parentDir.empty() )
+        utils::mkdir( parentDir );
 
     auto result = ::arrow::io::FileOutputStream::Open( path );
     STATUS_OK_OR_THROW_RUNTIME( result.status(), "Failed to open output stream " << path );
@@ -94,8 +102,10 @@ FileWriter makeParquetFileWriter( const std::string & path, const std::shared_pt
         },
         [writer, stream]()
         {
-            STATUS_OK_OR_THROW_RUNTIME( writer -> Close(), "Failed to close parquet writer" );
-            STATUS_OK_OR_THROW_RUNTIME( stream -> Close(), "Failed to close parquet output stream" );
+            auto writerStatus = writer -> Close();
+            auto streamStatus = stream -> Close();   // always close the stream, even if the footer flush failed
+            STATUS_OK_OR_THROW_RUNTIME( writerStatus, "Failed to close parquet writer" );
+            STATUS_OK_OR_THROW_RUNTIME( streamStatus, "Failed to close parquet output stream" );
         } };
 }
 
@@ -124,8 +134,10 @@ FileWriter makeIpcFileWriter( const std::string & path, const std::shared_ptr<::
         },
         [writer, stream]()
         {
-            STATUS_OK_OR_THROW_RUNTIME( writer -> Close(), "Failed to close arrow IPC writer" );
-            STATUS_OK_OR_THROW_RUNTIME( stream -> Close(), "Failed to close arrow output stream" );
+            auto writerStatus = writer -> Close();
+            auto streamStatus = stream -> Close();   // always close the stream, even if the writer close failed
+            STATUS_OK_OR_THROW_RUNTIME( writerStatus, "Failed to close arrow IPC writer" );
+            STATUS_OK_OR_THROW_RUNTIME( streamStatus, "Failed to close arrow output stream" );
         } };
 }
 
@@ -134,7 +146,8 @@ FileWriter makeIpcFileWriter( const std::string & path, const std::shared_ptr<::
 FileWriter makeSplitWriter( const std::string & dir, const std::shared_ptr<::arrow::Schema> & schema,
                             const FileWriterFactory & perColumn, const std::string & extension )
 {
-    utils::mkdir( dir );
+    if( !dir.empty() )
+        utils::mkdir( dir );
 
     auto subWriters = std::make_shared<std::vector<FileWriter>>();
     auto colSchemas = std::make_shared<std::vector<std::shared_ptr<::arrow::Schema>>>();
@@ -159,8 +172,14 @@ FileWriter makeSplitWriter( const std::string & dir, const std::shared_ptr<::arr
         },
         [subWriters]()
         {
+            std::exception_ptr firstError;
             for( auto & w : *subWriters )
-                w.close();
+            {
+                try { w.close(); }
+                catch( ... ) { if( !firstError ) firstError = std::current_exception(); }
+            }
+            if( firstError )
+                std::rethrow_exception( firstError );
         } };
 }
 
@@ -197,10 +216,13 @@ RecordBatchSink makeFileSink( bool writeArrowBinary, bool splitColumns,
     {
         if( current -> has_value() )
         {
-            ( *current ) -> close();
+            // Detach + reset BEFORE invoking the (user-supplied) visitor, so that a throwing
+            // visitor cannot cause a later onStop()->closeCurrent() to close this writer twice.
+            FileWriter w = std::move( current -> value() );
+            current -> reset();
+            w.close();
             if( fileVisitor )
                 fileVisitor( *currentPath );
-            current -> reset();
         }
     };
 
diff --git a/csp/adapters/_parquet_rb_writer.py b/csp/adapters/_parquet_rb_writer.py
deleted file mode 100644
index 56e980ebf..000000000
--- a/csp/adapters/_parquet_rb_writer.py
+++ /dev/null
@@ -1,202 +0,0 @@
-"""RecordBatch sink for parquet/arrow output.
-
-Receives RecordBatches from C++ (via the Arrow C Data Interface)
-and writes them using a pluggable writer backend.
-"""
-
-import os
-
-import pyarrow as pa
-import pyarrow.ipc as ipc
-import pyarrow.parquet as pq
-
-
-# ── Writer helpers ────────────────────────────────────────────────────
-
-
-class _IPCWriter:
-    """Wraps an IPC stream writer + underlying file sink."""
-
-    __slots__ = ("_writer", "_sink")
-
-    def __init__(self, writer, sink):
-        self._writer = writer
-        self._sink = sink
-
-    def write_batch(self, rb):
-        self._writer.write_batch(rb)
-
-    def close(self):
-        self._writer.close()
-        self._sink.close()
-
-
-class _SplitColumnsWriter:
-    """Dispatches each column of a RecordBatch to its own writer."""
-
-    __slots__ = ("_writers", "_schema")
-
-    def __init__(self, writers, schema):
-        self._writers = writers
-        self._schema = schema
-
-    def write_batch(self, rb):
-        for i, field in enumerate(self._schema):
-            col_rb = pa.RecordBatch.from_arrays([rb.column(i)], schema=pa.schema([field]))
-            self._writers[field.name].write_batch(col_rb)
-
-    def close(self):
-        for w in self._writers.values():
-            w.close()
-
-
-def _ensure_parent(path, allow_overwrite):
-    if not allow_overwrite and os.path.exists(path):
-        raise FileExistsError(f"File already exists: {path}")
-    parent = os.path.dirname(path)
-    if parent:
-        os.makedirs(parent, exist_ok=True)
-
-
-def _apply_metadata(schema, file_metadata, column_metadata):
-    """Attach file/column-level metadata to a pyarrow schema."""
-    if file_metadata:
-        existing = schema.metadata or {}
-        merged = {**existing, **{k.encode(): v.encode() for k, v in file_metadata.items()}}
-        schema = schema.with_metadata(merged)
-    if column_metadata:
-        fields = []
-        for field in schema:
-            if field.name in column_metadata:
-                meta = {k.encode(): v.encode() for k, v in column_metadata[field.name].items()}
-                existing = field.metadata or {}
-                fields.append(field.with_metadata({**existing, **meta}))
-            else:
-                fields.append(field)
-        schema = pa.schema(fields, metadata=schema.metadata)
-    return schema
-
-
-# ── Writer factories ──────────────────────────────────────────────────
-
-
-def _parquet_writer_factory(compression, allow_overwrite, file_metadata=None, column_metadata=None):
-    compression = compression or "none"
-
-    def factory(path, schema):
-        schema = _apply_metadata(schema, file_metadata, column_metadata)
-        _ensure_parent(path, allow_overwrite)
-        return pq.ParquetWriter(path, schema, compression=compression)
-
-    return factory
-
-
-def _arrow_ipc_writer_factory(compression, allow_overwrite):
-    def factory(path, schema):
-        _ensure_parent(path, allow_overwrite)
-        options = ipc.IpcWriteOptions()
-        if compression and compression != "none":
-            options = ipc.IpcWriteOptions(compression=pa.Codec(compression))
-        sink = pa.OSFile(path, "wb")
-        return _IPCWriter(ipc.new_stream(sink, schema, options=options), sink)
-
-    return factory
-
-
-def _split_columns_writer_factory(compression, allow_overwrite, write_arrow_binary, file_metadata=None):
-    compression = compression or "none"
-    ext = ".arrow" if write_arrow_binary else ".parquet"
-
-    def factory(dir_name, schema):
-        if dir_name:
-            os.makedirs(dir_name, exist_ok=True)
-        # Carry file-level metadata from the parent schema (or explicit param) to each per-column file
-        meta = schema.metadata
-        if file_metadata:
-            existing = meta or {}
-            meta = {**existing, **{k.encode(): v.encode() for k, v in file_metadata.items()}}
-        writers = {}
-        for field in schema:
-            col_schema = pa.schema([field], metadata=meta)
-            file_path = os.path.join(dir_name, field.name + ext)
-            if not allow_overwrite and os.path.exists(file_path):
-                raise FileExistsError(f"File already exists: {file_path}")
-            if write_arrow_binary:
-                sink = pa.OSFile(file_path, "wb")
-                writers[field.name] = _IPCWriter(ipc.new_stream(sink, col_schema), sink)
-            else:
-                writers[field.name] = pq.ParquetWriter(file_path, col_schema, compression=compression)
-        return _SplitColumnsWriter(writers, schema)
-
-    return factory
-
-
-# ── Unified sink ──────────────────────────────────────────────────────
-
-
-class RecordBatchSink:
-    """Receives RecordBatches from C++ and writes them via a pluggable backend.
-
-    writer_factory(path, schema) must return an object with write_batch() and close().
-    """
-
-    def __init__(self, writer_factory, file_visitor=None):
-        self._writer_factory = writer_factory
-        self._file_visitor = file_visitor
-        self._schema = None
-        self._writer = None
-        self._current_file = None
-
-    def on_start(self, schema_capsule):
-        self._schema = pa.Schema._import_from_c_capsule(schema_capsule)
-
-    def on_batch(self, schema_capsule, array_capsule):
-        rb = pa.RecordBatch._import_from_c_capsule(schema_capsule, array_capsule)
-        if self._writer is not None:
-            self._writer.write_batch(rb)
-
-    def on_file_change(self, new_path):
-        if self._writer is not None:
-            self._writer.close()
-            if self._file_visitor:
-                self._file_visitor(self._current_file)
-        if new_path:
-            self._writer = self._writer_factory(new_path, self._schema)
-            self._current_file = new_path
-        else:
-            self._writer = None
-
-    def on_stop(self):
-        if self._writer is not None:
-            self._writer.close()
-            if self._file_visitor:
-                self._file_visitor(self._current_file)
-            self._writer = None
-
-
-# ── Public factory functions ──────────────────────────────────────────
-
-
-def create_sink(file_name, compression, allow_overwrite, write_arrow_binary,
-                split_columns_to_files, file_visitor=None,
-                file_metadata=None, column_metadata=None):
-    """Factory function to create the appropriate sink based on config."""
-    if split_columns_to_files:
-        wf = _split_columns_writer_factory(compression, allow_overwrite, write_arrow_binary, file_metadata)
-    elif write_arrow_binary:
-        wf = _arrow_ipc_writer_factory(compression, allow_overwrite)
-    else:
-        wf = _parquet_writer_factory(compression, allow_overwrite, file_metadata, column_metadata)
-    return RecordBatchSink(wf, file_visitor)
-
-
-def create_sink_factory(compression, allow_overwrite, write_arrow_binary, file_visitor=None):
-    """Returns a callable(name) -> sink for dict basket writers.
-
-    Dict basket writers always use split_columns_to_files=True for their data,
-    and each basket gets its own sink identified by name.
-    """
-    def factory(name):
-        wf = _split_columns_writer_factory(compression, allow_overwrite, write_arrow_binary)
-        return RecordBatchSink(wf, file_visitor)
-    return factory
diff --git a/csp/adapters/output_adapters/parquet.py b/csp/adapters/output_adapters/parquet.py
index 24acf0b5f..ff59ad044 100644
--- a/csp/adapters/output_adapters/parquet.py
+++ b/csp/adapters/output_adapters/parquet.py
@@ -199,6 +199,11 @@ def _parquet_dict_basket_writer(
         return self._parquet_dict_basket_writer_node_def
 
     def publish_dict_basket(self, column_name, value, key_type, value_type):
+        if not self._split_columns_to_files:
+            raise ValueError(
+                f"Cannot publish dict basket '{column_name}': dict baskets require "
+                "split_columns_to_files=True (single-file parquet output is not supported for baskets)"
+            )
         if key_type is not str:
             raise NotImplementedError("Writing of baskets with non str key type is not supported")
         if self._filename_provider is not None:
@@ -224,38 +229,6 @@ def _create(self, engine, memo):
         if self._parquet_output_adapter_manager:
             return self._parquet_output_adapter_manager
 
-        from csp.adapters._parquet_rb_writer import create_sink, create_sink_factory
-
-        # Create main sink for the primary writer
-        file_name = self._properties.get("file_name", "")
-        compression = self._properties.get("compression", "snappy")
-        allow_overwrite = self._properties.get("allow_overwrite", False)
-        write_arrow_binary = self._properties.get("write_arrow_binary", False)
-        split_columns = self._properties.get("split_columns_to_files", False)
-        file_visitor = self._properties.get("file_visitor")
-        file_metadata = self._properties.get("file_metadata")
-        column_metadata = self._properties.get("column_metadata")
-
-        sink = create_sink(
-            file_name=file_name,
-            compression=compression,
-            allow_overwrite=allow_overwrite,
-            write_arrow_binary=write_arrow_binary,
-            split_columns_to_files=split_columns,
-            file_visitor=file_visitor,
-            file_metadata=file_metadata,
-            column_metadata=column_metadata,
-        )
-        self._properties["rb_sink"] = sink
-
-        # Sink factory for dict basket writers (no file_visitor — only main sink visits)
-        sink_factory = create_sink_factory(
-            compression=compression,
-            allow_overwrite=allow_overwrite,
-            write_arrow_binary=write_arrow_binary,
-        )
-        self._properties["rb_sink_factory"] = sink_factory
-
         self._parquet_output_adapter_manager = _parquetadapterimpl._parquet_output_adapter_manager(
             engine, self._properties
         )
diff --git a/csp/tests/adapters/test_parquet_output.py b/csp/tests/adapters/test_parquet_output.py
index 18ee01dd3..58867ae08 100644
--- a/csp/tests/adapters/test_parquet_output.py
+++ b/csp/tests/adapters/test_parquet_output.py
@@ -20,7 +20,6 @@
 from csp.adapters.output_adapters.parquet import ParquetOutputConfig
 from csp.adapters.parquet import ParquetReader, ParquetWriter
 
-
 START = datetime(2022, 1, 1, tzinfo=pytz.utc)
 
 
@@ -276,14 +275,19 @@ class TestOutputFileRotation(unittest.TestCase):
 
     def test_basic_rotation(self):
         with tempfile.TemporaryDirectory() as d:
+
             @csp.graph
             def g():
                 data = csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(10)])
-                rotate = csp.curve(str, [
-                    (timedelta(seconds=5), os.path.join(d, "part1.parquet")),
-                ])
+                rotate = csp.curve(
+                    str,
+                    [
+                        (timedelta(seconds=5), os.path.join(d, "part1.parquet")),
+                    ],
+                )
                 writer = ParquetWriter(
-                    os.path.join(d, "part0.parquet"), "csp_timestamp",
+                    os.path.join(d, "part0.parquet"),
+                    "csp_timestamp",
                     config=ParquetOutputConfig(allow_overwrite=True),
                     filename_provider=rotate,
                 )
@@ -297,15 +301,20 @@ def g():
 
     def test_multiple_rotations(self):
         with tempfile.TemporaryDirectory() as d:
+
             @csp.graph
             def g():
                 data = csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(9)])
-                rotate = csp.curve(str, [
-                    (timedelta(seconds=4), os.path.join(d, "p1.parquet")),
-                    (timedelta(seconds=7), os.path.join(d, "p2.parquet")),
-                ])
+                rotate = csp.curve(
+                    str,
+                    [
+                        (timedelta(seconds=4), os.path.join(d, "p1.parquet")),
+                        (timedelta(seconds=7), os.path.join(d, "p2.parquet")),
+                    ],
+                )
                 writer = ParquetWriter(
-                    os.path.join(d, "p0.parquet"), "csp_timestamp",
+                    os.path.join(d, "p0.parquet"),
+                    "csp_timestamp",
                     config=ParquetOutputConfig(allow_overwrite=True),
                     filename_provider=rotate,
                 )
@@ -322,12 +331,14 @@ def g():
     def test_file_visitor_called(self):
         visited = []
         with tempfile.TemporaryDirectory() as d:
+
             @csp.graph
             def g():
                 data = csp.curve(int, [(timedelta(seconds=1), 1), (timedelta(seconds=3), 2)])
                 rotate = csp.curve(str, [(timedelta(seconds=2), os.path.join(d, "p1.parquet"))])
                 writer = ParquetWriter(
-                    os.path.join(d, "p0.parquet"), "csp_timestamp",
+                    os.path.join(d, "p0.parquet"),
+                    "csp_timestamp",
                     config=ParquetOutputConfig(allow_overwrite=True),
                     filename_provider=rotate,
                     file_visitor=lambda f: visited.append(f),
@@ -340,15 +351,40 @@ def g():
             self.assertTrue(visited[0].endswith("p0.parquet"))
             self.assertTrue(visited[1].endswith("p1.parquet"))
 
+    def test_file_visitor_exception_surfaces_without_double_close(self):
+        """If file_visitor raises, that error surfaces cleanly (no double-close crash on teardown)."""
+        with tempfile.TemporaryDirectory() as d:
+
+            def boom(_f):
+                raise ValueError("visitor boom")
+
+            @csp.graph
+            def g():
+                data = csp.curve(int, [(timedelta(seconds=1), 1), (timedelta(seconds=3), 2)])
+                rotate = csp.curve(str, [(timedelta(seconds=2), os.path.join(d, "p1.parquet"))])
+                writer = ParquetWriter(
+                    os.path.join(d, "p0.parquet"),
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    filename_provider=rotate,
+                    file_visitor=boom,
+                )
+                writer.publish("value", data)
+
+            with self.assertRaisesRegex(Exception, "visitor boom"):
+                csp.run(g, starttime=START, endtime=timedelta(seconds=5))
+
     def test_rotation_no_initial_file(self):
         """filename_provider only, no initial file_name."""
         with tempfile.TemporaryDirectory() as d:
+
             @csp.graph
             def g():
                 data = csp.curve(int, [(timedelta(seconds=2), 1), (timedelta(seconds=3), 2)])
                 rotate = csp.curve(str, [(timedelta(seconds=1), os.path.join(d, "output.parquet"))])
                 writer = ParquetWriter(
-                    None, "csp_timestamp",
+                    None,
+                    "csp_timestamp",
                     config=ParquetOutputConfig(allow_overwrite=True),
                     filename_provider=rotate,
                 )
@@ -426,7 +462,8 @@ def test_split_scalars(self):
             @csp.graph
             def g():
                 writer = ParquetWriter(
-                    outdir, "csp_timestamp",
+                    outdir,
+                    "csp_timestamp",
                     config=ParquetOutputConfig(allow_overwrite=True),
                     split_columns_to_files=True,
                 )
@@ -453,7 +490,8 @@ def test_split_round_trip_via_csp(self):
             @csp.graph
             def g_write():
                 writer = ParquetWriter(
-                    outdir, "csp_timestamp",
+                    outdir,
+                    "csp_timestamp",
                     config=ParquetOutputConfig(allow_overwrite=True),
                     split_columns_to_files=True,
                 )
@@ -479,14 +517,20 @@ class S(csp.Struct):
             @csp.graph
             def g_write():
                 writer = ParquetWriter(
-                    outdir, "csp_timestamp",
+                    outdir,
+                    "csp_timestamp",
                     config=ParquetOutputConfig(allow_overwrite=True),
                     split_columns_to_files=True,
                 )
-                writer.publish_struct(csp.curve(S, [
-                    (timedelta(seconds=1), S(a=1, b=1.1)),
-                    (timedelta(seconds=2), S(a=2, b=2.2)),
-                ]))
+                writer.publish_struct(
+                    csp.curve(
+                        S,
+                        [
+                            (timedelta(seconds=1), S(a=1, b=1.1)),
+                            (timedelta(seconds=2), S(a=2, b=2.2)),
+                        ],
+                    )
+                )
 
             @csp.graph
             def g_read():
@@ -518,8 +562,27 @@ def g():
 class TestOutputDictBasket(unittest.TestCase):
     """Test dict basket output."""
 
+    def test_single_file_dict_basket_raises_clear_error(self):
+        """Publishing a dict basket on a single-file (non-split) writer fails fast with a clear error."""
+        with tempfile.TemporaryDirectory() as d:
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(
+                    os.path.join(d, "out.parquet"),
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    # split_columns_to_files defaults to False
+                )
+                basket = {"AAPL": csp.curve(float, [(timedelta(seconds=1), 1.0)])}
+                writer.publish_dict_basket("price", basket, str, float)
+
+            with self.assertRaisesRegex(ValueError, "split_columns_to_files"):
+                csp.run(g, starttime=START, endtime=timedelta(seconds=3))
+
     def test_basic_dict_basket(self):
         with tempfile.TemporaryDirectory() as d:
+
             @csp.graph
             def g_write():
                 writer = ParquetWriter(
@@ -557,6 +620,7 @@ def test_dict_basket_many_symbols(self):
         symbols = [f"SYM{i}" for i in range(n_symbols)]
 
         with tempfile.TemporaryDirectory() as d:
+
             @csp.graph
             def g_write():
                 writer = ParquetWriter(
@@ -565,10 +629,7 @@ def g_write():
                     config=ParquetOutputConfig(allow_overwrite=True),
                     split_columns_to_files=True,
                 )
-                basket = {
-                    sym: csp.curve(float, [(timedelta(seconds=1), float(i))])
-                    for i, sym in enumerate(symbols)
-                }
+                basket = {sym: csp.curve(float, [(timedelta(seconds=1), float(i))]) for i, sym in enumerate(symbols)}
                 writer.publish_dict_basket("val", basket, str, float)
 
             @csp.graph
@@ -600,10 +661,13 @@ def test_1d_float_array(self):
             @csp.graph
             def g_write():
                 writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
-                writer.publish("arr", csp.curve(
-                    csp.typing.Numpy1DArray[float],
-                    [(timedelta(seconds=i + 1), a) for i, a in enumerate(arrays)],
-                ))
+                writer.publish(
+                    "arr",
+                    csp.curve(
+                        csp.typing.Numpy1DArray[float],
+                        [(timedelta(seconds=i + 1), a) for i, a in enumerate(arrays)],
+                    ),
+                )
 
             @csp.graph
             def g_read():
@@ -624,10 +688,13 @@ def test_1d_int_array(self):
             @csp.graph
             def g_write():
                 writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
-                writer.publish("arr", csp.curve(
-                    csp.typing.Numpy1DArray[int],
-                    [(timedelta(seconds=i + 1), a) for i, a in enumerate(arrays)],
-                ))
+                writer.publish(
+                    "arr",
+                    csp.curve(
+                        csp.typing.Numpy1DArray[int],
+                        [(timedelta(seconds=i + 1), a) for i, a in enumerate(arrays)],
+                    ),
+                )
 
             @csp.graph
             def g_read():
@@ -725,6 +792,35 @@ def test_zstd(self):
     def test_none(self):
         self._write_and_check_codec("", "UNCOMPRESSED")
 
+    def test_compression_name_case_insensitive(self):
+        """Upper/mixed-case compression names resolve via Arrow (case-insensitive)."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=True, compression="ZSTD")
+                writer = ParquetWriter(fname, "csp_timestamp", config=config)
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(10)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=15))
+            codec = pyarrow.parquet.ParquetFile(fname).metadata.row_group(0).column(0).compression
+            self.assertEqual(codec, "ZSTD")
+
+    def test_invalid_compression_raises_clear_error(self):
+        """An unknown compression name fails with a clear error (not a cryptic one)."""
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=True, compression="not_a_codec")
+                writer = ParquetWriter(fname, "csp_timestamp", config=config)
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 1)]))
+
+            with self.assertRaisesRegex(Exception, "compression"):
+                csp.run(g, starttime=START, endtime=timedelta(seconds=3))
+
 
 class TestOutputBatchSize(unittest.TestCase):
     """Test batch_size controls row group flushing."""
@@ -732,6 +828,7 @@ class TestOutputBatchSize(unittest.TestCase):
     def _write_and_check_row_groups(self, batch_size, n_rows):
         """Write and verify row group count matches batch_size."""
         import math
+
         with tempfile.TemporaryDirectory() as d:
             fname = os.path.join(d, "out.parquet")
 
@@ -776,7 +873,8 @@ def test_file_metadata(self):
             @csp.graph
             def g():
                 writer = ParquetWriter(
-                    fname, "csp_timestamp",
+                    fname,
+                    "csp_timestamp",
                     config=ParquetOutputConfig(allow_overwrite=True),
                     file_metadata={"created_by": "test", "version": "1.0"},
                 )
@@ -795,7 +893,8 @@ def test_column_metadata(self):
             @csp.graph
             def g():
                 writer = ParquetWriter(
-                    fname, "csp_timestamp",
+                    fname,
+                    "csp_timestamp",
                     config=ParquetOutputConfig(allow_overwrite=True),
                     column_metadata={"x": {"units": "meters", "source": "sensor_1"}},
                 )
@@ -815,7 +914,8 @@ def test_file_metadata_in_split_mode(self):
             @csp.graph
             def g():
                 writer = ParquetWriter(
-                    outdir, "csp_timestamp",
+                    outdir,
+                    "csp_timestamp",
                     config=ParquetOutputConfig(allow_overwrite=True),
                     split_columns_to_files=True,
                     file_metadata={"author": "test_suite", "version": "2.0"},
@@ -853,7 +953,9 @@ def test_file_metadata_in_split_ipc_mode(self):
             def g():
                 config = ParquetOutputConfig(allow_overwrite=True, write_arrow_binary=True, compression="")
                 writer = ParquetWriter(
-                    outdir, "csp_timestamp", config=config,
+                    outdir,
+                    "csp_timestamp",
+                    config=config,
                     split_columns_to_files=True,
                     file_metadata={"source": "ipc_test"},
                 )
@@ -906,6 +1008,53 @@ def g():
 class TestOutputEdgeCases(unittest.TestCase):
     """Edge cases and special scenarios."""
 
+    def test_bare_relative_filename_no_directory(self):
+        """Writing to a bare relative filename (no directory component) must work.
+
+        Regression: the C++ sink called mkdir(dirname(path)); dirname("out.parquet")
+        is "" and mkdir("") fails with 'Invalid argument'.
+        """
+        with tempfile.TemporaryDirectory() as d:
+            cwd = os.getcwd()
+            os.chdir(d)
+            try:
+
+                @csp.graph
+                def g():
+                    writer = ParquetWriter(
+                        "out.parquet", "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True)
+                    )
+                    writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 7)]))
+
+                csp.run(g, starttime=START, endtime=timedelta(seconds=3))
+                df = pandas.read_parquet(os.path.join(d, "out.parquet"))
+                self.assertEqual(df["x"].tolist(), [7])
+            finally:
+                os.chdir(cwd)
+
+    def test_bare_relative_directory_split_columns(self):
+        """split_columns_to_files with a bare relative directory name must work."""
+        with tempfile.TemporaryDirectory() as d:
+            cwd = os.getcwd()
+            os.chdir(d)
+            try:
+
+                @csp.graph
+                def g():
+                    writer = ParquetWriter(
+                        "split_out",
+                        "csp_timestamp",
+                        config=ParquetOutputConfig(allow_overwrite=True),
+                        split_columns_to_files=True,
+                    )
+                    writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 7)]))
+
+                csp.run(g, starttime=START, endtime=timedelta(seconds=3))
+                df = pandas.read_parquet(os.path.join(d, "split_out", "x.parquet"))
+                self.assertEqual(df["x"].tolist(), [7])
+            finally:
+                os.chdir(cwd)
+
     def test_multiple_ticks_same_timestamp(self):
         """Multiple values at the same engine time produce multiple rows."""
         with tempfile.TemporaryDirectory() as d:
@@ -915,7 +1064,12 @@ def test_multiple_ticks_same_timestamp(self):
             def g():
                 writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
                 # csp.curve with same timedelta produces ticks at same engine time
-                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 10), (timedelta(seconds=1), 20), (timedelta(seconds=1), 30)]))
+                writer.publish(
+                    "x",
+                    csp.curve(
+                        int, [(timedelta(seconds=1), 10), (timedelta(seconds=1), 20), (timedelta(seconds=1), 30)]
+                    ),
+                )
 
             csp.run(g, starttime=START, endtime=timedelta(seconds=5))
             df = pandas.read_parquet(fname)
@@ -1016,7 +1170,10 @@ class FullStruct(csp.Struct):
             e: MyEnum
 
         v = FullStruct(
-            i=42, f=3.14, b=True, s="hello",
+            i=42,
+            f=3.14,
+            b=True,
+            s="hello",
             dt=datetime(2022, 6, 15, 12, 0, tzinfo=pytz.utc),
             td=timedelta(seconds=123),
             d=date(2022, 6, 15),
@@ -1045,17 +1202,24 @@ def g_read():
     def test_float_nan_round_trip(self):
         """NaN values round-trip correctly."""
         import math
+
         with tempfile.TemporaryDirectory() as d:
             fname = os.path.join(d, "out.parquet")
 
             @csp.graph
             def g():
                 writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
-                writer.publish("x", csp.curve(float, [
-                    (timedelta(seconds=1), 1.0),
-                    (timedelta(seconds=2), float("nan")),
-                    (timedelta(seconds=3), 3.0),
-                ]))
+                writer.publish(
+                    "x",
+                    csp.curve(
+                        float,
+                        [
+                            (timedelta(seconds=1), 1.0),
+                            (timedelta(seconds=2), float("nan")),
+                            (timedelta(seconds=3), 3.0),
+                        ],
+                    ),
+                )
 
             csp.run(g, starttime=START, endtime=timedelta(seconds=5))
             df = pandas.read_parquet(fname)
@@ -1065,6 +1229,7 @@ def g():
 
     def test_struct_with_unset_fields_produces_nulls(self):
         """Struct fields that are never set produce null cells."""
+
         class Partial(csp.Struct):
             a: int
             b: float
@@ -1089,6 +1254,7 @@ def g_write():
     def test_multi_batch_with_rotation(self):
         """Rows spanning multiple batches + file rotation land in correct files."""
         with tempfile.TemporaryDirectory() as d:
+
             @csp.graph
             def g():
                 # 20 rows, batch_size=4, rotate at row 10
@@ -1096,8 +1262,10 @@ def g():
                 rotate = csp.curve(str, [(timedelta(seconds=11), os.path.join(d, "p1.parquet"))])
                 config = ParquetOutputConfig(allow_overwrite=True, batch_size=4)
                 writer = ParquetWriter(
-                    os.path.join(d, "p0.parquet"), "csp_timestamp",
-                    config=config, filename_provider=rotate,
+                    os.path.join(d, "p0.parquet"),
+                    "csp_timestamp",
+                    config=config,
+                    filename_provider=rotate,
                 )
                 writer.publish("value", data)
 
@@ -1111,6 +1279,7 @@ def g():
             pf0 = pyarrow.parquet.ParquetFile(os.path.join(d, "p0.parquet"))
             pf1 = pyarrow.parquet.ParquetFile(os.path.join(d, "p1.parquet"))
             import math
+
             self.assertEqual(pf0.metadata.num_row_groups, math.ceil(10 / 4))  # 3
             self.assertEqual(pf1.metadata.num_row_groups, math.ceil(10 / 4))  # 3
 
@@ -1118,15 +1287,20 @@ def test_file_visitor_exact_contract(self):
         """file_visitor called once per closed file, in order, including final at stop()."""
         visited = []
         with tempfile.TemporaryDirectory() as d:
+
             @csp.graph
             def g():
                 data = csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(9)])
-                rotate = csp.curve(str, [
-                    (timedelta(seconds=4), os.path.join(d, "p1.parquet")),
-                    (timedelta(seconds=7), os.path.join(d, "p2.parquet")),
-                ])
+                rotate = csp.curve(
+                    str,
+                    [
+                        (timedelta(seconds=4), os.path.join(d, "p1.parquet")),
+                        (timedelta(seconds=7), os.path.join(d, "p2.parquet")),
+                    ],
+                )
                 writer = ParquetWriter(
-                    os.path.join(d, "p0.parquet"), "csp_timestamp",
+                    os.path.join(d, "p0.parquet"),
+                    "csp_timestamp",
                     config=ParquetOutputConfig(allow_overwrite=True),
                     filename_provider=rotate,
                     file_visitor=lambda f: visited.append(f),
@@ -1152,7 +1326,9 @@ def test_metadata_in_ipc_mode(self):
             def g():
                 config = ParquetOutputConfig(allow_overwrite=True, write_arrow_binary=True, compression="")
                 writer = ParquetWriter(
-                    fname, "csp_timestamp", config=config,
+                    fname,
+                    "csp_timestamp",
+                    config=config,
                     file_metadata={"author": "test"},
                     column_metadata={"x": {"units": "kg"}},
                 )
@@ -1166,79 +1342,5 @@ def g():
             self.assertEqual(x_field.metadata[b"units"], b"kg")
 
 
-class TestOutputSinkDirect(unittest.TestCase):
-    """Direct unit tests for the Python RecordBatchSink without the csp engine."""
-
-    def test_sink_parquet_lifecycle(self):
-        """Test on_start → on_file_change → on_batch → on_stop lifecycle."""
-        from csp.adapters._parquet_rb_writer import RecordBatchSink, _parquet_writer_factory
-
-        with tempfile.TemporaryDirectory() as d:
-            fname = os.path.join(d, "test.parquet")
-            wf = _parquet_writer_factory("snappy", allow_overwrite=True)
-            sink = RecordBatchSink(wf)
-
-            schema = pyarrow.schema([pyarrow.field("x", pyarrow.int64())])
-            # Simulate C++ calling the sink
-            sink.on_start(schema.__arrow_c_schema__())
-            sink.on_file_change(fname)
-            # Write a batch
-            rb = pyarrow.record_batch([pyarrow.array([1, 2, 3])], schema=schema)
-            schema_capsule, array_capsule = rb.__arrow_c_array__()
-            sink.on_batch(schema_capsule, array_capsule)
-            sink.on_stop()
-
-            # Verify file was written
-            df = pandas.read_parquet(fname)
-            self.assertEqual(df["x"].tolist(), [1, 2, 3])
-
-    def test_sink_ipc_lifecycle(self):
-        """Test IPC sink lifecycle."""
-        from csp.adapters._parquet_rb_writer import RecordBatchSink, _arrow_ipc_writer_factory
-
-        with tempfile.TemporaryDirectory() as d:
-            fname = os.path.join(d, "test.arrow")
-            wf = _arrow_ipc_writer_factory("", allow_overwrite=True)
-            sink = RecordBatchSink(wf)
-
-            schema = pyarrow.schema([pyarrow.field("val", pyarrow.float64())])
-            sink.on_start(schema.__arrow_c_schema__())
-            sink.on_file_change(fname)
-            rb = pyarrow.record_batch([pyarrow.array([1.1, 2.2])], schema=schema)
-            schema_capsule, array_capsule = rb.__arrow_c_array__()
-            sink.on_batch(schema_capsule, array_capsule)
-            sink.on_stop()
-
-            reader = pyarrow.ipc.open_stream(pyarrow.memory_map(fname))
-            table = reader.read_all()
-            self.assertEqual(table.column("val").to_pylist(), [1.1, 2.2])
-
-    def test_sink_file_visitor_called_on_rotation(self):
-        """file_visitor called when rotating and on stop."""
-        from csp.adapters._parquet_rb_writer import RecordBatchSink, _parquet_writer_factory
-
-        visited = []
-        with tempfile.TemporaryDirectory() as d:
-            wf = _parquet_writer_factory("snappy", allow_overwrite=True)
-            sink = RecordBatchSink(wf, file_visitor=lambda f: visited.append(f))
-
-            schema = pyarrow.schema([pyarrow.field("x", pyarrow.int64())])
-            sink.on_start(schema.__arrow_c_schema__())
-
-            f1 = os.path.join(d, "f1.parquet")
-            f2 = os.path.join(d, "f2.parquet")
-            sink.on_file_change(f1)
-            rb = pyarrow.record_batch([pyarrow.array([1])], schema=schema)
-            s, a = rb.__arrow_c_array__()
-            sink.on_batch(s, a)
-            sink.on_file_change(f2)  # closes f1, visits f1
-            rb2 = pyarrow.record_batch([pyarrow.array([2])], schema=schema)
-            s2, a2 = rb2.__arrow_c_array__()
-            sink.on_batch(s2, a2)
-            sink.on_stop()  # closes f2, visits f2
-
-            self.assertEqual(visited, [f1, f2])
-
-
 if __name__ == "__main__":
     unittest.main()

From 52cdce795c23cce07805cddef9b0bf1f925a6bee Mon Sep 17 00:00:00 2001
From: Arham Chopra <arham.chopra@cubistsystematic.com>
Date: Fri, 12 Jun 2026 16:57:47 -0400
Subject: [PATCH 05/11] Parquet output: drop dead manager index-sink, harden
 stop() order

Remove the unused manager-level m_indexSink/setIndexSink (per-basket index
sinks are unaffected) and stop all writers before destroying any in stop().

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>
---
 cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp | 8 +++-----
 cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h   | 2 --
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp
index 3b48561d6..64a55a0a6 100644
--- a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp
+++ b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp
@@ -27,22 +27,20 @@ void ParquetOutputAdapterManager::start( DateTime starttime, DateTime endtime )
     m_parquetWriter -> start();
     for( auto &&writer:m_dictBasketWriters )
     {
-        // Only overwrite per-basket index sink if a shared one was explicitly set
-        if( m_indexSink.onBatch )
-            writer -> setIndexSink( m_indexSink );
         writer -> start();
     }
 }
 
 void ParquetOutputAdapterManager::stop()
 {
+    // Stop all writers (flush + close their files) before destroying any of them, so a basket
+    // writer can never observe a destroyed sibling during teardown.
     m_parquetWriter -> stop();
-    m_parquetWriter = nullptr;
-
     for( auto &&writer:m_dictBasketWriters )
     {
         writer -> stop();
     }
+    m_parquetWriter = nullptr;
     m_dictBasketWriters.clear();
 }
 
diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h
index b6d2bf538..f6a1927dc 100644
--- a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h
+++ b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h
@@ -53,7 +53,6 @@ class ParquetOutputAdapterManager final : public csp::AdapterManager
     void scheduleEndCycle();
 
     void setSink( RecordBatchSink sink );
-    void setIndexSink( RecordBatchSink sink ) { m_indexSink = std::move( sink ); }
     void setSinkFactory( SinkFactory factory ) { m_sinkFactory = std::move( factory ); }
 
 private:
@@ -67,7 +66,6 @@ class ParquetOutputAdapterManager final : public csp::AdapterManager
     std::unordered_map<std::string, int>                        m_dictBasketWriterIndexByName;
     std::vector<std::unique_ptr<ParquetDictBasketOutputWriter>> m_dictBasketWriters;
     ParquetOutputFilenameAdapter                                *m_outputFilenameAdapter;
-    RecordBatchSink                                             m_indexSink;
     SinkFactory                                                 m_sinkFactory;
 };
 

From 4f95ec6fc332918f53bfc308cad39aa2fb21b8b1 Mon Sep 17 00:00:00 2001
From: Arham Chopra <arham.chopra@cubistsystematic.com>
Date: Fri, 12 Jun 2026 17:16:28 -0400
Subject: [PATCH 06/11] Parquet output: minor cleanups (scratch reset, batch
 guard, docs)

Encapsulate the scratch isSet reset behind StructField::clearIsSet, add a
debug-only length assert in buildRecordBatch, and document that file_visitor
runs synchronously on the engine thread.

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>
---
 .../adapters/parquet/ArrowBackedArrayBuilder.cpp |  3 +--
 cpp/csp/adapters/parquet/ParquetWriter.cpp       |  8 +++++++-
 cpp/csp/engine/Struct.h                          | 16 ++++++++++------
 cpp/csp/python/adapters/parquetadapterimpl.cpp   |  5 +++--
 csp/adapters/output_adapters/parquet.py          |  5 ++++-
 5 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
index c09114002..dde8b2576 100644
--- a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
+++ b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
@@ -114,8 +114,7 @@ void ArrowBackedArrayBuilder::handleRowFinished()
         m_writer -> writeNext( m_scratch.get() );
 
         // Clear the isSet bit so next cycle starts with null
-        uint8_t * mask = reinterpret_cast<uint8_t *>( m_scratch.get() ) + m_field -> maskOffset();
-        *mask &= ~m_field -> maskBitMask();
+        m_field -> clearIsSet( m_scratch.get() );
     }
     else
     {
diff --git a/cpp/csp/adapters/parquet/ParquetWriter.cpp b/cpp/csp/adapters/parquet/ParquetWriter.cpp
index 6d988e3fd..33bf35858 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetWriter.cpp
@@ -238,7 +238,13 @@ std::shared_ptr<::arrow::RecordBatch> ParquetWriter::buildRecordBatch()
     std::vector<std::shared_ptr<::arrow::Array>> columns;
     columns.reserve( m_columnBuilders.size() );
     for( auto && builder : m_columnBuilders )
-        columns.push_back( builder -> buildArray() );
+    {
+        auto column = builder -> buildArray();
+        // Every column must contain exactly the rows accumulated this chunk; a mismatch would build a
+        // corrupt RecordBatch (debug-only guard, compiled out in release).
+        CSP_ASSERT( column -> length() == static_cast<std::int64_t>( m_curChunkSize ) );
+        columns.push_back( std::move( column ) );
+    }
     return ::arrow::RecordBatch::Make( m_schema, m_curChunkSize, std::move( columns ) );
 }
 
diff --git a/cpp/csp/engine/Struct.h b/cpp/csp/engine/Struct.h
index 7c203a8b8..7082fefc9 100644
--- a/cpp/csp/engine/Struct.h
+++ b/cpp/csp/engine/Struct.h
@@ -77,6 +77,16 @@ class StructField
         (*m) |= m_noneMaskBitMask;
     }
 
+    //Marks this field "not set" without releasing/cleaning its current value.  The same capability is
+    //already reachable through the public maskOffset()/maskBitMask() accessors; this just encapsulates the
+    //bit poke.  Used by callers that reuse a struct as a per-row scratch buffer (e.g. the parquet
+    //ArrowBackedArrayBuilder appends the scratch value, then clears the bit so the next row defaults to null).
+    void clearIsSet( Struct * s ) const
+    {
+        uint8_t * m = reinterpret_cast<uint8_t *>( s ) + m_maskOffset;
+        (*m) &= ~m_maskBitMask;
+    }
+
     //copy methods need not deal with mask set/unset, only copy values
     virtual void copyFrom( const Struct * src, Struct * dest ) const = 0;
 
@@ -121,12 +131,6 @@ class StructField
         return reinterpret_cast<uint8_t *>( s ) + m_offset;
     }
 
-    void clearIsSet( Struct * s ) const
-    {
-        uint8_t * m = reinterpret_cast<uint8_t *>( s ) + m_maskOffset;
-        (*m) &= ~m_maskBitMask;
-    }
-
     void clearIsNone( Struct * s ) const
     {
         uint8_t * m = reinterpret_cast<uint8_t *>( s ) + m_maskOffset;
diff --git a/cpp/csp/python/adapters/parquetadapterimpl.cpp b/cpp/csp/python/adapters/parquetadapterimpl.cpp
index 39a275acd..acd774a71 100644
--- a/cpp/csp/python/adapters/parquetadapterimpl.cpp
+++ b/cpp/csp/python/adapters/parquetadapterimpl.cpp
@@ -686,8 +686,9 @@ csp::AdapterManager *create_parquet_output_adapter_manager( PyEngine *engine, co
     bool        allowOverwrite   = properties.get<bool>( "allow_overwrite" );
     std::string compression      = properties.get<std::string>( "compression" );
 
-    // Optional Python file_visitor: wrapped as a plain C++ callback, invoked (with the GIL
-    // held during the run) after each file is closed.  Only the main sink visits files.
+    // Optional Python file_visitor: wrapped as a plain C++ callback, invoked synchronously (with the
+    // GIL held during the run) after each file is closed, so a slow visitor blocks the engine.  Only
+    // the main sink visits files.
     std::function<void( const std::string & )> fileVisitor;
     DialectGenericType visitorDG;
     if( properties.tryGet( "file_visitor", visitorDG ) )
diff --git a/csp/adapters/output_adapters/parquet.py b/csp/adapters/output_adapters/parquet.py
index ff59ad044..e40d6401a 100644
--- a/csp/adapters/output_adapters/parquet.py
+++ b/csp/adapters/output_adapters/parquet.py
@@ -73,7 +73,10 @@ def __init__(
         file_name must be a folder into which the data will be written.
         :param file_metadata: optional str:str dict that will get written as file-level metadata
         :param column_metadata: optional dict of column : { str:str} that will get written as column-level metadata
-        :param file_visitor: optional callable that will be called, after a file is written, with the file name.
+        :param file_visitor: optional callable invoked with the file path after each file is closed (on rotation
+        and at shutdown). It runs synchronously on the engine thread with the GIL held, so a slow visitor (e.g.
+        uploading to remote storage) blocks the engine for its duration -- offload heavy work to a background
+        queue/thread if that matters. An exception raised by the visitor propagates out of csp.run.
         """
         super().__init__()
         config = ParquetOutputConfig() if config is None else config.copy()

From 53e2b2181fe7c21039c2d466630169a99a3c3239 Mon Sep 17 00:00:00 2001
From: Arham Chopra <arham.chopra@cubistsystematic.com>
Date: Fri, 12 Jun 2026 18:29:40 -0400
Subject: [PATCH 07/11] Fix Windows CI: release parquet/IPC file handles before
 tempdir cleanup

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>
---
 csp/tests/adapters/test_parquet_output.py | 60 +++++++++++++----------
 1 file changed, 35 insertions(+), 25 deletions(-)

diff --git a/csp/tests/adapters/test_parquet_output.py b/csp/tests/adapters/test_parquet_output.py
index 58867ae08..082e9439b 100644
--- a/csp/tests/adapters/test_parquet_output.py
+++ b/csp/tests/adapters/test_parquet_output.py
@@ -23,6 +23,18 @@
 START = datetime(2022, 1, 1, tzinfo=pytz.utc)
 
 
+def _read_ipc(path):
+    """Read an Arrow IPC stream fully into memory, leaving no open file handle.
+
+    ``pyarrow.memory_map`` keeps an OS handle open until GC, which blocks
+    ``tempfile.TemporaryDirectory`` cleanup on Windows ("file in use by another
+    process"). Reading the bytes up front and parsing from an in-memory buffer
+    avoids holding the file open.
+    """
+    with open(path, "rb") as fh:
+        return pyarrow.ipc.open_stream(pyarrow.py_buffer(fh.read()))
+
+
 class TestOutputScalarTypes(unittest.TestCase):
     """Test writing all scalar types to parquet and reading them back."""
 
@@ -410,7 +422,7 @@ def g():
                 writer.publish("y", csp.curve(float, [(timedelta(seconds=i + 1), i * 1.5) for i in range(5)]))
 
             csp.run(g, starttime=START, endtime=timedelta(seconds=10))
-            reader = pyarrow.ipc.open_stream(pyarrow.memory_map(fname))
+            reader = _read_ipc(fname)
             table = reader.read_all()
             self.assertEqual(table.column("x").to_pylist(), [0, 10, 20, 30, 40])
             self.assertEqual(table.column("y").to_pylist(), [0.0, 1.5, 3.0, 4.5, 6.0])
@@ -426,7 +438,7 @@ def g():
                 writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 42)]))
 
             csp.run(g, starttime=START, endtime=timedelta(seconds=5))
-            reader = pyarrow.ipc.open_stream(pyarrow.memory_map(fname))
+            reader = _read_ipc(fname)
             table = reader.read_all()
             self.assertEqual(table.column("x").to_pylist(), [42])
 
@@ -772,9 +784,8 @@ def g():
                 writer.publish("x", csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(n_rows)]))
 
             csp.run(g, starttime=START, endtime=timedelta(seconds=n_rows + 5))
-            pf = pyarrow.parquet.ParquetFile(fname)
-            # Check actual codec in row group metadata
-            actual_codec = pf.metadata.row_group(0).column(0).compression
+            # Read footer eagerly (no lingering file handle: Windows can't delete an open file)
+            actual_codec = pyarrow.parquet.read_metadata(fname).row_group(0).column(0).compression
             self.assertEqual(actual_codec, expected_codec)
             # Verify data integrity
             df = pandas.read_parquet(fname)
@@ -804,7 +815,7 @@ def g():
                 writer.publish("x", csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(10)]))
 
             csp.run(g, starttime=START, endtime=timedelta(seconds=15))
-            codec = pyarrow.parquet.ParquetFile(fname).metadata.row_group(0).column(0).compression
+            codec = pyarrow.parquet.read_metadata(fname).row_group(0).column(0).compression
             self.assertEqual(codec, "ZSTD")
 
     def test_invalid_compression_raises_clear_error(self):
@@ -839,9 +850,8 @@ def g():
                 writer.publish("x", csp.curve(int, [(timedelta(seconds=i + 1), i) for i in range(n_rows)]))
 
             csp.run(g, starttime=START, endtime=timedelta(seconds=n_rows + 5))
-            pf = pyarrow.parquet.ParquetFile(fname)
             expected_row_groups = math.ceil(n_rows / batch_size)
-            self.assertEqual(pf.metadata.num_row_groups, expected_row_groups)
+            self.assertEqual(pyarrow.parquet.read_metadata(fname).num_row_groups, expected_row_groups)
             # Verify data integrity
             df = pandas.read_parquet(fname)
             self.assertEqual(df["x"].tolist(), list(range(n_rows)))
@@ -881,8 +891,7 @@ def g():
                 writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 1)]))
 
             csp.run(g, starttime=START, endtime=timedelta(seconds=5))
-            pf = pyarrow.parquet.ParquetFile(fname)
-            metadata = pf.schema_arrow.metadata
+            metadata = pyarrow.parquet.read_schema(fname).metadata
             self.assertEqual(metadata[b"created_by"], b"test")
             self.assertEqual(metadata[b"version"], b"1.0")
 
@@ -901,8 +910,7 @@ def g():
                 writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 1)]))
 
             csp.run(g, starttime=START, endtime=timedelta(seconds=5))
-            pf = pyarrow.parquet.ParquetFile(fname)
-            x_field = pf.schema_arrow.field("x")
+            x_field = pyarrow.parquet.read_schema(fname).field("x")
             self.assertEqual(x_field.metadata[b"units"], b"meters")
             self.assertEqual(x_field.metadata[b"source"], b"sensor_1")
 
@@ -931,17 +939,17 @@ def g():
             self.assertTrue(os.path.isfile(x_file))
             self.assertTrue(os.path.isfile(ts_file))
 
-            pf_x = pyarrow.parquet.ParquetFile(x_file)
-            pf_ts = pyarrow.parquet.ParquetFile(ts_file)
+            schema_x = pyarrow.parquet.read_schema(x_file)
+            schema_ts = pyarrow.parquet.read_schema(ts_file)
 
             # File-level metadata on both files
-            self.assertEqual(pf_x.schema_arrow.metadata[b"author"], b"test_suite")
-            self.assertEqual(pf_x.schema_arrow.metadata[b"version"], b"2.0")
-            self.assertEqual(pf_ts.schema_arrow.metadata[b"author"], b"test_suite")
-            self.assertEqual(pf_ts.schema_arrow.metadata[b"version"], b"2.0")
+            self.assertEqual(schema_x.metadata[b"author"], b"test_suite")
+            self.assertEqual(schema_x.metadata[b"version"], b"2.0")
+            self.assertEqual(schema_ts.metadata[b"author"], b"test_suite")
+            self.assertEqual(schema_ts.metadata[b"version"], b"2.0")
 
             # Column-level metadata preserved on x
-            x_field = pf_x.schema_arrow.field("x")
+            x_field = schema_x.field("x")
             self.assertEqual(x_field.metadata[b"units"], b"kg")
 
     def test_file_metadata_in_split_ipc_mode(self):
@@ -965,7 +973,7 @@ def g():
 
             val_file = os.path.join(outdir, "val.arrow")
             self.assertTrue(os.path.isfile(val_file))
-            reader = pyarrow.ipc.open_stream(pyarrow.memory_map(val_file))
+            reader = _read_ipc(val_file)
             self.assertEqual(reader.schema.metadata[b"source"], b"ipc_test")
 
 
@@ -1276,12 +1284,14 @@ def g():
             self.assertEqual(df0["value"].tolist(), list(range(10)))
             self.assertEqual(df1["value"].tolist(), list(range(10, 20)))
             # Verify batch_size=4 → correct row group counts
-            pf0 = pyarrow.parquet.ParquetFile(os.path.join(d, "p0.parquet"))
-            pf1 = pyarrow.parquet.ParquetFile(os.path.join(d, "p1.parquet"))
             import math
 
-            self.assertEqual(pf0.metadata.num_row_groups, math.ceil(10 / 4))  # 3
-            self.assertEqual(pf1.metadata.num_row_groups, math.ceil(10 / 4))  # 3
+            self.assertEqual(
+                pyarrow.parquet.read_metadata(os.path.join(d, "p0.parquet")).num_row_groups, math.ceil(10 / 4)
+            )  # 3
+            self.assertEqual(
+                pyarrow.parquet.read_metadata(os.path.join(d, "p1.parquet")).num_row_groups, math.ceil(10 / 4)
+            )  # 3
 
     def test_file_visitor_exact_contract(self):
         """file_visitor called once per closed file, in order, including final at stop()."""
@@ -1335,7 +1345,7 @@ def g():
                 writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 1)]))
 
             csp.run(g, starttime=START, endtime=timedelta(seconds=5))
-            reader = pyarrow.ipc.open_stream(pyarrow.memory_map(fname))
+            reader = _read_ipc(fname)
             schema = reader.schema
             self.assertEqual(schema.metadata[b"author"], b"test")
             x_field = schema.field("x")

From 0002b0d51cf7bf79d893dee2bdc02d1ed63768d7 Mon Sep 17 00:00:00 2001
From: Arham Chopra <arham.chopra@cubistsystematic.com>
Date: Fri, 12 Jun 2026 18:38:38 -0400
Subject: [PATCH 08/11] Parquet output: remove dead code found in full audit

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>
---
 .../parquet/ArrowBackedArrayBuilder.cpp       |  5 -
 .../parquet/ArrowBackedArrayBuilder.h         |  1 -
 .../parquet/ArrowSingleColumnArrayBuilder.h   |  6 --
 cpp/csp/adapters/parquet/CMakeLists.txt       |  2 -
 .../parquet/FileNameGeneratorReplicator.h     | 95 -------------------
 cpp/csp/adapters/parquet/ParquetWriter.cpp    |  5 -
 cpp/csp/adapters/parquet/ParquetWriter.h      |  2 -
 .../adapters/parquet/RecordBatchWithFlag.h    | 21 ----
 8 files changed, 137 deletions(-)
 delete mode 100644 cpp/csp/adapters/parquet/FileNameGeneratorReplicator.h
 delete mode 100644 cpp/csp/adapters/parquet/RecordBatchWithFlag.h

diff --git a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
index dde8b2576..377b3b235 100644
--- a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
+++ b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
@@ -95,11 +95,6 @@ std::shared_ptr<::arrow::DataType> ArrowBackedArrayBuilder::getDataType()
     return m_writer -> dataTypes()[0];
 }
 
-std::shared_ptr<::arrow::ArrayBuilder> ArrowBackedArrayBuilder::getBuilder()
-{
-    return m_writer -> builder();
-}
-
 int64_t ArrowBackedArrayBuilder::length() const
 {
     return m_writer -> builder() -> length();
diff --git a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
index b0c970da8..92c56651e 100644
--- a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
+++ b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
@@ -39,7 +39,6 @@ class ArrowBackedArrayBuilder : public ArrowSingleColumnArrayBuilder
 
     // --- ArrowSingleColumnArrayBuilder interface ---
     std::shared_ptr<::arrow::DataType> getDataType() override;
-    std::shared_ptr<::arrow::ArrayBuilder> getBuilder() override;
     int64_t length() const override;
     void handleRowFinished() override;
     std::shared_ptr<::arrow::Array> buildArray() override;
diff --git a/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h b/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
index 00e564a8f..eb9f61cac 100644
--- a/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
+++ b/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
@@ -25,7 +25,6 @@ class ArrowSingleColumnArrayBuilder
     virtual ~ArrowSingleColumnArrayBuilder() {}
 
     virtual std::shared_ptr<::arrow::DataType> getDataType() = 0;
-    virtual std::shared_ptr<::arrow::ArrayBuilder> getBuilder() = 0;
 
     virtual int64_t length() const = 0;
 
@@ -64,11 +63,6 @@ class ListColumnArrayBuilder : public ArrowSingleColumnArrayBuilder
         return m_builderPtr -> type();
     }
 
-    virtual std::shared_ptr<::arrow::ArrayBuilder> getBuilder() override
-    {
-        return m_builderPtr;
-    }
-
     virtual int64_t length() const override
     {
         return m_builderPtr -> length();
diff --git a/cpp/csp/adapters/parquet/CMakeLists.txt b/cpp/csp/adapters/parquet/CMakeLists.txt
index a6312bb92..d89a0cd49 100644
--- a/cpp/csp/adapters/parquet/CMakeLists.txt
+++ b/cpp/csp/adapters/parquet/CMakeLists.txt
@@ -1,7 +1,6 @@
 set(PARQUET_HEADER_FILES
         ArrowBackedArrayBuilder.h
         ArrowSingleColumnArrayBuilder.h
-        FileNameGeneratorReplicator.h
         ParquetDictBasketOutputWriter.h
         ParquetInputAdapterManager.h
         ParquetOutputAdapter.h
@@ -12,7 +11,6 @@ set(PARQUET_HEADER_FILES
         RecordBatchFileSink.h
         RecordBatchSink.h
         RecordBatchStreamSource.h
-        RecordBatchWithFlag.h
 )
 
 set(PARQUET_SOURCE_FILES
diff --git a/cpp/csp/adapters/parquet/FileNameGeneratorReplicator.h b/cpp/csp/adapters/parquet/FileNameGeneratorReplicator.h
deleted file mode 100644
index fdbb53cf2..000000000
--- a/cpp/csp/adapters/parquet/FileNameGeneratorReplicator.h
+++ /dev/null
@@ -1,95 +0,0 @@
-#ifndef _IN_CSP_ADAPTERS_PARQUET_FileNameGeneratorReplicator_H
-#define _IN_CSP_ADAPTERS_PARQUET_FileNameGeneratorReplicator_H
-
-#include <csp/core/Generator.h>
-#include <csp/core/Time.h>
-
-#include <memory>
-#include <string>
-#include <vector>
-
-namespace csp::adapters::parquet
-{
-
-/**
- * Consumes a string generator (producing folder/file names) and caches the results so they can be
- * replayed by multiple child generators.  Each replica can optionally append a suffix.
- */
-class FileNameGeneratorReplicator
-{
-public:
-    using Ptr = std::shared_ptr<FileNameGeneratorReplicator>;
-    using GeneratorPtr = csp::Generator<std::string, csp::DateTime, csp::DateTime>::Ptr;
-
-    FileNameGeneratorReplicator( GeneratorPtr source )
-            : m_generatorPtr( source )
-    {
-    }
-
-    void init( csp::DateTime start, csp::DateTime end )
-    {
-        m_generatorPtr -> init( start, end );
-    }
-
-    const std::vector<std::string> &getFileNames(){ return m_fileNames; }
-
-    void consumeNextGeneratorFile()
-    {
-        std::string nextFile;
-        if( m_generatorPtr -> next( nextFile ) )
-        {
-            m_fileNames.push_back( std::move( nextFile ) );
-        }
-    }
-
-    GeneratorPtr getGeneratorReplica( const std::string &suffix = "" )
-    {
-        return std::make_shared<ChildGenerator>( *this, suffix );
-    }
-
-private:
-    class ChildGenerator : public csp::Generator<std::string, csp::DateTime, csp::DateTime>
-    {
-    public:
-        ChildGenerator( FileNameGeneratorReplicator &owner, const std::string &suffix )
-                : m_owner( owner ), m_suffix( suffix ), m_nextIndex( 0 )
-        {
-        }
-
-        void init( csp::DateTime, csp::DateTime ){}
-
-        bool next( std::string &value )
-        {
-            if( m_nextIndex < 0 )
-            {
-                return false;
-            }
-            const std::vector<std::string> &folders = m_owner.getFileNames();
-            if( m_nextIndex >= static_cast<int>(folders.size()) )
-            {
-                m_owner.consumeNextGeneratorFile();
-            }
-            if( m_nextIndex >= static_cast<int>(folders.size() ) )
-            {
-                m_nextIndex = -1;
-                return false;
-            }
-
-            value = folders[ m_nextIndex++ ] + m_suffix;
-            return true;
-        }
-
-    private:
-        FileNameGeneratorReplicator &m_owner;
-        const std::string           m_suffix;
-        int                         m_nextIndex = 0;
-    };
-
-private:
-    GeneratorPtr             m_generatorPtr;
-    std::vector<std::string> m_fileNames;
-};
-
-} // namespace csp::adapters::parquet
-
-#endif
diff --git a/cpp/csp/adapters/parquet/ParquetWriter.cpp b/cpp/csp/adapters/parquet/ParquetWriter.cpp
index 33bf35858..84b5b6c53 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetWriter.cpp
@@ -110,11 +110,6 @@ ListColumnParquetOutputAdapter *ParquetWriter::getListOutputAdapter(
     return res;
 }
 
-PushInputAdapter *ParquetWriter::getStatusAdapter()
-{
-    return nullptr;
-}
-
 void ParquetWriter::start()
 {
     std::vector<std::shared_ptr<::arrow::Field>> arrowFields;
diff --git a/cpp/csp/adapters/parquet/ParquetWriter.h b/cpp/csp/adapters/parquet/ParquetWriter.h
index 30d03b9a9..7869a5ae4 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.h
+++ b/cpp/csp/adapters/parquet/ParquetWriter.h
@@ -44,8 +44,6 @@ class ParquetWriter : public EndCycleListener
     StructParquetOutputHandler *getStructOutputHandler( CspTypePtr &type, csp::DictionaryPtr fieldMap );
     ListColumnParquetOutputHandler *getListOutputHandler( CspTypePtr &elemType, const std::string &columnName );
 
-    PushInputAdapter *getStatusAdapter();
-
     virtual void start();
     virtual void stop();
 
diff --git a/cpp/csp/adapters/parquet/RecordBatchWithFlag.h b/cpp/csp/adapters/parquet/RecordBatchWithFlag.h
deleted file mode 100644
index 7f0f688b7..000000000
--- a/cpp/csp/adapters/parquet/RecordBatchWithFlag.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#ifndef _IN_CSP_ADAPTERS_PARQUET_RecordBatchWithFlag_H
-#define _IN_CSP_ADAPTERS_PARQUET_RecordBatchWithFlag_H
-
-#include <arrow/record_batch.h>
-#include <memory>
-#include <string>
-#include <unordered_map>
-
-namespace csp::adapters::parquet
-{
-
-struct RecordBatchWithFlag
-{
-    std::shared_ptr<::arrow::RecordBatch> batch;
-    std::unordered_map<std::string, std::shared_ptr<::arrow::RecordBatch>> basketBatches;
-    bool                                  schemaChanged;
-};
-
-} // namespace csp::adapters::parquet
-
-#endif

From 9f413e2749daa37c60cbb24ccc72b05f47144dbb Mon Sep 17 00:00:00 2001
From: Arham Chopra <arham.chopra@cubistsystematic.com>
Date: Mon, 15 Jun 2026 16:50:51 -0400
Subject: [PATCH 09/11] Parquet output: harden teardown, basket metadata, and
 tests

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>
---
 .../parquet/ParquetDictBasketOutputWriter.cpp |  60 +++-
 .../parquet/ParquetOutputAdapterManager.cpp   |  24 +-
 cpp/csp/adapters/parquet/ParquetWriter.cpp    |  48 ++-
 cpp/csp/adapters/parquet/ParquetWriter.h      |   6 +
 .../adapters/parquet/RecordBatchFileSink.cpp  |  47 ++-
 csp/adapters/output_adapters/parquet.py       |   2 +
 csp/tests/adapters/test_parquet_output.py     | 287 ++++++++++++++++++
 7 files changed, 438 insertions(+), 36 deletions(-)

diff --git a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
index 908b09a6e..abb024032 100644
--- a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
@@ -1,6 +1,7 @@
 #include <csp/adapters/parquet/ParquetDictBasketOutputWriter.h>
 #include <csp/adapters/parquet/ParquetOutputAdapter.h>
 #include <arrow/record_batch.h>
+#include <exception>
 
 namespace csp::adapters::parquet
 {
@@ -19,7 +20,7 @@ void ParquetDictBasketOutputWriter::start()
     ParquetWriter::start();
     m_indexSchema = ::arrow::schema( { ::arrow::field(
         m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getColumnName(),
-        m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getDataType() ) } );
+        m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getDataType() ) }, getFileMetaData() );
     if( m_indexSink.onStart )
         m_indexSink.onStart( m_indexSchema );
     auto & fileName = m_adapterMgr.getFileName();
@@ -29,17 +30,31 @@ void ParquetDictBasketOutputWriter::start()
 
 void ParquetDictBasketOutputWriter::stop()
 {
+    // Exception-safe teardown: a failure writing/closing the index sink must not skip the value+symbol
+    // writer's stop() (which flushes the final batch and writes its footer) -- otherwise an index-side
+    // I/O error would leave the main basket data file unreadable. Attempt every step, rethrow first.
+    std::exception_ptr firstError;
+    auto guard = [&firstError]( auto && fn )
+    {
+        try { fn(); }
+        catch( ... ) { if( !firstError ) firstError = std::current_exception(); }
+    };
+
     auto && indexBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 );
     if( indexBuilder -> length() > 0 && m_indexSink.onBatch )
-    {
-        auto arr = indexBuilder -> buildArray();
-        auto rb  = ::arrow::RecordBatch::Make( m_indexSchema, arr -> length(), { arr } );
-        m_indexSink.onBatch( rb );
-    }
+        guard( [&]
+        {
+            auto arr = indexBuilder -> buildArray();
+            auto rb  = ::arrow::RecordBatch::Make( m_indexSchema, arr -> length(), { arr } );
+            m_indexSink.onBatch( rb );
+        } );
     if( m_indexSink.onStop )
-        m_indexSink.onStop();
+        guard( [&] { m_indexSink.onStop(); } );
 
-    ParquetWriter::stop();
+    guard( [&] { ParquetWriter::stop(); } );
+
+    if( firstError )
+        std::rethrow_exception( firstError );
 }
 
 void ParquetDictBasketOutputWriter::writeValue( const std::string &valueKey, const TimeSeriesProvider *ts )
@@ -74,17 +89,32 @@ void ParquetDictBasketOutputWriter::onEndCycle()
 
 void ParquetDictBasketOutputWriter::onFileNameChange( const std::string &fileName )
 {
-    ParquetWriter::onFileNameChange( fileName );
+    // Rotate the main (value+symbol) writer, then flush pending index data and rotate the index file.
+    // Each step is guarded so a failure in one cannot leave the index file un-rotated and
+    // desynchronised from the main file; the first error is rethrown after all steps are attempted.
+    std::exception_ptr firstError;
+    auto guard = [&firstError]( auto && fn )
+    {
+        try { fn(); }
+        catch( ... ) { if( !firstError ) firstError = std::current_exception(); }
+    };
+
+    guard( [&] { ParquetWriter::onFileNameChange( fileName ); } );
+
     // Flush any pending index data
     auto && indexBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 );
     if( indexBuilder -> length() > 0 && m_indexSink.onBatch )
-    {
-        auto arr = indexBuilder -> buildArray();
-        auto rb  = ::arrow::RecordBatch::Make( m_indexSchema, arr -> length(), { arr } );
-        m_indexSink.onBatch( rb );
-    }
+        guard( [&]
+        {
+            auto arr = indexBuilder -> buildArray();
+            auto rb  = ::arrow::RecordBatch::Make( m_indexSchema, arr -> length(), { arr } );
+            m_indexSink.onBatch( rb );
+        } );
     if( m_indexSink.onFileChange )
-        m_indexSink.onFileChange( fileName );
+        guard( [&] { m_indexSink.onFileChange( fileName ); } );
+
+    if( firstError )
+        std::rethrow_exception( firstError );
 }
 
 SingleColumnParquetOutputHandler *ParquetDictBasketOutputWriter::createScalarOutputHandler( CspTypePtr type, const std::string &name )
diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp
index 64a55a0a6..7a71bb77a 100644
--- a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp
+++ b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.cpp
@@ -4,6 +4,7 @@
 #include <csp/adapters/parquet/ParquetDictBasketOutputWriter.h>
 #include <csp/adapters/parquet/ParquetWriter.h>
 #include <csp/engine/Dictionary.h>
+#include <exception>
 
 namespace csp::adapters::parquet
 {
@@ -33,15 +34,26 @@ void ParquetOutputAdapterManager::start( DateTime starttime, DateTime endtime )
 
 void ParquetOutputAdapterManager::stop()
 {
-    // Stop all writers (flush + close their files) before destroying any of them, so a basket
-    // writer can never observe a destroyed sibling during teardown.
-    m_parquetWriter -> stop();
+    // Stop every writer (flush + close its files) before destroying any of them, so a basket writer
+    // can never observe a destroyed sibling during teardown. A failure stopping one writer must not
+    // skip the others (that would leave their files without footers / leak fds): attempt them all and
+    // rethrow the first error.
+    std::exception_ptr firstError;
+    auto stopWriter = [&firstError]( auto *writer )
+    {
+        try { writer -> stop(); }
+        catch( ... ) { if( !firstError ) firstError = std::current_exception(); }
+    };
+    if( m_parquetWriter )
+        stopWriter( m_parquetWriter.get() );
     for( auto &&writer:m_dictBasketWriters )
     {
-        writer -> stop();
+        stopWriter( writer.get() );
     }
     m_parquetWriter = nullptr;
     m_dictBasketWriters.clear();
+    if( firstError )
+        std::rethrow_exception( firstError );
 }
 
 DateTime ParquetOutputAdapterManager::processNextSimTimeSlice( DateTime time )
@@ -86,6 +98,10 @@ ParquetOutputAdapterManager::createDictOutputBasketWriter( const char *columnNam
 
     // Provide data sink via factory if available
     auto * writer = m_dictBasketWriters.back().get();
+    // Share the main writer's file_metadata, and claim any column_metadata the user attached to this
+    // basket's columns, so the basket files carry that metadata and the main writer does not later
+    // reject those keys as unmapped columns.
+    writer -> adoptMetadataFrom( *m_parquetWriter );
     if( m_sinkFactory )
     {
         writer -> setSink( m_sinkFactory( columnName ) );
diff --git a/cpp/csp/adapters/parquet/ParquetWriter.cpp b/cpp/csp/adapters/parquet/ParquetWriter.cpp
index 84b5b6c53..37018e3ea 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetWriter.cpp
@@ -5,6 +5,7 @@
 #include <arrow/array.h>
 #include <arrow/builder.h>
 #include <arrow/util/key_value_metadata.h>
+#include <exception>
 
 namespace csp::adapters::parquet
 {
@@ -175,15 +176,51 @@ void ParquetWriter::start()
 
 void ParquetWriter::stop()
 {
+    // Exception-safe teardown: a failure flushing the final batch must not skip onStop() (which
+    // writes the file footer/closes the stream), and a failure in onStop() must not be masked.
+    // Attempt both and rethrow the first error.
+    std::exception_ptr firstError;
     if( m_curChunkSize > 0 )
-        flushBatch();
+    {
+        try { flushBatch(); }
+        catch( ... ) { firstError = std::current_exception(); }
+    }
     if( m_sink.onStop )
-        m_sink.onStop();
+    {
+        try { m_sink.onStop(); }
+        catch( ... ) { if( !firstError ) firstError = std::current_exception(); }
+    }
     m_fileOpen = false;
+    if( firstError )
+        std::rethrow_exception( firstError );
+}
+
+void ParquetWriter::adoptMetadataFrom( ParquetWriter & source )
+{
+    // Used by dict-basket writers: take the main writer's file-level metadata, and move any column
+    // metadata the user attached to one of THIS writer's columns out of the source writer (so the
+    // basket files carry it and the main writer no longer rejects those keys as "unmapped").
+    if( !m_fileMetaData )
+        m_fileMetaData = source.m_fileMetaData;
+    for( auto && columnName : m_publishedColumnNames )
+    {
+        auto it = source.m_columnMetaData.find( columnName );
+        if( it != source.m_columnMetaData.end() )
+        {
+            m_columnMetaData[ columnName ] = it -> second;
+            source.m_columnMetaData.erase( it );
+        }
+    }
 }
 
 void ParquetWriter::onEndCycle()
 {
+    // NOTE: when no file is open -- e.g. a filename_provider ticked "" to "pause" output -- this body is
+    // skipped, so handleRowFinished() does NOT run and each scratch column's isSet bit is never cleared
+    // for that cycle. If a column ticks *while paused* and then does NOT tick on the cycle the file
+    // reopens, the stale paused-window value is emitted instead of null (the dropped tick "leaks" into
+    // the first row after resume). This matches the previous writer's behaviour. If dropping ticks while
+    // paused should instead null them out, clear each builder's scratch/isSet here in the else path.
     if( isFileOpen() ) [[likely]]
     {
         DateTime now;
@@ -236,8 +273,11 @@ std::shared_ptr<::arrow::RecordBatch> ParquetWriter::buildRecordBatch()
     {
         auto column = builder -> buildArray();
         // Every column must contain exactly the rows accumulated this chunk; a mismatch would build a
-        // corrupt RecordBatch (debug-only guard, compiled out in release).
-        CSP_ASSERT( column -> length() == static_cast<std::int64_t>( m_curChunkSize ) );
+        // corrupt RecordBatch. Enforced at runtime (not just in debug) so a future column-builder bug
+        // can never silently produce a malformed parquet/arrow file.
+        CSP_TRUE_OR_THROW_RUNTIME( column -> length() == static_cast<std::int64_t>( m_curChunkSize ),
+            "ParquetWriter column '" << builder -> getColumnName() << "' produced " << column -> length()
+            << " rows but the current chunk has " << m_curChunkSize );
         columns.push_back( std::move( column ) );
     }
     return ::arrow::RecordBatch::Make( m_schema, m_curChunkSize, std::move( columns ) );
diff --git a/cpp/csp/adapters/parquet/ParquetWriter.h b/cpp/csp/adapters/parquet/ParquetWriter.h
index 7869a5ae4..493d3006b 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.h
+++ b/cpp/csp/adapters/parquet/ParquetWriter.h
@@ -36,6 +36,12 @@ class ParquetWriter : public EndCycleListener
 
     void setSink( RecordBatchSink sink ) { m_sink = std::move( sink ); }
 
+    // Dict-basket writers adopt the main writer's file-level metadata and any column metadata that
+    // targets their own columns, so basket files carry the same metadata and the main writer does not
+    // reject those keys as "unmapped".
+    const std::shared_ptr<::arrow::KeyValueMetadata> & getFileMetaData() const { return m_fileMetaData; }
+    void adoptMetadataFrom( ParquetWriter & source );
+
     SingleColumnParquetOutputAdapter *getScalarOutputAdapter( CspTypePtr &type, const std::string &columnName );
     StructParquetOutputAdapter *getStructOutputAdapter( CspTypePtr &type, csp::DictionaryPtr fieldMap );
     ListColumnParquetOutputAdapter *getListOutputAdapter( CspTypePtr &elemType, const std::string &columnName );
diff --git a/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp b/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
index d3c934e4f..5ee0f17a2 100644
--- a/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
+++ b/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
@@ -8,7 +8,6 @@
 #include <arrow/table.h>
 #include <arrow/type.h>
 #include <arrow/util/compression.h>
-#include <arrow/util/config.h>
 #include <parquet/arrow/writer.h>
 
 #include <algorithm>
@@ -77,11 +76,10 @@ FileWriter makeParquetFileWriter( const std::string & path, const std::shared_pt
 
     ::parquet::WriterProperties::Builder props;
     props.compression( resolveCompression( compression ) );
-#if ARROW_VERSION_MAJOR >= 20
+    // csp builds against libparquet >= 16.0.0, where PARQUET_2_6 (the modern default, available since
+    // Arrow 6.0) is always present, so there is no need to gate on the Arrow version or fall back to
+    // the deprecated PARQUET_2_0.
     props.version( ::parquet::ParquetVersion::PARQUET_2_6 );
-#else
-    props.version( ::parquet::ParquetVersion::PARQUET_2_0 );
-#endif
     ::parquet::ArrowWriterProperties::Builder arrowProps;
     arrowProps.store_schema(); // preserve arrow (file/column) schema metadata in the parquet file
 
@@ -128,9 +126,13 @@ FileWriter makeIpcFileWriter( const std::string & path, const std::shared_ptr<::
     std::shared_ptr<::arrow::ipc::RecordBatchWriter> writer = result.MoveValueUnsafe();
 
     return FileWriter{
-        [writer]( const std::shared_ptr<::arrow::RecordBatch> & rb )
+        [writer, compression]( const std::shared_ptr<::arrow::RecordBatch> & rb )
         {
-            STATUS_OK_OR_THROW_RUNTIME( writer -> WriteRecordBatch( *rb ), "Failed to write arrow record batch" );
+            // Arrow validates which codecs IPC allows here and appends its own reason (e.g. "Only
+            // LZ4_FRAME and ZSTD compression allowed for IPC"); surface which codec was requested
+            // rather than a bare "failed to write". The allowed set is Arrow's to define, not ours.
+            STATUS_OK_OR_THROW_RUNTIME( writer -> WriteRecordBatch( *rb ),
+                "Failed to write Arrow IPC record batch with compression '" << compression << "'" );
         },
         [writer, stream]()
         {
@@ -154,11 +156,26 @@ FileWriter makeSplitWriter( const std::string & dir, const std::shared_ptr<::arr
     subWriters -> reserve( schema -> num_fields() );
     colSchemas -> reserve( schema -> num_fields() );
 
-    for( int i = 0; i < schema -> num_fields(); ++i )
+    try
+    {
+        for( int i = 0; i < schema -> num_fields(); ++i )
+        {
+            auto colSchema = ::arrow::schema( { schema -> field( i ) }, schema -> metadata() );
+            colSchemas -> push_back( colSchema );
+            subWriters -> push_back( perColumn( dir + "/" + schema -> field( i ) -> name() + extension, colSchema ) );
+        }
+    }
+    catch( ... )
     {
-        auto colSchema = ::arrow::schema( { schema -> field( i ) }, schema -> metadata() );
-        colSchemas -> push_back( colSchema );
-        subWriters -> push_back( perColumn( dir + "/" + schema -> field( i ) -> name() + extension, colSchema ) );
+        // A later column failed to open (e.g. allow_overwrite=false and that file already exists).
+        // Close the sub-writers already opened so we don't leak file descriptors or leave half-open
+        // files behind, then rethrow the original error.
+        for( auto & w : *subWriters )
+        {
+            try { w.close(); }
+            catch( ... ) {}
+        }
+        throw;
     }
 
     return FileWriter{
@@ -230,8 +247,12 @@ RecordBatchSink makeFileSink( bool writeArrowBinary, bool splitColumns,
     sink.onStart      = [schemaHolder]( const std::shared_ptr<::arrow::Schema> & schema ) { *schemaHolder = schema; };
     sink.onBatch      = [current]( const std::shared_ptr<::arrow::RecordBatch> & rb )
     {
-        if( current -> has_value() )
-            ( *current ) -> writeBatch( rb );
+        // A batch must only ever arrive while a file is open (the writer guards this via isFileOpen()
+        // before flushing). If the writer and sink ever desync, fail loudly instead of silently
+        // dropping the batch's rows.
+        CSP_TRUE_OR_THROW_RUNTIME( current -> has_value(),
+            "RecordBatchFileSink received a record batch with no open output file" );
+        ( *current ) -> writeBatch( rb );
     };
     sink.onFileChange = [factory, schemaHolder, current, currentPath, closeCurrent]( const std::string & path )
     {
diff --git a/csp/adapters/output_adapters/parquet.py b/csp/adapters/output_adapters/parquet.py
index e40d6401a..775304bdd 100644
--- a/csp/adapters/output_adapters/parquet.py
+++ b/csp/adapters/output_adapters/parquet.py
@@ -77,6 +77,8 @@ def __init__(
         and at shutdown). It runs synchronously on the engine thread with the GIL held, so a slow visitor (e.g.
         uploading to remote storage) blocks the engine for its duration -- offload heavy work to a background
         queue/thread if that matters. An exception raised by the visitor propagates out of csp.run.
+        Note: file_visitor is NOT invoked for dict-basket files (the value/symbol/index files written by
+        publish_dict_basket); it only covers the main writer's files (and any regular published columns).
         """
         super().__init__()
         config = ParquetOutputConfig() if config is None else config.copy()
diff --git a/csp/tests/adapters/test_parquet_output.py b/csp/tests/adapters/test_parquet_output.py
index 082e9439b..e5ebc983a 100644
--- a/csp/tests/adapters/test_parquet_output.py
+++ b/csp/tests/adapters/test_parquet_output.py
@@ -406,6 +406,64 @@ def g():
             df = pandas.read_parquet(os.path.join(d, "output.parquet"))
             self.assertEqual(df["value"].tolist(), [1, 2])
 
+    def test_file_visitor_split_mode_visits_directory(self):
+        """In split-column mode the file_visitor is invoked once, with the output *directory* path
+        (not each per-column file). Pins this contract so a future change is visible.
+        """
+        visited = []
+        with tempfile.TemporaryDirectory() as d:
+            outdir = os.path.join(d, "sp")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(
+                    outdir,
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                    file_visitor=lambda f: visited.append(f),
+                )
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 1)]))
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=3))
+            self.assertEqual(len(visited), 1)
+            self.assertEqual(os.path.normpath(visited[0]), os.path.normpath(outdir))
+
+    def test_rotate_to_empty_then_resume(self):
+        """A filename_provider may tick "" to pause output (close the current file, open none) and later
+        resume with a new path. A value that ticks while paused is retained and emitted on the first row
+        after the file reopens.
+
+        Scenario: A ticks 100 (file f0 open), then 999 while paused; B ticks 5 when f1 reopens; A does
+        not tick at reopen, so its first f1 row carries the paused-window value 999.
+        """
+        with tempfile.TemporaryDirectory() as d:
+
+            @csp.graph
+            def g():
+                a = csp.curve(int, [(timedelta(seconds=1), 100), (timedelta(seconds=3), 999)])
+                b = csp.curve(int, [(timedelta(seconds=4), 5)])
+                rotate = csp.curve(
+                    str,
+                    [(timedelta(seconds=2), ""), (timedelta(seconds=4), os.path.join(d, "f1.parquet"))],
+                )
+                writer = ParquetWriter(
+                    os.path.join(d, "f0.parquet"),
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    filename_provider=rotate,
+                )
+                writer.publish("A", a)
+                writer.publish("B", b)
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=6))
+
+            df0 = pandas.read_parquet(os.path.join(d, "f0.parquet"))
+            df1 = pandas.read_parquet(os.path.join(d, "f1.parquet"))
+            self.assertEqual(df0["A"].tolist(), [100])
+            self.assertEqual(df1["A"].tolist(), [999])
+            self.assertEqual(df1["B"].tolist(), [5])
+
 
 class TestOutputArrowIPC(unittest.TestCase):
     """Test Arrow IPC (binary arrow) output format."""
@@ -660,6 +718,164 @@ def g_read():
             for i, sym in enumerate(symbols):
                 self.assertEqual([v[1] for v in res[sym]], [float(i)])
 
+    def test_dict_basket_with_rotation(self):
+        """A dict basket whose filename_provider rotates mid-run lands data in the correct directory
+        and keeps the value/symbol/index files aligned across the rotation.
+        """
+        with tempfile.TemporaryDirectory() as d:
+
+            @csp.graph
+            def g():
+                rotate = csp.curve(str, [(timedelta(seconds=3), os.path.join(d, "b1"))])
+                writer = ParquetWriter(
+                    os.path.join(d, "b0"),
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                    filename_provider=rotate,
+                )
+                basket = {
+                    "AAPL": csp.curve(
+                        float,
+                        [(timedelta(seconds=1), 1.0), (timedelta(seconds=2), 2.0), (timedelta(seconds=4), 4.0)],
+                    )
+                }
+                writer.publish_dict_basket("price", basket, str, float)
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=6))
+
+            # Pre-rotation values in b0, post-rotation value in b1 -- both directories complete.
+            for sub in ("b0", "b1"):
+                for fname in ("price.parquet", "price__csp_symbol.parquet", "price__csp_value_count.parquet"):
+                    self.assertTrue(os.path.isfile(os.path.join(d, sub, fname)), f"{sub}/{fname} missing")
+            self.assertEqual(pandas.read_parquet(os.path.join(d, "b0", "price.parquet"))["price"].tolist(), [1.0, 2.0])
+            self.assertEqual(pandas.read_parquet(os.path.join(d, "b1", "price.parquet"))["price"].tolist(), [4.0])
+            # Symbol column stays aligned with the value column in each directory.
+            self.assertEqual(
+                pandas.read_parquet(os.path.join(d, "b0", "price__csp_symbol.parquet"))["price__csp_symbol"].tolist(),
+                ["AAPL", "AAPL"],
+            )
+
+    def test_dict_basket_with_rotation_round_trips(self):
+        """The rotated basket files read back correctly via ParquetReader for each segment."""
+        with tempfile.TemporaryDirectory() as d:
+
+            @csp.graph
+            def g_write():
+                rotate = csp.curve(str, [(timedelta(seconds=3), os.path.join(d, "b1"))])
+                writer = ParquetWriter(
+                    os.path.join(d, "b0"),
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                    filename_provider=rotate,
+                )
+                basket = {
+                    "AAPL": csp.curve(float, [(timedelta(seconds=1), 1.0), (timedelta(seconds=4), 4.0)]),
+                }
+                writer.publish_dict_basket("price", basket, str, float)
+
+            def read_segment(sub):
+                @csp.graph
+                def g_read():
+                    reader = ParquetReader(
+                        os.path.join(d, sub), time_column="csp_timestamp", split_columns_to_files=True
+                    )
+                    csp.add_graph_output("AAPL", reader.subscribe_dict_basket(float, "price", ["AAPL"])["AAPL"])
+
+                return csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=6))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=6))
+            self.assertEqual([v[1] for v in read_segment("b0")["AAPL"]], [1.0])
+            self.assertEqual([v[1] for v in read_segment("b1")["AAPL"]], [4.0])
+
+    def test_dict_basket_file_metadata(self):
+        """file_metadata is written to every dict-basket file (value, symbol, index) as well as the
+        main timestamp file, matching single-vs-split metadata behavior for regular columns.
+        """
+        with tempfile.TemporaryDirectory() as d:
+            bdir = os.path.join(d, "basket")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(
+                    bdir,
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                    file_metadata={"author": "meta_test"},
+                )
+                basket = {"AAPL": csp.curve(float, [(timedelta(seconds=1), 150.0)])}
+                writer.publish_dict_basket("price", basket, str, float)
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=3))
+            for fname in (
+                "price.parquet",
+                "price__csp_symbol.parquet",
+                "price__csp_value_count.parquet",
+                "csp_timestamp.parquet",
+            ):
+                meta = pyarrow.parquet.read_schema(os.path.join(bdir, fname)).metadata
+                self.assertIsNotNone(meta, f"{fname} is missing file_metadata")
+                self.assertEqual(meta[b"author"], b"meta_test", f"{fname} has wrong file_metadata")
+
+    def test_dict_basket_column_metadata(self):
+        """column_metadata keyed to a dict-basket column attaches to that basket's value file field."""
+        with tempfile.TemporaryDirectory() as d:
+            bdir = os.path.join(d, "basket")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(
+                    bdir,
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                    column_metadata={"price": {"units": "usd"}},
+                )
+                basket = {"AAPL": csp.curve(float, [(timedelta(seconds=1), 150.0)])}
+                writer.publish_dict_basket("price", basket, str, float)
+
+            csp.run(g, starttime=START, endtime=timedelta(seconds=3))
+            field = pyarrow.parquet.read_schema(os.path.join(bdir, "price.parquet")).field("price")
+            self.assertIsNotNone(field.metadata)
+            self.assertEqual(field.metadata[b"units"], b"usd")
+
+    def test_all_writers_flushed_when_file_visitor_raises(self):
+        """A throwing file_visitor must not prevent any sibling writer from being flushed and closed.
+        With a regular column and a dict basket, after the visitor error surfaces, every file -- the
+        regular column and the basket value/symbol/index files -- is still present and complete.
+        """
+        with tempfile.TemporaryDirectory() as d:
+            outdir = os.path.join(d, "out")
+
+            def boom(_f):
+                raise ValueError("visitor boom")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(
+                    outdir,
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                    file_visitor=boom,
+                )
+                writer.publish("reg", csp.curve(int, [(timedelta(seconds=1), 7), (timedelta(seconds=2), 8)]))
+                basket = {"AAPL": csp.curve(float, [(timedelta(seconds=1), 1.5), (timedelta(seconds=2), 2.5)])}
+                writer.publish_dict_basket("price", basket, str, float)
+
+            with self.assertRaisesRegex(Exception, "visitor boom"):
+                csp.run(g, starttime=START, endtime=timedelta(seconds=4))
+
+            self.assertEqual(pyarrow.parquet.read_table(os.path.join(outdir, "reg.parquet")).num_rows, 2)
+            self.assertEqual(
+                pyarrow.parquet.read_table(os.path.join(outdir, "price.parquet")).column("price").to_pylist(),
+                [1.5, 2.5],
+            )
+            for fname in ("price__csp_symbol.parquet", "price__csp_value_count.parquet"):
+                self.assertEqual(pyarrow.parquet.read_table(os.path.join(outdir, fname)).num_rows, 2)
+
 
 class TestOutputNumpyArrays(unittest.TestCase):
     """Test numpy array column output."""
@@ -832,6 +1048,25 @@ def g():
             with self.assertRaisesRegex(Exception, "compression"):
                 csp.run(g, starttime=START, endtime=timedelta(seconds=3))
 
+    def test_invalid_ipc_compression_raises_clear_error(self):
+        """A codec Arrow IPC does not support (e.g. snappy) fails with a message naming the codec.
+
+        Arrow only allows uncompressed/lz4/zstd for Arrow IPC streams and supplies its own
+        "Only LZ4_FRAME and ZSTD ..." reason; the writer surfaces which codec was requested. This
+        restriction is specific to write_arrow_binary=True -- parquet output accepts snappy.
+        """
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.arrow")
+
+            @csp.graph
+            def g():
+                config = ParquetOutputConfig(allow_overwrite=True, write_arrow_binary=True, compression="snappy")
+                writer = ParquetWriter(fname, "csp_timestamp", config=config)
+                writer.publish("x", csp.curve(int, [(timedelta(seconds=1), 1)]))
+
+            with self.assertRaisesRegex(Exception, "snappy"):
+                csp.run(g, starttime=START, endtime=timedelta(seconds=3))
+
 
 class TestOutputBatchSize(unittest.TestCase):
     """Test batch_size controls row group flushing."""
@@ -976,6 +1211,58 @@ def g():
             reader = _read_ipc(val_file)
             self.assertEqual(reader.schema.metadata[b"source"], b"ipc_test")
 
+    def test_split_and_single_metadata_equivalent(self):
+        """The PR's headline claim: file- and column-level metadata (and field types) are identical
+        between single-file and split-column modes. Writes the same data both ways and compares the
+        per-column schema field plus file metadata directly.
+        """
+        file_md = {"author": "me", "version": "2.0"}
+        col_md = {"x": {"units": "kg", "source": "sensor"}}
+        curve = [(timedelta(seconds=1), 1), (timedelta(seconds=2), 2)]
+
+        with tempfile.TemporaryDirectory() as d:
+            single = os.path.join(d, "single.parquet")
+            outdir = os.path.join(d, "split")
+
+            @csp.graph
+            def g_single():
+                w = ParquetWriter(
+                    single,
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    file_metadata=file_md,
+                    column_metadata=col_md,
+                )
+                w.publish("x", csp.curve(int, curve))
+
+            @csp.graph
+            def g_split():
+                w = ParquetWriter(
+                    outdir,
+                    "csp_timestamp",
+                    config=ParquetOutputConfig(allow_overwrite=True),
+                    split_columns_to_files=True,
+                    file_metadata=file_md,
+                    column_metadata=col_md,
+                )
+                w.publish("x", csp.curve(int, curve))
+
+            csp.run(g_single, starttime=START, endtime=timedelta(seconds=4))
+            csp.run(g_split, starttime=START, endtime=timedelta(seconds=4))
+
+            single_schema = pyarrow.parquet.read_schema(single)
+            split_x_schema = pyarrow.parquet.read_schema(os.path.join(outdir, "x.parquet"))
+
+            # File-level metadata identical.
+            self.assertEqual(single_schema.metadata, split_x_schema.metadata)
+            # Column 'x' field -- name, type, and column-level metadata -- identical.
+            self.assertEqual(single_schema.field("x"), split_x_schema.field("x"))
+            # Every per-column file in split mode carries the same file-level metadata.
+            for fname in ("x.parquet", "csp_timestamp.parquet"):
+                self.assertEqual(
+                    pyarrow.parquet.read_schema(os.path.join(outdir, fname)).metadata, single_schema.metadata
+                )
+
 
 class TestOutputAllowOverwrite(unittest.TestCase):
     """Test allow_overwrite behavior."""

From 9bf26033752ad34ed7c3e926544cd031469808e7 Mon Sep 17 00:00:00 2001
From: Arham Chopra <arham.chopra@cubistsystematic.com>
Date: Wed, 17 Jun 2026 10:48:50 -0400
Subject: [PATCH 10/11] Review fixes, and cleanup

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>
---
 cpp/csp/adapters/arrow/ArrowFieldWriter.cpp   |   6 +-
 .../parquet/ArrowBackedArrayBuilder.cpp       |  10 ++
 .../parquet/ArrowBackedArrayBuilder.h         |   1 +
 .../parquet/ArrowSingleColumnArrayBuilder.h   |   4 +
 .../parquet/ParquetDictBasketOutputWriter.cpp |   9 +-
 .../adapters/parquet/ParquetOutputAdapter.cpp |   3 +-
 cpp/csp/adapters/parquet/ParquetWriter.cpp    |   9 +-
 .../adapters/parquet/RecordBatchFileSink.cpp  |  15 ++-
 .../adapters/parquet/RecordBatchFileSink.h    |   3 +-
 .../python/adapters/parquetadapterimpl.cpp    |   7 +-
 csp/adapters/output_adapters/parquet.py       |   3 +
 csp/tests/adapters/test_parquet_output.py     | 117 ++++++++++++++++--
 12 files changed, 159 insertions(+), 28 deletions(-)

diff --git a/cpp/csp/adapters/arrow/ArrowFieldWriter.cpp b/cpp/csp/adapters/arrow/ArrowFieldWriter.cpp
index 53f9979d2..90e040d4b 100644
--- a/cpp/csp/adapters/arrow/ArrowFieldWriter.cpp
+++ b/cpp/csp/adapters/arrow/ArrowFieldWriter.cpp
@@ -345,7 +345,11 @@ CreatedFieldWriter createFieldWriter(
             std::vector<std::shared_ptr<::arrow::ArrayBuilder>> childBuilders;
             std::vector<std::unique_ptr<FieldWriter>> childWriters;
 
-            // Use fieldNames() for stable insertion order (fields() is sorted for memory layout)
+            // Ordering convention: csp writes struct child columns in declaration order, using
+            // fieldNames() rather than fields() (which is sorted by struct memory layout). This keeps
+            // nested-struct child order consistent with top-level column order and stable against
+            // internal field packing. The reader matches children by name, so either order round-trips,
+            // but declaration order is the on-disk contract.
             for( auto & subFieldName : nestedMeta -> fieldNames() )
             {
                 auto subField = nestedMeta -> field( subFieldName );
diff --git a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
index 377b3b235..6ab566f6f 100644
--- a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
+++ b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.cpp
@@ -88,6 +88,13 @@ void ArrowBackedArrayBuilder::init( const std::string & columnName, const Struct
 {
     auto created = csp::adapters::arrow::createFieldWriter( columnName, field );
     m_writer = std::move( created.writer );
+    // Reserve the first chunk up front so the per-row append path doesn't reallocate as rows accumulate.
+    m_writer -> reserve( getChunkSize() );
+}
+
+void ArrowBackedArrayBuilder::reserve( int64_t numRows )
+{
+    m_writer -> reserve( numRows );
 }
 
 std::shared_ptr<::arrow::DataType> ArrowBackedArrayBuilder::getDataType()
@@ -132,6 +139,9 @@ std::shared_ptr<::arrow::Array> ArrowBackedArrayBuilder::buildArray()
     auto arrays = m_writer -> finish();
     CSP_TRUE_OR_THROW_RUNTIME( arrays.size() == 1,
         "ArrowBackedArrayBuilder expected 1 array from FieldWriter, got " << arrays.size() );
+    // finish() resets the underlying builder's capacity; re-reserve for the next chunk so the per-row
+    // append path stays allocation-free (and keeps UnsafeAppend within reserved capacity).
+    m_writer -> reserve( getChunkSize() );
     return arrays[0];
 }
 
diff --git a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
index 92c56651e..359c09d03 100644
--- a/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
+++ b/cpp/csp/adapters/parquet/ArrowBackedArrayBuilder.h
@@ -42,6 +42,7 @@ class ArrowBackedArrayBuilder : public ArrowSingleColumnArrayBuilder
     int64_t length() const override;
     void handleRowFinished() override;
     std::shared_ptr<::arrow::Array> buildArray() override;
+    void reserve( int64_t numRows ) override;
 
     // --- Scratch mode API ---
     // Get the scratch struct for value setting
diff --git a/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h b/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
index eb9f61cac..efc98a783 100644
--- a/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
+++ b/cpp/csp/adapters/parquet/ArrowSingleColumnArrayBuilder.h
@@ -37,6 +37,10 @@ class ArrowSingleColumnArrayBuilder
     // Release the array from the currently built values
     virtual std::shared_ptr<::arrow::Array> buildArray() = 0;
 
+    // Pre-reserve builder capacity for numRows rows. Default no-op; fixed-width builders override to
+    // reserve their Arrow builder so the per-row append path avoids repeated reallocation.
+    virtual void reserve( int64_t /*numRows*/ ) {}
+
 private:
     const std::string   m_columnName;
     const std::uint32_t m_chunkSize;
diff --git a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
index abb024032..39d3fbf55 100644
--- a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
@@ -151,12 +151,15 @@ ParquetStructDictBasketOutputWriter::ParquetStructDictBasketOutputWriter( Parque
         : ParquetDictBasketOutputWriter( outputAdapterManager, columnName )
 {
 
-    // We don't support fieldMap for now, only default field map
+    // We don't support fieldMap for now, only default field map.
+    // Iterate fieldNames() (declaration order), NOT fields() (sorted by memory layout): csp writes
+    // struct columns -- both top-level and nested -- in declaration order, so the on-disk column order
+    // matches the struct definition and stays stable regardless of internal field packing.
     auto structMetaPtr = std::static_pointer_cast<const CspStructType>( cspTypePtr ) -> meta().get();
     DictionaryPtr dict = std::make_shared<Dictionary>();
-    for(auto&& field:structMetaPtr->fields())
+    for( auto && fieldName : structMetaPtr -> fieldNames() )
     {
-        dict->insert(field->fieldname(), columnName + "." + field->fieldname());
+        dict->insert( fieldName, columnName + "." + fieldName );
     }
     m_valueOutputAdapter = getStructOutputHandler( cspTypePtr, dict );
 }
diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp
index 0df2efadc..889cb27ac 100644
--- a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp
@@ -100,7 +100,8 @@ StructParquetOutputHandler::StructParquetOutputHandler( Engine *engine, ParquetW
             CSP_THROW( ValueError, "Struct field '" << structFieldName << "' not found" );
 
         if( structField -> type() -> type() == CspType::Type::DIALECT_GENERIC )
-            continue;
+            CSP_THROW( TypeError, "Writing of column " << structField -> fieldname()
+                << " of type DIALECT_GENERIC is not supported" );
 
         auto builder = createArrowBackedArrayBuilderForField( columnName, getChunkSize(), structField );
         m_arrowBuilders.push_back( builder.get() );
diff --git a/cpp/csp/adapters/parquet/ParquetWriter.cpp b/cpp/csp/adapters/parquet/ParquetWriter.cpp
index 37018e3ea..a39f4508c 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetWriter.cpp
@@ -215,12 +215,9 @@ void ParquetWriter::adoptMetadataFrom( ParquetWriter & source )
 
 void ParquetWriter::onEndCycle()
 {
-    // NOTE: when no file is open -- e.g. a filename_provider ticked "" to "pause" output -- this body is
-    // skipped, so handleRowFinished() does NOT run and each scratch column's isSet bit is never cleared
-    // for that cycle. If a column ticks *while paused* and then does NOT tick on the cycle the file
-    // reopens, the stale paused-window value is emitted instead of null (the dropped tick "leaks" into
-    // the first row after resume). This matches the previous writer's behaviour. If dropping ticks while
-    // paused should instead null them out, clear each builder's scratch/isSet here in the else path.
+    // When no file is open -- e.g. a filename_provider ticked "" to "pause" output -- this body is
+    // skipped, so handleRowFinished() does not run for that cycle. As a result, a value that ticked
+    // while paused is emitted on the first row after the file reopens, rather than being nulled.
     if( isFileOpen() ) [[likely]]
     {
         DateTime now;
diff --git a/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp b/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
index 5ee0f17a2..0c8d1e16a 100644
--- a/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
+++ b/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
@@ -20,6 +20,7 @@
 
 namespace csp::adapters::parquet
 {
+
 namespace
 {
 
@@ -70,7 +71,7 @@ std::shared_ptr<::arrow::io::OutputStream> openOutputStream( const std::string &
 }
 
 FileWriter makeParquetFileWriter( const std::string & path, const std::shared_ptr<::arrow::Schema> & schema,
-                                  const std::string & compression, bool allowOverwrite )
+                                  const std::string & compression, bool allowOverwrite, bool useDictionary )
 {
     auto stream = openOutputStream( path, allowOverwrite );
 
@@ -80,6 +81,11 @@ FileWriter makeParquetFileWriter( const std::string & path, const std::shared_pt
     // Arrow 6.0) is always present, so there is no need to gate on the Arrow version or fall back to
     // the deprecated PARQUET_2_0.
     props.version( ::parquet::ParquetVersion::PARQUET_2_6 );
+    // Dictionary encoding is on by default. For high-cardinality numeric columns it is a large write-time
+    // cost (hash table build) and can even grow files; allow it to be turned off (trading file size for
+    // write throughput). Low-cardinality columns still benefit from leaving it on (the default).
+    if( !useDictionary )
+        props.disable_dictionary();
     ::parquet::ArrowWriterProperties::Builder arrowProps;
     arrowProps.store_schema(); // preserve arrow (file/column) schema metadata in the parquet file
 
@@ -204,16 +210,17 @@ FileWriter makeSplitWriter( const std::string & dir, const std::shared_ptr<::arr
 
 RecordBatchSink makeFileSink( bool writeArrowBinary, bool splitColumns,
                               const std::string & compression, bool allowOverwrite,
-                              std::function<void( const std::string & )> fileVisitor )
+                              std::function<void( const std::string & )> fileVisitor,
+                              bool useDictionary )
 {
     const std::string extension = writeArrowBinary ? ".arrow" : ".parquet";
 
     // Per-file factory: format choice is the only difference.
     FileWriterFactory perFile =
-        [writeArrowBinary, compression, allowOverwrite]( const std::string & path, const std::shared_ptr<::arrow::Schema> & schema )
+        [writeArrowBinary, compression, allowOverwrite, useDictionary]( const std::string & path, const std::shared_ptr<::arrow::Schema> & schema )
         {
             return writeArrowBinary ? makeIpcFileWriter( path, schema, compression, allowOverwrite )
-                                    : makeParquetFileWriter( path, schema, compression, allowOverwrite );
+                                    : makeParquetFileWriter( path, schema, compression, allowOverwrite, useDictionary );
         };
 
     // Layout choice: single file vs. one file per column.
diff --git a/cpp/csp/adapters/parquet/RecordBatchFileSink.h b/cpp/csp/adapters/parquet/RecordBatchFileSink.h
index 9a6ff753f..9e61823c4 100644
--- a/cpp/csp/adapters/parquet/RecordBatchFileSink.h
+++ b/cpp/csp/adapters/parquet/RecordBatchFileSink.h
@@ -22,7 +22,8 @@ RecordBatchSink makeFileSink( bool writeArrowBinary,
                               bool splitColumns,
                               const std::string & compression,
                               bool allowOverwrite,
-                              std::function<void( const std::string & )> fileVisitor = {} );
+                              std::function<void( const std::string & )> fileVisitor = {},
+                              bool useDictionary = true );
 
 } // namespace csp::adapters::parquet
 
diff --git a/cpp/csp/python/adapters/parquetadapterimpl.cpp b/cpp/csp/python/adapters/parquetadapterimpl.cpp
index acd774a71..c5239b04e 100644
--- a/cpp/csp/python/adapters/parquetadapterimpl.cpp
+++ b/cpp/csp/python/adapters/parquetadapterimpl.cpp
@@ -685,6 +685,7 @@ csp::AdapterManager *create_parquet_output_adapter_manager( PyEngine *engine, co
     bool        splitColumns     = properties.get<bool>( "split_columns_to_files" );
     bool        allowOverwrite   = properties.get<bool>( "allow_overwrite" );
     std::string compression      = properties.get<std::string>( "compression" );
+    bool        useDictionary    = properties.get( "write_dictionary", true );
 
     // Optional Python file_visitor: wrapped as a plain C++ callback, invoked synchronously (with the
     // GIL held during the run) after each file is closed, so a slow visitor blocks the engine.  Only
@@ -707,13 +708,13 @@ csp::AdapterManager *create_parquet_output_adapter_manager( PyEngine *engine, co
     }
 
     auto * mgr = engine -> engine() -> createOwnedObject<ParquetOutputAdapterManager>( properties );
-    mgr -> setSink( makeFileSink( writeArrowBinary, splitColumns, compression, allowOverwrite, fileVisitor ) );
+    mgr -> setSink( makeFileSink( writeArrowBinary, splitColumns, compression, allowOverwrite, fileVisitor, useDictionary ) );
 
     // Dict-basket writers always write split-column files (one per value/symbol/index column),
     // without a file visitor.
-    mgr -> setSinkFactory( [writeArrowBinary, compression, allowOverwrite]( const std::string & ) -> RecordBatchSink
+    mgr -> setSinkFactory( [writeArrowBinary, compression, allowOverwrite, useDictionary]( const std::string & ) -> RecordBatchSink
     {
-        return makeFileSink( writeArrowBinary, /*splitColumns=*/true, compression, allowOverwrite, {} );
+        return makeFileSink( writeArrowBinary, /*splitColumns=*/true, compression, allowOverwrite, {}, useDictionary );
     } );
 
     return mgr;
diff --git a/csp/adapters/output_adapters/parquet.py b/csp/adapters/output_adapters/parquet.py
index 775304bdd..5c14f5de7 100644
--- a/csp/adapters/output_adapters/parquet.py
+++ b/csp/adapters/output_adapters/parquet.py
@@ -30,6 +30,8 @@ class ParquetOutputConfig(Struct):
     batch_size: int = 2**15
     compression: str
     write_arrow_binary: bool = False  # If true will write output as binary arrow data rather than parquet
+    write_dictionary: bool = True  # If false, disables parquet dictionary encoding (faster writes for
+    # high-cardinality numeric columns, at the cost of larger files; low-cardinality data benefits from True)
 
     def resolve_compression(self):
         if not hasattr(self, "compression"):
@@ -107,6 +109,7 @@ def __init__(
             "batch_size": config.batch_size,
             "compression": config.compression,
             "write_arrow_binary": config.write_arrow_binary,
+            "write_dictionary": config.write_dictionary,
             "split_columns_to_files": split_columns_to_files,
             "file_metadata": file_metadata,
             "column_metadata": column_metadata,
diff --git a/csp/tests/adapters/test_parquet_output.py b/csp/tests/adapters/test_parquet_output.py
index e5ebc983a..0382f4659 100644
--- a/csp/tests/adapters/test_parquet_output.py
+++ b/csp/tests/adapters/test_parquet_output.py
@@ -281,6 +281,73 @@ def g():
             self.assertEqual(df["f7"].iloc[0], "eight")
             self.assertEqual(df["f8"].iloc[0], True)
 
+    def test_struct_with_generic_field_raises(self):
+        """Publishing a struct with a DIALECT_GENERIC (object) field raises rather than silently
+        dropping the column.
+        """
+
+        class WithObj(csp.Struct):
+            x: int
+            obj: object
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+
+            @csp.graph
+            def g():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish_struct(csp.const(WithObj(x=1, obj={"a": 1})))
+
+            with self.assertRaisesRegex(Exception, "DIALECT_GENERIC"):
+                csp.run(g, starttime=START, endtime=timedelta(seconds=2))
+
+    def test_nested_struct_child_field_order_is_declaration_order(self):
+        """Nested-struct child columns are written in declaration order (not struct memory-layout
+        order), consistent with top-level column order. Pins the on-disk schema shape: a_flag (a 1-byte
+        bool declared first) must come first, which memory-layout packing -- which front-loads the
+        8-byte fields -- would never produce.
+        """
+
+        class Inner(csp.Struct):
+            a_flag: bool
+            z_price: float
+            m_count: int
+
+        class Outer(csp.Struct):
+            o_flag: bool
+            o_price: float
+            inner: Inner
+
+        with tempfile.TemporaryDirectory() as d:
+            fname = os.path.join(d, "out.parquet")
+            v = Outer(o_flag=True, o_price=1.5, inner=Inner(a_flag=False, z_price=2.5, m_count=3))
+
+            @csp.graph
+            def g_write():
+                writer = ParquetWriter(fname, "csp_timestamp", config=ParquetOutputConfig(allow_overwrite=True))
+                writer.publish_struct(csp.const(v))
+
+            csp.run(g_write, starttime=START, endtime=timedelta(seconds=2))
+
+            schema = pyarrow.parquet.read_schema(fname)
+            # Top-level columns in declaration order (after the timestamp column).
+            self.assertEqual(schema.names, ["csp_timestamp", "o_flag", "o_price", "inner"])
+            # Nested struct child fields in declaration order.
+            inner_type = schema.field("inner").type
+            self.assertEqual(
+                [inner_type.field(i).name for i in range(inner_type.num_fields)],
+                ["a_flag", "z_price", "m_count"],
+            )
+
+            # And it round-trips back to the original struct.
+            @csp.graph
+            def g_read():
+                reader = ParquetReader(fname, time_column="csp_timestamp")
+                csp.add_graph_output("data", reader.subscribe_all(Outer))
+
+            res = csp.run(g_read, starttime=START, endtime=START + timedelta(seconds=2))
+            self.assertEqual(res["data"][0][1], v)
+
 
 class TestOutputFileRotation(unittest.TestCase):
     """Test file rotation via filename_provider."""
@@ -1067,6 +1134,45 @@ def g():
             with self.assertRaisesRegex(Exception, "snappy"):
                 csp.run(g, starttime=START, endtime=timedelta(seconds=3))
 
+    def test_write_dictionary_disabled(self):
+        """write_dictionary=False disables parquet dictionary encoding; data still round-trips and the
+        column uses a non-dictionary encoding (default write_dictionary=True keeps dictionary on).
+        """
+        values = [float(i) for i in range(2000)]
+        with tempfile.TemporaryDirectory() as d:
+            on = os.path.join(d, "dict_on.parquet")
+            off = os.path.join(d, "dict_off.parquet")
+
+            def write(fname, use_dict):
+                @csp.graph
+                def g():
+                    config = ParquetOutputConfig(allow_overwrite=True, write_dictionary=use_dict)
+                    writer = ParquetWriter(fname, "csp_timestamp", config=config)
+                    writer.publish("x", csp.curve(float, [(timedelta(seconds=i + 1), v) for i, v in enumerate(values)]))
+
+                csp.run(g, starttime=START, endtime=timedelta(seconds=len(values) + 2))
+
+            write(on, True)
+            write(off, False)
+
+            # Both round-trip identically.
+            self.assertEqual(pandas.read_parquet(off)["x"].tolist(), values)
+            self.assertEqual(pandas.read_parquet(on)["x"].tolist(), values)
+
+            # The 'x' column encodings differ: dictionary on (default) uses PLAIN_DICTIONARY/RLE_DICTIONARY;
+            # with write_dictionary=False it must not.
+            def x_encodings(fname):
+                md = pyarrow.parquet.ParquetFile(fname).metadata
+                return set(md.row_group(0).column(1).encodings)
+
+            off_enc = x_encodings(off)
+            self.assertFalse(
+                any("DICTIONARY" in e for e in off_enc), f"dict-off file unexpectedly dictionary-encoded: {off_enc}"
+            )
+            self.assertTrue(
+                any("DICTIONARY" in e for e in x_encodings(on)), "dict-on (default) should use dictionary encoding"
+            )
+
 
 class TestOutputBatchSize(unittest.TestCase):
     """Test batch_size controls row group flushing."""
@@ -1212,10 +1318,7 @@ def g():
             self.assertEqual(reader.schema.metadata[b"source"], b"ipc_test")
 
     def test_split_and_single_metadata_equivalent(self):
-        """The PR's headline claim: file- and column-level metadata (and field types) are identical
-        between single-file and split-column modes. Writes the same data both ways and compares the
-        per-column schema field plus file metadata directly.
-        """
+        """Metadata and field types are identical between single-file and split-column modes."""
         file_md = {"author": "me", "version": "2.0"}
         col_md = {"x": {"units": "kg", "source": "sensor"}}
         curve = [(timedelta(seconds=1), 1), (timedelta(seconds=2), 2)]
@@ -1304,11 +1407,7 @@ class TestOutputEdgeCases(unittest.TestCase):
     """Edge cases and special scenarios."""
 
     def test_bare_relative_filename_no_directory(self):
-        """Writing to a bare relative filename (no directory component) must work.
-
-        Regression: the C++ sink called mkdir(dirname(path)); dirname("out.parquet")
-        is "" and mkdir("") fails with 'Invalid argument'.
-        """
+        """Writing to a bare relative filename with no directory component succeeds."""
         with tempfile.TemporaryDirectory() as d:
             cwd = os.getcwd()
             os.chdir(d)

From 04445fc16f85b9600e95243c658a788191964b5f Mon Sep 17 00:00:00 2001
From: Arham Chopra <arham.chopra@cubistsystematic.com>
Date: Tue, 23 Jun 2026 13:47:55 -0400
Subject: [PATCH 11/11] Address comments

Signed-off-by: Arham Chopra <arham.chopra@cubistsystematic.com>
---
 cpp/csp/adapters/parquet/CMakeLists.txt       |   3 +-
 .../parquet/ParquetDictBasketOutputWriter.cpp |  55 +++---
 .../parquet/ParquetDictBasketOutputWriter.h   |   8 +-
 .../adapters/parquet/ParquetOutputAdapter.cpp |  12 +-
 .../adapters/parquet/ParquetOutputAdapter.h   |   5 +-
 .../parquet/ParquetOutputAdapterManager.h     |   2 +
 cpp/csp/adapters/parquet/ParquetWriter.cpp    |  33 ++--
 cpp/csp/adapters/parquet/ParquetWriter.h      |   5 +-
 .../adapters/parquet/RecordBatchFileSink.h    |  30 ----
 ...dBatchFileSink.cpp => RecordBatchSink.cpp} | 162 +++++++++---------
 cpp/csp/adapters/parquet/RecordBatchSink.h    |  80 +++++++--
 .../python/adapters/parquetadapterimpl.cpp    |   8 +-
 csp/adapters/output_adapters/parquet.py       |  14 --
 13 files changed, 213 insertions(+), 204 deletions(-)
 delete mode 100644 cpp/csp/adapters/parquet/RecordBatchFileSink.h
 rename cpp/csp/adapters/parquet/{RecordBatchFileSink.cpp => RecordBatchSink.cpp} (65%)

diff --git a/cpp/csp/adapters/parquet/CMakeLists.txt b/cpp/csp/adapters/parquet/CMakeLists.txt
index d89a0cd49..60a0c79c2 100644
--- a/cpp/csp/adapters/parquet/CMakeLists.txt
+++ b/cpp/csp/adapters/parquet/CMakeLists.txt
@@ -8,7 +8,6 @@ set(PARQUET_HEADER_FILES
         ParquetOutputFilenameAdapter.h
         ParquetStatusUtils.h
         ParquetWriter.h
-        RecordBatchFileSink.h
         RecordBatchSink.h
         RecordBatchStreamSource.h
 )
@@ -21,7 +20,7 @@ set(PARQUET_SOURCE_FILES
         ParquetOutputAdapterManager.cpp
         ParquetOutputFilenameAdapter.cpp
         ParquetWriter.cpp
-        RecordBatchFileSink.cpp
+        RecordBatchSink.cpp
         ${PARQUET_HEADER_FILES}
         )
 
diff --git a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
index 39d3fbf55..faf79206d 100644
--- a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.cpp
@@ -21,11 +21,11 @@ void ParquetDictBasketOutputWriter::start()
     m_indexSchema = ::arrow::schema( { ::arrow::field(
         m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getColumnName(),
         m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 ) -> getDataType() ) }, getFileMetaData() );
-    if( m_indexSink.onStart )
-        m_indexSink.onStart( m_indexSchema );
+    if( m_indexSink )
+        m_indexSink -> onStart( m_indexSchema );
     auto & fileName = m_adapterMgr.getFileName();
-    if( !fileName.empty() && m_indexSink.onFileChange )
-        m_indexSink.onFileChange( fileName );
+    if( !fileName.empty() && m_indexSink )
+        m_indexSink -> onFileChange( fileName );
 }
 
 void ParquetDictBasketOutputWriter::stop()
@@ -40,16 +40,9 @@ void ParquetDictBasketOutputWriter::stop()
         catch( ... ) { if( !firstError ) firstError = std::current_exception(); }
     };
 
-    auto && indexBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 );
-    if( indexBuilder -> length() > 0 && m_indexSink.onBatch )
-        guard( [&]
-        {
-            auto arr = indexBuilder -> buildArray();
-            auto rb  = ::arrow::RecordBatch::Make( m_indexSchema, arr -> length(), { arr } );
-            m_indexSink.onBatch( rb );
-        } );
-    if( m_indexSink.onStop )
-        guard( [&] { m_indexSink.onStop(); } );
+    guard( [&] { flushIndexBatch(); } );
+    if( m_indexSink )
+        guard( [&] { m_indexSink -> onStop(); } );
 
     guard( [&] { ParquetWriter::stop(); } );
 
@@ -73,12 +66,8 @@ void ParquetDictBasketOutputWriter::onEndCycle()
         m_cycleIndexOutputAdapter -> writeValue<std::uint16_t>( m_nextCycleIndex );
         auto && indexBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 );
         indexBuilder -> handleRowFinished();
-        if( indexBuilder -> length() >= getChunkSize() && m_indexSink.onBatch )
-        {
-            auto arr = indexBuilder -> buildArray();
-            auto rb  = ::arrow::RecordBatch::Make( m_indexSchema, arr -> length(), { arr } );
-            m_indexSink.onBatch( rb );
-        }
+        if( indexBuilder -> length() >= getChunkSize() )
+            flushIndexBatch();
         m_nextCycleIndex = 0;
     }
     else
@@ -101,22 +90,26 @@ void ParquetDictBasketOutputWriter::onFileNameChange( const std::string &fileNam
 
     guard( [&] { ParquetWriter::onFileNameChange( fileName ); } );
 
-    // Flush any pending index data
-    auto && indexBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 );
-    if( indexBuilder -> length() > 0 && m_indexSink.onBatch )
-        guard( [&]
-        {
-            auto arr = indexBuilder -> buildArray();
-            auto rb  = ::arrow::RecordBatch::Make( m_indexSchema, arr -> length(), { arr } );
-            m_indexSink.onBatch( rb );
-        } );
-    if( m_indexSink.onFileChange )
-        guard( [&] { m_indexSink.onFileChange( fileName ); } );
+    // Flush any pending index data, then rotate the index file.
+    guard( [&] { flushIndexBatch(); } );
+    if( m_indexSink )
+        guard( [&] { m_indexSink -> onFileChange( fileName ); } );
 
     if( firstError )
         std::rethrow_exception( firstError );
 }
 
+void ParquetDictBasketOutputWriter::flushIndexBatch()
+{
+    auto && indexBuilder = m_cycleIndexOutputAdapter -> getColumnArrayBuilder( 0 );
+    if( indexBuilder -> length() > 0 && m_indexSink )
+    {
+        auto arr = indexBuilder -> buildArray();
+        auto rb  = ::arrow::RecordBatch::Make( m_indexSchema, arr -> length(), { arr } );
+        m_indexSink -> onBatch( rb );
+    }
+}
+
 SingleColumnParquetOutputHandler *ParquetDictBasketOutputWriter::createScalarOutputHandler( CspTypePtr type, const std::string &name )
 {
     m_allHandlers.push_back( std::make_unique<SingleColumnParquetOutputHandler>( m_engine, *this, type, name ) );
diff --git a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h
index 7362edb17..7a5e5529d 100644
--- a/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h
+++ b/cpp/csp/adapters/parquet/ParquetDictBasketOutputWriter.h
@@ -16,7 +16,7 @@ class ParquetDictBasketOutputWriter : public ParquetWriter
     void start() override;
     void stop() override;
 
-    void setIndexSink( RecordBatchSink sink ) { m_indexSink = std::move( sink ); }
+    void setIndexSink( RecordBatchSink sink ) { m_indexSink.emplace( std::move( sink ) ); }
 
     virtual void writeValue( const std::string &valueKey, const TimeSeriesProvider *ts );
 
@@ -27,11 +27,15 @@ class ParquetDictBasketOutputWriter : public ParquetWriter
     StructParquetOutputHandler *createStructOutputHandler( CspTypePtr type, const DictionaryPtr &fieldMap ) override;
 
 private:
+    // Build a RecordBatch from any pending index rows and hand it to the index sink. Shared by
+    // onEndCycle (chunk full), onFileNameChange (rotation), and stop (final flush).
+    void flushIndexBatch();
+
     SingleColumnParquetOutputHandler                    *m_symbolOutputAdapter;
     SingleColumnParquetOutputHandler                    *m_cycleIndexOutputAdapter;
     std::uint16_t                                       m_nextCycleIndex;
     std::vector<std::unique_ptr<ParquetOutputHandler>>  m_allHandlers;
-    RecordBatchSink                                     m_indexSink;
+    std::optional<RecordBatchSink>                      m_indexSink;
     std::shared_ptr<::arrow::Schema>                    m_indexSchema;
 };
 
diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp
index 889cb27ac..bc8fabc17 100644
--- a/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp
@@ -17,17 +17,15 @@ uint32_t ParquetOutputHandler::getChunkSize() const
 
 SingleColumnParquetOutputHandler::SingleColumnParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type,
                                                                     std::string columnName )
-        : ParquetOutputHandler( parquetWriter, type )
+        : ParquetOutputHandler( parquetWriter )
 {
     bool isBytes = false;
-    CspTypePtr effectiveType = type;
-
     if( type -> type() == CspType::TypeTraits::STRING )
     {
         isBytes = static_cast<const CspStringType &>( *type ).isBytes();
     }
 
-    auto arrowBuilder = createArrowBackedArrayBuilder( columnName, getChunkSize(), effectiveType, isBytes );
+    auto arrowBuilder = createArrowBackedArrayBuilder( columnName, getChunkSize(), type, isBytes );
     auto * scratch    = arrowBuilder -> scratch();
     auto   field      = arrowBuilder -> scratchField();
     m_columnArrayBuilder = arrowBuilder;
@@ -40,7 +38,7 @@ SingleColumnParquetOutputHandler::SingleColumnParquetOutputHandler( Engine *engi
         },
         [&]()
         {
-            CSP_THROW( TypeError, "Writing of " << m_type -> type().asString() << " to parquet is not supported" );
+            CSP_THROW( TypeError, "Writing of " << type -> type().asString() << " to parquet is not supported" );
         } );
 }
 
@@ -64,7 +62,7 @@ template void SingleColumnParquetOutputHandler::writeValue<std::uint16_t, void>(
 
 ListColumnParquetOutputHandler::ListColumnParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &elemType,
                                                                 const std::string &columnName )
-        : ParquetOutputHandler( parquetWriter, CspType::DIALECT_GENERIC() )
+        : ParquetOutputHandler( parquetWriter )
 {
     auto [ valueBuilder, writeItemsFn ] = csp::adapters::arrow::createListFieldWriter( elemType );
     m_columnArrayBuilder = std::make_shared<ListColumnArrayBuilder>(
@@ -86,7 +84,7 @@ void ListColumnParquetOutputAdapter::executeImpl()
 
 StructParquetOutputHandler::StructParquetOutputHandler( Engine *engine, ParquetWriter &parquetWriter, CspTypePtr &type,
                                                         DictionaryPtr fieldMap )
-        : ParquetOutputHandler( parquetWriter, type )
+        : ParquetOutputHandler( parquetWriter )
 {
     auto structMetaPtr = std::static_pointer_cast<const CspStructType>( type ) -> meta().get();
 
diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapter.h b/cpp/csp/adapters/parquet/ParquetOutputAdapter.h
index 8c6e26e4f..cfa29dfd6 100644
--- a/cpp/csp/adapters/parquet/ParquetOutputAdapter.h
+++ b/cpp/csp/adapters/parquet/ParquetOutputAdapter.h
@@ -27,8 +27,8 @@ class ParquetWriter;
 class ParquetOutputHandler
 {
 public:
-    ParquetOutputHandler( ParquetWriter &parquetWriter, CspTypePtr &type )
-            : m_type( type ), m_parquetWriter( parquetWriter )
+    ParquetOutputHandler( ParquetWriter &parquetWriter )
+            : m_parquetWriter( parquetWriter )
     {
     }
 
@@ -41,7 +41,6 @@ class ParquetOutputHandler
     virtual void writeValueFromTs( const TimeSeriesProvider *input ) = 0;
 
 protected:
-    CspTypePtr    &m_type;
     ParquetWriter &m_parquetWriter;
 };
 
diff --git a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h
index f6a1927dc..8423d451c 100644
--- a/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h
+++ b/cpp/csp/adapters/parquet/ParquetOutputAdapterManager.h
@@ -6,6 +6,8 @@
 #include <csp/core/Generator.h>
 #include <csp/engine/AdapterManager.h>
 #include <csp/engine/Dictionary.h>
+#include <functional>
+#include <memory>
 #include <string>
 #include <unordered_map>
 
diff --git a/cpp/csp/adapters/parquet/ParquetWriter.cpp b/cpp/csp/adapters/parquet/ParquetWriter.cpp
index a39f4508c..ea4feeb76 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.cpp
+++ b/cpp/csp/adapters/parquet/ParquetWriter.cpp
@@ -166,12 +166,18 @@ void ParquetWriter::start()
         CSP_THROW( ValueError, "parquet column metadata has unmapped column: '" << m_columnMetaData.begin() -> first << "'" );
 
     m_schema = ::arrow::schema( arrowFields, m_fileMetaData );
-    if( m_sink.onStart )
-        m_sink.onStart( m_schema );
-    auto & fileName = m_adapterMgr.getFileName();
-    m_fileOpen = !fileName.empty();
-    if( m_fileOpen && m_sink.onFileChange )
-        m_sink.onFileChange( fileName );
+    // Hand the schema to the sink, then open the first file (if any). m_fileOpen reflects whether a
+    // file is actually open after onFileChange -- an empty filename (no file / paused) leaves it false.
+    const std::string & fileName = m_adapterMgr.getFileName();
+    if( m_sink )
+    {
+        m_sink -> onStart( m_schema );
+        m_fileOpen = m_sink -> onFileChange( fileName );
+    }
+    else
+    {
+        m_fileOpen = !fileName.empty();
+    }
 }
 
 void ParquetWriter::stop()
@@ -185,9 +191,9 @@ void ParquetWriter::stop()
         try { flushBatch(); }
         catch( ... ) { firstError = std::current_exception(); }
     }
-    if( m_sink.onStop )
+    if( m_sink )
     {
-        try { m_sink.onStop(); }
+        try { m_sink -> onStop(); }
         catch( ... ) { if( !firstError ) firstError = std::current_exception(); }
     }
     m_fileOpen = false;
@@ -240,9 +246,10 @@ void ParquetWriter::onEndCycle()
 void ParquetWriter::onFileNameChange( const std::string &fileName )
 {
     flushBatch();
-    if( m_sink.onFileChange )
-        m_sink.onFileChange( fileName );
-    m_fileOpen = !fileName.empty();
+    if( m_sink )
+        m_fileOpen = m_sink -> onFileChange( fileName );
+    else
+        m_fileOpen = !fileName.empty();
 }
 
 SingleColumnParquetOutputHandler *ParquetWriter::createScalarOutputHandler( CspTypePtr type, const std::string &name )
@@ -286,8 +293,8 @@ void ParquetWriter::flushBatch()
     {
         if( !isFileOpen() ) [[unlikely]]
             CSP_THROW( csp::RuntimeException, "Trying to write to parquet/arrow file, when no file name was provided" );
-        if( m_sink.onBatch )
-            m_sink.onBatch( buildRecordBatch() );
+        if( m_sink )
+            m_sink -> onBatch( buildRecordBatch() );
         m_curChunkSize = 0;
     }
 }
diff --git a/cpp/csp/adapters/parquet/ParquetWriter.h b/cpp/csp/adapters/parquet/ParquetWriter.h
index 493d3006b..9a76d360b 100644
--- a/cpp/csp/adapters/parquet/ParquetWriter.h
+++ b/cpp/csp/adapters/parquet/ParquetWriter.h
@@ -6,6 +6,7 @@
 #include <csp/engine/Dictionary.h>
 #include <csp/engine/Engine.h>
 #include <arrow/record_batch.h>
+#include <memory>
 #include <string>
 #include <optional>
 #include <unordered_map>
@@ -34,7 +35,7 @@ class ParquetWriter : public EndCycleListener
 
     ~ParquetWriter();
 
-    void setSink( RecordBatchSink sink ) { m_sink = std::move( sink ); }
+    void setSink( RecordBatchSink sink ) { m_sink.emplace( std::move( sink ) ); }
 
     // Dict-basket writers adopt the main writer's file-level metadata and any column metadata that
     // targets their own columns, so basket files carry the same metadata and the main writer does not
@@ -84,7 +85,7 @@ class ParquetWriter : public EndCycleListener
     Adapters             m_adapters;
     PublishedColumnNames m_publishedColumnNames;
 
-    RecordBatchSink                                             m_sink;
+    std::optional<RecordBatchSink>                              m_sink;
     bool                                                        m_fileOpen = false;
 
     std::shared_ptr<::arrow::Schema>                            m_schema;
diff --git a/cpp/csp/adapters/parquet/RecordBatchFileSink.h b/cpp/csp/adapters/parquet/RecordBatchFileSink.h
deleted file mode 100644
index 9e61823c4..000000000
--- a/cpp/csp/adapters/parquet/RecordBatchFileSink.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#ifndef _IN_CSP_ADAPTERS_PARQUET_RecordBatchFileSink_H
-#define _IN_CSP_ADAPTERS_PARQUET_RecordBatchFileSink_H
-
-#include <csp/adapters/parquet/RecordBatchSink.h>
-#include <functional>
-#include <string>
-
-namespace csp::adapters::parquet
-{
-
-// Builds a RecordBatchSink that writes RecordBatches to files entirely in C++
-// (no Python / no Arrow C Data Interface hop on the hot path).
-//
-// One sink == one logical output target:
-//   - splitColumns == false : a single file (.parquet or .arrow) with all columns.
-//   - splitColumns == true  : a directory; each column is written to its own file.
-//
-// File rotation is just onFileChange (close current, open next). compression /
-// allowOverwrite / fileVisitor behave as on the Python sink. fileVisitor, if set,
-// is invoked with the path of each file after it is closed.
-RecordBatchSink makeFileSink( bool writeArrowBinary,
-                              bool splitColumns,
-                              const std::string & compression,
-                              bool allowOverwrite,
-                              std::function<void( const std::string & )> fileVisitor = {},
-                              bool useDictionary = true );
-
-} // namespace csp::adapters::parquet
-
-#endif // _IN_CSP_ADAPTERS_PARQUET_RecordBatchFileSink_H
diff --git a/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp b/cpp/csp/adapters/parquet/RecordBatchSink.cpp
similarity index 65%
rename from cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
rename to cpp/csp/adapters/parquet/RecordBatchSink.cpp
index 0c8d1e16a..40178fb46 100644
--- a/cpp/csp/adapters/parquet/RecordBatchFileSink.cpp
+++ b/cpp/csp/adapters/parquet/RecordBatchSink.cpp
@@ -1,4 +1,4 @@
-#include <csp/adapters/parquet/RecordBatchFileSink.h>
+#include <csp/adapters/parquet/RecordBatchSink.h>
 #include <csp/adapters/parquet/ParquetStatusUtils.h>
 #include <csp/core/FileUtils.h>
 
@@ -14,8 +14,7 @@
 #include <cctype>
 #include <exception>
 #include <memory>
-#include <optional>
-#include <unordered_map>
+#include <utility>
 #include <vector>
 
 namespace csp::adapters::parquet
@@ -24,18 +23,6 @@ namespace csp::adapters::parquet
 namespace
 {
 
-// A single open output file: write one batch, then close. Type-erased over format
-// so the rest of the machinery never branches on parquet-vs-ipc-vs-split.
-struct FileWriter
-{
-    std::function<void( const std::shared_ptr<::arrow::RecordBatch> & )> writeBatch;
-    std::function<void()>                                                close;
-};
-
-// (path, schema) -> open FileWriter.  The factory IS the format/layout choice.
-using FileWriterFactory =
-    std::function<FileWriter( const std::string &, const std::shared_ptr<::arrow::Schema> & )>;
-
 ::arrow::Compression::type resolveCompression( const std::string & name )
 {
     // csp treats "" / "none" as no compression; Arrow expects lower-case codec names.
@@ -70,8 +57,11 @@ std::shared_ptr<::arrow::io::OutputStream> openOutputStream( const std::string &
     return result.MoveValueUnsafe();
 }
 
-FileWriter makeParquetFileWriter( const std::string & path, const std::shared_ptr<::arrow::Schema> & schema,
-                                  const std::string & compression, bool allowOverwrite, bool useDictionary )
+} // namespace
+
+RecordBatchSink::FileWriter RecordBatchSink::makeParquetFileWriter(
+    const std::string & path, const std::shared_ptr<::arrow::Schema> & schema,
+    const std::string & compression, bool allowOverwrite, bool useDictionary )
 {
     auto stream = openOutputStream( path, allowOverwrite );
 
@@ -113,8 +103,9 @@ FileWriter makeParquetFileWriter( const std::string & path, const std::shared_pt
         } };
 }
 
-FileWriter makeIpcFileWriter( const std::string & path, const std::shared_ptr<::arrow::Schema> & schema,
-                              const std::string & compression, bool allowOverwrite )
+RecordBatchSink::FileWriter RecordBatchSink::makeIpcFileWriter(
+    const std::string & path, const std::shared_ptr<::arrow::Schema> & schema,
+    const std::string & compression, bool allowOverwrite )
 {
     auto stream = openOutputStream( path, allowOverwrite );
 
@@ -149,10 +140,12 @@ FileWriter makeIpcFileWriter( const std::string & path, const std::shared_ptr<::
         } };
 }
 
-// A split-columns writer is itself a FileWriter that fans out to one sub-writer per
-// column.  Each column file carries the parent file-level metadata.
-FileWriter makeSplitWriter( const std::string & dir, const std::shared_ptr<::arrow::Schema> & schema,
-                            const FileWriterFactory & perColumn, const std::string & extension )
+// A split-columns writer is itself a FileWriter that fans out to one sub-writer per column.
+// Each column file carries the parent file-level metadata.
+RecordBatchSink::FileWriter RecordBatchSink::makeSplitWriter(
+    const std::string & dir, const std::shared_ptr<::arrow::Schema> & schema,
+    bool writeArrowBinary, const std::string & compression, bool allowOverwrite,
+    bool useDictionary, const std::string & extension )
 {
     if( !dir.empty() )
         utils::mkdir( dir );
@@ -168,7 +161,10 @@ FileWriter makeSplitWriter( const std::string & dir, const std::shared_ptr<::arr
         {
             auto colSchema = ::arrow::schema( { schema -> field( i ) }, schema -> metadata() );
             colSchemas -> push_back( colSchema );
-            subWriters -> push_back( perColumn( dir + "/" + schema -> field( i ) -> name() + extension, colSchema ) );
+            auto colPath = dir + "/" + schema -> field( i ) -> name() + extension;
+            subWriters -> push_back( writeArrowBinary
+                ? makeIpcFileWriter( colPath, colSchema, compression, allowOverwrite )
+                : makeParquetFileWriter( colPath, colSchema, compression, allowOverwrite, useDictionary ) );
         }
     }
     catch( ... )
@@ -206,72 +202,72 @@ FileWriter makeSplitWriter( const std::string & dir, const std::shared_ptr<::arr
         } };
 }
 
-} // namespace
+RecordBatchSink::RecordBatchSink( bool writeArrowBinary, bool splitColumns,
+                                  std::string compression, bool allowOverwrite,
+                                  std::function<void( const std::string & )> fileVisitor,
+                                  bool useDictionary )
+    : m_writeArrowBinary( writeArrowBinary )
+    , m_splitColumns( splitColumns )
+    , m_compression( std::move( compression ) )
+    , m_allowOverwrite( allowOverwrite )
+    , m_fileVisitor( std::move( fileVisitor ) )
+    , m_useDictionary( useDictionary )
+    , m_extension( writeArrowBinary ? ".arrow" : ".parquet" )
+{
+}
 
-RecordBatchSink makeFileSink( bool writeArrowBinary, bool splitColumns,
-                              const std::string & compression, bool allowOverwrite,
-                              std::function<void( const std::string & )> fileVisitor,
-                              bool useDictionary )
+
+RecordBatchSink::FileWriter RecordBatchSink::openWriter( const std::string & path )
 {
-    const std::string extension = writeArrowBinary ? ".arrow" : ".parquet";
+    if( m_splitColumns )
+        return makeSplitWriter( path, m_schema, m_writeArrowBinary, m_compression, m_allowOverwrite,
+                                m_useDictionary, m_extension );
+    return m_writeArrowBinary ? makeIpcFileWriter( path, m_schema, m_compression, m_allowOverwrite )
+                              : makeParquetFileWriter( path, m_schema, m_compression, m_allowOverwrite, m_useDictionary );
+}
 
-    // Per-file factory: format choice is the only difference.
-    FileWriterFactory perFile =
-        [writeArrowBinary, compression, allowOverwrite, useDictionary]( const std::string & path, const std::shared_ptr<::arrow::Schema> & schema )
-        {
-            return writeArrowBinary ? makeIpcFileWriter( path, schema, compression, allowOverwrite )
-                                    : makeParquetFileWriter( path, schema, compression, allowOverwrite, useDictionary );
-        };
+void RecordBatchSink::closeCurrent()
+{
+    if( m_current.has_value() )
+    {
+        // Detach + reset BEFORE invoking the (user-supplied) visitor, so that a throwing visitor
+        // cannot cause a later onStop()->closeCurrent() to close this writer twice.
+        FileWriter w = std::move( *m_current );
+        m_current.reset();
+        w.close();
+        if( m_fileVisitor )
+            m_fileVisitor( m_currentPath );
+    }
+}
 
-    // Layout choice: single file vs. one file per column.
-    FileWriterFactory factory = perFile;
-    if( splitColumns )
-        factory = [perFile, extension]( const std::string & dir, const std::shared_ptr<::arrow::Schema> & schema )
-        {
-            return makeSplitWriter( dir, schema, perFile, extension );
-        };
+void RecordBatchSink::onStart( const std::shared_ptr<::arrow::Schema> & schema )
+{
+    m_schema = schema;
+}
 
-    // State shared across the sink callbacks.
-    auto schemaHolder = std::make_shared<std::shared_ptr<::arrow::Schema>>();
-    auto current      = std::make_shared<std::optional<FileWriter>>();
-    auto currentPath  = std::make_shared<std::string>();
+void RecordBatchSink::onBatch( const std::shared_ptr<::arrow::RecordBatch> & rb )
+{
+    // A batch must only ever arrive while a file is open (the writer guards this via isFileOpen()
+    // before flushing). If the writer and sink ever desync, fail loudly instead of silently
+    // dropping the batch's rows.
+    CSP_TRUE_OR_THROW_RUNTIME( m_current.has_value(),
+        "RecordBatchSink received a record batch with no open output file" );
+    m_current -> writeBatch( rb );
+}
 
-    auto closeCurrent = [current, currentPath, fileVisitor]()
-    {
-        if( current -> has_value() )
-        {
-            // Detach + reset BEFORE invoking the (user-supplied) visitor, so that a throwing
-            // visitor cannot cause a later onStop()->closeCurrent() to close this writer twice.
-            FileWriter w = std::move( current -> value() );
-            current -> reset();
-            w.close();
-            if( fileVisitor )
-                fileVisitor( *currentPath );
-        }
-    };
+bool RecordBatchSink::onFileChange( const std::string & path )
+{
+    closeCurrent();
+    if( path.empty() )
+        return false;
+    m_current     = openWriter( path );
+    m_currentPath = path;
+    return true;
+}
 
-    RecordBatchSink sink;
-    sink.onStart      = [schemaHolder]( const std::shared_ptr<::arrow::Schema> & schema ) { *schemaHolder = schema; };
-    sink.onBatch      = [current]( const std::shared_ptr<::arrow::RecordBatch> & rb )
-    {
-        // A batch must only ever arrive while a file is open (the writer guards this via isFileOpen()
-        // before flushing). If the writer and sink ever desync, fail loudly instead of silently
-        // dropping the batch's rows.
-        CSP_TRUE_OR_THROW_RUNTIME( current -> has_value(),
-            "RecordBatchFileSink received a record batch with no open output file" );
-        ( *current ) -> writeBatch( rb );
-    };
-    sink.onFileChange = [factory, schemaHolder, current, currentPath, closeCurrent]( const std::string & path )
-    {
-        closeCurrent();
-        if( !path.empty() )
-        {
-            *current     = factory( path, *schemaHolder );
-            *currentPath = path;
-        }
-    };
-    sink.onStop = closeCurrent;
-    return sink;
+void RecordBatchSink::onStop()
+{
+    closeCurrent();
 }
 
 } // namespace csp::adapters::parquet
diff --git a/cpp/csp/adapters/parquet/RecordBatchSink.h b/cpp/csp/adapters/parquet/RecordBatchSink.h
index c7b0036e7..1f219ee88 100644
--- a/cpp/csp/adapters/parquet/RecordBatchSink.h
+++ b/cpp/csp/adapters/parquet/RecordBatchSink.h
@@ -4,6 +4,7 @@
 #include <arrow/record_batch.h>
 #include <functional>
 #include <memory>
+#include <optional>
 #include <string>
 
 namespace arrow { class Schema; }
@@ -11,21 +12,74 @@ namespace arrow { class Schema; }
 namespace csp::adapters::parquet
 {
 
-// Callback interface for receiving RecordBatches from ParquetWriter.
-// Python side implements this via a sink object.
-struct RecordBatchSink
+// Writes the RecordBatches produced by ParquetWriter to files, entirely in C++ (no Python / no Arrow
+// C Data Interface hop on the hot path).
+//
+// One sink == one logical output target:
+//   - splitColumns == false : a single file (.parquet or .arrow) with all columns.
+//   - splitColumns == true  : a directory; each column is written to its own file.
+//
+// The writer drives the lifecycle: onStart() once with the schema, then onFileChange()/onBatch() as
+// data flows, then onStop() at teardown. File rotation is just onFileChange (close current, open
+// next). fileVisitor, if set, is invoked with the path of each file after it is closed.
+class RecordBatchSink
 {
-    using SchemaCallback     = std::function<void( const std::shared_ptr<::arrow::Schema> & )>;
-    using BatchCallback      = std::function<void( const std::shared_ptr<::arrow::RecordBatch> & )>;
-    using FileChangeCallback = std::function<void( const std::string & )>;
-    using StopCallback       = std::function<void()>;
-
-    SchemaCallback     onStart;       // called with schema at start()
-    BatchCallback      onBatch;       // called with each full RecordBatch
-    FileChangeCallback onFileChange;  // called when file rotation requested
-    StopCallback       onStop;        // called at stop()
+public:
+    RecordBatchSink( bool writeArrowBinary,
+                     bool splitColumns,
+                     std::string compression,
+                     bool allowOverwrite,
+                     std::function<void( const std::string & )> fileVisitor = {},
+                     bool useDictionary = true );
+
+    // Called once with the output schema before any batches.
+    void onStart( const std::shared_ptr<::arrow::Schema> & schema );
+    // Called with each full RecordBatch. A batch only arrives while a file is open.
+    void onBatch( const std::shared_ptr<::arrow::RecordBatch> & rb );
+    // Close the current file (if any) and open path (if non-empty). Returns true if a file is open
+    // after the call (path was non-empty), false otherwise.
+    bool onFileChange( const std::string & path );
+    // Close the current file (if any).
+    void onStop();
+
+private:
+    // A single open output file: write one batch, then close. Type-erased over format so the rest
+    // of the sink never branches on parquet-vs-ipc-vs-split.
+    struct FileWriter
+    {
+        std::function<void( const std::shared_ptr<::arrow::RecordBatch> & )> writeBatch;
+        std::function<void()>                                                close;
+    };
+
+    // Format/layout builders. Kept as plain functions (not a class hierarchy): each returns a
+    // FileWriter that writes one open file. They are static members (rather than .cpp-local free
+    // functions) only so they can name the private FileWriter type.
+    static FileWriter makeParquetFileWriter( const std::string & path, const std::shared_ptr<::arrow::Schema> & schema,
+                                             const std::string & compression, bool allowOverwrite, bool useDictionary );
+    static FileWriter makeIpcFileWriter( const std::string & path, const std::shared_ptr<::arrow::Schema> & schema,
+                                         const std::string & compression, bool allowOverwrite );
+    static FileWriter makeSplitWriter( const std::string & dir, const std::shared_ptr<::arrow::Schema> & schema,
+                                       bool writeArrowBinary, const std::string & compression, bool allowOverwrite,
+                                       bool useDictionary, const std::string & extension );
+
+    // (path) -> open FileWriter for m_schema, honoring this sink's format/layout/compression settings.
+    FileWriter openWriter( const std::string & path );
+    // Close the current file (if any), running fileVisitor after, exactly once.
+    void closeCurrent();
+
+    const bool                                  m_writeArrowBinary;
+    const bool                                  m_splitColumns;
+    const std::string                           m_compression;
+    const bool                                  m_allowOverwrite;
+    std::function<void( const std::string & )>  m_fileVisitor;
+    const bool                                  m_useDictionary;
+    const std::string                           m_extension;
+
+    std::shared_ptr<::arrow::Schema>            m_schema;
+    std::optional<FileWriter>                   m_current;
+    std::string                                 m_currentPath;
 };
 
 } // namespace csp::adapters::parquet
 
-#endif
+#endif // _IN_CSP_ADAPTERS_PARQUET_RecordBatchSink_H
diff --git a/cpp/csp/python/adapters/parquetadapterimpl.cpp b/cpp/csp/python/adapters/parquetadapterimpl.cpp
index c5239b04e..42af21aba 100644
--- a/cpp/csp/python/adapters/parquetadapterimpl.cpp
+++ b/cpp/csp/python/adapters/parquetadapterimpl.cpp
@@ -4,7 +4,6 @@
 #include <csp/adapters/parquet/ParquetDictBasketOutputWriter.h>
 #include <csp/adapters/parquet/ParquetStatusUtils.h>
 #include <csp/adapters/parquet/ParquetWriter.h>
-#include <csp/adapters/parquet/RecordBatchFileSink.h>
 #include <csp/adapters/parquet/RecordBatchSink.h>
 #include <csp/engine/PushInputAdapter.h>
 #include <csp/python/Conversions.h>
@@ -680,7 +679,7 @@ static PyObject *create_parquet_input_adapter_manager_native( PyObject *args )
 
 csp::AdapterManager *create_parquet_output_adapter_manager( PyEngine *engine, const Dictionary &properties )
 {
-    // Output file I/O is done entirely in C++ via makeFileSink (no per-batch Python hop).
+    // Output file I/O is done entirely in C++ via RecordBatchSink (no per-batch Python hop).
     bool        writeArrowBinary = properties.get<bool>( "write_arrow_binary" );
     bool        splitColumns     = properties.get<bool>( "split_columns_to_files" );
     bool        allowOverwrite   = properties.get<bool>( "allow_overwrite" );
@@ -708,13 +707,14 @@ csp::AdapterManager *create_parquet_output_adapter_manager( PyEngine *engine, co
     }
 
     auto * mgr = engine -> engine() -> createOwnedObject<ParquetOutputAdapterManager>( properties );
-    mgr -> setSink( makeFileSink( writeArrowBinary, splitColumns, compression, allowOverwrite, fileVisitor, useDictionary ) );
+    mgr -> setSink( RecordBatchSink( writeArrowBinary, splitColumns, compression, allowOverwrite, fileVisitor, useDictionary ) );
 
     // Dict-basket writers always write split-column files (one per value/symbol/index column),
     // without a file visitor.
     mgr -> setSinkFactory( [writeArrowBinary, compression, allowOverwrite, useDictionary]( const std::string & ) -> RecordBatchSink
     {
-        return makeFileSink( writeArrowBinary, /*splitColumns=*/true, compression, allowOverwrite, {}, useDictionary );
+        return RecordBatchSink( writeArrowBinary, /*splitColumns=*/true, compression, allowOverwrite,
+                                std::function<void( const std::string & )>{}, useDictionary );
     } );
 
     return mgr;
diff --git a/csp/adapters/output_adapters/parquet.py b/csp/adapters/output_adapters/parquet.py
index 5c14f5de7..4f67da55d 100644
--- a/csp/adapters/output_adapters/parquet.py
+++ b/csp/adapters/output_adapters/parquet.py
@@ -1,9 +1,7 @@
 import os
-from importlib.metadata import PackageNotFoundError, version as get_package_version
 from typing import Callable, Dict, Optional, TypeVar
 
 import numpy
-from packaging import version
 
 import csp
 from csp.impl.struct import Struct
@@ -39,19 +37,7 @@ def resolve_compression(self):
         return self
 
 
-def _get_default_parquet_version():
-    try:
-        if version.parse(get_package_version("pyarrow")) >= version.parse("6.0.1"):
-            return "2.6"
-    except PackageNotFoundError:
-        # Don't need to do anything in particular
-        ...
-    return "2.0"
-
-
 class ParquetWriter:
-    PARQUET_VERSION = _get_default_parquet_version()
-
     def __init__(
         self,
         file_name: Optional[str],