From ad6c2c1876e3454a5fcda88080a5c4d6ba959ddc Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Thu, 21 Aug 2025 17:58:55 +0100 Subject: [PATCH 01/11] Define __repr__ and __str__ methods for CSV/JSON options --- python/pyarrow/_csv.pyx | 84 ++++++++++++++++++++++++++++++++++++++++ python/pyarrow/_json.pyx | 22 +++++++++++ 2 files changed, 106 insertions(+) diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index ed9d20beb6b3..37fbc5b3ec1e 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -332,6 +332,26 @@ cdef class ReadOptions(_Weakrefable): except TypeError: return False + def __repr__(self): + return (f"") + + def __str__(self): + return (f"ReadOptions(" + f"use_threads={self.use_threads}, " + f"block_size={self.block_size}, " + f"skip_rows={self.skip_rows}, " + f"skip_rows_after_names={self.skip_rows_after_names}, " + f"column_names={self.column_names}, " + f"autogenerate_column_names={self.autogenerate_column_names}, " + f"encoding='{self.encoding}')") + cdef class ParseOptions(_Weakrefable): """ @@ -585,6 +605,24 @@ cdef class ParseOptions(_Weakrefable): except TypeError: return False + def __repr__(self): + return (f"") + + def __str__(self): + return (f"ParseOptions(" + f"delimiter={self.delimiter!r}, " + f"quote_char={self.quote_char!r}, " + f"double_quote={self.double_quote}, " + f"escape_char={self.escape_char!r}, " + f"newlines_in_values={self.newlines_in_values}, " + f"ignore_empty_lines={self.ignore_empty_lines})") + cdef class _ISO8601(_Weakrefable): """ @@ -1108,6 +1146,38 @@ cdef class ConvertOptions(_Weakrefable): except TypeError: return False + def __repr__(self): + return (f"") + + def __str__(self): + return (f"ConvertOptions(" + f"check_utf8={self.check_utf8}, " + f"column_types={self.column_types}, " + f"null_values={self.null_values}, " + f"true_values={self.true_values}, " + f"false_values={self.false_values}, " + f"decimal_point={self.decimal_point!r}, " + f"strings_can_be_null={self.strings_can_be_null}, " + f"quoted_strings_can_be_null={self.quoted_strings_can_be_null}, " + f"include_columns={self.include_columns}, " + f"include_missing_columns={self.include_missing_columns}, " + f"auto_dict_encode={self.auto_dict_encode}, " + f"auto_dict_max_cardinality={self.auto_dict_max_cardinality}, " + f"timestamp_parsers={self.timestamp_parsers})") + cdef _get_reader(input_file, ReadOptions read_options, shared_ptr[CInputStream]* out): @@ -1459,6 +1529,20 @@ cdef class WriteOptions(_Weakrefable): def validate(self): check_status(self.options.get().Validate()) + def __repr__(self): + return (f"") + + def __str__(self): + return (f"WriteOptions(" + f"include_header={self.include_header}, " + f"batch_size={self.batch_size}, " + f"delimiter={self.delimiter!r}, " + f"quoting_style='{self.quoting_style}')") + cdef _get_write_options(WriteOptions write_options, CCSVWriteOptions* out): if write_options is None: diff --git a/python/pyarrow/_json.pyx b/python/pyarrow/_json.pyx index 07e615dd5e48..edccfa97a64d 100644 --- a/python/pyarrow/_json.pyx +++ b/python/pyarrow/_json.pyx @@ -105,6 +105,16 @@ cdef class ReadOptions(_Weakrefable): except TypeError: return False + def __repr__(self): + return (f"") + + def __str__(self): + return (f"ReadOptions(" + f"use_threads={self.use_threads}, " + f"block_size={self.block_size})") + @staticmethod cdef ReadOptions wrap(CJSONReadOptions options): out = ReadOptions() @@ -244,6 +254,18 @@ cdef class ParseOptions(_Weakrefable): except TypeError: return False + def __repr__(self): + return (f"") + + def __str__(self): + return (f"ParseOptions(" + f"explicit_schema={self.explicit_schema}, " + f"newlines_in_values={self.newlines_in_values}, " + f"unexpected_field_behavior='{self.unexpected_field_behavior}')") + @staticmethod cdef ParseOptions wrap(CJSONParseOptions options): out = ParseOptions() From c64a8eed50ada248c0154d37ed976315f0ccdc67 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Mon, 25 Aug 2025 11:37:00 +0100 Subject: [PATCH 02/11] Add tests and add parens for repr --- python/pyarrow/_csv.pyx | 68 ++++++++++++------------- python/pyarrow/_json.pyx | 14 ++--- python/pyarrow/tests/test_csv.py | 85 +++++++++++++++++++++++++++++++ python/pyarrow/tests/test_json.py | 22 ++++++++ 4 files changed, 148 insertions(+), 41 deletions(-) diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index 37fbc5b3ec1e..1f71766a6757 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -333,14 +333,14 @@ cdef class ReadOptions(_Weakrefable): return False def __repr__(self): - return (f"") + return (f"") def __str__(self): return (f"ReadOptions(" @@ -606,13 +606,13 @@ cdef class ParseOptions(_Weakrefable): return False def __repr__(self): - return (f"") + return (f"") def __str__(self): return (f"ParseOptions(" @@ -1147,20 +1147,20 @@ cdef class ConvertOptions(_Weakrefable): return False def __repr__(self): - return (f"") + return (f"") def __str__(self): return (f"ConvertOptions(" @@ -1530,11 +1530,11 @@ cdef class WriteOptions(_Weakrefable): check_status(self.options.get().Validate()) def __repr__(self): - return (f"") + return (f"") def __str__(self): return (f"WriteOptions(" diff --git a/python/pyarrow/_json.pyx b/python/pyarrow/_json.pyx index edccfa97a64d..4577df49bbf1 100644 --- a/python/pyarrow/_json.pyx +++ b/python/pyarrow/_json.pyx @@ -106,9 +106,9 @@ cdef class ReadOptions(_Weakrefable): return False def __repr__(self): - return (f"") + return (f"") def __str__(self): return (f"ReadOptions(" @@ -255,10 +255,10 @@ cdef class ParseOptions(_Weakrefable): return False def __repr__(self): - return (f"") + return (f"") def __str__(self): return (f"ParseOptions(" diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index dce605c7156d..5ca4b6afd045 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -213,6 +213,26 @@ def test_read_options(pickle_module): opts.column_names = ('a', 'b') opts.validate() + expected_repr = ("") + assert repr(opts) == expected_repr + + expected_str = (f"ReadOptions(" + f"use_threads={opts.use_threads}, " + f"block_size={opts.block_size}, " + f"skip_rows={opts.skip_rows}, " + f"skip_rows_after_names={opts.skip_rows_after_names}, " + f"column_names={opts.column_names}, " + f"autogenerate_column_names={opts.autogenerate_column_names}, " + f"encoding='{opts.encoding}')") + assert str(opts) == expected_str + def test_parse_options(pickle_module): cls = ParseOptions @@ -273,6 +293,24 @@ def test_parse_options(pickle_module): opts.escape_char = "\r" opts.validate() + expected_repr = ("") + assert repr(opts) == expected_repr + + expected_str = (f"ParseOptions(" + f"delimiter={opts.delimiter!r}, " + f"quote_char={opts.quote_char!r}, " + f"double_quote={opts.double_quote}, " + f"escape_char={opts.escape_char!r}, " + f"newlines_in_values={opts.newlines_in_values}, " + f"ignore_empty_lines={opts.ignore_empty_lines})") + assert str(opts) == expected_str + def test_convert_options(pickle_module): cls = ConvertOptions @@ -354,6 +392,38 @@ def test_convert_options(pickle_module): assert opts.auto_dict_max_cardinality == 999 assert opts.timestamp_parsers == [ISO8601, '%Y-%m-%d'] + expected_repr = ("") + assert repr(opts) == expected_repr + + expected_str = (f"ConvertOptions(" + f"check_utf8={opts.check_utf8}, " + f"column_types={opts.column_types}, " + f"null_values={opts.null_values}, " + f"true_values={opts.true_values}, " + f"false_values={opts.false_values}, " + f"decimal_point={opts.decimal_point!r}, " + f"strings_can_be_null={opts.strings_can_be_null}, " + f"quoted_strings_can_be_null={opts.quoted_strings_can_be_null}, " + f"include_columns={opts.include_columns}, " + f"include_missing_columns={opts.include_missing_columns}, " + f"auto_dict_encode={opts.auto_dict_encode}, " + f"auto_dict_max_cardinality={opts.auto_dict_max_cardinality}, " + f"timestamp_parsers={opts.timestamp_parsers})") + assert str(opts) == expected_str + def test_write_options(): cls = WriteOptions @@ -378,6 +448,21 @@ def test_write_options(): opts.batch_size = 0 opts.validate() + expected_repr = ("") + assert repr(opts) == expected_repr + + # Test str + expected_str = (f"WriteOptions(" + f"include_header={opts.include_header}, " + f"batch_size={opts.batch_size}, " + f"delimiter={opts.delimiter!r}, " + f"quoting_style='{opts.quoting_style}')") + assert str(opts) == expected_str + class BaseTestCSV(abc.ABC): """Common tests which are shared by streaming and non streaming readers""" diff --git a/python/pyarrow/tests/test_json.py b/python/pyarrow/tests/test_json.py index c3f9fe333bd0..f2261836888a 100644 --- a/python/pyarrow/tests/test_json.py +++ b/python/pyarrow/tests/test_json.py @@ -80,6 +80,16 @@ def test_read_options(pickle_module): assert opts.block_size == 1234 assert opts.use_threads is False + expected_repr = ("") + assert repr(opts) == expected_repr + + expected_str = (f"ReadOptions(" + f"use_threads={opts.use_threads}, " + f"block_size={opts.block_size})") + assert str(opts) == expected_str + check_options_class_pickling(cls, pickler=pickle_module, block_size=1234, use_threads=False) @@ -94,6 +104,18 @@ def test_parse_options(pickle_module): opts.newlines_in_values = True assert opts.newlines_in_values is True + expected_repr = ("") + assert repr(opts) == expected_repr + + expected_str = (f"ParseOptions(" + f"explicit_schema={opts.explicit_schema}, " + f"newlines_in_values={opts.newlines_in_values}, " + f"unexpected_field_behavior='{opts.unexpected_field_behavior}')") + assert str(opts) == expected_str + schema = pa.schema([pa.field('foo', pa.int32())]) opts.explicit_schema = schema assert opts.explicit_schema == schema From f1bc6b753b550dbd7ac32880a7f841813055d194 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Mon, 1 Sep 2025 10:37:06 +0100 Subject: [PATCH 03/11] Update python/pyarrow/tests/test_csv.py Co-authored-by: Alenka Frim --- python/pyarrow/tests/test_csv.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index 5ca4b6afd045..e93d7ba76657 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -455,7 +455,6 @@ def test_write_options(): f"quoting_style='{opts.quoting_style}')>") assert repr(opts) == expected_repr - # Test str expected_str = (f"WriteOptions(" f"include_header={opts.include_header}, " f"batch_size={opts.batch_size}, " From 1a5a857bd932e92f741773d17d716d0941f3c956 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Mon, 1 Sep 2025 10:39:37 +0100 Subject: [PATCH 04/11] Add invalid_row_handler param --- python/pyarrow/_csv.pyx | 6 ++++-- python/pyarrow/tests/test_csv.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index 1f71766a6757..cc92ef382ea2 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -612,7 +612,8 @@ cdef class ParseOptions(_Weakrefable): f"double_quote={self.double_quote}, " f"escape_char={self.escape_char!r}, " f"newlines_in_values={self.newlines_in_values}, " - f"ignore_empty_lines={self.ignore_empty_lines})>") + f"ignore_empty_lines={self.ignore_empty_lines}, " + f"invalid_row_handler={self.invalid_row_handler})>") def __str__(self): return (f"ParseOptions(" @@ -621,7 +622,8 @@ cdef class ParseOptions(_Weakrefable): f"double_quote={self.double_quote}, " f"escape_char={self.escape_char!r}, " f"newlines_in_values={self.newlines_in_values}, " - f"ignore_empty_lines={self.ignore_empty_lines})") + f"ignore_empty_lines={self.ignore_empty_lines}, " + f"invalid_row_handler={self.invalid_row_handler})") cdef class _ISO8601(_Weakrefable): diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index e93d7ba76657..149407215f83 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -299,7 +299,8 @@ def test_parse_options(pickle_module): f"double_quote={opts.double_quote}, " f"escape_char={opts.escape_char!r}, " f"newlines_in_values={opts.newlines_in_values}, " - f"ignore_empty_lines={opts.ignore_empty_lines})>") + f"ignore_empty_lines={opts.ignore_empty_lines}, " + f"invalid_row_handler={opts.invalid_row_handler})>") assert repr(opts) == expected_repr expected_str = (f"ParseOptions(" @@ -308,7 +309,8 @@ def test_parse_options(pickle_module): f"double_quote={opts.double_quote}, " f"escape_char={opts.escape_char!r}, " f"newlines_in_values={opts.newlines_in_values}, " - f"ignore_empty_lines={opts.ignore_empty_lines})") + f"ignore_empty_lines={opts.ignore_empty_lines}, " + f"invalid_row_handler={opts.invalid_row_handler})") assert str(opts) == expected_str From ce20cdd630cdc03d6d4132eb37283dd31ac9fa0e Mon Sep 17 00:00:00 2001 From: AlenkaF Date: Thu, 23 Oct 2025 15:52:53 +0200 Subject: [PATCH 05/11] Update ConvertOptions --- python/pyarrow/_csv.pyx | 46 ++++++++++++------------------- python/pyarrow/tests/test_csv.py | 47 +++++++++++--------------------- 2 files changed, 34 insertions(+), 59 deletions(-) diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index cc92ef382ea2..598711e62b07 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -1148,37 +1148,27 @@ cdef class ConvertOptions(_Weakrefable): except TypeError: return False + def _repr_base(self): + return (f""" +check_utf8={self.check_utf8} +column_types={self.column_types} +null_values={self.null_values} +true_values={self.true_values} +false_values={self.false_values} +decimal_point={self.decimal_point!r} +strings_can_be_null={self.strings_can_be_null} +quoted_strings_can_be_null={self.quoted_strings_can_be_null} +include_columns={self.include_columns} +include_missing_columns={self.include_missing_columns} +auto_dict_encode={self.auto_dict_encode} +auto_dict_max_cardinality={self.auto_dict_max_cardinality} +timestamp_parsers={[str(i) for i in self.timestamp_parsers]}""") + def __repr__(self): - return (f"") + return (f"({self._repr_base()})") def __str__(self): - return (f"ConvertOptions(" - f"check_utf8={self.check_utf8}, " - f"column_types={self.column_types}, " - f"null_values={self.null_values}, " - f"true_values={self.true_values}, " - f"false_values={self.false_values}, " - f"decimal_point={self.decimal_point!r}, " - f"strings_can_be_null={self.strings_can_be_null}, " - f"quoted_strings_can_be_null={self.quoted_strings_can_be_null}, " - f"include_columns={self.include_columns}, " - f"include_missing_columns={self.include_missing_columns}, " - f"auto_dict_encode={self.auto_dict_encode}, " - f"auto_dict_max_cardinality={self.auto_dict_max_cardinality}, " - f"timestamp_parsers={self.timestamp_parsers})") + return (f"ConvertOptions({self._repr_base()})") cdef _get_reader(input_file, ReadOptions read_options, diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index 149407215f83..22f75f2c9c38 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -394,37 +394,22 @@ def test_convert_options(pickle_module): assert opts.auto_dict_max_cardinality == 999 assert opts.timestamp_parsers == [ISO8601, '%Y-%m-%d'] - expected_repr = ("") - assert repr(opts) == expected_repr - - expected_str = (f"ConvertOptions(" - f"check_utf8={opts.check_utf8}, " - f"column_types={opts.column_types}, " - f"null_values={opts.null_values}, " - f"true_values={opts.true_values}, " - f"false_values={opts.false_values}, " - f"decimal_point={opts.decimal_point!r}, " - f"strings_can_be_null={opts.strings_can_be_null}, " - f"quoted_strings_can_be_null={opts.quoted_strings_can_be_null}, " - f"include_columns={opts.include_columns}, " - f"include_missing_columns={opts.include_missing_columns}, " - f"auto_dict_encode={opts.auto_dict_encode}, " - f"auto_dict_max_cardinality={opts.auto_dict_max_cardinality}, " - f"timestamp_parsers={opts.timestamp_parsers})") - assert str(opts) == expected_str + expected_repr_inner = (""" +check_utf8=True +column_types={'a': DataType(null)} +null_values=['N', 'nn'] +true_values=['T', 'tt'] +false_values=['F', 'ff'] +decimal_point='.' +strings_can_be_null=False +quoted_strings_can_be_null=True +include_columns=[] +include_missing_columns=False +auto_dict_encode=False +auto_dict_max_cardinality=999 +timestamp_parsers=['ISO8601', '%Y-%m-%d']""") + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"ConvertOptions({expected_repr_inner})" def test_write_options(): From 3487bf996d968c1bc24d1ddb7fd6dfd3a8071fca Mon Sep 17 00:00:00 2001 From: AlenkaF Date: Thu, 23 Oct 2025 16:59:13 +0200 Subject: [PATCH 06/11] Update ParseOptions --- python/pyarrow/_csv.pyx | 28 ++++++++++++---------------- python/pyarrow/tests/test_csv.py | 30 +++++++++++------------------- 2 files changed, 23 insertions(+), 35 deletions(-) diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index 598711e62b07..aa65f62cc9cd 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -605,25 +605,21 @@ cdef class ParseOptions(_Weakrefable): except TypeError: return False + def _repr_base(self): + return (f""" +delimiter={self.delimiter!r} +quote_char={self.quote_char!r} +double_quote={self.double_quote} +escape_char={self.escape_char!r} +newlines_in_values={self.newlines_in_values} +ignore_empty_lines={self.ignore_empty_lines} +invalid_row_handler={getattr(self.invalid_row_handler, '__name__', self.invalid_row_handler)}""") + def __repr__(self): - return (f"") + return (f"({self._repr_base()})") def __str__(self): - return (f"ParseOptions(" - f"delimiter={self.delimiter!r}, " - f"quote_char={self.quote_char!r}, " - f"double_quote={self.double_quote}, " - f"escape_char={self.escape_char!r}, " - f"newlines_in_values={self.newlines_in_values}, " - f"ignore_empty_lines={self.ignore_empty_lines}, " - f"invalid_row_handler={self.invalid_row_handler})") + return (f"ParseOptions({self._repr_base()})") cdef class _ISO8601(_Weakrefable): diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index 22f75f2c9c38..c5001169a495 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -293,25 +293,17 @@ def test_parse_options(pickle_module): opts.escape_char = "\r" opts.validate() - expected_repr = ("") - assert repr(opts) == expected_repr - - expected_str = (f"ParseOptions(" - f"delimiter={opts.delimiter!r}, " - f"quote_char={opts.quote_char!r}, " - f"double_quote={opts.double_quote}, " - f"escape_char={opts.escape_char!r}, " - f"newlines_in_values={opts.newlines_in_values}, " - f"ignore_empty_lines={opts.ignore_empty_lines}, " - f"invalid_row_handler={opts.invalid_row_handler})") - assert str(opts) == expected_str + expected_repr_inner = r""" +delimiter=',' +quote_char='"' +double_quote=True +escape_char='\r' +newlines_in_values=False +ignore_empty_lines=True +invalid_row_handler=None""" + + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"ParseOptions({expected_repr_inner})" def test_convert_options(pickle_module): From cd8bb548f1060495c17d4b00de8b73c493fcd419 Mon Sep 17 00:00:00 2001 From: AlenkaF Date: Thu, 23 Oct 2025 17:10:55 +0200 Subject: [PATCH 07/11] Update WriteOptions --- python/pyarrow/_csv.pyx | 22 +++++++++++----------- python/pyarrow/tests/test_csv.py | 19 +++++++------------ 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index aa65f62cc9cd..b6ea7693b8a0 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -613,7 +613,8 @@ double_quote={self.double_quote} escape_char={self.escape_char!r} newlines_in_values={self.newlines_in_values} ignore_empty_lines={self.ignore_empty_lines} -invalid_row_handler={getattr(self.invalid_row_handler, '__name__', self.invalid_row_handler)}""") +invalid_row_handler={getattr(self.invalid_row_handler, '__name__', + self.invalid_row_handler)}""") def __repr__(self): return (f"({self._repr_base()})") @@ -1517,19 +1518,18 @@ cdef class WriteOptions(_Weakrefable): def validate(self): check_status(self.options.get().Validate()) + def _repr_base(self): + return (f""" +include_header={self.include_header} +batch_size={self.batch_size} +delimiter={self.delimiter!r} +quoting_style='{self.quoting_style}'""") + def __repr__(self): - return (f"") + return (f"({self._repr_base()})") def __str__(self): - return (f"WriteOptions(" - f"include_header={self.include_header}, " - f"batch_size={self.batch_size}, " - f"delimiter={self.delimiter!r}, " - f"quoting_style='{self.quoting_style}')") + return (f"WriteOptions({self._repr_base()})") cdef _get_write_options(WriteOptions write_options, CCSVWriteOptions* out): diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index c5001169a495..99106df103a3 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -427,19 +427,14 @@ def test_write_options(): opts.batch_size = 0 opts.validate() - expected_repr = ("") - assert repr(opts) == expected_repr + expected_repr_inner = """ +include_header=True +batch_size=0 +delimiter=',' +quoting_style='needed'""" - expected_str = (f"WriteOptions(" - f"include_header={opts.include_header}, " - f"batch_size={opts.batch_size}, " - f"delimiter={opts.delimiter!r}, " - f"quoting_style='{opts.quoting_style}')") - assert str(opts) == expected_str + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"WriteOptions({expected_repr_inner})" class BaseTestCSV(abc.ABC): From 9dee61c38d5c7b7d86e5f75ad89bf157b589d300 Mon Sep 17 00:00:00 2001 From: AlenkaF Date: Thu, 23 Oct 2025 18:35:12 +0200 Subject: [PATCH 08/11] Update json options --- python/pyarrow/_json.pyx | 28 ++++++++++++++-------------- python/pyarrow/tests/test_json.py | 31 ++++++++++++------------------- 2 files changed, 26 insertions(+), 33 deletions(-) diff --git a/python/pyarrow/_json.pyx b/python/pyarrow/_json.pyx index 4577df49bbf1..1cda41fb39dc 100644 --- a/python/pyarrow/_json.pyx +++ b/python/pyarrow/_json.pyx @@ -106,14 +106,14 @@ cdef class ReadOptions(_Weakrefable): return False def __repr__(self): - return (f"") + return (f"""( +use_threads={self.use_threads} +block_size={self.block_size})""") def __str__(self): - return (f"ReadOptions(" - f"use_threads={self.use_threads}, " - f"block_size={self.block_size})") + return (f"""ReadOptions( +use_threads={self.use_threads} +block_size={self.block_size})""") @staticmethod cdef ReadOptions wrap(CJSONReadOptions options): @@ -254,17 +254,17 @@ cdef class ParseOptions(_Weakrefable): except TypeError: return False + def _repr_base(self): + return (f""" +explicit_schema={self.explicit_schema} +newlines_in_values={self.newlines_in_values} +unexpected_field_behavior='{self.unexpected_field_behavior}'""") + def __repr__(self): - return (f"") + return (f"({self._repr_base()})") def __str__(self): - return (f"ParseOptions(" - f"explicit_schema={self.explicit_schema}, " - f"newlines_in_values={self.newlines_in_values}, " - f"unexpected_field_behavior='{self.unexpected_field_behavior}')") + return (f"ParseOptions({self._repr_base()})") @staticmethod cdef ParseOptions wrap(CJSONParseOptions options): diff --git a/python/pyarrow/tests/test_json.py b/python/pyarrow/tests/test_json.py index f2261836888a..e6247203d315 100644 --- a/python/pyarrow/tests/test_json.py +++ b/python/pyarrow/tests/test_json.py @@ -80,15 +80,12 @@ def test_read_options(pickle_module): assert opts.block_size == 1234 assert opts.use_threads is False - expected_repr = ("") - assert repr(opts) == expected_repr + expected_repr_inner = """ +use_threads=False +block_size=1234""" - expected_str = (f"ReadOptions(" - f"use_threads={opts.use_threads}, " - f"block_size={opts.block_size})") - assert str(opts) == expected_str + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"ReadOptions({expected_repr_inner})" check_options_class_pickling(cls, pickler=pickle_module, block_size=1234, @@ -104,17 +101,13 @@ def test_parse_options(pickle_module): opts.newlines_in_values = True assert opts.newlines_in_values is True - expected_repr = ("") - assert repr(opts) == expected_repr - - expected_str = (f"ParseOptions(" - f"explicit_schema={opts.explicit_schema}, " - f"newlines_in_values={opts.newlines_in_values}, " - f"unexpected_field_behavior='{opts.unexpected_field_behavior}')") - assert str(opts) == expected_str + expected_repr_inner = """ +explicit_schema=None +newlines_in_values=True +unexpected_field_behavior='infer'""" + + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"ParseOptions({expected_repr_inner})" schema = pa.schema([pa.field('foo', pa.int32())]) opts.explicit_schema = schema From 326185b420ad3029d1f239158c728e9de4dbfb4d Mon Sep 17 00:00:00 2001 From: AlenkaF Date: Thu, 23 Oct 2025 19:27:12 +0200 Subject: [PATCH 09/11] Update ReadOptions --- python/pyarrow/_csv.pyx | 28 ++++++++++++---------------- python/pyarrow/tests/test_csv.py | 31 ++++++++++++------------------- 2 files changed, 24 insertions(+), 35 deletions(-) diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index b6ea7693b8a0..7736a7ae7033 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -332,25 +332,21 @@ cdef class ReadOptions(_Weakrefable): except TypeError: return False + def _repr_base(self): + return (f""" +use_threads={self.use_threads} +block_size={self.block_size} +skip_rows={self.skip_rows} +skip_rows_after_names={self.skip_rows_after_names} +column_names={self.column_names} +autogenerate_column_names={self.autogenerate_column_names} +encoding='{self.encoding}'""") + def __repr__(self): - return (f"") + return (f"({self._repr_base()})") def __str__(self): - return (f"ReadOptions(" - f"use_threads={self.use_threads}, " - f"block_size={self.block_size}, " - f"skip_rows={self.skip_rows}, " - f"skip_rows_after_names={self.skip_rows_after_names}, " - f"column_names={self.column_names}, " - f"autogenerate_column_names={self.autogenerate_column_names}, " - f"encoding='{self.encoding}')") + return (f"ReadOptions({self._repr_base()})") cdef class ParseOptions(_Weakrefable): diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index 99106df103a3..918a49c4796e 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -213,25 +213,18 @@ def test_read_options(pickle_module): opts.column_names = ('a', 'b') opts.validate() - expected_repr = ("") - assert repr(opts) == expected_repr - - expected_str = (f"ReadOptions(" - f"use_threads={opts.use_threads}, " - f"block_size={opts.block_size}, " - f"skip_rows={opts.skip_rows}, " - f"skip_rows_after_names={opts.skip_rows_after_names}, " - f"column_names={opts.column_names}, " - f"autogenerate_column_names={opts.autogenerate_column_names}, " - f"encoding='{opts.encoding}')") - assert str(opts) == expected_str + expected_repr_inner = """ +use_threads=True +block_size=1048576 +skip_rows=0 +skip_rows_after_names=0 +column_names=['a', 'b'] +autogenerate_column_names=True +encoding='utf8'""" + + assert repr(opts) == f"({expected_repr_inner})" + assert str(opts) == f"ReadOptions({expected_repr_inner})" + def test_parse_options(pickle_module): From 19192c968730b893b3cd98136d317df2aecc50a9 Mon Sep 17 00:00:00 2001 From: AlenkaF Date: Fri, 24 Oct 2025 06:00:43 +0200 Subject: [PATCH 10/11] Linter fix --- python/pyarrow/tests/test_csv.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index 918a49c4796e..d811783673a9 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -226,7 +226,6 @@ def test_read_options(pickle_module): assert str(opts) == f"ReadOptions({expected_repr_inner})" - def test_parse_options(pickle_module): cls = ParseOptions skip_handler = InvalidRowHandler('skip') From 552771e5a30f58af28d0002aa0592d2d48bee9d1 Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Wed, 11 Mar 2026 04:45:26 +0000 Subject: [PATCH 11/11] Fix indentation --- python/pyarrow/_csv.pyx | 64 +++++++++++++++---------------- python/pyarrow/_json.pyx | 14 +++---- python/pyarrow/tests/test_csv.py | 62 +++++++++++++++--------------- python/pyarrow/tests/test_json.py | 10 ++--- 4 files changed, 75 insertions(+), 75 deletions(-) diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index 7736a7ae7033..79985530af17 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -334,13 +334,13 @@ cdef class ReadOptions(_Weakrefable): def _repr_base(self): return (f""" -use_threads={self.use_threads} -block_size={self.block_size} -skip_rows={self.skip_rows} -skip_rows_after_names={self.skip_rows_after_names} -column_names={self.column_names} -autogenerate_column_names={self.autogenerate_column_names} -encoding='{self.encoding}'""") + use_threads={self.use_threads}, + block_size={self.block_size}, + skip_rows={self.skip_rows}, + skip_rows_after_names={self.skip_rows_after_names}, + column_names={self.column_names}, + autogenerate_column_names={self.autogenerate_column_names}, + encoding={self.encoding!r}""") def __repr__(self): return (f"({self._repr_base()})") @@ -603,14 +603,14 @@ cdef class ParseOptions(_Weakrefable): def _repr_base(self): return (f""" -delimiter={self.delimiter!r} -quote_char={self.quote_char!r} -double_quote={self.double_quote} -escape_char={self.escape_char!r} -newlines_in_values={self.newlines_in_values} -ignore_empty_lines={self.ignore_empty_lines} -invalid_row_handler={getattr(self.invalid_row_handler, '__name__', - self.invalid_row_handler)}""") + delimiter={self.delimiter!r}, + quote_char={self.quote_char!r}, + double_quote={self.double_quote}, + escape_char={self.escape_char!r}, + newlines_in_values={self.newlines_in_values}, + ignore_empty_lines={self.ignore_empty_lines}, + invalid_row_handler={getattr(self.invalid_row_handler, '__name__', + self.invalid_row_handler)}""") def __repr__(self): return (f"({self._repr_base()})") @@ -1143,19 +1143,19 @@ cdef class ConvertOptions(_Weakrefable): def _repr_base(self): return (f""" -check_utf8={self.check_utf8} -column_types={self.column_types} -null_values={self.null_values} -true_values={self.true_values} -false_values={self.false_values} -decimal_point={self.decimal_point!r} -strings_can_be_null={self.strings_can_be_null} -quoted_strings_can_be_null={self.quoted_strings_can_be_null} -include_columns={self.include_columns} -include_missing_columns={self.include_missing_columns} -auto_dict_encode={self.auto_dict_encode} -auto_dict_max_cardinality={self.auto_dict_max_cardinality} -timestamp_parsers={[str(i) for i in self.timestamp_parsers]}""") + check_utf8={self.check_utf8}, + column_types={self.column_types}, + null_values={self.null_values}, + true_values={self.true_values}, + false_values={self.false_values}, + decimal_point={self.decimal_point!r}, + strings_can_be_null={self.strings_can_be_null}, + quoted_strings_can_be_null={self.quoted_strings_can_be_null}, + include_columns={self.include_columns}, + include_missing_columns={self.include_missing_columns}, + auto_dict_encode={self.auto_dict_encode}, + auto_dict_max_cardinality={self.auto_dict_max_cardinality}, + timestamp_parsers={[str(i) for i in self.timestamp_parsers]}""") def __repr__(self): return (f"({self._repr_base()})") @@ -1516,10 +1516,10 @@ cdef class WriteOptions(_Weakrefable): def _repr_base(self): return (f""" -include_header={self.include_header} -batch_size={self.batch_size} -delimiter={self.delimiter!r} -quoting_style='{self.quoting_style}'""") + include_header={self.include_header}, + batch_size={self.batch_size}, + delimiter={self.delimiter!r}, + quoting_style={self.quoting_style!r}""") def __repr__(self): return (f"({self._repr_base()})") diff --git a/python/pyarrow/_json.pyx b/python/pyarrow/_json.pyx index 1cda41fb39dc..f8373feeabe5 100644 --- a/python/pyarrow/_json.pyx +++ b/python/pyarrow/_json.pyx @@ -107,13 +107,13 @@ cdef class ReadOptions(_Weakrefable): def __repr__(self): return (f"""( -use_threads={self.use_threads} -block_size={self.block_size})""") + use_threads={self.use_threads}, + block_size={self.block_size})""") def __str__(self): return (f"""ReadOptions( -use_threads={self.use_threads} -block_size={self.block_size})""") + use_threads={self.use_threads}, + block_size={self.block_size})""") @staticmethod cdef ReadOptions wrap(CJSONReadOptions options): @@ -256,9 +256,9 @@ cdef class ParseOptions(_Weakrefable): def _repr_base(self): return (f""" -explicit_schema={self.explicit_schema} -newlines_in_values={self.newlines_in_values} -unexpected_field_behavior='{self.unexpected_field_behavior}'""") + explicit_schema={self.explicit_schema}, + newlines_in_values={self.newlines_in_values}, + unexpected_field_behavior={self.unexpected_field_behavior!r}""") def __repr__(self): return (f"({self._repr_base()})") diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py index d811783673a9..d608d2bee5eb 100644 --- a/python/pyarrow/tests/test_csv.py +++ b/python/pyarrow/tests/test_csv.py @@ -214,13 +214,13 @@ def test_read_options(pickle_module): opts.validate() expected_repr_inner = """ -use_threads=True -block_size=1048576 -skip_rows=0 -skip_rows_after_names=0 -column_names=['a', 'b'] -autogenerate_column_names=True -encoding='utf8'""" + use_threads=True, + block_size=1048576, + skip_rows=0, + skip_rows_after_names=0, + column_names=['a', 'b'], + autogenerate_column_names=True, + encoding='utf8'""" assert repr(opts) == f"({expected_repr_inner})" assert str(opts) == f"ReadOptions({expected_repr_inner})" @@ -286,13 +286,13 @@ def test_parse_options(pickle_module): opts.validate() expected_repr_inner = r""" -delimiter=',' -quote_char='"' -double_quote=True -escape_char='\r' -newlines_in_values=False -ignore_empty_lines=True -invalid_row_handler=None""" + delimiter=',', + quote_char='"', + double_quote=True, + escape_char='\r', + newlines_in_values=False, + ignore_empty_lines=True, + invalid_row_handler=None""" assert repr(opts) == f"({expected_repr_inner})" assert str(opts) == f"ParseOptions({expected_repr_inner})" @@ -379,19 +379,19 @@ def test_convert_options(pickle_module): assert opts.timestamp_parsers == [ISO8601, '%Y-%m-%d'] expected_repr_inner = (""" -check_utf8=True -column_types={'a': DataType(null)} -null_values=['N', 'nn'] -true_values=['T', 'tt'] -false_values=['F', 'ff'] -decimal_point='.' -strings_can_be_null=False -quoted_strings_can_be_null=True -include_columns=[] -include_missing_columns=False -auto_dict_encode=False -auto_dict_max_cardinality=999 -timestamp_parsers=['ISO8601', '%Y-%m-%d']""") + check_utf8=True, + column_types={'a': DataType(null)}, + null_values=['N', 'nn'], + true_values=['T', 'tt'], + false_values=['F', 'ff'], + decimal_point='.', + strings_can_be_null=False, + quoted_strings_can_be_null=True, + include_columns=[], + include_missing_columns=False, + auto_dict_encode=False, + auto_dict_max_cardinality=999, + timestamp_parsers=['ISO8601', '%Y-%m-%d']""") assert repr(opts) == f"({expected_repr_inner})" assert str(opts) == f"ConvertOptions({expected_repr_inner})" @@ -420,10 +420,10 @@ def test_write_options(): opts.validate() expected_repr_inner = """ -include_header=True -batch_size=0 -delimiter=',' -quoting_style='needed'""" + include_header=True, + batch_size=0, + delimiter=',', + quoting_style='needed'""" assert repr(opts) == f"({expected_repr_inner})" assert str(opts) == f"WriteOptions({expected_repr_inner})" diff --git a/python/pyarrow/tests/test_json.py b/python/pyarrow/tests/test_json.py index e6247203d315..8d5e6f43db06 100644 --- a/python/pyarrow/tests/test_json.py +++ b/python/pyarrow/tests/test_json.py @@ -81,8 +81,8 @@ def test_read_options(pickle_module): assert opts.use_threads is False expected_repr_inner = """ -use_threads=False -block_size=1234""" + use_threads=False, + block_size=1234""" assert repr(opts) == f"({expected_repr_inner})" assert str(opts) == f"ReadOptions({expected_repr_inner})" @@ -102,9 +102,9 @@ def test_parse_options(pickle_module): assert opts.newlines_in_values is True expected_repr_inner = """ -explicit_schema=None -newlines_in_values=True -unexpected_field_behavior='infer'""" + explicit_schema=None, + newlines_in_values=True, + unexpected_field_behavior='infer'""" assert repr(opts) == f"({expected_repr_inner})" assert str(opts) == f"ParseOptions({expected_repr_inner})"