diff --git a/extension/httpfs/include/http_filesystem.h b/extension/httpfs/include/http_filesystem.h index 4e429d16e..28ea4e720 100644 --- a/extension/httpfs/include/http_filesystem.h +++ b/extension/httpfs/include/http_filesystem.h @@ -27,8 +27,8 @@ #include #include "neug/compiler/common/case_insensitive_map.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/file_sys/file_system.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/vfs/file_system.h" namespace neug { namespace extension { diff --git a/extension/httpfs/include/s3_filesystem.h b/extension/httpfs/include/s3_filesystem.h index dd0d46de4..56b167c27 100644 --- a/extension/httpfs/include/s3_filesystem.h +++ b/extension/httpfs/include/s3_filesystem.h @@ -24,8 +24,8 @@ #include #include "glob_utils.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/file_sys/file_system.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/vfs/file_system.h" namespace neug { namespace extension { diff --git a/extension/httpfs/include/s3_options.h b/extension/httpfs/include/s3_options.h index c4b2226ae..1907f93ff 100644 --- a/extension/httpfs/include/s3_options.h +++ b/extension/httpfs/include/s3_options.h @@ -20,8 +20,8 @@ #include #include #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/options.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/read/common/schema.h" namespace neug { namespace extension { diff --git a/extension/httpfs/src/http_filesystem.cc b/extension/httpfs/src/http_filesystem.cc index 5d83b43e1..48a246d93 100644 --- a/extension/httpfs/src/http_filesystem.cc +++ b/extension/httpfs/src/http_filesystem.cc @@ -29,7 +29,7 @@ #include #include #include -#include "neug/utils/file_sys/file_system.h" +#include "neug/utils/io/vfs/file_system.h" namespace neug { namespace extension { diff --git a/extension/httpfs/src/s3_extension.cpp b/extension/httpfs/src/s3_extension.cpp index e4a954717..d9e5b9a9d 100644 --- a/extension/httpfs/src/s3_extension.cpp +++ b/extension/httpfs/src/s3_extension.cpp @@ -21,7 +21,7 @@ #include #include "neug/compiler/main/metadata_registry.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/file_sys/file_system.h" +#include "neug/utils/io/vfs/file_system.h" #include "s3_filesystem.h" #include "http_filesystem.h" #include "s3_options.h" diff --git a/extension/httpfs/src/s3_filesystem.cc b/extension/httpfs/src/s3_filesystem.cc index ba361410f..7a4a46ea1 100644 --- a/extension/httpfs/src/s3_filesystem.cc +++ b/extension/httpfs/src/s3_filesystem.cc @@ -20,7 +20,7 @@ #include #include #include "neug/utils/exception/exception.h" -#include "neug/utils/file_sys/file_system.h" +#include "neug/utils/io/vfs/file_system.h" namespace { diff --git a/extension/httpfs/src/s3_options.cc b/extension/httpfs/src/s3_options.cc index 13c90b9f4..e5713455f 100644 --- a/extension/httpfs/src/s3_options.cc +++ b/extension/httpfs/src/s3_options.cc @@ -26,7 +26,7 @@ #include #include #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/options.h" +#include "neug/utils/io/read/common/options.h" namespace neug { namespace extension { diff --git a/extension/httpfs/tests/s3_test.cpp b/extension/httpfs/tests/s3_test.cpp index 0b65ca330..fed8d878f 100644 --- a/extension/httpfs/tests/s3_test.cpp +++ b/extension/httpfs/tests/s3_test.cpp @@ -6,8 +6,8 @@ #include #include "s3_filesystem.h" #include "glob_utils.h" -#include "neug/utils/reader/schema.h" -#include "neug/utils/reader/options.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/read/common/options.h" #include "neug/utils/exception/exception.h" #include #include diff --git a/extension/parquet/include/parquet/arrow_options.h b/extension/parquet/include/parquet/arrow_options.h index abc1714c9..cdbad7af0 100644 --- a/extension/parquet/include/parquet/arrow_options.h +++ b/extension/parquet/include/parquet/arrow_options.h @@ -19,8 +19,8 @@ #include #include -#include "neug/utils/reader/options.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/read/common/schema.h" namespace neug { namespace reader { diff --git a/extension/parquet/include/parquet/arrow_reader.h b/extension/parquet/include/parquet/arrow_reader.h index 3f378087d..45666e471 100644 --- a/extension/parquet/include/parquet/arrow_reader.h +++ b/extension/parquet/include/parquet/arrow_reader.h @@ -21,7 +21,7 @@ #include #include -#include "neug/utils/reader/reader.h" +#include "neug/utils/io/reader.h" #include "parquet/arrow_options.h" namespace neug { diff --git a/extension/parquet/include/parquet/arrow_sniffer.h b/extension/parquet/include/parquet/arrow_sniffer.h index e696e2353..973fdfe9c 100644 --- a/extension/parquet/include/parquet/arrow_sniffer.h +++ b/extension/parquet/include/parquet/arrow_sniffer.h @@ -16,7 +16,7 @@ #include -#include "neug/utils/reader/sniffer.h" +#include "neug/utils/io/read/common/sniffer.h" #include "parquet/arrow_reader.h" namespace neug { diff --git a/extension/parquet/include/parquet/arrow_type_converter.h b/extension/parquet/include/parquet/arrow_type_converter.h index f8b169e51..f0b98646c 100644 --- a/extension/parquet/include/parquet/arrow_type_converter.h +++ b/extension/parquet/include/parquet/arrow_type_converter.h @@ -17,7 +17,7 @@ #include #include "neug/generated/proto/plan/basic_type.pb.h" -#include "neug/utils/reader/type_converter.h" +#include "neug/utils/io/read/common/type_converter.h" namespace arrow { class DataType; diff --git a/extension/parquet/include/parquet_export_function.h b/extension/parquet/include/parquet_export_function.h index 44bbd946d..b5b22e476 100644 --- a/extension/parquet/include/parquet_export_function.h +++ b/extension/parquet/include/parquet_export_function.h @@ -23,7 +23,7 @@ #include #include "neug/compiler/function/export/export_function.h" -#include "neug/utils/writer/writer.h" +#include "neug/utils/io/write/writer.h" namespace neug { namespace writer { diff --git a/extension/parquet/include/parquet_read_function.h b/extension/parquet/include/parquet_read_function.h index aca9f33ee..43177547b 100644 --- a/extension/parquet/include/parquet_read_function.h +++ b/extension/parquet/include/parquet_read_function.h @@ -21,8 +21,8 @@ #include "neug/compiler/function/read_function.h" #include "neug/compiler/main/metadata_registry.h" #include "neug/execution/execute/ops/batch/batch_update_utils.h" -#include "neug/utils/reader/schema.h" -#include "neug/utils/reader/sniffer.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/read/common/sniffer.h" #include "parquet/arrow_fs_resolver.h" #include "parquet/arrow_reader.h" #include "parquet/arrow_sniffer.h" diff --git a/extension/parquet/src/arrow_fs_resolver.cc b/extension/parquet/src/arrow_fs_resolver.cc index 8cd294413..a63a2230e 100644 --- a/extension/parquet/src/arrow_fs_resolver.cc +++ b/extension/parquet/src/arrow_fs_resolver.cc @@ -18,7 +18,7 @@ #include #include -#include "neug/utils/file_sys/file_system.h" +#include "neug/utils/io/vfs/file_system.h" namespace neug { namespace parquet { diff --git a/extension/parquet/src/arrow_options.cc b/extension/parquet/src/arrow_options.cc index 9af9acb48..a87e14ea3 100644 --- a/extension/parquet/src/arrow_options.cc +++ b/extension/parquet/src/arrow_options.cc @@ -21,7 +21,7 @@ #include #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/reader.h" +#include "neug/utils/io/reader.h" #include "parquet/arrow_type_converter.h" #include "parquet/expression_converter.h" diff --git a/extension/parquet/src/arrow_reader.cc b/extension/parquet/src/arrow_reader.cc index d23dea314..b82ea6c7b 100644 --- a/extension/parquet/src/arrow_reader.cc +++ b/extension/parquet/src/arrow_reader.cc @@ -27,7 +27,7 @@ #include "neug/execution/common/context.h" #include "neug/storages/loader/loader_utils.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/options.h" +#include "neug/utils/io/read/common/options.h" namespace neug { namespace reader { diff --git a/extension/parquet/src/arrow_sniffer.cc b/extension/parquet/src/arrow_sniffer.cc index 12b9d2eea..84ae5b57e 100644 --- a/extension/parquet/src/arrow_sniffer.cc +++ b/extension/parquet/src/arrow_sniffer.cc @@ -18,7 +18,7 @@ #include #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/schema.h" #include "parquet/arrow_type_converter.h" #include "neug/utils/result.h" diff --git a/extension/parquet/src/parquet_export_function.cc b/extension/parquet/src/parquet_export_function.cc index a2dd11433..e807baf24 100644 --- a/extension/parquet/src/parquet_export_function.cc +++ b/extension/parquet/src/parquet_export_function.cc @@ -31,7 +31,7 @@ #include "neug/compiler/main/metadata_registry.h" #include "neug/utils/exception/exception.h" #include "neug/utils/property/types.h" -#include "neug/utils/writer/writer.h" +#include "neug/utils/io/write/writer.h" #include "parquet/arrow_fs_resolver.h" #include "parquet_options.h" diff --git a/extension/parquet/src/parquet_options.cc b/extension/parquet/src/parquet_options.cc index 2c54f559d..d1d978be9 100644 --- a/extension/parquet/src/parquet_options.cc +++ b/extension/parquet/src/parquet_options.cc @@ -25,9 +25,9 @@ #include #include #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/options.h" -#include "neug/utils/reader/reader.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/reader.h" +#include "neug/utils/io/read/common/schema.h" namespace neug { namespace reader { diff --git a/extension/parquet/tests/parquet_test.cpp b/extension/parquet/tests/parquet_test.cpp index bacfd93ae..7be843f7c 100644 --- a/extension/parquet/tests/parquet_test.cpp +++ b/extension/parquet/tests/parquet_test.cpp @@ -30,9 +30,9 @@ #include "neug/execution/common/context.h" #include "neug/generated/proto/plan/basic_type.pb.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/options.h" +#include "neug/utils/io/read/common/options.h" #include "parquet/arrow_reader.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/schema.h" #include "../../extension/parquet/include/parquet_options.h" #include "../../extension/parquet/include/parquet_export_function.h" diff --git a/include/neug/compiler/function/export/export_function.h b/include/neug/compiler/function/export/export_function.h index 875c831c6..bc2b26373 100644 --- a/include/neug/compiler/function/export/export_function.h +++ b/include/neug/compiler/function/export/export_function.h @@ -27,7 +27,7 @@ #include "neug/compiler/function/function.h" #include "neug/execution/common/context.h" #include "neug/storages/graph/graph_interface.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/schema.h" namespace neug { namespace function { diff --git a/include/neug/compiler/function/export/json_export_function.h b/include/neug/compiler/function/export/json_export_function.h index 6c7030a58..3f2495f48 100644 --- a/include/neug/compiler/function/export/json_export_function.h +++ b/include/neug/compiler/function/export/json_export_function.h @@ -19,8 +19,8 @@ #include #include "neug/compiler/function/export/export_function.h" +#include "neug/utils/io/write/writer.h" #include "neug/utils/result.h" -#include "neug/utils/writer/writer.h" #include "rapidjson/document.h" namespace neug { diff --git a/include/neug/compiler/function/import/csv_read_function.h b/include/neug/compiler/function/import/csv_read_function.h index a274c5837..9a55756f2 100644 --- a/include/neug/compiler/function/import/csv_read_function.h +++ b/include/neug/compiler/function/import/csv_read_function.h @@ -22,10 +22,10 @@ #include "neug/compiler/main/metadata_registry.h" #include "neug/execution/execute/ops/batch/batch_update_utils.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/options.h" -#include "neug/utils/reader/reader.h" -#include "neug/utils/reader/schema.h" -#include "neug/utils/reader/sniffer.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/read/common/sniffer.h" +#include "neug/utils/io/reader.h" namespace neug { namespace function { struct CSVReadFunction { diff --git a/include/neug/compiler/function/import/json_read_function.h b/include/neug/compiler/function/import/json_read_function.h index 0ae10a18e..5b2e8793c 100644 --- a/include/neug/compiler/function/import/json_read_function.h +++ b/include/neug/compiler/function/import/json_read_function.h @@ -21,10 +21,10 @@ #include "neug/compiler/function/read_function.h" #include "neug/compiler/main/metadata_registry.h" #include "neug/execution/execute/ops/batch/batch_update_utils.h" -#include "neug/utils/reader/options.h" -#include "neug/utils/reader/reader.h" -#include "neug/utils/reader/schema.h" -#include "neug/utils/reader/sniffer.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/read/common/sniffer.h" +#include "neug/utils/io/reader.h" namespace neug { namespace function { diff --git a/include/neug/compiler/function/read_function.h b/include/neug/compiler/function/read_function.h index 3c43665c5..cfc8c74e1 100644 --- a/include/neug/compiler/function/read_function.h +++ b/include/neug/compiler/function/read_function.h @@ -23,8 +23,8 @@ #include "neug/compiler/function/table/table_function.h" #include "neug/execution/common/context.h" #include "neug/execution/execute/ops/batch/batch_update_utils.h" -#include "neug/utils/reader/reader.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/reader.h" namespace neug { namespace function { diff --git a/include/neug/compiler/main/metadata_manager.h b/include/neug/compiler/main/metadata_manager.h index b6d697219..680e7be2a 100644 --- a/include/neug/compiler/main/metadata_manager.h +++ b/include/neug/compiler/main/metadata_manager.h @@ -32,7 +32,7 @@ #include "neug/compiler/main/option_config.h" #include "neug/compiler/storage/buffer_manager/memory_manager.h" #include "neug/utils/api.h" -#include "neug/utils/file_sys/file_system.h" +#include "neug/utils/io/vfs/file_system.h" namespace neug { namespace common { diff --git a/include/neug/execution/execute/ops/batch/data_source.h b/include/neug/execution/execute/ops/batch/data_source.h index eaf5462a1..09853e97c 100644 --- a/include/neug/execution/execute/ops/batch/data_source.h +++ b/include/neug/execution/execute/ops/batch/data_source.h @@ -16,7 +16,7 @@ #include "neug/execution/execute/operator.h" #include "neug/execution/execute/ops/batch/batch_update_utils.h" -#include "neug/utils/reader/reader.h" +#include "neug/utils/io/reader.h" namespace neug { using namespace reader; diff --git a/include/neug/storages/allocators.h b/include/neug/storages/allocators.h index ba1d7477f..ea00dc2b3 100644 --- a/include/neug/storages/allocators.h +++ b/include/neug/storages/allocators.h @@ -27,7 +27,7 @@ #include "neug/storages/container/container_utils.h" #include "neug/storages/container/file_header.h" #include "neug/storages/container/mmap_container.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include "neug/utils/property/types.h" namespace neug { diff --git a/include/neug/storages/csr/mutable_csr.h b/include/neug/storages/csr/mutable_csr.h index 05d4aefe7..a1779154b 100644 --- a/include/neug/storages/csr/mutable_csr.h +++ b/include/neug/storages/csr/mutable_csr.h @@ -37,7 +37,7 @@ #include "neug/storages/csr/csr_view.h" #include "neug/storages/csr/nbr.h" #include "neug/storages/module/type_name.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include "neug/utils/property/types.h" #include "neug/utils/spinlock.h" diff --git a/include/neug/storages/loader/loader_utils.h b/include/neug/storages/loader/loader_utils.h index b5f577857..177554b68 100644 --- a/include/neug/storages/loader/loader_utils.h +++ b/include/neug/storages/loader/loader_utils.h @@ -25,9 +25,9 @@ #include #include "neug/execution/common/data_chunk.h" -#include "neug/storages/loader/csv_read_config.h" #include "neug/storages/loader/loading_config.h" #include "neug/utils/exception/exception.h" +#include "neug/utils/io/read/csv/csv_read_config.h" #include "neug/utils/string_utils.h" namespace neug { diff --git a/include/neug/utils/file_utils.h b/include/neug/utils/io/file/file_utils.h similarity index 100% rename from include/neug/utils/file_utils.h rename to include/neug/utils/io/file/file_utils.h diff --git a/include/neug/utils/reader/operator_precedence.h b/include/neug/utils/io/read/common/operator_precedence.h similarity index 100% rename from include/neug/utils/reader/operator_precedence.h rename to include/neug/utils/io/read/common/operator_precedence.h diff --git a/include/neug/utils/reader/options.h b/include/neug/utils/io/read/common/options.h similarity index 97% rename from include/neug/utils/reader/options.h rename to include/neug/utils/io/read/common/options.h index dd037fd4d..0ba646491 100644 --- a/include/neug/utils/reader/options.h +++ b/include/neug/utils/io/read/common/options.h @@ -22,10 +22,10 @@ #include #include "neug/compiler/common/case_insensitive_map.h" -#include "neug/storages/loader/csv_read_config.h" -#include "neug/storages/loader/json_read_config.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/read/csv/csv_read_config.h" +#include "neug/utils/io/read/json/json_read_config.h" namespace neug { namespace reader { diff --git a/include/neug/utils/io/read/common/read_state.h b/include/neug/utils/io/read/common/read_state.h new file mode 100644 index 000000000..79eed2da6 --- /dev/null +++ b/include/neug/utils/io/read/common/read_state.h @@ -0,0 +1,71 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "neug/compiler/common/cast.h" +#include "neug/utils/io/read/common/schema.h" + +namespace common { +class Expression; +} // namespace common + +namespace neug { +namespace reader { + +struct ReadLocalState { + virtual ~ReadLocalState() = default; + + template + TARGET& cast() { + return common::neug_dynamic_cast(*this); + } + + template + TARGET* ptrCast() { + return common::neug_dynamic_cast(this); + } + + template + const TARGET& constCast() const { + return common::neug_dynamic_cast(*this); + } + + template + const TARGET* constPtrCast() const { + return common::neug_dynamic_cast(this); + } +}; + +struct ReadSharedState { + ExternalSchema schema; + std::vector projectColumns; + std::shared_ptr<::common::Expression> skipRows; + + int columnNum() { + if (!schema.entry) { + return 0; + } + const auto& allColumns = schema.entry->columnNames; + return projectColumns.empty() ? allColumns.size() : projectColumns.size(); + } +}; + +} // namespace reader +} // namespace neug diff --git a/include/neug/utils/reader/row_expression_filter.h b/include/neug/utils/io/read/common/row_expression_filter.h similarity index 100% rename from include/neug/utils/reader/row_expression_filter.h rename to include/neug/utils/io/read/common/row_expression_filter.h diff --git a/include/neug/utils/reader/schema.h b/include/neug/utils/io/read/common/schema.h similarity index 100% rename from include/neug/utils/reader/schema.h rename to include/neug/utils/io/read/common/schema.h diff --git a/include/neug/utils/reader/sniffer.h b/include/neug/utils/io/read/common/sniffer.h similarity index 91% rename from include/neug/utils/reader/sniffer.h rename to include/neug/utils/io/read/common/sniffer.h index 354bbc65d..43ef37e73 100644 --- a/include/neug/utils/reader/sniffer.h +++ b/include/neug/utils/io/read/common/sniffer.h @@ -17,8 +17,9 @@ #include #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/reader.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/read/csv/csv_reader.h" +#include "neug/utils/io/read/json/json_reader.h" #include "neug/utils/result.h" namespace neug { diff --git a/include/neug/utils/reader/type_converter.h b/include/neug/utils/io/read/common/type_converter.h similarity index 100% rename from include/neug/utils/reader/type_converter.h rename to include/neug/utils/io/read/common/type_converter.h diff --git a/include/neug/storages/loader/csv_read_config.h b/include/neug/utils/io/read/csv/csv_read_config.h similarity index 100% rename from include/neug/storages/loader/csv_read_config.h rename to include/neug/utils/io/read/csv/csv_read_config.h diff --git a/include/neug/utils/io/read/csv/csv_reader.h b/include/neug/utils/io/read/csv/csv_reader.h new file mode 100644 index 000000000..bf4f66d74 --- /dev/null +++ b/include/neug/utils/io/read/csv/csv_reader.h @@ -0,0 +1,61 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "neug/execution/common/context.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/read/common/read_state.h" +#include "neug/utils/io/read/csv/csv_read_config.h" +#include "neug/utils/result.h" + +namespace neug { + +class IDataChunkSupplier; + +namespace execution { +class Context; +} + +namespace reader { + +class CsvReader { + public: + explicit CsvReader(std::shared_ptr sharedState, + std::unique_ptr optionsBuilder); + ~CsvReader(); + + void read(std::shared_ptr localState, + execution::Context& ctx); + + result> inferSchema(); + + private: + void full_read( + const std::vector>& suppliers, + execution::Context& output, const CsvReadConfig& output_config); + void batch_read( + const std::vector>& suppliers, + execution::Context& output); + + std::shared_ptr sharedState_; + std::unique_ptr optionsBuilder_; +}; + +} // namespace reader +} // namespace neug diff --git a/include/neug/storages/loader/json_read_config.h b/include/neug/utils/io/read/json/json_read_config.h similarity index 100% rename from include/neug/storages/loader/json_read_config.h rename to include/neug/utils/io/read/json/json_read_config.h diff --git a/include/neug/utils/io/read/json/json_reader.h b/include/neug/utils/io/read/json/json_reader.h new file mode 100644 index 000000000..b88011352 --- /dev/null +++ b/include/neug/utils/io/read/json/json_reader.h @@ -0,0 +1,61 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "neug/execution/common/context.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/read/common/read_state.h" +#include "neug/utils/io/read/json/json_read_config.h" +#include "neug/utils/result.h" + +namespace neug { + +class IDataChunkSupplier; + +namespace execution { +class Context; +} + +namespace reader { + +class JsonReader { + public: + explicit JsonReader(std::shared_ptr sharedState, + std::unique_ptr optionsBuilder); + ~JsonReader(); + + void read(std::shared_ptr localState, + execution::Context& ctx); + + result> inferSchema(); + + private: + void full_read( + const std::vector>& suppliers, + execution::Context& output, const JsonReadConfig& output_config); + void batch_read( + const std::vector>& suppliers, + execution::Context& output); + + std::shared_ptr sharedState_; + std::unique_ptr optionsBuilder_; +}; + +} // namespace reader +} // namespace neug diff --git a/include/neug/utils/io/reader.h b/include/neug/utils/io/reader.h new file mode 100644 index 000000000..c89a45ffd --- /dev/null +++ b/include/neug/utils/io/reader.h @@ -0,0 +1,20 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "neug/utils/io/read/common/read_state.h" +#include "neug/utils/io/read/csv/csv_reader.h" +#include "neug/utils/io/read/json/json_reader.h" diff --git a/include/neug/utils/io/output_stream.h b/include/neug/utils/io/stream/output_stream.h similarity index 100% rename from include/neug/utils/io/output_stream.h rename to include/neug/utils/io/stream/output_stream.h diff --git a/include/neug/utils/file_sys/file_system.h b/include/neug/utils/io/vfs/file_system.h similarity index 97% rename from include/neug/utils/file_sys/file_system.h rename to include/neug/utils/io/vfs/file_system.h index dd8d4cf44..51b81bc0e 100644 --- a/include/neug/utils/file_sys/file_system.h +++ b/include/neug/utils/io/vfs/file_system.h @@ -23,7 +23,7 @@ #include #include -#include "neug/utils/reader/reader.h" +#include "neug/utils/io/read/common/read_state.h" namespace neug { namespace fsys { diff --git a/include/neug/utils/writer/writer.h b/include/neug/utils/io/write/writer.h similarity index 97% rename from include/neug/utils/writer/writer.h rename to include/neug/utils/io/write/writer.h index fb44efcfe..399bad15f 100644 --- a/include/neug/utils/writer/writer.h +++ b/include/neug/utils/io/write/writer.h @@ -22,9 +22,9 @@ #include "neug/execution/execute/ops/batch/batch_update_utils.h" #include "neug/generated/proto/response/response.pb.h" #include "neug/storages/graph/graph_interface.h" -#include "neug/utils/io/output_stream.h" -#include "neug/utils/reader/options.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/stream/output_stream.h" namespace neug { diff --git a/include/neug/utils/property/column.h b/include/neug/utils/property/column.h index fa53c8c8b..535b231a4 100644 --- a/include/neug/utils/property/column.h +++ b/include/neug/utils/property/column.h @@ -40,7 +40,7 @@ #include "neug/storages/module/module.h" #include "neug/storages/module/type_name.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include "neug/utils/likely.h" #include "neug/utils/property/types.h" #include "neug/utils/serialization/out_archive.h" diff --git a/include/neug/utils/reader/reader.h b/include/neug/utils/reader/reader.h deleted file mode 100644 index 8def59721..000000000 --- a/include/neug/utils/reader/reader.h +++ /dev/null @@ -1,127 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -#include "neug/execution/common/context.h" -#include "neug/storages/loader/csv_read_config.h" -#include "neug/storages/loader/json_read_config.h" -#include "neug/utils/reader/options.h" -#include "neug/utils/reader/schema.h" - -namespace common { -class Expression; -} // namespace common - -namespace neug { - -class IDataChunkSupplier; - -namespace execution { -class Context; -} - -namespace reader { - -struct ReadLocalState { - virtual ~ReadLocalState() = default; - - template - TARGET& cast() { - return common::neug_dynamic_cast(*this); - } - - template - TARGET* ptrCast() { - return common::neug_dynamic_cast(this); - } - - template - const TARGET& constCast() const { - return common::neug_dynamic_cast(*this); - } - - template - const TARGET* constPtrCast() const { - return common::neug_dynamic_cast(this); - } -}; - -struct ReadSharedState { - ExternalSchema schema; - std::vector projectColumns; - std::shared_ptr<::common::Expression> skipRows; - - int columnNum() { - if (!schema.entry) { - return 0; - } - const auto& allColumns = schema.entry->columnNames; - return projectColumns.empty() ? allColumns.size() : projectColumns.size(); - } -}; - -class CsvReader { - public: - explicit CsvReader(std::shared_ptr sharedState, - std::unique_ptr optionsBuilder); - ~CsvReader(); - - void read(std::shared_ptr localState, - execution::Context& ctx); - - result> inferSchema(); - - private: - void full_read( - const std::vector>& suppliers, - execution::Context& output, const CsvReadConfig& output_config); - void batch_read( - const std::vector>& suppliers, - execution::Context& output); - - std::shared_ptr sharedState_; - std::unique_ptr optionsBuilder_; -}; - -class JsonReader { - public: - explicit JsonReader(std::shared_ptr sharedState, - std::unique_ptr optionsBuilder); - ~JsonReader(); - - void read(std::shared_ptr localState, - execution::Context& ctx); - - result> inferSchema(); - - private: - void full_read( - const std::vector>& suppliers, - execution::Context& output, const JsonReadConfig& output_config); - void batch_read( - const std::vector>& suppliers, - execution::Context& output); - - std::shared_ptr sharedState_; - std::unique_ptr optionsBuilder_; -}; - -} // namespace reader -} // namespace neug diff --git a/src/compiler/binder/bind/bind_file_scan.cpp b/src/compiler/binder/bind/bind_file_scan.cpp index cba91e288..708e7a36f 100644 --- a/src/compiler/binder/bind/bind_file_scan.cpp +++ b/src/compiler/binder/bind/bind_file_scan.cpp @@ -35,7 +35,7 @@ #include "neug/compiler/parser/scan_source.h" #include "neug/utils/exception/exception.h" #include "neug/utils/exception/message.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/schema.h" using namespace neug::parser; using namespace neug::binder; diff --git a/src/compiler/function/csv_export_function.cpp b/src/compiler/function/csv_export_function.cpp index bddd176c6..0e044e553 100644 --- a/src/compiler/function/csv_export_function.cpp +++ b/src/compiler/function/csv_export_function.cpp @@ -22,7 +22,7 @@ #include "neug/compiler/function/export/export_function.h" #include "neug/compiler/main/metadata_registry.h" -#include "neug/utils/writer/writer.h" +#include "neug/utils/io/write/writer.h" namespace neug { namespace function { diff --git a/src/compiler/function/json_export_function.cc b/src/compiler/function/json_export_function.cc index 1d460d7b1..c864056c2 100644 --- a/src/compiler/function/json_export_function.cc +++ b/src/compiler/function/json_export_function.cc @@ -16,7 +16,7 @@ #include "neug/compiler/function/export/json_export_function.h" -#include "neug/utils/io/output_stream.h" +#include "neug/utils/io/stream/output_stream.h" #include #include #include @@ -29,7 +29,7 @@ #include "neug/utils/exception/exception.h" #include "neug/utils/property/types.h" #include "neug/utils/result.h" -#include "neug/utils/writer/writer.h" +#include "neug/utils/io/write/writer.h" namespace neug { namespace writer { diff --git a/src/compiler/gopt/g_query_converter.cpp b/src/compiler/gopt/g_query_converter.cpp index 653ca7c4d..e31eab109 100644 --- a/src/compiler/gopt/g_query_converter.cpp +++ b/src/compiler/gopt/g_query_converter.cpp @@ -71,7 +71,7 @@ #include "neug/generated/proto/plan/expr.pb.h" #include "neug/generated/proto/plan/physical.pb.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/schema.h" namespace neug { namespace gopt { diff --git a/src/execution/execute/ops/batch/data_export.cc b/src/execution/execute/ops/batch/data_export.cc index 80d9c0190..225fe9266 100644 --- a/src/execution/execute/ops/batch/data_export.cc +++ b/src/execution/execute/ops/batch/data_export.cc @@ -18,8 +18,8 @@ #include "neug/compiler/main/metadata_registry.h" #include "neug/execution/execute/ops/batch/data_source.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/reader.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/reader.h" +#include "neug/utils/io/read/common/schema.h" namespace neug { namespace execution { diff --git a/src/execution/execute/ops/batch/data_source.cc b/src/execution/execute/ops/batch/data_source.cc index 86d34bf88..abe23668e 100644 --- a/src/execution/execute/ops/batch/data_source.cc +++ b/src/execution/execute/ops/batch/data_source.cc @@ -24,8 +24,8 @@ #include "neug/compiler/main/metadata_registry.h" #include "neug/execution/common/context.h" #include "neug/execution/execute/ops/batch/data_source.h" -#include "neug/utils/reader/reader.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/reader.h" +#include "neug/utils/io/read/common/schema.h" #include "neug/utils/result.h" diff --git a/src/main/neug_db.cc b/src/main/neug_db.cc index 951d9e5df..119a7319e 100644 --- a/src/main/neug_db.cc +++ b/src/main/neug_db.cc @@ -41,7 +41,7 @@ #include "neug/transaction/compact_transaction.h" #include "neug/transaction/wal/wal.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include "neug/utils/result.h" namespace neug { diff --git a/src/storages/container/anon_mmap_container.cc b/src/storages/container/anon_mmap_container.cc index cabedfad7..877f00169 100644 --- a/src/storages/container/anon_mmap_container.cc +++ b/src/storages/container/anon_mmap_container.cc @@ -57,7 +57,7 @@ #include #include "neug/storages/container/anon_mmap_container.h" #include "neug/storages/container/file_header.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" namespace neug { diff --git a/src/storages/container/container_utils.cc b/src/storages/container/container_utils.cc index 47dad1149..982b79018 100644 --- a/src/storages/container/container_utils.cc +++ b/src/storages/container/container_utils.cc @@ -26,7 +26,7 @@ #include "neug/storages/container/file_mmap_container.h" #include "neug/storages/container/i_container.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" namespace neug { diff --git a/src/storages/container/file_mmap_container.cc b/src/storages/container/file_mmap_container.cc index 0b447e56e..9d56b1e8a 100644 --- a/src/storages/container/file_mmap_container.cc +++ b/src/storages/container/file_mmap_container.cc @@ -24,7 +24,7 @@ #include #include "neug/storages/container/file_header.h" #include "neug/storages/container/file_mmap_container.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" namespace neug { diff --git a/src/storages/container/mmap_container.cc b/src/storages/container/mmap_container.cc index f216dab45..27178ce6b 100644 --- a/src/storages/container/mmap_container.cc +++ b/src/storages/container/mmap_container.cc @@ -25,7 +25,7 @@ #include "neug/storages/checkpoint.h" #include "neug/storages/container/file_header.h" #include "neug/storages/container/mmap_container.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include diff --git a/src/storages/csr/mutable_csr.cc b/src/storages/csr/mutable_csr.cc index ba436e9e3..4eb9adc50 100644 --- a/src/storages/csr/mutable_csr.cc +++ b/src/storages/csr/mutable_csr.cc @@ -35,7 +35,7 @@ #include "neug/storages/container/container_utils.h" #include "neug/storages/container/file_mmap_container.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include "neug/utils/property/types.h" #include "neug/utils/spinlock.h" diff --git a/src/storages/graph/checkpoint_file_manager.cc b/src/storages/graph/checkpoint_file_manager.cc index ff9727888..80906347e 100644 --- a/src/storages/graph/checkpoint_file_manager.cc +++ b/src/storages/graph/checkpoint_file_manager.cc @@ -17,7 +17,7 @@ #include -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include diff --git a/src/storages/graph/checkpoint_manifest.cc b/src/storages/graph/checkpoint_manifest.cc index 41f56da1b..693bfc93a 100644 --- a/src/storages/graph/checkpoint_manifest.cc +++ b/src/storages/graph/checkpoint_manifest.cc @@ -16,7 +16,7 @@ #include "neug/storages/checkpoint_manifest.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include #include diff --git a/src/storages/graph/edge_table.cc b/src/storages/graph/edge_table.cc index 1ef047c42..ea5145c46 100644 --- a/src/storages/graph/edge_table.cc +++ b/src/storages/graph/edge_table.cc @@ -34,7 +34,7 @@ #include "neug/storages/loader/loader_utils.h" #include "neug/storages/module/type_name.h" #include "neug/storages/module_descriptor.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include "neug/utils/property/types.h" namespace neug { diff --git a/src/storages/graph/property_graph.cc b/src/storages/graph/property_graph.cc index ff3a4296b..05b577fbd 100644 --- a/src/storages/graph/property_graph.cc +++ b/src/storages/graph/property_graph.cc @@ -32,7 +32,7 @@ #include "neug/storages/graph/schema.h" #include "neug/storages/module/module_broker.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include "neug/utils/indexers.h" #include "neug/utils/property/column.h" #include "neug/utils/property/types.h" diff --git a/src/storages/graph/vertex_table.cc b/src/storages/graph/vertex_table.cc index 3556f2e87..fd65acebc 100644 --- a/src/storages/graph/vertex_table.cc +++ b/src/storages/graph/vertex_table.cc @@ -19,7 +19,7 @@ #include "neug/storages/module/module_broker.h" #include "neug/storages/module/module_factory.h" #include "neug/storages/module_descriptor.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include "neug/utils/likely.h" namespace neug { diff --git a/src/transaction/update_transaction.cc b/src/transaction/update_transaction.cc index 6bd5865ce..4df050879 100644 --- a/src/transaction/update_transaction.cc +++ b/src/transaction/update_transaction.cc @@ -36,7 +36,7 @@ #include "neug/transaction/version_manager.h" #include "neug/transaction/wal/wal.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include "neug/utils/id_indexer.h" #include "neug/utils/likely.h" #include "neug/utils/property/column.h" diff --git a/src/utils/file_utils.cc b/src/utils/io/file/file_utils.cc similarity index 99% rename from src/utils/file_utils.cc rename to src/utils/io/file/file_utils.cc index 87e6a617f..c6d330ffb 100644 --- a/src/utils/file_utils.cc +++ b/src/utils/io/file/file_utils.cc @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" #include diff --git a/src/utils/reader/operator_precedence.cc b/src/utils/io/read/common/operator_precedence.cc similarity index 96% rename from src/utils/reader/operator_precedence.cc rename to src/utils/io/read/common/operator_precedence.cc index 0b34c952e..80dae83f6 100644 --- a/src/utils/reader/operator_precedence.cc +++ b/src/utils/io/read/common/operator_precedence.cc @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "neug/utils/reader/operator_precedence.h" +#include "neug/utils/io/read/common/operator_precedence.h" namespace neug { namespace reader { diff --git a/src/utils/reader/options.cc b/src/utils/io/read/common/options.cc similarity index 96% rename from src/utils/reader/options.cc rename to src/utils/io/read/common/options.cc index 05646e162..8ed84d887 100644 --- a/src/utils/reader/options.cc +++ b/src/utils/io/read/common/options.cc @@ -1,11 +1,11 @@ -#include "neug/utils/reader/options.h" +#include "neug/utils/io/read/common/options.h" #include "neug/compiler/common/assert.h" #include "neug/storages/loader/loader_utils.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/reader.h" -#include "neug/utils/reader/schema.h" -#include "neug/utils/reader/type_converter.h" +#include "neug/utils/io/reader.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/read/common/type_converter.h" #include #include diff --git a/src/utils/reader/row_expression_filter.cc b/src/utils/io/read/common/row_expression_filter.cc similarity index 98% rename from src/utils/reader/row_expression_filter.cc rename to src/utils/io/read/common/row_expression_filter.cc index 13c3b87ca..0b890c784 100644 --- a/src/utils/reader/row_expression_filter.cc +++ b/src/utils/io/read/common/row_expression_filter.cc @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "neug/utils/reader/row_expression_filter.h" +#include "neug/utils/io/read/common/row_expression_filter.h" #include #include @@ -23,7 +23,7 @@ #include "neug/generated/proto/plan/expr.pb.h" #include "neug/storages/loader/loader_utils.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/operator_precedence.h" +#include "neug/utils/io/read/common/operator_precedence.h" namespace neug { namespace reader { diff --git a/src/utils/reader/sniffer.cc b/src/utils/io/read/common/sniffer.cc similarity index 96% rename from src/utils/reader/sniffer.cc rename to src/utils/io/read/common/sniffer.cc index 0ca3e0283..cf277a8e5 100644 --- a/src/utils/reader/sniffer.cc +++ b/src/utils/io/read/common/sniffer.cc @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "neug/utils/reader/sniffer.h" +#include "neug/utils/io/read/common/sniffer.h" #include "neug/utils/result.h" diff --git a/src/utils/reader/type_converter.cc b/src/utils/io/read/common/type_converter.cc similarity index 98% rename from src/utils/reader/type_converter.cc rename to src/utils/io/read/common/type_converter.cc index 8c3447d1a..716140494 100644 --- a/src/utils/reader/type_converter.cc +++ b/src/utils/io/read/common/type_converter.cc @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "neug/utils/reader/type_converter.h" +#include "neug/utils/io/read/common/type_converter.h" #include diff --git a/src/utils/reader/csv_reader.cc b/src/utils/io/read/csv/csv_reader.cc similarity index 99% rename from src/utils/reader/csv_reader.cc rename to src/utils/io/read/csv/csv_reader.cc index d2436f415..269d094f6 100644 --- a/src/utils/reader/csv_reader.cc +++ b/src/utils/io/read/csv/csv_reader.cc @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "neug/utils/reader/reader.h" +#include "neug/utils/io/reader.h" #include "neug/execution/common/columns/container_types.h" @@ -39,10 +39,10 @@ #include "neug/generated/proto/plan/expr.pb.h" #include "neug/storages/loader/loader_utils.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/operator_precedence.h" -#include "neug/utils/reader/options.h" -#include "neug/utils/reader/schema.h" -#include "neug/utils/reader/type_converter.h" +#include "neug/utils/io/read/common/operator_precedence.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/read/common/type_converter.h" #include "neug/utils/result.h" namespace neug { diff --git a/src/utils/reader/json_reader.cc b/src/utils/io/read/json/json_reader.cc similarity index 98% rename from src/utils/reader/json_reader.cc rename to src/utils/io/read/json/json_reader.cc index eb8d45a38..0a2014977 100644 --- a/src/utils/reader/json_reader.cc +++ b/src/utils/io/read/json/json_reader.cc @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "neug/utils/reader/reader.h" +#include "neug/utils/io/reader.h" #include #include @@ -33,10 +33,10 @@ #include "neug/execution/common/types/value.h" #include "neug/storages/loader/loader_utils.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/options.h" -#include "neug/utils/reader/row_expression_filter.h" -#include "neug/utils/reader/schema.h" -#include "neug/utils/reader/type_converter.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/read/common/row_expression_filter.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/read/common/type_converter.h" #include "neug/utils/result.h" #include "neug/utils/service_utils.h" diff --git a/src/utils/io/output_stream.cc b/src/utils/io/stream/output_stream.cc similarity index 98% rename from src/utils/io/output_stream.cc rename to src/utils/io/stream/output_stream.cc index e6c4c9d1a..7cdc84dc5 100644 --- a/src/utils/io/output_stream.cc +++ b/src/utils/io/stream/output_stream.cc @@ -13,7 +13,7 @@ * limitations under the License. */ -#include "neug/utils/io/output_stream.h" +#include "neug/utils/io/stream/output_stream.h" #include #include diff --git a/src/utils/file_sys/file_system.cc b/src/utils/io/vfs/file_system.cc similarity index 97% rename from src/utils/file_sys/file_system.cc rename to src/utils/io/vfs/file_system.cc index b0242129a..4e3358c90 100644 --- a/src/utils/file_sys/file_system.cc +++ b/src/utils/io/vfs/file_system.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "neug/utils/file_sys/file_system.h" +#include "neug/utils/io/vfs/file_system.h" #include #include @@ -22,7 +22,7 @@ #include "neug/execution/execute/ops/batch/batch_update_utils.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/schema.h" namespace neug { namespace fsys { diff --git a/src/utils/writer/writer.cc b/src/utils/io/write/writer.cc similarity index 99% rename from src/utils/writer/writer.cc rename to src/utils/io/write/writer.cc index 3c841809e..d7dfe5f94 100644 --- a/src/utils/writer/writer.cc +++ b/src/utils/io/write/writer.cc @@ -13,13 +13,13 @@ * limitations under the License. */ -#include "neug/utils/writer/writer.h" +#include "neug/utils/io/write/writer.h" #include "neug/execution/common/operators/retrieve/sink.h" #include "neug/generated/proto/response/response.pb.h" #include "neug/utils/exception/exception.h" -#include "neug/utils/io/output_stream.h" +#include "neug/utils/io/stream/output_stream.h" #include "neug/utils/property/types.h" -#include "neug/utils/reader/options.h" +#include "neug/utils/io/read/common/options.h" #include #include diff --git a/tests/storage/alter_property_test.cc b/tests/storage/alter_property_test.cc index a2db7db6e..eca74e401 100644 --- a/tests/storage/alter_property_test.cc +++ b/tests/storage/alter_property_test.cc @@ -27,7 +27,7 @@ #include "neug/storages/csr/csr_base.h" #include "neug/storages/graph/property_graph.h" #include "neug/storages/graph/schema.h" -#include "neug/storages/loader/csv_read_config.h" +#include "neug/utils/io/read/csv/csv_read_config.h" #include "neug/storages/loader/loader_utils.h" #include "neug/utils/property/types.h" #include "unittest/utils.h" diff --git a/tests/utils/json_test.cpp b/tests/utils/json_test.cpp index 0c324b7d2..20e465e04 100644 --- a/tests/utils/json_test.cpp +++ b/tests/utils/json_test.cpp @@ -24,9 +24,9 @@ #include "neug/execution/common/columns/value_columns.h" #include "neug/execution/common/context.h" #include "neug/generated/proto/plan/basic_type.pb.h" -#include "neug/utils/reader/options.h" -#include "neug/utils/reader/reader.h" -#include "neug/utils/reader/schema.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/reader.h" +#include "neug/utils/io/read/common/schema.h" namespace neug { namespace test { diff --git a/tests/utils/test_file_utils.cc b/tests/utils/test_file_utils.cc index acd854f6a..189d5b47f 100644 --- a/tests/utils/test_file_utils.cc +++ b/tests/utils/test_file_utils.cc @@ -24,7 +24,7 @@ #include -#include "neug/utils/file_utils.h" +#include "neug/utils/io/file/file_utils.h" // Non-static helper from file_utils.cc, exposed for direct testing of the // sparse-aware fallback path (otherwise shadowed by clonefile/FICLONE on diff --git a/tests/utils/test_reader.h b/tests/utils/test_reader.h index 3ba74769c..7f0142337 100644 --- a/tests/utils/test_reader.h +++ b/tests/utils/test_reader.h @@ -31,10 +31,10 @@ #include "neug/execution/common/data_chunk.h" #include "neug/generated/proto/plan/basic_type.pb.h" #include "neug/generated/proto/plan/expr.pb.h" -#include "neug/utils/reader/options.h" -#include "neug/utils/reader/reader.h" -#include "neug/utils/reader/schema.h" -#include "neug/utils/reader/type_converter.h" +#include "neug/utils/io/read/common/options.h" +#include "neug/utils/io/read/common/schema.h" +#include "neug/utils/io/read/common/type_converter.h" +#include "neug/utils/io/reader.h" namespace neug { namespace test { diff --git a/tests/utils/test_sniffer.cc b/tests/utils/test_sniffer.cc index a29ddc7f3..3a1a0effc 100644 --- a/tests/utils/test_sniffer.cc +++ b/tests/utils/test_sniffer.cc @@ -15,7 +15,7 @@ #include -#include "neug/utils/reader/sniffer.h" +#include "neug/utils/io/read/common/sniffer.h" #include "test_reader.h" namespace neug { namespace test {