From e9346116e85be10fdda06734682d15cf4e09de2d Mon Sep 17 00:00:00 2001 From: wraymo Date: Fri, 16 Aug 2024 14:35:59 -0400 Subject: [PATCH 01/31] Add a CLP connector --- CMake/ExternalAntlr4Cpp.cmake | 175 ++++++++++ CMake/FindANTLR.cmake | 174 ++++++++++ CMake/resolve_dependency_modules/clp.cmake | 38 +++ .../msgpack-cxx.cmake | 34 ++ CMake/resolve_dependency_modules/spdlog.cmake | 35 ++ CMakeLists.txt | 22 ++ velox/connectors/CMakeLists.txt | 4 + velox/connectors/clp/CMakeLists.txt | 31 ++ velox/connectors/clp/ClpColumnHandle.h | 56 ++++ velox/connectors/clp/ClpConfig.h | 43 +++ velox/connectors/clp/ClpConnector.cpp | 61 ++++ velox/connectors/clp/ClpConnector.h | 75 +++++ velox/connectors/clp/ClpConnectorSplit.h | 32 ++ velox/connectors/clp/ClpDataSource.cpp | 176 ++++++++++ velox/connectors/clp/ClpDataSource.h | 106 ++++++ velox/connectors/clp/ClpTableHandle.cpp | 27 ++ velox/connectors/clp/ClpTableHandle.h | 60 ++++ .../connectors/clp/search_lib/CMakeLists.txt | 129 +++++++ velox/connectors/clp/search_lib/ClpCursor.cpp | 254 ++++++++++++++ velox/connectors/clp/search_lib/ClpCursor.h | 123 +++++++ .../clp/search_lib/ClpQueryRunner.cpp | 82 +++++ .../clp/search_lib/ClpQueryRunner.h | 85 +++++ .../clp/search_lib/ClpVectorLoader.cpp | 155 +++++++++ .../clp/search_lib/ClpVectorLoader.h | 53 +++ velox/connectors/clp/tests/CMakeLists.txt | 27 ++ .../connectors/clp/tests/ClpConnectorTest.cpp | 317 ++++++++++++++++++ .../connectors/clp/tests/examples/test_1.clps | Bin 0 -> 959 bytes .../clp/tests/examples/test_1.ndjson | 10 + .../connectors/clp/tests/examples/test_2.clps | Bin 0 -> 1576 bytes .../clp/tests/examples/test_2.ndjson | 10 + velox/docs/develop/connectors.rst | 31 +- 31 files changed, 2423 insertions(+), 2 deletions(-) create mode 100644 CMake/ExternalAntlr4Cpp.cmake create mode 100644 CMake/FindANTLR.cmake create mode 100644 CMake/resolve_dependency_modules/clp.cmake create mode 100644 CMake/resolve_dependency_modules/msgpack-cxx.cmake create mode 100644 CMake/resolve_dependency_modules/spdlog.cmake create mode 100644 velox/connectors/clp/CMakeLists.txt create mode 100644 velox/connectors/clp/ClpColumnHandle.h create mode 100644 velox/connectors/clp/ClpConfig.h create mode 100644 velox/connectors/clp/ClpConnector.cpp create mode 100644 velox/connectors/clp/ClpConnector.h create mode 100644 velox/connectors/clp/ClpConnectorSplit.h create mode 100644 velox/connectors/clp/ClpDataSource.cpp create mode 100644 velox/connectors/clp/ClpDataSource.h create mode 100644 velox/connectors/clp/ClpTableHandle.cpp create mode 100644 velox/connectors/clp/ClpTableHandle.h create mode 100644 velox/connectors/clp/search_lib/CMakeLists.txt create mode 100644 velox/connectors/clp/search_lib/ClpCursor.cpp create mode 100644 velox/connectors/clp/search_lib/ClpCursor.h create mode 100644 velox/connectors/clp/search_lib/ClpQueryRunner.cpp create mode 100644 velox/connectors/clp/search_lib/ClpQueryRunner.h create mode 100644 velox/connectors/clp/search_lib/ClpVectorLoader.cpp create mode 100644 velox/connectors/clp/search_lib/ClpVectorLoader.h create mode 100644 velox/connectors/clp/tests/CMakeLists.txt create mode 100644 velox/connectors/clp/tests/ClpConnectorTest.cpp create mode 100644 velox/connectors/clp/tests/examples/test_1.clps create mode 100644 velox/connectors/clp/tests/examples/test_1.ndjson create mode 100644 velox/connectors/clp/tests/examples/test_2.clps create mode 100644 velox/connectors/clp/tests/examples/test_2.ndjson diff --git a/CMake/ExternalAntlr4Cpp.cmake b/CMake/ExternalAntlr4Cpp.cmake new file mode 100644 index 00000000000..0b2b521e737 --- /dev/null +++ b/CMake/ExternalAntlr4Cpp.cmake @@ -0,0 +1,175 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: ExternalAntlr4Cpp.cmake taken from +# https://github.com/antlr/antlr4/blob/4.13.1/runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake + +cmake_minimum_required(VERSION 3.7) + +if(POLICY CMP0114) + cmake_policy(SET CMP0114 NEW) +endif() + +include(ExternalProject) + +set(ANTLR4_ROOT ${CMAKE_CURRENT_BINARY_DIR}/antlr4_runtime/src/antlr4_runtime) +set(ANTLR4_INCLUDE_DIRS ${ANTLR4_ROOT}/runtime/Cpp/runtime/src) +set(ANTLR4_GIT_REPOSITORY https://github.com/antlr/antlr4.git) +if(NOT DEFINED ANTLR4_TAG) + # Set to branch name to keep library updated at the cost of needing to rebuild + # after 'clean' Set to commit hash to keep the build stable and does not need + # to rebuild after 'clean' + set(ANTLR4_TAG master) +endif() + +# Ensure that the include dir already exists at configure time (to avoid cmake +# erroring on non-existent include dirs) +file(MAKE_DIRECTORY "${ANTLR4_INCLUDE_DIRS}") + +if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") + set(ANTLR4_OUTPUT_DIR ${ANTLR4_ROOT}/runtime/Cpp/dist/$(Configuration)) +elseif(${CMAKE_GENERATOR} MATCHES "Xcode.*") + set(ANTLR4_OUTPUT_DIR ${ANTLR4_ROOT}/runtime/Cpp/dist/$(CONFIGURATION)) +else() + set(ANTLR4_OUTPUT_DIR ${ANTLR4_ROOT}/runtime/Cpp/dist) +endif() + +if(MSVC) + set(ANTLR4_STATIC_LIBRARIES ${ANTLR4_OUTPUT_DIR}/antlr4-runtime-static.lib) + set(ANTLR4_SHARED_LIBRARIES ${ANTLR4_OUTPUT_DIR}/antlr4-runtime.lib) + set(ANTLR4_RUNTIME_LIBRARIES ${ANTLR4_OUTPUT_DIR}/antlr4-runtime.dll) +else() + set(ANTLR4_STATIC_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.a) + if(MINGW) + set(ANTLR4_SHARED_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.dll.a) + set(ANTLR4_RUNTIME_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.dll) + elseif(CYGWIN) + set(ANTLR4_SHARED_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.dll.a) + set(ANTLR4_RUNTIME_LIBRARIES + ${ANTLR4_OUTPUT_DIR}/cygantlr4-runtime-${ANTLR4_TAG}.dll) + elseif(APPLE) + set(ANTLR4_RUNTIME_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.dylib) + else() + set(ANTLR4_RUNTIME_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.so) + endif() +endif() + +if(${CMAKE_GENERATOR} MATCHES ".* Makefiles") + # This avoids 'warning: jobserver unavailable: using -j1. Add '+' to parent + # make rule.' + set(ANTLR4_BUILD_COMMAND $(MAKE)) +elseif(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") + set(ANTLR4_BUILD_COMMAND ${CMAKE_COMMAND} --build . --config $(Configuration) + --target) +elseif(${CMAKE_GENERATOR} MATCHES "Xcode.*") + set(ANTLR4_BUILD_COMMAND ${CMAKE_COMMAND} --build . --config $(CONFIGURATION) + --target) +else() + set(ANTLR4_BUILD_COMMAND ${CMAKE_COMMAND} --build . --target) +endif() + +if(NOT DEFINED ANTLR4_WITH_STATIC_CRT) + set(ANTLR4_WITH_STATIC_CRT ON) +endif() + +if(ANTLR4_ZIP_REPOSITORY) + ExternalProject_Add( + antlr4_runtime + PREFIX antlr4_runtime + URL ${ANTLR4_ZIP_REPOSITORY} + DOWNLOAD_DIR ${CMAKE_CURRENT_BINARY_DIR} + BUILD_COMMAND "" + BUILD_IN_SOURCE 1 + SOURCE_DIR ${ANTLR4_ROOT} + SOURCE_SUBDIR runtime/Cpp + CMAKE_CACHE_ARGS + -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} + -DWITH_STATIC_CRT:BOOL=${ANTLR4_WITH_STATIC_CRT} + -DDISABLE_WARNINGS:BOOL=ON + # -DCMAKE_CXX_STANDARD:STRING=17 # if desired, compile the runtime with a + # different C++ standard -DCMAKE_CXX_STANDARD:STRING=${CMAKE_CXX_STANDARD} + # alternatively, compile the runtime with the same C++ standard as the + # outer project + INSTALL_COMMAND "" + EXCLUDE_FROM_ALL 1) +else() + ExternalProject_Add( + antlr4_runtime + PREFIX antlr4_runtime + GIT_REPOSITORY ${ANTLR4_GIT_REPOSITORY} + GIT_TAG ${ANTLR4_TAG} + DOWNLOAD_DIR ${CMAKE_CURRENT_BINARY_DIR} + BUILD_COMMAND "" + BUILD_IN_SOURCE 1 + SOURCE_DIR ${ANTLR4_ROOT} + SOURCE_SUBDIR runtime/Cpp + CMAKE_CACHE_ARGS + -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} + -DWITH_STATIC_CRT:BOOL=${ANTLR4_WITH_STATIC_CRT} + -DDISABLE_WARNINGS:BOOL=ON + # -DCMAKE_CXX_STANDARD:STRING=17 # if desired, compile the runtime with a + # different C++ standard -DCMAKE_CXX_STANDARD:STRING=${CMAKE_CXX_STANDARD} + # alternatively, compile the runtime with the same C++ standard as the + # outer project + INSTALL_COMMAND "" + EXCLUDE_FROM_ALL 1) +endif() + +# Separate build step as rarely people want both +set(ANTLR4_BUILD_DIR ${ANTLR4_ROOT}) +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14.0") + # CMake 3.14 builds in above's SOURCE_SUBDIR when BUILD_IN_SOURCE is true + set(ANTLR4_BUILD_DIR ${ANTLR4_ROOT}/runtime/Cpp) +endif() + +ExternalProject_Add_Step( + antlr4_runtime build_static + COMMAND ${ANTLR4_BUILD_COMMAND} antlr4_static + # Depend on target instead of step (a custom command) to avoid running + # dependent steps concurrently + DEPENDS antlr4_runtime + BYPRODUCTS ${ANTLR4_STATIC_LIBRARIES} + EXCLUDE_FROM_MAIN 1 + WORKING_DIRECTORY ${ANTLR4_BUILD_DIR}) +ExternalProject_Add_StepTargets(antlr4_runtime build_static) + +add_library(antlr4_static STATIC IMPORTED) +add_dependencies(antlr4_static antlr4_runtime-build_static) +set_target_properties( + antlr4_static + PROPERTIES IMPORTED_LOCATION ${ANTLR4_STATIC_LIBRARIES}) +target_include_directories(antlr4_static INTERFACE ${ANTLR4_INCLUDE_DIRS}) + +ExternalProject_Add_Step( + antlr4_runtime build_shared + COMMAND ${ANTLR4_BUILD_COMMAND} antlr4_shared + # Depend on target instead of step (a custom command) to avoid running + # dependent steps concurrently + DEPENDS antlr4_runtime + BYPRODUCTS ${ANTLR4_SHARED_LIBRARIES} ${ANTLR4_RUNTIME_LIBRARIES} + EXCLUDE_FROM_MAIN 1 + WORKING_DIRECTORY ${ANTLR4_BUILD_DIR}) +ExternalProject_Add_StepTargets(antlr4_runtime build_shared) + +add_library(antlr4_shared SHARED IMPORTED) +add_dependencies(antlr4_shared antlr4_runtime-build_shared) +set_target_properties( + antlr4_shared + PROPERTIES IMPORTED_LOCATION ${ANTLR4_RUNTIME_LIBRARIES}) +target_include_directories(antlr4_shared INTERFACE ${ANTLR4_INCLUDE_DIRS}) + +if(ANTLR4_SHARED_LIBRARIES) + set_target_properties(antlr4_shared PROPERTIES IMPORTED_IMPLIB + ${ANTLR4_SHARED_LIBRARIES}) +endif() diff --git a/CMake/FindANTLR.cmake b/CMake/FindANTLR.cmake new file mode 100644 index 00000000000..e1d1cfc9d09 --- /dev/null +++ b/CMake/FindANTLR.cmake @@ -0,0 +1,174 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: FindANTLR.cmake taken from +# https://github.com/antlr/antlr4/blob/4.13.1/runtime/Cpp/cmake/FindANTLR.cmake + +# Set ANTLR version +set(ANTLR4_TAG 4.13.1) +add_definitions(-DANTLR4CPP_STATIC) + +# Define the JAR name and path +set(ANTLR_JAR_NAME antlr-${ANTLR4_TAG}-complete.jar) + +# Set the download directory +set(ANTLR_DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/antlr) +file(MAKE_DIRECTORY ${ANTLR_DOWNLOAD_DIR}) + +# Define the full path to the ANTLR JAR +set(ANTLR_EXECUTABLE ${ANTLR_DOWNLOAD_DIR}/${ANTLR_JAR_NAME}) + +# Download the ANTLR JAR if it does not exist +if(NOT EXISTS ${ANTLR_EXECUTABLE}) + message(STATUS "ANTLR JAR not found. Downloading ANTLR ${ANTLR4_TAG}...") + file( + DOWNLOAD https://www.antlr.org/download/${ANTLR_JAR_NAME} + ${ANTLR_EXECUTABLE} + SHOW_PROGRESS + EXPECTED_HASH + SHA256=bc13a9c57a8dd7d5196888211e5ede657cb64a3ce968608697e4f668251a8487 + TLS_VERIFY ON) +endif() + +# Include the ANTLR C++ runtime integration +include(ExternalAntlr4Cpp) + +find_package(Java 11 REQUIRED COMPONENTS Runtime) + +if(NOT ANTLR_EXECUTABLE) + find_program(ANTLR_EXECUTABLE NAMES antlr.jar antlr4.jar antlr-4.jar + antlr-${ANTLR4_TAG}-complete.jar) +endif() + +if(ANTLR_EXECUTABLE AND Java_JAVA_EXECUTABLE) + execute_process( + COMMAND ${Java_JAVA_EXECUTABLE} -jar ${ANTLR_EXECUTABLE} + OUTPUT_VARIABLE ANTLR_COMMAND_OUTPUT + ERROR_VARIABLE ANTLR_COMMAND_ERROR + RESULT_VARIABLE ANTLR_COMMAND_RESULT + OUTPUT_STRIP_TRAILING_WHITESPACE) + + if(ANTLR_COMMAND_RESULT EQUAL 0) + string(REGEX MATCH "Version [0-9]+(\\.[0-9]+)*" ANTLR_VERSION + ${ANTLR_COMMAND_OUTPUT}) + string(REPLACE "Version " "" ANTLR_VERSION ${ANTLR_VERSION}) + else() + message( + SEND_ERROR "Command '${Java_JAVA_EXECUTABLE} -jar ${ANTLR_EXECUTABLE}' " + "failed with the output '${ANTLR_COMMAND_ERROR}'") + endif() + + macro(ANTLR_TARGET Name InputFile) + set(ANTLR_OPTIONS LEXER PARSER LISTENER VISITOR) + set(ANTLR_ONE_VALUE_ARGS PACKAGE OUTPUT_DIRECTORY DEPENDS_ANTLR) + set(ANTLR_MULTI_VALUE_ARGS COMPILE_FLAGS DEPENDS) + cmake_parse_arguments( + ANTLR_TARGET + "${ANTLR_OPTIONS}" + "${ANTLR_ONE_VALUE_ARGS}" + "${ANTLR_MULTI_VALUE_ARGS}" + ${ARGN}) + set(ANTLR_${Name}_INPUT ${InputFile}) + + get_filename_component(ANTLR_INPUT ${InputFile} NAME_WE) + get_filename_component(ANTLR_INPUT_PARENT_DIR "${InputFile}" DIRECTORY) + + if(ANTLR_TARGET_OUTPUT_DIRECTORY) + set(ANTLR_${Name}_OUTPUT_DIR ${ANTLR_TARGET_OUTPUT_DIRECTORY}) + else() + set(ANTLR_${Name}_OUTPUT_DIR + ${CMAKE_CURRENT_BINARY_DIR}/antlr4cpp_generated_src/${ANTLR_INPUT}) + endif() + + unset(ANTLR_${Name}_CXX_OUTPUTS) + + if((ANTLR_TARGET_LEXER AND NOT ANTLR_TARGET_PARSER) + OR (ANTLR_TARGET_PARSER AND NOT ANTLR_TARGET_LEXER)) + list(APPEND ANTLR_${Name}_CXX_OUTPUTS + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}.h + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}.cpp) + set(ANTLR_${Name}_OUTPUTS + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}.interp + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}.tokens) + else() + list( + APPEND + ANTLR_${Name}_CXX_OUTPUTS + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Lexer.h + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Lexer.cpp + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Parser.h + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Parser.cpp) + list(APPEND ANTLR_${Name}_OUTPUTS + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Lexer.interp + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Lexer.tokens) + endif() + + if(ANTLR_TARGET_LISTENER) + list( + APPEND + ANTLR_${Name}_CXX_OUTPUTS + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}BaseListener.h + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}BaseListener.cpp + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Listener.h + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Listener.cpp) + list(APPEND ANTLR_TARGET_COMPILE_FLAGS -listener) + endif() + + if(ANTLR_TARGET_VISITOR) + list( + APPEND + ANTLR_${Name}_CXX_OUTPUTS + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}BaseVisitor.h + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}BaseVisitor.cpp + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Visitor.h + ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Visitor.cpp) + list(APPEND ANTLR_TARGET_COMPILE_FLAGS -visitor) + endif() + + if(ANTLR_TARGET_PACKAGE) + list(APPEND ANTLR_TARGET_COMPILE_FLAGS -package ${ANTLR_TARGET_PACKAGE}) + endif() + + list(APPEND ANTLR_${Name}_OUTPUTS ${ANTLR_${Name}_CXX_OUTPUTS}) + + if(ANTLR_TARGET_DEPENDS_ANTLR) + if(ANTLR_${ANTLR_TARGET_DEPENDS_ANTLR}_INPUT) + list(APPEND ANTLR_TARGET_DEPENDS + ${ANTLR_${ANTLR_TARGET_DEPENDS_ANTLR}_INPUT}) + list(APPEND ANTLR_TARGET_DEPENDS + ${ANTLR_${ANTLR_TARGET_DEPENDS_ANTLR}_OUTPUTS}) + else() + message( + SEND_ERROR "ANTLR target '${ANTLR_TARGET_DEPENDS_ANTLR}' not found") + endif() + endif() + + add_custom_command( + OUTPUT ${ANTLR_${Name}_OUTPUTS} + COMMAND + ${Java_JAVA_EXECUTABLE} -jar ${ANTLR_EXECUTABLE} ${InputFile} -o + ${ANTLR_${Name}_OUTPUT_DIR} -no-listener -Dlanguage=Cpp + ${ANTLR_TARGET_COMPILE_FLAGS} + DEPENDS ${InputFile} ${ANTLR_TARGET_DEPENDS} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMENT "Building ${Name} with ANTLR ${ANTLR_VERSION}") + endmacro(ANTLR_TARGET) + +endif(ANTLR_EXECUTABLE AND Java_JAVA_EXECUTABLE) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + ANTLR + REQUIRED_VARS ANTLR_EXECUTABLE Java_JAVA_EXECUTABLE + VERSION_VAR ANTLR_VERSION) diff --git a/CMake/resolve_dependency_modules/clp.cmake b/CMake/resolve_dependency_modules/clp.cmake new file mode 100644 index 00000000000..bf50d26d7c6 --- /dev/null +++ b/CMake/resolve_dependency_modules/clp.cmake @@ -0,0 +1,38 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +include_guard(GLOBAL) + +FetchContent_Declare( + clp + GIT_REPOSITORY https://github.com/y-scope/clp.git + GIT_TAG 7b1b169a89abdfe44c159d6200b168391b697877 + GIT_SUBMODULES "" GIT_SUBMODULES_RECURSE TRUE) + +FetchContent_MakeAvailable(clp) + +if(clp_POPULATED) + message(STATUS "Updating submodules for clp...") + execute_process( + COMMAND ${CMAKE_COMMAND} -E chdir "${clp_SOURCE_DIR}" git submodule update + --init --recursive + RESULT_VARIABLE submodule_update_result + OUTPUT_VARIABLE submodule_update_output + ERROR_VARIABLE submodule_update_error) + if(NOT ${submodule_update_result} EQUAL 0) + message(ERROR + "Failed to update submodules for clp:\n${submodule_update_error}") + else() + message(STATUS "Submodules for clp updated successfully.") + endif() +endif() diff --git a/CMake/resolve_dependency_modules/msgpack-cxx.cmake b/CMake/resolve_dependency_modules/msgpack-cxx.cmake new file mode 100644 index 00000000000..ce39b7cf167 --- /dev/null +++ b/CMake/resolve_dependency_modules/msgpack-cxx.cmake @@ -0,0 +1,34 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +include_guard(GLOBAL) + +set(VELOX_MSGPACK_BUILD_VERSION cpp-7.0.0) +set(VELOX_MSGPACK_BUILD_SHA256_CHECKSUM + 070881ebea9208cf7e731fd5a46a11404025b2f260ab9527e32dfcb7c689fbfc) +string( + CONCAT VELOX_MSGPACK_SOURCE_URL + "https://github.com/msgpack/msgpack-c/archive/refs/tags/" + "${VELOX_MSGPACK_BUILD_VERSION}.tar.gz") + +velox_resolve_dependency_url(MSGPACK) + +message(STATUS "Building msgpack-cxx from source") + +FetchContent_Declare( + msgpack-cxx + URL ${VELOX_MSGPACK_SOURCE_URL} + URL_HASH ${VELOX_MSGPACK_BUILD_SHA256_CHECKSUM} + OVERRIDE_FIND_PACKAGE EXCLUDE_FROM_ALL SYSTEM) + +FetchContent_MakeAvailable(msgpack-cxx) diff --git a/CMake/resolve_dependency_modules/spdlog.cmake b/CMake/resolve_dependency_modules/spdlog.cmake new file mode 100644 index 00000000000..022525fc134 --- /dev/null +++ b/CMake/resolve_dependency_modules/spdlog.cmake @@ -0,0 +1,35 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +include_guard(GLOBAL) + +set(VELOX_SPDLOG_BUILD_VERSION 1.12.0) +set(VELOX_SPDLOG_BUILD_SHA256_CHECKSUM + 4dccf2d10f410c1e2feaff89966bfc49a1abb29ef6f08246335b110e001e09a9) +string( + CONCAT VELOX_SPDLOG_SOURCE_URL + "https://github.com/gabime/spdlog/archive/refs/tags/" + "v${VELOX_SPDLOG_BUILD_VERSION}.tar.gz") + +velox_resolve_dependency_url(SPDLOG) + +message(STATUS "Building spdlog from source") + +FetchContent_Declare( + spdlog + URL ${VELOX_SPDLOG_SOURCE_URL} + URL_HASH ${VELOX_SPDLOG_BUILD_SHA256_CHECKSUM} + OVERRIDE_FIND_PACKAGE EXCLUDE_FROM_ALL SYSTEM) + +set(SPDLOG_FMT_EXTERNAL ON) +FetchContent_MakeAvailable(spdlog) diff --git a/CMakeLists.txt b/CMakeLists.txt index cc27d5e5623..39a481923b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,6 +127,7 @@ set(VELOX_GFLAGS_TYPE ) option(VELOX_ENABLE_EXEC "Build exec." ON) option(VELOX_ENABLE_AGGREGATES "Build aggregates." ON) +option(VELOX_ENABLE_CLP_CONNECTOR "Build CLP connector." ON) option(VELOX_ENABLE_HIVE_CONNECTOR "Build Hive connector." ON) option(VELOX_ENABLE_TPCH_CONNECTOR "Build TPC-H connector." ON) option(VELOX_ENABLE_PRESTO_FUNCTIONS "Build Presto SQL functions." ON) @@ -174,6 +175,7 @@ if(${VELOX_BUILD_MINIMAL} OR ${VELOX_BUILD_MINIMAL_WITH_DWIO}) set(VELOX_ENABLE_EXPRESSION ON) set(VELOX_ENABLE_EXEC OFF) set(VELOX_ENABLE_AGGREGATES OFF) + set(VELOX_ENABLE_CLP_CONNECTOR OFF) set(VELOX_ENABLE_HIVE_CONNECTOR OFF) set(VELOX_ENABLE_TPCH_CONNECTOR OFF) set(VELOX_ENABLE_SPARK_FUNCTIONS OFF) @@ -459,9 +461,11 @@ set(BOOST_INCLUDE_LIBRARIES context date_time filesystem + iostreams program_options regex system + url thread) velox_set_source(Boost) @@ -518,6 +522,24 @@ endif() velox_set_source(re2) velox_resolve_dependency(re2) +if(${VELOX_ENABLE_CLP_CONNECTOR}) + set(clp_SOURCE BUNDLED) + velox_resolve_dependency(clp) + + set(spdlog_SOURCE BUNDLED) + velox_resolve_dependency(spdlog) + + velox_set_source(msgpack-cxx) + velox_resolve_dependency(msgpack-cxx) + + velox_set_source(absl) + velox_resolve_dependency(absl) + + set(curl_SOURCE BUNDLED) + velox_resolve_dependency(curl) + find_package(ANTLR REQUIRED) +endif() + if(${VELOX_BUILD_PYTHON_PACKAGE}) find_package( Python 3.9 diff --git a/velox/connectors/CMakeLists.txt b/velox/connectors/CMakeLists.txt index 3cc600201f6..743ecbcaac0 100644 --- a/velox/connectors/CMakeLists.txt +++ b/velox/connectors/CMakeLists.txt @@ -17,6 +17,10 @@ velox_link_libraries(velox_connector velox_common_config velox_vector) add_subdirectory(fuzzer) +if(${VELOX_ENABLE_CLP_CONNECTOR}) + add_subdirectory(clp) +endif() + if(${VELOX_ENABLE_HIVE_CONNECTOR}) add_subdirectory(hive) endif() diff --git a/velox/connectors/clp/CMakeLists.txt b/velox/connectors/clp/CMakeLists.txt new file mode 100644 index 00000000000..8fc435d8bc4 --- /dev/null +++ b/velox/connectors/clp/CMakeLists.txt @@ -0,0 +1,31 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set(CLP_SRC_DIR ${clp_SOURCE_DIR}/components/core/src) +add_subdirectory(search_lib) + +velox_add_library( + velox_clp_connector + OBJECT + ClpConnector.cpp + ClpDataSource.cpp + ClpTableHandle.cpp) + +velox_link_libraries(velox_clp_connector + PRIVATE clp-s-search simdjson::simdjson velox_connector) + +target_compile_features(velox_clp_connector PRIVATE cxx_std_20) + +if(${VELOX_BUILD_TESTING}) + add_subdirectory(tests) +endif() diff --git a/velox/connectors/clp/ClpColumnHandle.h b/velox/connectors/clp/ClpColumnHandle.h new file mode 100644 index 00000000000..d7857d82724 --- /dev/null +++ b/velox/connectors/clp/ClpColumnHandle.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "velox/connectors/Connector.h" + +namespace facebook::velox::connector::clp { +class ClpColumnHandle : public ColumnHandle { + public: + ClpColumnHandle( + const std::string& columnName, + const std::string& originalColumnName, + const TypePtr& columnType, + bool nullable) + : columnName_(columnName), + originalColumnName_(originalColumnName), + columnType_(columnType), + nullable_(nullable) {} + + const std::string& columnName() const { + return columnName_; + } + + const std::string& originalColumnName() const { + return originalColumnName_; + } + + const TypePtr& columnType() const { + return columnType_; + } + + bool nullable() const { + return nullable_; + } + + private: + const std::string columnName_; + const std::string originalColumnName_; + const TypePtr columnType_; + const bool nullable_; +}; +} // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpConfig.h b/velox/connectors/clp/ClpConfig.h new file mode 100644 index 00000000000..5d6a254a81e --- /dev/null +++ b/velox/connectors/clp/ClpConfig.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "velox/common/config/Config.h" + +namespace facebook::velox::config { +class ConfigBase; +} + +namespace facebook::velox::connector::clp { +class ClpConfig { + public: + explicit ClpConfig(std::shared_ptr config) { + VELOX_CHECK_NOT_NULL(config, "Config is null for CLP initialization"); + config_ = std::move(config); + } + + [[nodiscard]] const std::shared_ptr& config() + const { + return config_; + } + + private: + std::shared_ptr config_; +}; +} // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpConnector.cpp b/velox/connectors/clp/ClpConnector.cpp new file mode 100644 index 00000000000..54ffe63a5ac --- /dev/null +++ b/velox/connectors/clp/ClpConnector.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "clp_s/TimestampPattern.hpp" + +#include "velox/connectors/clp/ClpConnector.h" +#include "velox/connectors/clp/ClpDataSource.h" + +namespace facebook::velox::connector::clp { +ClpConnector::ClpConnector( + const std::string& id, + std::shared_ptr config) + : Connector(id), config_(std::make_shared(config)) {} + +std::unique_ptr ClpConnector::createDataSource( + const RowTypePtr& outputType, + const std::shared_ptr& tableHandle, + const std::unordered_map< + std::string, + std::shared_ptr>& columnHandles, + ConnectorQueryCtx* connectorQueryCtx) { + return std::make_unique( + outputType, + tableHandle, + columnHandles, + connectorQueryCtx->memoryPool(), + config_); +} + +std::unique_ptr ClpConnector::createDataSink( + RowTypePtr inputType, + std::shared_ptr connectorInsertTableHandle, + ConnectorQueryCtx* connectorQueryCtx, + CommitStrategy commitStrategy) { + VELOX_NYI("createDataSink for ClpConnector is not implemented!"); +} + +ClpConnectorFactory::ClpConnectorFactory() + : ConnectorFactory(kClpConnectorName) { + clp_s::TimestampPattern::init(); +} + +ClpConnectorFactory::ClpConnectorFactory(const char* connectorName) + : ConnectorFactory(connectorName) { + clp_s::TimestampPattern::init(); +} + +} // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpConnector.h b/velox/connectors/clp/ClpConnector.h new file mode 100644 index 00000000000..ae9c28419ac --- /dev/null +++ b/velox/connectors/clp/ClpConnector.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "velox/connectors/Connector.h" +#include "velox/connectors/clp/ClpConfig.h" + +namespace facebook::velox::connector::clp { +class ClpConnector : public Connector { + public: + ClpConnector( + const std::string& id, + std::shared_ptr config); + + [[nodiscard]] const std::shared_ptr& + connectorConfig() const override { + return config_->config(); + } + + [[nodiscard]] bool canAddDynamicFilter() const override { + return false; + } + + std::unique_ptr createDataSource( + const RowTypePtr& outputType, + const std::shared_ptr& tableHandle, + const std::unordered_map< + std::string, + std::shared_ptr>& columnHandles, + ConnectorQueryCtx* connectorQueryCtx) override; + + bool supportsSplitPreload() override { + return false; + } + + std::unique_ptr createDataSink( + RowTypePtr inputType, + std::shared_ptr connectorInsertTableHandle, + ConnectorQueryCtx* connectorQueryCtx, + CommitStrategy commitStrategy) override; + + private: + std::shared_ptr config_; +}; + +class ClpConnectorFactory : public ConnectorFactory { + public: + static constexpr const char* kClpConnectorName = "clp"; + + ClpConnectorFactory(); + explicit ClpConnectorFactory(const char* connectorName); + + std::shared_ptr newConnector( + const std::string& id, + std::shared_ptr config, + folly::Executor* /*ioExecutor*/, + folly::Executor* /*cpuExecutor*/) override { + return std::make_shared(id, config); + } +}; +} // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpConnectorSplit.h b/velox/connectors/clp/ClpConnectorSplit.h new file mode 100644 index 00000000000..3b00a4b6b5e --- /dev/null +++ b/velox/connectors/clp/ClpConnectorSplit.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "velox/connectors/Connector.h" + +namespace facebook::velox::connector::clp { +struct ClpConnectorSplit : public connector::ConnectorSplit { + ClpConnectorSplit(const std::string& connectorId, const std::string& path) + : connector::ConnectorSplit(connectorId), path_(path) {} + + [[nodiscard]] std::string toString() const override { + return fmt::format("CLP Split: {}", path_); + } + + const std::string path_; +}; +} // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpDataSource.cpp b/velox/connectors/clp/ClpDataSource.cpp new file mode 100644 index 00000000000..7f3783cb1c8 --- /dev/null +++ b/velox/connectors/clp/ClpDataSource.cpp @@ -0,0 +1,176 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "velox/connectors/clp/ClpColumnHandle.h" +#include "velox/connectors/clp/ClpConnectorSplit.h" +#include "velox/connectors/clp/ClpDataSource.h" +#include "velox/connectors/clp/ClpTableHandle.h" +#include "velox/connectors/clp/search_lib/ClpCursor.h" +#include "velox/connectors/clp/search_lib/ClpVectorLoader.h" +#include "velox/vector/FlatVector.h" + +namespace facebook::velox::connector::clp { + +ClpDataSource::ClpDataSource( + const RowTypePtr& outputType, + const std::shared_ptr& tableHandle, + const std::unordered_map< + std::string, + std::shared_ptr>& columnHandles, + velox::memory::MemoryPool* pool, + std::shared_ptr& clpConfig) + : pool_(pool), outputType_(outputType) { + auto clpTableHandle = std::dynamic_pointer_cast(tableHandle); + storageType_ = clpTableHandle->storageType(); + if (auto query = clpTableHandle->kqlQuery(); query && !query->empty()) { + kqlQuery_ = *query; + } else { + kqlQuery_ = "*"; + } + + for (const auto& outputName : outputType->names()) { + auto columnHandle = columnHandles.find(outputName); + VELOX_CHECK( + columnHandle != columnHandles.end(), + "ColumnHandle not found for output name: {}", + outputName); + auto clpColumnHandle = + std::dynamic_pointer_cast(columnHandle->second); + VELOX_CHECK_NOT_NULL( + clpColumnHandle, + "ColumnHandle must be an instance of ClpColumnHandle for output name: {}", + outputName); + auto columnName = clpColumnHandle->originalColumnName(); + auto columnType = clpColumnHandle->columnType(); + addFieldsRecursively(columnType, columnName); + } +} + +void ClpDataSource::addFieldsRecursively( + const TypePtr& columnType, + const std::string& parentName) { + if (columnType->kind() == TypeKind::ROW) { + const auto& rowType = columnType->asRow(); + for (uint32_t i = 0; i < rowType.size(); ++i) { + const auto& childType = rowType.childAt(i); + const auto childName = parentName + "." + rowType.nameOf(i); + addFieldsRecursively(childType, childName); + } + } else { + search_lib::ColumnType clpColumnType = search_lib::ColumnType::Unknown; + switch (columnType->kind()) { + case TypeKind::BOOLEAN: + clpColumnType = search_lib::ColumnType::Boolean; + break; + case TypeKind::INTEGER: + case TypeKind::BIGINT: + case TypeKind::SMALLINT: + case TypeKind::TINYINT: + clpColumnType = search_lib::ColumnType::Integer; + break; + case TypeKind::DOUBLE: + case TypeKind::REAL: + clpColumnType = search_lib::ColumnType::Float; + break; + case TypeKind::VARCHAR: + clpColumnType = search_lib::ColumnType::String; + break; + case TypeKind::ARRAY: + clpColumnType = search_lib::ColumnType::Array; + break; + default: + VELOX_USER_FAIL("Type not supported: {}", columnType->name()); + } + fields_.emplace_back(search_lib::Field{clpColumnType, parentName}); + } +} + +void ClpDataSource::addSplit(std::shared_ptr split) { + auto clpSplit = std::dynamic_pointer_cast(split); + + if (storageType_ == ClpTableHandle::StorageType::kFS) { + cursor_ = std::make_unique( + clp_s::InputSource::Filesystem, clpSplit->path_); + } else if (storageType_ == ClpTableHandle::StorageType::kS3) { + cursor_ = std::make_unique( + clp_s::InputSource::Network, clpSplit->path_); + } + + cursor_->executeQuery(kqlQuery_, fields_); +} + +VectorPtr ClpDataSource::createVector( + const TypePtr& type, + size_t size, + const std::vector& projectedColumns, + const std::shared_ptr>& filteredRows, + size_t& readerIndex) { + if (type->kind() == TypeKind::ROW) { + std::vector children; + auto& rowType = type->as(); + for (uint32_t i = 0; i < rowType.size(); ++i) { + children.push_back(createVector( + rowType.childAt(i), + size, + projectedColumns, + filteredRows, + readerIndex)); + } + return std::make_shared( + pool_, type, nullptr, size, std::move(children)); + } + auto vector = BaseVector::create(type, size, pool_); + vector->setNulls(allocateNulls(size, pool_, bits::kNull)); + + VELOX_CHECK_LT( + readerIndex, projectedColumns.size(), "Reader index out of bounds"); + auto projectedColumn = projectedColumns[readerIndex]; + auto projectedType = fields_[readerIndex].type; + readerIndex++; + return std::make_shared( + pool_, + type, + size, + std::make_unique( + projectedColumn, projectedType, filteredRows), + std::move(vector)); +} + +std::optional ClpDataSource::next( + uint64_t size, + ContinueFuture& future) { + auto filteredRows = std::make_shared>(); + auto rowsScanned = cursor_->fetchNext(size, filteredRows); + auto rowsFiltered = filteredRows->size(); + if (rowsFiltered == 0) { + return nullptr; + } + completedRows_ += rowsScanned; + size_t readerIndex = 0; + const auto& projectedColumns = cursor_->getProjectedColumns(); + if (projectedColumns.size() != fields_.size()) { + VELOX_USER_FAIL( + "Projected columns size {} does not match fields size {}", + projectedColumns.size(), + fields_.size()); + } + return std::dynamic_pointer_cast(createVector( + outputType_, rowsFiltered, projectedColumns, filteredRows, readerIndex)); +} + +} // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpDataSource.h b/velox/connectors/clp/ClpDataSource.h new file mode 100644 index 00000000000..fd2be384884 --- /dev/null +++ b/velox/connectors/clp/ClpDataSource.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "velox/connectors/Connector.h" +#include "velox/connectors/clp/ClpConfig.h" +#include "velox/connectors/clp/ClpTableHandle.h" +#include "velox/connectors/clp/search_lib/ClpCursor.h" + +namespace facebook::velox::connector::clp { +class ClpDataSource : public DataSource { + public: + ClpDataSource( + const RowTypePtr& outputType, + const std::shared_ptr& tableHandle, + const std::unordered_map< + std::string, + std::shared_ptr>& columnHandles, + velox::memory::MemoryPool* pool, + std::shared_ptr& clpConfig); + + void addSplit(std::shared_ptr split) override; + + std::optional next(uint64_t size, velox::ContinueFuture& future) + override; + + void addDynamicFilter( + column_index_t outputChannel, + const std::shared_ptr& filter) override { + VELOX_NYI("Dynamic filters not supported by ClpConnector."); + } + + uint64_t getCompletedBytes() override { + return completedBytes_; + } + + uint64_t getCompletedRows() override { + return completedRows_; + } + + std::unordered_map runtimeStats() override { + return {}; + } + + private: + /** + * Recursively adds fields from the column type to the list of fields to be + * retrieved from the data source. + * + * @param columnType The type of the column. + * @param parentName The name of the parent field (used for nested fields). + */ + void addFieldsRecursively( + const TypePtr& columnType, + const std::string& parentName); + + /** + * Creates a Vector of the specified type and size. + * + * This method recursively creates vectors for complex types like ROW. For + * primitive types, it creates a LazyVector that will load the data from the + * underlying data source when it is accessed. + * + * @param type The type of the Vector to create. + * @param size The number of elements in the Vector. + * @param projectedColumns The readers the projected columns. + * @param filteredRows The rows to be read. + * @param readerIndex The index of the column reader. + * @return A Vector of the specified type and size. + */ + VectorPtr createVector( + const TypePtr& type, + size_t size, + const std::vector& projectedColumns, + const std::shared_ptr>& filteredRows, + size_t& readerIndex); + + ClpTableHandle::StorageType storageType_; + std::string kqlQuery_; + velox::memory::MemoryPool* pool_; + RowTypePtr outputType_; + std::set columnUntypedNames_; + uint64_t completedRows_{0}; + uint64_t completedBytes_{0}; + + std::vector fields_; + + std::unique_ptr cursor_; +}; +} // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpTableHandle.cpp b/velox/connectors/clp/ClpTableHandle.cpp new file mode 100644 index 00000000000..5ef7bfecf48 --- /dev/null +++ b/velox/connectors/clp/ClpTableHandle.cpp @@ -0,0 +1,27 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "velox/connectors/clp/ClpTableHandle.h" + +namespace facebook::velox::connector::clp { +std::string ClpTableHandle::toString() const { + return ConnectorTableHandle::toString(); +} + +folly::dynamic ClpTableHandle::serialize() const { + return ConnectorTableHandle::serialize(); +} +} // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpTableHandle.h b/velox/connectors/clp/ClpTableHandle.h new file mode 100644 index 00000000000..801a9ff163d --- /dev/null +++ b/velox/connectors/clp/ClpTableHandle.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "velox/connectors/Connector.h" + +namespace facebook::velox::connector::clp { +class ClpTableHandle : public ConnectorTableHandle { + public: + enum class StorageType { + kFS, + kS3, + }; + + ClpTableHandle( + const std::string& connectorId, + const std::string& tableName, + StorageType storageType, + std::shared_ptr kqlQuery) + : ConnectorTableHandle(connectorId), + tableName_(tableName), + storageType_(storageType), + kqlQuery_(std::move(kqlQuery)) {} + + [[nodiscard]] const std::string& tableName() const { + return tableName_; + } + + [[nodiscard]] const StorageType storageType() const { + return storageType_; + } + + [[nodiscard]] const std::shared_ptr& kqlQuery() const { + return kqlQuery_; + } + + std::string toString() const override; + + folly::dynamic serialize() const override; + + private: + const std::string tableName_; + const StorageType storageType_; + std::shared_ptr kqlQuery_; +}; +} // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/search_lib/CMakeLists.txt b/velox/connectors/clp/search_lib/CMakeLists.txt new file mode 100644 index 00000000000..3e41e905a70 --- /dev/null +++ b/velox/connectors/clp/search_lib/CMakeLists.txt @@ -0,0 +1,129 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(CLP_EXTERNAL_BINARY_DIR ${CMAKE_BINARY_DIR}/external/clp) +add_subdirectory(${clp_SOURCE_DIR}/components/core/src/clp/string_utils + ${CLP_EXTERNAL_BINARY_DIR}/string_utils) +set(YSTDLIB_CPP_BUILD_TESTING OFF) +add_subdirectory(${clp_SOURCE_DIR}/components/core/submodules/ystdlib-cpp + ${CLP_EXTERNAL_BINARY_DIR}/ystdlib-cpp EXCLUDE_FROM_ALL) + +string(LENGTH "${CMAKE_SOURCE_DIR}/" SOURCE_PATH_SIZE) + +antlr_target( + KqlParser + ${CLP_SRC_DIR}/clp_s/search/kql/Kql.g4 + LEXER + PARSER + VISITOR + PACKAGE + kql) + +set(CLP_SRC_FILES + ${ANTLR_KqlParser_CXX_OUTPUTS} + ${CLP_SRC_DIR}/clp_s/ArchiveReader.cpp + ${CLP_SRC_DIR}/clp_s/ArchiveReaderAdaptor.cpp + ${CLP_SRC_DIR}/clp_s/ColumnReader.cpp + ${CLP_SRC_DIR}/clp_s/DictionaryEntry.cpp + ${CLP_SRC_DIR}/clp_s/DictionaryWriter.cpp + ${CLP_SRC_DIR}/clp_s/FileReader.cpp + ${CLP_SRC_DIR}/clp_s/FileWriter.cpp + ${CLP_SRC_DIR}/clp_s/InputConfig.cpp + ${CLP_SRC_DIR}/clp_s/PackedStreamReader.cpp + ${CLP_SRC_DIR}/clp_s/ReaderUtils.cpp + ${CLP_SRC_DIR}/clp_s/Schema.cpp + ${CLP_SRC_DIR}/clp_s/SchemaMap.cpp + ${CLP_SRC_DIR}/clp_s/SchemaReader.cpp + ${CLP_SRC_DIR}/clp_s/SchemaTree.cpp + ${CLP_SRC_DIR}/clp_s/TimestampDictionaryReader.cpp + ${CLP_SRC_DIR}/clp_s/TimestampDictionaryWriter.cpp + ${CLP_SRC_DIR}/clp_s/TimestampEntry.cpp + ${CLP_SRC_DIR}/clp_s/TimestampPattern.cpp + ${CLP_SRC_DIR}/clp_s/Utils.cpp + ${CLP_SRC_DIR}/clp_s/VariableEncoder.cpp + ${CLP_SRC_DIR}/clp_s/VariableDecoder.cpp + ${CLP_SRC_DIR}/clp_s/ZstdCompressor.cpp + ${CLP_SRC_DIR}/clp_s/ZstdDecompressor.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/AndExpr.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/BooleanLiteral.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/ColumnDescriptor.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/ConstantProp.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/ConvertToExists.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/DateLiteral.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/EmptyExpr.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/Expression.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/FilterExpr.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/Integral.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/NarrowTypes.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/NullLiteral.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/OrExpr.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/OrOfAndForm.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/SearchUtils.cpp + ${CLP_SRC_DIR}/clp_s/search/ast/StringLiteral.cpp + ${CLP_SRC_DIR}/clp_s/search/clp_search/EncodedVariableInterpreter.cpp + ${CLP_SRC_DIR}/clp_s/search/clp_search/Grep.cpp + ${CLP_SRC_DIR}/clp_s/search/clp_search/Query.cpp + ${CLP_SRC_DIR}/clp_s/search/EvaluateTimestampIndex.cpp + ${CLP_SRC_DIR}/clp_s/search/kql/kql.cpp + ${CLP_SRC_DIR}/clp_s/search/Projection.cpp + ${CLP_SRC_DIR}/clp_s/search/QueryRunner.cpp + ${CLP_SRC_DIR}/clp_s/search/SchemaMatch.cpp + ${CLP_SRC_DIR}/clp/aws/AwsAuthenticationSigner.cpp + ${CLP_SRC_DIR}/clp/BoundedReader.cpp + ${CLP_SRC_DIR}/clp/CurlDownloadHandler.cpp + ${CLP_SRC_DIR}/clp/CurlGlobalInstance.cpp + ${CLP_SRC_DIR}/clp/Defs.h + ${CLP_SRC_DIR}/clp/FileReader.cpp + ${CLP_SRC_DIR}/clp/hash_utils.cpp + ${CLP_SRC_DIR}/clp/NetworkReader.cpp + ${CLP_SRC_DIR}/clp/ReaderInterface.cpp + ${CLP_SRC_DIR}/clp/Thread.cpp) + +velox_add_library( + clp-s-search + STATIC + ${CLP_SRC_FILES} + ClpVectorLoader.cpp + ClpVectorLoader.h + ClpCursor.cpp + ClpCursor.h + ClpQueryRunner.cpp + ClpQueryRunner.h) +target_compile_features(clp-s-search PRIVATE cxx_std_20) +target_compile_definitions(clp-s-search + PUBLIC SOURCE_PATH_SIZE=${SOURCE_PATH_SIZE}) +velox_include_directories( + clp-s-search + PUBLIC ${ANTLR_KqlParser_OUTPUT_DIR} + ${clp_SOURCE_DIR}/components/core/submodules ${CLP_SRC_DIR}) + +velox_link_libraries( + clp-s-search + PUBLIC msgpack-cxx spdlog::spdlog + PRIVATE + absl::flat_hash_map + antlr4_static + Boost::filesystem + Boost::iostreams + Boost::program_options + Boost::url + clp::string_utils + ${CURL_LIBRARIES} + glog::glog + OpenSSL::Crypto + simdjson::simdjson + velox_vector + ystdlib::containers + ystdlib::error_handling + zstd::zstd) diff --git a/velox/connectors/clp/search_lib/ClpCursor.cpp b/velox/connectors/clp/search_lib/ClpCursor.cpp new file mode 100644 index 00000000000..e2ae5276c95 --- /dev/null +++ b/velox/connectors/clp/search_lib/ClpCursor.cpp @@ -0,0 +1,254 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "clp_s/search/EvaluateTimestampIndex.hpp" +#include "clp_s/search/ast/ConvertToExists.hpp" +#include "clp_s/search/ast/EmptyExpr.hpp" +#include "clp_s/search/ast/NarrowTypes.hpp" +#include "clp_s/search/ast/OrOfAndForm.hpp" +#include "clp_s/search/ast/SearchUtils.hpp" +#include "clp_s/search/kql/kql.hpp" +#include "velox/connectors/clp/search_lib/ClpCursor.h" + +using namespace clp_s; +using namespace clp_s::search; +using namespace clp_s::search::ast; + +namespace facebook::velox::connector::clp::search_lib { +ClpCursor::ClpCursor(InputSource inputSource, std::string archivePath) + : errorCode_(ErrorCode::QueryNotInitialized), + inputSource_(inputSource), + archivePath_(std::move(archivePath)), + archiveReader_(std::make_shared()) {} + +ClpCursor::~ClpCursor() { + if (currentArchiveLoaded_) { + archiveReader_->close(); + } +} + +void ClpCursor::executeQuery( + const std::string& query, + const std::vector& outputColumns) { + query_ = query; + outputColumns_ = outputColumns; + errorCode_ = preprocessQuery(); +} + +uint64_t ClpCursor::fetchNext( + uint64_t numRows, + const std::shared_ptr>& filteredRowIndices) { + if (ErrorCode::Success != errorCode_) { + return 0; + } + + if (false == currentArchiveLoaded_) { + errorCode_ = loadArchive(); + if (ErrorCode::Success != errorCode_) { + return 0; + } + + archiveReader_->open_packed_streams(); + currentArchiveLoaded_ = true; + queryRunner_ = std::make_shared( + schemaMatch_, expr_, archiveReader_, false, projection_); + queryRunner_->global_init(); + } + + while (currentSchemaIndex_ < matchedSchemas_.size()) { + if (false == currentSchemaTableLoaded_) { + currentSchemaId_ = matchedSchemas_[currentSchemaIndex_]; + if (EvaluatedValue::False == + queryRunner_->schema_init(currentSchemaId_)) { + currentSchemaIndex_ += 1; + currentSchemaTableLoaded_ = false; + errorCode_ = ErrorCode::DictionaryNotFound; + continue; + } + + auto& reader = + archiveReader_->read_schema_table(currentSchemaId_, false, false); + reader.initialize_filter_with_column_map(queryRunner_.get()); + + errorCode_ = ErrorCode::Success; + currentSchemaTableLoaded_ = true; + } + + auto rowsScanned = queryRunner_->fetchNext(numRows, filteredRowIndices); + if (false == filteredRowIndices->empty()) { + return rowsScanned; + } + + currentSchemaIndex_ += 1; + currentSchemaTableLoaded_ = false; + } + + return 0; +} + +const std::vector& ClpCursor::getProjectedColumns() + const { + if (queryRunner_) { + return queryRunner_->getProjectedColumns(); + } + static std::vector kEmpty; + return kEmpty; +} + +ErrorCode ClpCursor::preprocessQuery() { + auto queryStream = std::istringstream(query_); + expr_ = kql::parse_kql_expression(queryStream); + if (nullptr == expr_) { + VLOG(2) << "Failed to parse query '" << query_ << "'"; + return ErrorCode::InvalidQuerySyntax; + } + + if (std::dynamic_pointer_cast(expr_)) { + VLOG(2) << "Query '" << query_ << "' is logically false"; + return ErrorCode::LogicalError; + } + + OrOfAndForm standardizePass; + if (expr_ = standardizePass.run(expr_); + std::dynamic_pointer_cast(expr_)) { + VLOG(2) << "Query '" << query_ << "' is logically false"; + return ErrorCode::LogicalError; + } + + NarrowTypes narrowPass; + if (expr_ = narrowPass.run(expr_); + std::dynamic_pointer_cast(expr_)) { + VLOG(2) << "Query '" << query_ << "' is logically false"; + return ErrorCode::LogicalError; + } + + ConvertToExists convertPass; + if (expr_ = convertPass.run(expr_); + std::dynamic_pointer_cast(expr_)) { + VLOG(2) << "Query '" << query_ << "' is logically false"; + return ErrorCode::LogicalError; + } + + return ErrorCode::Success; +} + +ErrorCode ClpCursor::loadArchive() { + auto networkAuthOption = inputSource_ == InputSource::Filesystem + ? NetworkAuthOption{.method = AuthMethod::None} + : NetworkAuthOption{.method = AuthMethod::S3PresignedUrlV4}; + + try { + archiveReader_->open( + get_path_object_for_raw_path(archivePath_), networkAuthOption); + } catch (std::exception& e) { + VLOG(2) << "Failed to open archive file: " << e.what(); + return ErrorCode::InternalError; + } + + auto timestampDict = archiveReader_->get_timestamp_dictionary(); + auto schemaTree = archiveReader_->get_schema_tree(); + auto schemaMap = archiveReader_->get_schema_map(); + + EvaluateTimestampIndex timestampIndex(timestampDict); + if (clp_s::EvaluatedValue::False == timestampIndex.run(expr_)) { + VLOG(2) << "No matching timestamp ranges for query '" << query_ << "'"; + return ErrorCode::InvalidTimestampRange; + } + + schemaMatch_ = std::make_shared(schemaTree, schemaMap); + if (expr_ = schemaMatch_->run(expr_); + std::dynamic_pointer_cast(expr_)) { + VLOG(2) << "No matching schemas for query '" << query_ << "'"; + return ErrorCode::SchemaNotFound; + } + + projection_ = std::make_shared( + outputColumns_.empty() ? ReturnAllColumns : ReturnSelectedColumns); + try { + for (auto const& column : outputColumns_) { + std::vector descriptorTokens; + std::string descriptorNamespace; + if (false == + tokenize_column_descriptor( + column.name, descriptorTokens, descriptorNamespace)) { + VLOG(2) << "Can not tokenize invalid column: '" << column.name << "'"; + return ErrorCode::InternalError; + } + + auto columnDescriptor = ColumnDescriptor::create_from_escaped_tokens( + descriptorTokens, descriptorNamespace); + switch (column.type) { + case ColumnType::String: + columnDescriptor->set_matching_types( + LiteralType::ClpStringT | LiteralType::VarStringT | + LiteralType::EpochDateT); + break; + case ColumnType::Integer: + columnDescriptor->set_matching_types(LiteralType::IntegerT); + break; + case ColumnType::Float: + columnDescriptor->set_matching_types(LiteralType::FloatT); + break; + case ColumnType::Boolean: + columnDescriptor->set_matching_types(LiteralType::BooleanT); + break; + case ColumnType::Array: + columnDescriptor->set_matching_types(LiteralType::ArrayT); + break; + default: + break; + } + + projection_->add_column(columnDescriptor); + } + } catch (TraceableException& e) { + VLOG(2) << e.what(); + return ErrorCode::InternalError; + } + projection_->resolve_columns(schemaTree); + archiveReader_->set_projection(projection_); + + archiveReader_->read_metadata(); + + matchedSchemas_.clear(); + for (auto schemaId : archiveReader_->get_schema_ids()) { + if (schemaMatch_->schema_matched(schemaId)) { + matchedSchemas_.push_back(schemaId); + } + } + + if (matchedSchemas_.empty()) { + return ErrorCode::SchemaNotFound; + } + + EvaluateTimestampIndex timestamp_index(timestampDict); + if (EvaluatedValue::False == timestamp_index.run(expr_)) { + VLOG(2) << "No matching timestamp ranges for query '" << query_ << "'"; + return ErrorCode::InvalidTimestampRange; + } + + archiveReader_->read_variable_dictionary(); + archiveReader_->read_log_type_dictionary(); + archiveReader_->read_array_dictionary(); + + currentSchemaIndex_ = 0; + currentSchemaTableLoaded_ = false; + return ErrorCode::Success; +} + +} // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ClpCursor.h b/velox/connectors/clp/search_lib/ClpCursor.h new file mode 100644 index 00000000000..bafd5f00c98 --- /dev/null +++ b/velox/connectors/clp/search_lib/ClpCursor.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "clp_s/ArchiveReader.hpp" +#include "clp_s/search/SchemaMatch.hpp" +#include "clp_s/search/ast/Expression.hpp" +#include "velox/connectors/clp/search_lib/ClpQueryRunner.h" + +namespace facebook::velox::connector::clp::search_lib { + +enum class ErrorCode { + Success, + QueryNotInitialized, + InvalidQuerySyntax, + SchemaNotFound, + LogicalError, + DictionaryNotFound, + InvalidTimestampRange, + InternalError +}; + +enum class ColumnType { String, Integer, Float, Array, Boolean, Unknown = -1 }; + +struct Field { + ColumnType type; + std::string name; +}; + +/** + * This class is a query execution interface that manages the lifecycle of a + * query on a CLP-S archive, including parsing and validating the query, loading + * the relevant schemas and archives, applying filters, and iterating over the + * results. It abstracts away the low-level details of archive access and schema + * matching while supporting projection and batch-oriented retrieval of filtered + * rows. + */ +class ClpCursor { + public: + // Constructor + explicit ClpCursor(clp_s::InputSource inputSource, std::string archivePath); + + // Destructor + ~ClpCursor(); + + /** + * Executes a query. This function parses, validates, and prepares the given + * query for execution. + * @param query The KQL query to execute. + * @param outputColumns A vector specifying the columns to be included in the + * query result. + */ + void executeQuery( + const std::string& query, + const std::vector& outputColumns); + + /** + * Fetches the next set of rows from the cursor.If the archive and schema are + * not yet loaded, this function will perform the necessary loading. + * @param numRows The maximum number of rows to fetch. + * @param filteredRowIndices A vector of row indices that match the filter. + * @return The number of rows scanned. + */ + uint64_t fetchNext( + uint64_t numRows, + const std::shared_ptr>& filteredRowIndices); + + /** + * Retrieves the projected columns + * @return A vector of BaseColumnReader pointers representing the projected + * columns. + */ + const std::vector& getProjectedColumns() const; + + private: + /** + * Preprocesses the query, performing parsing, validation, and optimization. + * @return The error code. + */ + ErrorCode preprocessQuery(); + + /** + * Loads the archive at the current index. + * @return The error code. + */ + ErrorCode loadArchive(); + + ErrorCode errorCode_; + + clp_s::InputSource inputSource_{clp_s::InputSource::Filesystem}; + std::string archivePath_; + std::string query_; + std::vector outputColumns_; + std::vector matchedSchemas_; + size_t currentSchemaIndex_{0}; + int32_t currentSchemaId_{-1}; + bool currentSchemaTableLoaded_{false}; + bool currentArchiveLoaded_{false}; + + std::shared_ptr expr_; + std::shared_ptr schemaMatch_; + std::shared_ptr queryRunner_; + std::shared_ptr projection_; + std::shared_ptr archiveReader_; +}; +} // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ClpQueryRunner.cpp b/velox/connectors/clp/search_lib/ClpQueryRunner.cpp new file mode 100644 index 00000000000..6c1395d6345 --- /dev/null +++ b/velox/connectors/clp/search_lib/ClpQueryRunner.cpp @@ -0,0 +1,82 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "velox/connectors/clp/search_lib/ClpQueryRunner.h" +#include "clp_s/search/clp_search/Grep.hpp" +#include "velox/vector/ComplexVector.h" + +using namespace clp_s; +using namespace clp_s::search; +using namespace clp_s::search::clp_search; + +namespace facebook::velox::connector::clp::search_lib { +void ClpQueryRunner::init( + clp_s::SchemaReader* schemaReader, + std::unordered_map const& columnMap) { + numMessages_ = schemaReader->get_num_messages(); + curMessage_ = 0; + clear_readers(); + + projectedColumns_.clear(); + auto matchingNodesList = projection_->get_ordered_matching_nodes(); + for (const auto& nodeIds : matchingNodesList) { + if (nodeIds.empty()) { + projectedColumns_.push_back(nullptr); + continue; + } + + // Try to find a matching column in columnMap + bool foundReader = false; + for (const auto nodeId : nodeIds) { + auto columnIt = columnMap.find(nodeId); + if (columnIt != columnMap.end()) { + projectedColumns_.push_back(columnIt->second); + foundReader = true; + break; + } + } + + if (!foundReader) { + projectedColumns_.push_back(nullptr); + } + } + + for (auto& [columnId, columnReader] : columnMap) { + initialize_reader(columnId, columnReader); + } +} + +uint64_t ClpQueryRunner::fetchNext( + uint64_t numRows, + const std::shared_ptr>& filteredRowIndices) { + size_t rowsfiltered = 0; + size_t rowsScanned = 0; + while (curMessage_ < numMessages_) { + if (filter(curMessage_)) { + filteredRowIndices->emplace_back(curMessage_); + rowsfiltered += 1; + } + + curMessage_ += 1; + rowsScanned += 1; + if (rowsfiltered >= numRows) { + break; + } + } + return rowsScanned; +} + +} // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ClpQueryRunner.h b/velox/connectors/clp/search_lib/ClpQueryRunner.h new file mode 100644 index 00000000000..643988f97f6 --- /dev/null +++ b/velox/connectors/clp/search_lib/ClpQueryRunner.h @@ -0,0 +1,85 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "clp_s/SchemaReader.hpp" +#include "clp_s/search/Projection.hpp" +#include "clp_s/search/QueryRunner.hpp" +#include "clp_s/search/SchemaMatch.hpp" +#include "clp_s/search/ast/Expression.hpp" +#include "velox/vector/FlatVector.h" + +namespace facebook::velox::connector::clp::search_lib { +/** + * This class extends the generic QueryRunner to support column projection and + * row filtering over CLP-S archives. It is used by the Velox CLP connector to + * efficiently identify matching rows and project relevant columns, which are + * then consumed by the ClpVectorLoader. + */ +class ClpQueryRunner : public clp_s::search::QueryRunner { + public: + // Constructor + ClpQueryRunner( + const std::shared_ptr& match, + const std::shared_ptr& expr, + const std::shared_ptr& archiveReader, + bool ignoreCase, + const std::shared_ptr& projection) + : clp_s::search::QueryRunner(match, expr, archiveReader, ignoreCase), + projection_(projection) {} + + /** + * Initializes the filter with schema information and column readers. + * @param schemaReader A pointer to the SchemaReader + * @param columnMap An unordered map associating column IDs with + * BaseColumnReader pointers. + */ + void init( + clp_s::SchemaReader* schemaReader, + std::unordered_map const& columnMap) + override; + + /** + * Fetches the next set of rows from the cursor. + * @param numRows The maximum number of rows to fetch. + * @param filteredRowIndices A vector to store the row indices that match the + * filter. + */ + uint64_t fetchNext( + uint64_t numRows, + const std::shared_ptr>& filteredRowIndices); + + /** + * @return A reference to the vector of BaseColumnReader pointers that + * represent the columns involved in the scanning operation. + */ + std::vector& getProjectedColumns() { + return projectedColumns_; + } + + private: + std::shared_ptr expr_; + std::shared_ptr schemaTree_; + std::shared_ptr projection_; + std::vector projectedColumns_; + + uint64_t curMessage_{}; + uint64_t numMessages_{}; +}; +} // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ClpVectorLoader.cpp b/velox/connectors/clp/search_lib/ClpVectorLoader.cpp new file mode 100644 index 00000000000..c05908bf131 --- /dev/null +++ b/velox/connectors/clp/search_lib/ClpVectorLoader.cpp @@ -0,0 +1,155 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "velox/connectors/clp/search_lib/ClpVectorLoader.h" +#include "velox/vector/ComplexVector.h" + +namespace facebook::velox::connector::clp::search_lib { +ClpVectorLoader::ClpVectorLoader( + clp_s::BaseColumnReader* columnReader, + ColumnType nodeType, + std::shared_ptr> filteredRowIndices) + : columnReader_(columnReader), + nodeType_(nodeType), + filteredRowIndices_(std::move(filteredRowIndices)) {} + +template +void ClpVectorLoader::populateData(RowSet rows, VectorPtr vector) { + if (columnReader_ == nullptr) { + for (int vectorIndex : rows) { + vector->setNull(vectorIndex, true); + } + return; + } + + for (int vectorIndex : rows) { + auto messageIndex = (*filteredRowIndices_)[vectorIndex]; + + if constexpr (std::is_same_v) { + auto string_value = + std::get(columnReader_->extract_value(messageIndex)); + vector->set(vectorIndex, StringView(string_value)); + } else { + vector->set( + vectorIndex, std::get(columnReader_->extract_value(messageIndex))); + } + + vector->setNull(vectorIndex, false); + } +} + +void ClpVectorLoader::loadInternal( + RowSet rows, + ValueHook* hook, + vector_size_t resultSize, + VectorPtr* result) { + if (!result) { + VELOX_USER_FAIL("vector is null"); + } + + auto vector = *result; + switch (nodeType_) { + case ColumnType::Integer: { + auto intVector = vector->asFlatVector(); + populateData(rows, intVector); + break; + } + case ColumnType::Float: { + auto floatVector = vector->asFlatVector(); + populateData(rows, floatVector); + break; + } + case ColumnType::Boolean: { + auto boolVector = vector->asFlatVector(); + populateData(rows, boolVector); + break; + } + case ColumnType::String: { + auto stringVector = vector->asFlatVector(); + populateData(rows, stringVector); + break; + } + case ColumnType::Array: { + auto arrayVector = std::dynamic_pointer_cast(vector); + auto elements = arrayVector->elements()->asFlatVector(); + vector_size_t elementIndex = 0; + + for (int vectorIndex : rows) { + auto messageIndex = (*filteredRowIndices_)[vectorIndex]; + + auto jsonString = + std::get(columnReader_->extract_value(messageIndex)); + + simdjson::padded_string padded(jsonString); + simdjson::ondemand::document doc; + try { + doc = arrayParser_->iterate(padded); + } catch (const simdjson::simdjson_error& e) { + VELOX_FAIL("JSON parse error at row {}: {}", vectorIndex, e.what()); + } + + simdjson::ondemand::array array; + try { + array = doc.get_array(); + } catch (const simdjson::simdjson_error& e) { + VELOX_FAIL( + "Expected JSON array at row {}: {}", vectorIndex, e.what()); + } + + std::vector arrayElements; + for (auto arrayElement : array) { + arrayElements.emplace_back( + simdjson::to_json_string(arrayElement).value()); + } + + if (elementIndex + arrayElements.size() > elements->size()) { + size_t newSize = std::max( + elementIndex + arrayElements.size(), + static_cast(elements->size()) * 2); + elements->resize(newSize); + } + + arrayVector->setOffsetAndSize( + vectorIndex, elementIndex, arrayElements.size()); + for (auto& arrayElement : arrayElements) { + elements->set(elementIndex++, StringView(arrayElement)); + } + + arrayVector->setNull(vectorIndex, false); + } + break; + } + default: + VELOX_FAIL("Unsupported column type"); + } +} + +// Explicit template instantiations for linker +template void ClpVectorLoader::populateData( + RowSet rows, + FlatVector* vector); +template void ClpVectorLoader::populateData( + RowSet rows, + FlatVector* vector); +template void ClpVectorLoader::populateData( + RowSet rows, + FlatVector* vector); +template void ClpVectorLoader::populateData( + RowSet rows, + FlatVector* vector); +} // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ClpVectorLoader.h b/velox/connectors/clp/search_lib/ClpVectorLoader.h new file mode 100644 index 00000000000..98779e404af --- /dev/null +++ b/velox/connectors/clp/search_lib/ClpVectorLoader.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "clp_s/ColumnReader.hpp" +#include "velox/connectors/clp/search_lib/ClpCursor.h" +#include "velox/vector/LazyVector.h" + +namespace facebook::velox::connector::clp::search_lib { +/** + * ClpVectorLoader is a custom Velox VectorLoader that populates Velox vectors + * from a CLP-based column reader. It supports various column types including + * integers, floats, booleans, strings, and arrays of strings. + */ +class ClpVectorLoader : public VectorLoader { + public: + ClpVectorLoader( + clp_s::BaseColumnReader* columnReader, + ColumnType nodeType, + std::shared_ptr> filteredRowIndices); + + private: + void loadInternal( + RowSet rows, + ValueHook* hook, + vector_size_t resultSize, + VectorPtr* result) override; + + template + void populateData(RowSet rows, VectorPtr vector); + + clp_s::BaseColumnReader* columnReader_; + ColumnType nodeType_; + std::shared_ptr> filteredRowIndices_; + + inline static thread_local std::unique_ptr + arrayParser_ = std::make_unique(); +}; +} // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/tests/CMakeLists.txt b/velox/connectors/clp/tests/CMakeLists.txt new file mode 100644 index 00000000000..6c0c2147366 --- /dev/null +++ b/velox/connectors/clp/tests/CMakeLists.txt @@ -0,0 +1,27 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +add_executable(velox_clp_connector_test ClpConnectorTest.cpp) + +add_test(velox_clp_connector_test velox_clp_connector_test) + +target_link_libraries( + velox_clp_connector_test + velox_clp_connector + velox_vector_test_lib + velox_exec_test_lib + GTest::gtest + GTest::gtest_main) + +file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/examples + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/velox/connectors/clp/tests/ClpConnectorTest.cpp b/velox/connectors/clp/tests/ClpConnectorTest.cpp new file mode 100644 index 00000000000..c7473bf9980 --- /dev/null +++ b/velox/connectors/clp/tests/ClpConnectorTest.cpp @@ -0,0 +1,317 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "velox/common/base/Fs.h" +#include "velox/common/base/tests/GTestUtils.h" +#include "velox/connectors/clp/ClpColumnHandle.h" +#include "velox/connectors/clp/ClpConnector.h" +#include "velox/connectors/clp/ClpConnectorSplit.h" +#include "velox/connectors/clp/ClpTableHandle.h" +#include "velox/exec/tests/utils/AssertQueryBuilder.h" +#include "velox/exec/tests/utils/OperatorTestBase.h" +#include "velox/exec/tests/utils/PlanBuilder.h" + +namespace { + +using namespace facebook::velox; +using namespace facebook::velox::connector::clp; + +using facebook::velox::exec::test::PlanBuilder; + +class ClpConnectorTest : public exec::test::OperatorTestBase { + public: + const std::string kClpConnectorId = "test-clp"; + + void SetUp() override { + OperatorTestBase::SetUp(); + connector::registerConnectorFactory( + std::make_shared()); + auto clpConnector = + connector::getConnectorFactory( + connector::clp::ClpConnectorFactory::kClpConnectorName) + ->newConnector( + kClpConnectorId, + std::make_shared( + std::unordered_map{ + {"clp.split-source", "local"}})); + connector::registerConnector(clpConnector); + } + + void TearDown() override { + connector::unregisterConnector(kClpConnectorId); + connector::unregisterConnectorFactory( + connector::clp::ClpConnectorFactory::kClpConnectorName); + OperatorTestBase::TearDown(); + } + + exec::Split makeClpSplit(const std::string& splitPath) { + return exec::Split( + std::make_shared(kClpConnectorId, splitPath)); + } + + RowVectorPtr getResults( + const core::PlanNodePtr& planNode, + std::vector&& splits) { + return exec::test::AssertQueryBuilder(planNode) + .splits(std::move(splits)) + .copyResults(pool()); + } + + static std::string getExampleFilePath(const std::string& filePath) { + std::string current_path = fs::current_path().string(); + return current_path + "/examples/" + filePath; + } +}; + +TEST_F(ClpConnectorTest, test1NoPushdown) { + auto plan = PlanBuilder() + .startTableScan() + .outputType( + ROW({"requestId", "userId", "method"}, + {VARCHAR(), VARCHAR(), VARCHAR()})) + .tableHandle(std::make_shared( + kClpConnectorId, + "test_1", + ClpTableHandle::StorageType::kFS, + nullptr)) + .assignments({ + {"requestId", + std::make_shared( + "requestId", "requestId", VARCHAR(), true)}, + {"userId", + std::make_shared( + "userId", "userId", VARCHAR(), true)}, + {"method", + std::make_shared( + "method", "method", VARCHAR(), true)}, + }) + .endTableScan() + .filter("method = 'GET'") + .planNode(); + + auto output = + getResults(plan, {makeClpSplit(getExampleFilePath("test_1.clps"))}); + auto expected = makeRowVector( + {// requestId + makeFlatVector( + {"req-100", "req-105", "req-107", "req-109", "req-102"}), + // userId + makeNullableFlatVector( + {"user201", "user204", "user202", "user203", std::nullopt}), + // method + makeFlatVector({ + "GET", + "GET", + "GET", + "GET", + "GET", + })}); + test::assertEqualVectors(expected, output); +} + +TEST_F(ClpConnectorTest, test1Pushdown) { + auto plan = PlanBuilder() + .startTableScan() + .outputType( + ROW({"requestId", "userId", "path"}, + {VARCHAR(), VARCHAR(), VARCHAR()})) + .tableHandle(std::make_shared( + kClpConnectorId, + "test_1", + ClpTableHandle::StorageType::kFS, + std::make_shared( + "method: \"POST\" AND status: 200"))) + .assignments({ + {"requestId", + std::make_shared( + "requestId", "requestId", VARCHAR(), true)}, + {"userId", + std::make_shared( + "userId", "userId", VARCHAR(), true)}, + {"path", + std::make_shared( + "path", "path", VARCHAR(), true)}, + }) + .endTableScan() + .planNode(); + + auto output = + getResults(plan, {makeClpSplit(getExampleFilePath("test_1.clps"))}); + auto expected = + makeRowVector({// requestId + makeFlatVector({"req-106"}), + // userId + makeNullableFlatVector({std::nullopt}), + // path + makeFlatVector({"/auth/login"})}); + test::assertEqualVectors(expected, output); +} + +TEST_F(ClpConnectorTest, test2NoPushdown) { + auto plan = + PlanBuilder(pool_.get()) + .startTableScan() + .outputType( + ROW({"timestamp", "event"}, + {VARCHAR(), + ROW({"type", "subtype", "severity"}, + {VARCHAR(), VARCHAR(), VARCHAR()})})) + .tableHandle(std::make_shared( + kClpConnectorId, + "test_2", + ClpTableHandle::StorageType::kFS, + nullptr)) + .assignments( + {{"timestamp", + std::make_shared( + "timestamp", "timestamp", VARCHAR(), true)}, + {"event", + std::make_shared( + "event", + "event", + ROW({"type", "subtype", "severity"}, + {VARCHAR(), VARCHAR(), VARCHAR()}), + true)}}) + .endTableScan() + .filter( + "event.severity IN ('WARNING', 'ERROR') AND " + "((event.type = 'network' AND event.subtype = 'connection') OR " + "(event.type = 'storage' AND event.subtype LIKE 'disk_usage%'))") + .planNode(); + + auto output = + getResults(plan, {makeClpSplit(getExampleFilePath("test_2.clps"))}); + auto expected = + makeRowVector({// timestamp + makeFlatVector({"2025-04-30T08:50:05Z"}), + // event + makeRowVector({ + // event.type + makeFlatVector({"storage"}), + // event.subtype + makeFlatVector({"disk_usage"}), + // event.severity + makeFlatVector({"WARNING"}), + })}); + test::assertEqualVectors(expected, output); +} + +TEST_F(ClpConnectorTest, test2Pushdown) { + auto plan = + PlanBuilder() + .startTableScan() + .outputType( + ROW({"timestamp", "event"}, + {VARCHAR(), + ROW({"type", "subtype", "severity"}, + {VARCHAR(), VARCHAR(), VARCHAR()})})) + .tableHandle(std::make_shared( + kClpConnectorId, + "test_2", + ClpTableHandle::StorageType::kFS, + std::make_shared( + "(event.severity: \"WARNING\" OR event.severity: \"ERROR\") AND " + "((event.type: \"network\" AND event.subtype: \"connection\") OR " + "(event.type: \"storage\" AND event.subtype: \"disk*\"))"))) + .assignments( + {{"timestamp", + std::make_shared( + "timestamp", "timestamp", VARCHAR(), true)}, + {"event", + std::make_shared( + "event", + "event", + ROW({"type", "subtype", "severity"}, + {VARCHAR(), VARCHAR(), VARCHAR()}), + true)}}) + .endTableScan() + .planNode(); + + auto output = + getResults(plan, {makeClpSplit(getExampleFilePath("test_2.clps"))}); + auto expected = + makeRowVector({// timestamp + makeFlatVector({"2025-04-30T08:50:05Z"}), + // event + makeRowVector({ + // event.type + makeFlatVector({"storage"}), + // event.subtype + makeFlatVector({"disk_usage"}), + // event.severity + makeFlatVector({"WARNING"}), + })}); + test::assertEqualVectors(expected, output); +} + +TEST_F(ClpConnectorTest, test2Hybrid) { + auto plan = + PlanBuilder(pool_.get()) + .startTableScan() + .outputType( + ROW({"timestamp", "event"}, + {VARCHAR(), + ROW({"type", "subtype", "severity"}, + {VARCHAR(), VARCHAR(), VARCHAR()})})) + .tableHandle(std::make_shared( + kClpConnectorId, + "test_2", + ClpTableHandle::StorageType::kFS, + std::make_shared( + "((event.type: \"network\" AND event.subtype: \"connection\") OR " + "(event.type: \"storage\" AND event.subtype: \"disk*\"))"))) + .assignments( + {{"timestamp", + std::make_shared( + "timestamp", "timestamp", VARCHAR(), true)}, + {"event", + std::make_shared( + "event", + "event", + ROW({"type", "subtype", "severity"}, + {VARCHAR(), VARCHAR(), VARCHAR()}), + true)}}) + .endTableScan() + .filter("upper(event.severity) IN ('WARNING', 'ERROR')") + .planNode(); + + auto output = + getResults(plan, {makeClpSplit(getExampleFilePath("test_2.clps"))}); + auto expected = + makeRowVector({// timestamp + makeFlatVector({"2025-04-30T08:50:05Z"}), + // event + makeRowVector({ + // event.type + makeFlatVector({"storage"}), + // event.subtype + makeFlatVector({"disk_usage"}), + // event.severity + makeFlatVector({"WARNING"}), + })}); + test::assertEqualVectors(expected, output); +} + +} // namespace + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + folly::Init init{&argc, &argv, false}; + return RUN_ALL_TESTS(); +} diff --git a/velox/connectors/clp/tests/examples/test_1.clps b/velox/connectors/clp/tests/examples/test_1.clps new file mode 100644 index 0000000000000000000000000000000000000000..9538d88c997f215f9e56d58ab18a27340cad9832 GIT binary patch literal 959 zcmeyXf7F11ftewnjR6eyGec-Xe_j}#!%AIWvp7vVG{=v)t%m$N;HgJa>eXXgq(D0S}DH{Q%&IIG7hD|F3xgUSM z(~+R99US*u=g5LRCUO4WEYX{$eXNrEmY3pw*DB!2p*{0NiWXi=2>hAv>G>)8`eWzcdiV(C-bP$a8)xI&22ty}urGP#!Z%rE^mzuZ#(juGOougnZgZesEW zmK**D0>9}ercCnKdFPaxQl;y+&`?J=Ul*5@^&%`ww%pn|e|B7ms@q+iXJ)LQ4>NUF zhOWNL&@g59yNh87ukM7tWSi8f&~)k3vI%;o*f!yFVnEI`Z(#Goi+=i~yjycIu) zv@X2!_wvVcu@VVT=Raj&@L*&(0MyFhz{t)JE!n8G+rh1%CPITuD8ZMZ$l%Q6%;$Be@r_Y#E>8`*Ao3l1ywo z{7h0z%g=-^O>qf*IS+i6a&D{^g0NvEi$a*2dbluSj0 znlk0ch}=(#w$3U=p>#t^oK%jJ^QKqldGY;zUpz0K@8@}*ugq2!00{Vq0pDSUUfBOP z`~M@DNtj;&w-61SyUi_=*7reHx-DaQP$lc|N%vUn5nfw_=L@Gq$H%6Amvh%WtX}mI z7Dfv7lfu_2mAZCj*VGj1F7DO*I6d*pknwe2 zMtRt@P;G^`OZ2dwEo+XMlTDa*GA~*JS)G~#yI~c;a1kgSHxvt~^p-t~lvf|pwV^lA z`_Ub4ck1sSt-~z*#wUPynWIJ45;x4;cc=kfgZdI`ZGWdIYWSl2Px4EB2T&*2K8}?n z&)E!xsj|T`lqFyRYcMP0j!0wq39pqND^EV)+E*8fdOsD{2>lvmhLQSM1A5z4=9%6L zL!efr>Fx)IEyS_UXJqe%+~%GeHqZ?El}lcO|C_5Oi#oiP1#h3~+^F!&D&418J*YiB z8OU?mMaDHoh_v!vfYLWS9b{B=r`4!!T6_YErvydG%7W zy@DE55mrixXHurf8j@ZZ<<eXf*4%d=I2i*Z{^QXZqw|_Ie{&*vD`vF zuaic=vRunJJy`c%a-Y^j^CdY#Vza_ENM`qiL5)aQitlzwgt^d5Po=Twbc?waJHbNW zjCU||)-*Ac@8vyL*h(o3C!$6!^e4L~h))nLep(PQ0B4N0UFe7C*O?7%DEVR9`Sfj)2J{YNVvR)5rz_ z*5Q!0Mw-gDfp&5xWwM=`qPbDae}X-f&Ln$K-nZO8OPxhQQp@XG=oeSnz3Lr!QfL!! z)*)aqh(QJepC@)T$+k{kBUDDlZwnj@>Mz0thxMbz=91gS#@cQ)>wLR+rlcS-wRz)i z-LAiTe$xts79>=`z!-+W3=DAd{Q&?1T!0ayu=6KJSv0D|g3)`R8?Y8=!gi<{qnAUk z4nj`BH1#9^C%xToiWVFsS9}_jC8DxUSP-HbA2`TGZoNBqj(uXObI;6`k~1;L@}1GrkyX(6n~!NakG|EQ z<80e?;%d+%suNFlWizsK#K%e{sp$`YA5Oo^lt#WbSMP8e@ZRrnWBT;Rf~^{;lcBHe zYBrLLUOrCl3~{=|q5ND`8&BPz;(60{voIpVAfIK;CLJ~I({S8(VM;P?VcluQFHyVs zt^6}Z+*r|F;1QHiX4ul|Yn91NS9#(aX&TyfewtbZhD+8DPI&t&q;H$z^5{_TNXc?AsdAUmcJ;_ z+;?hfV!G+1zOA5+{5z%FF^zB+wbrtOmpfpTV6rB;l*shJJwz;Jp6S*-vDNSo7Gyx+ zFiZsD(eXi%;i0tHpt!i*F@jJs@OItc>f%FV&{lhSxYKsU@da`A@Vwnf*uQ!Bj)kb8 z&>#nc0Ip<6mp)nbX=IEsFRN5G4*6>{z5u)*z~)C&qOtR3b3YW3N4qLSwN0_llKq zFuqS|K?A3Z#W>y8bUqj%85@h~J+(+|tfD>PV)wC7`=Hk~4ackrH{Vuwl539)*!vI&ka;03lBhREtg!}`DSbGWp literal 0 HcmV?d00001 diff --git a/velox/connectors/clp/tests/examples/test_2.ndjson b/velox/connectors/clp/tests/examples/test_2.ndjson new file mode 100644 index 00000000000..b6755864344 --- /dev/null +++ b/velox/connectors/clp/tests/examples/test_2.ndjson @@ -0,0 +1,10 @@ +{"timestamp": "2025-04-30T08:50:00Z", "event": {"type": "network", "subtype": "connection", "severity": "INFO", "details": {"source": {"ip": "192.168.3.10"}, "destination": {"ip": "10.0.1.5", "port": 80}}}} +{"timestamp": "2025-04-30T08:50:05Z", "event": {"type": "storage", "subtype": "disk_usage", "severity": "WARNING", "details": {"mount": "/var/log", "usage": {"percent": 92}}}} +{"timestamp": "2025-04-30T08:50:10Z", "event": {"type": "process", "subtype": "start", "severity": "INFO", "details": {"name": "cron", "id": 1234}}} +{"timestamp": "2025-04-30T08:50:15Z", "event": {"type": "authentication", "subtype": "failure", "severity": "ERROR", "user": {"name": "guest", "method": "ssh"}, "details": {"reason": "invalid password", "ip": "203.0.113.20"}}} +{"timestamp": "2025-04-30T08:50:20Z", "event": {"type": "service", "subtype": "status", "severity": "INFO", "details": {"name": "httpd", "state": "running", "uptime": {"seconds": 3600}}}} +{"timestamp": "2025-04-30T08:50:25Z", "event": {"type": "security", "subtype": "alert", "severity": "HIGH", "details": {"rule": {"id": "SQLI-001"}, "target": "/api/data", "payload": "SELECT * FROM users;"}}} +{"timestamp": "2025-04-30T08:50:30Z", "event": {"type": "memory", "subtype": "usage", "severity": "NORMAL", "details": {"total": {"gb": 16}, "used": {"gb": 8.5}, "free": {"gb": 7.5}}}} +{"timestamp": "2025-04-30T08:50:35Z", "event": {"type": "configuration", "subtype": "change", "severity": "INFO", "user": {"name": "admin"}, "details": {"component": "database", "setting": {"name": "timeout", "oldValue": 30, "newValue": 60}}}} +{"timestamp": "2025-04-30T08:50:40Z", "event": {"type": "file", "subtype": "access", "severity": "WARNING", "user": {"name": "user101"}, "details": {"operation": "read", "path": "/etc/shadow", "permissions": "rw-------"}}} +{"timestamp": "2025-04-30T08:50:45Z", "event": {"type": "backup", "subtype": "status", "severity": "INFO", "details": {"job": {"name": "daily_backup", "status": "completed"}, "duration": {"seconds": 1200}, "size": {"gb": 25.3}}}} diff --git a/velox/docs/develop/connectors.rst b/velox/docs/develop/connectors.rst index 2d7ac8a011f..b5b04a70c3a 100644 --- a/velox/docs/develop/connectors.rst +++ b/velox/docs/develop/connectors.rst @@ -29,8 +29,8 @@ Connector Interface * - Connector Factory - Enables creating instances of a particular connector. -Velox provides Hive and TPC-H Connectors out of the box. -Let's see how the above connector interfaces are implemented in the Hive Connector in detail below. +Velox provides Hive, CLP and TPC-H Connectors out of the box. +Let's examine the implementation details of both the Hive and CLP Connectors as examples Hive Connector -------------- @@ -121,3 +121,30 @@ This is the behavior when the proxy settings are enabled: 4. The no_proxy/NO_PROXY list is comma separated. 5. Use . or \*. to indicate domain suffix matching, e.g. `.foobar.com` will match `test.foobar.com` or `foo.foobar.com`. + +CLP Connector +------------- +The CLP Connector is used to read CLP archives stored in a local file system or S3. It implements similar +interfaces as the Hive Connector except for the `DataSink` interface. Here we only describe the `DataSource` +interface and the `ConnectorSplit` interface implementation since `Connector` and `ConnectorFactory` are +similar to the Hive Connector. + +ClpConnectorSplit +~~~~~~~~~~~~~~~~~ +The ClpConnectorSplit describes a data chunk using `split_path`. For now, only archive format is supported. +The `split_path` is the path to the archive file. + +ClpDataSource +~~~~~~~~~~~~~ +The ClpDataSource implements the `addSplit` API that consumes a ClpConnectorSplit and `next` API that +processes the split and returns a batch of rows. + +During initialization, it records the KQL query and archive source (S3 or local). It then iterates through each +output column, accessing its handle to get its type and original name, and specifically for row types, it +recursively traverses the nested structure to process each field; for non-row types, it directly maps the +Velox column type to a CLP column type. + +When a split is added, a `ClpCursor` is created with the archive path and input source. The query is parsed +and simplified into an AST. On `next`, the cursor finds matching row indices and, if any exist, `ClpDataSource` +recursively creates a row vector composed of lazy vectors, which use CLP column readers to decode and load data +as needed during execution. \ No newline at end of file From 0977d33be94633325d222ff0ce387982581adbcc Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 19:26:10 +0000 Subject: [PATCH 02/31] update cmake files --- CMake/ExternalAntlr4Cpp.cmake | 175 ------------------ CMake/FindANTLR.cmake | 174 ----------------- .../antlr4-runtime.cmake | 35 ++++ CMake/resolve_dependency_modules/clp.cmake | 45 +++-- CMake/resolve_dependency_modules/date.cmake | 38 ++++ .../msgpack-cxx.cmake | 7 +- .../nlohmann_json.cmake | 32 ++++ .../resolve_dependency_modules/simdjson.cmake | 3 +- CMake/resolve_dependency_modules/spdlog.cmake | 7 +- .../ystdlib_cpp.cmake | 24 +++ CMakeLists.txt | 27 ++- velox/connectors/clp/CMakeLists.txt | 3 +- .../connectors/clp/search_lib/CMakeLists.txt | 109 +---------- 13 files changed, 191 insertions(+), 488 deletions(-) delete mode 100644 CMake/ExternalAntlr4Cpp.cmake delete mode 100644 CMake/FindANTLR.cmake create mode 100644 CMake/resolve_dependency_modules/antlr4-runtime.cmake create mode 100644 CMake/resolve_dependency_modules/date.cmake create mode 100644 CMake/resolve_dependency_modules/nlohmann_json.cmake create mode 100644 CMake/resolve_dependency_modules/ystdlib_cpp.cmake diff --git a/CMake/ExternalAntlr4Cpp.cmake b/CMake/ExternalAntlr4Cpp.cmake deleted file mode 100644 index 0b2b521e737..00000000000 --- a/CMake/ExternalAntlr4Cpp.cmake +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE: ExternalAntlr4Cpp.cmake taken from -# https://github.com/antlr/antlr4/blob/4.13.1/runtime/Cpp/cmake/ExternalAntlr4Cpp.cmake - -cmake_minimum_required(VERSION 3.7) - -if(POLICY CMP0114) - cmake_policy(SET CMP0114 NEW) -endif() - -include(ExternalProject) - -set(ANTLR4_ROOT ${CMAKE_CURRENT_BINARY_DIR}/antlr4_runtime/src/antlr4_runtime) -set(ANTLR4_INCLUDE_DIRS ${ANTLR4_ROOT}/runtime/Cpp/runtime/src) -set(ANTLR4_GIT_REPOSITORY https://github.com/antlr/antlr4.git) -if(NOT DEFINED ANTLR4_TAG) - # Set to branch name to keep library updated at the cost of needing to rebuild - # after 'clean' Set to commit hash to keep the build stable and does not need - # to rebuild after 'clean' - set(ANTLR4_TAG master) -endif() - -# Ensure that the include dir already exists at configure time (to avoid cmake -# erroring on non-existent include dirs) -file(MAKE_DIRECTORY "${ANTLR4_INCLUDE_DIRS}") - -if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(ANTLR4_OUTPUT_DIR ${ANTLR4_ROOT}/runtime/Cpp/dist/$(Configuration)) -elseif(${CMAKE_GENERATOR} MATCHES "Xcode.*") - set(ANTLR4_OUTPUT_DIR ${ANTLR4_ROOT}/runtime/Cpp/dist/$(CONFIGURATION)) -else() - set(ANTLR4_OUTPUT_DIR ${ANTLR4_ROOT}/runtime/Cpp/dist) -endif() - -if(MSVC) - set(ANTLR4_STATIC_LIBRARIES ${ANTLR4_OUTPUT_DIR}/antlr4-runtime-static.lib) - set(ANTLR4_SHARED_LIBRARIES ${ANTLR4_OUTPUT_DIR}/antlr4-runtime.lib) - set(ANTLR4_RUNTIME_LIBRARIES ${ANTLR4_OUTPUT_DIR}/antlr4-runtime.dll) -else() - set(ANTLR4_STATIC_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.a) - if(MINGW) - set(ANTLR4_SHARED_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.dll.a) - set(ANTLR4_RUNTIME_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.dll) - elseif(CYGWIN) - set(ANTLR4_SHARED_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.dll.a) - set(ANTLR4_RUNTIME_LIBRARIES - ${ANTLR4_OUTPUT_DIR}/cygantlr4-runtime-${ANTLR4_TAG}.dll) - elseif(APPLE) - set(ANTLR4_RUNTIME_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.dylib) - else() - set(ANTLR4_RUNTIME_LIBRARIES ${ANTLR4_OUTPUT_DIR}/libantlr4-runtime.so) - endif() -endif() - -if(${CMAKE_GENERATOR} MATCHES ".* Makefiles") - # This avoids 'warning: jobserver unavailable: using -j1. Add '+' to parent - # make rule.' - set(ANTLR4_BUILD_COMMAND $(MAKE)) -elseif(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(ANTLR4_BUILD_COMMAND ${CMAKE_COMMAND} --build . --config $(Configuration) - --target) -elseif(${CMAKE_GENERATOR} MATCHES "Xcode.*") - set(ANTLR4_BUILD_COMMAND ${CMAKE_COMMAND} --build . --config $(CONFIGURATION) - --target) -else() - set(ANTLR4_BUILD_COMMAND ${CMAKE_COMMAND} --build . --target) -endif() - -if(NOT DEFINED ANTLR4_WITH_STATIC_CRT) - set(ANTLR4_WITH_STATIC_CRT ON) -endif() - -if(ANTLR4_ZIP_REPOSITORY) - ExternalProject_Add( - antlr4_runtime - PREFIX antlr4_runtime - URL ${ANTLR4_ZIP_REPOSITORY} - DOWNLOAD_DIR ${CMAKE_CURRENT_BINARY_DIR} - BUILD_COMMAND "" - BUILD_IN_SOURCE 1 - SOURCE_DIR ${ANTLR4_ROOT} - SOURCE_SUBDIR runtime/Cpp - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} - -DWITH_STATIC_CRT:BOOL=${ANTLR4_WITH_STATIC_CRT} - -DDISABLE_WARNINGS:BOOL=ON - # -DCMAKE_CXX_STANDARD:STRING=17 # if desired, compile the runtime with a - # different C++ standard -DCMAKE_CXX_STANDARD:STRING=${CMAKE_CXX_STANDARD} - # alternatively, compile the runtime with the same C++ standard as the - # outer project - INSTALL_COMMAND "" - EXCLUDE_FROM_ALL 1) -else() - ExternalProject_Add( - antlr4_runtime - PREFIX antlr4_runtime - GIT_REPOSITORY ${ANTLR4_GIT_REPOSITORY} - GIT_TAG ${ANTLR4_TAG} - DOWNLOAD_DIR ${CMAKE_CURRENT_BINARY_DIR} - BUILD_COMMAND "" - BUILD_IN_SOURCE 1 - SOURCE_DIR ${ANTLR4_ROOT} - SOURCE_SUBDIR runtime/Cpp - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} - -DWITH_STATIC_CRT:BOOL=${ANTLR4_WITH_STATIC_CRT} - -DDISABLE_WARNINGS:BOOL=ON - # -DCMAKE_CXX_STANDARD:STRING=17 # if desired, compile the runtime with a - # different C++ standard -DCMAKE_CXX_STANDARD:STRING=${CMAKE_CXX_STANDARD} - # alternatively, compile the runtime with the same C++ standard as the - # outer project - INSTALL_COMMAND "" - EXCLUDE_FROM_ALL 1) -endif() - -# Separate build step as rarely people want both -set(ANTLR4_BUILD_DIR ${ANTLR4_ROOT}) -if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.14.0") - # CMake 3.14 builds in above's SOURCE_SUBDIR when BUILD_IN_SOURCE is true - set(ANTLR4_BUILD_DIR ${ANTLR4_ROOT}/runtime/Cpp) -endif() - -ExternalProject_Add_Step( - antlr4_runtime build_static - COMMAND ${ANTLR4_BUILD_COMMAND} antlr4_static - # Depend on target instead of step (a custom command) to avoid running - # dependent steps concurrently - DEPENDS antlr4_runtime - BYPRODUCTS ${ANTLR4_STATIC_LIBRARIES} - EXCLUDE_FROM_MAIN 1 - WORKING_DIRECTORY ${ANTLR4_BUILD_DIR}) -ExternalProject_Add_StepTargets(antlr4_runtime build_static) - -add_library(antlr4_static STATIC IMPORTED) -add_dependencies(antlr4_static antlr4_runtime-build_static) -set_target_properties( - antlr4_static - PROPERTIES IMPORTED_LOCATION ${ANTLR4_STATIC_LIBRARIES}) -target_include_directories(antlr4_static INTERFACE ${ANTLR4_INCLUDE_DIRS}) - -ExternalProject_Add_Step( - antlr4_runtime build_shared - COMMAND ${ANTLR4_BUILD_COMMAND} antlr4_shared - # Depend on target instead of step (a custom command) to avoid running - # dependent steps concurrently - DEPENDS antlr4_runtime - BYPRODUCTS ${ANTLR4_SHARED_LIBRARIES} ${ANTLR4_RUNTIME_LIBRARIES} - EXCLUDE_FROM_MAIN 1 - WORKING_DIRECTORY ${ANTLR4_BUILD_DIR}) -ExternalProject_Add_StepTargets(antlr4_runtime build_shared) - -add_library(antlr4_shared SHARED IMPORTED) -add_dependencies(antlr4_shared antlr4_runtime-build_shared) -set_target_properties( - antlr4_shared - PROPERTIES IMPORTED_LOCATION ${ANTLR4_RUNTIME_LIBRARIES}) -target_include_directories(antlr4_shared INTERFACE ${ANTLR4_INCLUDE_DIRS}) - -if(ANTLR4_SHARED_LIBRARIES) - set_target_properties(antlr4_shared PROPERTIES IMPORTED_IMPLIB - ${ANTLR4_SHARED_LIBRARIES}) -endif() diff --git a/CMake/FindANTLR.cmake b/CMake/FindANTLR.cmake deleted file mode 100644 index e1d1cfc9d09..00000000000 --- a/CMake/FindANTLR.cmake +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE: FindANTLR.cmake taken from -# https://github.com/antlr/antlr4/blob/4.13.1/runtime/Cpp/cmake/FindANTLR.cmake - -# Set ANTLR version -set(ANTLR4_TAG 4.13.1) -add_definitions(-DANTLR4CPP_STATIC) - -# Define the JAR name and path -set(ANTLR_JAR_NAME antlr-${ANTLR4_TAG}-complete.jar) - -# Set the download directory -set(ANTLR_DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/antlr) -file(MAKE_DIRECTORY ${ANTLR_DOWNLOAD_DIR}) - -# Define the full path to the ANTLR JAR -set(ANTLR_EXECUTABLE ${ANTLR_DOWNLOAD_DIR}/${ANTLR_JAR_NAME}) - -# Download the ANTLR JAR if it does not exist -if(NOT EXISTS ${ANTLR_EXECUTABLE}) - message(STATUS "ANTLR JAR not found. Downloading ANTLR ${ANTLR4_TAG}...") - file( - DOWNLOAD https://www.antlr.org/download/${ANTLR_JAR_NAME} - ${ANTLR_EXECUTABLE} - SHOW_PROGRESS - EXPECTED_HASH - SHA256=bc13a9c57a8dd7d5196888211e5ede657cb64a3ce968608697e4f668251a8487 - TLS_VERIFY ON) -endif() - -# Include the ANTLR C++ runtime integration -include(ExternalAntlr4Cpp) - -find_package(Java 11 REQUIRED COMPONENTS Runtime) - -if(NOT ANTLR_EXECUTABLE) - find_program(ANTLR_EXECUTABLE NAMES antlr.jar antlr4.jar antlr-4.jar - antlr-${ANTLR4_TAG}-complete.jar) -endif() - -if(ANTLR_EXECUTABLE AND Java_JAVA_EXECUTABLE) - execute_process( - COMMAND ${Java_JAVA_EXECUTABLE} -jar ${ANTLR_EXECUTABLE} - OUTPUT_VARIABLE ANTLR_COMMAND_OUTPUT - ERROR_VARIABLE ANTLR_COMMAND_ERROR - RESULT_VARIABLE ANTLR_COMMAND_RESULT - OUTPUT_STRIP_TRAILING_WHITESPACE) - - if(ANTLR_COMMAND_RESULT EQUAL 0) - string(REGEX MATCH "Version [0-9]+(\\.[0-9]+)*" ANTLR_VERSION - ${ANTLR_COMMAND_OUTPUT}) - string(REPLACE "Version " "" ANTLR_VERSION ${ANTLR_VERSION}) - else() - message( - SEND_ERROR "Command '${Java_JAVA_EXECUTABLE} -jar ${ANTLR_EXECUTABLE}' " - "failed with the output '${ANTLR_COMMAND_ERROR}'") - endif() - - macro(ANTLR_TARGET Name InputFile) - set(ANTLR_OPTIONS LEXER PARSER LISTENER VISITOR) - set(ANTLR_ONE_VALUE_ARGS PACKAGE OUTPUT_DIRECTORY DEPENDS_ANTLR) - set(ANTLR_MULTI_VALUE_ARGS COMPILE_FLAGS DEPENDS) - cmake_parse_arguments( - ANTLR_TARGET - "${ANTLR_OPTIONS}" - "${ANTLR_ONE_VALUE_ARGS}" - "${ANTLR_MULTI_VALUE_ARGS}" - ${ARGN}) - set(ANTLR_${Name}_INPUT ${InputFile}) - - get_filename_component(ANTLR_INPUT ${InputFile} NAME_WE) - get_filename_component(ANTLR_INPUT_PARENT_DIR "${InputFile}" DIRECTORY) - - if(ANTLR_TARGET_OUTPUT_DIRECTORY) - set(ANTLR_${Name}_OUTPUT_DIR ${ANTLR_TARGET_OUTPUT_DIRECTORY}) - else() - set(ANTLR_${Name}_OUTPUT_DIR - ${CMAKE_CURRENT_BINARY_DIR}/antlr4cpp_generated_src/${ANTLR_INPUT}) - endif() - - unset(ANTLR_${Name}_CXX_OUTPUTS) - - if((ANTLR_TARGET_LEXER AND NOT ANTLR_TARGET_PARSER) - OR (ANTLR_TARGET_PARSER AND NOT ANTLR_TARGET_LEXER)) - list(APPEND ANTLR_${Name}_CXX_OUTPUTS - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}.h - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}.cpp) - set(ANTLR_${Name}_OUTPUTS - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}.interp - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}.tokens) - else() - list( - APPEND - ANTLR_${Name}_CXX_OUTPUTS - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Lexer.h - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Lexer.cpp - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Parser.h - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Parser.cpp) - list(APPEND ANTLR_${Name}_OUTPUTS - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Lexer.interp - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Lexer.tokens) - endif() - - if(ANTLR_TARGET_LISTENER) - list( - APPEND - ANTLR_${Name}_CXX_OUTPUTS - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}BaseListener.h - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}BaseListener.cpp - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Listener.h - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Listener.cpp) - list(APPEND ANTLR_TARGET_COMPILE_FLAGS -listener) - endif() - - if(ANTLR_TARGET_VISITOR) - list( - APPEND - ANTLR_${Name}_CXX_OUTPUTS - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}BaseVisitor.h - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}BaseVisitor.cpp - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Visitor.h - ${ANTLR_${Name}_OUTPUT_DIR}/${ANTLR_INPUT}Visitor.cpp) - list(APPEND ANTLR_TARGET_COMPILE_FLAGS -visitor) - endif() - - if(ANTLR_TARGET_PACKAGE) - list(APPEND ANTLR_TARGET_COMPILE_FLAGS -package ${ANTLR_TARGET_PACKAGE}) - endif() - - list(APPEND ANTLR_${Name}_OUTPUTS ${ANTLR_${Name}_CXX_OUTPUTS}) - - if(ANTLR_TARGET_DEPENDS_ANTLR) - if(ANTLR_${ANTLR_TARGET_DEPENDS_ANTLR}_INPUT) - list(APPEND ANTLR_TARGET_DEPENDS - ${ANTLR_${ANTLR_TARGET_DEPENDS_ANTLR}_INPUT}) - list(APPEND ANTLR_TARGET_DEPENDS - ${ANTLR_${ANTLR_TARGET_DEPENDS_ANTLR}_OUTPUTS}) - else() - message( - SEND_ERROR "ANTLR target '${ANTLR_TARGET_DEPENDS_ANTLR}' not found") - endif() - endif() - - add_custom_command( - OUTPUT ${ANTLR_${Name}_OUTPUTS} - COMMAND - ${Java_JAVA_EXECUTABLE} -jar ${ANTLR_EXECUTABLE} ${InputFile} -o - ${ANTLR_${Name}_OUTPUT_DIR} -no-listener -Dlanguage=Cpp - ${ANTLR_TARGET_COMPILE_FLAGS} - DEPENDS ${InputFile} ${ANTLR_TARGET_DEPENDS} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMENT "Building ${Name} with ANTLR ${ANTLR_VERSION}") - endmacro(ANTLR_TARGET) - -endif(ANTLR_EXECUTABLE AND Java_JAVA_EXECUTABLE) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( - ANTLR - REQUIRED_VARS ANTLR_EXECUTABLE Java_JAVA_EXECUTABLE - VERSION_VAR ANTLR_VERSION) diff --git a/CMake/resolve_dependency_modules/antlr4-runtime.cmake b/CMake/resolve_dependency_modules/antlr4-runtime.cmake new file mode 100644 index 00000000000..a5d34f4b768 --- /dev/null +++ b/CMake/resolve_dependency_modules/antlr4-runtime.cmake @@ -0,0 +1,35 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +include_guard(GLOBAL) + +set(VELOX_ANTLR4_RUNTIME_VERSION 4.13.2) +set(VELOX_ANTLR4_RUNTIME_BUILD_SHA256_CHECKSUM + 9f18272a9b32b622835a3365f850dd1063d60f5045fb1e12ce475ae6e18a35bb) +set(VELOX_ANTLR4_RUNTIME_SOURCE_URL + "https://github.com/antlr/antlr4/archive/refs/tags/${VELOX_ANTLR4_RUNTIME_VERSION}.tar.gz" +) + +velox_resolve_dependency_url(ANTLR4_RUNTIME) + +message(STATUS "Building antlr4-runtime from source") +FetchContent_Declare( + antlr4-runtime + URL ${VELOX_ANTLR4_RUNTIME_URL} + URL_HASH ${VELOX_ANTLR4_RUNTIME_BUILD_SHA256_CHECKSUM} + SOURCE_SUBDIR runtime/Cpp OVERRIDE_FIND_PACKAGE) + +set(ANTLR4_INSTALL + ON + CACHE BOOL "" FORCE) +FetchContent_MakeAvailable(antlr4-runtime) diff --git a/CMake/resolve_dependency_modules/clp.cmake b/CMake/resolve_dependency_modules/clp.cmake index bf50d26d7c6..5dd1abddfaf 100644 --- a/CMake/resolve_dependency_modules/clp.cmake +++ b/CMake/resolve_dependency_modules/clp.cmake @@ -16,23 +16,32 @@ include_guard(GLOBAL) FetchContent_Declare( clp GIT_REPOSITORY https://github.com/y-scope/clp.git - GIT_TAG 7b1b169a89abdfe44c159d6200b168391b697877 - GIT_SUBMODULES "" GIT_SUBMODULES_RECURSE TRUE) + GIT_TAG 04b44c58ffa0e2c2884fa3eed134b3e4d856ae4a) -FetchContent_MakeAvailable(clp) +set(CLP_BUILD_TESTING + OFF + CACHE BOOL "Build CLP tests") +set(CLP_BUILD_EXECUTABLES + OFF + CACHE BOOL "Build CLP executables") +set(CLP_BUILD_CLP_S_ARCHIVEWRITER + OFF + CACHE BOOL "Build CLP-S archive writer") +set(CLP_BUILD_CLP_S_JSONCONSTRUCTOR + OFF + CACHE BOOL "Build CLP-S JSON constructor") +set(CLP_BUILD_CLP_S_REDUCER_DEPENDENCIES + OFF + CACHE BOOL "Build CLP-S reducer dependencies") +set(CLP_BUILD_CLP_S_SEARCH_SQL + OFF + CACHE BOOL "Build CLP-S search SQL") +set(CLP_BUILD_CLP_REGEX_UTILS + OFF + CACHE BOOL "Build CLP regex utils") -if(clp_POPULATED) - message(STATUS "Updating submodules for clp...") - execute_process( - COMMAND ${CMAKE_COMMAND} -E chdir "${clp_SOURCE_DIR}" git submodule update - --init --recursive - RESULT_VARIABLE submodule_update_result - OUTPUT_VARIABLE submodule_update_output - ERROR_VARIABLE submodule_update_error) - if(NOT ${submodule_update_result} EQUAL 0) - message(ERROR - "Failed to update submodules for clp:\n${submodule_update_error}") - else() - message(STATUS "Submodules for clp updated successfully.") - endif() -endif() +FetchContent_Populate(clp) + +list(APPEND CMAKE_MODULE_PATH "${clp_SOURCE_DIR}/components/core/cmake/Modules") +add_subdirectory(${clp_SOURCE_DIR}/components/core + ${clp_BINARY_DIR}/components/core) diff --git a/CMake/resolve_dependency_modules/date.cmake b/CMake/resolve_dependency_modules/date.cmake new file mode 100644 index 00000000000..2619a0b6394 --- /dev/null +++ b/CMake/resolve_dependency_modules/date.cmake @@ -0,0 +1,38 @@ +# Copyright (c) Facebook, Inc. and its affiliates.Add commentMore actions +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +include_guard(GLOBAL) + +set(VELOX_DATE_BUILD_VERSION 3.0.1) +set(VELOX_DATE_BUILD_SHA256_CHECKSUM + 7a390f200f0ccd207e8cff6757e04817c1a0aec3e327b006b7eb451c57ee3538) +set(VELOX_DATE_SOURCE_URL + "https://github.com/HowardHinnant/date/archive/refs/tags/v${DATE_BUILD_VERSION}.tar.gz" +) + +velox_resolve_dependency_url(DATE) + +# Optionally set CMake variables *before* make-available +set(CMAKE_INSTALL_MESSAGE + LAZY + CACHE STRING "" FORCE) + +message(STATUS "Building date from source") + +FetchContent_Declare( + date + URL ${DATE_SOURCE_URL} + URL_HASH ${DATE_BUILD_SHA256_CHECKSUM} + OVERRIDE_FIND_PACKAGE) + +FetchContent_MakeAvailable(date) diff --git a/CMake/resolve_dependency_modules/msgpack-cxx.cmake b/CMake/resolve_dependency_modules/msgpack-cxx.cmake index ce39b7cf167..69733e8e8b5 100644 --- a/CMake/resolve_dependency_modules/msgpack-cxx.cmake +++ b/CMake/resolve_dependency_modules/msgpack-cxx.cmake @@ -16,10 +16,9 @@ include_guard(GLOBAL) set(VELOX_MSGPACK_BUILD_VERSION cpp-7.0.0) set(VELOX_MSGPACK_BUILD_SHA256_CHECKSUM 070881ebea9208cf7e731fd5a46a11404025b2f260ab9527e32dfcb7c689fbfc) -string( - CONCAT VELOX_MSGPACK_SOURCE_URL - "https://github.com/msgpack/msgpack-c/archive/refs/tags/" - "${VELOX_MSGPACK_BUILD_VERSION}.tar.gz") +set(VELOX_MSGPACK_SOURCE_URL + "https://github.com/msgpack/msgpack-c/archive/refs/tags/${VELOX_MSGPACK_BUILD_VERSION}.tar.gz" +) velox_resolve_dependency_url(MSGPACK) diff --git a/CMake/resolve_dependency_modules/nlohmann_json.cmake b/CMake/resolve_dependency_modules/nlohmann_json.cmake new file mode 100644 index 00000000000..b07953733d2 --- /dev/null +++ b/CMake/resolve_dependency_modules/nlohmann_json.cmake @@ -0,0 +1,32 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +include_guard(GLOBAL) + +set(VELOX_NLOHMANN_JSON_BUILD_VERSION 3.11.3) +set(VELOX_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM + 0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406) +set(VELOX_NLOHMANN_JSON_SOURCE_URL + "https://github.com/nlohmann/json/archive/refs/tags/v${VELOX_NLOHMANN_JSON_BUILD_VERSION}.tar.gz" +) + +FetchContent_Declare( + nlohmann_json + URL ${VELOX_NLOHMANN_JSON_SOURCE_URL} + URL_HASH ${VELOX_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}) + +set(JSON_BuildTests + OFF + CACHE INTERNAL "") + +FetchContent_MakeAvailable(nlohmann_json) diff --git a/CMake/resolve_dependency_modules/simdjson.cmake b/CMake/resolve_dependency_modules/simdjson.cmake index a0c55c3758a..5d28789747e 100644 --- a/CMake/resolve_dependency_modules/simdjson.cmake +++ b/CMake/resolve_dependency_modules/simdjson.cmake @@ -27,7 +27,8 @@ message(STATUS "Building simdjson from source") FetchContent_Declare( simdjson URL ${VELOX_SIMDJSON_SOURCE_URL} - URL_HASH ${VELOX_SIMDJSON_BUILD_SHA256_CHECKSUM}) + URL_HASH ${VELOX_SIMDJSON_BUILD_SHA256_CHECKSUM} + OVERRIDE_FIND_PACKAGE) if(${VELOX_SIMDJSON_SKIPUTF8VALIDATION}) set(SIMDJSON_SKIPUTF8VALIDATION ON) diff --git a/CMake/resolve_dependency_modules/spdlog.cmake b/CMake/resolve_dependency_modules/spdlog.cmake index 022525fc134..a7a1ddfe698 100644 --- a/CMake/resolve_dependency_modules/spdlog.cmake +++ b/CMake/resolve_dependency_modules/spdlog.cmake @@ -16,10 +16,9 @@ include_guard(GLOBAL) set(VELOX_SPDLOG_BUILD_VERSION 1.12.0) set(VELOX_SPDLOG_BUILD_SHA256_CHECKSUM 4dccf2d10f410c1e2feaff89966bfc49a1abb29ef6f08246335b110e001e09a9) -string( - CONCAT VELOX_SPDLOG_SOURCE_URL - "https://github.com/gabime/spdlog/archive/refs/tags/" - "v${VELOX_SPDLOG_BUILD_VERSION}.tar.gz") +set(VELOX_SPDLOG_SOURCE_URL + "https://github.com/gabime/spdlog/archive/refs/tags/v${VELOX_SPDLOG_BUILD_VERSION}.tar.gz" +) velox_resolve_dependency_url(SPDLOG) diff --git a/CMake/resolve_dependency_modules/ystdlib_cpp.cmake b/CMake/resolve_dependency_modules/ystdlib_cpp.cmake new file mode 100644 index 00000000000..1473aad723d --- /dev/null +++ b/CMake/resolve_dependency_modules/ystdlib_cpp.cmake @@ -0,0 +1,24 @@ +# Copyright (c) Facebook, Inc. and its affiliates.Add commentMore actions +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +include_guard(GLOBAL) + +FetchContent_Declare( + ystdlib_cpp + GIT_REPOSITORY https://github.com/y-scope/ystdlib-cpp.git + GIT_TAG d80cf86e1a1f2dae6421978c8ee353408368f424 + GIT_SUBMODULES "" GIT_SUBMODULES_RECURSE TRUE) + +FetchContent_Populate(ystdlib_cpp) + +set(CLP_YSTDLIB_SOURCE_DIRECTORY "${ystdlib_cpp_SOURCE_DIR}") diff --git a/CMakeLists.txt b/CMakeLists.txt index 39a481923b7..a665beb1292 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -523,21 +523,32 @@ velox_set_source(re2) velox_resolve_dependency(re2) if(${VELOX_ENABLE_CLP_CONNECTOR}) - set(clp_SOURCE BUNDLED) - velox_resolve_dependency(clp) + set(ystdlib_cpp_SOURCE BUNDLED) + velox_resolve_dependency(ystdlib_cpp) + + velox_set_source(absl) + velox_resolve_dependency(absl) + + velox_set_source(nlohmann_json) + velox_resolve_dependency(nlohmann_json) - set(spdlog_SOURCE BUNDLED) + velox_set_source(simdjson) + velox_resolve_dependency(simdjson) + + velox_set_source(date) + velox_resolve_dependency(date) + + velox_set_source(spdlog) velox_resolve_dependency(spdlog) velox_set_source(msgpack-cxx) velox_resolve_dependency(msgpack-cxx) - velox_set_source(absl) - velox_resolve_dependency(absl) + velox_set_source(antlr4-runtime) + velox_resolve_dependency(antlr4-runtime) - set(curl_SOURCE BUNDLED) - velox_resolve_dependency(curl) - find_package(ANTLR REQUIRED) + set(clp_SOURCE BUNDLED) + velox_resolve_dependency(clp) endif() if(${VELOX_BUILD_PYTHON_PACKAGE}) diff --git a/velox/connectors/clp/CMakeLists.txt b/velox/connectors/clp/CMakeLists.txt index 8fc435d8bc4..2d5bcdef5e4 100644 --- a/velox/connectors/clp/CMakeLists.txt +++ b/velox/connectors/clp/CMakeLists.txt @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -set(CLP_SRC_DIR ${clp_SOURCE_DIR}/components/core/src) +#set(CLP_SRC_DIR ${clp_SOURCE_DIR}/components/core/src) add_subdirectory(search_lib) velox_add_library( @@ -23,7 +23,6 @@ velox_add_library( velox_link_libraries(velox_clp_connector PRIVATE clp-s-search simdjson::simdjson velox_connector) - target_compile_features(velox_clp_connector PRIVATE cxx_std_20) if(${VELOX_BUILD_TESTING}) diff --git a/velox/connectors/clp/search_lib/CMakeLists.txt b/velox/connectors/clp/search_lib/CMakeLists.txt index 3e41e905a70..b84869e2da6 100644 --- a/velox/connectors/clp/search_lib/CMakeLists.txt +++ b/velox/connectors/clp/search_lib/CMakeLists.txt @@ -11,85 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -set(CLP_EXTERNAL_BINARY_DIR ${CMAKE_BINARY_DIR}/external/clp) -add_subdirectory(${clp_SOURCE_DIR}/components/core/src/clp/string_utils - ${CLP_EXTERNAL_BINARY_DIR}/string_utils) -set(YSTDLIB_CPP_BUILD_TESTING OFF) -add_subdirectory(${clp_SOURCE_DIR}/components/core/submodules/ystdlib-cpp - ${CLP_EXTERNAL_BINARY_DIR}/ystdlib-cpp EXCLUDE_FROM_ALL) - -string(LENGTH "${CMAKE_SOURCE_DIR}/" SOURCE_PATH_SIZE) - -antlr_target( - KqlParser - ${CLP_SRC_DIR}/clp_s/search/kql/Kql.g4 - LEXER - PARSER - VISITOR - PACKAGE - kql) - -set(CLP_SRC_FILES - ${ANTLR_KqlParser_CXX_OUTPUTS} - ${CLP_SRC_DIR}/clp_s/ArchiveReader.cpp - ${CLP_SRC_DIR}/clp_s/ArchiveReaderAdaptor.cpp - ${CLP_SRC_DIR}/clp_s/ColumnReader.cpp - ${CLP_SRC_DIR}/clp_s/DictionaryEntry.cpp - ${CLP_SRC_DIR}/clp_s/DictionaryWriter.cpp - ${CLP_SRC_DIR}/clp_s/FileReader.cpp - ${CLP_SRC_DIR}/clp_s/FileWriter.cpp - ${CLP_SRC_DIR}/clp_s/InputConfig.cpp - ${CLP_SRC_DIR}/clp_s/PackedStreamReader.cpp - ${CLP_SRC_DIR}/clp_s/ReaderUtils.cpp - ${CLP_SRC_DIR}/clp_s/Schema.cpp - ${CLP_SRC_DIR}/clp_s/SchemaMap.cpp - ${CLP_SRC_DIR}/clp_s/SchemaReader.cpp - ${CLP_SRC_DIR}/clp_s/SchemaTree.cpp - ${CLP_SRC_DIR}/clp_s/TimestampDictionaryReader.cpp - ${CLP_SRC_DIR}/clp_s/TimestampDictionaryWriter.cpp - ${CLP_SRC_DIR}/clp_s/TimestampEntry.cpp - ${CLP_SRC_DIR}/clp_s/TimestampPattern.cpp - ${CLP_SRC_DIR}/clp_s/Utils.cpp - ${CLP_SRC_DIR}/clp_s/VariableEncoder.cpp - ${CLP_SRC_DIR}/clp_s/VariableDecoder.cpp - ${CLP_SRC_DIR}/clp_s/ZstdCompressor.cpp - ${CLP_SRC_DIR}/clp_s/ZstdDecompressor.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/AndExpr.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/BooleanLiteral.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/ColumnDescriptor.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/ConstantProp.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/ConvertToExists.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/DateLiteral.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/EmptyExpr.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/Expression.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/FilterExpr.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/Integral.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/NarrowTypes.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/NullLiteral.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/OrExpr.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/OrOfAndForm.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/SearchUtils.cpp - ${CLP_SRC_DIR}/clp_s/search/ast/StringLiteral.cpp - ${CLP_SRC_DIR}/clp_s/search/clp_search/EncodedVariableInterpreter.cpp - ${CLP_SRC_DIR}/clp_s/search/clp_search/Grep.cpp - ${CLP_SRC_DIR}/clp_s/search/clp_search/Query.cpp - ${CLP_SRC_DIR}/clp_s/search/EvaluateTimestampIndex.cpp - ${CLP_SRC_DIR}/clp_s/search/kql/kql.cpp - ${CLP_SRC_DIR}/clp_s/search/Projection.cpp - ${CLP_SRC_DIR}/clp_s/search/QueryRunner.cpp - ${CLP_SRC_DIR}/clp_s/search/SchemaMatch.cpp - ${CLP_SRC_DIR}/clp/aws/AwsAuthenticationSigner.cpp - ${CLP_SRC_DIR}/clp/BoundedReader.cpp - ${CLP_SRC_DIR}/clp/CurlDownloadHandler.cpp - ${CLP_SRC_DIR}/clp/CurlGlobalInstance.cpp - ${CLP_SRC_DIR}/clp/Defs.h - ${CLP_SRC_DIR}/clp/FileReader.cpp - ${CLP_SRC_DIR}/clp/hash_utils.cpp - ${CLP_SRC_DIR}/clp/NetworkReader.cpp - ${CLP_SRC_DIR}/clp/ReaderInterface.cpp - ${CLP_SRC_DIR}/clp/Thread.cpp) - velox_add_library( clp-s-search STATIC @@ -100,30 +21,14 @@ velox_add_library( ClpCursor.h ClpQueryRunner.cpp ClpQueryRunner.h) -target_compile_features(clp-s-search PRIVATE cxx_std_20) -target_compile_definitions(clp-s-search - PUBLIC SOURCE_PATH_SIZE=${SOURCE_PATH_SIZE}) -velox_include_directories( - clp-s-search - PUBLIC ${ANTLR_KqlParser_OUTPUT_DIR} - ${clp_SOURCE_DIR}/components/core/submodules ${CLP_SRC_DIR}) velox_link_libraries( clp-s-search - PUBLIC msgpack-cxx spdlog::spdlog + PUBLIC clp_s::archive_reader PRIVATE - absl::flat_hash_map - antlr4_static - Boost::filesystem - Boost::iostreams - Boost::program_options - Boost::url - clp::string_utils - ${CURL_LIBRARIES} - glog::glog - OpenSSL::Crypto - simdjson::simdjson - velox_vector - ystdlib::containers - ystdlib::error_handling - zstd::zstd) + clp_s::io + clp_s::clp_dependencies + clp_s::search + clp_s::search::kql + velox_vector) +target_compile_features(clp-s-search PRIVATE cxx_std_20) \ No newline at end of file From 7fb63098a2b1968918a145b817a7c15f0c93c7b9 Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 19:46:23 +0000 Subject: [PATCH 03/31] add clp.storage-type config --- velox/connectors/clp/CMakeLists.txt | 1 + velox/connectors/clp/ClpConfig.cpp | 40 ++++++++++++++++++++++++++ velox/connectors/clp/ClpConfig.h | 9 ++++++ velox/connectors/clp/ClpDataSource.cpp | 6 ++-- velox/connectors/clp/ClpDataSource.h | 2 +- velox/connectors/clp/ClpTableHandle.h | 12 -------- 6 files changed, 54 insertions(+), 16 deletions(-) create mode 100644 velox/connectors/clp/ClpConfig.cpp diff --git a/velox/connectors/clp/CMakeLists.txt b/velox/connectors/clp/CMakeLists.txt index 2d5bcdef5e4..5cf5366aa9e 100644 --- a/velox/connectors/clp/CMakeLists.txt +++ b/velox/connectors/clp/CMakeLists.txt @@ -17,6 +17,7 @@ add_subdirectory(search_lib) velox_add_library( velox_clp_connector OBJECT + ClpConfig.cpp ClpConnector.cpp ClpDataSource.cpp ClpTableHandle.cpp) diff --git a/velox/connectors/clp/ClpConfig.cpp b/velox/connectors/clp/ClpConfig.cpp new file mode 100644 index 00000000000..22f6aa4a6f0 --- /dev/null +++ b/velox/connectors/clp/ClpConfig.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ClpConfig.h" + +namespace facebook::velox::connector::clp { + +namespace { + +ClpConfig::StorageType stringToStorageType(const std::string& strValue) { + auto upperValue = boost::algorithm::to_upper_copy(strValue); + if (upperValue == "FS") { + return ClpConfig::StorageType::kFS; + } + if (upperValue == "S3") { + return ClpConfig::StorageType::kS3; + } + VELOX_UNSUPPORTED("Unsupported storage type: {}.", strValue); +} + +} // namespace + +ClpConfig::StorageType ClpConfig::storageType() const { + return stringToStorageType(config_->get(kStorageType, "FS")); +} + +} // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpConfig.h b/velox/connectors/clp/ClpConfig.h index 5d6a254a81e..2e90d801bcb 100644 --- a/velox/connectors/clp/ClpConfig.h +++ b/velox/connectors/clp/ClpConfig.h @@ -27,6 +27,13 @@ class ConfigBase; namespace facebook::velox::connector::clp { class ClpConfig { public: + enum class StorageType { + kFS, + kS3, + }; + + static constexpr const char* kStorageType = "clp.storage-type"; + explicit ClpConfig(std::shared_ptr config) { VELOX_CHECK_NOT_NULL(config, "Config is null for CLP initialization"); config_ = std::move(config); @@ -37,6 +44,8 @@ class ClpConfig { return config_; } + StorageType storageType() const; + private: std::shared_ptr config_; }; diff --git a/velox/connectors/clp/ClpDataSource.cpp b/velox/connectors/clp/ClpDataSource.cpp index 7f3783cb1c8..d3f648a0808 100644 --- a/velox/connectors/clp/ClpDataSource.cpp +++ b/velox/connectors/clp/ClpDataSource.cpp @@ -36,7 +36,7 @@ ClpDataSource::ClpDataSource( std::shared_ptr& clpConfig) : pool_(pool), outputType_(outputType) { auto clpTableHandle = std::dynamic_pointer_cast(tableHandle); - storageType_ = clpTableHandle->storageType(); + storageType_ = clpConfig->storageType(); if (auto query = clpTableHandle->kqlQuery(); query && !query->empty()) { kqlQuery_ = *query; } else { @@ -103,10 +103,10 @@ void ClpDataSource::addFieldsRecursively( void ClpDataSource::addSplit(std::shared_ptr split) { auto clpSplit = std::dynamic_pointer_cast(split); - if (storageType_ == ClpTableHandle::StorageType::kFS) { + if (storageType_ == ClpConfig::StorageType::kFS) { cursor_ = std::make_unique( clp_s::InputSource::Filesystem, clpSplit->path_); - } else if (storageType_ == ClpTableHandle::StorageType::kS3) { + } else if (storageType_ == ClpConfig::StorageType::kS3) { cursor_ = std::make_unique( clp_s::InputSource::Network, clpSplit->path_); } diff --git a/velox/connectors/clp/ClpDataSource.h b/velox/connectors/clp/ClpDataSource.h index fd2be384884..a6c536073cb 100644 --- a/velox/connectors/clp/ClpDataSource.h +++ b/velox/connectors/clp/ClpDataSource.h @@ -91,7 +91,7 @@ class ClpDataSource : public DataSource { const std::shared_ptr>& filteredRows, size_t& readerIndex); - ClpTableHandle::StorageType storageType_; + ClpConfig::StorageType storageType_; std::string kqlQuery_; velox::memory::MemoryPool* pool_; RowTypePtr outputType_; diff --git a/velox/connectors/clp/ClpTableHandle.h b/velox/connectors/clp/ClpTableHandle.h index 801a9ff163d..a1ec00b329c 100644 --- a/velox/connectors/clp/ClpTableHandle.h +++ b/velox/connectors/clp/ClpTableHandle.h @@ -21,29 +21,18 @@ namespace facebook::velox::connector::clp { class ClpTableHandle : public ConnectorTableHandle { public: - enum class StorageType { - kFS, - kS3, - }; - ClpTableHandle( const std::string& connectorId, const std::string& tableName, - StorageType storageType, std::shared_ptr kqlQuery) : ConnectorTableHandle(connectorId), tableName_(tableName), - storageType_(storageType), kqlQuery_(std::move(kqlQuery)) {} [[nodiscard]] const std::string& tableName() const { return tableName_; } - [[nodiscard]] const StorageType storageType() const { - return storageType_; - } - [[nodiscard]] const std::shared_ptr& kqlQuery() const { return kqlQuery_; } @@ -54,7 +43,6 @@ class ClpTableHandle : public ConnectorTableHandle { private: const std::string tableName_; - const StorageType storageType_; std::shared_ptr kqlQuery_; }; } // namespace facebook::velox::connector::clp From e4a54a6243d1505bcf397c694b7107a1ff13355d Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 19:50:59 +0000 Subject: [PATCH 04/31] add a cmake message --- CMake/resolve_dependency_modules/antlr4-runtime.cmake | 1 + CMake/resolve_dependency_modules/nlohmann_json.cmake | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CMake/resolve_dependency_modules/antlr4-runtime.cmake b/CMake/resolve_dependency_modules/antlr4-runtime.cmake index a5d34f4b768..7535071dd6d 100644 --- a/CMake/resolve_dependency_modules/antlr4-runtime.cmake +++ b/CMake/resolve_dependency_modules/antlr4-runtime.cmake @@ -23,6 +23,7 @@ set(VELOX_ANTLR4_RUNTIME_SOURCE_URL velox_resolve_dependency_url(ANTLR4_RUNTIME) message(STATUS "Building antlr4-runtime from source") + FetchContent_Declare( antlr4-runtime URL ${VELOX_ANTLR4_RUNTIME_URL} diff --git a/CMake/resolve_dependency_modules/nlohmann_json.cmake b/CMake/resolve_dependency_modules/nlohmann_json.cmake index b07953733d2..288eeac20ad 100644 --- a/CMake/resolve_dependency_modules/nlohmann_json.cmake +++ b/CMake/resolve_dependency_modules/nlohmann_json.cmake @@ -20,6 +20,8 @@ set(VELOX_NLOHMANN_JSON_SOURCE_URL "https://github.com/nlohmann/json/archive/refs/tags/v${VELOX_NLOHMANN_JSON_BUILD_VERSION}.tar.gz" ) +message(STATUS "Building nlohmann_json from source") + FetchContent_Declare( nlohmann_json URL ${VELOX_NLOHMANN_JSON_SOURCE_URL} From b0e521151c83ef5530f2ac567b31ab3070a2e531 Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 19:57:26 +0000 Subject: [PATCH 05/31] fix lint errors --- CMake/resolve_dependency_modules/ystdlib_cpp.cmake | 9 ++++----- velox/connectors/clp/search_lib/CMakeLists.txt | 2 +- velox/docs/develop/connectors.rst | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/CMake/resolve_dependency_modules/ystdlib_cpp.cmake b/CMake/resolve_dependency_modules/ystdlib_cpp.cmake index 1473aad723d..aa188279908 100644 --- a/CMake/resolve_dependency_modules/ystdlib_cpp.cmake +++ b/CMake/resolve_dependency_modules/ystdlib_cpp.cmake @@ -1,4 +1,4 @@ -# Copyright (c) Facebook, Inc. and its affiliates.Add commentMore actions +# Copyright (c) Facebook, Inc. and its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,10 +14,9 @@ include_guard(GLOBAL) FetchContent_Declare( - ystdlib_cpp - GIT_REPOSITORY https://github.com/y-scope/ystdlib-cpp.git - GIT_TAG d80cf86e1a1f2dae6421978c8ee353408368f424 - GIT_SUBMODULES "" GIT_SUBMODULES_RECURSE TRUE) + ystdlib_cpp + GIT_REPOSITORY https://github.com/y-scope/ystdlib-cpp.git + GIT_TAG d80cf86e1a1f2dae6421978c8ee353408368f424) FetchContent_Populate(ystdlib_cpp) diff --git a/velox/connectors/clp/search_lib/CMakeLists.txt b/velox/connectors/clp/search_lib/CMakeLists.txt index b84869e2da6..0f7eefc58e0 100644 --- a/velox/connectors/clp/search_lib/CMakeLists.txt +++ b/velox/connectors/clp/search_lib/CMakeLists.txt @@ -31,4 +31,4 @@ velox_link_libraries( clp_s::search clp_s::search::kql velox_vector) -target_compile_features(clp-s-search PRIVATE cxx_std_20) \ No newline at end of file +target_compile_features(clp-s-search PRIVATE cxx_std_20) diff --git a/velox/docs/develop/connectors.rst b/velox/docs/develop/connectors.rst index b5b04a70c3a..27e94d9a1c2 100644 --- a/velox/docs/develop/connectors.rst +++ b/velox/docs/develop/connectors.rst @@ -147,4 +147,4 @@ Velox column type to a CLP column type. When a split is added, a `ClpCursor` is created with the archive path and input source. The query is parsed and simplified into an AST. On `next`, the cursor finds matching row indices and, if any exist, `ClpDataSource` recursively creates a row vector composed of lazy vectors, which use CLP column readers to decode and load data -as needed during execution. \ No newline at end of file +as needed during execution. From b05bba5e2d671d8aed2c9ec4e1c90f1f5d330d15 Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 20:03:05 +0000 Subject: [PATCH 06/31] add config to docs --- CMake/resolve_dependency_modules/date.cmake | 2 +- velox/docs/configs.rst | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CMake/resolve_dependency_modules/date.cmake b/CMake/resolve_dependency_modules/date.cmake index 2619a0b6394..fb607ec5198 100644 --- a/CMake/resolve_dependency_modules/date.cmake +++ b/CMake/resolve_dependency_modules/date.cmake @@ -1,4 +1,4 @@ -# Copyright (c) Facebook, Inc. and its affiliates.Add commentMore actions +# Copyright (c) Facebook, Inc. and its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/velox/docs/configs.rst b/velox/docs/configs.rst index 9c19b600e06..291ecdd03b5 100644 --- a/velox/docs/configs.rst +++ b/velox/docs/configs.rst @@ -900,6 +900,21 @@ These semantics are similar to the `Apache Hadoop-Aws module /oauth2/token`. +CLP Connector +----------------------------- +.. list-table:: + :widths: 20 10 10 70 + :header-rows: 1 + + * - Property Name + - Type + - Default Value + - Description + * - clp.storage-type + - string + - LOCAL + - The storage type of CLP archives, **Allowed values:** ``FS``, ``S3``. + Presto-specific Configuration ----------------------------- .. list-table:: From 6e366142cdc150346d4ec572300854a897733600 Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 20:16:42 +0000 Subject: [PATCH 07/31] fix nlohmann_json --- CMake/resolve_dependency_modules/nlohmann_json.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMake/resolve_dependency_modules/nlohmann_json.cmake b/CMake/resolve_dependency_modules/nlohmann_json.cmake index 288eeac20ad..c98681d87e7 100644 --- a/CMake/resolve_dependency_modules/nlohmann_json.cmake +++ b/CMake/resolve_dependency_modules/nlohmann_json.cmake @@ -20,6 +20,8 @@ set(VELOX_NLOHMANN_JSON_SOURCE_URL "https://github.com/nlohmann/json/archive/refs/tags/v${VELOX_NLOHMANN_JSON_BUILD_VERSION}.tar.gz" ) +velox_resolve_dependency_url(NLOHMANN_JSON) + message(STATUS "Building nlohmann_json from source") FetchContent_Declare( From 46754a1b9de15e06b39e5524451a364f1f6773ca Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 20:22:09 +0000 Subject: [PATCH 08/31] fix date --- CMake/resolve_dependency_modules/date.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMake/resolve_dependency_modules/date.cmake b/CMake/resolve_dependency_modules/date.cmake index fb607ec5198..275aa6e0a08 100644 --- a/CMake/resolve_dependency_modules/date.cmake +++ b/CMake/resolve_dependency_modules/date.cmake @@ -17,7 +17,7 @@ set(VELOX_DATE_BUILD_VERSION 3.0.1) set(VELOX_DATE_BUILD_SHA256_CHECKSUM 7a390f200f0ccd207e8cff6757e04817c1a0aec3e327b006b7eb451c57ee3538) set(VELOX_DATE_SOURCE_URL - "https://github.com/HowardHinnant/date/archive/refs/tags/v${DATE_BUILD_VERSION}.tar.gz" + "https://github.com/HowardHinnant/date/archive/refs/tags/v${VELOX_DATE_BUILD_VERSION}.tar.gz" ) velox_resolve_dependency_url(DATE) @@ -31,8 +31,8 @@ message(STATUS "Building date from source") FetchContent_Declare( date - URL ${DATE_SOURCE_URL} - URL_HASH ${DATE_BUILD_SHA256_CHECKSUM} + URL ${VELOX_DATE_SOURCE_URL} + URL_HASH ${VELOX_DATE_BUILD_SHA256_CHECKSUM} OVERRIDE_FIND_PACKAGE) FetchContent_MakeAvailable(date) From e74745df3462c2992526552c5febc0b5e5d6af17 Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 20:49:57 +0000 Subject: [PATCH 09/31] fix antlr4 --- CMake/resolve_dependency_modules/antlr4-runtime.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMake/resolve_dependency_modules/antlr4-runtime.cmake b/CMake/resolve_dependency_modules/antlr4-runtime.cmake index 7535071dd6d..86bc799ce71 100644 --- a/CMake/resolve_dependency_modules/antlr4-runtime.cmake +++ b/CMake/resolve_dependency_modules/antlr4-runtime.cmake @@ -26,7 +26,7 @@ message(STATUS "Building antlr4-runtime from source") FetchContent_Declare( antlr4-runtime - URL ${VELOX_ANTLR4_RUNTIME_URL} + URL ${VELOX_ANTLR4_RUNTIME_SOURCE_URL} URL_HASH ${VELOX_ANTLR4_RUNTIME_BUILD_SHA256_CHECKSUM} SOURCE_SUBDIR runtime/Cpp OVERRIDE_FIND_PACKAGE) From 85acfe08d58f14902be8e3686757fc62c2532101 Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 20:53:19 +0000 Subject: [PATCH 10/31] fix nlohmann_json --- CMake/resolve_dependency_modules/nlohmann_json.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMake/resolve_dependency_modules/nlohmann_json.cmake b/CMake/resolve_dependency_modules/nlohmann_json.cmake index c98681d87e7..1b0298f6618 100644 --- a/CMake/resolve_dependency_modules/nlohmann_json.cmake +++ b/CMake/resolve_dependency_modules/nlohmann_json.cmake @@ -27,7 +27,8 @@ message(STATUS "Building nlohmann_json from source") FetchContent_Declare( nlohmann_json URL ${VELOX_NLOHMANN_JSON_SOURCE_URL} - URL_HASH ${VELOX_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM}) + URL_HASH ${VELOX_NLOHMANN_JSON_BUILD_SHA256_CHECKSUM} + OVERRIDE_FIND_PACKAGE) set(JSON_BuildTests OFF From 864519c5d3e7076f61c0f6404880609e3eeb2575 Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 20:54:29 +0000 Subject: [PATCH 11/31] fix docs --- velox/docs/configs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/velox/docs/configs.rst b/velox/docs/configs.rst index 291ecdd03b5..e7864ac2fd2 100644 --- a/velox/docs/configs.rst +++ b/velox/docs/configs.rst @@ -912,7 +912,7 @@ CLP Connector - Description * - clp.storage-type - string - - LOCAL + - FS - The storage type of CLP archives, **Allowed values:** ``FS``, ``S3``. Presto-specific Configuration From 0686e0fe8c9463ae05389b535a93d8efab20a449 Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 21:35:26 +0000 Subject: [PATCH 12/31] fix msgpack-cxx --- CMake/resolve_dependency_modules/msgpack-cxx.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMake/resolve_dependency_modules/msgpack-cxx.cmake b/CMake/resolve_dependency_modules/msgpack-cxx.cmake index 69733e8e8b5..9f20800b74e 100644 --- a/CMake/resolve_dependency_modules/msgpack-cxx.cmake +++ b/CMake/resolve_dependency_modules/msgpack-cxx.cmake @@ -30,4 +30,6 @@ FetchContent_Declare( URL_HASH ${VELOX_MSGPACK_BUILD_SHA256_CHECKSUM} OVERRIDE_FIND_PACKAGE EXCLUDE_FROM_ALL SYSTEM) +set(MSGPACK_USE_BOOST OFF) + FetchContent_MakeAvailable(msgpack-cxx) From e9f90462239a57c4be361dd9ffe2e54e8d702622 Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 22:33:51 +0000 Subject: [PATCH 13/31] try fix --- CMake/resolve_dependency_modules/boost/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMake/resolve_dependency_modules/boost/CMakeLists.txt b/CMake/resolve_dependency_modules/boost/CMakeLists.txt index 69018a0f3a6..2b2ec3b8091 100644 --- a/CMake/resolve_dependency_modules/boost/CMakeLists.txt +++ b/CMake/resolve_dependency_modules/boost/CMakeLists.txt @@ -42,7 +42,8 @@ endif() FetchContent_Declare( Boost URL ${VELOX_BOOST_SOURCE_URL} - URL_HASH ${VELOX_BOOST_BUILD_SHA256_CHECKSUM}) + URL_HASH ${VELOX_BOOST_BUILD_SHA256_CHECKSUM} + OVERRIDE_FIND_PACKAGE) # Configure the file before adding the header only libs configure_file(${CMAKE_CURRENT_LIST_DIR}/FindBoost.cmake.in From f965510afae3ca3a1a6b40fbb4fa244fb4575bca Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 25 Jun 2025 22:58:53 +0000 Subject: [PATCH 14/31] try fix --- CMake/resolve_dependency_modules/boost/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMake/resolve_dependency_modules/boost/CMakeLists.txt b/CMake/resolve_dependency_modules/boost/CMakeLists.txt index 2b2ec3b8091..69018a0f3a6 100644 --- a/CMake/resolve_dependency_modules/boost/CMakeLists.txt +++ b/CMake/resolve_dependency_modules/boost/CMakeLists.txt @@ -42,8 +42,7 @@ endif() FetchContent_Declare( Boost URL ${VELOX_BOOST_SOURCE_URL} - URL_HASH ${VELOX_BOOST_BUILD_SHA256_CHECKSUM} - OVERRIDE_FIND_PACKAGE) + URL_HASH ${VELOX_BOOST_BUILD_SHA256_CHECKSUM}) # Configure the file before adding the header only libs configure_file(${CMAKE_CURRENT_LIST_DIR}/FindBoost.cmake.in From 741e8b39ddcb995a6695c2737a432bb6f300358d Mon Sep 17 00:00:00 2001 From: wraymo Date: Thu, 26 Jun 2025 02:25:01 +0000 Subject: [PATCH 15/31] update ystdlib_cpp --- CMake/resolve_dependency_modules/ystdlib_cpp.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMake/resolve_dependency_modules/ystdlib_cpp.cmake b/CMake/resolve_dependency_modules/ystdlib_cpp.cmake index aa188279908..a47b24416c8 100644 --- a/CMake/resolve_dependency_modules/ystdlib_cpp.cmake +++ b/CMake/resolve_dependency_modules/ystdlib_cpp.cmake @@ -16,7 +16,7 @@ include_guard(GLOBAL) FetchContent_Declare( ystdlib_cpp GIT_REPOSITORY https://github.com/y-scope/ystdlib-cpp.git - GIT_TAG d80cf86e1a1f2dae6421978c8ee353408368f424) + GIT_TAG 0ae886c6a7ee706a3c6e1950262b63d72f71fe63) FetchContent_Populate(ystdlib_cpp) From cf58de14d7402b9f9528efd94ebc87cc37b88fc8 Mon Sep 17 00:00:00 2001 From: wraymo Date: Thu, 26 Jun 2025 14:51:24 +0000 Subject: [PATCH 16/31] try fix --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a665beb1292..36e701d88f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -461,6 +461,7 @@ set(BOOST_INCLUDE_LIBRARIES context date_time filesystem + headers iostreams program_options regex From 82a1a9108d7845ac0d650da4d8d05a7d746b1832 Mon Sep 17 00:00:00 2001 From: wraymo Date: Fri, 27 Jun 2025 15:14:49 +0000 Subject: [PATCH 17/31] use system boost --- .github/workflows/linux-build-base.yml | 1 + CMakeLists.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-build-base.yml b/.github/workflows/linux-build-base.yml index a0b86504a3e..6aa766ed9ef 100644 --- a/.github/workflows/linux-build-base.yml +++ b/.github/workflows/linux-build-base.yml @@ -53,6 +53,7 @@ jobs: env: VELOX_DEPENDENCY_SOURCE: BUNDLED ICU_SOURCE: SYSTEM + Boost_SOURCE: SYSTEM MAKEFLAGS: MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3 run: | if [[ "${USE_CLANG}" = "true" ]]; then export CC=/usr/bin/clang-15; export CXX=/usr/bin/clang++-15; fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 36e701d88f4..a665beb1292 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -461,7 +461,6 @@ set(BOOST_INCLUDE_LIBRARIES context date_time filesystem - headers iostreams program_options regex From dc3d9f87e5555919a86fc0d04ce15dd5c7c7ae8a Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 30 Jun 2025 20:08:54 +0000 Subject: [PATCH 18/31] update clp --- CMake/resolve_dependency_modules/clp.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMake/resolve_dependency_modules/clp.cmake b/CMake/resolve_dependency_modules/clp.cmake index 5dd1abddfaf..e91e1b79850 100644 --- a/CMake/resolve_dependency_modules/clp.cmake +++ b/CMake/resolve_dependency_modules/clp.cmake @@ -16,7 +16,7 @@ include_guard(GLOBAL) FetchContent_Declare( clp GIT_REPOSITORY https://github.com/y-scope/clp.git - GIT_TAG 04b44c58ffa0e2c2884fa3eed134b3e4d856ae4a) + GIT_TAG 35208950df5acf38ff8f0c560abee65a61b28703) set(CLP_BUILD_TESTING OFF From bd1cefc340ad24f3480bf3b68dc42d5239b2e32b Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 2 Jul 2025 22:54:22 +0000 Subject: [PATCH 19/31] address comments --- velox/connectors/clp/ClpColumnHandle.h | 2 + velox/connectors/clp/ClpConfig.h | 2 + velox/connectors/clp/ClpConnector.cpp | 1 + velox/connectors/clp/ClpConnector.h | 2 + velox/connectors/clp/ClpConnectorSplit.h | 2 + velox/connectors/clp/ClpDataSource.cpp | 12 +-- velox/connectors/clp/ClpDataSource.h | 40 ++++---- velox/connectors/clp/ClpTableHandle.cpp | 2 + velox/connectors/clp/ClpTableHandle.h | 2 + velox/connectors/clp/search_lib/ClpCursor.cpp | 2 + velox/connectors/clp/search_lib/ClpCursor.h | 96 ++++++++++--------- .../clp/search_lib/ClpQueryRunner.cpp | 5 +- .../clp/search_lib/ClpQueryRunner.h | 67 +++++++------ .../clp/search_lib/ClpVectorLoader.cpp | 5 + .../clp/search_lib/ClpVectorLoader.h | 15 +-- 15 files changed, 142 insertions(+), 113 deletions(-) diff --git a/velox/connectors/clp/ClpColumnHandle.h b/velox/connectors/clp/ClpColumnHandle.h index d7857d82724..a4a1b8702f5 100644 --- a/velox/connectors/clp/ClpColumnHandle.h +++ b/velox/connectors/clp/ClpColumnHandle.h @@ -19,6 +19,7 @@ #include "velox/connectors/Connector.h" namespace facebook::velox::connector::clp { + class ClpColumnHandle : public ColumnHandle { public: ClpColumnHandle( @@ -53,4 +54,5 @@ class ClpColumnHandle : public ColumnHandle { const TypePtr columnType_; const bool nullable_; }; + } // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpConfig.h b/velox/connectors/clp/ClpConfig.h index 2e90d801bcb..fa0d4de7699 100644 --- a/velox/connectors/clp/ClpConfig.h +++ b/velox/connectors/clp/ClpConfig.h @@ -25,6 +25,7 @@ class ConfigBase; } namespace facebook::velox::connector::clp { + class ClpConfig { public: enum class StorageType { @@ -49,4 +50,5 @@ class ClpConfig { private: std::shared_ptr config_; }; + } // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpConnector.cpp b/velox/connectors/clp/ClpConnector.cpp index 54ffe63a5ac..d381716f1de 100644 --- a/velox/connectors/clp/ClpConnector.cpp +++ b/velox/connectors/clp/ClpConnector.cpp @@ -20,6 +20,7 @@ #include "velox/connectors/clp/ClpDataSource.h" namespace facebook::velox::connector::clp { + ClpConnector::ClpConnector( const std::string& id, std::shared_ptr config) diff --git a/velox/connectors/clp/ClpConnector.h b/velox/connectors/clp/ClpConnector.h index ae9c28419ac..59efe8d8bb7 100644 --- a/velox/connectors/clp/ClpConnector.h +++ b/velox/connectors/clp/ClpConnector.h @@ -20,6 +20,7 @@ #include "velox/connectors/clp/ClpConfig.h" namespace facebook::velox::connector::clp { + class ClpConnector : public Connector { public: ClpConnector( @@ -72,4 +73,5 @@ class ClpConnectorFactory : public ConnectorFactory { return std::make_shared(id, config); } }; + } // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpConnectorSplit.h b/velox/connectors/clp/ClpConnectorSplit.h index 3b00a4b6b5e..cbe13e6318b 100644 --- a/velox/connectors/clp/ClpConnectorSplit.h +++ b/velox/connectors/clp/ClpConnectorSplit.h @@ -19,6 +19,7 @@ #include "velox/connectors/Connector.h" namespace facebook::velox::connector::clp { + struct ClpConnectorSplit : public connector::ConnectorSplit { ClpConnectorSplit(const std::string& connectorId, const std::string& path) : connector::ConnectorSplit(connectorId), path_(path) {} @@ -29,4 +30,5 @@ struct ClpConnectorSplit : public connector::ConnectorSplit { const std::string path_; }; + } // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpDataSource.cpp b/velox/connectors/clp/ClpDataSource.cpp index d3f648a0808..3de0a97392e 100644 --- a/velox/connectors/clp/ClpDataSource.cpp +++ b/velox/connectors/clp/ClpDataSource.cpp @@ -163,12 +163,12 @@ std::optional ClpDataSource::next( completedRows_ += rowsScanned; size_t readerIndex = 0; const auto& projectedColumns = cursor_->getProjectedColumns(); - if (projectedColumns.size() != fields_.size()) { - VELOX_USER_FAIL( - "Projected columns size {} does not match fields size {}", - projectedColumns.size(), - fields_.size()); - } + VELOX_CHECK_EQ( + projectedColumns.size(), + fields_.size(), + "Projected columns size {} does not match fields size {}", + projectedColumns.size(), + fields_.size()); return std::dynamic_pointer_cast(createVector( outputType_, rowsFiltered, projectedColumns, filteredRows, readerIndex)); } diff --git a/velox/connectors/clp/ClpDataSource.h b/velox/connectors/clp/ClpDataSource.h index a6c536073cb..3382e128b73 100644 --- a/velox/connectors/clp/ClpDataSource.h +++ b/velox/connectors/clp/ClpDataSource.h @@ -24,6 +24,7 @@ #include "velox/connectors/clp/search_lib/ClpCursor.h" namespace facebook::velox::connector::clp { + class ClpDataSource : public DataSource { public: ClpDataSource( @@ -59,31 +60,27 @@ class ClpDataSource : public DataSource { } private: - /** - * Recursively adds fields from the column type to the list of fields to be - * retrieved from the data source. - * - * @param columnType The type of the column. - * @param parentName The name of the parent field (used for nested fields). - */ + /// Recursively adds fields from the column type to the list of fields to be + /// retrieved from the data source. + /// + /// @param columnType The type of the column. + /// @param parentName The name of the parent field (used for nested fields). void addFieldsRecursively( const TypePtr& columnType, const std::string& parentName); - /** - * Creates a Vector of the specified type and size. - * - * This method recursively creates vectors for complex types like ROW. For - * primitive types, it creates a LazyVector that will load the data from the - * underlying data source when it is accessed. - * - * @param type The type of the Vector to create. - * @param size The number of elements in the Vector. - * @param projectedColumns The readers the projected columns. - * @param filteredRows The rows to be read. - * @param readerIndex The index of the column reader. - * @return A Vector of the specified type and size. - */ + /// Creates a Vector of the specified type and size. + /// + /// This method recursively creates vectors for complex types like ROW. For + /// primitive types, it creates a LazyVector that will load the data from the + /// underlying data source when it is accessed. + /// + /// @param type The type of the Vector to create. + /// @param size The number of elements in the Vector. + /// @param projectedColumns The readers of the projected columns. + /// @param filteredRows The rows to be read. + /// @param readerIndex The index of the column reader. + /// @return A Vector of the specified type and size. VectorPtr createVector( const TypePtr& type, size_t size, @@ -103,4 +100,5 @@ class ClpDataSource : public DataSource { std::unique_ptr cursor_; }; + } // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpTableHandle.cpp b/velox/connectors/clp/ClpTableHandle.cpp index 5ef7bfecf48..0ef944768d8 100644 --- a/velox/connectors/clp/ClpTableHandle.cpp +++ b/velox/connectors/clp/ClpTableHandle.cpp @@ -17,6 +17,7 @@ #include "velox/connectors/clp/ClpTableHandle.h" namespace facebook::velox::connector::clp { + std::string ClpTableHandle::toString() const { return ConnectorTableHandle::toString(); } @@ -24,4 +25,5 @@ std::string ClpTableHandle::toString() const { folly::dynamic ClpTableHandle::serialize() const { return ConnectorTableHandle::serialize(); } + } // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/ClpTableHandle.h b/velox/connectors/clp/ClpTableHandle.h index a1ec00b329c..94c917ed1b9 100644 --- a/velox/connectors/clp/ClpTableHandle.h +++ b/velox/connectors/clp/ClpTableHandle.h @@ -19,6 +19,7 @@ #include "velox/connectors/Connector.h" namespace facebook::velox::connector::clp { + class ClpTableHandle : public ConnectorTableHandle { public: ClpTableHandle( @@ -45,4 +46,5 @@ class ClpTableHandle : public ConnectorTableHandle { const std::string tableName_; std::shared_ptr kqlQuery_; }; + } // namespace facebook::velox::connector::clp diff --git a/velox/connectors/clp/search_lib/ClpCursor.cpp b/velox/connectors/clp/search_lib/ClpCursor.cpp index e2ae5276c95..4fae9626c21 100644 --- a/velox/connectors/clp/search_lib/ClpCursor.cpp +++ b/velox/connectors/clp/search_lib/ClpCursor.cpp @@ -16,6 +16,7 @@ #include +#include "clp_s/ArchiveReader.hpp" #include "clp_s/search/EvaluateTimestampIndex.hpp" #include "clp_s/search/ast/ConvertToExists.hpp" #include "clp_s/search/ast/EmptyExpr.hpp" @@ -23,6 +24,7 @@ #include "clp_s/search/ast/OrOfAndForm.hpp" #include "clp_s/search/ast/SearchUtils.hpp" #include "clp_s/search/kql/kql.hpp" + #include "velox/connectors/clp/search_lib/ClpCursor.h" using namespace clp_s; diff --git a/velox/connectors/clp/search_lib/ClpCursor.h b/velox/connectors/clp/search_lib/ClpCursor.h index bafd5f00c98..b8420d61d6b 100644 --- a/velox/connectors/clp/search_lib/ClpCursor.h +++ b/velox/connectors/clp/search_lib/ClpCursor.h @@ -19,22 +19,34 @@ #include #include -#include "clp_s/ArchiveReader.hpp" -#include "clp_s/search/SchemaMatch.hpp" -#include "clp_s/search/ast/Expression.hpp" #include "velox/connectors/clp/search_lib/ClpQueryRunner.h" +namespace clp_s { +enum class InputSource : uint8_t; +class ArchiveReader; +class BaseColumnReader; + +namespace search { +class Projection; +class SchemaMatch; + +namespace ast { +class Expression; +} // namespace ast +} // namespace search +} // namespace clp_s + namespace facebook::velox::connector::clp::search_lib { enum class ErrorCode { - Success, - QueryNotInitialized, - InvalidQuerySyntax, - SchemaNotFound, - LogicalError, DictionaryNotFound, + InternalError, + InvalidQuerySyntax, InvalidTimestampRange, - InternalError + LogicalError, + QueryNotInitialized, + SchemaNotFound, + Success }; enum class ColumnType { String, Integer, Float, Array, Boolean, Unknown = -1 }; @@ -44,62 +56,52 @@ struct Field { std::string name; }; -/** - * This class is a query execution interface that manages the lifecycle of a - * query on a CLP-S archive, including parsing and validating the query, loading - * the relevant schemas and archives, applying filters, and iterating over the - * results. It abstracts away the low-level details of archive access and schema - * matching while supporting projection and batch-oriented retrieval of filtered - * rows. - */ +/// This class is a query execution interface that manages the lifecycle of a +/// query on a CLP-S archive, including parsing and validating the query, +/// loading the relevant schemas and archives, applying filters, and iterating +/// over the results. It abstracts away the low-level details of archive access +/// and schema matching while supporting projection and batch-oriented retrieval +/// of filtered rows. class ClpCursor { public: - // Constructor explicit ClpCursor(clp_s::InputSource inputSource, std::string archivePath); - - // Destructor ~ClpCursor(); - /** - * Executes a query. This function parses, validates, and prepares the given - * query for execution. - * @param query The KQL query to execute. - * @param outputColumns A vector specifying the columns to be included in the - * query result. - */ + /// Executes a query. This function parses, validates, and prepares the given + /// query for execution. + /// + /// @param query The KQL query to execute. + /// @param outputColumns A vector specifying the columns to be included in the + /// query result. void executeQuery( const std::string& query, const std::vector& outputColumns); - /** - * Fetches the next set of rows from the cursor.If the archive and schema are - * not yet loaded, this function will perform the necessary loading. - * @param numRows The maximum number of rows to fetch. - * @param filteredRowIndices A vector of row indices that match the filter. - * @return The number of rows scanned. - */ + /// Fetches the next set of rows from the cursor. If the archive and schema + /// are not yet loaded, this function will perform the necessary loading. + /// + /// @param numRows The maximum number of rows to fetch. + /// @param filteredRowIndices A vector of row indices that match the filter. + /// @return The number of rows scanned. uint64_t fetchNext( uint64_t numRows, const std::shared_ptr>& filteredRowIndices); - /** - * Retrieves the projected columns - * @return A vector of BaseColumnReader pointers representing the projected - * columns. - */ + /// Retrieves the projected columns. + /// + /// @return A vector of BaseColumnReader pointers representing the projected + /// columns. const std::vector& getProjectedColumns() const; private: - /** - * Preprocesses the query, performing parsing, validation, and optimization. - * @return The error code. - */ + /// Preprocesses the query, performing parsing, validation, and optimization. + /// + /// @return The error code. ErrorCode preprocessQuery(); - /** - * Loads the archive at the current index. - * @return The error code. - */ + /// Loads the archive at the current index. + /// + /// @return The error code. ErrorCode loadArchive(); ErrorCode errorCode_; diff --git a/velox/connectors/clp/search_lib/ClpQueryRunner.cpp b/velox/connectors/clp/search_lib/ClpQueryRunner.cpp index 6c1395d6345..d36479a27da 100644 --- a/velox/connectors/clp/search_lib/ClpQueryRunner.cpp +++ b/velox/connectors/clp/search_lib/ClpQueryRunner.cpp @@ -15,8 +15,7 @@ */ #include "velox/connectors/clp/search_lib/ClpQueryRunner.h" -#include "clp_s/search/clp_search/Grep.hpp" -#include "velox/vector/ComplexVector.h" +#include "clp_s/search/Projection.hpp" using namespace clp_s; using namespace clp_s::search; @@ -54,7 +53,7 @@ void ClpQueryRunner::init( } } - for (auto& [columnId, columnReader] : columnMap) { + for (const auto& [columnId, columnReader] : columnMap) { initialize_reader(columnId, columnReader); } } diff --git a/velox/connectors/clp/search_lib/ClpQueryRunner.h b/velox/connectors/clp/search_lib/ClpQueryRunner.h index 643988f97f6..c0dec586bd6 100644 --- a/velox/connectors/clp/search_lib/ClpQueryRunner.h +++ b/velox/connectors/clp/search_lib/ClpQueryRunner.h @@ -16,22 +16,31 @@ #pragma once -#include - -#include "clp_s/SchemaReader.hpp" -#include "clp_s/search/Projection.hpp" #include "clp_s/search/QueryRunner.hpp" -#include "clp_s/search/SchemaMatch.hpp" -#include "clp_s/search/ast/Expression.hpp" -#include "velox/vector/FlatVector.h" + +namespace clp_s { +class SchemaReader; +class SchemaTree; +class ArchiveReader; +class BaseColumnReader; + +namespace search { +class SchemaMatch; +class Projection; + +namespace ast { +class Expression; +} // namespace ast + +} // namespace search +} // namespace clp_s namespace facebook::velox::connector::clp::search_lib { -/** - * This class extends the generic QueryRunner to support column projection and - * row filtering over CLP-S archives. It is used by the Velox CLP connector to - * efficiently identify matching rows and project relevant columns, which are - * then consumed by the ClpVectorLoader. - */ + +/// This class extends the generic QueryRunner to support column projection and +/// row filtering over CLP-S archives. It is used by the Velox-CLP connector to +/// efficiently identify matching rows and project relevant columns, which are +/// then consumed by the ClpVectorLoader. class ClpQueryRunner : public clp_s::search::QueryRunner { public: // Constructor @@ -44,32 +53,29 @@ class ClpQueryRunner : public clp_s::search::QueryRunner { : clp_s::search::QueryRunner(match, expr, archiveReader, ignoreCase), projection_(projection) {} - /** - * Initializes the filter with schema information and column readers. - * @param schemaReader A pointer to the SchemaReader - * @param columnMap An unordered map associating column IDs with - * BaseColumnReader pointers. - */ + /// Initializes the filter with schema information and column readers. + /// + /// @param schemaReader A pointer to the SchemaReader. + /// @param columnMap An unordered map associating column IDs with + /// BaseColumnReader pointers. void init( clp_s::SchemaReader* schemaReader, std::unordered_map const& columnMap) override; - /** - * Fetches the next set of rows from the cursor. - * @param numRows The maximum number of rows to fetch. - * @param filteredRowIndices A vector to store the row indices that match the - * filter. - */ + /// Fetches the next set of rows from the cursor. + /// + /// @param numRows The maximum number of rows to fetch. + /// @param filteredRowIndices A vector to store the row indices that match the + /// filter. + /// @return The number of rows scanned. uint64_t fetchNext( uint64_t numRows, const std::shared_ptr>& filteredRowIndices); - /** - * @return A reference to the vector of BaseColumnReader pointers that - * represent the columns involved in the scanning operation. - */ - std::vector& getProjectedColumns() { + /// @return A reference to the vector of BaseColumnReader pointers that + /// represent the columns involved in the scanning operation. + const std::vector& getProjectedColumns() { return projectedColumns_; } @@ -82,4 +88,5 @@ class ClpQueryRunner : public clp_s::search::QueryRunner { uint64_t curMessage_{}; uint64_t numMessages_{}; }; + } // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ClpVectorLoader.cpp b/velox/connectors/clp/search_lib/ClpVectorLoader.cpp index c05908bf131..87f91c70c35 100644 --- a/velox/connectors/clp/search_lib/ClpVectorLoader.cpp +++ b/velox/connectors/clp/search_lib/ClpVectorLoader.cpp @@ -16,10 +16,14 @@ #include +#include "clp_s/ColumnReader.hpp" + #include "velox/connectors/clp/search_lib/ClpVectorLoader.h" #include "velox/vector/ComplexVector.h" +#include "velox/vector/FlatVector.h" namespace facebook::velox::connector::clp::search_lib { + ClpVectorLoader::ClpVectorLoader( clp_s::BaseColumnReader* columnReader, ColumnType nodeType, @@ -152,4 +156,5 @@ template void ClpVectorLoader::populateData( template void ClpVectorLoader::populateData( RowSet rows, FlatVector* vector); + } // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ClpVectorLoader.h b/velox/connectors/clp/search_lib/ClpVectorLoader.h index 98779e404af..fb9bd4ac2d1 100644 --- a/velox/connectors/clp/search_lib/ClpVectorLoader.h +++ b/velox/connectors/clp/search_lib/ClpVectorLoader.h @@ -16,16 +16,18 @@ #pragma once -#include "clp_s/ColumnReader.hpp" #include "velox/connectors/clp/search_lib/ClpCursor.h" #include "velox/vector/LazyVector.h" +namespace clp_s { +class BaseColumnReader; +} // namespace clp_s + namespace facebook::velox::connector::clp::search_lib { -/** - * ClpVectorLoader is a custom Velox VectorLoader that populates Velox vectors - * from a CLP-based column reader. It supports various column types including - * integers, floats, booleans, strings, and arrays of strings. - */ + +/// ClpVectorLoader is a custom Velox VectorLoader that populates Velox vectors +/// from a CLP-based column reader. It supports various column types including +/// integers, floats, booleans, strings, and arrays of strings. class ClpVectorLoader : public VectorLoader { public: ClpVectorLoader( @@ -50,4 +52,5 @@ class ClpVectorLoader : public VectorLoader { inline static thread_local std::unique_ptr arrayParser_ = std::make_unique(); }; + } // namespace facebook::velox::connector::clp::search_lib From 952a58df2ffab9bb7fe5d9e566312c04d7800713 Mon Sep 17 00:00:00 2001 From: wraymo Date: Thu, 3 Jul 2025 00:32:37 +0000 Subject: [PATCH 20/31] update clp --- CMake/resolve_dependency_modules/clp.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMake/resolve_dependency_modules/clp.cmake b/CMake/resolve_dependency_modules/clp.cmake index e91e1b79850..fbdb5c2deb0 100644 --- a/CMake/resolve_dependency_modules/clp.cmake +++ b/CMake/resolve_dependency_modules/clp.cmake @@ -16,7 +16,8 @@ include_guard(GLOBAL) FetchContent_Declare( clp GIT_REPOSITORY https://github.com/y-scope/clp.git - GIT_TAG 35208950df5acf38ff8f0c560abee65a61b28703) + GIT_TAG 0798100389bd5231b520ec48ab186275795e3790 + GIT_SHALLOW TRUE) set(CLP_BUILD_TESTING OFF From 453af87c2b7c4ff0e1252fa3e684c90bdcd18748 Mon Sep 17 00:00:00 2001 From: wraymo Date: Thu, 3 Jul 2025 02:52:35 +0000 Subject: [PATCH 21/31] update tests and add array type --- CMake/resolve_dependency_modules/clp.cmake | 3 - .../clp/search_lib/ClpVectorLoader.cpp | 6 ++ velox/connectors/clp/tests/CMakeLists.txt | 1 + .../connectors/clp/tests/ClpConnectorTest.cpp | 54 ++++++++---------- .../connectors/clp/tests/examples/test_2.clps | Bin 1576 -> 1947 bytes .../clp/tests/examples/test_2.ndjson | 20 +++---- 6 files changed, 42 insertions(+), 42 deletions(-) diff --git a/CMake/resolve_dependency_modules/clp.cmake b/CMake/resolve_dependency_modules/clp.cmake index fbdb5c2deb0..b1cf6e191a2 100644 --- a/CMake/resolve_dependency_modules/clp.cmake +++ b/CMake/resolve_dependency_modules/clp.cmake @@ -25,9 +25,6 @@ set(CLP_BUILD_TESTING set(CLP_BUILD_EXECUTABLES OFF CACHE BOOL "Build CLP executables") -set(CLP_BUILD_CLP_S_ARCHIVEWRITER - OFF - CACHE BOOL "Build CLP-S archive writer") set(CLP_BUILD_CLP_S_JSONCONSTRUCTOR OFF CACHE BOOL "Build CLP-S JSON constructor") diff --git a/velox/connectors/clp/search_lib/ClpVectorLoader.cpp b/velox/connectors/clp/search_lib/ClpVectorLoader.cpp index 87f91c70c35..a299d582df9 100644 --- a/velox/connectors/clp/search_lib/ClpVectorLoader.cpp +++ b/velox/connectors/clp/search_lib/ClpVectorLoader.cpp @@ -89,6 +89,12 @@ void ClpVectorLoader::loadInternal( break; } case ColumnType::Array: { + if (columnReader_ == nullptr) { + for (int vectorIndex : rows) { + vector->setNull(vectorIndex, true); + } + return; + } auto arrayVector = std::dynamic_pointer_cast(vector); auto elements = arrayVector->elements()->asFlatVector(); vector_size_t elementIndex = 0; diff --git a/velox/connectors/clp/tests/CMakeLists.txt b/velox/connectors/clp/tests/CMakeLists.txt index 6c0c2147366..7b8331bb8d4 100644 --- a/velox/connectors/clp/tests/CMakeLists.txt +++ b/velox/connectors/clp/tests/CMakeLists.txt @@ -17,6 +17,7 @@ add_test(velox_clp_connector_test velox_clp_connector_test) target_link_libraries( velox_clp_connector_test + clp_s_archive_writer velox_clp_connector velox_vector_test_lib velox_exec_test_lib diff --git a/velox/connectors/clp/tests/ClpConnectorTest.cpp b/velox/connectors/clp/tests/ClpConnectorTest.cpp index c7473bf9980..62df85d357f 100644 --- a/velox/connectors/clp/tests/ClpConnectorTest.cpp +++ b/velox/connectors/clp/tests/ClpConnectorTest.cpp @@ -86,10 +86,7 @@ TEST_F(ClpConnectorTest, test1NoPushdown) { ROW({"requestId", "userId", "method"}, {VARCHAR(), VARCHAR(), VARCHAR()})) .tableHandle(std::make_shared( - kClpConnectorId, - "test_1", - ClpTableHandle::StorageType::kFS, - nullptr)) + kClpConnectorId, "test_1", nullptr)) .assignments({ {"requestId", std::make_shared( @@ -134,7 +131,6 @@ TEST_F(ClpConnectorTest, test1Pushdown) { .tableHandle(std::make_shared( kClpConnectorId, "test_1", - ClpTableHandle::StorageType::kFS, std::make_shared( "method: \"POST\" AND status: 200"))) .assignments({ @@ -173,10 +169,7 @@ TEST_F(ClpConnectorTest, test2NoPushdown) { ROW({"type", "subtype", "severity"}, {VARCHAR(), VARCHAR(), VARCHAR()})})) .tableHandle(std::make_shared( - kClpConnectorId, - "test_2", - ClpTableHandle::StorageType::kFS, - nullptr)) + kClpConnectorId, "test_2", nullptr)) .assignments( {{"timestamp", std::make_shared( @@ -224,7 +217,6 @@ TEST_F(ClpConnectorTest, test2Pushdown) { .tableHandle(std::make_shared( kClpConnectorId, "test_2", - ClpTableHandle::StorageType::kFS, std::make_shared( "(event.severity: \"WARNING\" OR event.severity: \"ERROR\") AND " "((event.type: \"network\" AND event.subtype: \"connection\") OR " @@ -267,12 +259,11 @@ TEST_F(ClpConnectorTest, test2Hybrid) { .outputType( ROW({"timestamp", "event"}, {VARCHAR(), - ROW({"type", "subtype", "severity"}, - {VARCHAR(), VARCHAR(), VARCHAR()})})) + ROW({"type", "subtype", "severity", "tags"}, + {VARCHAR(), VARCHAR(), VARCHAR(), ARRAY(VARCHAR())})})) .tableHandle(std::make_shared( kClpConnectorId, "test_2", - ClpTableHandle::StorageType::kFS, std::make_shared( "((event.type: \"network\" AND event.subtype: \"connection\") OR " "(event.type: \"storage\" AND event.subtype: \"disk*\"))"))) @@ -284,27 +275,32 @@ TEST_F(ClpConnectorTest, test2Hybrid) { std::make_shared( "event", "event", - ROW({"type", "subtype", "severity"}, - {VARCHAR(), VARCHAR(), VARCHAR()}), + ROW({"type", "subtype", "severity", "tags"}, + {VARCHAR(), VARCHAR(), VARCHAR(), ARRAY(VARCHAR())}), true)}}) .endTableScan() .filter("upper(event.severity) IN ('WARNING', 'ERROR')") .planNode(); - auto output = - getResults(plan, {makeClpSplit(getExampleFilePath("test_2.clps"))}); - auto expected = - makeRowVector({// timestamp - makeFlatVector({"2025-04-30T08:50:05Z"}), - // event - makeRowVector({ - // event.type - makeFlatVector({"storage"}), - // event.subtype - makeFlatVector({"disk_usage"}), - // event.severity - makeFlatVector({"WARNING"}), - })}); + auto output = getResults( + plan, + {makeClpSplit( + getExampleFilePath("test_2.clps"))}); + auto expected = makeRowVector( + {// timestamp + makeFlatVector({"2025-04-30T08:50:05Z"}), + // event + makeRowVector({// event.type + makeFlatVector({"storage"}), + // event.subtype + makeFlatVector({"disk_usage"}), + // event.severity + makeFlatVector({"WARNING"}), + // event.tags + makeArrayVector( + {{"\"filesystem\"", "\"monitoring\""}})}) + + }); test::assertEqualVectors(expected, output); } diff --git a/velox/connectors/clp/tests/examples/test_2.clps b/velox/connectors/clp/tests/examples/test_2.clps index a4c84a4a9101cb6a0bf2b9e730aed0fdafb7f8a1..ad773162d3af70c22fe438bbaa4342120afecb2a 100644 GIT binary patch delta 1877 zcmV-b2dem}44V%U{V&BZ0ssR5JqZ8+00000n+K607m+XskuV2;G64Vp00000D77#B z09Z{407er+E^vw39CB5a6*Phy8-4iDHCrSimQzC*LtWfBh*}&IkQ+)`#w#) zwyE-*3o{RJBUPk-INFT@1FwKzi~}=Q05AbVA>f)fVi13=h6Y1Z+6j%xlo`e~uXXW3 zv@5pxYe9@_-nRYPda&bD|Awhcrph-PB~_ct2-`&D5FYS7B={VWM}h{Dr~Mr~Q6zFa z4gzt9By~aO#YU%Xr>=Qi@6WpM9(uIulA7A$tlH|9QPQ!0N~d-HKb>Vtuh`FME6y$c zcol8`LrK@BtxhQhK>-T@$rNr%ZY~+=>bRSif%70Ky(!!V$Vtt+(}=kTD77#B09fS= z0Cp8aFOVlm2mk;7|Ns9F02?8@r5(21*Zgg_m77Q^@4kpZJ^EMSvWEKBTGA|$gpig3 zH2^aJHvlJpVU)+BlhX)QERT=yA33xw`bYjFc>Hf1ZDq5r)XIMY6&#%8&`cxL0j)ZK z|4(?+G|d$R0?;YHpJeDthA?HEwg9Lo`5n#Pu6ElN7LaY~WT{!tukc?vTDtK+0hMuY zlm7~`O4%CAD2W58>K>HDf(+nMehVsqDtjA^ldBeg*%Y~K0;uXaK5QA~0jMi=d)nEI zioups-EkJ8btM6b1tmazkQf2Jqu|%s+|Yo_GC2M#38PdG%r-Hn<+MR`HpFS$2Fk8a z<$|M@fdLUq-O5bawiQOGP|gTVZ3>bO;KHi04G};ni4_s3(}iKb1HgH{sXJfMcf{XxPF;K4zY`B=Ue~ITn zQSBjPKwpqEv`#2`{;`Qykg9>5z=83`YfqgCBwMK2($YgvTpSFZ}>modN(P0s#P60{{wt z00;m80097K00YPeC;0T2TM7XZKrFd#?_WCGHGjDaM;fB*vo0NH`W zKqeqvFd#?_Wb!jzFkm1TpkQze8DIf6000000000ewJ-euSal5mb{9+~AdlQ2lPrfT z6xotb09#~#znldq-xNUrKmbGl5Vx9EiwlL7TRr7dX&tG# z#*N)J>{`?i4qSs0+!9n8D3fhO3IBPF@~yyS%FKIQ2*s?xk^%x#a1@bnM1mMvCy>KX zi=1oRB8DzX7vQN^%Q01^iNR;ZU|XVu-w%HD>uVcOl5?V*?`J=CQ0h-k3_DAIV$W=h zXLlP1%7ij4R0P+1%xXbPl&^r2AQmE~3gt0n=GlYd$m#FuftY?az%fTA5E2v3gTv&( zv0g{U!&#rxhLWHNW2_TWVg9fB4}TD2It`V-W0*|70E-L^c13`xm)94PB$AmKS@lJq z_x|ia)joF{j@@)xA(Z-ijX_I)gJOyEReA=W_C#s@A`pfpO}jdgZW?}hZ^Nb z#$mJpU;&dA1wwxd000000000ewJ-euSak&e;tC-hfTzs?06r1fuq!s6{Ox7ihahNz z;6R$35fO^4TIWR_C7WZV861cpaNehe4&Q5{CPYyEaHiNCqk4hhtVn)0Nl5WvZ7DK< zvpQIW%&-AxW!I1^L6nv~YX_>$aUNM};JnYeGqg`hSFO0D?nS~vj%*>3twuu>mDD{Vu9%AUQ+o9B-Kxs?%CPePNQ~$pK zS|mIWL>7O{=)~aHd0Vbb*&AEicHCIWF&4X88UIkKT5AP(Tc0)sBv&F0pDekK4EgbO z>Tb@PHS^4vFV7VxMts=tc47Hw(V;=pvp;+8Oaox3a8$P&z%!B#5hQq$bE@6iB$j9% zh;A*dsnJroUa{|xNLPFuSk%wav*E(?l)f9UNphDiubloz^M06;Vm0M2%`HluLN!J{2j{ z#Ke&ixgUzQ&O}itsgT@GD#xk*Iq$!o*Yo=P@jS2R^LetBit)-{>CFrPU@(IQ-=U&X z^wd%qm=p$@H}Incs3Q7Tz|KVh=T1|Lm`%NqrB>5K22{#8d)Yk^7*3%z z9aolLcSg&aR!18^wYhcGJ~>~5{;?bv4j zq_=u`$Q0&Q$5Nu#;zgOUyxuCL1z-Rxuqa`Vio>{(<8uA-(@)uU<+*~Mxq?dmwt9(v zs18Pt)^eAg({p_oR4Ft(`eeV0F!4D@@{!MO?5?(k7Eo_pGU|4|yKAJ}=Do^)-)y^w zY+G_sv~1lFPh)=Cm*aGlgsl$|sAs$eMejJ8@R*>h*`VNvW^vu2Aly*y8>{lb+j3Kz z_DAimjPH2h-9t(B_N!Nf8F7VPF-Za|NPK;aQ~4)*C{>Zby}4Byeh{0mpWS3?$&54)cgERMoz;x=r6)KqBgm8cbW-z+4if6Dsdp_e#iB+jW=H3iL<_D` z2^Q(_!JQGxq+R={T00$|)OdIAcHgHq0aC+;9kgkAQ^Lap!~xPL*Me6TB7NBn2NkM_ z_n+&eFql54M2yMEhBxCe_GdIy6BPCiwvv*_)2);^H4pWjQ_SH+I?0p#vFS-3Wf2L9 zE$;22-B@e;wrB8puJ!Ib``s(RL3qgbRa8fVWY2sBzBDvqukVoGKt9$#XaG5}6x%W} z(Nf*0`R#E|VOCUpqsK9=j(@uUQ0MtVKg>~v0EP$%EFb`OIVk`@fDO<>Bxbo}$qUBh z7zosHs2ZUT8Zd2&hN#uh+tZMfzphFQK*YR1WsC|xO*CH==p(Mf8jG27>WpL@i(wOo z7n=PqU+Fjyk^Y+zp>qr}>)5%y%9N}A0@?<4O!^F9VN5qp`8pt7~HT;zi)jV8@8daQnaWvH=HBES-NEDygcXEW6_=qkJ9XC~J za~s@p%CmYtdnRj-D)MsRxNW5e(cn#gY_mK4=hWv_7_o zK=;Jgj9SBZCHpSE#9LXhImEW?+amGG0lQKW{=)L712d<+}b_X0t65G@KZQU^x~TYExDr zsS*@XQbF>#SD||mWFHoFwxLZNb9bG!@^{( zkvBRo1lswHSJqvy!ry;i-cG7IH)!VrAJP< Date: Thu, 3 Jul 2025 02:57:25 +0000 Subject: [PATCH 22/31] fix cmake --- velox/connectors/clp/search_lib/CMakeLists.txt | 3 ++- velox/connectors/clp/tests/CMakeLists.txt | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/velox/connectors/clp/search_lib/CMakeLists.txt b/velox/connectors/clp/search_lib/CMakeLists.txt index 0f7eefc58e0..db683298cc6 100644 --- a/velox/connectors/clp/search_lib/CMakeLists.txt +++ b/velox/connectors/clp/search_lib/CMakeLists.txt @@ -26,8 +26,9 @@ velox_link_libraries( clp-s-search PUBLIC clp_s::archive_reader PRIVATE - clp_s::io + clp_s::archive_writer clp_s::clp_dependencies + clp_s::io clp_s::search clp_s::search::kql velox_vector) diff --git a/velox/connectors/clp/tests/CMakeLists.txt b/velox/connectors/clp/tests/CMakeLists.txt index 7b8331bb8d4..6c0c2147366 100644 --- a/velox/connectors/clp/tests/CMakeLists.txt +++ b/velox/connectors/clp/tests/CMakeLists.txt @@ -17,7 +17,6 @@ add_test(velox_clp_connector_test velox_clp_connector_test) target_link_libraries( velox_clp_connector_test - clp_s_archive_writer velox_clp_connector velox_vector_test_lib velox_exec_test_lib From d249799697d1ea7712b3162b0fbc75cb08223b50 Mon Sep 17 00:00:00 2001 From: wraymo Date: Thu, 3 Jul 2025 12:47:31 +0000 Subject: [PATCH 23/31] fix --- CMake/resolve_dependency_modules/clp.cmake | 3 +-- velox/connectors/clp/tests/ClpConnectorTest.cpp | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CMake/resolve_dependency_modules/clp.cmake b/CMake/resolve_dependency_modules/clp.cmake index b1cf6e191a2..6315adb7927 100644 --- a/CMake/resolve_dependency_modules/clp.cmake +++ b/CMake/resolve_dependency_modules/clp.cmake @@ -16,8 +16,7 @@ include_guard(GLOBAL) FetchContent_Declare( clp GIT_REPOSITORY https://github.com/y-scope/clp.git - GIT_TAG 0798100389bd5231b520ec48ab186275795e3790 - GIT_SHALLOW TRUE) + GIT_TAG 0798100389bd5231b520ec48ab186275795e3790) set(CLP_BUILD_TESTING OFF diff --git a/velox/connectors/clp/tests/ClpConnectorTest.cpp b/velox/connectors/clp/tests/ClpConnectorTest.cpp index 62df85d357f..bb351984b64 100644 --- a/velox/connectors/clp/tests/ClpConnectorTest.cpp +++ b/velox/connectors/clp/tests/ClpConnectorTest.cpp @@ -282,10 +282,8 @@ TEST_F(ClpConnectorTest, test2Hybrid) { .filter("upper(event.severity) IN ('WARNING', 'ERROR')") .planNode(); - auto output = getResults( - plan, - {makeClpSplit( - getExampleFilePath("test_2.clps"))}); + auto output = + getResults(plan, {makeClpSplit(getExampleFilePath("test_2.clps"))}); auto expected = makeRowVector( {// timestamp makeFlatVector({"2025-04-30T08:50:05Z"}), From 9411387c88505ef82052ac0ba4e3cd478ab56561 Mon Sep 17 00:00:00 2001 From: wraymo Date: Fri, 4 Jul 2025 00:36:44 +0000 Subject: [PATCH 24/31] address review comments --- CMake/resolve_dependency_modules/clp.cmake | 14 ++++++------ CMakeLists.txt | 22 +++++++++---------- velox/connectors/clp/ClpConfig.cpp | 4 +++- velox/connectors/clp/ClpConfig.h | 2 -- velox/connectors/clp/ClpDataSource.h | 5 ++++- .../connectors/clp/search_lib/CMakeLists.txt | 7 +++--- velox/connectors/clp/search_lib/ClpCursor.cpp | 2 +- .../clp/search_lib/ClpQueryRunner.cpp | 2 +- velox/connectors/clp/tests/CMakeLists.txt | 2 +- velox/docs/develop/connectors.rst | 14 ++++++------ 10 files changed, 38 insertions(+), 36 deletions(-) diff --git a/CMake/resolve_dependency_modules/clp.cmake b/CMake/resolve_dependency_modules/clp.cmake index 6315adb7927..6859f41ae08 100644 --- a/CMake/resolve_dependency_modules/clp.cmake +++ b/CMake/resolve_dependency_modules/clp.cmake @@ -18,12 +18,9 @@ FetchContent_Declare( GIT_REPOSITORY https://github.com/y-scope/clp.git GIT_TAG 0798100389bd5231b520ec48ab186275795e3790) -set(CLP_BUILD_TESTING - OFF - CACHE BOOL "Build CLP tests") -set(CLP_BUILD_EXECUTABLES +set(CLP_BUILD_CLP_REGEX_UTILS OFF - CACHE BOOL "Build CLP executables") + CACHE BOOL "Build CLP regex utils") set(CLP_BUILD_CLP_S_JSONCONSTRUCTOR OFF CACHE BOOL "Build CLP-S JSON constructor") @@ -33,9 +30,12 @@ set(CLP_BUILD_CLP_S_REDUCER_DEPENDENCIES set(CLP_BUILD_CLP_S_SEARCH_SQL OFF CACHE BOOL "Build CLP-S search SQL") -set(CLP_BUILD_CLP_REGEX_UTILS +set(CLP_BUILD_EXECUTABLES OFF - CACHE BOOL "Build CLP regex utils") + CACHE BOOL "Build CLP executables") +set(CLP_BUILD_TESTING + OFF + CACHE BOOL "Build CLP tests") FetchContent_Populate(clp) diff --git a/CMakeLists.txt b/CMakeLists.txt index a665beb1292..ba0734c6ee4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -523,29 +523,29 @@ velox_set_source(re2) velox_resolve_dependency(re2) if(${VELOX_ENABLE_CLP_CONNECTOR}) - set(ystdlib_cpp_SOURCE BUNDLED) - velox_resolve_dependency(ystdlib_cpp) - velox_set_source(absl) velox_resolve_dependency(absl) + velox_set_source(antlr4-runtime) + velox_resolve_dependency(antlr4-runtime) + + velox_set_source(date) + velox_resolve_dependency(date) + + velox_set_source(msgpack-cxx) + velox_resolve_dependency(msgpack-cxx) + velox_set_source(nlohmann_json) velox_resolve_dependency(nlohmann_json) velox_set_source(simdjson) velox_resolve_dependency(simdjson) - velox_set_source(date) - velox_resolve_dependency(date) - velox_set_source(spdlog) velox_resolve_dependency(spdlog) - velox_set_source(msgpack-cxx) - velox_resolve_dependency(msgpack-cxx) - - velox_set_source(antlr4-runtime) - velox_resolve_dependency(antlr4-runtime) + set(ystdlib_cpp_SOURCE BUNDLED) + velox_resolve_dependency(ystdlib_cpp) set(clp_SOURCE BUNDLED) velox_resolve_dependency(clp) diff --git a/velox/connectors/clp/ClpConfig.cpp b/velox/connectors/clp/ClpConfig.cpp index 22f6aa4a6f0..7dc3d4a90f0 100644 --- a/velox/connectors/clp/ClpConfig.cpp +++ b/velox/connectors/clp/ClpConfig.cpp @@ -14,7 +14,9 @@ * limitations under the License. */ -#include "ClpConfig.h" +#include + +#include "velox/connectors/clp/ClpConfig.h" namespace facebook::velox::connector::clp { diff --git a/velox/connectors/clp/ClpConfig.h b/velox/connectors/clp/ClpConfig.h index fa0d4de7699..3f1cabc8f8a 100644 --- a/velox/connectors/clp/ClpConfig.h +++ b/velox/connectors/clp/ClpConfig.h @@ -16,8 +16,6 @@ #pragma once -#include - #include "velox/common/config/Config.h" namespace facebook::velox::config { diff --git a/velox/connectors/clp/ClpDataSource.h b/velox/connectors/clp/ClpDataSource.h index 3382e128b73..768f2b61021 100644 --- a/velox/connectors/clp/ClpDataSource.h +++ b/velox/connectors/clp/ClpDataSource.h @@ -20,9 +20,12 @@ #include "velox/connectors/Connector.h" #include "velox/connectors/clp/ClpConfig.h" -#include "velox/connectors/clp/ClpTableHandle.h" #include "velox/connectors/clp/search_lib/ClpCursor.h" +namespace clp_s { +class BaseColumnReader; +} // namespace clp_s + namespace facebook::velox::connector::clp { class ClpDataSource : public DataSource { diff --git a/velox/connectors/clp/search_lib/CMakeLists.txt b/velox/connectors/clp/search_lib/CMakeLists.txt index db683298cc6..7f163d0aa2a 100644 --- a/velox/connectors/clp/search_lib/CMakeLists.txt +++ b/velox/connectors/clp/search_lib/CMakeLists.txt @@ -14,13 +14,12 @@ velox_add_library( clp-s-search STATIC - ${CLP_SRC_FILES} - ClpVectorLoader.cpp - ClpVectorLoader.h ClpCursor.cpp ClpCursor.h ClpQueryRunner.cpp - ClpQueryRunner.h) + ClpQueryRunner.h + ClpVectorLoader.cpp + ClpVectorLoader.h) velox_link_libraries( clp-s-search diff --git a/velox/connectors/clp/search_lib/ClpCursor.cpp b/velox/connectors/clp/search_lib/ClpCursor.cpp index 4fae9626c21..8d436cf615d 100644 --- a/velox/connectors/clp/search_lib/ClpCursor.cpp +++ b/velox/connectors/clp/search_lib/ClpCursor.cpp @@ -108,7 +108,7 @@ const std::vector& ClpCursor::getProjectedColumns() if (queryRunner_) { return queryRunner_->getProjectedColumns(); } - static std::vector kEmpty; + static std::vector const kEmpty; return kEmpty; } diff --git a/velox/connectors/clp/search_lib/ClpQueryRunner.cpp b/velox/connectors/clp/search_lib/ClpQueryRunner.cpp index d36479a27da..f110fc6d00a 100644 --- a/velox/connectors/clp/search_lib/ClpQueryRunner.cpp +++ b/velox/connectors/clp/search_lib/ClpQueryRunner.cpp @@ -15,13 +15,13 @@ */ #include "velox/connectors/clp/search_lib/ClpQueryRunner.h" -#include "clp_s/search/Projection.hpp" using namespace clp_s; using namespace clp_s::search; using namespace clp_s::search::clp_search; namespace facebook::velox::connector::clp::search_lib { + void ClpQueryRunner::init( clp_s::SchemaReader* schemaReader, std::unordered_map const& columnMap) { diff --git a/velox/connectors/clp/tests/CMakeLists.txt b/velox/connectors/clp/tests/CMakeLists.txt index 6c0c2147366..4e53803374b 100644 --- a/velox/connectors/clp/tests/CMakeLists.txt +++ b/velox/connectors/clp/tests/CMakeLists.txt @@ -18,8 +18,8 @@ add_test(velox_clp_connector_test velox_clp_connector_test) target_link_libraries( velox_clp_connector_test velox_clp_connector - velox_vector_test_lib velox_exec_test_lib + velox_vector_test_lib GTest::gtest GTest::gtest_main) diff --git a/velox/docs/develop/connectors.rst b/velox/docs/develop/connectors.rst index 27e94d9a1c2..b35613d0313 100644 --- a/velox/docs/develop/connectors.rst +++ b/velox/docs/develop/connectors.rst @@ -125,18 +125,18 @@ This is the behavior when the proxy settings are enabled: CLP Connector ------------- The CLP Connector is used to read CLP archives stored in a local file system or S3. It implements similar -interfaces as the Hive Connector except for the `DataSink` interface. Here we only describe the `DataSource` -interface and the `ConnectorSplit` interface implementation since `Connector` and `ConnectorFactory` are +interfaces as the Hive Connector except for the ``DataSink`` interface. Here we only describe the ``DataSource`` +interface and the ``ConnectorSplit`` interface implementation since `Connector` and ``ConnectorFactory`` are similar to the Hive Connector. ClpConnectorSplit ~~~~~~~~~~~~~~~~~ -The ClpConnectorSplit describes a data chunk using `split_path`. For now, only archive format is supported. -The `split_path` is the path to the archive file. +The ClpConnectorSplit describes a data chunk using ``path``. For now, only archive format is supported. +The ``path`` is the path to the archive file. ClpDataSource ~~~~~~~~~~~~~ -The ClpDataSource implements the `addSplit` API that consumes a ClpConnectorSplit and `next` API that +The ClpDataSource implements the ``addSplit`` API that consumes a ClpConnectorSplit and ``next`` API that processes the split and returns a batch of rows. During initialization, it records the KQL query and archive source (S3 or local). It then iterates through each @@ -144,7 +144,7 @@ output column, accessing its handle to get its type and original name, and speci recursively traverses the nested structure to process each field; for non-row types, it directly maps the Velox column type to a CLP column type. -When a split is added, a `ClpCursor` is created with the archive path and input source. The query is parsed -and simplified into an AST. On `next`, the cursor finds matching row indices and, if any exist, `ClpDataSource` +When a split is added, a ``ClpCursor`` is created with the archive path and input source. The query is parsed +and simplified into an AST. On ``next``, the cursor finds matching row indices and, if any exist, ``ClpDataSource`` recursively creates a row vector composed of lazy vectors, which use CLP column readers to decode and load data as needed during execution. From f3b0576cfc3cce7869a9107e5b844bddc60c1e7d Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 7 Jul 2025 14:04:33 +0000 Subject: [PATCH 25/31] address review comments --- .github/workflows/linux-build-base.yml | 2 +- velox/connectors/clp/ClpConfig.cpp | 2 +- velox/connectors/clp/ClpConfig.h | 2 +- velox/connectors/clp/ClpDataSource.cpp | 22 ++++++++-------- velox/connectors/clp/ClpDataSource.h | 10 ++++---- velox/connectors/clp/search_lib/ClpCursor.h | 11 ++++---- .../clp/search_lib/ClpQueryRunner.h | 5 ++-- .../clp/search_lib/ClpVectorLoader.h | 6 ++--- velox/docs/develop/connectors.rst | 25 +++++++++---------- 9 files changed, 41 insertions(+), 44 deletions(-) diff --git a/.github/workflows/linux-build-base.yml b/.github/workflows/linux-build-base.yml index 6aa766ed9ef..49529a21473 100644 --- a/.github/workflows/linux-build-base.yml +++ b/.github/workflows/linux-build-base.yml @@ -52,8 +52,8 @@ jobs: - name: Build Artifact env: VELOX_DEPENDENCY_SOURCE: BUNDLED - ICU_SOURCE: SYSTEM Boost_SOURCE: SYSTEM + ICU_SOURCE: SYSTEM MAKEFLAGS: MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3 run: | if [[ "${USE_CLANG}" = "true" ]]; then export CC=/usr/bin/clang-15; export CXX=/usr/bin/clang++-15; fi diff --git a/velox/connectors/clp/ClpConfig.cpp b/velox/connectors/clp/ClpConfig.cpp index 7dc3d4a90f0..adbb8f0fc47 100644 --- a/velox/connectors/clp/ClpConfig.cpp +++ b/velox/connectors/clp/ClpConfig.cpp @@ -25,7 +25,7 @@ namespace { ClpConfig::StorageType stringToStorageType(const std::string& strValue) { auto upperValue = boost::algorithm::to_upper_copy(strValue); if (upperValue == "FS") { - return ClpConfig::StorageType::kFS; + return ClpConfig::StorageType::kFs; } if (upperValue == "S3") { return ClpConfig::StorageType::kS3; diff --git a/velox/connectors/clp/ClpConfig.h b/velox/connectors/clp/ClpConfig.h index 3f1cabc8f8a..ce4808eef62 100644 --- a/velox/connectors/clp/ClpConfig.h +++ b/velox/connectors/clp/ClpConfig.h @@ -27,7 +27,7 @@ namespace facebook::velox::connector::clp { class ClpConfig { public: enum class StorageType { - kFS, + kFs, kS3, }; diff --git a/velox/connectors/clp/ClpDataSource.cpp b/velox/connectors/clp/ClpDataSource.cpp index 3de0a97392e..6bc4d1d1414 100644 --- a/velox/connectors/clp/ClpDataSource.cpp +++ b/velox/connectors/clp/ClpDataSource.cpp @@ -103,7 +103,7 @@ void ClpDataSource::addFieldsRecursively( void ClpDataSource::addSplit(std::shared_ptr split) { auto clpSplit = std::dynamic_pointer_cast(split); - if (storageType_ == ClpConfig::StorageType::kFS) { + if (storageType_ == ClpConfig::StorageType::kFs) { cursor_ = std::make_unique( clp_s::InputSource::Filesystem, clpSplit->path_); } else if (storageType_ == ClpConfig::StorageType::kS3) { @@ -115,27 +115,27 @@ void ClpDataSource::addSplit(std::shared_ptr split) { } VectorPtr ClpDataSource::createVector( - const TypePtr& type, - size_t size, + const TypePtr& vectorType, + size_t vectorSize, const std::vector& projectedColumns, const std::shared_ptr>& filteredRows, size_t& readerIndex) { - if (type->kind() == TypeKind::ROW) { + if (vectorType->kind() == TypeKind::ROW) { std::vector children; - auto& rowType = type->as(); + auto& rowType = vectorType->as(); for (uint32_t i = 0; i < rowType.size(); ++i) { children.push_back(createVector( rowType.childAt(i), - size, + vectorSize, projectedColumns, filteredRows, readerIndex)); } return std::make_shared( - pool_, type, nullptr, size, std::move(children)); + pool_, vectorType, nullptr, vectorSize, std::move(children)); } - auto vector = BaseVector::create(type, size, pool_); - vector->setNulls(allocateNulls(size, pool_, bits::kNull)); + auto vector = BaseVector::create(vectorType, vectorSize, pool_); + vector->setNulls(allocateNulls(vectorSize, pool_, bits::kNull)); VELOX_CHECK_LT( readerIndex, projectedColumns.size(), "Reader index out of bounds"); @@ -144,8 +144,8 @@ VectorPtr ClpDataSource::createVector( readerIndex++; return std::make_shared( pool_, - type, - size, + vectorType, + vectorSize, std::make_unique( projectedColumn, projectedType, filteredRows), std::move(vector)); diff --git a/velox/connectors/clp/ClpDataSource.h b/velox/connectors/clp/ClpDataSource.h index 768f2b61021..c9ab8ea7d0f 100644 --- a/velox/connectors/clp/ClpDataSource.h +++ b/velox/connectors/clp/ClpDataSource.h @@ -66,7 +66,7 @@ class ClpDataSource : public DataSource { /// Recursively adds fields from the column type to the list of fields to be /// retrieved from the data source. /// - /// @param columnType The type of the column. + /// @param columnType /// @param parentName The name of the parent field (used for nested fields). void addFieldsRecursively( const TypePtr& columnType, @@ -78,15 +78,15 @@ class ClpDataSource : public DataSource { /// primitive types, it creates a LazyVector that will load the data from the /// underlying data source when it is accessed. /// - /// @param type The type of the Vector to create. - /// @param size The number of elements in the Vector. + /// @param vectorType + /// @param vectorSize /// @param projectedColumns The readers of the projected columns. /// @param filteredRows The rows to be read. /// @param readerIndex The index of the column reader. /// @return A Vector of the specified type and size. VectorPtr createVector( - const TypePtr& type, - size_t size, + const TypePtr& vectorType, + size_t vectorSize, const std::vector& projectedColumns, const std::shared_ptr>& filteredRows, size_t& readerIndex); diff --git a/velox/connectors/clp/search_lib/ClpCursor.h b/velox/connectors/clp/search_lib/ClpCursor.h index b8420d61d6b..bf254fa788a 100644 --- a/velox/connectors/clp/search_lib/ClpCursor.h +++ b/velox/connectors/clp/search_lib/ClpCursor.h @@ -56,12 +56,11 @@ struct Field { std::string name; }; -/// This class is a query execution interface that manages the lifecycle of a -/// query on a CLP-S archive, including parsing and validating the query, -/// loading the relevant schemas and archives, applying filters, and iterating -/// over the results. It abstracts away the low-level details of archive access -/// and schema matching while supporting projection and batch-oriented retrieval -/// of filtered rows. +/// A query execution interface that manages the lifecycle of a query on a CLP-S +/// archive, including parsing and validating the query, loading the relevant +/// schemas and archives, applying filters, and iterating over the results. It +/// abstracts away the low-level details of archive access and schema matching +/// while supporting projection and batch-oriented retrieval of filtered rows. class ClpCursor { public: explicit ClpCursor(clp_s::InputSource inputSource, std::string archivePath); diff --git a/velox/connectors/clp/search_lib/ClpQueryRunner.h b/velox/connectors/clp/search_lib/ClpQueryRunner.h index c0dec586bd6..aec15b24ede 100644 --- a/velox/connectors/clp/search_lib/ClpQueryRunner.h +++ b/velox/connectors/clp/search_lib/ClpQueryRunner.h @@ -37,13 +37,12 @@ class Expression; namespace facebook::velox::connector::clp::search_lib { -/// This class extends the generic QueryRunner to support column projection and -/// row filtering over CLP-S archives. It is used by the Velox-CLP connector to +/// Extends the generic QueryRunner to support column projection and row +/// filtering over CLP-S archives. It is used by the Velox-CLP connector to /// efficiently identify matching rows and project relevant columns, which are /// then consumed by the ClpVectorLoader. class ClpQueryRunner : public clp_s::search::QueryRunner { public: - // Constructor ClpQueryRunner( const std::shared_ptr& match, const std::shared_ptr& expr, diff --git a/velox/connectors/clp/search_lib/ClpVectorLoader.h b/velox/connectors/clp/search_lib/ClpVectorLoader.h index fb9bd4ac2d1..ab57b52dfd9 100644 --- a/velox/connectors/clp/search_lib/ClpVectorLoader.h +++ b/velox/connectors/clp/search_lib/ClpVectorLoader.h @@ -25,9 +25,9 @@ class BaseColumnReader; namespace facebook::velox::connector::clp::search_lib { -/// ClpVectorLoader is a custom Velox VectorLoader that populates Velox vectors -/// from a CLP-based column reader. It supports various column types including -/// integers, floats, booleans, strings, and arrays of strings. +/// A custom Velox VectorLoader that populates Velox vectors from a CLP-based +/// column reader. It supports various column types including integers, floats, +/// booleans, strings, and arrays of strings. class ClpVectorLoader : public VectorLoader { public: ClpVectorLoader( diff --git a/velox/docs/develop/connectors.rst b/velox/docs/develop/connectors.rst index b35613d0313..8fd5ba46eaa 100644 --- a/velox/docs/develop/connectors.rst +++ b/velox/docs/develop/connectors.rst @@ -29,8 +29,8 @@ Connector Interface * - Connector Factory - Enables creating instances of a particular connector. -Velox provides Hive, CLP and TPC-H Connectors out of the box. -Let's examine the implementation details of both the Hive and CLP Connectors as examples +Velox provides Hive, CLP, and TPC-H Connectors out of the box. +Let's examine the implementation details of both the Hive and CLP Connectors as examples. Hive Connector -------------- @@ -124,27 +124,26 @@ This is the behavior when the proxy settings are enabled: CLP Connector ------------- -The CLP Connector is used to read CLP archives stored in a local file system or S3. It implements similar +The CLP Connector is used to read CLP archives stored on a local file system or S3. It implements similar interfaces as the Hive Connector except for the ``DataSink`` interface. Here we only describe the ``DataSource`` interface and the ``ConnectorSplit`` interface implementation since `Connector` and ``ConnectorFactory`` are similar to the Hive Connector. ClpConnectorSplit ~~~~~~~~~~~~~~~~~ -The ClpConnectorSplit describes a data chunk using ``path``. For now, only archive format is supported. -The ``path`` is the path to the archive file. +``ClpConnectorSplit`` describes a data chunk using ``path``, which is the path to the archive file. ClpDataSource ~~~~~~~~~~~~~ -The ClpDataSource implements the ``addSplit`` API that consumes a ClpConnectorSplit and ``next`` API that +``ClpDataSource`` implements the ``addSplit`` API that consumes a ``ClpConnectorSplit`` and ``next`` API that processes the split and returns a batch of rows. -During initialization, it records the KQL query and archive source (S3 or local). It then iterates through each -output column, accessing its handle to get its type and original name, and specifically for row types, it -recursively traverses the nested structure to process each field; for non-row types, it directly maps the -Velox column type to a CLP column type. +During initialization, it records the KQL query and archive source (S3 or local). It then iterates through +each output column, accessing its handle to get its type and original name. For row types, it recursively +traverses the nested structure to process each field; for non-row types, it directly maps the Velox column +type to a CLP column type. When a split is added, a ``ClpCursor`` is created with the archive path and input source. The query is parsed -and simplified into an AST. On ``next``, the cursor finds matching row indices and, if any exist, ``ClpDataSource`` -recursively creates a row vector composed of lazy vectors, which use CLP column readers to decode and load data -as needed during execution. +and simplified into an AST. On ``next``, the cursor finds matching row indices and, if any exist, +``ClpDataSource`` recursively creates a row vector composed of lazy vectors, which use CLP column readers to +decode and load data as needed during execution. \ No newline at end of file From e4f5b39054f768354820df03e18ec0166f8e845f Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 7 Jul 2025 14:31:11 +0000 Subject: [PATCH 26/31] add new line in connectors.rst --- velox/docs/develop/connectors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/velox/docs/develop/connectors.rst b/velox/docs/develop/connectors.rst index 8fd5ba46eaa..21a486e84a5 100644 --- a/velox/docs/develop/connectors.rst +++ b/velox/docs/develop/connectors.rst @@ -146,4 +146,4 @@ type to a CLP column type. When a split is added, a ``ClpCursor`` is created with the archive path and input source. The query is parsed and simplified into an AST. On ``next``, the cursor finds matching row indices and, if any exist, ``ClpDataSource`` recursively creates a row vector composed of lazy vectors, which use CLP column readers to -decode and load data as needed during execution. \ No newline at end of file +decode and load data as needed during execution. From 01b51dd01a0681a6d5e3f6ffe24c2eb9d8af794c Mon Sep 17 00:00:00 2001 From: wraymo Date: Mon, 7 Jul 2025 21:58:25 +0000 Subject: [PATCH 27/31] address review comments --- velox/connectors/clp/ClpConnector.cpp | 2 +- velox/connectors/clp/search_lib/ClpQueryRunner.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/velox/connectors/clp/ClpConnector.cpp b/velox/connectors/clp/ClpConnector.cpp index d381716f1de..25f90171c27 100644 --- a/velox/connectors/clp/ClpConnector.cpp +++ b/velox/connectors/clp/ClpConnector.cpp @@ -24,7 +24,7 @@ namespace facebook::velox::connector::clp { ClpConnector::ClpConnector( const std::string& id, std::shared_ptr config) - : Connector(id), config_(std::make_shared(config)) {} + : Connector(id), config_(std::make_shared(std::move(config))) {} std::unique_ptr ClpConnector::createDataSource( const RowTypePtr& outputType, diff --git a/velox/connectors/clp/search_lib/ClpQueryRunner.cpp b/velox/connectors/clp/search_lib/ClpQueryRunner.cpp index f110fc6d00a..a1c9032ebd5 100644 --- a/velox/connectors/clp/search_lib/ClpQueryRunner.cpp +++ b/velox/connectors/clp/search_lib/ClpQueryRunner.cpp @@ -61,8 +61,8 @@ void ClpQueryRunner::init( uint64_t ClpQueryRunner::fetchNext( uint64_t numRows, const std::shared_ptr>& filteredRowIndices) { - size_t rowsfiltered = 0; - size_t rowsScanned = 0; + size_t rowsfiltered{0}; + size_t rowsScanned{0}; while (curMessage_ < numMessages_) { if (filter(curMessage_)) { filteredRowIndices->emplace_back(curMessage_); From 01d8efa375ebbc5cb0e2455edb6e691a4128e9a0 Mon Sep 17 00:00:00 2001 From: wraymo Date: Tue, 8 Jul 2025 03:24:18 +0000 Subject: [PATCH 28/31] address review comments --- velox/connectors/clp/search_lib/ClpCursor.cpp | 1 + velox/connectors/clp/search_lib/ClpCursor.h | 10 +++++----- velox/connectors/clp/search_lib/ClpQueryRunner.h | 11 +++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/velox/connectors/clp/search_lib/ClpCursor.cpp b/velox/connectors/clp/search_lib/ClpCursor.cpp index 8d436cf615d..bab940c92c3 100644 --- a/velox/connectors/clp/search_lib/ClpCursor.cpp +++ b/velox/connectors/clp/search_lib/ClpCursor.cpp @@ -32,6 +32,7 @@ using namespace clp_s::search; using namespace clp_s::search::ast; namespace facebook::velox::connector::clp::search_lib { + ClpCursor::ClpCursor(InputSource inputSource, std::string archivePath) : errorCode_(ErrorCode::QueryNotInitialized), inputSource_(inputSource), diff --git a/velox/connectors/clp/search_lib/ClpCursor.h b/velox/connectors/clp/search_lib/ClpCursor.h index bf254fa788a..83d0a9f4477 100644 --- a/velox/connectors/clp/search_lib/ClpCursor.h +++ b/velox/connectors/clp/search_lib/ClpCursor.h @@ -25,16 +25,16 @@ namespace clp_s { enum class InputSource : uint8_t; class ArchiveReader; class BaseColumnReader; +} // namespace clp_s -namespace search { +namespace clp_s::search { class Projection; class SchemaMatch; +} // namespace clp_s::search -namespace ast { +namespace clp_s::search::ast { class Expression; -} // namespace ast -} // namespace search -} // namespace clp_s +} // namespace clp_s::search::ast namespace facebook::velox::connector::clp::search_lib { diff --git a/velox/connectors/clp/search_lib/ClpQueryRunner.h b/velox/connectors/clp/search_lib/ClpQueryRunner.h index aec15b24ede..181d09848e9 100644 --- a/velox/connectors/clp/search_lib/ClpQueryRunner.h +++ b/velox/connectors/clp/search_lib/ClpQueryRunner.h @@ -23,17 +23,16 @@ class SchemaReader; class SchemaTree; class ArchiveReader; class BaseColumnReader; +} // namespace clp_s -namespace search { +namespace clp_s::search { class SchemaMatch; class Projection; +} // namespace clp_s::search -namespace ast { +namespace lp_s::search::ast { class Expression; -} // namespace ast - -} // namespace search -} // namespace clp_s +} // namespace lp_s::search::ast namespace facebook::velox::connector::clp::search_lib { From 17fc6e4a046fc1d2e8da81aa51faa9b67a9c89f4 Mon Sep 17 00:00:00 2001 From: wraymo Date: Wed, 9 Jul 2025 21:17:36 +0000 Subject: [PATCH 29/31] address review comments --- velox/connectors/clp/search_lib/ClpCursor.h | 1 + velox/connectors/clp/search_lib/ClpVectorLoader.cpp | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/velox/connectors/clp/search_lib/ClpCursor.h b/velox/connectors/clp/search_lib/ClpCursor.h index 83d0a9f4477..43db49c7841 100644 --- a/velox/connectors/clp/search_lib/ClpCursor.h +++ b/velox/connectors/clp/search_lib/ClpCursor.h @@ -121,4 +121,5 @@ class ClpCursor { std::shared_ptr projection_; std::shared_ptr archiveReader_; }; + } // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ClpVectorLoader.cpp b/velox/connectors/clp/search_lib/ClpVectorLoader.cpp index a299d582df9..be73e842d73 100644 --- a/velox/connectors/clp/search_lib/ClpVectorLoader.cpp +++ b/velox/connectors/clp/search_lib/ClpVectorLoader.cpp @@ -62,9 +62,7 @@ void ClpVectorLoader::loadInternal( ValueHook* hook, vector_size_t resultSize, VectorPtr* result) { - if (!result) { - VELOX_USER_FAIL("vector is null"); - } + VELOX_CHECK_NOT_NULL(result, "result vector must not be null"); auto vector = *result; switch (nodeType_) { From b10d1aa07629c3ecb108342c78c3ce463735867d Mon Sep 17 00:00:00 2001 From: wraymo Date: Thu, 10 Jul 2025 03:21:47 +0000 Subject: [PATCH 30/31] fix --- velox/connectors/clp/search_lib/ClpQueryRunner.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/velox/connectors/clp/search_lib/ClpQueryRunner.h b/velox/connectors/clp/search_lib/ClpQueryRunner.h index 181d09848e9..eb8d87a56c5 100644 --- a/velox/connectors/clp/search_lib/ClpQueryRunner.h +++ b/velox/connectors/clp/search_lib/ClpQueryRunner.h @@ -30,7 +30,7 @@ class SchemaMatch; class Projection; } // namespace clp_s::search -namespace lp_s::search::ast { +namespace clp_s::search::ast { class Expression; } // namespace lp_s::search::ast From a65d8186beeb502bc181c081552cde4ced5a38ac Mon Sep 17 00:00:00 2001 From: wraymo Date: Thu, 10 Jul 2025 13:21:38 +0000 Subject: [PATCH 31/31] fix lint --- velox/connectors/clp/search_lib/ClpQueryRunner.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/velox/connectors/clp/search_lib/ClpQueryRunner.h b/velox/connectors/clp/search_lib/ClpQueryRunner.h index eb8d87a56c5..97c558e096e 100644 --- a/velox/connectors/clp/search_lib/ClpQueryRunner.h +++ b/velox/connectors/clp/search_lib/ClpQueryRunner.h @@ -32,7 +32,7 @@ class Projection; namespace clp_s::search::ast { class Expression; -} // namespace lp_s::search::ast +} // namespace clp_s::search::ast namespace facebook::velox::connector::clp::search_lib {