diff --git a/CMake/resolve_dependency_modules/clp.cmake b/CMake/resolve_dependency_modules/clp.cmake index 93d57bf4da6..996c6b9f75d 100644 --- a/CMake/resolve_dependency_modules/clp.cmake +++ b/CMake/resolve_dependency_modules/clp.cmake @@ -16,7 +16,7 @@ include_guard(GLOBAL) FetchContent_Declare( clp GIT_REPOSITORY https://github.com/y-scope/clp.git - GIT_TAG 19cd534e629d746395efc64343a60f768b0c9a2d) + GIT_TAG bfd4f60ffe9c5d69618cc8416ec6729c76ee9862) set(CLP_BUILD_CLP_REGEX_UTILS OFF diff --git a/CMake/resolve_dependency_modules/ystdlib_cpp.cmake b/CMake/resolve_dependency_modules/ystdlib_cpp.cmake index a47b24416c8..931df80ead0 100644 --- a/CMake/resolve_dependency_modules/ystdlib_cpp.cmake +++ b/CMake/resolve_dependency_modules/ystdlib_cpp.cmake @@ -21,3 +21,4 @@ FetchContent_Declare( FetchContent_Populate(ystdlib_cpp) set(CLP_YSTDLIB_SOURCE_DIRECTORY "${ystdlib_cpp_SOURCE_DIR}") +include_directories(${ystdlib_cpp_SOURCE_DIR}/src) diff --git a/velox/connectors/clp/ClpDataSource.cpp b/velox/connectors/clp/ClpDataSource.cpp index 07570409111..ac038869bf9 100644 --- a/velox/connectors/clp/ClpDataSource.cpp +++ b/velox/connectors/clp/ClpDataSource.cpp @@ -22,6 +22,7 @@ #include "velox/connectors/clp/ClpTableHandle.h" #include "velox/connectors/clp/search_lib/ClpS3AuthProviderBase.h" #include "velox/connectors/clp/search_lib/archive/ClpArchiveCursor.h" +#include "velox/connectors/clp/search_lib/ir/ClpIrCursor.h" #include "velox/vector/FlatVector.h" namespace facebook::velox::connector::clp { @@ -116,6 +117,9 @@ void ClpDataSource::addSplit(std::shared_ptr split) { if (ClpConnectorSplit::SplitType::kArchive == clpSplit->type_) { cursor_ = std::make_unique(inputSource, splitPath); + } else if (ClpConnectorSplit::SplitType::kIr == clpSplit->type_) { + cursor_ = + std::make_unique(inputSource, splitPath, true); } else { VELOX_UNSUPPORTED( "Unsupported split type: {}", static_cast(clpSplit->type_)); diff --git a/velox/connectors/clp/search_lib/CMakeLists.txt b/velox/connectors/clp/search_lib/CMakeLists.txt index 2dd05a96cf3..bb9452b8cc2 100644 --- a/velox/connectors/clp/search_lib/CMakeLists.txt +++ b/velox/connectors/clp/search_lib/CMakeLists.txt @@ -22,7 +22,11 @@ velox_add_library( ClpS3AuthProviderBase.h) add_subdirectory(archive) +add_subdirectory(ir) -velox_link_libraries(clp-s-search PUBLIC clp-s-archive-search) +velox_link_libraries( + clp-s-search + PUBLIC clp-s-archive-search + PUBLIC clp-s-ir-search) target_compile_features(clp-s-search PRIVATE cxx_std_20) diff --git a/velox/connectors/clp/search_lib/ir/CMakeLists.txt b/velox/connectors/clp/search_lib/ir/CMakeLists.txt new file mode 100644 index 00000000000..35a49d59261 --- /dev/null +++ b/velox/connectors/clp/search_lib/ir/CMakeLists.txt @@ -0,0 +1,33 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +velox_add_library( + clp-s-ir-search + STATIC + ClpIrCursor.cpp + ClpIrCursor.h + ClpIrUnitHandler.h + ClpIrVectorLoader.cpp + ClpIrVectorLoader.h) + +velox_link_libraries( + clp-s-ir-search + PUBLIC clp_s::archive_reader + PRIVATE + # Once the IR-stream-related targets are exported, this should be updated to + # use more fine-grained dependencies. + clp_s::clp_dependencies + clp_s::io + clp_s::search + clp_s::search::kql + velox_vector) diff --git a/velox/connectors/clp/search_lib/ir/ClpIrCursor.cpp b/velox/connectors/clp/search_lib/ir/ClpIrCursor.cpp new file mode 100644 index 00000000000..124fa5e0b06 --- /dev/null +++ b/velox/connectors/clp/search_lib/ir/ClpIrCursor.cpp @@ -0,0 +1,211 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "clp_s/ColumnReader.hpp" +#include "clp_s/InputConfig.hpp" + +#include "ffi/ir_stream/search/QueryHandler.hpp" +#include "velox/connectors/clp/search_lib/ir/ClpIrCursor.h" +#include "velox/connectors/clp/search_lib/ir/ClpIrVectorLoader.h" + +using namespace clp_s; + +namespace facebook::velox::connector::clp::search_lib { + +uint64_t ClpIrCursor::fetchNext(uint64_t numRows) { + readerIndex_ = 0; + if (ErrorCode::Success != errorCode_) { + return 0; + } + + if (false == currentSplitLoaded_) { + errorCode_ = loadSplit(); + if (ErrorCode::Success != errorCode_) { + return 0; + } + } + + auto deserializeResult = deserialize(numRows); + if (deserializeResult.has_error()) { + auto error = deserializeResult.error(); + VELOX_FAIL( + "IR file {} might be broken, failed to deserialize. {}: {}", + this->splitPath_, + error.category().name(), + error.message()); + } + return irDeserializer_->get_num_log_events_deserialized(); +} + +size_t ClpIrCursor::getNumFilteredRows() const { + return irDeserializer_->get_ir_unit_handler().getFilteredLogEvents()->size(); +} + +VectorPtr ClpIrCursor::createVector( + memory::MemoryPool* pool, + const TypePtr& vectorType, + size_t vectorSize) { + VELOX_CHECK_EQ( + projectedColumnIdxNodeIdMap_.size(), + outputColumns_.size(), + "Projected columns size {} does not match fields size {}", + projectedColumnIdxNodeIdMap_.size(), + outputColumns_.size()); + return createVectorHelper(pool, vectorType, vectorSize); +} + +ErrorCode ClpIrCursor::loadSplit() { + auto networkAuthOption = inputSource_ == InputSource::Filesystem + ? NetworkAuthOption{.method = AuthMethod::None} + : NetworkAuthOption{.method = AuthMethod::S3PresignedUrlV4}; + + auto irHandler = ClpIrUnitHandler{}; + + auto projections = splitFieldsToNamesAndTypes(); + auto queryHandlerResult{QueryHandlerType::create( + projectionResolutionCallback_, + std::move(expr_), + projections, + ignoreCase_)}; + if (!queryHandlerResult) { + VLOG(2) << "Failed to create query handler for deserialization."; + return ErrorCode::InternalError; + } + auto queryHandler = std::move(queryHandlerResult).value(); + + auto irPath = Path{.source = inputSource_, .path = splitPath_}; + irReader_ = try_create_reader(irPath, networkAuthOption); + if (nullptr == irReader_) { + VLOG(2) << "Failed to open kv-ir stream \"" << splitPath_ + << "\" for reading."; + return ErrorCode::InternalError; + } + + auto deserializerResult = ::clp::ffi::ir_stream::make_deserializer( + *irReader_, std::move(irHandler), std::move(queryHandler)); + if (!deserializerResult) { + VLOG(2) << "Failed to create deserializer for deserialization."; + return ErrorCode::InternalError; + } + irDeserializer_ = std::make_shared< + ::clp::ffi::ir_stream::Deserializer>( + std::move(deserializerResult).value()); + + currentSplitLoaded_ = true; + return ErrorCode::Success; +} + +std::vector> +ClpIrCursor::splitFieldsToNamesAndTypes() const { + auto result = std::vector< + std::pair>{}; + for (size_t i{0}; i < outputColumns_.size(); ++i) { + auto column = outputColumns_[i]; + search::ast::literal_type_bitmask_t literalType; + switch (column.type) { + case ColumnType::Array: + literalType = search::ast::LiteralType::ArrayT; + break; + case ColumnType::Boolean: + literalType = search::ast::LiteralType::BooleanT; + break; + case ColumnType::Float: + literalType = search::ast::LiteralType::FloatT; + break; + case ColumnType::Integer: + literalType = search::ast::LiteralType::IntegerT; + break; + case ColumnType::String: + literalType = search::ast::LiteralType::VarStringT | + search::ast::LiteralType::ClpStringT; + break; + case ColumnType::Timestamp: + // TODO: IR timestamp support pending; constrain to Unknown to avoid + // mismatched projections. + literalType = search::ast::LiteralType::EpochDateT; + break; + default: + literalType = search::ast::LiteralType::UnknownT; + break; + } + result.emplace_back(column.name, literalType); + } + return result; +} + +ystdlib::error_handling::Result ClpIrCursor::deserialize( + uint64_t numRows) { + irDeserializer_->get_ir_unit_handler().clearFilteredLogEvents(); + uint64_t cnt{0}; + while (cnt < numRows) { + auto deserializeResult = + irDeserializer_->deserialize_next_ir_unit(*irReader_); + if (deserializeResult.has_error()) { + auto error = deserializeResult.error(); + if (std::errc::result_out_of_range == error || + irDeserializer_->is_stream_completed()) { + break; + } + return error; + } + if (::clp::ffi::ir_stream::IrUnitType::LogEvent == + deserializeResult.value()) { + ++cnt; + } + } + return ystdlib::error_handling::success(); +} + +VectorPtr ClpIrCursor::createVectorHelper( + memory::MemoryPool* pool, + const TypePtr& vectorType, + size_t vectorSize) { + if (vectorType->kind() == TypeKind::ROW) { + std::vector children; + auto& rowType = vectorType->as(); + for (uint32_t i = 0; i < rowType.size(); ++i) { + children.push_back( + createVectorHelper(pool, rowType.childAt(i), vectorSize)); + } + return std::make_shared( + pool, vectorType, nullptr, vectorSize, std::move(children)); + } + auto vector = BaseVector::create(vectorType, vectorSize, pool); + vector->setNulls(allocateNulls(vectorSize, pool, bits::kNull)); + VELOX_CHECK_LT( + readerIndex_, outputColumns_.size(), "Reader index out of bounds"); + auto projectedColumn = outputColumns_[readerIndex_]; + auto projectedColumnType = projectedColumn.type; + auto it = projectedColumnIdxNodeIdMap_.find(readerIndex_); + bool isResolved = it != projectedColumnIdxNodeIdMap_.end(); + ::clp::ffi::SchemaTree::Node::id_t projectedColumnNodeId; + if (isResolved) { + projectedColumnNodeId = it->second; + } + readerIndex_++; + return std::make_shared( + pool, + vectorType, + vectorSize, + std::make_unique( + isResolved, + projectedColumnType, + projectedColumnNodeId, + irDeserializer_->get_ir_unit_handler().getFilteredLogEvents()), + std::move(vector)); +} + +} // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ir/ClpIrCursor.h b/velox/connectors/clp/search_lib/ir/ClpIrCursor.h new file mode 100644 index 00000000000..a132bc3ba35 --- /dev/null +++ b/velox/connectors/clp/search_lib/ir/ClpIrCursor.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "ffi/ir_stream/Deserializer.hpp" +#include "velox/connectors/clp/search_lib/BaseClpCursor.h" +#include "velox/connectors/clp/search_lib/ir/ClpIrUnitHandler.h" + +namespace facebook::velox::connector::clp::search_lib { + +class ClpIrCursor final : public BaseClpCursor { + public: + explicit ClpIrCursor( + clp_s::InputSource inputSource, + std::string_view splitPath, + bool ignoreCase) + : BaseClpCursor(inputSource, splitPath), ignoreCase_(ignoreCase) {} + + ClpIrCursor(const ClpIrCursor&) = delete; + ClpIrCursor& operator=(const ClpIrCursor&) = delete; + ClpIrCursor(ClpIrCursor&&) = delete; + ClpIrCursor& operator=(ClpIrCursor&&) = delete; + + uint64_t fetchNext(uint64_t numRows) override; + + size_t getNumFilteredRows() const override; + + VectorPtr createVector( + memory::MemoryPool* pool, + const TypePtr& vectorType, + size_t vectorSize) override; + + protected: + ErrorCode loadSplit() override; + + private: + std::function( + bool, + ::clp::ffi::SchemaTree::Node::id_t, + std::pair)> + projectionResolutionCallback_ = + [this]( + [[maybe_unused]] bool isAutoGenerated, + [[maybe_unused]] ::clp::ffi::SchemaTree::Node::id_t nodeId, + [[maybe_unused]] std::pair + projected_key_and_index) + -> ystdlib::error_handling::Result { + projectedColumnIdxNodeIdMap_.insert( + {projected_key_and_index.second, nodeId}); + return ystdlib::error_handling::success(); + }; + using QueryHandlerType = ::clp::ffi::ir_stream::search::QueryHandler< + decltype(projectionResolutionCallback_)>; + bool ignoreCase_; + std::shared_ptr< + ::clp::ffi::ir_stream::Deserializer> + irDeserializer_; + std::shared_ptr<::clp::ReaderInterface> irReader_{nullptr}; + std::unordered_map + projectedColumnIdxNodeIdMap_; + size_t readerIndex_{0}; + + std::vector< + std::pair> + splitFieldsToNamesAndTypes() const; + + ystdlib::error_handling::Result deserialize(uint64_t numRows); + + VectorPtr createVectorHelper( + memory::MemoryPool* pool, + const TypePtr& vectorType, + size_t vectorSize); +}; + +} // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ir/ClpIrUnitHandler.h b/velox/connectors/clp/search_lib/ir/ClpIrUnitHandler.h new file mode 100644 index 00000000000..e4c2ca77fe3 --- /dev/null +++ b/velox/connectors/clp/search_lib/ir/ClpIrUnitHandler.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "clp_s/SchemaTree.hpp" +#include "ffi/ir_stream/Deserializer.hpp" +#include "velox/common/base/Exceptions.h" +#include "velox/connectors/clp/search_lib/ir/ClpIrUnitHandler.h" + +namespace facebook::velox::connector::clp::search_lib { + +class ClpIrUnitHandler { + public: + ClpIrUnitHandler() { + filteredLogEvents_ = std::make_shared< + std::vector>>(); + } + + // Destructor + ~ClpIrUnitHandler() = default; + + // Methods implementing `IrUnitHandlerInterface` + [[nodiscard]] auto handle_log_event( + ::clp::ffi::KeyValuePairLogEvent log_event, + size_t log_event_idx) -> ::clp::ffi::ir_stream::IRErrorCode { + filteredLogEvents_->push_back( + std::make_unique<::clp::ffi::KeyValuePairLogEvent>( + std::move(log_event))); + return ::clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; + } + + [[nodiscard]] auto handle_utc_offset_change( + [[maybe_unused]] ::clp::UtcOffset utc_offset_old, + [[maybe_unused]] ::clp::UtcOffset utc_offset_new) + -> ::clp::ffi::ir_stream::IRErrorCode { + return ::clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; + } + + [[nodiscard]] auto handle_schema_tree_node_insertion( + [[maybe_unused]] bool is_auto_generated, + [[maybe_unused]] ::clp::ffi::SchemaTree::NodeLocator + schema_tree_node_locator, + [[maybe_unused]] std::shared_ptr<::clp::ffi::SchemaTree const> const& + schema_tree) -> ::clp::ffi::ir_stream::IRErrorCode { + return ::clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; + } + + [[nodiscard]] auto handle_end_of_stream() + -> ::clp::ffi::ir_stream::IRErrorCode { + return ::clp::ffi::ir_stream::IRErrorCode::IRErrorCode_Success; + } + + std::shared_ptr< + const std::vector>> + getFilteredLogEvents() const { + return filteredLogEvents_; + } + + void clearFilteredLogEvents() { + filteredLogEvents_->clear(); + } + + private: + std::shared_ptr< + std::vector>> + filteredLogEvents_; +}; + +} // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ir/ClpIrVectorLoader.cpp b/velox/connectors/clp/search_lib/ir/ClpIrVectorLoader.cpp new file mode 100644 index 00000000000..c082aecb85d --- /dev/null +++ b/velox/connectors/clp/search_lib/ir/ClpIrVectorLoader.cpp @@ -0,0 +1,135 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "velox/connectors/clp/search_lib/ir/ClpIrVectorLoader.h" +#include "velox/connectors/clp/search_lib/BaseClpCursor.h" + +namespace facebook::velox::connector::clp::search_lib { + +void ClpIrVectorLoader::loadInternal( + RowSet rows, + ValueHook* hook, + vector_size_t resultSize, + VectorPtr* result) { + auto vector = *result; + for (int vectorIndex : rows) { + vector->setNull(vectorIndex, true); + if (!isResolved_) { + continue; + } + auto& logEvent = filteredLogEvents_->at(vectorIndex); + // TODO: also need to support auto-generated keys + auto userGenNodeIdValueMap = logEvent->get_user_gen_node_id_value_pairs(); + auto const value_it{userGenNodeIdValueMap.find(nodeId_)}; + if (userGenNodeIdValueMap.end() == value_it || + false == value_it->second.has_value()) { + continue; + } + auto const& value{value_it->second}; + switch (nodeType_) { + case ColumnType::String: { + auto stringVector = vector->asFlatVector(); + if (value->is()) { + auto stringValue = value->get_immutable_view(); + stringVector->set(vectorIndex, StringView(stringValue)); + } else if (value->is<::clp::ir::EightByteEncodedTextAst>()) { + auto decodeResult = + value->get_immutable_view<::clp::ir::EightByteEncodedTextAst>() + .decode_and_unparse(); + if (!decodeResult.has_value()) { + continue; + } + stringVector->set(vectorIndex, StringView(decodeResult.value())); + } else if (value->is<::clp::ir::FourByteEncodedTextAst>()) { + auto decodeResult = + value->get_immutable_view<::clp::ir::FourByteEncodedTextAst>() + .decode_and_unparse(); + if (!decodeResult.has_value()) { + continue; + } + stringVector->set(vectorIndex, StringView(decodeResult.value())); + } else { + continue; + } + vector->setNull(vectorIndex, false); + break; + } + case ColumnType::Integer: { + auto intVector = vector->asFlatVector(); + intVector->set( + vectorIndex, value->get_immutable_view<::clp::ffi::value_int_t>()); + vector->setNull(vectorIndex, false); + break; + } + case ColumnType::Float: { + auto floatVector = vector->asFlatVector(); + floatVector->set( + vectorIndex, + value->get_immutable_view<::clp::ffi::value_float_t>()); + vector->setNull(vectorIndex, false); + break; + } + case ColumnType::Boolean: { + auto boolVector = vector->asFlatVector(); + boolVector->set( + vectorIndex, value->get_immutable_view<::clp::ffi::value_bool_t>()); + vector->setNull(vectorIndex, false); + break; + } + case ColumnType::Array: { + auto arrayVector = std::dynamic_pointer_cast(vector); + std::string jsonString; + if (value->is<::clp::ir::EightByteEncodedTextAst>()) { + auto decodeResult = + value->get_immutable_view<::clp::ir::EightByteEncodedTextAst>() + .decode_and_unparse(); + if (!decodeResult.has_value()) { + continue; + } + jsonString = std::move(decodeResult.value()); + } else { + auto decodeResult = + value->get_immutable_view<::clp::ir::FourByteEncodedTextAst>() + .decode_and_unparse(); + if (!decodeResult.has_value()) { + continue; + } + jsonString = std::move(decodeResult.value()); + } + + size_t numElements{0ULL}; + auto elements = arrayVector->elements()->asFlatVector(); + auto obj = arrayParser_.iterate(jsonString); + std::vector rawElements; + for (auto arrayElement : obj.get_array()) { + auto raw_element = simdjson::to_json_string(arrayElement).value(); + rawElements.emplace_back(raw_element); + } + elements->resize(rawElements.size()); + for (auto& raw_element : rawElements) { + elements->set(numElements++, StringView(raw_element)); + } + arrayVector->setOffsetAndSize(vectorIndex, 0ULL, numElements); + arrayVector->setNull(vectorIndex, false); + break; + } + default: + VELOX_FAIL("Unsupported column type"); + } + } +} + +} // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/search_lib/ir/ClpIrVectorLoader.h b/velox/connectors/clp/search_lib/ir/ClpIrVectorLoader.h new file mode 100644 index 00000000000..bdbf3fd8a58 --- /dev/null +++ b/velox/connectors/clp/search_lib/ir/ClpIrVectorLoader.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "connectors/clp/search_lib/BaseClpCursor.h" +#include "ffi/ir_stream/Deserializer.hpp" +#include "velox/vector/FlatVector.h" +#include "velox/vector/LazyVector.h" + +namespace facebook::velox::connector::clp::search_lib { + +class ClpIrVectorLoader : public VectorLoader { + public: + ClpIrVectorLoader( + bool isResolved, + ColumnType nodeType, + ::clp::ffi::SchemaTree::Node::id_t nodeId, + const std::shared_ptr< + const std::vector>>& + filteredLogEvents) + : isResolved_(isResolved), + nodeType_(nodeType), + nodeId_(nodeId), + filteredLogEvents_(filteredLogEvents) {} + + private: + simdjson::ondemand::parser arrayParser_; + + bool isResolved_; + ColumnType nodeType_; + ::clp::ffi::SchemaTree::Node::id_t nodeId_; + std::shared_ptr< + const std::vector>> + filteredLogEvents_; + + void loadInternal( + RowSet rows, + ValueHook* hook, + vector_size_t resultSize, + VectorPtr* result) override; +}; + +} // namespace facebook::velox::connector::clp::search_lib diff --git a/velox/connectors/clp/tests/ClpConnectorTest.cpp b/velox/connectors/clp/tests/ClpConnectorTest.cpp index 0e20dbb23b9..3bec10d9ef5 100644 --- a/velox/connectors/clp/tests/ClpConnectorTest.cpp +++ b/velox/connectors/clp/tests/ClpConnectorTest.cpp @@ -132,6 +132,31 @@ TEST_F(ClpConnectorTest, test1NoPushdown) { "GET", })}); test::assertEqualVectors(expected, output); + + // The IR stream will be deserialized in order, so the expected vector is + // different + auto irExpected = makeRowVector( + {// requestId + makeFlatVector( + {"req-100", "req-102", "req-105", "req-107", "req-109"}), + // userId + makeNullableFlatVector( + {"user201", std::nullopt, "user204", "user202", "user203"}), + // method + makeFlatVector({ + "GET", + "GET", + "GET", + "GET", + "GET", + })}); + auto irOutput = getResults( + plan, + {makeClpSplit( + getExampleFilePath("test_1_ir.clps"), + ClpConnectorSplit::SplitType::kIr, + kqlQuery)}); + test::assertEqualVectors(irExpected, irOutput); } TEST_F(ClpConnectorTest, test1Pushdown) { @@ -172,6 +197,14 @@ TEST_F(ClpConnectorTest, test1Pushdown) { // path makeFlatVector({"/auth/login"})}); test::assertEqualVectors(expected, output); + + auto irOutput = getResults( + plan, + {makeClpSplit( + getExampleFilePath("test_1_ir.clps"), + ClpConnectorSplit::SplitType::kIr, + kqlQuery)}); + test::assertEqualVectors(expected, irOutput); } TEST_F(ClpConnectorTest, test2NoPushdown) { @@ -223,6 +256,50 @@ TEST_F(ClpConnectorTest, test2NoPushdown) { makeFlatVector({"WARNING"}), })}); test::assertEqualVectors(expected, output); + + // IR stream currently does not support TIMESTAMP type; will merge into the + // plan above as soon as this feature is implemented + auto irPlan = + PlanBuilder(pool_.get()) + .startTableScan() + .outputType( + ROW({"event"}, + {ROW( + {"type", "subtype", "severity", "tags"}, + {VARCHAR(), VARCHAR(), VARCHAR(), ARRAY(VARCHAR())})})) + .tableHandle( + std::make_shared(kClpConnectorId, "test_2")) + .assignments( + {{"event", + std::make_shared( + "event", + "event", + ROW({"type", "subtype", "severity", "tags"}, + {VARCHAR(), VARCHAR(), VARCHAR(), ARRAY(VARCHAR())}))}}) + .endTableScan() + .filter( + "event.severity IN ('WARNING', 'ERROR') AND " + "((event.type = 'network' AND event.subtype = 'connection') OR " + "(event.type = 'storage' AND event.subtype LIKE 'disk_usage%'))") + .planNode(); + auto irOutput = getResults( + irPlan, + {makeClpSplit( + getExampleFilePath("test_2_ir.clps"), + ClpConnectorSplit::SplitType::kIr, + kqlQuery)}); + auto irExpected = makeRowVector( + {// event + makeRowVector( + {// event.type + makeFlatVector({"storage"}), + // event.subtype + makeFlatVector({"disk_usage"}), + // event.severity + makeFlatVector({"WARNING"}), + // event.tags + makeArrayVector({{"\"backup\"", "\"daily\""}})})}); + test::assertEqualVectors(irExpected, irOutput); } TEST_F(ClpConnectorTest, test2Pushdown) { @@ -272,6 +349,46 @@ TEST_F(ClpConnectorTest, test2Pushdown) { makeFlatVector({"WARNING"}), })}); test::assertEqualVectors(expected, output); + + // IR stream currently does not support TIMESTAMP type; will merge into the + // plan above as soon as this feature is implemented + auto irPlan = + PlanBuilder(pool_.get()) + .startTableScan() + .outputType( + ROW({"event"}, + {ROW( + {"type", "subtype", "severity", "tags"}, + {VARCHAR(), VARCHAR(), VARCHAR(), ARRAY(VARCHAR())})})) + .tableHandle( + std::make_shared(kClpConnectorId, "test_2")) + .assignments( + {{"event", + std::make_shared( + "event", + "event", + ROW({"type", "subtype", "severity", "tags"}, + {VARCHAR(), VARCHAR(), VARCHAR(), ARRAY(VARCHAR())}))}}) + .endTableScan() + .planNode(); + auto irOutput = getResults( + irPlan, + {makeClpSplit( + getExampleFilePath("test_2_ir.clps"), + ClpConnectorSplit::SplitType::kIr, + kqlQuery)}); + auto irExpected = makeRowVector( + {// event + makeRowVector({// event.type + makeFlatVector({"storage"}), + // event.subtype + makeFlatVector({"disk_usage"}), + // event.severity + makeFlatVector({"WARNING"}), + // event.tags + makeArrayVector( + {{"\"filesystem\"", "\"monitoring\""}})})}); + test::assertEqualVectors(irExpected, irOutput); } TEST_F(ClpConnectorTest, test2Hybrid) { diff --git a/velox/connectors/clp/tests/examples/test_1_ir.clps b/velox/connectors/clp/tests/examples/test_1_ir.clps new file mode 100644 index 00000000000..182de479b49 Binary files /dev/null and b/velox/connectors/clp/tests/examples/test_1_ir.clps differ diff --git a/velox/connectors/clp/tests/examples/test_2_ir.clps b/velox/connectors/clp/tests/examples/test_2_ir.clps new file mode 100644 index 00000000000..06c3691ef34 Binary files /dev/null and b/velox/connectors/clp/tests/examples/test_2_ir.clps differ