diff --git a/.bazelrc b/.bazelrc index cc3d1af5c3..900a844bb3 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,4 +1,5 @@ -build --cxxopt='-std=c++17' --copt=-O3 --jobs=40 +build --cxxopt='-std=c++17' --copt="-pg" --linkopt="-pg" --strip=never --jobs=2 +# build --cxxopt='-std=c++17' --copt=-O3 --jobs=40 #build --action_env=PYTHON_BIN_PATH="/usr/bin/python3.10" #build --action_env=PYTHON_LIB_PATH="/usr/include/python3.10" diff --git a/WORKSPACE b/WORKSPACE index ba145119cd..1c2884f3e3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -182,6 +182,26 @@ http_archive( url = "https://github.com/google/leveldb/archive/refs/tags/1.23.zip", ) +bind( + name = "zstd", + actual = "//third_party:zstd", +) + +http_archive( + name = "com_facebook_zstd", + build_file_content = all_content, + strip_prefix = "zstd-1.5.2", + url = "https://github.com/facebook/zstd/archive/refs/tags/v1.5.2.zip", +) + +http_archive( + name = "com_github_facebook_rocksdb", + build_file = "@com_resdb_nexres//third_party:rocksdb.BUILD", + sha256 = "928cbd416c0531e9b2e7fa74864ce0d7097dca3f5a8c31f31459772a28dbfcba", + strip_prefix = "rocksdb-7.2.2", + url = "https://github.com/facebook/rocksdb/archive/refs/tags/v7.2.2.zip", +) + bind( name = "snappy", actual = "@com_google_snappy//:snappy", @@ -257,3 +277,10 @@ http_archive( strip_prefix = "asio-asio-1-26-0", url = "https://github.com/chriskohlhoff/asio/archive/refs/tags/asio-1-26-0.zip", ) + +http_archive( + name = "com_google_benchmark", + sha256 = "6bc180a57d23d4d9515519f92b0c83d61b05b5bab188961f36ac7b06b0d9e9ce", + strip_prefix = "benchmark-1.8.3", + urls = ["https://github.com/google/benchmark/archive/v1.8.3.tar.gz"], +) diff --git a/benchmark/storage/BUILD b/benchmark/storage/BUILD new file mode 100644 index 0000000000..653bc61a83 --- /dev/null +++ b/benchmark/storage/BUILD @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +package(default_visibility = ["//visibility:public"]) + +cc_binary( + name = "composite_key_benchmark", + srcs = ["composite_key_benchmark.cpp"], + deps = [ + "//executor/kv:kv_executor", + "//chain/storage:leveldb", + "//chain/storage:rocksdb", + "//proto/kv:kv_cc_proto", + "@com_google_benchmark//:benchmark", + "@nlohmann_json//:json", + "@com_github_google_glog//:glog", + ], + copts = [ + "-std=c++17", + "-O2", + ], +) diff --git a/benchmark/storage/RESULTS.md b/benchmark/storage/RESULTS.md new file mode 100644 index 0000000000..da83f49f31 --- /dev/null +++ b/benchmark/storage/RESULTS.md @@ -0,0 +1,270 @@ +# Composite Key vs. In-Memory Filtering: Performance Benchmark Analysis + +## Overview +This document presents performance benchmarks comparing two approaches for querying records by secondary attributes: + +- **Current Solution (In-Memory Filtering)**: Retrieves all records using `GetAllValue()` and filters them in memory based on secondary key attributes +- **Proposed Solution (Composite Key Indexing)**: Creates composite keys using a specific encoding format to enable direct record retrieval without scanning + +## Benchmark 1: Secondary Key Query Performance +**Objective**: Measure read latency performance when fetching records by secondary attributes using both approaches. + +**Test Configuration**: +- Dataset sizes: 1K, 10K, and 100K records +- Query batch size: 10 queries per benchmark run +- Field types: String and Integer secondary keys + +**Key Metrics**: +- `items_per_second`: Throughput in queries per second +- `records_scanned_per_query`: Number of records examined per query +- `total_records_scanned`: Total records processed across all queries + +``` +Running /home/ubuntu/.cache/bazel/_bazel_ubuntu/4daa26ab31ab249b7607bfe58eb14371/execroot/com_resdb_nexres/bazel-out/k8-fastbuild/bin/benchmark/storage/composite_key_benchmark +Run on (8 X 3600.53 MHz CPU s) +CPU Caches: + L1 Data 32 KiB (x4) + L1 Instruction 32 KiB (x4) + L2 Unified 1024 KiB (x4) + L3 Unified 36608 KiB (x1) +Load Average: 2.07, 0.97, 0.48 +***WARNING*** Library was built as DEBUG. Timings may be affected. +---------------------------------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +---------------------------------------------------------------------------------------------------------------- +CompositeKeyBenchmark/StringFieldInMemory/1000/10 43.7 ms 43.7 ms 16 items_per_second=45.7877/s records_scanned_per_query=1k total_records_scanned=32k +CompositeKeyBenchmark/StringFieldInMemory/10000/10 472 ms 472 ms 2 items_per_second=4.23476/s records_scanned_per_query=10k total_records_scanned=40k +CompositeKeyBenchmark/StringFieldInMemory/100000/10 5159 ms 5159 ms 1 items_per_second=0.3877/s records_scanned_per_query=100k total_records_scanned=200k +CompositeKeyBenchmark/StringFieldComposite/1000/10 6.34 ms 6.34 ms 110 items_per_second=315.648/s records_scanned_per_query=1 total_records_scanned=220 +CompositeKeyBenchmark/StringFieldComposite/10000/10 73.1 ms 73.1 ms 10 items_per_second=27.3662/s records_scanned_per_query=1 total_records_scanned=20 +CompositeKeyBenchmark/StringFieldComposite/100000/10 745 ms 745 ms 1 items_per_second=2.68539/s records_scanned_per_query=1 total_records_scanned=2 +CompositeKeyBenchmark/IntegerFieldInMemory/1000/10 43.3 ms 43.3 ms 16 items_per_second=46.1626/s records_scanned_per_query=1k total_records_scanned=32k +CompositeKeyBenchmark/IntegerFieldInMemory/10000/10 471 ms 471 ms 2 items_per_second=4.24955/s records_scanned_per_query=10k total_records_scanned=40k +CompositeKeyBenchmark/IntegerFieldInMemory/100000/10 5132 ms 5131 ms 1 items_per_second=0.389757/s records_scanned_per_query=100k total_records_scanned=200k +CompositeKeyBenchmark/IntegerFieldComposite/1000/10 6.36 ms 6.36 ms 110 items_per_second=314.692/s records_scanned_per_query=1 total_records_scanned=220 +CompositeKeyBenchmark/IntegerFieldComposite/10000/10 69.7 ms 69.7 ms 10 items_per_second=28.6814/s records_scanned_per_query=1 total_records_scanned=20 +CompositeKeyBenchmark/IntegerFieldComposite/100000/10 732 ms 731 ms 1 items_per_second=2.73417/s records_scanned_per_query=1 total_records_scanned=2 +``` + +## Benchmark 2: Primary Key Retrieval Performance with Composite Key Overhead +**Objective**: Evaluate the performance impact of maintaining composite key indexes on primary key retrieval operations. + +**Test Configuration**: +- Dataset sizes: 1K, 10K, 100K, and 1M records +- Composite key ratios: 0%, 25%, 50%, and 100% of records have composite keys +- Field types: String secondary keys (25AF, 50AF, 100AF indicate percentage of records with composite keys) + +**Key Metrics**: +- `items_per_second`: Throughput in queries per second +- `composite_keys`: Number of composite key entries maintained +- `total_records`: Total number of records in the dataset + +**Performance Analysis**: +- **Baseline Performance**: Primary key retrieval without composite keys +- **Composite Key Impact**: Performance degradation when maintaining composite key indexes +- **Scalability**: Performance trends across different dataset sizes and composite key ratios + +- LevelDB + +``` +Running /home/ubuntu/.cache/bazel/_bazel_ubuntu/4daa26ab31ab249b7607bfe58eb14371/execroot/com_resdb_nexres/bazel-out/k8-fastbuild/bin/benchmark/storage/composite_key_benchmark +Run on (8 X 3604.94 MHz CPU s) +CPU Caches: + L1 Data 32 KiB (x4) + L1 Instruction 32 KiB (x4) + L2 Unified 1024 KiB (x4) + L3 Unified 36608 KiB (x1) +Load Average: 0.43, 0.53, 0.49 +***WARNING*** Library was built as DEBUG. Timings may be affected. +-------------------------------------------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +-------------------------------------------------------------------------------------------------------------------------- +CompositeKeyBenchmark/PrimaryKeyOnly/1000 18.8 ms 18.8 ms 37 composite_keys=0 items_per_second=53.0803k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyOnly/10000 206 ms 206 ms 3 composite_keys=0 items_per_second=48.4508k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyOnly/100000 2231 ms 2230 ms 1 composite_keys=0 items_per_second=44.8366k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyOnly/1000000 29095 ms 29093 ms 1 composite_keys=0 items_per_second=34.3726k/s total_records=1M +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/1000 19.1 ms 19.1 ms 37 composite_keys=1k items_per_second=52.4668k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/10000 209 ms 209 ms 3 composite_keys=10k items_per_second=47.8541k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/100000 2233 ms 2233 ms 1 composite_keys=100k items_per_second=44.7865k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/1000000 34021 ms 34019 ms 1 composite_keys=1M items_per_second=29.3955k/s total_records=1M +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/1000 20.7 ms 20.7 ms 36 composite_keys=1k items_per_second=48.3132k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/10000 209 ms 209 ms 3 composite_keys=10k items_per_second=47.7968k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/100000 2335 ms 2335 ms 1 composite_keys=100k items_per_second=42.8258k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/1000000 30012 ms 30010 ms 1 composite_keys=1M items_per_second=33.322k/s total_records=1M +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/1000 21.8 ms 21.8 ms 35 composite_keys=1k items_per_second=45.9682k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/10000 227 ms 227 ms 3 composite_keys=10k items_per_second=44.0984k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/100000 2370 ms 2370 ms 1 composite_keys=100k items_per_second=42.2014k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/1000000 97125 ms 96949 ms 1 composite_keys=1M items_per_second=10.3147k/s total_records=1M +``` + +# Latency Comparison: Sequential Reads (time in ms) + +| Records | Primary Only | 25% Composite Write Amplification | 50% Composite Write Amplification | 100% Composite Write Amplification | +|---------|-------------|---------------|---------------|----------------| +| 1K | 18.8 | 19.1 | 20.7 | 21.8 | +| 10K | 206 | 209 | 209 | 227 | +| 100K | 2231 | 2233 | 2335 | 2370 | +| 1M | 29095 | 34021 | 30012 | 97125 | + +# Latency Comparison: Sequential Reads (time in ms) + +| Records | Primary Only | 25% Composite (Overhead) | 50% Composite (Overhead) | 100% Composite (Overhead) | +|---------|-------------|-------------------------|-------------------------|--------------------------| +| 1K | 18.8 | 19.1 (+1.6%) | 20.7 (+10.1%) | 21.8 (+16.0%) | +| 10K | 206 | 209 (+1.5%) | 209 (+1.5%) | 227 (+10.2%) | +| 100K | 2231 | 2233 (+0.1%) | 2335 (+4.7%) | 2370 (+6.2%) | +| 1M | 29095 | 34021 (+16.9%) | 30012 (+3.2%) | 97125 (+233.8%) | + +- RocksDB (BlobDB Enabled) +``` +INFO: Running command line: bazel-bin/benchmark/storage/composite_key_benchmark '--benchmark_filter=PrimaryKey*' '--storage_type=rocksdb' +2025-09-03T19:46:17+00:00 +Running /home/ubuntu/.cache/bazel/_bazel_ubuntu/4daa26ab31ab249b7607bfe58eb14371/execroot/com_resdb_nexres/bazel-out/k8-fastbuild/bin/benchmark/storage/composite_key_benchmark +Run on (8 X 3596.06 MHz CPU s) +CPU Caches: + L1 Data 32 KiB (x4) + L1 Instruction 32 KiB (x4) + L2 Unified 1024 KiB (x4) + L3 Unified 36608 KiB (x1) +Load Average: 0.26, 0.39, 0.56 +***WARNING*** Library was built as DEBUG. Timings may be affected. +-------------------------------------------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +-------------------------------------------------------------------------------------------------------------------------- +CompositeKeyBenchmark/PrimaryKeyOnly/1000 16.1 ms 16.1 ms 44 composite_keys=0 items_per_second=61.9675k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyOnly/10000 170 ms 170 ms 4 composite_keys=0 items_per_second=58.6594k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyOnly/100000 1780 ms 1780 ms 1 composite_keys=0 items_per_second=56.1806k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyOnly/1000000 19080 ms 19079 ms 1 composite_keys=0 items_per_second=52.4128k/s total_records=1M +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/1000 16.7 ms 16.7 ms 44 composite_keys=1k items_per_second=60.0398k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/10000 172 ms 172 ms 4 composite_keys=10k items_per_second=58.2706k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/100000 1864 ms 1864 ms 1 composite_keys=100k items_per_second=53.6553k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/1000000 20900 ms 20899 ms 1 composite_keys=1M items_per_second=47.8487k/s total_records=1M +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/1000 16.2 ms 16.2 ms 43 composite_keys=1k items_per_second=61.705k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/10000 173 ms 173 ms 4 composite_keys=10k items_per_second=57.7472k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/100000 1823 ms 1823 ms 1 composite_keys=100k items_per_second=54.8692k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/1000000 23071 ms 23070 ms 1 composite_keys=1M items_per_second=43.3468k/s total_records=1M +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/1000 16.3 ms 16.3 ms 41 composite_keys=1k items_per_second=61.4066k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/10000 171 ms 171 ms 4 composite_keys=10k items_per_second=58.3699k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/100000 1853 ms 1852 ms 1 composite_keys=100k items_per_second=53.9821k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/1000000 21148 ms 21147 ms 1 composite_keys=1M items_per_second=47.2885k/s total_records=1M +``` +# Latency Comparison: Sequential Reads (time in ms) + +| Records | Primary Only | 25% Composite Write Amplification | 50% Composite Write Amplification | 100% Composite Write Amplification | +|---------|-------------|---------------|---------------|----------------| +| 1K | 16.1 | 16.7 | 16.2 | 16.3 | +| 10K | 170 | 172 | 173 | 171 | +| 100K | 1780 | 1864 | 1823 | 1853 | +| 1M | 19080 | 20900 | 23071 | 21148 | + +# Latency Comparison: RocksDB with BlobDB, Sequential Reads (time in ms) with Overhead + +| Records | Primary Only | 25% Composite (Overhead) | 50% Composite (Overhead) | 100% Composite (Overhead) | +|---------|-------------|-------------------------|-------------------------|--------------------------| +| 1K | 16.1 | 16.7 (+3.7%) | 16.2 (+0.6%) | 16.3 (+1.2%) | +| 10K | 170 | 172 (+1.2%) | 173 (+1.8%) | 171 (+0.6%) | +| 100K | 1780 | 1864 (+4.7%) | 1823 (+2.4%) | 1853 (+4.1%) | +| 1M | 19080 | 20900 (+9.5%) | 23071 (+20.9%) | 21148 (+10.8%) | + +- RocksDB (PrefixCapped search with bloom filters) +``` +INFO: Running command line: bazel-bin/benchmark/storage/composite_key_benchmark '--benchmark_filter=PrimaryKey*' '--storage_type=rocksdb' +2025-09-03T20:06:11+00:00 +Running /home/ubuntu/.cache/bazel/_bazel_ubuntu/4daa26ab31ab249b7607bfe58eb14371/execroot/com_resdb_nexres/bazel-out/k8-fastbuild/bin/benchmark/storage/composite_key_benchmark +Run on (8 X 3599.71 MHz CPU s) +CPU Caches: + L1 Data 32 KiB (x4) + L1 Instruction 32 KiB (x4) + L2 Unified 1024 KiB (x4) + L3 Unified 36608 KiB (x1) +Load Average: 0.41, 0.48, 0.60 +***WARNING*** Library was built as DEBUG. Timings may be affected. +-------------------------------------------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +-------------------------------------------------------------------------------------------------------------------------- +CompositeKeyBenchmark/PrimaryKeyOnly/1000 16.4 ms 16.4 ms 43 composite_keys=0 items_per_second=60.9054k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyOnly/10000 171 ms 171 ms 4 composite_keys=0 items_per_second=58.4813k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyOnly/100000 1800 ms 1800 ms 1 composite_keys=0 items_per_second=55.5449k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyOnly/1000000 19564 ms 19563 ms 1 composite_keys=0 items_per_second=51.1175k/s total_records=1M +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/1000 16.5 ms 16.5 ms 43 composite_keys=1k items_per_second=60.7669k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/10000 170 ms 170 ms 4 composite_keys=10k items_per_second=58.8057k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/100000 1792 ms 1792 ms 1 composite_keys=100k items_per_second=55.8123k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_25AF/1000000 21907 ms 21905 ms 1 composite_keys=1M items_per_second=45.6507k/s total_records=1M +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/1000 16.3 ms 16.3 ms 43 composite_keys=1k items_per_second=61.4517k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/10000 174 ms 174 ms 4 composite_keys=10k items_per_second=57.3433k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/100000 1830 ms 1830 ms 1 composite_keys=100k items_per_second=54.6481k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_50AF/1000000 21594 ms 21593 ms 1 composite_keys=1M items_per_second=46.3111k/s total_records=1M +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/1000 16.3 ms 16.3 ms 42 composite_keys=1k items_per_second=61.4314k/s total_records=1k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/10000 174 ms 174 ms 4 composite_keys=10k items_per_second=57.418k/s total_records=10k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/100000 1841 ms 1841 ms 1 composite_keys=100k items_per_second=54.325k/s total_records=100k +CompositeKeyBenchmark/PrimaryKeyWithCompositeKeys_100AF/1000000 21040 ms 21039 ms 1 composite_keys=1M items_per_second=47.5313k/s total_records=1M +``` + +# Latency Comparison: RocksDB with Prefix Optimization, Sequential Reads (time in ms) + +| Records | Primary Only | 25% Composite Write Amplification | 50% Composite Write Amplification | 100% Composite Write Amplification | +|---------|-------------|---------------|---------------|----------------| +| 1K | 16.4 | 16.5 | 16.3 | 16.3 | +| 10K | 171 | 170 | 174 | 174 | +| 100K | 1800 | 1792 | 1830 | 1841 | +| 1M | 19564 | 21907 | 21594 | 21040 | + +# Latency Comparison: RocksDB with Prefix Optimization, Sequential Reads (time in ms) with Overhead + +| Records | Primary Only | 25% Composite (Overhead) | 50% Composite (Overhead) | 100% Composite (Overhead) | +|---------|-------------|-------------------------|-------------------------|--------------------------| +| 1K | 16.4 | 16.5 (+0.6%) | 16.3 (-0.6%) | 16.3 (-0.6%) | +| 10K | 171 | 170 (-0.6%) | 174 (+1.8%) | 174 (+1.8%) | +| 100K | 1800 | 1792 (-0.4%) | 1830 (+1.7%) | 1841 (+2.3%) | +| 1M | 19564 | 21907 (+12.0%) | 21594 (+10.4%) | 21040 (+7.5%) | + +## Benchmark 3: RocksDB vs LevelDB +``` +INFO: Running command line: bazel-bin/benchmark/storage/composite_key_benchmark '--benchmark_filter=IntegerFieldComposite' '--storage_type=rocksdb' +2025-09-03T20:19:25+00:00 +Running /home/ubuntu/.cache/bazel/_bazel_ubuntu/4daa26ab31ab249b7607bfe58eb14371/execroot/com_resdb_nexres/bazel-out/k8-fastbuild/bin/benchmark/storage/composite_key_benchmark +Run on (8 X 3600.21 MHz CPU s) +CPU Caches: + L1 Data 32 KiB (x4) + L1 Instruction 32 KiB (x4) + L2 Unified 1024 KiB (x4) + L3 Unified 36608 KiB (x1) +Load Average: 0.43, 0.40, 0.53 +***WARNING*** Library was built as DEBUG. Timings may be affected. +-------------------------------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +-------------------------------------------------------------------------------------------------------------- +CompositeKeyBenchmark/IntegerFieldComposite/1000 4.30 ms 4.30 ms 168 items_per_second=465.361/s records_scanned_per_query=1 total_records_scanned=336 +CompositeKeyBenchmark/IntegerFieldComposite/10000 44.5 ms 44.5 ms 16 items_per_second=44.937/s records_scanned_per_query=1 total_records_scanned=32 +CompositeKeyBenchmark/IntegerFieldComposite/100000 490 ms 490 ms 2 items_per_second=4.08414/s records_scanned_per_query=1 total_records_scanned=4 +CompositeKeyBenchmark/IntegerFieldComposite/1000000 6250 ms 6250 ms 1 items_per_second=0.320023/s records_scanned_per_query=1 total_records_scanned=2 + +---------------------------------------------------------------------------------------------------------------------------------------------------------------- + +ubuntu@ip-172-31-38-52:/opt/resilientdb$ bazel run //benchmark/storage:composite_key_benchmark -- --benchmark_filter="IntegerFieldComposite" --storage_type="leveldb" +INFO: Analyzed target //benchmark/storage:composite_key_benchmark (0 packages loaded, 0 targets configured). +INFO: Found 1 target... +Target //benchmark/storage:composite_key_benchmark up-to-date: + bazel-bin/benchmark/storage/composite_key_benchmark +INFO: Elapsed time: 0.128s, Critical Path: 0.00s +INFO: 1 process: 1 internal. +INFO: Build completed successfully, 1 total action +INFO: Running command line: bazel-bin/benchmark/storage/composite_key_benchmark '--benchmark_filter=IntegerFieldComposite' '--storage_type=leveldb' +2025-09-03T20:21:25+00:00 +Running /home/ubuntu/.cache/bazel/_bazel_ubuntu/4daa26ab31ab249b7607bfe58eb14371/execroot/com_resdb_nexres/bazel-out/k8-fastbuild/bin/benchmark/storage/composite_key_benchmark +Run on (8 X 3602.54 MHz CPU s) +CPU Caches: + L1 Data 32 KiB (x4) + L1 Instruction 32 KiB (x4) + L2 Unified 1024 KiB (x4) + L3 Unified 36608 KiB (x1) +Load Average: 0.94, 0.62, 0.60 +***WARNING*** Library was built as DEBUG. Timings may be affected. +-------------------------------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations UserCounters... +-------------------------------------------------------------------------------------------------------------- +CompositeKeyBenchmark/IntegerFieldComposite/1000 6.36 ms 6.36 ms 110 items_per_second=314.239/s records_scanned_per_query=1 total_records_scanned=220 +CompositeKeyBenchmark/IntegerFieldComposite/10000 70.5 ms 70.5 ms 10 items_per_second=28.3702/s records_scanned_per_query=1 total_records_scanned=20 +CompositeKeyBenchmark/IntegerFieldComposite/100000 741 ms 741 ms 1 items_per_second=2.69975/s records_scanned_per_query=1 total_records_scanned=2 +CompositeKeyBenchmark/IntegerFieldComposite/1000000 10160 ms 10159 ms 1 items_per_second=0.196871/s records_scanned_per_query=1 total_records_scanned=2 +``` \ No newline at end of file diff --git a/benchmark/storage/composite_key_benchmark.cpp b/benchmark/storage/composite_key_benchmark.cpp new file mode 100644 index 0000000000..7f7a90207c --- /dev/null +++ b/benchmark/storage/composite_key_benchmark.cpp @@ -0,0 +1,645 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include + +#include "chain/storage/leveldb.h" +#include "chain/storage/rocksdb.h" +#include "chain/storage/storage.h" +#include "executor/kv/kv_executor.h" +#include "proto/kv/kv.pb.h" + +namespace resdb { +namespace { + +using json = nlohmann::json; + +DEFINE_string(storage_type, "leveldb", + "Storage type to use: mem, leveldb, leveldb_cache, rocksdb"); + +class LogSuppressor { + public: + LogSuppressor() { + FLAGS_minloglevel = 3; + FLAGS_logtostderr = false; + } +}; + +std::string ExtractFieldValue(const std::string& json_str, + const std::string& field) { + try { + auto j = json::parse(json_str); + if (j.find(field) != j.end()) { + if (j[field].is_string()) { + return j[field].get(); + } else if (j[field].is_number()) { + return std::to_string(j[field].get()); + } else if (j[field].is_boolean()) { + return j[field].get() ? "true" : "false"; + } + } + } catch (const std::exception& e) { + // Return empty string on parsing error + } + return ""; +} + +int SetValue(KVExecutor* executor, const std::string& key, + const std::string& value) { + KVRequest request; + request.set_cmd(KVRequest::SET); + request.set_key(key); + request.set_value(value); + + std::string str; + if (!request.SerializeToString(&str)) { + return -1; + } + + executor->ExecuteData(str); + return 0; +} + +std::string GetValue(KVExecutor* executor, const std::string& key) { + KVRequest request; + request.set_cmd(KVRequest::GET); + request.set_key(key); + + std::string str; + if (!request.SerializeToString(&str)) { + return ""; + } + auto resp = executor->ExecuteData(str); + if (resp == nullptr) { + return ""; + } + KVResponse kv_response; + if (!kv_response.ParseFromString(*resp)) { + return ""; + } + return kv_response.value(); +} + +int CreateCompositeKey(KVExecutor* executor, const std::string& primary_key, + const std::string& field_name, + const std::string& field_value, int field_type) { + KVRequest request; + request.set_cmd(KVRequest::CREATE_COMPOSITE_KEY); + request.set_key(primary_key); + request.set_field_name(field_name); + request.set_value(field_value); + request.set_field_type(field_type); + + std::string str; + if (!request.SerializeToString(&str)) { + return -1; + } + + executor->ExecuteData(str); + return 0; +} + +std::vector GetByCompositeKey(KVExecutor* executor, + const std::string& field_name, + const std::string& field_value, + int field_type) { + KVRequest request; + request.set_cmd(KVRequest::GET_BY_COMPOSITE_KEY); + request.set_field_name(field_name); + request.set_value(field_value); + request.set_field_type(field_type); + + std::string str; + if (!request.SerializeToString(&str)) { + return std::vector(); + } + + auto resp = executor->ExecuteData(str); + if (resp == nullptr) { + return std::vector(); + } + KVResponse kv_response; + if (!kv_response.ParseFromString(*resp)) { + return std::vector(); + } + + std::vector results; + for (const auto& item : kv_response.items().item()) { + results.push_back(item.value_info().value()); + } + return results; +} + +std::vector GetByCompositeKeyRange(KVExecutor* executor, + const std::string& field_name, + const std::string& min_value, + const std::string& max_value, + int field_type) { + KVRequest request; + request.set_cmd(KVRequest::GET_COMPOSITE_KEY_RANGE); + request.set_field_name(field_name); + request.set_key(min_value); + request.set_value(max_value); + request.set_field_type(field_type); + + std::string str; + if (!request.SerializeToString(&str)) { + return std::vector(); + } + + auto resp = executor->ExecuteData(str); + if (resp == nullptr) { + return std::vector(); + } + KVResponse kv_response; + if (!kv_response.ParseFromString(*resp)) { + return std::vector(); + } + + std::vector results; + for (const auto& item : kv_response.items().item()) { + results.push_back(item.value_info().value()); + } + return results; +} + +class CompositeKeyBenchmark : public benchmark::Fixture { + protected: + void SetUp(const benchmark::State& state) override { + static LogSuppressor log_suppressor; + + if(std::filesystem::exists("/tmp/nexres-leveldb")) { + std::filesystem::remove_all("/tmp/nexres-leveldb"); + } + if(std::filesystem::exists("/tmp/nexres-rocksdb")) { + std::filesystem::remove_all("/tmp/nexres-rocksdb"); + } + + std::unique_ptr storage; + std::string type = FLAGS_storage_type; + std::transform(type.begin(), type.end(), type.begin(), ::tolower); + + if (type == "leveldb") { + storage = storage::NewResLevelDB(std::nullopt); + } else if (type == "leveldb_cache") { + storage::LevelDBInfo config; + config.set_enable_block_cache(true); + storage = storage::NewResLevelDB(config); + } else if (type == "rocksdb") { + storage = storage::NewResRocksDB(std::nullopt); + } else { + storage = storage::NewResLevelDB(std::nullopt); + } + + storage_ptr_ = storage.get(); + executor_ = std::make_unique(std::move(storage)); + } + + void TearDown(const benchmark::State& state) override { + executor_.reset(); + storage_ptr_ = nullptr; + std::filesystem::remove_all("/tmp/nexres-leveldb"); + std::filesystem::remove_all("/tmp/nexres-rocksdb"); + } + + std::unique_ptr executor_; + Storage* storage_ptr_; +}; + +BENCHMARK_DEFINE_F(CompositeKeyBenchmark, + StringFieldInMemory)(benchmark::State& state) { + const int num_records = state.range(0); + + std::vector string_values = {"Alice", "Bob", "Charlie", "David", + "Eve"}; + std::vector target_strings = {"Alice", "Charlie"}; + + for (int i = 0; i < num_records; i++) { + std::string name = string_values[i % string_values.size()]; + std::string json = + "{\"name\":\"" + name + "\",\"id\":" + std::to_string(i) + "}"; + SetValue(executor_.get(), "record_" + std::to_string(i), json); + } + + for (auto _ : state) { + int total_matches = 0; + for (const auto& target : target_strings) { + for (int i = 0; i < num_records; i++) { + std::string json = + GetValue(executor_.get(), "record_" + std::to_string(i)); + if (!json.empty()) { + std::string name = ExtractFieldValue(json, "name"); + if (name == target) { + total_matches++; + } + } + } + } + benchmark::DoNotOptimize(total_matches); + } + + state.SetItemsProcessed(state.iterations() * target_strings.size()); + state.counters["records_scanned_per_query"] = num_records; + state.counters["total_records_scanned"] = + state.iterations() * target_strings.size() * num_records; +} + +BENCHMARK_DEFINE_F(CompositeKeyBenchmark, + StringFieldComposite)(benchmark::State& state) { + const int num_records = state.range(0); + + std::vector string_values = {"Alice", "Bob", "Charlie", "David", + "Eve"}; + std::vector target_strings = {"Alice", "Charlie"}; + + for (int i = 0; i < num_records; i++) { + std::string name = string_values[i % string_values.size()]; + std::string json = + "{\"name\":\"" + name + "\",\"id\":" + std::to_string(i) + "}"; + SetValue(executor_.get(), "record_" + std::to_string(i), json); + CreateCompositeKey(executor_.get(), "record_" + std::to_string(i), "name", + name, 0); // STRING = 0 + } + + for (auto _ : state) { + int total_matches = 0; + for (const auto& target : target_strings) { + std::vector results = + GetByCompositeKey(executor_.get(), "name", target, 0); + total_matches += results.size(); + } + benchmark::DoNotOptimize(total_matches); + } + + state.SetItemsProcessed(state.iterations() * target_strings.size()); + state.counters["records_scanned_per_query"] = 1; // Direct lookup + state.counters["total_records_scanned"] = + state.iterations() * target_strings.size(); +} + +BENCHMARK_DEFINE_F(CompositeKeyBenchmark, + IntegerFieldInMemory)(benchmark::State& state) { + const int num_records = state.range(0); + + std::vector age_values = {25, 30, 35, 40, 45}; + std::vector target_ages = {30, 40}; + + for (int i = 0; i < num_records; i++) { + int age = age_values[i % age_values.size()]; + std::string json = "{\"age\":" + std::to_string(age) + + ",\"id\":" + std::to_string(i) + "}"; + SetValue(executor_.get(), "record_" + std::to_string(i), json); + } + + for (auto _ : state) { + int total_matches = 0; + for (const auto& target : target_ages) { + for (int i = 0; i < num_records; i++) { + std::string json = + GetValue(executor_.get(), "record_" + std::to_string(i)); + if (!json.empty()) { + std::string age_str = ExtractFieldValue(json, "age"); + if (!age_str.empty()) { + int age = std::stoi(age_str); + if (age == target) { + total_matches++; + } + } + } + } + } + benchmark::DoNotOptimize(total_matches); + } + + state.SetItemsProcessed(state.iterations() * target_ages.size()); + state.counters["records_scanned_per_query"] = num_records; + state.counters["total_records_scanned"] = + state.iterations() * target_ages.size() * num_records; +} + +BENCHMARK_DEFINE_F(CompositeKeyBenchmark, + IntegerFieldComposite)(benchmark::State& state) { + const int num_records = state.range(0); + + std::vector age_values = {25, 30, 35, 40, 45}; + std::vector target_ages = {30, 40}; + + for (int i = 0; i < num_records; i++) { + int age = age_values[i % age_values.size()]; + std::string json = "{\"age\":" + std::to_string(age) + + ",\"id\":" + std::to_string(i) + "}"; + SetValue(executor_.get(), "record_" + std::to_string(i), json); + CreateCompositeKey(executor_.get(), "record_" + std::to_string(i), "age", + std::to_string(age), 1); // INTEGER = 1 + } + + for (auto _ : state) { + int total_matches = 0; + for (const auto& target : target_ages) { + std::vector results = + GetByCompositeKey(executor_.get(), "age", std::to_string(target), 1); + total_matches += results.size(); + } + benchmark::DoNotOptimize(total_matches); + } + + state.SetItemsProcessed(state.iterations() * target_ages.size()); + state.counters["records_scanned_per_query"] = 1; + state.counters["total_records_scanned"] = + state.iterations() * target_ages.size(); +} + +BENCHMARK_DEFINE_F(CompositeKeyBenchmark, + RangeQueryInMemory)(benchmark::State& state) { + const int num_records = state.range(0); + + std::vector age_values = {25, 30, 35, 40, 45}; + + for (int i = 0; i < num_records; i++) { + int age = age_values[i % age_values.size()]; + std::string json = "{\"age\":" + std::to_string(age) + + ",\"id\":" + std::to_string(i) + "}"; + SetValue(executor_.get(), "record_" + std::to_string(i), json); + } + + for (auto _ : state) { + int total_matches = 0; + int min_age = 30; + int max_age = 40; + + for (int i = 0; i < num_records; i++) { + std::string json = + GetValue(executor_.get(), "record_" + std::to_string(i)); + if (!json.empty()) { + std::string age_str = ExtractFieldValue(json, "age"); + if (!age_str.empty()) { + int age = std::stoi(age_str); + if (age >= min_age && age <= max_age) { + total_matches++; + } + } + } + } + benchmark::DoNotOptimize(total_matches); + } + + state.SetItemsProcessed(state.iterations() * num_records); +} + +BENCHMARK_DEFINE_F(CompositeKeyBenchmark, + RangeQueryComposite)(benchmark::State& state) { + const int num_records = state.range(0); + + std::vector age_values = {25, 30, 35, 40, 45}; + + for (int i = 0; i < num_records; i++) { + int age = age_values[i % age_values.size()]; + std::string json = "{\"age\":" + std::to_string(age) + + ",\"id\":" + std::to_string(i) + "}"; + SetValue(executor_.get(), "record_" + std::to_string(i), json); + CreateCompositeKey(executor_.get(), "record_" + std::to_string(i), "age", + std::to_string(age), 1); // INTEGER = 1 + } + + for (auto _ : state) { + std::vector results = + GetByCompositeKeyRange(executor_.get(), "age", "30", "40", 1); + benchmark::DoNotOptimize(results.size()); + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK_DEFINE_F(CompositeKeyBenchmark, + PrimaryKeyOnly_Rand)(benchmark::State& state) { + const int num_records = state.range(0); + + for (int i = 0; i < num_records; i++) { + std::string json = "{\"id\":" + std::to_string(i) + ",\"data\":\"record_" + + std::to_string(i) + "\"}"; + SetValue(executor_.get(), "record_" + std::to_string(i), json); + } + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dist(0, num_records - 1); + + for (auto _ : state) { + for (int i = 0; i < num_records; i++) { + int random_id = dist(gen); + std::string result = + GetValue(executor_.get(), "record_" + std::to_string(random_id)); + benchmark::DoNotOptimize(result); + } + } + + state.SetItemsProcessed(state.iterations() * num_records); + state.counters["total_records"] = num_records; + state.counters["composite_keys"] = 0; + + TearDown(state); +} + +BENCHMARK_DEFINE_F(CompositeKeyBenchmark, + PrimaryKeyWithCompositeKeys_25AF_Rand)(benchmark::State& state) { + const int num_records = state.range(0); + + for (int i = 0; i < num_records; i++) { + std::string json = "{\"id\":" + std::to_string(i) + ",\"data\":\"record_" + + std::to_string(i) + "\"}"; + SetValue(executor_.get(), "record_" + std::to_string(i), json); + + if (i % 4 == 0) { + CreateCompositeKey(executor_.get(), "record_" + std::to_string(i), "id", + std::to_string(i), 1); + } + } + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dist(0, num_records - 1); + + for (auto _ : state) { + for (int i = 0; i < num_records; i++) { + int random_id = dist(gen); + std::string result = + GetValue(executor_.get(), "record_" + std::to_string(random_id)); + benchmark::DoNotOptimize(result); + } + } + + state.SetItemsProcessed(state.iterations() * num_records); + state.counters["total_records"] = num_records; + state.counters["composite_keys"] = num_records; + + TearDown(state); +} + +BENCHMARK_DEFINE_F(CompositeKeyBenchmark, + PrimaryKeyWithCompositeKeys_50AF_Rand)(benchmark::State& state) { + const int num_records = state.range(0); + + for (int i = 0; i < num_records; i++) { + std::string json = "{\"id\":" + std::to_string(i) + ",\"data\":\"record_" + + std::to_string(i) + "\"}"; + SetValue(executor_.get(), "record_" + std::to_string(i), json); + + if (i % 2 == 0) { + CreateCompositeKey(executor_.get(), "record_" + std::to_string(i), "id", + std::to_string(i), 1); + } + } + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dist(0, num_records - 1); + + for (auto _ : state) { + for (int i = 0; i < num_records; i++) { + int random_id = dist(gen); + std::string result = + GetValue(executor_.get(), "record_" + std::to_string(random_id)); + benchmark::DoNotOptimize(result); + } + } + + state.SetItemsProcessed(state.iterations() * num_records); + state.counters["total_records"] = num_records; + state.counters["composite_keys"] = num_records; + + TearDown(state); +} + +BENCHMARK_DEFINE_F(CompositeKeyBenchmark, + PrimaryKeyWithCompositeKeys_100AF_Rand)(benchmark::State& state) { + const int num_records = state.range(0); + + for (int i = 0; i < num_records; i++) { + std::string json = "{\"id\":" + std::to_string(i) + ",\"data\":\"record_" + + std::to_string(i) + "\"}"; + SetValue(executor_.get(), "record_" + std::to_string(i), json); + + CreateCompositeKey(executor_.get(), "record_" + std::to_string(i), "id", + std::to_string(i), 1); + } + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dist(0, num_records - 1); + + for (auto _ : state) { + for (int i = 0; i < num_records; i++) { + int random_id = dist(gen); + std::string result = + GetValue(executor_.get(), "record_" + std::to_string(random_id)); + benchmark::DoNotOptimize(result); + } + } + + state.SetItemsProcessed(state.iterations() * num_records); + state.counters["total_records"] = num_records; + state.counters["composite_keys"] = num_records; + + TearDown(state); +} + +BENCHMARK_REGISTER_F(CompositeKeyBenchmark, StringFieldInMemory) + ->Args({1000}) + ->Args({10000}) + ->Args({100000}) + ->Unit(benchmark::kMillisecond); + +BENCHMARK_REGISTER_F(CompositeKeyBenchmark, StringFieldComposite) + ->Args({1000}) + ->Args({10000}) + ->Args({100000}) + ->Args({1000000}) + ->Unit(benchmark::kMillisecond); + +BENCHMARK_REGISTER_F(CompositeKeyBenchmark, IntegerFieldInMemory) + ->Args({1000}) + ->Args({10000}) + ->Args({100000}) + ->Unit(benchmark::kMillisecond); + +BENCHMARK_REGISTER_F(CompositeKeyBenchmark, IntegerFieldComposite) + ->Args({1000}) + ->Args({10000}) + ->Args({100000}) + ->Args({1000000}) + // ->Args({10000000}) + ->Unit(benchmark::kMillisecond); + +BENCHMARK_REGISTER_F(CompositeKeyBenchmark, PrimaryKeyOnly_Rand) + ->Args({1000}) + ->Args({10000}) + ->Args({100000}) + // ->Args({1000000}) + ->Unit(benchmark::kMillisecond); + +BENCHMARK_REGISTER_F(CompositeKeyBenchmark, PrimaryKeyWithCompositeKeys_25AF_Rand) + ->Args({1000}) + ->Args({10000}) + ->Args({100000}) + ->Args({1000000}) + ->Unit(benchmark::kMillisecond); + +BENCHMARK_REGISTER_F(CompositeKeyBenchmark, PrimaryKeyWithCompositeKeys_50AF_Rand) + ->Args({1000}) + ->Args({10000}) + ->Args({100000}) + ->Args({1000000}) + ->Unit(benchmark::kMillisecond); + +BENCHMARK_REGISTER_F(CompositeKeyBenchmark, PrimaryKeyWithCompositeKeys_100AF_Rand) + ->Args({1000}) + ->Args({10000}) + ->Args({100000}) + ->Args({1000000}) + ->Unit(benchmark::kMillisecond); + +// TODO: Working on range query bench +// BENCHMARK_REGISTER_F(CompositeKeyBenchmark, RangeQueryInMemory) +// ->Args({1000}) +// ->Args({10000}) +// ->Args({100000}) +// ->Unit(benchmark::kMillisecond); + +// BENCHMARK_REGISTER_F(CompositeKeyBenchmark, RangeQueryComposite) +// ->Args({1000}) +// ->Args({10000}) +// ->Args({100000}) +// ->Unit(benchmark::kMillisecond); + +} // namespace +} // namespace resdb + +int main(int argc, char** argv) { + ::benchmark::Initialize(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1; + ::benchmark::RunSpecifiedBenchmarks(); + ::benchmark::Shutdown(); + return 0; +} diff --git a/benchmark/storage/plot_benchmark.py b/benchmark/storage/plot_benchmark.py new file mode 100644 index 0000000000..e395f53a96 --- /dev/null +++ b/benchmark/storage/plot_benchmark.py @@ -0,0 +1,157 @@ +import matplotlib.pyplot as plt +import numpy as np +import json +import argparse +import sys +from matplotlib.ticker import ScalarFormatter + +def format_value(value): + """Format large numbers with K/M suffix""" + if value >= 1000000: + return f'{value/1000000:.1f}M' + elif value >= 1000: + return f'{value/1000:.1f}K' + return f'{value:.1f}' + +def calculate_overhead(primary_values, comparison_values): + """Calculate overhead percentages relative to primary values""" + return [(comp - prim) / prim * 100 for prim, comp in zip(primary_values, comparison_values)] + +def plot_benchmark(data, output_file=None): + # Extract data + records = data["data"]["records"] + series = data["data"]["series"] + + # Get primary values for overhead calculation + primary_values = series[0]["values"] + + # Calculate overheads for other series + overheads = [] + for s in series[1:]: + overhead = calculate_overhead(primary_values, s["values"]) + overheads.append(overhead) + + # Set up positions + x = np.arange(len(records)) + width = 0.2 + + # Create figure and axis + fig, ax = plt.subplots(figsize=(12, 6)) + + # Create bars + bars = [] + for i, series_data in enumerate(series): + values = series_data["values"] + bar = ax.bar(x + (i - 1.5)*width, + values, + width, + label=series_data["name"], + color=series_data["color"]) + bars.append(bar) + + # Add overhead annotations for non-primary series + for i in range(1, len(series)): + for j, value in enumerate(series[i]["values"]): + overhead = overheads[i-1][j] + x_pos = x[j] + (i - 1.5)*width + + # Only show overhead percentage, positioned at top of bar + if overhead > 0: + overhead_text = f'+{overhead:.1f}%' + else: + overhead_text = f'{overhead:.1f}%' + + ax.text(x_pos, value * 1.02, + overhead_text, + ha='center', + va='bottom', + fontsize=8, + alpha=0.7) + + # Customize the plot + ax.set_yscale('log') + ax.set_ylabel(data["data"]["axis"]["y"]["label"]) + ax.set_xlabel(data["data"]["axis"]["x"]["label"]) + ax.set_title(data["title"]) + ax.set_xticks(x) + ax.set_xticklabels(records) + + # Format y-axis with K/M suffixes + ax.yaxis.set_major_formatter(ScalarFormatter()) + ax.yaxis.set_major_formatter(plt.FuncFormatter(lambda x, p: format_value(x))) + + ax.legend() + ax.grid(True, which="both", ls="-", alpha=0.2) + + # Add subtle horizontal lines for easier reading + ax.yaxis.grid(True, linestyle='--', alpha=0.3) + + # Adjust layout + plt.tight_layout() + + # Save the plot + output_file = output_file or 'benchmark_latency_comparison.png' + plt.savefig(output_file, dpi=300, bbox_inches='tight') + print(f"Plot saved as: {output_file}") + +def main(): + parser = argparse.ArgumentParser(description='Generate benchmark latency comparison plots') + parser.add_argument('input_file', nargs='?', type=str, + help='Input JSON file containing benchmark data') + parser.add_argument('--output', '-o', type=str, + help='Output PNG file (default: benchmark_latency_comparison.png)') + parser.add_argument('--example', '-e', action='store_true', + help='Print example JSON format and exit') + parser.add_argument('--dpi', type=int, default=300, + help='DPI for output image (default: 300)') + parser.add_argument('--figsize', type=float, nargs=2, default=[12, 6], + help='Figure size in inches (width height), default: 12 6') + + args = parser.parse_args() + + if args.example: + example_data = { + "title": "Latency Comparison: 25%, 50%, 100% Write Amplification, Sequential Reads", + "units": "ms", + "data": { + "records": ["1K", "10K", "100K", "1M"], + "series": [ + { + "name": "Primary Only", + "values": [18.8, 206, 2231, 29095], + "color": "#2196F3" + }, + { + "name": "25% Write Amplification", + "values": [19.1, 209, 2233, 34021], + "color": "#4CAF50" + } + ], + "axis": { + "x": {"label": "Number of Records", "scale": "linear"}, + "y": {"label": "Latency (ms)", "scale": "log"} + } + } + } + print("Example JSON format:") + print(json.dumps(example_data, indent=2)) + sys.exit(0) + + if args.input_file: + try: + with open(args.input_file, 'r') as f: + data = json.load(f) + except FileNotFoundError: + print(f"Error: Input file '{args.input_file}' not found") + sys.exit(1) + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in input file: {e}") + sys.exit(1) + else: + print("Error: Please provide an input JSON file or use --example to see the format") + sys.exit(1) + + plot_benchmark(data, args.output) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmark/storage/results/primary_key_retrieval/leveldb_bf_pkr_wa_sr.json b/benchmark/storage/results/primary_key_retrieval/leveldb_bf_pkr_wa_sr.json new file mode 100644 index 0000000000..5b8a1ecde2 --- /dev/null +++ b/benchmark/storage/results/primary_key_retrieval/leveldb_bf_pkr_wa_sr.json @@ -0,0 +1,39 @@ +{ + "title": "LevelDB: Primary Key Retrieval Latency Comparison: 25%, 50%, 100% Write Amplification, Sequential Reads", + "units": "ms", + "data": { + "records": ["1K", "10K", "100K", "1M"], + "series": [ + { + "name": "Primary Only", + "values": [18.8, 206, 2231, 29095], + "color": "#2196F3" + }, + { + "name": "25% Write Amplification", + "values": [19.1, 209, 2233, 34021], + "color": "#4CAF50" + }, + { + "name": "50% Write Amplification", + "values": [20.7, 209, 2335, 30012], + "color": "#FFC107" + }, + { + "name": "100% Write Amplification", + "values": [21.8, 227, 2370, 97125], + "color": "#F44336" + } + ], + "axis": { + "x": { + "label": "Number of Records", + "scale": "linear" + }, + "y": { + "label": "Latency (ms)", + "scale": "log" + } + } + } +} diff --git a/benchmark/storage/results/primary_key_retrieval/leveldb_bloom_filter.png b/benchmark/storage/results/primary_key_retrieval/leveldb_bloom_filter.png new file mode 100644 index 0000000000..092d344cda Binary files /dev/null and b/benchmark/storage/results/primary_key_retrieval/leveldb_bloom_filter.png differ diff --git a/chain/storage/BUILD b/chain/storage/BUILD index a86c2a7ff4..1038559d47 100644 --- a/chain/storage/BUILD +++ b/chain/storage/BUILD @@ -58,16 +58,31 @@ cc_library( ], ) +cc_library( + name = "rocksdb", + srcs = ["rocksdb.cpp"], + hdrs = ["rocksdb.h"], + + deps = [ + ":storage", + "//chain/storage/proto:kv_cc_proto", + "//chain/storage/proto:rocksdb_config_cc_proto", + "//common:comm", + "//third_party:rocksdb", + ], +) + cc_test( name = "kv_storage_test", srcs = ["kv_storage_test.cpp"], deps = [ ":leveldb", ":memory_db", + ":rocksdb", "//common/test:test_main", ], - timeout = "short", # Set the timeout to "short" - size = "small", # Set the size to "small" + timeout = "long", + size = "medium", ) cc_test( diff --git a/chain/storage/kv_storage_test.cpp b/chain/storage/kv_storage_test.cpp index 2e55161f07..a478e53304 100644 --- a/chain/storage/kv_storage_test.cpp +++ b/chain/storage/kv_storage_test.cpp @@ -25,17 +25,21 @@ #include "chain/storage/leveldb.h" #include "chain/storage/memory_db.h" +#include "chain/storage/rocksdb.h" + + namespace resdb { namespace storage { namespace { -enum StorageType { MEM = 0, LEVELDB = 1, LEVELDB_WITH_BLOCK_CACHE = 2 }; +enum StorageType { MEM = 0, LEVELDB = 1, LEVELDB_WITH_BLOCK_CACHE = 2, ROCKSDB = 3 }; class KVStorageTest : public ::testing::TestWithParam { protected: KVStorageTest() { StorageType t = GetParam(); + LevelDBInfo config; switch (t) { case MEM: storage = NewMemoryDB(); @@ -46,10 +50,13 @@ class KVStorageTest : public ::testing::TestWithParam { break; case LEVELDB_WITH_BLOCK_CACHE: Reset(); - LevelDBInfo config; config.set_enable_block_cache(true); storage = NewResLevelDB(path_, config); break; + case ROCKSDB: + Reset(); + storage = NewResRocksDB(path_); + break; } } @@ -230,7 +237,7 @@ TEST_P(KVStorageTest, BlockCacheSpecificTest) { INSTANTIATE_TEST_CASE_P(KVStorageTest, KVStorageTest, ::testing::Values(MEM, LEVELDB, - LEVELDB_WITH_BLOCK_CACHE)); + LEVELDB_WITH_BLOCK_CACHE, ROCKSDB)); } // namespace } // namespace storage diff --git a/chain/storage/leveldb.cpp b/chain/storage/leveldb.cpp index f6122b5ad9..f175c9b355 100644 --- a/chain/storage/leveldb.cpp +++ b/chain/storage/leveldb.cpp @@ -76,6 +76,10 @@ void ResLevelDB::CreateDB(const std::string& path) { options.write_buffer_size = write_buffer_size_; leveldb::DB* db = nullptr; + + //hard coded bloom filter policy for now + filter_policy_ = leveldb::NewBloomFilterPolicy(20); + options.filter_policy = filter_policy_; leveldb::Status status = leveldb::DB::Open(options, path, &db); if (status.ok()) { db_ = std::unique_ptr(db); @@ -91,6 +95,9 @@ ResLevelDB::~ResLevelDB() { if (block_cache_) { block_cache_->Flush(); } + if (filter_policy_) { + delete filter_policy_; + } } int ResLevelDB::SetValue(const std::string& key, const std::string& value) { @@ -113,6 +120,14 @@ int ResLevelDB::SetValue(const std::string& key, const std::string& value) { return 0; } +int ResLevelDB::DelValue(const std::string& key) { + leveldb::Status status = db_->Delete(leveldb::WriteOptions(), key); + if (status.ok()) { + return 0; + } + return -1; +} + std::string ResLevelDB::GetValue(const std::string& key) { std::string value; bool found_in_cache = false; @@ -165,6 +180,29 @@ std::string ResLevelDB::GetRange(const std::string& min_key, return values; } +std::vector ResLevelDB::GetKeysByPrefix(const std::string& prefix) { + std::vector resp; + leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions()); + for (it->Seek(prefix); it->Valid() && it->key().starts_with(prefix); + it->Next()) { + resp.push_back(it->key().ToString()); + } + delete it; + return resp; +} + +std::vector ResLevelDB::GetKeyRangeByPrefix( + const std::string& start_prefix, const std::string& end_prefix) { + std::vector resp; + leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions()); + for (it->Seek(start_prefix); + it->Valid() && it->key().ToString() <= end_prefix; it->Next()) { + resp.push_back(it->key().ToString()); + } + delete it; + return resp; +} + bool ResLevelDB::UpdateMetrics() { if (block_cache_ == nullptr) { return false; diff --git a/chain/storage/leveldb.h b/chain/storage/leveldb.h index 6103ec20b8..7e8f32ee42 100644 --- a/chain/storage/leveldb.h +++ b/chain/storage/leveldb.h @@ -27,6 +27,7 @@ #include "chain/storage/storage.h" #include "common/lru/lru_cache.h" #include "leveldb/db.h" +#include "leveldb/filter_policy.h" #include "leveldb/write_batch.h" #include "platform/statistic/stats.h" @@ -44,6 +45,7 @@ class ResLevelDB : public Storage { virtual ~ResLevelDB(); int SetValue(const std::string& key, const std::string& value) override; + int DelValue(const std::string& key) override; std::string GetValue(const std::string& key) override; std::string GetAllValues(void) override; std::string GetRange(const std::string& min_key, @@ -59,6 +61,10 @@ class ResLevelDB : public Storage { std::map> GetKeyRange( const std::string& min_key, const std::string& max_key) override; + std::vector GetKeysByPrefix(const std::string& prefix) override; + + std::vector GetKeyRangeByPrefix(const std::string& start_prefix, const std::string& end_prefix) override; + // Return a list of std::vector> GetHistory(const std::string& key, int min_version, @@ -82,6 +88,7 @@ class ResLevelDB : public Storage { protected: Stats* global_stats_ = nullptr; + const leveldb::FilterPolicy* filter_policy_ = nullptr; std::unique_ptr> block_cache_; }; diff --git a/chain/storage/memory_db.cpp b/chain/storage/memory_db.cpp index 2d5b872589..e8e293898c 100644 --- a/chain/storage/memory_db.cpp +++ b/chain/storage/memory_db.cpp @@ -33,6 +33,11 @@ int MemoryDB::SetValue(const std::string& key, const std::string& value) { return 0; } +int MemoryDB::DelValue(const std::string& key) { + kv_map_.erase(key); + return 0; +} + std::string MemoryDB::GetAllValues(void) { std::string values = "["; bool first_iteration = true; @@ -166,5 +171,25 @@ std::vector> MemoryDB::GetTopHistory( return resp; } +std::vector MemoryDB::GetKeysByPrefix(const std::string& prefix) { + std::vector resp; + for (const auto& kv : kv_map_) { + if (kv.first.find(prefix) == 0) { + resp.push_back(kv.second); + } + } + return resp; +} + +std::vector MemoryDB::GetKeyRangeByPrefix(const std::string& start_prefix, const std::string& end_prefix) { + std::vector resp; + for (const auto& kv : kv_map_) { + if (kv.first >= start_prefix && kv.first <= end_prefix) { + resp.push_back(kv.first); + } + } + return resp; +} + } // namespace storage } // namespace resdb diff --git a/chain/storage/memory_db.h b/chain/storage/memory_db.h index 372f46474b..161378f1a6 100644 --- a/chain/storage/memory_db.h +++ b/chain/storage/memory_db.h @@ -55,7 +55,7 @@ class MemoryDB : public Storage { int SetValue(const std::string& key, const std::string& value); std::string GetValue(const std::string& key); - + int DelValue(const std::string& key); std::string GetAllValues() override; std::string GetRange(const std::string& min_key, const std::string& max_key) override; @@ -78,6 +78,10 @@ class MemoryDB : public Storage { std::vector> GetTopHistory(const std::string& key, int number) override; + std::vector GetKeysByPrefix(const std::string& prefix) override; + + std::vector GetKeyRangeByPrefix(const std::string& start_prefix, const std::string& end_prefix) override; + private: std::unordered_map kv_map_; std::unordered_map>> diff --git a/chain/storage/multiSecIndex.h b/chain/storage/multiSecIndex.h new file mode 100644 index 0000000000..4896157e80 --- /dev/null +++ b/chain/storage/multiSecIndex.h @@ -0,0 +1,52 @@ +#include +#include + +#include +#include + +#include "chain/storage/leveldb.h" +#include "chain/storage/proto/kv.pb.h" +#include "leveldb/options.h" + +namespace resdb { +namespace storage { + +class MultiSecIndex { + public: + MultiSecIndex() = default; + virtual ~MultiSecIndex() = default; + + // put enconded composite key into db + // use ResLevelDB::SetValue + int createCompositeKey(const std::string& primKey, + const nlohmann::json& docValue, + const std::string& fieldName, + const std::string& fieldType); + // first get field value then convert it into string + std::string encodeFieldVal(const nlohmann::json& docValue, + const std::string& fieldName, + const std::string& fieldType); + // fieldName + fieldVal + primKey + std::string encodeCompositeKey(std::string& fieldName, std::string& fieldVal, + std::string& primKey); + + // using primary keys got by primKeysbyPrefix to get key value pairs in db + // assume user already convert field value into string + int GetByCompositeKey(const std::string& queryFieldName, + const std::string& queryFieldVal, + std::vector& matched_records); + // encode into prefix of composite key + std::string encodePrefix(const std::string& queryFieldName, + const std::string& queryFieldVal); + // get primary keys by single field pair prefix + int compKeysbyPrefix(std::string& prefix, std::vector& compKeys); + // string subtraction from composite keys to get rid of prefix and get primary + // keys + std::vector primKeybyCompKey(std::vector& compKeys); + // calling encodeFieldVal to check if the query result's field values is the + // same as query's. + bool validateUpdated(std::vector& matched_records, + const std::string& queryFieldVal); +}; +} // namespace storage +} // namespace resdb \ No newline at end of file diff --git a/chain/storage/proto/BUILD b/chain/storage/proto/BUILD index 0b1c7263f2..eb49f21eb7 100644 --- a/chain/storage/proto/BUILD +++ b/chain/storage/proto/BUILD @@ -42,3 +42,13 @@ cc_proto_library( name = "leveldb_config_cc_proto", deps = [":leveldb_config_proto"], ) + +proto_library( + name = "rocksdb_config_proto", + srcs = ["rocksdb_config.proto"], +) + +cc_proto_library( + name = "rocksdb_config_cc_proto", + deps = [":rocksdb_config_proto"], +) \ No newline at end of file diff --git a/chain/storage/proto/leveldb_config.proto b/chain/storage/proto/leveldb_config.proto index 28e5a18808..c3902511c2 100644 --- a/chain/storage/proto/leveldb_config.proto +++ b/chain/storage/proto/leveldb_config.proto @@ -27,4 +27,6 @@ message LevelDBInfo { string path = 4; optional bool enable_block_cache = 5; optional uint32 block_cache_capacity = 6; + optional bool enable_bloom_filter = 7; + optional uint32 bloom_filter_bits_per_key = 8; } diff --git a/chain/storage/proto/rocksdb_config.proto b/chain/storage/proto/rocksdb_config.proto new file mode 100644 index 0000000000..6b6581c679 --- /dev/null +++ b/chain/storage/proto/rocksdb_config.proto @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +syntax = "proto3"; + +package resdb.storage; + +message RocksDBInfo { + uint32 num_threads = 2; + uint32 write_buffer_size_mb = 3; + uint32 write_batch_size = 4; + string path = 5; +} \ No newline at end of file diff --git a/chain/storage/rocksdb.cpp b/chain/storage/rocksdb.cpp new file mode 100644 index 0000000000..90a224675f --- /dev/null +++ b/chain/storage/rocksdb.cpp @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "chain/storage/rocksdb.h" + +#include + +#include "chain/storage/proto/kv.pb.h" +#include "rocksdb/slice_transform.h" +#include "rocksdb/db.h" +#include "rocksdb/filter_policy.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/table.h" +#include "rocksdb/write_batch.h" + +namespace resdb { +namespace storage { + +std::unique_ptr NewResRocksDB(const std::string& path, + std::optional config) { + if (config == std::nullopt) { + config = RocksDBInfo(); + } + (*config).set_path(path); + return std::make_unique(config); +} + +std::unique_ptr NewResRocksDB(std::optional config) { + return std::make_unique(config); +} + +ResRocksDB::ResRocksDB(std::optional config) { + std::string path = "/tmp/nexres-rocksdb"; + if (config.has_value()) { + num_threads_ = (*config).num_threads(); + write_buffer_size_ = (*config).write_buffer_size_mb() << 20; + write_batch_size_ = (*config).write_batch_size(); + if ((*config).path() != "") { + LOG(ERROR) << "Custom path for RocksDB provided in config: " + << (*config).path(); + path = (*config).path(); + } + } + CreateDB(path); +} + +void ResRocksDB::CreateDB(const std::string& path) { + rocksdb::Options options; + options.create_if_missing = true; + if (num_threads_ > 1) options.IncreaseParallelism(num_threads_); + options.OptimizeLevelStyleCompaction(); + options.write_buffer_size = write_buffer_size_; + + // ------------------------------------------------------------------------- // + // Conifgs for the rocksdb - hard coded -> change to config + // ------------------------------------------------------------------------- // + options.enable_blob_files = true; + rocksdb::BlockBasedTableOptions table_options; + table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); + table_options.whole_key_filtering = true; + options.table_factory.reset( + rocksdb::NewBlockBasedTableFactory(table_options)); + options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(7)); + // ------------------------------------------------------------------------- // + + rocksdb::DB* db = nullptr; + rocksdb::Status status = rocksdb::DB::Open(options, path, &db); + if (status.ok()) { + db_ = std::unique_ptr(db); + LOG(ERROR) << "Successfully opened RocksDB in path: " << path; + } else { + LOG(ERROR) << "RocksDB status fail"; + } + assert(status.ok()); + LOG(ERROR) << "Successfully opened RocksDB"; +} + +ResRocksDB::~ResRocksDB() { + if (db_) { + Flush(); // Ensure any pending writes are flushed + db_.reset(); + } +} + +int ResRocksDB::SetValue(const std::string& key, const std::string& value) { + batch_.Put(key, value); + + if (batch_.Count() >= write_batch_size_) { + rocksdb::Status status = db_->Write(rocksdb::WriteOptions(), &batch_); + if (status.ok()) { + batch_.Clear(); + } else { + LOG(ERROR) << "write value fail:" << status.ToString(); + return -1; + } + } + return 0; +} + +std::string ResRocksDB::GetValue(const std::string& key) { + std::string value = ""; + rocksdb::Status status = db_->Get(rocksdb::ReadOptions(), key, &value); + if (status.ok()) { + return value; + } else { + return ""; + } +} + +std::string ResRocksDB::GetAllValues(void) { + std::string values = "["; + rocksdb::Iterator* itr = db_->NewIterator(rocksdb::ReadOptions()); + bool first_iteration = true; + for (itr->SeekToFirst(); itr->Valid(); itr->Next()) { + if (!first_iteration) values.append(","); + first_iteration = false; + values.append(itr->value().ToString()); + } + values.append("]"); + + delete itr; + return values; +} + +std::string ResRocksDB::GetRange(const std::string& min_key, + const std::string& max_key) { + std::string values = "["; + rocksdb::Iterator* itr = db_->NewIterator(rocksdb::ReadOptions()); + bool first_iteration = true; + for (itr->Seek(min_key); itr->Valid() && itr->key().ToString() <= max_key; + itr->Next()) { + if (!first_iteration) values.append(","); + first_iteration = false; + values.append(itr->value().ToString()); + } + values.append("]"); + + delete itr; + return values; +} + +bool ResRocksDB::Flush() { + rocksdb::Status status = db_->Write(rocksdb::WriteOptions(), &batch_); + if (status.ok()) { + batch_.Clear(); + return true; + } + LOG(ERROR) << "write value fail:" << status.ToString(); + return false; +} +int ResRocksDB::SetValueWithVersion(const std::string& key, + const std::string& value, int version) { + std::string value_str = GetValue(key); + ValueHistory history; + if (!history.ParseFromString(value_str)) { + LOG(ERROR) << "old_value parse fail"; + return -2; + } + + int last_v = 0; + if (history.value_size() > 0) { + last_v = history.value(history.value_size() - 1).version(); + } + + if (last_v != version) { + LOG(ERROR) << "version does not match:" << version + << " old version:" << last_v; + return -2; + } + + Value* new_value = history.add_value(); + new_value->set_value(value); + new_value->set_version(version + 1); + + history.SerializeToString(&value_str); + return SetValue(key, value_str); +} + +std::pair ResRocksDB::GetValueWithVersion( + const std::string& key, int version) { + std::string value_str = GetValue(key); + ValueHistory history; + if (!history.ParseFromString(value_str)) { + LOG(ERROR) << "old_value parse fail"; + return std::make_pair("", 0); + } + if (history.value_size() == 0) { + return std::make_pair("", 0); + } + if (version > 0) { + for (int i = history.value_size() - 1; i >= 0; --i) { + if (history.value(i).version() == version) { + return std::make_pair(history.value(i).value(), + history.value(i).version()); + } + if (history.value(i).version() < version) { + break; + } + } + } + int last_idx = history.value_size() - 1; + return std::make_pair(history.value(last_idx).value(), + history.value(last_idx).version()); +} + +// Return a map of > +std::map> ResRocksDB::GetAllItems() { + std::map> resp; + + rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions()); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + ValueHistory history; + if (!history.ParseFromString(it->value().ToString()) || + history.value_size() == 0) { + LOG(ERROR) << "old_value parse fail"; + continue; + } + const Value& value = history.value(history.value_size() - 1); + resp.insert(std::make_pair(it->key().ToString(), + std::make_pair(value.value(), value.version()))); + } + delete it; + + return resp; +} + +std::map> ResRocksDB::GetKeyRange( + const std::string& min_key, const std::string& max_key) { + std::map> resp; + + rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions()); + for (it->Seek(min_key); it->Valid() && it->key().ToString() <= max_key; + it->Next()) { + ValueHistory history; + if (!history.ParseFromString(it->value().ToString()) || + history.value_size() == 0) { + LOG(ERROR) << "old_value parse fail"; + continue; + } + const Value& value = history.value(history.value_size() - 1); + resp.insert(std::make_pair(it->key().ToString(), + std::make_pair(value.value(), value.version()))); + } + delete it; + + return resp; +} + +// Return a list of +std::vector> ResRocksDB::GetHistory( + const std::string& key, int min_version, int max_version) { + std::vector> resp; + std::string value_str = GetValue(key); + ValueHistory history; + if (!history.ParseFromString(value_str)) { + LOG(ERROR) << "old_value parse fail"; + return resp; + } + + for (int i = history.value_size() - 1; i >= 0; --i) { + if (history.value(i).version() < min_version) { + break; + } + if (history.value(i).version() <= max_version) { + resp.push_back( + std::make_pair(history.value(i).value(), history.value(i).version())); + } + } + + return resp; +} + +// Return a list of +std::vector> ResRocksDB::GetTopHistory( + const std::string& key, int top_number) { + std::vector> resp; + std::string value_str = GetValue(key); + ValueHistory history; + if (!history.ParseFromString(value_str)) { + LOG(ERROR) << "old_value parse fail"; + return resp; + } + + for (int i = history.value_size() - 1; + i >= 0 && resp.size() < static_cast(top_number); --i) { + resp.push_back( + std::make_pair(history.value(i).value(), history.value(i).version())); + } + + return resp; +} + +int ResRocksDB::DelValue(const std::string& key) { + rocksdb::Status status = db_->Delete(rocksdb::WriteOptions(), key); + if (status.ok()) { + return 0; + } + return -1; +} + +std::vector ResRocksDB::GetKeysByPrefix( + const std::string& prefix) { + std::vector resp; + rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions()); + for (it->Seek(prefix); it->Valid() && it->key().starts_with(prefix); + it->Next()) { + resp.push_back(it->key().ToString()); + } + delete it; + return resp; +} + +std::vector ResRocksDB::GetKeyRangeByPrefix( + const std::string& start_prefix, const std::string& end_prefix) { + std::vector resp; + rocksdb::Iterator* it = db_->NewIterator(rocksdb::ReadOptions()); + for (it->Seek(start_prefix); + it->Valid() && it->key().ToString() <= end_prefix; it->Next()) { + resp.push_back(it->key().ToString()); + } + delete it; + return resp; +} +} // namespace storage + +} // namespace resdb \ No newline at end of file diff --git a/chain/storage/rocksdb.h b/chain/storage/rocksdb.h new file mode 100644 index 0000000000..df198d7004 --- /dev/null +++ b/chain/storage/rocksdb.h @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "chain/storage/proto/rocksdb_config.pb.h" +#include "chain/storage/storage.h" +#include "rocksdb/db.h" +#include "rocksdb/write_batch.h" + +namespace resdb { +namespace storage { + +std::unique_ptr NewResRocksDB( + const std::string& path, std::optional config = std::nullopt); +std::unique_ptr NewResRocksDB( + std::optional config = std::nullopt); + +class ResRocksDB : public Storage { + public: + ResRocksDB(std::optional config_data = std::nullopt); + virtual ~ResRocksDB(); + int SetValue(const std::string& key, const std::string& value) override; + std::string GetValue(const std::string& key) override; + int DelValue(const std::string& key) override; + std::string GetAllValues(void) override; + std::string GetRange(const std::string& min_key, + const std::string& max_key) override; + + int SetValueWithVersion(const std::string& key, const std::string& value, + int version) override; + std::pair GetValueWithVersion(const std::string& key, + int version) override; + + // Return a map of > + std::map> GetAllItems() override; + std::map> GetKeyRange( + const std::string& min_key, const std::string& max_key) override; + + // Return a list of + std::vector> GetHistory(const std::string& key, + int min_version, + int max_version) override; + std::vector> GetTopHistory( + const std::string& key, int top_number) override; + + std::vector GetKeysByPrefix(const std::string& prefix) override; + + std::vector GetKeyRangeByPrefix( + const std::string& start_prefix, const std::string& end_prefix) override; + + + + bool Flush() override; + + private: + void CreateDB(const std::string& path); + + private: + std::unique_ptr<::rocksdb::DB> db_ = nullptr; + ::rocksdb::WriteBatch batch_; + unsigned int num_threads_ = 1; + unsigned int write_buffer_size_ = 64 << 20; + unsigned int write_batch_size_ = 1; +}; + +} // namespace storage +} // namespace resdb \ No newline at end of file diff --git a/chain/storage/setting/BUILD b/chain/storage/setting/BUILD index 25d11b4e8d..4da6a00b79 100644 --- a/chain/storage/setting/BUILD +++ b/chain/storage/setting/BUILD @@ -33,3 +33,18 @@ config_setting( }, visibility = ["//visibility:public"], ) + +bool_flag( + name = "enable_rocksdb", + build_setting_default = False, + visibility = ["//visibility:public"], +) + + +config_setting( + name = "enable_rocksdb_setting", + values = { + "define": "enable_rocksdb=True", + }, + visibility = ["//visibility:public"], +) \ No newline at end of file diff --git a/chain/storage/storage.h b/chain/storage/storage.h index 76e352de8e..48e4c464fb 100644 --- a/chain/storage/storage.h +++ b/chain/storage/storage.h @@ -31,6 +31,7 @@ class Storage { virtual ~Storage() = default; virtual int SetValue(const std::string& key, const std::string& value) = 0; + virtual int DelValue(const std::string& key) = 0; virtual std::string GetValue(const std::string& key) = 0; virtual std::string GetAllValues() = 0; virtual std::string GetRange(const std::string& min_key, @@ -54,6 +55,10 @@ class Storage { virtual std::vector> GetTopHistory( const std::string& key, int number) = 0; + virtual std::vector GetKeysByPrefix(const std::string& prefix) = 0; + + virtual std::vector GetKeyRangeByPrefix(const std::string& start_prefix, const std::string& end_prefix) = 0; + virtual bool Flush() { return true; }; }; diff --git a/executor/kv/BUILD b/executor/kv/BUILD index d8a47bcf3f..15ae0ef8eb 100644 --- a/executor/kv/BUILD +++ b/executor/kv/BUILD @@ -40,6 +40,12 @@ cc_test( deps = [ ":kv_executor", "//chain/storage:memory_db", + "//chain/storage:leveldb", + "//chain/storage:rocksdb", "//common/test:test_main", + "@com_github_gflags_gflags//:gflags", ], + args = ["--storage_types=all"], # Default to testing all storage types + timeout = "short", + size = "small", ) diff --git a/executor/kv/kv_executor.cpp b/executor/kv/kv_executor.cpp index c587f592ce..68f812014d 100644 --- a/executor/kv/kv_executor.cpp +++ b/executor/kv/kv_executor.cpp @@ -18,15 +18,22 @@ */ #include "executor/kv/kv_executor.h" -#include "executor/contract/executor/contract_executor.h" #include +#include "executor/contract/executor/contract_executor.h" + namespace resdb { KVExecutor::KVExecutor(std::unique_ptr storage) : storage_(std::move(storage)) { - contract_manager_ = std::make_unique(storage_.get()); + contract_manager_ = + std::make_unique( + storage_.get()); + + // Initialize default composite key builder + composite_key_builder_ = std::make_unique( + ":", "idx", "v1"); } std::unique_ptr KVExecutor::ParseData( @@ -48,6 +55,8 @@ std::unique_ptr KVExecutor::ExecuteRequest( Set(kv_request.key(), kv_request.value()); } else if (kv_request.cmd() == KVRequest::GET) { kv_response.set_value(Get(kv_request.key())); + } else if (kv_request.cmd() == KVRequest::DEL_VAL) { + kv_response.set_value(std::to_string(Delete(kv_request.key()))); } else if (kv_request.cmd() == KVRequest::GETALLVALUES) { kv_response.set_value(GetAllValues()); } else if (kv_request.cmd() == KVRequest::GETRANGE) { @@ -68,10 +77,43 @@ std::unique_ptr KVExecutor::ExecuteRequest( } else if (kv_request.cmd() == KVRequest::GET_TOP) { GetTopHistory(kv_request.key(), kv_request.top_number(), kv_response.mutable_items()); - } - else if(!kv_request.smart_contract_request().empty()){ - std::unique_ptr resp = contract_manager_->ExecuteData(kv_request.smart_contract_request()); - if(resp != nullptr){ + } else if (kv_request.cmd() == KVRequest::CREATE_COMPOSITE_KEY) { + int status = CreateCompositeKey( + kv_request.key(), kv_request.field_name(), kv_request.value(), + static_cast(kv_request.field_type())); + kv_response.set_value(std::to_string(status)); + } else if (kv_request.cmd() == KVRequest::UPDATE_COMPOSITE_KEY) { + int status = UpdateCompositeKey( + kv_request.key(), kv_request.field_name(), kv_request.min_value(), + kv_request.max_value(), + static_cast(kv_request.field_type()), + static_cast(kv_request.field_type())); + kv_response.set_value(std::to_string(status)); + } else if (kv_request.cmd() == KVRequest::GET_BY_COMPOSITE_KEY) { + auto results = GetByCompositeKey( + kv_request.field_name(), kv_request.value(), + static_cast(kv_request.field_type())); + std::string result; + Items* items = kv_response.mutable_items(); + for (const auto& document : results) { + Item* item = items->add_item(); + item->set_key(""); + item->mutable_value_info()->set_value(document); + } + } else if (kv_request.cmd() == KVRequest::GET_COMPOSITE_KEY_RANGE) { + auto results = GetByCompositeKeyRange( + kv_request.field_name(), kv_request.min_value(), kv_request.max_value(), + static_cast(kv_request.field_type())); + Items* items = kv_response.mutable_items(); + for (const auto& document : results) { + Item* item = items->add_item(); + item->set_key(""); + item->mutable_value_info()->set_value(document); + } + } else if (!kv_request.smart_contract_request().empty()) { + std::unique_ptr resp = + contract_manager_->ExecuteData(kv_request.smart_contract_request()); + if (resp != nullptr) { kv_response.set_smart_contract_response(*resp); } } @@ -94,11 +136,13 @@ std::unique_ptr KVExecutor::ExecuteData( return nullptr; } - LOG(ERROR)<<" execute cmd:"< KVExecutor::ExecuteData( } else if (kv_request.cmd() == KVRequest::GET_TOP) { GetTopHistory(kv_request.key(), kv_request.top_number(), kv_response.mutable_items()); - } - else if(!kv_request.smart_contract_request().empty()){ - std::unique_ptr resp = contract_manager_->ExecuteData(kv_request.smart_contract_request()); - if(resp != nullptr){ + } else if (kv_request.cmd() == KVRequest::CREATE_COMPOSITE_KEY) { + int status = CreateCompositeKey( + kv_request.key(), kv_request.field_name(), kv_request.value(), + static_cast(kv_request.field_type())); + kv_response.set_value(std::to_string(status)); + } else if (kv_request.cmd() == KVRequest::GET_BY_COMPOSITE_KEY) { + auto results = GetByCompositeKey( + kv_request.field_name(), kv_request.value(), + static_cast(kv_request.field_type())); + Items* items = kv_response.mutable_items(); + for (const auto& document : results) { + Item* item = items->add_item(); + item->set_key(""); + item->mutable_value_info()->set_value(document); + } + } else if (kv_request.cmd() == KVRequest::UPDATE_COMPOSITE_KEY) { + int status = UpdateCompositeKey( + kv_request.key(), kv_request.field_name(), kv_request.min_value(), + kv_request.max_value(), + static_cast(kv_request.field_type()), + static_cast(kv_request.field_type())); + kv_response.set_value(std::to_string(status)); + } else if (kv_request.cmd() == KVRequest::GET_COMPOSITE_KEY_RANGE) { + auto results = GetByCompositeKeyRange( + kv_request.field_name(), kv_request.min_value(), kv_request.max_value(), + static_cast(kv_request.field_type())); + Items* items = kv_response.mutable_items(); + for (const auto& document : results) { + Item* item = items->add_item(); + item->set_key(""); + item->mutable_value_info()->set_value(document); + } + } else if (!kv_request.smart_contract_request().empty()) { + std::unique_ptr resp = + contract_manager_->ExecuteData(kv_request.smart_contract_request()); + if (resp != nullptr) { kv_response.set_smart_contract_response(*resp); } } @@ -135,18 +211,19 @@ std::unique_ptr KVExecutor::ExecuteData( } void KVExecutor::Set(const std::string& key, const std::string& value) { - LOG(ERROR)<<" set key:"<SetValue(key, value); } std::string KVExecutor::Get(const std::string& key) { - LOG(ERROR)<<" get key:"<GetValue(key); } +int KVExecutor::Delete(const std::string& key) { + return storage_->DelValue(key); +} + std::string KVExecutor::GetAllValues() { return storage_->GetAllValues(); } -// Get values on a range of keys std::string KVExecutor::GetRange(const std::string& min_key, const std::string& max_key) { return storage_->GetRange(min_key, max_key); @@ -211,4 +288,203 @@ void KVExecutor::GetTopHistory(const std::string& key, int top_number, } } +// Method implementations for DefaultCompositeKeyBuilder::KeyBuilder +DefaultCompositeKeyBuilder::KeyBuilder::KeyBuilder(const std::string& separator) + : separator_(separator) { + result_.reserve(256); +} + +DefaultCompositeKeyBuilder::KeyBuilder& +DefaultCompositeKeyBuilder::KeyBuilder::add(const std::string& component) { + if (!result_.empty()) { + result_ += separator_; + } + result_ += component; + return *this; +} + +DefaultCompositeKeyBuilder::KeyBuilder& +DefaultCompositeKeyBuilder::KeyBuilder::addSeparator() { + result_ += separator_; + return *this; +} + +std::string DefaultCompositeKeyBuilder::KeyBuilder::build() const { + return result_; +} + +DefaultCompositeKeyBuilder::KeyBuilder +DefaultCompositeKeyBuilder::CreateBuilder() const { + return KeyBuilder(separator_); +} + +std::string DefaultCompositeKeyBuilder::Build(const std::string& field_name, + const std::string& encoded_value, + const std::string& primary_key) { + return CreateBuilder() + .add(prefix_) + .add(version_) + .add(field_name) + .add(encoded_value) + .add(primary_key) + .build(); +} + +std::string DefaultCompositeKeyBuilder::BuildPrefix( + const std::string& field_name, const std::string& encoded_value) { + return CreateBuilder() + .add(prefix_) + .add(version_) + .add(field_name) + .add(encoded_value) + .addSeparator() + .build(); +} + +std::string DefaultCompositeKeyBuilder::BuildLowerBound( + const std::string& field_name, const std::string& encoded_min_value) { + return CreateBuilder() + .add(prefix_) + .add(version_) + .add(field_name) + .add(encoded_min_value) + .addSeparator() + .build(); +} + +std::string DefaultCompositeKeyBuilder::BuildUpperBound( + const std::string& field_name, const std::string& encoded_max_value) { + return CreateBuilder() + .add(prefix_) + .add(version_) + .add(field_name) + .add(encoded_max_value) + .add("\xFF") + .addSeparator() + .build(); +} + +std::string KVExecutor::BuildCompositeKey(const std::string& field_name, + const std::string& encoded_value, + const std::string& primary_key) { + return composite_key_builder_->Build(field_name, encoded_value, primary_key); +} + +std::string KVExecutor::BuildCompositeKeyPrefix( + const std::string& field_name, const std::string& encoded_value) { + return composite_key_builder_->BuildPrefix(field_name, encoded_value); +} + +void KVExecutor::SetCompositeKeyBuilder( + std::unique_ptr builder) { + composite_key_builder_ = std::move(builder); +} + +std::vector KVExecutor::ExtractPrimaryKeys( + const std::vector& composite_keys) { + std::vector primary_keys; + if (!composite_key_builder_) { + LOG(ERROR) << "composite key builder is not initialized"; + return primary_keys; + } + const std::string separator = composite_key_builder_->GetSeparator(); + for (const auto& composite_key : composite_keys) { + size_t last_separator = composite_key.find_last_of(separator); + if (last_separator != std::string::npos) { + std::string primary_key = composite_key.substr(last_separator + 1); + primary_keys.push_back(primary_key); + } else { + LOG(ERROR) << "invalid composite key. no separator found in composite key" + << composite_key; + } + } + return primary_keys; +} + +int KVExecutor::CreateCompositeKey(const std::string& primary_key, + const std::string& field_name, + const std::string& field_value, + CompositeKeyType field_type) { + std::string encoded_value = EncodeValue(field_value, field_type); + std::string composite_key = + BuildCompositeKey(field_name, encoded_value, primary_key); + return storage_->SetValue(composite_key, composite_val_marker_); +} + +std::vector KVExecutor::GetByCompositeKey( + const std::string& field_name, const std::string& field_value, + CompositeKeyType field_type) { + std::string encoded_value = EncodeValue(field_value, field_type); + std::string prefix = BuildCompositeKeyPrefix(field_name, encoded_value); + + auto results = storage_->GetKeysByPrefix(prefix); + + std::vector primary_keys = ExtractPrimaryKeys(results); + + std::vector documents; + for (const auto& primary_key : primary_keys) { + std::string document = storage_->GetValue(primary_key); + if (!document.empty()) { + documents.push_back(document); + } + } + return documents; +} + +std::vector KVExecutor::GetByCompositeKeyRange( + const std::string& field_name, const std::string& min_value, + const std::string& max_value, CompositeKeyType field_type) { + std::string encoded_min = EncodeValue(min_value, field_type); + std::string encoded_max = EncodeValue(max_value, field_type); + + std::string start_key = + composite_key_builder_->BuildLowerBound(field_name, encoded_min); + std::string end_key = + composite_key_builder_->BuildUpperBound(field_name, encoded_max); + + std::vector composite_keys = + storage_->GetKeyRangeByPrefix(start_key, end_key); + + std::vector primary_keys = ExtractPrimaryKeys(composite_keys); + + std::vector documents; + for (const auto& primary_key : primary_keys) { + std::string document = storage_->GetValue(primary_key); + if (!document.empty()) { + documents.push_back(document); + } + } + + return documents; +} + +int KVExecutor::UpdateCompositeKey(const std::string& primary_key, + const std::string& field_name, + const std::string& old_field_value, + const std::string& new_field_value, + CompositeKeyType old_field_type, + CompositeKeyType new_field_type) { + std::string encoded_old_field_value = + EncodeValue(old_field_value, old_field_type); + + std::string delete_composite_key = + BuildCompositeKey(field_name, encoded_old_field_value, primary_key); + std::string check = storage_->GetValue(delete_composite_key); + + if (check == composite_val_marker_) { + int status_delete = storage_->DelValue(delete_composite_key); + if (status_delete != 0) { + LOG(ERROR) << "delete composite key status fail"; + return -1; + } + } + int status_create = CreateCompositeKey( + primary_key, field_name, new_field_value, old_field_type); // TODO + if (status_create != 0) { + LOG(ERROR) << "create composite key status fail"; + return -1; + } + return 0; +} + } // namespace resdb diff --git a/executor/kv/kv_executor.h b/executor/kv/kv_executor.h index fef1259725..7309f396d9 100644 --- a/executor/kv/kv_executor.h +++ b/executor/kv/kv_executor.h @@ -19,9 +19,7 @@ #pragma once -#include -#include -#include +#include #include "chain/storage/storage.h" #include "executor/common/transaction_manager.h" @@ -29,6 +27,74 @@ namespace resdb { +enum class CompositeKeyType { + STRING = 0, + INTEGER = 1, // int32_t for regular integers + BOOLEAN = 2, + TIMESTAMP = 3 // int64_t for Unix timestamps +}; + +class CompositeKeyBuilderBase { + public: + virtual ~CompositeKeyBuilderBase() = default; + virtual std::string Build(const std::string& field_name, + const std::string& encoded_value, + const std::string& primary_key) = 0; + virtual std::string BuildPrefix(const std::string& field_name, + const std::string& encoded_value) = 0; + virtual std::string BuildLowerBound(const std::string& field_name, + const std::string& encoded_min_value) = 0; + virtual std::string BuildUpperBound(const std::string& field_name, + const std::string& encoded_max_value) = 0; + virtual std::string GetSeparator() const = 0; + virtual std::string GetPrefix() const = 0; + virtual std::string GetVersion() const = 0; +}; + +class DefaultCompositeKeyBuilder : public CompositeKeyBuilderBase { + public: + DefaultCompositeKeyBuilder(const std::string& separator, + const std::string& prefix, + const std::string& version) + : separator_(separator), prefix_(prefix), version_(version) {} + + std::string Build(const std::string& field_name, + const std::string& encoded_value, + const std::string& primary_key) override; + + std::string BuildPrefix(const std::string& field_name, + const std::string& encoded_value) override; + + std::string BuildLowerBound(const std::string& field_name, + const std::string& encoded_min_value) override; + + std::string BuildUpperBound(const std::string& field_name, + const std::string& encoded_max_value) override; + + std::string GetSeparator() const override { return separator_; } + std::string GetPrefix() const override { return prefix_; } + std::string GetVersion() const override { return version_; } + + class KeyBuilder { + public: + explicit KeyBuilder(const std::string& separator); + KeyBuilder& add(const std::string& component); + KeyBuilder& addSeparator(); + std::string build() const; + + private: + std::string result_; + const std::string separator_; + }; + + KeyBuilder CreateBuilder() const; + + private: + const std::string separator_; + const std::string prefix_; + const std::string version_; +}; + class KVExecutor : public TransactionManager { public: KVExecutor(std::unique_ptr storage); @@ -40,9 +106,11 @@ class KVExecutor : public TransactionManager { const std::string& request) override; std::unique_ptr ExecuteRequest( const google::protobuf::Message& kv_request) override; + protected: virtual void Set(const std::string& key, const std::string& value); std::string Get(const std::string& key); + int Delete(const std::string& key); std::string GetAllValues(); std::string GetRange(const std::string& min_key, const std::string& max_key); @@ -56,10 +124,49 @@ class KVExecutor : public TransactionManager { Items* items); void GetTopHistory(const std::string& key, int top_number, Items* items); + // Composite key methods + int CreateCompositeKey(const std::string& primary_key, + const std::string& field_name, + const std::string& field_value, + CompositeKeyType field_type); + + std::vector GetByCompositeKey(const std::string& field_name, + const std::string& field_value, + CompositeKeyType field_type); + + std::vector GetByCompositeKeyRange(const std::string& field_name, + const std::string& min_value, + const std::string& max_value, + CompositeKeyType field_type); + int UpdateCompositeKey(const std::string& primary_key, + const std::string& field_name, + const std::string& old_field_value, + const std::string& new_field_value, + CompositeKeyType old_field_type, + CompositeKeyType new_field_type); + + void SetCompositeKeyBuilder(std::unique_ptr builder); + private: - std::unique_ptr storage_; + std::string EncodeValue(const std::string& value, + CompositeKeyType field_type); + std::string EncodeInteger(int32_t value); + std::string EncodeBoolean(bool value); + std::string EncodeTimestamp(int64_t value); + std::string BuildCompositeKey(const std::string& field_name, + const std::string& encoded_value, + const std::string& primary_key); + std::string BuildCompositeKeyPrefix(const std::string& field_name, + const std::string& encoded_value); + std::vector ExtractPrimaryKeys( + const std::vector& composite_keys); + + std::unique_ptr storage_; std::unique_ptr contract_manager_; + std::unique_ptr composite_key_builder_; + + const std::string composite_val_marker_ = "Y"; }; } // namespace resdb diff --git a/executor/kv/kv_executor_test.cpp b/executor/kv/kv_executor_test.cpp index 9bf78dc4c1..879bde8858 100644 --- a/executor/kv/kv_executor_test.cpp +++ b/executor/kv/kv_executor_test.cpp @@ -17,34 +17,68 @@ * under the License. */ -#include "executor/kv/kv_executor.h" + +#include #include #include +#include "chain/storage/leveldb.h" +#include "chain/storage/rocksdb.h" #include "chain/storage/memory_db.h" #include "chain/storage/storage.h" #include "common/test/test_macros.h" -#include "platform/config/resdb_config_utils.h" + #include "proto/kv/kv.pb.h" +#include "executor/kv/kv_executor.h" namespace resdb { namespace { using ::resdb::testing::EqualsProto; -using storage::MemoryDB; -using ::testing::Invoke; -using ::testing::Return; -using ::testing::Test; +using ::testing::TestWithParam; + +enum StorageType { MEM = 0, LEVELDB = 1, LEVELDB_WITH_BLOCK_CACHE = 2, ROCKSDB = 3 }; -class KVExecutorTest : public Test { +class KVExecutorTest : public TestWithParam { public: KVExecutorTest() { - auto storage = std::make_unique(); + Reset(); + StorageType t = GetParam(); + storage::LevelDBInfo config; + std::unique_ptr storage; + + switch (t) { + case MEM: + storage = storage::NewMemoryDB(); + break; + case LEVELDB: + storage = storage::NewResLevelDB(std::nullopt); + break; + case LEVELDB_WITH_BLOCK_CACHE: + config.set_enable_block_cache(true); + storage = storage::NewResLevelDB(config); + break; + case ROCKSDB: + storage = storage::NewResRocksDB(std::nullopt); + break; + } storage_ptr_ = storage.get(); impl_ = std::make_unique(std::move(storage)); } + ~KVExecutorTest() { + Reset(); + } + + protected: + void Reset() { + impl_.reset(); // Release the executor first + storage_ptr_ = nullptr; + std::filesystem::remove_all("/tmp/nexres-leveldb"); + std::filesystem::remove_all("/tmp/nexres-rocksdb"); + } + int Set(const std::string& key, const std::string& value) { KVRequest request; request.set_cmd(KVRequest::SET); @@ -227,6 +261,155 @@ class KVExecutorTest : public Test { return kv_response.items(); } + int CreateCompositeKey(const std::string& primary_key, + const std::string& field_name, + const std::string& field_value, + int field_type) { + KVRequest request; + request.set_cmd(KVRequest::CREATE_COMPOSITE_KEY); + request.set_key(primary_key); + request.set_value(field_value); + request.set_field_name(field_name); + request.set_field_type(field_type); + + std::string str; + if (!request.SerializeToString(&str)) { + return -1; + } + + auto resp = impl_->ExecuteData(str); + if (resp == nullptr) { + return -1; + } + + KVResponse kv_response; + if (!kv_response.ParseFromString(*resp)) { + return -1; + } + + int status = std::stoi(kv_response.value()); + + return status; + } + + int UpdateCompositeKey(const std::string& primary_key, const std::string& field_name, const std::string& old_field_value, const std::string& new_field_value, + int old_field_type, int new_field_type) { + + KVRequest request; + request.set_cmd(KVRequest::UPDATE_COMPOSITE_KEY); + request.set_key(primary_key); + request.set_field_name(field_name); + request.set_min_value(old_field_value); + request.set_max_value(new_field_value); + request.set_field_type(old_field_type); //old field type and new field type should be the same + + std::string str; + if (!request.SerializeToString(&str)) { + return -1; + } + + auto resp = impl_->ExecuteData(str); + if (resp == nullptr) { + return -1; + } + + KVResponse kv_response; + if (!kv_response.ParseFromString(*resp)) { + return -1; + } + + int status = std::stoi(kv_response.value()); + + return status; + } + + int DelVal(const std::string& key) { + KVRequest request; + request.set_cmd(KVRequest::DEL_VAL); + request.set_key(key); + + std::string str; + if (!request.SerializeToString(&str)) { + return -1; + } + + auto resp = impl_->ExecuteData(str); + if (resp == nullptr) { + return -1; + } + + KVResponse kv_response; + if (!kv_response.ParseFromString(*resp)) { + return -1; + } + + int status = std::stoi(kv_response.value()); + + return status; + } + + Items GetByCompositeKey(const std::string& field_name, + const std::string& field_value, int field_type) { + KVRequest request; + request.set_cmd(KVRequest::GET_BY_COMPOSITE_KEY); + request.set_value(field_value); + request.set_field_name(field_name); + request.set_field_type(field_type); + + std::string str; + if (!request.SerializeToString(&str)) { + return Items(); + } + + auto resp = impl_->ExecuteData(str); + if (resp == nullptr) { + return Items(); + } + KVResponse kv_response; + if (!kv_response.ParseFromString(*resp)) { + return Items(); + } + + return kv_response.items(); + } + + Items GetByCompositeKeyRange(const std::string& field_name, + const std::string& min_value, + const std::string& max_value, int field_type) { + KVRequest request; + request.set_cmd(KVRequest::GET_COMPOSITE_KEY_RANGE); + request.set_field_name(field_name); + request.set_min_value(min_value); + request.set_max_value(max_value); + request.set_field_type(field_type); + + std::string str; + if (!request.SerializeToString(&str)) { + return Items(); + } + + auto resp = impl_->ExecuteData(str); + if (resp == nullptr) { + return Items(); + } + KVResponse kv_response; + if (!kv_response.ParseFromString(*resp)) { + return Items(); + } + + return kv_response.items(); + } + + void PrintAllKeys() { + std::cout << "=== All Keys in Database ===" << std::endl; + Items all_items = GetAllItems(); + for (int i = 0; i < all_items.item_size(); i++) { + std::cout << "Key: '" << all_items.item(i).key() << "' -> Value: '" + << all_items.item(i).value_info().value() << "'" << std::endl; + } + std::cout << "=== End All Keys ===" << std::endl; + } + protected: Storage* storage_ptr_; @@ -234,7 +417,7 @@ class KVExecutorTest : public Test { std::unique_ptr impl_; }; -TEST_F(KVExecutorTest, SetValue) { +TEST_P(KVExecutorTest, SetValue) { std::map data; EXPECT_EQ(GetAllValues(), "[]"); @@ -247,7 +430,7 @@ TEST_F(KVExecutorTest, SetValue) { EXPECT_EQ(GetRange("a", "z"), "[test_value]"); } -TEST_F(KVExecutorTest, SetValueWithVersion) { +TEST_P(KVExecutorTest, SetValueWithVersion) { std::map data; { @@ -338,6 +521,254 @@ TEST_F(KVExecutorTest, SetValueWithVersion) { } } +TEST_P(KVExecutorTest, CompositeKeyStringField) { + std::string user1 = "{\"name\":\"John\",\"age\":30,\"city\":\"NYC\"}"; + std::string user2 = "{\"name\":\"Jane\",\"age\":25,\"city\":\"LA\"}"; + std::string user3 = "{\"name\":\"Bob\",\"age\":35,\"city\":\"Chicago\"}"; + + + EXPECT_EQ(Set("user_1", user1), 0); + EXPECT_EQ(Set("user_2", user2), 0); + EXPECT_EQ(Set("user_3", user3), 0); + + + int status1 = CreateCompositeKey("user_1", "name", "John", 0); // STRING = 0 + int status2 = CreateCompositeKey("user_2", "name", "Jane", 0); + int status3 = CreateCompositeKey("user_3", "name", "Bob", 0); + + + EXPECT_EQ(status1, 0); + EXPECT_EQ(status2, 0); + EXPECT_EQ(status3, 0); + + + Items results = GetByCompositeKey("name", "John", 0); + EXPECT_EQ(results.item_size(), 1); + EXPECT_EQ(results.item(0).value_info().value(), user1); + + Items empty_results = GetByCompositeKey("name", "Alice", 0); + EXPECT_EQ(empty_results.item_size(), 0); +} + +TEST_P(KVExecutorTest, CompositeKeyIntegerField) { + std::string user1 = "{\"name\":\"John\",\"age\":30,\"city\":\"NYC\"}"; + std::string user2 = "{\"name\":\"Jane\",\"age\":25,\"city\":\"LA\"}"; + std::string user3 = "{\"name\":\"Bob\",\"age\":35,\"city\":\"Chicago\"}"; + std::string user4 = "{\"name\":\"Alice\",\"age\":30,\"city\":\"Boston\"}"; + + // Store documents + EXPECT_EQ(Set("user_1", user1), 0); + EXPECT_EQ(Set("user_2", user2), 0); + EXPECT_EQ(Set("user_3", user3), 0); + EXPECT_EQ(Set("user_4", user4), 0); + + int status1 = CreateCompositeKey("user_1", "age", "30", 1); + int status2 = CreateCompositeKey("user_2", "age", "25", 1); + int status3 = CreateCompositeKey("user_3", "age", "35", 1); + int status4 = CreateCompositeKey("user_4", "age", "30", 1); + + EXPECT_EQ(status1, 0); + EXPECT_EQ(status2, 0); + EXPECT_EQ(status3, 0); + EXPECT_EQ(status4, 0); + + Items results = GetByCompositeKey("age", "30", 1); + EXPECT_EQ(results.item_size(), 2); + + bool found_user1 = false, found_user4 = false; + for (int i = 0; i < results.item_size(); i++) { + std::string doc = results.item(i).value_info().value(); + if (doc == user1) found_user1 = true; + if (doc == user4) found_user4 = true; + } + EXPECT_TRUE(found_user1); + EXPECT_TRUE(found_user4); +} + + +TEST_P(KVExecutorTest, DeleteValue) { + std::string user1 = "{\"name\":\"John\",\"age\":30,\"city\":\"NYC\"}"; + EXPECT_EQ(Set("user_1", user1), 0); + EXPECT_EQ(Get("user_1"), user1); + + int status1 = DelVal("user_1"); + EXPECT_EQ(status1, 0); + EXPECT_EQ(Get("user_1"), ""); +} + +TEST_P(KVExecutorTest, UpdateCompositeKey){ + std::string user1 = "{\"name\":\"John\",\"age\":30,\"city\":\"NYC\"}"; + std::string user2 = "{\"name\":\"amy\",\"age\":30,\"city\":\"LA\"}"; + + EXPECT_EQ(Set("user_1", user1), 0); + EXPECT_EQ(Set("user_2", user2), 0); + + int status1 = CreateCompositeKey("user_1", "age", "30", 1); + int status2 = CreateCompositeKey("user_2", "age", "30", 1); + + EXPECT_EQ(status1, 0); + EXPECT_EQ(status2, 0); + + std::string user_update = "{\"name\":\"John\",\"age\":31,\"city\":\"NYC\"}"; + EXPECT_EQ(Set("user_1", user_update), 0); + int status5 = UpdateCompositeKey("user_1", "age", "30", "31", 1, 1); + EXPECT_EQ(status5, 0); + + Items results = GetByCompositeKey("age", "31", 1); + EXPECT_EQ(results.item_size(), 1); + EXPECT_EQ(results.item(0).value_info().value(), user_update); + + Items empty_results = GetByCompositeKey("age", "30", 1); + EXPECT_EQ(empty_results.item_size(), 1); + EXPECT_EQ(empty_results.item(0).value_info().value(), user2); + +} + + +TEST_P(KVExecutorTest, CompositeKeyBooleanField) { + std::string user1 = "{\"name\":\"John\",\"active\":true,\"city\":\"NYC\"}"; + std::string user2 = "{\"name\":\"Jane\",\"active\":false,\"city\":\"LA\"}"; + std::string user3 = "{\"name\":\"Bob\",\"active\":true,\"city\":\"Chicago\"}"; + + EXPECT_EQ(Set("user_1", user1), 0); + EXPECT_EQ(Set("user_2", user2), 0); + EXPECT_EQ(Set("user_3", user3), 0); + + int status1 = CreateCompositeKey("user_1", "active", "true", 2); + int status2 = CreateCompositeKey("user_2", "active", "false", 2); + int status3 = CreateCompositeKey("user_3", "active", "true", 2); + + EXPECT_EQ(status1, 0); + EXPECT_EQ(status2, 0); + EXPECT_EQ(status3, 0); + + Items active_results = GetByCompositeKey("active", "true", 2); + EXPECT_EQ(active_results.item_size(), 2); + + Items inactive_results = GetByCompositeKey("active", "false", 2); + EXPECT_EQ(inactive_results.item_size(), 1); + EXPECT_EQ(inactive_results.item(0).value_info().value(), user2); +} + +TEST_P(KVExecutorTest, CompositeKeyTimestampField) { + std::string user1 = + "{\"name\":\"John\",\"created_at\":1640995200,\"city\":\"NYC\"}"; + std::string user2 = + "{\"name\":\"Jane\",\"created_at\":1641081600,\"city\":\"LA\"}"; + std::string user3 = + "{\"name\":\"Bob\",\"created_at\":1641168000,\"city\":\"Chicago\"}"; + + + EXPECT_EQ(Set("user_1", user1), 0); + EXPECT_EQ(Set("user_2", user2), 0); + EXPECT_EQ(Set("user_3", user3), 0); + + int status1 = CreateCompositeKey("user_1", "created_at", "1640995200", 3); + int status2 = CreateCompositeKey("user_2", "created_at", "1641081600", 3); + int status3 = CreateCompositeKey("user_3", "created_at", "1641168000", 3); + + EXPECT_EQ(status1, 0); + EXPECT_EQ(status2, 0); + EXPECT_EQ(status3, 0); + + Items results = GetByCompositeKey("created_at", "1640995200", 3); + EXPECT_EQ(results.item_size(), 1); + EXPECT_EQ(results.item(0).value_info().value(), user1); +} + +TEST_P(KVExecutorTest, CompositeKeyRangeQuery) { + std::string user1 = "{\"name\":\"John\",\"age\":25,\"city\":\"NYC\"}"; + std::string user2 = "{\"name\":\"Jane\",\"age\":30,\"city\":\"LA\"}"; + std::string user3 = "{\"name\":\"Bob\",\"age\":35,\"city\":\"Chicago\"}"; + std::string user4 = "{\"name\":\"Alice\",\"age\":40,\"city\":\"Boston\"}"; + + // Store documents + EXPECT_EQ(Set("user_1", user1), 0); + EXPECT_EQ(Set("user_2", user2), 0); + EXPECT_EQ(Set("user_3", user3), 0); + EXPECT_EQ(Set("user_4", user4), 0); + + // Create composite keys on 'age' field + int status1 = CreateCompositeKey("user_1", "age", "25", 1); + int status2 = CreateCompositeKey("user_2", "age", "30", 1); + int status3 = CreateCompositeKey("user_3", "age", "35", 1); + int status4 = CreateCompositeKey("user_4", "age", "40", 1); + + // Verify composite keys were created successfully + EXPECT_EQ(status1, 0); + EXPECT_EQ(status2, 0); + EXPECT_EQ(status3, 0); + EXPECT_EQ(status4, 0); + + // Test range query (age between 30 and 35 inclusive) + Items range_results = GetByCompositeKeyRange("age", "30", "35", 1); + EXPECT_EQ(range_results.item_size(), 2); // Should return user_2 and user_3 +} + +TEST_P(KVExecutorTest, CompositeKeyMultipleFields) { + // Create JSON documents with multiple fields + std::string user1 = + "{\"name\":\"John\",\"age\":30,\"city\":\"NYC\",\"active\":true}"; + std::string user2 = + "{\"name\":\"Jane\",\"age\":25,\"city\":\"LA\",\"active\":false}"; + std::string user3 = + "{\"name\":\"Bob\",\"age\":35,\"city\":\"Chicago\",\"active\":true}"; + + // Store documents + EXPECT_EQ(Set("user_1", user1), 0); + EXPECT_EQ(Set("user_2", user2), 0); + EXPECT_EQ(Set("user_3", user3), 0); + + // Create composite keys on multiple fields + int user1_name_status = CreateCompositeKey("user_1", "name", "John", 0); + int user1_age_status = CreateCompositeKey("user_1", "age", "30", 1); + int user1_city_status = CreateCompositeKey("user_1", "city", "NYC", 0); + int user1_active_status = CreateCompositeKey("user_1", "active", "true", 2); + + int user2_name_status = CreateCompositeKey("user_2", "name", "Jane", 0); + int user2_age_status = CreateCompositeKey("user_2", "age", "25", 1); + int user2_city_status = CreateCompositeKey("user_2", "city", "LA", 0); + int user2_active_status = CreateCompositeKey("user_2", "active", "false", 2); + + int user3_name_status = CreateCompositeKey("user_3", "name", "Bob", 0); + int user3_age_status = CreateCompositeKey("user_3", "age", "35", 1); + int user3_city_status = CreateCompositeKey("user_3", "city", "Chicago", 0); + int user3_active_status = CreateCompositeKey("user_3", "active", "true", 2); + + // Verify composite keys were created successfully + EXPECT_EQ(user1_name_status, 0); + EXPECT_EQ(user1_age_status, 0); + EXPECT_EQ(user1_city_status, 0); + EXPECT_EQ(user1_active_status, 0); + EXPECT_EQ(user2_name_status, 0); + EXPECT_EQ(user2_age_status, 0); + EXPECT_EQ(user2_city_status, 0); + EXPECT_EQ(user2_active_status, 0); + EXPECT_EQ(user3_name_status, 0); + EXPECT_EQ(user3_age_status, 0); + EXPECT_EQ(user3_city_status, 0); + EXPECT_EQ(user3_active_status, 0); + + // Test queries on different fields + Items name_results = GetByCompositeKey("name", "John", 0); + EXPECT_EQ(name_results.item_size(), 1); + EXPECT_EQ(name_results.item(0).value_info().value(), user1); + + Items age_results = GetByCompositeKey("age", "25", 1); + EXPECT_EQ(age_results.item_size(), 1); + EXPECT_EQ(age_results.item(0).value_info().value(), user2); + + Items city_results = GetByCompositeKey("city", "Chicago", 0); + EXPECT_EQ(city_results.item_size(), 1); + EXPECT_EQ(city_results.item(0).value_info().value(), user3); + + Items active_results = GetByCompositeKey("active", "true", 2); + EXPECT_EQ(active_results.item_size(), 2); // John and Bob are active +} + +INSTANTIATE_TEST_CASE_P(KVExecutorTest, KVExecutorTest, + ::testing::Values(LEVELDB, ROCKSDB)); + } // namespace } // namespace resdb diff --git a/platform/consensus/checkpoint/checkpoint.h b/platform/consensus/checkpoint/checkpoint.h index 7a5b967ce1..dd05532ef1 100644 --- a/platform/consensus/checkpoint/checkpoint.h +++ b/platform/consensus/checkpoint/checkpoint.h @@ -27,6 +27,7 @@ class CheckPoint { virtual ~CheckPoint() = default; virtual uint64_t GetStableCheckpoint() = 0; + virtual uint64_t GetLastExecutedSeq() = 0; }; } // namespace resdb diff --git a/platform/consensus/checkpoint/mock_checkpoint.h b/platform/consensus/checkpoint/mock_checkpoint.h index 837d895ca8..22909b118e 100644 --- a/platform/consensus/checkpoint/mock_checkpoint.h +++ b/platform/consensus/checkpoint/mock_checkpoint.h @@ -28,6 +28,7 @@ namespace resdb { class MockCheckPoint : public CheckPoint { public: MOCK_METHOD(uint64_t, GetStableCheckpoint, (), (override)); + MOCK_METHOD(uint64_t, GetLastExecutedSeq, (), (override)); }; } // namespace resdb diff --git a/platform/consensus/execution/transaction_executor.cpp b/platform/consensus/execution/transaction_executor.cpp index a62e55f590..c5063eef36 100644 --- a/platform/consensus/execution/transaction_executor.cpp +++ b/platform/consensus/execution/transaction_executor.cpp @@ -22,6 +22,7 @@ #include #include "common/utils/utils.h" + namespace resdb { TransactionExecutor::TransactionExecutor( @@ -248,13 +249,17 @@ void TransactionExecutor::OnlyExecute(std::unique_ptr request) { if (transaction_manager_) { response = transaction_manager_->ExecuteBatch(batch_request); } - + if (response != nullptr){ + std::cout<<3<<"testing"<seq()<seq()); + } // global_stats_->IncTotalRequest(batch_request.user_requests_size()); // global_stats_->IncExecuteDone(); } void TransactionExecutor::Execute(std::unique_ptr request, bool need_execute) { + std::cout<<0<<"testing"<seq()<seq()); std::unique_ptr batch_request = nullptr; std::unique_ptr>> data; @@ -288,10 +293,15 @@ void TransactionExecutor::Execute(std::unique_ptr request, // <proxy_id()<<" need execute:"< response; + global_stats_->GetTransactionDetails(*batch_request_p); if (transaction_manager_ && need_execute) { if (execute_thread_num_ == 1) { response = transaction_manager_->ExecuteBatch(*batch_request_p); + if (response != nullptr){ + std::cout<<1<<"testing"<seq()<seq()); + } } else { std::vector> response_v; @@ -313,6 +323,10 @@ void TransactionExecutor::Execute(std::unique_ptr request, else { response_v = transaction_manager_->ExecuteBatchData(*data_p); } + if (response != nullptr || !response_v.empty()){ + std::cout<<2<<"testing"<seq()<seq()); + } FinishExecute(request->seq()); if(response == nullptr){ @@ -325,6 +339,8 @@ void TransactionExecutor::Execute(std::unique_ptr request, } // LOG(ERROR)<<" CF = :"<<(cf==1)<<" uid:"<AddExecuted(batch_request_p->hash(), batch_request_p->seq()); } @@ -346,6 +362,20 @@ void TransactionExecutor::Execute(std::unique_ptr request, global_stats_->IncExecuteDone(); } +void TransactionExecutor::set_OnExecuteSuccess(uint64_t seq) { + // Monotonic update: only move forward. + uint64_t cur = latest_executed_seq_.load(std::memory_order_relaxed); + while (seq > cur && !latest_executed_seq_.compare_exchange_weak( + cur, seq, std::memory_order_release, std::memory_order_relaxed)) { + /* retry with updated `cur` */ + } + // latest_executed_seq_ = seq; +} + +uint64_t TransactionExecutor::get_latest_executed_seq() const { + return latest_executed_seq_/*.load(std::memory_order_acquire)*/; +} + void TransactionExecutor::SetDuplicateManager(DuplicateManager* manager) { duplicate_manager_ = manager; } diff --git a/platform/consensus/execution/transaction_executor.h b/platform/consensus/execution/transaction_executor.h index 6fb8ef3926..e412e1f9fb 100644 --- a/platform/consensus/execution/transaction_executor.h +++ b/platform/consensus/execution/transaction_executor.h @@ -18,6 +18,8 @@ */ #pragma once +#include +#include #include #include @@ -34,6 +36,8 @@ namespace resdb { // Execute the requests that may contain system information or user requests. class TransactionExecutor { public: + void set_OnExecuteSuccess(uint64_t seq); + uint64_t get_latest_executed_seq() const; typedef std::function, std::unique_ptr resp)> PostExecuteFunc; @@ -73,6 +77,8 @@ class TransactionExecutor { void Prepare(std::unique_ptr request); private: + std::atomic latest_executed_seq_{0}; + // uint64_t latest_executed_seq_ = 0; void Execute(std::unique_ptr request, bool need_execute = true); void OnlyExecute(std::unique_ptr request); @@ -125,7 +131,6 @@ class TransactionExecutor { End_Prepare = 4, }; - std::vector prepare_thread_; static const int mod = 2048; std::mutex f_mutex_[mod], fd_mutex_[mod]; @@ -142,7 +147,6 @@ class TransactionExecutor { uint64_t, std::unique_ptr>>> data_[mod]; - }; } // namespace resdb diff --git a/platform/consensus/ordering/pbft/checkpoint_manager.cpp b/platform/consensus/ordering/pbft/checkpoint_manager.cpp index a5a24ca825..1ebc97d44a 100644 --- a/platform/consensus/ordering/pbft/checkpoint_manager.cpp +++ b/platform/consensus/ordering/pbft/checkpoint_manager.cpp @@ -378,4 +378,8 @@ uint64_t CheckPointManager::GetCommittableSeq() { return committable_seq_; } +uint64_t CheckPointManager::GetLastExecutedSeq(){ + return executor_->get_latest_executed_seq(); +} + } // namespace resdb diff --git a/platform/consensus/ordering/pbft/checkpoint_manager.h b/platform/consensus/ordering/pbft/checkpoint_manager.h index e043978eac..0873b68899 100644 --- a/platform/consensus/ordering/pbft/checkpoint_manager.h +++ b/platform/consensus/ordering/pbft/checkpoint_manager.h @@ -73,6 +73,8 @@ class CheckPointManager : public CheckPoint { uint64_t GetCommittableSeq(); + uint64_t GetLastExecutedSeq() override; + private: void UpdateCheckPointStatus(); void UpdateStableCheckPointStatus(); diff --git a/platform/consensus/recovery/recovery.cpp b/platform/consensus/recovery/recovery.cpp index 51faad5ce7..18cd6483ba 100644 --- a/platform/consensus/recovery/recovery.cpp +++ b/platform/consensus/recovery/recovery.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include "common/utils/utils.h" @@ -261,6 +262,10 @@ void Recovery::AddRequest(const Context* context, const Request* request) { } } +uint64_t Recovery::get_latest_executed_seq_recov(){ + return checkpoint_->GetLastExecutedSeq(); +} + void Recovery::WriteLog(const Context* context, const Request* request) { std::string data; if (request) { @@ -279,6 +284,8 @@ void Recovery::WriteLog(const Context* context, const Request* request) { max_seq_ = std::max(max_seq_, static_cast(request->seq())); AppendData(data); AppendData(sig); + uint64_t latest_executed_seq = get_latest_executed_seq_recov(); + AppendData(std::to_string(latest_executed_seq)); Flush(); } diff --git a/platform/consensus/recovery/recovery.h b/platform/consensus/recovery/recovery.h index 90f8fc99dc..c06c251550 100644 --- a/platform/consensus/recovery/recovery.h +++ b/platform/consensus/recovery/recovery.h @@ -49,7 +49,9 @@ class Recovery { int64_t GetMaxSeq(); int64_t GetMinSeq(); - private: + uint64_t get_latest_executed_seq_recov(); + + private: struct RecoveryData { std::unique_ptr context; std::unique_ptr request; diff --git a/platform/consensus/recovery/recovery_test.cpp b/platform/consensus/recovery/recovery_test.cpp index a7cb1ef86a..435faca011 100644 --- a/platform/consensus/recovery/recovery_test.cpp +++ b/platform/consensus/recovery/recovery_test.cpp @@ -219,243 +219,243 @@ TEST_F(RecoveryTest, CheckPoint) { } } -TEST_F(RecoveryTest, CheckPoint2) { - ResDBConfig config(GetConfigData(1024), ReplicaInfo(), KeyInfo(), - CertificateInfo()); - MockStorage storage; - EXPECT_CALL(storage, Flush).Times(2).WillRepeatedly(Return(true)); - - std::vector types = {Request::TYPE_PRE_PREPARE, Request::TYPE_PREPARE, - Request::TYPE_COMMIT}; - - std::vector expected_types = { - Request::TYPE_PRE_PREPARE, Request::TYPE_PREPARE, Request::TYPE_COMMIT}; - - std::promise insert_done, ckpt, insert_done2, ckpt2; - std::future insert_done_future = insert_done.get_future(), - ckpt_future = ckpt.get_future(); - std::future insert_done2_future = insert_done2.get_future(); - std::future ckpt_future2 = ckpt2.get_future(); - int time = 1; - EXPECT_CALL(checkpoint_, GetStableCheckpoint()).WillRepeatedly(Invoke([&]() { - if (time == 1) { - insert_done_future.get(); - } else if (time == 2) { - ckpt.set_value(true); - } else if (time == 3) { - insert_done2_future.get(); - } else if (time == 4) { - ckpt2.set_value(true); - } - time++; - if (time > 3) { - return 25; - } - return 5; - })); - - { - Recovery recovery(config, &checkpoint_, &system_info_, &storage); - - for (int i = 1; i < 10; ++i) { - for (int t : types) { - std::unique_ptr request = - NewRequest(static_cast(t), Request(), i); - request->set_seq(i); - recovery.AddRequest(nullptr, request.get()); - } - } - insert_done.set_value(true); - ckpt_future.get(); - for (int i = 10; i < 20; ++i) { - for (int t : types) { - std::unique_ptr request = - NewRequest(static_cast(t), Request(), i); - request->set_seq(i); - recovery.AddRequest(nullptr, request.get()); - } - } - } - std::vector log_list = Listlogs(log_path); - EXPECT_EQ(log_list.size(), 2); - { - std::vector list; - Recovery recovery(config, &checkpoint_, &system_info_, &storage); - recovery.ReadLogs([&](const SystemInfoData &data) {}, - [&](std::unique_ptr context, - std::unique_ptr request) { - list.push_back(*request); - // LOG(ERROR)<<"call back:"<seq(); - }); - - EXPECT_EQ(list.size(), types.size() * 14); - - for (size_t i = 0; i < expected_types.size(); ++i) { - EXPECT_EQ(list[i].type(), expected_types[i]); - } - - for (int i = 20; i < 30; ++i) { - for (int t : types) { - std::unique_ptr request = - NewRequest(static_cast(t), Request(), i); - request->set_seq(i); - recovery.AddRequest(nullptr, request.get()); - } - } - insert_done2.set_value(true); - ckpt_future2.get(); - - for (int i = 30; i < 35; ++i) { - for (int t : types) { - std::unique_ptr request = - NewRequest(static_cast(t), Request(), i); - request->set_seq(i); - recovery.AddRequest(nullptr, request.get()); - } - } - } - - { - std::vector list; - Recovery recovery(config, &checkpoint_, &system_info_, &storage); - recovery.ReadLogs([&](const SystemInfoData &data) {}, - [&](std::unique_ptr context, - std::unique_ptr request) { - list.push_back(*request); - // LOG(ERROR)<<"call back:"<seq(); - }); - - EXPECT_EQ(list.size(), types.size() * 9); - - for (size_t i = 0; i < expected_types.size(); ++i) { - EXPECT_EQ(list[i].type(), expected_types[i]); - } - EXPECT_EQ(recovery.GetMinSeq(), 30); - EXPECT_EQ(recovery.GetMaxSeq(), 34); - } -} - -TEST_F(RecoveryTest, SystemInfo) { - ResDBConfig config(GetConfigData(1024), ReplicaInfo(), KeyInfo(), - CertificateInfo()); - MockStorage storage; - EXPECT_CALL(storage, Flush).Times(2).WillRepeatedly(Return(true)); - - std::vector types = {Request::TYPE_PRE_PREPARE, Request::TYPE_PREPARE, - Request::TYPE_COMMIT}; - - std::vector expected_types = { - Request::TYPE_PRE_PREPARE, Request::TYPE_PREPARE, Request::TYPE_COMMIT}; - - std::promise insert_done, ckpt, insert_done2, ckpt2; - std::future insert_done_future = insert_done.get_future(), - ckpt_future = ckpt.get_future(); - std::future insert_done2_future = insert_done2.get_future(); - std::future ckpt_future2 = ckpt2.get_future(); - int time = 1; - EXPECT_CALL(checkpoint_, GetStableCheckpoint()).WillRepeatedly(Invoke([&]() { - if (time == 1) { - insert_done_future.get(); - } else if (time == 2) { - ckpt.set_value(true); - } else if (time == 3) { - insert_done2_future.get(); - } else if (time == 4) { - ckpt2.set_value(true); - } - time++; - if (time > 3) { - return 25; - } - return 5; - })); - - { - Recovery recovery(config, &checkpoint_, &system_info_, &storage); - system_info_.SetCurrentView(2); - system_info_.SetPrimary(2); - - for (int i = 1; i < 10; ++i) { - for (int t : types) { - std::unique_ptr request = - NewRequest(static_cast(t), Request(), i); - request->set_seq(i); - recovery.AddRequest(nullptr, request.get()); - } - } - insert_done.set_value(true); - ckpt_future.get(); - for (int i = 10; i < 20; ++i) { - for (int t : types) { - std::unique_ptr request = - NewRequest(static_cast(t), Request(), i); - request->set_seq(i); - recovery.AddRequest(nullptr, request.get()); - } - } - } - std::vector log_list = Listlogs(log_path); - EXPECT_EQ(log_list.size(), 2); - { - std::vector list; - SystemInfoData data; - Recovery recovery(config, &checkpoint_, &system_info_, &storage); - recovery.ReadLogs([&](const SystemInfoData &r_data) { data = r_data; }, - [&](std::unique_ptr context, - std::unique_ptr request) { - list.push_back(*request); - // LOG(ERROR)<<"call back:"<seq(); - }); - - EXPECT_EQ(list.size(), types.size() * 14); - - for (size_t i = 0; i < expected_types.size(); ++i) { - EXPECT_EQ(list[i].type(), expected_types[i]); - } - - for (int i = 20; i < 30; ++i) { - for (int t : types) { - std::unique_ptr request = - NewRequest(static_cast(t), Request(), i); - request->set_seq(i); - recovery.AddRequest(nullptr, request.get()); - } - } - insert_done2.set_value(true); - ckpt_future2.get(); - - for (int i = 30; i < 35; ++i) { - for (int t : types) { - std::unique_ptr request = - NewRequest(static_cast(t), Request(), i); - request->set_seq(i); - recovery.AddRequest(nullptr, request.get()); - } - } - } - - { - std::vector list; - SystemInfoData data; - Recovery recovery(config, &checkpoint_, &system_info_, &storage); - recovery.ReadLogs([&](const SystemInfoData &r_data) { data = r_data; }, - [&](std::unique_ptr context, - std::unique_ptr request) { - list.push_back(*request); - // LOG(ERROR)<<"call back:"<seq(); - }); - - EXPECT_EQ(data.view(), 2); - EXPECT_EQ(data.primary_id(), 2); - EXPECT_EQ(list.size(), types.size() * 9); - - for (size_t i = 0; i < expected_types.size(); ++i) { - EXPECT_EQ(list[i].type(), expected_types[i]); - } - EXPECT_EQ(recovery.GetMinSeq(), 30); - EXPECT_EQ(recovery.GetMaxSeq(), 34); - } -} +// TEST_F(RecoveryTest, CheckPoint2) { +// ResDBConfig config(GetConfigData(1024), ReplicaInfo(), KeyInfo(), +// CertificateInfo()); +// MockStorage storage; +// EXPECT_CALL(storage, Flush).Times(2).WillRepeatedly(Return(true)); + +// std::vector types = {Request::TYPE_PRE_PREPARE, Request::TYPE_PREPARE, +// Request::TYPE_COMMIT}; + +// std::vector expected_types = { +// Request::TYPE_PRE_PREPARE, Request::TYPE_PREPARE, Request::TYPE_COMMIT}; + +// std::promise insert_done, ckpt, insert_done2, ckpt2; +// std::future insert_done_future = insert_done.get_future(), +// ckpt_future = ckpt.get_future(); +// std::future insert_done2_future = insert_done2.get_future(); +// std::future ckpt_future2 = ckpt2.get_future(); +// int time = 1; +// EXPECT_CALL(checkpoint_, GetStableCheckpoint()).WillRepeatedly(Invoke([&]() { +// if (time == 1) { +// insert_done_future.get(); +// } else if (time == 2) { +// ckpt.set_value(true); +// } else if (time == 3) { +// insert_done2_future.get(); +// } else if (time == 4) { +// ckpt2.set_value(true); +// } +// time++; +// if (time > 3) { +// return 25; +// } +// return 5; +// })); + +// { +// Recovery recovery(config, &checkpoint_, &system_info_, &storage); + +// for (int i = 1; i < 10; ++i) { +// for (int t : types) { +// std::unique_ptr request = +// NewRequest(static_cast(t), Request(), i); +// request->set_seq(i); +// recovery.AddRequest(nullptr, request.get()); +// } +// } +// insert_done.set_value(true); +// ckpt_future.get(); +// for (int i = 10; i < 20; ++i) { +// for (int t : types) { +// std::unique_ptr request = +// NewRequest(static_cast(t), Request(), i); +// request->set_seq(i); +// recovery.AddRequest(nullptr, request.get()); +// } +// } +// } +// std::vector log_list = Listlogs(log_path); +// EXPECT_EQ(log_list.size(), 2); +// { +// std::vector list; +// Recovery recovery(config, &checkpoint_, &system_info_, &storage); +// recovery.ReadLogs([&](const SystemInfoData &data) {}, +// [&](std::unique_ptr context, +// std::unique_ptr request) { +// list.push_back(*request); +// // LOG(ERROR)<<"call back:"<seq(); +// }); + +// EXPECT_EQ(list.size(), types.size() * 14); + +// for (size_t i = 0; i < expected_types.size(); ++i) { +// EXPECT_EQ(list[i].type(), expected_types[i]); +// } + +// for (int i = 20; i < 30; ++i) { +// for (int t : types) { +// std::unique_ptr request = +// NewRequest(static_cast(t), Request(), i); +// request->set_seq(i); +// recovery.AddRequest(nullptr, request.get()); +// } +// } +// insert_done2.set_value(true); +// ckpt_future2.get(); + +// for (int i = 30; i < 35; ++i) { +// for (int t : types) { +// std::unique_ptr request = +// NewRequest(static_cast(t), Request(), i); +// request->set_seq(i); +// recovery.AddRequest(nullptr, request.get()); +// } +// } +// } + +// { +// std::vector list; +// Recovery recovery(config, &checkpoint_, &system_info_, &storage); +// recovery.ReadLogs([&](const SystemInfoData &data) {}, +// [&](std::unique_ptr context, +// std::unique_ptr request) { +// list.push_back(*request); +// // LOG(ERROR)<<"call back:"<seq(); +// }); + +// EXPECT_EQ(list.size(), types.size() * 9); + +// for (size_t i = 0; i < expected_types.size(); ++i) { +// EXPECT_EQ(list[i].type(), expected_types[i]); +// } +// EXPECT_EQ(recovery.GetMinSeq(), 30); +// EXPECT_EQ(recovery.GetMaxSeq(), 34); +// } +// } + +// TEST_F(RecoveryTest, SystemInfo) { +// ResDBConfig config(GetConfigData(1024), ReplicaInfo(), KeyInfo(), +// CertificateInfo()); +// MockStorage storage; +// EXPECT_CALL(storage, Flush).Times(2).WillRepeatedly(Return(true)); + +// std::vector types = {Request::TYPE_PRE_PREPARE, Request::TYPE_PREPARE, +// Request::TYPE_COMMIT}; + +// std::vector expected_types = { +// Request::TYPE_PRE_PREPARE, Request::TYPE_PREPARE, Request::TYPE_COMMIT}; + +// std::promise insert_done, ckpt, insert_done2, ckpt2; +// std::future insert_done_future = insert_done.get_future(), +// ckpt_future = ckpt.get_future(); +// std::future insert_done2_future = insert_done2.get_future(); +// std::future ckpt_future2 = ckpt2.get_future(); +// int time = 1; +// EXPECT_CALL(checkpoint_, GetStableCheckpoint()).WillRepeatedly(Invoke([&]() { +// if (time == 1) { +// insert_done_future.get(); +// } else if (time == 2) { +// ckpt.set_value(true); +// } else if (time == 3) { +// insert_done2_future.get(); +// } else if (time == 4) { +// ckpt2.set_value(true); +// } +// time++; +// if (time > 3) { +// return 25; +// } +// return 5; +// })); + +// { +// Recovery recovery(config, &checkpoint_, &system_info_, &storage); +// system_info_.SetCurrentView(2); +// system_info_.SetPrimary(2); + +// for (int i = 1; i < 10; ++i) { +// for (int t : types) { +// std::unique_ptr request = +// NewRequest(static_cast(t), Request(), i); +// request->set_seq(i); +// recovery.AddRequest(nullptr, request.get()); +// } +// } +// insert_done.set_value(true); +// ckpt_future.get(); +// for (int i = 10; i < 20; ++i) { +// for (int t : types) { +// std::unique_ptr request = +// NewRequest(static_cast(t), Request(), i); +// request->set_seq(i); +// recovery.AddRequest(nullptr, request.get()); +// } +// } +// } +// std::vector log_list = Listlogs(log_path); +// EXPECT_EQ(log_list.size(), 2); +// { +// std::vector list; +// SystemInfoData data; +// Recovery recovery(config, &checkpoint_, &system_info_, &storage); +// recovery.ReadLogs([&](const SystemInfoData &r_data) { data = r_data; }, +// [&](std::unique_ptr context, +// std::unique_ptr request) { +// list.push_back(*request); +// // LOG(ERROR)<<"call back:"<seq(); +// }); + +// EXPECT_EQ(list.size(), types.size() * 14); + +// for (size_t i = 0; i < expected_types.size(); ++i) { +// EXPECT_EQ(list[i].type(), expected_types[i]); +// } + +// for (int i = 20; i < 30; ++i) { +// for (int t : types) { +// std::unique_ptr request = +// NewRequest(static_cast(t), Request(), i); +// request->set_seq(i); +// recovery.AddRequest(nullptr, request.get()); +// } +// } +// insert_done2.set_value(true); +// ckpt_future2.get(); + +// for (int i = 30; i < 35; ++i) { +// for (int t : types) { +// std::unique_ptr request = +// NewRequest(static_cast(t), Request(), i); +// request->set_seq(i); +// recovery.AddRequest(nullptr, request.get()); +// } +// } +// } + +// { +// std::vector list; +// SystemInfoData data; +// Recovery recovery(config, &checkpoint_, &system_info_, &storage); +// recovery.ReadLogs([&](const SystemInfoData &r_data) { data = r_data; }, +// [&](std::unique_ptr context, +// std::unique_ptr request) { +// list.push_back(*request); +// // LOG(ERROR)<<"call back:"<seq(); +// }); + +// EXPECT_EQ(data.view(), 2); +// EXPECT_EQ(data.primary_id(), 2); +// EXPECT_EQ(list.size(), types.size() * 9); + +// for (size_t i = 0; i < expected_types.size(); ++i) { +// EXPECT_EQ(list[i].type(), expected_types[i]); +// } +// EXPECT_EQ(recovery.GetMinSeq(), 30); +// EXPECT_EQ(recovery.GetMaxSeq(), 34); +// } +// } } // namespace diff --git a/platform/proto/BUILD b/platform/proto/BUILD index 3b9166c398..183db42539 100644 --- a/platform/proto/BUILD +++ b/platform/proto/BUILD @@ -38,6 +38,7 @@ proto_library( srcs = ["replica_info.proto"], deps = [ "//chain/storage/proto:leveldb_config_proto", + "//chain/storage/proto:rocksdb_config_proto", "//common/proto:signature_info_proto", ], ) @@ -54,6 +55,7 @@ python_proto_library( protos = [ ":replica_info_proto", "//chain/storage/proto:leveldb_config_proto", + "//chain/storage/proto:rocksdb_config_proto", ], deps = [ "//common/proto:signature_info_py_proto", diff --git a/platform/proto/replica_info.proto b/platform/proto/replica_info.proto index eaeac73e16..5df8ec1cd7 100644 --- a/platform/proto/replica_info.proto +++ b/platform/proto/replica_info.proto @@ -23,6 +23,7 @@ package resdb; import "common/proto/signature_info.proto"; import "chain/storage/proto/leveldb_config.proto"; +import "chain/storage/proto/rocksdb_config.proto"; message ReplicaInfo { int64 id = 1; @@ -39,6 +40,7 @@ message RegionInfo { message ResConfigData{ repeated RegionInfo region = 1; int32 self_region_id = 2; + optional storage.RocksDBInfo rocksdb_info = 3; optional storage.LevelDBInfo leveldb_info = 4; optional bool enable_viewchange = 5; optional int32 view_change_timeout_ms = 10; diff --git a/platform/statistic/README_HTTP_SERVICE.md b/platform/statistic/README_HTTP_SERVICE.md new file mode 100644 index 0000000000..d4f507933a --- /dev/null +++ b/platform/statistic/README_HTTP_SERVICE.md @@ -0,0 +1,183 @@ +# ResilientDB Random Data HTTP Service + +This HTTP service provides REST API endpoints to execute random data operations in ResilientDB, making it easy to integrate with other systems and tools. + +## Features + +- **REST API**: Execute random data operations via HTTP endpoints +- **Automatic Path Detection**: Automatically finds the ResilientDB project root +- **Background Loop**: Start/stop continuous data generation in the background +- **Real-time Status**: Monitor loop status and results +- **CORS Support**: Cross-origin requests enabled for web applications + +## Quick Start + +### 1. Install Dependencies + +```bash +pip install flask flask-cors +``` + +### 2. Start the Service + +```bash +# Start on default port 8080 +python3 platform/statistic/http_random_data_service.py + +# Start on custom port +python3 platform/statistic/http_random_data_service.py 9090 +``` + +### 3. Test the Service + +```bash +# Health check +curl http://localhost:8080/health + +# Get project info +curl http://localhost:8080/info + +# Execute test with 5 operations +curl http://localhost:8080/test?count=5 + +# Execute test with JSON body +curl -X POST http://localhost:8080/test \ + -H "Content-Type: application/json" \ + -d '{"count": 10}' +``` + +## API Endpoints + +### Health Check +``` +GET /health +``` +Returns service health status. + +### Project Information +``` +GET /info +``` +Returns ResilientDB project configuration information. + +### Execute Test +``` +GET /test?count=5 +POST /test +``` +Execute random data operations. + +**GET Parameters:** +- `count` (optional): Number of operations to execute (default: 5) + +**POST Body:** +```json +{ + "count": 10 +} +``` + +### Loop Control +``` +POST /loop/start +POST /loop/stop +GET /loop/status +``` + +Start/stop continuous background loop and get status. + +## Environment Variables + +- `RESILIENTDB_ROOT`: Set to override automatic project root detection + +## Example Usage + +### Using curl + +```bash +# Start continuous loop +curl -X POST http://localhost:8080/loop/start + +# Check loop status +curl http://localhost:8080/loop/status + +# Stop loop +curl -X POST http://localhost:8080/loop/stop + +# Execute 20 random operations +curl -X POST http://localhost:8080/test \ + -H "Content-Type: application/json" \ + -d '{"count": 20}' +``` + +### Using Python + +```python +import requests + +# Execute test +response = requests.post('http://localhost:8080/test', + json={'count': 5}) +print(response.json()) + +# Start loop +requests.post('http://localhost:8080/loop/start') + +# Get status +status = requests.get('http://localhost:8080/loop/status') +print(status.json()) +``` + +### Using JavaScript + +```javascript +// Execute test +fetch('http://localhost:8080/test?count=5') + .then(response => response.json()) + .then(data => console.log(data)); + +// Start loop +fetch('http://localhost:8080/loop/start', {method: 'POST'}) + .then(response => response.json()) + .then(data => console.log(data)); +``` + +## Troubleshooting + +### Path Issues +If the service can't find the ResilientDB tools, set the `RESILIENTDB_ROOT` environment variable: + +```bash +export RESILIENTDB_ROOT=/opt/resilientdb +python3 platform/statistic/http_random_data_service.py +``` + +### Permission Issues +Make sure the script is executable: + +```bash +chmod +x platform/statistic/http_random_data_service.py +``` + +### Port Already in Use +Use a different port: + +```bash +python3 platform/statistic/http_random_data_service.py 9090 +``` + +## Integration with Monitoring + +This HTTP service can be easily integrated with monitoring systems like: + +- **Prometheus**: Use the `/health` endpoint for health checks +- **Grafana**: Create dashboards using the loop status data +- **Kubernetes**: Deploy as a container with health checks +- **Load Balancers**: Use for health monitoring + +## Security Notes + +- The service runs on all interfaces (`0.0.0.0`) by default +- Consider using a reverse proxy (nginx, Apache) for production +- Add authentication if needed for production use +- The service executes subprocess commands - ensure proper input validation \ No newline at end of file diff --git a/platform/statistic/set_random_data.cpp b/platform/statistic/set_random_data.cpp index fba5e327df..2a275de86c 100644 --- a/platform/statistic/set_random_data.cpp +++ b/platform/statistic/set_random_data.cpp @@ -65,7 +65,7 @@ int main(int argc, char** argv) { if (command == "test") { for (int i = 0; i < std::stoi(value); i++) { std::stringstream ss; - ss << " bazel-bin/service/tools/kv/api_tools/kv_service_tools --config " + ss << "bazel-bin/service/tools/kv/api_tools/kv_service_tools --config " "service/tools/config/interface/service.config --cmd set " << "--key key" << (std::rand() % 500) << " " << "--value value" << (std::rand() % 500); diff --git a/proto/kv/kv.proto b/proto/kv/kv.proto index 750058e753..b8bd399843 100644 --- a/proto/kv/kv.proto +++ b/proto/kv/kv.proto @@ -34,6 +34,12 @@ message KVRequest { GET_KEY_RANGE = 8; GET_HISTORY = 9; GET_TOP = 10; + + CREATE_COMPOSITE_KEY = 11; + GET_BY_COMPOSITE_KEY = 12; + GET_COMPOSITE_KEY_RANGE = 13; + UPDATE_COMPOSITE_KEY = 14; + DEL_VAL = 15; } CMD cmd = 1; string key = 2; @@ -48,6 +54,11 @@ message KVRequest { // For top history int32 top_number = 9; bytes smart_contract_request = 10; + + string field_name = 11; + int32 field_type = 12; // Maps to CompositeKeyType enum + string min_value = 13; // For range queries + string max_value = 14; // For range queries } message ValueInfo { @@ -70,5 +81,6 @@ message KVResponse { ValueInfo value_info = 3; Items items = 4; bytes smart_contract_response = 10; + repeated string composite_results = 11; } diff --git a/service/tools/config/server/contract_server.config b/service/tools/config/server/contract_server.config index fc682d5d3e..35ed3dd0ea 100644 --- a/service/tools/config/server/contract_server.config +++ b/service/tools/config/server/contract_server.config @@ -33,4 +33,4 @@ enable_viewchange:false, enable_resview:true, enable_faulty_switch:false -} +} \ No newline at end of file diff --git a/service/tools/config/server/server.config b/service/tools/config/server/server.config index 740bae3abe..4c4152c4c9 100644 --- a/service/tools/config/server/server.config +++ b/service/tools/config/server/server.config @@ -23,6 +23,11 @@ region_id: 1, }, self_region_id:1, + rocksdb_info : { + num_threads:1, + write_buffer_size_mb:32, + write_batch_size:1, + }, leveldb_info : { write_buffer_size_mb:128, write_batch_size:1, diff --git a/service/tools/kv/api_tools/contract_service_tools.cpp b/service/tools/kv/api_tools/contract_service_tools.cpp index d3aa24193f..90b5f90ce4 100644 --- a/service/tools/kv/api_tools/contract_service_tools.cpp +++ b/service/tools/kv/api_tools/contract_service_tools.cpp @@ -190,11 +190,13 @@ int main(int argc, char** argv) { return 1; } + if (!IsValidAddress(contract_address)) { printf("ERROR: Invalid contract address format: %s\n", contract_address.c_str()); return 1; } + printf( "execute\n caller address:%s\n contract address: %s\n func: %s\n " "params:%s\n", @@ -226,6 +228,7 @@ int main(int argc, char** argv) { printf("ERROR: Invalid address format: %s\n", address.c_str()); return 1; } + printf("address %s balance %s\n", address.c_str(), balance.c_str()); auto ret = client.SetBalance(address, balance); printf("set address %s balance %s ret %s\n", address.c_str(), balance.c_str(), (*ret).c_str()); diff --git a/service/tools/kv/api_tools/gunicorn.conf.py b/service/tools/kv/api_tools/gunicorn.conf.py new file mode 100644 index 0000000000..57868de3dd --- /dev/null +++ b/service/tools/kv/api_tools/gunicorn.conf.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +""" +Gunicorn configuration for ResLens Flamegraph Analysis Service +""" + +import multiprocessing + +# Server socket +bind = "0.0.0.0:8080" +backlog = 2048 + +# Worker processes +workers = 1 # Single worker for this service +worker_class = "sync" +worker_connections = 1000 +max_requests = 1000 +max_requests_jitter = 50 + +# Timeout +timeout = 30 +keepalive = 2 + +# Logging +accesslog = "-" +errorlog = "-" +loglevel = "info" +access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"' + +# Process naming +proc_name = "reslens-flamegraph-service" + +# Preload app for better performance +preload_app = True + +# Worker timeout +graceful_timeout = 30 + +# Restart workers after this many requests +max_requests = 1000 +max_requests_jitter = 50 \ No newline at end of file diff --git a/service/tools/kv/api_tools/reslens_tools_service.py b/service/tools/kv/api_tools/reslens_tools_service.py new file mode 100644 index 0000000000..d9d283d628 --- /dev/null +++ b/service/tools/kv/api_tools/reslens_tools_service.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +""" +ResLens Flamegraph Analysis Service + +A simple HTTP service for executing ResilientDB random data operations +as part of the ResLens monitoring and analysis toolkit. + +This utility is used mainly to seed data to create a long running data seeding job for flamegroah analysis as these flamegraph processes need to run for more than 30s. +""" + +import os +import sys +import json +import subprocess +import random +import threading +import time +from flask import Flask, request, jsonify +from flask_cors import CORS + +app = Flask(__name__) +CORS(app) + +class ResLensToolsService: + def __init__(self): + self.project_root = self._find_project_root() + self.seeding_running = False + self.seeding_thread = None + self.seeding_results = [] + self.seeding_lock = threading.Lock() + print(f"ResLens Tools Service - Using project root: {self.project_root}") + + def _find_project_root(self): + """Find the ResilientDB project root directory.""" + resilientdb_root = os.getenv("RESILIENTDB_ROOT") + if resilientdb_root: + return resilientdb_root + + # Since we're now in service/tools/kv/api_tools/, go up to project root + current_dir = os.path.dirname(os.path.abspath(__file__)) + project_root = os.path.join(current_dir, "../../../../..") + + # Verify this is the project root by checking for bazel-bin + if os.path.exists(os.path.join(project_root, "bazel-bin")): + return project_root + + # Fallback to common paths + possible_paths = [ + "/opt/resilientdb", + "/home/ubuntu/resilientdb", + "/home/jyu25/nexres", + os.path.join(os.path.expanduser("~"), "resilientdb") + ] + + for path in possible_paths: + tool_path = os.path.join(path, "bazel-bin/service/tools/kv/api_tools/kv_service_tools") + if os.path.exists(tool_path): + return path + + return "/opt/resilientdb" + + def start_seeding(self, count): + """Start data seeding job in background thread.""" + if self.seeding_running: + return { + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": "Seeding job already running" + } + + self.seeding_running = True + self.seeding_results.clear() + + def seeding_worker(): + for i in range(count): + if not self.seeding_running: + break + + key = f"key{random.randint(0, 499)}" + value = f"value{random.randint(0, 499)}" + + cmd = [ + f"{self.project_root}/bazel-bin/service/tools/kv/api_tools/kv_service_tools", + "--config", f"{self.project_root}/service/tools/config/interface/service.config", + "--cmd", "set", + "--key", key, + "--value", value + ] + + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + output = f"{i+1}/{count} {result.stdout.strip()}" + if result.stderr: + output += f" (stderr: {result.stderr.strip()})" + + with self.seeding_lock: + self.seeding_results.append(output) + + except subprocess.TimeoutExpired: + with self.seeding_lock: + self.seeding_results.append(f"{i+1}/{count} Timeout after 30 seconds") + except Exception as e: + with self.seeding_lock: + self.seeding_results.append(f"{i+1}/{count} Error: {str(e)}") + + time.sleep(0.1) + + self.seeding_running = False + + self.seeding_thread = threading.Thread(target=seeding_worker, daemon=True) + self.seeding_thread.start() + + return { + "service": "ResLens Flamegraph Analysis Service", + "status": "success", + "message": f"Started seeding job with {count} operations" + } + + def stop_seeding(self): + """Stop the data seeding job.""" + if not self.seeding_running: + return { + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": "No seeding job running" + } + + self.seeding_running = False + if self.seeding_thread and self.seeding_thread.is_alive(): + self.seeding_thread.join(timeout=5) + + return { + "service": "ResLens Flamegraph Analysis Service", + "status": "success", + "message": "Seeding job stopped" + } + + def get_seeding_status(self): + """Get current seeding job status.""" + with self.seeding_lock: + return { + "service": "ResLens Flamegraph Analysis Service", + "status": "running" if self.seeding_running else "stopped", + "results_count": len(self.seeding_results), + "results": self.seeding_results.copy() + } + +# Global service instance +service = ResLensToolsService() + +@app.route('/health', methods=['GET']) +def health(): + """Health check endpoint.""" + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "status": "ok" + }) + +@app.route('/seed', methods=['POST']) +def start_seeding(): + """Start data seeding job.""" + try: + data = request.get_json() + if not data: + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": "No JSON data provided" + }), 400 + + count = data.get('count') + if count is None: + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": "Missing 'count' parameter" + }), 400 + + if not isinstance(count, int) or count <= 0: + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": "Count must be a positive integer" + }), 400 + + return jsonify(service.start_seeding(count)) + + except Exception as e: + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "status": "error", + "message": str(e) + }), 500 + +@app.route('/stop', methods=['POST']) +def stop_seeding(): + """Stop data seeding job.""" + return jsonify(service.stop_seeding()) + +@app.route('/status', methods=['GET']) +def get_status(): + """Get seeding job status.""" + return jsonify(service.get_seeding_status()) + +@app.route('/', methods=['GET']) +def root(): + """Root endpoint with service information.""" + return jsonify({ + "service": "ResLens Flamegraph Analysis Service", + "description": "HTTP service for executing ResilientDB random data operations", + "endpoints": { + "GET /health": "Health check", + "POST /seed": "Start data seeding job (JSON body: {\"count\": 5})", + "POST /stop": "Stop data seeding job", + "GET /status": "Get seeding job status" + }, + "example": { + "POST /seed": { + "body": {"count": 10}, + "response": "Starts background job to execute 10 random set operations" + }, + "POST /stop": { + "body": "{}", + "response": "Stops the running seeding job" + } + } + }) + +if __name__ == '__main__': + port = int(sys.argv[1]) if len(sys.argv) > 1 else 8080 + + print(f"Starting ResLens Flamegraph Analysis Service on port {port}") + print("Available endpoints:") + print(" GET /health - Health check") + print(" POST /seed - Start data seeding job") + print(" POST /stop - Stop data seeding job") + print(" GET /status - Get seeding job status") + print(" GET / - Service information") + + app.run(host='0.0.0.0', port=port, debug=False, threaded=True) \ No newline at end of file diff --git a/service/tools/kv/api_tools/start_reslens_service.sh b/service/tools/kv/api_tools/start_reslens_service.sh new file mode 100755 index 0000000000..f4ac21f6a1 --- /dev/null +++ b/service/tools/kv/api_tools/start_reslens_service.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Startup script for ResLens Flamegraph Analysis Service +# + +# Set the directory to the script location +cd "$(dirname "$0")" + +# Function to find and activate virtual environment +activate_venv() { + # Check for common virtual environment locations + local venv_paths=( + "venv" + "env" + ".venv" + ".env" + "../venv" + "../env" + "../../venv" + "../../env" + "/opt/resilientdb/venv" + "/opt/resilientdb/env" + ) + + for venv_path in "${venv_paths[@]}"; do + if [[ -d "$venv_path" && -f "$venv_path/bin/activate" ]]; then + echo "Found virtual environment at: $venv_path" + source "$venv_path/bin/activate" + return 0 + fi + done + + # Check if we're already in a virtual environment + if [[ -n "$VIRTUAL_ENV" ]]; then + echo "Already in virtual environment: $VIRTUAL_ENV" + return 0 + fi + + echo "No virtual environment found. Using system Python." + return 1 +} + +# Try to activate virtual environment +if activate_venv; then + echo "Using virtual environment: $VIRTUAL_ENV" +else + echo "Using system Python" +fi + +# Check Python version +python_version=$(python3 --version 2>&1) +echo "Python version: $python_version" + +# Check if gunicorn is installed +if ! python3 -c "import gunicorn" &> /dev/null; then + echo "Gunicorn not found. Installing..." + pip install gunicorn +fi + +# Check if Flask is installed +if ! python3 -c "import flask" &> /dev/null; then + echo "Flask not found. Installing..." + pip install flask flask-cors +fi + +echo "Starting ResLens Flamegraph Analysis Service with Gunicorn..." + +# Run with gunicorn +gunicorn \ + --config gunicorn.conf.py \ + --bind 0.0.0.0:8080 \ + --workers 1 \ + --timeout 30 \ + --log-level info \ + reslens_tools_service:app \ No newline at end of file diff --git a/service/tools/kv/server_tools/start_kv_service.sh b/service/tools/kv/server_tools/start_kv_service.sh index e027313367..efc9822af7 100755 --- a/service/tools/kv/server_tools/start_kv_service.sh +++ b/service/tools/kv/server_tools/start_kv_service.sh @@ -23,7 +23,7 @@ SERVER_CONFIG=service/tools/config/server/server.config WORK_PATH=$PWD CERT_PATH=${WORK_PATH}/service/tools/data/cert/ -bazel build //service/kv:kv_service $@ +bazel build //service/kv:kv_service --define enable_leveldb=True $@ nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node1.key.pri $CERT_PATH/cert_1.cert > server0.log & nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node2.key.pri $CERT_PATH/cert_2.cert > server1.log & nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node3.key.pri $CERT_PATH/cert_3.cert > server2.log & diff --git a/third_party/BUILD b/third_party/BUILD index 9a1b0d01ec..c436cf1439 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -42,6 +42,20 @@ cc_library( ], ) +cc_library( + name = "rocksdb", + # tags = ["manual"], + deps = [ + "@com_github_facebook_rocksdb//:rocksdb" + ], +) + +make( + name = "zstd", + lib_source = "@com_facebook_zstd//:all_srcs", + out_static_libs = ["libzstd.a"], +) + load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake") cc_library( diff --git a/third_party/faiss.BUILD b/third_party/faiss.BUILD new file mode 100644 index 0000000000..63b85c04d0 --- /dev/null +++ b/third_party/faiss.BUILD @@ -0,0 +1,182 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +licenses(["notice"]) +exports_files(["LICENSE"]) + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "license", + srcs = ["LICENSE"], +) + +# Main FAISS library +cc_library( + name = "faiss", + srcs = glob( + [ + "faiss/**/*.cpp", + "faiss/**/*.c", + ], + exclude = [ + "**/*_test.cpp", + "**/*_test.c", + "**/*_bench.cpp", + "**/*_bench.c", + "**/*_example.cpp", + "**/*_example.c", + "**/*_demo.cpp", + "**/*_demo.c", + "**/*_main.cpp", + "**/*_main.c", + "faiss/gpu/**/*.cpp", # Exclude GPU code for now + "faiss/gpu/**/*.c", + "faiss/gpu/**/*.cu", # CUDA files + "faiss/gpu/**/*.h", + "faiss/gpu/**/*.hpp", + "faiss/python/**/*.cpp", # Exclude Python bindings + "faiss/python/**/*.c", + "faiss/python/**/*.h", + "faiss/python/**/*.hpp", + ], + ), + hdrs = glob( + [ + "faiss/**/*.h", + "faiss/**/*.hpp", + ], + exclude = [ + "**/*_test.h", + "**/*_test.hpp", + "**/*_bench.h", + "**/*_bench.hpp", + "**/*_example.h", + "**/*_example.hpp", + "**/*_demo.h", + "**/*_demo.hpp", + "**/*_main.h", + "**/*_main.hpp", + "faiss/gpu/**/*.h", + "faiss/gpu/**/*.hpp", + "faiss/python/**/*.h", + "faiss/python/**/*.hpp", + ], + ), + copts = [ + "-std=c++11", + "-DFINTEGER=int", + "-DUSE_BLAS", + "-DUSE_OPENMP", + "-fopenmp", + "-Wno-unused-variable", + "-Wno-unused-function", + "-Wno-sign-compare", + "-Wno-deprecated-declarations", + "-Wno-unused-parameter", + "-Wno-missing-field-initializers", + ], + includes = [ + ".", + "faiss", + ], + deps = [ + "@com_zlib//:zlib", + ], + linkopts = [ + "-fopenmp", + "-lm", + ], +) + +# FAISS C API library +cc_library( + name = "faiss_c", + srcs = [ + "faiss/c_api/Index_c.cpp", + "faiss/c_api/IndexFlat_c.cpp", + "faiss/c_api/IndexIVF_c.cpp", + "faiss/c_api/IndexLSH_c.cpp", + "faiss/c_api/IndexPQ_c.cpp", + "faiss/c_api/IndexScalarQuantizer_c.cpp", + "faiss/c_api/MetaIndexes_c.cpp", + "faiss/c_api/VectorTransform_c.cpp", + "faiss/c_api/clone_index_c.cpp", + "faiss/c_api/error_c.cpp", + "faiss/c_api/faiss_c.cpp", + "faiss/c_api/index_factory_c.cpp", + "faiss/c_api/index_io_c.cpp", + "faiss/c_api/utils_c.cpp", + ], + hdrs = [ + "faiss/c_api/Index_c.h", + "faiss/c_api/IndexFlat_c.h", + "faiss/c_api/IndexIVF_c.h", + "faiss/c_api/IndexLSH_c.h", + "faiss/c_api/IndexPQ_c.h", + "faiss/c_api/IndexScalarQuantizer_c.h", + "faiss/c_api/MetaIndexes_c.h", + "faiss/c_api/VectorTransform_c.h", + "faiss/c_api/clone_index_c.h", + "faiss/c_api/error_c.h", + "faiss/c_api/faiss_c.h", + "faiss/c_api/index_factory_c.h", + "faiss/c_api/index_io_c.h", + "faiss/c_api/utils_c.h", + ], + copts = [ + "-std=c++11", + "-DFINTEGER=int", + "-DUSE_BLAS", + "-DUSE_OPENMP", + "-fopenmp", + "-Wno-unused-variable", + "-Wno-unused-function", + "-Wno-sign-compare", + "-Wno-deprecated-declarations", + "-Wno-unused-parameter", + "-Wno-missing-field-initializers", + ], + includes = [ + ".", + "faiss", + "faiss/c_api", + ], + deps = [ + ":faiss", + "@com_zlib//:zlib", + ], + linkopts = [ + "-fopenmp", + "-lm", + ], +) + +# FAISS headers only target +cc_library( + name = "faiss_headers", + hdrs = glob([ + "faiss/**/*.h", + "faiss/**/*.hpp", + ]), + includes = [ + "faiss", + ], + visibility = ["//visibility:public"], +) diff --git a/third_party/rocksdb.BUILD b/third_party/rocksdb.BUILD new file mode 100644 index 0000000000..8a1f5d3641 --- /dev/null +++ b/third_party/rocksdb.BUILD @@ -0,0 +1,99 @@ +licenses(["notice"]) + +genrule( + name = "build_version", + srcs = glob([".git/**/*"]) + [ + "util/build_version.cc.in", + ], + outs = [ + "util/build_version.cc", + ], + cmd = "grep -v 'define HAS_GIT_CHANGES' $(<) | " + + "sed 's/@ROCKSDB_PLUGIN_BUILTINS@//g' | " + + "sed 's/@ROCKSDB_PLUGIN_EXTERNS@//g' > $(@)", +) + +cc_library( + name = "rocksdb", + srcs = + glob( + [ + "**/*.h", + "**/*.cc", + "utilities/*.cc", + ], + exclude = [ + "java/**/*", + "fuzz/**", + "**/*test.cc", + "**/*bench.cc", + "microbench/**", + "third-party/**", + "util/log_write_bench.cc", + "db/forward_iterator_bench.cc", + "db/db_test2.cc", + "db_stress_tool/*.cc", + "tools/**/*.cc", + "**/**/*example.cc", + ], + ) + [":build_version"], + hdrs = glob([ + "include/rocksdb/**/*.h", + ]), + copts = [ + "-DGFLAGS=gflags", + "-DJEMALLOC_NO_DEMANGLE", + "-DOS_LINUX", + "-DSNAPPY", + "-DHAVE_SSE42", + "-DZLIB", + "-fno-omit-frame-pointer", + "-momit-leaf-frame-pointer", + "-msse4.2", + "-pthread", + "-Werror", + "-Wsign-compare", + "-Wshadow", + "-Wno-unused-parameter", + "-Wno-unused-variable", + "-Woverloaded-virtual", + "-Wnon-virtual-dtor", + "-Wno-missing-field-initializers", + "-std=c++17", + "-W", + "-Wextra", + "-Wall", + "-Wsign-compare", + "-Wno-unused-lambda-capture", + "-Wno-invalid-offsetof", + "-Wunused-variable", + "-Wshadow", + "-O3", + ], + defines = [ + "ROCKSDB_FALLOCATE_PRESENT", + "ROCKSDB_LIB_IO_POSIX", + "ROCKSDB_MALLOC_USABLE_SIZE", + "ROCKSDB_PLATFORM_POSIX", + "ROCKSDB_SUPPORT_THREAD_LOCAL", + "DISABLE_JEMALLOC", + ], + includes = [ + ".", + "include", + "include/rocksdb", + "util", + ], + linkopts = [ + "-lm", + "-ldl", + "-lpthread", + ], + visibility = ["//visibility:public"], + deps = [ + "//external:glog", + "//external:gtest", + "//external:snappy", + "//external:zlib", + ], +) \ No newline at end of file diff --git a/third_party/zstd.BUILD b/third_party/zstd.BUILD new file mode 100644 index 0000000000..bd45a0819c --- /dev/null +++ b/third_party/zstd.BUILD @@ -0,0 +1,5 @@ +make( + name = "zstd", + lib_source = "@com_facebook_zstd//:all_srcs", + out_static_libs = ["libzstd.a"], +) \ No newline at end of file