diff --git a/benchmark/storage/BUILD b/benchmark/storage/BUILD new file mode 100644 index 0000000000..16d2234ff4 --- /dev/null +++ b/benchmark/storage/BUILD @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +package(default_visibility = ["//visibility:private"]) + +cc_binary( + name = "composite_key_benchmark", + srcs = ["composite_key_benchmark.cpp"], + deps = [ + "//chain/storage:composite_key_codec", + "//chain/storage:leveldb", + ], +) diff --git a/benchmark/storage/composite_key_benchmark.cpp b/benchmark/storage/composite_key_benchmark.cpp new file mode 100644 index 0000000000..f20a729861 --- /dev/null +++ b/benchmark/storage/composite_key_benchmark.cpp @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Microbenchmark: prefix-scan via composite keys vs full scan + app-side +// filter. Both paths run against the same LevelDB instance. + +#include +#include +#include +#include +#include +#include + +#include "chain/storage/composite_key_codec.h" +#include "chain/storage/leveldb.h" + +using ::resdb::storage::EncodeCompositeKey; +using ::resdb::storage::EncodeCompositeKeyPrefix; +using ::resdb::storage::NewResLevelDB; +using ::resdb::Storage; +using Clock = std::chrono::high_resolution_clock; + +namespace { + +constexpr const char* kIndexName = "byCity"; +constexpr const char* kTargetCity = "Davis"; +constexpr const char* kOtherCity = "Other"; + +struct Row { + int n_records; + int selectivity_pct; + int matching; + double old_way_ms; + double new_way_ms; +}; + +// `matching` records get city="Davis"; the rest get city="Other". Every +// record also gets a composite-key index entry. +void SeedDatabase(Storage* storage, int n, int matching) { + for (int i = 0; i < n; ++i) { + std::string key = "user:" + std::to_string(i); + std::string city = (i < matching) ? kTargetCity : kOtherCity; + storage->SetValueWithVersion(key, city, 0); + storage->CreateCompositeKey(EncodeCompositeKey(kIndexName, {city}, key)); + } +} + +// OLD path: scan everything, filter in code. +int CountByScan(Storage* storage) { + auto all = storage->GetAllItems(); + int count = 0; + for (const auto& kv : all) { + if (kv.second.first == kTargetCity) ++count; + } + return count; +} + +// NEW path: prefix scan the byCity:Davis index. +int CountByPrefixScan(Storage* storage) { + return storage + ->GetByCompositeKeyPrefix( + EncodeCompositeKeyPrefix(kIndexName, {kTargetCity})) + .size(); +} + +template +double TimeMs(F&& f) { + auto start = Clock::now(); + f(); + auto end = Clock::now(); + return std::chrono::duration(end - start).count(); +} + +Row RunOne(int n, int selectivity_pct, int iters) { + int matching = (n * selectivity_pct) / 100; + + std::string path = "/tmp/composite_key_bench_n" + std::to_string(n) + "_s" + + std::to_string(selectivity_pct); + std::filesystem::remove_all(path); + auto storage = NewResLevelDB(path); + + SeedDatabase(storage.get(), n, matching); + + // Warmup pass. + int warm_a = CountByScan(storage.get()); + int warm_b = CountByPrefixScan(storage.get()); + if (warm_a != matching || warm_b != matching) { + std::cerr << "WARN: warmup counts disagree: scan=" << warm_a + << " prefix=" << warm_b << " expected=" << matching << "\n"; + } + + double old_total = 0, new_total = 0; + for (int i = 0; i < iters; ++i) { + old_total += TimeMs([&] { CountByScan(storage.get()); }); + new_total += TimeMs([&] { CountByPrefixScan(storage.get()); }); + } + + return Row{n, selectivity_pct, matching, old_total / iters, + new_total / iters}; +} + +void PrintTable(const std::vector& rows) { + std::cout + << "\n============================================================" + << "================\n"; + std::cout << "Composite Key Benchmark -- ResilientDB Storage Layer\n"; + std::cout << " OLD: GetAllItems() + filter in app code (expected O(N))\n"; + std::cout << " NEW: GetByCompositeKeyPrefix() (expected O(log N + M))\n"; + std::cout + << "============================================================" + << "================\n"; + std::cout << std::left << std::setw(10) << "Records" << std::setw(13) + << "Selectivity" << std::setw(11) << "Matching" << std::setw(13) + << "OLD (ms)" << std::setw(13) << "NEW (ms)" << "Speedup\n"; + std::cout + << "------------------------------------------------------------" + << "----------------\n"; + for (const auto& r : rows) { + double speedup = (r.new_way_ms > 0) ? r.old_way_ms / r.new_way_ms : 0; + std::cout << std::left << std::setw(10) << r.n_records << std::setw(13) + << (std::to_string(r.selectivity_pct) + "%") << std::setw(11) + << r.matching << std::setw(13) << std::fixed + << std::setprecision(2) << r.old_way_ms << std::setw(13) + << r.new_way_ms << std::fixed << std::setprecision(1) << speedup + << "x\n"; + } + std::cout + << "============================================================" + << "================\n\n"; +} + +} // namespace + +int main(int /*argc*/, char** /*argv*/) { + std::cout << "Running composite key benchmark...\n"; + std::cout << "(Each (N, selectivity) pair is averaged over 5 iterations " + << "after a warmup.)\n\n"; + + const int iters = 5; + std::vector rows; + for (int n : {1000, 10000, 100000}) { + for (int sel : {1, 10}) { + std::cout << " N=" << n << " selectivity=" << sel << "% ..." + << std::flush; + rows.push_back(RunOne(n, sel, iters)); + std::cout << " done.\n"; + } + } + + PrintTable(rows); + return 0; +} diff --git a/chain/storage/BUILD b/chain/storage/BUILD index 45aa6e9f5b..7ade54cc7e 100644 --- a/chain/storage/BUILD +++ b/chain/storage/BUILD @@ -48,6 +48,7 @@ cc_library( srcs = ["leveldb.cpp"], hdrs = ["leveldb.h"], deps = [ + ":composite_key_codec", ":storage", "//chain/storage/proto:kv_cc_proto", "//chain/storage/proto:leveldb_config_cc_proto", @@ -64,6 +65,7 @@ cc_test( timeout = "short", # Set the timeout to "short" srcs = ["kv_storage_test.cpp"], deps = [ + ":composite_key_codec", ":leveldb", ":memory_db", "//common/test:test_main", @@ -82,6 +84,21 @@ cc_test( ], ) +cc_library( + name = "composite_key_codec", + srcs = ["composite_key_codec.cpp"], + hdrs = ["composite_key_codec.h"], +) + +cc_test( + name = "composite_key_codec_test", + srcs = ["composite_key_codec_test.cpp"], + deps = [ + ":composite_key_codec", + "//common/test:test_main", + ], +) + cc_library( name = "duckdb_storage", srcs = ["duckdb.cpp"], @@ -103,4 +120,4 @@ cc_binary( "//common:glog", "//chain/storage/proto:duckdb_config_cc_proto", ], -) \ No newline at end of file +) diff --git a/chain/storage/composite_key_codec.cpp b/chain/storage/composite_key_codec.cpp new file mode 100644 index 0000000000..781fa99543 --- /dev/null +++ b/chain/storage/composite_key_codec.cpp @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + #include "chain/storage/composite_key_codec.h" + + #include + + namespace resdb { + namespace storage { + +std::string EncodeCompositeKey(const std::string& index_name, + const std::vector& attributes, + const std::string& primary_key) { + if (index_name.find(kCompositeKeyDelim) != std::string::npos || + primary_key.find(kCompositeKeyDelim) != std::string::npos) { + return ""; + } + for (const auto& attr : attributes) { + if (attr.find(kCompositeKeyDelim) != std::string::npos) { + return ""; + } + } + + std::string result; + result.append(kCompositeKeyNamespace); + result.push_back(kCompositeKeyDelim); + result.append(index_name); + result.push_back(kCompositeKeyDelim); + + for (const auto& attr : attributes) { + result.append(attr); + result.push_back(kCompositeKeyDelim); + } + + result.append(primary_key); + return result; +} + +std::string EncodeCompositeKeyPrefix( + const std::string& index_name, + const std::vector& attribute_prefix) { + if (index_name.find(kCompositeKeyDelim) != std::string::npos) { + return ""; + } + for (const auto& attr : attribute_prefix) { + if (attr.find(kCompositeKeyDelim) != std::string::npos) { + return ""; + } + } + + std::string result; + result.append(kCompositeKeyNamespace); + result.push_back(kCompositeKeyDelim); + result.append(index_name); + result.push_back(kCompositeKeyDelim); + + for (size_t i = 0; i < attribute_prefix.size(); ++i) { + if (i > 0) result.push_back(kCompositeKeyDelim); + result.append(attribute_prefix[i]); + } + + // Trailing delimiter makes this a strict byte prefix of the full key. + result.push_back(kCompositeKeyDelim); + return result; +} + +bool DecodeCompositeKey(const std::string& encoded, + std::string* index_name, + std::vector* attributes, + std::string* primary_key) { + if (!index_name || !attributes || !primary_key) { + return false; + } + + size_t ns_len = std::strlen(kCompositeKeyNamespace); + if (encoded.size() < ns_len + 1 || + encoded.compare(0, ns_len, kCompositeKeyNamespace) != 0) { + return false; + } + + std::string rest = encoded.substr(ns_len + 1); + size_t pos = 0; + std::vector parts; + while (pos < rest.size()) { + size_t next = rest.find(kCompositeKeyDelim, pos); + if (next == std::string::npos) { + parts.push_back(rest.substr(pos)); + break; + } + parts.push_back(rest.substr(pos, next - pos)); + pos = next + 1; + } + + // Need at least index_name and primary_key. + if (parts.size() < 2) { + return false; + } + + *index_name = parts[0]; + *primary_key = parts.back(); + attributes->clear(); + for (size_t i = 1; i < parts.size() - 1; ++i) { + attributes->push_back(parts[i]); + } + return true; +} + + } // namespace storage + } // namespace resdb \ No newline at end of file diff --git a/chain/storage/composite_key_codec.h b/chain/storage/composite_key_codec.h new file mode 100644 index 0000000000..bc992be82c --- /dev/null +++ b/chain/storage/composite_key_codec.h @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +namespace resdb { +namespace storage { + +// Reserved prefix for index entries; keeps them out of the user key space. +constexpr char kCompositeKeyNamespace[] = "ck"; + +// Field separator. Null byte is illegal in user attributes, so it is safe. +constexpr char kCompositeKeyDelim = '\0'; + +// Layout: "ck" \0 index_name \0 attr_1 \0 ... \0 primary_key +// Returns "" if any input contains the delimiter. +std::string EncodeCompositeKey(const std::string& index_name, + const std::vector& attributes, + const std::string& primary_key); + +// Same layout without primary_key; ends with a trailing delimiter so the +// result is a strict byte prefix of the corresponding full key. +std::string EncodeCompositeKeyPrefix( + const std::string& index_name, + const std::vector& attribute_prefix); + +// Returns false if input is malformed. +bool DecodeCompositeKey(const std::string& encoded, + std::string* index_name, + std::vector* attributes, + std::string* primary_key); + +} // namespace storage +} // namespace resdb diff --git a/chain/storage/composite_key_codec_test.cpp b/chain/storage/composite_key_codec_test.cpp new file mode 100644 index 0000000000..30509f5f98 --- /dev/null +++ b/chain/storage/composite_key_codec_test.cpp @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include "chain/storage/composite_key_codec.h" + +using namespace resdb::storage; + +TEST(CompositeKeyCodecTest, RoundTrip) { + std::string encoded = + EncodeCompositeKey("byOwner", {"alice", "active"}, "user:1"); + EXPECT_FALSE(encoded.empty()); + + std::string idx, pk; + std::vector attrs; + ASSERT_TRUE(DecodeCompositeKey(encoded, &idx, &attrs, &pk)); + EXPECT_EQ(idx, "byOwner"); + EXPECT_EQ(attrs.size(), 2u); + EXPECT_EQ(attrs[0], "alice"); + EXPECT_EQ(attrs[1], "active"); + EXPECT_EQ(pk, "user:1"); +} + +TEST(CompositeKeyCodecTest, RejectDelimInInput) { + // C-string literals truncate at '\0'; build the strings explicitly. + std::string bad_index = std::string("in") + kCompositeKeyDelim + "dex"; + EXPECT_TRUE(EncodeCompositeKey(bad_index, {"attr"}, "pk").empty()); + + std::string bad_attr = std::string("at") + kCompositeKeyDelim + "tr"; + EXPECT_TRUE(EncodeCompositeKey("idx", {bad_attr}, "pk").empty()); + + std::string bad_pk = std::string("p") + kCompositeKeyDelim + "k"; + EXPECT_TRUE(EncodeCompositeKey("idx", {"attr"}, bad_pk).empty()); +} + +TEST(CompositeKeyCodecTest, PrefixIsStrictBytePrefix) { + std::string full = EncodeCompositeKey("byOwner", {"alice"}, "user:1"); + std::string prefix = EncodeCompositeKeyPrefix("byOwner", {"alice"}); + + EXPECT_FALSE(full.empty()); + EXPECT_FALSE(prefix.empty()); + EXPECT_EQ(full.find(prefix), 0u); + EXPECT_LT(prefix.size(), full.size()); +} + +TEST(CompositeKeyCodecTest, DecodeMalformed) { + std::string idx, pk; + std::vector attrs; + + std::string no_ns = std::string("no_prefix") + kCompositeKeyDelim + "key"; + EXPECT_FALSE(DecodeCompositeKey(no_ns, &idx, &attrs, &pk)); + + std::string only_one = std::string("ck") + kCompositeKeyDelim + "alone"; + EXPECT_FALSE(DecodeCompositeKey(only_one, &idx, &attrs, &pk)); +} + +TEST(CompositeKeyCodecTest, EmptyAttributes) { + std::string encoded = EncodeCompositeKey("byId", {}, "user:1"); + EXPECT_FALSE(encoded.empty()); + + std::string idx, pk; + std::vector attrs; + ASSERT_TRUE(DecodeCompositeKey(encoded, &idx, &attrs, &pk)); + EXPECT_EQ(idx, "byId"); + EXPECT_EQ(attrs.size(), 0u); + EXPECT_EQ(pk, "user:1"); +} diff --git a/chain/storage/kv_storage_test.cpp b/chain/storage/kv_storage_test.cpp index 343cdbf9bd..b6719072c1 100644 --- a/chain/storage/kv_storage_test.cpp +++ b/chain/storage/kv_storage_test.cpp @@ -23,6 +23,7 @@ #include +#include "chain/storage/composite_key_codec.h" #include "chain/storage/leveldb.h" #include "chain/storage/memory_db.h" @@ -305,6 +306,118 @@ TEST_P(KVStorageTest, BlockCacheSpecificTest) { } } +// Composite key tests run against all three backends. Keys are built via the +// codec because C-string literals would truncate at the embedded '\0'. + +TEST_P(KVStorageTest, CreateAndRetrieveCompositeKey) { + std::string ck1 = EncodeCompositeKey("byCity", {"Davis"}, "user:1"); + std::string ck2 = EncodeCompositeKey("byCity", {"Davis"}, "user:2"); + std::string ck3 = EncodeCompositeKey("byCity", {"Davis"}, "user:3"); + + EXPECT_EQ(storage->CreateCompositeKey(ck1), 0); + EXPECT_EQ(storage->CreateCompositeKey(ck2), 0); + EXPECT_EQ(storage->CreateCompositeKey(ck3), 0); + + std::string davis_prefix = EncodeCompositeKeyPrefix("byCity", {"Davis"}); + auto results = storage->GetByCompositeKeyPrefix(davis_prefix); + + EXPECT_EQ(results.size(), 3u); + EXPECT_EQ(results[0], ck1); + EXPECT_EQ(results[1], ck2); + EXPECT_EQ(results[2], ck3); +} + +TEST_P(KVStorageTest, PrefixScanOrdering) { + // Inserted out of order; backend must return them sorted within a prefix. + std::string sf_ck = EncodeCompositeKey("byCity", {"SF"}, "user:3"); + std::string davis_ck1 = EncodeCompositeKey("byCity", {"Davis"}, "user:1"); + std::string davis_ck2 = EncodeCompositeKey("byCity", {"Davis"}, "user:2"); + std::string nyc_ck = EncodeCompositeKey("byCity", {"NYC"}, "user:4"); + + EXPECT_EQ(storage->CreateCompositeKey(sf_ck), 0); + EXPECT_EQ(storage->CreateCompositeKey(davis_ck1), 0); + EXPECT_EQ(storage->CreateCompositeKey(davis_ck2), 0); + EXPECT_EQ(storage->CreateCompositeKey(nyc_ck), 0); + + std::string davis_prefix = EncodeCompositeKeyPrefix("byCity", {"Davis"}); + auto davis_results = storage->GetByCompositeKeyPrefix(davis_prefix); + EXPECT_EQ(davis_results.size(), 2u); + EXPECT_EQ(davis_results[0], davis_ck1); + EXPECT_EQ(davis_results[1], davis_ck2); + + std::string sf_prefix = EncodeCompositeKeyPrefix("byCity", {"SF"}); + auto sf_results = storage->GetByCompositeKeyPrefix(sf_prefix); + EXPECT_EQ(sf_results.size(), 1u); + EXPECT_EQ(sf_results[0], sf_ck); +} + +TEST_P(KVStorageTest, DeleteRemovesEntry) { + std::string ck = EncodeCompositeKey("byCity", {"Davis"}, "user:1"); + std::string davis_prefix = EncodeCompositeKeyPrefix("byCity", {"Davis"}); + + EXPECT_EQ(storage->CreateCompositeKey(ck), 0); + EXPECT_EQ(storage->GetByCompositeKeyPrefix(davis_prefix).size(), 1u); + + EXPECT_EQ(storage->DeleteCompositeKey(ck), 0); + EXPECT_EQ(storage->GetByCompositeKeyPrefix(davis_prefix).size(), 0u); +} + +TEST_P(KVStorageTest, UpdateIsAtomic) { + std::string old_ck = EncodeCompositeKey("byCity", {"Davis"}, "user:1"); + std::string new_ck = EncodeCompositeKey("byCity", {"SF"}, "user:1"); + std::string davis_prefix = EncodeCompositeKeyPrefix("byCity", {"Davis"}); + std::string sf_prefix = EncodeCompositeKeyPrefix("byCity", {"SF"}); + + EXPECT_EQ(storage->CreateCompositeKey(old_ck), 0); + EXPECT_EQ(storage->GetByCompositeKeyPrefix(davis_prefix).size(), 1u); + + EXPECT_EQ(storage->UpdateCompositeKey(old_ck, new_ck), 0); + + EXPECT_EQ(storage->GetByCompositeKeyPrefix(davis_prefix).size(), 0u); + auto sf_results = storage->GetByCompositeKeyPrefix(sf_prefix); + EXPECT_EQ(sf_results.size(), 1u); + EXPECT_EQ(sf_results[0], new_ck); +} + +TEST_P(KVStorageTest, EmptyPrefixScanReturnsNothing) { + EXPECT_EQ(storage->CreateCompositeKey( + EncodeCompositeKey("byCity", {"Davis"}, "user:1")), + 0); + EXPECT_EQ(storage->CreateCompositeKey( + EncodeCompositeKey("byCity", {"Davis"}, "user:2")), + 0); + + std::string nyc_prefix = EncodeCompositeKeyPrefix("byCity", {"NYC"}); + auto results = storage->GetByCompositeKeyPrefix(nyc_prefix); + EXPECT_EQ(results.size(), 0u); +} + +// GetAllItems must not return composite-key markers. Also checks that the +// filter is exact ("ck\0...") and doesn't drop user keys like "ck_balance". +TEST_P(KVStorageTest, GetAllItemsExcludesCompositeKeys) { + EXPECT_EQ(storage->SetValueWithVersion("user:1", "alice_data", 0), 0); + EXPECT_EQ(storage->SetValueWithVersion("user:2", "bob_data", 0), 0); + EXPECT_EQ(storage->SetValueWithVersion("ck_balance", "real_user_data", 0), 0); + + EXPECT_EQ(storage->CreateCompositeKey( + EncodeCompositeKey("byCity", {"Davis"}, "user:1")), + 0); + EXPECT_EQ(storage->CreateCompositeKey( + EncodeCompositeKey("byCity", {"SF"}, "user:2")), + 0); + + auto all = storage->GetAllItems(); + EXPECT_EQ(all.size(), 3u); + EXPECT_TRUE(all.count("user:1")); + EXPECT_TRUE(all.count("user:2")); + EXPECT_TRUE(all.count("ck_balance")); + + // Markers are hidden, not deleted. + auto davis = storage->GetByCompositeKeyPrefix( + EncodeCompositeKeyPrefix("byCity", {"Davis"})); + EXPECT_EQ(davis.size(), 1u); +} + INSTANTIATE_TEST_CASE_P(KVStorageTest, KVStorageTest, ::testing::Values(MEM, LEVELDB, LEVELDB_WITH_BLOCK_CACHE)); diff --git a/chain/storage/leveldb.cpp b/chain/storage/leveldb.cpp index de49376e75..fc0a09b5fa 100644 --- a/chain/storage/leveldb.cpp +++ b/chain/storage/leveldb.cpp @@ -23,7 +23,9 @@ #include #include +#include +#include "chain/storage/composite_key_codec.h" #include "chain/storage/proto/kv.pb.h" #include "leveldb/options.h" @@ -321,8 +323,16 @@ ResLevelDB::GetAllItemsWithSeq() { std::map> ResLevelDB::GetAllItems() { std::map> resp; + // Hide composite-key index entries from user-facing scans. + const std::string ck_prefix = + std::string(kCompositeKeyNamespace) + kCompositeKeyDelim; + leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions()); for (it->SeekToFirst(); it->Valid(); it->Next()) { + if (it->key().size() >= ck_prefix.size() && + memcmp(it->key().data(), ck_prefix.data(), ck_prefix.size()) == 0) { + continue; + } ValueHistory history; if (!history.ParseFromString(it->value().ToString()) || history.value_size() == 0) { @@ -342,9 +352,17 @@ std::map> ResLevelDB::GetKeyRange( const std::string& min_key, const std::string& max_key) { std::map> resp; + // Hide composite-key index entries from user-facing scans. + const std::string ck_prefix = + std::string(kCompositeKeyNamespace) + kCompositeKeyDelim; + leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions()); for (it->Seek(min_key); it->Valid() && it->key().ToString() <= max_key; it->Next()) { + if (it->key().size() >= ck_prefix.size() && + memcmp(it->key().data(), ck_prefix.data(), ck_prefix.size()) == 0) { + continue; + } ValueHistory history; if (!history.ParseFromString(it->value().ToString()) || history.value_size() == 0) { @@ -440,5 +458,44 @@ uint64_t ResLevelDB::GetLastCheckpoint() { return GetLastCheckpointInternal(); } +int ResLevelDB::CreateCompositeKey(const std::string& composite_key) { + leveldb::Status s = db_->Put(leveldb::WriteOptions(), composite_key, ""); + return s.ok() ? 0 : -1; +} + +int ResLevelDB::DeleteCompositeKey(const std::string& composite_key) { + leveldb::Status s = db_->Delete(leveldb::WriteOptions(), composite_key); + return s.ok() ? 0 : -1; +} + +int ResLevelDB::UpdateCompositeKey(const std::string& old_composite_key, + const std::string& new_composite_key) { + // WriteBatch makes delete+put atomic. + leveldb::WriteBatch batch; + batch.Delete(old_composite_key); + batch.Put(new_composite_key, ""); + leveldb::Status s = db_->Write(leveldb::WriteOptions(), &batch); + return s.ok() ? 0 : -1; +} + +std::vector ResLevelDB::GetByCompositeKeyPrefix( + const std::string& prefix) { + std::vector out; + std::unique_ptr it( + db_->NewIterator(leveldb::ReadOptions())); + + // Seek lands on the first key >= prefix; sorted order lets us stop on the + // first non-match. + for (it->Seek(prefix); it->Valid(); it->Next()) { + leveldb::Slice key = it->key(); + if (key.size() < prefix.size() || + memcmp(key.data(), prefix.data(), prefix.size()) != 0) { + break; + } + out.emplace_back(key.data(), key.size()); + } + return out; +} + } // namespace storage } // namespace resdb diff --git a/chain/storage/leveldb.h b/chain/storage/leveldb.h index 67ec7a40c2..db378bf519 100644 --- a/chain/storage/leveldb.h +++ b/chain/storage/leveldb.h @@ -72,6 +72,14 @@ class ResLevelDB : public Storage { std::vector> GetTopHistory( const std::string& key, int top_number) override; + // Composite key operations for secondary indexing + int CreateCompositeKey(const std::string& composite_key) override; + int DeleteCompositeKey(const std::string& composite_key) override; + std::vector GetByCompositeKeyPrefix( + const std::string& prefix) override; + int UpdateCompositeKey(const std::string& old_composite_key, + const std::string& new_composite_key) override; + bool UpdateMetrics(); bool Flush() override; diff --git a/chain/storage/memory_db.cpp b/chain/storage/memory_db.cpp index aac3bc8c7e..b009eb127c 100644 --- a/chain/storage/memory_db.cpp +++ b/chain/storage/memory_db.cpp @@ -201,5 +201,37 @@ std::vector> MemoryDB::GetTopHistory( return resp; } +int MemoryDB::CreateCompositeKey(const std::string& composite_key) { + ck_map_[composite_key] = ""; + return 0; +} + +int MemoryDB::DeleteCompositeKey(const std::string& composite_key) { + ck_map_.erase(composite_key); + return 0; +} + +std::vector MemoryDB::GetByCompositeKeyPrefix( + const std::string& prefix) { + std::vector out; + // std::map is sorted; lower_bound + walk mirrors the LevelDB iterator path. + for (auto it = ck_map_.lower_bound(prefix); it != ck_map_.end(); ++it) { + const std::string& key = it->first; + if (key.size() < prefix.size() || + key.compare(0, prefix.size(), prefix) != 0) { + break; + } + out.push_back(key); + } + return out; +} + +int MemoryDB::UpdateCompositeKey(const std::string& old_composite_key, + const std::string& new_composite_key) { + ck_map_.erase(old_composite_key); + ck_map_[new_composite_key] = ""; + return 0; +} + } // namespace storage } // namespace resdb diff --git a/chain/storage/memory_db.h b/chain/storage/memory_db.h index ec138c7aeb..ce398a131f 100644 --- a/chain/storage/memory_db.h +++ b/chain/storage/memory_db.h @@ -83,12 +83,25 @@ class MemoryDB : public Storage { std::vector> GetTopHistory(const std::string& key, int number) override; + // Composite key operations for secondary indexing. + // Backed by an ordered std::map to mirror LevelDB's sorted key layout, which + // is what enables efficient prefix scans. + int CreateCompositeKey(const std::string& composite_key) override; + int DeleteCompositeKey(const std::string& composite_key) override; + std::vector GetByCompositeKeyPrefix( + const std::string& prefix) override; + int UpdateCompositeKey(const std::string& old_composite_key, + const std::string& new_composite_key) override; + private: std::unordered_map kv_map_; std::unordered_map>> kv_map_with_v_; std::unordered_map>> kv_map_with_seq_; + // Sorted map for composite keys; preserves lexicographic order so prefix + // scans return matching keys deterministically (matches LevelDB behavior). + std::map ck_map_; }; } // namespace storage diff --git a/chain/storage/storage.h b/chain/storage/storage.h index 7a0bfe5362..415ea70386 100644 --- a/chain/storage/storage.h +++ b/chain/storage/storage.h @@ -59,6 +59,13 @@ class Storage { virtual std::vector> GetTopHistory( const std::string& key, int number) = 0; + // Composite key operations for secondary indexing + virtual int CreateCompositeKey(const std::string& composite_key) = 0; + virtual int DeleteCompositeKey(const std::string& composite_key) = 0; + virtual std::vector GetByCompositeKeyPrefix( + const std::string& prefix) = 0; + virtual int UpdateCompositeKey(const std::string& old_composite_key, + const std::string& new_composite_key) = 0; // Default no-op SQL execution for non-SQL backends. virtual std::string ExecuteSQL(const std::string& sql_string) { return ""; }