From ce5ae5df0fe6f57cdce4d90166f4ea859a347024 Mon Sep 17 00:00:00 2001 From: eshabram Date: Thu, 13 Nov 2025 23:46:01 -0800 Subject: [PATCH 01/41] Ignore generated configs and certs --- .gitignore | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6d1b093fe9..8ddd00706b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ deploy/kv_server/output/ .vscode/ .DS_Store bazel-* -*.log +**/*.log *.bak venv sdk_validator/venv @@ -20,4 +20,8 @@ apache_release resdb/ 100*_db/ gmon.out -.history/ \ No newline at end of file +.history/ +**/*.cert +service/tools/config/server/server.config +service/tools/config/interface/service.config + From 6d64edaad9cacc1cc58de5c7f634795117af6f2c Mon Sep 17 00:00:00 2001 From: eshabram Date: Thu, 13 Nov 2025 23:46:39 -0800 Subject: [PATCH 02/41] Ignore generated configs/certs --- service/tools/config/interface/service.config | 3 -- service/tools/config/server/server.config | 38 ------------------ service/tools/data/cert/admin.key.pri | 2 - service/tools/data/cert/admin.key.pub | 2 - service/tools/data/cert/cert_1.cert | Bin 166 -> 0 bytes service/tools/data/cert/cert_2.cert | 7 ---- service/tools/data/cert/cert_3.cert | 7 ---- service/tools/data/cert/cert_4.cert | 6 --- service/tools/data/cert/cert_5.cert | 8 ---- service/tools/data/cert/cert_6.cert | Bin 166 -> 0 bytes service/tools/data/cert/cert_7.cert | 6 --- service/tools/data/cert/cert_8.cert | Bin 166 -> 0 bytes service/tools/data/cert/cert_9.cert | 7 ---- service/tools/data/cert/node1.key.pri | 2 - service/tools/data/cert/node1.key.pub | 3 -- service/tools/data/cert/node2.key.pri | 2 - service/tools/data/cert/node2.key.pub | 2 - service/tools/data/cert/node3.key.pri | Bin 36 -> 0 bytes service/tools/data/cert/node3.key.pub | 3 -- service/tools/data/cert/node4.key.pri | Bin 36 -> 0 bytes service/tools/data/cert/node4.key.pub | 2 - service/tools/data/cert/node5.key.pri | 2 - service/tools/data/cert/node5.key.pub | 3 -- service/tools/data/cert/node6.key.key.pri | 2 - service/tools/data/cert/node6.key.key.pub | 2 - service/tools/data/cert/node6.key.pri | 2 - service/tools/data/cert/node6.key.pub | 2 - service/tools/data/cert/node7.key.key.pri | 2 - service/tools/data/cert/node7.key.key.pub | 2 - service/tools/data/cert/node7.key.pri | 2 - service/tools/data/cert/node7.key.pub | 2 - service/tools/data/cert/node8.key.key.pri | 2 - service/tools/data/cert/node8.key.key.pub | 2 - service/tools/data/cert/node8.key.pri | 2 - service/tools/data/cert/node8.key.pub | 2 - service/tools/data/cert/node9.key.key.pri | 2 - service/tools/data/cert/node9.key.key.pub | 2 - service/tools/data/cert/node9.key.pri | 2 - service/tools/data/cert/node9.key.pub | 2 - 39 files changed, 137 deletions(-) delete mode 100644 service/tools/config/interface/service.config delete mode 100644 service/tools/config/server/server.config delete mode 100644 service/tools/data/cert/admin.key.pri delete mode 100644 service/tools/data/cert/admin.key.pub delete mode 100644 service/tools/data/cert/cert_1.cert delete mode 100644 service/tools/data/cert/cert_2.cert delete mode 100644 service/tools/data/cert/cert_3.cert delete mode 100644 service/tools/data/cert/cert_4.cert delete mode 100644 service/tools/data/cert/cert_5.cert delete mode 100644 service/tools/data/cert/cert_6.cert delete mode 100644 service/tools/data/cert/cert_7.cert delete mode 100644 service/tools/data/cert/cert_8.cert delete mode 100644 service/tools/data/cert/cert_9.cert delete mode 100644 service/tools/data/cert/node1.key.pri delete mode 100644 service/tools/data/cert/node1.key.pub delete mode 100644 service/tools/data/cert/node2.key.pri delete mode 100644 service/tools/data/cert/node2.key.pub delete mode 100644 service/tools/data/cert/node3.key.pri delete mode 100644 service/tools/data/cert/node3.key.pub delete mode 100644 service/tools/data/cert/node4.key.pri delete mode 100644 service/tools/data/cert/node4.key.pub delete mode 100644 service/tools/data/cert/node5.key.pri delete mode 100644 service/tools/data/cert/node5.key.pub delete mode 100644 service/tools/data/cert/node6.key.key.pri delete mode 100644 service/tools/data/cert/node6.key.key.pub delete mode 100644 service/tools/data/cert/node6.key.pri delete mode 100644 service/tools/data/cert/node6.key.pub delete mode 100644 service/tools/data/cert/node7.key.key.pri delete mode 100644 service/tools/data/cert/node7.key.key.pub delete mode 100644 service/tools/data/cert/node7.key.pri delete mode 100644 service/tools/data/cert/node7.key.pub delete mode 100644 service/tools/data/cert/node8.key.key.pri delete mode 100644 service/tools/data/cert/node8.key.key.pub delete mode 100644 service/tools/data/cert/node8.key.pri delete mode 100644 service/tools/data/cert/node8.key.pub delete mode 100644 service/tools/data/cert/node9.key.key.pri delete mode 100644 service/tools/data/cert/node9.key.key.pub delete mode 100644 service/tools/data/cert/node9.key.pri delete mode 100644 service/tools/data/cert/node9.key.pub diff --git a/service/tools/config/interface/service.config b/service/tools/config/interface/service.config deleted file mode 100644 index d357ba7350..0000000000 --- a/service/tools/config/interface/service.config +++ /dev/null @@ -1,3 +0,0 @@ -5 127.0.0.1 10005 - - diff --git a/service/tools/config/server/server.config b/service/tools/config/server/server.config deleted file mode 100644 index 740bae3abe..0000000000 --- a/service/tools/config/server/server.config +++ /dev/null @@ -1,38 +0,0 @@ -{ - region : { - replica_info : { - id:1, - ip:"127.0.0.1", - port: 10001, - }, - replica_info : { - id:2, - ip:"127.0.0.1", - port: 10002, - }, - replica_info : { - id:3, - ip:"127.0.0.1", - port: 10003, - }, - replica_info : { - id:4, - ip:"127.0.0.1", - port: 10004, - }, - region_id: 1, - }, - self_region_id:1, - leveldb_info : { - write_buffer_size_mb:128, - write_batch_size:1, - enable_block_cache: true, - block_cache_capacity: 100 - }, - require_txn_validation:true, - enable_viewchange:false, - enable_resview:true, - enable_faulty_switch:false -} - - diff --git a/service/tools/data/cert/admin.key.pri b/service/tools/data/cert/admin.key.pri deleted file mode 100644 index 7629a70a96..0000000000 --- a/service/tools/data/cert/admin.key.pri +++ /dev/null @@ -1,2 +0,0 @@ - - |quwq&07!6fa's| \ No newline at end of file diff --git a/service/tools/data/cert/admin.key.pub b/service/tools/data/cert/admin.key.pub deleted file mode 100644 index 4924810534..0000000000 --- a/service/tools/data/cert/admin.key.pub +++ /dev/null @@ -1,2 +0,0 @@ - - u>,Nb+Rm.dQ|;V;݅  \ No newline at end of file diff --git a/service/tools/data/cert/cert_1.cert b/service/tools/data/cert/cert_1.cert deleted file mode 100644 index cb2ffd016e8ff7b2c275972455ad78f84b380d7e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 166 zcmV;X09pSEBnlvPK8>s_>deW_PGT!kC=NeUyT%MRs10T2QZ0U`-8GB++TE-)@JD3MMQ zLK;AUU4cmw{c)RSsEc_aPjoX(s?l9yNYbI%XgT!nKCp{6eD7Y}>rXh<=r`Ftvsfyo UI{|Fu@uX(T5m#Wu7u*0K0WVEKHvj+t diff --git a/service/tools/data/cert/cert_2.cert b/service/tools/data/cert/cert_2.cert deleted file mode 100644 index 81b8743532..0000000000 --- a/service/tools/data/cert/cert_2.cert +++ /dev/null @@ -1,7 +0,0 @@ - -$ - u>,Nb+Rm.dQ|;V;݅ | -6 -$ - 󗑉EjK' =zp@rE" 127.0.0.1(NB@,'GJ|R;'m@:#9ĝ'[ JZ}_,Iʟ9'n)% -n|G  \ No newline at end of file diff --git a/service/tools/data/cert/cert_3.cert b/service/tools/data/cert/cert_3.cert deleted file mode 100644 index f7f811218d..0000000000 --- a/service/tools/data/cert/cert_3.cert +++ /dev/null @@ -1,7 +0,0 @@ - -$ - u>,Nb+Rm.dQ|;V;݅ | -6 -$ - |a9 #Y,Nb+Rm.dQ|;V;݅ | -6 -$ - { aLx2٢=y{֐6]gw  \ No newline at end of file diff --git a/service/tools/data/cert/cert_5.cert b/service/tools/data/cert/cert_5.cert deleted file mode 100644 index b04bf7a912..0000000000 --- a/service/tools/data/cert/cert_5.cert +++ /dev/null @@ -1,8 +0,0 @@ - -$ - u>,Nb+Rm.dQ|;V;݅ ~ -8 -$ - HY -1hܘhT]%%꡽eyV" 127.0.0.1(NB@P$4L vt`#+gX7i -L]൮! ]k!d{qP  \ No newline at end of file diff --git a/service/tools/data/cert/cert_6.cert b/service/tools/data/cert/cert_6.cert deleted file mode 100644 index 92e27c8883c80cb59234ec2f367ce158204fab92..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 166 zcmV;X09pSEBnlvPK8>s_>deW_PGT!kKLPa<>L^3%tG)6@;L^C!wMlm)tGC4CuIYmN4HxL651|kVDGB++TE-)@JD3(qV zLK;Bx91H=1+SB770@jI__B3;KvEh%r%MvBpdaQgU1fmDhtcqS=VZs^B(>&L&ICoxX U2)ojt=evWiey#{H9gF}V21Hmp+W-In diff --git a/service/tools/data/cert/cert_7.cert b/service/tools/data/cert/cert_7.cert deleted file mode 100644 index 982a0b91eb..0000000000 --- a/service/tools/data/cert/cert_7.cert +++ /dev/null @@ -1,6 +0,0 @@ - -$ - u>,Nb+Rm.dQ|;V;݅ | -6 -$ - 374850F7F3BBE65DF6483A6D4D90AFAE" 127.0.0.1(NB@\~nMm;$iUBKEwcڃ;G#ӞݽQ眮zf6  \ No newline at end of file diff --git a/service/tools/data/cert/cert_8.cert b/service/tools/data/cert/cert_8.cert deleted file mode 100644 index e392ad442c7be38fd0a2b47c47c918324558c206..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 166 zcmV;X09pSEBnlvPK8>s_>deW_PGT!kKLPa<>L^3%tG)6@;L^C!wMlm)tGC4CuIYmN4HxL652qFnFGB++TE-)@JD40$X zLK;AoJbTkmQM1UY0Pn0;3(_djzHU>0G*0B!v=$(lNPMEHSQ$e~r+~F57wYu7K&Yd! UW%rKzZWmbTwUlQD1M&?Z2t9s0GXMYp diff --git a/service/tools/data/cert/cert_9.cert b/service/tools/data/cert/cert_9.cert deleted file mode 100644 index feb0ef7ed6..0000000000 --- a/service/tools/data/cert/cert_9.cert +++ /dev/null @@ -1,7 +0,0 @@ - -$ - u>,Nb+Rm.dQ|;V;݅ | -6 -$ - 194BE86D2924FE3D367F164293D9EBC7 " 127.0.0.1(NB@jTq`z=G_ F.ܒl"t.LtTj/q@;" -PJx \ No newline at end of file diff --git a/service/tools/data/cert/node1.key.pri b/service/tools/data/cert/node1.key.pri deleted file mode 100644 index f6306d8043..0000000000 --- a/service/tools/data/cert/node1.key.pri +++ /dev/null @@ -1,2 +0,0 @@ - - %A^fK@W7c|K% \ No newline at end of file diff --git a/service/tools/data/cert/node1.key.pub b/service/tools/data/cert/node1.key.pub deleted file mode 100644 index 583c105cad..0000000000 --- a/service/tools/data/cert/node1.key.pub +++ /dev/null @@ -1,3 +0,0 @@ - - A/{?"XU,lErI -#A \ No newline at end of file diff --git a/service/tools/data/cert/node2.key.pri b/service/tools/data/cert/node2.key.pri deleted file mode 100644 index b5e05751e2..0000000000 --- a/service/tools/data/cert/node2.key.pri +++ /dev/null @@ -1,2 +0,0 @@ - - @ diff --git a/service/tools/data/cert/node4.key.pub b/service/tools/data/cert/node4.key.pub deleted file mode 100644 index 25f8b40f80..0000000000 --- a/service/tools/data/cert/node4.key.pub +++ /dev/null @@ -1,2 +0,0 @@ - - { aLx2٢ Date: Thu, 13 Nov 2025 23:51:56 -0800 Subject: [PATCH 03/41] logs into folder --- .gitignore | 2 ++ logs/.gitkeep | 0 service/tools/kv/server_tools/start_kv_service.sh | 11 +++++------ 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 logs/.gitkeep diff --git a/.gitignore b/.gitignore index 8ddd00706b..2e95b095e0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,6 @@ gmon.out **/*.cert service/tools/config/server/server.config service/tools/config/interface/service.config +service/tools/config/server.config +service/tools/data/cert/* diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/service/tools/kv/server_tools/start_kv_service.sh b/service/tools/kv/server_tools/start_kv_service.sh index e027313367..c0e1759d79 100755 --- a/service/tools/kv/server_tools/start_kv_service.sh +++ b/service/tools/kv/server_tools/start_kv_service.sh @@ -24,9 +24,8 @@ WORK_PATH=$PWD CERT_PATH=${WORK_PATH}/service/tools/data/cert/ bazel build //service/kv:kv_service $@ -nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node1.key.pri $CERT_PATH/cert_1.cert > server0.log & -nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node2.key.pri $CERT_PATH/cert_2.cert > server1.log & -nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node3.key.pri $CERT_PATH/cert_3.cert > server2.log & -nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node4.key.pri $CERT_PATH/cert_4.cert > server3.log & - -nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node5.key.pri $CERT_PATH/cert_5.cert > client.log & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node1.key.pri $CERT_PATH/cert_1.cert > logs/server0.log & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node2.key.pri $CERT_PATH/cert_2.cert > logs/server1.log & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node3.key.pri $CERT_PATH/cert_3.cert > logs/server2.log & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node4.key.pri $CERT_PATH/cert_4.cert > logs/server3.log & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node5.key.pri $CERT_PATH/cert_5.cert > logs/client.log & From 25bd05e0f42e215f0b99d44dcaed54a552b30f5e Mon Sep 17 00:00:00 2001 From: eshabram Date: Mon, 17 Nov 2025 23:54:37 -0800 Subject: [PATCH 04/41] temp --- .gitignore | 1 + platform/config/resdb_config.h | 7 ++++ platform/learner/BUILD | 21 ++++++++++ platform/learner/learner.config | 15 +++++++ platform/learner/learner.cpp | 20 ++++++++++ platform/learner/learner.h | 14 +++++++ platform/learner/learner_main.cpp | 8 ++++ .../server_tools/start_kv_service_learner.sh | 40 +++++++++++++++++++ 8 files changed, 126 insertions(+) create mode 100644 platform/learner/BUILD create mode 100644 platform/learner/learner.config create mode 100644 platform/learner/learner.cpp create mode 100644 platform/learner/learner.h create mode 100644 platform/learner/learner_main.cpp create mode 100755 service/tools/kv/server_tools/start_kv_service_learner.sh diff --git a/.gitignore b/.gitignore index 2e95b095e0..d333fd9dfc 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ service/tools/config/server/server.config service/tools/config/interface/service.config service/tools/config/server.config service/tools/data/cert/* +**/*AGENTS.md diff --git a/platform/config/resdb_config.h b/platform/config/resdb_config.h index 9867c6d5ab..58a36c27e2 100644 --- a/platform/config/resdb_config.h +++ b/platform/config/resdb_config.h @@ -121,6 +121,10 @@ class ResDBConfig { uint32_t GetViewchangeCommitTimeout() const; void SetViewchangeCommitTimeout(uint64_t timeout_ms); + // learner block_size functions + int GetBlockSize() const { return block_size_; } + void SetBlockSize(int bs) { block_size_ = bs; } + private: ResConfigData config_data_; std::vector replicas_; @@ -147,6 +151,9 @@ class ResDBConfig { uint32_t input_worker_num_ = 5; uint32_t output_worker_num_ = 5; uint32_t client_batch_num_ = 100; + + // learner + uint32_t block_size_ = 100; }; } // namespace resdb diff --git a/platform/learner/BUILD b/platform/learner/BUILD new file mode 100644 index 0000000000..7d85e7cdee --- /dev/null +++ b/platform/learner/BUILD @@ -0,0 +1,21 @@ +package(default_visibility = ["//visibility:public"]) + +cc_binary( + name = "learner", + srcs = [ + "learner.cpp", + "learner_main.cpp", + "learner.h", + ], + deps = [ + "//platform/config:resdb_config_utils", + "//common:comm", + + "//platform/networkstrate:consensus_manager", + "//platform/networkstrate:replica_communicator", + + "//platform/proto:resdb_cc_proto", + "//platform/proto:replica_info_cc_proto", + ], +) + diff --git a/platform/learner/learner.config b/platform/learner/learner.config new file mode 100644 index 0000000000..91d0e63966 --- /dev/null +++ b/platform/learner/learner.config @@ -0,0 +1,15 @@ +{ + "node_id": 6, + "ip": "127.0.0.1", + "port": 25000, + + "replicas": [ + { "id": 1, "ip": "127.0.0.1", "port": 20001 }, + { "id": 2, "ip": "127.0.0.1", "port": 20002 }, + { "id": 3, "ip": "127.0.0.1", "port": 20003 }, + { "id": 4, "ip": "127.0.0.1", "port": 20004 } + ], + + "block_size": 100 +} + diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp new file mode 100644 index 0000000000..8876214fbf --- /dev/null +++ b/platform/learner/learner.cpp @@ -0,0 +1,20 @@ +#include "learner/learner.h" + +Learner::Learner(const std::string& config_path) + : config_(resdb::ResDBConfig::FromFile(config_path)), + manager_(config_) { + + int bs = config_.GetBlockSize(); + std::cout << "Learner block_size = " << bs << std::endl; + + manager_.RegisterMessageHandler( + [&](const resdb::ResDBMessage& msg){ + // no-op for now + } + ); +} + +void Learner::Run() { + manager_.Run(); +} + diff --git a/platform/learner/learner.h b/platform/learner/learner.h new file mode 100644 index 0000000000..5829edbf69 --- /dev/null +++ b/platform/learner/learner.h @@ -0,0 +1,14 @@ +#pragma once +#include "server/connection_manager.h" +#include "common/config/resdb_config.h" + +class Learner { +public: + explicit Learner(const std::string& config_path); + void Run(); + +private: + resdb::ResDBConfig config_; + resdb::ConnectionManager manager_; +}; + diff --git a/platform/learner/learner_main.cpp b/platform/learner/learner_main.cpp new file mode 100644 index 0000000000..1d416c477c --- /dev/null +++ b/platform/learner/learner_main.cpp @@ -0,0 +1,8 @@ +#include "learner/learner.h" + +int main(int argc, char** argv) { + Learner learner("config/learner.config"); + learner.Run(); + return 0; +} + diff --git a/service/tools/kv/server_tools/start_kv_service_learner.sh b/service/tools/kv/server_tools/start_kv_service_learner.sh new file mode 100755 index 0000000000..bfa13f1d5c --- /dev/null +++ b/service/tools/kv/server_tools/start_kv_service_learner.sh @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +killall -9 kv_service +killall -9 learner + +SERVER_PATH=./bazel-bin/service/kv/kv_service +SERVER_CONFIG=service/tools/config/server/server.config +WORK_PATH=$PWD +CERT_PATH=${WORK_PATH}/service/tools/data/cert/ +LEARNER_PATH=./bazel-bin/src/learner/learner +LEARNER_CONFIG=$PWD/learner/learner.config + +# start both builds +bazel build //learner:learner $@ +bazel build //service/kv:kv_service $@ + +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node1.key.pri $CERT_PATH/cert_1.cert > logs/server0.log & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node2.key.pri $CERT_PATH/cert_2.cert > logs/server1.log & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node3.key.pri $CERT_PATH/cert_3.cert > logs/server2.log & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node4.key.pri $CERT_PATH/cert_4.cert > logs/server3.log & +nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node5.key.pri $CERT_PATH/cert_5.cert > logs/client.log & + +# learner +nohup $LEARNER_PATH -config $LEARNER_CONFIG > logs/learner.log & From f79e15313e1bbf48b21904ef0d14753f8f7b24de Mon Sep 17 00:00:00 2001 From: eshabram Date: Tue, 18 Nov 2025 03:27:24 -0800 Subject: [PATCH 05/41] learners listening. Reps talking --- entrypoint.sh | 17 ++- platform/config/resdb_config.cpp | 24 +++ platform/config/resdb_config.h | 5 +- .../consensus/ordering/pbft/commitment.cpp | 9 +- platform/learner/BUILD | 23 +-- platform/learner/learner.config | 1 - platform/learner/learner.cpp | 141 ++++++++++++++++-- platform/learner/learner.h | 42 +++++- platform/learner/learner_main.cpp | 27 +++- platform/networkstrate/consensus_manager.cpp | 5 + platform/proto/replica_info.proto | 6 +- .../server_tools/start_kv_service_learner.sh | 6 +- 12 files changed, 262 insertions(+), 44 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index dbd4ca2c0e..3b9a856eaf 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -19,5 +19,18 @@ # under the License. # -./service/tools/kv/server_tools/start_kv_service.sh -tail -f /dev/null +./service/tools/kv/server_tools/start_kv_service_learner.sh + +if [[ "$2" =~ ^[0-9]+$ ]]; then + lines=$2 +else + lines=2 +fi + +if [ "$1" = "-w" ]; then + watch -n 1 tail -n "$lines" logs/*.log +elif [ "$1" = "-l" ]; then + tail -f logs/learner.log +else + tail -f /dev/null +fi diff --git a/platform/config/resdb_config.cpp b/platform/config/resdb_config.cpp index 23129a2584..1ed26c0ad4 100644 --- a/platform/config/resdb_config.cpp +++ b/platform/config/resdb_config.cpp @@ -58,6 +58,14 @@ ResDBConfig::ResDBConfig(const ResConfigData& config_data, break; } } + learners_.clear(); + for (const auto& learner : config_data_.learner_info()) { + learners_.push_back(learner); + } + + if (config_data_.block_size() > 0) { + block_size_ = config_data_.block_size(); + } if (config_data_.view_change_timeout_ms() == 0) { config_data_.set_view_change_timeout_ms(viewchange_commit_timeout_ms_); } @@ -95,11 +103,27 @@ void ResDBConfig::SetConfigData(const ResConfigData& config_data) { break; } } + learners_.clear(); + for (const auto& learner : config_data_.learner_info()) { + learners_.push_back(learner); + } + if (config_data_.block_size() > 0) { + block_size_ = config_data_.block_size(); + } if (config_data_.view_change_timeout_ms() == 0) { config_data_.set_view_change_timeout_ms(viewchange_commit_timeout_ms_); } } +const std::vector& ResDBConfig::GetLearnerInfos() const { + return learners_; +} + +void ResDBConfig::AddLearnerInfo(const ReplicaInfo& learner) { + *config_data_.add_learner_info() = learner; + learners_.push_back(learner); +} + KeyInfo ResDBConfig::GetPrivateKey() const { return private_key_; } CertificateInfo ResDBConfig::GetPublicKeyCertificateInfo() const { diff --git a/platform/config/resdb_config.h b/platform/config/resdb_config.h index 58a36c27e2..8c9be091fc 100644 --- a/platform/config/resdb_config.h +++ b/platform/config/resdb_config.h @@ -121,9 +121,11 @@ class ResDBConfig { uint32_t GetViewchangeCommitTimeout() const; void SetViewchangeCommitTimeout(uint64_t timeout_ms); - // learner block_size functions + // learner int GetBlockSize() const { return block_size_; } void SetBlockSize(int bs) { block_size_ = bs; } + const std::vector& GetLearnerInfos() const; + void AddLearnerInfo(const ReplicaInfo&); private: ResConfigData config_data_; @@ -154,6 +156,7 @@ class ResDBConfig { // learner uint32_t block_size_ = 100; + std::vector learners_; }; } // namespace resdb diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index 4d048331eb..9990cd584f 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -317,11 +317,16 @@ int Commitment::PostProcessExecutedMsg() { request.set_type(Request::TYPE_RESPONSE); request.set_sender_id(config_.GetSelfInfo().id()); request.set_current_view(batch_resp->current_view()); - request.set_proxy_id(batch_resp->proxy_id()); - request.set_primary_id(batch_resp->primary_id()); + request.set_proxy_id(batch_resp->proxy_id()); + request.set_primary_id(batch_resp->primary_id()); // LOG(ERROR)<<"send back to proxy:"<proxy_id(); batch_resp->SerializeToString(request.mutable_data()); replica_communicator_->SendMessage(request, request.proxy_id()); + // Also mirror the executed request to every learner so they can consume + // the committed stream. + for (const auto& learner : config_.GetLearnerInfos()) { + replica_communicator_->SendMessage(request, learner.id()); + } } return 0; } diff --git a/platform/learner/BUILD b/platform/learner/BUILD index 7d85e7cdee..93bc617fa3 100644 --- a/platform/learner/BUILD +++ b/platform/learner/BUILD @@ -1,21 +1,22 @@ package(default_visibility = ["//visibility:public"]) -cc_binary( - name = "learner", +cc_library( + name = "learner_lib", srcs = [ "learner.cpp", - "learner_main.cpp", - "learner.h", ], + hdrs = ["learner.h"], deps = [ - "//platform/config:resdb_config_utils", - "//common:comm", - - "//platform/networkstrate:consensus_manager", - "//platform/networkstrate:replica_communicator", - + "//common:json", + "//platform/common/network:tcp_socket", "//platform/proto:resdb_cc_proto", - "//platform/proto:replica_info_cc_proto", ], ) +cc_binary( + name = "learner", + srcs = ["learner_main.cpp"], + deps = [ + ":learner_lib", + ], +) diff --git a/platform/learner/learner.config b/platform/learner/learner.config index 91d0e63966..124bff3a21 100644 --- a/platform/learner/learner.config +++ b/platform/learner/learner.config @@ -12,4 +12,3 @@ "block_size": 100 } - diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index 8876214fbf..eb54cc2f96 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -1,20 +1,139 @@ -#include "learner/learner.h" +#include "platform/learner/learner.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "platform/common/network/socket.h" +#include "platform/common/network/tcp_socket.h" +#include "platform/proto/resdb.pb.h" + +namespace { + +std::string DefaultConfigPath() { return "platform/learner/learner.config"; } + +LearnerConfig ParseLearnerConfig(const std::string& config_path) { + std::ifstream config_stream(config_path); + if (!config_stream.is_open()) { + throw std::runtime_error("Unable to open learner config: " + config_path); + } + + nlohmann::json config_json = + nlohmann::json::parse(config_stream, nullptr, true, true); + LearnerConfig config; + config.ip = config_json.value("ip", config.ip); + config.port = config_json.value("port", 0); + config.block_size = config_json.value("block_size", 0); + + if (config.port <= 0) { + throw std::runtime_error("Learner config must include a valid port"); + } + if (config.block_size <= 0) { + throw std::runtime_error("Learner config must include a positive block_size"); + } + return config; +} + +} // namespace Learner::Learner(const std::string& config_path) - : config_(resdb::ResDBConfig::FromFile(config_path)), - manager_(config_) { + : config_(LoadConfig(config_path)) {} - int bs = config_.GetBlockSize(); - std::cout << "Learner block_size = " << bs << std::endl; +LearnerConfig Learner::LoadConfig(const std::string& config_path) { + const std::string path = config_path.empty() ? DefaultConfigPath() : config_path; + return ParseLearnerConfig(path); +} + +void Learner::Run() { + resdb::TcpSocket server; + if (server.Listen(config_.ip, config_.port) != 0) { + throw std::runtime_error("Learner failed to bind to " + config_.ip + ":" + + std::to_string(config_.port)); + } - manager_.RegisterMessageHandler( - [&](const resdb::ResDBMessage& msg){ - // no-op for now + is_running_.store(true); + metrics_thread_ = std::thread(&Learner::MetricsLoop, this); + + while (is_running_.load()) { + auto client = server.Accept(); + if (!client) { + continue; } - ); + std::thread(&Learner::HandleClient, this, std::move(client)).detach(); + } } -void Learner::Run() { - manager_.Run(); +void Learner::HandleClient(std::unique_ptr socket) const { + std::cout << "[Learner] replica connected" << std::endl; + while (true) { + void* buffer = nullptr; + size_t len = 0; + int ret = socket->Recv(&buffer, &len); + if (ret <= 0) { + if (buffer != nullptr) { + free(buffer); + } + break; + } + + std::string payload(static_cast(buffer), len); + free(buffer); + ProcessBroadcast(payload); + } + std::cout << "[Learner] replica disconnected" << std::endl; +} + +void Learner::ProcessBroadcast(const std::string& payload) const { + resdb::ResDBMessage envelope; + if (!envelope.ParseFromString(payload)) { + ++total_messages_; + total_bytes_.fetch_add(payload.size()); + last_type_.store(-1); + last_seq_.store(0); + last_sender_.store(-1); + last_payload_bytes_.store(payload.size()); + return; + } + + resdb::Request request; + if (request.ParseFromString(envelope.data())) { + ++total_messages_; + total_bytes_.fetch_add(payload.size()); + last_type_.store(request.type()); + last_seq_.store(request.seq()); + last_sender_.store(request.sender_id()); + last_payload_bytes_.store(request.data().size()); + return; + } + + ++total_messages_; + total_bytes_.fetch_add(envelope.data().size()); + last_type_.store(-1); + last_seq_.store(0); + last_sender_.store(-1); + last_payload_bytes_.store(envelope.data().size()); } +void Learner::MetricsLoop() const { + while (is_running_.load()) { + PrintMetrics(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } +} + +void Learner::PrintMetrics() const { + std::cout << "[Learner] broadcasts=" << total_messages_.load() + << " bytes=" << total_bytes_.load() + << " last_type=" << last_type_.load() + << " last_seq=" << last_seq_.load() + << " sender=" << last_sender_.load() + << " payload=" << last_payload_bytes_.load() << "B" + << " (listening on " << config_.ip << ":" << config_.port + << ", block_size=" << config_.block_size << ")" << std::endl; +} diff --git a/platform/learner/learner.h b/platform/learner/learner.h index 5829edbf69..86ce87a4cb 100644 --- a/platform/learner/learner.h +++ b/platform/learner/learner.h @@ -1,14 +1,40 @@ #pragma once -#include "server/connection_manager.h" -#include "common/config/resdb_config.h" +#include +#include +#include +#include + +namespace resdb { +class Socket; +class Request; +} + +struct LearnerConfig { + std::string ip = "127.0.0.1"; + int port = 0; + int block_size = 0; +}; class Learner { -public: - explicit Learner(const std::string& config_path); - void Run(); + public: + explicit Learner(const std::string& config_path); + void Run(); private: - resdb::ResDBConfig config_; - resdb::ConnectionManager manager_; -}; + static LearnerConfig LoadConfig(const std::string& config_path); + void HandleClient(std::unique_ptr socket) const; + void ProcessBroadcast(const std::string& payload) const; + void MetricsLoop() const; + void PrintMetrics() const; +private: + LearnerConfig config_; + std::atomic is_running_{false}; + mutable std::atomic total_messages_{0}; + mutable std::atomic total_bytes_{0}; + mutable std::atomic last_type_{-1}; + mutable std::atomic last_seq_{0}; + mutable std::atomic last_sender_{-1}; + mutable std::atomic last_payload_bytes_{0}; + mutable std::thread metrics_thread_; +}; diff --git a/platform/learner/learner_main.cpp b/platform/learner/learner_main.cpp index 1d416c477c..9c74ec790c 100644 --- a/platform/learner/learner_main.cpp +++ b/platform/learner/learner_main.cpp @@ -1,8 +1,27 @@ -#include "learner/learner.h" +#include "platform/learner/learner.h" + +#include +#include +#include + +std::string ResolveConfigPath(int argc, char** argv) { + std::string config_path = "platform/learner/learner.config"; + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "-config" && i + 1 < argc) { + config_path = argv[i + 1]; + break; + } + } + return config_path; +} int main(int argc, char** argv) { - Learner learner("config/learner.config"); - learner.Run(); + try { + Learner learner(ResolveConfigPath(argc, argv)); + learner.Run(); + } catch (const std::exception& e) { + std::cerr << "Failed to start learner: " << e.what() << std::endl; + return 1; + } return 0; } - diff --git a/platform/networkstrate/consensus_manager.cpp b/platform/networkstrate/consensus_manager.cpp index b3fb106253..399f6a5181 100644 --- a/platform/networkstrate/consensus_manager.cpp +++ b/platform/networkstrate/consensus_manager.cpp @@ -47,6 +47,11 @@ ConsensusManager::ConsensusManager(const ResDBConfig& config) config_.GetPrivateKey(), config_.GetPublicKeyCertificateInfo()); } bc_client_ = GetReplicaClient(config_.GetReplicaInfos(), true); + + // Add learners + for (const auto& learner : config_.GetLearnerInfos()) { + AddNewClient(learner); + } } ConsensusManager::~ConsensusManager() { diff --git a/platform/proto/replica_info.proto b/platform/proto/replica_info.proto index eaeac73e16..668c19205d 100644 --- a/platform/proto/replica_info.proto +++ b/platform/proto/replica_info.proto @@ -58,12 +58,16 @@ message ResConfigData{ optional bool enable_resview = 23; optional bool enable_faulty_switch = 24; -// for hotstuff. + // for hotstuff. optional bool use_chain_hotstuff = 9; optional int32 max_client_complaint_num = 21; optional int32 duplicate_check_frequency_useconds = 22; + + // learner + optional int32 block_size = 33; + repeated ReplicaInfo learner_info = 34; } message ReplicaStates { diff --git a/service/tools/kv/server_tools/start_kv_service_learner.sh b/service/tools/kv/server_tools/start_kv_service_learner.sh index bfa13f1d5c..e390e9a0fe 100755 --- a/service/tools/kv/server_tools/start_kv_service_learner.sh +++ b/service/tools/kv/server_tools/start_kv_service_learner.sh @@ -23,11 +23,11 @@ SERVER_PATH=./bazel-bin/service/kv/kv_service SERVER_CONFIG=service/tools/config/server/server.config WORK_PATH=$PWD CERT_PATH=${WORK_PATH}/service/tools/data/cert/ -LEARNER_PATH=./bazel-bin/src/learner/learner -LEARNER_CONFIG=$PWD/learner/learner.config +LEARNER_PATH=./bazel-bin/platform/learner/learner +LEARNER_CONFIG=$PWD/platform/learner/learner.config # start both builds -bazel build //learner:learner $@ +bazel build //platform/learner:learner $@ bazel build //service/kv:kv_service $@ nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node1.key.pri $CERT_PATH/cert_1.cert > logs/server0.log & From e16452489a924454e4d1c2be02a9272033f8ce6c Mon Sep 17 00:00:00 2001 From: eshabram Date: Tue, 18 Nov 2025 10:10:57 -0800 Subject: [PATCH 06/41] save --- .gitignore | 9 ++---- entrypoint.sh | 2 ++ service/tools/config/interface/service.config | 2 ++ service/tools/config/server.config | 5 ++++ service/tools/config/server/server.config | 28 +++++++++++++++++++ 5 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 service/tools/config/interface/service.config create mode 100644 service/tools/config/server.config create mode 100644 service/tools/config/server/server.config diff --git a/.gitignore b/.gitignore index d333fd9dfc..e0ab3dd4e6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,13 +4,13 @@ scripts/deploy/config_out/ deploy/kv_server/output/ .idea/ .vscode/ -.DS_Store +**/*.DS_Store bazel-* **/*.log *.bak venv sdk_validator/venv -__pycache__ +**/*__pycache__ MODULE.* apache_release *.out.* @@ -21,10 +21,5 @@ resdb/ 100*_db/ gmon.out .history/ -**/*.cert -service/tools/config/server/server.config -service/tools/config/interface/service.config -service/tools/config/server.config -service/tools/data/cert/* **/*AGENTS.md diff --git a/entrypoint.sh b/entrypoint.sh index 3b9a856eaf..96810abd87 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -31,6 +31,8 @@ if [ "$1" = "-w" ]; then watch -n 1 tail -n "$lines" logs/*.log elif [ "$1" = "-l" ]; then tail -f logs/learner.log +elif [ "$1" = "-a" ]; then + tail -f logs/*.log else tail -f /dev/null fi diff --git a/service/tools/config/interface/service.config b/service/tools/config/interface/service.config new file mode 100644 index 0000000000..2214ed91a0 --- /dev/null +++ b/service/tools/config/interface/service.config @@ -0,0 +1,2 @@ + +5 127.0.0.1 20005 diff --git a/service/tools/config/server.config b/service/tools/config/server.config new file mode 100644 index 0000000000..16ace299bd --- /dev/null +++ b/service/tools/config/server.config @@ -0,0 +1,5 @@ + +1 127.0.0.1 20001 +2 127.0.0.1 20002 +3 127.0.0.1 20003 +4 127.0.0.1 20004 diff --git a/service/tools/config/server/server.config b/service/tools/config/server/server.config new file mode 100644 index 0000000000..1cf8416f85 --- /dev/null +++ b/service/tools/config/server/server.config @@ -0,0 +1,28 @@ +{ + "region": [ + { + "replicaInfo": [ + { + "id": "1", + "ip": "127.0.0.1", + "port": 20001 + }, + { + "id": "2", + "ip": "127.0.0.1", + "port": 20002 + }, + { + "id": "3", + "ip": "127.0.0.1", + "port": 20003 + }, + { + "id": "4", + "ip": "127.0.0.1", + "port": 20004 + } + ] + } + ] +} \ No newline at end of file From 25e5f34a150deaf890d5852ce71bc86a140c0bbd Mon Sep 17 00:00:00 2001 From: eshabram Date: Tue, 18 Nov 2025 12:19:16 -0800 Subject: [PATCH 07/41] fixed config gen and keys --- .gitignore | 5 --- service/tools/config/interface/service.config | 2 + service/tools/config/server.config | 5 +++ service/tools/config/server/server.config | 36 ++++++++++++++++++ service/tools/data/cert/admin.key.pri | 2 + service/tools/data/cert/admin.key.pub | 2 + service/tools/data/cert/cert_1.cert | Bin 0 -> 167 bytes service/tools/data/cert/cert_2.cert | 7 ++++ service/tools/data/cert/cert_3.cert | 7 ++++ service/tools/data/cert/cert_4.cert | 6 +++ service/tools/data/cert/cert_5.cert | 7 ++++ service/tools/data/cert/cert_6.cert | Bin 0 -> 166 bytes service/tools/data/cert/cert_7.cert | 6 +++ service/tools/data/cert/cert_8.cert | Bin 0 -> 166 bytes service/tools/data/cert/cert_9.cert | 7 ++++ service/tools/data/cert/node1.key.pri | 2 + service/tools/data/cert/node1.key.pub | 3 ++ service/tools/data/cert/node2.key.pri | 2 + service/tools/data/cert/node2.key.pub | 2 + service/tools/data/cert/node3.key.pri | Bin 0 -> 36 bytes service/tools/data/cert/node3.key.pub | 3 ++ service/tools/data/cert/node4.key.pri | Bin 0 -> 36 bytes service/tools/data/cert/node4.key.pub | 2 + service/tools/data/cert/node5.key.pri | 2 + service/tools/data/cert/node5.key.pub | 3 ++ service/tools/data/cert/node6.key.key.pri | 2 + service/tools/data/cert/node6.key.key.pub | 2 + service/tools/data/cert/node6.key.pri | 2 + service/tools/data/cert/node6.key.pub | 2 + service/tools/data/cert/node7.key.key.pri | 2 + service/tools/data/cert/node7.key.key.pub | 2 + service/tools/data/cert/node7.key.pri | 2 + service/tools/data/cert/node7.key.pub | 2 + service/tools/data/cert/node8.key.key.pri | 2 + service/tools/data/cert/node8.key.key.pub | 2 + service/tools/data/cert/node8.key.pri | 2 + service/tools/data/cert/node8.key.pub | 2 + service/tools/data/cert/node9.key.key.pri | 2 + service/tools/data/cert/node9.key.key.pub | 2 + service/tools/data/cert/node9.key.pri | 2 + service/tools/data/cert/node9.key.pub | 2 + .../tools/kv/server_tools/generate_config.sh | 27 +++++++++++++ 42 files changed, 165 insertions(+), 5 deletions(-) create mode 100644 service/tools/config/interface/service.config create mode 100644 service/tools/config/server.config create mode 100644 service/tools/config/server/server.config create mode 100644 service/tools/data/cert/admin.key.pri create mode 100644 service/tools/data/cert/admin.key.pub create mode 100644 service/tools/data/cert/cert_1.cert create mode 100644 service/tools/data/cert/cert_2.cert create mode 100644 service/tools/data/cert/cert_3.cert create mode 100644 service/tools/data/cert/cert_4.cert create mode 100644 service/tools/data/cert/cert_5.cert create mode 100644 service/tools/data/cert/cert_6.cert create mode 100644 service/tools/data/cert/cert_7.cert create mode 100644 service/tools/data/cert/cert_8.cert create mode 100644 service/tools/data/cert/cert_9.cert create mode 100644 service/tools/data/cert/node1.key.pri create mode 100644 service/tools/data/cert/node1.key.pub create mode 100644 service/tools/data/cert/node2.key.pri create mode 100644 service/tools/data/cert/node2.key.pub create mode 100644 service/tools/data/cert/node3.key.pri create mode 100644 service/tools/data/cert/node3.key.pub create mode 100644 service/tools/data/cert/node4.key.pri create mode 100644 service/tools/data/cert/node4.key.pub create mode 100644 service/tools/data/cert/node5.key.pri create mode 100644 service/tools/data/cert/node5.key.pub create mode 100644 service/tools/data/cert/node6.key.key.pri create mode 100644 service/tools/data/cert/node6.key.key.pub create mode 100644 service/tools/data/cert/node6.key.pri create mode 100644 service/tools/data/cert/node6.key.pub create mode 100644 service/tools/data/cert/node7.key.key.pri create mode 100644 service/tools/data/cert/node7.key.key.pub create mode 100644 service/tools/data/cert/node7.key.pri create mode 100644 service/tools/data/cert/node7.key.pub create mode 100644 service/tools/data/cert/node8.key.key.pri create mode 100644 service/tools/data/cert/node8.key.key.pub create mode 100644 service/tools/data/cert/node8.key.pri create mode 100644 service/tools/data/cert/node8.key.pub create mode 100644 service/tools/data/cert/node9.key.key.pri create mode 100644 service/tools/data/cert/node9.key.key.pub create mode 100644 service/tools/data/cert/node9.key.pri create mode 100644 service/tools/data/cert/node9.key.pub diff --git a/.gitignore b/.gitignore index d333fd9dfc..16ae8eda2e 100644 --- a/.gitignore +++ b/.gitignore @@ -21,10 +21,5 @@ resdb/ 100*_db/ gmon.out .history/ -**/*.cert -service/tools/config/server/server.config -service/tools/config/interface/service.config -service/tools/config/server.config -service/tools/data/cert/* **/*AGENTS.md diff --git a/service/tools/config/interface/service.config b/service/tools/config/interface/service.config new file mode 100644 index 0000000000..2214ed91a0 --- /dev/null +++ b/service/tools/config/interface/service.config @@ -0,0 +1,2 @@ + +5 127.0.0.1 20005 diff --git a/service/tools/config/server.config b/service/tools/config/server.config new file mode 100644 index 0000000000..16ace299bd --- /dev/null +++ b/service/tools/config/server.config @@ -0,0 +1,5 @@ + +1 127.0.0.1 20001 +2 127.0.0.1 20002 +3 127.0.0.1 20003 +4 127.0.0.1 20004 diff --git a/service/tools/config/server/server.config b/service/tools/config/server/server.config new file mode 100644 index 0000000000..7442dce7bd --- /dev/null +++ b/service/tools/config/server/server.config @@ -0,0 +1,36 @@ +{ + "region": [ + { + "replicaInfo": [ + { + "id": "1", + "ip": "127.0.0.1", + "port": 20001 + }, + { + "id": "2", + "ip": "127.0.0.1", + "port": 20002 + }, + { + "id": "3", + "ip": "127.0.0.1", + "port": 20003 + }, + { + "id": "4", + "ip": "127.0.0.1", + "port": 20004 + } + ] + } + ], + "learnerInfo": [ + { + "id": 6, + "ip": "127.0.0.1", + "port": 25000 + } + ], + "block_size": 100 +} \ No newline at end of file diff --git a/service/tools/data/cert/admin.key.pri b/service/tools/data/cert/admin.key.pri new file mode 100644 index 0000000000..7629a70a96 --- /dev/null +++ b/service/tools/data/cert/admin.key.pri @@ -0,0 +1,2 @@ + + |quwq&07!6fa's| \ No newline at end of file diff --git a/service/tools/data/cert/admin.key.pub b/service/tools/data/cert/admin.key.pub new file mode 100644 index 0000000000..4924810534 --- /dev/null +++ b/service/tools/data/cert/admin.key.pub @@ -0,0 +1,2 @@ + + u>,Nb+Rm.dQ|;V;݅  \ No newline at end of file diff --git a/service/tools/data/cert/cert_1.cert b/service/tools/data/cert/cert_1.cert new file mode 100644 index 0000000000000000000000000000000000000000..a0a6ede63660a252e643bc0eaf3a2fbe923d5861 GIT binary patch literal 167 zcmV;Y09gMDBnlvPK8>s_>deW_PGT!kC=NeUyT%MRs10T2QZ0U`-8GB++TE-)@JD50DI z5<(h4$9#w*JvDy_aT8s<5%yRwj}o^`!nwJuv$qHf!3GilbUFw5-eIw$aDWY@PJpHM VB@XSKyW#0cM<|L7$t6$,Nb+Rm.dQ|;V;݅ } +7 +$ + 󗑉EjK' =zp@rE" 127.0.0.1(B@Ol +Ɖ瀥5&eJXBO.kP'yᆰy:kI}m=.5 $  \ No newline at end of file diff --git a/service/tools/data/cert/cert_3.cert b/service/tools/data/cert/cert_3.cert new file mode 100644 index 0000000000..58899e5754 --- /dev/null +++ b/service/tools/data/cert/cert_3.cert @@ -0,0 +1,7 @@ + +$ + u>,Nb+Rm.dQ|;V;݅ } +7 +$ + |a9 #Y,Nb+Rm.dQ|;V;݅ } +7 +$ + { aLx2٢~l(}Z{yqaYX^0$N'C"Tz}\-  \ No newline at end of file diff --git a/service/tools/data/cert/cert_5.cert b/service/tools/data/cert/cert_5.cert new file mode 100644 index 0000000000..a1ebad5af6 --- /dev/null +++ b/service/tools/data/cert/cert_5.cert @@ -0,0 +1,7 @@ + +$ + u>,Nb+Rm.dQ|;V;݅  +9 +$ + HY +1hܘhT]%%꡽eyV" 127.0.0.1(B@! VH` NTô0 1oN9PTjk|UUt@&n}*W  \ No newline at end of file diff --git a/service/tools/data/cert/cert_6.cert b/service/tools/data/cert/cert_6.cert new file mode 100644 index 0000000000000000000000000000000000000000..92e27c8883c80cb59234ec2f367ce158204fab92 GIT binary patch literal 166 zcmV;X09pSEBnlvPK8>s_>deW_PGT!kKLPa<>L^3%tG)6@;L^C!wMlm)tGC4CuIYmN4HxL651|kVDGB++TE-)@JD3(qV zLK;Bx91H=1+SB770@jI__B3;KvEh%r%MvBpdaQgU1fmDhtcqS=VZs^B(>&L&ICoxX U2)ojt=evWiey#{H9gF}V21Hmp+W-In literal 0 HcmV?d00001 diff --git a/service/tools/data/cert/cert_7.cert b/service/tools/data/cert/cert_7.cert new file mode 100644 index 0000000000..982a0b91eb --- /dev/null +++ b/service/tools/data/cert/cert_7.cert @@ -0,0 +1,6 @@ + +$ + u>,Nb+Rm.dQ|;V;݅ | +6 +$ + 374850F7F3BBE65DF6483A6D4D90AFAE" 127.0.0.1(NB@\~nMm;$iUBKEwcڃ;G#ӞݽQ眮zf6  \ No newline at end of file diff --git a/service/tools/data/cert/cert_8.cert b/service/tools/data/cert/cert_8.cert new file mode 100644 index 0000000000000000000000000000000000000000..e392ad442c7be38fd0a2b47c47c918324558c206 GIT binary patch literal 166 zcmV;X09pSEBnlvPK8>s_>deW_PGT!kKLPa<>L^3%tG)6@;L^C!wMlm)tGC4CuIYmN4HxL652qFnFGB++TE-)@JD40$X zLK;AoJbTkmQM1UY0Pn0;3(_djzHU>0G*0B!v=$(lNPMEHSQ$e~r+~F57wYu7K&Yd! UW%rKzZWmbTwUlQD1M&?Z2t9s0GXMYp literal 0 HcmV?d00001 diff --git a/service/tools/data/cert/cert_9.cert b/service/tools/data/cert/cert_9.cert new file mode 100644 index 0000000000..feb0ef7ed6 --- /dev/null +++ b/service/tools/data/cert/cert_9.cert @@ -0,0 +1,7 @@ + +$ + u>,Nb+Rm.dQ|;V;݅ | +6 +$ + 194BE86D2924FE3D367F164293D9EBC7 " 127.0.0.1(NB@jTq`z=G_ F.ܒl"t.LtTj/q@;" +PJx \ No newline at end of file diff --git a/service/tools/data/cert/node1.key.pri b/service/tools/data/cert/node1.key.pri new file mode 100644 index 0000000000..f6306d8043 --- /dev/null +++ b/service/tools/data/cert/node1.key.pri @@ -0,0 +1,2 @@ + + %A^fK@W7c|K% \ No newline at end of file diff --git a/service/tools/data/cert/node1.key.pub b/service/tools/data/cert/node1.key.pub new file mode 100644 index 0000000000..583c105cad --- /dev/null +++ b/service/tools/data/cert/node1.key.pub @@ -0,0 +1,3 @@ + + A/{?"XU,lErI +#A \ No newline at end of file diff --git a/service/tools/data/cert/node2.key.pri b/service/tools/data/cert/node2.key.pri new file mode 100644 index 0000000000..b5e05751e2 --- /dev/null +++ b/service/tools/data/cert/node2.key.pri @@ -0,0 +1,2 @@ + + @ literal 0 HcmV?d00001 diff --git a/service/tools/data/cert/node4.key.pub b/service/tools/data/cert/node4.key.pub new file mode 100644 index 0000000000..25f8b40f80 --- /dev/null +++ b/service/tools/data/cert/node4.key.pub @@ -0,0 +1,2 @@ + + { aLx2٢ Date: Tue, 18 Nov 2025 20:39:25 -0800 Subject: [PATCH 08/41] replicas write to learners when modified --- .../consensus/ordering/pbft/commitment.cpp | 8 +++++ .../ordering/pbft/message_manager.cpp | 34 +++++++++++++++++++ platform/learner/learner.cpp | 31 +++++++++++++---- platform/learner/learner.h | 3 ++ .../networkstrate/replica_communicator.cpp | 12 +++---- platform/proto/resdb.proto | 4 ++- 6 files changed, 76 insertions(+), 16 deletions(-) diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index 9990cd584f..4299402a66 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -322,9 +322,17 @@ int Commitment::PostProcessExecutedMsg() { // LOG(ERROR)<<"send back to proxy:"<proxy_id(); batch_resp->SerializeToString(request.mutable_data()); replica_communicator_->SendMessage(request, request.proxy_id()); + uint64_t set_cnt = batch_resp->set_count(); + uint64_t read_cnt = batch_resp->read_count(); + uint64_t delete_cnt = batch_resp->delete_count(); + // Also mirror the executed request to every learner so they can consume // the committed stream. for (const auto& learner : config_.GetLearnerInfos()) { + LOG(ERROR) << "Forward commit to learner id " << learner.id() + << " seq " << request.seq() + << " set=" << set_cnt << " read=" << read_cnt + << " delete=" << delete_cnt; replica_communicator_->SendMessage(request, learner.id()); } } diff --git a/platform/consensus/ordering/pbft/message_manager.cpp b/platform/consensus/ordering/pbft/message_manager.cpp index cc5e187c67..46eda527ec 100644 --- a/platform/consensus/ordering/pbft/message_manager.cpp +++ b/platform/consensus/ordering/pbft/message_manager.cpp @@ -22,6 +22,7 @@ #include #include "common/utils/utils.h" +#include "proto/kv/kv.pb.h" namespace resdb { @@ -48,6 +49,39 @@ MessageManager::MessageManager( resp_msg->set_seq(request->seq()); resp_msg->set_current_view(request->current_view()); resp_msg->set_primary_id(GetCurrentPrimary()); + uint64_t set_cnt = 0; + uint64_t read_cnt = 0; + uint64_t delete_cnt = 0; + BatchUserRequest batch_request; + if (batch_request.ParseFromString(request->data())) { + for (const auto& user_req : batch_request.user_requests()) { + resdb::KVRequest kv_request; + if (!kv_request.ParseFromString(user_req.request().data())) { + continue; + } + switch (kv_request.cmd()) { + case resdb::KVRequest::SET: + case resdb::KVRequest::SET_WITH_VERSION: + ++set_cnt; + break; + case resdb::KVRequest::GET: + case resdb::KVRequest::GETALLVALUES: + case resdb::KVRequest::GETRANGE: + case resdb::KVRequest::GET_WITH_VERSION: + case resdb::KVRequest::GET_ALL_ITEMS: + case resdb::KVRequest::GET_KEY_RANGE: + case resdb::KVRequest::GET_HISTORY: + case resdb::KVRequest::GET_TOP: + ++read_cnt; + break; + default: + break; + } + } + } + resp_msg->set_set_count(set_cnt); + resp_msg->set_read_count(read_cnt); + resp_msg->set_delete_count(delete_cnt); if (transaction_executor_->NeedResponse() && resp_msg->proxy_id() != 0) { queue_.Push(std::move(resp_msg)); diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index eb54cc2f96..299e8861a8 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -63,7 +63,7 @@ void Learner::Run() { while (is_running_.load()) { auto client = server.Accept(); if (!client) { - continue; + continue; } std::thread(&Learner::HandleClient, this, std::move(client)).detach(); } @@ -76,17 +76,17 @@ void Learner::HandleClient(std::unique_ptr socket) const { size_t len = 0; int ret = socket->Recv(&buffer, &len); if (ret <= 0) { - if (buffer != nullptr) { - free(buffer); - } - break; + if (buffer != nullptr) { + free(buffer); + } + break; } std::string payload(static_cast(buffer), len); free(buffer); ProcessBroadcast(payload); } - std::cout << "[Learner] replica disconnected" << std::endl; + // std::cout << "[Learner] replica disconnected" << std::endl; } void Learner::ProcessBroadcast(const std::string& payload) const { @@ -109,6 +109,12 @@ void Learner::ProcessBroadcast(const std::string& payload) const { last_seq_.store(request.seq()); last_sender_.store(request.sender_id()); last_payload_bytes_.store(request.data().size()); + resdb::BatchUserResponse batch_response; + if (batch_response.ParseFromString(request.data())) { + total_sets_.fetch_add(batch_response.set_count()); + total_reads_.fetch_add(batch_response.read_count()); + total_deletes_.fetch_add(batch_response.delete_count()); + } return; } @@ -121,8 +127,16 @@ void Learner::ProcessBroadcast(const std::string& payload) const { } void Learner::MetricsLoop() const { + uint64_t last_messages = 0; + uint64_t last_bytes = 0; while (is_running_.load()) { - PrintMetrics(); + uint64_t current_messages = total_messages_.load(); + uint64_t current_bytes = total_bytes_.load(); + if (current_messages != last_messages || current_bytes != last_bytes) { + PrintMetrics(); + last_messages = current_messages; + last_bytes = current_bytes; + } std::this_thread::sleep_for(std::chrono::seconds(1)); } } @@ -130,6 +144,9 @@ void Learner::MetricsLoop() const { void Learner::PrintMetrics() const { std::cout << "[Learner] broadcasts=" << total_messages_.load() << " bytes=" << total_bytes_.load() + << " sets=" << total_sets_.load() + << " reads=" << total_reads_.load() + << " deletes=" << total_deletes_.load() << " last_type=" << last_type_.load() << " last_seq=" << last_seq_.load() << " sender=" << last_sender_.load() diff --git a/platform/learner/learner.h b/platform/learner/learner.h index 86ce87a4cb..91e98e2293 100644 --- a/platform/learner/learner.h +++ b/platform/learner/learner.h @@ -32,6 +32,9 @@ class Learner { std::atomic is_running_{false}; mutable std::atomic total_messages_{0}; mutable std::atomic total_bytes_{0}; + mutable std::atomic total_sets_{0}; + mutable std::atomic total_reads_{0}; + mutable std::atomic total_deletes_{0}; mutable std::atomic last_type_{-1}; mutable std::atomic last_seq_{0}; mutable std::atomic last_sender_{-1}; diff --git a/platform/networkstrate/replica_communicator.cpp b/platform/networkstrate/replica_communicator.cpp index f1521acb49..44bb8fee02 100644 --- a/platform/networkstrate/replica_communicator.cpp +++ b/platform/networkstrate/replica_communicator.cpp @@ -206,16 +206,12 @@ int ReplicaCommunicator::SendMessage(const google::protobuf::Message& message) { int ReplicaCommunicator::SendMessage(const google::protobuf::Message& message, const ReplicaInfo& replica_info) { - return SendSingleMessage(message, replica_info); - - if (is_use_long_conn_) { - std::string data = NetChannel::GetRawMessageString(message, verifier_); - BroadcastData broadcast_data; - broadcast_data.add_data()->swap(data); - return SendMessageFromPool(broadcast_data, {replica_info}); - } else { + // Learners/clients aren't part of the replica pool, so send directly on the + // configured port instead of the long-connection (+10000) channel. + if (!IsInPool(replica_info)) { return SendMessageInternal(message, {replica_info}); } + return SendSingleMessage(message, replica_info); } int ReplicaCommunicator::SendBatchMessage( diff --git a/platform/proto/resdb.proto b/platform/proto/resdb.proto index 47edac3833..111dfdb145 100644 --- a/platform/proto/resdb.proto +++ b/platform/proto/resdb.proto @@ -131,6 +131,9 @@ message BatchUserResponse { uint64 local_id = 7; bytes hash = 8; int32 primary_id = 9; + uint64 set_count = 10; + uint64 read_count = 11; + uint64 delete_count = 12; } message HeartBeatInfo{ @@ -206,4 +209,3 @@ message QueryResponse { message CustomQueryResponse { bytes resp_str = 1; } - From 4ea1e80986a5898f4d140007669202855df8879c Mon Sep 17 00:00:00 2001 From: eshabram Date: Tue, 18 Nov 2025 21:24:04 -0800 Subject: [PATCH 09/41] add headers --- platform/learner/learner.cpp | 19 +++++++++++++++++++ platform/learner/learner.h | 19 +++++++++++++++++++ platform/learner/learner_main.cpp | 19 +++++++++++++++++++ .../wallet_tool/cpp/server_config0.config | 19 +++++++++++++++++++ service/utxo/config/server_config.config | 19 +++++++++++++++++++ service/utxo/config/utxo_config.config | 18 ++++++++++++++++++ 6 files changed, 113 insertions(+) diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index 299e8861a8..cfd7dfe056 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + #include "platform/learner/learner.h" #include diff --git a/platform/learner/learner.h b/platform/learner/learner.h index 91e98e2293..13ee383af6 100644 --- a/platform/learner/learner.h +++ b/platform/learner/learner.h @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + #pragma once #include #include diff --git a/platform/learner/learner_main.cpp b/platform/learner/learner_main.cpp index 9c74ec790c..678fad3879 100644 --- a/platform/learner/learner_main.cpp +++ b/platform/learner/learner_main.cpp @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + #include "platform/learner/learner.h" #include diff --git a/service/tools/utxo/wallet_tool/cpp/server_config0.config b/service/tools/utxo/wallet_tool/cpp/server_config0.config index ee48298e41..17308dfc68 100644 --- a/service/tools/utxo/wallet_tool/cpp/server_config0.config +++ b/service/tools/utxo/wallet_tool/cpp/server_config0.config @@ -1 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + 1 127.0.0.1 10001 diff --git a/service/utxo/config/server_config.config b/service/utxo/config/server_config.config index ec284753a5..3ab099068e 100644 --- a/service/utxo/config/server_config.config +++ b/service/utxo/config/server_config.config @@ -1,3 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + { region : { replica_info : { diff --git a/service/utxo/config/utxo_config.config b/service/utxo/config/utxo_config.config index 237ad28225..31a9e7656c 100644 --- a/service/utxo/config/utxo_config.config +++ b/service/utxo/config/utxo_config.config @@ -1,3 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# { genesis_transactions: { From 7d3f8b8200da273b77382cb79cc71c4842ee35fa Mon Sep 17 00:00:00 2001 From: eshabram Date: Tue, 18 Nov 2025 21:39:06 -0800 Subject: [PATCH 10/41] add license --- api/ip_address.config | 19 +++++++++++++++++++ .../api_tools/example_contract/token.json | 1 + .../kv/api_tools/example_contract/token.json | 17 +++++++++++++++++ .../utxo/wallet_tool/cpp/client_config.config | 19 +++++++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/api/ip_address.config b/api/ip_address.config index e119673ead..f0837c4df6 100644 --- a/api/ip_address.config +++ b/api/ip_address.config @@ -1 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + 5 44.193.63.142 17005 \ No newline at end of file diff --git a/service/tools/contract/api_tools/example_contract/token.json b/service/tools/contract/api_tools/example_contract/token.json index eaeaa36ff9..ebac04a78b 100644 --- a/service/tools/contract/api_tools/example_contract/token.json +++ b/service/tools/contract/api_tools/example_contract/token.json @@ -1,3 +1,4 @@ + { "contracts": { "token.sol:Token": { diff --git a/service/tools/kv/api_tools/example_contract/token.json b/service/tools/kv/api_tools/example_contract/token.json index eaeaa36ff9..b030794123 100644 --- a/service/tools/kv/api_tools/example_contract/token.json +++ b/service/tools/kv/api_tools/example_contract/token.json @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + { "contracts": { "token.sol:Token": { diff --git a/service/tools/utxo/wallet_tool/cpp/client_config.config b/service/tools/utxo/wallet_tool/cpp/client_config.config index e1bcc90547..c7b0cd3144 100644 --- a/service/tools/utxo/wallet_tool/cpp/client_config.config +++ b/service/tools/utxo/wallet_tool/cpp/client_config.config @@ -1 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + 5 127.0.0.1 10005 From a96fc72c0bcd3bb54ca10b1fe830e82baf7b0ada Mon Sep 17 00:00:00 2001 From: eshabram Date: Tue, 18 Nov 2025 21:51:37 -0800 Subject: [PATCH 11/41] changes license ignore --- .licenserc.yaml | 2 ++ .../kv/api_tools/example_contract/token.json | 17 ----------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/.licenserc.yaml b/.licenserc.yaml index badb8a63cd..fd21279d99 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -14,6 +14,8 @@ header: - 'dev/.rat-excludes' - 'documents/doxygen/.gitignore' - 'third_party/loc_script/src/index.js' + - '**/*.json' + - '**/*.config' diff --git a/service/tools/kv/api_tools/example_contract/token.json b/service/tools/kv/api_tools/example_contract/token.json index b030794123..eaeaa36ff9 100644 --- a/service/tools/kv/api_tools/example_contract/token.json +++ b/service/tools/kv/api_tools/example_contract/token.json @@ -1,20 +1,3 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - { "contracts": { "token.sol:Token": { From 5069734d807c206eeb148395fc63443a63fb5a72 Mon Sep 17 00:00:00 2001 From: eshabram Date: Tue, 18 Nov 2025 21:57:06 -0800 Subject: [PATCH 12/41] license --- .licenserc.yaml | 1 + logs/.gitkeep | 18 ++++++++++++++++++ platform/learner/BUILD | 19 +++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/.licenserc.yaml b/.licenserc.yaml index fd21279d99..2196156526 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -16,6 +16,7 @@ header: - 'third_party/loc_script/src/index.js' - '**/*.json' - '**/*.config' + - '**/*.conf' diff --git a/logs/.gitkeep b/logs/.gitkeep index e69de29bb2..5fb255256d 100644 --- a/logs/.gitkeep +++ b/logs/.gitkeep @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# \ No newline at end of file diff --git a/platform/learner/BUILD b/platform/learner/BUILD index 93bc617fa3..2a623e9431 100644 --- a/platform/learner/BUILD +++ b/platform/learner/BUILD @@ -1,3 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + package(default_visibility = ["//visibility:public"]) cc_library( From 366edc75687a80d28443188285f1d4da6fdfaf19 Mon Sep 17 00:00:00 2001 From: eshabram Date: Tue, 18 Nov 2025 22:00:58 -0800 Subject: [PATCH 13/41] license --- .licenserc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.licenserc.yaml b/.licenserc.yaml index 2196156526..068bb2193b 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -5,7 +5,7 @@ header: paths-ignore: - '.*' - - '.**/**' + - '.**/*' - '**/*.sol' - CNAME - DISCLAIMER From e6c09ba9ca32dca55995c08b36927735b899f1a5 Mon Sep 17 00:00:00 2001 From: eshabram Date: Tue, 18 Nov 2025 22:03:30 -0800 Subject: [PATCH 14/41] license --- .licenserc.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.licenserc.yaml b/.licenserc.yaml index 068bb2193b..3d01696f77 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -6,6 +6,13 @@ header: paths-ignore: - '.*' - '.**/*' + - "ecosystem/**/.gitignore" + - "ecosystem/**/.npmrc" + - "ecosystem/**/.prettierignore" + - "ecosystem/**/.prettierrc" + - "ecosystem/**/.bazelrc" + - "ecosystem/**/.licenserc.yaml" + - "ecosystem/**/dev/.rat-excludes" - '**/*.sol' - CNAME - DISCLAIMER From 709e5de779e41958d464f061a315cf09fd0a7857 Mon Sep 17 00:00:00 2001 From: eshabram Date: Tue, 18 Nov 2025 22:06:11 -0800 Subject: [PATCH 15/41] license --- .licenserc.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.licenserc.yaml b/.licenserc.yaml index 3d01696f77..417cbdf85e 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -6,13 +6,7 @@ header: paths-ignore: - '.*' - '.**/*' - - "ecosystem/**/.gitignore" - - "ecosystem/**/.npmrc" - - "ecosystem/**/.prettierignore" - - "ecosystem/**/.prettierrc" - - "ecosystem/**/.bazelrc" - - "ecosystem/**/.licenserc.yaml" - - "ecosystem/**/dev/.rat-excludes" + - "ecosystem/**/.*" - '**/*.sol' - CNAME - DISCLAIMER From dc602c1022aa7a8c05c6dbbfe93eb788588f3318 Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 20 Nov 2025 21:52:35 -0800 Subject: [PATCH 16/41] Adding new API endpoint for client to call to. --- api/kv_operation.py | 8 ++++++++ api/pybind_kv_service.cpp | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/api/kv_operation.py b/api/kv_operation.py index f18c846018..8bc1c647cb 100644 --- a/api/kv_operation.py +++ b/api/kv_operation.py @@ -44,3 +44,11 @@ def get_value(key: str or int or float, config_path: str = current_dir + "/ip_ad :return: A string of the key's corresponding value. """ return pybind_kv.get(str(key), os.path.abspath(config_path)) + +def get_value_ro(key: str | int | float, config_path: str = current_dir + "/ip_address.config") -> str: + """ + :param key: The key of the value you want to get in key value pair. + :param config_path: Default is connect to the main chain, users can specify the path to connect to their local blockchain without tracking the read. + :return: A string of the key's corresponding value. + """ + return pybind_kv.get_readonly(str(key), os.path.abspath(config_path)) diff --git a/api/pybind_kv_service.cpp b/api/pybind_kv_service.cpp index 013683fbd7..44741430fd 100644 --- a/api/pybind_kv_service.cpp +++ b/api/pybind_kv_service.cpp @@ -48,6 +48,18 @@ std::string get(std::string key, std::string config_path) { } } +std::string get_ro(std::string key, std::string config_path) { + ResDBConfig config = GenerateResDBConfig(config_path); + config.SetClientTimeoutMs(100000); + KVClient client(config); + auto result_ptr = client.GetReadOnly(key); + if (result_ptr) { + return *result_ptr; + } else { + return ""; + } +} + bool set(std::string key, std::string value, std::string config_path) { ResDBConfig config = GenerateResDBConfig(config_path); config.SetClientTimeoutMs(100000); @@ -62,5 +74,6 @@ bool set(std::string key, std::string value, std::string config_path) { PYBIND11_MODULE(pybind_kv, m) { m.def("get", &get, "A function that gets a value from the key-value store"); + m.def("get_readonly", &get_ro, "A function that gets a value from the key-value store with no tracking"); m.def("set", &set, "A function that sets a value in the key-value store"); } From d0092fc287f72fd7a7ea59ef50b5b958ba27a859 Mon Sep 17 00:00:00 2001 From: Nic Date: Sat, 22 Nov 2025 18:40:17 -0800 Subject: [PATCH 17/41] KV Client readonly prep --- interface/kv/kv_client.cpp | 13 +++++++++++++ interface/kv/kv_client.h | 1 + proto/kv/kv.proto | 1 + 3 files changed, 15 insertions(+) diff --git a/interface/kv/kv_client.cpp b/interface/kv/kv_client.cpp index 25526395b5..077f3ac45c 100644 --- a/interface/kv/kv_client.cpp +++ b/interface/kv/kv_client.cpp @@ -47,6 +47,19 @@ std::unique_ptr KVClient::Get(const std::string& key) { return std::make_unique(response.value()); } +std::unique_ptr KVClient::GetReadOnly(const std::string& key) { + KVRequest request; + request.set_cmd(KVRequest::GET_READ_ONLY); + request.set_key(key); + KVResponse response; + int ret = SendRequest(request, &response); + if (ret != 0) { + LOG(ERROR) << "send request fail, ret:" << ret; + return nullptr; + } + return std::make_unique(response.value()); +} + std::unique_ptr KVClient::GetAllValues() { KVRequest request; request.set_cmd(KVRequest::GETALLVALUES); diff --git a/interface/kv/kv_client.h b/interface/kv/kv_client.h index 52cbcab19d..cc3cc9435b 100644 --- a/interface/kv/kv_client.h +++ b/interface/kv/kv_client.h @@ -58,6 +58,7 @@ class KVClient : public TransactionConstructor { // above. int Set(const std::string& key, const std::string& data); std::unique_ptr Get(const std::string& key); + std::unique_ptr GetReadOnly(const std::string& key); std::unique_ptr GetAllValues(); std::unique_ptr GetRange(const std::string& min_key, const std::string& max_key); diff --git a/proto/kv/kv.proto b/proto/kv/kv.proto index 750058e753..99893309b7 100644 --- a/proto/kv/kv.proto +++ b/proto/kv/kv.proto @@ -34,6 +34,7 @@ message KVRequest { GET_KEY_RANGE = 8; GET_HISTORY = 9; GET_TOP = 10; + GET_READ_ONLY = 11; } CMD cmd = 1; string key = 2; From dd7bcda9ba43d2ba71c70f999c67899abb2afb3a Mon Sep 17 00:00:00 2001 From: netzworld Date: Sat, 22 Nov 2025 23:18:30 -0800 Subject: [PATCH 18/41] Exposed readonly function in kv_operation Bound readonly function in pybind_kv_service.cpp Setup KV executor for RO reqs. --- api/kv_operation.py | 4 ++-- api/pybind_kv_service.cpp | 2 +- executor/kv/kv_executor.cpp | 10 ++++++++++ executor/kv/kv_executor.h | 1 + 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/api/kv_operation.py b/api/kv_operation.py index 8bc1c647cb..3efcb0bbbf 100644 --- a/api/kv_operation.py +++ b/api/kv_operation.py @@ -45,10 +45,10 @@ def get_value(key: str or int or float, config_path: str = current_dir + "/ip_ad """ return pybind_kv.get(str(key), os.path.abspath(config_path)) -def get_value_ro(key: str | int | float, config_path: str = current_dir + "/ip_address.config") -> str: +def get_value_readonly(key: str | int | float, config_path: str = current_dir + "/ip_address.config") -> str: """ :param key: The key of the value you want to get in key value pair. :param config_path: Default is connect to the main chain, users can specify the path to connect to their local blockchain without tracking the read. :return: A string of the key's corresponding value. """ - return pybind_kv.get_readonly(str(key), os.path.abspath(config_path)) + return pybind_kv.get_value_readonly(str(key), os.path.abspath(config_path)) diff --git a/api/pybind_kv_service.cpp b/api/pybind_kv_service.cpp index 44741430fd..1454ebbbe5 100644 --- a/api/pybind_kv_service.cpp +++ b/api/pybind_kv_service.cpp @@ -74,6 +74,6 @@ bool set(std::string key, std::string value, std::string config_path) { PYBIND11_MODULE(pybind_kv, m) { m.def("get", &get, "A function that gets a value from the key-value store"); - m.def("get_readonly", &get_ro, "A function that gets a value from the key-value store with no tracking"); + m.def("get_value_readonly", &get_ro, "A function that gets a value from the key-value store with no tracking"); m.def("set", &set, "A function that sets a value in the key-value store"); } diff --git a/executor/kv/kv_executor.cpp b/executor/kv/kv_executor.cpp index c587f592ce..3703a635e1 100644 --- a/executor/kv/kv_executor.cpp +++ b/executor/kv/kv_executor.cpp @@ -48,6 +48,8 @@ std::unique_ptr KVExecutor::ExecuteRequest( Set(kv_request.key(), kv_request.value()); } else if (kv_request.cmd() == KVRequest::GET) { kv_response.set_value(Get(kv_request.key())); + } else if (kv_request.cmd() == KVRequest::GET_READ_ONLY) { + kv_response.set_value(GetReadOnly(kv_request.key())); } else if (kv_request.cmd() == KVRequest::GETALLVALUES) { kv_response.set_value(GetAllValues()); } else if (kv_request.cmd() == KVRequest::GETRANGE) { @@ -99,6 +101,8 @@ std::unique_ptr KVExecutor::ExecuteData( Set(kv_request.key(), kv_request.value()); } else if (kv_request.cmd() == KVRequest::GET) { kv_response.set_value(Get(kv_request.key())); + } else if (kv_request.cmd() == KVRequest::GET_READ_ONLY) { + kv_response.set_value(GetReadOnly(kv_request.key())); } else if (kv_request.cmd() == KVRequest::GETALLVALUES) { kv_response.set_value(GetAllValues()); } else if (kv_request.cmd() == KVRequest::GETRANGE) { @@ -144,6 +148,12 @@ std::string KVExecutor::Get(const std::string& key) { return storage_->GetValue(key); } +std::string KVExecutor::GetReadOnly(const std::string& key) { + LOG(ERROR)<<" get read only key:"<GetValue(key); +} + std::string KVExecutor::GetAllValues() { return storage_->GetAllValues(); } // Get values on a range of keys diff --git a/executor/kv/kv_executor.h b/executor/kv/kv_executor.h index fef1259725..f7a85e2c6f 100644 --- a/executor/kv/kv_executor.h +++ b/executor/kv/kv_executor.h @@ -43,6 +43,7 @@ class KVExecutor : public TransactionManager { protected: virtual void Set(const std::string& key, const std::string& value); std::string Get(const std::string& key); + std::string GetReadOnly(const std::string& key); std::string GetAllValues(); std::string GetRange(const std::string& min_key, const std::string& max_key); From 693cd85b241b9b41e2c2ea7ac4533474562c5746 Mon Sep 17 00:00:00 2001 From: netzworld Date: Sun, 23 Nov 2025 13:30:11 -0800 Subject: [PATCH 19/41] License match --- .licenserc.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.licenserc.yaml b/.licenserc.yaml index badb8a63cd..36a11e92b3 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -5,7 +5,8 @@ header: paths-ignore: - '.*' - - '.**/**' + - '.**/*' + - "ecosystem/**/.*" - '**/*.sol' - CNAME - DISCLAIMER @@ -14,8 +15,10 @@ header: - 'dev/.rat-excludes' - 'documents/doxygen/.gitignore' - 'third_party/loc_script/src/index.js' + - '**/*.json' + - '**/*.config' + - '**/*.conf' comment: on-failure - From 60e4e94b234c088e32fe7b2c9ad818b2c5db28ed Mon Sep 17 00:00:00 2001 From: Amey Bable Date: Wed, 26 Nov 2025 17:24:14 -0800 Subject: [PATCH 20/41] Add learner routing path + config + KV read-only support --- interface/kv/kv_client.cpp | 2 +- platform/config/resdb_config.cpp | 21 +++++++++++++++++++++ platform/config/resdb_config.h | 14 ++++++++++++++ platform/proto/replica_info.proto | 3 +++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/interface/kv/kv_client.cpp b/interface/kv/kv_client.cpp index 077f3ac45c..a4bde62bc3 100644 --- a/interface/kv/kv_client.cpp +++ b/interface/kv/kv_client.cpp @@ -52,7 +52,7 @@ std::unique_ptr KVClient::GetReadOnly(const std::string& key) { request.set_cmd(KVRequest::GET_READ_ONLY); request.set_key(key); KVResponse response; - int ret = SendRequest(request, &response); + int ret = SendReadOnlyRequest(request, &response); if (ret != 0) { LOG(ERROR) << "send request fail, ret:" << ret; return nullptr; diff --git a/platform/config/resdb_config.cpp b/platform/config/resdb_config.cpp index 23129a2584..f4863d6b30 100644 --- a/platform/config/resdb_config.cpp +++ b/platform/config/resdb_config.cpp @@ -58,6 +58,16 @@ ResDBConfig::ResDBConfig(const ResConfigData& config_data, break; } } + + //NEW: populate Learners_ from config_data + for (const auto& region : config_data.region()) { + if (region.region_id() == config_data.self_region_id()) { + for (const auto& learner : region.learner_info()) { + learners_.push_back(learner); + } + } + } + if (config_data_.view_change_timeout_ms() == 0) { config_data_.set_view_change_timeout_ms(viewchange_commit_timeout_ms_); } @@ -95,6 +105,17 @@ void ResDBConfig::SetConfigData(const ResConfigData& config_data) { break; } } + // NEW: keep Learners_ in sync with config_data_ + learners_.clear(); + for (const auto& region : config_data.region()) { + if (region.region_id() == config_data.self_region_id()) { + for (const auto& learner : region.learner_info()) { + learners_.push_back(learner); + } + break; + } + } + if (config_data_.view_change_timeout_ms() == 0) { config_data_.set_view_change_timeout_ms(viewchange_commit_timeout_ms_); } diff --git a/platform/config/resdb_config.h b/platform/config/resdb_config.h index 9867c6d5ab..889c165d2e 100644 --- a/platform/config/resdb_config.h +++ b/platform/config/resdb_config.h @@ -50,6 +50,16 @@ class ResDBConfig { // Each replica infomation, including the binding urls(or ip,port). const std::vector& GetReplicaInfos() const; + // NEW: Access Learner Nodes + const std::vector& GetLearnerInfos() const { + return learners_; + } + + // NEW: Add Learner Node + void AddLearnerInfo(const ReplicaInfo& info) { + learners_.push_back(info); + } + ResConfigData GetConfigData() const; // The current replica infomation, including the binding urls(or ip,port). @@ -124,6 +134,10 @@ class ResDBConfig { private: ResConfigData config_data_; std::vector replicas_; + + // NEW: Learner nodes for read only fast path + std::vector learners_; + ReplicaInfo self_info_; const KeyInfo private_key_; const CertificateInfo public_key_cert_info_; diff --git a/platform/proto/replica_info.proto b/platform/proto/replica_info.proto index eaeac73e16..5f91f3186f 100644 --- a/platform/proto/replica_info.proto +++ b/platform/proto/replica_info.proto @@ -34,6 +34,9 @@ message ReplicaInfo { message RegionInfo { repeated ReplicaInfo replica_info = 1; int32 region_id = 2; + +// NEW: Learner nodes for read-only routing + repeated ReplicaInfo learner_info = 3; } message ResConfigData{ From 85df29d4e743daf3a4eeada41abd347325616422 Mon Sep 17 00:00:00 2001 From: pprabhu2727 Date: Wed, 26 Nov 2025 21:34:47 -0800 Subject: [PATCH 21/41] Transaction constructor changes --- interface/rdbc/transaction_constructor.cpp | 41 ++++++++++++++++++++++ interface/rdbc/transaction_constructor.h | 8 +++++ 2 files changed, 49 insertions(+) diff --git a/interface/rdbc/transaction_constructor.cpp b/interface/rdbc/transaction_constructor.cpp index be42224aaf..afea31871b 100644 --- a/interface/rdbc/transaction_constructor.cpp +++ b/interface/rdbc/transaction_constructor.cpp @@ -85,5 +85,46 @@ int TransactionConstructor::SendRequest( } return -1; } +// NEW: Send a read-only request. If learner nodes are configured, route the +// request to a learner; otherwise fall back to a regular replica. + +int TransactionConstructor::SendReadOnlyRequest( + const google::protobuf::Message& message, Request::Type type) { + const auto& learners = config_.GetLearnerInfos(); + if (!learners.empty()) { + NetChannel::SetDestReplicaInfo(learners[0]); + } else { + NetChannel::SetDestReplicaInfo(config_.GetReplicaInfos()[0]); + } + + return NetChannel::SendRequest(message, type, false); +} + +// NEW: Send a read-only request and wait for a response. Routed to learners +// when available, with fallback to replicas. +int TransactionConstructor::SendReadOnlyRequest( + const google::protobuf::Message& message, + google::protobuf::Message* response, Request::Type type) { + const auto& learners = config_.GetLearnerInfos(); + if (!learners.empty()) { + NetChannel::SetDestReplicaInfo(learners[0]); + } else { + NetChannel::SetDestReplicaInfo(config_.GetReplicaInfos()[0]); + } + + int ret = NetChannel::SendRequest(message, type, true); + if (ret == 0) { + std::string resp_str; + int recv_ret = NetChannel::RecvRawMessageData(&resp_str); + if (recv_ret >= 0) { + if (!response->ParseFromString(resp_str)) { + LOG(ERROR) << "parse response fail:" << resp_str.size(); + return -2; + } + return 0; + } + } + return -1; +} } // namespace resdb diff --git a/interface/rdbc/transaction_constructor.h b/interface/rdbc/transaction_constructor.h index 39754a846c..bb05b6173a 100644 --- a/interface/rdbc/transaction_constructor.h +++ b/interface/rdbc/transaction_constructor.h @@ -39,7 +39,15 @@ class TransactionConstructor : public NetChannel { int SendRequest(const google::protobuf::Message& message, google::protobuf::Message* response, Request::Type type = Request::TYPE_CLIENT_REQUEST); + + // NEW: Send read-only request (routed to learners if available). + int SendReadOnlyRequest(const google::protobuf::Message& message, + Request::Type type = Request::TYPE_CLIENT_REQUEST); + // NEW: Send read-only request and wait for a response. + int SendReadOnlyRequest(const google::protobuf::Message& message, + google::protobuf::Message* response, + Request::Type type = Request::TYPE_CLIENT_REQUEST); private: absl::StatusOr GetResponseData(const Response& response); From 1b24d460a5d31c675454052ba829fb1214ea4a50 Mon Sep 17 00:00:00 2001 From: Ryan Nelson Date: Mon, 1 Dec 2025 16:41:06 -0800 Subject: [PATCH 22/41] starting request workarounds --- platform/consensus/ordering/pbft/commitment.cpp | 17 +++++++++++++++++ .../consensus/ordering/pbft/message_manager.cpp | 12 ++++++++++++ .../consensus/ordering/pbft/message_manager.h | 5 +++++ 3 files changed, 34 insertions(+) diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index 4d048331eb..fff1f7cda3 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -322,6 +322,23 @@ int Commitment::PostProcessExecutedMsg() { // LOG(ERROR)<<"send back to proxy:"<proxy_id(); batch_resp->SerializeToString(request.mutable_data()); replica_communicator_->SendMessage(request, request.proxy_id()); + uint64_t set_cnt = batch_resp->set_count(); + uint64_t read_cnt = batch_resp->read_count(); + uint64_t delete_cnt = batch_resp->delete_count(); + + if (request.seq % 50 == 0) { + std::vector> reqs_to_learner = message_manager_->getLearnerUpdateRequests(); + } + + // Also mirror the executed request to every learner so they can consume + // the committed stream. + for (const auto& learner : config_.GetLearnerInfos()) { + LOG(ERROR) << "Forward commit to learner id " << learner.id() + << " seq " << request.seq() + << " set=" << set_cnt << " read=" << read_cnt + << " delete=" << delete_cnt; + replica_communicator_->SendMessage(request, learner.id()); + } } return 0; } diff --git a/platform/consensus/ordering/pbft/message_manager.cpp b/platform/consensus/ordering/pbft/message_manager.cpp index cc5e187c67..974abfdbf3 100644 --- a/platform/consensus/ordering/pbft/message_manager.cpp +++ b/platform/consensus/ordering/pbft/message_manager.cpp @@ -48,6 +48,14 @@ MessageManager::MessageManager( resp_msg->set_seq(request->seq()); resp_msg->set_current_view(request->current_view()); resp_msg->set_primary_id(GetCurrentPrimary()); + + // copying request for learner update sake + auto request_copy = std::unique_ptr(request->New()); + request_copy->CopyFrom(*request); + + // adds request to the vector of requests waiting to be sent to learners + learner_update_requests_.push_back(std::move(request_copy)); + if (transaction_executor_->NeedResponse() && resp_msg->proxy_id() != 0) { queue_.Push(std::move(resp_msg)); @@ -290,4 +298,8 @@ LockFreeCollectorPool* MessageManager::GetCollectorPool() { return collector_pool_.get(); } +std::vector> MessageManager::getLearnerUpdateRequests() { + return std::move(learner_update_requests_); +} + } // namespace resdb diff --git a/platform/consensus/ordering/pbft/message_manager.h b/platform/consensus/ordering/pbft/message_manager.h index c51fd22a5c..ddb6337f68 100644 --- a/platform/consensus/ordering/pbft/message_manager.h +++ b/platform/consensus/ordering/pbft/message_manager.h @@ -112,6 +112,9 @@ class MessageManager { LockFreeCollectorPool* GetCollectorPool(); + // function to retrieved stored learner updates + std::vector> getLearnerUpdateRequests(); + private: bool IsValidMsg(const Request& request); @@ -139,6 +142,8 @@ class MessageManager { std::mutex lct_lock_; std::map last_committed_time_; + + std::vector> learner_update_requests_; }; } // namespace resdb From a3912dc678a2e38f603b65ce79f53f5c7e86848f Mon Sep 17 00:00:00 2001 From: eshabram Date: Mon, 1 Dec 2025 19:37:52 -0800 Subject: [PATCH 23/41] save --- api/ip_address.config | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/api/ip_address.config b/api/ip_address.config index f0837c4df6..0450a6c2f7 100644 --- a/api/ip_address.config +++ b/api/ip_address.config @@ -1,20 +1 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -5 44.193.63.142 17005 \ No newline at end of file +5 127.0.0.1 20005 From ea7ab2f4690251de8e2e142cd5117b41843fe337 Mon Sep 17 00:00:00 2001 From: eshabram Date: Mon, 1 Dec 2025 21:49:54 -0800 Subject: [PATCH 24/41] save --- api/learner-test.py | 8 ++++++++ platform/config/resdb_config.h | 12 +----------- platform/consensus/ordering/pbft/commitment.cpp | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) create mode 100644 api/learner-test.py diff --git a/api/learner-test.py b/api/learner-test.py new file mode 100644 index 0000000000..4bff83f5b1 --- /dev/null +++ b/api/learner-test.py @@ -0,0 +1,8 @@ +import sys, os, time +# Your path to ResilientDB api folder +sys.path.append(os.getcwd()) +from kv_operation import set_value, get_value + +print(set_value("test", "111225")) +time.sleep(0.5) +print(f"Key 'test' value: {get_value('test')}") \ No newline at end of file diff --git a/platform/config/resdb_config.h b/platform/config/resdb_config.h index 5fd8bed841..b90a7bf800 100644 --- a/platform/config/resdb_config.h +++ b/platform/config/resdb_config.h @@ -50,16 +50,6 @@ class ResDBConfig { // Each replica infomation, including the binding urls(or ip,port). const std::vector& GetReplicaInfos() const; - // NEW: Access Learner Nodes - const std::vector& GetLearnerInfos() const { - return learners_; - } - - // NEW: Add Learner Node - void AddLearnerInfo(const ReplicaInfo& info) { - learners_.push_back(info); - } - ResConfigData GetConfigData() const; // The current replica infomation, including the binding urls(or ip,port). @@ -142,7 +132,7 @@ class ResDBConfig { std::vector replicas_; // NEW: Learner nodes for read only fast path - std::vector learners_; + // std::vector learners_; ReplicaInfo self_info_; const KeyInfo private_key_; diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index 4d57f39606..8c3fbf8349 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -326,7 +326,7 @@ int Commitment::PostProcessExecutedMsg() { uint64_t read_cnt = batch_resp->read_count(); uint64_t delete_cnt = batch_resp->delete_count(); - if (request.seq % 50 == 0) { + if (request.seq() % 50 == 0) { std::vector> reqs_to_learner = message_manager_->getLearnerUpdateRequests(); } From 92f90814d696cdb36147e0f3f27de5f526e06f9c Mon Sep 17 00:00:00 2001 From: Ryan Nelson Date: Mon, 1 Dec 2025 23:11:06 -0800 Subject: [PATCH 25/41] config and committment edits --- platform/config/resdb_config.h | 7 +------ platform/consensus/ordering/pbft/commitment.cpp | 6 +++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/platform/config/resdb_config.h b/platform/config/resdb_config.h index 5fd8bed841..26d3ff5b40 100644 --- a/platform/config/resdb_config.h +++ b/platform/config/resdb_config.h @@ -51,9 +51,7 @@ class ResDBConfig { const std::vector& GetReplicaInfos() const; // NEW: Access Learner Nodes - const std::vector& GetLearnerInfos() const { - return learners_; - } + const std::vector& GetLearnerInfos() const; // NEW: Add Learner Node void AddLearnerInfo(const ReplicaInfo& info) { @@ -140,9 +138,6 @@ class ResDBConfig { private: ResConfigData config_data_; std::vector replicas_; - - // NEW: Learner nodes for read only fast path - std::vector learners_; ReplicaInfo self_info_; const KeyInfo private_key_; diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index 4d57f39606..852315a151 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -317,8 +317,8 @@ int Commitment::PostProcessExecutedMsg() { request.set_type(Request::TYPE_RESPONSE); request.set_sender_id(config_.GetSelfInfo().id()); request.set_current_view(batch_resp->current_view()); - request.set_proxy_id(batch_resp->proxy_id()); - request.set_primary_id(batch_resp->primary_id()); + request.set_proxy_id(batch_resp->proxy_id()); + request.set_primary_id(batch_resp->primary_id()); // LOG(ERROR)<<"send back to proxy:"<proxy_id(); batch_resp->SerializeToString(request.mutable_data()); replica_communicator_->SendMessage(request, request.proxy_id()); @@ -326,7 +326,7 @@ int Commitment::PostProcessExecutedMsg() { uint64_t read_cnt = batch_resp->read_count(); uint64_t delete_cnt = batch_resp->delete_count(); - if (request.seq % 50 == 0) { + if (request.seq() % 50 == 0) { std::vector> reqs_to_learner = message_manager_->getLearnerUpdateRequests(); } From a927c4582c0301494ab14a506750898ddafa5d77 Mon Sep 17 00:00:00 2001 From: Ryan Nelson Date: Mon, 1 Dec 2025 23:15:09 -0800 Subject: [PATCH 26/41] missed a func --- platform/config/resdb_config.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/platform/config/resdb_config.h b/platform/config/resdb_config.h index 26d3ff5b40..f4b144c1a8 100644 --- a/platform/config/resdb_config.h +++ b/platform/config/resdb_config.h @@ -54,9 +54,7 @@ class ResDBConfig { const std::vector& GetLearnerInfos() const; // NEW: Add Learner Node - void AddLearnerInfo(const ReplicaInfo& info) { - learners_.push_back(info); - } + void AddLearnerInfo(const ReplicaInfo& info); ResConfigData GetConfigData() const; From 114c1b6a56aa5c1ee4b055c47e3b43887e6fbc46 Mon Sep 17 00:00:00 2001 From: eshabram Date: Tue, 2 Dec 2025 01:13:34 -0800 Subject: [PATCH 27/41] readonly sending to learner from client --- api/learner-test.py | 10 +-- entrypoint.sh | 1 + platform/consensus/ordering/pbft/BUILD | 2 + .../ordering/pbft/response_manager.cpp | 61 +++++++++++++++++++ .../ordering/pbft/response_manager.h | 4 ++ platform/learner/BUILD | 1 + platform/learner/learner.cpp | 18 ++++-- platform/learner/learner.h | 3 +- 8 files changed, 91 insertions(+), 9 deletions(-) diff --git a/api/learner-test.py b/api/learner-test.py index 4bff83f5b1..6d3086c44d 100644 --- a/api/learner-test.py +++ b/api/learner-test.py @@ -1,8 +1,10 @@ -import sys, os, time -# Your path to ResilientDB api folder +import sys, os, time, subprocess + +subprocess.run("bazel build :pybind_kv_so", shell=True) sys.path.append(os.getcwd()) -from kv_operation import set_value, get_value + +from kv_operation import set_value, get_value, get_value_readonly print(set_value("test", "111225")) time.sleep(0.5) -print(f"Key 'test' value: {get_value('test')}") \ No newline at end of file +print(f"Key 'test' value: {get_value_readonly('test')}") \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index 96810abd87..7962b7884a 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -19,6 +19,7 @@ # under the License. # +# ./service/tools/kv/server_tools/generate_config.sh ./service/tools/kv/server_tools/start_kv_service_learner.sh if [[ "$2" =~ ^[0-9]+$ ]]; then diff --git a/platform/consensus/ordering/pbft/BUILD b/platform/consensus/ordering/pbft/BUILD index 6f4fdff709..6d11f659ff 100644 --- a/platform/consensus/ordering/pbft/BUILD +++ b/platform/consensus/ordering/pbft/BUILD @@ -45,6 +45,8 @@ cc_library( ":lock_free_collector_pool", ":transaction_utils", "//platform/networkstrate:replica_communicator", + "//interface/rdbc:net_channel", + "//proto/kv:kv_cc_proto", ], ) diff --git a/platform/consensus/ordering/pbft/response_manager.cpp b/platform/consensus/ordering/pbft/response_manager.cpp index f490c00399..f9c947741c 100644 --- a/platform/consensus/ordering/pbft/response_manager.cpp +++ b/platform/consensus/ordering/pbft/response_manager.cpp @@ -22,6 +22,8 @@ #include #include "common/utils/utils.h" +#include "interface/rdbc/net_channel.h" +#include "proto/kv/kv.pb.h" namespace resdb { @@ -72,6 +74,14 @@ ResponseManager::ResponseManager(const ResDBConfig& config, } global_stats_ = Stats::GetGlobalStats(); send_num_ = 0; + + if (config_.GetPublicKeyCertificateInfo() + .public_key() + .public_key_info() + .type() == CertificateKeyInfo::CLIENT && + !config_.GetLearnerInfos().empty()) { + learner_info_ = config_.GetLearnerInfos()[0]; + } } ResponseManager::~ResponseManager() { @@ -102,6 +112,10 @@ std::vector> ResponseManager::FetchContextList( int ResponseManager::NewUserRequest(std::unique_ptr context, std::unique_ptr user_request) { + if (ForwardReadOnlyToLearner(context.get(), user_request.get())) { + return 0; + } + if (!user_request->need_response()) { context->client = nullptr; } @@ -407,4 +421,51 @@ void ResponseManager::MonitoringClientTimeOut() { } } } + +bool ResponseManager::ForwardReadOnlyToLearner(Context* context, + Request* user_request) { + if (!learner_info_.has_value() || context == nullptr || + context->client == nullptr || user_request == nullptr) { + return false; + } + if (user_request->type() != Request::TYPE_CLIENT_REQUEST) { + return false; + } + + KVRequest kv_request; + if (!kv_request.ParseFromString(user_request->data())) { + return false; + } + if (kv_request.cmd() != KVRequest::GET_READ_ONLY) { + return false; + } + + NetChannel learner_channel(learner_info_->ip(), learner_info_->port()); + learner_channel.SetRecvTimeout(100000); // 100ms + + std::string payload; + if (!kv_request.SerializeToString(&payload)) { + return false; + } + + if (learner_channel.SendRawMessageData(payload) != 0) { + return false; + } + + // Await learner response; if nothing returns, fall back to PBFT. + std::string learner_resp; + if (learner_channel.RecvRawMessageData(&learner_resp) <= 0) { + return false; + } + + KVResponse learner_response; + if (!learner_response.ParseFromString(learner_resp)) { + return false; + } + + if (context->client->SendRawMessage(learner_response) != 0) { + return false; + } + return true; +} } // namespace resdb diff --git a/platform/consensus/ordering/pbft/response_manager.h b/platform/consensus/ordering/pbft/response_manager.h index 2972171eba..53df26a244 100644 --- a/platform/consensus/ordering/pbft/response_manager.h +++ b/platform/consensus/ordering/pbft/response_manager.h @@ -19,6 +19,8 @@ #pragma once #include +#include +#include #include "platform/config/resdb_config.h" #include "platform/consensus/ordering/pbft/lock_free_collector_pool.h" @@ -82,6 +84,7 @@ class ResponseManager { void ResponseTimer(std::string hash); void MonitoringClientTimeOut(); std::unique_ptr GetTimeOutRequest(std::string hash); + bool ForwardReadOnlyToLearner(Context* context, Request* user_request); private: ResDBConfig config_; @@ -104,6 +107,7 @@ class ResponseManager { sem_t request_sent_signal_; uint64_t highest_seq_; uint64_t highest_seq_primary_id_; + std::optional learner_info_; }; } // namespace resdb diff --git a/platform/learner/BUILD b/platform/learner/BUILD index 2a623e9431..0d8997c1ad 100644 --- a/platform/learner/BUILD +++ b/platform/learner/BUILD @@ -29,6 +29,7 @@ cc_library( "//common:json", "//platform/common/network:tcp_socket", "//platform/proto:resdb_cc_proto", + "//proto/kv:kv_cc_proto", ], ) diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index cfd7dfe056..ce43d2b4b0 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -32,6 +32,7 @@ #include "platform/common/network/socket.h" #include "platform/common/network/tcp_socket.h" #include "platform/proto/resdb.pb.h" +#include "proto/kv/kv.pb.h" namespace { @@ -103,21 +104,29 @@ void Learner::HandleClient(std::unique_ptr socket) const { std::string payload(static_cast(buffer), len); free(buffer); - ProcessBroadcast(payload); + if (!ProcessBroadcast(payload)) { + break; + } } // std::cout << "[Learner] replica disconnected" << std::endl; } -void Learner::ProcessBroadcast(const std::string& payload) const { +bool Learner::ProcessBroadcast(const std::string& payload) const { resdb::ResDBMessage envelope; if (!envelope.ParseFromString(payload)) { + resdb::KVRequest kv_request; + if (kv_request.ParseFromString(payload)) { + std::cout << "[Learner] received read-only request for key '" + << kv_request.key() << "' (no response yet)" << std::endl; + return false; + } ++total_messages_; total_bytes_.fetch_add(payload.size()); last_type_.store(-1); last_seq_.store(0); last_sender_.store(-1); last_payload_bytes_.store(payload.size()); - return; + return true; } resdb::Request request; @@ -134,7 +143,7 @@ void Learner::ProcessBroadcast(const std::string& payload) const { total_reads_.fetch_add(batch_response.read_count()); total_deletes_.fetch_add(batch_response.delete_count()); } - return; + return true; } ++total_messages_; @@ -143,6 +152,7 @@ void Learner::ProcessBroadcast(const std::string& payload) const { last_seq_.store(0); last_sender_.store(-1); last_payload_bytes_.store(envelope.data().size()); + return true; } void Learner::MetricsLoop() const { diff --git a/platform/learner/learner.h b/platform/learner/learner.h index 13ee383af6..50ae05b371 100644 --- a/platform/learner/learner.h +++ b/platform/learner/learner.h @@ -42,7 +42,8 @@ class Learner { private: static LearnerConfig LoadConfig(const std::string& config_path); void HandleClient(std::unique_ptr socket) const; - void ProcessBroadcast(const std::string& payload) const; + // Returns true if the connection should remain open, false if it should be closed. + bool ProcessBroadcast(const std::string& payload) const; void MetricsLoop() const; void PrintMetrics() const; From 886ee6125eb098dd15fa044c2eee1a0267f4832d Mon Sep 17 00:00:00 2001 From: Ryan Nelson Date: Tue, 2 Dec 2025 23:00:59 -0800 Subject: [PATCH 28/41] replica message creation (so tested fr) --- .../consensus/ordering/pbft/commitment.cpp | 116 ++++++++++++++++-- platform/consensus/ordering/pbft/commitment.h | 5 + platform/proto/resdb.proto | 11 ++ 3 files changed, 121 insertions(+), 11 deletions(-) diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index 852315a151..2056e5c938 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -23,6 +23,8 @@ #include #include "common/utils/utils.h" +#include "common/crypto/hash.h" + #include "platform/consensus/ordering/pbft/transaction_utils.h" namespace resdb { @@ -326,19 +328,20 @@ int Commitment::PostProcessExecutedMsg() { uint64_t read_cnt = batch_resp->read_count(); uint64_t delete_cnt = batch_resp->delete_count(); - if (request.seq() % 50 == 0) { - std::vector> reqs_to_learner = message_manager_->getLearnerUpdateRequests(); + // send an update to learners after every block size execution + if (request.seq() % config_.GetBlockSize() == 0) { + SendUpdateToLearners(request.seq()); } - // Also mirror the executed request to every learner so they can consume - // the committed stream. - for (const auto& learner : config_.GetLearnerInfos()) { - LOG(ERROR) << "Forward commit to learner id " << learner.id() - << " seq " << request.seq() - << " set=" << set_cnt << " read=" << read_cnt - << " delete=" << delete_cnt; - replica_communicator_->SendMessage(request, learner.id()); - } + // // Also mirror the executed request to every learner so they can consume + // // the committed stream. + // for (const auto& learner : config_.GetLearnerInfos()) { + // LOG(ERROR) << "Forward commit to learner id " << learner.id() + // << " seq " << request.seq() + // << " set=" << set_cnt << " read=" << read_cnt + // << " delete=" << delete_cnt; + // replica_communicator_->SendMessage(request, learner.id()); + // } } return 0; } @@ -347,4 +350,95 @@ DuplicateManager* Commitment::GetDuplicateManager() { return duplicate_manager_.get(); } +void Commitment::SendUpdateToLearners(int seq_num) { + + // get stored requests from message manager + std::vector> reqs_to_learner = message_manager_->getLearnerUpdateRequests(); + + std::string raw_data; + + // adds all requests into one batch variable + RequestBatch batch; + for (const auto& r : reqs_to_learner) { + *batch.add_requests()->Swap(r.get()); + } + + // turn the entire batch variable into string + batch.SerializeToString(&raw_data); + + // generate hash of entire batch + str::string block_hash = CalculateSHA256Hash(raw_data); + + // generate correct row of Vandermode matrix + A_i = gen_A_row(config_.GetSelfInfo().id()); + + // perform message distribution algorithm + vector F_i; + + uint32_t iter = 0; + uint32_t c_ik = 0; + for (int d = 0; d < raw_data.size(); d++) { // data MUST BE A MULTIPLE OF m + c_ik = (c_ik + A_i[iter] * raw_data[d]) % p; + + iter++; + if (iter == m) { + F_i.push_back(c_ik); + + c_ik = 0; + iter = 0; + } + } + + // add data to learner update message and send it + LearnerUpdate learnerUpdate; + learnerUpdate.mutable_data()->Reserve(static_cast(F_i.size())); + for (uint32_t v : F_i) { + learnerUpdate.add_data(v); + } + learnerUpdate.set_block_hash(block_hash); + learnerUpdate.set_seq(seq_num); + learnerUpdate.set_sender_id(config_.GetSelfInfo().id()); + + for (const auto& learner : config_.GetLearnerInfos()) { + replica_communicator_->SendMessage(learnerUpdate, learner.id()); + } + +} + +vector Commitment::gen_A_row() { + + int n = config_.GetReplicaNum(); + int m = config_.GetMinClientReceiveNum(); + int p = 257; + + vector A_i(m); + + int i = config_.GetSelfInfo().id(); + + for (int j = 0; j < m; j++) { + + int exp = j; + int base = i; + int mod = p; + int result = 1; + + while (exp > 0) { + if (exp & 1) { + result = (result * base) % mod; + } + + base = (base * base) % mod; + exp >>= 1; + } + + A[j] = result; + + } + + return A; + +} + +} + } // namespace resdb diff --git a/platform/consensus/ordering/pbft/commitment.h b/platform/consensus/ordering/pbft/commitment.h index 03d77cf3d2..a00837a415 100644 --- a/platform/consensus/ordering/pbft/commitment.h +++ b/platform/consensus/ordering/pbft/commitment.h @@ -59,6 +59,10 @@ class Commitment { protected: virtual int PostProcessExecutedMsg(); + // distributed message functions + vector Commitment::gen_A_row(); + vector Commitment::SendUpdateToLearners(int seq_num); + protected: ResDBConfig config_; MessageManager* message_manager_; @@ -74,6 +78,7 @@ class Commitment { std::mutex mutex_; std::unique_ptr duplicate_manager_; + }; } // namespace resdb diff --git a/platform/proto/resdb.proto b/platform/proto/resdb.proto index 111dfdb145..e810752e5e 100644 --- a/platform/proto/resdb.proto +++ b/platform/proto/resdb.proto @@ -91,6 +91,17 @@ message Request { bytes data_hash = 26; } +message RequestBatch { + repeated Request requests = 1; +} + +message LearnerUpdate { + repeated uint32 data = 1; + bytes block_hash = 2; + uint64 seq = 3; + uint32 sender_id = 4; +} + // The response message containing response message ResponseData { bytes data = 1; From 0e34a6d71bfbb6f3ae063aaa1d424f0a20989dca Mon Sep 17 00:00:00 2001 From: eshabram Date: Wed, 3 Dec 2025 00:57:15 -0800 Subject: [PATCH 29/41] readonly to learner works --- .gitignore | 3 +- platform/config/resdb_config.h | 2 +- .../ordering/pbft/response_manager.cpp | 8 +- platform/learner/BUILD | 2 + platform/learner/learner.config | 12 +- platform/learner/learner.cpp | 107 +++++++++++++++++- platform/learner/learner.h | 16 ++- platform/learner/learner_main.cpp | 4 + 8 files changed, 129 insertions(+), 25 deletions(-) diff --git a/.gitignore b/.gitignore index e0ab3dd4e6..c3dee08e87 100644 --- a/.gitignore +++ b/.gitignore @@ -17,9 +17,10 @@ apache_release *.data.* *.pb.* .cache/ +.bazel-cache/ resdb/ 100*_db/ +*_db/ gmon.out .history/ **/*AGENTS.md - diff --git a/platform/config/resdb_config.h b/platform/config/resdb_config.h index b90a7bf800..931a8ec937 100644 --- a/platform/config/resdb_config.h +++ b/platform/config/resdb_config.h @@ -159,7 +159,7 @@ class ResDBConfig { uint32_t client_batch_num_ = 100; // learner - uint32_t block_size_ = 100; + uint32_t block_size_ = 10; std::vector learners_; }; diff --git a/platform/consensus/ordering/pbft/response_manager.cpp b/platform/consensus/ordering/pbft/response_manager.cpp index f9c947741c..2876bfbcb4 100644 --- a/platform/consensus/ordering/pbft/response_manager.cpp +++ b/platform/consensus/ordering/pbft/response_manager.cpp @@ -75,11 +75,7 @@ ResponseManager::ResponseManager(const ResDBConfig& config, global_stats_ = Stats::GetGlobalStats(); send_num_ = 0; - if (config_.GetPublicKeyCertificateInfo() - .public_key() - .public_key_info() - .type() == CertificateKeyInfo::CLIENT && - !config_.GetLearnerInfos().empty()) { + if (!config_.GetLearnerInfos().empty()) { learner_info_ = config_.GetLearnerInfos()[0]; } } @@ -463,7 +459,7 @@ bool ResponseManager::ForwardReadOnlyToLearner(Context* context, return false; } - if (context->client->SendRawMessage(learner_response) != 0) { + if (context->client->SendRawMessageData(learner_resp) != 0) { return false; } return true; diff --git a/platform/learner/BUILD b/platform/learner/BUILD index 0d8997c1ad..bd38e094f4 100644 --- a/platform/learner/BUILD +++ b/platform/learner/BUILD @@ -26,10 +26,12 @@ cc_library( ], hdrs = ["learner.h"], deps = [ + "//chain/storage:leveldb", "//common:json", "//platform/common/network:tcp_socket", "//platform/proto:resdb_cc_proto", "//proto/kv:kv_cc_proto", + "@com_github_google_glog//:glog", ], ) diff --git a/platform/learner/learner.config b/platform/learner/learner.config index 124bff3a21..213adcfc04 100644 --- a/platform/learner/learner.config +++ b/platform/learner/learner.config @@ -1,14 +1,6 @@ { - "node_id": 6, "ip": "127.0.0.1", "port": 25000, - - "replicas": [ - { "id": 1, "ip": "127.0.0.1", "port": 20001 }, - { "id": 2, "ip": "127.0.0.1", "port": 20002 }, - { "id": 3, "ip": "127.0.0.1", "port": 20003 }, - { "id": 4, "ip": "127.0.0.1", "port": 20004 } - ], - - "block_size": 100 + "db_path": "25000_db/", + "block_size": 10 } diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index ce43d2b4b0..e144d316ab 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -27,8 +27,10 @@ #include #include +#include #include +#include "chain/storage/leveldb.h" #include "platform/common/network/socket.h" #include "platform/common/network/tcp_socket.h" #include "platform/proto/resdb.pb.h" @@ -50,6 +52,22 @@ LearnerConfig ParseLearnerConfig(const std::string& config_path) { config.ip = config_json.value("ip", config.ip); config.port = config_json.value("port", 0); config.block_size = config_json.value("block_size", 0); + config.db_path = config_json.value("db_path", ""); + if (config.db_path.empty() && config.port > 0) { + config.db_path = std::to_string(config.port) + "_db/"; + } + + if (config_json.contains("replicas") && config_json["replicas"].is_array()) { + for (const auto& replica_json : config_json["replicas"]) { + resdb::ReplicaInfo replica_info; + replica_info.set_id(replica_json.value("id", 0)); + replica_info.set_ip(replica_json.value("ip", "127.0.0.1")); + replica_info.set_port(replica_json.value("port", 0)); + if (replica_info.port() != 0) { + config.replicas.push_back(replica_info); + } + } + } if (config.port <= 0) { throw std::runtime_error("Learner config must include a valid port"); @@ -63,7 +81,9 @@ LearnerConfig ParseLearnerConfig(const std::string& config_path) { } // namespace Learner::Learner(const std::string& config_path) - : config_(LoadConfig(config_path)) {} + : config_(LoadConfig(config_path)) { + InitializeStorage(); +} LearnerConfig Learner::LoadConfig(const std::string& config_path) { const std::string path = config_path.empty() ? DefaultConfigPath() : config_path; @@ -104,20 +124,32 @@ void Learner::HandleClient(std::unique_ptr socket) const { std::string payload(static_cast(buffer), len); free(buffer); - if (!ProcessBroadcast(payload)) { + if (!ProcessBroadcast(socket.get(), payload)) { break; } } // std::cout << "[Learner] replica disconnected" << std::endl; } -bool Learner::ProcessBroadcast(const std::string& payload) const { +bool Learner::ProcessBroadcast(resdb::Socket* socket, + const std::string& payload) const { resdb::ResDBMessage envelope; if (!envelope.ParseFromString(payload)) { resdb::KVRequest kv_request; if (kv_request.ParseFromString(payload)) { - std::cout << "[Learner] received read-only request for key '" - << kv_request.key() << "' (no response yet)" << std::endl; + if (HandleReadOnlyRequest(socket, kv_request)) { + LOG(INFO) << "[Learner] served read-only key '" << kv_request.key() + << "' from local snapshot"; + return false; + } + LOG(INFO) << "[Learner] no cached value for key '" + << kv_request.key() << "', letting replica handle"; + ++total_messages_; + total_bytes_.fetch_add(payload.size()); + last_type_.store(kv_request.cmd()); + last_seq_.store(0); + last_sender_.store(-1); + last_payload_bytes_.store(payload.size()); return false; } ++total_messages_; @@ -131,6 +163,18 @@ bool Learner::ProcessBroadcast(const std::string& payload) const { resdb::Request request; if (request.ParseFromString(envelope.data())) { + resdb::KVRequest kv_request; + if (kv_request.ParseFromString(request.data()) && + kv_request.cmd() == resdb::KVRequest::GET_READ_ONLY) { + if (HandleReadOnlyRequest(socket, kv_request)) { + LOG(INFO) << "[Learner] served read-only key '" << kv_request.key() + << "' from local snapshot"; + return false; + } + LOG(INFO) << "[Learner] no cached value for key '" + << kv_request.key() << "', letting replica handle"; + } + ++total_messages_; total_bytes_.fetch_add(payload.size()); last_type_.store(request.type()); @@ -170,6 +214,39 @@ void Learner::MetricsLoop() const { } } +bool Learner::HandleReadOnlyRequest(resdb::Socket* socket, + const resdb::KVRequest& request) const { + if (request.cmd() != resdb::KVRequest::GET_READ_ONLY || storage_ == nullptr) { + return false; + } + + if (known_keys_.find(request.key()) == known_keys_.end()) { + return false; + } + + std::string value = storage_->GetValue(request.key()); + if (value.empty()) { + return false; + } + + resdb::KVResponse response; + response.set_key(request.key()); + response.set_value(value); + + std::string resp_str; + if (!response.SerializeToString(&resp_str)) { + LOG(ERROR) << "[Learner] failed to serialize KVResponse"; + return false; + } + + if (socket->Send(resp_str) < 0) { + LOG(ERROR) << "[Learner] failed to send read-only response"; + return false; + } + + return true; +} + void Learner::PrintMetrics() const { std::cout << "[Learner] broadcasts=" << total_messages_.load() << " bytes=" << total_bytes_.load() @@ -181,5 +258,23 @@ void Learner::PrintMetrics() const { << " sender=" << last_sender_.load() << " payload=" << last_payload_bytes_.load() << "B" << " (listening on " << config_.ip << ":" << config_.port - << ", block_size=" << config_.block_size << ")" << std::endl; + << ", block_size=" << config_.block_size + << ", db_path=" << config_.db_path << ")" << std::endl; +} + +void Learner::InitializeStorage() { + if (config_.db_path.empty()) { + config_.db_path = std::to_string(config_.port) + "_db/"; + } + storage_ = resdb::storage::NewResLevelDB(config_.db_path); + if (!storage_) { + throw std::runtime_error("Learner failed to initialize storage at " + + config_.db_path); + } + known_keys_.clear(); + storage_->SetValue("test", "from learner db"); + storage_->Flush(); + known_keys_.insert("test"); + LOG(INFO) << "[Learner] Initialized local DB at " << config_.db_path + << " with test key"; } diff --git a/platform/learner/learner.h b/platform/learner/learner.h index 50ae05b371..ab1f95853c 100644 --- a/platform/learner/learner.h +++ b/platform/learner/learner.h @@ -22,6 +22,12 @@ #include #include #include +#include +#include + +#include "chain/storage/storage.h" +#include "proto/kv/kv.pb.h" +#include "platform/proto/replica_info.pb.h" namespace resdb { class Socket; @@ -32,6 +38,8 @@ struct LearnerConfig { std::string ip = "127.0.0.1"; int port = 0; int block_size = 0; + std::string db_path; + std::vector replicas; }; class Learner { @@ -43,9 +51,13 @@ class Learner { static LearnerConfig LoadConfig(const std::string& config_path); void HandleClient(std::unique_ptr socket) const; // Returns true if the connection should remain open, false if it should be closed. - bool ProcessBroadcast(const std::string& payload) const; + bool ProcessBroadcast(resdb::Socket* socket, + const std::string& payload) const; + bool HandleReadOnlyRequest(resdb::Socket* socket, + const resdb::KVRequest& request) const; void MetricsLoop() const; void PrintMetrics() const; + void InitializeStorage(); private: LearnerConfig config_; @@ -60,4 +72,6 @@ class Learner { mutable std::atomic last_sender_{-1}; mutable std::atomic last_payload_bytes_{0}; mutable std::thread metrics_thread_; + mutable std::unique_ptr storage_; + std::unordered_set known_keys_; }; diff --git a/platform/learner/learner_main.cpp b/platform/learner/learner_main.cpp index 678fad3879..911664c223 100644 --- a/platform/learner/learner_main.cpp +++ b/platform/learner/learner_main.cpp @@ -23,6 +23,8 @@ #include #include +#include + std::string ResolveConfigPath(int argc, char** argv) { std::string config_path = "platform/learner/learner.config"; for (int i = 1; i < argc; ++i) { @@ -36,6 +38,8 @@ std::string ResolveConfigPath(int argc, char** argv) { int main(int argc, char** argv) { try { + google::InitGoogleLogging(argv[0]); + FLAGS_logtostderr = 1; Learner learner(ResolveConfigPath(argc, argv)); learner.Run(); } catch (const std::exception& e) { From b986e339a232fbc288a534c32f012d69afef5c94 Mon Sep 17 00:00:00 2001 From: eshabram Date: Wed, 3 Dec 2025 12:48:12 -0800 Subject: [PATCH 30/41] fix learner recv --- platform/learner/learner.cpp | 46 ++++++++++++------------------------ 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index e144d316ab..3cf4fcd83a 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -35,6 +35,7 @@ #include "platform/common/network/tcp_socket.h" #include "platform/proto/resdb.pb.h" #include "proto/kv/kv.pb.h" +#include "platform/statistic/stats.h" namespace { @@ -83,6 +84,7 @@ LearnerConfig ParseLearnerConfig(const std::string& config_path) { Learner::Learner(const std::string& config_path) : config_(LoadConfig(config_path)) { InitializeStorage(); + resdb::Stats::GetGlobalStats()->Stop(); } LearnerConfig Learner::LoadConfig(const std::string& config_path) { @@ -110,7 +112,7 @@ void Learner::Run() { } void Learner::HandleClient(std::unique_ptr socket) const { - std::cout << "[Learner] replica connected" << std::endl; + // std::cout << "[Learner] replica connected" << std::endl; while (true) { void* buffer = nullptr; size_t len = 0; @@ -138,12 +140,9 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, resdb::KVRequest kv_request; if (kv_request.ParseFromString(payload)) { if (HandleReadOnlyRequest(socket, kv_request)) { - LOG(INFO) << "[Learner] served read-only key '" << kv_request.key() - << "' from local snapshot"; + LOG(INFO) << "served read-only key=" << kv_request.key(); return false; } - LOG(INFO) << "[Learner] no cached value for key '" - << kv_request.key() << "', letting replica handle"; ++total_messages_; total_bytes_.fetch_add(payload.size()); last_type_.store(kv_request.cmd()); @@ -163,18 +162,6 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, resdb::Request request; if (request.ParseFromString(envelope.data())) { - resdb::KVRequest kv_request; - if (kv_request.ParseFromString(request.data()) && - kv_request.cmd() == resdb::KVRequest::GET_READ_ONLY) { - if (HandleReadOnlyRequest(socket, kv_request)) { - LOG(INFO) << "[Learner] served read-only key '" << kv_request.key() - << "' from local snapshot"; - return false; - } - LOG(INFO) << "[Learner] no cached value for key '" - << kv_request.key() << "', letting replica handle"; - } - ++total_messages_; total_bytes_.fetch_add(payload.size()); last_type_.store(request.type()); @@ -248,18 +235,15 @@ bool Learner::HandleReadOnlyRequest(resdb::Socket* socket, } void Learner::PrintMetrics() const { - std::cout << "[Learner] broadcasts=" << total_messages_.load() - << " bytes=" << total_bytes_.load() - << " sets=" << total_sets_.load() - << " reads=" << total_reads_.load() - << " deletes=" << total_deletes_.load() - << " last_type=" << last_type_.load() - << " last_seq=" << last_seq_.load() - << " sender=" << last_sender_.load() - << " payload=" << last_payload_bytes_.load() << "B" - << " (listening on " << config_.ip << ":" << config_.port - << ", block_size=" << config_.block_size - << ", db_path=" << config_.db_path << ")" << std::endl; + LOG(INFO) << "msgs=" << total_messages_.load() + << " bytes=" << total_bytes_.load() + << " sets=" << total_sets_.load() + << " reads=" << total_reads_.load() + << " deletes=" << total_deletes_.load() + << " last_type=" << last_type_.load() + << " last_seq=" << last_seq_.load() + << " sender=" << last_sender_.load() + << " payload=" << last_payload_bytes_.load() << "B"; } void Learner::InitializeStorage() { @@ -275,6 +259,6 @@ void Learner::InitializeStorage() { storage_->SetValue("test", "from learner db"); storage_->Flush(); known_keys_.insert("test"); - LOG(INFO) << "[Learner] Initialized local DB at " << config_.db_path - << " with test key"; + VLOG(1) << "[Learner] Initialized local DB at " << config_.db_path + << " with test key"; } From 0d8daab197012c50ffbddd524a7e72771ce9c8fa Mon Sep 17 00:00:00 2001 From: eshabram Date: Wed, 3 Dec 2025 13:49:42 -0800 Subject: [PATCH 31/41] error to infor log --- platform/consensus/ordering/pbft/commitment.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index 8c3fbf8349..f37b1ab3a0 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -317,8 +317,8 @@ int Commitment::PostProcessExecutedMsg() { request.set_type(Request::TYPE_RESPONSE); request.set_sender_id(config_.GetSelfInfo().id()); request.set_current_view(batch_resp->current_view()); - request.set_proxy_id(batch_resp->proxy_id()); - request.set_primary_id(batch_resp->primary_id()); + request.set_proxy_id(batch_resp->proxy_id()); + request.set_primary_id(batch_resp->primary_id()); // LOG(ERROR)<<"send back to proxy:"<proxy_id(); batch_resp->SerializeToString(request.mutable_data()); replica_communicator_->SendMessage(request, request.proxy_id()); @@ -333,7 +333,7 @@ int Commitment::PostProcessExecutedMsg() { // Also mirror the executed request to every learner so they can consume // the committed stream. for (const auto& learner : config_.GetLearnerInfos()) { - LOG(ERROR) << "Forward commit to learner id " << learner.id() + LOG(INFO) << "Forward commit to learner id " << learner.id() << " seq " << request.seq() << " set=" << set_cnt << " read=" << read_cnt << " delete=" << delete_cnt; From 68b4d3a7e2d454619aa841b546524040712b239c Mon Sep 17 00:00:00 2001 From: Ryan Nelson Date: Wed, 3 Dec 2025 14:02:08 -0800 Subject: [PATCH 32/41] merged elliots learner code pt2 --- platform/config/resdb_config.h | 12 ------------ .../consensus/ordering/pbft/commitment.cpp | 18 ------------------ 2 files changed, 30 deletions(-) diff --git a/platform/config/resdb_config.h b/platform/config/resdb_config.h index 4e824af89b..931a8ec937 100644 --- a/platform/config/resdb_config.h +++ b/platform/config/resdb_config.h @@ -50,15 +50,6 @@ class ResDBConfig { // Each replica infomation, including the binding urls(or ip,port). const std::vector& GetReplicaInfos() const; -<<<<<<< HEAD - // NEW: Access Learner Nodes - const std::vector& GetLearnerInfos() const; - - // NEW: Add Learner Node - void AddLearnerInfo(const ReplicaInfo& info); - -======= ->>>>>>> origin/feature/readonly/learner-node ResConfigData GetConfigData() const; // The current replica infomation, including the binding urls(or ip,port). @@ -139,12 +130,9 @@ class ResDBConfig { private: ResConfigData config_data_; std::vector replicas_; -<<<<<<< HEAD -======= // NEW: Learner nodes for read only fast path // std::vector learners_; ->>>>>>> origin/feature/readonly/learner-node ReplicaInfo self_info_; const KeyInfo private_key_; diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index f96ec5d9fa..a369ef1385 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -328,7 +328,6 @@ int Commitment::PostProcessExecutedMsg() { uint64_t read_cnt = batch_resp->read_count(); uint64_t delete_cnt = batch_resp->delete_count(); -<<<<<<< HEAD // send an update to learners after every block size execution if (request.seq() % config_.GetBlockSize() == 0) { SendUpdateToLearners(request.seq()); @@ -343,21 +342,6 @@ int Commitment::PostProcessExecutedMsg() { // << " delete=" << delete_cnt; // replica_communicator_->SendMessage(request, learner.id()); // } -======= - if (request.seq() % 50 == 0) { - std::vector> reqs_to_learner = message_manager_->getLearnerUpdateRequests(); - } - - // Also mirror the executed request to every learner so they can consume - // the committed stream. - for (const auto& learner : config_.GetLearnerInfos()) { - LOG(INFO) << "Forward commit to learner id " << learner.id() - << " seq " << request.seq() - << " set=" << set_cnt << " read=" << read_cnt - << " delete=" << delete_cnt; - replica_communicator_->SendMessage(request, learner.id()); - } ->>>>>>> origin/feature/readonly/learner-node } return 0; } @@ -455,6 +439,4 @@ vector Commitment::gen_A_row() { } -} - } // namespace resdb From 6f5759654261186b8674631883d47c6c5249613f Mon Sep 17 00:00:00 2001 From: Ryan Nelson Date: Wed, 3 Dec 2025 23:06:07 -0800 Subject: [PATCH 33/41] start learner decode work --- platform/consensus/ordering/pbft/BUILD | 3 +- .../consensus/ordering/pbft/commitment.cpp | 20 +-- platform/consensus/ordering/pbft/commitment.h | 4 +- platform/learner/learner.config | 10 +- platform/learner/learner.cpp | 164 ++++++++++++++++-- platform/learner/learner.h | 7 + 6 files changed, 180 insertions(+), 28 deletions(-) diff --git a/platform/consensus/ordering/pbft/BUILD b/platform/consensus/ordering/pbft/BUILD index 6d11f659ff..2f2c5de00d 100644 --- a/platform/consensus/ordering/pbft/BUILD +++ b/platform/consensus/ordering/pbft/BUILD @@ -105,7 +105,8 @@ cc_library( hdrs = ["commitment.h"], deps = [ ":message_manager", - ":response_manager", + ":response_manager", + "//common/crypto:hash", "//common/utils", "//platform/common/queue:batch_queue", "//platform/config:resdb_config", diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index a369ef1385..7426b79362 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -360,28 +360,28 @@ void Commitment::SendUpdateToLearners(int seq_num) { // adds all requests into one batch variable RequestBatch batch; for (const auto& r : reqs_to_learner) { - *batch.add_requests()->Swap(r.get()); + batch.add_requests()->Swap(r.get()); } // turn the entire batch variable into string batch.SerializeToString(&raw_data); // generate hash of entire batch - str::string block_hash = CalculateSHA256Hash(raw_data); + std::string block_hash = resdb::utils::CalculateSHA256Hash(raw_data); // generate correct row of Vandermode matrix - A_i = gen_A_row(config_.GetSelfInfo().id()); + std::vector A_i = gen_A_row(); // perform message distribution algorithm - vector F_i; + std::vector F_i; uint32_t iter = 0; uint32_t c_ik = 0; for (int d = 0; d < raw_data.size(); d++) { // data MUST BE A MULTIPLE OF m - c_ik = (c_ik + A_i[iter] * raw_data[d]) % p; + c_ik = (c_ik + A_i[iter] * raw_data[d]) % 257; iter++; - if (iter == m) { + if (iter == A_i.size()) { F_i.push_back(c_ik); c_ik = 0; @@ -405,13 +405,13 @@ void Commitment::SendUpdateToLearners(int seq_num) { } -vector Commitment::gen_A_row() { +std::vector Commitment::gen_A_row() { int n = config_.GetReplicaNum(); int m = config_.GetMinClientReceiveNum(); int p = 257; - vector A_i(m); + std::vector A_i(m); int i = config_.GetSelfInfo().id(); @@ -431,11 +431,11 @@ vector Commitment::gen_A_row() { exp >>= 1; } - A[j] = result; + A_i[j] = result; } - return A; + return A_i; } diff --git a/platform/consensus/ordering/pbft/commitment.h b/platform/consensus/ordering/pbft/commitment.h index a00837a415..4cdd4933df 100644 --- a/platform/consensus/ordering/pbft/commitment.h +++ b/platform/consensus/ordering/pbft/commitment.h @@ -60,8 +60,8 @@ class Commitment { virtual int PostProcessExecutedMsg(); // distributed message functions - vector Commitment::gen_A_row(); - vector Commitment::SendUpdateToLearners(int seq_num); + std::vector gen_A_row(); + void SendUpdateToLearners(int seq_num); protected: ResDBConfig config_; diff --git a/platform/learner/learner.config b/platform/learner/learner.config index 213adcfc04..363cabf1a7 100644 --- a/platform/learner/learner.config +++ b/platform/learner/learner.config @@ -1,6 +1,12 @@ { "ip": "127.0.0.1", "port": 25000, - "db_path": "25000_db/", - "block_size": 10 + "block_size": 10, + "replicas": [ + { "id": 1, "ip": "127.0.0.1", "port": 20000 }, + { "id": 2, "ip": "127.0.0.1", "port": 20001 }, + { "id": 3, "ip": "127.0.0.1", "port": 20002 }, + { "id": 4, "ip": "127.0.0.1", "port": 20003 } + ] } + \ No newline at end of file diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index 3cf4fcd83a..c216cb5aa7 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -134,9 +135,12 @@ void Learner::HandleClient(std::unique_ptr socket) const { } bool Learner::ProcessBroadcast(resdb::Socket* socket, - const std::string& payload) const { + const std::string& payload) const { + resdb::ResDBMessage envelope; if (!envelope.ParseFromString(payload)) { + + LOG(INFO) << "HERE 1"; resdb::KVRequest kv_request; if (kv_request.ParseFromString(payload)) { if (HandleReadOnlyRequest(socket, kv_request)) { @@ -160,20 +164,32 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, return true; } - resdb::Request request; + LOG(INFO) << "HERE 2"; + resdb::LearnerUpdate request; if (request.ParseFromString(envelope.data())) { + + if (request.sender_id() == 0) return true; + + LOG(INFO) << "FFFFFFFFFFFFFFFFFFFFFFFFF"; + // LOG(INFO) << "DATA: " << request.data(); + LOG(INFO) << "HASH: " << request.block_hash(); + LOG(INFO) << "SEQ: " << request.seq(); + LOG(INFO) << "S_ID: " << request.sender_id(); + + HandleLearnerUpdate(request); + ++total_messages_; - total_bytes_.fetch_add(payload.size()); - last_type_.store(request.type()); - last_seq_.store(request.seq()); - last_sender_.store(request.sender_id()); - last_payload_bytes_.store(request.data().size()); - resdb::BatchUserResponse batch_response; - if (batch_response.ParseFromString(request.data())) { - total_sets_.fetch_add(batch_response.set_count()); - total_reads_.fetch_add(batch_response.read_count()); - total_deletes_.fetch_add(batch_response.delete_count()); - } + // total_bytes_.fetch_add(payload.size()); + // last_type_.store(request.type()); + // last_seq_.store(request.seq()); + // last_sender_.store(request.sender_id()); + // last_payload_bytes_.store(request.data().size()); + // resdb::BatchUserResponse batch_response; + // if (batch_response.ParseFromString(request.data())) { + // total_sets_.fetch_add(batch_response.set_count()); + // total_reads_.fetch_add(batch_response.read_count()); + // total_deletes_.fetch_add(batch_response.delete_count()); + // } return true; } @@ -186,6 +202,128 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, return true; } +void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { + + std::string block_hash = request.block_hash(); + int seq = request.seq(); + int sender_id = request.sender_id(); + + int blockIndex = seq / config_.block_size() - 1; + + // fill sequence status with "havent started" if variable doesnt exist yet + while (sequence_status.size() < blockIndex + 1) { + sequence_status.push_back(0); + } + + switch (sequence_status[blockIndex]) { + case 0: + hashCounts.push_back(std::tuple(blockIndex, block_hash, 1)); + learnerUpdates.push_back(learnerUpdate); + sequence_status[blockIndex] = 1; + + if (1 >= TEMP_VAR_MIN_NEEDED) { + sequence_status[blockIndex] = 2; + } + + break; + case 1: + learnerUpdates.push_back(learnerUpdate); + + std::tuple *curr_hc = nullptr; + for (std::tuple& hc : hashCounts) { + if (std::get<0>(hc) != blockIndex) continue; + + if (std::get<1>(hc) == block_hash) { + curr_hc = &hc; + break; + } + } + + if (curr_hc == nullptr) { + hashCounts.push_back(std::tuple(blockIndex, block_hash, 1)); + curr_hc = &hashCounts.back(); + } + + if (std::get<2>(curr_hc) >= TEMP_VAR_MIN_NEEDED) { + sequence_status[blockIndex] = 2; + } + break; + + case 2: + std::tuple *valid_hc = nullptr; + for (std::tuple& hc : hashCounts) { + if (std::get<0>(hc) != blockIndex) continue; + + if (std::get<2>(hc) >= TEMP_VAR_MIN_NEEDED) { + valid_hc = &hc; + break; + } + } + + if (std::get<1>(valid_hc) == block_hash) { + + std::vector lus; + for (resdb::LearnerUpdate lu : learnerUpdates) { + if (lu.block_hash == block_hash) { + lus.push_back(lu); + } + } + + vector choice(lus.size(), false); + for (int i = 0; i < TEMP_VAR_MIN_NEEDED-1; i++) { + choice[i] = true; + } + + while (GetNextCombination(choice)) { + + + + } + + } + + break; + + + } + +} + +bool Learner::GetNextCombination(vector &choice) { + int stage = 0; + int counter = 0; + for (int i = choice.size() - 1; i >= 0; i--) { + if (stage == 0) { + if (choice[i] == 1) { + stage = 1; + counter = 1; + choice[i] = 0; + } + } else if (stage == 1) { + if (choice[i] == 1) { + counter++; + choice[i] = 0; + } else { + stage = 2; + } + } else if (stage == 2) { + if (choice[i] = 1) { + choice[i] = 0; + counter++; + + while (counter > 0) { + choice[i+counter] = 1; + counter--; + } + + stage = 3; + } + } + } + + return stage == 3; +} + void Learner::MetricsLoop() const { uint64_t last_messages = 0; uint64_t last_bytes = 0; diff --git a/platform/learner/learner.h b/platform/learner/learner.h index ab1f95853c..1a20f023e7 100644 --- a/platform/learner/learner.h +++ b/platform/learner/learner.h @@ -24,6 +24,7 @@ #include #include #include +#include #include "chain/storage/storage.h" #include "proto/kv/kv.pb.h" @@ -59,6 +60,8 @@ class Learner { void PrintMetrics() const; void InitializeStorage(); + void HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate); + private: LearnerConfig config_; std::atomic is_running_{false}; @@ -74,4 +77,8 @@ class Learner { mutable std::thread metrics_thread_; mutable std::unique_ptr storage_; std::unordered_set known_keys_; + + std::vector sequence_status; // 0 = not started, 1 = unknown valid hash, 2 = known valid hash, 3 = completed + std::vector learnerUpdates; + std::vector> hashCounts; }; From 9f9e85c984b8eaccfce78babe6d3fa02c6abc986 Mon Sep 17 00:00:00 2001 From: Ryan Nelson Date: Thu, 4 Dec 2025 01:09:25 -0800 Subject: [PATCH 34/41] at least no compile issues --- .../consensus/ordering/pbft/commitment.cpp | 7 + platform/learner/BUILD | 1 + platform/learner/learner.cpp | 202 ++++++++++++++++-- platform/learner/learner.h | 15 +- platform/proto/resdb.proto | 1 + 5 files changed, 202 insertions(+), 24 deletions(-) diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index 7426b79362..6ba2cb95ce 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -375,6 +375,12 @@ void Commitment::SendUpdateToLearners(int seq_num) { // perform message distribution algorithm std::vector F_i; + uint32_t excess_bytes = 0; + while (raw_data.size() % 257 != 0) { + raw_data += "0"; + excess_bytes++; + } + uint32_t iter = 0; uint32_t c_ik = 0; for (int d = 0; d < raw_data.size(); d++) { // data MUST BE A MULTIPLE OF m @@ -398,6 +404,7 @@ void Commitment::SendUpdateToLearners(int seq_num) { learnerUpdate.set_block_hash(block_hash); learnerUpdate.set_seq(seq_num); learnerUpdate.set_sender_id(config_.GetSelfInfo().id()); + learnerUpdate.set_excess_bytes(excess_bytes); for (const auto& learner : config_.GetLearnerInfos()) { replica_communicator_->SendMessage(learnerUpdate, learner.id()); diff --git a/platform/learner/BUILD b/platform/learner/BUILD index bd38e094f4..852205e4f7 100644 --- a/platform/learner/BUILD +++ b/platform/learner/BUILD @@ -32,6 +32,7 @@ cc_library( "//platform/proto:resdb_cc_proto", "//proto/kv:kv_cc_proto", "@com_github_google_glog//:glog", + "//common/crypto:hash", ], ) diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index c216cb5aa7..ca479c17fe 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -37,6 +37,7 @@ #include "platform/proto/resdb.pb.h" #include "proto/kv/kv.pb.h" #include "platform/statistic/stats.h" +#include "common/crypto/hash.h" namespace { @@ -69,6 +70,8 @@ LearnerConfig ParseLearnerConfig(const std::string& config_path) { config.replicas.push_back(replica_info); } } + config.total_replicas = config.replicas.size(); + config.min_needed = (config.total_replicas - 1) / 3 + 1; } if (config.port <= 0) { @@ -112,7 +115,7 @@ void Learner::Run() { } } -void Learner::HandleClient(std::unique_ptr socket) const { +void Learner::HandleClient(std::unique_ptr socket) { // std::cout << "[Learner] replica connected" << std::endl; while (true) { void* buffer = nullptr; @@ -135,7 +138,7 @@ void Learner::HandleClient(std::unique_ptr socket) const { } bool Learner::ProcessBroadcast(resdb::Socket* socket, - const std::string& payload) const { + const std::string& payload) { resdb::ResDBMessage envelope; if (!envelope.ParseFromString(payload)) { @@ -164,7 +167,6 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, return true; } - LOG(INFO) << "HERE 2"; resdb::LearnerUpdate request; if (request.ParseFromString(envelope.data())) { @@ -175,6 +177,7 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, LOG(INFO) << "HASH: " << request.block_hash(); LOG(INFO) << "SEQ: " << request.seq(); LOG(INFO) << "S_ID: " << request.sender_id(); + LOG(INFO) << "excess_bytes: " << request.excess_bytes(); HandleLearnerUpdate(request); @@ -204,11 +207,12 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { - std::string block_hash = request.block_hash(); - int seq = request.seq(); - int sender_id = request.sender_id(); + std::string block_hash = learnerUpdate.block_hash(); + int c_seq = learnerUpdate.seq(); + int sender_id = learnerUpdate.sender_id(); + int excess_bytes = learnerUpdate.excess_bytes(); - int blockIndex = seq / config_.block_size() - 1; + int blockIndex = c_seq / config_.block_size - 1; // fill sequence status with "havent started" if variable doesnt exist yet while (sequence_status.size() < blockIndex + 1) { @@ -216,17 +220,18 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { } switch (sequence_status[blockIndex]) { - case 0: + case 0: { hashCounts.push_back(std::tuple(blockIndex, block_hash, 1)); learnerUpdates.push_back(learnerUpdate); sequence_status[blockIndex] = 1; - if (1 >= TEMP_VAR_MIN_NEEDED) { + if (1 >= config_.min_needed) { sequence_status[blockIndex] = 2; } break; - case 1: + } + case 1: { learnerUpdates.push_back(learnerUpdate); std::tuple *curr_hc = nullptr; @@ -244,52 +249,207 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { curr_hc = &hashCounts.back(); } - if (std::get<2>(curr_hc) >= TEMP_VAR_MIN_NEEDED) { + if (std::get<2>(*curr_hc) >= config_.min_needed) { sequence_status[blockIndex] = 2; } break; - - case 2: + } + case 2: { std::tuple *valid_hc = nullptr; for (std::tuple& hc : hashCounts) { if (std::get<0>(hc) != blockIndex) continue; - if (std::get<2>(hc) >= TEMP_VAR_MIN_NEEDED) { + if (std::get<2>(hc) >= config_.min_needed) { valid_hc = &hc; break; } } - if (std::get<1>(valid_hc) == block_hash) { + if (std::get<1>(*valid_hc) == block_hash) { std::vector lus; for (resdb::LearnerUpdate lu : learnerUpdates) { - if (lu.block_hash == block_hash) { + if (lu.block_hash() == block_hash) { lus.push_back(lu); } } - vector choice(lus.size(), false); - for (int i = 0; i < TEMP_VAR_MIN_NEEDED-1; i++) { + lus.push_back(learnerUpdate); + + std::vector choice(lus.size(), false); + for (int i = 0; i < config_.min_needed; i++) { choice[i] = true; } while (GetNextCombination(choice)) { + if (!choice[choice.size()-1]) continue; // include the current received message + + std::vector inds; + std::vector rep_ids; + for (int i = 0; i < choice.size(); i++) { + if (choice[i]) { + inds.push_back(i); + rep_ids.push_back(lus[i].sender_id()); + } + } + + std::vector> A = gen_A(rep_ids); + + std::vector> Ainv = invertMatrix(A, 257); + + std::string raw_bytes; + for (auto i_iter = 0; i_iter < learnerUpdate.data().size(); i_iter++) { // iterate through each block of m bytes + + for (auto b_iter = 0; b_iter < config_.min_needed; b_iter++) { // iterate through each byte - + uint32_t mid = 0; + for (int m_iter = 0; m_iter < config_.min_needed; m_iter++) { // do dot product + + mid = (mid + Ainv[b_iter][m_iter] * lus[inds[m_iter]].data()[i_iter]) % 257; + + } + + raw_bytes += (char)mid; + + } + + } + + for (int i = 0; i < excess_bytes; i++) { + raw_bytes.pop_back(); + } + + std::string final_hash = resdb::utils::CalculateSHA256Hash(raw_bytes); + + if (final_hash == block_hash) { // we have reconstructed the batch + LOG(INFO) << "RECONSTRUCTED" << c_seq; + + resdb::RequestBatch batch; + if (batch.ParseFromArray(raw_bytes.data(), static_cast(raw_bytes.size()))) { + std::vector out; + for (int i = 0; i < batch.requests_size(); ++i) { + out.push_back(batch.requests(i)); // copies each Request + LOG(INFO) << out[i].seq(); + } + + } + + sequence_status[blockIndex] = 3; + } } } - break; + } + + } +} +uint32_t Learner::modpow(uint32_t a, uint32_t e, uint32_t p) { + uint32_t r = 1; + while (e > 0) { + if (e & 1) r = (r * a) % p; + a = (a * a) % p; + e >>= 1; } + return r; +} + +uint32_t Learner::modinv(uint32_t x, uint32_t p) { + return modpow(x, p - 2, p); +} + +std::vector> Learner::invertMatrix(std::vector> A, int p) { + int n = A.size(); + + // Form augmented matrix [A | I] + std::vector> aug(n, std::vector(2*n)); + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + aug[i][j] = A[i][j] % p; + + aug[i][n+j] = 0; + if (i == j) aug[i][n+j] = 1; + } + + } + + // Gauss-Jordan + for (int col = 0; col < n; col++) { + + // Find pivot row + int pivot = col; + while (pivot < n && aug[pivot][col] == 0) pivot++; + if (pivot == n) throw std::runtime_error("Matrix is not invertible mod p"); + + swap(aug[col], aug[pivot]); + + // Normalize pivot row + int32_t inv = modinv(aug[col][col], p); + for (int j = 0; j < 2*n; j++) + aug[col][j] = (aug[col][j] * inv) % p; + + // Eliminate other rows + for (int i = 0; i < n; i++) { + if (i == col) continue; + int32_t factor = aug[i][col]; + for (int j = 0; j < 2*n; j++) { + int32_t temp = (aug[i][j] - factor * aug[col][j]) % p; + if (temp < 0) temp += p; + aug[i][j] = temp; + } + } + + } + + // Extract inverse matrix + std::vector> invA(n, std::vector(n)); + for (int i = 0; i < n; i++) + for (int j = 0; j < n; j++) + invA[i][j] = aug[i][n+j]; + + return invA; +} + +std::vector> Learner::gen_A(std::vector inds) { + + int n = inds.size(); + int m = config_.min_needed; + int p = 257; + + std::vector> A(n,std::vector(m)); + + for (int i = 0; i < n; i++) { + + for (int j = 0; j < m; j++) { + + int exp = j; + int base = inds[i]; + int mod = p; + int result = 1; + + while (exp > 0) { + if (exp & 1) { + result = (result * base) % mod; + } + + base = (base * base) % mod; + exp >>= 1; + } + + A[i][j] = result; + + } + + } + + return A; } -bool Learner::GetNextCombination(vector &choice) { +bool Learner::GetNextCombination(std::vector &choice) { int stage = 0; int counter = 0; for (int i = choice.size() - 1; i >= 0; i--) { diff --git a/platform/learner/learner.h b/platform/learner/learner.h index 1a20f023e7..a7890fa7db 100644 --- a/platform/learner/learner.h +++ b/platform/learner/learner.h @@ -28,7 +28,9 @@ #include "chain/storage/storage.h" #include "proto/kv/kv.pb.h" +#include "platform/proto/resdb.pb.h" #include "platform/proto/replica_info.pb.h" +#include "common/crypto/hash.h" namespace resdb { class Socket; @@ -41,6 +43,8 @@ struct LearnerConfig { int block_size = 0; std::string db_path; std::vector replicas; + int total_replicas = 0; + int min_needed = 0; }; class Learner { @@ -50,10 +54,10 @@ class Learner { private: static LearnerConfig LoadConfig(const std::string& config_path); - void HandleClient(std::unique_ptr socket) const; + void HandleClient(std::unique_ptr socket); // Returns true if the connection should remain open, false if it should be closed. bool ProcessBroadcast(resdb::Socket* socket, - const std::string& payload) const; + const std::string& payload); bool HandleReadOnlyRequest(resdb::Socket* socket, const resdb::KVRequest& request) const; void MetricsLoop() const; @@ -61,6 +65,11 @@ class Learner { void InitializeStorage(); void HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate); + uint32_t modpow(uint32_t a, uint32_t e, uint32_t p); + uint32_t modinv(uint32_t x, uint32_t p); + std::vector> invertMatrix(std::vector> A, int p); + std::vector> gen_A(std::vector inds); + bool GetNextCombination(std::vector &choice); private: LearnerConfig config_; @@ -80,5 +89,5 @@ class Learner { std::vector sequence_status; // 0 = not started, 1 = unknown valid hash, 2 = known valid hash, 3 = completed std::vector learnerUpdates; - std::vector> hashCounts; + std::vector> hashCounts; }; diff --git a/platform/proto/resdb.proto b/platform/proto/resdb.proto index e810752e5e..10b05fb68f 100644 --- a/platform/proto/resdb.proto +++ b/platform/proto/resdb.proto @@ -100,6 +100,7 @@ message LearnerUpdate { bytes block_hash = 2; uint64 seq = 3; uint32 sender_id = 4; + uint32 excess_bytes = 5; } // The response message containing response From e5165208ff6bf58911a35194304e8d9e025b0727 Mon Sep 17 00:00:00 2001 From: Ryan Nelson Date: Fri, 5 Dec 2025 23:28:53 -0800 Subject: [PATCH 35/41] bugfixing in progress --- api/learner-test.py | 6 +- entrypoint.sh | 2 +- .../consensus/ordering/pbft/commitment.cpp | 12 +- platform/learner/learner.cpp | 107 ++++++++++-------- platform/learner/learner.h | 2 +- scripts/deploy/config/pbft.config | 2 +- service/tools/config/server/server.config | 4 +- 7 files changed, 81 insertions(+), 54 deletions(-) diff --git a/api/learner-test.py b/api/learner-test.py index 6d3086c44d..88a3101445 100644 --- a/api/learner-test.py +++ b/api/learner-test.py @@ -5,6 +5,8 @@ from kv_operation import set_value, get_value, get_value_readonly -print(set_value("test", "111225")) -time.sleep(0.5) +for i in range(0,10): + print(set_value("test", "342")) + time.sleep(0.2) + print(f"Key 'test' value: {get_value_readonly('test')}") \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index 7962b7884a..5f45e48292 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -19,7 +19,7 @@ # under the License. # -# ./service/tools/kv/server_tools/generate_config.sh +./service/tools/kv/server_tools/generate_config.sh ./service/tools/kv/server_tools/start_kv_service_learner.sh if [[ "$2" =~ ^[0-9]+$ ]]; then diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index 6ba2cb95ce..5776bc6247 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -239,6 +239,7 @@ int Commitment::ProcessProposeMsg(std::unique_ptr context, // If receive 2f+1 prepare message, broadcast a commit message. int Commitment::ProcessPrepareMsg(std::unique_ptr context, std::unique_ptr request) { + if (context == nullptr || context->signature.signature().empty()) { LOG(ERROR) << "user request doesn't contain signature, reject"; return -2; @@ -330,6 +331,7 @@ int Commitment::PostProcessExecutedMsg() { // send an update to learners after every block size execution if (request.seq() % config_.GetBlockSize() == 0) { + LOG(ERROR) << "SENDING LEARNER UPDATE"; SendUpdateToLearners(request.seq()); } @@ -366,6 +368,13 @@ void Commitment::SendUpdateToLearners(int seq_num) { // turn the entire batch variable into string batch.SerializeToString(&raw_data); + // std::ofstream out("learner_resTEST" + std::to_string(config_.GetSelfInfo().id()) + ".txt"); + // for (int i = 0; i < raw_data.size(); i++) { + // for (int j=1; j < 256; j*=2) { + // out << (raw_data[i] & j) << " "; + // } + // } + // generate hash of entire batch std::string block_hash = resdb::utils::CalculateSHA256Hash(raw_data); @@ -376,13 +385,14 @@ void Commitment::SendUpdateToLearners(int seq_num) { std::vector F_i; uint32_t excess_bytes = 0; - while (raw_data.size() % 257 != 0) { + while (raw_data.size() % A_i.size() != 0) { raw_data += "0"; excess_bytes++; } uint32_t iter = 0; uint32_t c_ik = 0; + for (int d = 0; d < raw_data.size(); d++) { // data MUST BE A MULTIPLE OF m c_ik = (c_ik + A_i[iter] * raw_data[d]) % 257; diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index ca479c17fe..7e43f020fc 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -143,7 +143,6 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, resdb::ResDBMessage envelope; if (!envelope.ParseFromString(payload)) { - LOG(INFO) << "HERE 1"; resdb::KVRequest kv_request; if (kv_request.ParseFromString(payload)) { if (HandleReadOnlyRequest(socket, kv_request)) { @@ -172,12 +171,11 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, if (request.sender_id() == 0) return true; - LOG(INFO) << "FFFFFFFFFFFFFFFFFFFFFFFFF"; // LOG(INFO) << "DATA: " << request.data(); - LOG(INFO) << "HASH: " << request.block_hash(); - LOG(INFO) << "SEQ: " << request.seq(); + // LOG(INFO) << "HASH: " << request.block_hash(); + // LOG(INFO) << "SEQ: " << request.seq(); LOG(INFO) << "S_ID: " << request.sender_id(); - LOG(INFO) << "excess_bytes: " << request.excess_bytes(); + // LOG(INFO) << "excess_bytes: " << request.excess_bytes(); HandleLearnerUpdate(request); @@ -207,6 +205,8 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { + LOG(INFO) << "START HANDLE"; + std::string block_hash = learnerUpdate.block_hash(); int c_seq = learnerUpdate.seq(); int sender_id = learnerUpdate.sender_id(); @@ -221,6 +221,7 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { switch (sequence_status[blockIndex]) { case 0: { + LOG(INFO) << "CASE 0"; hashCounts.push_back(std::tuple(blockIndex, block_hash, 1)); learnerUpdates.push_back(learnerUpdate); sequence_status[blockIndex] = 1; @@ -232,39 +233,51 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { break; } case 1: { - learnerUpdates.push_back(learnerUpdate); + } + case 2: { + LOG(INFO) << "CASE 1/2: " << sequence_status[blockIndex]; - std::tuple *curr_hc = nullptr; - for (std::tuple& hc : hashCounts) { - if (std::get<0>(hc) != blockIndex) continue; + std::tuple *valid_hc = nullptr; - if (std::get<1>(hc) == block_hash) { - curr_hc = &hc; - break; - } - } + if (sequence_status[blockIndex] == 1) { + learnerUpdates.push_back(learnerUpdate); - if (curr_hc == nullptr) { - hashCounts.push_back(std::tuple(blockIndex, block_hash, 1)); - curr_hc = &hashCounts.back(); - } + std::tuple *curr_hc = nullptr; + for (std::tuple& hc : hashCounts) { + if (std::get<0>(hc) != blockIndex) continue; - if (std::get<2>(*curr_hc) >= config_.min_needed) { - sequence_status[blockIndex] = 2; - } - break; - } - case 2: { - std::tuple *valid_hc = nullptr; - for (std::tuple& hc : hashCounts) { - if (std::get<0>(hc) != blockIndex) continue; + if (std::get<1>(hc) == block_hash) { + std::get<2>(hc) = std::get<2>(hc) + 1; + curr_hc = &hc; + break; + } + } + + if (curr_hc == nullptr) { + hashCounts.push_back(std::tuple(blockIndex, block_hash, 1)); + curr_hc = &hashCounts.back(); + } - if (std::get<2>(hc) >= config_.min_needed) { - valid_hc = &hc; + if (std::get<2>(*curr_hc) >= config_.min_needed) { + sequence_status[blockIndex] = 2; + valid_hc = curr_hc; + } else { break; } } + if (valid_hc == nullptr) { // will run if input case: 2 + for (std::tuple& hc : hashCounts) { + if (std::get<0>(hc) != blockIndex) continue; + + if (std::get<2>(hc) >= config_.min_needed && std::get<1>(hc) == block_hash) { // only set if hash is valid and hash matches received hash + learnerUpdates.push_back(learnerUpdate); + valid_hc = &hc; + break; + } + } + } + if (std::get<1>(*valid_hc) == block_hash) { std::vector lus; @@ -274,14 +287,14 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { } } - lus.push_back(learnerUpdate); - + bool first = true; std::vector choice(lus.size(), false); for (int i = 0; i < config_.min_needed; i++) { choice[i] = true; } - while (GetNextCombination(choice)) { + while (GetNextCombination(choice, first)) { + first = false; if (!choice[choice.size()-1]) continue; // include the current received message std::vector inds; @@ -298,7 +311,7 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { std::vector> Ainv = invertMatrix(A, 257); std::string raw_bytes; - for (auto i_iter = 0; i_iter < learnerUpdate.data().size(); i_iter++) { // iterate through each block of m bytes + for (auto i_iter = 0; i_iter < learnerUpdate.data().size(); i_iter++) { // iterate through each block of m bytes (assumption, lengths are equal) for (auto b_iter = 0; b_iter < config_.min_needed; b_iter++) { // iterate through each byte @@ -309,7 +322,7 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { } - raw_bytes += (char)mid; + raw_bytes.push_back(static_cast(mid)); } @@ -319,6 +332,13 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { raw_bytes.pop_back(); } + // std::ofstream out("learner_resTEST.txt"); + // for (int i = 0; i < raw_bytes.size(); i++) { + // for (int j=1; j < 256; j*=2) { + // out << (raw_bytes[i] & j) << " "; + // } + // } + std::string final_hash = resdb::utils::CalculateSHA256Hash(raw_bytes); if (final_hash == block_hash) { // we have reconstructed the batch @@ -449,39 +469,34 @@ std::vector> Learner::gen_A(std::vector inds) { } -bool Learner::GetNextCombination(std::vector &choice) { +bool Learner::GetNextCombination(std::vector &choice, bool first) { + if (first) return true; + int stage = 0; int counter = 0; for (int i = choice.size() - 1; i >= 0; i--) { if (stage == 0) { if (choice[i] == 1) { - stage = 1; - counter = 1; + counter++; choice[i] = 0; + } else { + stage = 1; } } else if (stage == 1) { if (choice[i] == 1) { counter++; choice[i] = 0; - } else { stage = 2; - } - } else if (stage == 2) { - if (choice[i] = 1) { - choice[i] = 0; - counter++; while (counter > 0) { choice[i+counter] = 1; counter--; } - - stage = 3; } } } - return stage == 3; + return stage == 2; } void Learner::MetricsLoop() const { diff --git a/platform/learner/learner.h b/platform/learner/learner.h index a7890fa7db..9e32ad53aa 100644 --- a/platform/learner/learner.h +++ b/platform/learner/learner.h @@ -69,7 +69,7 @@ class Learner { uint32_t modinv(uint32_t x, uint32_t p); std::vector> invertMatrix(std::vector> A, int p); std::vector> gen_A(std::vector inds); - bool GetNextCombination(std::vector &choice); + bool GetNextCombination(std::vector &choice, bool first); private: LearnerConfig config_; diff --git a/scripts/deploy/config/pbft.config b/scripts/deploy/config/pbft.config index 93e2ba29e1..16145bdc44 100644 --- a/scripts/deploy/config/pbft.config +++ b/scripts/deploy/config/pbft.config @@ -1,5 +1,5 @@ { - "clientBatchNum": 100, + "clientBatchNum": 10, "enable_viewchange": true, "recovery_enabled": true, "max_client_complaint_num":10, diff --git a/service/tools/config/server/server.config b/service/tools/config/server/server.config index 7442dce7bd..f8e0008e93 100644 --- a/service/tools/config/server/server.config +++ b/service/tools/config/server/server.config @@ -27,10 +27,10 @@ ], "learnerInfo": [ { - "id": 6, + "id": 0, "ip": "127.0.0.1", "port": 25000 } ], - "block_size": 100 + "block_size": 10 } \ No newline at end of file From 74eb2d8c4c95789a6b1aacfd8ccf9b1dcb2e708d Mon Sep 17 00:00:00 2001 From: Ryan Nelson Date: Sat, 6 Dec 2025 17:33:04 -0800 Subject: [PATCH 36/41] working encode/decode --- .../consensus/ordering/pbft/commitment.cpp | 14 ++------ platform/learner/learner.cpp | 33 ++++++------------- platform/learner/learner.h | 2 ++ 3 files changed, 15 insertions(+), 34 deletions(-) diff --git a/platform/consensus/ordering/pbft/commitment.cpp b/platform/consensus/ordering/pbft/commitment.cpp index 5776bc6247..78ef4df98a 100644 --- a/platform/consensus/ordering/pbft/commitment.cpp +++ b/platform/consensus/ordering/pbft/commitment.cpp @@ -331,7 +331,6 @@ int Commitment::PostProcessExecutedMsg() { // send an update to learners after every block size execution if (request.seq() % config_.GetBlockSize() == 0) { - LOG(ERROR) << "SENDING LEARNER UPDATE"; SendUpdateToLearners(request.seq()); } @@ -368,13 +367,6 @@ void Commitment::SendUpdateToLearners(int seq_num) { // turn the entire batch variable into string batch.SerializeToString(&raw_data); - // std::ofstream out("learner_resTEST" + std::to_string(config_.GetSelfInfo().id()) + ".txt"); - // for (int i = 0; i < raw_data.size(); i++) { - // for (int j=1; j < 256; j*=2) { - // out << (raw_data[i] & j) << " "; - // } - // } - // generate hash of entire batch std::string block_hash = resdb::utils::CalculateSHA256Hash(raw_data); @@ -394,8 +386,8 @@ void Commitment::SendUpdateToLearners(int seq_num) { uint32_t c_ik = 0; for (int d = 0; d < raw_data.size(); d++) { // data MUST BE A MULTIPLE OF m - c_ik = (c_ik + A_i[iter] * raw_data[d]) % 257; - + c_ik = (c_ik + A_i[iter] * (uint8_t)static_cast(raw_data[d])) % 257; + iter++; if (iter == A_i.size()) { F_i.push_back(c_ik); @@ -404,7 +396,7 @@ void Commitment::SendUpdateToLearners(int seq_num) { iter = 0; } } - + // add data to learner update message and send it LearnerUpdate learnerUpdate; learnerUpdate.mutable_data()->Reserve(static_cast(F_i.size())); diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index 7e43f020fc..b537431d2e 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -71,7 +72,7 @@ LearnerConfig ParseLearnerConfig(const std::string& config_path) { } } config.total_replicas = config.replicas.size(); - config.min_needed = (config.total_replicas - 1) / 3 + 1; + config.min_needed = (config.total_replicas - 1) / 3 + 2; } if (config.port <= 0) { @@ -171,26 +172,13 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, if (request.sender_id() == 0) return true; - // LOG(INFO) << "DATA: " << request.data(); - // LOG(INFO) << "HASH: " << request.block_hash(); - // LOG(INFO) << "SEQ: " << request.seq(); - LOG(INFO) << "S_ID: " << request.sender_id(); - // LOG(INFO) << "excess_bytes: " << request.excess_bytes(); - HandleLearnerUpdate(request); ++total_messages_; - // total_bytes_.fetch_add(payload.size()); - // last_type_.store(request.type()); - // last_seq_.store(request.seq()); - // last_sender_.store(request.sender_id()); - // last_payload_bytes_.store(request.data().size()); - // resdb::BatchUserResponse batch_response; - // if (batch_response.ParseFromString(request.data())) { - // total_sets_.fetch_add(batch_response.set_count()); - // total_reads_.fetch_add(batch_response.read_count()); - // total_deletes_.fetch_add(batch_response.delete_count()); - // } + total_bytes_.fetch_add(payload.size()); + + + return true; } @@ -205,7 +193,7 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { - LOG(INFO) << "START HANDLE"; + std::lock_guard lock(m); std::string block_hash = learnerUpdate.block_hash(); int c_seq = learnerUpdate.seq(); @@ -221,7 +209,6 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { switch (sequence_status[blockIndex]) { case 0: { - LOG(INFO) << "CASE 0"; hashCounts.push_back(std::tuple(blockIndex, block_hash, 1)); learnerUpdates.push_back(learnerUpdate); sequence_status[blockIndex] = 1; @@ -235,8 +222,6 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { case 1: { } case 2: { - LOG(INFO) << "CASE 1/2: " << sequence_status[blockIndex]; - std::tuple *valid_hc = nullptr; if (sequence_status[blockIndex] == 1) { @@ -342,7 +327,8 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { std::string final_hash = resdb::utils::CalculateSHA256Hash(raw_bytes); if (final_hash == block_hash) { // we have reconstructed the batch - LOG(INFO) << "RECONSTRUCTED" << c_seq; + + LOG(INFO) << "RECONSTRUCTED " << c_seq << " FROM " << rep_ids[0] << " " << rep_ids[1] << " " << rep_ids[2]; resdb::RequestBatch batch; if (batch.ParseFromArray(raw_bytes.data(), static_cast(raw_bytes.size()))) { @@ -355,6 +341,7 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { } sequence_status[blockIndex] = 3; + return; } } diff --git a/platform/learner/learner.h b/platform/learner/learner.h index 9e32ad53aa..e0b3cecab0 100644 --- a/platform/learner/learner.h +++ b/platform/learner/learner.h @@ -25,6 +25,7 @@ #include #include #include +#include #include "chain/storage/storage.h" #include "proto/kv/kv.pb.h" @@ -90,4 +91,5 @@ class Learner { std::vector sequence_status; // 0 = not started, 1 = unknown valid hash, 2 = known valid hash, 3 = completed std::vector learnerUpdates; std::vector> hashCounts; + std::mutex m; }; From ba2501c469dde283b48e9d95f228ed2c875828e5 Mon Sep 17 00:00:00 2001 From: Ryan Nelson Date: Sat, 6 Dec 2025 19:02:18 -0800 Subject: [PATCH 37/41] working system? --- api/learner-test.py | 9 ++++++-- platform/learner/learner.cpp | 40 ++++++++++++++++++++++++++++-------- platform/learner/learner.h | 1 + 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/api/learner-test.py b/api/learner-test.py index 88a3101445..424eb831d5 100644 --- a/api/learner-test.py +++ b/api/learner-test.py @@ -6,7 +6,12 @@ from kv_operation import set_value, get_value, get_value_readonly for i in range(0,10): - print(set_value("test", "342")) + print(set_value("INC_TEST", "0")) + time.sleep(0.2) + +for i in range(0,10): + print(set_value("INC_TEST", str(i+1))) + time.sleep(0.2) + print(f"Key 'INC_TEST' value: {get_value_readonly('INC_TEST')}") time.sleep(0.2) -print(f"Key 'test' value: {get_value_readonly('test')}") \ No newline at end of file diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index b537431d2e..1e984d68e0 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -72,7 +72,7 @@ LearnerConfig ParseLearnerConfig(const std::string& config_path) { } } config.total_replicas = config.replicas.size(); - config.min_needed = (config.total_replicas - 1) / 3 + 2; + config.min_needed = (config.total_replicas - 1) / 3 + 1; } if (config.port <= 0) { @@ -176,8 +176,6 @@ bool Learner::ProcessBroadcast(resdb::Socket* socket, ++total_messages_; total_bytes_.fetch_add(payload.size()); - - return true; } @@ -328,14 +326,17 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { if (final_hash == block_hash) { // we have reconstructed the batch - LOG(INFO) << "RECONSTRUCTED " << c_seq << " FROM " << rep_ids[0] << " " << rep_ids[1] << " " << rep_ids[2]; + LOG(INFO) << "RECONSTRUCTED " << c_seq << " FROM " << rep_ids[0] << " " << rep_ids[1]; resdb::RequestBatch batch; if (batch.ParseFromArray(raw_bytes.data(), static_cast(raw_bytes.size()))) { - std::vector out; + resdb::BatchUserRequest batch_request; for (int i = 0; i < batch.requests_size(); ++i) { - out.push_back(batch.requests(i)); // copies each Request - LOG(INFO) << out[i].seq(); + batch_request.ParseFromString(batch.requests()[i].data()); + + for (int j = 0; j < batch_request.user_requests().size(); j++) { + ExecuteRequest(batch_request.user_requests()[j].request().data()); + } } } @@ -354,6 +355,26 @@ void Learner::HandleLearnerUpdate(resdb::LearnerUpdate learnerUpdate) { } + +void Learner::ExecuteRequest(std::string requestStr) { + resdb::KVRequest kv_request; + if (!kv_request.ParseFromString(requestStr)) { + return; + } + + if (kv_request.cmd() == resdb::KVRequest::SET) { + storage_->SetValue(kv_request.key(), kv_request.value()); + if (known_keys_.find(kv_request.key()) == known_keys_.end()) { + known_keys_.insert(kv_request.key()); + } + } else if (kv_request.cmd() == resdb::KVRequest::SET_WITH_VERSION) { + storage_->SetValueWithVersion(kv_request.key(), kv_request.value(), kv_request.version()); + if (known_keys_.find(kv_request.key()) == known_keys_.end()) { + known_keys_.insert(kv_request.key()); + } + } +} + uint32_t Learner::modpow(uint32_t a, uint32_t e, uint32_t p) { uint32_t r = 1; while (e > 0) { @@ -504,15 +525,18 @@ void Learner::MetricsLoop() const { bool Learner::HandleReadOnlyRequest(resdb::Socket* socket, const resdb::KVRequest& request) const { if (request.cmd() != resdb::KVRequest::GET_READ_ONLY || storage_ == nullptr) { + LOG(INFO) << "A"; return false; } if (known_keys_.find(request.key()) == known_keys_.end()) { + LOG(INFO) << "B"; return false; } std::string value = storage_->GetValue(request.key()); if (value.empty()) { + LOG(INFO) << "C"; return false; } @@ -561,4 +585,4 @@ void Learner::InitializeStorage() { known_keys_.insert("test"); VLOG(1) << "[Learner] Initialized local DB at " << config_.db_path << " with test key"; -} +} \ No newline at end of file diff --git a/platform/learner/learner.h b/platform/learner/learner.h index e0b3cecab0..567bfaf2d0 100644 --- a/platform/learner/learner.h +++ b/platform/learner/learner.h @@ -71,6 +71,7 @@ class Learner { std::vector> invertMatrix(std::vector> A, int p); std::vector> gen_A(std::vector inds); bool GetNextCombination(std::vector &choice, bool first); + void ExecuteRequest(std::string requestStr); private: LearnerConfig config_; From 3e037a6432d1e531c811001ba3f5f7b3fc204b97 Mon Sep 17 00:00:00 2001 From: Ryan Nelson Date: Sat, 6 Dec 2025 19:03:18 -0800 Subject: [PATCH 38/41] removed some learner logs --- platform/learner/learner.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/platform/learner/learner.cpp b/platform/learner/learner.cpp index 1e984d68e0..7a53544400 100644 --- a/platform/learner/learner.cpp +++ b/platform/learner/learner.cpp @@ -525,18 +525,15 @@ void Learner::MetricsLoop() const { bool Learner::HandleReadOnlyRequest(resdb::Socket* socket, const resdb::KVRequest& request) const { if (request.cmd() != resdb::KVRequest::GET_READ_ONLY || storage_ == nullptr) { - LOG(INFO) << "A"; return false; } if (known_keys_.find(request.key()) == known_keys_.end()) { - LOG(INFO) << "B"; return false; } std::string value = storage_->GetValue(request.key()); if (value.empty()) { - LOG(INFO) << "C"; return false; } From 072e172cbea743bb62c898f8de315216490f3030 Mon Sep 17 00:00:00 2001 From: netzworld Date: Sun, 7 Dec 2025 16:58:07 -0800 Subject: [PATCH 39/41] test+plots --- api/learner-test.py | 642 +++++++++++++++++++++++++++- api/plots/exp1_latency_median.pdf | Bin 0 -> 14579 bytes api/plots/exp1_latency_p95.pdf | Bin 0 -> 16019 bytes api/plots/exp1_throughput.pdf | Bin 0 -> 14798 bytes api/plots/exp2_read_throughput.pdf | Bin 0 -> 15344 bytes api/plots/exp2_write_throughput.pdf | Bin 0 -> 16777 bytes api/plots/exp3_latency.pdf | Bin 0 -> 15409 bytes api/plots/exp3_staleness.pdf | Bin 0 -> 14389 bytes 8 files changed, 631 insertions(+), 11 deletions(-) create mode 100644 api/plots/exp1_latency_median.pdf create mode 100644 api/plots/exp1_latency_p95.pdf create mode 100644 api/plots/exp1_throughput.pdf create mode 100644 api/plots/exp2_read_throughput.pdf create mode 100644 api/plots/exp2_write_throughput.pdf create mode 100644 api/plots/exp3_latency.pdf create mode 100644 api/plots/exp3_staleness.pdf diff --git a/api/learner-test.py b/api/learner-test.py index 424eb831d5..d9865db168 100644 --- a/api/learner-test.py +++ b/api/learner-test.py @@ -1,17 +1,637 @@ -import sys, os, time, subprocess +# import sys, os, time, subprocess -subprocess.run("bazel build :pybind_kv_so", shell=True) -sys.path.append(os.getcwd()) +# subprocess.run("bazel build :pybind_kv_so", shell=True) +# sys.path.append(os.getcwd()) from kv_operation import set_value, get_value, get_value_readonly +import sys, os, time, statistics, threading, queue, subprocess, matplotlib.pyplot as plt + +# Build and import the kv_operation module +def build_kv_binding(): + subprocess.run("bazel build :pybind_kv_so", shell=True, check=True) + sys.path.append(os.getcwd()) + from kv_operation import set_value, get_value, get_value_readonly + return set_value, get_value, get_value_readonly + +# Simple stats helper +def summarize_latencies(latencies): + if not latencies: + return {} + return { + "count": len(latencies), + "mean": statistics.mean(latencies), + "median": statistics.median(latencies), + "p95": statistics.quantiles(latencies, n=20)[18], # approx 95th + "p99": statistics.quantiles(latencies, n=100)[98], # approx 99th + "min": min(latencies), + "max": max(latencies), + } + +# Generic worker to send operations and record per-op latency +def run_client_worker(op_func, num_requests, key_prefix, thread_id, result_queue): + latencies = [] + successes = 0 + for i in range(num_requests): + key = f"{key_prefix}_{thread_id}_{i}" + # Some experiments will treat op_func as GET, some as SET, etc. + start = time.time() + try: + op_func(key) + successes += 1 + except Exception as e: + # Log or ignore for pseudocode + pass + end = time.time() + latencies.append(end - start) + # Push results back to main thread + result_queue.put((successes, latencies)) + +# Experiment 1: Read Latency Comparison +def exp1_read_latency(set_value, get_value, get_value_readonly): + """ + Compare read latency for PBFT vs learner under different concurrency levels. + Assumes KV already populated with some keys. + """ + # Pre-populate a test key so both paths can read something + base_key = "EXP1_KEY" + set_value(base_key, "warmup_value") + + client_concurrency_levels = [1, 4, 8, 16, 32] + num_requests_per_client = 100 + + results = [] + + for num_clients in client_concurrency_levels: + print(f"\n=== Exp1: {num_clients} clients ===") + + for mode in ["pbft", "learner"]: + if mode == "pbft": + read_func = lambda k: get_value(k) + else: + read_func = lambda k: get_value_readonly(k) + + # All clients read same key to stress the path + def op_func(_ignored_key): + # Always read base_key + return read_func(base_key) + + result_queue = queue.Queue() + threads = [] + + start_wall = time.time() + for cid in range(num_clients): + t = threading.Thread( + target=run_client_worker, + args=(op_func, num_requests_per_client, "EXP1", cid, result_queue) + ) + t.start() + threads.append(t) + + for t in threads: + t.join() + end_wall = time.time() + + # Aggregate latencies + all_latencies = [] + total_success = 0 + while not result_queue.empty(): + successes, lat = result_queue.get() + total_success += successes + all_latencies.extend(lat) + + stats = summarize_latencies(all_latencies) + throughput = total_success / (end_wall - start_wall) + + results.append({ + "mode": mode, + "num_clients": num_clients, + "throughput": throughput, + "latency_stats": stats, + }) + + print(f"Mode={mode}, clients={num_clients}") + print(f" throughput={throughput:.2f} reads/sec") + print(f" latency: {stats}") + + return results + +def run_mixed_client_worker( + set_value, + read_func, + key_prefix, + thread_id, + read_ratio, + total_ops, + result_queue, +): + """ + A worker that issues a mix of reads and writes on a *single existing key*. + This avoids calling get_value_readonly() on keys that were never written. + """ + latencies_reads = [] + latencies_writes = [] + num_reads = 0 + num_writes = 0 + + # Use one key per client, pre-populated before measurement + key = f"{key_prefix}_{thread_id}" + + for i in range(total_ops): + is_read = (i % 100) < int(read_ratio * 100) + + if is_read: + start = time.time() + try: + read_func(key) + num_reads += 1 + except Exception: + # Optional: log, but don't kill the worker + pass + end = time.time() + latencies_reads.append(end - start) + else: + value = str(i) + start = time.time() + try: + set_value(key, value) + num_writes += 1 + except Exception: + pass + end = time.time() + latencies_writes.append(end - start) + + result_queue.put({ + "reads": num_reads, + "writes": num_writes, + "latencies_reads": latencies_reads, + "latencies_writes": latencies_writes, + }) + +# Experiment 2: Mixed Read/Write Workload +def exp2_mixed_workload(set_value, get_value, get_value_readonly): + """ + Measure write throughput and latency under different read/write mixes, + comparing PBFT vs learner for reads. + Now: + - pre-populates one key per client, + - only reads/writes existing keys. + """ + mixes = [0.0, 0.5, 0.9, 0.99] # fraction of reads + num_clients = 8 + total_ops_per_client = 200 + + results = [] + + # Pre-populate keys so both PBFT and learner know about them + for cid in range(num_clients): + key = f"EXP2_{cid}" + try: + set_value(key, "0") + except Exception as e: + print(f"Pre-populate failed for {key}: {e}") + + for read_ratio in mixes: + print(f"\n=== Exp2: read_ratio={read_ratio} ===") + + for mode in ["pbft", "learner"]: + if mode == "pbft": + read_func = lambda k: get_value(k) + else: + read_func = lambda k: get_value_readonly(k) + + result_queue = queue.Queue() + threads = [] + + start_wall = time.time() + for cid in range(num_clients): + t = threading.Thread( + target=run_mixed_client_worker, + args=( + set_value, + read_func, + "EXP2", + cid, + read_ratio, + total_ops_per_client, + result_queue, + ), + ) + t.start() + threads.append(t) + + for t in threads: + t.join() + end_wall = time.time() + + # Aggregate + total_reads = 0 + total_writes = 0 + all_read_lat = [] + all_write_lat = [] + while not result_queue.empty(): + r = result_queue.get() + total_reads += r["reads"] + total_writes += r["writes"] + all_read_lat.extend(r["latencies_reads"]) + all_write_lat.extend(r["latencies_writes"]) + + duration = end_wall - start_wall + read_throughput = total_reads / duration if duration > 0 else 0 + write_throughput = total_writes / duration if duration > 0 else 0 + + stats_reads = summarize_latencies(all_read_lat) + stats_writes = summarize_latencies(all_write_lat) + + results.append({ + "mode": mode, + "read_ratio": read_ratio, + "read_throughput": read_throughput, + "write_throughput": write_throughput, + "latency_reads": stats_reads, + "latency_writes": stats_writes, + }) + + print(f"Mode={mode}, read_ratio={read_ratio}") + print(f" read_throughput={read_throughput:.2f} rps") + print(f" write_throughput={write_throughput:.2f} wps") + print(f" read latency: {stats_reads}") + print(f" write latency: {stats_writes}") + + return results + +# Experiment 3: Staleness and Fallback Rate +def exp3_staleness_and_fallback(set_value, get_value, get_value_readonly): + """ + Measure how often learner reads differ from PBFT reads (staleness) + and how often learner fails and forces fallback. + """ + base_key = "EXP3_KEY" + + num_rounds = 100 + sleep_between_writes = 0.1 # simulate blocks committing over time + + num_equal = 0 + num_stale = 0 + num_fallback = 0 + learner_latencies = [] + pbft_latencies = [] + + for i in range(num_rounds): + # Write a fresh value through PBFT (normal write) + new_value = f"value_{i}" + set_value(base_key, new_value) + time.sleep(sleep_between_writes) + + # Ground truth read via PBFT + start_p = time.time() + truth = get_value(base_key) + end_p = time.time() + pbft_latencies.append(end_p - start_p) + + # Try learner read + start_l = time.time() + try: + learner_val = get_value_readonly(base_key) + end_l = time.time() + learner_latencies.append(end_l - start_l) + + if learner_val == truth: + num_equal += 1 + else: + # learner responded but value != PBFT truth ⇒ stale + num_stale += 1 + except Exception: + # interpret as fallback / failure to serve from learner + end_l = time.time() + learner_latencies.append(end_l - start_l) + num_fallback += 1 + + total = num_rounds + stats_learner = summarize_latencies(learner_latencies) + stats_pbft = summarize_latencies(pbft_latencies) + + print("\n=== Exp3: Staleness & Fallback ===") + print(f"Total trials: {total}") + print(f" equal (fresh) : {num_equal} ({num_equal/total:.2%})") + print(f" stale responses : {num_stale} ({num_stale/total:.2%})") + print(f" fallbacks : {num_fallback} ({num_fallback/total:.2%})") + print(f" learner latency : {stats_learner}") + print(f" PBFT latency : {stats_pbft}") + + return { + "total": total, + "equal": num_equal, + "stale": num_stale, + "fallback": num_fallback, + "latency_learner": stats_learner, + "latency_pbft": stats_pbft, + } + +# Experiment 4: Fault Injection Sanity Check +def exp4_fault_injection_sanity(set_value, get_value, get_value_readonly): + """ + Assumes that on the server side you have configured one replica + to send bad digests to the learner. + + From the client side, we: + - write monotone increasing integers to a key, + - read via PBFT as ground truth, + - read via learner, and + - check for impossible values. + """ + key = "EXP4_KEY" + num_rounds = 50 + + impossible_values = [] + mismatches = 0 + fallbacks = 0 + + for i in range(num_rounds): + # Write monotone integer + set_value(key, str(i)) + time.sleep(0.1) + + # Ground truth + truth = get_value(key) + + try: + val = get_value_readonly(key) + except Exception: + # learner refused due to bad digests or staleness + fallbacks += 1 + continue + + if val != truth: + mismatches += 1 + # If you never wrote val at all, this is impossible + try: + int_val = int(val) + if int_val < 0 or int_val > i: + impossible_values.append(val) + except ValueError: + # non-integer, clearly impossible in this setup + impossible_values.append(val) + + print("\n=== Exp4: Fault Injection Sanity ===") + print(f"Total rounds: {num_rounds}") + print(f" mismatches (learner != PBFT): {mismatches}") + print(f" fallbacks (exceptions) : {fallbacks}") + print(f" impossible values seen : {impossible_values}") + + # In a correct implementation, impossible_values should be empty. + return { + "mismatches": mismatches, + "fallbacks": fallbacks, + "impossible_values": impossible_values, + } + +def plot_exp1_latency(exp1_results, metric="median"): + """ + exp1_results: list of dicts from exp1_read_latency() + each dict has: + "mode": "pbft" or "learner" + "num_clients": int + "throughput": float + "latency_stats": {"median": ..., "p95": ..., ...} + + metric: which stat to plot: "median", "p95", "p99", etc. + """ + modes = ["pbft", "learner"] + num_clients = sorted(set(r["num_clients"] for r in exp1_results)) + + plt.figure() + for mode in modes: + ys = [] + for c in num_clients: + entry = next( + r for r in exp1_results + if r["mode"] == mode and r["num_clients"] == c + ) + ys.append(entry["latency_stats"][metric]) + plt.plot(num_clients, ys, marker="o", label=mode) + + plt.xlabel("Number of clients") + plt.ylabel(f"Read latency ({metric}) [seconds]") + plt.title(f"Experiment 1: {metric} read latency vs clients") + plt.legend() + plt.grid(True) + plt.tight_layout() + + filename = os.path.join(PLOTS_DIR, f"exp1_latency_{metric}.pdf") + plt.savefig(filename, dpi=300) + + +def plot_exp1_throughput(exp1_results): + modes = ["pbft", "learner"] + num_clients = sorted(set(r["num_clients"] for r in exp1_results)) + + plt.figure() + for mode in modes: + ys = [] + for c in num_clients: + entry = next( + r for r in exp1_results + if r["mode"] == mode and r["num_clients"] == c + ) + ys.append(entry["throughput"]) + plt.plot(num_clients, ys, marker="o", label=mode) + + plt.xlabel("Number of clients") + plt.ylabel("Read throughput [ops/sec]") + plt.title("Experiment 1: read throughput vs clients") + plt.legend() + plt.grid(True) + plt.tight_layout() + + filename = os.path.join(PLOTS_DIR, "exp1_throughput.pdf") + plt.savefig(filename, dpi=300) + + +def plot_exp2_write_throughput(exp2_results): + """ + exp2_results: list of dicts from exp2_mixed_workload(), each dict: + "mode": "pbft" or "learner" + "read_ratio": float + "read_throughput": float + "write_throughput": float + "latency_reads": {...} + "latency_writes": {...} + """ + modes = ["pbft", "learner"] + read_ratios = sorted(set(r["read_ratio"] for r in exp2_results)) + + plt.figure() + for mode in modes: + ys = [] + for rr in read_ratios: + entry = next( + r for r in exp2_results + if r["mode"] == mode and r["read_ratio"] == rr + ) + ys.append(entry["write_throughput"]) + plt.plot(read_ratios, ys, marker="o", label=mode) + + plt.xlabel("Read ratio (fraction of operations that are reads)") + plt.ylabel("Write throughput [ops/sec]") + plt.title("Experiment 2: write throughput vs read ratio") + plt.legend() + plt.grid(True) + plt.tight_layout() + + filename = os.path.join(PLOTS_DIR, "exp2_write_throughput.pdf") + plt.savefig(filename, dpi=300) + + + +def plot_exp2_read_throughput(exp2_results): + modes = ["pbft", "learner"] + read_ratios = sorted(set(r["read_ratio"] for r in exp2_results)) + + plt.figure() + for mode in modes: + ys = [] + for rr in read_ratios: + entry = next( + r for r in exp2_results + if r["mode"] == mode and r["read_ratio"] == rr + ) + ys.append(entry["read_throughput"]) + plt.plot(read_ratios, ys, marker="o", label=mode) + + plt.xlabel("Read ratio (fraction of operations that are reads)") + plt.ylabel("Read throughput [ops/sec]") + plt.title("Experiment 2: read throughput vs read ratio") + plt.legend() + plt.grid(True) + plt.tight_layout() + + filename = os.path.join(PLOTS_DIR, "exp2_read_throughput.pdf") + plt.savefig(filename, dpi=300) + + +def plot_exp3_staleness(exp3_results): + """ + Visual summary of how learner behaves: + - fraction of fresh reads (equal to PBFT) + - fraction of stale reads (value != PBFT) + - fraction of fallbacks (learner couldn't serve) + """ + total = exp3_results["total"] + equal = exp3_results["equal"] + stale = exp3_results["stale"] + fallback = exp3_results["fallback"] + + labels = ["Fresh (equal)", "Stale", "Fallback"] + counts = [equal, stale, fallback] + fractions = [c / total if total > 0 else 0 for c in counts] + + plt.figure() + x = range(len(labels)) + plt.bar(x, fractions) + plt.xticks(x, labels, rotation=15) + plt.ylim(0, 1.0) + + plt.ylabel("Fraction of reads") + plt.title("Experiment 3: Learner read outcomes (fresh vs stale vs fallback)") + plt.grid(axis="y") + plt.tight_layout() + + filename = os.path.join(PLOTS_DIR, "exp3_staleness.pdf") + plt.savefig(filename, dpi=300) + + +def plot_exp3_latency(exp3_results): + """ + Compare learner vs PBFT latency from Experiment 3 + using median and p95 stats. + """ + stats_learner = exp3_results["latency_learner"] + stats_pbft = exp3_results["latency_pbft"] + + metrics = ["median", "p95"] + x = range(len(metrics)) + + pbft_vals = [stats_pbft[m] for m in metrics] + learner_vals = [stats_learner[m] for m in metrics] + + width = 0.35 + + plt.figure() + plt.bar([xi - width/2 for xi in x], pbft_vals, width, label="pbft") + plt.bar([xi + width/2 for xi in x], learner_vals, width, label="learner") + + plt.xticks(list(x), metrics) + plt.ylabel("Latency [seconds]") + plt.title("Experiment 3: Latency comparison (PBFT vs learner)") + plt.legend() + plt.grid(axis="y") + plt.tight_layout() + + filename = os.path.join(PLOTS_DIR, "exp3_latency.pdf") + plt.savefig(filename, dpi=300) + + +def plot_exp4_fault_summary(exp4_results): + """ + Visual summary of learner behavior under faulty replica digests. + Shows: + - mismatches between learner and PBFT + - fallbacks (learner refused to serve) + - impossible values (should be 0 in a correct system) + """ + mismatches = exp4_results["mismatches"] + fallbacks = exp4_results["fallbacks"] + impossible_count = len(exp4_results["impossible_values"]) + + labels = ["Mismatches", "Fallbacks", "Impossible values"] + values = [mismatches, fallbacks, impossible_count] + + plt.figure() + x = range(len(labels)) + plt.bar(x, values) + plt.xticks(x, labels, rotation=15) + + plt.ylabel("Count") + plt.title("Experiment 4: Learner behavior under faulty digest injection") + plt.grid(axis="y") + plt.tight_layout() + + filename = os.path.join(PLOTS_DIR, "exp4_fault_summary.pdf") + plt.savefig(filename, dpi=300) + + +PLOTS_DIR = "plots" + +def ensure_plots_dir(): + os.makedirs(PLOTS_DIR, exist_ok=True) + +if __name__ == "__main__": + ensure_plots_dir() + + set_value, get_value, get_value_readonly = build_kv_binding() + + # Run experiments + exp1_results = exp1_read_latency(set_value, get_value, get_value_readonly) + plot_exp1_latency(exp1_results, metric="median") + plot_exp1_latency(exp1_results, metric="p95") + plot_exp1_throughput(exp1_results) + + exp2_results = exp2_mixed_workload(set_value, get_value, get_value_readonly) + plot_exp2_write_throughput(exp2_results) + plot_exp2_read_throughput(exp2_results) + + exp3_results = exp3_staleness_and_fallback(set_value, get_value, get_value_readonly) + plot_exp3_staleness(exp3_results) + plot_exp3_latency(exp3_results) + + # Only run exp4 when there is a faulty-replica setup ready + # exp4_results = exp4_fault_injection_sanity(set_value, get_value, get_value_readonly) + # plot_exp4_fault_summary(exp4_results) + + + + + -for i in range(0,10): - print(set_value("INC_TEST", "0")) - time.sleep(0.2) -for i in range(0,10): - print(set_value("INC_TEST", str(i+1))) - time.sleep(0.2) - print(f"Key 'INC_TEST' value: {get_value_readonly('INC_TEST')}") - time.sleep(0.2) diff --git a/api/plots/exp1_latency_median.pdf b/api/plots/exp1_latency_median.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e7117ed32bc70a369f72dda680734cdec9a0a962 GIT binary patch literal 14579 zcmd^mc{o*H^mih14P`8aj)aVNzSqr=dB_l#gecQBgloDaP39r#X(J=dL4RqNO8`R=^=7iU*)e zRR}bMg1j8|Bb1dPr1?Q_3WU_Or`daWxj;x0dl!lygaIAMkg6(zLUn>WD*iG+$BRmX zu=EZ{%bgDUDULLVxOk}@NYgc=+0!TxZjoSOPoq(MsSqB%L?F#v?Va4HE)ZequCJG) z8HEPz0n2LX09Gi0Gzh8h30Tlrsx_BteaQ9~cF=zc0P_uV?@IyPEwYa!Q~bRAeI0>3 zF#lkEBZ`xoy@ppHSP=#OaRiitB80`^6$oH@q$UsyjP`>_i|naWsb1g;4R`%j8Sw3& z+B2k3U1+WldTB)+HxD2$gw*i>O3

dO1;Gw*6?n6njrZVD4?pX6hc|+QH`~Lzunc z!``zK3RE&PlQk3*qL3mdY!x7)XDnB(|8+)X^hokGn!H#%XMg2}506G#vV~I0GGo1n zpX&R|%s#xWb|xInc~0D-Ix=Ow^@Gr{=c{6{-)xD9(-!eQJ6M(FtEC7Q+YXa!JNFs+ zJ#E=LlzymK$vrM}FNY?kYWmKu!B3d&*of)R*Y*$&w^+Z^n!9Fd{RS#O@K~vIZ}j&1 z%UQSEo00@eUThJLG;Sob0q{+1>j3AogoKi?q+t^bXa#9?xQ3 z-e?`>A1gVrd#v@u#FL#h;n5P>U-~#p`SZg}4r)9#wuw$NcjB)O&ilgjK$LlXiNxH6 zfyN;X+Im#y#Q^4!V$SzS#f?3a%$y=&m>m5StnkQMW3g?kv!%=@wRGIY!>;Y{Yd?y# zOHdaQN#DY!;)~b99_MCuwth;rTWQdKTtvF0MYVC<(InC+tGpc<)gk5JX+uBW%!`&ki&ONj(`2j#Ed!x2W1cyj6GUsvrEsDF3G)vSF`E66>lbnE& z{Cl#MZr3@qIxB?}k$q>?M;=?{)l&F{!wt@!uRr@37s3%JaHKr^_Ko2po&#EUQ1?_~ zW78$Jtx-WrmOTCXv>OJ>iy~=*#Lw0I>^&-EQKntjgLQkb)ZuZ1Kop!B9mz}fI6~vNur;5jRZ2Z=k*r#oJvnSahsL5Ee zT-^zBlVZ*@6qAW~UPTA)8pxxg2)2xIPyBwLMYLSpQ}pW z@S0t8XS1AVHM^TlU3PZIRR~XhVH0gEu!pbbrk!1mX}fOrzGhyj_;YpzTR#_8I5|+( zM(p<{s6zCykzM&5^(-=4FuL7+xixFK$J^l$zZ9RAf;~N$!2u3U zP{jX40Q4vb6Zb&2|IipjLPa!)e*X~#jasX@Y9LOQ_EXZzD~A?TP182_-i_ztIa7L? zCUw#-3PJU;3f?COc2Yu<&1d|!AD zhy2K$eynIx-e@S~+E;d`XUom7Oz||SoX3|OK96ay8Qj!#YCR_drhWx2OSl1p#r!u{ zDoF;uEMlCRGr1bND~`QTKDnbcf9ywpD8l`Fw==Ts}pwE)@P&Z0~HscPlDF z^QoOu-rE`%HRGWT3N&f&g1&IS!zXJ`AaW|B8a%G+6`y=*DE0;=sqZna-$;1m>N}D< zaMApkq5JIz!zX+XR6lg@%ea3#>fyZNh&kz!jC(SHCtKTn<2@bGiK}e|mt32zWCn^B(E{}}} zdZBwrQqSA}613}V*P}kOpzme_TjswrPv6@u!a!j#s9Ls*Skzx^qe_Yal^G2brW&EJ zwr0Nf)ThF&RawuD^-D>-J}%rVsJ%a<%FxP8Gzpg)e(d#o@6wYWl|0&+-9up zeWl*HK6v${aX9J6vv?MR#4E01pN2}PT?%%$%(iJeI2rpEyT$G@o-yI?yjgpjbM0pj zZB1#v}Uvepr=m^)YU>AvUHoQO&1+k`oc z9LF5Kfl+$DtvB0yXC>99mQstg-!r5n)2y^t^FRf8`^hsGr#6tDDUk6riqq+!GdUbB zNzqq$tZ1^C{%W0R@5nDUcSP{noaf>D18MGu{rPs5l#ZIWidnZ_=G$=m zsz;#ulcp^^*STV)GW^6$=Pqu%v`xV-RbqHLcpxv``EU?fp9d9oxVXu+WhJ3Kwc2If zgB$}ai6DKWteUC=WjUEK?p4 z``^05R^UufCRb5r?RXr)9CZz6fg)l}^w6U#5h0d0ZBV7csOA>lIt-4e>F_#7NQ07W z361zx+Odw+v^2kXIJAR#g=mamIs=B_^Vc$l2&BJe2)BEYAp**DiJ?%m-#xjj9^C5; zMZzhp4(o+-pwgD$2O$zKIFp1(-krXcn99d1=8FWi888E%{+2QGSAGz0s!y#$aaMm7 z_*S!`>)kD>`n;$w3*F?0{HxU@Dk>!$9K}~6(X*uJf!P#%f#gva-`(Hjc1GylzdWSR zACmcEZ$N1C#s`$`+}KUGGA$kzK1k(k)F!AX^I&=;)^0;pv-A%Zq>=? z?B<<*EW`59tm66WP+reV^L}58IXu7i@fimkdZYi6FM2GaxP~$!kXmoas$zQ60DUCs zjmH_OGq<*$4)ajmAFY+S)nzE=-o35u+!Hq^U50mwx!Mk%_jJ6Vv!kK&E|-x}%IY~S zc9#m=JHa2v+HT+J{<_9k%$$dT!r;^OGKKw>IS~lEQ6?xHOR|m*E8?7*#c{E9dNu8h zT4I?TTUI-~47Orb+mNrwc11Eumbr!1E#d-ODfA{@KK_ghk6g3|>+TIJ&Nr?rVZhL@ zuLJ&xWPcI(@jFQ=Ad)p7j8H}GTHmjB_x$+^LeJc>s+4Twr1xS?oqQtMf!X5XSDhRB zt(_p}Av44JyI;!J@Lil465X5@%V8gx?A=v7LOqqRZXqDO&5~`3Fnio;@7kXGhF!Y_ zT@?CPIlAT6k(#J+wtS`W{+G|2S?Z$BaE8+=YSSP;^TO$-1k9^(#bl{du~T3A85UW@Umx31RsY2d7qU@Ks;19h?=_t{#_wHnIzLnL517eHp&*b^uNE`c_^) zjU06aKC#x4w25eV_%<>ae4zyv}HE8{81Q5plW6$hQMYN-#0!n z!{;W0Bv*@ApFHZ%n?AUHY&~zL%Q2znP78BvKccKR|EWPN13-ty68_tR86~Q7Gox2C zPaTXntp=$Dwpr|2)4#CKpeLY{L$oQ_!<02dA}TOyy+rQIgaq0#M; zGKUFaJG?$i*MG$AY{TFekCHFnRy^{+=`#?B5g7a%lQ1&XKh6xC;F!R?n%LgdTjw>i zHOCgZA96>KSlCNaQZuqFIo6=VH_55YCX4wIzi&L6-&<`Jl+x6@kTtg1%ygD~LBT=8 zHhN>*8J_nlXWV(MN^gH4$rWV#-C^4lQOSZm!{m5GA9M4)1 zYIllaZfYp3)8njr$W+D|TrQCt8toFC1$nNmhXl1svtxv`L~@yuo8cHM(J$DtnA3}Mr=NHFP(M8O&2aH2Fo(Q8W_Cde?J-8$>3B0t=5`3 z6)GP5ZB#jR2L*j|fxz|1i4c%^;d}TF@pHU#WBdnEN18siI#s+rK(bKUxQbtUYgW^) z@zRx-l+5a~-DCE@b)<||b}F&fZ+v7+Dsu8Ei1Ro1m+>}iOaE*-I2T}*c0`w(g6Jvs z6ZRN-=vlj7O6SDocib~E&`v2c-TFzW_dT0b{E_ravfd{>zH}I%xLLky@MVq0s=eh5 zdC+P$J<2;VG7>NrV&`>`N3VBnW3t$Kk%3tbVO8a|NK70|HH zAw|HCDRw~m!kD{)b+^#AXQ|kR(Q{uTAKkRetPV4rn%myhHvfZJRN8r-fhrlGBsAeK zaGQ3L5-&42pdB(kBR(t6tlH*`ktWP|IreyvYdS5Vuxh^0Cw-wYl?FyIf7DtRaMiE!iIt3?xV63yX@Vp44T<*oUC?MV;?Xh z^<7fgz6r7I3jpAab^TlrF++oDJaK$(A^xoKw zsU?s@31{xQd!KDXzLYZSVW1)guuJi8tRW7(-9W9@I&{vC<*4Kr z@OI<=v%6etC8(8BgDN&6?Ag*(Y^^5iDNeaaTc42W3>1fru1Uhli30nlciynXe?CY4 zG_@w})I{-c#VTL>eoXTrdrGFO z!SI=nX?rprQf95$hdn0wA;w@Ii=Dv!96j4L0Myy;X*-@+VImGnvDEaI)` z6N&sp+5Rl@Nn`n!>oZS}A3B_{LZ$v${6I%T^SU^l>+%;m_NObZLGhV&r2AY-4oTx< z)pXI_jpI1m_%KKVe?OjdCi`}+w;gkHS$N^u%j8>YC`{ukZTq&3;C6&2thoPtYux8~ z){Ecv5*g@?0T4suQUC2MRd$lNYf+s2-w{WlV1LVsXQ`XVlYSh#Jf0`Q!jl)+c&)&3 z6*oFWzd<-AvwYJxT)diGSdMs*Mm>YI~;0TjlJ&OJo?)MjWI7TQ5|2Mciv6%~qr#Y1q%E8j!c1?Nb@w_P7g1f}|Rh(I< zCaGS*u;e&RL%WGA#}^XjJ1?tl5x-HTPg+}KLS)(LYRHMrEi@QPe|?9dd?S!_`N^u2 zNsWV6o;WR`R^`Zo^AiHb^GLtc3+X}>?#5!XLF2}Up#oX!p7=g*tGw)chEKFYwl-g> zKHB$$klXC{sY0UdzJWm0(?7Ql#{e|_jd)l>ISBbE9EvSPCsHjVIYt9M39YDWc^S(7 z>eo|Hm@LupI3q5-$LtEPl5`rSxnwMvnW}d8ak(8STK5D~L`=>*Mk{UAGt_fT(W?**6VI_Mv(3sjmnkE^ zD0i#Bf4A5AX`@!Wcl@`Q*R1B>cD}Jmdv6q0l7gOJZ+=TDEKYxRul9g7dh&-Bs#NF; zyO*})s_>Z7qTV}gc9AVJh;GG(Ph5tf#J3CtWdLKa*uQx3BTqK)t%o@K{SdZ!*jctw z<=3>Tz}pKIDXH2z+;-FGmyt0$4qHJt3cBiU#J8wLyePTU649pD-?%NhIVrX>MLyz? zmfR@1PPs00>e2^M(YfN5rZTn%$le2MWWPKKr-+Tz3-#FxG}sG_T3+=RyLc$jtgyCd zOV3E`f@-~=Jd6Bco%Y~;qo+#}hUQWP-Da{pJFf&?HB`D-cvaYAr{7-5ebz2Fw~4l~ zHGR3#!$YoK$schO`^fLb)=mqZa=`$C-hLYouY~WGJK7_tHaf?}lTH_HR?aJIYLP?? zC+3R!$}ro%$+<%F-MGu={HYfFiP#%hedo7#^h&euAGB30`@w1EeEpzQn{WP;qSaHS zE(Y!Gq${UxXIQz?PVn6-3~zg2ywmv3ZtPnzk^p;=gZ1uvZv5E}?ESp^%uLO4f`op2 z{rK*K($ODGa(;*X7;FmznD;l~nr`~Rwfve_a4SS!F_Rh0%n0>h|0d2Q8gR3S0RWks zpHS1VY0L-nI>K*ZGWaBw&rv19JmhBR85IS0=92=O#Dpq7+$hRO(6&-F4z;*3fy4l%;tFZ;B#4qI@0;2o!_iXl0*&bFW zgB%#C5@|f!RsAeK_Rg_FCNw((X&B%dEbxo}`I`iuOrnCw`1U*E2oy~1pX!D5lE2;M zD(dBD_Mwr}l&sAInxi7juTRkC%G3LJiK7Q75q0cMmv|@1k$M+yS%^KI%Epx3z`L+C zlF!~BEgbb}3f*3N_B=v6`nvHA8IRswb>4F{!>uMUFg#bb zOTNK!6?x;J;^gs}GY93bu}0s{&Nz+`v4{~os{SEQH~z?%_2!mP$E2{ncU+7~a&SOM zpB^XakX(T2rfKX}G~amNM(qLh5Q&s5QhgO|`uS{oY2J^XIO5%HGOzboTRA>f-8h-{ zKo0Xd$a0$YsjDH312>ibZR{y0FOCwQ@__AfA1N|F+tecPhGr^PE4o4K6ZffFA@&{f z%KUc|CHO?r8lQ7>ojJBov>+~*@NDI){Oz6sY}~_TwR>-d2<%wro-)H!WH=u3dZFaQ z^*i0&n?xpqA28S)28astHx69hRNt3beRXx4>~y&E`R9(AvQeW8-K!t6v2w+#(X5B_ z;!Znf*n@tSXO z(gWl5FVI3RH@YcL)=t=#sAQ`*72DKF@U1g^YA^1>y*(SFQ0}KfgvN9%GwqY878|g< z3AhM7v51Q&3Lmkk^Tu7#i6{8-e?5y#q($zEzkr_bllR=$W~AueK4Dq>aWqZT)<;2V z9kN(>`ZIxfovB;#M>V~g@6TBiB-lA}sI}?ieRI3|6Wvrb!qegBx1vYB z#+8hX&3$9uXSXMsfesl!KQ!?#{IgM#KDQ?QSET#vE`(!i#+2_~ta6>Yds&msKjhme z#s8H4*p)%5MpqDZg1yk zri_FmU)0BtH=D@>r88@;GuTCNb1bbe(bmK70N#hltmyAvbzk0s3AXZh>ZB#4dKcHCzRAB*TF2RSINw-GJhm@r&AKF?E~ z;(Vn2fku^tYw)3BGq-_rK74f|_*HMD-{cnDT6^fx-9Di3gMm~GkQfI27p8zWC2@mO`oxUD54A0E$iSWXTU&qJZH?=&5WQy;S)L}A z{gIq{LF}cOYEZU9Zq-0{%YNKgM!CFSaAn0US*F!QZ#KvMEhpj?#>a9iEe1e{BQib@t;Jo@Qb8@iJ?CgV# z^&zZG7K*~%Xq{K-bQ$aJ`5#QDbv?zX#zHZ($FJB0~l)Pu=??9uc z4#H~yxL|DW2@>!$mQTGnUSl!oP9BZIpdoo2c&UOWl5ikx4gd|1c?U0HWKZ*Tqi5zR zpir>og8zS7wg@E5oC!!61POUd*?ihmkY5L<6aJgV_>+V{YPmT(Q$R8xoc*^4f`4ZB z^R{;ciHSfyZ#YZPgW?R54e3o^Hy2kL$OZ(L6kj(lCkO{#galE1y&$9)6$nK;09rIx zUkU|*2a-Cw1yCUP6`P}%hZhxse@N4Yh~U~7QUvReArg4T9DMB^-6=HsDkLyy>3VS~ z5_m6Yk3c$rDIla1gam6igFHiE#BQJhGkHKrPY6i`>jR!3G#dDKUmyYC0z&#j$N-=@ zAV(mCJP5KeK!PE1x0@5q6{v=uX!&cVUu*bfKk3T82B~4roFcwg|1~30%`fP3359@)~TDLI@QI40vnzgjpAtmH;n89 z=@LO`EDrOlI6uMa9*BTK{l6%P|J!-6zzlJ~P(czYq=;372v`Dw1Y!bF5d~pDXu^_E z^nwR`!uR3+SOR*n#uDNF|EGfZc^?bz1YnTaGD7X;NSo&Bz5PWeC&=;&rZ-Y}K20RvU!4paJngoyv0niC(3=nW}T^v{! z58MKl1dqc(cu*?>Wx+MQVBmfPx)R_TUZ1{cumS-OK(PpTO*|MwSIST3;F@kf@JcxN z0tKrGW*Zg_u0f&i3%uN7LEwRi^pUWB|7`#U7pK4giQzjiSAg@MX7F>ne%4^iz&o~I zX222Au-J6=VLSMZRahpvF#&^uNBuJk%oGZRpx;}XfCL=Q&jK65q7A@y@Oz=_8rTJF z6TcL?J-{~cv(Rm2$tGZX__e@%(XYWAuxlcrWfsflDVf;6*4R)N~(GkKd#lXb?fPo_}n92#l&?6J<&w&f1 z)0`ptPnTdSdei{hMJKz#5sBXN0Cv6D@&uXai!Ca6h8A01z-278Kp+BM0E|I^hyb>Q zKKkEChFx0l*MJ7cP{7yEptiKOF9cjFeI7pux3u7_?Jak zCrd2;ON8b4$+r>geZV!?0V5#dqHh1~O27BB?_y;B`85UdA8e)L;}eVqd}cXkE;yh*1vjsZEHiGseIQXf5i)J?cR>G*i!y;PrAkf3r@Ty zJWJg^f7MD%XS*4;VAJt4U2}O<}w-%jy%g~cmPF~(}-~R{JkFz z1qa7vG&F#&%V{_OAD7b<;X~r@WAP|BOMMv)3rE^zG#v2{UE%Qn5-smXf&=~UV+m-& zAM+7#f9xBIK={)(fEoP39}x}5;NRCEVhMlj2N4Hw>GFO=7`yyFRuTQDP8I*O7nCCI zPuUa||J+LwoB^?H4HE88J(J+$#qxec0AH8WNPp}Hc=o{=>YsizUwbzXiZA`+L^HP_ taMT8;U$U1M>;~!4RG;eX1<~Cg9a8ww?0so;w*$aFnuL&$(B7$o_#at-Vb=fv literal 0 HcmV?d00001 diff --git a/api/plots/exp1_latency_p95.pdf b/api/plots/exp1_latency_p95.pdf new file mode 100644 index 0000000000000000000000000000000000000000..5463a15bea4cd8f3cd30747ec80f65c71de22c22 GIT binary patch literal 16019 zcmd^mc{r6%_;l@8z0n=9%Z7nS1X0p1D6W=YB*CRMn->(m0q%!4Tp` zB@B%~A>3?^!Q|x;NRyN9Bm`2$nquwZ=72yNSUZqB5g0&Vgiut3k;rxsq3kaW)ZNGw z1eQvGG}~`;jATne$gDo9`A{^BDb^Gc0=HUWU`?TrJje(<^aw*5J6hX0ksT0(A8$R} zY>i12#33-OsybkWIqwOj!U`+npqKXNUEE`8O z3+4my560Ic**RJ7bMpZsqQF0nfRdI)U~zb90_Yy80wxApdm@Od>?xDUZr}+Gk^U+S z==D$O>5|9}6h{R5$B61qE?~Y0q`C_bf-1?@&5i`I?Md+@_aPqhj-loScb@0zqCMw(yvC|~IQNGwx9;f? z6voPbf9jPuJRw}|!FumgCs9h7_^5SkzHzQAY}Q5WUF}JOC&AVwv*u5ca>4`VjdhbR zpL*O}Hj#NZ$2k^irVteCzD4>>ebN5l7#X2GKHW{C*{wUZ$umxIY>!VMD;?f_8mOe> zkM~+f-+DVE*OH?|jv?ViW!D1!U4}o;X3FcGP1~5qWF03J!cr_xIBXuCaLaDV4Qmhh z(pWW}<7=h=#M$fP?Nc5-3KII?*Xf3Y^M34X5K7RdY!aVO8VF_OQ{p^rrNS1k;1_MX z4jxWu@_ma=F{ErkJ-+76FjDxXlP=CmcYt}z9`!bqXas#^@5WaqndhVKHb2p~ODa^q z>mRa@<@RPwa|>UjVKa9`cn@1ly?Zj|_O|#I3zxY%wUSe4w(Y`43&OTa1$@^`j2}lF zo<^6!`7W1bHEgM?J!J^JmcTZK;@4|nyrroXp-&w~lX zMWw^7{V3AdrTaxMIuqZGol( z4lmqNaHCjD?D109`xS}UR_E*ugMGr7$c@U`B}SW^rgRZ8W?wZOFC2*~gdaGGQ{9~y z?nU?<5_3mMez-yuz-mmk5KcauOz6b&QJ< zTw3Y5n*P`N8n?6a+N(3no|PRn2~e?ede3cUp#v{QSp-#Z*NL%doT%rz#2j{FV5?8* z&2(+|O_C8;cqAioss<6)vkcUk#!QqCvi54cO1tMeed%tKGw-IopNs$PsLGm-(#MWE zu6U@MTXM*I$>oi8d7{j&I{QPKdj*s~;_i}s&Ss`Q?PHfux_|OIuUk!roBc}u!HtR{ zC%rO{JnwYa?)H4RX8E}qLW;y0qIG(XKN}lzdxh~Eob8LBOHi!u*_#8OC*&vNGMnpE z+>M%pLl}zn48I=s&&(X(Vo9F1vU}KPrX0^AFSReuGy2sHFCQe`jvoG~Ea?~S=|&>y z_wU})95vN(eR<-2;lTU%7>5VXOQedHg1RIKfuzgblI8_Td%h3nHg9JpNrVsvrcT^c z*sbB=FEnuNJ-ztseFQvS&kwgw?{?aO&rK7fwJtZc*feceuDV{{YD8atQ;$tE z`E+n@YnY_BSg3$;mX`sZ!Wk;9b@bsDIE1WC6T-`c=86c@)HpI%(v>abJo{_d!27FTl* zb9sMv3jI9!zFhd}gz)9MtxJ0!Lt2k#cC)w^$xf>0iC7372$Q^Zhtc?9Y>ol{_X(!b=M!fu#&?S9SwA>3`euiMi$YjOW;-E$Fbqp~aM8>%1aZlSo-xxIYYQ<__cj1w)`5i-fLBs(H2 znpK+d$)NDSyTmiL5w|^in!PQ!EgIPJzI3^)x1MfrB4p%FvH3*6O=}~sMo1W_)owdJ zi*jhSGhTRgI<|O6VOi+)4;K{g-IK2)>LrGyGao$$6Ms{ZvzR~ewkn@l{ivpqbbK$9 zVQ4Q;Pss8e^~$Tr;I<&46XGAE>=tVh&etv5cQkoia4u8bfy6q-Aa0fYCk`;V+ek59j6L>E;xJV%OjU2o9QNJ0-8_iKnr^;0kLuVKs&n0uyFA0(zG$V`EaYkv(&J~ z%cKJvWj>KSJ0_MHFhg67AQ14wgZ+R5RCfotK(q{+_!A>g{UlU*2x0k89SvV8iw6Gj zKm4TL!z%V#1RG=bRWW(`mn({faXb3&N3e5dl%!EauUK7xk&m0P-In7oTzF!fbY@(T zE5^WcSk(+GG7y-3bg3OxpJdijULxmWU6v>#@U zEf2O#C%Z)D*UrL^^TQoJg|KAve=D%IH-J}P2vT`!C6_)~>!76na;r2&%sp!$(DU?_ z`pdACiVMvyWts(7-slRwL+#RX8P{qdbUJ#Bqz>hqJlA!;*Yx_b$BC*o=YfO=_b#-7 zzJ>< zbDw|v!QpZ7=DxPTexum8kBFa(5_*>0Z-)i>zS20gOVi!!2I4?YZ|8uq?|0*&olD;t z=IRa#&=42|yVhtCi~0*SD#YlJ8PGsrib2wwYM1)2e#+imne_bJps2{iMgD$XwPOjD zy5`1$QMlN^a}zV}C0AzUT)OP-Us_})hQ5g#>*leU&OF`Up>Am|+W0Qee$Sn8L*iy; zf?j^w4auBgX&;VfyRIJk!nwD)J6-SzUV4KJ7F_1vRV-@-93kLg;v7~MFv zUH*vbf$?!>jnEyoiJQkpls8OhKX9}Ni9eynpgcW$yY1A5>A~m6T5d7tC3VV)*%wbz zjHJn%F9->?%;wI#$?9|7*R@bAAu^}g7mW1~opx+``=sS{)bjHPMx9GH9fdxbaGZHYuXt{q&a<8p4ClM{9&9gZ*#WVass_D1%~@aW@`EFhSkInWAK{2Ab$m zdYHf2-NUF7epGu0R|5toqhd3W!ncnUZH5@}sIXc;R@>43s_msV!#crG-gp`eK_|2| z3=xQb%@FS3DnkU6;SYua(4KXY#V#D1bp--R>ut91r65W={F?klUa>{-5#1kqRA4F| zty{|DRinWSbUs|e%wO$;ctb661B$KcE6=yub-mNoqK)YnzN|bmYU5^65-G0`v9T4V zN1_*qAwvr>_^e%L9Xt+xliVMq@u2Xf7Po)mt0Ue4?c172dpWS%s}oH-vzubsTGR*% z@|>7HkxhG0RqMU_><2IiO?vXfSn{oMzi2yFw8%v&F3s_yh4#4%j zPrcK60}mNXD5xck@WeKntyeI-tAjoh_0A#nxB>e|hZ@3ZUa#jq@@Zg42aP4h0F>$+F-?CVB-Arnp-0)x)qYXtUJ z=0qSIL>ZuPjM3`q>tWZFOfL#;)~xMrQ58yL-N|C}#?O4c($-8_=9{}NNHBD)cM8g7 zExIETp6o9iKYl+k26(delR=8816u}_?q9!N zPUu@aR~eJ6A2lP?`WP;d?2{}kTx{PwXkmx2e`&1Sc>l}ojqv>Wmx4Ru!dR_?quqN8 zM#xtqH?MfdcbPGNAS_%oKeDOsfo|_XUI*zxCR?Y}24X8Y+!9_A;Z=Cu*h~YJ!4^m< zua84`nq<$lMq=KM%SMY{4g2tAP|}-tmSd8Y5Fx}fwjk~sKscW{b@BuKRuAPB%i#q< zwW@Iu#I_B*y@oc^sc!<0-t(pimhI;9+?S#(4HtSiA%1OBia|kXR=Sa)d!FL;@q)B- zZ;l?8q9HFDI0=LMi}j~qAQ!1B#5PF`LeA`*6?PX?zaeG+;l)jc6fpzcv*p;G-JJd! zZ0NY65s||5oBsF1_I)J3o;?2maYMxj(doM}gfexF5DniWkc!=FEn^#5tA<{mIzhRn zz`Tq2^#LrkqFiLXWeQ5{RlWnikG79I5!@Lu`=Dp)gfqYBWo^d<9&H~}&{>S?VlNb2i?;IVl z(~oba2zD%_-tIh4` z^YR6Y9e+v?O9RkhiTMBC!4!;i$c;)YgJax>(;v-T?mB67V7U6Lo3n~ho_KXAx(Rir zE$Z$Y9ZRm`L9KU=+&mhEi{z6bRbde#iseIz(?aEz6zLLK0gYK!%oTCf+=ux+ z84LRof|#T9XEInHoFOicApPo|6|+~oQotw|?|dt^uZe8Pm%^pYX+0LrlbdajX8z!X z;q>G@hUu7Fcn+&A|J;V#O#56l8|dFDty7(fGbxOUbl3^M&ok^`!*4a#bBpdKpNVS1 zsj23Wyw2S4YSjkC{d!Bg6Lh6ao5;p|N592|KRHcd63@fU#?5vgDD7r73q8s8h4@W# zVXW6hD0mOKcQ*9Rc5EYttJ6w{XqAyTqVaOj+>7(3z}<>YivRZbUU%KwlV@8#?_Z`2 zT0Hcs_F&0WdZVX2Q^j#py`-T5qj)7k%r{fAcxy_vI=0Sw6KC&yeG9|Z?;h2gaMzu8 z+&0^Bxy(j2;-ZVE!ac@b-4Bh#-g@{0mk(EZGj3tA;z9ufmBGg*mAp%9KDn2t?Zu8~ zy3)`o4Lpp-68>AKdY6*ByP1=ydAUZ2u-S^BTW)b7he0ocghS!KwPV?{=m!dD<)^)nzv}jC-244NE zWv*wab&&x$wlSV1C7~m+)z?*$RmN7HwQ&Ry88;Ng#3m$}v2H{MZkJS8h!*mc`M&LJ zW`C8rZ%k|dO48U4W5WfbTxpwqmLc1^GB{@xGMu^0OYVInN@gW{)-Z1ms$j%s(Al2R z!raY$8!rUcXY=Ccu*iN;;cQd?+M-83Vu<`z-JY~7ko}U~zM(CCA&5!(TU#XK2n0CB zyp!QQdn>zz)w8^9hG78`alIQTnWhc-9WP7Y_Ik^xNX@>?J@awN(KOL2s7L3sna|N6 z#iYDs@9CI*zOVG;c612BvN3W}5@n|v-jV&6_O=_%d~cm!5GdOC@Y~^38WN;|I*ES+ zIwR^hYY{ArD<;YtV6cJxyJ7(_h}eWVZ8gR*9mvF5eo}oqQ~a(YJZ8+=wnGM&lIcwPE9Q?q|lOSI4yfdKC-hgV+rF5X)? z9xT^Y#&?td!|M}Cz2417ZDM#l>B5G@a>tydEuQi1c^->x9=-N8xbv=6V%2%w4~u(y zyOx$21jX!^Xo!*q!b21O;>4g9CC9}8t_Du&X9zC{Gbnc1W5fvaZnk|cMzxR45a+Al z0Z#@3LdOFxl#2{AeJEsfu>CecAmh148NK{Wvqv1`jLzI`bc&~ZbR>96uol7v>^Xen zviQoIpYPiY6EBDQx3&qeJbY(7v`CQ4tgLKZ&Tbwmcy?A| z>VUPAxla3rJ6Edgl{O3+69;Z6?A;FA^M%G-X@Ey8<}a2H-muEG5yduzXWEQHoE3RE zg3yeg@-PU~&#id$o$2ThHdGcwM}9`l|i4ZycYaMv?^kg({7E6Haa6 zO$a@yEl`poIq%L&s9Bf7<;nVSo@GYD&#+cCo&0frW2HHl(pK(o+~X-X63lGq*~5Yg z`*~iQ4vU#p+*t75d6#*E2iknxm_zNN*qz$be3Jdq7O9gS=CBp=8Tzf(L9$b&VX7}r z)-vesT(FVl!>lk z*MF;H^!(u6=B{+*=L=YQhZB(o94Q7YbHntA3&XYQfr@2awbBgbT2&`(wHH(E#A+A` zurImB0&O!kQyr#meczhL+w2WpqT)aIOmMUfrNeCm-Rv?B-75bov$6Ed3f)n@Q+6~| zK?|3n|6&Cz7~G9uM6ZDme#@ym`LUNs|ed~%E!(ko8H-1zyP z(+Le6CxhRR?Fb^`YRK4Pk@;@e$1|aCx#x$UCX}k+5*>+WS+U~z+XNo#5K`R5;g#t_ z_1GnLR|XPAMJJ_`K*00?3xRiDzY@1^JM-}eq7>>92 z6|?WNp6o@&mqequjaPr5bR0~_0l}K}OI^mV|x@s+?{HpF8S>ASAxROoj zOK8LK6r;$D`_Aq;UC1}0#(jT^2uA~W%Kpv53GUI*Q^-N?)n0*I3GN}&%Q~7 zTp>EFa9ChNvKSd#ud@Csn`E%%asRml6zgFPgUBmWS=LW$-Z4gezGn34!^XI)Qw6Vs zC+X<-ZMCPt8V&q}!T!amLeHSp6=?5k6%3E69p|qSWDb+R_5GYm+lH7#N1fLhvvG$K z+DP-lM7P$;H!2tORDp&>UK zFb$1I{kPvz*iYoBN3jilhn+?Ed6|_zkKHjIwS2B{JY9g1Gu@}&myn*Zx$%4?j+O%BVf~i20G_1HPdr|9RTO$;zy-@C>NDjULp(0?IW2tu zkS$|*bjSzwl!p3fpf5DxFVv?NMdSj<21d_&0iv3R1Lh0b%j%X^@|b%yGAHP9{FS_V zZNA)&iW4R38I|46T24?XqF3}7fvxNwMGA&q8Rkj7L#hZ^Wjvvv3N99Ow3aeAA^T5kl=$)_kR&wH$Twik(`?N%YF6welz+;{IJ-V?XWvNJ ziejUu6r^=`kTqiIEvFBfBYo#vBVAK&yX)|Jc8F6MXH?|Ed`Q45E=djz|fTff}w z<20(G=MK7y?eu)L`?0C|ZC-DJ=CQ*rZjs;3w7Y}Iht)3%N2TTMkWbHU?bro-eJNGY zL!80-UCK?0$F>8F2k#{1G}2^_v_aj#7+tkc{b$PSRAZ#;!d{NFvxY(*w8q|mv8y&?DR*u zv&(dno~J!&sD%dX#eyH4{_|HaJh+NrLbD8ZiB%}&%PHW4pBE|F_x#fPln$fDdEQd)E~z4oF3vkrFh)bTf>|<>XUCJSx$!-@ z?%HYo2U+fLZ>v79weI|}1!lnzlhbz=@!t_AE#?`TOs|SJ?#KI47kvfh0<;o%9E9>?WvylpRSd1CxDu9tAJJIU6D)LuOz+j!&?oXAld zVtXY;lFim}*LB~>(Aw^+r8(A3&4L!6KW#LbIDLVJB4~l;zW^^-!fgYTUp=?VP24(x zw+u9X0L9UOsA%lpc>PN{9E?KXWRsd1S=+pQ>^Np3$aCer-xV_4Rw2m5|878rg0wTk z6`t)veB~eSJ+Gge;W))8PR_!_yZu6tN!b)-@pk+Gm(1u1Qc%N& z)*D>!je<3Et4)QTen`d?-N8FBwix9+7|k9%-WssCKIb}X^I~~P(VG^5OM+%R!WpWy zn8t?jtqGme#rB8JCOFkz@(E0p=#^?VV=~${Ec^cAe8x$sTkAvaB_~{j37CfRo>l&s zt`TwO%N7$eM9+JEE%)$HgJ?f*{{c-l;wedQ!|ij}-Dvpuz&5oZWq*;FBw}MFW$wj7 zcS-tkU%1TuJ>nCGEX-{mDQLfHC3)`qmqg$KaV;-6*fY)@70e1xB|N8po4MDOhneGb zY5kGA{yf^7onz+d@^r`jCsv9+meo9awq4-8U(=t;0l4FjyM4^xcxx#`Ee{4|mZ~m^ zxj_5tFKiPfE{v``V`*bv&mN{kv3Q*xo`#S-Sp6pYhGYE;EIQx?(}lAV&s@z$l}r#? zwg&hN?~lH7f_=x6Hgv@H7PHwGZ2T53HZjDp`iTB*`18aT;Vs>BM2VcUvrR`D8UCJ za?w+sQm#k4^kkj8r_2gwN84N9b;!uq@S?G?#cvEptqz6IkRc5qjF$NeN2wR3#i2s| zVg148+(2yYnEd_xO2-fP3ssoC{J)JN+~wuFkMESG*o}urN?hh2sVzm=zV4A-X{}If zKr0VtN|_~kH}`jNc|O3P>C4qy`{grhH#Q$;;j&BIUQ>(MVR>oM^pkn)T-P1B;Y*db z?T;#C)`m{8G#1kx^!r}f?631+@yO%RL~#*Y_=Q>jcRP#-av7V-Or|{YFP08pmT>2p z`_`i}{hp~zSpEf#Y16>5GJkO;jxEC(fuDs4>$tAjtHZACdP}{AQVfZ2Cpp@|?BBTa znWu=EnXTj7W&hSdC@Gj<+Sb@JolZ%u>6+En{Jj?>&P%Ybi;v>^K!YV3*i-f|P9%7} zyPSR~wn>E`joM3^)f<})1HT`d3dAMor*XD1JCD)@yox{yEo3730?Sx8ihtFl^r^Bt z@a%-w7fH_Kr}DUZiopBLb+by9+vf`3yPX%T+&dh}52L{#E%1o@3vjG(sSaR&7L^Z+ zuob23KCz6``>!9nFuhVFtW(A6OK%$<)ZM);Kczm{>hehABM}ql1@1;A5t)``D~_D( zM*8)S@J#Q--lb+$Z@Qb7 z=g^0X8OD{ycvi6AKhh?vZdncrqY0QcSQpBMD!W8g^L=JF$G&Rq&th6&6 z>9ap`_rvA;55s#*1?vt6-;NVXo;8Zi6?$W==$kB^S~>Kr;}~u%;kJ~gUqyK}vM{DL z)IQ{Vol{Z;U5Y%N%CnwaLzT^f-A{6icNY&aDqg%p-Z*Sq=AT$DG!}}m+07@Z{4h@q z&g1o!%}@JFaxJ!L4Mjbp64lWx6yClY~FzgZ>pM ztVTpr!<>=(tUXCTN_vKds-_aEq+`~mUdGmBPslxbcv4gxtv$dFjnuXNQO4l$FeKDQ zmE>vb;p9$n^FW|~L_8aN*-)tQ*3cLLF6djkf~e(vYr9@;zi%~CSqhE9pb=6yaNCEL z0e9VK3;-G+Tp1cd&zj=lL=9(_Mxmfh4*dU9vV~;f6$Qgd{sb#I|lOZe#=$WuS(T0sZJB zWWk6=2qIW*HXhct&Lj$TJR%nM^LcepBDlJ?1~i}x1kw)l3I+pV+Q6KhKn8kpK_Fcb zNHSmwFpEH=!A|c1m;kK7kX{I+H;^Hikq-iS5`?OO_-y1sCp(HG5E8UZehu}e>h0$b z>;KxtFPf#Q;eSyh(2eEI2Yf>BA$lwUy_#cXAo~APg8h7t1+-9t z`uLgfSTwK`NC2`pkU&R7S%i!Xcm^hq2ZPH%36DWwhqul50G!MM~i zILKqbvVam^hDgnc0J9K4JpqjY6I>k^2gb#N4FXGq+Tjp*kjnyLK{+*HAUXn72v81< zPgNQuK)?eKEDRbG5866zr`iuR5)OJmK_Y_KhNcGPAW`)K4Y!(LcrZn3OGv)| z765~*U0}eBp*Ij$fb*Yb@KaqsbD%PC_xMW>I2klFZ7Tba9sI^BG$*Pt0fT~C{nHD? z6bc2SzWdPu5o|a=6J!XhHUQbd?};jFU>A^0{F12l0NKRPM75P4HUZhguL{>4nN|&fh7N^B(5_64l?5du1kfV2&Kd!Jj-i%p5I~`;rC)Yz3N=^^tL_p6`>Z|z z4@y1q0S3Ils#>k}1`7r%!GP1k{}}Hi0{>$?ssqNa4*BmlTM*e#*QE{?%<91ZzK0tB ze6NiD6%YLL6&F%P84w|Dtr2L*iy)OT|0@~r+5eJZ|5q~TgDZ#-$`uuVaZM2SKe+jq zMM!dgR6_Ik$+sS~MS*7^EIlyAReAi|p8D=5?W+6!`NbOYA1J2o<^jOMPp~D6kOGbZ z`~!sb^W!co8u;)3h*)=xQsxLmu&K=}s3B*xbksiI1|9vzY9z5c=rR^(v)3g^`$PinVkMx3>^Q29;+)c&gB|L-LbqTGhU|v=-JcggT2u@F@qGhu_y0Cl0pIRH7=AYZ+wLxI6c;BO zgf~_ijgpp;L^x6??w)&)NY{VfNxOMCh{2Emb=!H_f^PwTZRT!gkFbG&E!pm87l4#H zfRh_p75Z>;x9T1Y3f%8tfU`wm@o20RN=6ig5*7Ppt@wD5>|ww+%D{k={qYAd3Lflt z_K1J#AnXeLA;>@K&^SB*S8M99vd|*=oenDt?O$u^h`_h4sRJGzY@ol>;ZV@t{aYP) zAgt{V%njP7f2YHspZe~pz$?zSjf}-UWdUD|Exm+ytal8 z4aB*&4zRqI4)6l-#P4lUAXa%TFDMX_4Giu#UQiernYFy2uxQBmey79YKrrvxIs(8p zYwKk2fAE4s5&n=f4ukrW4w%~7{&2F82mPHFJQm>IwRON;|EQDwQ@;TCt+hWC0S$T0 z-^U) literal 0 HcmV?d00001 diff --git a/api/plots/exp1_throughput.pdf b/api/plots/exp1_throughput.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4f9c3aaa09be67356a0dabac5a89439b2f3efe4e GIT binary patch literal 14798 zcmd^mc|25K_z@1Evpmmpo_n76Ip?{O##%b^XayWXvY;Qz zFGZjs6y#yMAEBZGA$5r#kRS&XRzz+IP0bssi?rBuO-8}n9GS%C|hh_)l zf%ymh8&d6^DHLl0ra)n$`~&(K?v zH<7-L%e#|Vms(YXb%+Luh-mwY=*~uu=WH!MZLQF7V~d=4BxNZkF}t@e<>u+?=QE=% zQ(qt1&&&=df1k}%&dYAN7B?<@FUjDl$LHC7v1!i^^|D92Oe+aGWZm*!z`0spC(HWC zo6_)!QM|il)ntx$?4g$H!4p!yFQP^V8c~`3MV}mK=k%N6i~B80lCK@> z!7KY;9@IKjrWw6*t@!tfy~4p9=EVNP?>BE9@yk%FjZe-yZ}9ZUs8R1gfP&h#<6Ij1 zdbx&gM@jqEJ$t|HKxlc;dPD}XC5^pHX~SeRhm=bKCGOyX!(HpYi>L69k8dbC^6C87 z`aNN+pTg40UOzeKb5Z)z-iok|gTrMRXY}p*!V{!22=6!Oc}l7{hZ5f?+)o~XY7b2` zD5)j5<~eV*Ezxw_R4aQWO+r3`O-y;UM7kzsyUUfknZ}*Y28YlemOa#Q;HQ*H%R*-} zJ{mQvggTh5RN11?onvbF6TQxUV;ro$=!IJr9AKlaceWdJ zBOX(2-nwJW_{$(ehh3eozrMh(exj)~sB^YIakw(-?A+(0uSsKU6d&#VTpAlcU$-b% z+QTkYg^;cA5*3C-WLrL)ZJ$^jcJfx%9)p)0t3y35JZ+sj`q}ennM&$jo2_!aXN7%d z@=5UfssE(Y{!sr&u+`#7q&8{(&Iph31GSMHpvF_7` z8#N6%QPCwar|!nrZ5mHoE_o+p*u$jDW>o%dld?`0vyl`ed3UP8tCh|-w#Hg# zgs|K`cQ#=Ex_14E>xN~XDKje_5=6U>+VzBY2ufj3 z{#dtmeDz7GAuNSgHDT4po1Yvexsy+vr4(Km+1vBIyJo35`|0()`Da6$3U6{xj}NuH zdGAC@J3Q0tt51s)4KjOR;US)Gbbq#Rj`(fR_+jekBI0l?h0Urt=sq zdsMj!k?}Cla9JnPuG~hl(#w7+LM(TgugcNqRT2pmS6KbmV$H)kM!V`*#3^6eDPjDs z!W)D`k_CzesKp%HdEYJ9X}i=f$EqT~nd=IRTlKohBCf&Wkmj_mJ(oX-u2OfdIqp>? z7BiOmspoqPe|$_o!TqA!8#8A;P9JX@Nd=^S9rAFc-w{O>v2x*BkF!rVB#9M&eO2b1 zGIHjeAo`9-`N4>p)wvnGXAl?rO7qT?u3Po`@iC8!;rXPfT!YzKTTAUy+~Gdjbza{6U+Gtk!SQk36(Z`{MC^cBX}PKKf}#ophdUk2q28r`Qm; zJ6*F9u`w0WC<*TAd~VDWTC_c5T@}BpoVTD)-J$S->{Y>g*&j)-yIi^70aYW1y8SS? zG`>zcpROq)%|D0Ry*!|$muQz#$5Z!3c+D-hl`9BhvBx5N#`wM3&b5BX*f)kd=AnLB zOI*`W0D4rAtb^U*de@+_BzS0mIH=P4W!R*lgoR(DQH9e(GWEn0UazApEgu@l{Lbj~ zf_UK=GZ&M&UEGgu1m5|6*ejs1iTbK{;<2cF*r67|#uAUrvNMq{h#hrSiknNVPUY?H zoGTIF>NMd~Gts^w_^x6&(mbzYZHcRV$MNKtWD2L#$qw?ge~sWI&No+m0Do4!gH0f` zKJ%c5vC5jIE*-eeOk*L}ybF)62d<6h^?LJgR5R>e8ybfdJa6X0W42tj)J5v@OhL!T zbytLjz2fXj_yyO+SDB?<$-I7IXKrCfXSPUg>#gHYo-17Ys?*YTVI*8je%%XIbCLb9U<&8D=9?-JDO7;ck z?44;sRVQ22mlP`QxdFB}Ked0MBIGZ~4fM|TZ-ayEyulQci#@ok}B;uvcJjl2GIHJux zAl7hhEe8{(79yb^U<8B3{GBV+Xnoo;aSqMtRE=#*jy+dN+uo8jGW)oGBE>a2ui`zQ zmk6KZhm$LFM7|YJ9E|zyhK6W9+NYfHy24S-=-E02y0m9@U$FP#wCb~nGq*zPTyN+W zq`fc@e}&ql=Q^raPiS?b4W;(ynLROZDXSekOFM9<(WNi3sw}i|hB#zK%9nA8A@HPV zy9ih6h>l-v%g%RdvXtppj4-s}OQ=3uGhsQEsd{=9qssDG={*5!=S*I#@t!JVz7^y89 zMJ(zchEX+E-<=f=6t*QqVRgk!@3{{-n@f|Q9D6J!Id)2iW(C18FFjxG{D6WV zSNEoKyFc-4z1yB4`VgCSLt-j>w04e;uGki~ z!7gdd$dEew*!C(XtCI-_v{}_B-d}G##6I!($^QClD{_-tm8Bi7zNV8E+}DJPi`Kut z^!7z|kBdgz=c}@kQ@TATur#R&r`nef>j$G}pF}RxKYhhX{KKE{U#!e%oi3i)IgO(&sYoeSFDi^0G>A z`?Q!2q1gw|T?6RNHdMmIf2$oGo`Z%36iI zWbI_(+xC`Mb3818PYiPyIf^-a6QlfAsW-)Q$1Qj3YHBr>viq5`Oq22+%>yOmt!bHg zlj}%N6v%iw)jr)n^USK|=#y7?Ea|dIK5Cuu6XfUXTz3cYf81Umm6m}UiJmq(P0bfQD$d_OXprG&enOe72o+iD;N$0uzSdMcpEX2&8{z2)ApV zAp*+e2SY(~Y6B&ob3_Y~JUeN(Ft;8onaLH@X2J}- zq+G6*AJRg)3=X|9{x$iGrevgDSet(^o568)KUvj1}| zK6}$qN7}A$ayvq{Rb76j#~+yVe2;HXlW;9{D;HMmZjyOxPHh}Vy*5Erg$L6kxmpQz zhs~$Qp)c&7`14{3a*Ix4XE*QE1DR!wrX^3uf_UBXXS}}_taAI>$7kew=#}0JzLO(~ z1r^jGfw&qAHdT|``sgFkuUs>wGVg9qKkm9^|4FT+&5qB)Dl0d)bB*7A<2ble+{tF( zqMO|%o$Ym<<(!6wu`9o5u{)OFCIn}XwUyoL{>p76ZpOnzVepcDk;4AToCt(nC}R|E zS&WVj8{&eR`6=-=x)ts9TH;BoHmxB_S=MGY=a5S)ilA4K@{yS` zJaQ*p*>lJNPI<(}0&RR&$V1RWI~FSm0}y-#XzkFen@iuAdB(bQraDw89aUQ!(od7I@- zHAG=vjuK;}&V^5YdMxKlI?DBW6(LewVC1ume-Pn#*7(6mmUT4sIh%pcqS|*xB_UyU z!7dZqiPRUtd&_+3qBl14dTX3fSKt$G8I!rN`iyZwadrmT#4~rx#nFQFV=wkv%QMjz z6P$#>{lojGYOEZkCC>4h6oP!a@x6qns7}7T!{pN|tY@T+4UU#zH@5QxZsS157Y<2Y z&bSg-9weXT+S zdka&?&2^>1@-{@h!bVA-lR1>`Cn)!(p0HagR&^M9P9}#noKKH$-0IdCk5F>xSSH7N z`9yvlN9`j1{g71bS1uFH~N~89MtpyS(y`4KAr+gOiyt%i}-YB7tF53J# z^?K`Zl0fP7P#9r_VPxOv_%xrh43d0jy;a&#AKrw4wIgeJlN^r;J++_vvSK#WYQvuz z#4-VNXe{CHGnnCNbuL!)O4iAPA?a$6nqQmwPVUEZd-Z#KJ6DM|1h|^81xkkcg$h6K zc+CA&n;_sgl)9HJZ7ZI1_#aF`U|M@~bquU1Pss0#_Vfo4n0u$oYKTz$e$jPv4fby5jVh2*eBw{)2z>J#sp18XgHB&T4=DHiXLP*Ql3uEIFlPy+pql3ldR6oavdn6XYwTlTMe6&!fly6Z&fp0crA;{-jU?8Q@ro35DU4r44b*s?uZ`d z_NA8z;(SINJ|bLJIaRk@Y^w*YI#eS2-M{KMP&Wm0p0?NMUmI`|dd+fetz0~Tz=yGH zWqrfhz-hIwJ6G5wJV-Xai=Ccjem}4IS@Fv*Uqwy%_s=dJc{k%^p0qEdL;s_N-`al~Fv{|>HxAu@g%aXpk8?#Qno0 zq)pa$*Fvv+TyE+asQWxA`or}+7#!|CYw)P$8T&q0zb4ZQ_KjC>OG<3)j^A-)gJ!{n zZ#de<4HrJ^u`l;cI9=zWUl2JSbBRB_@7++*l+J}-0jrM_9p_uJOj;W2N%gBU&k2-X zJM@sN_pQIRZs}xvi?~SBoN=HWb%2JDRNvixV4UYE_xdYAvO$!qE(*urP%#hZWsNK3F>C|boy z$=&Pkbv#*;c*JH@JMz?bSv3<&wQu#0d@G-z8+1JdU$RA&d8*W;cbtynbZCQ)>I*V75H=c!2RPp{YLj|bmpGCEA~Bbx9p_%XW9YdOH~ zlX)zXr~t&rg1QgfuV)KSOm(XJuhrnN-BESbdY2c4UVZweJeKp=Hlc~y8I0ZCM;4IO zC@V)ySbj&&jBkxl-qi3T+bh{w)<~rdyv?yg`Dzk}m9Evrm6Tw=)Axvti%;^+$;gbj zQUmj0zG~m(R#9K2 z_e_LkhWGwqVTZ-sgz+A3DenovO^BBcYzXNS)@t{1!)|LRtkLDT-?+4xBjCDZYS2l? zfMm#Rbqyq_Rg@AYq_sYEX-pFg_aq++HZb9b3Eac|!@7wnKZ&c&71kKYt!QTHNc2Q( zI9Hj#w&b=WrxcTJnE^XI@$Zwgb_`hHLMyeV&jm>Yd>dAY+fGH_o+EI!+7o<}E`1N) zE^&cZZiN3J>PW-;7WOx7ECX zdf%>)iae5#FYB4+`l&-7#kK6a24C`UxY}!uz}gnmsbSvnp&{Rqzx6RK4WTO{lYuQh5wH+ik%R-s99S%TQt|_yVM&O;Zp#c|a41t00l%~U1xAs;QcSCmlkk>+R&5O5 zr*4l0cwXQG;xH3lnV>;5VB@d-X-6ybvVwh$Lq?erpCwqgv^ij;3DX{SJ+9=6P7CPx z9loH4eL-QPL7^p*1Is5bFLSi}Hb!vA^A0cb2{6wYa*8J(xn1L&K!4{%@RnV5nQy%V zmw$Y=P{wK{fhTXWhG`cjTEA#|e%$-9+;)I;HY~8AasA4cSEl`62+CQdr46$=b^Qh1 zM`g!%Qk*UIo7iur-EmN3?>8m&<*RNLLnwV>A}cc_hyDi>S2eyJxeU$GWD~N^qQd`8 zuA^YM{)M?I-7lXubO>sbW$-Dn(nz0pNlViwN{18ga~(YK!rh)ADWi>yyDB-|j(B$@ z>?Qy7z@x-sOF6q~OrG%iPJO}|PZ_?_jQ(ou45u5N)jU+RXgAcJv#+}ZqSbM) z<)}>I3*W%8-H>hW3hi|Io^atU)$cUBzxU=WG(9a9Ufj;+>YJI5h+!p19b zDiO8HdYf@n+ITkQ(Y;s8B0pXrf0*QsKQ~@5c;fX^7L9cdOju(Ey0HI%!wroa+`#C4 zy@SA`Dn>=_h^`1%x%T~-W+QuSl9T>m=KJ{FiH+2036e)c=?l$JqXg&o?(HpwMYGu) zEWUHbW~(nt7Hw~~e&Ak&(?j-F6C;ycZbXgb9tyV1_Io?j(N^8 zqK$6V7QX6KPT0sARvP_5<2>TE=tIe@)3T3~$!SLNFV-fdj~+T4wM4b%No0RVUDKKf zog4C(I`$_Jxlw$k9SL6fF@f=XY?_YScHvf?uW$6%z*j|*GE>T`J@>IT6$j^>zf8W% zO3 zM&-<0?uxE$Sz|0Xky1g!Mt^>%En`JGnWgP6qjKA0wJHT;*Bf87JE^=`*Fe`UR<}g- zxWpNYVw>a?Gnr!Y^XtwvRpq@_kLtA|JtMz`jj@@1+wsae{;lEh!dUdoTC=;#$0PJU z@6qnJLcf{SLKO*pV)xMAv^+ShD7W`s+rAUc(}-?j-3LyCAjQ{A1Z9F+u-JbQ^>b5*w$j!?kY;{o+K}_QRIY&FrrGHzS+XLY^1qH;1(8 zKCV|f*%TdqD^@<_ke1vq`o7BjpvnAqqM~04nj4B&)FOKiaLayr7)%u(suAj=2-Hyo zhApo8i02*hGtH^a-Pkh}KDVXDTYj1RVV(AXy~F8+QO~}_3OY|GyLDdiziObImvdFb zb%*z!O?#~zZ!3wmt!VgkrH6-nhlM}nHn!FK`Q}b@o$G?W1l|4Ct{zd}Ew;CZxLfO- zl88>v-Jp_@)6l#LF?c#vlqSPUd3ELrohH1~>*Bd){MqoESUrc=_jHR=st(!^i)T4Z z9c~zB{PwW31C74xBYj|d0L-qnK%GUsaa7Kbz&o4n6VJ=xiwIF>`bIth*^HZ z4&V*`Dh-}YatE{P>+gsoP=Nd6$zDh|=36;uZZAKp7o8lhY-Q%#6dGc7W1RlwdO{zs z;_v}#$bI&PeBL+Y6S|k~nu|Z0Ou-c1#5*plC!enx&KdS<2-;eG{^F{-FC|5VFY4Ey z7PSzN$keL9)Z8Ckm)JUS)nWJ1MCXdre!;1-UGjAn%gMq6#5bp=GY`sNV>?-vl6VTS z-aJh3sQSB%ZIMSltu?cNI^Kxrc}9d8#{~EW_UUqv4$1kNh)rQPqxnYrgthzC10`dV zNj0VPsi&XYi!x?=A{5J&WX5(|S=v3=BK#)4Rt_`fZ!tyx&{cPQ6>c)?+sGphUK}+t zwsytkKGKP-6ccm5o0@T)E$BM&4_xP}h1j>xsPLDcmgHL>U;mVgGxOM9(d>v+!V{L4 zSzFx%R)FK;>OHpu1-7qoiJe}WYcLu(HdpxW#=Y)tvGs2PYMBfU6Oe|%|HEoo-b9ba zs=o40o9tAu!^Nj|NwT5CbKNT&SFmx0tI@3nGa}LDd|MGkxFrPl@(;C9^ljbzS9fDB%7UA>BV(^BgVYc(a@OaP_!Np=yeHLxJ^u zNxn4(k0=t3Tw7Bx3fH|=6`>IwizG_4yZJgSZxk*>S3D%|Y)4_8I*|k#|JU=# z)ASQNBQK%Hz2)8Zwiyy#+Q%&l-Veu%+IT5Qtw9!uOnoG;^lC za}Bq)+()YqWXW43`PTI|^Lkfd&@3f74ZSK^72I{!D|zjc#O_r<8*EO0G5=s0H`R7i zdEj*Eb%(vGSruX9D{HPU-4*b?v@TG;>dT(a;UpPJJHF8Ofv+}@3CfwPZ|*t%smpIXj*FIV9f%S^Fkz4hV#1*RL1}ms z5*Jt#jZX{As%?xw`t8Wt+%j9<647BUT4{aadc1hbdve?*@fW6B{8JQCOZ&T<_v1zq zugiM}+$yE^5b%eI`l>CQc1cOx*V zM0_L+vfV5sr{0pQ%_r#ACY6IP2djf8NpA+~P0!aRt_@^cYEBgC9_@acpt@sI2VDU3 zFdyHvmYq*~gUXkv3y=6-Tj3sHcpujC6zpfctNa~hv}6gNR}+hU`u6zZ;rkv%Q^jn% zW@eYBZ!=5$Q$Og1zG8lT%ZOD(YEZnXKXQg9CR*mQTGag%b01TRyEklcG;g}56NLuO zLy!iPA9)NO57xFIAW)0yZAWwVq{jUXaui42jz^lnR|*tt;Yu2cs)L;{t5+|!&Lod5xJLjg3<1cbDQ zkf09-2nixfLBGx*m=d(*3L(KBgFD~=LV7}IG??vZfH%Mgg!F-szF?mLjD;VF2?4Q_ z$X(9%bSEG?BZhQg5a>dn^J};l3>aeo|CiAL1Mf<4^alF*r*TsEwu1vpNf@9IxCi(~ z9s?g1Xi_|NsSL?c2&Bc&EC^c#0kzI{>h6xNRM@gi=~Op!IAcihV@L#{u{g{^aegAg z-4FqV`hSrS|F`X6ft}%iC4%@^h=?UZ1S|nT0yDNE5d~quER7|h7zq#fgrCFZu>|z| z6{`rB|34+f&*xZB7EW**KNB8{21WuafQSPLG(;joAY=$Mh=K5+b457eF%X7?M}Pzo z@B|2t2aSN;Aw?W0O9Tv{Kpwur&#@qZfMG>YA8wNb=5zvao(5kT!eA8<@D*kP779)T zG?q~p4+NiY1C#~*GV);ghEXIk5?+zSxRQX=CIC7CjR6AA_lpDl;=v9YmIT+sL3nT_ z0%gHhM#8}52n;2_SGa%1puq|RJOITa;6Cx74nrwFnS-wk`+-Sd%FgV`?21pD)fw=;l|1^W2!}aqDh74?3ENB6zh=#>xun*h8 zZ>+*HF^mZq6kO}SR$!)3CG1l|X;WGav*e>8vg|EMY;ua!pZ3qp6Ml$d`5F;oJhMFx!mVPKp9Xtod- z(D~eg2^#~idqFJxZ<96{aYjKq2=l|q&CmZB*rkG|>>&)p^}y3Pm;xCz2Z(Wi3Yua# z4KQ2`vLozr7&#|k*Yi1N5M)1}a|Q3ve9jF_8S^=J2n+?NfdKvhY@9Lb!oG>I=?6OZ zgun?LgGz(Il*!0>L%1Jp&$~1bMZ{>~hpYS12OUgd^X2C~9B_e*+OX3DX2Hn&K;VRw zk@E%i2~b2v%3#)p^L?5)cYy_yl4K~K?4zz2R;D&f1NI%v1s6${}T1- z7Ol<|3~^{rt7;?P>gj7!Uq5iUm0~dX{U)oIn!y0B{hrEEliAp}M&~O&Z)DduDjxRl zjDApR5V7{l^UtLJ#m?Ri)smKAx_&Pww;6BxR1^#8w;9)4ARANu((ijexw_Anj= zvn%{AF+R8P0U(ND{Xb%a0Ra6~SrdFR1EcQE03Lg~deB{+Z6RN*0ve^DC7L$7 zNTl0;&lEgpj?xGuz{&PLcHk47h1ERm9Uxm6vbx*C0Y44T z*T2&+1b7}^OaoA6F-;M;xy3XhllF-4Z24Pz;D!KR{Y5l*=Xntgg8|$wqM8 zZE-m~>MttT)@9yau88})+OTpl!J%|$I^@I!f&sA{q$`mI;e# zgg(=j-l2o|A4%Ke#sB~S literal 0 HcmV?d00001 diff --git a/api/plots/exp2_read_throughput.pdf b/api/plots/exp2_read_throughput.pdf new file mode 100644 index 0000000000000000000000000000000000000000..86d06968467aeb33164d0349bfefb789c7fa7d7c GIT binary patch literal 15344 zcmd^mcT^Nh&@V_3mmon<6mb!W0-G0>BsoY9N)jb4L88loh>55mL2?q1s6+ulR76FB zYA!w_tXd(s%c1~q_8ldk`Z`u z4GaZG!aePd!sO-Q2-6c@L^wj#mTc?h=>$g@+By-v;b_pn7_O)YBYHSM4P}1opyBC3 zhGQrV5atGUM~U`ixb(`Ux*u87gltPD!m%qAhPGrfk>ml#L6UR#!>z!yY8rqPq8}NK&~^tTsI2CytGPDZ`Zsb=e=`8l4N*@b0_s-CM;H^mJ$*>_ zfFFo{Fuxwr!Npd^(+|vu1b-|ZDJ28PU~y7-Fg!vPFa~;i!wDnaHiG+EMnJ@ ziHPG@MdrH~?C|$G&4R|IPf6t`J%9gIOgrcL+Iw|+3jao%6&m9$K=)@f`1{rk~16SbhLfbJV)w@;G6SCRqDn7`m*t5|@W#^^TJ~X(i z$l!u-&a}P3y58GmrGEhOl@P!ZmJDj2s(nNK93;yk^ub7GLD z#oIvMO($th=BEM*W%OvrFwA7u@$@&FY?f#PQL=^Xxtc@nfnrSGX-q!21cEVW?5j+~ zjBL7=x#}8KH`ogvtzpabx6!CST$gZ4)<#_al0w3Pz*vW<8{$W!d-F%gdRtUaYaOyP zOzL4FU2NBS%A6c$a9*d$MNVoGrQlE2uy_=&S8w_DG<#}iL2-@Q?%WSJ66A~8zNK)_ z^NhA3t){-=t$x7|xen9oa~kx@E$*UulR)DiIfNZ&ar~lr>0@n^ye=zmSiCE1v4D+O z(uh<)dib=HP*QS6YUK&Wub&GVmNdG4_Rl{VEx(Y{CAjSy!7oA%YFCzc010@ zBzexd+m*DZ>Cv~gAH9Vt-#FBneoJOT)kIjb5z@?^lx!AKU>6KYh3ulQ^ z#Z#yEf3G-t?oy{-C)PBYP??<0u~SMs@aRcBv0C?d1v~6G?<U4R^x&F9A0j7JG6U`op?MX zSYmWdeI#q?&CI?QoBf4x$8R6!D-@u4w#`j-fB(2^0{a1vZ7j}T-?R#kGt%0TBl~EJ z4h&7~VK+WW?=QYbMUK;7vZ6HCkI0%m9VfJ_(Y>lrmi^V!fU2;JT!(FEn1mRYm|5-0 zl&?DRb*srX8;<5p*e0r~pb};P_OgT`WVdj=g`VtD4r|b>Qbk{{k68 z_qC4;C3Ls9)gqWiX;0ISUwTG!$hb?v~EP9Fwl}*=Q`Zg~sEO>Rp?Uy~Uapl^2zFhv$(f)UL zbgGB3L!uX8QC&z^(WE*1Qa)pUXU@dW;g;D{x0s^3d9Gu8TuvXunDY3RN^Bhs zxoXdcsy?xi%bKopQqq61U5YH`l{*yTeJZ2j5-jc3`DV8pS|u5;bp_rcg|*!#wOjCA z&ZM#Qks{M)x~_L0zPdy@ey7cKD5?JL`L=H|W2S^+ao1QpN2?Lde&Tt4VRi`Ek;{kU6?C7H1wG3c|`bJmel*r zt12SY|E1H^;&kNVN18eIz${upe0$l0m)UKJVha&Fy@jxQ{8 zlEYsoO!RQu&E}jM?9i~b5NUcF;wXD_(ulBiBVMoQaOrbl^*g|*qK5J*~p1qc#Z|ELx;L?20sX7y$$h8Z7 z3NeX3brUW3UUo3m%ixyBp$1|D#@6bEoOrw3Vb$Z+#``j|i{`cyo=F+w$V7+B{@H2F z9Wi0oI4sC}lYNvPC(IhZ-0o%-wB_^u5|NB7>?B9q`{N0&r+l~!$|}Z9I|VE|OS!gR zD0lNy?rYt{af3BNB*|OA=xfoAVp%Dhc%fIH0!FeD9Z&fiYjYrjPnEPfcWl7-#NTn+ z`Y=t0Up?9kn^G>Pje8%k@T&UW0{+;=k6RZ*k3V1D%XWHTgW_2louAf>RD=iZJk|(L z8uQ<>!&GApk%r|+lO7xvZ;HH*J&2UX7;2%$H^2hTZy!Qd@FCkfI2+MeX;r(aG+q^A ztT}v)bjyZuqOPO;W!sDWbTs_oJc(2Yg0?zq2*MNAO%T>>g&;iAXqBKKl=lOPayPcE zx;sONjCR|2)8G{y0S^O(Ub4jS61*OhZlP~IqWMgJ-os?~I>3TaX?u>$#%sdo_tJGRx`Y;Ntbo6bl8nk0=a2|H`NbG2n_85M${ zt@Ij#(3o}jSeyYN4tTQm6QPR81KWm`?q9uHjeq*}Y)xFMe$0D;*2i2sQ~go}1Fo| zLz6QLTrT1W<2ySoGtT&MCXQ~K*v6Ucbe8wI!}8aSKhIn4rXoQqpbkU8{kH{EFxK&C zQeqmO*m5ZA(fdo?CyWn_)_(VNRW&XYuMI~%M4oPox&2zln)6s_>&?U0tmLnplCrW) zJrQ;%B1%ZHdL(&PpxT-&RUxxebDkNUA+A=?S=5`osFD=AF-HG=Hgo-H!p|{8z=MHu z)>|(X(8}d|-iWC@^f2O0<5cFborvWw$TPfbQUBa%c6tHLaMUw0pV^-8)21p06?d)1 z4R4ib)MgS)OJky)_HflV=6eoQ_Ar}=pWysLSkhXY=ywx1C+pEaAO3n5rU}j2Wurr| z$xa^Ad@*d{!*Olr?OR>sz+H*`UbnXp#!*)%hqOlRJ zm=PuBpCeJeJ*`#)^T3v!qko~Hg>L%~QY}07s_X75^WB$j*r`QbaPwBUOW&_M*F@-V z;Hr0<%jnO(j=_iv1P#@kJ36i8TY2xJSM}w6n8_UXztvZWgY?VCDC7EBE9M9`{Yc_d7GtoVkRlQr1S!KX2 z?khTQanm%JL`OTwW+{(Tz(R8_CwW%;f1+Nm;1DT>8aMZz>=$B2EH^E>-2Ey^c ztwph;PuWbc&4w^ASD(<>#gHg`nA?2ge)|!_i%NH2k|bM47&d!}oE_>hO;37!+$C1~ zfY?YL=ZR%DC*hca6TS8s3(WT41?8zwNd?(qufljQLop5CH}MM8dYK&av8o6H z)mE2upLD+1WaH-7ZgRz;t^Br-;GTg5gVVcJORg+oNqcr*S=8Rd;G1}{*;S_`Y9_W| z%jKaDV-=q?t_*TpexB{k@60jkY-=I3uxDp-*IYl@$2R!h|BzPATtcS+U;DCQpagN0 z1QSxW>N!5c@qF{nYe9R1Y|CAxg5MF*eaqj@^BkHxy~l%;$Ia6}%g%jztn9x{ zg~iUGzWk8cFJUEjQ;7 zLUuQ2N8DJklHe)X>&@}i)tDdT!@Fh#=D3!{#V0&$X{Cy0mA)G9`p|cA(CBSi_C@i% zuKY)b%tEX=_75vcT}A7U-lemAc{j0GVQ6k2zpv~(6=tcxQt5SUh%xA!5KhL@a-QQ6 zhP`oQgo&Lyq;#(^{I-hJRxOssHrh&-fGVN%pfIO^6u3Kk6P!n_A~l>>ZD%@dY&!&# zg@$=rsnA0OnPS(m-=s=U68k}^t-2B$J2vztdBJvPK1gJwx$VR%LS$^tCbpbFo^GA*bf}&>PBJ9?LN{?dY;56gnKsjr1|~5%<#VPW)^%`qkGW z;j|_j5%#pio6qeUkM*qHA`ppEAV4y{4Gy3uVi(8lO&c*HRxsfzWjVkr`z#*QJbvZ-xvtwb$#;Tv=f3Xi@Ba3aj$h32 z8x>JfL8d7DI<{WwF>;)A;Ed&@ezxGEAe~~jBU%i<;A#KV&A9HdIXw6dS5V(jQ21of z`D&q2hPhIDC;O!-ya$eRoZctkVBVN>g7N9wO)iP#56*b+z09RtJ002l6LNX8*k!q& zy~`OVU779rs_OOm$mjcZql8Q0fvs&jnL6K^jC{q*<*D0DH%Akcjkbti-k`6 zrkfde9F;bWm=K1F7543d$$p{2D>YP!T8G*w7~YPeN3pb9hi*5o^M6<9#1o-&W%-lV z*Dt$!dDM-?aj9`)2%iKY3zKKEr;-}kPMmw~;eZzsS4YH`3oZ1(KAaAJvt?oQNm8XM zHrbhQh8ZJXv}5OE9ekRXI9E;9NCV~@t2>$Q;CcQED|kO;e+ay*f_Q~xMlMn^WLmwCKp9wJv{HdI-Fj}M#sSymsTriZC)Y8 z$>D-i*6Xgblnl6Y^YG6dRf07vN?*bokEIz$XWw`A%I`+J7BP8BMMP9!qs)4=ArhPg zBAL`qUa_G+Bm4!N1=c^i&&n?3aZ6-W;n2=asbU_O235vP7KwA##{xekA(;ij#TKJ~>tWg3 z$8w2#=)!AaKC0xyruqAXaxU&2PBG5Vmwde~`SRq+Q_(aEP0yl6dYjv~Mrz!UEa*L& zD6<*KWzw5?tT;9>fs0YqNz)9=oZr&suY#+OB4nrDZSb<8Yp)E+%P%#q-AtsN++aN< zJBHmK6irkATr~3YH^!o+!_rjbMg?S{aLE5QmI?+0wgx22@DJD-cz}<2^|SchlQBQf zmQH5vr02-;Yq_3l&%lNX)NbYrPp;avgpE>?2u>69S7|~@c;U4J&nq9)HdC}p*A=+I za9C@xe~bJMJ%^yo=CGZ&;#()_=8UZ>d?sn%u^-rDUCCl!QCj1rU`vl{^5=f@8EGFL zr+N8+&NM9p*W#nEXJ*xmRbn^}iv!;|mgbiEJ$;CZqZ-Z9&{ATl+}Q zyKj|}vbp%H_cr9nHHDEb@wzPjn9Gy4J~HBmd_raUsNkFRu!q4{0hf=&A~(iqoKs4Q z4Of8x&4hdqdJsAUzQbjgm!BU zx5$RI$3)zUlMFqnCNYj`ly3~0EB?UG|Fxu}wQ}P_#NhGGd%yIB5Cz7Xc!zAcn{Bzr z&C7iRicb2OEE9Y}F z@IEYj#M0@uEPwaL)-Ttdav0y)uqE_1rpx=K=;MPLRXo0Ut)qwBJfnY@@9zoqIHYkw zFy?aMZuzXd)(&CVtBdLUByl?1w`teNq#Xy2UCr#kU5dDg(RQ4^r&WiA>9x44fOAsVWtHL+?xvREX^LcY)Q4- zG|YL##K z78OO@17{dqce!f+=THBqerGx%`#90uVD{(Pa1#x1ht%^PYR)1Dldw5jJmFBWOWTv@>HHgCn1n76ZZ zjCT7(zU7$GlpYu5=mPy99WvKin)#idMH!H1#(=xViPk`I3k20wY-x?}N0( z&j}btq->9sBzoKo7{;?+zn*B4A6*`ii@f-y%8i@&n@5Y<^fx!ar~iG(ydGB}X{1e}Q)ar;z4udy|>v?E>lQVZY?>~D8#i@_ld2b6WhL2v{aS1f;46Sy=E z_mY?IIkrcN>@XP_z4sE|SY0LD{#CEca_cR{MwIesj-+|AZ}VUWr*}OXwV_(0bx=O1 zZgcY?CQgUsUH9tXyR9#NJ^0Zg{!{l&xzUR?RgOm#a_Yion3~FI%>sVZGzaR`e?9zo zJXu`Gp6mR4;M?8Cc)4u$8>Ta)q6?Lym-c#be_HBQoqfk}Ls0%X6}G8>DCu<^7GrK; zjRDBO({NsK)PP+PenUA|Pz+0KCptU89KpkFktSwtPQxqg_{LBmmDU_MP7gfse!(&N(V1QNcYj>p=TBz1tiCNS0}ZP|doFTD6;-jY57{&4gf+^e=O? zZFC){4SE@c5LnDX@P^!A-YouIi~LlL)ror#S3{Y^`=WGicW)uCK?~j7O6B%XrSCk0 z`D^x#M)Sd_5J&|bp;7A)8qSEo26oOf3*0}I_CzB5401#}f8Otm>^;c;;Ly3M1cB6f z1qI~x(5geg(6`8jlBI;1w*L0I64wR{3;Z%Y6XX)SG>J! z?LjOp;Li&RY;_|#l2?l)7bj;j2w(-5M3RfA18D1J>+K9j_!CK4Y z7YGrsLpn)i$8K4VEH1x(q)z(XEMPNu6!u(eege8N3X%~BC4<|Pwq<|)5qWeLppl9nx z;RuexfDPxrA=DK9pclJVa6A(E|02QuZ^yv^b;JUN1@X^t8H@}ZkHNzT024^dAmM0W z%rOKcCE);_(0!;q29H|FG15@`|5Jkfx{m>Ep#+WbE8)PqY!ng_fD9HSFc3ioE-ekN zfx_d!Wq2Xu(4h9k&4>T5s1A~AkP#OR{X&FEO5|p7Fx{m<~1cXb2{?M2NVEXYg zD|Jv#VFn`&gK~%nh$$%HQ5Z^J9AJE94A2(LODTgL9a>t3l5o-lN=^X29S`d9C^TSj zWnL_p7YED(h5+@$!f_y%0m6cENR%BoI4`tW;#=%QSMF(BRel_1XXNE}#0Kr5sJR1yK$LW*91 zhZg@^`lS=Ua!4p(-C>j*D*u)!YrS$GYV%Km^a5H{DE~LG^7oC4IvfSY-Ll z9WaBG_KtAMn`AHs#fg9=qf|RV-icCj2D-gcaseMwKqVM3|A0NpU3XwMR!SalUy>;-fPD#3p9bl}D5%00;UtvbBbX;+;dc6DwN9E8tN+IqvWl(}IP zX9xV_N-f1Vg4azeS1Vq1l?vcRS81U5O9~Zm;HJQK2(*R)Zv`GuN-B(8a4gvo8YzCpK3>qQqZQ&?rV1zRIe{BZ*^uNt8 z|7$bo3m2#%lq)L!MlB?!Rcihv5t7$ml@K4l=+=Ym61WB$p$Ax8D+0>BUu{=>_ODNb z5dWYr4Nnq)2fy%@3|ta;2k-}=>etsa7!+{Y{}EH{9;3__0%uWQP*6v_*Va)dPCs(J zm8$#d$4$CpO1h)io_q3VjDE(w*Uzt8+_OnXzjECFam=F{-N6d$H3jZvA-?wrL0gZipY-J%Qc~pfDiJ=(JKW9b^H!O2x9`38# zl@oQw-ct1k6&(Bf3jcq{J@l0Px%4aFk|A_l93Vgo>@K7?lnOyJ__p$2`aCaNZjMxt>5l1d|ySS$uBiIf&WB1Oc0dsh5N zL`N8KhRl zWP#S!0bufH9k{};ofnCOj;YtQLjzB~whoB~5PoetBmhBw)&X<^893NPB2B6v6^+if!!KVakXpxqNP!06^R|a51 z{a0TZ)Za3dK?6{~rmqYR!a;wICj;%@*3=O&f43uGp_Af2`x0;vE?HA24dIGE>cA6& zU)v6a{Cj;-V1d`theF~Zoc8B<7y>}_|J0F5wk~c&66MQW6BmCXWxsCh=?R%jir3Zl fa0I)0&|?L9c#~~OWQuu0VZl=c6B1H4(1869M?;-a literal 0 HcmV?d00001 diff --git a/api/plots/exp2_write_throughput.pdf b/api/plots/exp2_write_throughput.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6b3c8d042a9c6af2b8806b486c4694c5b6902469 GIT binary patch literal 16777 zcmd^nc|4R|_;(@8*kVx1G6-2`AI6q_31LVmvW=zehNKNq$&!6bi><6hq>_qc-%BL4 zpirVjQpxY!qn@Xj-~0HypZC9ad`|bdulp?5cAx8e&V5eerrNr4Xn7n=ym$~%R0TsL zPzZ0w127d81k&QL4;g{fa-cYPc)KEyrVg%Te*^{;AR*M$U}P_6sG#B>6?DD5C2n#)d`~LnNu7nWCU)RVCq1jko~+6c<2^}GP zQxJPWx7xaZ6><;-fi&<0ENK3^YW=zzAZ-6&2fbPVh;N8{KQiEMnSCUQ?C%}m=LF<| z_y_$PlbziiG`)jBk0|hmBcS9J5m+2vo&cIhY5~DOZGQxDnLQ0JFK=*%hKl~F4CwK1 z?HQ51Tq$k{^sgRu-93Q32&AqDP=Yqu$=jI>vF%UsBRhD)f->(}8~VeE8}D~s?}`@? zz4hYpi-ZVc-rYymk?PkgI>*2@-?%f~*l}V^ReJgoxo7hFK=aNnrW`lBmPVC)n zYgrCAvUV7>eiLS)VI`i}`C0I0r@~yhnHghDs*BZ3cvGgsj)mS~K@0bWN_-S2`0-AtEti?+qggkMrR zlIDgyU>C9_PgiCGhT_-#`iR1TNs)X5n6wOUk9{ZN`MC`DeBT4v@v)-Tn@EH4TTZMy z5UubSj*GmfYE-cak5rE#ISEMLz_x701nM0aUY}pq(EN&r5bdycv-T0Q&vZT8@qEq* zy9x=xK$*Hi**cve66Ug(RNEdO!y6LOqU8K}y@y-Ix7*3d-#`1N{>j|EuRq@T9Y!}k zxp4ad&pc<8&Cy%jCNJeLKB}K##;Z33$65*PUTm%Jdpahz-Ez_8526pkJw2)=*7RERy+-JH=hChM~BQC@uIamc^vcX2u_f0S}tCITN3TuCy^6`Sj4&#txS2YZs^2 z8!W7~ExcC|=@66JC|xJus*XGM;%Q8xCTHHDGxAHB_vMs66V?j-V%4f8-?^YoS7-_m!Qs4-RadlEZ8L0SBz$Z&bFBpH_brBI8qhQY0FKT zYFAR_+f%nP3J#qzG%q!ns4dA;$~L_^_hvTEj42QX54_(MceP=0JWGj0%3>cPR8mA` zF)t2;O$@~KWu7UJP@0{6=JIU(P_^ZR(!;w-H%n#$B=6rW(&|zi#Jotk75&xohuV#$ zGQs=ATOC!Er+SIv*Uly)PDxy7jcdMH=B7ILT;yZ>1H}>Bm-15=!+&b2lxs5P7|$#8 zlwXo5`{{(w6kcHQ33SAqSQM03^zibLN!sK7PPE9Wqx)!XSbT5kA#F*Ofop1KO4<`E zwiTIQxqck~;oRqzUZz7&H$JW2%s^&;`0)mh*38)tBbic9&&CmIcaoeB?&-n6*|zr3 z`DeYWG4jrukQ;vdf}yg>{w?9u+Xc+GieTTpY1?accnfAem@;2@H=5fQBWX04^j7qS z@wfByA)1@oanan8G9%n@R|iqgt=5_`t>@5hk|Z}8FP!R-NL)N&;eM9gk)9>^pmt?qp$Ll$lY}r0e5QX^!WKExoQku)EWX z(#(RR7OVC@=wK+6Y3&c5D<6HvmUngfiO|nGD%{fMad#^vU*tWNswBJeJyHG~XZ|eJ z`9PJyMGK?0^1_Wv-bYaT752YndpBF#Fsik$S5=|It@g&o4$Yope%IPWJn|oJt2i6` z#;{WF&gSa<@XCXqWLx%3y6qtSe96QZ)!Qt^v%4`se|vT9!suC*7DsPUi5Zoa$1f^E z-RdZt!>(-}tal9^h?vRXIQDE4Y(TCOEi#t1@lYn(N_YCQ*6S#Z{ZqGUaZ|%`pGptv zzLDxRwv`pj^Ws~$c)PRUZv73#?-onzIkw?XLrCCP82g0@s4)+UgJ=Z|?l(}N21|&z z7h(GkjYhy!M1vstAHmYN{VsbAf{mf;qLd2Vt0gtFB+>o{G3;DfWtS+D7wk{NynL9UB+oPB8T!(ze2iKM&8{|E(2OmuA&ed0k=8HNX3Og91^jCBQ(DeMKb4 zU@`yAnQEM&AA<;+)_kU>-ntX7RW9so&mQ|Z&@`3q5m#6<1NRkxyM8{yoGb97*uljV zUVS=3>#4o+<@YtN>bqWTk*7%cT??Nf>yH8#EC* z-26r|2MaA;7#+OVFnr$c(A}1U&r=@WJKge4am0dHBz-WRz?H6ZaBHou&}3<6PLW%y zwe+Auw6QHbx$b=Zl=V>a@Y(6>`$nG1*0#;p-FhG(yk}@!y75^{ct0ue?PKEH^_1>! zJ{3_B!LRj>Na*_n6d`u!^>#cr4_-7M-1cpeeztaxAPt56iMxJ5AL3f{6;sPzgJPy% zGOA_XNcZTw#ctt=pD!i2-Z2R?_HVX#D9695?XO9F{9NH)u1?}}X3Lv8L#7w+^5j|B z>g5?dG)Y%JL81tJ@ZlJE+51M(cw~J4eAnW7&YeQ6Gv8feYLrgj-&G~zhd{O-Mi*Zd7j*_H`${kQ~X8D77Di(DaqR>iVx3*3|>xlIT}nd;6j}|THNf`Mn~vMyz9ED zA;VBeCw31mtyI|n|1o5Kxbpry!S~$fOZO=GX)7JXB<<(C5bwZJR$ut;(W^F4NA`sWi5N^*hLj;uBFNVU<{}$S%3NTm!&>Z3lFopc;#(zM<+2lB>F4y%;47qe2&C%SHC?!WKAOU9+tc^ z;0aBAZ5J5Uy0wA4g99sEood;U+mOiCq(e|u;lex<->8JT%NX#?<$2_Nk=Ny7q;}nu zCw<(rkEIz}%qw3`gmHToee?fb%s&$bkalZz!~k-?XIsPYiqZ6aZKL%;5xip#Laf-s;5)F?#{+157>>36POpY zSzIe|Q+z*9bl$t)_nmW>hy@o7g+X)j6$)FEI}r$bP^KsxL%gmoBkYR03Bn5YWL;=^#@n4RuY~qoTy4j-xc>!r1=S4Fg++;Osv$U zalpnI;quDdsQ$s13Ql<8{3{{Rq$pO0Q}I5%#Uox9V>c}YCU;sfO%uMJwYJ;%?4eQb z9zIw3felXXnU9FgUeUJjvY3F9tL9dEs4TW{N@ZOV!rvlywmBB_c3d%D@?zBVmjT&8 z;&G1mtb`a5-m$OJ!C{1x*^`H->9+W3EZGiy71FsoE{@pB!q;o&IFbiqOsN z-2R#w8uDg;-x;`Raa`{^5gDA~UUc>KCK1390 zkq{lhoM$MLR|xTNCBaPW4hIFN*cu%)W99_qiYk)?@%uwqxc^r2q@})OWT9y6J8Z1f zMd>4#g7{=VG!gbnCa8`gUXw_X&3TuST6TE0B*Bzix*24-@4Rw`8!NWnH6TUvCt=op&WSQiR&RW>$2ZB=T0xk3LrIDn&f;Jb92XH43Z64dqo$7jm|E2TN0@KgaHlnLFh^2w=utfZSk6@}KL$7*u z=7BMueU~49Jl}bkw0o%fyZ1pYQh{`JB)S20tR?Qw8$(-e--za0cGvc*TsbPg*Cze& znY&Rj;%b$HsZ%1AwiNj?MM1q8R?G!y?fmw_?yRqxDG^L@yFO;IK0HSJIf4wS?JH%! z{aO{HQM&D|lxBmM8Gi=12A9KFJa2xk=_TujFU_Xj&to9Mv1I!pXfpMipTCJZJKgGGNmlQ0>Q@t^USXjGo>!eZ@LoNz{QhU3%1b-27~ebb9OS?g7r1Is{(Vk<9%hDMyhs6Q4^c zbAcGCLo-h2LVS}B$X;~|vNP;3uG=Jc2ridblA6fxeS0c(UN6}*o?WL?j#abIBVhp( z%5-#dnqI(&dtT$v7mw>-eyr>8z!}gGhz4F=(`#;QW^k4s1hz5WZ|Wj;iPcxN(zV8x z`dT<5hzu;(6B1L>tXMhG;li@2U*kpm6&AN1&+fl#9h}hIzmzs6YHs$GlrQh7X?tdC zXBO8-)vSZu)@AoT5oL4I{qHjgN8Dz>X03NRW`Mbq|2A0!zKbnDfWs#D;q8Ntb;CB@ zDlvmz-*sKc65;IUoHYkGhnzvYr@OIPHiJFRu*j&g5_eA~V&V1dD&n@A)AU=Xtr zbJTu}V=83LyO;_ME@{9!EN%_qPX`P}+Gys12j)JZ`ma;tK35b%(~`aC zjh?o@VzKuKYBj&&+){c+Tx?rklF2bqt>P;`aDLlFuY5IN*$|j~uJNE@am-|VKF_7+ zpGL}Nb+7dE+RROL=e1{>wYM}8n>J=$Eu%Afo|#`G+GJI%Lm`q;c%kNK26Eo1CGAC>xaA4A0kI`y;nhBpU)6o2*>x#!xMjE^h$Z*cVqFKi%k_C`Bca1e?diu;Is-g=|LlDU=O+ik02j^Kwe~ zbtZD(m-m|tC&M4nunrBZizckaiQ*c4H-K5=bRnHMDg-uH-1wRE{m<*CW}j#TZ`NdU zGorU~3iC(NtW?Xm2m0glXx3mB$Uh%&}iB3FN z{y{Kn&tD7^3;8jslDLzMzOzJN?{Fprrsgk(?-aYjEjz|@7VCr5}tu@ZO0$di#koWBt|+TVjE;Z%%Z8MS!%AQD^d;+0a6ub<#0C z4l?XnvA=-Fs}|3?&62w3OQtyHBM~N&=6dxX5d9ySBx8;x7s>cs@c7bgh~i*a)P$#v zMybDN3vFmOpB?3%92p553$^z?%%$JIu_<0;vtU@@g?&rIy$ko2d`~HN-sHa~Fg<)I ztv9f7zheTg|N5vwsr<2n@-}_^N-q+zjiXn-pX#_{pL+MC(e%QO-p+47>4l_RzR^%6 z4XlbLtYPx16Q|5g4_4BS?8*}RDn_r?>4K3W%zHaM^B~ncu|k}@3lHmg9u_$scDho0 zXv1^~gR9ez34#}%dz2v{#4>lpEs1pOPQ80F<&zu1Uxu{=F6hD$oRq_Vd7~2Vix1hO zepjYC7PNfk_Rl?V93q~N3~g=^WNv?FKDaMwi| z?dH=VN#oNKq>uqg;@0Lbl#Zr6;y8Tjjh8b)Tv`X2SSmi>1^aX?@-5H&(9@K1EnKP_ z@i;41x^S!D6GK9Vk2JjM@?ahI8@nfk-tx3i*0sBUAuiKJCs`~_4(xtqxby|!U*n z{~bfj+!fO2Y0jjJlf}cQ-mju-^B(Fg&VeT;Q${QaBED5P@hQnDiNqbXT$n#=E@8+sETBmmrTr>s0Gs#0+*fwr+~ny(yR9eIQwp69qT# zPWCN|4^4tIYPss|!LjBwwFGP8AI1=~((l#z*weR`hv(*%kg7S!>&NMApDT^vc80~S zd-!sD^xQYb!XI`DH1tLTWTEk>{~jz=O^6(ID7Jw`*l|QifK}y-MA7lMpC?MjFAFko zT@Grxk>j+1108D6C=i)iA^Zauqb_?gLo8Uc9wqBTFbF-ZVQH{OZEvQL$juFQ`d@o_ zRJIyBhh1zuBX~Qpd7OTlw6`o^eEo-wySLevvpJQORQaenFyQNhdEd^VoFWtSF6}ma zzkUP!>*IwJQ(n`7$JX1|qo`jd{{EN&7AaS)mZ0%#Kxg=Kt?WQNt?7&~R)((58!9Ge zFQ29?9TdO1fh`TyEZNU@GCo?%$bK@->9x3pNr~Dvv0GIJ#Ek`}3JfN0Mr_#3T*Ft% z6Zgp~w}OZzJsU2>H4RyN;i(7Bj zep@PF>eb7hpu-7N@fo)S^SG%^l<8(wbsZd+-I<_W%a8L>*lY`&smxQ;|E5S|e`d^gg;3)21Hr`g&1YM5q2h zlhT>ixTxC+auG+gWk=DER33#*7kv^ES}1O7E@x^$_8;Pu`O*_k78$AMf9}BB=)gN_ zRT>~tcqGU?x2|B@vyrGJwR(R!2DziUT_O8NFI|s)wUEH)KA-0Kfsep%x zzn#Q>8`nEZLY+*_U#>mlBHg9qiMWI9@PEDiiKT7@Um!vMz&;P}*hQu%qx7u)dXqi$ghT;AW;FH3)T*jBOpC!4v; z&BKzNe%U<*%+qDAhFx96YZvdOSi4cq!>e<{I~#VH?7F`P`(A{|%TnNIv!~XbC*6@{ zfP25WnR!Mq|IhC;Q=gQN|6DKYf7G9bS!lpKg*CVz7E3UP*462_uUt6=d#js5MZ0Qe zl3O>xTi2Jj$+m^cim}F}2*_)(I-xurJ#y_`92uRh5okmew2qyY0jHbM{nCf{L0Rx;bQ$5b2q{L1IiOPrU6Bk5XwPUA0Tb)7|6t z4~0%kMekuP&lmET*$Yq36)YLy??=68MVUKhy)NC*^VX3ZHuYu%fBoT^4`zwBbv{!m z?T47V-F|N6YkB3wn-HRRwTzqHU9sj&%|$!Kdb`hXB1g>`rwbXfY))6j{Y z*?T<2z2;m{c&1FRT%*+n($*ox4`=7I4$IwOJaaES`VvBRPxYJlBDcDiSajIS4X2OG^m$s1s#_oooJ{dqfuDkBg`WAQ zv9u-jma@4NbN+~y8XT{Gjplc~)kp5xIB9!bHC>~*c;6#&_$H&L4q~nxJJK=o75=IU zh%sHOREIb(%Pm;$SX_j@NJQcJ+zti}?&xc}F$6!J?|H~`lvBH7^3jw2a-RD;jTH}e zO zeO~H>r^7F+&y0MJzCJd#@PmH8{oXS)bVvi3p%vCJ4jIQ8aA;BYpgi1^AC9dVQ+ZHW zo>{rdMiJWAvFI~SU zWU;C-)bQbg-ILK&X>lj`>6y@Xq9lTH*2bF_lYWI~%ZJX(`0&pD=+>J0u;Hed%1atz z(*T7EYxpvQy@?|MJW5!{eZ@r=c17atKS7!7<_vQIBd>Lal$F&wehHVirXp#l1mvB} z{V%Uq*J-$7zol@;X&DHoCdYA4(_o1Pd{bP*#1IcAtsy8j?IR|yxrQI~Gl{^SJ*YSE zae6tFXh#}Wf1|_|dnHs-V=mne(Lb@?QfJuApl!+cUV)Bz=ammd#tf3!*tr0K{ER2T zT{hc26Zu~F99x$u_<;t4G=R*S7&N}a1sHs<8lhKnPQRM7(JXv%?_@YGW!ELH7N&!v z>%(5hAVt1rBl*K`vT{m)*QY$wW_RV?2Cus=`>`;S*VA7d-fwM`Q?AiETk^sCq)^q4 zp;!SJ4F+jo8w`34R>hkUIlwB=w87k|w`nvPYv^EKK_2ijv zFCAo1JA2EEbI9ptXlkX%SR}%6JHM<(dw~v|&$Cl97ykgPhrCGrFw|t8*O0O~lySYK zqCnqx-^XNC6NzpLFQ%sm-@2Ivt|O|l5PRh*{JjnCF^0RRfvflceYNr;YS+4TaNkxs z=Sw@2%10l0m(7+l?)mm}{Utq%l-2q{|LI}N`|hcm>5-Za{^VadV>2^tOBrqQ0SC(f za|bVfDAM`)Q?%S1`~YG=8ae#RWAJzw5~`z3_IL7g_n~Y;d3h8H z0xsb1pPVfW2{C61_VRq_`4yH!DK&g1k&3J2u3;N z4MS4g{Kz2h0yg}Ef87H?-XHLc^d|?By%0z;wH;`2*U8(%+Y15hW6?npYX3x_xL^0nOb|ga2S5W&Adr9&s5b->9GL*zxPwyypluHX(i4FM z-2sMt5NI?Q#sN3LhCrME1TqkTJcK|7ft``yOaO9^yEDZNs0<1Ke;zNOTJiVBjeou5 z4?9zh^1m1rSj@x0)gP#JP3G0`cY+Q~5HUa>P!G_J90vNHs^#FLzpOPB3~BW{3yx!e zgCg!u8eXm*WWYGmoI>`rgfhksK~#wlXeITnGkqS6a7Pu(^ z1@h1pdX5DN9E4E-^`SP2AQTW3mub+IDhyTu23;X0Afcc{Kx3(O@j&q9Hb7a>FEtNV zD=-R*)Pz?cQm;e+MhJjTKx2S_%l+a&zj!cQV8P~3=oubd6@jv#D>Y%Das;Xppexir z)o73c0S`d2FsM&Fs6$oCZ|0yY)qkL#aL^43QW3;9BpP%DiE0<9yX6GK0}-h;A^rZF z0SqoTfdLXjPav)U=fBf0r|t93F3=-588ly z<9C82C!p}ahJaVd2WTWB7!av`0R|<)AL+MG{Jug;0d|K`uTcJvM78zubEwQe3Gxfb zs?hb{t>J${8XW{00uUr5U_~m;0fB*Lwt(h{00v#o{V`r+pvedbJ%5ea zz~a<`P6*7e*tk6WW1vV3nsP>9s1XYq&A|{zZD4uOqwXC7C0u~JESIGQ6`)XRMQWS@ z5t4f23eJ!%=iI=n1m$2LA^=~fKJ`F=p@5q6L;%}Rb6()pSbYuvkK^9{^ltITwfkV-}Tm2n=b_J1t~9SejCLRU4lKY|$~=r3;mWf78j6%mr-H{Zt4pa$-NLXCll%X<5_ zKK0q}vdaPg_qob{p`5O_AAlCW!I~mM4ulr)2N2rtBZyc4VE*e!g=d@wM>v8_XI@nY z`O&~ohy4EW!Q1IZ!;81*ebtSIa9#IRj+^~V__!;t=Ib^V!(HX0!B65IR~bcbe*5&s z_QuO*a=je^ius*gK2DUVGM+0vz2iYS=O+j+5_fznwHDFcA;57x{0N_x#6BrBXnJ0{}Lo zdibvcH~_``TUiU7Foos-+W}bj@$ja2xH}>OvGQn?yn-ykjY9G9S3)8^|9K|w?dK{5 zLju(99N+{_to~We$Jqtp2mxU)=ig0$qSOxDy}h)dW_CN$dBv-gJKN~gymm9fNJnygmgjtLxa#R^h0?4qM<>U1|!DbG#riy6tRMa zhr+-L8i4>_(UmlSwpP*ET@(uXezT$+n9;7Hp)dgcuPO({2k7^&Y&ZpI zoc)^yz{1M*&=?{V&i*ckQGf==6*M$q6%9*NSV;p@JAmT;u8RV@=U115P{+!03dEKA zLt)Vht7$keWm-`eOMoIC^!rB#D69fNmMdsD6hJ~NX&5LXL%)CP;_$0j4{r<5fSZ`RRXEX{2;j|Uy6d|niHw_JKE?P;$L5B}ll!InaD`^Svpe-2OLlP=8MVk{gnqLITn)iH{(Xe7*fB&R{-} z{9t@zl8c9JCn_6Bz-i1l}6K#zZE z&xk~Jqq@UU%OmP~c!K%D5xSm03ECuQZx<3Iwl9@Ja`b`)oV{(`LAF!e@bt^}stUoJ zNMW^G<%3P?y*dxqj%{tbn^d}+E!ophclhVAs2ReZWFZ-rZR%?RwMDyyDUk_n-j8O! z2Ywy>YWZ{C+$eJJ==O>Q-RBX^b8&SDV#mRG1!glZiDEYXq6$w#_j8{u!|JAQZaC~j z(Xp@W`(Uedr6NtTaHH=DHm^XNbn{*O9J{kzkKJ$$Os>U9UUEv7ykBKSmYqxAYiKs; zKANX^$Gmy|8*d>`L$iBt<~Dq+xLtdfX0$^J>%dTP$sSgYs=hPbIUD%`4*G4aecx6# zn7aP+qLR43kxH`=uelB<^H;7Mc3BNQs~IlT+B=swHbknNDV|xo^O?-KNkeXAQu7lv zAAE?3sM9y?*ay#@_AjI&SyS}&-#P@am zM$73tFP}}crF!JX%C(vs9GXpbmKTp=9!cDt5yfu-`z-iLyB_|W;gkCTuX7%P34Mhe z@(8KX?Wl7xZOR82DgJv#L!Ok(ZbMi-n*7ld5pd#?y&$HBs8 zuyJ>`knT8Bj?U&?tl;lyo!P7M=1d@a-&By4c-{FsP4~9++^f2omlRnzG!Wbu)4x>* zmkQ5Sm^WqT>Xr;vlvNTmlDccQQ)g}WgJ{$%nYFKS4JwpG#)m7VPZG{Z%Sa~-?s$F@ z-Dk=jQI5?$6#&cKX3$gdMb~&|j@(JrGtyag#Nt?yb2^+g z{554H7ya{S`>gtto^5gG&`dtb=(Cr0yEa)8Tf`@2M5Q0SZP6jUntQN)BdJwxP;5|G znL$dzw)>&$5VwVXCf_^hRK*K5hxCPx)zuzEO2fRTi!59jOo)+RD%wwF7@XnG)#Sk2 z*y2f!-RVa-&sAzL<@b7bsQL^(>kO#j%)+FGe=-*`FqjUpe3m*r$Ab)&G7&UR4;Jz{ zi55Svn~Ac}5Ms_>;6BaNZcta;Db2Vu#%Nm7;_Xm&&#s1}N`8z7Ukzj)uMfQ1|K-G+ zOJ2AW*3ZfHK7(>wr;n$Z1Ssz@Pd=#hrKIJi)qH8d4q@}jnTWxQx+mB<8 zy=`{JH556S*TJ2|iWSxzx_RDQ^uAo|wHuyyCONID9v{1%S730)BA-jJMp`Th9T^r$;~x15#>Yt)#%#OE|}TftnUB21U>YTl&I0^3t_ zQ8CDSFT0*)KTh*gkncjFF!*22o#x;mq8;4+9~$J*S2#H|9t%9%e|R`mVs#P|ilgHu zEUjTbQMIo6ki*Oc(FRpz6v~jBL1Tt&PT0KDOQIuT;W5cg|8?3-w%6yH+;P^TGjQLw zLv8#{X`icjwh73SB>UYihkZW8d4RueiWkHCP)xsxheAx)TAWZCHtK}H9T$jri?i;E;&h*lLGSu3%L}g-w$1|%SL;lXUoyYWcnllo^jnzU)=$kBw|bpeWbko!tD&R$ z-$Uqf)KEhGw?2)#ZgDoiIhcFT%c!h*v8ZOAvh_)Q94B{9*#)Z9Ifp11`H(e7g|cwT zY@bEOu`yAeWHaBV+SV9}!SMY3^Bu^>4C~g)QpG)2DRlzF!m)G&qZ7^)@&7Ft2B)|i zX?CrGf#+i6Ti9FOG?+}}UUjwT)D9Nj4#x6Mqt0MCEkl>Im@=_Ml9OX(6B=I@+`iSC zReh7EF<0lb@KuhhSO%P+5X;{(p`nnw!us&5P!O^YPy_S@?8Fy#XaQVG&sGCeIgc4$Jx!PxUIHDhV*Jf2W zSUM&>n#i5sSex^u*fpwjHwG21n8~xrLH00Z{nxLTlqQFg6Fv7h_MY36UykfdT2S3! z;oIvQowOjnpK{Q#H}kEap!^k~5|YvV3CGKx@*LxF8i(7z_61%2g5>`Ah3UD5nhTvN zLojBg-dAG_7_24ItQWV2$2C&&5550u3(gi6+tL^l@n~Gw!wo{&j6ST|Cs_AvVjv#m zYoH?u1Ta@ff-Lix#tWZ-|aX4E1SHZ;sXIsYYJx+desf~N`rTmzEl?Y?pEtnJMf?ZYXoQ>5<6BgtT$FMmJr;K;%$^os*V_MXHmN4BW! z)80Kc#G=!nL;cLsOeZKG2U>o942_*~I`WZj3MX9rrZvlV4@E6ccEdru`Q zkOiZ}L|Z=>PQA%{d{DDz_S$xdX}!mh7>d*f_m;PPt*;V)K8s^EOt|VU_T@z>xnIGd z)?$Z_lZy$Z*dung$&4A_!|KM{92>rR>S)RMCOsP8-`LH5FwIjSz+j$5H1DaTA(z#Y zp+b|7B`QyPXSUc5P`A+0Jp|rY=pKz-h40r%GN!PItqq?!pNp8f{CbaSPd?Gc=tDhc zSM9d#wh?cc9X&1`()u34FTsC%$ zyM5|#%E6<4yr!jPuPwX8Y`aQ$H=VrZ8KBYEwvGD+XRK73ubBDV<;_=iC^(#!cr_h7 zoSo`=G>E9rjSM?l+~(f72H$)7mYZNpmZ7Ll{2pw^HD!I=RPfBJ%DNf+p@c7jvk`}% zFK*{LHn2wR1cTvEdsaHigSI*=l&6UKZ@a}*V$Fc-LR$3Vuy{-4b*vRq5o4x@dc6h~ zVqLu#Stg9^=;UcaV->ZW-en1Cl9H_9Ba|u!*3pK}j+gB(444>1V+2y^2n21mRtSVA ztXd%Ko+W|sNb_ZZLQ%eV^c}zC-$;joYF^-PF2sPyHjYI17mo8P}AJ3hrNzOD$oDyq$$SaZ= zkSQ*H&Gp`ptqa`sg@sXb{kMwsyq9NQh;B`ZWp|8B^64)gA)k*IT=Y-vv1a>-pFL@_ zZ^PreM*VvP+!TgboITDq5!%S7?0L)L{7NoaSnopSaD-DU8&lxEmig0d@#wc>N=Z`Z zV?TZylJh4V=X%eMj}zk`os|s=#fRli1b$qziK4M+|8!PV=hm16eDgYiesiY}XWxYH zzwJ*Iy&=uxtC^*tz$@1EPWIx4EVJVBylkSmPm$WCvEmCS-t6Bi|F^y{bf63xyNdTu z)l4~FTa4p9Ap$YA?X$R#sO}Yc*N@MyGG)n_86B_0Z0qF?*~NiMDIJk0$-Wv=AFKI^ z{OWz!UHBC(BD_0jeKd9AB0h)ChIi>hprgztA@;l7(m6IOas zF_*W-k7MFx&dVOT5Fj8o*^0N5N>+Uhe@P_9v=v-PY2WG9o&ww9dY@U2r}~8pud!0c zEq&rCp%nDE=tcjn$BI>2^LC867~nMfpo>ZBJ*sHu?AeO$Fam$g z%t#EL%{XpwY+{DjLl!~2C1QK-xF1jI(~YAWdD7iZ2t9XMoMZbLWxJJ*2I=513<3Aw zK0}pgNN!eNJ2c9-H~YcVnVvx6?x(fiy$@;;i)3qKP%X$~?TOWI4DERiMYP@Achyei z;!y=V+swe|Td{ExYL&z3AH*u{sS0IEBD+4bqtD4|7j|8~pEIkO7QvQiGL^%A_ZZ>l z2qO5-z%|aQm#S!uYunz+Xtt2eg|c`wxE)86_zUyRF4)|CZvNr@44UPD_o)JQXW{8} z6)c)wdQEG_)fu!WQY=dnqYs*t62+L4KS(M8>br)ssE)J%=% zrPGY$ty;(yLi-m|PW2t7vd9);Kc{@|-Cf?xZXFZI^Np~eH#^$zDHged-2XY|%@#~E zny1^rkl>J$KCr9ju8w@Lt`&yzYxn<%OM?v0N`YYf1-bf*t99uc>i@ z1{V`*Ez`%qhzsg)^?)9$-Rp-I_ZvR;f5UG!@TkZCF!SyysJK>&+JO|O5)V% zk!RN&@JyaVhp-*pn6b-m#G~Ne({G-o-xe6UJ+b<9CPW9XuIe>6HrGGN1OnSA|Gc`` zzSFgrv@*3u7YEw8A_&auN|R5gWmvPXM}=>ZQ=Lr`^Huz@`FQS=TQ)(-ZBG_6Mz>m+ z&k_q2oHXsDH}~XlPpRe{YzoM4>=NE;?KFzyZr8`92H~eZHus=dAqbSq=L$YSjOJn7& zouaUOw6gs68%LVP$szMQJBU+1+Gb`&O4oNS>^)0IgLF_QVKtyL?hbbYd@b{$rN%lK zY|vElGyn#18_-7`M!7zOv#?hN>TY34mE6Z~&1T>{Y?h#Y`z1xbZJ1@fkJO35Udyv- z4-b1J>F<^q&gThSTxu+GnR7%S4f-VVLl#E?K69eL=W zJM1d9Np$5&+hC<2Xe7thx#oVF4{YoCJE^P;)oz?pbcUse<8Vs<{r>7C8B&fFY?HZ@&zJ~wSDe#seE^sV5Pj7g-=I1qiy$C z*_tcL7EPH4V-CD`CcUnDsLa~Dx!aylwQl%J)atdCJo>R0opbN)6d$98d%V2_J^ zg*{)ido^y9(mhl1fomoPZYpK5t9cUsWQt8H?pW%T?LOx`zuh-PaxwqV;SXO34$--mruJ*<|i(%Eb@e zb6P%M_kXQ-dP+DG6Vlc$vbJm7Vt5X(oLf`V_A~$9aPh$L?Gw8lJ!}j+*4;dJ%T;~d zumxf8it5fSupQs%%#{v!#GqI4`ryoOc{L+BCU7i*NceGyt`WFi(#;cRln0CS9@cKz zy$@ZmSHZ7M*2eL3ftKINi$~A9UR>b%nmCdn`b4b8qCf4(MuD`LKm(DoEV&sUc6=Q} z7LPCcrdX4co(@ThO%Ti_eZx{_w?w1zV_qT~txE32n6)>T(#{kK)K zt)rl9#w@sWPRiVDI4UIfB+2&ddx{Ngk$h~;CdUY+iSk(O7l92-M%#Y61bg)`z(mEQFR#|EG2+Qz71p>##9%JKVwTW?1*rKOYz{iV66I!=?fhQw!ks_*q=Ydo99 zsJI=DH{;4OTRZ)94Ls^;gKoImjh+Strb_)=hn)@P&br9dG2>z13N1w1r<^7_t-K#E z+#73mHTO(R{d)f$SNm`_ual^^OOD<3%I}Kn%a1KG?iV^j=coru50|1=@dm4!RmU-d z177=xP1X%TlSOUC0dej~@7Oy1b790ta$i&rTL7xu+&R%`sE#g~-K^%l2i{o}JQXQ@SkR1(>{ z?$y4tMO;kW!pT{+;`Y{M3Ow8%I1S^T23z^yTQ~Rp+*~1E!=e5yrs+@?F+QjMpie;$ z;*FHW(o%1&(ZyYqgJDbYMO%`3)S|-hEcjxj}+lCG}KwugJPg88W6( zi}gH*T%`S>km)oe``%q<@#iM;93Rzyp#m4PWJaPO(dCC(VcxyxVHtXhoT% zdVD7Lc9oX>%;Q+&zi4K;p+ut0pmXm7av4@1@uX2HF8$z*_|c*x;Wl{zQzMTb%7naA zMc;8PzfC}1QSIhRG^uJS9e=2tx{WEOCh?1A0qnhKpG0oL_Mr^oITQIe8`Ce09XT4$ zpxXQ_ZutJa4#88pH{=WNA4pYNkL0zupL*y@Qb-CftCriYJy`aF*7hJx+}$`rPUh`K z9|xw6^6>nE5@PLo664q!`@tO}ScA}bhP%(DPko(dy}Ynbk&fQzz%&$YHD|h}1g=IT z$IuVhad@zwb>*|uTgMWAo+ug17GdVj4rsle=gh)|3emqO9FtzLWdR$fE*F+19;Dff zl=H#sheT;u>F-gqJ8L9%gJqxIY(JmMW@DGo^Y@}fs!q3!F?}T3mHCY^PHxz}&Ayz& zxvZqdN7a!T*Bk_XsX{u(B=5Sg+weUj3-9cMxf37AAN`LpIy56`zt#Nt^A0$q`|GR+ zg=1j3R|X&E_8WH`T+urw`E z+a`XqMxU^u$V`#h)ZK^!b2i`bMe4gclFH2hLP;OXxy06|HeOh5p)Qrkyh{`OCi4j2 z(}k%*B(BzCi>D^7?VWZ8$)DQDGNEtlymcg5!eh#cp49a?M0U==29)l7 z;WP?Wd{1Yhbf6suvkD+CPc)=7!#Re0VfNXWS+>_I@2E8aw-+muPwVJ%IZUJ8M8+5# zwSnKv>uyGG-#Mr82`G_Oha<5TMDovpuuY3{}oh$BaD`#s# zJUP66`?tPulGsSI(4Zs#JxBi6*4O;RE*}Z7$Zss#_IM~5=Zg24Z*B46T zU(6*7c+6yYJ-ixp%}Duj{xxAwQ{R1(`)%E-cZl||wSBw#n45TO4PQhxrrY_2~y#eq{07S!6I9+@ijf+^GPFPAUJ)`3$H-rM{}NQV&_gv|rPx;}}ag2vZ&TV^E6 zG@m6*W_1#qX9UXmdgM!Y^>E*mhcTP;s#<3v__y};Opom-^f9;)vS;m0zRk5^`V3(Q zX4ymwFmSU`^_E*sXd+YSw#U96ht|eWBx6qG*PeeaRg#}Gt!k6m@%*%5L9d9h$rxjE zd$WGFz5ZNwW`==wa{BvYWt!8G4aq8MyHIVx6ZjafTo%bu(E^FiCcYil?yaeH&3d$Z z%;BzRl+39;?B#``o}cY_Q}abiMueUqpLHNDoN``XW9fVAL<;@zW&~GyH+s_iw0)z` zhqSK4Ywx@N+$_-k!kIrgc-N&e9!?LXhUkX#`;?mZec>f=HAFj~%aY@8c9*;q6d%*j zd%nEDvE`nq?bk2sE#DoDqO%ZmFzjk@6U_$&H}YwLUp+ zi&+e8s?O%dH{9DYdIKbt-_OEbbqeh zR=?z(VNJ4#;gK4V)`I?9&vIkyPUJJ9*3p>_9rT0&sB;wtOCSS?^8N?x7(AFf^zjM6 z`U~}(MNjyc4pE6I%C?sN9Z?aMHzufa6{&+fimwloBAV8hU;6Kbv132xE{s0n;K7pOl3UnH1__b5ndVjjH?>Z4cA@Txec?La zD74OCUWKndL4sE#rS&-%XU>WJqIsvz;-9T~o4eDCpN;EPdE>t75Pk!}gUK_DMMh&G z?-olx-KZNF*dj6++(Kt@=;6oJL{fQkeF~Gt+FL!_r^8(@J$Fvu9`$;0U~M}aD`%`a z)%I2PsS9wqJ+*I=uDCZo$Dl%=vqT-=KHz2jTHO+^?`($4@&6PwE&BLS#=6ea{nt2L zi`Orlk{KRreu)xtyE#DW+c05Ys+y_MR=l@Kf>+S!k)ya9*Un6|LWQrYB79WWI^8jm zY_$o)6OWD16N|WfCcm3mgXh#$-8ejj?|T6vff~6xt`IfhEAO?x$5`oL?}T;n=hrEs z_J5~{Z#4b=-dL+eh0f~Iy$6-B^6h3ERDH=B?2&l2ZnW{x#+Oz^3JI35J*W)Stdqaz?40EYtmtp9WhstI=h{$8u~ zaSpbqp8r@T=cwtW15qCqOT`Uuu?MYjJ{8g1yZLffW2D2Gk>&>ymIr6~n$;x~TQeQF z3i2O_>`Dr9Qrfszv|@AotFp0q?pw*O$9h{dYb4x*j}%*Y3|~CN+a!Wx8J8J9n^(J` z`a+T0V{GzMYG4F6B{ zZKn_crn%BxKkK_r-M13Gvp2FLMJ)3(@pPfs8w<6dOog*G!vmcMu%l@e^1i`Um9>bH zw$Lyiesv_g%qtY+T2yvsGm- z{^BFv_qNyvXr8_n?&1SXwaPz`CJYR`hdS1{Trfx}f8FF=HeJrTXZ|PSgZEF8(ckV#N{sao!i6wnx9rCt^$PtDt03;aekU`2X zl;QkuX7w);4597e>PiCn$xuSF9US`A+tT;Fh;pR>S3CiG> zMDg%;fn&kZOc06U4M%vB!KA2%L5=E8A(3DRSCHfkN4R?U1G+De7YACAXzdYX4~W^> z+tZs2#{;q^9Qx^B2TTtR<#-vvmB83$a3uimE#U+d4BY<1dt4@B^BI&=l(#wV88%4 zA`qk+g2ZUV9uF6)JJ28GOMlO?rY-5Otc8C~=C?(stk)f<5jJ-hOaY} zHBLYS-9RHiH}Yuc=O`^lAH5}2B4G&YUsaHk4zlGvoHfX9o+KbR!h%ZjvVtncjsdh8 zfeB&JzbCB!iWPQnJQDf;qrm=O`@w*PzyfOq$?tF_j1n9V5$0jsd(-f%^DWa3E0tg@hCUEDo;(IwB~+6+uoN9=r`W zFt{RAaA-K1fP;Yo#{*l&;Xo(g^;86}fT9u*014{Q6?%>V1$+&l2--t^53&y3@!Dbh&s6;C`MFQb> z3&zC(-+>`O?XYkhxGDiLSfLDIM9ZslwZO@SDO7m zBVnN%B%~rpY-noG6%^WfK*KE+7!FL4))Lb1zZD?hQWt12W9SJa6_6Zi4w=EP<@$96 zO9qYxf9nCO2$(?LFeE-?2Y-kP5oyK*dV*U0(+eaN5(%R{Tke1WT+gop8N!kcKz8tF zq3Igf1!NPy6`DOjHu0;_Y-QOdAba?|KyuOUK_8H9{3_7Q@kkt4Lm(?;19S>O39JcZ z7a(l?UVhocuPdY!uKz0GGDs=rf*ZOz(pb5ed%@+bM4U|ZW zYr1e0wCMy9&InFRjRiF`I9OR4%@PiNL7~+g;b4(L4Y+YQdO0Tk?$E)~)A;^y>S$W% zf?PU!dEljRfu;=(1AZR3MOrfwoR-W2248ZCU=3+>S16V()u0#z;^)#06qdl;!3_)q z2rz3J*%MgNQq2nv+yjk9hJ*E5szCt@9AwjOyn$^k*8r7v91ptkfdkh{d;V|G#4e9X zqb$eHWd;0-pUcBjpr8s`!$4R9Ka|q?fr2J5q^0^&NM4qJ3aogEPK&kRWm~=j;TOER zw0`{IxaH9U;JD>M1AzrCanfQo@M2Ks|9(#eHTV_Xbiq5d#Qg7bsGDEUHBi5!iGRMZ zLTG6KCWNyi90hfZ&_MrBHh{1HWyAbWHt1t0h!MJ~sr@D`B;B$!|B47r=`SLr(7%X~ zo_gLWQH*GaDk8zcqbw4(7wR(1EH5TpXCo0fbajeu@?A>3c`ER0GRoBdQ&|; zoZ$W#1r$<2Q4a1-rTX~pKp?#Sd8Xh^ag%`|077%|a|T~!{oc&S#TD)ZK^?NouPy*9 zZ2%8%vNrSyo3!>0G!l&ih(!^J#A0zu@<>G~BvMM|H_Zf4NUkv8ER|ru>n{I+poIg* z?h5~h2BAaf4^CdDp+No;u&Wg`MF<70q+x;bSJ0Gzvs+0c0QmJ64SI3^XbW!eEBiwt zq0`+Jd?*}1iz{eI^j~8jQMkY5L!&^H__HkvMS$K6=6;>Zd<9OU9w&=f(CTR~Gq{bjpIMJxbYEBWwH6#271usi!}TqP9#ueM4MAY0K^ z2?x2@Kl@XH0OblA0R{Q(Kly;^{>_K~+cpSF(9dswv;}+1zibWcm%w*FEBO?maJ!Fn9t?LPEz>7xsSuqZf>4 literal 0 HcmV?d00001 diff --git a/api/plots/exp3_staleness.pdf b/api/plots/exp3_staleness.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7a06e89298997068c2ecf86af5c5e7ebfdbd683f GIT binary patch literal 14389 zcmb_@2{@Ep^my4DTSyY=l_cxz!`QNCUxsAQHkPtwhNKOZC|hiE=ic|c_ug~vIp>~p?tRaQn`-IEqUCT1@q!mnekB48 zp&(!9{Rm}c2x)QHj|?F-ov2P;zU~mx)XAMpfiR#02~t%>kbPX?jtajG(DC)5LReY{ zq?L*DezFS{l3#k%4x;Whr#ex|5N-)z>O`fI{e2)j{D?rBdpNmz`nW@c<+uL6F6Lw^ zWDC%0=>Ss5K~xB-=M6~ESgtjfYdy&RH*(N_E&!$*rrw_ns9Pc*Ng`8x1N>dUdSLni zej~E0r;~8?BeT6hRLQ-{mD+=h@g!26ay0>z4eWgKO{dyaC`P% z$-H}7IWGI2Ye}MJ{K>&6&H~PJMzzNm3I!Y5s5OioI@6n@pV-eV415{OyE#2)^kwj< z<&VSu97QJWcP_LO!e47-GDCJF6m}+wZ!#PoyvTNH7pP@kyWduGD%O4PPD4|$$8(Fk zr~{D)yLDKG3n=Jty8}Vv?J4bcJD;{Df1LjAc_olDSNAM$ji{}0O}BMSn})D(f%F_( zN=S0*QQgohy^-u6B{rAw9MhM%ge?%L+FNTP9lh@1w&3AoZ#cA%NgHrvP2F&h>DE`1 zz-Jg=@%7GFShLt{ge{vt_992v3(CxpUf8hRV8kM4s?`1ckaz7)Ql_m*Zt-G~9e+$R zu{iOjqNq^wC_2gM+nsr{^Ovp+?me1P((j?-g)K5wb8mVm; zEHaN`GhdGkux;zh*&f>?_BznYyW{m|vw$DRDi@SDKNLu%uhmR$I@I&3+_(L3bggc8=}XS^ z9R#h|Z$*jZIbqE68pMf8yW`;j6!IP`N{O?`|ned7MoR1 zclfCG*~U6-A5ILPwqcx63u#iy@1=CLevC2gEauK8U30RK`WUVyQaC4kTT5Xxm)Idl zKdNsVkCT_=7mn|yQUROwbN}+uk9vtA%5IW7z`r$n1DCdBgeCvvy?)XMjw5DL)QO$^ z`izM)s=K`j&S#C2!guB}n(aR_x~I>5SEljoam}md&y2L2`s^lOa<9*tj_(P}<3;W- z`6l$J_i4=b{!M!EgqF^a=2uHEe9kwxldk!IoOZl_=0V}rh*_>()}t+3#anI`9OLd& z&9_lKsQPUD(}d%7VzPE-`OCoP7I(rc;*nQkj~v~v=DRpwWs_Ny*D4g3GS$eob7Y;i zA?d_^)iKuc*8PQSoYG=_$2Z6Oefi-Qcsq~qoVnvb(+~cup4=zA9L;BKzBs-tmoX~x zn84O7uzRzxHN&QCc^@ttHQJ`7SvMl~m!YF=b{J+0+5ZiLt=ozXuYd-d{699kQF|3< z4aC9JbxBg0;q{`bS>m?72hp57StXfNiHnYB5IzU3ILefS3+JAgryd&-;f*(?3}{(l z#h;(b+4rp#RhMejR9+;%_on~-jW2}H(OJOnW_j5X$rJvYFf3kv56ZN-Yz^<#@b`%K zI>`vha62{Ch@@6#zSgy+ZH8?jGMWai$&n?Zab#zw@JARzNl42!D;BlQUPfGPGr~7H zZsOM91q7Mt&_u^N5>~-1EFQminWn2c|DuWP;mPIpF2s6cgBpvjV20D0oQLTY$6)_E zm^**1Y2{zv_9qb-*krGe7#6n*D)1zwzb>e=jXf|QVYp=dvh{FAbIu%Pk-(tDFH@|+ z@D~GXT2Q2Dp4rVgaTAn-#MGgC%F&sPRdF^`7CBq*5WEL}Se5QTB zR-Re|;i0gX1oq7aZKk&ZiQDB53K(sv?{2fARCSq>hVECLNl0**O-Z<3BQ2UyZ&Qob zG_ii~dQE+uMPbGoY(ejRl<{Mob%*v>-jn|N<#XHisD9xx@4c+HSd{#K>#a(>fe#~^LvvR3wA}idZ+(|O=j^OZeR-l^LVWbBaNkDl z{mGSv*5)E{xP((DMyLErE>0_Xb-KB|w#iP3e3v-f#qT_keYCGl$KG0^{=+FZ#flL# zq5vzwC@(W#CU-zCh^u$UCEKq&yQ;e`i#);0t&^wjy+W46x#WF6@z;@sD9r2qhW1{h z{3BbH9klk0jIi#G+~$%ZFg&QfZdAY4!zLo>kT#?G#B^E9k#!UOFZVayV!e^tp(N>6 zJVqtS`3Rg56=|BjKK1U(vjZBPbH&o)GrG^hvHlVh9u4oGG`)@c`7)ZxAoiw*=;zl( zK0R`d)#i%Y&aTG(1)k^j7|)t+czn0+9>@AGUfP}4Ca;fsEO z#Kp_F5uTQ*Lx~5D2Jo2_l?++5i`ujo@@+m_>=mT`qfx zi@l_l4Qy)S<#)uLUBnoW=s9AX~Kp%_y;FTqjRlPIwq?hXc z2(6Tz?yn;sKHS;GHGcP_``bOD9`*y*yj`yA=r=xoz-eR@&o-~M&b=HrvGM1L&U^QJ zzjGUlTJX?e7<>#^!7zF3fAb4VAOL58!ZDrK(P2ScRkJ)RDxh1_)ubhw!oHo&88`#r zin0}0Z|*oF&Dh4`dHOnQ3G^XaHabg+M<&9HW$)%S*O~;1=nw=SL{<=l!LG(Jm=NQc zL^)aypH@Zf+0?K0;M%ov!n64kmGNoDaZ{qrkNLKw1*M6J6}vU|+qgn*ugwkXAABw2 z=F6LXEwU}~9J^EadB2{5L7z)80*ir3omQ-qgt@cU4(p%Q8uskn=q}gK?Bbd6h}i5C zWzSa<9Z-19+-f%}i{livye<);6Z$yz91lLU1Hb4O6euM*Dl zDQ?NY?sAfMiK)>>vt*1?ud1-_Abzh!oT3Pk$1QawBJ;Mze89#?UXnVJ8MIO6V-vwv zB3@+(dP5>bHs@w0w(Rn5Nkk~Rbu-EE-hJ)LXQa?trALYqOvHQ;xf;0bSb=iW6~z%( zqmBZxw7$nC{b_T;eii$kyBa4oQbpS4GRitm68S4<2O|lrM$ykl#%K9FrI4hmEjAaA z2k<5hY#QFgo8o>#@RjT0JnPRhHrwcMkWTQ3CE{1XL6U(_y&7Bp@CLife@$KJJWSd% zQ2pKafF|jNRCOe}0d=e;?(RDSd)|Ymn=2e{+A3c?Drakxb~vKyT(r1q`HPeZ(Qp7;G*0rz9CdOtl%=>+#a@k#kXV#T5Yk2EEV)&r8Mr%CLqA)JTeLLR+ z{sDJqVaMU_TWfC$T4*I78E=fZ(Q!ShTI-RjiP5*6$%e9)4L;_A`xX+To*bnzOWnXt zCr)?mDeYpniagBwmAIfgH{9bT8m{QmGadPEE4CiP+u>+Hbj(T_-2J-WI)LZqmb-U4 zsG(bvdi)H_#*R0AG5JaDw`mWk_GimhduOCRRmF8vr{vKijOxW`$>406;?3#RI@pI! z>v?)+>zWuh|M0I~kGpnYTbb3i3%8xMqR)C!RPHhL7*5s`d+PXVy(TaAWZlAIr9{J? zSBCE&QwuD;|JkoRa~F0b+nbI|>BQY=>?#6pqgZt=Ml>7a6O; z&jKH_i!_ILnX!b5p9wl6^rpL?`;|6<-+eG+A4$?#G}Xi}6J;(OEqQ3#B{t+>;(nQH z9zhNU9Y%EmvWNI&a|=@v1by#Jq|EM4@;=Y0-6_ki(d!jIj|pWxx+!&cz@TStXeX{FD2RT@Qv_`A^EqXoXsN*o##3^R>^20h? znUi)n@LRF!ySy-sRlE1bU$*%LGq<0_iRWDw8Oe_id7jtXBl#kS_wXW@`;NHlhr3-a z&a%6F7gPR|N-UkI9!*#cnQ@H=nL*q*na3cG3PF4+X#C7Q_OmE_=COM4CJhc}liFgt zy$7ABb+Hw)Sk4o>1t%K5VO*-8T0s&cj2!19^Sg7t1=b7Z%?v$tzIi3v4ym|}w=I4! zUrp?&;;qJn@^b7Cs>9ZC(Mi5VDXC!}r?ql<6KeCMt)HI6_L+T1&x)0jJ|MFH`QB6Z zJo^2ra@R1%1NRth-rP&dS9w0UOC(Tnn$BYB1j+KN2sN;n3R7^7!mZ^E-?#(u-i?KJ zw!42L@~(!QfG)?QmbIlEA!XtjVG-^jsgU>jdT66oNm`_!)|QO5=UZRvGp-Tu-$>`M z48oj0Jb}V0T)9dckv9az#@uzud_n9~)3(`z!smh(Em7SpBKg6WMHNm~J}i9W*iuR5 zn=~JM+K|D!?s|~EQF0}F@RPFpPwsoXt{Qu9@HBE)d#Auf?y4<<19VtLCoIFQ;y6K< zRmtlu6eE8MFukhcNUHiO&KNG~JHG3&cEM#Fw zIYFO{zPm`^>~JLnrdD7;)9tS1qlbu=N9WY z+a#fLp>Tq0HWD(CFyCGO5$c;_m54r;lrQaf(d%os0g8+1hX!Bj&^fg+j?jj7^O+&u z@xj5s;ZR54!#uiu>zmGtZrTzSc+qb0ZO{C@#e?BWowo&V3QxX0l-d*6xX(GBpR)Gc z3(4!l2jpye1r=W=U>k?7eh=@s>zGn?(r|KqS5N1+pNt}sZr|vzl1^BLCaglcwBwX` z8AUlXj~Hi(&51FpcDi9C3A4T~&%8)AkFB7SReWJjo`*$_gq|7$x}N zd54$+LM(FzJrYUB?$&!IQ9pSQDAMeOd|TYOf)lR@US6-r|MFw@kpI<*j(JVLFZ;ec za2_CDhzxCR*}~TT!TiNMK`FbkviWCD}MoII@!TGv%vta4LZ_rjd`JYQwk zR)peLI%}mDQlVF&HY%ofqnXegt@fukTh#=Ayy3p_oWa$_8QuA>+qySulcexz@sh}Z zL~(2Lmx@P|A8{QHf9K;$5SP+MCKQX$b|F3;i+sOfcHn7psU|MPgLs@BE0rg-<*@-F z-A{_I^74y1>^DwtDx>8YkrG>|E61_79P)4E-o)`Rjqltx4t*n*#hd=InGWZKRO>!z zZa5#$^&Y=y^<$dCS!5n7ao)%G{#%EP8(fS$!tv?VV)j-ga=bjAcy*)B8XMV=stSjn zLSyle)A1c&!G&Ax z%Lkn6#eMEb45-*`S(hg1gRRqKxx^t8ZhtUzCK<(Ux7#%4;`kM(r}sZFMSr< zcxk-gZTQ$)293>bbV#G)`Z3s51jI(B&E7zI->VRKRLzKRl?dxOC$io7_U46(jmKSH9CcW1QqU?bFp>RPyr*2SebZsm1z2@e=(uyT5!&aC*qTI)&(z z1Gi&_ZyY&geI;mW@Y!R@&`}l4L$}g?2Unc&ora(V^)I7ebT_sNMCsg?z23b)Nr4;1XWpH3F#mjLA|H#U z`|iCs_S~kHU=4h2G%+jfUY(yKV{7TDoZLcEH8*+f2!s7|#X+2YSj?K*S39G=d}GO5 zaFD0NH#!jp8jt#KW2s_7>)D_zWaJulsWD>9 zorLBQ#z~TGNx;b3kL&krw=d;zDJiVJ zCIgSHb*x9xE{6Yom5ieon&4M+u8eKv1k2N$Ob=sc>gv3uY;yMU8S3Hz@oUT+si5j;Ff35w|cYRNXFCQK?5N)%BAbFmFgq> zF9>?h{g}*=x8L_72=(+&@qxS3KfM2HmWL&jfoKGUL$Su|gsUZ=kJNxq@@pQoy$f6S z{`XV-NomeCZ^}4R>7b?)cwlg=@C`<0RfXWTa7^X&Mg?$Lc>l0mP*6R2k&u(YbVEyX zgI;r;d8KQM(=mM=-55D;@685YmCEuO=4Z}^-8c~WAu&JgYMZ%?QP<~});Hg8dVOS% z{F7HR5e^LJ&BFFLg>uCy-Eo}jJFvscjC~tA+!Pz)S&N|D;YBkEM%{Mblqkhv{IpYQ z?|0-i@-wO8Ak@WeyKbd>T-#SE95g1$s`2baQdO|15!*yGi%Xk~-Ju(|F0K(aq^3ni zSygj3J{INRbVz3@@au_Ag-V3Pa=9H?1s@k5(iRD**!1bXZ9`(!)6QdN$KKn2+{*S) zNMi2D+7=;u2Rgi?6B(fitJrngaYSBlP+_9n3zN{b3!5!yz5Vdp;tkfG-PxlIIN{2T zMlHb`JXA+Zbh0YD4vfg?$7?;@7{A5znoESzPF+J?mw4TBk&|NSR;Bi-X%134NoP+iY8}!GYTBs7iuj_oZ zcQBudEVG>ksosxo1{WJD<>eF$dznxicI>lpzpE(H$=dw& z<})5r6~l(pcd;FmH#;9&>XdB^BJD20w#=)`qc@NwopKb_&h6Jub zPS%Wybz1AI1cThJEk?#8YwKI;^)B1%&0kJS)z^wo89P>@F%w=BudKQo-5fGbi1f~8 z-Z3naE8g~KgJN+bL$zD_(>)`OwIXLEqxQ0wUKjD2w&hF8*-|(t*oS)AiZXZ3dQ;5& z-q`ysY&kDo#tTVA{H$A|2`R>I5asZbM9bIC!W-r+MJ zk*g-clVbDbI*MQpx&=C&9;dWMc5=medsi8*QZJEU%dCivdB z_=4j;sfo~A5*47upT=c01jBJ^%pmWtc~ z#*6%0MFq=0RoqC9(Y98#2qsx_H@`kp(9kOS;6@@36qR zuR>voJ|Mgs`++zHh4}PO_CdPm7anll=-a?}kV;BavatwkJ#*UP_BeIEEa^F~{Lmrt z=|}6D^Lam#!ga4#TZ%rNOv4ma;N6*;NV&B`IYS4V!*USn^ZFE1&2*R&;8#EM@m zOREM`|7c`$a>qomo9*#r&zjhvQyJ1dvW-^EB%uL?k7sAI4$IzRiMW@Rd=|0AGIHZ_ z^-q^~M<4sT$-)Zi{wS>H7ZquGJ|r;oxh@Csh)kf_)*0+hG~dW`A?+9Hq2lqW#QI9= z%&WPslFL7zMae%|Ngc ztXywP>m2Te^6Lv6h@V}1!*C>Ybg}5u?fbpGTeo}+X`mwHBpGHnVvtRI$Kqz z^vo%@Yp+~Vq|Xd3_Oi9GvT&YLquRW^9F+;l?5%!xKHsD66&4-#iuufO>0WQEAvFs~ z&&3p<75FK5M&#MS)OBqMJ;fYu1>6f!k}pQ;-=GEED|*RK){om4sidhl7uY=#=Myk| z>LljQwJQxHS4L5hhlX{mQk>#^EH`6$V{oT+MNj8l$mw8G=Z(6l6HV~n@I4n9OAX%> zeH}eck@eo!X{2zVYuu_}dMHuE{-B(M0J1=M<_m#Qz|6B?T1~g+hY$A`%B_S~K?cJe z8y^oIX>XpcwQ#Jz(Cec{c>4BLb;RKJsG{NF`31&(jokl&8QW)ehOPNto$Ia(qr;Lp(blU=)w@>r>xv{aJwAUbtEcA`hxIaO)2E^wp(Gb z`Ht!%wE94{tW`>2V_zFDr51x`DA#H3Q_im8ZnR_LbxqlNzXsZ7A3JaP**amSvqEVg zwzAA^pGtO3@!Gv1KPnqT4Ql5d9uK8RiM#NfnGXH1jYLq&T7TPO+&}MZ>A(eP zKmM78Zq130%(umqU(s1Mo%mIL6_@bX+c*-4UkM;n!c7Npb;tX^GbP9#t_ZhxeuCEN zl2%r01b4W-Hx*3{7nXA|r(9mErrmJWadY0TGtzKen-s@8Nrxmlajn8C4w87IyF4K% zIIP`W6RTUz-Dq~|hwb<&T(WT{PYdgTp|xReqLHF=*+{`tx7oR+zUxw-X>q#qZ|AEk zl9|fO;P?ohkh2ds3uw*FcOgf)0Unf=3K`6->jM5xKyn%=j$-Pqpn) z$RLyKo$Ws#v`2MYiafLnFH00nnEN%d@n5BGq}?a-8R(cwtQd8eR^dixD+ zzK!0U5;^z>;CkX^%Ey5w^W28yO`$AnEfs`&M|!7{R7`esQ~5DZ^6{;k*70d?Q=X5x z`jl_X2KN_+_eld!!G6YSr5`BcHEZ||wlcV8>L->CJ@PG?DP`IF?dRIe-4@Az@&~Zu zpRKf%MM+DIL~1xu$jdDwGczqqX)W@8C(8hHCm#yzNc<^OO%Er3Fc@j*wA{wv@dzY5 zMvF{w@%Qwj`uam?IHn~*Nb>+^DlJD7#sM6wv6DB*bktZm_0o8arR+voGzx=;WO3jQ z4J{9D^3fRJl0k|ijKau?>hDQQfs{j`;C%u9|8ChMkT7wkATtwWKGO0u5lC$xkjMz< zZT^!l`U`|WYI(Z3kwMlaoMveY!C%8EeoihR0~D;!4^GGQBD+zSoBp2e9#oKa2_DJ* zp1!US4%~YMll^@mq^}QH6!j2jQ9b<0WCYR;q;5iZV3{a@SUea)rVWNKSY3R*e0?AS z*eu#$Nf4a!WC+0*MCK3?-1|B~NN3OqLb`%M5Yi2SMuBNOAfzYAtQy7uKgeR>k4~CSe!5G%K_QS*D@E{!k zje;4VfCB|gL{xy}<-s%1Z9G6O4;MTJ!VvKYP#^+OPCOn=0ydjGP-uAtKmZD~;TnFA z1qEa*%Y*UooJ3&d2?|RvxTY-zE02I{mmb04Ja^V1Y|`aR4tKSOP2&9*2YQpjH6Pf@@mA!2Jj`Ccrg}pC&Yzfq(|U zPy~z<55~}#@{2gQrs)ri2?sx*U>3n-!%Kr}P-yai(Jd7O9xRbI66Wte4M5=16d15# z_zg@Ipd20!tHCdE{i=b;z%A-;GvMSwCy>qwlMm~`zet6FG-U!a!K4131ttoGLeSnV zPe24li z(q8yO;F^-wqCgt72)jnw2-qirj2l`z04VHID-i6d zrB)EwE=w(%BgHSF{O6||@bfPxr31Fc67qlE!-V~Mua5rR#sB?d4y2|!=!A4}g3z$P zK&oT@$8O-td%4?ho&`p^jN%`>zbVu|K$ru6^n{o9i^3&0yYwR!q=Ba^9CLvE0&|D< z3yuKb7HLE-=Xiqv;-A5q;8z%MWVI87Kz?4nR4-3wC=e@$M#;&`Kps@8A4L&~^#1#u zoUgyTBmxOyA=dyG@T-pBhxxg>LC#Jt2LPI1Q-H2C1fIS=TJTR`c4{eNP~d!u0hSSk z!{HTVQSuTfl!WAOoC)$LyCHxf!y|x2T>b+N1rHR|4f-1f$6@dvk1ee3-9+8{V=GNI8bOTYGpq( zfB}L2Yb+X#0zvEw7zz)P@K?YvSXkiy8Vhj18*&8<2aNPe7y;g{EBeWU;Bo~FgZ_i} zK+M4Vt?Z`&4E_oj77dK*N*El0tb`HZFD~%k-!eyGi6FpO0mI>7llSj_I0F1NY6VOI z^~XB!s6W>W9Q7Zx;OX!Wj{{zI1r890{INCy>JR-x5dafbjwRqg@Umh)f;n zm^|!aR=|iLa{4z64U~5!E#PEFT)AGbWnjnfYd)&KlcyKipZ4=UbI)KhurCmj Date: Sun, 7 Dec 2025 18:25:45 -0800 Subject: [PATCH 40/41] updated learner-test --- api/learner-test.py | 178 ++++++++++++++++++++----------- api/plots/exp4_fault_summary.pdf | Bin 0 -> 15402 bytes 2 files changed, 116 insertions(+), 62 deletions(-) create mode 100644 api/plots/exp4_fault_summary.pdf diff --git a/api/learner-test.py b/api/learner-test.py index d9865db168..93c23e4a5b 100644 --- a/api/learner-test.py +++ b/api/learner-test.py @@ -339,51 +339,86 @@ def exp4_fault_injection_sanity(set_value, get_value, get_value_readonly): - read via learner, and - check for impossible values. """ - key = "EXP4_KEY" - num_rounds = 50 - - impossible_values = [] - mismatches = 0 - fallbacks = 0 - - for i in range(num_rounds): - # Write monotone integer - set_value(key, str(i)) - time.sleep(0.1) - - # Ground truth - truth = get_value(key) - - try: + num_rounds = 100 + fallback_thresholds = [0.1, 0.01, 0.001, 0.0001] # seconds + all_results = [] + + for th_idx, th in enumerate(fallback_thresholds): + key = f"EXP4_KEY_{th_idx}" + + print(f"Testing with fallback threshold: {th} seconds") + impossible_values = [] + mismatches = 0 + fallbacks = 0 + + pbft_latencies = [] + learner_latencies = [] + highest_seen = -1 + + for i in range(num_rounds): + # Write monotone integer + set_value(key, str(i)) + time.sleep(0.1) + + # Ground truth + time0 = time.time() + truth = get_value(key) + time1 = time.time() + # Took pbft this time to get value + pbft_latencies.append(time1 - time0) + + time2 = time.time() val = get_value_readonly(key) - except Exception: - # learner refused due to bad digests or staleness - fallbacks += 1 - continue - - if val != truth: - mismatches += 1 - # If you never wrote val at all, this is impossible + time3 = time.time() + # Took learner this time to get value + learner_latency = time3 - time2 + learner_latencies.append(learner_latency) + + if learner_latency > th: + # If learner took too long, consider it a fallback + fallbacks += 1 + + if val != truth: + mismatches += 1 + # If you never wrote val at all, this is impossible try: int_val = int(val) - if int_val < 0 or int_val > i: - impossible_values.append(val) except ValueError: # non-integer, clearly impossible in this setup impossible_values.append(val) + continue + + if int_val < 0 or int_val > i: + impossible_values.append(val) - print("\n=== Exp4: Fault Injection Sanity ===") - print(f"Total rounds: {num_rounds}") - print(f" mismatches (learner != PBFT): {mismatches}") - print(f" fallbacks (exceptions) : {fallbacks}") - print(f" impossible values seen : {impossible_values}") - - # In a correct implementation, impossible_values should be empty. - return { - "mismatches": mismatches, - "fallbacks": fallbacks, - "impossible_values": impossible_values, - } + # Rollback check: learner should not move backwards in value + if int_val < highest_seen: + impossible_values.append( + f"rollback: was {highest_seen}, now {int_val}" + ) + else: + highest_seen = max(highest_seen, int_val) + + print("\n=== Exp4: Fault Injection (Timing-based) ===") + print(f"Threshold: {th} seconds") + print(f"Total rounds: {num_rounds}") + print(f"Fallback reads (slow): {fallbacks}") + print(f"Mismatches total: {mismatches}") + print(f"Impossible values: {impossible_values}") + print(f"Learner latencies: {summarize_latencies(learner_latencies)}") + print(f"PBFT latencies: {summarize_latencies(pbft_latencies)}") + + all_results.append({ + "threshold": th, + "num_rounds": num_rounds, + "mismatches": mismatches, + "fallbacks": fallbacks, + "impossible_values": impossible_values, + "learner_latencies": summarize_latencies(learner_latencies), + "pbft_latencies": summarize_latencies(pbft_latencies), + }) + + return all_results def plot_exp1_latency(exp1_results, metric="median"): """ @@ -574,30 +609,49 @@ def plot_exp3_latency(exp3_results): def plot_exp4_fault_summary(exp4_results): """ Visual summary of learner behavior under faulty replica digests. - Shows: - - mismatches between learner and PBFT - - fallbacks (learner refused to serve) - - impossible values (should be 0 in a correct system) + + exp4_results is a list of dicts, each like: + { + "threshold": th, + "num_rounds": num_rounds, + "mismatches": mismatches, + "fallbacks": fallbacks, + "impossible_values": [...], + "learner_latencies": {...}, + "pbft_latencies": {...}, + } """ - mismatches = exp4_results["mismatches"] - fallbacks = exp4_results["fallbacks"] - impossible_count = len(exp4_results["impossible_values"]) + import matplotlib.pyplot as plt + import os + + # Extract per-threshold stats + thresholds = [r["threshold"] for r in exp4_results] + mismatches = [r["mismatches"] for r in exp4_results] + fallbacks = [r["fallbacks"] for r in exp4_results] + impossible_counts = [len(r["impossible_values"]) for r in exp4_results] - labels = ["Mismatches", "Fallbacks", "Impossible values"] - values = [mismatches, fallbacks, impossible_count] + x = list(range(len(thresholds))) + width = 0.25 plt.figure() - x = range(len(labels)) - plt.bar(x, values) - plt.xticks(x, labels, rotation=15) + # Grouped bars: one group per threshold + plt.bar([i - width for i in x], mismatches, width, label="Mismatches") + plt.bar(x, fallbacks, width, label="Fallbacks") + plt.bar([i + width for i in x], impossible_counts, width, label="Impossible values") + + # X-axis labels = thresholds + plt.xticks(x, [str(th) for th in thresholds], rotation=15) + plt.xlabel("Fallback threshold (seconds)") plt.ylabel("Count") plt.title("Experiment 4: Learner behavior under faulty digest injection") + plt.legend() plt.grid(axis="y") plt.tight_layout() filename = os.path.join(PLOTS_DIR, "exp4_fault_summary.pdf") plt.savefig(filename, dpi=300) + plt.close() PLOTS_DIR = "plots" @@ -611,22 +665,22 @@ def ensure_plots_dir(): set_value, get_value, get_value_readonly = build_kv_binding() # Run experiments - exp1_results = exp1_read_latency(set_value, get_value, get_value_readonly) - plot_exp1_latency(exp1_results, metric="median") - plot_exp1_latency(exp1_results, metric="p95") - plot_exp1_throughput(exp1_results) + # exp1_results = exp1_read_latency(set_value, get_value, get_value_readonly) + # plot_exp1_latency(exp1_results, metric="median") + # plot_exp1_latency(exp1_results, metric="p95") + # plot_exp1_throughput(exp1_results) - exp2_results = exp2_mixed_workload(set_value, get_value, get_value_readonly) - plot_exp2_write_throughput(exp2_results) - plot_exp2_read_throughput(exp2_results) + # exp2_results = exp2_mixed_workload(set_value, get_value, get_value_readonly) + # plot_exp2_write_throughput(exp2_results) + # plot_exp2_read_throughput(exp2_results) - exp3_results = exp3_staleness_and_fallback(set_value, get_value, get_value_readonly) - plot_exp3_staleness(exp3_results) - plot_exp3_latency(exp3_results) + # exp3_results = exp3_staleness_and_fallback(set_value, get_value, get_value_readonly) + # plot_exp3_staleness(exp3_results) + # plot_exp3_latency(exp3_results) # Only run exp4 when there is a faulty-replica setup ready - # exp4_results = exp4_fault_injection_sanity(set_value, get_value, get_value_readonly) - # plot_exp4_fault_summary(exp4_results) + exp4_results = exp4_fault_injection_sanity(set_value, get_value, get_value_readonly) + plot_exp4_fault_summary(exp4_results) diff --git a/api/plots/exp4_fault_summary.pdf b/api/plots/exp4_fault_summary.pdf new file mode 100644 index 0000000000000000000000000000000000000000..135adb757fa6f1727aff1cb87469c01d21eb666f GIT binary patch literal 15402 zcmb_@2{@J87k3$wW6GRF2O;B`4<|$Bc_vfHJe2V`Bu$brnWxB*AsLH^43&8nMT4Y? zq6oQ_l=AKO(7j!q|93s#^YQFw@3Y@MthM%f*ZS?f*DGwKrXhur#=?ZlpTWx-U??~e z?&Ej_CNB?1m>u_Zfg@BMNDiJpZg7N=gPV[R+nz!eo?F5b>iL)qUtX!v-O;23fP zgr%Y55f>*CTxRX3dJsv|l;l8ifn(Pyj2uWL7os;D2mOR0Ox+!vJ-prE`1PkmA16~6 z68sRDR!swt;u1uHBecB$2`cNk>UypXxBZPA)SnDMbVJk=T>y1!R zdU`J2ZX|a&YJElx4^O}^9HHR}grMf);1J8n-{yma3~ z<@7Q3`D<)X%`pFiprXhkI(2i;D@j>efe(IuS2W^?P8AzHq9Ej!uNFj;-WdO#i(z`U ztEcW=(Bjyb|ML6WvNM_!sqkfnOw5v%=n14$lw0LP_bY@RIuXbMV?(Oju!RQ_QeC%dZ}5>wId_6Z>`cR2tq*-8?ga zBk}&I$CQ?|=;E>Z1nWeBJtoNe3>7x+t7+c%UDTYf1+3KQ?x@sLcde3ueHaV zN1t%x;{Ksq<&KPE)U)%~?~Ak_Tdb$iZfV0kIBS`ba-( zgZ5~s@z)$1$CgYEv%=YIw-)otj~Z5G6;eKxH(D3mxXM3UWu$~MNm^biC!(y5T2*rP zhnAMh8&{gG+|wSrpe5M)@VQHr=(L&OWcr&&E~vr-g%av67*2l626JB%->#t2ht}sb z8QhJIXz38oHwB)S5R=%MDMZ(1a}js4=JGb-@JXSH8}~YSa!xAPr17XG_Fboo=Nh`~ zS$XSSvC2N@YT;|D6<@?pMB#|BsFgFEaqiZdtZb!J<`!zNo}||LewsLi>8QMT8Kv1P zdZWcJ_CrV{=W@Ra9|up3Tk;Kq3zesHt%_Ci z!_1^gCY`YILdROpwa!r~obY(W=TowKI_o~~8N?4CWg6vYB9UrM`4*Ki=kfRs!mfq+ z*AlIREp8LZ^jusYg^e|=6Hm5ic-m>`WKM?0-jj7;LZha)1p0c(b5C)wdcoh9s~e~(k5iI}YFq-{|1M}a}FFTI+cXKK|(+YG1b zV|h5OL}Syn7v#my(e2fYUZKwMe|)!=_jq9dp|4tP+7We(^0bSJB)* zf{&v!2NdHv&-9GVyeYvB3G`>_nLn1P)?I$(jN^GBDQ&UGU?y_)s`zl}!|vP@H`9(( z+r4xm?vQ@Lb)YGhbAFFB4Yhft_@c`2?U6V$-k6;*Woh_X?L*^Xx2E{`Vg;hiX=gnr zBJyFfVM5+l1)>i%46@~ouxj1k`OWM6gyYhY7VJ zvveHq55^UnZg-c_p9{kBc=G3P~(dAPq$3mkSPS=l>k=+_kAZuEd@7M2`1 z?BP+$V#9nndemyTUGhU-Px1pXXSDOc9lQ`&KS z7PPi~T1DJ9q4&fhOY7*JW-d9z`3@pdr?&i>fCRSD-3;?{uIlsqx88n-4oQr>lWvGi ztFNH8+R?#%hj8F-$7fBc)c)C1ybT}yesC2Zr?bFa>1K?o%d4sEOYtaO&|SH{`V(zs zFqPa4VXNal18my8!nCi6hd6J zNPbw6C9P=KSvP!%Hj0zianSB*lEU~q!dKheY=%9&;o%|X_fzX4-|op~%UsOQ;g_Dm zhhLpduJEgI9-mAi7FN9~ZK=s1a4C33KMH%4pg#Yc#!jK+^2H~OBUBTQJ_gFNEAI2T zG~B_pc-sH=&4#VTw_jYRHar!^TO3m$W37HNzRGX(X!fzYlJZ}kR4+3X7%w=*n}rV^ z%~mZdN`IW7W4_9VLxIN=>*G0>$A4o}jA1rUk zC6SWTz8lJ&~LpWYh*nU?28;2vix7t{0 zsm;5PIH8rtRHu1mvvOlUkI$}PSBcfu`UmcbERMMcb`8JD{>uG>yL|$3j_a0v-A$6+ z0OT4_SXr50zLV@fpvpsV+rR4|hq=LrpfNb$IR3+d;Ec7sx1hE+PI3NJx>6}RpHccg z@5k!R2x9~_^^+%i=UX)PDz+!g?Xx)9`ySi3AZIn%F{txu@^%!gP`nl9K{rS=$1vvb zW>BSLK3cgdW^l=I8R@tC1x{kmDMxwOX!gBB4$Du|*DLPybGmAE%~5M++~oG1$BVh* zy_%24CF=}EM|8&X-Db6I0__aFJPz*+;EYrr@j4?7@tksO=Y_m)vF;)z^Q&ib zKHt9cbBn6JSv>{e{tiOc{jMzPzeT1$(8$ybXQUa*5tFBSxvFTKA~?w# zaB*~qXh-9#LV5))w<+!N)MxJhwEG(4HS87~HxKQfG@&8TvO&Rcn-CV}26mmMh%^c` zF@|6*J(%^>nJu+6Ky%+xrX#kx3(H2x(H7k3Y4*Eb$JC76t~#&Rvi+4Q)pxUl zEImITN1P09VL8-&i=K;dPr=jhRBE5lD$xLYTRUn_6)qm<$uT+Z7#&xxyO|;-><{+( zJy*+<)7QynsTm%789qh*`5oUZ1qvYuy@5guQf3n*Ch2-pqZn0}6vL%=G=Ce-`KxeW z!^P)k#zckRo!>pmrG6x>LC?yRKLMK@cIMrJZ&mi9oadmc>r3nW^yoJ!Q$w7NbNMGn z`!#H>L_6Mxx$du;q^;%%OtOiO9wGO7RfpEh4nzw&}IG+oHV@*$)dnT4C_?- z{mg?CD;Ll&kLcNYnv|W`BX6g6aC(|fGg{CooqK9Rnf{$ltGjhn>M?a{<+;V0-V^k5 zW6zIt-=HhGcu!8uwQ`1JBJIr`DZt;oSiJD2;K@;y!LOAP!k@IBL|}-bbMBpQN4j4p z{CpluqZ@zCUEr^m*S#M}+c%l+S9f$aAeMW?95h%m;ux-PyUn=cv!}YMn1ABK_lMi= zF&s_vfi=*fkf}RA-}8+4rDNBBs$N>(Fuw`$sc; z4R3kdw7Ill91gvd6YrC=Q$1E|av(debbc4%xwHw6wgoR`VapAdD8)rusM9-uUw zGH3E?m*=5Sj?X&fqS=?R)2zJ<$5M`-3}83BUNvbpAYeUE!M^K!rDu@xNY7r@n@lmH zY5oGn%cZa-$n}XQnQ{dYlFJVx^om zZXsmpb?u!cykGoZ++V|wy;zlCKJ}PN@yr(8pSIg52oKuKZ4@2`{olfad{;Y?vGF_S zO7oUSb4{Whmm|NdJ~rv)*sdg8drR2SNr(!8`bvm;_B9DtAadG`Xt5$`7_Qk`@lu=P zZ2Bv^z|cP4PL~7Bm_1GD=JyIalNr0!@e1;+=qJKE_9Gj&1w3(m8huCLRke`GfJWNz zW42EZ#A$j>YhSzzW%DZg=KsB%!Rz}|c7wnZ@3r5sM@^-bH@i%5CU;nFQ!uXAMV(4` z@0ll>*R(J9oTuWED7Ex`ZZD%-TJ{Yw&(?o%dwo#A-FEz{ms7EZPS@~VCVl;+?aONP zZnfAsuAgTHfs_2sVjy6~NJim!-QV5-CC5zs)1On63)`K2|C=Pn=aU$`i8CSG&2dQy_>U`q+et z#u*ZlXapGhj)yBE5AGaOx_k9%E&j>!nTDhcgM7bI>q}F; zj=NuKSlCOKUh)g3#4tESB>FxopYYC!<6aF+9kisI$A3LJjO6mZCeW5->mRcPaOK~v}~eiPR#t5G08x}Y33OQe5?TH z)K~G~Q2e?4+2iw6yNJrGw&P#<)f=aU;k@)*kBlAXvfhLpz8y&7zqyaiUnNsnnq6Sv zop{lXOr!GZg3Bhxz9ouRr^|ECyg6(mML}K^kOvw|`0o`|Fp`T?6JVSngd-OAE(-bb zYm`a3&cC=uohfFdce)m{cZl_@CL=24`h;-B2l`5=ELV z!Y8ut=gYzzaFB6|YgR{X%X&vDQlJwdd~bz?`SZHWnrn$hmkP$c$Ha-{h@Z#};*$K( zjXxxsq%aA8WnvQDb0s&W_kdS#3T(gYeHux&`j^h^`m%kE+9nrxQqb@Divk5tmCJV* z?4NelzgI4lF*w)dSvB|05jY!`CZh3l`ms-^XP4ML#1STq zeAd~g1K3i>cTVkOOLsfN^TK&`neJz#wIBrvQh-Pp!Y1&pf{Ctohtl@3DGr;<4;C&B z9yd8S-t^t)sH#bccvCd06M3pPq5h4oEt_9>Po3ShL-Iu@r4LzW9FJ;@i4|6?eU?5a zP-{z)u9D@`Tx39Ji>nn6l-|$#s*)B?mte4v$IyC;@N)tY((<^H>DDU+v~uO%w_+-t z-o`wcY|5+-Q;D3#g+{qntuKt{W|q*jM|>_^VQ||0iN1zb#Y?N5>b=qywb>N2ii9|~ zz3g{6$K4!v+fUuUvGp2{nOfS3*{-OPd&L);)Y=sd^}p>)(~~mq^fu)=ypnQZFl1=N4|PBGpD;h8~!AXSq}s>5x3_nz0p@*Xk@;oQPtj#R?LnS3(l9U+?Cm+ zfoXBr!TM;at($t+4`R~}?A4=!HI{;xZaS*Pp7-=uxJ~m&Z@z=@sExhVb3Xe~-VF>! zTp;vmL&T98rNHVtfBDwt9>7fJd;Phl7zz**h1mq|)sI(Zrbcb2oQh(J4N} zFXf2jRreq}-Fy0N+)~HbrLI(@C-eB+noD2OO!Z1+QXiCJPcYlt4)C+3c!`H8Bh@0!rBbqPP1yJOb>%SH55q(ZY_2P`lS67^IRA$ zL+x>mJ+!GJcAS=UI!@1w;+1Z{B1-i!@yd>}+B&NAA;)6IGuyibgrD zKhNF2GLUaP(A!Pu-jSEX*>K~;2=nMdu#Hy3e9C~p?!Hx{vyv|3M3}Jhp`l~5tS?yj zu7yg3I#eE&KKH=|J+k^OlFMfP)RJSDbO=}Z)SVXQEyY*aYiPT9p3dRL+_xNTh|z8> zSPb?K%=auf_-=Xk77R0z*c>H%i0{t*=UdWF*-op+p8p}CWNf9h()}>7AO6PCC-Wj|g{Nydd;#ySZ0)0S(Goqj#(lttV_`s$P2uRn=R(nA`Iz3t{tRhFf~ne7Uo0~}uSa|uRu0~&!suw&T4`h9=Rj(U zuam+sozdi(>->gxYIHGoy!2OF9n56)tWi|l>G3YuYu+zhCWsBevfxUIzM&9_|DT%IE zk>=K7Z13G#%@|T6oD~}77IG2pwW9;hrB;;@&7;PbwKcKtr4IEL;W4fr3iMDwrYOuN zUJd;?V37nEspp#n_b3T^YIUEewf^Anef21xNokHe`iWnicC0w>!{--*U>W#eKfgQy zjd?b^q3Ysq+xCtMj%}qn$12aD$t{=MQ)g;AxG4A6-o+NBak!Lr{62;5Z6sPz#RAsk z_ZM|{Uz@i*X5W9WO1a)xUHK>7a$x!lb=gh?6$M@>fJWI(fJTLW0$T`jJMB=8>55QT zx8RcB?wFuebL9PP{AIzH1!T`Pv{byZ?`?95`LFAmbtPitTZ9;vxAa>xB; z6|qT9aU4gU z>RB9cu6=imU@pf?%b~vSV$Z?pDylL$)Ao#`(MM*STqbV~%Wdo6y=O}(arP^?5MUM{ z?yEPL`q_AVInXNQlqRza>`A%*ZqJv!UTr%?H7-@mF)u~K4Mj~gJ3hci7wAM|Po4xvz_4rum`5esMBJX@>D^=YZ*_Nw(REiNL9|_CCj1wMKVzCkpK33k}S+S$+Lz z`Sz+`gxugwo@=}3Umv^pD6s3WV-lzT)|h8v#ZyP6tsnF3f1Zr#nk@Praj)J!z44sh z{PKZEgWrBq^NYEDqaaEOa14dtgm$SX$gzPJMfHS1p3qkzYQ;fUv>1NL$LWcuN%OEJ z{9Ge@=*ZL1=;_eNTH$fp`3f30rmN$*KAdk5tk}cf7v@Lj^mZ zD|2v40ng_G<)E8RZ&y6i?*CHiNkgr{W+2ll}BfBBPFECrZ?Mr}fE6pZR)X;6%Pw&A-h zn}a`;xN*hk7Oj5LTK*z1X@{`)EzxlW8$S9BF>g$p>b4w4$p~A& zv!BwC3^tlZaoMv44iE3Vr-}VsWb)TMOG?gc`Rj<8tyC(zTq%%70qvkMo7j5k8})dB z)%)HE!y%iecQ^9W#mL|IaYnV5J}KQ@_jTT4%AvGgmn9*BPfx=e)kuR>k45jHf$LR2 z3mB;aSB=beR0vn;^xHh}uEJ^~M%!d#(~sVan<_aGW>pZhF!5wq?Cd)Qbc<{CZ341P z;U05>!L82g?}z15_fkhUB>bgv1vbM!BAg#DF?P`;+d%5g&h*^r6DQ-gD0DoJeRjXA zkNbkgO{wDhM^a^3knE=SQ~k;k&!(_%Q+3m{z%pFv?hRJKwZ;^QjBwwlqYs-nSH z!GQ+X5zP7aGtwzKNi+AL?#xzN_OB0?&&+wx2cFt$-+?6GnrvW?0`fp@=6o9$-^T>l zQ=QKYWuO@vydiIR{&FN~^{DVwTE>gW9??;*bBPyJ_3URaI=vD$GptbDD^%B@P1sRl zBtv89uE&VUD%5?M`tFX4d|eQsVuUt3p?loQ3#-O6ARke1b(YiM8^S-iIF-kRxx3tS z+@QNRl=C9@2=T?>tqNiuJAbW2TfSUJ6!8*|$JZb8g)+8>p9LWwQc!yeNMzO9zfJ<`m7M}^(pz@^_C%;B#1u1X`XVd&_zq)w7r3s(}K(N(7? zxqVuCT24t?wfyIVGA*lZFJ_pDSDU=5@#tv1J8J!~TP@Z%b|w1VHnSDO_ckdD`sc1E zp}y@jYmz&6LHnzn`ZH_Pho5T5DxNR&KI$U0VbN73qjv`FBl?$Mk7c|5V$uthnW4Za z1sH?DYyzrFndlNb;EZGbFx$(RuXL00????nw^wVElGQbs?LVR3L`3VHw1U?aJZi6t z?NlssqhnOl9Q-*`S#HHvXA`y%jNw&)pVVR(PJzUUq-@Q z1SUFoo;q-LIdD!|Rt5-^o(M86Y%AIOWFlr&vBO`AM(U)-P{`rQ-0N{Kmy@_WmM(e? zUkk3(lPfK(-0f-TZzpot+O2*+{~%q@mupX0O&X~a$!%u2<&xy7C%v(+TnfXH4>5cpx@P;e%z&)I*hjK%pI+& zjMn3}veiEsO*isXZ`opDA?c9g8l8F z*e#_i3x@apDlLt<1CEN=Z*%G)?I*+#Ru6RVj0lfiX`4K>XGyq9<$3&v%zl%OC9Z0Y zL8IDOhFny-3!*GQ_8EFuk>-w?q4~t@iTejEg#I zN$E4Es#HEjG$+X`YNC2VX7SNp`LrTa{8xng+d1}Ec2PCCWsOB6y*$%tE*a^(dQBK*Jl8jF7B3FasqML_us;@Y7cJW(({)@%z z-N{G_3qc7aZUQ%vZBR%Xhbp+72pPe#YxE(+B`0pOqii+_h)Q z5B)COfAvF1wl}+zLb%!4`p`TD>7&%yoO=X#YCqPMq{XRQDVhbFn6vb}j4bc$6L@x6 zav@Y(JVL-I=HjkIDHrd$kTE>djT@<^SK=yT3Xt(%YCJjl59c2e3`ja~u3a4QY@&g$ z`^uxn=lL;r&J=D%(No}t0(wGk##l=kYZIxJw>J(-dvr zBK?m{?3lH^u8^VJQ*P5P%+9U%&_T$J`9KC*y2f8Y20o=>neLF_ zZN3Y`7KaVj5(qE7RCtd@neDGMZ3NoYs^H4IIu*Lcb+RUn8L$Z|B%3U0WX`Yh5TiW10f44%F zF5v9SB-~eCe#mdHG|74TLY%~<-4o5#aHrSzWmkJ{DYm1O$MdBu(*wIk``P?k(I~1~ zjh<2Y{AQLeo9%4Q>3i-p!v$^Qm(BmOO8zuhCpRA7P~&=7A-_3#c6&$VR*R4y4P9q- zTbJ#IC)34+o!BE6&%PHl!OP|CxM?;^EInU6eo4ZY^V7cR0c&#RuF5|Ethi1dDX$HBh zy>v$>w}!roMF@P&NAQH*WMC2hu0?vH#^lDim%Z(}2&(LA7$XFE~)$2mKjSRt5P8Y(5 zhImUe+al~QO>{gEHaq&2qeDqpraQx)`AXpfKF!2nN7@ zX%Kb~IZ%3XUoo=2@uN9bzhCA-#kyIWiQD^7!8Wb(W-35Eo=jk8DyG^fjrtLhtZL7KL z?#I)Q7g7}rMedV0(IaKJzMb^!>VopiaYYZ=XRNUg&}<`}tmQ|jo8*2V4Yq7y_v@o_ z&ech&o^1E2`c%Em;@i)yxteBaf651l;vbi>7P3r^Lq@1L_`9r^^o@!Pf~SvAcD>aJ%{x&8i#`+pgw9Y{!T;>Uy=_Ij)K(Q1dcEba3qlfqM25m0%64}8;4%&ud)_OEQLa%QE(|NI6y(kfC~c@8u&60PYg|=??57YkmHS|kx0l< zfd9WswlD-loDm3v24TjK27nl4b#D+g3F)3A z-~__00e`+w?6aqfD`~w*^l)=0ff#4-(}n2a;|#|F=N;@q^noLMya7|BV?J;k8isHM zG171xu(kexP#kFPLhcBitU393`gp_f;A~G7jt53S9WVuluCBG=P=J#j9H9>{ zgd=DTLpX!paD*!y0V2uW;Rp{n;s_jZ6prwOBfQ`UZ!j~U295&KMi9Y-fCxAu0FDR* zG6Nh0!4bznj2sA@|6ve%+=2qX)mOBw6hLW7z=to(^vQKyaH{ zKxKa?D4Lvr2CPFfKtH6=&=r@egRj<_n2<1p<*y=;3y7lka8mYm^K=1(BTPvyUgl6i z-yw+15nu?5{ymWWmjgHi$0L#dKN9T!bsP*h6T<=(1wrv}S&S?kkHNzT0IA8yBH?J@ zlrSKC3Q9OYC-fX@j{$LAY<&O%uq?k4 zqzG#|0O`TM6Is?kFCd-xEs^yB(urS*tSjp}0qMi<38IVq8;k+z#;*i%jz{9a8Uk7& z9iWg1vS3Xhy#T)G_w-9Ae&vu*z`Dc8IaK~Fk=J_dIn?Iw1nC8|s!;wghd zu30M`0k6?o=_t?^s00H}0hj@D4=*@a7jnrPyf$kkAD}?%CE%?9z#~8Rg#-IZ?){Iu zBcF8wa>zC7UT=NcU!HG$ZX)Ck$>aFL!HpGp9LVbdZCR^bbEfMQkbtVL)stQ2It2mX zRavVIgyYu58U*=I@(9O)ima8$o)(xXsQRA|seaje$hm8Pmu9W;KhL3Iez`Yg)bAMJ z?@y}`s>+}d!pQ-Sf(Aw?qyJ|!aP|35GtB>N27LtuHH30S#owrfs9UGz9}*#!{-}h6 z`9~!rr(evh`RcW=%MiLA&Jcb9FC8Qu@~2k;z_*h5Sr4TL#{Hkps^Ie}2&L@Or=wt+ zUqgVVIHM!4;y$*#TK-#*YP6qjBI~nMW*w9WExZ;g<1PE1bXh(ov{D0Mf zA9%2s8|t9b&JA@a0%T@3v_m5|(twovf zV*t9^SO+1Tjde1A>I*;~8}%7_YyOy51`9c;f6pr;^QT;7WdD?(Eb0#(Ldrr2V#9c{ zc*v#wdk$F$9d4{6fX_xY)DbYUf3yRt`)4~@fLi|D7XTlbje3d#S1^CJ1E%DUzA^y0 zY#fgOxee&=Z`}YI1fbrAI)LAPlRb&(;Nj^)B!AOu>JjV$Y!e(|;^PCERkCZ+_I3pu dKFF++9g;uEfk+~oF>rc Date: Mon, 15 Dec 2025 12:03:39 -0800 Subject: [PATCH 41/41] Add license header to learner-test.py Added Apache License header to learner-test.py --- api/learner-test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/api/learner-test.py b/api/learner-test.py index 93c23e4a5b..8f6f2b18ec 100644 --- a/api/learner-test.py +++ b/api/learner-test.py @@ -1,8 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + + # import sys, os, time, subprocess # subprocess.run("bazel build :pybind_kv_so", shell=True) # sys.path.append(os.getcwd()) + + from kv_operation import set_value, get_value, get_value_readonly import sys, os, time, statistics, threading, queue, subprocess, matplotlib.pyplot as plt