From cc1e86248ef416f7b88b5a609312bec994a11089 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 13 May 2022 16:57:26 +0200 Subject: [PATCH 1/8] introduce register payment method on OracleLightningNetwork.py --- LICNENSE => LICENSE | 0 pickhardtpayments/OracleLightningNetwork.py | 5 +++++ pickhardtpayments/SyncSimulatedPaymentSession.py | 1 + pickhardtpayments/UncertaintyChannel.py | 2 +- 4 files changed, 7 insertions(+), 1 deletion(-) rename LICNENSE => LICENSE (100%) diff --git a/LICNENSE b/LICENSE similarity index 100% rename from LICNENSE rename to LICENSE diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 2be2969..d4de2cb 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -71,3 +71,8 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base mincut, _ = nx.minimum_cut(test_network, source, destination) return mincut + + def register_payment(self, channel: OracleChannel, actual_liquidity: int): + # TODO register payment + print("=> register payment function here") + diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 1be5c23..03cb098 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -249,6 +249,7 @@ def _attempt_payments(self, payments): if success: self._uncertainty_network.allocate_amount_on_path( attempt["path"], attempt["amount"]) + self._oracle.register_payment(attempt["path"], attempt["amount"]) def _evaluate_attempts(self, payments): """ diff --git a/pickhardtpayments/UncertaintyChannel.py b/pickhardtpayments/UncertaintyChannel.py index b855bc8..518680a 100644 --- a/pickhardtpayments/UncertaintyChannel.py +++ b/pickhardtpayments/UncertaintyChannel.py @@ -81,7 +81,7 @@ def conditional_capacity(self, respect_inflight=True): def allocate_amount(self, amt: int): """ - assign or remove ammount that is assigned to be `in_flight`. + assign or remove amount that is assigned to be `in_flight`. """ self.in_flight += amt if self.in_flight < 0: From 48322532a20e3af5a744a56c61301e41a23773d7 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 13 May 2022 17:37:14 +0200 Subject: [PATCH 2/8] adding settlement_payment in OracleLightningNetwork first stub of a method as described in #16 Still needs to be tested --- pickhardtpayments/OracleChannel.py | 4 ++++ pickhardtpayments/OracleLightningNetwork.py | 20 ++++++++++++++++--- .../SyncSimulatedPaymentSession.py | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index a94963f..c6cf926 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -37,3 +37,7 @@ def can_forward(self, amt: int): return True else: return False + + # setter for actual liquidity + def set_actual_liquidity(self, amt: int): + self._actual_liquidity = amt diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index d4de2cb..0ea1a29 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -72,7 +72,21 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base mincut, _ = nx.minimum_cut(test_network, source, destination) return mincut - def register_payment(self, channel: OracleChannel, actual_liquidity: int): - # TODO register payment - print("=> register payment function here") + def settle_payment(self, path: OracleChannel, payment_amount: int): + """ + receives a channel and a payment amount and adjusts the balances of the channels along the path. + + settle_payment should only be called after send_onion terminated successfully! + There is currently no further exception handling if along the path channels do not provide + enough liquidity! + # TODO testing + """ + for channel in path: + settlement_channel = self.get_channel(channel.src, channel.dest, channel.short_channel_id) + # print("path on UncertaintyNetwork: {}".format(channel.short_channel_id)) + # print("path on OracleLN: {}".format(settlement_channel.short_channel_id)) + if settlement_channel.actual_liquidity > payment_amount: + settlement_channel.set_actual_liquidity = settlement_channel.actual_liquidity - payment_amount + else: + print("=== CHANNEL EXHAUSTED ===") diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 03cb098..0df20ee 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -249,7 +249,7 @@ def _attempt_payments(self, payments): if success: self._uncertainty_network.allocate_amount_on_path( attempt["path"], attempt["amount"]) - self._oracle.register_payment(attempt["path"], attempt["amount"]) + self._oracle.settle_payment(attempt["path"], attempt["amount"]) def _evaluate_attempts(self, payments): """ From afa6a7120e35eded6babcc09ff92774b6d669526 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 13 May 2022 22:53:00 +0200 Subject: [PATCH 3/8] rewriting setter for `actual_liquidity` using properties (python way) --- pickhardtpayments/OracleChannel.py | 14 +++++++------- pickhardtpayments/OracleLightningNetwork.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index c6cf926..61e59fa 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -17,18 +17,22 @@ def __init__(self, channel: Channel, actual_liquidity: int = None): self._actual_liquidity = random.randint(0, self.capacity) def __str__(self): - return super().__str__()+" actual Liquidity: {}".format(self.actual_liquidity) + return super().__str__() + " actual Liquidity: {}".format(self.actual_liquidity) @property def actual_liquidity(self): """ Tells us the actual liquidity according to the oracle. - This is usful for experiments but must of course not be used in routing and is also - not a vailable if mainnet remote channels are being used. + This is useful for experiments but must of course not be used in routing and is also + not available if mainnet remote channels are being used. """ return self._actual_liquidity + @actual_liquidity.setter + def actual_liquidity(self, amt: int): + self._actual_liquidity = amt + def can_forward(self, amt: int): """ check if the oracle channel can forward a certain amount @@ -37,7 +41,3 @@ def can_forward(self, amt: int): return True else: return False - - # setter for actual liquidity - def set_actual_liquidity(self, amt: int): - self._actual_liquidity = amt diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 0ea1a29..351c60a 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -86,7 +86,7 @@ def settle_payment(self, path: OracleChannel, payment_amount: int): # print("path on UncertaintyNetwork: {}".format(channel.short_channel_id)) # print("path on OracleLN: {}".format(settlement_channel.short_channel_id)) if settlement_channel.actual_liquidity > payment_amount: - settlement_channel.set_actual_liquidity = settlement_channel.actual_liquidity - payment_amount + settlement_channel.actual_liquidity = settlement_channel.actual_liquidity - payment_amount else: print("=== CHANNEL EXHAUSTED ===") From 4ecadee08bfa090219047cc9277ed92b53191ae4 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 16 May 2022 16:19:04 +0200 Subject: [PATCH 4/8] adjust channels in both directions when payment is made --- pickhardtpayments/OracleLightningNetwork.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 351c60a..de57199 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -13,7 +13,7 @@ def __init__(self, channel_graph: ChannelGraph): for src, dest, short_channel_id, channel in channel_graph.network.edges(data="channel", keys=True): oracle_channel = None - # If Channel in oposite direction already exists with liquidity information match the channel + # If Channel in opposite direction already exists with liquidity information match the channel if self._network.has_edge(dest, src): if short_channel_id in self._network[dest][src]: capacity = channel.capacity @@ -39,10 +39,10 @@ def send_onion(self, path, amt): oracle_channel = self.get_channel( channel.src, channel.dest, channel.short_channel_id) success_of_probe = oracle_channel.can_forward( - channel.in_flight+amt) + channel.in_flight + amt) # print(channel,amt,success_of_probe) channel.update_knowledge(amt, success_of_probe) - if success_of_probe == False: + if not success_of_probe: return False, channel return True, None @@ -55,7 +55,7 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base """ test_network = nx.DiGraph() for src, dest, channel in self.network.edges(data="channel"): - #liqudity = 0 + # liquidity = 0 # for channel in channels: if channel.base_fee > base_fee: continue @@ -74,19 +74,19 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base def settle_payment(self, path: OracleChannel, payment_amount: int): """ - receives a channel and a payment amount and adjusts the balances of the channels along the path. + receives a dictionary with channels and payment amounts and adjusts the balances of the channels along the path. - settle_payment should only be called after send_onion terminated successfully! - There is currently no further exception handling if along the path channels do not provide - enough liquidity! + settle_payment should only be called after all send_onions for a payment terminated successfully! # TODO testing """ for channel in path: settlement_channel = self.get_channel(channel.src, channel.dest, channel.short_channel_id) - # print("path on UncertaintyNetwork: {}".format(channel.short_channel_id)) - # print("path on OracleLN: {}".format(settlement_channel.short_channel_id)) + return_settlement_channel = self.get_channel(channel.dest, channel.src, channel.short_channel_id) if settlement_channel.actual_liquidity > payment_amount: + # decrease channel balance in sending channel by amount settlement_channel.actual_liquidity = settlement_channel.actual_liquidity - payment_amount + # increase channel balance in the other direction by amount + return_settlement_channel.actual_liquidity = return_settlement_channel.actual_liquidity + payment_amount else: - print("=== CHANNEL EXHAUSTED ===") - + raise Exception("""Channel liquidity on Channel {} is lower than payment amount. + \nPayment cannot settle.""".format(channel.short_channel_id)) From a131eaf1761f148412e700288ddf4b6a6ea9e98e Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 16 May 2022 16:25:07 +0200 Subject: [PATCH 5/8] Settlement of onions with partial payments only after all onions were routed successfully --- .../SyncSimulatedPaymentSession.py | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 0df20ee..031fe5e 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -7,6 +7,7 @@ from typing import List import time import networkx as nx +import json DEFAULT_BASE_THRESHOLD = 0 @@ -234,13 +235,15 @@ def _estimate_payment_statistics(self, paths): return payments - def _attempt_payments(self, payments): + def _attempt_payments(self, payments, settled_onions): """ we attempt all planned payments and test the success against the oracle in particular this method changes - depending on the outcome of each payment - our belief about the uncertainty - in the UncertaintyNetwork + in the UncertaintyNetwork. + successful onions are collected to be transacted on the OracleNetwork if complete payment can be delivered """ # test actual payment attempts + for key, attempt in payments.items(): success, erring_channel = self._oracle.send_onion( attempt["path"], attempt["amount"]) @@ -249,7 +252,10 @@ def _attempt_payments(self, payments): if success: self._uncertainty_network.allocate_amount_on_path( attempt["path"], attempt["amount"]) - self._oracle.settle_payment(attempt["path"], attempt["amount"]) + settled_onions.append(payments[key]) + + + def _evaluate_attempts(self, payments): """ @@ -310,7 +316,7 @@ def _evaluate_attempts(self, payments): print("expected to deliver {:10} sats \t({:4.2f}%)".format( int(expected_sats_to_deliver), fraction)) fraction = (amt-residual_amt)*100./(amt) - print("actually deliverd {:10} sats \t({:4.2f}%)".format( + print("actually delivered {:10} sats \t({:4.2f}%)".format( amt-residual_amt, fraction)) print("deviation: {:4.2f}".format( (amt-residual_amt)/(expected_sats_to_deliver+1))) @@ -348,7 +354,8 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): # This is the main payment loop. It is currently blocking and synchronous but may be # implemented in a concurrent way. Also we stop after 10 rounds which is pretty arbitrary # a better stop criteria would be if we compute infeasable flows or if the probabilities - # are to low or residual amounts decrease to slowly + # are too low or residual amounts decrease to slowly + settled_onions = [] while amt > 0 and cnt < 10: print("Round number: ", cnt+1) print("Try to deliver", amt, "satoshi:") @@ -360,8 +367,8 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): # compute some statistics about candidate paths payments = self._estimate_payment_statistics(paths) - # matke attempts and update our information about the UncertaintyNetwork - self._attempt_payments(payments) + # make attempts and update our information about the UncertaintyNetwork and track settled onions + self._attempt_payments(payments, settled_onions) # run some simple statistics and depict them amt, paid_fees, num_paths, number_failed_paths = self._evaluate_attempts( @@ -373,6 +380,17 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): total_number_failed_paths += number_failed_paths total_fees += paid_fees cnt += 1 + + # When residual amount is 0 / enough successful onions have been found, then settle payment. Else drop onions. + if amt == 0: + # print("{} onions to settle.".format(len(settled_onions))) + for onion in settled_onions: + try: + self._oracle.settle_payment(onion["path"], onion["amount"]) + except Exception as e: + print(e) + return -1 + end = time.time() entropy_end = self._uncertainty_network.entropy() print("SUMMARY:") From 0b0e868234a2072c030f587004f8ef667178d16b Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 16 May 2022 17:59:44 +0200 Subject: [PATCH 6/8] deletion of unused import --- .../SyncSimulatedPaymentSession.py | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 031fe5e..876be4e 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -3,12 +3,9 @@ from ortools.graph import pywrapgraph - from typing import List import time import networkx as nx -import json - DEFAULT_BASE_THRESHOLD = 0 @@ -48,7 +45,8 @@ def _prepare_integer_indices_for_nodes(self): self._mcf_id[node_id] = k self._node_key[k] = node_id - def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, base_fee: int = DEFAULT_BASE_THRESHOLD): + def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, + base_fee: int = DEFAULT_BASE_THRESHOLD): """ computes the uncertainty network given our prior belief and prepares the min cost flow solver @@ -102,7 +100,7 @@ def _next_hop(self, path): The path is a list of node ids. Each call returns a tuple src, dest of an edge in the path """ for i in range(1, len(path)): - src = path[i-1] + src = path[i - 1] dest = path[i] yield (src, dest) @@ -113,9 +111,9 @@ def _make_channel_path(self, G: nx.MultiDiGraph, path: List[str]): min cost flow solver produced """ channel_path = [] - bottleneck = 2**63 + bottleneck = 2 ** 63 for src, dest in self._next_hop(path): - w = 2**63 + w = 2 ** 63 c = None flow = 0 for sid in G[src][dest].keys(): @@ -197,7 +195,7 @@ def _generate_candidate_paths(self, src, dest, amt, mu: int = 100_000_000, base: self._prepare_mcf_solver(src, dest, amt, mu, base) start = time.time() - #print("solving mcf...") + # print("solving mcf...") status = self._min_cost_flow.Solve() if status != self._min_cost_flow.OPTIMAL: @@ -207,7 +205,7 @@ def _generate_candidate_paths(self, src, dest, amt, mu: int = 100_000_000, base: paths = self._disect_flow_to_paths(src, dest) end = time.time() - return paths, end-start + return paths, end - start def _estimate_payment_statistics(self, paths): """ @@ -254,9 +252,6 @@ def _attempt_payments(self, payments, settled_onions): attempt["path"], attempt["amount"]) settled_onions.append(payments[key]) - - - def _evaluate_attempts(self, payments): """ helper function to collect statistics about attempts and print them @@ -287,7 +282,7 @@ def _evaluate_attempts(self, payments): total_fees += fee expected_sats_to_deliver += probability * amount print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5}".format( - probability*100, amount, len(path), int(fee*1000_000/amount))) + probability * 100, amount, len(path), int(fee * 1000_000 / amount))) paid_fees += fee if has_failed_attempt: @@ -305,21 +300,21 @@ def _evaluate_attempts(self, payments): total_fees += fee expected_sats_to_deliver += probability * amount print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5} ".format( - probability*100, amount, len(path), int(fee*1000_000/amount))) + probability * 100, amount, len(path), int(fee * 1000_000 / amount))) number_failed_paths += 1 residual_amt += amount print("\nAttempt Summary:") print("=================") print("\nTried to deliver {:10} sats".format(amt)) - fraction = expected_sats_to_deliver*100./amt + fraction = expected_sats_to_deliver * 100. / amt print("expected to deliver {:10} sats \t({:4.2f}%)".format( int(expected_sats_to_deliver), fraction)) - fraction = (amt-residual_amt)*100./(amt) + fraction = (amt - residual_amt) * 100. / (amt) print("actually delivered {:10} sats \t({:4.2f}%)".format( - amt-residual_amt, fraction)) + amt - residual_amt, fraction)) print("deviation: {:4.2f}".format( - (amt-residual_amt)/(expected_sats_to_deliver+1))) + (amt - residual_amt) / (expected_sats_to_deliver + 1))) print("planned_fee: {:8.3f} sat".format(total_fees)) print("paid fees: {:8.3f} sat".format(paid_fees)) return residual_amt, paid_fees, len(payments), number_failed_paths @@ -357,7 +352,7 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): # are too low or residual amounts decrease to slowly settled_onions = [] while amt > 0 and cnt < 10: - print("Round number: ", cnt+1) + print("Round number: ", cnt + 1) print("Try to deliver", amt, "satoshi:") # transfer to a min cost flow problem and rund the solver @@ -399,10 +394,10 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): print("Number of onions sent: ", number_number_of_onions) print("Number of failed onions: ", total_number_failed_paths) print("Failure rate: {:4.2f}% ".format( - total_number_failed_paths*100./number_number_of_onions)) + total_number_failed_paths * 100. / number_number_of_onions)) print("total runtime (including inefficient memory managment): {:4.3f} sec".format( - end-start)) - print("Learnt entropy: {:5.2f} bits".format(entropy_start-entropy_end)) + end - start)) + print("Learnt entropy: {:5.2f} bits".format(entropy_start - entropy_end)) print("Fees for successfull delivery: {:8.3f} sat --> {} ppm".format( - total_fees, int(total_fees*1000*1000/full_amt))) + total_fees, int(total_fees * 1000 * 1000 / full_amt))) print("used mu:", mu) From 6bba3199a672a70c1b654834ddd9448740f70862 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 17 May 2022 12:55:31 +0200 Subject: [PATCH 7/8] verbosity option for pickhardt_pay statistics (round_verbosity) can be added for each round of payment attempts and/or can be added as a final summary (summary). --- examples/basicexample.py | 27 ++- .../SyncSimulatedPaymentSession.py | 154 +++++++++++------- 2 files changed, 104 insertions(+), 77 deletions(-) diff --git a/examples/basicexample.py b/examples/basicexample.py index 6f9c467..2722121 100644 --- a/examples/basicexample.py +++ b/examples/basicexample.py @@ -3,30 +3,29 @@ from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork from pickhardtpayments.SyncSimulatedPaymentSession import SyncSimulatedPaymentSession - -#we first need to import the chanenl graph from c-lightning jsondump -#you can get your own data set via: +# we first need to import the chanenl graph from c-lightning jsondump +# you can get your own data set via: # $: lightning-cli listchannels > listchannels20220412.json # alternatively you can go to https://ln.rene-pickhardt.de to find a data dump channel_graph = ChannelGraph("listchannels20220412.json") uncertainty_network = UncertaintyNetwork(channel_graph) oracle_lightning_network = OracleLightningNetwork(channel_graph) -#we create ourselves a payment session which in this case operates by sending out the onions -#sequentially -payment_session = SyncSimulatedPaymentSession(oracle_lightning_network, - uncertainty_network, - prune_network=False) +# we create ourselves a payment session which in this case operates by sending out the onions +# sequentially +payment_session = SyncSimulatedPaymentSession(oracle_lightning_network, + uncertainty_network, + prune_network=False) -#we need to make sure we forget all learnt information on the Uncertainty Nework +# we need to make sure we forget all learnt information on the Uncertainty Nework payment_session.forget_information() -#we run the simulation of pickhardt payments and track all the results +# we run the simulation of pickhardt payments and track all the results -#Rene Pickhardt's public node key +# Rene Pickhardt's public node key RENE = "03efccf2c383d7bf340da9a3f02e2c23104a0e4fe8ac1a880c8e2dc92fbdacd9df" -#Carsten Otto's public node key +# Carsten Otto's public node key C_OTTO = "027ce055380348d7812d2ae7745701c9f93e70c1adeb2657f053f91df4f2843c71" -tested_amount = 10_000_000 #10 million sats +tested_amount = 10_000_000 # 10 million sats -payment_session.pickhardt_pay(RENE,C_OTTO, tested_amount,mu=0,base=0) +payment_session.pickhardt_pay(RENE, C_OTTO, tested_amount, mu=0, base=0, round_verbosity=False, summary=False) diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 876be4e..25dba23 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -12,13 +12,13 @@ class SyncSimulatedPaymentSession(): """ - A PaymentSesssion is used to create the min cost flow problem from the UncertaintyNetwork + A PaymentSession is used to create the min cost flow problem from the UncertaintyNetwork This happens by adding several parallel arcs coming from the piece wise linearization of the UncertaintyChannel to the min_cost_flow object. The main API call ist `pickhardt_pay` which invokes a sequential loop to conduct trial and error - attmpts. The loop could easily send out all onions concurrently but this does not make sense + attempts. The loop could easily send out all onions concurrently but this does not make sense against the simulated OracleLightningNetwork. """ @@ -36,7 +36,7 @@ def _prepare_integer_indices_for_nodes(self): necessary for the OR-lib by google and the min cost flow solver - let's initialize the look up tables for node_ids to integers from [0,...,#number of nodes] + let's initialize the look-up tables for node_ids to integers from [0,...,#number of nodes] this is necessary because of the API of the Google Operations Research min cost flow solver """ self._mcf_id = {} @@ -50,10 +50,11 @@ def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, """ computes the uncertainty network given our prior belief and prepares the min cost flow solver - This function can define a value for \mu to control how heavily we combine the uncertainty cost and fees - Also the function supports only taking channels into account that don't charge a base_fee higher or equal to `base` + This function can define a value for mu to control how heavily we combine the uncertainty cost and fees Also + the function supports only taking channels into account that don't charge a base_fee higher or equal to `base` - returns the instantiated min_cost_flow object from the google OR-lib that contains the piecewise linearized problem + returns the instantiated min_cost_flow object from the Google OR-lib that contains the piecewise linearized + problem """ self._min_cost_flow = pywrapgraph.SimpleMinCostFlow() self._arc_to_channel = {} @@ -63,7 +64,7 @@ def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, if channel.base_fee > base_fee: continue # FIXME: Remove Magic Number for pruning - # Prune channels away thay have too low success probability! This is a huge runtime boost + # Prune channels away they have too low success probability! This is a huge runtime boost # However the pruning would be much better to work on quantiles of normalized cost # So as soon as we have better Scaling, Centralization and feature engineering we can # probably have a more focused pruning @@ -95,7 +96,7 @@ def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, def _next_hop(self, path): """ - generator to iterate through edges indext by node id of paths + generator to iterate through edges indexed by node id of paths The path is a list of node ids. Each call returns a tuple src, dest of an edge in the path """ @@ -106,7 +107,7 @@ def _next_hop(self, path): def _make_channel_path(self, G: nx.MultiDiGraph, path: List[str]): """ - network x returns a path as a list of node_ids. However we need a list of `UncertaintyChannels` + network x returns a path as a list of node_ids. However, we need a list of `UncertaintyChannels` Since the graph has parallel edges it is quite some work to get the actual channels that the min cost flow solver produced """ @@ -138,9 +139,8 @@ def _disect_flow_to_paths(self, s, d): progress but this is a mere conjecture at this point. I expect quite a bit of research will be necessary to resolve this issue. """ - total_flow = {} - # first collect all linearized edges which are assigned a non zero flow put them into a networkx graph + # first collect all linearized edges which are assigned a non-zero flow put them into a networkx graph G = nx.MultiDiGraph() for i in range(self._min_cost_flow.NumArcs()): flow = self._min_cost_flow.Flow(i) # *QUANTIZATION @@ -181,14 +181,15 @@ def _disect_flow_to_paths(self, s, d): def _generate_candidate_paths(self, src, dest, amt, mu: int = 100_000_000, base: int = DEFAULT_BASE_THRESHOLD): """ - computes the optimal payment split to deliver `amt` from `src` to `dest` and updates our belief about the liquidity + computes the optimal payment split to deliver `amt` from `src` to `dest` and updates our belief about the + liquidity This is one step within the payment loop. - Retuns the residual amount of the `amt` that could ne be delivered and the paid fees + Returns the residual amount of the `amt` that could not be delivered and the paid fees (on a per channel base not including fees for downstream fees) for the delivered amount - the function also prints some results an statistics about the paths of the flow to stdout. + the function also prints some results on statistics about the paths of the flow to stdout. """ # First we prepare the min cost flow by getting arcs from the uncertainty network @@ -223,7 +224,7 @@ def _estimate_payment_statistics(self, paths): payments[i] = { "routing_fee": fee, "probability": probability, "path": path, "amount": amount} - # to correctly compute conditional probabilities of non disjoint paths in the same set of paths + # to correctly compute conditional probabilities of non-disjoint paths in the same set of paths self._uncertainty_network.allocate_amount_on_path(path, amount) # remove allocated amounts for all planned onions before doing actual attempts @@ -252,9 +253,9 @@ def _attempt_payments(self, payments, settled_onions): attempt["path"], attempt["amount"]) settled_onions.append(payments[key]) - def _evaluate_attempts(self, payments): + def _evaluate_attempts(self, payments, round_verbosity=False): """ - helper function to collect statistics about attempts and print them + helper function to collect statistics about attempts and prints them if round_verbosity is True returns the `residual` amount that could not have been delivered and some statistics """ @@ -264,11 +265,14 @@ def _evaluate_attempts(self, payments): number_failed_paths = 0 expected_sats_to_deliver = 0 amt = 0 - print("\nStatistics about {} candidate onions:\n".format(len(payments))) - has_failed_attempt = False - print("successful attempts:") - print("--------------------") + + # print summary + if round_verbosity: + print("\nStatistics about {} candidate onions:\n".format(len(payments))) + print("successful attempts:") + print("--------------------") + for attempt in payments.values(): success = attempt["success"] if success == False: @@ -281,13 +285,20 @@ def _evaluate_attempts(self, payments): amt += amount total_fees += fee expected_sats_to_deliver += probability * amount - print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5}".format( - probability * 100, amount, len(path), int(fee * 1000_000 / amount))) + + # print summary + if round_verbosity: + print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5}".format( + probability * 100, amount, len(path), int(fee * 1000_000 / amount))) + paid_fees += fee if has_failed_attempt: - print("\nfailed attempts:") - print("----------------") + # print summary + if round_verbosity: + print("\nfailed attempts:") + print("----------------") + for attempt in payments.values(): success = attempt["success"] if success: @@ -299,24 +310,30 @@ def _evaluate_attempts(self, payments): amt += amount total_fees += fee expected_sats_to_deliver += probability * amount - print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5} ".format( - probability * 100, amount, len(path), int(fee * 1000_000 / amount))) + + # print summary + if round_verbosity: + print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5} ".format( + probability * 100, amount, len(path), int(fee * 1000_000 / amount))) + number_failed_paths += 1 residual_amt += amount - print("\nAttempt Summary:") - print("=================") - print("\nTried to deliver {:10} sats".format(amt)) - fraction = expected_sats_to_deliver * 100. / amt - print("expected to deliver {:10} sats \t({:4.2f}%)".format( - int(expected_sats_to_deliver), fraction)) - fraction = (amt - residual_amt) * 100. / (amt) - print("actually delivered {:10} sats \t({:4.2f}%)".format( - amt - residual_amt, fraction)) - print("deviation: {:4.2f}".format( - (amt - residual_amt) / (expected_sats_to_deliver + 1))) - print("planned_fee: {:8.3f} sat".format(total_fees)) - print("paid fees: {:8.3f} sat".format(paid_fees)) + # print summary + if round_verbosity: + print("\nAttempt Summary:") + print("=================") + print("\nTried to deliver {:10} sats".format(amt)) + fraction = expected_sats_to_deliver * 100. / amt + print("expected to deliver {:10} sats \t({:4.2f}%)".format( + int(expected_sats_to_deliver), fraction)) + fraction = (amt - residual_amt) * 100. / (amt) + print("actually delivered {:10} sats \t({:4.2f}%)".format( + amt - residual_amt, fraction)) + print("deviation: {:4.2f}".format( + (amt - residual_amt) / (expected_sats_to_deliver + 1))) + print("planned_fee: {:8.3f} sat".format(total_fees)) + print("paid fees: {:8.3f} sat".format(paid_fees)) return residual_amt, paid_fees, len(payments), number_failed_paths def forget_information(self): @@ -332,10 +349,13 @@ def activate_network_wide_uncertainty_reduction(self, n): self._uncertainty_network.activate_network_wide_uncertainty_reduction( n, self._oracle) - def pickhardt_pay(self, src, dest, amt, mu=1, base=0): + def pickhardt_pay(self, src, dest, amt, mu=1, base=0, round_verbosity=False, summary=False): """ conduct one experiment! might need to call oracle.reset_uncertainty_network() first - I could not put it here as some experiments require sharing of liqudity information + I could not put it here as some experiments require sharing of liquidity information + + round_verbosity prints statistics for every round of payment attempts + summary prints final statistics for payment attempts """ entropy_start = self._uncertainty_network.entropy() @@ -347,15 +367,17 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): total_number_failed_paths = 0 # This is the main payment loop. It is currently blocking and synchronous but may be - # implemented in a concurrent way. Also we stop after 10 rounds which is pretty arbitrary - # a better stop criteria would be if we compute infeasable flows or if the probabilities + # implemented in a concurrent way. Also, we stop after 10 rounds which is pretty arbitrary + # a better stop criteria would be if we compute infeasible flows or if the probabilities # are too low or residual amounts decrease to slowly settled_onions = [] while amt > 0 and cnt < 10: - print("Round number: ", cnt + 1) - print("Try to deliver", amt, "satoshi:") + # print summary + if round_verbosity: + print("Round number: ", cnt + 1) + print("Try to deliver", amt, "satoshi:") - # transfer to a min cost flow problem and rund the solver + # transfer to a min cost flow problem and run the solver paths, runtime = self._generate_candidate_paths( src, dest, amt, mu, base) @@ -367,9 +389,12 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): # run some simple statistics and depict them amt, paid_fees, num_paths, number_failed_paths = self._evaluate_attempts( - payments) - print("Runtime of flow computation: {:4.2f} sec ".format(runtime)) - print("\n================================================================\n") + payments, round_verbosity) + + # print summary + if round_verbosity: + print("Runtime of flow computation: {:4.2f} sec ".format(runtime)) + print("\n================================================================\n") number_number_of_onions += num_paths total_number_failed_paths += number_failed_paths @@ -388,16 +413,19 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): end = time.time() entropy_end = self._uncertainty_network.entropy() - print("SUMMARY:") - print("========") - print("Rounds of mcf-computations: ", cnt) - print("Number of onions sent: ", number_number_of_onions) - print("Number of failed onions: ", total_number_failed_paths) - print("Failure rate: {:4.2f}% ".format( - total_number_failed_paths * 100. / number_number_of_onions)) - print("total runtime (including inefficient memory managment): {:4.3f} sec".format( - end - start)) - print("Learnt entropy: {:5.2f} bits".format(entropy_start - entropy_end)) - print("Fees for successfull delivery: {:8.3f} sat --> {} ppm".format( - total_fees, int(total_fees * 1000 * 1000 / full_amt))) - print("used mu:", mu) + + # print summary + if summary: + print("SUMMARY:") + print("========") + print("Rounds of mcf-computations: ", cnt) + print("Number of onions sent: ", number_number_of_onions) + print("Number of failed onions: ", total_number_failed_paths) + print("Failure rate: {:4.2f}% ".format( + total_number_failed_paths * 100. / number_number_of_onions)) + print("total runtime (including inefficient memory management): {:4.3f} sec".format( + end - start)) + print("Learnt entropy: {:5.2f} bits".format(entropy_start - entropy_end)) + print("Fees for successful delivery: {:8.3f} sat --> {} ppm".format( + total_fees, int(total_fees * 1000 * 1000 / full_amt))) + print("used mu:", mu) From 96f49d986945b29b74550951747feea88258218b Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 17 May 2022 12:56:04 +0200 Subject: [PATCH 8/8] verbosity option for pickhardt_pay statistics (round_verbosity) can be added for each round of payment attempts and/or can be added as a final summary (summary). --- examples/basicexample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/basicexample.py b/examples/basicexample.py index 2722121..a0c3b98 100644 --- a/examples/basicexample.py +++ b/examples/basicexample.py @@ -28,4 +28,4 @@ C_OTTO = "027ce055380348d7812d2ae7745701c9f93e70c1adeb2657f053f91df4f2843c71" tested_amount = 10_000_000 # 10 million sats -payment_session.pickhardt_pay(RENE, C_OTTO, tested_amount, mu=0, base=0, round_verbosity=False, summary=False) +payment_session.pickhardt_pay(RENE, C_OTTO, tested_amount, mu=0, base=0, round_verbosity=True, summary=True)