From 519185f1c5ff7b2e20977f2f8e48d57363d8e8b8 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 13 May 2022 15:46:47 +0200 Subject: [PATCH 01/38] add .gitignore --- .gitgnore | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .gitgnore diff --git a/.gitgnore b/.gitgnore new file mode 100644 index 0000000..fa47571 --- /dev/null +++ b/.gitgnore @@ -0,0 +1,7 @@ +.DS_Store +__pycache__ +.idea +pickhardtpayments/pickhardtpayments.egg-info + +# not including large file with Channel Graph +listchannels*.json \ No newline at end of file From cc1e86248ef416f7b88b5a609312bec994a11089 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 13 May 2022 16:57:26 +0200 Subject: [PATCH 02/38] introduce register payment method on OracleLightningNetwork.py --- LICNENSE => LICENSE | 0 pickhardtpayments/OracleLightningNetwork.py | 5 +++++ pickhardtpayments/SyncSimulatedPaymentSession.py | 1 + pickhardtpayments/UncertaintyChannel.py | 2 +- 4 files changed, 7 insertions(+), 1 deletion(-) rename LICNENSE => LICENSE (100%) diff --git a/LICNENSE b/LICENSE similarity index 100% rename from LICNENSE rename to LICENSE diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 2be2969..d4de2cb 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -71,3 +71,8 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base mincut, _ = nx.minimum_cut(test_network, source, destination) return mincut + + def register_payment(self, channel: OracleChannel, actual_liquidity: int): + # TODO register payment + print("=> register payment function here") + diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 1be5c23..03cb098 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -249,6 +249,7 @@ def _attempt_payments(self, payments): if success: self._uncertainty_network.allocate_amount_on_path( attempt["path"], attempt["amount"]) + self._oracle.register_payment(attempt["path"], attempt["amount"]) def _evaluate_attempts(self, payments): """ diff --git a/pickhardtpayments/UncertaintyChannel.py b/pickhardtpayments/UncertaintyChannel.py index b855bc8..518680a 100644 --- a/pickhardtpayments/UncertaintyChannel.py +++ b/pickhardtpayments/UncertaintyChannel.py @@ -81,7 +81,7 @@ def conditional_capacity(self, respect_inflight=True): def allocate_amount(self, amt: int): """ - assign or remove ammount that is assigned to be `in_flight`. + assign or remove amount that is assigned to be `in_flight`. """ self.in_flight += amt if self.in_flight < 0: From 48322532a20e3af5a744a56c61301e41a23773d7 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 13 May 2022 17:37:14 +0200 Subject: [PATCH 03/38] adding settlement_payment in OracleLightningNetwork first stub of a method as described in #16 Still needs to be tested --- pickhardtpayments/OracleChannel.py | 4 ++++ pickhardtpayments/OracleLightningNetwork.py | 20 ++++++++++++++++--- .../SyncSimulatedPaymentSession.py | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index a94963f..c6cf926 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -37,3 +37,7 @@ def can_forward(self, amt: int): return True else: return False + + # setter for actual liquidity + def set_actual_liquidity(self, amt: int): + self._actual_liquidity = amt diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index d4de2cb..0ea1a29 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -72,7 +72,21 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base mincut, _ = nx.minimum_cut(test_network, source, destination) return mincut - def register_payment(self, channel: OracleChannel, actual_liquidity: int): - # TODO register payment - print("=> register payment function here") + def settle_payment(self, path: OracleChannel, payment_amount: int): + """ + receives a channel and a payment amount and adjusts the balances of the channels along the path. + + settle_payment should only be called after send_onion terminated successfully! + There is currently no further exception handling if along the path channels do not provide + enough liquidity! + # TODO testing + """ + for channel in path: + settlement_channel = self.get_channel(channel.src, channel.dest, channel.short_channel_id) + # print("path on UncertaintyNetwork: {}".format(channel.short_channel_id)) + # print("path on OracleLN: {}".format(settlement_channel.short_channel_id)) + if settlement_channel.actual_liquidity > payment_amount: + settlement_channel.set_actual_liquidity = settlement_channel.actual_liquidity - payment_amount + else: + print("=== CHANNEL EXHAUSTED ===") diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 03cb098..0df20ee 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -249,7 +249,7 @@ def _attempt_payments(self, payments): if success: self._uncertainty_network.allocate_amount_on_path( attempt["path"], attempt["amount"]) - self._oracle.register_payment(attempt["path"], attempt["amount"]) + self._oracle.settle_payment(attempt["path"], attempt["amount"]) def _evaluate_attempts(self, payments): """ From afa6a7120e35eded6babcc09ff92774b6d669526 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 13 May 2022 22:53:00 +0200 Subject: [PATCH 04/38] rewriting setter for `actual_liquidity` using properties (python way) --- pickhardtpayments/OracleChannel.py | 14 +++++++------- pickhardtpayments/OracleLightningNetwork.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index c6cf926..61e59fa 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -17,18 +17,22 @@ def __init__(self, channel: Channel, actual_liquidity: int = None): self._actual_liquidity = random.randint(0, self.capacity) def __str__(self): - return super().__str__()+" actual Liquidity: {}".format(self.actual_liquidity) + return super().__str__() + " actual Liquidity: {}".format(self.actual_liquidity) @property def actual_liquidity(self): """ Tells us the actual liquidity according to the oracle. - This is usful for experiments but must of course not be used in routing and is also - not a vailable if mainnet remote channels are being used. + This is useful for experiments but must of course not be used in routing and is also + not available if mainnet remote channels are being used. """ return self._actual_liquidity + @actual_liquidity.setter + def actual_liquidity(self, amt: int): + self._actual_liquidity = amt + def can_forward(self, amt: int): """ check if the oracle channel can forward a certain amount @@ -37,7 +41,3 @@ def can_forward(self, amt: int): return True else: return False - - # setter for actual liquidity - def set_actual_liquidity(self, amt: int): - self._actual_liquidity = amt diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 0ea1a29..351c60a 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -86,7 +86,7 @@ def settle_payment(self, path: OracleChannel, payment_amount: int): # print("path on UncertaintyNetwork: {}".format(channel.short_channel_id)) # print("path on OracleLN: {}".format(settlement_channel.short_channel_id)) if settlement_channel.actual_liquidity > payment_amount: - settlement_channel.set_actual_liquidity = settlement_channel.actual_liquidity - payment_amount + settlement_channel.actual_liquidity = settlement_channel.actual_liquidity - payment_amount else: print("=== CHANNEL EXHAUSTED ===") From 4ecadee08bfa090219047cc9277ed92b53191ae4 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 16 May 2022 16:19:04 +0200 Subject: [PATCH 05/38] adjust channels in both directions when payment is made --- pickhardtpayments/OracleLightningNetwork.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 351c60a..de57199 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -13,7 +13,7 @@ def __init__(self, channel_graph: ChannelGraph): for src, dest, short_channel_id, channel in channel_graph.network.edges(data="channel", keys=True): oracle_channel = None - # If Channel in oposite direction already exists with liquidity information match the channel + # If Channel in opposite direction already exists with liquidity information match the channel if self._network.has_edge(dest, src): if short_channel_id in self._network[dest][src]: capacity = channel.capacity @@ -39,10 +39,10 @@ def send_onion(self, path, amt): oracle_channel = self.get_channel( channel.src, channel.dest, channel.short_channel_id) success_of_probe = oracle_channel.can_forward( - channel.in_flight+amt) + channel.in_flight + amt) # print(channel,amt,success_of_probe) channel.update_knowledge(amt, success_of_probe) - if success_of_probe == False: + if not success_of_probe: return False, channel return True, None @@ -55,7 +55,7 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base """ test_network = nx.DiGraph() for src, dest, channel in self.network.edges(data="channel"): - #liqudity = 0 + # liquidity = 0 # for channel in channels: if channel.base_fee > base_fee: continue @@ -74,19 +74,19 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base def settle_payment(self, path: OracleChannel, payment_amount: int): """ - receives a channel and a payment amount and adjusts the balances of the channels along the path. + receives a dictionary with channels and payment amounts and adjusts the balances of the channels along the path. - settle_payment should only be called after send_onion terminated successfully! - There is currently no further exception handling if along the path channels do not provide - enough liquidity! + settle_payment should only be called after all send_onions for a payment terminated successfully! # TODO testing """ for channel in path: settlement_channel = self.get_channel(channel.src, channel.dest, channel.short_channel_id) - # print("path on UncertaintyNetwork: {}".format(channel.short_channel_id)) - # print("path on OracleLN: {}".format(settlement_channel.short_channel_id)) + return_settlement_channel = self.get_channel(channel.dest, channel.src, channel.short_channel_id) if settlement_channel.actual_liquidity > payment_amount: + # decrease channel balance in sending channel by amount settlement_channel.actual_liquidity = settlement_channel.actual_liquidity - payment_amount + # increase channel balance in the other direction by amount + return_settlement_channel.actual_liquidity = return_settlement_channel.actual_liquidity + payment_amount else: - print("=== CHANNEL EXHAUSTED ===") - + raise Exception("""Channel liquidity on Channel {} is lower than payment amount. + \nPayment cannot settle.""".format(channel.short_channel_id)) From a131eaf1761f148412e700288ddf4b6a6ea9e98e Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 16 May 2022 16:25:07 +0200 Subject: [PATCH 06/38] Settlement of onions with partial payments only after all onions were routed successfully --- .../SyncSimulatedPaymentSession.py | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 0df20ee..031fe5e 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -7,6 +7,7 @@ from typing import List import time import networkx as nx +import json DEFAULT_BASE_THRESHOLD = 0 @@ -234,13 +235,15 @@ def _estimate_payment_statistics(self, paths): return payments - def _attempt_payments(self, payments): + def _attempt_payments(self, payments, settled_onions): """ we attempt all planned payments and test the success against the oracle in particular this method changes - depending on the outcome of each payment - our belief about the uncertainty - in the UncertaintyNetwork + in the UncertaintyNetwork. + successful onions are collected to be transacted on the OracleNetwork if complete payment can be delivered """ # test actual payment attempts + for key, attempt in payments.items(): success, erring_channel = self._oracle.send_onion( attempt["path"], attempt["amount"]) @@ -249,7 +252,10 @@ def _attempt_payments(self, payments): if success: self._uncertainty_network.allocate_amount_on_path( attempt["path"], attempt["amount"]) - self._oracle.settle_payment(attempt["path"], attempt["amount"]) + settled_onions.append(payments[key]) + + + def _evaluate_attempts(self, payments): """ @@ -310,7 +316,7 @@ def _evaluate_attempts(self, payments): print("expected to deliver {:10} sats \t({:4.2f}%)".format( int(expected_sats_to_deliver), fraction)) fraction = (amt-residual_amt)*100./(amt) - print("actually deliverd {:10} sats \t({:4.2f}%)".format( + print("actually delivered {:10} sats \t({:4.2f}%)".format( amt-residual_amt, fraction)) print("deviation: {:4.2f}".format( (amt-residual_amt)/(expected_sats_to_deliver+1))) @@ -348,7 +354,8 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): # This is the main payment loop. It is currently blocking and synchronous but may be # implemented in a concurrent way. Also we stop after 10 rounds which is pretty arbitrary # a better stop criteria would be if we compute infeasable flows or if the probabilities - # are to low or residual amounts decrease to slowly + # are too low or residual amounts decrease to slowly + settled_onions = [] while amt > 0 and cnt < 10: print("Round number: ", cnt+1) print("Try to deliver", amt, "satoshi:") @@ -360,8 +367,8 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): # compute some statistics about candidate paths payments = self._estimate_payment_statistics(paths) - # matke attempts and update our information about the UncertaintyNetwork - self._attempt_payments(payments) + # make attempts and update our information about the UncertaintyNetwork and track settled onions + self._attempt_payments(payments, settled_onions) # run some simple statistics and depict them amt, paid_fees, num_paths, number_failed_paths = self._evaluate_attempts( @@ -373,6 +380,17 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): total_number_failed_paths += number_failed_paths total_fees += paid_fees cnt += 1 + + # When residual amount is 0 / enough successful onions have been found, then settle payment. Else drop onions. + if amt == 0: + # print("{} onions to settle.".format(len(settled_onions))) + for onion in settled_onions: + try: + self._oracle.settle_payment(onion["path"], onion["amount"]) + except Exception as e: + print(e) + return -1 + end = time.time() entropy_end = self._uncertainty_network.entropy() print("SUMMARY:") From 0b0e868234a2072c030f587004f8ef667178d16b Mon Sep 17 00:00:00 2001 From: Sebastian Date: Mon, 16 May 2022 17:59:44 +0200 Subject: [PATCH 07/38] deletion of unused import --- .../SyncSimulatedPaymentSession.py | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 031fe5e..876be4e 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -3,12 +3,9 @@ from ortools.graph import pywrapgraph - from typing import List import time import networkx as nx -import json - DEFAULT_BASE_THRESHOLD = 0 @@ -48,7 +45,8 @@ def _prepare_integer_indices_for_nodes(self): self._mcf_id[node_id] = k self._node_key[k] = node_id - def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, base_fee: int = DEFAULT_BASE_THRESHOLD): + def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, + base_fee: int = DEFAULT_BASE_THRESHOLD): """ computes the uncertainty network given our prior belief and prepares the min cost flow solver @@ -102,7 +100,7 @@ def _next_hop(self, path): The path is a list of node ids. Each call returns a tuple src, dest of an edge in the path """ for i in range(1, len(path)): - src = path[i-1] + src = path[i - 1] dest = path[i] yield (src, dest) @@ -113,9 +111,9 @@ def _make_channel_path(self, G: nx.MultiDiGraph, path: List[str]): min cost flow solver produced """ channel_path = [] - bottleneck = 2**63 + bottleneck = 2 ** 63 for src, dest in self._next_hop(path): - w = 2**63 + w = 2 ** 63 c = None flow = 0 for sid in G[src][dest].keys(): @@ -197,7 +195,7 @@ def _generate_candidate_paths(self, src, dest, amt, mu: int = 100_000_000, base: self._prepare_mcf_solver(src, dest, amt, mu, base) start = time.time() - #print("solving mcf...") + # print("solving mcf...") status = self._min_cost_flow.Solve() if status != self._min_cost_flow.OPTIMAL: @@ -207,7 +205,7 @@ def _generate_candidate_paths(self, src, dest, amt, mu: int = 100_000_000, base: paths = self._disect_flow_to_paths(src, dest) end = time.time() - return paths, end-start + return paths, end - start def _estimate_payment_statistics(self, paths): """ @@ -254,9 +252,6 @@ def _attempt_payments(self, payments, settled_onions): attempt["path"], attempt["amount"]) settled_onions.append(payments[key]) - - - def _evaluate_attempts(self, payments): """ helper function to collect statistics about attempts and print them @@ -287,7 +282,7 @@ def _evaluate_attempts(self, payments): total_fees += fee expected_sats_to_deliver += probability * amount print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5}".format( - probability*100, amount, len(path), int(fee*1000_000/amount))) + probability * 100, amount, len(path), int(fee * 1000_000 / amount))) paid_fees += fee if has_failed_attempt: @@ -305,21 +300,21 @@ def _evaluate_attempts(self, payments): total_fees += fee expected_sats_to_deliver += probability * amount print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5} ".format( - probability*100, amount, len(path), int(fee*1000_000/amount))) + probability * 100, amount, len(path), int(fee * 1000_000 / amount))) number_failed_paths += 1 residual_amt += amount print("\nAttempt Summary:") print("=================") print("\nTried to deliver {:10} sats".format(amt)) - fraction = expected_sats_to_deliver*100./amt + fraction = expected_sats_to_deliver * 100. / amt print("expected to deliver {:10} sats \t({:4.2f}%)".format( int(expected_sats_to_deliver), fraction)) - fraction = (amt-residual_amt)*100./(amt) + fraction = (amt - residual_amt) * 100. / (amt) print("actually delivered {:10} sats \t({:4.2f}%)".format( - amt-residual_amt, fraction)) + amt - residual_amt, fraction)) print("deviation: {:4.2f}".format( - (amt-residual_amt)/(expected_sats_to_deliver+1))) + (amt - residual_amt) / (expected_sats_to_deliver + 1))) print("planned_fee: {:8.3f} sat".format(total_fees)) print("paid fees: {:8.3f} sat".format(paid_fees)) return residual_amt, paid_fees, len(payments), number_failed_paths @@ -357,7 +352,7 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): # are too low or residual amounts decrease to slowly settled_onions = [] while amt > 0 and cnt < 10: - print("Round number: ", cnt+1) + print("Round number: ", cnt + 1) print("Try to deliver", amt, "satoshi:") # transfer to a min cost flow problem and rund the solver @@ -399,10 +394,10 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): print("Number of onions sent: ", number_number_of_onions) print("Number of failed onions: ", total_number_failed_paths) print("Failure rate: {:4.2f}% ".format( - total_number_failed_paths*100./number_number_of_onions)) + total_number_failed_paths * 100. / number_number_of_onions)) print("total runtime (including inefficient memory managment): {:4.3f} sec".format( - end-start)) - print("Learnt entropy: {:5.2f} bits".format(entropy_start-entropy_end)) + end - start)) + print("Learnt entropy: {:5.2f} bits".format(entropy_start - entropy_end)) print("Fees for successfull delivery: {:8.3f} sat --> {} ppm".format( - total_fees, int(total_fees*1000*1000/full_amt))) + total_fees, int(total_fees * 1000 * 1000 / full_amt))) print("used mu:", mu) From 8dffa28ea2de5c4781d558b8cb6634a272b65291 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 17 May 2022 17:17:57 +0200 Subject: [PATCH 08/38] shell for a Payment Class --- pickhardtpayments/Attempt.py | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 pickhardtpayments/Attempt.py diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py new file mode 100644 index 0000000..82db812 --- /dev/null +++ b/pickhardtpayments/Attempt.py @@ -0,0 +1,40 @@ + + +class Attempt(): + """ + An OracleChannel us used in experiments and Simulations to form the (Oracle)LightningNetwork. + + It contains a ground truth about the Liquidity of a channel + """ + + def __init__(self, channel: Channel, actual_liquidity: int = None): + super().__init__(channel.cln_jsn) + self._actual_liquidity = actual_liquidity + if actual_liquidity is None or actual_liquidity >= self.capacity or actual_liquidity < 0: + self._actual_liquidity = random.randint(0, self.capacity) + + def __str__(self): + return super().__str__() + " actual Liquidity: {}".format(self.actual_liquidity) + + @property + def actual_liquidity(self): + """ + Tells us the actual liquidity according to the oracle. + + This is useful for experiments but must of course not be used in routing and is also + not available if mainnet remote channels are being used. + """ + return self._actual_liquidity + + @actual_liquidity.setter + def actual_liquidity(self, amt: int): + self._actual_liquidity = amt + + def can_forward(self, amt: int): + """ + check if the oracle channel can forward a certain amount + """ + if amt <= self.actual_liquidity: + return True + else: + return False From 8df2c57bf4c69686cb33eca88363f594898d2c4b Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 17 May 2022 17:25:59 +0200 Subject: [PATCH 09/38] Revert "shell for a Payment Class" This reverts commit 8dffa28ea2de5c4781d558b8cb6634a272b65291. --- pickhardtpayments/Attempt.py | 40 ------------------------------------ 1 file changed, 40 deletions(-) delete mode 100644 pickhardtpayments/Attempt.py diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py deleted file mode 100644 index 82db812..0000000 --- a/pickhardtpayments/Attempt.py +++ /dev/null @@ -1,40 +0,0 @@ - - -class Attempt(): - """ - An OracleChannel us used in experiments and Simulations to form the (Oracle)LightningNetwork. - - It contains a ground truth about the Liquidity of a channel - """ - - def __init__(self, channel: Channel, actual_liquidity: int = None): - super().__init__(channel.cln_jsn) - self._actual_liquidity = actual_liquidity - if actual_liquidity is None or actual_liquidity >= self.capacity or actual_liquidity < 0: - self._actual_liquidity = random.randint(0, self.capacity) - - def __str__(self): - return super().__str__() + " actual Liquidity: {}".format(self.actual_liquidity) - - @property - def actual_liquidity(self): - """ - Tells us the actual liquidity according to the oracle. - - This is useful for experiments but must of course not be used in routing and is also - not available if mainnet remote channels are being used. - """ - return self._actual_liquidity - - @actual_liquidity.setter - def actual_liquidity(self, amt: int): - self._actual_liquidity = amt - - def can_forward(self, amt: int): - """ - check if the oracle channel can forward a certain amount - """ - if amt <= self.actual_liquidity: - return True - else: - return False From c03fe39c502be930d958f7a8a5802e66cb67d24d Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 17 May 2022 17:32:50 +0200 Subject: [PATCH 10/38] shell for a Payment Class for issue #20 --- .gitignore | 7 ++++++ pickhardtpayments/Payment.py | 44 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 .gitignore create mode 100644 pickhardtpayments/Payment.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5fca83e --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.DS_Store +.idea +__pycache__ +docs + +# too large to upload +listchannels*.json \ No newline at end of file diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py new file mode 100644 index 0000000..93d242f --- /dev/null +++ b/pickhardtpayments/Payment.py @@ -0,0 +1,44 @@ +class Payment: + """ + Payment stores the information about an amount of sats to be delivered from source to destination. + + When sending an amount of sats from sender to receiver, a payment is usually split up and sent across + several paths, to increase the probability of being successfully delivered. + The PaymentClass holds all necessary information about a payment. + It also holds information that helps to calculate performance statistics about the payment. + + :param int _total_amount: The total amount of sats to be delivered from source address to destination address. + :param str _sender: sender address for the payment. + :param str _receiver: receiver address for the payment. + :param list _attempts: returns a list of Attempts + :param list _successful_attempts: returns a list of successful Attempts + :param bool _successful: returns True if the total_amount of the payment could be delivered successfully. + """ + + def __init__(self): + self._attempts = [] + self._successful_attempts = [] + + @property + def total_amount(self): + return self._total_amount + + @property + def src(self): + return self._sender + + @property + def dest(self): + return self._receiver + + @property + def attempts(self): + return self._attempts + + @property + def successful_attempts(self): + return self._successful_attempts + + @property + def successful(self): + return self._successful From 717015026e9ae8c4b4f4069b538739a8bca86f66 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 17 May 2022 18:10:33 +0200 Subject: [PATCH 11/38] shell for as Attempt Class for issue #20 --- pickhardtpayments/Attempt.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 pickhardtpayments/Attempt.py diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py new file mode 100644 index 0000000..d0fddc8 --- /dev/null +++ b/pickhardtpayments/Attempt.py @@ -0,0 +1,11 @@ +class Attempt: + """ + An Attempt describes a path (a set of channels) of an amount from sender to receiver. + + :param list _path: a list of UncertaintyChannel objects from sender to receiver + :param int _amount: the amount to be transferred from source to destination + :param int _routing_fee: the rounting fee in msat + :param float _probability: the success probability from + :param int _runtime: the number of milliseconds for the path to be found + :param boolean _success: the flag to describe if the path succeeded (True) or failed (False) for the amount in flight + """ From 16bad26e595bf6994fe0154b8cd98f9426b7f690 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 20 May 2022 02:31:27 +0200 Subject: [PATCH 12/38] Indroduction of Payment and Attempt class, refactoring. --- examples/basicexample.py | 2 +- pickhardtpayments/Attempt.py | 80 ++++++- pickhardtpayments/Channel.py | 2 +- pickhardtpayments/OracleLightningNetwork.py | 4 + pickhardtpayments/Payment.py | 83 ++++++- .../SyncSimulatedPaymentSession.py | 219 ++++++++++-------- 6 files changed, 280 insertions(+), 110 deletions(-) diff --git a/examples/basicexample.py b/examples/basicexample.py index 6f9c467..8823ad8 100644 --- a/examples/basicexample.py +++ b/examples/basicexample.py @@ -29,4 +29,4 @@ C_OTTO = "027ce055380348d7812d2ae7745701c9f93e70c1adeb2657f053f91df4f2843c71" tested_amount = 10_000_000 #10 million sats -payment_session.pickhardt_pay(RENE,C_OTTO, tested_amount,mu=0,base=0) +payment_session.pickhardt_pay(RENE, C_OTTO, tested_amount, mu=0, base=0) diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py index d0fddc8..627bf14 100644 --- a/pickhardtpayments/Attempt.py +++ b/pickhardtpayments/Attempt.py @@ -1,11 +1,83 @@ +from enum import Enum + +from pickhardtpayments import Channel + + +class SettlementStatus(Enum): + PLANNED = 1 + INFLIGHT = 2 + ARRIVED = 3 + FAILED = 4 + SETTLED = 5 + + class Attempt: """ An Attempt describes a path (a set of channels) of an amount from sender to receiver. :param list _path: a list of UncertaintyChannel objects from sender to receiver :param int _amount: the amount to be transferred from source to destination - :param int _routing_fee: the rounting fee in msat - :param float _probability: the success probability from - :param int _runtime: the number of milliseconds for the path to be found - :param boolean _success: the flag to describe if the path succeeded (True) or failed (False) for the amount in flight + :param int _routing_fee: the routing fee in msat + :param float _probability: estimated success probability before the attempt + :param float _start: time of sending out the onion + :param float _end: time of getting back the onion + :param SettlementStatus _status: the flag to describe if the path failed, succeeded or was used for settlement """ + + def __init__(self, path: list, amount: int = 0): + self._paths = None + self._routing_fee = None + self._probability = None + if not isinstance(path, list): + raise ValueError("path needs to be a collection of Channels") + for channel in path: + if not isinstance(channel, Channel): + raise ValueError("path needs to be a collection of Channels") + self._path = path + self._status = SettlementStatus.PLANNED + self._amount = amount + + def __str__(self): + return "Path with {} channels to deliver {} sats and status {}.".format(len(self._path), + self._amount, self._status.name) + + @property + def path(self): + return self._path + + @path.setter + def path(self, attempts: list): + if not isinstance(attempts, list): + raise ValueError("path needs to be a collection of Channels") + for attempt in attempts: + if not isinstance(attempt, Channel): + raise ValueError("path needs to be a collection of Channels") + self._paths = attempts + + @property + def amount(self): + return self._amount + + @property + def status(self): + return self._status + + @status.setter + def status(self, value: SettlementStatus): + self._status = value + + @property + def routing_fee(self): + return self._routing_fee + + @routing_fee.setter + def routing_fee(self, value: int): + self._routing_fee = value + + @property + def probability(self): + return self._probability + + @probability.setter + def probability(self, value: int): + self._probability = value diff --git a/pickhardtpayments/Channel.py b/pickhardtpayments/Channel.py index 91a7e03..3dbd4d0 100644 --- a/pickhardtpayments/Channel.py +++ b/pickhardtpayments/Channel.py @@ -25,7 +25,7 @@ class Channel(): """ Stores the public available information of a channel. - The `Channel` Class is intended to be read only and internatlly stores + The `Channel` Class is intended to be read only and internally stores the data from c-lightning's `lightning-cli listchannels` command as a json. If you retrieve data from a different implementation I suggest to overload the constructor and transform the information into the given json format diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index de57199..ef2c12c 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -35,6 +35,10 @@ def network(self): return self._network def send_onion(self, path, amt): + """ + + :rtype: object + """ for channel in path: oracle_channel = self.get_channel( channel.src, channel.dest, channel.short_channel_id) diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index 93d242f..572087a 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -1,3 +1,9 @@ +import logging +import time +from typing import List + +import Attempt + class Payment: """ Payment stores the information about an amount of sats to be delivered from source to destination. @@ -8,20 +14,31 @@ class Payment: It also holds information that helps to calculate performance statistics about the payment. :param int _total_amount: The total amount of sats to be delivered from source address to destination address. + :param float _fee: fee in sats for successful delivery and settlement of payment + :param float _ppm: fee in ppm for successful delivery and settlement of payment :param str _sender: sender address for the payment. :param str _receiver: receiver address for the payment. :param list _attempts: returns a list of Attempts - :param list _successful_attempts: returns a list of successful Attempts :param bool _successful: returns True if the total_amount of the payment could be delivered successfully. + :param float _start: time when payment was initiated + :param float _end: time when payment was finished, either by being aborted or by successful settlement """ - def __init__(self): - self._attempts = [] - self._successful_attempts = [] + def __init__(self, sender, receiver, amount: int = 1): + self._ppm = None + self._fee = None + self._end = None + self._sender = sender + self._receiver = receiver + self._total_amount = amount + self._attempts = list() + self._start = time.time() - @property - def total_amount(self): - return self._total_amount + def __str__(self): + return "Payment with {} attempts to deliver {} sats from {} to {}".format(len(self._attempts), + self._total_amount, + self._sender[-8:], + self._receiver[-8:]) @property def src(self): @@ -31,14 +48,62 @@ def src(self): def dest(self): return self._receiver + @property + def total_amount(self): + return self._total_amount + + # can later be set at successful settlement or failure + @property + def end(self): + return self._end + @property def attempts(self): return self._attempts + def add_attempt(self, attempt: Attempt): + self._attempts.append(attempt) + @property - def successful_attempts(self): - return self._successful_attempts + def fee(self): + return self._fee + + @property + def ppm(self): + return self._ppm + + @property + def inflight_attempts(self): + inflight_attempts = [] + try: + for attempt in self._attempts: + if attempt.status == Attempt.SettlementStatus.INFLIGHT: + inflight_attempts.append(attempt) + return inflight_attempts + except ValueError: + logging.warning("ValueError in Payment.inflight_attempts") + + return inflight_attempts + + @property + def failed_attempts(self): + failed_attempts = [] + try: + for attempt in self._attempts: + if attempt.status == Attempt.SettlementStatus.FAILED: + failed_attempts.append(attempt) + return failed_attempts + except ValueError: + return [] @property def successful(self): return self._successful + + @property + def sender(self): + return self._sender + + @property + def receiver(self): + return self._receiver diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 876be4e..77b7495 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -1,3 +1,9 @@ +import logging +import sys + +from Attempt import Attempt, SettlementStatus +from Payment import Payment +from pickhardtpayments import Channel from .UncertaintyNetwork import UncertaintyNetwork from .OracleLightningNetwork import OracleLightningNetwork @@ -10,6 +16,21 @@ DEFAULT_BASE_THRESHOLD = 0 +def set_logger(): + # Set Logger + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s | %(levelname)s | %(message)s') # ,'%y-%m-%d %H:%M:%S' + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(logging.DEBUG) + stdout_handler.setFormatter(formatter) + file_handler = logging.FileHandler('pickhardt_pay.log') + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + logger.addHandler(stdout_handler) + + class SyncSimulatedPaymentSession(): """ A PaymentSesssion is used to create the min cost flow problem from the UncertaintyNetwork @@ -107,7 +128,7 @@ def _next_hop(self, path): def _make_channel_path(self, G: nx.MultiDiGraph, path: List[str]): """ network x returns a path as a list of node_ids. However we need a list of `UncertaintyChannels` - Since the graph has parallel edges it is quite some work to get the actual channels that the + Since the graph has parallel edges it is quite some work to get the actual channels that the min cost flow solver produced """ channel_path = [] @@ -128,19 +149,19 @@ def _make_channel_path(self, G: nx.MultiDiGraph, path: List[str]): return channel_path, bottleneck - def _disect_flow_to_paths(self, s, d): + def _dissect_flow_to_paths(self, s, d): """ - A standard algorithm to disect a flow into several paths. + A standard algorithm to dissect a flow into several paths. - FIXME: Note that this disection while accurate is probably not optimal in practise. + FIXME: Note that this dissection while accurate is probably not optimal in practise. As noted in our Probabilistic payment delivery paper the payment process is a bernoulli trial - and I assume it makes sense to disect the flow into paths of similar likelihood to make most + and I assume it makes sense to dissect the flow into paths of similar likelihood to make most progress but this is a mere conjecture at this point. I expect quite a bit of research will be necessary to resolve this issue. """ total_flow = {} - # first collect all linearized edges which are assigned a non zero flow put them into a networkx graph + # first collect all linearized edges which are assigned a non-zero flow put them into a networkx graph G = nx.MultiDiGraph() for i in range(self._min_cost_flow.NumArcs()): flow = self._min_cost_flow.Flow(i) # *QUANTIZATION @@ -153,14 +174,14 @@ def _disect_flow_to_paths(self, s, d): G[src][dest][channel.short_channel_id]["flow"] += flow else: # FIXME: cost is not reflecting exactly the piecewise linearization - # Probably not such a big issue as we just disect flow + # Probably not such a big issue as we just dissect flow G.add_edge(src, dest, key=channel.short_channel_id, flow=flow, channel=channel, weight=channel.combined_linearized_unit_cost()) used_flow = 1 channel_paths = [] # allocate flow to shortest / cheapest paths from src to dest as long as this is possible - # decrease flow along those edges. This is a standard mechanism to disect a flow int paths + # decrease flow along those edges. This is a standard mechanism to dissect a flow into paths while used_flow > 0: path = None try: @@ -168,7 +189,7 @@ def _disect_flow_to_paths(self, s, d): except: break channel_path, used_flow = self._make_channel_path(G, path) - channel_paths.append((channel_path, used_flow)) + channel_paths.append(Attempt(channel_path, used_flow)) # reduce the flow from the selected path for pos, hop in enumerate(self._next_hop(path)): @@ -179,21 +200,23 @@ def _disect_flow_to_paths(self, s, d): G.remove_edge(src, dest, key=channel.short_channel_id) return channel_paths - def _generate_candidate_paths(self, src, dest, amt, mu: int = 100_000_000, base: int = DEFAULT_BASE_THRESHOLD): + def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, base: int = DEFAULT_BASE_THRESHOLD): """ - computes the optimal payment split to deliver `amt` from `src` to `dest` and updates our belief about the liquidity + computes the optimal payment split to deliver `amt` from `src` to `dest` and updates our belief about the + liquidity This is one step within the payment loop. - Retuns the residual amount of the `amt` that could ne be delivered and the paid fees + Returns the residual amount of the `amt` that could not be delivered and the paid fees (on a per channel base not including fees for downstream fees) for the delivered amount - the function also prints some results an statistics about the paths of the flow to stdout. + the function also prints some results on statistics about the paths of the flow to stdout. """ + # initialisation of List of Attempts for this round. + attempts_in_round = List[Attempt] # First we prepare the min cost flow by getting arcs from the uncertainty network self._prepare_mcf_solver(src, dest, amt, mu, base) - start = time.time() # print("solving mcf...") status = self._min_cost_flow.Solve() @@ -203,37 +226,32 @@ def _generate_candidate_paths(self, src, dest, amt, mu: int = 100_000_000, base: print(f'Status: {status}') exit(1) - paths = self._disect_flow_to_paths(src, dest) + attempts_in_round = self._dissect_flow_to_paths(src, dest) end = time.time() - return paths, end - start + return attempts_in_round, end - start - def _estimate_payment_statistics(self, paths): + def _estimate_payment_statistics(self, attempts): """ estimates the success probability of paths and computes fees (without paying downstream fees) @returns the statistics in the `payments` dictionary """ - # FIXME: Decide if an `Payments` or `Attempt` class shall be used - payments = {} # compute fees and probabilities of candidate paths for evaluation - for i, onion in enumerate(paths): - path, amount = onion - fee, probability = self._uncertainty_network.get_features_of_candidate_path( - path, amount) - payments[i] = { - "routing_fee": fee, "probability": probability, "path": path, "amount": amount} + for attempt in attempts: + attempt.routing_fee, attempt.probability = self._uncertainty_network.get_features_of_candidate_path( + attempt.path, attempt.amount) + # logging.debug("fee: {attempt.routing_fee} msat, p = {attempt.probability:.4%}, amount: {attempt.amount}") - # to correctly compute conditional probabilities of non disjoint paths in the same set of paths - self._uncertainty_network.allocate_amount_on_path(path, amount) + # to correctly compute conditional probabilities of non-disjoint paths in the same set of paths + self._uncertainty_network.allocate_amount_on_path(attempt.path, attempt.amount) # remove allocated amounts for all planned onions before doing actual attempts - for key, attempt in payments.items(): + for attempt in attempts: self._uncertainty_network.allocate_amount_on_path( - attempt["path"], -attempt["amount"]) + attempt.path, -attempt.amount) - return payments - def _attempt_payments(self, payments, settled_onions): + def _attempt_payments(self, attempts: List[Attempt]): """ we attempt all planned payments and test the success against the oracle in particular this method changes - depending on the outcome of each payment - our belief about the uncertainty @@ -241,18 +259,20 @@ def _attempt_payments(self, payments, settled_onions): successful onions are collected to be transacted on the OracleNetwork if complete payment can be delivered """ # test actual payment attempts - - for key, attempt in payments.items(): + for attempt in attempts: success, erring_channel = self._oracle.send_onion( - attempt["path"], attempt["amount"]) - payments[key]["success"] = success - payments[key]["erring_channel"] = erring_channel + attempt.path, attempt.amount) if success: + attempt.status = SettlementStatus.INFLIGHT self._uncertainty_network.allocate_amount_on_path( - attempt["path"], attempt["amount"]) - settled_onions.append(payments[key]) + attempt.path, attempt.amount) + # unnecessary, because information is in attempt (Status INFLIGHT) + # settled_onions.append(payments[key]) + else: + attempt.status = SettlementStatus.FAILED + #logging.debug('Failed Channel: {}'.format(erring_channel)) - def _evaluate_attempts(self, payments): + def _evaluate_attempts(self, attempts: List[Attempt]): """ helper function to collect statistics about attempts and print them @@ -264,20 +284,22 @@ def _evaluate_attempts(self, payments): number_failed_paths = 0 expected_sats_to_deliver = 0 amt = 0 - print("\nStatistics about {} candidate onions:\n".format(len(payments))) + inflight_attempts = [] + failed_attempts = [] + print("\nStatistics about {} candidate onions:\n".format(len(attempts))) + for attempt in attempts: + if attempt.status == SettlementStatus.INFLIGHT: + inflight_attempts.append(attempt) + if attempt.status == SettlementStatus.FAILED: + failed_attempts.append(attempt) - has_failed_attempt = False print("successful attempts:") print("--------------------") - for attempt in payments.values(): - success = attempt["success"] - if success == False: - has_failed_attempt = True - continue - fee = attempt["routing_fee"] / 1000. - probability = attempt["probability"] - path = attempt["path"] - amount = attempt["amount"] + for inflight_attempt in inflight_attempts: + fee = inflight_attempt.routing_fee / 1000. + probability = inflight_attempt.probability + path = inflight_attempt.path + amount = inflight_attempt.amount amt += amount total_fees += fee expected_sats_to_deliver += probability * amount @@ -285,39 +307,34 @@ def _evaluate_attempts(self, payments): probability * 100, amount, len(path), int(fee * 1000_000 / amount))) paid_fees += fee - if has_failed_attempt: - print("\nfailed attempts:") - print("----------------") - for attempt in payments.values(): - success = attempt["success"] - if success: - continue - fee = attempt["routing_fee"] / 1000. - probability = attempt["probability"] - path = attempt["path"] - amount = attempt["amount"] - amt += amount - total_fees += fee - expected_sats_to_deliver += probability * amount - print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5} ".format( - probability * 100, amount, len(path), int(fee * 1000_000 / amount))) - number_failed_paths += 1 - residual_amt += amount + print("\nfailed attempts:") + print("----------------") + for failed_attempt in failed_attempts: + fee = failed_attempt.routing_fee / 1000. + probability = failed_attempt.probability + path = failed_attempt.path + amount = failed_attempt.amount + amt += amount + total_fees += fee + expected_sats_to_deliver += probability * amount + print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5} ".format( + probability * 100, amount, len(path), int(fee * 1000_000 / amount))) + residual_amt += amount print("\nAttempt Summary:") print("=================") - print("\nTried to deliver {:10} sats".format(amt)) + print("\nTried to deliver \t{:10} sats".format(amt)) fraction = expected_sats_to_deliver * 100. / amt print("expected to deliver {:10} sats \t({:4.2f}%)".format( int(expected_sats_to_deliver), fraction)) fraction = (amt - residual_amt) * 100. / (amt) - print("actually delivered {:10} sats \t({:4.2f}%)".format( + print("actually delivered \t{:10} sats \t({:4.2f}%)".format( amt - residual_amt, fraction)) - print("deviation: {:4.2f}".format( + print("deviation: \t\t{:4.2f}".format( (amt - residual_amt) / (expected_sats_to_deliver + 1))) - print("planned_fee: {:8.3f} sat".format(total_fees)) - print("paid fees: {:8.3f} sat".format(paid_fees)) - return residual_amt, paid_fees, len(payments), number_failed_paths + print("planned_fee: \t{:8.3f} sat".format(total_fees)) + print("paid fees: \t\t{:8.3f} sat".format(paid_fees)) + return residual_amt, paid_fees, len(attempts), len(failed_attempts) def forget_information(self): """ @@ -338,50 +355,62 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): I could not put it here as some experiments require sharing of liqudity information """ + + set_logger() + + # Setup entropy_start = self._uncertainty_network.entropy() start = time.time() full_amt = amt cnt = 0 total_fees = 0 - number_number_of_onions = 0 + number_of_onions = 0 total_number_failed_paths = 0 + # Initialise Payment + # currently with underscore to not mix up with existing variable 'payment' + _payment = Payment(src, dest, amt) + # This is the main payment loop. It is currently blocking and synchronous but may be - # implemented in a concurrent way. Also we stop after 10 rounds which is pretty arbitrary - # a better stop criteria would be if we compute infeasable flows or if the probabilities + # implemented in a concurrent way. Also, we stop after 10 rounds which is pretty arbitrary + # a better stop criteria would be if we compute infeasible flows or if the probabilities # are too low or residual amounts decrease to slowly - settled_onions = [] while amt > 0 and cnt < 10: print("Round number: ", cnt + 1) print("Try to deliver", amt, "satoshi:") - # transfer to a min cost flow problem and rund the solver - paths, runtime = self._generate_candidate_paths( - src, dest, amt, mu, base) + # transfer to a min cost flow problem and run the solver + # paths is the lists of channels, runtime the time it took to calculate all candidates in this round + attempts_in_round, runtime = self._generate_candidate_paths( + _payment.sender, _payment.receiver, amt, mu, base) # compute some statistics about candidate paths - payments = self._estimate_payment_statistics(paths) + self._estimate_payment_statistics(attempts_in_round) + + # make attempts, try to send onion and register if success or not + # update our information about the UncertaintyNetwork + self._attempt_payments(attempts_in_round) - # make attempts and update our information about the UncertaintyNetwork and track settled onions - self._attempt_payments(payments, settled_onions) + # add attempts to payment + for attempt in attempts_in_round: + _payment.attempts.append(attempt) # run some simple statistics and depict them amt, paid_fees, num_paths, number_failed_paths = self._evaluate_attempts( - payments) + attempts_in_round) print("Runtime of flow computation: {:4.2f} sec ".format(runtime)) print("\n================================================================\n") - number_number_of_onions += num_paths + number_of_onions += num_paths total_number_failed_paths += number_failed_paths total_fees += paid_fees cnt += 1 # When residual amount is 0 / enough successful onions have been found, then settle payment. Else drop onions. if amt == 0: - # print("{} onions to settle.".format(len(settled_onions))) - for onion in settled_onions: + for onion in _payment.inflight_attempts: try: - self._oracle.settle_payment(onion["path"], onion["amount"]) + self._oracle.settle_payment(onion.path, onion.amount) except Exception as e: print(e) return -1 @@ -390,14 +419,14 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): entropy_end = self._uncertainty_network.entropy() print("SUMMARY:") print("========") - print("Rounds of mcf-computations: ", cnt) - print("Number of onions sent: ", number_number_of_onions) - print("Number of failed onions: ", total_number_failed_paths) + print("Rounds of mcf-computations:\t", cnt) + print("Number of onions sent:\t\t", number_of_onions) + print("Number of failed onions:\t", total_number_failed_paths) print("Failure rate: {:4.2f}% ".format( - total_number_failed_paths * 100. / number_number_of_onions)) - print("total runtime (including inefficient memory managment): {:4.3f} sec".format( + total_number_failed_paths * 100. / number_of_onions)) + print("total runtime (including inefficient memory management): {:4.3f} sec".format( end - start)) print("Learnt entropy: {:5.2f} bits".format(entropy_start - entropy_end)) - print("Fees for successfull delivery: {:8.3f} sat --> {} ppm".format( + print("Fees for successful delivery: {:8.3f} sat --> {} ppm".format( total_fees, int(total_fees * 1000 * 1000 / full_amt))) print("used mu:", mu) From 8727759b0bf614d47a2d5593ca99e383be6d2c57 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 20 May 2022 17:34:01 +0200 Subject: [PATCH 13/38] Introduction of PaymentClass and AttemptClass, refactoring. --- pickhardtpayments/Attempt.py | 71 ++++++-- pickhardtpayments/OracleLightningNetwork.py | 5 +- pickhardtpayments/Payment.py | 164 ++++++++++++++---- .../SyncSimulatedPaymentSession.py | 84 +++++---- pickhardtpayments/UncertaintyChannel.py | 4 +- pickhardtpayments/UncertaintyNetwork.py | 20 +-- 6 files changed, 248 insertions(+), 100 deletions(-) diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py index 627bf14..7e1d8de 100644 --- a/pickhardtpayments/Attempt.py +++ b/pickhardtpayments/Attempt.py @@ -15,19 +15,23 @@ class Attempt: """ An Attempt describes a path (a set of channels) of an amount from sender to receiver. - :param list _path: a list of UncertaintyChannel objects from sender to receiver - :param int _amount: the amount to be transferred from source to destination - :param int _routing_fee: the routing fee in msat - :param float _probability: estimated success probability before the attempt - :param float _start: time of sending out the onion - :param float _end: time of getting back the onion - :param SettlementStatus _status: the flag to describe if the path failed, succeeded or was used for settlement + When sending an amount of sats from sender to receiver, a payment is usually split up and sent across + several paths, to increase the probability of being successfully delivered. Each of this path is referred to as an + Attempt. + An Attempt consists of a list of Channels (class:Channel) and the amount in sats to be sent through this path. + + :param path: a list of UncertaintyChannel objects from sender to receiver + :type path: list[UncertaintyChannel] + :param amount: the amount to be transferred from source to destination + :type amount: int """ - def __init__(self, path: list, amount: int = 0): + def __init__(self, path: list[Channel], amount: int = 0): + """Constructor method + """ self._paths = None - self._routing_fee = None - self._probability = None + self._routing_fee = -1 + self._probability = -1 if not isinstance(path, list): raise ValueError("path needs to be a collection of Channels") for channel in path: @@ -43,10 +47,20 @@ def __str__(self): @property def path(self): + """Returns the path of the attempt. + + :return: the list of UncertaintyChannels that the path consists of + :rtype: list[UncertaintyChannel] + """ return self._path @path.setter def path(self, attempts: list): + """Sets the path that was set up for a certain amount + + :param attempts: a List of Channels from UncertaintyGraph that an amount was attempted to be routed through + :type attempts: list[UncertaintyChannel] + """ if not isinstance(attempts, list): raise ValueError("path needs to be a collection of Channels") for attempt in attempts: @@ -56,28 +70,65 @@ def path(self, attempts: list): @property def amount(self): + """Returns the amount of the attempt. + + :return: the amount that was tried to send in this Attempt + :rtype: int + """ return self._amount @property def status(self): + """Returns the status of the attempt. + + :return: returns the state of the attempt + :rtype: SettlementStatus + """ return self._status @status.setter def status(self, value: SettlementStatus): + """Sets the status of the attempt. + + A flag to describe if the path failed, succeeded or was used for settlement (see enum SettlementStatus) + + :param value: Current state of the Attempt + :type value: SettlementStatus + """ self._status = value @property def routing_fee(self): + """Returns the accrued routing fee in msat requested for this path. + + :return: accrued routing fees for this attempt in msat + :rtype: int + """ return self._routing_fee @routing_fee.setter def routing_fee(self, value: int): + """Sets the accrued routing fee in msat requested for this path + + :param value: accrued routing fees for this attempt in msat + :type value: int + """ self._routing_fee = value @property def probability(self): + """Returns estimated success probability before the attempt + + :return: estimated success probability before the attempt + :rtype: float + """ return self._probability @probability.setter def probability(self, value: int): + """Sets the estimated success probability before the attempt + + :param value: estimated success probability before the attempt + :type value: float + """ self._probability = value diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index ef2c12c..8b9d052 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -1,3 +1,5 @@ +from typing import List + from .ChannelGraph import ChannelGraph from .OracleChannel import OracleChannel import networkx as nx @@ -76,7 +78,7 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base mincut, _ = nx.minimum_cut(test_network, source, destination) return mincut - def settle_payment(self, path: OracleChannel, payment_amount: int): + def settle_payment(self, path: List[OracleChannel], payment_amount: int): """ receives a dictionary with channels and payment amounts and adjusts the balances of the channels along the path. @@ -94,3 +96,4 @@ def settle_payment(self, path: OracleChannel, payment_amount: int): else: raise Exception("""Channel liquidity on Channel {} is lower than payment amount. \nPayment cannot settle.""".format(channel.short_channel_id)) + return 0 diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index 572087a..0076c9b 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -1,9 +1,9 @@ import logging import time -from typing import List import Attempt + class Payment: """ Payment stores the information about an amount of sats to be delivered from source to destination. @@ -13,26 +13,26 @@ class Payment: The PaymentClass holds all necessary information about a payment. It also holds information that helps to calculate performance statistics about the payment. - :param int _total_amount: The total amount of sats to be delivered from source address to destination address. - :param float _fee: fee in sats for successful delivery and settlement of payment - :param float _ppm: fee in ppm for successful delivery and settlement of payment - :param str _sender: sender address for the payment. - :param str _receiver: receiver address for the payment. - :param list _attempts: returns a list of Attempts - :param bool _successful: returns True if the total_amount of the payment could be delivered successfully. - :param float _start: time when payment was initiated - :param float _end: time when payment was finished, either by being aborted or by successful settlement + :param sender: sender address for the payment + :type sender: class:`str` + :param receiver: receiver address for the payment + :type receiver: class:`str` + :param total_amount: The total amount of sats to be delivered from source address to destination address. + :type total_amount: class:`int` """ - def __init__(self, sender, receiver, amount: int = 1): - self._ppm = None - self._fee = None - self._end = None + def __init__(self, sender, receiver, total_amount: int = 1): + """Constructor method + """ + self._successful = False + self._ppm = -1 + self._fee = -1 + self._end_time = -1 self._sender = sender self._receiver = receiver - self._total_amount = amount + self._total_amount = total_amount self._attempts = list() - self._start = time.time() + self._start_time = time.time() def __str__(self): return "Payment with {} attempts to deliver {} sats from {} to {}".format(len(self._attempts), @@ -41,39 +41,109 @@ def __str__(self): self._receiver[-8:]) @property - def src(self): + def sender(self): + """Returns the address of the sender of the payment. + + :return: sender address for the payment + :rtype: str + """ return self._sender @property - def dest(self): + def receiver(self): + """Returns the address of the receiver of the payment. + + :return: receiver address for the payment + :rtype: str + """ return self._receiver @property def total_amount(self): + """Returns the amount to be sent with this payment. + + :return: The total amount of sats to be delivered from source address to destination address. + :rtype: int + """ return self._total_amount - # can later be set at successful settlement or failure @property - def end(self): - return self._end + def start_time(self): + """Returns the time when Payment object was instantiated. + + :return: time in seconds from epoch to instantiation of Payment object. + :rtype: float + """ + return self._start_time + + @property + def end_time(self): + """Time when payment was finished, either by being aborted or by successful settlement + + Returns the time when all Attempts in Payment did settle or fail. + + :return: time in seconds from epoch to successful payment or failure of payment. + :rtype: float + """ + return self._end_time + + @end_time.setter + def end_time(self, timestamp): + """Set time when payment was finished, either by being aborted or by successful settlement + + Sets end_time time in seconds from epoch. Should be called when the Payment failed or when Payment + settled successfully. + + :param timestamp: time in seconds from epoch + :type timestamp: float + """ + self._end_time = timestamp @property def attempts(self): + """Returns all onions that were built and are associated with this Payment. + + :return: A list of Attempts of this payment. + :rtype: list[Attempt] + """ return self._attempts - def add_attempt(self, attempt: Attempt): - self._attempts.append(attempt) + def add_attempts(self, attempts: list[Attempt]): + """Adds Attempts (onions) that have been made to settle the Payment to the Payment object. + + :param attempts: a list of attempts that belong to this Payment + :type: list[Attempt] + """ + self._attempts.extend(attempts) @property def fee(self): - return self._fee + """Returns the fees that accrued for this payment. It's the sum of the routing fees of all settled onions. + + :return: fee in sats for successful attempts of Payment + :rtype: float + """ + fee = 0 + for attempt in self.settled_attempts: + fee += attempt.routing_fee + return fee @property def ppm(self): - return self._ppm + """Returns the fees that accrued for this payment. It's the sum of the routing fees of all settled onions. + + :return: fee in ppm for successful delivery and settlement of payment + :rtype: float + """ + return self.fee * 1000 / self.total_amount @property def inflight_attempts(self): + """Returns all onions that can successfully be routed. + + :return: A list of successful Attempts of this Payment, which could be settled. + :rtype: list[Attempt] + """ inflight_attempts = [] try: for attempt in self._attempts: @@ -82,11 +152,15 @@ def inflight_attempts(self): return inflight_attempts except ValueError: logging.warning("ValueError in Payment.inflight_attempts") - - return inflight_attempts + return [] @property def failed_attempts(self): + """Returns all onions that failed, where no delivery of the (partial) amount was possible. + + :return: A list of failed Attempts of this Payment, Attempts that could be not settled. + :rtype: list[Attempt] + """ failed_attempts = [] try: for attempt in self._attempts: @@ -94,16 +168,40 @@ def failed_attempts(self): failed_attempts.append(attempt) return failed_attempts except ValueError: + logging.warning("ValueError in Payment.failed_attempts") + return [] + + @property + def settled_attempts(self): + """Returns all onions that were successfully routed and then settled. + + :return: A list of Attempts of this Payment, which were successfully settled. + :rtype: list[Attempt] + """ + settled_attempts = [] + try: + for attempt in self._attempts: + if attempt.status == Attempt.SettlementStatus.SETTLED: + settled_attempts.append(attempt) + return settled_attempts + except ValueError: + logging.warning("ValueError in Payment.settled_attempts") return [] @property def successful(self): + """Returns True if the total_amount of the payment could be delivered successfully. + + :return: True if Payment settled successfully, else False. + :rtype: bool + """ return self._successful - @property - def sender(self): - return self._sender + @successful.setter + def successful(self, value): + """Sets flag if all inflight attempts of the payment could settle successfully. - @property - def receiver(self): - return self._receiver + :param value: True if Payment settled successfully, else False. + :type: bool + """ + self._successful = value diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 77b7495..67ae088 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -3,7 +3,6 @@ from Attempt import Attempt, SettlementStatus from Payment import Payment -from pickhardtpayments import Channel from .UncertaintyNetwork import UncertaintyNetwork from .OracleLightningNetwork import OracleLightningNetwork @@ -20,7 +19,7 @@ def set_logger(): # Set Logger logger = logging.getLogger() logger.setLevel(logging.DEBUG) - formatter = logging.Formatter('%(asctime)s | %(levelname)s | %(message)s') # ,'%y-%m-%d %H:%M:%S' + formatter = logging.Formatter('%(asctime)s.%(msecs)03d | %(levelname)s | %(message)s', datefmt='%H:%M:%S') stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setLevel(logging.DEBUG) stdout_handler.setFormatter(formatter) @@ -31,15 +30,16 @@ def set_logger(): logger.addHandler(stdout_handler) -class SyncSimulatedPaymentSession(): +# noinspection PyPep8Naming +class SyncSimulatedPaymentSession: """ - A PaymentSesssion is used to create the min cost flow problem from the UncertaintyNetwork + A PaymentSession is used to create the min cost flow problem from the UncertaintyNetwork This happens by adding several parallel arcs coming from the piece wise linearization of the UncertaintyChannel to the min_cost_flow object. The main API call ist `pickhardt_pay` which invokes a sequential loop to conduct trial and error - attmpts. The loop could easily send out all onions concurrently but this does not make sense + attempts. The loop could easily send out all onions concurrently but this does not make sense against the simulated OracleLightningNetwork. """ @@ -56,8 +56,7 @@ def _prepare_integer_indices_for_nodes(self): """ necessary for the OR-lib by google and the min cost flow solver - - let's initialize the look up tables for node_ids to integers from [0,...,#number of nodes] + let's initialize the look-up tables for node_ids to integers from [0,...,#number of nodes] this is necessary because of the API of the Google Operations Research min cost flow solver """ self._mcf_id = {} @@ -71,10 +70,11 @@ def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, """ computes the uncertainty network given our prior belief and prepares the min cost flow solver - This function can define a value for \mu to control how heavily we combine the uncertainty cost and fees - Also the function supports only taking channels into account that don't charge a base_fee higher or equal to `base` + This function can define a value for mu to control how heavily we combine the uncertainty cost and fees Also + the function supports only taking channels into account that don't charge a base_fee higher or equal to `base` - returns the instantiated min_cost_flow object from the google OR-lib that contains the piecewise linearized problem + returns the instantiated min_cost_flow object from the Google OR-lib that contains the piecewise linearized + problem """ self._min_cost_flow = pywrapgraph.SimpleMinCostFlow() self._arc_to_channel = {} @@ -84,7 +84,7 @@ def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, if channel.base_fee > base_fee: continue # FIXME: Remove Magic Number for pruning - # Prune channels away thay have too low success probability! This is a huge runtime boost + # Prune channels away that have too low success probability! This is a huge runtime boost # However the pruning would be much better to work on quantiles of normalized cost # So as soon as we have better Scaling, Centralization and feature engineering we can # probably have a more focused pruning @@ -116,18 +116,18 @@ def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, def _next_hop(self, path): """ - generator to iterate through edges indext by node id of paths + generator to iterate through edges indexed by node id of paths The path is a list of node ids. Each call returns a tuple src, dest of an edge in the path """ for i in range(1, len(path)): src = path[i - 1] dest = path[i] - yield (src, dest) + yield src, dest def _make_channel_path(self, G: nx.MultiDiGraph, path: List[str]): """ - network x returns a path as a list of node_ids. However we need a list of `UncertaintyChannels` + network x returns a path as a list of node_ids. However, we need a list of `UncertaintyChannels` Since the graph has parallel edges it is quite some work to get the actual channels that the min cost flow solver produced """ @@ -159,8 +159,6 @@ def _dissect_flow_to_paths(self, s, d): progress but this is a mere conjecture at this point. I expect quite a bit of research will be necessary to resolve this issue. """ - total_flow = {} - # first collect all linearized edges which are assigned a non-zero flow put them into a networkx graph G = nx.MultiDiGraph() for i in range(self._min_cost_flow.NumArcs()): @@ -183,7 +181,6 @@ def _dissect_flow_to_paths(self, s, d): # allocate flow to shortest / cheapest paths from src to dest as long as this is possible # decrease flow along those edges. This is a standard mechanism to dissect a flow into paths while used_flow > 0: - path = None try: path = nx.shortest_path(G, s, d) except: @@ -200,7 +197,8 @@ def _dissect_flow_to_paths(self, s, d): G.remove_edge(src, dest, key=channel.short_channel_id) return channel_paths - def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, base: int = DEFAULT_BASE_THRESHOLD): + def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, + base: int = DEFAULT_BASE_THRESHOLD): """ computes the optimal payment split to deliver `amt` from `src` to `dest` and updates our belief about the liquidity @@ -212,9 +210,6 @@ def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, the function also prints some results on statistics about the paths of the flow to stdout. """ - # initialisation of List of Attempts for this round. - attempts_in_round = List[Attempt] - # First we prepare the min cost flow by getting arcs from the uncertainty network self._prepare_mcf_solver(src, dest, amt, mu, base) start = time.time() @@ -234,7 +229,7 @@ def _estimate_payment_statistics(self, attempts): """ estimates the success probability of paths and computes fees (without paying downstream fees) - @returns the statistics in the `payments` dictionary + @returns the statistics in the Payment """ # compute fees and probabilities of candidate paths for evaluation for attempt in attempts: @@ -250,7 +245,6 @@ def _estimate_payment_statistics(self, attempts): self._uncertainty_network.allocate_amount_on_path( attempt.path, -attempt.amount) - def _attempt_payments(self, attempts: List[Attempt]): """ we attempt all planned payments and test the success against the oracle in particular this @@ -270,7 +264,6 @@ def _attempt_payments(self, attempts: List[Attempt]): # settled_onions.append(payments[key]) else: attempt.status = SettlementStatus.FAILED - #logging.debug('Failed Channel: {}'.format(erring_channel)) def _evaluate_attempts(self, attempts: List[Attempt]): """ @@ -281,7 +274,6 @@ def _evaluate_attempts(self, attempts: List[Attempt]): total_fees = 0 paid_fees = 0 residual_amt = 0 - number_failed_paths = 0 expected_sats_to_deliver = 0 amt = 0 inflight_attempts = [] @@ -327,7 +319,7 @@ def _evaluate_attempts(self, attempts: List[Attempt]): fraction = expected_sats_to_deliver * 100. / amt print("expected to deliver {:10} sats \t({:4.2f}%)".format( int(expected_sats_to_deliver), fraction)) - fraction = (amt - residual_amt) * 100. / (amt) + fraction = (amt - residual_amt) * 100. / amt print("actually delivered \t{:10} sats \t({:4.2f}%)".format( amt - residual_amt, fraction)) print("deviation: \t\t{:4.2f}".format( @@ -352,24 +344,22 @@ def activate_network_wide_uncertainty_reduction(self, n): def pickhardt_pay(self, src, dest, amt, mu=1, base=0): """ conduct one experiment! might need to call oracle.reset_uncertainty_network() first - I could not put it here as some experiments require sharing of liqudity information + I could not put it here as some experiments require sharing of liquidity information """ set_logger() + logging.info('*** new pickhardt payment ***') # Setup entropy_start = self._uncertainty_network.entropy() - start = time.time() - full_amt = amt cnt = 0 total_fees = 0 - number_of_onions = 0 total_number_failed_paths = 0 # Initialise Payment # currently with underscore to not mix up with existing variable 'payment' - _payment = Payment(src, dest, amt) + payment = Payment(src, dest, amt) # This is the main payment loop. It is currently blocking and synchronous but may be # implemented in a concurrent way. Also, we stop after 10 rounds which is pretty arbitrary @@ -382,7 +372,7 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): # transfer to a min cost flow problem and run the solver # paths is the lists of channels, runtime the time it took to calculate all candidates in this round attempts_in_round, runtime = self._generate_candidate_paths( - _payment.sender, _payment.receiver, amt, mu, base) + payment.sender, payment.receiver, amt, mu, base) # compute some statistics about candidate paths self._estimate_payment_statistics(attempts_in_round) @@ -392,41 +382,47 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): self._attempt_payments(attempts_in_round) # add attempts to payment - for attempt in attempts_in_round: - _payment.attempts.append(attempt) + payment.add_attempts(attempts_in_round) # run some simple statistics and depict them amt, paid_fees, num_paths, number_failed_paths = self._evaluate_attempts( attempts_in_round) + print("Runtime of flow computation: {:4.2f} sec ".format(runtime)) print("\n================================================================\n") - number_of_onions += num_paths total_number_failed_paths += number_failed_paths total_fees += paid_fees cnt += 1 - # When residual amount is 0 / enough successful onions have been found, then settle payment. Else drop onions. if amt == 0: - for onion in _payment.inflight_attempts: + for onion in payment.inflight_attempts: try: self._oracle.settle_payment(onion.path, onion.amount) + onion.status = SettlementStatus.SETTLED except Exception as e: print(e) return -1 + payment.successful = True + payment.end_time = time.time() - end = time.time() entropy_end = self._uncertainty_network.entropy() print("SUMMARY:") print("========") print("Rounds of mcf-computations:\t", cnt) - print("Number of onions sent:\t\t", number_of_onions) - print("Number of failed onions:\t", total_number_failed_paths) + print("Number of attempts made:\t", len(payment.attempts)) + print("Number of failed attempts:\t", len(payment.failed_attempts)) print("Failure rate: {:4.2f}% ".format( - total_number_failed_paths * 100. / number_of_onions)) - print("total runtime (including inefficient memory management): {:4.3f} sec".format( - end - start)) + len(payment.failed_attempts) * 100. / len(payment.attempts))) + print("total Payment lifetime (including inefficient memory management): {:4.3f} sec".format( + payment.end_time - payment.start_time)) print("Learnt entropy: {:5.2f} bits".format(entropy_start - entropy_end)) print("Fees for successful delivery: {:8.3f} sat --> {} ppm".format( - total_fees, int(total_fees * 1000 * 1000 / full_amt))) + payment.fee/1000, int(payment.fee * 1000 / payment.total_amount))) print("used mu:", mu) + + +# TODO cleanup +# TODO calculate total fee per payment in payment class +# TODO calculate total ppm per payment in payment class +# TODO monitor successful flag in payment diff --git a/pickhardtpayments/UncertaintyChannel.py b/pickhardtpayments/UncertaintyChannel.py index 518680a..d80e757 100644 --- a/pickhardtpayments/UncertaintyChannel.py +++ b/pickhardtpayments/UncertaintyChannel.py @@ -21,11 +21,11 @@ class UncertaintyChannel(Channel): Most importantly the class stores our belief about the liquidity information of a channel. This is done by reducing the uncertainty interval from [0,`capacity`] to [`min_liquidity`, `max_liquidity`]. - Additionally we need to know how many sats we currently have allocated via outstanding onions + Additionally, we need to know how many sats we currently have allocated via outstanding onions to the channel which is stored in `inflight`. The most important API call is the `get_piecewise_linearized_costs` function that computes the - pieceweise linearized cost for a channel rising from uncertainty as well as routing fees. + piecewise linearized cost for a channel rising from uncertainty as well as routing fees. """ TOTAL_NUMBER_OF_SATS = 21_000_000 * 100_000_000 diff --git a/pickhardtpayments/UncertaintyNetwork.py b/pickhardtpayments/UncertaintyNetwork.py index dd1d417..4337d1f 100644 --- a/pickhardtpayments/UncertaintyNetwork.py +++ b/pickhardtpayments/UncertaintyNetwork.py @@ -10,7 +10,7 @@ class UncertaintyNetwork(ChannelGraph): """ - The UncertaintayNetwork is the main data structure to store our belief about the + The UncertaintyNetwork is the main data structure to store our belief about the Liquidity in the channels of the ChannelGraph. Most of its functionality comes from the UncertaintyChannel. Most notably the ability @@ -67,10 +67,10 @@ def reset_uncertainty_network(self): def activate_network_wide_uncertainty_reduction(self, n, oracle: OracleLightningNetwork): """ - With the help of an `OracleLightningNetwork` probes all chennels `n` times to reduce uncertainty. + With the help of an `OracleLightningNetwork` probes all channels `n` times to reduce uncertainty. While one can do this on mainnet by probing we can do this very quickly in simulation - at virtually no cost. Thus this API call needs to be taken with caution when using a different + at virtually no cost. Thus, this API call needs to be taken with caution when using a different oracle. """ for src, dest, channel in self.network.edges(data="channel"): @@ -78,13 +78,13 @@ def activate_network_wide_uncertainty_reduction(self, n, oracle: OracleLightning # FIXME: refactor to new code base. The following call will break! def activate_foaf_uncertainty_reduction(self, src, dest): - ego_netwok = set() + ego_network = set() foaf_network = set() out_set = set() edges = self.__channel_graph.out_edges(self.__node_key_to_id[src]) for edge in edges: - ego_netwok.add("{}x{}".format(edge[0], edge[1])) + ego_network.add("{}x{}".format(edge[0], edge[1])) out_set.add(edge[1]) for node in out_set: @@ -95,7 +95,7 @@ def activate_foaf_uncertainty_reduction(self, src, dest): in_set = set() edges = self.__channel_graph.in_edges(self.__node_key_to_id[dest]) for edge in edges: - ego_netwok.add("{}x{}".format(edge[0], edge[1])) + ego_network.add("{}x{}".format(edge[0], edge[1])) in_set.add(edge[0]) for node in in_set: @@ -103,12 +103,12 @@ def activate_foaf_uncertainty_reduction(self, src, dest): for edge in edges: foaf_network.add("{}x{}".format(edge[0], edge[1])) - # print(len(ego_netwok)) + # print(len(ego_network)) for k, arc in self.__arcs.items(): # print(k) vals = k.split("x") key = "{}x{}".format(vals[0], vals[1]) - if key in ego_netwok: + if key in ego_network: l = arc.get_actual_liquidity() arc.update_knowledge(l-1) arc.update_knowledge(l+1) @@ -119,6 +119,6 @@ def activate_foaf_uncertainty_reduction(self, src, dest): l = arc.get_actual_liquidity() arc.update_knowledge(l-1) arc.update_knowledge(l+1) - #print(key, arc.entropy()) - print("channels with full knowlege: ", len(ego_netwok)) + # print(key, arc.entropy()) + print("channels with full knowledge: ", len(ego_network)) print("channels with 2 Bits of less entropy: ", len(foaf_network)) From f6dad4622d676e3656fa08a9f36f7cf86ddae0ef Mon Sep 17 00:00:00 2001 From: Sebastian Date: Thu, 26 May 2022 12:56:06 +0200 Subject: [PATCH 14/38] error fixing, semantic changes and refinement of Payment ant Attempt class --- pickhardtpayments/Attempt.py | 39 ++++----- pickhardtpayments/OracleChannel.py | 10 ++- pickhardtpayments/Payment.py | 84 ++++++++----------- .../SyncSimulatedPaymentSession.py | 68 +++++++-------- 4 files changed, 89 insertions(+), 112 deletions(-) diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py index 7e1d8de..a2efc50 100644 --- a/pickhardtpayments/Attempt.py +++ b/pickhardtpayments/Attempt.py @@ -3,7 +3,7 @@ from pickhardtpayments import Channel -class SettlementStatus(Enum): +class AttemptStatus(Enum): PLANNED = 1 INFLIGHT = 2 ARRIVED = 3 @@ -29,7 +29,6 @@ class Attempt: def __init__(self, path: list[Channel], amount: int = 0): """Constructor method """ - self._paths = None self._routing_fee = -1 self._probability = -1 if not isinstance(path, list): @@ -38,12 +37,16 @@ def __init__(self, path: list[Channel], amount: int = 0): if not isinstance(channel, Channel): raise ValueError("path needs to be a collection of Channels") self._path = path - self._status = SettlementStatus.PLANNED + self._status = AttemptStatus.PLANNED self._amount = amount def __str__(self): - return "Path with {} channels to deliver {} sats and status {}.".format(len(self._path), - self._amount, self._status.name) + description = "Path with {} channels to deliver {} sats and status {}.".format(len(self._path), + self._amount, self._status.name) + if self._routing_fee > 0: + description += "\nsuccess probability of {:6.2f}% , fee of {:8.3f} sat and a ppm of {:5} ".format( + self._probability * 100, self._routing_fee/1000, int(self._routing_fee * 1000 / self._amount)) + return description @property def path(self): @@ -54,20 +57,6 @@ def path(self): """ return self._path - @path.setter - def path(self, attempts: list): - """Sets the path that was set up for a certain amount - - :param attempts: a List of Channels from UncertaintyGraph that an amount was attempted to be routed through - :type attempts: list[UncertaintyChannel] - """ - if not isinstance(attempts, list): - raise ValueError("path needs to be a collection of Channels") - for attempt in attempts: - if not isinstance(attempt, Channel): - raise ValueError("path needs to be a collection of Channels") - self._paths = attempts - @property def amount(self): """Returns the amount of the attempt. @@ -82,18 +71,18 @@ def status(self): """Returns the status of the attempt. :return: returns the state of the attempt - :rtype: SettlementStatus + :rtype: AttemptStatus """ return self._status @status.setter - def status(self, value: SettlementStatus): + def status(self, value: AttemptStatus): """Sets the status of the attempt. A flag to describe if the path failed, succeeded or was used for settlement (see enum SettlementStatus) :param value: Current state of the Attempt - :type value: SettlementStatus + :type value: AttemptStatus """ self._status = value @@ -126,9 +115,11 @@ def probability(self): @probability.setter def probability(self, value: int): - """Sets the estimated success probability before the attempt + """Sets the estimated success probability of the attempt. + + This is calculated as product of the channels' success probabilities as determined in the UncertaintyGraph. - :param value: estimated success probability before the attempt + :param value: estimated success probability of the attempt :type value: float """ self._probability = value diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index 61e59fa..65d7c31 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -31,7 +31,15 @@ def actual_liquidity(self): @actual_liquidity.setter def actual_liquidity(self, amt: int): - self._actual_liquidity = amt + """Sets the liquidity of a channel in the Oracle Network + + :param amt: amount to be assigned to channel liquidity + :type amt: int + """ + if 0 <= amt <= self.capacity: + self._actual_liquidity = amt + else: + raise ValueError("Oops! The amount to be assigned to channel liquidity is negative or higher than capacity") def can_forward(self, amt: int): """ diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index 0076c9b..b4c11f8 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -25,14 +25,14 @@ def __init__(self, sender, receiver, total_amount: int = 1): """Constructor method """ self._successful = False - self._ppm = -1 - self._fee = -1 - self._end_time = -1 + self._ppm = None + self._fee = None + self._start_time = time.time() + self._end_time = None self._sender = sender self._receiver = receiver self._total_amount = total_amount self._attempts = list() - self._start_time = time.time() def __str__(self): return "Payment with {} attempts to deliver {} sats from {} to {}".format(len(self._attempts), @@ -117,16 +117,30 @@ def add_attempts(self, attempts: list[Attempt]): self._attempts.extend(attempts) @property - def fee(self): + def settlement_fees(self): """Returns the fees that accrued for this payment. It's the sum of the routing fees of all settled onions. :return: fee in sats for successful attempts of Payment :rtype: float """ - fee = 0 - for attempt in self.settled_attempts: - fee += attempt.routing_fee - return fee + settlement_fees = 0 + for attempt in self.filter_attempts(Attempt.AttemptStatus.SETTLED): + settlement_fees += attempt.routing_fee + return settlement_fees + + @property + def planned_fees(self): + """Returns the fees for all Attempts for this payment, that are still outstanding/planned. + + It's the sum of the routing fees of all planned attempts. + + :return: fee in sats for planned attempts of Payment + :rtype: float + """ + planned_fees = 0 + for attempt in self.filter_attempts(Attempt.AttemptStatus.PLANNED): + planned_fees += attempt.routing_fee + return planned_fees @property def ppm(self): @@ -137,57 +151,25 @@ def ppm(self): """ return self.fee * 1000 / self.total_amount - @property - def inflight_attempts(self): - """Returns all onions that can successfully be routed. + def filter_attempts(self, flag: Attempt.AttemptStatus): + """Returns all onions with the given state. + + :param flag: the state of the attempts that sould be filtered for + :type: Attempt.AttemptStatus :return: A list of successful Attempts of this Payment, which could be settled. :rtype: list[Attempt] """ - inflight_attempts = [] + filtered_attempts = [] try: for attempt in self._attempts: - if attempt.status == Attempt.SettlementStatus.INFLIGHT: - inflight_attempts.append(attempt) - return inflight_attempts + if attempt.status == flag: + filtered_attempts.append(attempt) + return filtered_attempts except ValueError: - logging.warning("ValueError in Payment.inflight_attempts") + logging.warning("ValueError in Payment.filtered_attempts") return [] - @property - def failed_attempts(self): - """Returns all onions that failed, where no delivery of the (partial) amount was possible. - - :return: A list of failed Attempts of this Payment, Attempts that could be not settled. - :rtype: list[Attempt] - """ - failed_attempts = [] - try: - for attempt in self._attempts: - if attempt.status == Attempt.SettlementStatus.FAILED: - failed_attempts.append(attempt) - return failed_attempts - except ValueError: - logging.warning("ValueError in Payment.failed_attempts") - return [] - - @property - def settled_attempts(self): - """Returns all onions that were successfully routed and then settled. - - :return: A list of Attempts of this Payment, which were successfully settled. - :rtype: list[Attempt] - """ - settled_attempts = [] - try: - for attempt in self._attempts: - if attempt.status == Attempt.SettlementStatus.SETTLED: - settled_attempts.append(attempt) - return settled_attempts - except ValueError: - logging.warning("ValueError in Payment.settled_attempts") - return [] - @property def successful(self): """Returns True if the total_amount of the payment could be delivered successfully. diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 67ae088..3495895 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -1,7 +1,7 @@ import logging import sys -from Attempt import Attempt, SettlementStatus +from Attempt import Attempt, AttemptStatus from Payment import Payment from .UncertaintyNetwork import UncertaintyNetwork from .OracleLightningNetwork import OracleLightningNetwork @@ -176,7 +176,7 @@ def _dissect_flow_to_paths(self, s, d): G.add_edge(src, dest, key=channel.short_channel_id, flow=flow, channel=channel, weight=channel.combined_linearized_unit_cost()) used_flow = 1 - channel_paths = [] + attempts = [] # allocate flow to shortest / cheapest paths from src to dest as long as this is possible # decrease flow along those edges. This is a standard mechanism to dissect a flow into paths @@ -186,7 +186,7 @@ def _dissect_flow_to_paths(self, s, d): except: break channel_path, used_flow = self._make_channel_path(G, path) - channel_paths.append(Attempt(channel_path, used_flow)) + attempts.append(Attempt(channel_path, used_flow)) # reduce the flow from the selected path for pos, hop in enumerate(self._next_hop(path)): @@ -195,7 +195,7 @@ def _dissect_flow_to_paths(self, s, d): G[src][dest][channel.short_channel_id]["flow"] -= used_flow if G[src][dest][channel.short_channel_id]["flow"] == 0: G.remove_edge(src, dest, key=channel.short_channel_id) - return channel_paths + return attempts def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, base: int = DEFAULT_BASE_THRESHOLD): @@ -225,7 +225,7 @@ def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, end = time.time() return attempts_in_round, end - start - def _estimate_payment_statistics(self, attempts): + def _estimate_payment_statistics(self, attempts: List[Attempt]): """ estimates the success probability of paths and computes fees (without paying downstream fees) @@ -257,13 +257,13 @@ def _attempt_payments(self, attempts: List[Attempt]): success, erring_channel = self._oracle.send_onion( attempt.path, attempt.amount) if success: - attempt.status = SettlementStatus.INFLIGHT + attempt.status = AttemptStatus.ARRIVED self._uncertainty_network.allocate_amount_on_path( attempt.path, attempt.amount) # unnecessary, because information is in attempt (Status INFLIGHT) # settled_onions.append(payments[key]) else: - attempt.status = SettlementStatus.FAILED + attempt.status = AttemptStatus.FAILED def _evaluate_attempts(self, attempts: List[Attempt]): """ @@ -276,22 +276,22 @@ def _evaluate_attempts(self, attempts: List[Attempt]): residual_amt = 0 expected_sats_to_deliver = 0 amt = 0 - inflight_attempts = [] + arrived_attempts = [] failed_attempts = [] print("\nStatistics about {} candidate onions:\n".format(len(attempts))) for attempt in attempts: - if attempt.status == SettlementStatus.INFLIGHT: - inflight_attempts.append(attempt) - if attempt.status == SettlementStatus.FAILED: + if attempt.status == AttemptStatus.ARRIVED: + arrived_attempts.append(attempt) + if attempt.status == AttemptStatus.FAILED: failed_attempts.append(attempt) print("successful attempts:") print("--------------------") - for inflight_attempt in inflight_attempts: - fee = inflight_attempt.routing_fee / 1000. - probability = inflight_attempt.probability - path = inflight_attempt.path - amount = inflight_attempt.amount + for arrived_attempt in arrived_attempts: + fee = arrived_attempt.routing_fee / 1000. + probability = arrived_attempt.probability + path = arrived_attempt.path + amount = arrived_attempt.amount amt += amount total_fees += fee expected_sats_to_deliver += probability * amount @@ -369,24 +369,22 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): print("Round number: ", cnt + 1) print("Try to deliver", amt, "satoshi:") + sub_payment = Payment(payment.sender, payment.receiver, amt) # transfer to a min cost flow problem and run the solver # paths is the lists of channels, runtime the time it took to calculate all candidates in this round - attempts_in_round, runtime = self._generate_candidate_paths( + paths, runtime = self._generate_candidate_paths( payment.sender, payment.receiver, amt, mu, base) + sub_payment.add_attempts(paths) # compute some statistics about candidate paths - self._estimate_payment_statistics(attempts_in_round) - + self._estimate_payment_statistics(sub_payment.attempts) # make attempts, try to send onion and register if success or not # update our information about the UncertaintyNetwork - self._attempt_payments(attempts_in_round) - - # add attempts to payment - payment.add_attempts(attempts_in_round) + self._attempt_payments(sub_payment.attempts) # run some simple statistics and depict them amt, paid_fees, num_paths, number_failed_paths = self._evaluate_attempts( - attempts_in_round) + sub_payment.attempts) print("Runtime of flow computation: {:4.2f} sec ".format(runtime)) print("\n================================================================\n") @@ -394,12 +392,16 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): total_number_failed_paths += number_failed_paths total_fees += paid_fees cnt += 1 + + # add attempts of sub_payment to payment + payment.add_attempts(sub_payment.attempts) + # When residual amount is 0 / enough successful onions have been found, then settle payment. Else drop onions. if amt == 0: - for onion in payment.inflight_attempts: + for onion in payment.filter_attempts(AttemptStatus.ARRIVED): try: self._oracle.settle_payment(onion.path, onion.amount) - onion.status = SettlementStatus.SETTLED + onion.status = AttemptStatus.SETTLED except Exception as e: print(e) return -1 @@ -411,18 +413,12 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): print("========") print("Rounds of mcf-computations:\t", cnt) print("Number of attempts made:\t", len(payment.attempts)) - print("Number of failed attempts:\t", len(payment.failed_attempts)) + print("Number of failed attempts:\t", len(payment.filter_attempts(AttemptStatus.FAILED))) print("Failure rate: {:4.2f}% ".format( - len(payment.failed_attempts) * 100. / len(payment.attempts))) + len(payment.filter_attempts(AttemptStatus.FAILED)) * 100. / len(payment.attempts))) print("total Payment lifetime (including inefficient memory management): {:4.3f} sec".format( payment.end_time - payment.start_time)) print("Learnt entropy: {:5.2f} bits".format(entropy_start - entropy_end)) - print("Fees for successful delivery: {:8.3f} sat --> {} ppm".format( - payment.fee/1000, int(payment.fee * 1000 / payment.total_amount))) + print("fee for settlement of delivery: {:8.3f} sat --> {} ppm".format( + payment.settlement_fees/1000, int(payment.settlement_fees * 1000 / payment.total_amount))) print("used mu:", mu) - - -# TODO cleanup -# TODO calculate total fee per payment in payment class -# TODO calculate total ppm per payment in payment class -# TODO monitor successful flag in payment From 40967ffb12348511a42258ec1aa7e24849254613 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Thu, 26 May 2022 14:07:12 +0200 Subject: [PATCH 15/38] adding types in methods for Attempt and Payment Class and amending relative paths on imports --- pickhardtpayments/Attempt.py | 21 +++++++--------- pickhardtpayments/Channel.py | 4 ++-- pickhardtpayments/OracleLightningNetwork.py | 4 ++-- pickhardtpayments/Payment.py | 24 +++++++++---------- .../SyncSimulatedPaymentSession.py | 17 +++++++------ pickhardtpayments/UncertaintyChannel.py | 4 ++-- pickhardtpayments/UncertaintyNetwork.py | 8 ++++--- 7 files changed, 39 insertions(+), 43 deletions(-) diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py index a2efc50..b73dd48 100644 --- a/pickhardtpayments/Attempt.py +++ b/pickhardtpayments/Attempt.py @@ -1,6 +1,6 @@ from enum import Enum -from pickhardtpayments import Channel +import UncertaintyChannel class AttemptStatus(Enum): @@ -26,17 +26,12 @@ class Attempt: :type amount: int """ - def __init__(self, path: list[Channel], amount: int = 0): + def __init__(self, path: list[UncertaintyChannel], amount: int = 0): """Constructor method """ self._routing_fee = -1 self._probability = -1 - if not isinstance(path, list): - raise ValueError("path needs to be a collection of Channels") - for channel in path: - if not isinstance(channel, Channel): - raise ValueError("path needs to be a collection of Channels") - self._path = path + self._path = path self._status = AttemptStatus.PLANNED self._amount = amount @@ -49,7 +44,7 @@ def __str__(self): return description @property - def path(self): + def path(self) -> list[UncertaintyChannel]: """Returns the path of the attempt. :return: the list of UncertaintyChannels that the path consists of @@ -58,7 +53,7 @@ def path(self): return self._path @property - def amount(self): + def amount(self) -> int: """Returns the amount of the attempt. :return: the amount that was tried to send in this Attempt @@ -67,7 +62,7 @@ def amount(self): return self._amount @property - def status(self): + def status(self) -> AttemptStatus: """Returns the status of the attempt. :return: returns the state of the attempt @@ -87,7 +82,7 @@ def status(self, value: AttemptStatus): self._status = value @property - def routing_fee(self): + def routing_fee(self) -> int: """Returns the accrued routing fee in msat requested for this path. :return: accrued routing fees for this attempt in msat @@ -105,7 +100,7 @@ def routing_fee(self, value: int): self._routing_fee = value @property - def probability(self): + def probability(self) -> float: """Returns estimated success probability before the attempt :return: estimated success probability before the attempt diff --git a/pickhardtpayments/Channel.py b/pickhardtpayments/Channel.py index 3dbd4d0..055face 100644 --- a/pickhardtpayments/Channel.py +++ b/pickhardtpayments/Channel.py @@ -1,4 +1,4 @@ -class ChannelFields(): +class ChannelFields: """ These are the values describing public data about channels that is either available via gossip or via the Bitcoin Blockchain. Their format is taken from the c-lighting @@ -21,7 +21,7 @@ class ChannelFields(): SHORT_CHANNEL_ID = 'short_channel_id' -class Channel(): +class Channel: """ Stores the public available information of a channel. diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 8b9d052..7834e11 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -1,7 +1,7 @@ from typing import List -from .ChannelGraph import ChannelGraph -from .OracleChannel import OracleChannel +from pickhardtpayments.ChannelGraph import ChannelGraph +from pickhardtpayments.OracleChannel import OracleChannel import networkx as nx DEFAULT_BASE_THRESHOLD = 0 diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index b4c11f8..3d2cb8b 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -41,7 +41,7 @@ def __str__(self): self._receiver[-8:]) @property - def sender(self): + def sender(self) -> str: """Returns the address of the sender of the payment. :return: sender address for the payment @@ -50,7 +50,7 @@ def sender(self): return self._sender @property - def receiver(self): + def receiver(self) -> str: """Returns the address of the receiver of the payment. :return: receiver address for the payment @@ -59,7 +59,7 @@ def receiver(self): return self._receiver @property - def total_amount(self): + def total_amount(self) -> int: """Returns the amount to be sent with this payment. :return: The total amount of sats to be delivered from source address to destination address. @@ -68,7 +68,7 @@ def total_amount(self): return self._total_amount @property - def start_time(self): + def start_time(self) -> float: """Returns the time when Payment object was instantiated. :return: time in seconds from epoch to instantiation of Payment object. @@ -77,7 +77,7 @@ def start_time(self): return self._start_time @property - def end_time(self): + def end_time(self) -> float: """Time when payment was finished, either by being aborted or by successful settlement Returns the time when all Attempts in Payment did settle or fail. @@ -100,7 +100,7 @@ def end_time(self, timestamp): self._end_time = timestamp @property - def attempts(self): + def attempts(self) -> list[Attempt]: """Returns all onions that were built and are associated with this Payment. :return: A list of Attempts of this payment. @@ -117,7 +117,7 @@ def add_attempts(self, attempts: list[Attempt]): self._attempts.extend(attempts) @property - def settlement_fees(self): + def settlement_fees(self) -> float: """Returns the fees that accrued for this payment. It's the sum of the routing fees of all settled onions. :return: fee in sats for successful attempts of Payment @@ -129,7 +129,7 @@ def settlement_fees(self): return settlement_fees @property - def planned_fees(self): + def planned_fees(self) -> float: """Returns the fees for all Attempts for this payment, that are still outstanding/planned. It's the sum of the routing fees of all planned attempts. @@ -143,7 +143,7 @@ def planned_fees(self): return planned_fees @property - def ppm(self): + def ppm(self) -> float: """Returns the fees that accrued for this payment. It's the sum of the routing fees of all settled onions. :return: fee in ppm for successful delivery and settlement of payment @@ -151,10 +151,10 @@ def ppm(self): """ return self.fee * 1000 / self.total_amount - def filter_attempts(self, flag: Attempt.AttemptStatus): + def filter_attempts(self, flag: Attempt.AttemptStatus) -> list[Attempt]: """Returns all onions with the given state. - :param flag: the state of the attempts that sould be filtered for + :param flag: the state of the attempts that should be filtered for :type: Attempt.AttemptStatus :return: A list of successful Attempts of this Payment, which could be settled. @@ -171,7 +171,7 @@ def filter_attempts(self, flag: Attempt.AttemptStatus): return [] @property - def successful(self): + def successful(self) -> bool: """Returns True if the total_amount of the payment could be delivered successfully. :return: True if Payment settled successfully, else False. diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 3495895..4ad3a56 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -1,14 +1,13 @@ import logging import sys -from Attempt import Attempt, AttemptStatus -from Payment import Payment -from .UncertaintyNetwork import UncertaintyNetwork -from .OracleLightningNetwork import OracleLightningNetwork +from pickhardtpayments.Attempt import Attempt, AttemptStatus +from pickhardtpayments.Payment import Payment +from pickhardtpayments.UncertaintyNetwork import UncertaintyNetwork +from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork from ortools.graph import pywrapgraph -from typing import List import time import networkx as nx @@ -125,7 +124,7 @@ def _next_hop(self, path): dest = path[i] yield src, dest - def _make_channel_path(self, G: nx.MultiDiGraph, path: List[str]): + def _make_channel_path(self, G: nx.MultiDiGraph, path: list[str]): """ network x returns a path as a list of node_ids. However, we need a list of `UncertaintyChannels` Since the graph has parallel edges it is quite some work to get the actual channels that the @@ -225,7 +224,7 @@ def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, end = time.time() return attempts_in_round, end - start - def _estimate_payment_statistics(self, attempts: List[Attempt]): + def _estimate_payment_statistics(self, attempts: list[Attempt]): """ estimates the success probability of paths and computes fees (without paying downstream fees) @@ -245,7 +244,7 @@ def _estimate_payment_statistics(self, attempts: List[Attempt]): self._uncertainty_network.allocate_amount_on_path( attempt.path, -attempt.amount) - def _attempt_payments(self, attempts: List[Attempt]): + def _attempt_payments(self, attempts: list[Attempt]): """ we attempt all planned payments and test the success against the oracle in particular this method changes - depending on the outcome of each payment - our belief about the uncertainty @@ -265,7 +264,7 @@ def _attempt_payments(self, attempts: List[Attempt]): else: attempt.status = AttemptStatus.FAILED - def _evaluate_attempts(self, attempts: List[Attempt]): + def _evaluate_attempts(self, attempts: list[Attempt]): """ helper function to collect statistics about attempts and print them diff --git a/pickhardtpayments/UncertaintyChannel.py b/pickhardtpayments/UncertaintyChannel.py index d80e757..acd25fb 100644 --- a/pickhardtpayments/UncertaintyChannel.py +++ b/pickhardtpayments/UncertaintyChannel.py @@ -1,5 +1,5 @@ -from .Channel import Channel -from .OracleLightningNetwork import OracleLightningNetwork +from pickhardtpayments.Channel import Channel +from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork from math import log2 as log diff --git a/pickhardtpayments/UncertaintyNetwork.py b/pickhardtpayments/UncertaintyNetwork.py index 4337d1f..8829da1 100644 --- a/pickhardtpayments/UncertaintyNetwork.py +++ b/pickhardtpayments/UncertaintyNetwork.py @@ -1,6 +1,7 @@ -from .ChannelGraph import ChannelGraph -from .UncertaintyChannel import UncertaintyChannel -from .OracleLightningNetwork import OracleLightningNetwork +from pickhardtpayments.ChannelGraph import ChannelGraph +from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork +from pickhardtpayments.UncertaintyChannel import UncertaintyChannel + from typing import List import networkx as nx @@ -43,6 +44,7 @@ def entropy(self): def get_features_of_candidate_path(self, path: List[UncertaintyChannel], amt: int) -> (float, float): """ returns the routing fees and probability of a candidate path + :rtype: object """ probability = 1 routing_fees = 0 From 5903cd2379ae4c8e9233d2be7b7fef3f1661ee2d Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 29 May 2022 13:22:59 +0200 Subject: [PATCH 16/38] initializes fee and probability w/ none (instead of -1) #20 --- pickhardtpayments/Attempt.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py index b73dd48..ef13b4c 100644 --- a/pickhardtpayments/Attempt.py +++ b/pickhardtpayments/Attempt.py @@ -1,6 +1,6 @@ from enum import Enum -import UncertaintyChannel +from pickhardtpayments import Channel class AttemptStatus(Enum): @@ -20,17 +20,17 @@ class Attempt: Attempt. An Attempt consists of a list of Channels (class:Channel) and the amount in sats to be sent through this path. - :param path: a list of UncertaintyChannel objects from sender to receiver - :type path: list[UncertaintyChannel] + :param path: a list of Channel objects from sender to receiver + :type path: list[Channel] :param amount: the amount to be transferred from source to destination :type amount: int """ - def __init__(self, path: list[UncertaintyChannel], amount: int = 0): + def __init__(self, path: list[Channel], amount: int = 0): """Constructor method """ - self._routing_fee = -1 - self._probability = -1 + self._routing_fee = None + self._probability = None self._path = path self._status = AttemptStatus.PLANNED self._amount = amount @@ -44,11 +44,11 @@ def __str__(self): return description @property - def path(self) -> list[UncertaintyChannel]: + def path(self) -> list[Channel]: """Returns the path of the attempt. - :return: the list of UncertaintyChannels that the path consists of - :rtype: list[UncertaintyChannel] + :return: the list of Channels that the path consists of + :rtype: list[Channel] """ return self._path From c1c61a0ff8ede50033522184be7cce8ec551e390 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 29 May 2022 13:25:12 +0200 Subject: [PATCH 17/38] adjustment of relative paths for modules in import --- examples/basicexample.py | 8 ++++---- pickhardtpayments/ChannelGraph.py | 8 ++++---- pickhardtpayments/OracleChannel.py | 2 +- pickhardtpayments/OracleLightningNetwork.py | 3 ++- pickhardtpayments/Payment.py | 8 ++++---- pickhardtpayments/SyncSimulatedPaymentSession.py | 16 +++++++++++----- pickhardtpayments/__init__.py | 6 +++--- 7 files changed, 29 insertions(+), 22 deletions(-) diff --git a/examples/basicexample.py b/examples/basicexample.py index 8823ad8..2dc412a 100644 --- a/examples/basicexample.py +++ b/examples/basicexample.py @@ -1,7 +1,7 @@ -from pickhardtpayments.ChannelGraph import ChannelGraph -from pickhardtpayments.UncertaintyNetwork import UncertaintyNetwork -from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork -from pickhardtpayments.SyncSimulatedPaymentSession import SyncSimulatedPaymentSession +from ChannelGraph import ChannelGraph +from UncertaintyNetwork import UncertaintyNetwork +from OracleLightningNetwork import OracleLightningNetwork +from SyncSimulatedPaymentSession import SyncSimulatedPaymentSession #we first need to import the chanenl graph from c-lightning jsondump diff --git a/pickhardtpayments/ChannelGraph.py b/pickhardtpayments/ChannelGraph.py index 0c8a865..c2de74e 100644 --- a/pickhardtpayments/ChannelGraph.py +++ b/pickhardtpayments/ChannelGraph.py @@ -1,21 +1,21 @@ import networkx as nx import json -from .Channel import Channel +from Channel import Channel -class ChannelGraph(): +class ChannelGraph: """ Represents the public information about the Lightning Network that we see from Gossip and the Bitcoin Blockchain. - The channels of the Channel Graph are directed and identiried uniquly by a triple consisting of + The channels of the Channel Graph are directed and identified uniquely by a triple consisting of (source_node_id, destination_node_id, short_channel_id). This allows the ChannelGraph to also contain parallel channels. """ def _get_channel_json(self, filename: str): """ - extracts the dictionary from the file that contains lightnig-cli listchannels json string + extracts the dictionary from the file that contains lightning-cli listchannels json string """ f = open(filename) return json.load(f)["channels"] diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index 65d7c31..524c895 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -1,4 +1,4 @@ -from .Channel import Channel +from Channel import Channel import random diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 7834e11..664b008 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -1,5 +1,6 @@ from typing import List +from pickhardtpayments.Channel import Channel from pickhardtpayments.ChannelGraph import ChannelGraph from pickhardtpayments.OracleChannel import OracleChannel import networkx as nx @@ -78,7 +79,7 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base mincut, _ = nx.minimum_cut(test_network, source, destination) return mincut - def settle_payment(self, path: List[OracleChannel], payment_amount: int): + def settle_payment(self, path: List[Channel], payment_amount: int): """ receives a dictionary with channels and payment amounts and adjusts the balances of the channels along the path. diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index 3d2cb8b..f735433 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -1,7 +1,7 @@ import logging import time -import Attempt +from Attempt import Attempt, AttemptStatus class Payment: @@ -124,7 +124,7 @@ def settlement_fees(self) -> float: :rtype: float """ settlement_fees = 0 - for attempt in self.filter_attempts(Attempt.AttemptStatus.SETTLED): + for attempt in self.filter_attempts(AttemptStatus.SETTLED): settlement_fees += attempt.routing_fee return settlement_fees @@ -138,7 +138,7 @@ def planned_fees(self) -> float: :rtype: float """ planned_fees = 0 - for attempt in self.filter_attempts(Attempt.AttemptStatus.PLANNED): + for attempt in self.filter_attempts(AttemptStatus.PLANNED): planned_fees += attempt.routing_fee return planned_fees @@ -151,7 +151,7 @@ def ppm(self) -> float: """ return self.fee * 1000 / self.total_amount - def filter_attempts(self, flag: Attempt.AttemptStatus) -> list[Attempt]: + def filter_attempts(self, flag: AttemptStatus) -> list[Attempt]: """Returns all onions with the given state. :param flag: the state of the attempts that should be filtered for diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 4ad3a56..84305ac 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -1,10 +1,18 @@ +""" +SyncSimulatedPaymentSession.py +==================================== +The core module of the pickhardt payment project. +An example payment is executed and statistics are run. +""" + import logging import sys from pickhardtpayments.Attempt import Attempt, AttemptStatus from pickhardtpayments.Payment import Payment -from pickhardtpayments.UncertaintyNetwork import UncertaintyNetwork -from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork +from UncertaintyNetwork import UncertaintyNetwork +from OracleLightningNetwork import OracleLightningNetwork + from ortools.graph import pywrapgraph @@ -29,7 +37,6 @@ def set_logger(): logger.addHandler(stdout_handler) -# noinspection PyPep8Naming class SyncSimulatedPaymentSession: """ A PaymentSession is used to create the min cost flow problem from the UncertaintyNetwork @@ -371,8 +378,7 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): sub_payment = Payment(payment.sender, payment.receiver, amt) # transfer to a min cost flow problem and run the solver # paths is the lists of channels, runtime the time it took to calculate all candidates in this round - paths, runtime = self._generate_candidate_paths( - payment.sender, payment.receiver, amt, mu, base) + paths, runtime = self._generate_candidate_paths(payment.sender, payment.receiver, amt, mu, base) sub_payment.add_attempts(paths) # compute some statistics about candidate paths diff --git a/pickhardtpayments/__init__.py b/pickhardtpayments/__init__.py index b956334..09f14bd 100644 --- a/pickhardtpayments/__init__.py +++ b/pickhardtpayments/__init__.py @@ -1,6 +1,6 @@ -from .Channel import Channel, ChannelFields -from .UncertaintyChannel import UncertaintyChannel -from .OracleChannel import OracleChannel +from pickhardtpayments.Channel import Channel, ChannelFields +from pickhardtpayments.UncertaintyChannel import UncertaintyChannel +from pickhardtpayments.OracleChannel import OracleChannel __version__ = "0.0.2" __all__ = [ From 54e905c77d4467b7f3fd9dc44ac132c40a60a04c Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 29 May 2022 16:38:50 +0200 Subject: [PATCH 18/38] extending .gitignore --- .gitgnore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitgnore b/.gitgnore index fa47571..67c35ee 100644 --- a/.gitgnore +++ b/.gitgnore @@ -2,6 +2,10 @@ __pycache__ .idea pickhardtpayments/pickhardtpayments.egg-info +pickhardtpayments/__pycache__ # not including large file with Channel Graph -listchannels*.json \ No newline at end of file +listchannels*.json + +# Help Files - not yet included +docs From 3e69225200da902153119d50f7e3ac606d53ec25 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 29 May 2022 17:04:47 +0200 Subject: [PATCH 19/38] correcting tipo for gitignore --- .gitgnore | 11 ----------- .gitignore | 12 ++++++++---- 2 files changed, 8 insertions(+), 15 deletions(-) delete mode 100644 .gitgnore diff --git a/.gitgnore b/.gitgnore deleted file mode 100644 index 67c35ee..0000000 --- a/.gitgnore +++ /dev/null @@ -1,11 +0,0 @@ -.DS_Store -__pycache__ -.idea -pickhardtpayments/pickhardtpayments.egg-info -pickhardtpayments/__pycache__ - -# not including large file with Channel Graph -listchannels*.json - -# Help Files - not yet included -docs diff --git a/.gitignore b/.gitignore index 5fca83e..67c35ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ .DS_Store -.idea __pycache__ -docs +.idea +pickhardtpayments/pickhardtpayments.egg-info +pickhardtpayments/__pycache__ -# too large to upload -listchannels*.json \ No newline at end of file +# not including large file with Channel Graph +listchannels*.json + +# Help Files - not yet included +docs From bf14533c077831038d9d07c821ceaa9c9292f7cf Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 29 May 2022 17:28:12 +0200 Subject: [PATCH 20/38] init for documentation --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 67c35ee..972cd48 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,3 @@ pickhardtpayments/__pycache__ # not including large file with Channel Graph listchannels*.json - -# Help Files - not yet included -docs From 97d01cb7e5e9fe07d5cf6dd7d218c7b3edef32a9 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 29 May 2022 18:14:39 +0200 Subject: [PATCH 21/38] configuration updated --- docs/source/conf.py | 243 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 docs/source/conf.py diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..9e20375 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,243 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../../')) +from datetime import datetime +import subprocess + +# -- Project information ----------------------------------------------------- + +project = 'Pickhardt Payments Package' +copyright = '2022, Rene Pickhardt' +author = 'Rene Pickhardt' + +# The full version, including alpha/beta/rc tags +# release = '0.0.0' +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = subprocess.check_output('git describe --always --dirty=-modded --abbrev=7'.split()).decode('ASCII') +# The full version, including alpha/beta/rc tags. +release = version + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.duration', + 'sphinx.ext.doctest', + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.intersphinx', +] + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3/', None), + 'sphinx': ('https://www.sphinx-doc.org/en/master/', None), +} + +intersphinx_disabled_domains = ['std'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ['.rst', '.md'] + +# The encoding of source files. +# +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'release-notes'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# +# show_authors = False + +# The name of the Pygments style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. +# " v documentation" by default. +# +html_title = 'Pickhardt Payments ' + version + +# A shorter title for the navigation bar. Default is the same as html_title. +# +html_short_title = html_title + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# -- Options for EPUB output +epub_show_urls = 'footnote' + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If not None, a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +# The empty string is equivalent to '%b %d, %Y'. +# +html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# +# html_additional_pages = {} + +# If false, no module index is generated. +# +# html_domain_indices = True + +# If false, no index is generated. +# +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# +html_show_sourcelink = False + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'pickhardtpaymentsdoc' \ No newline at end of file From c8bfeb89b0d13b35238e5dcda2e1843aab109d56 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 29 May 2022 23:29:30 +0200 Subject: [PATCH 22/38] correction in filter function for Attempt.Status #24 --- pickhardtpayments/Payment.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index f735433..602dc5f 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -129,16 +129,16 @@ def settlement_fees(self) -> float: return settlement_fees @property - def planned_fees(self) -> float: - """Returns the fees for all Attempts for this payment, that are still outstanding/planned. + def arrived_fees(self) -> float: + """Returns the fees for all Attempts/onions for this payment, that arrived but have not yet been settled. - It's the sum of the routing fees of all planned attempts. + It's the sum of the routing fees of all arrived attempts. - :return: fee in sats for planned attempts of Payment + :return: fee in sats for arrived attempts of Payment :rtype: float """ planned_fees = 0 - for attempt in self.filter_attempts(AttemptStatus.PLANNED): + for attempt in self.filter_attempts(AttemptStatus.ARRIVED): planned_fees += attempt.routing_fee return planned_fees @@ -163,7 +163,7 @@ def filter_attempts(self, flag: AttemptStatus) -> list[Attempt]: filtered_attempts = [] try: for attempt in self._attempts: - if attempt.status == flag: + if attempt.status.value == flag.value: filtered_attempts.append(attempt) return filtered_attempts except ValueError: From fd232b50ebe693c83486f759d01dc58d64836c7c Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 18 Jun 2022 18:15:15 +0200 Subject: [PATCH 23/38] changing imports back to initial call --- examples/basicexample.py | 8 ++++---- pickhardtpayments/ChannelGraph.py | 2 +- pickhardtpayments/OracleChannel.py | 2 +- pickhardtpayments/OracleLightningNetwork.py | 7 +++---- .../SyncSimulatedPaymentSession.py | 8 ++++---- pickhardtpayments/UncertaintyChannel.py | 4 ++-- pickhardtpayments/UncertaintyNetwork.py | 6 +++--- pickhardtpayments/__init__.py | 17 +++++++++++++---- 8 files changed, 31 insertions(+), 23 deletions(-) diff --git a/examples/basicexample.py b/examples/basicexample.py index 2dc412a..8823ad8 100644 --- a/examples/basicexample.py +++ b/examples/basicexample.py @@ -1,7 +1,7 @@ -from ChannelGraph import ChannelGraph -from UncertaintyNetwork import UncertaintyNetwork -from OracleLightningNetwork import OracleLightningNetwork -from SyncSimulatedPaymentSession import SyncSimulatedPaymentSession +from pickhardtpayments.ChannelGraph import ChannelGraph +from pickhardtpayments.UncertaintyNetwork import UncertaintyNetwork +from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork +from pickhardtpayments.SyncSimulatedPaymentSession import SyncSimulatedPaymentSession #we first need to import the chanenl graph from c-lightning jsondump diff --git a/pickhardtpayments/ChannelGraph.py b/pickhardtpayments/ChannelGraph.py index c2de74e..4751c5b 100644 --- a/pickhardtpayments/ChannelGraph.py +++ b/pickhardtpayments/ChannelGraph.py @@ -1,6 +1,6 @@ import networkx as nx import json -from Channel import Channel +from .Channel import Channel class ChannelGraph: diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index 524c895..65d7c31 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -1,4 +1,4 @@ -from Channel import Channel +from .Channel import Channel import random diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 664b008..bc2cb22 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -1,8 +1,7 @@ from typing import List - -from pickhardtpayments.Channel import Channel -from pickhardtpayments.ChannelGraph import ChannelGraph -from pickhardtpayments.OracleChannel import OracleChannel +from .ChannelGraph import ChannelGraph +from .OracleChannel import OracleChannel +from .Channel import Channel import networkx as nx DEFAULT_BASE_THRESHOLD = 0 diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 84305ac..23eaea4 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -8,10 +8,10 @@ import logging import sys -from pickhardtpayments.Attempt import Attempt, AttemptStatus -from pickhardtpayments.Payment import Payment -from UncertaintyNetwork import UncertaintyNetwork -from OracleLightningNetwork import OracleLightningNetwork +from .Attempt import Attempt, AttemptStatus +from .Payment import Payment +from .UncertaintyNetwork import UncertaintyNetwork +from .OracleLightningNetwork import OracleLightningNetwork from ortools.graph import pywrapgraph diff --git a/pickhardtpayments/UncertaintyChannel.py b/pickhardtpayments/UncertaintyChannel.py index acd25fb..d80e757 100644 --- a/pickhardtpayments/UncertaintyChannel.py +++ b/pickhardtpayments/UncertaintyChannel.py @@ -1,5 +1,5 @@ -from pickhardtpayments.Channel import Channel -from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork +from .Channel import Channel +from .OracleLightningNetwork import OracleLightningNetwork from math import log2 as log diff --git a/pickhardtpayments/UncertaintyNetwork.py b/pickhardtpayments/UncertaintyNetwork.py index 8829da1..ef21c1d 100644 --- a/pickhardtpayments/UncertaintyNetwork.py +++ b/pickhardtpayments/UncertaintyNetwork.py @@ -1,6 +1,6 @@ -from pickhardtpayments.ChannelGraph import ChannelGraph -from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork -from pickhardtpayments.UncertaintyChannel import UncertaintyChannel +from .ChannelGraph import ChannelGraph +from .UncertaintyChannel import UncertaintyChannel +from .OracleLightningNetwork import OracleLightningNetwork from typing import List diff --git a/pickhardtpayments/__init__.py b/pickhardtpayments/__init__.py index 09f14bd..95453c8 100644 --- a/pickhardtpayments/__init__.py +++ b/pickhardtpayments/__init__.py @@ -1,6 +1,11 @@ -from pickhardtpayments.Channel import Channel, ChannelFields -from pickhardtpayments.UncertaintyChannel import UncertaintyChannel -from pickhardtpayments.OracleChannel import OracleChannel +from .Channel import Channel, ChannelFields +from .UncertaintyChannel import UncertaintyChannel +from .OracleChannel import OracleChannel +from .UncertaintyNetwork import UncertaintyNetwork +from .OracleLightningNetwork import OracleLightningNetwork +from .ChannelGraph import ChannelGraph +from .SyncSimulatedPaymentSession import SyncSimulatedPaymentSession + __version__ = "0.0.2" __all__ = [ @@ -8,4 +13,8 @@ "ChannelFields", "UncertaintyChannel", "OracleChannel", -] + "UncertaintyNetwork", + "OracleLightningNetwork", + "ChannelGraph", + "SyncSimulatedPaymentSession" +] \ No newline at end of file From 0730a91160aef6021a400acd47125e65b1827377 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 18 Jun 2022 19:55:57 +0200 Subject: [PATCH 24/38] fixing issues from partial review https://github.com/renepickhardt/pickhardtpayments/pull/24#pullrequestreview-1010067780_ --- pickhardtpayments/Attempt.py | 29 ++++++++++++++------- pickhardtpayments/OracleChannel.py | 2 +- pickhardtpayments/OracleLightningNetwork.py | 2 +- pickhardtpayments/Payment.py | 2 +- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py index ef13b4c..d62c3bd 100644 --- a/pickhardtpayments/Attempt.py +++ b/pickhardtpayments/Attempt.py @@ -1,22 +1,22 @@ from enum import Enum -from pickhardtpayments import Channel +from Channel import Channel class AttemptStatus(Enum): PLANNED = 1 INFLIGHT = 2 - ARRIVED = 3 - FAILED = 4 - SETTLED = 5 + ARRIVED = 4 + FAILED = 8 + SETTLED = 16 class Attempt: """ - An Attempt describes a path (a set of channels) of an amount from sender to receiver. + An Attempt describes a path (a list of channels) of an amount from sender to receiver. When sending an amount of sats from sender to receiver, a payment is usually split up and sent across - several paths, to increase the probability of being successfully delivered. Each of this path is referred to as an + several paths, to increase the probability of being successfully delivered. Each of these paths is referred to as an Attempt. An Attempt consists of a list of Channels (class:Channel) and the amount in sats to be sent through this path. @@ -31,14 +31,25 @@ def __init__(self, path: list[Channel], amount: int = 0): """ self._routing_fee = None self._probability = None - self._path = path self._status = AttemptStatus.PLANNED - self._amount = amount + + if amount >= 0: + self._amount = amount + else: + raise ValueError("amount for payment attempts needs to be positive") + + i = 1 + valid_path = True + while i < len(path): + valid_path = valid_path and (path[i - 1].dest == path[i].src) + i += 1 + if valid_path: + self._path = path def __str__(self): description = "Path with {} channels to deliver {} sats and status {}.".format(len(self._path), self._amount, self._status.name) - if self._routing_fee > 0: + if self._routing_fee and self._routing_fee > 0: description += "\nsuccess probability of {:6.2f}% , fee of {:8.3f} sat and a ppm of {:5} ".format( self._probability * 100, self._routing_fee/1000, int(self._routing_fee * 1000 / self._amount)) return description diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index 65d7c31..04dacaa 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -39,7 +39,7 @@ def actual_liquidity(self, amt: int): if 0 <= amt <= self.capacity: self._actual_liquidity = amt else: - raise ValueError("Oops! The amount to be assigned to channel liquidity is negative or higher than capacity") + raise ValueError(f"Liquidity for channel {self.short_channel_id} cannot be set. Amount {amt} is negative or higher than capacity") def can_forward(self, amt: int): """ diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index bc2cb22..8321994 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -80,7 +80,7 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base def settle_payment(self, path: List[Channel], payment_amount: int): """ - receives a dictionary with channels and payment amounts and adjusts the balances of the channels along the path. + receives a List of channels and payment amount and adjusts the balances of the channels along the path. settle_payment should only be called after all send_onions for a payment terminated successfully! # TODO testing diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index 602dc5f..fd7aeb6 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -1,7 +1,7 @@ import logging import time -from Attempt import Attempt, AttemptStatus +from .Attempt import Attempt, AttemptStatus class Payment: From f0bda8b2fbf558b2d654d50955827c495bcc6d9e Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 21 Jun 2022 11:43:45 +0200 Subject: [PATCH 25/38] deleting docs folder and moving _estimate_payment_statistics into attempt class --- pickhardtpayments/Attempt.py | 55 ++++++++++--------- .../SyncSimulatedPaymentSession.py | 11 ++-- pickhardtpayments/UncertaintyChannel.py | 2 +- pickhardtpayments/UncertaintyNetwork.py | 2 +- pickhardtpayments/__init__.py | 2 + 5 files changed, 40 insertions(+), 32 deletions(-) diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py index d62c3bd..593d90b 100644 --- a/pickhardtpayments/Attempt.py +++ b/pickhardtpayments/Attempt.py @@ -1,6 +1,7 @@ from enum import Enum -from Channel import Channel +from pickhardtpayments import Channel +from pickhardtpayments import UncertaintyChannel class AttemptStatus(Enum): @@ -15,10 +16,14 @@ class Attempt: """ An Attempt describes a path (a list of channels) of an amount from sender to receiver. + # TODO Describe life cycle of an Attempt + When sending an amount of sats from sender to receiver, a payment is usually split up and sent across several paths, to increase the probability of being successfully delivered. Each of these paths is referred to as an Attempt. An Attempt consists of a list of Channels (class:Channel) and the amount in sats to be sent through this path. + When an Attempt is instantiated, the given amount is allocated to the inflight amount in the channels of the + path and the AttemptStatus is set to PLANNED. :param path: a list of Channel objects from sender to receiver :type path: list[Channel] @@ -26,13 +31,9 @@ class Attempt: :type amount: int """ - def __init__(self, path: list[Channel], amount: int = 0): + def __init__(self, path: list[UncertaintyChannel], amount: int = 0): """Constructor method """ - self._routing_fee = None - self._probability = None - self._status = AttemptStatus.PLANNED - if amount >= 0: self._amount = amount else: @@ -46,6 +47,18 @@ def __init__(self, path: list[Channel], amount: int = 0): if valid_path: self._path = path + channel: UncertaintyChannel + self._routing_fee = 0 + self._probability = 1 + for channel in path: + self._routing_fee += channel.routing_cost_msat(amount) + self._probability *= channel.success_probability(amount) + # When Attempt is created, all amounts are set inflight. Needs to be updated with AttemptStatus change! + # This is to correctly compute conditional probabilities of non-disjoint paths in the same set of paths + # channel.in_flight(amount) + channel.allocate_amount(amount) + self._status = AttemptStatus.PLANNED + def __str__(self): description = "Path with {} channels to deliver {} sats and status {}.".format(len(self._path), self._amount, self._status.name) @@ -90,6 +103,17 @@ def status(self, value: AttemptStatus): :param value: Current state of the Attempt :type value: AttemptStatus """ + # remove allocated amounts when Attempt status changes from PLANNED + if self._status == AttemptStatus.PLANNED and not value == AttemptStatus.PLANNED: + for channel in self._path: + channel.allocate_amount(-self._amount) + + if self._status == AttemptStatus.PLANNED and value == AttemptStatus.ARRIVED: + # TODO write amount from inflight to min_liquidity/max_liquidity + # for channel in self._path: + # channel.allocate_amount(-self._amount) + pass + self._status = value @property @@ -101,15 +125,6 @@ def routing_fee(self) -> int: """ return self._routing_fee - @routing_fee.setter - def routing_fee(self, value: int): - """Sets the accrued routing fee in msat requested for this path - - :param value: accrued routing fees for this attempt in msat - :type value: int - """ - self._routing_fee = value - @property def probability(self) -> float: """Returns estimated success probability before the attempt @@ -119,13 +134,3 @@ def probability(self) -> float: """ return self._probability - @probability.setter - def probability(self, value: int): - """Sets the estimated success probability of the attempt. - - This is calculated as product of the channels' success probabilities as determined in the UncertaintyGraph. - - :param value: estimated success probability of the attempt - :type value: float - """ - self._probability = value diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 23eaea4..486c821 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -191,6 +191,7 @@ def _dissect_flow_to_paths(self, s, d): path = nx.shortest_path(G, s, d) except: break + print("used_flow = ", used_flow) channel_path, used_flow = self._make_channel_path(G, path) attempts.append(Attempt(channel_path, used_flow)) @@ -232,6 +233,7 @@ def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, return attempts_in_round, end - start def _estimate_payment_statistics(self, attempts: list[Attempt]): + exit(0) # FIXME: can go """ estimates the success probability of paths and computes fees (without paying downstream fees) @@ -239,9 +241,6 @@ def _estimate_payment_statistics(self, attempts: list[Attempt]): """ # compute fees and probabilities of candidate paths for evaluation for attempt in attempts: - attempt.routing_fee, attempt.probability = self._uncertainty_network.get_features_of_candidate_path( - attempt.path, attempt.amount) - # logging.debug("fee: {attempt.routing_fee} msat, p = {attempt.probability:.4%}, amount: {attempt.amount}") # to correctly compute conditional probabilities of non-disjoint paths in the same set of paths self._uncertainty_network.allocate_amount_on_path(attempt.path, attempt.amount) @@ -263,9 +262,13 @@ def _attempt_payments(self, attempts: list[Attempt]): success, erring_channel = self._oracle.send_onion( attempt.path, attempt.amount) if success: + # TODO: let this happen in Payment class? Or in Attempt class - with status change as settlement attempt.status = AttemptStatus.ARRIVED + # handling amounts on path happens in Attempt Class. self._uncertainty_network.allocate_amount_on_path( attempt.path, attempt.amount) + + # unnecessary, because information is in attempt (Status INFLIGHT) # settled_onions.append(payments[key]) else: @@ -381,8 +384,6 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): paths, runtime = self._generate_candidate_paths(payment.sender, payment.receiver, amt, mu, base) sub_payment.add_attempts(paths) - # compute some statistics about candidate paths - self._estimate_payment_statistics(sub_payment.attempts) # make attempts, try to send onion and register if success or not # update our information about the UncertaintyNetwork self._attempt_payments(sub_payment.attempts) diff --git a/pickhardtpayments/UncertaintyChannel.py b/pickhardtpayments/UncertaintyChannel.py index d80e757..826c0af 100644 --- a/pickhardtpayments/UncertaintyChannel.py +++ b/pickhardtpayments/UncertaintyChannel.py @@ -263,7 +263,7 @@ def update_knowledge(self, amt: int, success_of_probe): """ updates our knowledge about the channel if we tried to probe it for amount `amt` - This API works ony if we have an Oracle that allows to ask the actual liquidity of a channel + This API works only if we have an Oracle that allows to ask the actual liquidity of a channel In mainnet Lightning our oracle will not work on a per_channel level. This will change the data flow. Here for simplicity of the simulation we make use of the Oracle on a per channel level """ diff --git a/pickhardtpayments/UncertaintyNetwork.py b/pickhardtpayments/UncertaintyNetwork.py index ef21c1d..bcb1ff1 100644 --- a/pickhardtpayments/UncertaintyNetwork.py +++ b/pickhardtpayments/UncertaintyNetwork.py @@ -51,7 +51,7 @@ def get_features_of_candidate_path(self, path: List[UncertaintyChannel], amt: in for channel in path: routing_fees += channel.routing_cost_msat(amt) probability *= channel.success_probability(amt) - return routing_fees, probability + return probability def allocate_amount_on_path(self, path: List[UncertaintyChannel], amt: int): """ diff --git a/pickhardtpayments/__init__.py b/pickhardtpayments/__init__.py index 95453c8..2197034 100644 --- a/pickhardtpayments/__init__.py +++ b/pickhardtpayments/__init__.py @@ -1,5 +1,7 @@ from .Channel import Channel, ChannelFields from .UncertaintyChannel import UncertaintyChannel +from .Channel import Channel, ChannelFields +from .UncertaintyChannel import UncertaintyChannel from .OracleChannel import OracleChannel from .UncertaintyNetwork import UncertaintyNetwork from .OracleLightningNetwork import OracleLightningNetwork From 29990a24b3dd9de3c967f604d5f339e6ff112ec8 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 21 Jun 2022 11:43:45 +0200 Subject: [PATCH 26/38] deleting docs folder and moving _estimate_payment_statistics into attempt class --- pickhardtpayments/Attempt.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py index 593d90b..3906caa 100644 --- a/pickhardtpayments/Attempt.py +++ b/pickhardtpayments/Attempt.py @@ -1,6 +1,6 @@ from enum import Enum -from pickhardtpayments import Channel +from Channel import Channel from pickhardtpayments import UncertaintyChannel @@ -103,18 +103,19 @@ def status(self, value: AttemptStatus): :param value: Current state of the Attempt :type value: AttemptStatus """ - # remove allocated amounts when Attempt status changes from PLANNED - if self._status == AttemptStatus.PLANNED and not value == AttemptStatus.PLANNED: - for channel in self._path: - channel.allocate_amount(-self._amount) + if not self._status == value: + # remove allocated amounts when Attempt status changes from PLANNED + if self._status == AttemptStatus.PLANNED and not value == AttemptStatus.INFLIGHT: + for channel in self._path: + channel.allocate_amount(-self._amount) - if self._status == AttemptStatus.PLANNED and value == AttemptStatus.ARRIVED: - # TODO write amount from inflight to min_liquidity/max_liquidity - # for channel in self._path: - # channel.allocate_amount(-self._amount) - pass + if self._status == AttemptStatus.INFLIGHT and value == AttemptStatus.ARRIVED: + # TODO write amount from inflight to min_liquidity/max_liquidity + # for channel in self._path: + # channel.allocate_amount(-self._amount) + pass - self._status = value + self._status = value @property def routing_fee(self) -> int: @@ -133,4 +134,3 @@ def probability(self) -> float: :rtype: float """ return self._probability - From 081d75621157d84f977d99330ae78e0c109143b3 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 21 Jun 2022 12:18:13 +0200 Subject: [PATCH 27/38] removed doc folder --- docs/source/conf.py | 243 -------------------------------------------- 1 file changed, 243 deletions(-) delete mode 100644 docs/source/conf.py diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index 9e20375..0000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,243 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys -sys.path.insert(0, os.path.abspath('../../')) -from datetime import datetime -import subprocess - -# -- Project information ----------------------------------------------------- - -project = 'Pickhardt Payments Package' -copyright = '2022, Rene Pickhardt' -author = 'Rene Pickhardt' - -# The full version, including alpha/beta/rc tags -# release = '0.0.0' -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = subprocess.check_output('git describe --always --dirty=-modded --abbrev=7'.split()).decode('ASCII') -# The full version, including alpha/beta/rc tags. -release = version - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - - -# -- General configuration --------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.duration', - 'sphinx.ext.doctest', - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.intersphinx', -] - -intersphinx_mapping = { - 'python': ('https://docs.python.org/3/', None), - 'sphinx': ('https://www.sphinx-doc.org/en/master/', None), -} - -intersphinx_disabled_domains = ['std'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = ['.rst', '.md'] - -# The encoding of source files. -# -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'release-notes'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# -# show_authors = False - -# The name of the Pygments style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_rtd_theme' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. -# " v documentation" by default. -# -html_title = 'Pickhardt Payments ' + version - -# A shorter title for the navigation bar. Default is the same as html_title. -# -html_short_title = html_title - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# -# html_logo = None - -# The name of an image file (relative to this directory) to use as a favicon of -# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# -- Options for EPUB output -epub_show_urls = 'footnote' - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# -# html_extra_path = [] - -# If not None, a 'Last updated on:' timestamp is inserted at every page -# bottom, using the given strftime format. -# The empty string is equivalent to '%b %d, %Y'. -# -html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# -# html_additional_pages = {} - -# If false, no module index is generated. -# -# html_domain_indices = True - -# If false, no index is generated. -# -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# -html_show_sourcelink = False - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' -# -# html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# 'ja' uses this config value. -# 'zh' user can custom change `jieba` dictionary path. -# -# html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -# -# html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'pickhardtpaymentsdoc' \ No newline at end of file From c404105175e22c9caf5d63f8fdac9ab205852c6b Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 21 Jun 2022 17:58:01 +0200 Subject: [PATCH 28/38] adding CHANGELOG --- .gitignore | 5 ++++- CHANGELOG.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md diff --git a/.gitignore b/.gitignore index 972cd48..52cff06 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ pickhardtpayments/pickhardtpayments.egg-info pickhardtpayments/__pycache__ # not including large file with Channel Graph -listchannels*.json +*.json +examples/.json + +pickhardt_pay.log \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..41d06cf --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + +## [Unreleased] + +## [0.1.0] - 2022-06-21 +### Added + - introduction of an Attempt Class and a Payment Class ([#28]) + - introduction of an AttemptStatus to describe the state of the Attempt ([#28]) + - settle_payment in OracleLightingNetwork is added ([#28]) + - logging added in SyncSimulatedPaymentSession ([#28]) + +### Changed + - calculation of fees and probabilities is moved from SyncSimulatedPaymentSession to Attempt class ([#28]) + +### Deprecated + +### Removed + +### Fixed + +### EXPERIMENTAL From 3c850feed0ac682ce8dc830f0168d4e7b76e04fc Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 21 Jun 2022 22:44:00 +0200 Subject: [PATCH 29/38] Cleaning up after Attempt and Payment PR review --- pickhardtpayments/Payment.py | 10 +-- .../SyncSimulatedPaymentSession.py | 74 ++++++------------- pickhardtpayments/UncertaintyNetwork.py | 12 --- 3 files changed, 23 insertions(+), 73 deletions(-) diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index fd7aeb6..df6d71a 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -160,15 +160,7 @@ def filter_attempts(self, flag: AttemptStatus) -> list[Attempt]: :return: A list of successful Attempts of this Payment, which could be settled. :rtype: list[Attempt] """ - filtered_attempts = [] - try: - for attempt in self._attempts: - if attempt.status.value == flag.value: - filtered_attempts.append(attempt) - return filtered_attempts - except ValueError: - logging.warning("ValueError in Payment.filtered_attempts") - return [] + return [attempt for attempt in self._attempts if attempt.status.value == flag.value] @property def successful(self) -> bool: diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 486c821..6359a27 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -7,6 +7,7 @@ import logging import sys +from typing import List from .Attempt import Attempt, AttemptStatus from .Payment import Payment @@ -131,7 +132,7 @@ def _next_hop(self, path): dest = path[i] yield src, dest - def _make_channel_path(self, G: nx.MultiDiGraph, path: list[str]): + def _make_channel_path(self, G: nx.MultiDiGraph, path: List[str]): """ network x returns a path as a list of node_ids. However, we need a list of `UncertaintyChannels` Since the graph has parallel edges it is quite some work to get the actual channels that the @@ -191,7 +192,6 @@ def _dissect_flow_to_paths(self, s, d): path = nx.shortest_path(G, s, d) except: break - print("used_flow = ", used_flow) channel_path, used_flow = self._make_channel_path(G, path) attempts.append(Attempt(channel_path, used_flow)) @@ -232,24 +232,6 @@ def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, end = time.time() return attempts_in_round, end - start - def _estimate_payment_statistics(self, attempts: list[Attempt]): - exit(0) # FIXME: can go - """ - estimates the success probability of paths and computes fees (without paying downstream fees) - - @returns the statistics in the Payment - """ - # compute fees and probabilities of candidate paths for evaluation - for attempt in attempts: - - # to correctly compute conditional probabilities of non-disjoint paths in the same set of paths - self._uncertainty_network.allocate_amount_on_path(attempt.path, attempt.amount) - - # remove allocated amounts for all planned onions before doing actual attempts - for attempt in attempts: - self._uncertainty_network.allocate_amount_on_path( - attempt.path, -attempt.amount) - def _attempt_payments(self, attempts: list[Attempt]): """ we attempt all planned payments and test the success against the oracle in particular this @@ -274,7 +256,7 @@ def _attempt_payments(self, attempts: list[Attempt]): else: attempt.status = AttemptStatus.FAILED - def _evaluate_attempts(self, attempts: list[Attempt]): + def _evaluate_attempts(self, payment: Payment): """ helper function to collect statistics about attempts and print them @@ -287,40 +269,28 @@ def _evaluate_attempts(self, attempts: list[Attempt]): amt = 0 arrived_attempts = [] failed_attempts = [] - print("\nStatistics about {} candidate onions:\n".format(len(attempts))) - for attempt in attempts: - if attempt.status == AttemptStatus.ARRIVED: - arrived_attempts.append(attempt) - if attempt.status == AttemptStatus.FAILED: - failed_attempts.append(attempt) - + print("\nStatistics about {} candidate onions:\n".format(len(payment.attempts))) print("successful attempts:") print("--------------------") - for arrived_attempt in arrived_attempts: - fee = arrived_attempt.routing_fee / 1000. - probability = arrived_attempt.probability - path = arrived_attempt.path - amount = arrived_attempt.amount - amt += amount - total_fees += fee - expected_sats_to_deliver += probability * amount + for arrived_attempt in payment.filter_attempts(AttemptStatus.ARRIVED): + amt += arrived_attempt.amount + total_fees += arrived_attempt.routing_fee / 1000. + expected_sats_to_deliver += arrived_attempt.probability * arrived_attempt.amount print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5}".format( - probability * 100, amount, len(path), int(fee * 1000_000 / amount))) - paid_fees += fee + arrived_attempt.probability * 100, arrived_attempt.amount, len(arrived_attempt.path), + int(arrived_attempt.routing_fee * 1000 / arrived_attempt.amount))) + paid_fees += arrived_attempt.routing_fee print("\nfailed attempts:") print("----------------") - for failed_attempt in failed_attempts: - fee = failed_attempt.routing_fee / 1000. - probability = failed_attempt.probability - path = failed_attempt.path - amount = failed_attempt.amount - amt += amount - total_fees += fee - expected_sats_to_deliver += probability * amount - print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5} ".format( - probability * 100, amount, len(path), int(fee * 1000_000 / amount))) - residual_amt += amount + for failed_attempt in payment.filter_attempts(AttemptStatus.FAILED): + amt += failed_attempt.amount + total_fees += failed_attempt.routing_fee / 1000. + expected_sats_to_deliver += failed_attempt.probability * failed_attempt.amount + print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5}".format( + failed_attempt.probability * 100, failed_attempt.amount, len(failed_attempt.path), + int(failed_attempt.routing_fee * 1000 / failed_attempt.amount))) + residual_amt += failed_attempt.amount print("\nAttempt Summary:") print("=================") @@ -335,7 +305,7 @@ def _evaluate_attempts(self, attempts: list[Attempt]): (amt - residual_amt) / (expected_sats_to_deliver + 1))) print("planned_fee: \t{:8.3f} sat".format(total_fees)) print("paid fees: \t\t{:8.3f} sat".format(paid_fees)) - return residual_amt, paid_fees, len(attempts), len(failed_attempts) + return residual_amt, paid_fees, len(payment.attempts), len(failed_attempts) def forget_information(self): """ @@ -350,7 +320,7 @@ def activate_network_wide_uncertainty_reduction(self, n): self._uncertainty_network.activate_network_wide_uncertainty_reduction( n, self._oracle) - def pickhardt_pay(self, src, dest, amt, mu=1, base=0): + def pickhardt_pay(self, src, dest, amt, mu=1, base=DEFAULT_BASE_THRESHOLD): """ conduct one experiment! might need to call oracle.reset_uncertainty_network() first I could not put it here as some experiments require sharing of liquidity information @@ -390,7 +360,7 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=0): # run some simple statistics and depict them amt, paid_fees, num_paths, number_failed_paths = self._evaluate_attempts( - sub_payment.attempts) + sub_payment) print("Runtime of flow computation: {:4.2f} sec ".format(runtime)) print("\n================================================================\n") diff --git a/pickhardtpayments/UncertaintyNetwork.py b/pickhardtpayments/UncertaintyNetwork.py index bcb1ff1..dea4663 100644 --- a/pickhardtpayments/UncertaintyNetwork.py +++ b/pickhardtpayments/UncertaintyNetwork.py @@ -41,18 +41,6 @@ def entropy(self): """ return sum(channel.entropy() for src, dest, channel in self.network.edges(data="channel")) - def get_features_of_candidate_path(self, path: List[UncertaintyChannel], amt: int) -> (float, float): - """ - returns the routing fees and probability of a candidate path - :rtype: object - """ - probability = 1 - routing_fees = 0 - for channel in path: - routing_fees += channel.routing_cost_msat(amt) - probability *= channel.success_probability(amt) - return probability - def allocate_amount_on_path(self, path: List[UncertaintyChannel], amt: int): """ allocates `amt` to all channels of the path of `UncertaintyChannels` From 111a6918c96e7cb80315ac084ff62da685e1b8cf Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 22 Jun 2022 11:27:13 +0200 Subject: [PATCH 30/38] changed filter function in payment to generator --- pickhardtpayments/Payment.py | 4 +++- pickhardtpayments/SyncSimulatedPaymentSession.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index df6d71a..bf9e5d4 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -160,7 +160,9 @@ def filter_attempts(self, flag: AttemptStatus) -> list[Attempt]: :return: A list of successful Attempts of this Payment, which could be settled. :rtype: list[Attempt] """ - return [attempt for attempt in self._attempts if attempt.status.value == flag.value] + for attempt in self._attempts: + if attempt.status.value == flag.value: + yield attempt @property def successful(self) -> bool: diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 6359a27..12d9056 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -389,9 +389,9 @@ def pickhardt_pay(self, src, dest, amt, mu=1, base=DEFAULT_BASE_THRESHOLD): print("========") print("Rounds of mcf-computations:\t", cnt) print("Number of attempts made:\t", len(payment.attempts)) - print("Number of failed attempts:\t", len(payment.filter_attempts(AttemptStatus.FAILED))) + print("Number of failed attempts:\t", len(list(payment.filter_attempts(AttemptStatus.FAILED)))) print("Failure rate: {:4.2f}% ".format( - len(payment.filter_attempts(AttemptStatus.FAILED)) * 100. / len(payment.attempts))) + len(list(payment.filter_attempts(AttemptStatus.FAILED))) * 100. / len(payment.attempts))) print("total Payment lifetime (including inefficient memory management): {:4.3f} sec".format( payment.end_time - payment.start_time)) print("Learnt entropy: {:5.2f} bits".format(entropy_start - entropy_end)) From c320189a072d3f08dff07718b44461afc587e014 Mon Sep 17 00:00:00 2001 From: Rene Pickhardt Date: Mon, 30 May 2022 03:22:00 +0200 Subject: [PATCH 31/38] added setter to OracleChannel and fixed imports in init.py --- pickhardtpayments/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pickhardtpayments/__init__.py b/pickhardtpayments/__init__.py index 2197034..5e2c60f 100644 --- a/pickhardtpayments/__init__.py +++ b/pickhardtpayments/__init__.py @@ -1,7 +1,5 @@ from .Channel import Channel, ChannelFields from .UncertaintyChannel import UncertaintyChannel -from .Channel import Channel, ChannelFields -from .UncertaintyChannel import UncertaintyChannel from .OracleChannel import OracleChannel from .UncertaintyNetwork import UncertaintyNetwork from .OracleLightningNetwork import OracleLightningNetwork @@ -19,4 +17,4 @@ "OracleLightningNetwork", "ChannelGraph", "SyncSimulatedPaymentSession" -] \ No newline at end of file +] From de2511183ed62db296a1fc3d8e9ca5a3d1d7d637 Mon Sep 17 00:00:00 2001 From: nassersaazi Date: Mon, 20 Jun 2022 01:22:12 +0300 Subject: [PATCH 32/38] Fix typos in documentation --- examples/PickhardtPaymentsExample.ipynb | 20 +++++++++---------- examples/basicexample.py | 4 ++-- pickhardtpayments/Channel.py | 4 ++-- pickhardtpayments/ChannelGraph.py | 2 +- pickhardtpayments/OracleChannel.py | 2 +- .../SyncSimulatedPaymentSession.py | 6 +++--- pickhardtpayments/UncertaintyChannel.py | 6 +++--- readme.md | 10 +++++----- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/examples/PickhardtPaymentsExample.ipynb b/examples/PickhardtPaymentsExample.ipynb index 6ec0161..0585ae0 100644 --- a/examples/PickhardtPaymentsExample.ipynb +++ b/examples/PickhardtPaymentsExample.ipynb @@ -8,7 +8,7 @@ "\n", "Example code demonstrating how to use the `pickhardtpayments` package in python. You need to install the library via `pip install pickhardtpayments` or you can download the full source code at https://ln.rene-pickhardt.de or a copy from github at: https://www.github.com/renepickhardt/pickhardtpayments\n", "\n", - "Of course you can use the classes int the library to create your own async payment loop or you could exchange the Oracle to talk to the actual Lightning network by wrapping against your favourite node implementation and exposing the the `send_onion` call. \n", + "Of course you can use the classes in the library to create your own async payment loop or you could exchange the Oracle to talk to the actual Lightning network by wrapping against your favourite node implementation and exposing the the `send_onion` call. \n", "\n", "This example assumes a randomly generated Oracle to conduct payments in a simulated way. For this you will need an actual channelgraph which you can get for example with `lightning-cli listchannels > listchannels20220412.json`\n", "\n", @@ -39,14 +39,14 @@ "#Carsten Otto's public node key\n", "C_OTTO = \"027ce055380348d7812d2ae7745701c9f93e70c1adeb2657f053f91df4f2843c71\"\n", "\n", - "#we first need to import the chanenl graph from c-lightning jsondump\n", + "#we first need to import the chanenl graph from core lightning jsondump\n", "#you can get your own data set via:\n", "# $: lightning-cli listchannels > listchannels20220412.json\n", "# alternatively you can go to https://ln.rene-pickhardt.de to find a data dump\n", "channel_graph = ChannelGraph(\"listchannels20220412.json\")\n", "\n", - "#we now create ourself an Oracle. This is Simulated Network that assumes unformly distribution \n", - "#of the liquidity for the channels on the `channel_graph`. Of course one could create ones own \n", + "#we now create an Oracle. This is a Simulated Network that assumes uniform distribution \n", + "#of the liquidity for the channels on the `channel_graph`. Of course one could create one's own \n", "#oracle (for example a wrapper to an existing lightning network node / implementation)\n", "oracle_lightning_network = OracleLightningNetwork(channel_graph)" ] @@ -66,7 +66,7 @@ ], "source": [ "# Since we randomly generated our Oracle but also since we control it we can compute\n", - "# The maximum possible amout that can be payed between two nodes\n", + "# The maximum possible amount that can be payed between two nodes\n", "maximum_payable_amount =oracle_lightning_network.theoretical_maximum_payable_amount(RENE,C_OTTO,1000)\n", "print(maximum_payable_amount, \"sats would be possible on this oracle to deliver if including 1 sat basefee channels\")\n" ] @@ -86,7 +86,7 @@ ], "source": [ "#Of course we want to restrict ourselves to the zeroBaseFee part of the network.\n", - "#Therefor we compute the theoretical maximum payable amount for that subgraph\n", + "#Therefore we compute the theoretical maximum payable amount for that subgraph\n", "maximum_payable_amount =oracle_lightning_network.theoretical_maximum_payable_amount(RENE,C_OTTO,0)\n", "print(maximum_payable_amount, \"sats possible on this oracle on the zeroBaseFeeGraph\")" ] @@ -307,16 +307,16 @@ } ], "source": [ - "# We chose an amount that is 50% of half the theoretic maximum to demonstrate the the\n", + "# We choose an amount that is half the theoretical maximum to demonstrate the\n", "# minimum cost flow solver with Bayesian updates on the Uncertainty Network finds the\n", "# liquidity rather quickly\n", "tested_amount = int(maximum_payable_amount/2)\n", "\n", - "# From the channel graph we can derrive our initial Uncertainty Network which is the main data structure\n", + "# From the channel graph we can derive our initial Uncertainty Network which is the main data structure\n", "# that we maintain in order to deliver sats from one node to another\n", "uncertainty_network = UncertaintyNetwork(channel_graph)\n", "\n", - "#we create ourselves a payment session which in this case operates by sending out the onions\n", + "#we create a payment session which in this case operates by sending out the onions\n", "#sequentially \n", "payment_session = SyncSimulatedPaymentSession(oracle_lightning_network, \n", " uncertainty_network,\n", @@ -335,7 +335,7 @@ "source": [ "## Optimizing for Fees\n", "\n", - "controlling mu we can decide how much we wish to focuse on lower fees. However we will see that it will be much harder to deliver the same amount in the sense that we need to send out more onions and also have more failed attampts. Consiquantly we expect to need more time." + "controlling mu we can decide how much we wish to focus on lower fees. However ,we will see that it will be much harder to deliver the same amount in the sense that we need to send out more onions and also have more failed attampts. Consequently we expect to need more time." ] }, { diff --git a/examples/basicexample.py b/examples/basicexample.py index 8823ad8..dcf6b7b 100644 --- a/examples/basicexample.py +++ b/examples/basicexample.py @@ -4,7 +4,7 @@ from pickhardtpayments.SyncSimulatedPaymentSession import SyncSimulatedPaymentSession -#we first need to import the chanenl graph from c-lightning jsondump +#we first need to import the channel graph from core lightning jsondump #you can get your own data set via: # $: lightning-cli listchannels > listchannels20220412.json # alternatively you can go to https://ln.rene-pickhardt.de to find a data dump @@ -12,7 +12,7 @@ uncertainty_network = UncertaintyNetwork(channel_graph) oracle_lightning_network = OracleLightningNetwork(channel_graph) -#we create ourselves a payment session which in this case operates by sending out the onions +#we create a payment session which in this case operates by sending out the onions #sequentially payment_session = SyncSimulatedPaymentSession(oracle_lightning_network, uncertainty_network, diff --git a/pickhardtpayments/Channel.py b/pickhardtpayments/Channel.py index 055face..93eae38 100644 --- a/pickhardtpayments/Channel.py +++ b/pickhardtpayments/Channel.py @@ -1,7 +1,7 @@ class ChannelFields: """ These are the values describing public data about channels that is either available - via gossip or via the Bitcoin Blockchain. Their format is taken from the c-lighting + via gossip or via the Bitcoin Blockchain. Their format is taken from the core lighting API. If you use a different implementation I suggest to write a wrapper around the `ChannelFields` and `Channel` class """ @@ -26,7 +26,7 @@ class Channel: Stores the public available information of a channel. The `Channel` Class is intended to be read only and internally stores - the data from c-lightning's `lightning-cli listchannels` command as a json. + the data from core lightning's `lightning-cli listchannels` command as a json. If you retrieve data from a different implementation I suggest to overload the constructor and transform the information into the given json format """ diff --git a/pickhardtpayments/ChannelGraph.py b/pickhardtpayments/ChannelGraph.py index 4751c5b..578b60b 100644 --- a/pickhardtpayments/ChannelGraph.py +++ b/pickhardtpayments/ChannelGraph.py @@ -22,7 +22,7 @@ def _get_channel_json(self, filename: str): def __init__(self, lightning_cli_listchannels_json_file: str): """ - Importing the channel_graph from c-lightning listchannels command the file can be received by + Importing the channel_graph from core lightning listchannels command the file can be received by #$ lightning-cli listchannels > listchannels.json """ diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index 04dacaa..48c287f 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -5,7 +5,7 @@ class OracleChannel(Channel): """ - An OracleChannel us used in experiments and Simulations to form the (Oracle)LightningNetwork. + An OracleChannel is used in experiments and Simulations to form the (Oracle)LightningNetwork. It contains a ground truth about the Liquidity of a channel """ diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 12d9056..53a7b85 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -46,7 +46,7 @@ class SyncSimulatedPaymentSession: UncertaintyChannel to the min_cost_flow object. The main API call ist `pickhardt_pay` which invokes a sequential loop to conduct trial and error - attempts. The loop could easily send out all onions concurrently but this does not make sense + attmpts. The loop could easily send out all onions concurrently but this does not make sense against the simulated OracleLightningNetwork. """ @@ -160,7 +160,7 @@ def _dissect_flow_to_paths(self, s, d): """ A standard algorithm to dissect a flow into several paths. - FIXME: Note that this dissection while accurate is probably not optimal in practise. + FIXME: Note that this disection while accurate is probably not optimal in practise. As noted in our Probabilistic payment delivery paper the payment process is a bernoulli trial and I assume it makes sense to dissect the flow into paths of similar likelihood to make most progress but this is a mere conjecture at this point. I expect quite a bit of research will be @@ -212,7 +212,7 @@ def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, This is one step within the payment loop. - Returns the residual amount of the `amt` that could not be delivered and the paid fees + Retuns the residual amount of the `amt` that could ne be delivered and the paid fees (on a per channel base not including fees for downstream fees) for the delivered amount the function also prints some results on statistics about the paths of the flow to stdout. diff --git a/pickhardtpayments/UncertaintyChannel.py b/pickhardtpayments/UncertaintyChannel.py index 826c0af..890fb03 100644 --- a/pickhardtpayments/UncertaintyChannel.py +++ b/pickhardtpayments/UncertaintyChannel.py @@ -13,9 +13,9 @@ class UncertaintyChannel(Channel): UncertaintyNetwork. Since we optimize for reliability via a probability estimate for liquidity that is based - on the capacity of the channel the class contains the `capacity` as seen in the funding tx output. + on the capacity of the channel, the class contains the `capacity` as seen in the funding tx output. - As we also optimize for fees and want to be able to compute the fees of a flow the classe + As we also optimize for fees and want to be able to compute the fees of a flow ,the class contains information for the feerate (`ppm`) and the base_fee (`base`). Most importantly the class stores our belief about the liquidity information of a channel. @@ -91,7 +91,7 @@ def allocate_amount(self, amt: int): # FIXME: store timestamps when using setters so that we know when we learnt our belief def forget_information(self): """ - resets the information that we belief to have about the channel. + resets the information that we believe to have about the channel. """ self.min_liquidity = 0 self.max_liquidity = self.capacity diff --git a/readme.md b/readme.md index df97158..9f6fd51 100644 --- a/readme.md +++ b/readme.md @@ -4,9 +4,9 @@ The `pickhardtpayments` package is a collection of classes and interfaces that h ## What are Pickhardt Payments? -Pickhardt Payments are the method of deliverying satoshis from on Lightning network Node to another by using [probabilistic payment delivery](https://arxiv.org/abs/2103.08576) in a round based `payment loop` that updeates our `belief` of the remote `liquidity` in the `Uncertainty Network` and generates [optimally reliable and cheap payment flows](https://arxiv.org/abs/2107.05322) in every round by solving a [piece wise linearized min integer cost flow problem](https://github.com/renepickhardt/mpp-splitter/blob/pickhardt-payments-simulation-dev/Minimal%20Linearized%20min%20cost%20flow%20example%20for%20MPP.ipynb) with a seperable cost function. +Pickhardt Payments are the method of deliverying satoshis from one Lightning network Node to another by using [probabilistic payment delivery](https://arxiv.org/abs/2103.08576) in a round based `payment loop` that updates our `belief` of the remote `liquidity` in the `Uncertainty Network` and generates [optimally reliable and cheap payment flows](https://arxiv.org/abs/2107.05322) in every round by solving a [piece wise linearized min integer cost flow problem](https://github.com/renepickhardt/mpp-splitter/blob/pickhardt-payments-simulation-dev/Minimal%20Linearized%20min%20cost%20flow%20example%20for%20MPP.ipynb) with a separable cost function. -As of now the two main features of the cost function are the `linearized_uncertainty_unit_cost` (effectively proportional to `1/channel_capacity`) and the `linearized_routing_unit_cost` (effectivly just the `ppm`). +As of now the two main features of the cost function are the `linearized_uncertainty_unit_cost` (effectively proportional to `1/channel_capacity`) and the `linearized_routing_unit_cost` (effectively just the `ppm`). ## Depenencies @@ -20,7 +20,7 @@ The dependencies can be found at: ## build and install -Onestep install is via pip by typing `pip install pickhardtpayments` to your command line +One step install is via pip by typing `pip install pickhardtpayments` to your command line If you want to build and install the library yourself you can do: @@ -43,7 +43,7 @@ from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork from pickhardtpayments.SyncSimulatedPaymentSession import SyncSimulatedPaymentSession -#we first need to import the chanenl graph from c-lightning jsondump +#we first need to import the chanenl graph from core lightning jsondump #you can get your own data set via: # $: lightning-cli listchannels > listchannels20220412.json # alternatively you can go to https://ln.rene-pickhardt.de to find a data dump @@ -51,7 +51,7 @@ channel_graph = ChannelGraph("listchannels20220412.json") uncertainty_network = UncertaintyNetwork(channel_graph) oracle_lightning_network = OracleLightningNetwork(channel_graph) -#we create ourselves a payment session which in this case operates by sending out the onions +#we create a payment session which in this case operates by sending out the onions #sequentially payment_session = SyncSimulatedPaymentSession(oracle_lightning_network, uncertainty_network, From 40fb9be009a9fd167c954009ad79992f54b561fe Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 13 May 2022 22:53:00 +0200 Subject: [PATCH 33/38] rewriting setter for `actual_liquidity` using properties (python way) --- pickhardtpayments/OracleChannel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index 48c287f..ca45d9e 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -48,4 +48,4 @@ def can_forward(self, amt: int): if amt <= self.actual_liquidity: return True else: - return False + return False \ No newline at end of file From 54ccdc3690b67e031e0501ba4e72aa42bb35b128 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 20 May 2022 02:31:27 +0200 Subject: [PATCH 34/38] Indroduction of Payment and Attempt class, refactoring. --- pickhardtpayments/Channel.py | 6 +++--- pickhardtpayments/Payment.py | 6 ------ .../SyncSimulatedPaymentSession.py | 20 +++++++++---------- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/pickhardtpayments/Channel.py b/pickhardtpayments/Channel.py index 93eae38..7ae4a5f 100644 --- a/pickhardtpayments/Channel.py +++ b/pickhardtpayments/Channel.py @@ -1,7 +1,7 @@ class ChannelFields: """ These are the values describing public data about channels that is either available - via gossip or via the Bitcoin Blockchain. Their format is taken from the core lighting + via gossip or via the Bitcoin Blockchain. Their format is taken from the c-lighting API. If you use a different implementation I suggest to write a wrapper around the `ChannelFields` and `Channel` class """ @@ -25,8 +25,8 @@ class Channel: """ Stores the public available information of a channel. - The `Channel` Class is intended to be read only and internally stores - the data from core lightning's `lightning-cli listchannels` command as a json. + The `Channel` Class is intended to be read only and internatlly stores + the data from c-lightning's `lightning-cli listchannels` command as a json. If you retrieve data from a different implementation I suggest to overload the constructor and transform the information into the given json format """ diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index bf9e5d4..91f1af1 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -1,9 +1,3 @@ -import logging -import time - -from .Attempt import Attempt, AttemptStatus - - class Payment: """ Payment stores the information about an amount of sats to be delivered from source to destination. diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 53a7b85..91cd84a 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -14,7 +14,6 @@ from .UncertaintyNetwork import UncertaintyNetwork from .OracleLightningNetwork import OracleLightningNetwork - from ortools.graph import pywrapgraph import time @@ -45,8 +44,8 @@ class SyncSimulatedPaymentSession: This happens by adding several parallel arcs coming from the piece wise linearization of the UncertaintyChannel to the min_cost_flow object. - The main API call ist `pickhardt_pay` which invokes a sequential loop to conduct trial and error - attmpts. The loop could easily send out all onions concurrently but this does not make sense + The main API call is `pickhardt_pay` which invokes a sequential loop to conduct trial and error + attempts. The loop could easily send out all onions concurrently but this does not make sense against the simulated OracleLightningNetwork. """ @@ -160,7 +159,7 @@ def _dissect_flow_to_paths(self, s, d): """ A standard algorithm to dissect a flow into several paths. - FIXME: Note that this disection while accurate is probably not optimal in practise. + FIXME: Note that this dissection while accurate is probably not optimal in practise. As noted in our Probabilistic payment delivery paper the payment process is a bernoulli trial and I assume it makes sense to dissect the flow into paths of similar likelihood to make most progress but this is a mere conjecture at this point. I expect quite a bit of research will be @@ -193,7 +192,7 @@ def _dissect_flow_to_paths(self, s, d): except: break channel_path, used_flow = self._make_channel_path(G, path) - attempts.append(Attempt(channel_path, used_flow)) + channel_paths.append((channel_path, used_flow)) # reduce the flow from the selected path for pos, hop in enumerate(self._next_hop(path)): @@ -217,6 +216,7 @@ def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, the function also prints some results on statistics about the paths of the flow to stdout. """ + # First we prepare the min cost flow by getting arcs from the uncertainty network self._prepare_mcf_solver(src, dest, amt, mu, base) start = time.time() @@ -298,14 +298,14 @@ def _evaluate_attempts(self, payment: Payment): fraction = expected_sats_to_deliver * 100. / amt print("expected to deliver {:10} sats \t({:4.2f}%)".format( int(expected_sats_to_deliver), fraction)) - fraction = (amt - residual_amt) * 100. / amt - print("actually delivered \t{:10} sats \t({:4.2f}%)".format( + fraction = (amt - residual_amt) * 100. / (amt) + print("actually delivered {:10} sats \t({:4.2f}%)".format( amt - residual_amt, fraction)) print("deviation: \t\t{:4.2f}".format( (amt - residual_amt) / (expected_sats_to_deliver + 1))) - print("planned_fee: \t{:8.3f} sat".format(total_fees)) - print("paid fees: \t\t{:8.3f} sat".format(paid_fees)) - return residual_amt, paid_fees, len(payment.attempts), len(failed_attempts) + print("planned_fee: {:8.3f} sat".format(total_fees)) + print("paid fees: {:8.3f} sat".format(paid_fees)) + return residual_amt, paid_fees, len(payments), number_failed_paths def forget_information(self): """ From a40e61afe6cdb990b807ea15fd66a34bb72cf334 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 20 May 2022 17:34:01 +0200 Subject: [PATCH 35/38] Introduction of PaymentClass and AttemptClass, refactoring. --- pickhardtpayments/OracleLightningNetwork.py | 2 -- pickhardtpayments/Payment.py | 7 ++++++ .../SyncSimulatedPaymentSession.py | 25 ++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 8321994..9426a5b 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -1,7 +1,5 @@ -from typing import List from .ChannelGraph import ChannelGraph from .OracleChannel import OracleChannel -from .Channel import Channel import networkx as nx DEFAULT_BASE_THRESHOLD = 0 diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index 91f1af1..c1a5881 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -1,3 +1,9 @@ +import logging +import time +from typing import List + +import Attempt + class Payment: """ Payment stores the information about an amount of sats to be delivered from source to destination. @@ -27,6 +33,7 @@ def __init__(self, sender, receiver, total_amount: int = 1): self._receiver = receiver self._total_amount = total_amount self._attempts = list() + self._start = time.time() def __str__(self): return "Payment with {} attempts to deliver {} sats from {} to {}".format(len(self._attempts), diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 91cd84a..929ec27 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -216,6 +216,8 @@ def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, the function also prints some results on statistics about the paths of the flow to stdout. """ + # initialisation of List of Attempts for this round. + attempts_in_round = List[Attempt] # First we prepare the min cost flow by getting arcs from the uncertainty network self._prepare_mcf_solver(src, dest, amt, mu, base) @@ -232,7 +234,28 @@ def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, end = time.time() return attempts_in_round, end - start - def _attempt_payments(self, attempts: list[Attempt]): + def _estimate_payment_statistics(self, attempts): + """ + estimates the success probability of paths and computes fees (without paying downstream fees) + + @returns the statistics in the `payments` dictionary + """ + # compute fees and probabilities of candidate paths for evaluation + for attempt in attempts: + attempt.routing_fee, attempt.probability = self._uncertainty_network.get_features_of_candidate_path( + attempt.path, attempt.amount) + # logging.debug("fee: {attempt.routing_fee} msat, p = {attempt.probability:.4%}, amount: {attempt.amount}") + + # to correctly compute conditional probabilities of non-disjoint paths in the same set of paths + self._uncertainty_network.allocate_amount_on_path(attempt.path, attempt.amount) + + # remove allocated amounts for all planned onions before doing actual attempts + for attempt in attempts: + self._uncertainty_network.allocate_amount_on_path( + attempt.path, -attempt.amount) + + + def _attempt_payments(self, attempts: List[Attempt]): """ we attempt all planned payments and test the success against the oracle in particular this method changes - depending on the outcome of each payment - our belief about the uncertainty From 1a05a217c245ef1c796bb70cd7097d385f3ed989 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 29 May 2022 13:25:12 +0200 Subject: [PATCH 36/38] adjustment of relative paths for modules in import --- examples/basicexample.py | 8 ++++---- pickhardtpayments/ChannelGraph.py | 2 +- pickhardtpayments/OracleChannel.py | 2 +- pickhardtpayments/OracleLightningNetwork.py | 2 +- pickhardtpayments/Payment.py | 1 + 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/basicexample.py b/examples/basicexample.py index dcf6b7b..72b442a 100644 --- a/examples/basicexample.py +++ b/examples/basicexample.py @@ -1,7 +1,7 @@ -from pickhardtpayments.ChannelGraph import ChannelGraph -from pickhardtpayments.UncertaintyNetwork import UncertaintyNetwork -from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork -from pickhardtpayments.SyncSimulatedPaymentSession import SyncSimulatedPaymentSession +from ChannelGraph import ChannelGraph +from UncertaintyNetwork import UncertaintyNetwork +from OracleLightningNetwork import OracleLightningNetwork +from SyncSimulatedPaymentSession import SyncSimulatedPaymentSession #we first need to import the channel graph from core lightning jsondump diff --git a/pickhardtpayments/ChannelGraph.py b/pickhardtpayments/ChannelGraph.py index 578b60b..adb9f08 100644 --- a/pickhardtpayments/ChannelGraph.py +++ b/pickhardtpayments/ChannelGraph.py @@ -1,6 +1,6 @@ import networkx as nx import json -from .Channel import Channel +from Channel import Channel class ChannelGraph: diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index ca45d9e..ea11691 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -1,4 +1,4 @@ -from .Channel import Channel +from Channel import Channel import random diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 9426a5b..881f5a6 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -76,7 +76,7 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base mincut, _ = nx.minimum_cut(test_network, source, destination) return mincut - def settle_payment(self, path: List[Channel], payment_amount: int): + def settle_payment(self, path: List[OracleChannel], payment_amount: int): """ receives a List of channels and payment amount and adjusts the balances of the channels along the path. diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index c1a5881..45e69a6 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -4,6 +4,7 @@ import Attempt + class Payment: """ Payment stores the information about an amount of sats to be delivered from source to destination. From d494c0dd73875b012a1c2b99b1738e290c5ca206 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sat, 18 Jun 2022 18:15:15 +0200 Subject: [PATCH 37/38] changing imports back to initial call --- examples/basicexample.py | 8 ++++---- pickhardtpayments/ChannelGraph.py | 2 +- pickhardtpayments/OracleChannel.py | 2 +- pickhardtpayments/__init__.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/basicexample.py b/examples/basicexample.py index 72b442a..dcf6b7b 100644 --- a/examples/basicexample.py +++ b/examples/basicexample.py @@ -1,7 +1,7 @@ -from ChannelGraph import ChannelGraph -from UncertaintyNetwork import UncertaintyNetwork -from OracleLightningNetwork import OracleLightningNetwork -from SyncSimulatedPaymentSession import SyncSimulatedPaymentSession +from pickhardtpayments.ChannelGraph import ChannelGraph +from pickhardtpayments.UncertaintyNetwork import UncertaintyNetwork +from pickhardtpayments.OracleLightningNetwork import OracleLightningNetwork +from pickhardtpayments.SyncSimulatedPaymentSession import SyncSimulatedPaymentSession #we first need to import the channel graph from core lightning jsondump diff --git a/pickhardtpayments/ChannelGraph.py b/pickhardtpayments/ChannelGraph.py index adb9f08..578b60b 100644 --- a/pickhardtpayments/ChannelGraph.py +++ b/pickhardtpayments/ChannelGraph.py @@ -1,6 +1,6 @@ import networkx as nx import json -from Channel import Channel +from .Channel import Channel class ChannelGraph: diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index ea11691..ca45d9e 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -1,4 +1,4 @@ -from Channel import Channel +from .Channel import Channel import random diff --git a/pickhardtpayments/__init__.py b/pickhardtpayments/__init__.py index 5e2c60f..95453c8 100644 --- a/pickhardtpayments/__init__.py +++ b/pickhardtpayments/__init__.py @@ -17,4 +17,4 @@ "OracleLightningNetwork", "ChannelGraph", "SyncSimulatedPaymentSession" -] +] \ No newline at end of file From fc1e8d5be2a8be63736529da0a9d34285fe275da Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 22 Jun 2022 13:04:20 +0200 Subject: [PATCH 38/38] another rebasing hassle ;) --- pickhardtpayments/OracleLightningNetwork.py | 2 ++ pickhardtpayments/Payment.py | 10 ++++------ pickhardtpayments/SyncSimulatedPaymentSession.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 881f5a6..eb8aacc 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -1,3 +1,5 @@ +from typing import List + from .ChannelGraph import ChannelGraph from .OracleChannel import OracleChannel import networkx as nx diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py index 45e69a6..348c9ed 100644 --- a/pickhardtpayments/Payment.py +++ b/pickhardtpayments/Payment.py @@ -1,8 +1,7 @@ -import logging import time from typing import List -import Attempt +from .Attempt import Attempt, AttemptStatus class Payment: @@ -34,7 +33,6 @@ def __init__(self, sender, receiver, total_amount: int = 1): self._receiver = receiver self._total_amount = total_amount self._attempts = list() - self._start = time.time() def __str__(self): return "Payment with {} attempts to deliver {} sats from {} to {}".format(len(self._attempts), @@ -102,7 +100,7 @@ def end_time(self, timestamp): self._end_time = timestamp @property - def attempts(self) -> list[Attempt]: + def attempts(self) -> List[Attempt]: """Returns all onions that were built and are associated with this Payment. :return: A list of Attempts of this payment. @@ -110,7 +108,7 @@ def attempts(self) -> list[Attempt]: """ return self._attempts - def add_attempts(self, attempts: list[Attempt]): + def add_attempts(self, attempts: List[Attempt]): """Adds Attempts (onions) that have been made to settle the Payment to the Payment object. :param attempts: a list of attempts that belong to this Payment @@ -153,7 +151,7 @@ def ppm(self) -> float: """ return self.fee * 1000 / self.total_amount - def filter_attempts(self, flag: AttemptStatus) -> list[Attempt]: + def filter_attempts(self, flag: AttemptStatus) -> List[Attempt]: """Returns all onions with the given state. :param flag: the state of the attempts that should be filtered for diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 929ec27..7043525 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -192,7 +192,7 @@ def _dissect_flow_to_paths(self, s, d): except: break channel_path, used_flow = self._make_channel_path(G, path) - channel_paths.append((channel_path, used_flow)) + attempts.append(Attempt(channel_path, used_flow)) # reduce the flow from the selected path for pos, hop in enumerate(self._next_hop(path)): @@ -328,7 +328,7 @@ def _evaluate_attempts(self, payment: Payment): (amt - residual_amt) / (expected_sats_to_deliver + 1))) print("planned_fee: {:8.3f} sat".format(total_fees)) print("paid fees: {:8.3f} sat".format(paid_fees)) - return residual_amt, paid_fees, len(payments), number_failed_paths + return residual_amt, paid_fees, len(payment.attempts), len(failed_attempts) def forget_information(self): """