Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 16 additions & 14 deletions src/Data/Models/Rebalance.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,16 @@ namespace NodeGuard.Data.Models;

public enum RebalanceStatus
{
Pending,
Probing,
InFlight,
Succeeded,
Failed,
NoRoute,
Timeout,
InsufficientBalance,
ExceededFeeLimit
Pending = 0,
// 1 was Probing — removed (rebalances go straight to SendPaymentV2, no probe). The value is
// intentionally left as a gap so the other statuses keep their stored int values.
InFlight = 2,
Succeeded = 3,
Failed = 4,
NoRoute = 5,
Timeout = 6,
InsufficientBalance = 7,
ExceededFeeLimit = 8
}

public class Rebalance : Entity
Expand Down Expand Up @@ -132,12 +133,13 @@ public class Rebalance : Entity
public int TimeoutSeconds { get; set; } = 60;

/// <summary>
/// Multiplier applied to the probe amount after each failure. Range: (0, 1) exclusive.
/// 0.5 = halve (default); 0.8 = next try is 20% smaller (finer-grained, more iterations);
/// 0.3 = next try is 70% smaller (gives up faster). When null, the runtime falls back
/// to <c>Constants.REBALANCE_PROBE_BACKOFF_RATIO</c>.
/// Multiplier applied to the rebalanced amount on each retry attempt. Range: (0, 1].
/// 1 = never shrink (every attempt retries the full requested amount); 0.8 = each retry is
/// 20% smaller; 0.5 = halve each time. The amount for attempt n is
/// RequestedAmountSats × ratio^(n-1), floored at <c>Constants.REBALANCE_MIN_AMOUNT_SATS</c>.
/// When null, the runtime falls back to <c>Constants.REBALANCE_AMOUNT_BACKOFF_RATIO</c>.
/// </summary>
public double? ProbeBackoffRatio { get; set; }
public double? AmountBackoffRatio { get; set; }

/// <summary>
/// Maximum number of attempts (including the first try). When null, falls back to
Expand Down
2 changes: 1 addition & 1 deletion src/Data/Repositories/Interfaces/IRebalanceRepository.cs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public interface IRebalanceRepository

/// <summary>
/// Returns rebalances the monitor job should reconcile against LND:
/// non-terminal rows (Pending/Probing/InFlight) plus recently-marked terminal failures
/// non-terminal rows (Pending/InFlight) plus recently-marked terminal failures
/// within <paramref name="recentTerminalWindow"/>. Only rows with a stored payment hash
/// are returned — without a hash there is nothing to look up in LND.
/// The recent-terminal sweep exists because the catch-all in RebalanceService can mark
Expand Down
1 change: 0 additions & 1 deletion src/Data/Repositories/RebalanceRepository.cs
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,6 @@ public async Task<List<Rebalance>> GetReconcilable(TimeSpan recentTerminalWindow
.Include(r => r.Node)
.Where(r => r.PaymentHashHex != null
&& (r.Status == RebalanceStatus.Pending
|| r.Status == RebalanceStatus.Probing
|| r.Status == RebalanceStatus.InFlight
|| ((r.Status == RebalanceStatus.Failed
|| r.Status == RebalanceStatus.Timeout
Expand Down
33 changes: 18 additions & 15 deletions src/Helpers/Constants.cs
Original file line number Diff line number Diff line change
Expand Up @@ -179,22 +179,25 @@ public class Constants
public static double REBALANCE_RETRY_BACKOFF_MULTIPLIER = 2.0;

/// <summary>
/// Smallest amount the probing binary search will halve down to before giving up.
/// Smallest amount a rebalance attempt will shrink down to before giving up (the floor
/// for the per-attempt amount backoff).
/// </summary>
public static long REBALANCE_MIN_PROBE_AMOUNT_SATS = 10_000;
public static long REBALANCE_MIN_AMOUNT_SATS = 10_000;

/// <summary>
/// Multiplier applied to the probe amount after each failure. Range: (0, 1) exclusive.
/// 0.5 (default) = halve the amount each iteration; 0.8 = next try is 20% smaller
/// (more iterations, finer granularity); 0.3 = next try is 70% smaller (fewer
/// iterations, gives up faster).
/// Multiplier applied to the rebalanced amount on each retry attempt. Range: (0, 1].
/// 1 = never shrink (every attempt retries the full requested amount); 0.8 = each retry
/// is 20% smaller than the previous attempt (partial rebalancing); 0.5 = halve each time.
/// The amount for attempt n is RequestedAmountSats × ratio^(n-1), floored at
/// REBALANCE_MIN_AMOUNT_SATS.
/// </summary>
public static double REBALANCE_PROBE_BACKOFF_RATIO = 0.8;
public static double REBALANCE_AMOUNT_BACKOFF_RATIO = 0.8;

/// <summary>
/// Maximum number of QueryRoutes-returned routes the prober will try per amount level.
/// Maximum number of partial payments (MPP shards) LND may split a rebalance into. 1
/// disables splitting; higher values let LND complete an amount across several routes.
/// </summary>
public static int REBALANCE_MAX_PROBE_ROUTES_PER_AMOUNT = 6;
public static uint REBALANCE_MAX_PARTS = 32;

/// <summary>
/// How far back the MonitorRebalancesJob sweeps for already-terminal-but-possibly-wrong
Expand Down Expand Up @@ -428,14 +431,14 @@ static Constants()
var rebBackoff = Environment.GetEnvironmentVariable("REBALANCE_RETRY_BACKOFF_MULTIPLIER");
if (rebBackoff != null) REBALANCE_RETRY_BACKOFF_MULTIPLIER = double.Parse(rebBackoff, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture);

var rebMinProbe = Environment.GetEnvironmentVariable("REBALANCE_MIN_PROBE_AMOUNT_SATS");
if (rebMinProbe != null) REBALANCE_MIN_PROBE_AMOUNT_SATS = long.Parse(rebMinProbe);
var rebMinAmount = Environment.GetEnvironmentVariable("REBALANCE_MIN_AMOUNT_SATS");
if (rebMinAmount != null) REBALANCE_MIN_AMOUNT_SATS = long.Parse(rebMinAmount);

var rebProbeBackoff = Environment.GetEnvironmentVariable("REBALANCE_PROBE_BACKOFF_RATIO");
if (rebProbeBackoff != null) REBALANCE_PROBE_BACKOFF_RATIO = double.Parse(rebProbeBackoff, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture);
var rebProbeBackoff = Environment.GetEnvironmentVariable("REBALANCE_AMOUNT_BACKOFF_RATIO");
if (rebProbeBackoff != null) REBALANCE_AMOUNT_BACKOFF_RATIO = double.Parse(rebProbeBackoff, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture);

var rebMaxProbeRoutes = Environment.GetEnvironmentVariable("REBALANCE_MAX_PROBE_ROUTES_PER_AMOUNT");
if (rebMaxProbeRoutes != null) REBALANCE_MAX_PROBE_ROUTES_PER_AMOUNT = int.Parse(rebMaxProbeRoutes);
var rebMaxParts = Environment.GetEnvironmentVariable("REBALANCE_MAX_PARTS");
if (rebMaxParts != null) REBALANCE_MAX_PARTS = uint.Parse(rebMaxParts);

var rebReconcileWindow = Environment.GetEnvironmentVariable("REBALANCE_RECONCILE_TERMINAL_WINDOW_HOURS");
if (rebReconcileWindow != null) REBALANCE_RECONCILE_TERMINAL_WINDOW_HOURS = int.Parse(rebReconcileWindow);
Expand Down
14 changes: 7 additions & 7 deletions src/Jobs/MonitorRebalancesJob.cs
Original file line number Diff line number Diff line change
Expand Up @@ -125,11 +125,11 @@ private async Task ReconcileAsync(Rebalance rebalance, CancellationToken ct)
// LND returned NotFound (or the call errored). The right action depends on which
// non-terminal state the row is in:
//
// - Pending/Probing: the local ExecuteAsync persisted PaymentHashHex right after
// AddInvoice, but SendPaymentV2 doesn't fire until after the probe succeeds —
// so "no record in LND" is the EXPECTED state during this window. Flipping the
// row to Failed here would kill an in-progress probe. Leave it alone; the local
// execution is the source of truth until it dispatches SendPaymentV2.
// - Pending: the local ExecuteAsync persisted PaymentHashHex right after AddInvoice,
// but SendPaymentV2 hasn't fired yet — so "no record in LND" is the EXPECTED state
// during this window. Flipping the row to Failed here would kill an in-progress
// attempt. Leave it alone; the local execution is the source of truth until it
// dispatches SendPaymentV2.
//
// - InFlight: SendpaymentV2 has been invoked, so LND should know.
// If it doesn't, the payment never reached LND (process crashed mid-dispatch,
Expand Down Expand Up @@ -157,7 +157,7 @@ await _auditService.LogSystemAsync(
NewStatus = rebalance.Status.ToString(),
});
}
else if (rebalance.Status is RebalanceStatus.Pending or RebalanceStatus.Probing)
else if (rebalance.Status is RebalanceStatus.Pending)
{
var window = TimeSpan.FromHours(Constants.REBALANCE_RECONCILE_TERMINAL_WINDOW_HOURS);
if (rebalance.UpdateDatetime < DateTimeOffset.UtcNow - window)
Expand Down Expand Up @@ -198,7 +198,7 @@ await _auditService.LogSystemAsync(
return;
}

if (rebalance.Status is RebalanceStatus.Pending or RebalanceStatus.Probing)
if (rebalance.Status is RebalanceStatus.Pending)
{
_logger.LogDebug(
"Rebalance {RebalanceId} (status={Status}) has a payment record in LND (status={LndStatus}, hash={PaymentHashHex}); local execution still owns it, leaving as-is",
Expand Down
Loading
Loading