From fc412f6fc9de391826eb198213d2c446eb8a459e Mon Sep 17 00:00:00 2001 From: Gabriel Martinez Rodriguez Date: Wed, 6 Nov 2024 15:05:20 +0100 Subject: [PATCH 1/6] feat(txm): purge transactions --- .../transaction-manager/lib/Transaction.ts | 18 +++++++++++++++++- .../lib/TransactionManager.ts | 8 ++++++++ .../lib/TransactionRepository.ts | 12 +++++++++++- packages/transaction-manager/lib/db/types.ts | 2 ++ .../migrations/Migration20241111223000.js | 11 +++++++++++ 5 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 packages/transaction-manager/migrations/Migration20241111223000.js diff --git a/packages/transaction-manager/lib/Transaction.ts b/packages/transaction-manager/lib/Transaction.ts index bcdd1260c2..6775a2d689 100644 --- a/packages/transaction-manager/lib/Transaction.ts +++ b/packages/transaction-manager/lib/Transaction.ts @@ -50,6 +50,10 @@ export class Transaction { readonly attempts: Attempt[] + createdAt: Date + + updatedAt: Date + /** * Stores additional information for the transaction. * Enables originators to provide extra details, such as gas limits, which can be leveraged by customizable services. @@ -66,6 +70,8 @@ export class Transaction { deadline, status, attempts, + createdAt, + updatedAt, metadata, }: { intentId?: UUID @@ -77,6 +83,8 @@ export class Transaction { deadline?: number status?: TransactionStatus attempts?: Attempt[] + createdAt?: Date + updatedAt?: Date metadata?: Record }) { this.intentId = intentId ?? createUUID() @@ -88,11 +96,14 @@ export class Transaction { this.deadline = deadline this.status = status ?? TransactionStatus.Pending this.attempts = attempts ?? [] + this.createdAt = createdAt ?? new Date() + this.updatedAt = updatedAt ?? new Date() this.metadata = metadata } addAttempt(attempt: Attempt): void { this.attempts.push(attempt) + this.updatedAt = new Date() } removeAttempt(hash: Hash): void { @@ -100,6 +111,7 @@ export class Transaction { if (index > -1) { this.attempts.splice(index, 1) } + this.updatedAt = new Date() } getInAirAttempts(): Attempt[] { @@ -114,7 +126,7 @@ export class Transaction { changeStatus(status: TransactionStatus): void { this.status = status - + this.updatedAt = new Date() eventBus.emit(Topics.TransactionStatusChanged, { transaction: this, }) @@ -140,6 +152,8 @@ export class Transaction { status: this.status, attempts: JSON.stringify(this.attempts, bigIntReplacer), metadata: this.metadata ? JSON.stringify(this.metadata, bigIntReplacer) : undefined, + createdAt: this.createdAt.getTime(), + updatedAt: this.updatedAt.getTime(), } } @@ -149,6 +163,8 @@ export class Transaction { args: JSON.parse(row.args, bigIntReviver), attempts: JSON.parse(row.attempts, bigIntReviver), metadata: row.metadata ? JSON.parse(row.metadata, bigIntReviver) : undefined, + createdAt: new Date(row.createdAt), + updatedAt: new Date(row.updatedAt), }) } } diff --git a/packages/transaction-manager/lib/TransactionManager.ts b/packages/transaction-manager/lib/TransactionManager.ts index d3f790cca6..4a4e07ae81 100644 --- a/packages/transaction-manager/lib/TransactionManager.ts +++ b/packages/transaction-manager/lib/TransactionManager.ts @@ -68,6 +68,12 @@ export type TransactionManagerConfig = { */ blockTime?: bigint + /** + * The time (in milliseconds) after which finalized transactions are purged from the database. + * Defaults to 2 minutes. + */ + finalizedTransactionPurgeTime?: number + /** * The gas estimator to use for estimating the gas limit of a transaction. * You can provide your own implementation to override the default one. @@ -99,6 +105,7 @@ export class TransactionManager { public readonly maxPriorityFeePerGas: bigint public readonly rpcAllowDebug: boolean public readonly blockTime: bigint + public readonly finalizedTransactionPurgeTime: number constructor(_config: TransactionManagerConfig) { this.collectors = [] @@ -136,6 +143,7 @@ export class TransactionManager { this.rpcAllowDebug = _config.rpcAllowDebug || false this.blockTime = _config.blockTime || 2n + this.finalizedTransactionPurgeTime = _config.finalizedTransactionPurgeTime || 2 * 60 * 1000 } /** diff --git a/packages/transaction-manager/lib/TransactionRepository.ts b/packages/transaction-manager/lib/TransactionRepository.ts index 2301115c4a..200043cebc 100644 --- a/packages/transaction-manager/lib/TransactionRepository.ts +++ b/packages/transaction-manager/lib/TransactionRepository.ts @@ -1,6 +1,7 @@ import { unknownToError } from "@happychain/common" import type { UUID } from "@happychain/common" import { type Result, ResultAsync } from "neverthrow" +import { Topics, eventBus } from "./EventBus.js" import { NotFinalizedStatuses, Transaction } from "./Transaction.js" import type { TransactionManager } from "./TransactionManager.js" import { db } from "./db/driver.js" @@ -20,8 +21,9 @@ export class TransactionRepository { .where("status", "in", NotFinalizedStatuses) .selectAll() .execute() - + this.notFinalizedTransactions = transactionRows.map((row) => Transaction.fromDbRow(row)) + eventBus.on(Topics.NewBlock, this.purgeFinalizedTransactions.bind(this)) } getNotFinalizedTransactions(): Transaction[] { @@ -103,4 +105,12 @@ export class TransactionRepository { (n) => !this.notFinalizedTransactions.some((t) => t.attempts.some((a) => a.nonce === n)), ) } + + async purgeFinalizedTransactions() { + await db + .deleteFrom("transaction") + .where("status", "not in", NotFinalizedStatuses) + .where("updatedAt", "<", Date.now() - this.transactionManager.finalizedTransactionPurgeTime) + .execute() + } } diff --git a/packages/transaction-manager/lib/db/types.ts b/packages/transaction-manager/lib/db/types.ts index 7f545486f4..e5b71b1589 100644 --- a/packages/transaction-manager/lib/db/types.ts +++ b/packages/transaction-manager/lib/db/types.ts @@ -13,6 +13,8 @@ export interface TransactionTable { status: TransactionStatus attempts: string metadata: string | undefined + createdAt: number + updatedAt: number } export interface Database { diff --git a/packages/transaction-manager/migrations/Migration20241111223000.js b/packages/transaction-manager/migrations/Migration20241111223000.js new file mode 100644 index 0000000000..8a411633bf --- /dev/null +++ b/packages/transaction-manager/migrations/Migration20241111223000.js @@ -0,0 +1,11 @@ +export async function up(db) { + await db.schema + .alterTable("transaction") + .addColumn("createdAt", "integer") + .execute() + + await db.schema + .alterTable("transaction") + .addColumn("updatedAt", "integer") + .execute() +} From e658ff7b9c72b30ec00e3aa137f16f288efed0d1 Mon Sep 17 00:00:00 2001 From: Gabriel Martinez Rodriguez Date: Tue, 12 Nov 2024 11:25:48 +0100 Subject: [PATCH 2/6] chore(txm): format --- .../transaction-manager/lib/TransactionRepository.ts | 2 +- .../migrations/Migration20241111223000.js | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/transaction-manager/lib/TransactionRepository.ts b/packages/transaction-manager/lib/TransactionRepository.ts index 200043cebc..b9cf299c09 100644 --- a/packages/transaction-manager/lib/TransactionRepository.ts +++ b/packages/transaction-manager/lib/TransactionRepository.ts @@ -21,7 +21,7 @@ export class TransactionRepository { .where("status", "in", NotFinalizedStatuses) .selectAll() .execute() - + this.notFinalizedTransactions = transactionRows.map((row) => Transaction.fromDbRow(row)) eventBus.on(Topics.NewBlock, this.purgeFinalizedTransactions.bind(this)) } diff --git a/packages/transaction-manager/migrations/Migration20241111223000.js b/packages/transaction-manager/migrations/Migration20241111223000.js index 8a411633bf..df7a643ebf 100644 --- a/packages/transaction-manager/migrations/Migration20241111223000.js +++ b/packages/transaction-manager/migrations/Migration20241111223000.js @@ -1,11 +1,5 @@ export async function up(db) { - await db.schema - .alterTable("transaction") - .addColumn("createdAt", "integer") - .execute() + await db.schema.alterTable("transaction").addColumn("createdAt", "integer").execute() - await db.schema - .alterTable("transaction") - .addColumn("updatedAt", "integer") - .execute() + await db.schema.alterTable("transaction").addColumn("updatedAt", "integer").execute() } From 79dbb9ac50aafdfc0682ccec287fd455458d452b Mon Sep 17 00:00:00 2001 From: Gabriel Martinez Rodriguez Date: Tue, 12 Nov 2024 11:51:56 +0100 Subject: [PATCH 3/6] chore(txm): self review --- .../transaction-manager/lib/TransactionRepository.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/transaction-manager/lib/TransactionRepository.ts b/packages/transaction-manager/lib/TransactionRepository.ts index b9cf299c09..c116be3dba 100644 --- a/packages/transaction-manager/lib/TransactionRepository.ts +++ b/packages/transaction-manager/lib/TransactionRepository.ts @@ -74,6 +74,11 @@ export class TransactionRepository { .execute(), unknownToError, ) + + this.notFinalizedTransactions = this.notFinalizedTransactions = this.notFinalizedTransactions.filter( + (transaction) => NotFinalizedStatuses.includes(transaction.status), + ) + return result.map(() => undefined) } @@ -91,6 +96,11 @@ export class TransactionRepository { }), unknownToError, ) + + this.notFinalizedTransactions = this.notFinalizedTransactions = this.notFinalizedTransactions.filter( + (transaction) => NotFinalizedStatuses.includes(transaction.status), + ) + return result } From 337423523b18be59d9941af2c5634c62d9a922d3 Mon Sep 17 00:00:00 2001 From: Gabriel Martinez Rodriguez Date: Wed, 20 Nov 2024 12:06:14 +0100 Subject: [PATCH 4/6] chore(txm): pr comments --- packages/transaction-manager/lib/TransactionRepository.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/transaction-manager/lib/TransactionRepository.ts b/packages/transaction-manager/lib/TransactionRepository.ts index c116be3dba..7bb1bd47d5 100644 --- a/packages/transaction-manager/lib/TransactionRepository.ts +++ b/packages/transaction-manager/lib/TransactionRepository.ts @@ -75,8 +75,8 @@ export class TransactionRepository { unknownToError, ) - this.notFinalizedTransactions = this.notFinalizedTransactions = this.notFinalizedTransactions.filter( - (transaction) => NotFinalizedStatuses.includes(transaction.status), + this.notFinalizedTransactions = this.notFinalizedTransactions.filter((transaction) => + NotFinalizedStatuses.includes(transaction.status), ) return result.map(() => undefined) @@ -97,8 +97,8 @@ export class TransactionRepository { unknownToError, ) - this.notFinalizedTransactions = this.notFinalizedTransactions = this.notFinalizedTransactions.filter( - (transaction) => NotFinalizedStatuses.includes(transaction.status), + this.notFinalizedTransactions = this.notFinalizedTransactions.filter((transaction) => + NotFinalizedStatuses.includes(transaction.status), ) return result From 26f7762b6676bf1145dbd1207e5e3712c5530170 Mon Sep 17 00:00:00 2001 From: Gabriel Martinez Rodriguez Date: Tue, 3 Dec 2024 10:50:24 +0100 Subject: [PATCH 5/6] feat(txm): if finalizedTransactionPurgeTime is 0, txm dont purge finalized transaction --- packages/transaction-manager/lib/TransactionManager.ts | 1 + packages/transaction-manager/lib/TransactionRepository.ts | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/transaction-manager/lib/TransactionManager.ts b/packages/transaction-manager/lib/TransactionManager.ts index 4a4e07ae81..f6ef369326 100644 --- a/packages/transaction-manager/lib/TransactionManager.ts +++ b/packages/transaction-manager/lib/TransactionManager.ts @@ -70,6 +70,7 @@ export type TransactionManagerConfig = { /** * The time (in milliseconds) after which finalized transactions are purged from the database. + * If finalizedTransactionPurgeTime is 0, finalized transactions are not purged from the database. * Defaults to 2 minutes. */ finalizedTransactionPurgeTime?: number diff --git a/packages/transaction-manager/lib/TransactionRepository.ts b/packages/transaction-manager/lib/TransactionRepository.ts index 7bb1bd47d5..9f1f2ee3ce 100644 --- a/packages/transaction-manager/lib/TransactionRepository.ts +++ b/packages/transaction-manager/lib/TransactionRepository.ts @@ -23,7 +23,10 @@ export class TransactionRepository { .execute() this.notFinalizedTransactions = transactionRows.map((row) => Transaction.fromDbRow(row)) - eventBus.on(Topics.NewBlock, this.purgeFinalizedTransactions.bind(this)) + + if (this.transactionManager.finalizedTransactionPurgeTime > 0) { + eventBus.on(Topics.NewBlock, this.purgeFinalizedTransactions.bind(this)) + } } getNotFinalizedTransactions(): Transaction[] { From ef383c33b51e9db06b707bd9b86aa7b68b849333 Mon Sep 17 00:00:00 2001 From: Gabriel Martinez Rodriguez Date: Tue, 3 Dec 2024 10:56:57 +0100 Subject: [PATCH 6/6] chore(txm): added inline doc to explain why we use integer instead of DATETIME in sqlite --- .../migrations/Migration20241111223000.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/transaction-manager/migrations/Migration20241111223000.js b/packages/transaction-manager/migrations/Migration20241111223000.js index df7a643ebf..97e24c6ba7 100644 --- a/packages/transaction-manager/migrations/Migration20241111223000.js +++ b/packages/transaction-manager/migrations/Migration20241111223000.js @@ -1,3 +1,8 @@ +/* + SQLite does not have native time types. The SQL interface allows arbitrary type names including "DATE" and "DATETIME", + but this is invalid in this API, and results in "NUMERIC" affinity instead of "INTEGER" affinity, + which is the one we want here +*/ export async function up(db) { await db.schema.alterTable("transaction").addColumn("createdAt", "integer").execute()