From 50b7de9a7a5b721333960e8159ed03056bbf3467 Mon Sep 17 00:00:00 2001 From: cabrador Date: Fri, 29 Aug 2025 09:41:22 +0000 Subject: [PATCH 1/5] Refactor metadata --- Makefile | 6 +- cmd/util-db/clone/cloner.go | 14 +- cmd/util-db/clone/db.go | 9 +- cmd/util-db/clone/patch.go | 10 +- cmd/util-db/db/{update.go => scrape.go} | 48 +- cmd/util-db/generate/db_hash.go | 2 +- cmd/util-db/info/db_hash.go | 2 +- cmd/util-db/main.go | 4 +- cmd/util-db/merge/merge.go | 20 +- cmd/util-db/metadata/generate.go | 25 +- cmd/util-db/metadata/insert.go | 7 +- cmd/util-db/metadata/print.go | 8 +- cmd/util-db/metadata/remove.go | 12 +- {utildb => cmd/util-db/update}/update.go | 423 ++++++----- cmd/util-db/update/update_test.go | 413 +++++++++++ cmd/util-db/validate/validate.go | 12 +- utildb/merger.go | 12 +- utildb/utils.go | 81 +- utildb/utils_test.go | 13 - utildb/validator.go | 23 +- utils/config.go | 15 +- utils/hash_provider.go | 100 +++ utils/hash_provider_test.go | 164 +++++ utils/metadata.go | 897 +++++++---------------- utils/metadata_test.go | 801 +------------------- utils/test_utils.go | 8 +- 26 files changed, 1328 insertions(+), 1801 deletions(-) rename cmd/util-db/db/{update.go => scrape.go} (50%) rename {utildb => cmd/util-db/update}/update.go (60%) create mode 100644 cmd/util-db/update/update_test.go diff --git a/Makefile b/Makefile index d38d89791..2ee8edd03 100644 --- a/Makefile +++ b/Makefile @@ -118,8 +118,8 @@ install-dev-tools: @go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.1.2 format: - @goimports -w ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db/compact ./cmd/util-db/info ./cmd/util-db/metadata ./cmd/util-db/merge ./cmd/util-db/validate ./cmd/util-db/generate ./cmd/util-db/clone ./logger ./register - @gofmt -s -d -w ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db/compact ./cmd/util-db/info ./cmd/util-db/metadata ./cmd/util-db/merge ./cmd/util-db/validate ./cmd/util-db/generate ./cmd/util-db/clone ./logger ./register + @goimports -w ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db ./logger ./register + @gofmt -s -d -w ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db ./logger ./register check: - @golangci-lint run -c .golangci.yml ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db/compact ./cmd/util-db/info ./cmd/util-db/metadata ./cmd/util-db/merge ./cmd/util-db/validate ./cmd/util-db/generate ./cmd/util-db/clone ./logger ./register + @golangci-lint run -c .golangci.yml ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db ./logger ./register diff --git a/cmd/util-db/clone/cloner.go b/cmd/util-db/clone/cloner.go index 4e634c269..7a3f9be41 100644 --- a/cmd/util-db/clone/cloner.go +++ b/cmd/util-db/clone/cloner.go @@ -113,11 +113,15 @@ func (c *cloner) clone() error { } if c.typ != utils.CustomType { - sourceMD := utils.NewAidaDbMetadata(c.sourceDb, c.cfg.LogLevel) - chainID := sourceMD.GetChainID() - - if err = utils.ProcessCloneLikeMetadata(c.cloneDb, c.typ, c.cfg.LogLevel, c.cfg.First, c.cfg.Last, chainID); err != nil { - return err + srcMd := utils.NewAidaDbMetadata(c.sourceDb, c.cfg.LogLevel) + md := utils.NewAidaDbMetadata(c.cloneDb, c.cfg.LogLevel) + err = md.SetDbType(utils.CloneType) + if err != nil { + return fmt.Errorf("cannot set db type for clone db; %v", err) + } + err = md.GenerateMetadata(srcMd.GetChainID()) + if err != nil { + return fmt.Errorf("cannot generate metadata for clone db; %v", err) } } diff --git a/cmd/util-db/clone/db.go b/cmd/util-db/clone/db.go index 3f6484e77..65d48c603 100644 --- a/cmd/util-db/clone/db.go +++ b/cmd/util-db/clone/db.go @@ -53,14 +53,15 @@ func cloneDbAction(ctx *cli.Context) error { if err != nil { return err } + defer func() { + utildb.MustCloseDB(aidaDb) + utildb.MustCloseDB(targetDb) + }() err = clone(cfg, aidaDb, targetDb, utils.CloneType) if err != nil { return err } - utildb.MustCloseDB(aidaDb) - utildb.MustCloseDB(targetDb) - - return utildb.PrintMetadata(cfg.TargetDb) + return utildb.PrintMetadata(targetDb) } diff --git a/cmd/util-db/clone/patch.go b/cmd/util-db/clone/patch.go index 5806d72c0..4e65532c4 100644 --- a/cmd/util-db/clone/patch.go +++ b/cmd/util-db/clone/patch.go @@ -73,15 +73,17 @@ func clonePatchAction(ctx *cli.Context) error { return err } + defer func() { + utildb.MustCloseDB(aidaDb) + utildb.MustCloseDB(targetDb) + }() + err = createPatchClone(cfg, aidaDb, targetDb, firstEpoch, lastEpoch) if err != nil { return err } - utildb.MustCloseDB(aidaDb) - utildb.MustCloseDB(targetDb) - - return utildb.PrintMetadata(cfg.TargetDb) + return utildb.PrintMetadata(targetDb) } // createPatchClone creates aida-db clonePatchCommand diff --git a/cmd/util-db/db/update.go b/cmd/util-db/db/scrape.go similarity index 50% rename from cmd/util-db/db/update.go rename to cmd/util-db/db/scrape.go index c9722360d..43a7ca6e1 100644 --- a/cmd/util-db/db/update.go +++ b/cmd/util-db/db/scrape.go @@ -17,40 +17,48 @@ package db import ( + "fmt" + "github.com/0xsoniclabs/aida/logger" - "github.com/0xsoniclabs/aida/utildb" "github.com/0xsoniclabs/aida/utils" + "github.com/0xsoniclabs/substate/db" "github.com/urfave/cli/v2" ) -// UpdateCommand downloads aida-db and new patches -var UpdateCommand = cli.Command{ - Action: update, - Name: "update", - Usage: "download aida-db patches", +var ScrapeCommand = cli.Command{ + Action: scrapePrepare, + Name: "scrape", + Usage: "Stores state hashes into TargetDb for given range", + ArgsUsage: " ", Flags: []cli.Flag{ - &utils.AidaDbFlag, + &utils.TargetDbFlag, &utils.ChainIDFlag, + &utils.ClientDbFlag, &logger.LogLevelFlag, - &utils.CompactDbFlag, - &utils.DbTmpFlag, - &utils.UpdateTypeFlag, - &utils.SubstateEncodingFlag, }, - Description: ` -Updates aida-db by downloading patches from aida-db generation server. -`, } -// update updates aida-db by downloading patches from aida-db generation server. -func update(ctx *cli.Context) error { - cfg, err := utils.NewConfig(ctx, utils.NoArgs) +// scrapePrepare stores state hashes into Target for given range +func scrapePrepare(ctx *cli.Context) error { + cfg, argErr := utils.NewConfig(ctx, utils.BlockRangeArgs) + if argErr != nil { + return argErr + } + + log := logger.NewLogger(cfg.LogLevel, "UtilDb-Scrape") + log.Infof("Scraping for range %d-%d", cfg.First, cfg.Last) + + database, err := db.NewDefaultBaseDB(cfg.TargetDb) if err != nil { - return err + return fmt.Errorf("error opening stateHash leveldb %s: %v", cfg.TargetDb, err) } - if err = utildb.Update(cfg); err != nil { + defer database.Close() + + err = utils.StateAndBlockHashScraper(ctx.Context, cfg.ChainID, cfg.ClientDb, database, cfg.First, cfg.Last, log) + if err != nil { return err } - return utildb.PrintMetadata(cfg.AidaDb) + log.Infof("Scraping finished") + return nil } diff --git a/cmd/util-db/generate/db_hash.go b/cmd/util-db/generate/db_hash.go index 6b22aa835..22048c02a 100644 --- a/cmd/util-db/generate/db_hash.go +++ b/cmd/util-db/generate/db_hash.go @@ -28,7 +28,7 @@ func generateDbHashAction(ctx *cli.Context) error { return err } - aidaDb, err := db.NewDefaultBaseDB(cfg.AidaDb) + aidaDb, err := db.NewDefaultSubstateDB(cfg.AidaDb) if err != nil { return fmt.Errorf("cannot open db; %v", err) } diff --git a/cmd/util-db/info/db_hash.go b/cmd/util-db/info/db_hash.go index dd65aa4a9..d9efb97c6 100644 --- a/cmd/util-db/info/db_hash.go +++ b/cmd/util-db/info/db_hash.go @@ -41,7 +41,7 @@ var printDbHashCommand = cli.Command{ func printDbHashAction(ctx *cli.Context) error { var force = ctx.Bool(flags.ForceFlag.Name) - aidaDb, err := db.NewReadOnlyBaseDB(ctx.String(utils.AidaDbFlag.Name)) + aidaDb, err := db.NewReadOnlySubstateDB(ctx.String(utils.AidaDbFlag.Name)) if err != nil { return fmt.Errorf("cannot open db; %v", err) } diff --git a/cmd/util-db/main.go b/cmd/util-db/main.go index 24e2d592e..7c2fd6b6b 100644 --- a/cmd/util-db/main.go +++ b/cmd/util-db/main.go @@ -22,13 +22,13 @@ import ( "github.com/0xsoniclabs/aida/cmd/util-db/clone" "github.com/0xsoniclabs/aida/cmd/util-db/compact" - "github.com/0xsoniclabs/aida/cmd/util-db/db" "github.com/0xsoniclabs/aida/cmd/util-db/generate" "github.com/0xsoniclabs/aida/cmd/util-db/info" "github.com/0xsoniclabs/aida/cmd/util-db/merge" "github.com/0xsoniclabs/aida/cmd/util-db/metadata" "github.com/0xsoniclabs/aida/cmd/util-db/primer" "github.com/0xsoniclabs/aida/cmd/util-db/scrape" + "github.com/0xsoniclabs/aida/cmd/util-db/update" "github.com/0xsoniclabs/aida/cmd/util-db/validate" "github.com/urfave/cli/v2" ) @@ -47,7 +47,7 @@ var UtilDbApp = cli.App{ &validate.Command, &metadata.Command, &generate.Command, - &db.UpdateCommand, + &update.Command, &scrape.Command, //Priming only diff --git a/cmd/util-db/merge/merge.go b/cmd/util-db/merge/merge.go index 34e002eae..945c9fbc5 100644 --- a/cmd/util-db/merge/merge.go +++ b/cmd/util-db/merge/merge.go @@ -58,13 +58,13 @@ func mergeAction(ctx *cli.Context) error { sourcePaths[i] = ctx.Args().Get(i) } - targetDb, err := db.NewDefaultBaseDB(cfg.AidaDb) + targetDb, err := db.NewDefaultSubstateDB(cfg.AidaDb) if err != nil { return fmt.Errorf("cannot open db; %v", err) } var ( - dbs []db.BaseDB + dbs []db.SubstateDB md *utils.AidaDbMetadata ) @@ -73,17 +73,15 @@ func mergeAction(ctx *cli.Context) error { if err != nil { return fmt.Errorf("cannot open source databases: %w", err) } - md, err = utils.ProcessMergeMetadata(cfg, targetDb, dbs, sourcePaths) - if err != nil { - return err - } - // todo this should not be necessary - do not close aida-db in metadata - targetDb, err = db.NewDefaultBaseDB(cfg.AidaDb) - if err != nil { - return fmt.Errorf("cannot re-open db: %w", err) + // merge metadata from all source dbs + targetMD := utils.NewAidaDbMetadata(targetDb, cfg.LogLevel) + for _, db := range dbs { + sourceMD := utils.NewAidaDbMetadata(db, cfg.LogLevel) + if err := targetMD.Merge(sourceMD); err != nil { + return fmt.Errorf("cannot merge metadata: %w", err) + } } - for _, database := range dbs { utildb.MustCloseDB(database) } diff --git a/cmd/util-db/metadata/generate.go b/cmd/util-db/metadata/generate.go index 5fc22b828..c6817261d 100644 --- a/cmd/util-db/metadata/generate.go +++ b/cmd/util-db/metadata/generate.go @@ -41,26 +41,25 @@ func generateAction(ctx *cli.Context) error { return argErr } - base, err := db.NewDefaultBaseDB(cfg.AidaDb) + aidaDb, err := db.NewDefaultSubstateDB(cfg.AidaDb) if err != nil { return err } - - sdb := db.MakeDefaultSubstateDBFromBaseDB(base) - fb, lb, ok := utils.FindBlockRangeInSubstate(sdb) + defer utildb.MustCloseDB(aidaDb) + fb, lb, ok := utils.FindBlockRangeInSubstate(aidaDb) if !ok { return errors.New("cannot find block range in substate") } - md := utils.NewAidaDbMetadata(base, "INFO") - md.FirstBlock = fb - md.LastBlock = lb - if err = md.SetFreshMetadata(cfg.ChainID); err != nil { + md := utils.NewAidaDbMetadata(aidaDb, "INFO") + if err = md.SetFirstBlock(fb); err != nil { return err } - - utildb.MustCloseDB(base) - - return utildb.PrintMetadata(cfg.AidaDb) - + if err = md.SetLastBlock(lb); err != nil { + return err + } + if err = md.SetChainID(cfg.ChainID); err != nil { + return err + } + return utildb.PrintMetadata(aidaDb) } diff --git a/cmd/util-db/metadata/insert.go b/cmd/util-db/metadata/insert.go index d3cfde29d..c27d83f66 100644 --- a/cmd/util-db/metadata/insert.go +++ b/cmd/util-db/metadata/insert.go @@ -59,17 +59,16 @@ func insertAction(ctx *cli.Context) (finalErr error) { valArg := ctx.Args().Get(1) // open db - base, err := db.NewDefaultBaseDB(aidaDbPath) + aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath) if err != nil { return err } defer func() { - finalErr = errors.Join(finalErr, base.Close()) + finalErr = errors.Join(finalErr, aidaDb.Close()) }() - md := utils.NewAidaDbMetadata(base, "INFO") - + md := utils.NewAidaDbMetadata(aidaDb, "INFO") switch db.MetadataPrefix + keyArg { case utils.FirstBlockPrefix: val, err = strconv.ParseUint(valArg, 10, 64) diff --git a/cmd/util-db/metadata/print.go b/cmd/util-db/metadata/print.go index 60db4e07b..386412eb1 100644 --- a/cmd/util-db/metadata/print.go +++ b/cmd/util-db/metadata/print.go @@ -19,6 +19,7 @@ package metadata import ( "github.com/0xsoniclabs/aida/utildb" "github.com/0xsoniclabs/aida/utils" + "github.com/0xsoniclabs/substate/db" "github.com/urfave/cli/v2" ) @@ -37,5 +38,10 @@ func printAction(ctx *cli.Context) error { return argErr } - return utildb.PrintMetadata(cfg.AidaDb) + aidaDb, err := db.NewReadOnlySubstateDB(cfg.AidaDb) + if err != nil { + return err + } + + return utildb.PrintMetadata(aidaDb) } diff --git a/cmd/util-db/metadata/remove.go b/cmd/util-db/metadata/remove.go index 4eea59e99..0bb37376c 100644 --- a/cmd/util-db/metadata/remove.go +++ b/cmd/util-db/metadata/remove.go @@ -41,18 +41,14 @@ Removes block and epoch range and ChainID from metadata for given AidaDb. // is applied onto it func removeAction(ctx *cli.Context) (finalErr error) { aidaDbPath := ctx.String(utils.AidaDbFlag.Name) - - // open db - base, err := db.NewDefaultBaseDB(aidaDbPath) + aidaDb, err := db.NewReadOnlySubstateDB(aidaDbPath) if err != nil { return err } - defer func() { - finalErr = errors.Join(finalErr, base.Close()) + finalErr = errors.Join(finalErr, aidaDb.Close()) }() - md := utils.NewAidaDbMetadata(base, "DEBUG") - md.DeleteMetadata() - + md := utils.NewAidaDbMetadata(aidaDb, "DEBUG") + md.Delete() return nil } diff --git a/utildb/update.go b/cmd/util-db/update/update.go similarity index 60% rename from utildb/update.go rename to cmd/util-db/update/update.go index d39735335..45b21bced 100644 --- a/utildb/update.go +++ b/cmd/util-db/update/update.go @@ -14,13 +14,14 @@ // You should have received a copy of the GNU Lesser General Public License // along with Aida. If not, see . -package utildb +package update import ( "archive/tar" "bufio" "compress/gzip" - "errors" + "crypto/md5" + "encoding/hex" "fmt" "io" "net/http" @@ -32,32 +33,74 @@ import ( "time" "github.com/0xsoniclabs/aida/logger" + "github.com/0xsoniclabs/aida/utildb" "github.com/0xsoniclabs/aida/utils" "github.com/0xsoniclabs/substate/db" + "github.com/cockroachdb/errors" + "github.com/urfave/cli/v2" ) +// Command downloads aida-db and new patches +var Command = cli.Command{ + Action: updateAction, + Name: "update", + Usage: "download aida-db patches", + Flags: []cli.Flag{ + &utils.AidaDbFlag, + &utils.ChainIDFlag, + &logger.LogLevelFlag, + &utils.CompactDbFlag, + &utils.DbTmpFlag, + &utils.UpdateTypeFlag, + &utils.SubstateEncodingFlag, + }, + Description: ` +Updates aida-db by downloading patches from aida-db generation server. +`, +} + const ( maxNumberOfDownloadAttempts = 5 - firstMainnetPatchFileName = "5577-46750.tar.gz" - firstTestnetPatchFileName = "" // todo fill with first testnet patch once lachesis patch for testnet is released stateHashPatchFileName = "state-hashes_0-68940000" ) -// Update implements updating command to be called from various commands and automatically downloads aida-db patches. -func Update(cfg *utils.Config) error { +// updateAction updates aida-db by downloading patches from aida-db generation server. +func updateAction(ctx *cli.Context) error { + cfg, err := utils.NewConfig(ctx, utils.NoArgs) + if err != nil { + return err + } + if err = update(cfg); err != nil { + return err + } + + aidaDb, err := db.NewReadOnlySubstateDB(cfg.AidaDb) + if err != nil { + return fmt.Errorf("failed to open aida-db: %v", err) + } + return utildb.PrintMetadata(aidaDb) +} + +// update implements updating command to be called from various commands and automatically downloads aida-db patches. +func update(cfg *utils.Config) error { log := logger.NewLogger(cfg.LogLevel, "DB Update") - start := time.Now() - targetDbFirstBlock, targetDbLastBlock, err := getTargetDbBlockRange(cfg) + targetDB, err := db.NewReadOnlySubstateDB(cfg.AidaDb) + if err != nil { + return err + } + md := utils.NewAidaDbMetadata(targetDB, cfg.LogLevel) + err = md.GenerateMetadata(cfg.ChainID) if err != nil { - return fmt.Errorf("unable retrieve aida-db metadata; %v", err) + return fmt.Errorf("unable generate aida-db metadata; %v", err) } - log.Noticef("First block of your AidaDb: #%v", targetDbFirstBlock) - log.Noticef("Last block of your AidaDb: #%v", targetDbLastBlock) + log.Noticef("First block of your AidaDb: #%v", md.GetFirstBlock()) + log.Noticef("Last block of your AidaDb: #%v", md.GetLastBlock()) + start := time.Now() // retrieve available patches from aida-db generation server - patches, err := retrievePatchesToDownload(cfg, targetDbFirstBlock, targetDbLastBlock) + patches, err := retrievePatchesToDownload(cfg, md) if err != nil { return fmt.Errorf("unable to prepare list of aida-db patches for download; %v", err) } @@ -76,7 +119,7 @@ func Update(cfg *utils.Config) error { log.Noticef("These patches are in que for download:%v", str) // we need to know whether Db is new for metadata - err = patchesDownloader(cfg, patches, targetDbFirstBlock, targetDbLastBlock) + err = patchesDownloader(cfg, patches, md) if err != nil { return err } @@ -87,7 +130,7 @@ func Update(cfg *utils.Config) error { } // getTargetDbBlockRange initialize aidaMetadata of targetDB -func getTargetDbBlockRange(cfg *utils.Config) (uint64, uint64, error) { +func getTargetDbBlockRange(cfg *utils.Config) (firstAidaDbBlock uint64, lastAidaDbBlock uint64, finalErr error) { // load stats of current aida-db to download just latest patches _, err := os.Stat(cfg.AidaDb) if err != nil { @@ -103,7 +146,9 @@ func getTargetDbBlockRange(cfg *utils.Config) (uint64, uint64, error) { if err != nil { return 0, 0, err } - defer sdb.Close() + defer func() { + finalErr = errors.Join(finalErr, sdb.Close()) + }() firstAidaDbBlock, lastAidaDbBlock, ok := utils.FindBlockRangeInSubstate(sdb) if !ok { return 0, 0, fmt.Errorf("cannot find blocks in substate; is substate present in given db? %v", cfg.AidaDb) @@ -113,7 +158,7 @@ func getTargetDbBlockRange(cfg *utils.Config) (uint64, uint64, error) { } // patchesDownloader processes patch names to download then download them in pipelined process -func patchesDownloader(cfg *utils.Config, patches []utils.PatchJson, firstBlock, lastBlock uint64) error { +func patchesDownloader(cfg *utils.Config, patches []utils.PatchJson, md utils.Metadata) error { // create channel to push patch labels trough channel patchesChan := pushPatchToChanel(patches) @@ -124,7 +169,7 @@ func patchesDownloader(cfg *utils.Config, patches []utils.PatchJson, firstBlock, decompressedPatchChan, errDecompressChan := decompressPatch(cfg, downloadedPatchChan, errChan) // merge decompressed patches - err := mergePatch(cfg, decompressedPatchChan, errDecompressChan, firstBlock, lastBlock) + err := mergePatch(cfg, decompressedPatchChan, errDecompressChan, md) if err != nil { return err } @@ -133,16 +178,14 @@ func patchesDownloader(cfg *utils.Config, patches []utils.PatchJson, firstBlock, } // mergePatch takes decompressed patches and merges them into aida-db -func mergePatch(cfg *utils.Config, decompressChan chan string, errChan chan error, firstAidaDbBlock, lastAidaDbBlock uint64) error { +func mergePatch(cfg *utils.Config, decompressChan chan string, errChan chan error, md utils.Metadata) error { var ( - err error - patchDb db.BaseDB - targetMD *utils.AidaDbMetadata - isNewDb bool - log = logger.NewLogger(cfg.LogLevel, "aida-merge-patch") + err error + isNewDb bool + log = logger.NewLogger(cfg.LogLevel, "aida-merge-patch") ) - if lastAidaDbBlock == 0 { + if md.GetLastBlock() == 0 { isNewDb = true } @@ -151,103 +194,84 @@ func mergePatch(cfg *utils.Config, decompressChan chan string, errChan chan erro for { select { case err, ok := <-errChan: - { - if ok { - return err - } + if ok { + return err } case extractedPatchPath, ok := <-decompressChan: - { - // no more data then return - if !ok { - return nil - } - - // firstRun is triggered only when applying first patch - // distinction is necessary because if targetDb was empty we can move patch directly into targetPath - // before opening database for writing - if firstRun { - firstRun = false - // first patch to empty database is moved to target right away - // this way we can skip iteration and metadata inserts - if isNewDb { - log.Noticef("AIDA-DB was empty - directly saving first patch") - // move extracted patch to target location - first attempting with os.Rename because it is fastest - if err = os.Rename(extractedPatchPath, cfg.AidaDb); err != nil { - // attempting with deep copy - needed when moving across different disks - if err2 := utils.CopyDir(extractedPatchPath, cfg.AidaDb); err2 != nil { - return fmt.Errorf("unable to move patch into aida-db target; %v (%v)", err2, err) - } - } - } - - // open targetDB only after there is already first patch or any existing previous data - targetDb, err := db.NewDefaultBaseDB(cfg.AidaDb) - if err != nil { - return fmt.Errorf("can't open aidaDb; %v", err) - } - targetMD = utils.NewAidaDbMetadata(targetDb, cfg.LogLevel) - - errOldAida := targetMD.UpdateMetadataInOldAidaDb(cfg.ChainID, firstAidaDbBlock, lastAidaDbBlock) - if errOldAida != nil { - log.Warningf("error UpdateMetadataInOldAidaDb; %v", errOldAida) - } + // no more data then return + if !ok { + return nil + } - defer func() { - if err = targetMD.Db.Close(); err != nil { - log.Warningf("patchesDownloader: cannot close targetDb; %v", err) + // firstRun is triggered only when applying first patch + // distinction is necessary because if targetDb was empty we can move patch directly into targetPath + // before opening database for writing + if firstRun { + firstRun = false + // first patch to empty database is moved to target right away + // this way we can skip iteration and metadata inserts + if isNewDb { + log.Noticef("AIDA-DB was empty - directly saving first patch") + // move extracted patch to target location - first attempting with os.Rename because it is fastest + if err = os.Rename(extractedPatchPath, cfg.AidaDb); err != nil { + // attempting with deep copy - needed when moving across different disks + if err2 := utils.CopyDir(extractedPatchPath, cfg.AidaDb); err2 != nil { + return fmt.Errorf("unable to move patch into aida-db target; %v (%v)", err2, err) } - }() - - // patch was already applied before opening targetDb hence we don't need to merge it anymore - if isNewDb { - continue } } - - // merge newly extracted patch - patchDb, err = db.NewReadOnlyBaseDB(extractedPatchPath) - if err != nil { - return fmt.Errorf("cannot open targetDb; %v", err) - } - - // we only check metadata if not applying stateHashPatch - if !strings.Contains(extractedPatchPath, stateHashPatchFileName) { - err = targetMD.CheckUpdateMetadata(cfg, patchDb) - if err != nil { + // patch was already applied before opening targetDb hence we don't need to merge it anymore + if !isNewDb { + if err = mergeToExistingAidaDb(cfg, md, extractedPatchPath); err != nil { return err } } + } + } + } +} - m := NewMerger(cfg, targetMD.Db, []db.BaseDB{patchDb}, []string{extractedPatchPath}, nil) +func mergeToExistingAidaDb(cfg *utils.Config, md utils.Metadata, extractedPatchPath string) error { + // merge newly extracted patch + patchDb, err := db.NewReadOnlySubstateDB(extractedPatchPath) + if err != nil { + return fmt.Errorf("cannot open patchDb; %v", err) + } - err = m.Merge() - if err != nil { - return fmt.Errorf("unable to merge %v; %v", extractedPatchPath, err) - } + // we only check metadata if not applying stateHashPatch + if !strings.Contains(extractedPatchPath, stateHashPatchFileName) { + patchMD := utils.NewAidaDbMetadata(patchDb, cfg.LogLevel) + // patches contain chainID in metadata + if err = patchMD.GenerateMetadata(0); err != nil { + return fmt.Errorf("cannot generate patch metadata; %v", err) + } + if err = md.Merge(patchMD); err != nil { + return fmt.Errorf("cannot merge patch metadata; %v", err) + } + } - // we only set metadata if not applying stateHashPatch - if strings.Contains(extractedPatchPath, stateHashPatchFileName) { - err = targetMD.SetHasHashPatch() - if err != nil { - return fmt.Errorf("cannot set has-hash-patch; %v", err) - } - } else { - err = targetMD.SetAll() - if err != nil { - return fmt.Errorf("cannot set metadata; %v", err) - } - } - m.CloseSourceDbs() + m := utildb.NewMerger(cfg, md.GetDb(), []db.SubstateDB{patchDb}, []string{extractedPatchPath}, nil) - // remove patch - err = os.RemoveAll(extractedPatchPath) - if err != nil { - return err - } - } + err = m.Merge() + if err != nil { + return fmt.Errorf("unable to merge %v; %v", extractedPatchPath, err) + } + + // we only set metadata if not applying stateHashPatch + if strings.Contains(extractedPatchPath, stateHashPatchFileName) { + err = md.SetHasHashPatch() + if err != nil { + return fmt.Errorf("cannot set has-hash-patch; %v", err) } } + m.CloseSourceDbs() + + // remove patch + err = os.RemoveAll(extractedPatchPath) + if err != nil { + return err + } + return nil } // decompressPatch takes tar.gz archives and decompresses them, then sends them for further processing @@ -262,38 +286,33 @@ func decompressPatch(cfg *utils.Config, patchChan chan utils.PatchJson, errChan for { select { case err, ok := <-errChan: - { - if ok { - errDecompressChan <- err - return - } + if ok { + errDecompressChan <- err + return } case patch, ok := <-patchChan: - { - if !ok { - return - } - log.Debugf("Decompressing %v...", patch.FileName) + if !ok { + return + } + log.Debugf("Decompressing %v...", patch.FileName) - compressedPatchPath := filepath.Join(cfg.DbTmp, patch.FileName) - err := extractTarGz(compressedPatchPath, cfg.DbTmp) - if err != nil { - errDecompressChan <- err - return - } + compressedPatchPath := filepath.Join(cfg.DbTmp, patch.FileName) + err := extractTarGz(compressedPatchPath, cfg.DbTmp) + if err != nil { + errDecompressChan <- err + return + } - // extracted patch is folder without the .tar.gz extension - extractedPatchPath := strings.TrimSuffix(compressedPatchPath, ".tar.gz") + // extracted patch is folder without the .tar.gz extension + extractedPatchPath := strings.TrimSuffix(compressedPatchPath, ".tar.gz") - decompressedPatchChan <- extractedPatchPath - // remove compressed patch - err = os.RemoveAll(compressedPatchPath) - if err != nil { - errDecompressChan <- err - return - } + decompressedPatchChan <- extractedPatchPath + // remove compressed patch + err = os.RemoveAll(compressedPatchPath) + if err != nil { + errDecompressChan <- err + return } - } } }() @@ -327,15 +346,15 @@ func downloadPatch(cfg *utils.Config, patchesChan chan utils.PatchJson) (chan ut log.Debugf("Finished downloading %s!", patch.FileName) log.Debugf("Calculating %s md5...", patch.FileName) - md5, err := calculateMD5Sum(compressedPatchPath) + sum, err := calculateMD5Sum(compressedPatchPath) if err != nil { errChan <- fmt.Errorf("archive %v; unable to calculate md5sum; %v", patch.FileName, err) return } // Compare whether downloaded file matches expected md5 - if strings.Compare(md5, patch.TarHash) != 0 { - errChan <- fmt.Errorf("archive %v doesn't have matching md5; archive %v, expected %v", patch.FileName, md5, patch.TarHash) + if strings.Compare(sum, patch.TarHash) != 0 { + errChan <- fmt.Errorf("archive %v doesn't have matching md5; archive %v, expected %v", patch.FileName, sum, patch.TarHash) return } @@ -345,6 +364,35 @@ func downloadPatch(cfg *utils.Config, patchesChan chan utils.PatchJson) (chan ut return downloadedPatchChan, errChan } +// calculateMD5Sum calculates MD5 hash of given file +func calculateMD5Sum(filePath string) (md5Sum string, finalErr error) { + // Open the file + file, err := os.Open(filePath) + if err != nil { + return "", fmt.Errorf("unable open file %s; %v", filePath, err.Error()) + } + defer func() { + finalErr = errors.Join(finalErr, file.Close()) + }() + + // Create a new MD5 hash instance + hash := md5.New() + + // Copy the file content into the hash instance + _, err = io.Copy(hash, file) + if err != nil { + return "", fmt.Errorf("unable to calculate md5; %v", err) + } + + // Calculate the MD5 checksum as a byte slice + checksum := hash.Sum(nil) + + // Convert the checksum to a hexadecimal string + md5sum := hex.EncodeToString(checksum) + + return md5sum, nil +} + // pushPatchToChanel used to pipe strings into channel func pushPatchToChanel(strings []utils.PatchJson) chan utils.PatchJson { ch := make(chan utils.PatchJson, 1) @@ -358,15 +406,13 @@ func pushPatchToChanel(strings []utils.PatchJson) chan utils.PatchJson { } // retrievePatchesToDownload retrieves all available patches from aida-db generation server. -func retrievePatchesToDownload(cfg *utils.Config, targetDbFirstBlock uint64, targetDbLastBlock uint64) ([]utils.PatchJson, error) { +func retrievePatchesToDownload(cfg *utils.Config, md utils.Metadata) ([]utils.PatchJson, error) { if cfg.UpdateType != "stable" && cfg.UpdateType != "nightly" { return nil, fmt.Errorf("please choose correct data-type with --data-type flag (stable/nightly)") } var includeNightly = cfg.UpdateType == "nightly" - var isAddingLachesisPatch = false - // download list of available availablePatches availablePatches, err := utils.DownloadPatchesJson() if err != nil { @@ -388,13 +434,8 @@ func retrievePatchesToDownload(cfg *utils.Config, targetDbFirstBlock uint64, tar } } // skip every patch which is sooner than previous last block - if patch.ToBlock <= targetDbLastBlock { - // if patch is lachesis and user has not got it in their db we download it - if patch.ToBlock == utils.FirstOperaBlock-1 && targetDbFirstBlock == utils.FirstOperaBlock { - isAddingLachesisPatch = true - } else { - continue - } + if patch.ToBlock <= md.GetLastBlock() { + continue } // add all stable patches and nightly only if user wants to download nightly @@ -403,14 +444,6 @@ func retrievePatchesToDownload(cfg *utils.Config, targetDbFirstBlock uint64, tar } } - // if user has second patch already in their db, we have to re-download it again and delete old update-set key - if isAddingLachesisPatch && targetDbFirstBlock == utils.FirstOperaBlock { - patchesToDownload, err = appendFirstPatch(cfg, availablePatches, patchesToDownload) - if err != nil { - return nil, err - } - } - sort.Sort(ByToBlock(patchesToDownload)) return patchesToDownload, nil @@ -429,61 +462,8 @@ func (a ByToBlock) Less(i, j int) bool { return a[i].ToBlock < a[j].ToBlock } -// appendFirstPatch finds whether user is downloading fresh new db or updating an existing one. -// If updating an existing one, first patch is appended to download and first update-set is deleted -func appendFirstPatch(cfg *utils.Config, availablePatches []utils.PatchJson, patchesToDownload []utils.PatchJson) ([]utils.PatchJson, error) { - var expectedFileName string - - if cfg.ChainID == utils.MainnetChainID { - expectedFileName = firstMainnetPatchFileName - } else if cfg.ChainID == utils.TestnetChainID { - expectedFileName = firstTestnetPatchFileName - } else { - return nil, errors.New("please choose chain-id with --chainid") - } - - // did we already append first patch? - for _, patch := range patchesToDownload { - if patch.FileName == expectedFileName { - - // first patch was already appended - that means user is downloading fresh db - return patchesToDownload, nil - } - } - - for _, patch := range availablePatches { - if patch.FileName == expectedFileName { - patchesToDownload = append(patchesToDownload, patch) - // we need to remove first update-set for data consistency - err := deleteOperaWorldStateFromUpdateSet(cfg.AidaDb) - if err != nil { - return nil, err - } - break - } - } - - return patchesToDownload, nil -} - -// deleteOperaWorldStateFromUpdateSet when user has already merged second patch, and we are prepending lachesis patch. -// This situation could happen due to lachesis patch being implemented later than rest of the Db -func deleteOperaWorldStateFromUpdateSet(dbPath string) error { - updateDb, err := db.NewDefaultUpdateDB(dbPath) - if err != nil { - return fmt.Errorf("cannot open update-db; %v", err) - } - - err = updateDb.DeleteUpdateSet(utils.FirstOperaBlock - 1) - if err != nil { - return err - } - - return updateDb.Close() -} - // downloadFile downloads file - used for downloading individual patches. -func downloadFile(filePath string, parentPath string, url string) error { +func downloadFile(filePath string, parentPath string, url string) (finalErr error) { // Create parent directories if they don't exist err := os.MkdirAll(parentPath, 0755) if err != nil { @@ -495,7 +475,9 @@ func downloadFile(filePath string, parentPath string, url string) error { if err != nil { return fmt.Errorf("error opening file: %v", err) } - defer file.Close() + defer func() { + finalErr = errors.Join(finalErr, file.Close()) + }() // Get the current file size fileInfo, err := file.Stat() @@ -542,11 +524,11 @@ func getFileContentsFromUrl(url string, startSize int64, out *bufio.Writer) erro startSize += written } - return fmt.Errorf("failed after %v attempts; %s", maxNumberOfDownloadAttempts, err.Error()) + return fmt.Errorf("failed after %v attempts; %s", maxNumberOfDownloadAttempts, err) } // downloadFileContents downloads file contents from given start -func downloadFileContents(url string, startSize int64, out *bufio.Writer) (int64, error) { +func downloadFileContents(url string, startSize int64, out *bufio.Writer) (size int64, finalErr error) { // Set the "Range" header to resume the download from the current size req, err := http.NewRequest("GET", url, nil) if err != nil { @@ -561,7 +543,9 @@ func downloadFileContents(url string, startSize int64, out *bufio.Writer) (int64 if err != nil { return 0, fmt.Errorf("error making request: %v", err) } - defer resp.Body.Close() + defer func() { + finalErr = errors.Join(finalErr, resp.Body.Close()) + }() // Check server response again if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusRequestedRangeNotSatisfiable { @@ -573,20 +557,27 @@ func downloadFileContents(url string, startSize int64, out *bufio.Writer) (int64 } // extractTarGz extracts tar file contents into location of output folder -func extractTarGz(tarGzFile, outputFolder string) error { +func extractTarGz(tarGzFile, outputFolder string) (finalErr error) { // Open the tar.gz file file, err := os.Open(tarGzFile) if err != nil { return err } - defer file.Close() + defer func() { + finalErr = errors.Join(finalErr, file.Close()) + }() // Create the gzip readerÏ gr, err := gzip.NewReader(file) if err != nil { return err } - defer gr.Close() + defer func() { + err = gr.Close() + if err != nil { + finalErr = errors.Join(finalErr, err) + } + }() // Create the tar reader tr := tar.NewReader(gr) @@ -610,13 +601,13 @@ func extractTarGz(tarGzFile, outputFolder string) error { // Make sure that path does not contain ".." if strings.Contains(targetPath, "..") { - return fmt.Errorf("Tarfile is attempting to use path containing ..: %s", targetPath) + return fmt.Errorf("tarfile is attempting to use path containing ..: %s", targetPath) } // Make sure that output file does not overwrite existing files _, err = os.Stat(targetPath) if err == nil || os.IsExist(err) { - return fmt.Errorf("Tarfile is attempting to overwrite existing file. This may have happened due to previous failed attempt to extract the file - consider removing the folder %s", targetPath) + return fmt.Errorf("tarfile is attempting to overwrite existing file. This may have happened due to previous failed attempt to extract the file - consider removing the folder %s", targetPath) } // Check if it's a directory diff --git a/cmd/util-db/update/update_test.go b/cmd/util-db/update/update_test.go new file mode 100644 index 000000000..c4f8b6159 --- /dev/null +++ b/cmd/util-db/update/update_test.go @@ -0,0 +1,413 @@ +// Copyright 2025 Sonic Labs +// This file is part of Aida Testing Infrastructure for Sonic +// +// Aida is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Aida is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Aida. If not, see . + +package update + +import ( + "context" + "math" + "math/big" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/0xsoniclabs/aida/logger" + "github.com/0xsoniclabs/substate/substate" + "github.com/0xsoniclabs/substate/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/holiman/uint256" + "github.com/stretchr/testify/assert" + + "github.com/0xsoniclabs/aida/utils" + "github.com/0xsoniclabs/substate/db" + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" +) + +func TestUpdate_UpdateDbCommand(t *testing.T) { + aidaDbPath := t.TempDir() + "/aida-db" + aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath) + require.NoError(t, err) + + // Put substate with max latest block to avoid any updating + ss := utils.GetTestSubstate("pb") + ss.Block = math.MaxUint64 + ss.Env.Number = math.MaxUint64 + err = aidaDb.PutSubstate(ss) + require.NoError(t, err) + + err = aidaDb.Close() + require.NoError(t, err) + + app := cli.NewApp() + app.Commands = []*cli.Command{&Command} + + args := utils.NewArgs("test"). + Arg(Command.Name). + Flag(utils.AidaDbFlag.Name, aidaDbPath). + Flag(logger.LogLevelFlag.Name, "CRITICAL"). + Flag(utils.ChainIDFlag.Name, int(utils.SonicMainnetChainID)). + Flag(utils.DbTmpFlag.Name, t.TempDir()). + Flag(utils.SubstateEncodingFlag.Name, "protobuf"). + Build() + err = app.Run(args) + require.NoError(t, err) +} + +func TestCmd_UpdateCommand(t *testing.T) { + // given + tmpDir := t.TempDir() + aidaDbPath := filepath.Join(tmpDir, "aida-db") + tmpDbPath := filepath.Join(tmpDir, "tmp-db") + require.NoError(t, os.Mkdir(tmpDbPath, os.ModePerm)) + + app := cli.NewApp() + app.Commands = []*cli.Command{&Command} + + args := utils.NewArgs("test"). + Arg(Command.Name). + Flag(utils.AidaDbFlag.Name, aidaDbPath). + Flag(utils.ChainIDFlag.Name, int(utils.MainnetChainID)). + Flag(utils.DbTmpFlag.Name, tmpDbPath). + Flag(utils.UpdateTypeFlag.Name, "stable"). + Flag(utils.SubstateEncodingFlag.Name, "protobuf"). + Build() + + // Create a context with cancellation to control the app execution + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Channel to communicate the result of app.Run + resultChan := make(chan error, 1) + + // Run the app in a goroutine + go func() { + err := app.Run(args) + resultChan <- err + }() + + // Monitor tmp-db folder for gz files + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + timeout := time.NewTimer(1 * time.Minute) // 1 minute timeout + defer timeout.Stop() + + for { + select { + case err := <-resultChan: + // App completed naturally + assert.NoError(t, err) + return + case <-ticker.C: + // Check for gz files in tmp-db folder + if hasGzFile(tmpDbPath) { + // Found gz file, cancel context and terminate test successfully + cancel() + t.Log("Found gz file in tmp-db, terminating test early") + return + } + case <-timeout.C: + // Timeout reached + cancel() + t.Fatal("Test timed out waiting for gz file or completion") + case <-ctx.Done(): + // Context cancelled + return + } + } +} + +// hasGzFile checks if there are any .gz files in the specified directory +func hasGzFile(dir string) bool { + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".gz") { + return true + } + } + return false +} + +func TestUpdate_getTargetDbBlockRange(t *testing.T) { + tests := []struct { + name string + wantFirst uint64 + wantLast uint64 + wantErr string + setup func(t *testing.T) *utils.Config + }{ + { + name: "db does not exist", + setup: func(t *testing.T) *utils.Config { + return &utils.Config{AidaDb: t.TempDir() + "/nonexistent-db"} + }, + wantFirst: 0, wantLast: 0, wantErr: "", + }, + { + name: "db exists but no substates", + setup: func(t *testing.T) *utils.Config { + aidaDbPath := t.TempDir() + "/aida-db" + aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath) + require.NoError(t, err) + require.NoError(t, aidaDb.Close()) + return &utils.Config{AidaDb: aidaDbPath, SubstateEncoding: "pb"} + }, + wantFirst: 0, wantLast: 0, wantErr: "cannot find blocks in substate", + }, + { + name: "db exists with substates", + setup: func(t *testing.T) *utils.Config { + aidaDbPath := t.TempDir() + "/aida-db" + aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath) + require.NoError(t, err) + ss := utils.GetTestSubstate("pb") + ss.Block = 100 + ss.Env.Number = 100 + require.NoError(t, aidaDb.PutSubstate(ss)) + require.NoError(t, aidaDb.Close()) + return &utils.Config{AidaDb: aidaDbPath, SubstateEncoding: "pb"} + }, + wantFirst: 100, wantLast: 100, wantErr: "", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := test.setup(t) + first, last, err := getTargetDbBlockRange(cfg) + assert.Equal(t, test.wantFirst, first) + assert.Equal(t, test.wantLast, last) + if test.wantErr != "" { + assert.ErrorContains(t, err, test.wantErr) + } else { + assert.NoError(t, err) + } + + }) + } +} + +func TestUpdate_CalculateMD5Sum(t *testing.T) { + name := t.TempDir() + "/testfile" + f, err := os.Create(name) + require.NoError(t, err) + _, err = f.Write([]byte("test")) + require.NoError(t, err) + require.NoError(t, f.Close()) + md5Sum, err := calculateMD5Sum(name) + require.NoError(t, err) + require.Equal(t, md5Sum, "098f6bcd4621d373cade4e832627b4f6") +} + +func TestUpdate_pushToChanel(t *testing.T) { + patches := []utils.PatchJson{ + {FileName: "patch1.tar.gz", ToBlock: 10}, + {FileName: "patch2.tar.gz", ToBlock: 20}, + } + + ch := pushPatchToChanel(patches) + + var received []utils.PatchJson + for patch := range ch { + received = append(received, patch) + } + + assert.Equal(t, patches, received) +} + +func TestUpdate_retrievePatchesToDownload(t *testing.T) { + utils.AidaDbRepositoryUrl = utils.AidaDbRepositorySonicUrl + defer func() { + utils.AidaDbRepositoryUrl = "" + }() + patches, err := retrievePatchesToDownload(&utils.Config{ + ChainID: utils.SonicMainnetChainID, + UpdateType: "nightly", + }, 28_000_000) + require.NoError(t, err) + require.NotEmpty(t, patches) +} + +func TestUpdate_update(t *testing.T) { + aidaDbPath := t.TempDir() + "/aida-db" + utils.AidaDbRepositoryUrl = utils.AidaDbRepositoryTestUrl + defer func() { + utils.AidaDbRepositoryUrl = "" + }() + err := update(&utils.Config{ + AidaDb: aidaDbPath, + UpdateType: "nightly", + DbTmp: t.TempDir(), + }) + require.NoError(t, err) + aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath) + require.NoError(t, err) + ss := aidaDb.GetFirstSubstate() + assert.Equal(t, uint64(1), ss.Block) + ss, err = aidaDb.GetLastSubstate() + require.NoError(t, err) + assert.Equal(t, uint64(210080), ss.Block) +} +func TestUpdate_update_downloadFails(t *testing.T) { + aidaDbPath := t.TempDir() + "/aida-db" + utils.AidaDbRepositoryUrl = "https://unknownrepository.com" + defer func() { + utils.AidaDbRepositoryUrl = "" + }() + err := update(&utils.Config{ + AidaDb: aidaDbPath, + UpdateType: "nightly", + DbTmp: t.TempDir(), + }) + require.ErrorContains(t, err, "unable to download patches.json") +} + +func TestUpdate_mergeToExistingAidaDb_ClassicPatch(t *testing.T) { + // Create patch with a substate + want, patchPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema) + // Open target db and fill it + targetDb, err := db.NewDefaultSubstateDB(t.TempDir() + "/target-db") + require.NoError(t, err) + txType := int32(substate.SetCodeTxType) + err = targetDb.PutSubstate(&substate.Substate{ + InputSubstate: make(substate.WorldState), + OutputSubstate: make(substate.WorldState), + Env: &substate.Env{ + Difficulty: new(big.Int).SetUint64(1), + BaseFee: new(big.Int).SetUint64(1), + }, + Message: substate.NewMessage( + 1, + true, + new(big.Int).SetUint64(1), + 1, + types.Address{1}, + new(types.Address), new(big.Int).SetUint64(1), []byte{1}, nil, &txType, + types.AccessList{{Address: types.Address{1}, StorageKeys: []types.Hash{{1}, {2}}}}, new(big.Int).SetUint64(1), + new(big.Int).SetUint64(1), new(big.Int).SetUint64(1), make([]types.Hash, 0), + []types.SetCodeAuthorization{ + {ChainID: *uint256.NewInt(1), Address: types.Address{1}, Nonce: 1, V: 1, R: *uint256.NewInt(1), S: *uint256.NewInt(1)}, + }), + Result: new(substate.Result), + Block: 0, + Transaction: 0, + }) + require.NoError(t, err) + // Set correct metadata block range + targetMD := utils.NewAidaDbMetadata(targetDb, "CRITICAL") + err = targetMD.SetBlockRange(0, want.Block-1) + require.NoError(t, err) + err = targetMD.SetChainID(utils.SonicMainnetChainID) + require.NoError(t, err) + + cfg := &utils.Config{ + ChainID: utils.SonicMainnetChainID, + LogLevel: "CRITICAL", + } + err = mergeToExistingAidaDb(cfg, targetMD, patchPath) + require.NoError(t, err) + // Check that merge has happened + got, err := targetDb.GetSubstate(want.Block, want.Transaction) + require.NoError(t, err) + assert.NoError(t, want.Equal(got)) +} + +func TestUpdate_mergeToExistingAidaDb_StateHashPatch(t *testing.T) { + ss, targetDbPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema) + // Create patch with a state root + patchPath := t.TempDir() + stateHashPatchFileName + patchDb, err := db.NewDefaultBaseDB(patchPath) + require.NoError(t, err) + wantHash := common.Hash{0x12} + err = utils.SaveStateRoot(patchDb, hexutil.EncodeUint64(ss.Block), wantHash.String()) + require.NoError(t, err) + err = patchDb.Close() + require.NoError(t, err) + + // Create target db + targetDb, err := db.NewDefaultBaseDB(targetDbPath) + require.NoError(t, err) + + // Set correct metadata block range + targetMD := utils.NewAidaDbMetadata(targetDb, "CRITICAL") + require.NoError(t, err) + err = targetMD.SetChainID(utils.SonicMainnetChainID) + require.NoError(t, err) + + cfg := &utils.Config{ + ChainID: utils.SonicMainnetChainID, + LogLevel: "CRITICAL", + } + err = mergeToExistingAidaDb(cfg, targetMD, patchPath) + require.NoError(t, err) + hp := utils.MakeHashProvider(targetDb) + + gotHash, err := hp.GetStateRootHash(int(ss.Block)) + require.NoError(t, err) + require.Zero(t, wantHash.Cmp(gotHash)) +} + +func TestUpdate_mergeToExistingAidaDb_BlocksDoesNotAlign(t *testing.T) { + want, patchPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema) + _, targetPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema) + targetDb, err := db.NewDefaultBaseDB(targetPath) + require.NoError(t, err) + targetMD := utils.NewAidaDbMetadata(targetDb, "CRITICAL") + // set wrong block range to target db + err = targetMD.SetBlockRange(0, want.Block-1000) + require.NoError(t, err) + err = targetMD.SetChainID(utils.SonicMainnetChainID) + require.NoError(t, err) + + cfg := &utils.Config{ + ChainID: utils.SonicMainnetChainID, + LogLevel: "CRITICAL", + } + err = mergeToExistingAidaDb(cfg, targetMD, patchPath) + require.ErrorContains(t, err, "metadata blocks does not align") +} + +func TestUpdate_retrievePatchesToDownload_MustChooseUpdateType(t *testing.T) { + _, err := retrievePatchesToDownload(&utils.Config{ + UpdateType: "", // empty + }, 0) + require.ErrorContains(t, err, "please choose correct data-type") +} + +func TestByToBlock_CanBeUsedToSortByToBlock(t *testing.T) { + patches := []utils.PatchJson{ + {FileName: "patch1.tar.gz", ToBlock: 10}, + {FileName: "patch2.tar.gz", ToBlock: 20}, + {FileName: "patch3.tar.gz", ToBlock: 15}, + } + + expected := []utils.PatchJson{ + {FileName: "patch1.tar.gz", ToBlock: 10}, + {FileName: "patch3.tar.gz", ToBlock: 15}, + {FileName: "patch2.tar.gz", ToBlock: 20}, + } + + sort.Sort(ByToBlock(patches)) + assert.Equal(t, expected, patches) +} diff --git a/cmd/util-db/validate/validate.go b/cmd/util-db/validate/validate.go index 9098d40f3..8e8ccf0ac 100644 --- a/cmd/util-db/validate/validate.go +++ b/cmd/util-db/validate/validate.go @@ -47,19 +47,19 @@ func validateAction(ctx *cli.Context) error { return fmt.Errorf("cannot parse config; %v", err) } - aidaDb, err := db.NewReadOnlyBaseDB(cfg.AidaDb) + aidaDb, err := db.NewReadOnlySubstateDB(cfg.AidaDb) if err != nil { return fmt.Errorf("cannot open db; %v", err) } defer utildb.MustCloseDB(aidaDb) - md := utils.NewAidaDbMetadata(aidaDb, "INFO") - md.ChainId = md.GetChainID() - if md.ChainId == 0 { + if md.GetChainID() == 0 { log.Warning("cannot find db-hash in your aida-db metadata, this operation is needed because db-hash was not found inside your aida-db; please make sure you specified correct chain-id with flag --%v", utils.ChainIDFlag.Name) - md.ChainId = cfg.ChainID + if err = md.SetChainID(cfg.ChainID); err != nil { + return fmt.Errorf("cannot set chain-id inside aida-db metadata; %v", err) + } } // validation only makes sense if user has pure AidaDb @@ -74,7 +74,7 @@ func validateAction(ctx *cli.Context) error { if len(expectedHash) == 0 { // we want to save the hash inside metadata saveHash = true - expectedHash, err = utildb.FindDbHashOnline(md.ChainId, log, md) + expectedHash, err = utildb.FindDbHashOnline(md.GetChainID(), log, md) if err != nil { return fmt.Errorf("validation cannot be performed - could not find expected db hash; %v", err) } diff --git a/utildb/merger.go b/utildb/merger.go index 0a1e3a367..a6cf2574e 100644 --- a/utildb/merger.go +++ b/utildb/merger.go @@ -30,15 +30,15 @@ import ( type Merger struct { cfg *utils.Config log logger.Logger - targetDb db.BaseDB - sourceDbs []db.BaseDB + targetDb db.SubstateDB + sourceDbs []db.SubstateDB sourceDbPaths []string md *utils.AidaDbMetadata start time.Time } // NewMerger returns new instance of Merger -func NewMerger(cfg *utils.Config, targetDb db.BaseDB, sourceDbs []db.BaseDB, sourceDbPaths []string, md *utils.AidaDbMetadata) *Merger { +func NewMerger(cfg *utils.Config, targetDb db.SubstateDB, sourceDbs []db.SubstateDB, sourceDbPaths []string, md *utils.AidaDbMetadata) *Merger { return &Merger{ cfg: cfg, log: logger.NewLogger(cfg.LogLevel, "aida-db-Merger"), @@ -55,13 +55,9 @@ func (m *Merger) FinishMerge() error { if !m.cfg.SkipMetadata { // merge type db does not have epoch calculations yet m.md.Db = m.targetDb - err := m.md.SetAll() - if err != nil { - return err - } MustCloseDB(m.targetDb) - err = PrintMetadata(m.cfg.AidaDb) + err := PrintMetadata(m.targetDb) if err != nil { return err } diff --git a/utildb/utils.go b/utildb/utils.go index d5b894494..536818ac0 100644 --- a/utildb/utils.go +++ b/utildb/utils.go @@ -17,11 +17,9 @@ package utildb import ( - "crypto/md5" "encoding/hex" "errors" "fmt" - "io" "math/big" "os" "strconv" @@ -34,26 +32,25 @@ import ( "github.com/0xsoniclabs/substate/substate" "github.com/0xsoniclabs/substate/types" "github.com/0xsoniclabs/substate/updateset" - "github.com/Fantom-foundation/lachesis-base/common/bigendian" "github.com/holiman/uint256" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) // OpenSourceDatabases opens all databases required for merge -func OpenSourceDatabases(sourceDbPaths []string) ([]db.BaseDB, error) { +func OpenSourceDatabases(sourceDbPaths []string) ([]db.SubstateDB, error) { if len(sourceDbPaths) < 1 { return nil, fmt.Errorf("no source database were specified") } - var sourceDbs []db.BaseDB + var sourceDbs []db.SubstateDB for i := 0; i < len(sourceDbPaths); i++ { path := sourceDbPaths[i] _, err := os.Stat(path) if os.IsNotExist(err) { return nil, fmt.Errorf("source database %s; doesn't exist", path) } - db, err := db.NewReadOnlyBaseDB(path) + db, err := db.NewReadOnlySubstateDB(path) if err != nil { return nil, fmt.Errorf("source database %s; error: %v", path, err) } @@ -74,33 +71,6 @@ func MustCloseDB(db db.BaseDB) { } } -// calculateMD5Sum calculates MD5 hash of given file -func calculateMD5Sum(filePath string) (string, error) { - // Open the file - file, err := os.Open(filePath) - if err != nil { - return "", fmt.Errorf("unable open file %s; %v", filePath, err.Error()) - } - defer file.Close() - - // Create a new MD5 hash instance - hash := md5.New() - - // Copy the file content into the hash instance - _, err = io.Copy(hash, file) - if err != nil { - return "", fmt.Errorf("unable to calculate md5; %v", err) - } - - // Calculate the MD5 checksum as a byte slice - checksum := hash.Sum(nil) - - // Convert the checksum to a hexadecimal string - md5sum := hex.EncodeToString(checksum) - - return md5sum, nil -} - // GetDbSize retrieves database size func GetDbSize(db db.BaseDB) uint64 { var count uint64 @@ -113,19 +83,13 @@ func GetDbSize(db db.BaseDB) uint64 { } // PrintMetadata from given AidaDb -func PrintMetadata(pathToDb string) error { +func PrintMetadata(aidaDb db.SubstateDB) error { log := logger.NewLogger("INFO", "Print-Metadata") - base, err := db.NewReadOnlyBaseDB(pathToDb) - if err != nil { - return err - } - defer MustCloseDB(base) - - md := utils.NewAidaDbMetadata(base, "INFO") + md := utils.NewAidaDbMetadata(aidaDb, "INFO") log.Notice("AIDA-DB INFO:") - if err = printDbType(md); err != nil { + if err := printDbType(md); err != nil { return err } @@ -172,30 +136,20 @@ func PrintMetadata(pathToDb string) error { } // printUpdateSetInfo from given AidaDb -func printUpdateSetInfo(m *utils.AidaDbMetadata) { +func printUpdateSetInfo(m utils.Metadata) { log := logger.NewLogger("INFO", "Print-Metadata") log.Notice("UPDATE-SET INFO:") - intervalBytes, err := m.Db.Get([]byte(db.UpdatesetIntervalKey)) - if err != nil { - log.Warning("Value for update-set interval does not exist in given Dbs metadata") - } else { - log.Infof("Interval: %v blocks", bigendian.BytesToUint64(intervalBytes)) - } + interval := m.GetUpdatesetInterval() + log.Infof("Interval: %v blocks", interval) - sizeBytes, err := m.Db.Get([]byte(db.UpdatesetSizeKey)) - if err != nil { - log.Warning("Value for update-set size does not exist in given Dbs metadata") - } else { - u := bigendian.BytesToUint64(sizeBytes) - - log.Infof("Size: %.1f MB", float64(u)/float64(1_000_000)) - } + size := m.GetUpdatesetSize() + log.Infof("Size: %.1f MB", float64(size)/float64(1_000_000)) } // printDbType from given AidaDb -func printDbType(m *utils.AidaDbMetadata) error { +func printDbType(m utils.Metadata) error { t := m.GetDbType() var typePrint string @@ -220,12 +174,19 @@ func printDbType(m *utils.AidaDbMetadata) error { func GenerateTestAidaDb(t *testing.T) db.BaseDB { tmpDir := t.TempDir() + "/testAidaDb" - database, err := db.NewDefaultBaseDB(tmpDir) + database, err := db.NewDefaultSubstateDB(tmpDir) if err != nil { t.Fatalf("error opening stateHash leveldb %s: %v", tmpDir, err) } md := utils.NewAidaDbMetadata(database, "ERROR") - err = md.SetAllMetadata(1, 50, 1, 50, 250, []byte("0x0"), 1) + err = errors.Join( + md.SetFirstBlock(1), + md.SetLastBlock(50), + md.SetFirstEpoch(1), + md.SetLastEpoch(50), + md.SetChainID(utils.MainnetChainID), + md.SetDbHash([]byte("0x0")), + ) assert.NoError(t, err) // write substates to the database diff --git a/utildb/utils_test.go b/utildb/utils_test.go index 566c8fd08..d79fd5ef8 100644 --- a/utildb/utils_test.go +++ b/utildb/utils_test.go @@ -17,7 +17,6 @@ package utildb import ( - "os" "testing" "github.com/0xsoniclabs/aida/utils" @@ -74,15 +73,3 @@ func TestUtils_OpenSourceDatabases_Error(t *testing.T) { } } - -func TestUtils_CalculateMD5Sum(t *testing.T) { - name := t.TempDir() + "/testfile" - f, err := os.Create(name) - require.NoError(t, err) - _, err = f.Write([]byte("test")) - require.NoError(t, err) - require.NoError(t, f.Close()) - md5Sum, err := calculateMD5Sum(name) - require.NoError(t, err) - require.Equal(t, md5Sum, "098f6bcd4621d373cade4e832627b4f6") -} diff --git a/utildb/validator.go b/utildb/validator.go index 0530d5428..e983c55d5 100644 --- a/utildb/validator.go +++ b/utildb/validator.go @@ -20,6 +20,7 @@ import ( "crypto/md5" "encoding/hex" "errors" + "fmt" "sync" "time" @@ -45,7 +46,7 @@ type validator struct { } // FindDbHashOnline if user has no dbHash inside his AidaDb metadata -func FindDbHashOnline(chainId utils.ChainID, log logger.Logger, md *utils.AidaDbMetadata) ([]byte, error) { +func FindDbHashOnline(chainId utils.ChainID, log logger.Logger, md utils.Metadata) ([]byte, error) { var url string if chainId == utils.SonicMainnetChainID { @@ -62,26 +63,12 @@ func FindDbHashOnline(chainId utils.ChainID, log logger.Logger, md *utils.AidaDb return nil, err } - md.LastBlock = md.GetLastBlock() - - if md.LastBlock == 0 { - log.Warning("your aida-db seems to have empty metadata; looking for block range in substate") - } - - var ok bool - - md.FirstBlock, md.LastBlock, ok = utils.FindBlockRangeInSubstate(db.MakeDefaultSubstateDBFromBaseDB(md.Db)) - if !ok { - return nil, errors.New("cannot find block range in substate") - } - - err = md.SetBlockRange(md.FirstBlock, md.LastBlock) + err = md.GenerateMetadata(chainId) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot generate metadata; %v", err) } - for _, patch := range patches { - if patch.ToBlock == md.LastBlock { + if patch.ToBlock == md.GetLastBlock() { return hex.DecodeString(patch.DbHash) } } diff --git a/utils/config.go b/utils/config.go index dff8f10c4..dc745d1a9 100644 --- a/utils/config.go +++ b/utils/config.go @@ -121,6 +121,8 @@ const ( AidaDbRepositoryHoleskyUrl = "https://storage.googleapis.com/aida-repository-public/holesky/aida-patches" AidaDbRepositoryHoodiUrl = "https://storage.googleapis.com/aida-repository-public/hoodi/aida-patches" AidaDbRepositorySepoliaUrl = "https://storage.googleapis.com/aida-repository-public/sepolia/aida-patches" + + AidaDbRepositoryTestUrl = "https://storage.googleapis.com/aida-repository-public/sonic-dummy/aida-patches" ) const maxLastBlock = math.MaxUint64 - 1 // we decrease the value by one because params are always +1 @@ -711,7 +713,7 @@ func (cc *configContext) getMdBlockRange() (uint64, uint64, uint64, error) { } // read meta data - aidaDb, err := db.NewReadOnlyBaseDB(cc.cfg.AidaDb) + aidaDb, err := db.NewDefaultSubstateDB(cc.cfg.AidaDb) if err != nil { cc.log.Warningf("Cannot open AidaDB; %v", err) return defaultFirst, defaultLast, defaultLastPatch, nil @@ -723,18 +725,13 @@ func (cc *configContext) getMdBlockRange() (uint64, uint64, uint64, error) { }() md := NewAidaDbMetadata(aidaDb, cc.cfg.LogLevel) - err = md.getBlockRange() - if err != nil { - cc.log.Warning(err) - return defaultFirst, defaultLast, defaultLastPatch, nil - } cc.hasMetadata = true - lastPatchBlock, err := getPatchFirstBlock(md.LastBlock) + lastPatchBlock, err := getPatchFirstBlock(md.GetLastBlock()) if err != nil { cc.log.Warningf("Cannot get first block of the last patch of given AidaDB; %v", err) } - return md.FirstBlock, md.LastBlock, lastPatchBlock, nil + return md.GetFirstBlock(), md.GetLastBlock(), lastPatchBlock, nil } // adjustBlockRange finds overlap between metadata block range and block range specified by user in command line @@ -774,7 +771,7 @@ func (cc *configContext) setChainId() error { cc.log.Warningf("ChainID (--%v) was not set; looking for it in AidaDb", ChainIDFlag.Name) // we check if AidaDb was set with err == nil - if aidaDb, err := db.OpenBaseDB(cc.cfg.AidaDb); err == nil { + if aidaDb, err := db.NewDefaultSubstateDB(cc.cfg.AidaDb); err == nil { md := NewAidaDbMetadata(aidaDb, cc.cfg.LogLevel) cc.cfg.ChainID = md.GetChainID() diff --git a/utils/hash_provider.go b/utils/hash_provider.go index 34a5c7f8a..550006654 100644 --- a/utils/hash_provider.go +++ b/utils/hash_provider.go @@ -23,9 +23,11 @@ import ( "context" "encoding/binary" "fmt" + "os" "strconv" "strings" + "github.com/0xsoniclabs/aida/logger" "github.com/0xsoniclabs/substate/db" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rpc" @@ -102,6 +104,104 @@ func (p *hashProvider) GetStateRootHash(number int) (common.Hash, error) { return common.BytesToHash(stateRoot), nil } +// StateAndBlockHashScraper scrapes state and block hashes from a node and saves them to a leveldb database +func StateAndBlockHashScraper(ctx context.Context, chainId ChainID, clientDb string, db db.BaseDB, firstBlock, lastBlock uint64, log logger.Logger) error { + client, err := getClient(ctx, chainId, clientDb, log) + if err != nil { + return err + } + defer client.Close() + + var i = firstBlock + + // If firstBlock is 0, we need to get the state root for block 1 and save it as the state root for block 0 + // this is because the correct state root for block 0 is not available from the rpc node (at least in fantom mainnet and testnet) + if firstBlock == 0 { + block, err := getBlockByNumber(client, "0x1") + if err != nil { + return err + } + + if block == nil { + return fmt.Errorf("block 1 not found") + } + + err = SaveStateRoot(db, "0x0", block["stateRoot"].(string)) + if err != nil { + return err + } + err = SaveBlockHash(db, "0x1", block["hash"].(string)) + if err != nil { + return err + } + i++ + } + + for ; i <= lastBlock; i++ { + blockNumber := fmt.Sprintf("0x%x", i) + block, err := getBlockByNumber(client, blockNumber) + if err != nil { + return err + } + + if block == nil { + return fmt.Errorf("block %d not found", i) + } + + err = SaveStateRoot(db, blockNumber, block["stateRoot"].(string)) + if err != nil { + return err + } + err = SaveBlockHash(db, blockNumber, block["hash"].(string)) + if err != nil { + return err + } + + if i%10000 == 0 { + log.Infof("Scraping block %d done!\n", i) + } + } + + return nil +} + +// getClient returns a rpc/ipc client +func getClient(ctx context.Context, chainId ChainID, clientDb string, log logger.Logger) (*rpc.Client, error) { + var client *rpc.Client + var err error + + // try both sonic and geth ipcs + ipcPaths := []string{ + clientDb + "/sonic.ipc", + clientDb + "/geth.ipc", + } + for _, ipcPath := range ipcPaths { + _, errIpc := os.Stat(ipcPath) + if errIpc == nil { + // ipc file exists + client, err = rpc.DialIPC(ctx, ipcPath) + if err != nil { + return nil, fmt.Errorf("failed to connect to IPC at %s: %v", ipcPath, err) + } + log.Infof("Connected to IPC at %s", ipcPath) + return client, err + } + } + + // if ipc file does not exist, try to connect to RPC + var provider string + provider, err = GetProvider(chainId) + if err != nil { + return nil, err + } + client, err = rpc.Dial(provider) + if err != nil { + return nil, fmt.Errorf("failed to connect to the RPC client at %s: %v", provider, err) + } + log.Infof("Connected to RPC at %s", provider) + return client, nil +} + // SaveStateRoot saves the state root hash to the database func SaveStateRoot(db db.BaseDB, blockNumber string, stateRoot string) error { fullPrefix := StateRootHashPrefix + blockNumber diff --git a/utils/hash_provider_test.go b/utils/hash_provider_test.go index d858f90f3..01ba09025 100644 --- a/utils/hash_provider_test.go +++ b/utils/hash_provider_test.go @@ -17,20 +17,136 @@ package utils import ( + "context" "errors" "fmt" "math/rand" + "os" "strconv" "testing" + "github.com/0xsoniclabs/aida/logger" "github.com/0xsoniclabs/substate/db" "github.com/Fantom-foundation/lachesis-base/common/bigendian" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rpc" "github.com/stretchr/testify/assert" "github.com/syndtr/goleveldb/leveldb" "go.uber.org/mock/gomock" ) +// TestStateHash_ZeroHasSameStateHashAsOne tests that the state hash of block 0 is the same as the state hash of block 1 +func TestStateHash_ZeroHasSameStateHashAsOne(t *testing.T) { + tmpDir := t.TempDir() + "/blockHashes" + database, err := db.NewDefaultBaseDB(tmpDir) + if err != nil { + t.Fatalf("error opening stateHash leveldb %s: %v", tmpDir, err) + } + log := logger.NewLogger("info", "Test state hash") + + err = StateAndBlockHashScraper(context.TODO(), TestnetChainID, "", database, 0, 1, log) + if err != nil { + t.Fatalf("error scraping state hashes: %v", err) + } + err = database.Close() + if err != nil { + t.Fatalf("error closing stateHash leveldb %s: %v", tmpDir, err) + } + + database, err = db.NewReadOnlyBaseDB(tmpDir) + if err != nil { + t.Fatalf("error opening stateHash leveldb %s: %v", tmpDir, err) + } + defer func(database db.BaseDB) { + e := database.Close() + if e != nil { + t.Fatalf("error closing stateHash leveldb %s: %v", tmpDir, e) + } + }(database) + + shp := MakeHashProvider(database) + + hashZero, err := shp.GetStateRootHash(0) + if err != nil { + t.Fatalf("error getting state hash for block 0: %v", err) + } + + hashOne, err := shp.GetStateRootHash(1) + if err != nil { + t.Fatalf("error getting state hash for block 1: %v", err) + } + + if hashZero != hashOne { + t.Fatalf("state hash of block 0 (%s) is not the same as the state hash of block 1 (%s)", hashZero.Hex(), hashOne.Hex()) + } +} + +func TestStateHash_Log(t *testing.T) { + tmpDir := t.TempDir() + "/blockHashes" + database, err := db.NewDefaultBaseDB(tmpDir) + if err != nil { + t.Fatalf("error opening stateHash leveldb %s: %v", tmpDir, err) + } + + ctrl := gomock.NewController(t) + log := logger.NewMockLogger(ctrl) + log.EXPECT().Infof("Connected to RPC at %s", RPCTestnet) + log.EXPECT().Infof("Scraping block %d done!\n", uint64(10000)) + + err = StateAndBlockHashScraper(context.TODO(), TestnetChainID, "", database, 9990, 10100, log) + if err != nil { + t.Fatalf("error scraping state hashes: %v", err) + } +} + +// TestStateHash_ZeroHasSameStateHashAsOne tests that the state hash of block 0 is different to the state hash of block 100 +// we are expecting that at least some storage has changed between block and block 100 +func TestStateHash_ZeroHasDifferentStateHashAfterHundredBlocks(t *testing.T) { + tmpDir := t.TempDir() + "/blockHashes" + database, err := db.NewDefaultBaseDB(tmpDir) + if err != nil { + t.Fatalf("error opening stateHash leveldb %s: %v", tmpDir, err) + } + log := logger.NewLogger("info", "Test state hash") + + err = StateAndBlockHashScraper(context.TODO(), TestnetChainID, "", database, 0, 100, log) + if err != nil { + t.Fatalf("error scraping state hashes: %v", err) + } + err = database.Close() + if err != nil { + t.Fatalf("error closing stateHash leveldb %s: %v", tmpDir, err) + } + + database, err = db.NewReadOnlyBaseDB(tmpDir) + if err != nil { + t.Fatalf("error opening stateHash leveldb %s: %v", tmpDir, err) + } + defer func(database db.BaseDB) { + e := database.Close() + if e != nil { + t.Fatalf("error closing stateHash leveldb %s: %v", tmpDir, e) + } + }(database) + + shp := MakeHashProvider(database) + + hashZero, err := shp.GetStateRootHash(0) + if err != nil { + t.Fatalf("error getting state hash for block 0: %v", err) + } + + hashHundred, err := shp.GetStateRootHash(100) + if err != nil { + t.Fatalf("error getting state hash for block 100: %v", err) + } + + // block 0 should have a different state hash than block 100 + if hashZero == hashHundred { + t.Fatalf("state hash of block 0 (%s) is the same as the state hash of block 100 (%s)", hashZero.Hex(), hashHundred.Hex()) + } +} + func TestStateHash_KeyToUint64(t *testing.T) { type args struct { hexBytes []byte @@ -58,6 +174,54 @@ func TestStateHash_KeyToUint64(t *testing.T) { } } +func Test_getClient(t *testing.T) { + type args struct { + ctx context.Context + chainId ChainID + ipcPath string + } + log := logger.NewLogger("info", "Test state hash") + tests := []struct { + name string + args args + want *rpc.Client + wantErr bool + }{ + {"testGetClientRpcSonicMainnet", args{context.Background(), SonicMainnetChainID, ""}, &rpc.Client{}, false}, + {"testGetClientRpcOperaMainnet", args{context.Background(), MainnetChainID, ""}, &rpc.Client{}, false}, + {"testGetClientRpcTestnet", args{context.Background(), TestnetChainID, ""}, &rpc.Client{}, false}, + {"testGetClientIpcNonExistant", args{context.Background(), TestnetChainID, "/non-existant-path"}, nil, false}, + {"testGetClientRpcUnknownChainId", args{context.Background(), 88888, ""}, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getClient(tt.args.ctx, tt.args.chainId, tt.args.ipcPath, log) + if (err != nil) != tt.wantErr { + t.Errorf("getClient() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.want != nil && got == nil { + t.Errorf("getClient() got nil, want non-nil") + } + }) + } +} + +func TestStateHash_GetClientIpcFail(t *testing.T) { + tmpIpcPath := t.TempDir() + // create this file + if err := os.WriteFile(tmpIpcPath+"/geth.ipc", []byte("test"), 0644); err != nil { + t.Fatalf("error creating ipc file %s: %v", tmpIpcPath+"/geth.ipc", err) + } + + log := logger.NewLogger("info", "Test state hash") + _, err := getClient(context.Background(), TestnetChainID, tmpIpcPath, log) + if err == nil { + t.Fatalf("expected error when trying to connect to ipc file %s, but got nil", tmpIpcPath) + } + assert.Equal(t, fmt.Sprintf("failed to connect to IPC at %v/geth.ipc: dial unix %v/geth.ipc: connect: connection refused", tmpIpcPath, tmpIpcPath), err.Error()) +} + func TestStateHash_GetStateRootHash(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/utils/metadata.go b/utils/metadata.go index b9a74f3b7..209bdad95 100644 --- a/utils/metadata.go +++ b/utils/metadata.go @@ -43,6 +43,21 @@ const ( CustomType ) +func (t AidaDbType) String() string { + switch t { + case GenType: + return "Generate" + case CloneType: + return "Clone" + case PatchType: + return "Patch" + case NoType: + return "NoType" + default: + return "unknown db type" + } +} + const ( FirstBlockPrefix = db.MetadataPrefix + "fb" LastBlockPrefix = db.MetadataPrefix + "lb" @@ -71,355 +86,190 @@ type PatchJson struct { Nightly bool } -// AidaDbMetadata holds any information about AidaDb needed for putting it into the Db -type AidaDbMetadata struct { - Db db.BaseDB - log logger.Logger - FirstBlock, LastBlock uint64 - FirstEpoch, LastEpoch uint64 - ChainId ChainID - DbType AidaDbType - timestamp uint64 +type Metadata interface { + // GenerateMetadata generates new or updates metadata in AidaDb. + GenerateMetadata(chainID ChainID) error + // Merge merges source metadata into target metadata if its possible. + Merge(Metadata) error + // Delete deletes metadata from AidaDb. + Delete() error + + // Getters + GetFirstBlock() uint64 + GetLastBlock() uint64 + GetFirstEpoch() uint64 + GetLastEpoch() uint64 + GetChainID() ChainID + GetTimestamp() uint64 + GetDbType() AidaDbType + GetDbHash() []byte + GetUpdatesetInterval() uint64 + GetUpdatesetSize() uint64 + + // Setters + SetFirstBlock(uint64) error + SetLastBlock(uint64) error + SetFirstEpoch(uint64) error + SetLastEpoch(uint64) error + SetChainID(ChainID) error + SetTimestamp() error + SetDbType(AidaDbType) error + SetDbHash([]byte) error + SetHasHashPatch() error + SetUpdatesetInterval(uint64) error + SetUpdatesetSize(uint64) error + GetDb() db.SubstateDB } -// todo we need to check block alignment and chainID match before any merging - -// NewAidaDbMetadata creates new instance of AidaDbMetadata -func NewAidaDbMetadata(db db.BaseDB, logLevel string) *AidaDbMetadata { - return &AidaDbMetadata{ - Db: db, - log: logger.NewLogger(logLevel, "aida-metadata"), - } +// AidaDbMetadata holds any information about AidaDb needed for putting it into the Db +type AidaDbMetadata struct { + Db db.SubstateDB + log logger.Logger + FirstBlock, LastBlock *uint64 + FirstEpoch, LastEpoch *uint64 + updateSetSize, updateSetInterval *uint64 + ChainId ChainID + DbType AidaDbType + timestamp *uint64 + dbHash []byte } -// ProcessPatchLikeMetadata decides whether patch is new or not. If so the DbType is Set to GenType, otherwise its PatchType. -// Then it inserts all given metadata -func ProcessPatchLikeMetadata(aidaDb db.BaseDB, logLevel string, firstBlock, lastBlock, firstEpoch, lastEpoch uint64, chainID ChainID, isNew bool, dbHash []byte) error { - var ( - dbType AidaDbType - err error - ) - - // if this is brand-new patch, it should be treated as a gen type db - if isNew { - dbType = GenType - } else { - dbType = PatchType - } - - md := NewAidaDbMetadata(aidaDb, logLevel) - - if err = md.SetFirstBlock(firstBlock); err != nil { - return err - } - if err = md.SetLastBlock(lastBlock); err != nil { - return err - } - - if err = md.SetFirstEpoch(firstEpoch); err != nil { - return err - } - if err = md.SetLastEpoch(lastEpoch); err != nil { - return err - } - - if err = md.SetChainID(chainID); err != nil { - return err - } - - if err = md.SetDbType(dbType); err != nil { - return err - } - - if err = md.SetTimestamp(); err != nil { - return err - } - - err = md.SetDbHash(dbHash) - if err != nil { - return err - } - - md.log.Notice("Metadata added successfully") - - return nil +func (md *AidaDbMetadata) GetDb() db.SubstateDB { + return md.Db } -// ProcessCloneLikeMetadata inserts every metadata from sourceDb, only epochs are excluded. -// We can't be certain if given epoch is whole -func ProcessCloneLikeMetadata(aidaDb db.BaseDB, typ AidaDbType, logLevel string, firstBlock, lastBlock uint64, chainID ChainID) error { - var err error - - md := NewAidaDbMetadata(aidaDb, logLevel) - - firstBlock, lastBlock = md.compareBlocks(firstBlock, lastBlock) - - if err = md.SetFirstBlock(firstBlock); err != nil { - return err - } - if err = md.SetLastBlock(lastBlock); err != nil { - return err - } - - if err = md.SetChainID(chainID); err != nil { - return err - } - - if err = md.findEpochs(); err != nil { - return err - } - - if err = md.SetFirstEpoch(md.FirstEpoch); err != nil { - return err - } - - if err = md.SetLastEpoch(md.LastEpoch); err != nil { - return err +func (md *AidaDbMetadata) GenerateMetadata(chainId ChainID) error { + if chainId == 0 { + chainId = md.GetChainID() + if chainId == 0 { + md.log.Warningf("ChainID was nor set neither found in metadata - metadata generation will be incomplete") + } } - if err = md.SetDbType(typ); err != nil { - return err - } + fss := md.Db.GetFirstSubstate() + // if there is no substate, we cannot find blocks and epochs + if fss == nil { + md.log.Warningf("Your AidaDb does not contain any substate, metadata will be incomplete") + } else { + lss, err := md.Db.GetLastSubstate() + if err != nil { + return fmt.Errorf("cannot get last substate; %v", err) + } + err = md.SetFirstBlock(fss.Block) + if err != nil { + return fmt.Errorf("cannot set first block; %v", err) + } + err = md.SetLastBlock(lss.Block) + if err != nil { + return fmt.Errorf("cannot set last block; %v", err) + } - if err = md.SetTimestamp(); err != nil { - return err + // Epoch numbers can only be found if chainID is known and AidaDB has substates + if chainId != 0 { + err = md.findEpochs() + if err != nil { + return err + } + } } - md.log.Notice("Metadata added successfully") return nil } -func ProcessGenLikeMetadata(aidaDb db.BaseDB, firstBlock uint64, lastBlock uint64, firstEpoch uint64, lastEpoch uint64, chainID ChainID, logLevel string, dbHash []byte) error { - md := NewAidaDbMetadata(aidaDb, logLevel) - return md.genMetadata(firstBlock, lastBlock, firstEpoch, lastEpoch, chainID, dbHash) -} - -// genMetadata inserts metadata into newly generated AidaDb. -// If generate is used onto an existing AidaDb it updates last block, last epoch and timestamp. -func (md *AidaDbMetadata) genMetadata(firstBlock uint64, lastBlock uint64, firstEpoch uint64, lastEpoch uint64, chainID ChainID, dbHash []byte) error { - var err error - - firstBlock, lastBlock = md.compareBlocks(firstBlock, lastBlock) - - if err = md.SetFirstBlock(firstBlock); err != nil { - return err - } - if err = md.SetLastBlock(lastBlock); err != nil { - return err - } - - firstEpoch, lastEpoch = md.compareEpochs(firstEpoch, lastEpoch) - - if err = md.SetFirstEpoch(firstEpoch); err != nil { - return err - } - if err = md.SetLastEpoch(lastEpoch); err != nil { - return err - } - - if err = md.SetChainID(chainID); err != nil { - return err - } - - if err = md.SetDbType(GenType); err != nil { - return err - } - - if err = md.SetTimestamp(); err != nil { - return err - } - - if err = md.SetDbHash(dbHash); err != nil { - return err +// NewAidaDbMetadata creates new instance of AidaDbMetadata +func NewAidaDbMetadata(db db.SubstateDB, logLevel string) Metadata { + return &AidaDbMetadata{ + Db: db, + log: logger.NewLogger(logLevel, "aida-metadata"), } - - return nil } -// ProcessMergeMetadata decides the type according to the types of merged Dbs and inserts every metadata -func ProcessMergeMetadata(cfg *Config, aidaDb db.BaseDB, sourceDbs []db.BaseDB, paths []string) (*AidaDbMetadata, error) { - var ( - err error - ok bool - ) - - targetMD := NewAidaDbMetadata(aidaDb, cfg.LogLevel) - - for i, database := range sourceDbs { - md := NewAidaDbMetadata(database, cfg.LogLevel) - md.GetMetadata() - - // todo do we need to check whether blocks align? - - // Get chainID of first source database - if targetMD.ChainId == 0 { - targetMD.ChainId = md.ChainId +func (md *AidaDbMetadata) Merge(src Metadata) error { + targetChainID := md.GetChainID() + srcChainID := src.GetChainID() + if targetChainID != 0 { + if targetChainID != srcChainID { + return fmt.Errorf("cannot merge dbs with different chainIDs; target db chainID %v, source db chainID %v", targetChainID, srcChainID) } - - // if chain ids doesn't match, we should not be merging - if md.ChainId != targetMD.ChainId { - md.log.Critical("ChainIDs in Dbs metadata does not match!") - } - - hasNoBlockRangeInMetadata := md.FirstBlock == 0 && md.LastBlock == 0 - - // if database had no metadata we will look for blocks in substate - if hasNoBlockRangeInMetadata { - // we need to close database before opening substate - if err = database.Close(); err != nil { - return nil, fmt.Errorf("cannot close database; %v", err) - } - - sdb := db.MakeDefaultSubstateDBFromBaseDB(database) - md.FirstBlock, md.LastBlock, ok = FindBlockRangeInSubstate(sdb) - if !ok { - md.log.Warningf("Cannot find blocks in substate; is substate present in given database? %v", paths[i]) - } else { - md.log.Noticef("Found block range inside substate of %v (%v-%v)", paths[i], md.FirstBlock, md.LastBlock) - } - } else { - ok = true - } - - // only check blocks when merged database has metadata or substate - if ok { - if md.FirstBlock < targetMD.FirstBlock || hasNoBlockRangeInMetadata { - targetMD.FirstBlock = md.FirstBlock - } - - if md.LastBlock > targetMD.LastBlock { - targetMD.LastBlock = md.LastBlock - } - } - - // set first - if targetMD.DbType == NoType { - targetMD.DbType = md.DbType - continue - } - - if targetMD.DbType == GenType && (md.DbType == PatchType || md.DbType == GenType) { - targetMD.DbType = GenType - continue - } - - if targetMD.DbType == PatchType { - switch md.DbType { - case GenType: - targetMD.DbType = GenType - continue - case PatchType: - targetMD.DbType = PatchType - continue - case CloneType: - targetMD.DbType = CloneType - // we cannot merge patch with smaller first block onto clone because... todo explain + error - if targetMD.FirstBlock < md.FirstBlock { - return nil, errors.New("cannot prepend patch on clone") - } - continue - default: - targetMD.DbType = GenType - } + } else { + if srcChainID == 0 { + return errors.New("cannot merge dbs with no chainIDs in metadata; you can set chainID manually using the util-db insert cmd") } - - if targetMD.DbType == CloneType && md.DbType == PatchType { - targetMD.DbType = CloneType - // we cannot merge patch with smaller first block onto clone because... todo explain + error - if md.FirstBlock < targetMD.FirstBlock { - return nil, errors.New("cannot prepend patch on clone") - } - continue + err := md.SetChainID(srcChainID) + if err != nil { + return fmt.Errorf("cannot set chainID while merging dbs; %v", err) } - - return nil, fmt.Errorf("cannot merge %v with %v", targetMD.getVerboseDbType(), md.getVerboseDbType()) } - // if source dbs had neither metadata nor substate, we try to find the block range inside substate of targetDb - if targetMD.FirstBlock == 0 && targetMD.LastBlock == 0 { - // we must close database before accessing substate - if err = targetMD.Db.Close(); err != nil { - return nil, fmt.Errorf("cannot close targetDb; %v", err) - } - sdb := db.MakeDefaultSubstateDBFromBaseDB(targetMD.Db) - targetMD.FirstBlock, targetMD.LastBlock, ok = FindBlockRangeInSubstate(sdb) - if !ok { - targetMD.log.Warningf("Cannot find block range in substate of AidaDb (%v); this will in corrupted metadata but will not affect data itself", cfg.AidaDb) - } else { - targetMD.log.Noticef("Found block range inside substate of AidaDb %v (%v-%v)", cfg.AidaDb, targetMD.FirstBlock, targetMD.LastBlock) + // Set DbType + targetDbType := md.GetDbType() + srcDbType := src.GetDbType() + switch targetDbType { + case NoType: + targetDbType = srcDbType + case GenType: + switch srcDbType { + // GetType and PatchType can be merged onto GenType + case GenType: + break + case PatchType: + break + default: + targetDbType = CustomType } + default: + targetDbType = CustomType } - - if targetMD.ChainId == 0 { - targetMD.log.Warningf("your dbs does not have chain-id, Setting value from config (%v)", cfg.ChainID) - targetMD.ChainId = cfg.ChainID + err := md.SetDbType(targetDbType) + if err != nil { + return fmt.Errorf("cannot merge db type: %v", err) } + // Find block range + targetFirstBlock := md.GetFirstBlock() + srcFirstBlock := src.GetFirstBlock() + targetLastBlock := md.GetLastBlock() + srcLastBlock := src.GetLastBlock() - if err = targetMD.findEpochs(); err != nil { - return nil, err + // Source is a subset of target + if targetFirstBlock < srcFirstBlock && targetLastBlock > srcLastBlock { + return fmt.Errorf("source db (%v-%v) is subset of target db (%v-%v)", srcFirstBlock, srcLastBlock, targetFirstBlock, targetLastBlock) } - - return targetMD, nil -} - -// GetMetadata from given Db and save it -func (md *AidaDbMetadata) GetMetadata() { - md.FirstBlock = md.GetFirstBlock() - - md.LastBlock = md.GetLastBlock() - - md.FirstEpoch = md.GetFirstEpoch() - - md.LastEpoch = md.GetLastEpoch() - - md.DbType = md.GetDbType() - - md.timestamp = md.GetTimestamp() - - md.ChainId = md.GetChainID() -} - -// compareBlocks from given Db and return them -func (md *AidaDbMetadata) compareBlocks(firstBlock uint64, lastBlock uint64) (uint64, uint64) { - var ( - dbFirst, dbLast uint64 - ) - - dbFirst = md.GetFirstBlock() - if (dbFirst != 0 && dbFirst < firstBlock) || firstBlock == 0 { - firstBlock = dbFirst + // Target is a subset of source + if targetFirstBlock > srcFirstBlock && targetLastBlock < srcLastBlock { + return fmt.Errorf("target db (%v-%v) is subset of source db (%v-%v)", targetFirstBlock, targetLastBlock, srcFirstBlock, srcLastBlock) } - dbLast = md.GetLastBlock() - - if dbLast > lastBlock || lastBlock == 0 { - lastBlock = dbLast + blocksOk := false + // Check alignment - dbs can overlap but cannot have gaps + // Target is before source + if targetLastBlock+1 >= srcFirstBlock { + err = md.SetLastBlock(srcLastBlock) + if err != nil { + return fmt.Errorf("cannot merge last block: %v", err) + } + blocksOk = true } - return firstBlock, lastBlock -} - -// compareEpochs from given Db and return them -func (md *AidaDbMetadata) compareEpochs(firstEpoch uint64, lastEpoch uint64) (uint64, uint64) { - var ( - dbFirst, dbLast uint64 - ) - - dbFirst = md.GetFirstEpoch() - if (dbFirst != 0 && dbFirst < firstEpoch) || firstEpoch == 0 { - firstEpoch = dbFirst + // Target is after source + if srcLastBlock+1 >= targetFirstBlock { + err = md.SetFirstBlock(srcFirstBlock) + if err != nil { + return fmt.Errorf("cannot merge first block: %v", err) + } + blocksOk = true } - dbLast = md.GetLastEpoch() - if dbLast > lastEpoch || lastEpoch == 0 { - lastEpoch = dbLast + if !blocksOk { + return fmt.Errorf("blocks does not align; target db (%v-%v), source db (%v-%v)", targetFirstBlock, targetLastBlock, srcFirstBlock, srcLastBlock) } - - return firstEpoch, lastEpoch + return md.findEpochs() } // GetFirstBlock and return it func (md *AidaDbMetadata) GetFirstBlock() uint64 { + if md.FirstBlock != nil { + return *md.FirstBlock + } firstBlockBytes, err := md.Db.Get([]byte(FirstBlockPrefix)) if err != nil { if errors.Is(err, leveldb.ErrNotFound) { @@ -429,11 +279,16 @@ func (md *AidaDbMetadata) GetFirstBlock() uint64 { return 0 } - return bigendian.BytesToUint64(firstBlockBytes) + blk := bigendian.BytesToUint64(firstBlockBytes) + md.FirstBlock = &blk + return blk } // GetLastBlock and return it func (md *AidaDbMetadata) GetLastBlock() uint64 { + if md.LastBlock != nil { + return *md.LastBlock + } lastBlockBytes, err := md.Db.Get([]byte(LastBlockPrefix)) if err != nil { if errors.Is(err, leveldb.ErrNotFound) { @@ -442,26 +297,32 @@ func (md *AidaDbMetadata) GetLastBlock() uint64 { md.log.Criticalf("cannot get last block from metadata; %v", err) return 0 } - - return bigendian.BytesToUint64(lastBlockBytes) + blk := bigendian.BytesToUint64(lastBlockBytes) + md.LastBlock = &blk + return blk } -// GetFirstEpoch and return it func (md *AidaDbMetadata) GetFirstEpoch() uint64 { + if md.FirstEpoch != nil { + return *md.FirstEpoch + } firstEpochBytes, err := md.Db.Get([]byte(FirstEpochPrefix)) if err != nil { if errors.Is(err, leveldb.ErrNotFound) { return 0 } md.log.Criticalf("cannot get first epoch from metadata; %v", err) - return 0 + return 00 } - - return bigendian.BytesToUint64(firstEpochBytes) + blk := bigendian.BytesToUint64(firstEpochBytes) + md.FirstEpoch = &blk + return blk } -// GetLastEpoch and return it func (md *AidaDbMetadata) GetLastEpoch() uint64 { + if md.LastEpoch != nil { + return *md.LastEpoch + } lastEpochBytes, err := md.Db.Get([]byte(LastEpochPrefix)) if err != nil { if errors.Is(err, leveldb.ErrNotFound) { @@ -470,12 +331,16 @@ func (md *AidaDbMetadata) GetLastEpoch() uint64 { md.log.Criticalf("cannot get last epoch from metadata; %v", err) return 0 } - - return bigendian.BytesToUint64(lastEpochBytes) + blk := bigendian.BytesToUint64(lastEpochBytes) + md.LastEpoch = &blk + return blk } // GetChainID and return it func (md *AidaDbMetadata) GetChainID() ChainID { + if md.ChainId != 0 { + return md.ChainId + } chainIDBytes, err := md.Db.Get([]byte(ChainIDPrefix)) if err != nil { if errors.Is(err, leveldb.ErrNotFound) { @@ -487,14 +352,18 @@ func (md *AidaDbMetadata) GetChainID() ChainID { // chainID used to be 2 bytes long, now it is 8 bytes long if len(chainIDBytes) == 2 { - return ChainID(bigendian.BytesToUint16(chainIDBytes)) + md.ChainId = ChainID(bigendian.BytesToUint16(chainIDBytes)) + } else { + md.ChainId = ChainID(bigendian.BytesToUint64(chainIDBytes)) } - - return ChainID(bigendian.BytesToUint64(chainIDBytes)) + return md.ChainId } // GetTimestamp and return it func (md *AidaDbMetadata) GetTimestamp() uint64 { + if md.timestamp != nil { + return *md.timestamp + } byteTimestamp, err := md.Db.Get([]byte(TimestampPrefix)) if err != nil { if errors.Is(err, leveldb.ErrNotFound) { @@ -503,12 +372,16 @@ func (md *AidaDbMetadata) GetTimestamp() uint64 { md.log.Criticalf("cannot get timestamp from metadata; %v", err) return 0 } - - return bigendian.BytesToUint64(byteTimestamp) + timestamp := bigendian.BytesToUint64(byteTimestamp) + md.timestamp = ×tamp + return timestamp } // GetDbType and return it func (md *AidaDbMetadata) GetDbType() AidaDbType { + if md.DbType != NoType { + return md.DbType + } byteDbType, err := md.Db.Get([]byte(TypePrefix)) if err != nil { if errors.Is(err, leveldb.ErrNotFound) { @@ -518,156 +391,99 @@ func (md *AidaDbMetadata) GetDbType() AidaDbType { return 0 } - return AidaDbType(byteDbType[0]) + md.DbType = AidaDbType(byteDbType[0]) + return md.DbType } -// SetFirstBlock in given Db func (md *AidaDbMetadata) SetFirstBlock(firstBlock uint64) error { firstBlockBytes := db.BlockToBytes(firstBlock) - if err := md.Db.Put([]byte(FirstBlockPrefix), firstBlockBytes); err != nil { return fmt.Errorf("cannot put first block; %v", err) } - - md.FirstBlock = firstBlock - + md.FirstBlock = &firstBlock md.log.Info("METADATA: First block saved successfully") - return nil } -// SetLastBlock in given Db func (md *AidaDbMetadata) SetLastBlock(lastBlock uint64) error { lastBlockBytes := db.BlockToBytes(lastBlock) - if err := md.Db.Put([]byte(LastBlockPrefix), lastBlockBytes); err != nil { return fmt.Errorf("cannot put last block; %v", err) } - - md.LastBlock = lastBlock - + md.LastBlock = &lastBlock md.log.Info("METADATA: Last block saved successfully") - return nil } // SetFirstEpoch in given Db func (md *AidaDbMetadata) SetFirstEpoch(firstEpoch uint64) error { firstEpochBytes := db.BlockToBytes(firstEpoch) - if err := md.Db.Put([]byte(FirstEpochPrefix), firstEpochBytes); err != nil { return fmt.Errorf("cannot put first epoch; %v", err) } - md.log.Info("METADATA: First epoch saved successfully") - + md.FirstEpoch = &firstEpoch return nil } // SetLastEpoch in given Db func (md *AidaDbMetadata) SetLastEpoch(lastEpoch uint64) error { lastEpochBytes := db.BlockToBytes(lastEpoch) - if err := md.Db.Put([]byte(LastEpochPrefix), lastEpochBytes); err != nil { return fmt.Errorf("cannot put last epoch; %v", err) } - md.log.Info("METADATA: Last epoch saved successfully") - + md.LastEpoch = &lastEpoch return nil } // SetChainID in given Db func (md *AidaDbMetadata) SetChainID(chainID ChainID) error { chainIDBytes := bigendian.Uint64ToBytes(uint64(chainID)) - if err := md.Db.Put([]byte(ChainIDPrefix), chainIDBytes); err != nil { return fmt.Errorf("cannot put chain-id; %v", err) } - - md.ChainId = chainID - md.log.Info("METADATA: ChainID saved successfully") - + md.ChainId = chainID return nil } -// SetTimestamp in given Db func (md *AidaDbMetadata) SetTimestamp() error { createTime := make([]byte, 8) - binary.BigEndian.PutUint64(createTime, uint64(time.Now().Unix())) if err := md.Db.Put([]byte(TimestampPrefix), createTime); err != nil { return fmt.Errorf("cannot put timestamp into db metadata; %v", err) } - md.log.Info("METADATA: Creation timestamp saved successfully") - + timestamp := binary.BigEndian.Uint64(createTime) + md.timestamp = ×tamp return nil } -// SetDbType in given Db func (md *AidaDbMetadata) SetDbType(dbType AidaDbType) error { dbTypeBytes := make([]byte, 1) dbTypeBytes[0] = byte(dbType) - if err := md.Db.Put([]byte(TypePrefix), dbTypeBytes); err != nil { return fmt.Errorf("cannot put db-type into aida-db; %v", err) } - md.DbType = dbType - md.log.Info("METADATA: DB Type saved successfully") - - return nil -} - -// SetAll in given Db -func (md *AidaDbMetadata) SetAll() error { - var err error - - if err = md.SetFirstBlock(md.FirstBlock); err != nil { - return err - } - - if err = md.SetLastBlock(md.LastBlock); err != nil { - return err - } - - if err = md.SetFirstEpoch(md.FirstEpoch); err != nil { - return err - } - - if err = md.SetLastEpoch(md.LastEpoch); err != nil { - return err - } - - if err = md.SetChainID(md.ChainId); err != nil { - return err - } - - if err = md.SetDbType(md.DbType); err != nil { - return err - } - - if err = md.SetTimestamp(); err != nil { - return err - } + md.DbType = dbType return nil } -// SetDbHash in given Db func (md *AidaDbMetadata) SetDbHash(dbHash []byte) error { if err := md.Db.Put([]byte(DbHashPrefix), dbHash); err != nil { return fmt.Errorf("cannot put metadata; %v", err) } - md.log.Info("METADATA: Db hash saved successfully") - + md.dbHash = dbHash return nil } -// GetDbHash and return it func (md *AidaDbMetadata) GetDbHash() []byte { + if len(md.dbHash) > 0 { + return md.dbHash + } dbHash, err := md.Db.Get([]byte(DbHashPrefix)) if err != nil { if errors.Is(err, leveldb.ErrNotFound) { @@ -676,49 +492,10 @@ func (md *AidaDbMetadata) GetDbHash() []byte { md.log.Criticalf("cannot get Db hash from metadata; %v", err) return nil } - + md.dbHash = dbHash return dbHash } -// SetAllMetadata in given Db -func (md *AidaDbMetadata) SetAllMetadata(firstBlock uint64, lastBlock uint64, firstEpoch uint64, lastEpoch uint64, chainID ChainID, dbHash []byte, dbType AidaDbType) error { - var err error - - if err = md.SetFirstBlock(firstBlock); err != nil { - return err - } - - if err = md.SetLastBlock(lastBlock); err != nil { - return err - } - - if err = md.SetFirstEpoch(firstEpoch); err != nil { - return err - } - - if err = md.SetLastEpoch(lastEpoch); err != nil { - return err - } - - if err = md.SetChainID(chainID); err != nil { - return err - } - - if err = md.SetDbType(dbType); err != nil { - return err - } - - if err = md.SetTimestamp(); err != nil { - return err - } - - if err = md.SetDbHash(dbHash); err != nil { - return err - } - - return nil -} - // findEpochs for block range in metadata func (md *AidaDbMetadata) findEpochs() error { var ( @@ -732,131 +509,49 @@ func (md *AidaDbMetadata) findEpochs() error { return nil } - md.FirstEpoch, err = FindEpochNumber(md.FirstBlock, md.ChainId) + firstEpoch, err := FindEpochNumber(md.GetFirstBlock(), md.ChainId) if err != nil { - return err + return fmt.Errorf("cannot find first epoch; %v", err) } - // if first block is 0 we can be sure the block begins an epoch so no need to check that - if md.FirstBlock != 0 { + if md.GetFirstBlock() != 0 { // we need to check if block is really first block of an epoch - firstEpochMinus, err = FindEpochNumber(md.FirstBlock-1, md.ChainId) + firstEpochMinus, err = FindEpochNumber(md.GetFirstBlock(), md.ChainId) if err != nil { return err } - if firstEpochMinus >= md.FirstEpoch { - md.log.Warningf("first block of db is not beginning of an epoch; setting first epoch to 0") - md.FirstEpoch = 0 + if firstEpochMinus >= md.GetFirstEpoch() { + md.log.Warningf("first block of db is not beginning of an epoch") } else { - md.log.Noticef("Found first epoch #%v", md.FirstEpoch) + md.log.Noticef("Found first epoch #%v", md.GetFirstEpoch()) + err = md.SetFirstEpoch(firstEpoch) + if err != nil { + return fmt.Errorf("cannot set first epoch; %v", err) + } } } - md.LastEpoch, err = FindEpochNumber(md.LastBlock, md.ChainId) + lastEpoch, err := FindEpochNumber(md.GetLastEpoch(), md.ChainId) if err != nil { - return err + return fmt.Errorf("cannot find last epoch; %v", err) } - // we need to check if block is really last block of an epoch - lastEpochPlus, err = FindEpochNumber(md.LastBlock+1, md.ChainId) + lastEpochPlus, err = FindEpochNumber(md.GetLastEpoch()+1, md.ChainId) if err != nil { return err } - if lastEpochPlus <= md.LastEpoch { - md.log.Warningf("last block block of db is not end of an epoch; setting last epoch to 0") - md.LastEpoch = 0 + if lastEpochPlus <= md.GetLastEpoch() { + md.log.Warningf("last block block of db is not end of an epoch") } else { md.log.Noticef("Found last epoch #%v", md.LastEpoch) - } - - return nil -} - -// CheckUpdateMetadata goes through metadata of updated AidaDb and its patch, -// looks if blocks and epoch align and if chainIDs are same for both Dbs -func (md *AidaDbMetadata) CheckUpdateMetadata(cfg *Config, patchDb db.BaseDB) error { - var ( - err error - ignoreBlockAlignment, isLachesisPatch bool - ) - - patchMD := NewAidaDbMetadata(patchDb, cfg.LogLevel) - - patchMD.GetMetadata() - - // if we are updating existing AidaDb and this Db does not have metadata, we go through substate to find - // blocks and epochs, chainID is Set from user via chain-id flag and db type in this case will always be genType - md.GetMetadata() - if md.LastBlock == 0 { - if err = md.SetFreshMetadata(cfg.ChainID); err != nil { - return fmt.Errorf("cannot set fresh metadata for existing AidaDb; %v", err) - } - } - // we check if patch is lachesis with first condition - // we also need to check that metadata were set with second condition - if patchMD.FirstBlock == 0 { - if patchMD.LastBlock == 0 { - var ok bool - - sdb := db.MakeDefaultSubstateDBFromBaseDB(patchDb) - patchMD.FirstBlock, patchMD.LastBlock, ok = FindBlockRangeInSubstate(sdb) - if !ok { - return errors.New("patch does not contain metadata and block range was not found in substate") - } - - md.FirstEpoch, err = FindEpochNumber(md.FirstBlock, md.ChainId) - if err != nil { - return err - } - md.LastEpoch, err = FindEpochNumber(md.LastBlock, md.ChainId) - if err != nil { - return err - } - } - // we need to check again whether first block is still 0 after substate search - if patchMD.FirstBlock == 0 { - ignoreBlockAlignment = true - isLachesisPatch = true + err = md.SetLastEpoch(lastEpoch) + if err != nil { + return fmt.Errorf("cannot set last epoch; %v", err) } } - // we ignore block alignment also for first patch - this exception is for a situation when user has first patch - // and lachesis is being installed, so first patch is getting replaced - if patchMD.FirstBlock == 4564026 { - ignoreBlockAlignment = true - } - - // the patch is usable only if its FirstBlock is within targetDbs block range - // and if its last block is bigger than tarGetDBs last block - if patchMD.FirstBlock > md.LastBlock+1 || patchMD.FirstBlock < md.FirstBlock || patchMD.LastBlock <= md.LastBlock { - // if patch is lachesis patch, we continue with merge - - if !ignoreBlockAlignment { - return fmt.Errorf("metadata blocks does not align; aida-db %v-%v, patch %v-%v", md.FirstBlock, md.LastBlock, patchMD.FirstBlock, patchMD.LastBlock) - } - - } - - // if chainIDs doesn't match, we can't patch the DB - if md.ChainId != patchMD.ChainId { - return fmt.Errorf("metadata chain-ids does not match; aida-db: %v, patch: %v", md.ChainId, patchMD.ChainId) - } - - if isLachesisPatch { - // we set the first block and epoch to 0 - // last block and epoch stays - md.FirstBlock = 0 - md.FirstEpoch = 0 - } else if md.LastBlock < patchMD.LastBlock { - // this condition is needed when we try to overwrite the first patch, then we dont want to overwrite the metadata - // if patch is not lachesis hence is being appended, we take last block and epoch from it - // first block and epoch stays - md.LastBlock = patchMD.LastBlock - md.LastEpoch = patchMD.LastEpoch - } - return nil } @@ -881,7 +576,7 @@ func (md *AidaDbMetadata) SetFreshMetadata(chainID ChainID) error { return err } - _, err = getPatchFirstBlock(md.LastBlock) + _, err = getPatchFirstBlock(md.GetLastBlock()) if err != nil { md.log.Warning("Uncertain AidaDbType.") if err = md.SetDbType(NoType); err != nil { @@ -909,93 +604,50 @@ func (md *AidaDbMetadata) SetBlockRange(firstBlock uint64, lastBlock uint64) err return nil } -func (md *AidaDbMetadata) DeleteMetadata() { +func (md *AidaDbMetadata) Delete() error { var err error if err = md.Db.Delete([]byte(ChainIDPrefix)); err != nil { - md.log.Criticalf("cannot delete chain-id; %v", err) + return fmt.Errorf("cannot delete chain-id; %v", err) } else { md.log.Debugf("ChainID deleted successfully") } if err = md.Db.Delete([]byte(FirstBlockPrefix)); err != nil { - md.log.Criticalf("cannot delete first block; %v", err) + return fmt.Errorf("cannot delete first block; %v", err) } else { md.log.Debugf("First block deleted successfully") } if err = md.Db.Delete([]byte(LastBlockPrefix)); err != nil { - md.log.Criticalf("cannot delete last block; %v", err) + return fmt.Errorf("cannot delete last block; %v", err) } else { md.log.Debugf("Last block deleted successfully") } if err = md.Db.Delete([]byte(FirstEpochPrefix)); err != nil { - md.log.Criticalf("cannot delete first epoch; %v", err) + return fmt.Errorf("cannot delete first epoch; %v", err) } else { md.log.Debugf("First epoch deleted successfully") } if err = md.Db.Delete([]byte(LastEpochPrefix)); err != nil { - md.log.Criticalf("cannot delete last epoch; %v", err) + return fmt.Errorf("cannot delete last epoch; %v", err) } else { md.log.Debugf("Last epoch deleted successfully") } if err = md.Db.Delete([]byte(TypePrefix)); err != nil { - md.log.Criticalf("cannot delete db type; %v", err) + return fmt.Errorf("cannot delete db type; %v", err) } else { md.log.Debugf("Timestamp deleted successfully") } if err = md.Db.Delete([]byte(TimestampPrefix)); err != nil { - md.log.Criticalf("cannot delete creation timestamp; %v", err) + return fmt.Errorf("cannot delete creation timestamp; %v", err) } else { md.log.Debugf("Timestamp deleted successfully") } -} - -// UpdateMetadataInOldAidaDb Sets metadata necessary for update in old aida-db, which doesn't have any metadata -func (md *AidaDbMetadata) UpdateMetadataInOldAidaDb(chainId ChainID, firstAidaDbBlock uint64, lastAidaDbBlock uint64) error { - var err error - - // Set chainid if it doesn't exist - inCID := md.GetChainID() - if inCID == 0 { - err = md.SetChainID(chainId) - if err != nil { - return err - } - } - - // Set first block if it doesn't exist - inFB := md.GetFirstBlock() - if inFB == 0 { - err = md.SetFirstBlock(firstAidaDbBlock) - if err != nil { - return err - } - } - - // Set last block if it doesn't exist - inLB := md.GetLastBlock() - if inLB == 0 { - err = md.SetLastBlock(lastAidaDbBlock) - if err != nil { - return err - } - } - - // anything apart from clone db is always gentype db - inType := md.GetDbType() - if inType != CloneType { - inType = GenType - } - - err = md.SetDbType(inType) - if err != nil { - return err - } return nil } @@ -1020,22 +672,6 @@ func FindBlockRangeInSubstate(db db.SubstateDB) (uint64, uint64, bool) { return firstBlock, lastBlock, true } -func (md *AidaDbMetadata) getVerboseDbType() string { - switch md.DbType { - case GenType: - return "Generate" - case CloneType: - return "Clone" - case PatchType: - return "Patch" - case NoType: - return "NoType" - - default: - return "unknown db type" - } -} - // DownloadPatchesJson downloads list of available patches from aida-db generation server. func DownloadPatchesJson() (data []PatchJson, err error) { // Make the HTTP GET request @@ -1084,20 +720,6 @@ func getPatchFirstBlock(lastPatchBlock uint64) (uint64, error) { } -// getBlockRange returns first and last block inside metadata. -// If last block is zero, it looks for block range in substate, and tries to get even the epoch range -func (md *AidaDbMetadata) getBlockRange() error { - md.FirstBlock = md.GetFirstBlock() - md.LastBlock = md.GetLastBlock() - - // check if AidaDb has block range - if md.LastBlock == 0 { - return errors.New("given aida-db does not contain metadata; please generate them using util-db metadata generate") - } - - return nil -} - // HasStateHashPatch checks whether given db has already acquired patch with StateHashes. func HasStateHashPatch(path string) (bool, error) { db, err := geth_leveldb.New(path, 1024, 100, "profiling", true) @@ -1153,3 +775,38 @@ func (md *AidaDbMetadata) SetUpdatesetSize(val uint64) error { md.log.Info("METADATA: Updateset size saved successfully") return nil } +func (md *AidaDbMetadata) GetUpdatesetInterval() uint64 { + if md.updateSetInterval != nil { + return *md.updateSetInterval + } + bytes, err := md.Db.Get([]byte(db.UpdatesetIntervalKey)) + if err != nil { + if errors.Is(err, leveldb.ErrNotFound) { + return 0 + } + md.log.Criticalf("cannot get updateset interval from metadata; %v", err) + return 0 + } + + interval := bigendian.BytesToUint64(bytes) + md.updateSetInterval = &interval + return interval +} + +func (md *AidaDbMetadata) GetUpdatesetSize() uint64 { + if md.updateSetSize != nil { + return *md.updateSetSize + } + bytes, err := md.Db.Get([]byte(db.UpdatesetSizeKey)) + if err != nil { + if errors.Is(err, leveldb.ErrNotFound) { + return 0 + } + md.log.Criticalf("cannot get updateset size from metadata; %v", err) + return 0 + } + + size := bigendian.BytesToUint64(bytes) + md.updateSetSize = &size + return size +} diff --git a/utils/metadata_test.go b/utils/metadata_test.go index 05c118d45..1a97c933e 100644 --- a/utils/metadata_test.go +++ b/utils/metadata_test.go @@ -63,89 +63,6 @@ func TestGetPatchFirstBlock_Positive(t *testing.T) { } } -func TestAidaDbMetadata_SetAll(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockErr := errors.New("mock error") - - // Case 1: No errors - mockDb := db.NewMockBaseDB(ctrl) - md := NewAidaDbMetadata(mockDb, "ERROR") - md.FirstBlock = 100 - md.LastBlock = 200 - md.FirstEpoch = 10 - md.LastEpoch = 20 - md.ChainId = MainnetChainID - md.DbType = GenType - - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(7) // 7 metadata fields - err := md.SetAll() - assert.NoError(t, err) - - // Case 2: Error with SetFirstBlock - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAll() - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 3: Error with SetLastBlock - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(1) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAll() - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 4: Error with SetFirstEpoch - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(2) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAll() - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 5: Error with SetLastEpoch - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(3) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAll() - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 6: Error with SetChainId - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(4) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAll() - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 7: Error with SetDbType - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(5) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAll() - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 8: Error with SetTimestamp - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(6) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAll() - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) -} - func TestAidaDbMetadata_SetDbHash(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -154,14 +71,14 @@ func TestAidaDbMetadata_SetDbHash(t *testing.T) { dbHash := []byte("hash123") // Case 1: Success - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Put([]byte(DbHashPrefix), dbHash).Return(nil) err := md.SetDbHash(dbHash) assert.NoError(t, err) // Case 2: Error - mockDb = db.NewMockBaseDB(ctrl) + mockDb = db.NewMockSubstateDB(ctrl) md = NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Put([]byte(DbHashPrefix), dbHash).Return(mockErr) err = md.SetDbHash(dbHash) @@ -176,277 +93,33 @@ func TestAidaDbMetadata_GetDbHash(t *testing.T) { expectedHash := []byte("hash123") // Case 1: Success - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Get([]byte(DbHashPrefix)).Return(expectedHash, nil) hash := md.GetDbHash() assert.Equal(t, expectedHash, hash) // Case 2: Not found error - mockDb = db.NewMockBaseDB(ctrl) + mockDb = db.NewMockSubstateDB(ctrl) md = NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Get([]byte(DbHashPrefix)).Return(nil, leveldb.ErrNotFound) hash = md.GetDbHash() assert.Nil(t, hash) // Case 3: Other error - mockDb = db.NewMockBaseDB(ctrl) + mockDb = db.NewMockSubstateDB(ctrl) md = NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Get([]byte(DbHashPrefix)).Return(nil, errors.New("other error")) hash = md.GetDbHash() assert.Nil(t, hash) } -func TestAidaDbMetadata_SetAllMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockErr := errors.New("mock error") - dbHash := []byte("hash123") - - // Case 1: Success - mockDb := db.NewMockBaseDB(ctrl) - md := NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(8) // 8 metadata fields including hash - err := md.SetAllMetadata(100, 200, 10, 20, MainnetChainID, dbHash, GenType) - assert.NoError(t, err) - - // Case 2: Error with SetFirstBlock - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAllMetadata(100, 200, 10, 20, MainnetChainID, dbHash, GenType) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 3: Error with SetLastBlock - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(1) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAllMetadata(100, 200, 10, 20, MainnetChainID, dbHash, GenType) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 4: Error with SetFirstEpoch - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(2) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAllMetadata(100, 200, 10, 20, MainnetChainID, dbHash, GenType) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 5: Error with SetLastEpoch - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(3) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAllMetadata(100, 200, 10, 20, MainnetChainID, dbHash, GenType) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 6: Error with SetChainId - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(4) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAllMetadata(100, 200, 10, 20, MainnetChainID, dbHash, GenType) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 7: Error with SetDbType - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(5) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAllMetadata(100, 200, 10, 20, MainnetChainID, dbHash, GenType) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 8: Error with SetTimestamp - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(6) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAllMetadata(100, 200, 10, 20, MainnetChainID, dbHash, GenType) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 9: Error with SetDbHash - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(7) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.SetAllMetadata(100, 200, 10, 20, MainnetChainID, dbHash, GenType) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) -} - -func TestAidaDbMetadata_CheckUpdateMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - // Case 1: Success - cfg := &Config{ - LogLevel: "ERROR", - ChainID: MainnetChainID, - } - mockAidaDb := db.NewMockBaseDB(ctrl) - mockPatchDb := db.NewMockBaseDB(ctrl) - md := NewAidaDbMetadata(mockAidaDb, "ERROR") - mockAidaDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(4564026), nil).AnyTimes() - mockPatchDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(4564026), nil).AnyTimes() - err := md.CheckUpdateMetadata(cfg, mockPatchDb) - assert.NoError(t, err) - - // Case 2: Last block is Zero - cfg = &Config{ - LogLevel: "ERROR", - ChainID: ChainID(0), - } - mockAidaDb = db.NewMockBaseDB(ctrl) - mockPatchDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockAidaDb, "ERROR") - mockAidaDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(0), nil).AnyTimes() - mockPatchDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(0), nil).AnyTimes() - err = md.CheckUpdateMetadata(cfg, mockPatchDb) - assert.Error(t, err) - - // Case 3: Block not aligned - cfg = &Config{ - LogLevel: "ERROR", - ChainID: MainnetChainID, - } - mockAidaDb = db.NewMockBaseDB(ctrl) - mockPatchDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockAidaDb, "ERROR") - mockAidaDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(741852), nil).AnyTimes() - mockPatchDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(123456), nil).AnyTimes() - err = md.CheckUpdateMetadata(cfg, mockPatchDb) - assert.Error(t, err) - - // Case 4: ChainID mismatch - cfg = &Config{ - LogLevel: "ERROR", - ChainID: MainnetChainID, - } - mockAidaDb = db.NewMockBaseDB(ctrl) - mockPatchDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockAidaDb, "ERROR") - mockAidaDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(4564026), nil).AnyTimes() - mockPatchDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(bigendian.Uint64ToBytes(123456), nil).AnyTimes() - mockPatchDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(4564026), nil).AnyTimes() - err = md.CheckUpdateMetadata(cfg, mockPatchDb) - assert.Error(t, err) - - // Case 5 - cfg = &Config{ - LogLevel: "ERROR", - ChainID: MainnetChainID, - } - mockAidaDb = db.NewMockBaseDB(ctrl) - mockPatchDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockAidaDb, "ERROR") - mockAidaDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(bigendian.Uint64ToBytes(4564026), nil).AnyTimes() - mockAidaDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(4564025), nil).AnyTimes() - mockPatchDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(bigendian.Uint64ToBytes(4564026), nil).AnyTimes() - mockPatchDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(4564026), nil).AnyTimes() - err = md.CheckUpdateMetadata(cfg, mockPatchDb) - assert.NoError(t, err) - - // Case 6 - cfg = &Config{ - LogLevel: "ERROR", - ChainID: MainnetChainID, - } - mockAidaDb = db.NewMockBaseDB(ctrl) - mockPatchDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockAidaDb, "ERROR") - mockAidaDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(4564026), nil).AnyTimes() - mockPatchDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(bigendian.Uint64ToBytes(0), nil).AnyTimes() - mockPatchDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(4564026), nil).AnyTimes() - err = md.CheckUpdateMetadata(cfg, mockPatchDb) - assert.NoError(t, err) -} - -func TestAidaDbMetadata_SetFreshMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - // case 1 - mockDb := db.NewMockBaseDB(ctrl) - md := NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(3) - err := md.SetFreshMetadata(MainnetChainID) - assert.NoError(t, err) - - // case 2 - err = md.SetFreshMetadata(0) - assert.Error(t, err) - - // case 3 - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(errors.New("error")) - err = md.SetFreshMetadata(MainnetChainID) - assert.Error(t, err) - - // case 4 - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(errors.New("error")) - err = md.SetFreshMetadata(MainnetChainID) - assert.Error(t, err) - - // case 5 - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(2) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(errors.New("error")) - err = md.SetFreshMetadata(MainnetChainID) - assert.Error(t, err) -} - -func TestAidaDbMetadata_SetBlockRange(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockErr := errors.New("mock error") - - // Case 1: Success - mockDb := db.NewMockBaseDB(ctrl) - md := NewAidaDbMetadata(mockDb, "ERROR") - - mockDb.EXPECT().Put([]byte(FirstBlockPrefix), gomock.Any()).Return(nil) - mockDb.EXPECT().Put([]byte(LastBlockPrefix), gomock.Any()).Return(nil) - - err := md.SetBlockRange(100, 200) - assert.NoError(t, err) - - // Case 2: Error with SetFirstBlock - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - - mockDb.EXPECT().Put([]byte(FirstBlockPrefix), gomock.Any()).Return(mockErr) - - err = md.SetBlockRange(100, 200) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 3: Error with SetLastBlock - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - - mockDb.EXPECT().Put([]byte(FirstBlockPrefix), gomock.Any()).Return(nil) - mockDb.EXPECT().Put([]byte(LastBlockPrefix), gomock.Any()).Return(mockErr) - - err = md.SetBlockRange(100, 200) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) -} - func TestAidaDbMetadata_DeleteMetadata(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() // Case 1: Success - all deletes succeed - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Delete([]byte(ChainIDPrefix)).Return(nil) @@ -457,10 +130,10 @@ func TestAidaDbMetadata_DeleteMetadata(t *testing.T) { mockDb.EXPECT().Delete([]byte(TypePrefix)).Return(nil) mockDb.EXPECT().Delete([]byte(TimestampPrefix)).Return(nil) - md.DeleteMetadata() // Should not panic or return error + md.Delete() // Should not panic or return error // Case 2: Some deletes fail - should log errors but not fail - mockDb = db.NewMockBaseDB(ctrl) + mockDb = db.NewMockSubstateDB(ctrl) md = NewAidaDbMetadata(mockDb, "ERROR") mockErr := errors.New("delete error") @@ -472,432 +145,33 @@ func TestAidaDbMetadata_DeleteMetadata(t *testing.T) { mockDb.EXPECT().Delete([]byte(TypePrefix)).Return(mockErr) mockDb.EXPECT().Delete([]byte(TimestampPrefix)).Return(mockErr) - md.DeleteMetadata() // Should not panic despite errors -} - -func TestAidaDbMetadata_UpdateMetadataInOldAidaDb(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockErr := errors.New("mock error") - - // Case 1: No existing metadata, all values should be set - mockDb := db.NewMockBaseDB(ctrl) - md := NewAidaDbMetadata(mockDb, "ERROR") - - // Set expectations for checking current values (all return not found) - mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Get([]byte(TypePrefix)).Return(nil, errors.New("not found")) - - // Set expectations for setting new values - mockDb.EXPECT().Put([]byte(ChainIDPrefix), gomock.Any()).Return(nil) - mockDb.EXPECT().Put([]byte(FirstBlockPrefix), gomock.Any()).Return(nil) - mockDb.EXPECT().Put([]byte(LastBlockPrefix), gomock.Any()).Return(nil) - mockDb.EXPECT().Put([]byte(TypePrefix), gomock.Any()).Return(nil) - - err := md.UpdateMetadataInOldAidaDb(MainnetChainID, 100, 200) - assert.NoError(t, err) - - // Case 2: Error when setting ChainID - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - - mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(ChainIDPrefix), gomock.Any()).Return(mockErr) - - err = md.UpdateMetadataInOldAidaDb(MainnetChainID, 100, 200) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 3: Error when setting FirstBlock - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - - mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(ChainIDPrefix), gomock.Any()).Return(nil) - mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(FirstBlockPrefix), gomock.Any()).Return(mockErr) - - err = md.UpdateMetadataInOldAidaDb(MainnetChainID, 100, 200) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 4: Error when setting LastBlock - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - - mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(ChainIDPrefix), gomock.Any()).Return(nil) - mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(FirstBlockPrefix), gomock.Any()).Return(nil) - mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(LastBlockPrefix), gomock.Any()).Return(mockErr) - - err = md.UpdateMetadataInOldAidaDb(MainnetChainID, 100, 200) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 5: Error when setting DbType - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - - mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(ChainIDPrefix), gomock.Any()).Return(nil) - mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(FirstBlockPrefix), gomock.Any()).Return(nil) - mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(LastBlockPrefix), gomock.Any()).Return(nil) - mockDb.EXPECT().Get([]byte(TypePrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(TypePrefix), gomock.Any()).Return(mockErr) - - err = md.UpdateMetadataInOldAidaDb(MainnetChainID, 100, 200) - assert.Error(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // Case 6: Some metadata already exists - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - - // Chain ID already exists - mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(bigendian.Uint64ToBytes(uint64(MainnetChainID)), nil) - // First block needs to be set - mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(FirstBlockPrefix), gomock.Any()).Return(nil) - // Last block already exists - mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(bigendian.Uint64ToBytes(999), nil) - // Type still needs to be set - mockDb.EXPECT().Get([]byte(TypePrefix)).Return(nil, errors.New("not found")) - mockDb.EXPECT().Put([]byte(TypePrefix), gomock.Any()).Return(nil) - - err = md.UpdateMetadataInOldAidaDb(MainnetChainID, 100, 200) - assert.NoError(t, err) -} - -func TestNewAidaDbMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockDb := db.NewMockBaseDB(ctrl) - metadata := NewAidaDbMetadata(mockDb, "ERROR") - assert.Equal(t, mockDb, metadata.Db) -} - -func TestProcessPatchLikeMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockErr := errors.New("mock error") - - // no error - mockDb := db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - err := ProcessPatchLikeMetadata(mockDb, "ERROR", 0, 0, 0, 0, 0, true, nil) - assert.NoError(t, err) - - // no error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - err = ProcessPatchLikeMetadata(mockDb, "ERROR", 0, 0, 0, 0, 0, false, nil) - assert.NoError(t, err) - - // SetFirstBlock error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessPatchLikeMetadata(mockDb, "ERROR", 0, 0, 0, 0, 0, true, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetLastBlock error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(1) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessPatchLikeMetadata(mockDb, "ERROR", 0, 0, 0, 0, 0, true, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetFirstEpoch error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(2) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessPatchLikeMetadata(mockDb, "ERROR", 0, 0, 0, 0, 0, true, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetLastEpoch error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(3) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessPatchLikeMetadata(mockDb, "ERROR", 0, 0, 0, 0, 0, true, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetChainID error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(4) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessPatchLikeMetadata(mockDb, "ERROR", 0, 0, 0, 0, 0, true, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetDbType error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(5) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessPatchLikeMetadata(mockDb, "ERROR", 0, 0, 0, 0, 0, true, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetTimestamp error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(6) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessPatchLikeMetadata(mockDb, "ERROR", 0, 0, 0, 0, 0, true, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetDbHash error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(7) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessPatchLikeMetadata(mockDb, "ERROR", 0, 0, 0, 0, 0, true, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) -} - -func TestProcessCloneLikeMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockErr := errors.New("mock error") - - // no error - mockDb := db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - err := ProcessCloneLikeMetadata(mockDb, NoType, "ERROR", 0, 0, MainnetChainID) - assert.NoError(t, err) - - // SetFirstBlock error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessCloneLikeMetadata(mockDb, NoType, "ERROR", 0, 0, MainnetChainID) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetLastBlock error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(1) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessCloneLikeMetadata(mockDb, NoType, "ERROR", 0, 0, MainnetChainID) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetFirstEpoch error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(2) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessCloneLikeMetadata(mockDb, NoType, "ERROR", 0, 0, MainnetChainID) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetLastEpoch error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(3) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessCloneLikeMetadata(mockDb, NoType, "ERROR", 0, 0, MainnetChainID) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetChainID error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(4) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessCloneLikeMetadata(mockDb, NoType, "ERROR", 0, 0, MainnetChainID) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetDbType error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(5) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessCloneLikeMetadata(mockDb, NoType, "ERROR", 0, 0, MainnetChainID) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetTimestamp error - mockDb = db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(6) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = ProcessCloneLikeMetadata(mockDb, NoType, "ERROR", 0, 0, MainnetChainID) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) -} - -func TestProcessGenLikeMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockDb := db.NewMockBaseDB(ctrl) - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - err := ProcessGenLikeMetadata(mockDb, 0, 0, 0, 0, MainnetChainID, "ERROR", nil) - assert.NoError(t, err) -} - -func TestAidaDbMetadata_genMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockErr := errors.New("mock error") - - // no error - mockDb := db.NewMockBaseDB(ctrl) - md := NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - err := md.genMetadata(0, 0, 0, 0, MainnetChainID, nil) - assert.NoError(t, err) - - // SetFirstBlock error - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.genMetadata(0, 0, 0, 0, MainnetChainID, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetLastBlock error - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(1) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.genMetadata(0, 0, 0, 0, MainnetChainID, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetFirstEpoch error - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(2) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.genMetadata(0, 0, 0, 0, MainnetChainID, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetLastEpoch error - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(3) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.genMetadata(0, 0, 0, 0, MainnetChainID, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetChainID error - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(4) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.genMetadata(0, 0, 0, 0, MainnetChainID, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetDbType error - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(5) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.genMetadata(0, 0, 0, 0, MainnetChainID, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetTimestamp error - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(6) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.genMetadata(0, 0, 0, 0, MainnetChainID, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - - // SetDbHash error - mockDb = db.NewMockBaseDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).AnyTimes() - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).Times(7) - mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(mockErr) - err = md.genMetadata(0, 0, 0, 0, MainnetChainID, nil) - assert.NotNil(t, err) - assert.Contains(t, err.Error(), mockErr.Error()) - + md.Delete() // Should not panic despite errors } -func TestAidaDbMetadata_getVerboseDbType(t *testing.T) { +func TestAidaDbMetadata_AidaDbTyString(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockDb := db.NewMockBaseDB(ctrl) - md := NewAidaDbMetadata(mockDb, "ERROR") - md.DbType = GenType - dbType := md.getVerboseDbType() - assert.Equal(t, "Generate", dbType) + typ := GenType + assert.Equal(t, "Generate", typ.String()) - md.DbType = CloneType - dbType = md.getVerboseDbType() - assert.Equal(t, "Clone", dbType) + typ = CloneType + assert.Equal(t, "Clone", typ.String()) - md.DbType = PatchType - dbType = md.getVerboseDbType() - assert.Equal(t, "Patch", dbType) + typ = PatchType + assert.Equal(t, "Patch", typ.String()) - md.DbType = NoType - dbType = md.getVerboseDbType() - assert.Equal(t, "NoType", dbType) + typ = NoType + assert.Equal(t, "NoType", typ.String()) - md.DbType = 99 - dbType = md.getVerboseDbType() - assert.Equal(t, "unknown db type", dbType) + typ = 99 + assert.Equal(t, "unknown db type", typ.String()) } - -func TestAidaDbMetadata_getBlockRange(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockDb := db.NewMockBaseDB(ctrl) - md := NewAidaDbMetadata(mockDb, "ERROR") - - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(99), nil).Times(2) - err := md.getBlockRange() - assert.NoError(t, err) - assert.Equal(t, uint64(99), md.FirstBlock) - assert.Equal(t, uint64(99), md.LastBlock) - - mockDb.EXPECT().Get(gomock.Any()).Return(bigendian.Uint64ToBytes(0), nil).Times(2) - err = md.getBlockRange() - assert.Error(t, err) -} - func TestAidaDbMetadata_SetHasHashPatch(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil) @@ -909,7 +183,7 @@ func TestAidaDbMetadata_SetUpdatesetInterval(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil) @@ -925,7 +199,7 @@ func TestAidaDbMetadata_SetUpdatesetSize(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil) @@ -998,7 +272,7 @@ func TestAidaDbMetadata_GetFirstBlock(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(bigendian.Uint64ToBytes(100), nil) firstBlock := md.GetFirstBlock() @@ -1017,7 +291,7 @@ func TestAidaDbMetadata_GetLastBlock(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(bigendian.Uint64ToBytes(100), nil) lastBlock := md.GetLastBlock() @@ -1036,7 +310,7 @@ func TestAidaDbMetadata_GetFirstEpoch(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Get([]byte(FirstEpochPrefix)).Return(bigendian.Uint64ToBytes(100), nil) data := md.GetFirstEpoch() @@ -1055,7 +329,7 @@ func TestAidaDbMetadata_GetLastEpoch(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Get([]byte(LastEpochPrefix)).Return(bigendian.Uint64ToBytes(100), nil) data := md.GetLastEpoch() @@ -1074,13 +348,13 @@ func TestAidaDbMetadata_GetChainID(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(bigendian.Uint64ToBytes(100), nil) data := md.GetChainID() assert.Equal(t, ChainID(100), data) - mockDb = db.NewMockBaseDB(ctrl) + mockDb = db.NewMockSubstateDB(ctrl) md = NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return([]byte{0, 1}, nil) data = md.GetChainID() @@ -1099,7 +373,7 @@ func TestAidaDbMetadata_GetTimestamp(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Get([]byte(TimestampPrefix)).Return(bigendian.Uint64ToBytes(100), nil) data := md.GetTimestamp() @@ -1118,7 +392,7 @@ func TestAidaDbMetadata_GetDbType(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - mockDb := db.NewMockBaseDB(ctrl) + mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") mockDb.EXPECT().Get([]byte(TypePrefix)).Return(bigendian.Uint64ToBytes(100), nil) data := md.GetDbType() @@ -1158,19 +432,6 @@ func TestHasStateHashPatch(t *testing.T) { } -func TestProcessMergeMetadata(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - cfg := &Config{} - mockAidaDb := db.NewMockBaseDB(ctrl) - mockSourceDb := db.NewMockBaseDB(ctrl) - mockSourceDb.EXPECT().Get(gomock.Any()).Return([]byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8}, nil).AnyTimes() - out, err := ProcessMergeMetadata(cfg, mockAidaDb, []db.BaseDB{mockSourceDb}, []string{}) - assert.Error(t, err) - assert.Nil(t, out) -} - func Test_FindEpochNumber_IsSkippedForEthereumChainIDs(t *testing.T) { for chainID := range EthereumChainIDs { md := &AidaDbMetadata{ChainId: chainID} diff --git a/utils/test_utils.go b/utils/test_utils.go index 3e2ec0761..d6292765f 100644 --- a/utils/test_utils.go +++ b/utils/test_utils.go @@ -17,7 +17,6 @@ package utils import ( - "encoding/hex" "fmt" "math/big" "strconv" @@ -106,10 +105,11 @@ func CreateTestSubstateDb(t *testing.T, encoding substateDb.SubstateEncodingSche require.NoError(t, err) md := NewAidaDbMetadata(db, "CRITICAL") - dbHash, err := hex.DecodeString("a0d4f7616f3007bf8c02f816a60b2526") - require.NoError(t, err) + // todo might cause failing tests + //dbHash, err := hex.DecodeString("a0d4f7616f3007bf8c02f816a60b2526") + //require.NoError(t, err) - require.NoError(t, md.genMetadata(ss.Block-1, ss.Block+1, 0, 0, SonicMainnetChainID, dbHash)) + require.NoError(t, md.GenerateMetadata(SonicMainnetChainID)) require.NoError(t, db.Close()) From 2f704c6115f2d9bbcaabbfdb2fcbe1afc247fe4d Mon Sep 17 00:00:00 2001 From: cabrador Date: Mon, 8 Sep 2025 06:14:57 +0000 Subject: [PATCH 2/5] Fix md set --- utils/config_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/utils/config_test.go b/utils/config_test.go index ce804044f..bcf2aacda 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -17,6 +17,7 @@ package utils import ( + "errors" "flag" "fmt" "math" @@ -930,13 +931,19 @@ func createFakeAidaDb(cfg *Config) error { ) // open fake aidaDB - testDb, err := db.NewDefaultBaseDB(cfg.AidaDb) + testDb, err := db.NewDefaultSubstateDB(cfg.AidaDb) if err != nil { return fmt.Errorf("cannot open patch db; %v", err) } // create fake metadata - err = ProcessPatchLikeMetadata(testDb, cfg.LogLevel, firstBlock, lastBlock, firstEpoch, lastEpoch, cfg.ChainID, true, nil) + md := NewAidaDbMetadata(testDb, "CRITICAL") + err = errors.Join( + md.SetFirstBlock(firstBlock), + md.SetLastBlock(lastBlock), + md.SetFirstEpoch(firstEpoch), + md.SetLastEpoch(lastEpoch), + ) if err != nil { return fmt.Errorf("cannot create a metadata; %v", err) } From 439813a89c0cbdc321749cdaa3784b6c105ce9d9 Mon Sep 17 00:00:00 2001 From: cabrador Date: Mon, 8 Sep 2025 06:39:25 +0000 Subject: [PATCH 3/5] Fix tests --- utils/config_test.go | 1 + utils/metadata.go | 43 +++++++++++-------- utils/metadata_test.go | 95 +++++++++++++++++++++++++++--------------- 3 files changed, 88 insertions(+), 51 deletions(-) diff --git a/utils/config_test.go b/utils/config_test.go index bcf2aacda..934039a18 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -943,6 +943,7 @@ func createFakeAidaDb(cfg *Config) error { md.SetLastBlock(lastBlock), md.SetFirstEpoch(firstEpoch), md.SetLastEpoch(lastEpoch), + md.SetChainID(cfg.ChainID), ) if err != nil { return fmt.Errorf("cannot create a metadata; %v", err) diff --git a/utils/metadata.go b/utils/metadata.go index 209bdad95..360611bf3 100644 --- a/utils/metadata.go +++ b/utils/metadata.go @@ -139,7 +139,13 @@ func (md *AidaDbMetadata) GetDb() db.SubstateDB { } func (md *AidaDbMetadata) GenerateMetadata(chainId ChainID) error { - if chainId == 0 { + if chainId != 0 { + // todo is this what we want, should not user set chain-id outside this func? + err := md.SetChainID(chainId) + if err != nil { + return err + } + } else { chainId = md.GetChainID() if chainId == 0 { md.log.Warningf("ChainID was nor set neither found in metadata - metadata generation will be incomplete") @@ -506,6 +512,9 @@ func (md *AidaDbMetadata) findEpochs() error { // Finding epoch number calls rpc method eth_getBlockByNumber. // Ethereum does not provide information about epoch number in their RPC interface. if IsEthereumNetwork(md.ChainId) { + zero := uint64(0) + md.FirstEpoch = &zero + md.LastEpoch = &zero return nil } @@ -605,51 +614,51 @@ func (md *AidaDbMetadata) SetBlockRange(firstBlock uint64, lastBlock uint64) err } func (md *AidaDbMetadata) Delete() error { - var err error + var finalErr error - if err = md.Db.Delete([]byte(ChainIDPrefix)); err != nil { - return fmt.Errorf("cannot delete chain-id; %v", err) + if err := md.Db.Delete([]byte(ChainIDPrefix)); err != nil { + finalErr = errors.Join(finalErr, fmt.Errorf("cannot delete chain-id; %v", err)) } else { md.log.Debugf("ChainID deleted successfully") } - if err = md.Db.Delete([]byte(FirstBlockPrefix)); err != nil { - return fmt.Errorf("cannot delete first block; %v", err) + if err := md.Db.Delete([]byte(FirstBlockPrefix)); err != nil { + finalErr = errors.Join(finalErr, fmt.Errorf("cannot delete first block; %v", err)) } else { md.log.Debugf("First block deleted successfully") } - if err = md.Db.Delete([]byte(LastBlockPrefix)); err != nil { - return fmt.Errorf("cannot delete last block; %v", err) + if err := md.Db.Delete([]byte(LastBlockPrefix)); err != nil { + finalErr = errors.Join(finalErr, fmt.Errorf("cannot delete last block; %v", err)) } else { md.log.Debugf("Last block deleted successfully") } - if err = md.Db.Delete([]byte(FirstEpochPrefix)); err != nil { - return fmt.Errorf("cannot delete first epoch; %v", err) + if err := md.Db.Delete([]byte(FirstEpochPrefix)); err != nil { + finalErr = errors.Join(finalErr, fmt.Errorf("cannot delete first epoch; %v", err)) } else { md.log.Debugf("First epoch deleted successfully") } - if err = md.Db.Delete([]byte(LastEpochPrefix)); err != nil { - return fmt.Errorf("cannot delete last epoch; %v", err) + if err := md.Db.Delete([]byte(LastEpochPrefix)); err != nil { + finalErr = errors.Join(finalErr, fmt.Errorf("cannot delete last epoch; %v", err)) } else { md.log.Debugf("Last epoch deleted successfully") } - if err = md.Db.Delete([]byte(TypePrefix)); err != nil { - return fmt.Errorf("cannot delete db type; %v", err) + if err := md.Db.Delete([]byte(TypePrefix)); err != nil { + finalErr = errors.Join(finalErr, fmt.Errorf("cannot delete db type; %v", err)) } else { md.log.Debugf("Timestamp deleted successfully") } - if err = md.Db.Delete([]byte(TimestampPrefix)); err != nil { - return fmt.Errorf("cannot delete creation timestamp; %v", err) + if err := md.Db.Delete([]byte(TimestampPrefix)); err != nil { + finalErr = errors.Join(finalErr, fmt.Errorf("cannot delete creation timestamp; %v", err)) } else { md.log.Debugf("Timestamp deleted successfully") } - return nil + return finalErr } // FindBlockRangeInSubstate if AidaDb does not yet have metadata diff --git a/utils/metadata_test.go b/utils/metadata_test.go index 1a97c933e..0f44427c6 100644 --- a/utils/metadata_test.go +++ b/utils/metadata_test.go @@ -25,6 +25,7 @@ import ( "github.com/Fantom-foundation/lachesis-base/common/bigendian" gethleveldb "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/syndtr/goleveldb/leveldb" "go.uber.org/mock/gomock" ) @@ -130,7 +131,8 @@ func TestAidaDbMetadata_DeleteMetadata(t *testing.T) { mockDb.EXPECT().Delete([]byte(TypePrefix)).Return(nil) mockDb.EXPECT().Delete([]byte(TimestampPrefix)).Return(nil) - md.Delete() // Should not panic or return error + err := md.Delete() + require.NoError(t, err) // Case 2: Some deletes fail - should log errors but not fail mockDb = db.NewMockSubstateDB(ctrl) @@ -145,7 +147,8 @@ func TestAidaDbMetadata_DeleteMetadata(t *testing.T) { mockDb.EXPECT().Delete([]byte(TypePrefix)).Return(mockErr) mockDb.EXPECT().Delete([]byte(TimestampPrefix)).Return(mockErr) - md.Delete() // Should not panic despite errors + err = md.Delete() + require.ErrorContains(t, err, mockErr.Error()) } func TestAidaDbMetadata_AidaDbTyString(t *testing.T) { @@ -274,17 +277,22 @@ func TestAidaDbMetadata_GetFirstBlock(t *testing.T) { mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(bigendian.Uint64ToBytes(100), nil) - firstBlock := md.GetFirstBlock() - assert.Equal(t, uint64(100), firstBlock) mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(nil, errors.New("mock error")) - firstBlock = md.GetFirstBlock() + firstBlock := md.GetFirstBlock() assert.Equal(t, uint64(0), firstBlock) mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(nil, leveldb.ErrNotFound) firstBlock = md.GetFirstBlock() assert.Equal(t, uint64(0), firstBlock) + + mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(bigendian.Uint64ToBytes(100), nil) + firstBlock = md.GetFirstBlock() + assert.Equal(t, uint64(100), firstBlock) + + // cached - no mock call + firstBlock = md.GetFirstBlock() + assert.Equal(t, uint64(100), firstBlock) } func TestAidaDbMetadata_GetLastBlock(t *testing.T) { @@ -293,17 +301,22 @@ func TestAidaDbMetadata_GetLastBlock(t *testing.T) { mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(bigendian.Uint64ToBytes(100), nil) + + mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(nil, leveldb.ErrNotFound) lastBlock := md.GetLastBlock() - assert.Equal(t, uint64(100), lastBlock) + assert.Equal(t, uint64(0), lastBlock) mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(nil, errors.New("mock error")) lastBlock = md.GetLastBlock() assert.Equal(t, uint64(0), lastBlock) - mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(nil, leveldb.ErrNotFound) + mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(bigendian.Uint64ToBytes(100), nil) lastBlock = md.GetLastBlock() - assert.Equal(t, uint64(0), lastBlock) + assert.Equal(t, uint64(100), lastBlock) + + // cached - no mock call + lastBlock = md.GetLastBlock() + assert.Equal(t, uint64(100), lastBlock) } func TestAidaDbMetadata_GetFirstEpoch(t *testing.T) { @@ -312,17 +325,22 @@ func TestAidaDbMetadata_GetFirstEpoch(t *testing.T) { mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get([]byte(FirstEpochPrefix)).Return(bigendian.Uint64ToBytes(100), nil) - data := md.GetFirstEpoch() - assert.Equal(t, uint64(100), data) mockDb.EXPECT().Get([]byte(FirstEpochPrefix)).Return(nil, errors.New("mock error")) - data = md.GetFirstEpoch() + data := md.GetFirstEpoch() assert.Equal(t, uint64(0), data) mockDb.EXPECT().Get([]byte(FirstEpochPrefix)).Return(nil, leveldb.ErrNotFound) data = md.GetFirstEpoch() assert.Equal(t, uint64(0), data) + + mockDb.EXPECT().Get([]byte(FirstEpochPrefix)).Return(bigendian.Uint64ToBytes(100), nil) + data = md.GetFirstEpoch() + assert.Equal(t, uint64(100), data) + + // cached - no mock call + data = md.GetFirstEpoch() + assert.Equal(t, uint64(100), data) } func TestAidaDbMetadata_GetLastEpoch(t *testing.T) { @@ -331,17 +349,22 @@ func TestAidaDbMetadata_GetLastEpoch(t *testing.T) { mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get([]byte(LastEpochPrefix)).Return(bigendian.Uint64ToBytes(100), nil) - data := md.GetLastEpoch() - assert.Equal(t, uint64(100), data) mockDb.EXPECT().Get([]byte(LastEpochPrefix)).Return(nil, errors.New("mock error")) - data = md.GetLastEpoch() + data := md.GetLastEpoch() assert.Equal(t, uint64(0), data) mockDb.EXPECT().Get([]byte(LastEpochPrefix)).Return(nil, leveldb.ErrNotFound) data = md.GetLastEpoch() assert.Equal(t, uint64(0), data) + + mockDb.EXPECT().Get([]byte(LastEpochPrefix)).Return(bigendian.Uint64ToBytes(100), nil) + data = md.GetLastEpoch() + assert.Equal(t, uint64(100), data) + + // cached - no mock call + data = md.GetLastEpoch() + assert.Equal(t, uint64(100), data) } func TestAidaDbMetadata_GetChainID(t *testing.T) { @@ -350,23 +373,22 @@ func TestAidaDbMetadata_GetChainID(t *testing.T) { mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(bigendian.Uint64ToBytes(100), nil) - data := md.GetChainID() - assert.Equal(t, ChainID(100), data) - - mockDb = db.NewMockSubstateDB(ctrl) - md = NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return([]byte{0, 1}, nil) - data = md.GetChainID() - assert.Equal(t, ChainID(1), data) mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(nil, errors.New("mock error")) - data = md.GetChainID() + data := md.GetChainID() assert.Equal(t, ChainID(0), data) mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(nil, leveldb.ErrNotFound) data = md.GetChainID() assert.Equal(t, ChainID(0), data) + + mockDb.EXPECT().Get([]byte(ChainIDPrefix)).Return(bigendian.Uint64ToBytes(100), nil) + data = md.GetChainID() + assert.Equal(t, ChainID(100), data) + + // cached - no mock call + data = md.GetChainID() + assert.Equal(t, ChainID(100), data) } func TestAidaDbMetadata_GetTimestamp(t *testing.T) { @@ -375,17 +397,22 @@ func TestAidaDbMetadata_GetTimestamp(t *testing.T) { mockDb := db.NewMockSubstateDB(ctrl) md := NewAidaDbMetadata(mockDb, "ERROR") - mockDb.EXPECT().Get([]byte(TimestampPrefix)).Return(bigendian.Uint64ToBytes(100), nil) - data := md.GetTimestamp() - assert.Equal(t, uint64(100), data) mockDb.EXPECT().Get([]byte(TimestampPrefix)).Return(nil, errors.New("mock error")) - data = md.GetTimestamp() + data := md.GetTimestamp() assert.Equal(t, uint64(0), data) mockDb.EXPECT().Get([]byte(TimestampPrefix)).Return(nil, leveldb.ErrNotFound) data = md.GetTimestamp() assert.Equal(t, uint64(0), data) + + mockDb.EXPECT().Get([]byte(TimestampPrefix)).Return(bigendian.Uint64ToBytes(100), nil) + data = md.GetTimestamp() + assert.Equal(t, uint64(100), data) + + // cached - no mock call + data = md.GetTimestamp() + assert.Equal(t, uint64(100), data) } func TestAidaDbMetadata_GetDbType(t *testing.T) { @@ -437,7 +464,7 @@ func Test_FindEpochNumber_IsSkippedForEthereumChainIDs(t *testing.T) { md := &AidaDbMetadata{ChainId: chainID} assert.NoError(t, md.findEpochs()) // Epochs must be unchange - assert.Equal(t, md.FirstEpoch, uint64(0)) - assert.Equal(t, md.LastEpoch, uint64(0)) + assert.Equal(t, md.GetFirstEpoch(), uint64(0)) + assert.Equal(t, md.GetLastEpoch(), uint64(0)) } } From eb2b8ced40db25270d18150a683cc40fb3554d70 Mon Sep 17 00:00:00 2001 From: cabrador Date: Mon, 8 Sep 2025 10:23:36 +0000 Subject: [PATCH 4/5] Fix tests --- cmd/util-db/clone/clone_test.go | 4 +- cmd/util-db/generate/generate_test.go | 2 +- cmd/util-db/info/info_test.go | 17 +- cmd/util-db/merge/merge.go | 7 +- cmd/util-db/merge/merge_test.go | 23 +- cmd/util-db/update/update.go | 19 +- cmd/util-db/update/update_test.go | 24 +- cmd/util-db/validate/validate.go | 2 +- cmd/util-db/validate/validate_test.go | 10 +- utildb/merger.go | 5 +- utils/config.go | 15 +- utils/metadata.go | 85 ++++-- utils/metadata_mock.go | 405 ++++++++++++++++++++++++++ utils/metadata_test.go | 71 ++--- utils/rpc.go | 22 +- utils/rpc_test.go | 12 - utils/test_utils.go | 10 +- 17 files changed, 575 insertions(+), 158 deletions(-) create mode 100644 utils/metadata_mock.go diff --git a/cmd/util-db/clone/clone_test.go b/cmd/util-db/clone/clone_test.go index aa6694df6..375824ce7 100644 --- a/cmd/util-db/clone/clone_test.go +++ b/cmd/util-db/clone/clone_test.go @@ -47,8 +47,8 @@ func TestClone(t *testing.T) { dbc string wantErr string }{ - {"NoType", utils.NoType, "", "clone failed for NoType: incorrect clone type: 0"}, - {"GenType", utils.GenType, "", "clone failed for GenType: incorrect clone type: 1"}, + {"NoType", utils.NoType, "", "clone failed for NoType: incorrect clone type: NoType"}, + {"GenType", utils.GenType, "", "clone failed for GenType: incorrect clone type: Generate"}, {"PatchType", utils.PatchType, "", ""}, {"CloneType", utils.CloneType, "", ""}, {"CustomTypeAll", utils.CustomType, "all", ""}, diff --git a/cmd/util-db/generate/generate_test.go b/cmd/util-db/generate/generate_test.go index 6b662c1ea..178cc9286 100644 --- a/cmd/util-db/generate/generate_test.go +++ b/cmd/util-db/generate/generate_test.go @@ -85,7 +85,7 @@ func TestGenerateDbHash_Command(t *testing.T) { err := app.Run(argsBuilder.Build()) require.NoError(t, err) - aidaDb, err := db.NewDefaultBaseDB(path) + aidaDb, err := db.NewDefaultSubstateDB(path) require.NoError(t, err) md := utils.NewAidaDbMetadata(aidaDb, "CRITICAL") got := md.GetDbHash() diff --git a/cmd/util-db/info/info_test.go b/cmd/util-db/info/info_test.go index dc3bf7aca..0b376e83b 100644 --- a/cmd/util-db/info/info_test.go +++ b/cmd/util-db/info/info_test.go @@ -48,20 +48,7 @@ func TestInfo_PrintCount(t *testing.T) { wantErr string } - aidaDbPath := t.TempDir() + "aida-db" - - aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath) - if err != nil { - t.Fatal(err) - } - if aidaDb == nil { - t.Fatal("aidaDb is nil") - } - err = aidaDb.Close() - if err != nil { - t.Fatal(err) - } - + _, aidaDbPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema) tests := []testCase{ { name: "IntegrationTest", @@ -69,7 +56,7 @@ func TestInfo_PrintCount(t *testing.T) { "info", "count", "--aida-db", aidaDbPath, "--db-component=all", - "1", "2", + "37534834", "37534834", }, wantErr: "", }, diff --git a/cmd/util-db/merge/merge.go b/cmd/util-db/merge/merge.go index 945c9fbc5..a35fab340 100644 --- a/cmd/util-db/merge/merge.go +++ b/cmd/util-db/merge/merge.go @@ -63,10 +63,7 @@ func mergeAction(ctx *cli.Context) error { return fmt.Errorf("cannot open db; %v", err) } - var ( - dbs []db.SubstateDB - md *utils.AidaDbMetadata - ) + var dbs []db.SubstateDB if !cfg.SkipMetadata { dbs, err = utildb.OpenSourceDatabases(sourcePaths) @@ -91,7 +88,7 @@ func mergeAction(ctx *cli.Context) error { return err } - m := utildb.NewMerger(cfg, targetDb, dbs, sourcePaths, md) + m := utildb.NewMerger(cfg, targetDb, dbs, sourcePaths) if err = m.Merge(); err != nil { return err } diff --git a/cmd/util-db/merge/merge_test.go b/cmd/util-db/merge/merge_test.go index bd4296d8b..1e79bda5e 100644 --- a/cmd/util-db/merge/merge_test.go +++ b/cmd/util-db/merge/merge_test.go @@ -27,14 +27,20 @@ import ( ) func TestMerge_Command(t *testing.T) { + ss, aidaDbPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema) + path1 := t.TempDir() + "/sdb1" sdb1, err := db.NewDefaultSubstateDB(path1) require.NoError(t, err) s1 := utils.GetTestSubstate("pb") - s1.Block = 10 - s1.Transaction = 2 + s1.Block = ss.Block + 1 + s1.Transaction = 1 err = sdb1.PutSubstate(s1) require.NoError(t, err) + + md := utils.NewAidaDbMetadata(sdb1, "CRITICAL") + require.NoError(t, md.SetChainID(utils.SonicMainnetChainID)) + err = sdb1.Close() require.NoError(t, err) @@ -42,14 +48,17 @@ func TestMerge_Command(t *testing.T) { sdb2, err := db.NewDefaultSubstateDB(path2) require.NoError(t, err) s2 := utils.GetTestSubstate("pb") - s2.Block = 20 - s2.Transaction = 3 + s2.Block = ss.Block + 2 + s2.Transaction = 2 err = sdb2.PutSubstate(s2) require.NoError(t, err) + + md = utils.NewAidaDbMetadata(sdb2, "CRITICAL") + require.NoError(t, md.SetChainID(utils.SonicMainnetChainID)) + err = sdb2.Close() require.NoError(t, err) - _, aidaDbPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema) app := cli.NewApp() app.Action = mergeAction app.Flags = Command.Flags @@ -69,11 +78,11 @@ func TestMerge_Command(t *testing.T) { aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath) require.NoError(t, err) - gotS1, err := aidaDb.GetSubstate(10, 2) + gotS1, err := aidaDb.GetSubstate(s1.Block, s1.Transaction) require.NoError(t, err) require.NoError(t, gotS1.Equal(s1)) - gotS2, err := aidaDb.GetSubstate(20, 3) + gotS2, err := aidaDb.GetSubstate(s2.Block, s2.Transaction) require.NoError(t, err) require.NoError(t, gotS2.Equal(s2)) } diff --git a/cmd/util-db/update/update.go b/cmd/util-db/update/update.go index 45b21bced..1868c7b0b 100644 --- a/cmd/util-db/update/update.go +++ b/cmd/util-db/update/update.go @@ -82,13 +82,17 @@ func updateAction(ctx *cli.Context) error { } // update implements updating command to be called from various commands and automatically downloads aida-db patches. -func update(cfg *utils.Config) error { +func update(cfg *utils.Config) (finalErr error) { log := logger.NewLogger(cfg.LogLevel, "DB Update") - targetDB, err := db.NewReadOnlySubstateDB(cfg.AidaDb) + targetDB, err := db.NewDefaultSubstateDB(cfg.AidaDb) if err != nil { return err } + defer func() { + finalErr = errors.Join(err, targetDB.Close()) + }() + md := utils.NewAidaDbMetadata(targetDB, cfg.LogLevel) err = md.GenerateMetadata(cfg.ChainID) if err != nil { @@ -233,7 +237,7 @@ func mergePatch(cfg *utils.Config, decompressChan chan string, errChan chan erro func mergeToExistingAidaDb(cfg *utils.Config, md utils.Metadata, extractedPatchPath string) error { // merge newly extracted patch - patchDb, err := db.NewReadOnlySubstateDB(extractedPatchPath) + patchDb, err := db.NewDefaultSubstateDB(extractedPatchPath) if err != nil { return fmt.Errorf("cannot open patchDb; %v", err) } @@ -250,7 +254,7 @@ func mergeToExistingAidaDb(cfg *utils.Config, md utils.Metadata, extractedPatchP } } - m := utildb.NewMerger(cfg, md.GetDb(), []db.SubstateDB{patchDb}, []string{extractedPatchPath}, nil) + m := utildb.NewMerger(cfg, md.GetDb(), []db.SubstateDB{patchDb}, []string{extractedPatchPath}) err = m.Merge() if err != nil { @@ -419,17 +423,12 @@ func retrievePatchesToDownload(cfg *utils.Config, md utils.Metadata) ([]utils.Pa return nil, fmt.Errorf("unable to download patches.json: %v", err) } - hasStateHashPatch, err := utils.HasStateHashPatch(cfg.AidaDb) - if err != nil { - return nil, err - } - // list of availablePatches to be downloaded var patchesToDownload = make([]utils.PatchJson, 0) for _, patch := range availablePatches { if patch.FileName == stateHashPatchFileName+".tar.gz" { - if !hasStateHashPatch { + if !md.HasHashPatch() { patchesToDownload = append(patchesToDownload, patch) } } diff --git a/cmd/util-db/update/update_test.go b/cmd/util-db/update/update_test.go index c4f8b6159..fd9c492dc 100644 --- a/cmd/util-db/update/update_test.go +++ b/cmd/util-db/update/update_test.go @@ -18,6 +18,7 @@ package update import ( "context" + "errors" "math" "math/big" "os" @@ -34,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/holiman/uint256" "github.com/stretchr/testify/assert" + "go.uber.org/mock/gomock" "github.com/0xsoniclabs/aida/utils" "github.com/0xsoniclabs/substate/db" @@ -48,8 +50,8 @@ func TestUpdate_UpdateDbCommand(t *testing.T) { // Put substate with max latest block to avoid any updating ss := utils.GetTestSubstate("pb") - ss.Block = math.MaxUint64 - ss.Env.Number = math.MaxUint64 + ss.Block = math.MaxInt64 + ss.Env.Number = math.MaxInt64 err = aidaDb.PutSubstate(ss) require.NoError(t, err) @@ -237,6 +239,10 @@ func TestUpdate_pushToChanel(t *testing.T) { } func TestUpdate_retrievePatchesToDownload(t *testing.T) { + ctrl := gomock.NewController(t) + md := utils.NewMockMetadata(ctrl) + md.EXPECT().GetLastBlock().Return(uint64(100)).Times(2) + utils.AidaDbRepositoryUrl = utils.AidaDbRepositorySonicUrl defer func() { utils.AidaDbRepositoryUrl = "" @@ -244,7 +250,7 @@ func TestUpdate_retrievePatchesToDownload(t *testing.T) { patches, err := retrievePatchesToDownload(&utils.Config{ ChainID: utils.SonicMainnetChainID, UpdateType: "nightly", - }, 28_000_000) + }, md) require.NoError(t, err) require.NotEmpty(t, patches) } @@ -316,7 +322,7 @@ func TestUpdate_mergeToExistingAidaDb_ClassicPatch(t *testing.T) { require.NoError(t, err) // Set correct metadata block range targetMD := utils.NewAidaDbMetadata(targetDb, "CRITICAL") - err = targetMD.SetBlockRange(0, want.Block-1) + err = errors.Join(targetMD.SetFirstBlock(0), targetMD.SetLastBlock(want.Block-1)) require.NoError(t, err) err = targetMD.SetChainID(utils.SonicMainnetChainID) require.NoError(t, err) @@ -346,7 +352,7 @@ func TestUpdate_mergeToExistingAidaDb_StateHashPatch(t *testing.T) { require.NoError(t, err) // Create target db - targetDb, err := db.NewDefaultBaseDB(targetDbPath) + targetDb, err := db.NewDefaultSubstateDB(targetDbPath) require.NoError(t, err) // Set correct metadata block range @@ -371,11 +377,11 @@ func TestUpdate_mergeToExistingAidaDb_StateHashPatch(t *testing.T) { func TestUpdate_mergeToExistingAidaDb_BlocksDoesNotAlign(t *testing.T) { want, patchPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema) _, targetPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema) - targetDb, err := db.NewDefaultBaseDB(targetPath) + targetDb, err := db.NewDefaultSubstateDB(targetPath) require.NoError(t, err) targetMD := utils.NewAidaDbMetadata(targetDb, "CRITICAL") // set wrong block range to target db - err = targetMD.SetBlockRange(0, want.Block-1000) + err = errors.Join(targetMD.SetFirstBlock(0), targetMD.SetLastBlock(want.Block-1000)) require.NoError(t, err) err = targetMD.SetChainID(utils.SonicMainnetChainID) require.NoError(t, err) @@ -385,13 +391,13 @@ func TestUpdate_mergeToExistingAidaDb_BlocksDoesNotAlign(t *testing.T) { LogLevel: "CRITICAL", } err = mergeToExistingAidaDb(cfg, targetMD, patchPath) - require.ErrorContains(t, err, "metadata blocks does not align") + require.ErrorContains(t, err, "cannot merge dbs with gap") } func TestUpdate_retrievePatchesToDownload_MustChooseUpdateType(t *testing.T) { _, err := retrievePatchesToDownload(&utils.Config{ UpdateType: "", // empty - }, 0) + }, nil) require.ErrorContains(t, err, "please choose correct data-type") } diff --git a/cmd/util-db/validate/validate.go b/cmd/util-db/validate/validate.go index 8e8ccf0ac..2590d2db2 100644 --- a/cmd/util-db/validate/validate.go +++ b/cmd/util-db/validate/validate.go @@ -47,7 +47,7 @@ func validateAction(ctx *cli.Context) error { return fmt.Errorf("cannot parse config; %v", err) } - aidaDb, err := db.NewReadOnlySubstateDB(cfg.AidaDb) + aidaDb, err := db.NewDefaultSubstateDB(cfg.AidaDb) if err != nil { return fmt.Errorf("cannot open db; %v", err) } diff --git a/cmd/util-db/validate/validate_test.go b/cmd/util-db/validate/validate_test.go index 447ed378a..f150c5ebb 100644 --- a/cmd/util-db/validate/validate_test.go +++ b/cmd/util-db/validate/validate_test.go @@ -67,7 +67,7 @@ func TestCmd_ValidateCommandError(t *testing.T) { Arg(Command.Name), wantErr: fmt.Sprintf("your db type (%v) cannot be validated", utils.NoType), setup: func(aidaDbPath string) { - aidaDb, err := db.NewDefaultBaseDB(aidaDbPath) + aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath) require.NoError(t, err) md := utils.NewAidaDbMetadata(aidaDb, "CRITICAL") err = md.SetDbType(utils.NoType) @@ -82,9 +82,11 @@ func TestCmd_ValidateCommandError(t *testing.T) { Arg(Command.Name), wantErr: "could not find expected db hash", setup: func(aidaDbPath string) { - aidaDb, err := db.NewDefaultBaseDB(aidaDbPath) + aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath) require.NoError(t, err) md := utils.NewAidaDbMetadata(aidaDb, "CRITICAL") + err = md.SetDbType(utils.GenType) + require.NoError(t, err) err = md.SetDbHash([]byte{}) require.NoError(t, err) err = aidaDb.Close() @@ -97,9 +99,11 @@ func TestCmd_ValidateCommandError(t *testing.T) { Arg(Command.Name), wantErr: "hashes are different", setup: func(aidaDbPath string) { - aidaDb, err := db.NewDefaultBaseDB(aidaDbPath) + aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath) require.NoError(t, err) md := utils.NewAidaDbMetadata(aidaDb, "CRITICAL") + err = md.SetDbType(utils.GenType) + require.NoError(t, err) err = md.SetDbHash([]byte("wrong-hash")) require.NoError(t, err) err = aidaDb.Close() diff --git a/utildb/merger.go b/utildb/merger.go index a6cf2574e..4811417bc 100644 --- a/utildb/merger.go +++ b/utildb/merger.go @@ -33,19 +33,17 @@ type Merger struct { targetDb db.SubstateDB sourceDbs []db.SubstateDB sourceDbPaths []string - md *utils.AidaDbMetadata start time.Time } // NewMerger returns new instance of Merger -func NewMerger(cfg *utils.Config, targetDb db.SubstateDB, sourceDbs []db.SubstateDB, sourceDbPaths []string, md *utils.AidaDbMetadata) *Merger { +func NewMerger(cfg *utils.Config, targetDb db.SubstateDB, sourceDbs []db.SubstateDB, sourceDbPaths []string) *Merger { return &Merger{ cfg: cfg, log: logger.NewLogger(cfg.LogLevel, "aida-db-Merger"), targetDb: targetDb, sourceDbs: sourceDbs, sourceDbPaths: sourceDbPaths, - md: md, start: time.Now(), } } @@ -54,7 +52,6 @@ func NewMerger(cfg *utils.Config, targetDb db.SubstateDB, sourceDbs []db.Substat func (m *Merger) FinishMerge() error { if !m.cfg.SkipMetadata { // merge type db does not have epoch calculations yet - m.md.Db = m.targetDb MustCloseDB(m.targetDb) err := PrintMetadata(m.targetDb) diff --git a/utils/config.go b/utils/config.go index dc745d1a9..ed4e22a94 100644 --- a/utils/config.go +++ b/utils/config.go @@ -770,14 +770,17 @@ func (cc *configContext) setChainId() error { if cc.cfg.ChainID == UnknownChainID { cc.log.Warningf("ChainID (--%v) was not set; looking for it in AidaDb", ChainIDFlag.Name) - // we check if AidaDb was set with err == nil - if aidaDb, err := db.NewDefaultSubstateDB(cc.cfg.AidaDb); err == nil { - md := NewAidaDbMetadata(aidaDb, cc.cfg.LogLevel) + // only open aidaDb if it exists + if _, err := os.Stat(cc.cfg.AidaDb); err == nil { + // we check if AidaDb was set with err == nil + if aidaDb, err := db.NewDefaultSubstateDB(cc.cfg.AidaDb); err == nil { + md := NewAidaDbMetadata(aidaDb, cc.cfg.LogLevel) - cc.cfg.ChainID = md.GetChainID() + cc.cfg.ChainID = md.GetChainID() - if err = aidaDb.Close(); err != nil { - return fmt.Errorf("cannot close db; %v", err) + if err = aidaDb.Close(); err != nil { + return fmt.Errorf("cannot close db; %v", err) + } } } diff --git a/utils/metadata.go b/utils/metadata.go index 360611bf3..d6abece3d 100644 --- a/utils/metadata.go +++ b/utils/metadata.go @@ -23,13 +23,14 @@ import ( "fmt" "io" "net/http" - "os" + "strconv" + "strings" "time" "github.com/0xsoniclabs/aida/logger" "github.com/0xsoniclabs/substate/db" + "github.com/0xsoniclabs/substate/substate" "github.com/Fantom-foundation/lachesis-base/common/bigendian" - geth_leveldb "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/syndtr/goleveldb/leveldb" ) @@ -86,6 +87,8 @@ type PatchJson struct { Nightly bool } +//go:generate mockgen -source metadata.go -destination metadata_mock.go -package utils + type Metadata interface { // GenerateMetadata generates new or updates metadata in AidaDb. GenerateMetadata(chainID ChainID) error @@ -105,6 +108,7 @@ type Metadata interface { GetDbHash() []byte GetUpdatesetInterval() uint64 GetUpdatesetSize() uint64 + HasHashPatch() bool // Setters SetFirstBlock(uint64) error @@ -134,6 +138,18 @@ type AidaDbMetadata struct { dbHash []byte } +func (md *AidaDbMetadata) HasHashPatch() bool { + _, err := md.Db.Get([]byte(HasStateHashPatchPrefix)) + if err != nil { + if !errors.Is(err, leveldb.ErrNotFound) { + md.log.Criticalf("error getting 'HasHashPatch' from db metadata: %v", err) + } + return false + } + + return true +} + func (md *AidaDbMetadata) GetDb() db.SubstateDB { return md.Db } @@ -245,6 +261,14 @@ func (md *AidaDbMetadata) Merge(src Metadata) error { return fmt.Errorf("target db (%v-%v) is subset of source db (%v-%v)", targetFirstBlock, targetLastBlock, srcFirstBlock, srcLastBlock) } + if srcFirstBlock < targetFirstBlock && srcLastBlock+1 < targetFirstBlock { + return fmt.Errorf("cannot merge dbs with gap; target db (%v-%v), source db (%v-%v)", targetFirstBlock, targetLastBlock, srcFirstBlock, srcLastBlock) + } + + if targetFirstBlock < srcFirstBlock && targetLastBlock+1 < srcFirstBlock { + return fmt.Errorf("cannot merge dbs with gap; target db (%v-%v), source db (%v-%v)", targetFirstBlock, targetLastBlock, srcFirstBlock, srcLastBlock) + } + blocksOk := false // Check alignment - dbs can overlap but cannot have gaps // Target is before source @@ -279,7 +303,12 @@ func (md *AidaDbMetadata) GetFirstBlock() uint64 { firstBlockBytes, err := md.Db.Get([]byte(FirstBlockPrefix)) if err != nil { if errors.Is(err, leveldb.ErrNotFound) { - return 0 + fs := md.Db.GetFirstSubstate() + if fs != nil { + blk := fs.Block + md.FirstBlock = &blk + return blk + } } md.log.Criticalf("cannot get first block from metadata; %v", err) return 0 @@ -298,7 +327,13 @@ func (md *AidaDbMetadata) GetLastBlock() uint64 { lastBlockBytes, err := md.Db.Get([]byte(LastBlockPrefix)) if err != nil { if errors.Is(err, leveldb.ErrNotFound) { - return 0 + var ls *substate.Substate + ls, err = md.Db.GetLastSubstate() + if ls != nil { + blk := ls.Block + md.LastBlock = &blk + return blk + } } md.log.Criticalf("cannot get last block from metadata; %v", err) return 0 @@ -564,6 +599,20 @@ func (md *AidaDbMetadata) findEpochs() error { return nil } +// FindEpochNumber via RPC request GetBlockByNumber +func FindEpochNumber(blockNumber uint64, chainId ChainID) (uint64, error) { + hex := strconv.FormatUint(blockNumber, 16) + + blockStr := "0x" + hex + + num, err := getEpochByNumber(blockStr, chainId) + if err != nil && strings.Contains(err.Error(), "rpc does not know this block") { + // block is unknown + return 0, nil + } + return num, err +} + // SetFreshMetadata for an existing AidaDb without metadata func (md *AidaDbMetadata) SetFreshMetadata(chainID ChainID) error { var err error @@ -729,34 +778,6 @@ func getPatchFirstBlock(lastPatchBlock uint64) (uint64, error) { } -// HasStateHashPatch checks whether given db has already acquired patch with StateHashes. -func HasStateHashPatch(path string) (bool, error) { - db, err := geth_leveldb.New(path, 1024, 100, "profiling", true) - if err != nil { - // if AidaDb does not exist force downloading the state hash patch - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - return false, fmt.Errorf("cannot open aida-db to check if it already has state hash patch; %v", err) - } - - _, getErr := db.Get([]byte(HasStateHashPatchPrefix)) - - err = db.Close() - if err != nil { - return false, fmt.Errorf("cannot close aida-db after checking if it already has state hash patch; %v", err) - } - - if getErr != nil { - if errors.Is(getErr, leveldb.ErrNotFound) { - return false, nil - } - return false, err - } - - return true, nil -} - // SetHasHashPatch marks AidaDb that it already has HashPatch merged so it will not get downloaded next update. func (md *AidaDbMetadata) SetHasHashPatch() error { return md.Db.Put([]byte(HasStateHashPatchPrefix), []byte{1}) diff --git a/utils/metadata_mock.go b/utils/metadata_mock.go new file mode 100644 index 000000000..794629e21 --- /dev/null +++ b/utils/metadata_mock.go @@ -0,0 +1,405 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: metadata.go +// +// Generated by this command: +// +// mockgen -source metadata.go -destination metadata_mock.go -package utils +// + +// Package utils is a generated GoMock package. +package utils + +import ( + reflect "reflect" + + db "github.com/0xsoniclabs/substate/db" + gomock "go.uber.org/mock/gomock" +) + +// MockMetadata is a mock of Metadata interface. +type MockMetadata struct { + ctrl *gomock.Controller + recorder *MockMetadataMockRecorder + isgomock struct{} +} + +// MockMetadataMockRecorder is the mock recorder for MockMetadata. +type MockMetadataMockRecorder struct { + mock *MockMetadata +} + +// NewMockMetadata creates a new mock instance. +func NewMockMetadata(ctrl *gomock.Controller) *MockMetadata { + mock := &MockMetadata{ctrl: ctrl} + mock.recorder = &MockMetadataMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMetadata) EXPECT() *MockMetadataMockRecorder { + return m.recorder +} + +// Delete mocks base method. +func (m *MockMetadata) Delete() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete") + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockMetadataMockRecorder) Delete() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockMetadata)(nil).Delete)) +} + +// GenerateMetadata mocks base method. +func (m *MockMetadata) GenerateMetadata(chainID ChainID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateMetadata", chainID) + ret0, _ := ret[0].(error) + return ret0 +} + +// GenerateMetadata indicates an expected call of GenerateMetadata. +func (mr *MockMetadataMockRecorder) GenerateMetadata(chainID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateMetadata", reflect.TypeOf((*MockMetadata)(nil).GenerateMetadata), chainID) +} + +// GetChainID mocks base method. +func (m *MockMetadata) GetChainID() ChainID { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetChainID") + ret0, _ := ret[0].(ChainID) + return ret0 +} + +// GetChainID indicates an expected call of GetChainID. +func (mr *MockMetadataMockRecorder) GetChainID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChainID", reflect.TypeOf((*MockMetadata)(nil).GetChainID)) +} + +// GetDb mocks base method. +func (m *MockMetadata) GetDb() db.SubstateDB { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDb") + ret0, _ := ret[0].(db.SubstateDB) + return ret0 +} + +// GetDb indicates an expected call of GetDb. +func (mr *MockMetadataMockRecorder) GetDb() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDb", reflect.TypeOf((*MockMetadata)(nil).GetDb)) +} + +// GetDbHash mocks base method. +func (m *MockMetadata) GetDbHash() []byte { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDbHash") + ret0, _ := ret[0].([]byte) + return ret0 +} + +// GetDbHash indicates an expected call of GetDbHash. +func (mr *MockMetadataMockRecorder) GetDbHash() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDbHash", reflect.TypeOf((*MockMetadata)(nil).GetDbHash)) +} + +// GetDbType mocks base method. +func (m *MockMetadata) GetDbType() AidaDbType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDbType") + ret0, _ := ret[0].(AidaDbType) + return ret0 +} + +// GetDbType indicates an expected call of GetDbType. +func (mr *MockMetadataMockRecorder) GetDbType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDbType", reflect.TypeOf((*MockMetadata)(nil).GetDbType)) +} + +// GetFirstBlock mocks base method. +func (m *MockMetadata) GetFirstBlock() uint64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFirstBlock") + ret0, _ := ret[0].(uint64) + return ret0 +} + +// GetFirstBlock indicates an expected call of GetFirstBlock. +func (mr *MockMetadataMockRecorder) GetFirstBlock() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFirstBlock", reflect.TypeOf((*MockMetadata)(nil).GetFirstBlock)) +} + +// GetFirstEpoch mocks base method. +func (m *MockMetadata) GetFirstEpoch() uint64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFirstEpoch") + ret0, _ := ret[0].(uint64) + return ret0 +} + +// GetFirstEpoch indicates an expected call of GetFirstEpoch. +func (mr *MockMetadataMockRecorder) GetFirstEpoch() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFirstEpoch", reflect.TypeOf((*MockMetadata)(nil).GetFirstEpoch)) +} + +// GetLastBlock mocks base method. +func (m *MockMetadata) GetLastBlock() uint64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLastBlock") + ret0, _ := ret[0].(uint64) + return ret0 +} + +// GetLastBlock indicates an expected call of GetLastBlock. +func (mr *MockMetadataMockRecorder) GetLastBlock() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastBlock", reflect.TypeOf((*MockMetadata)(nil).GetLastBlock)) +} + +// GetLastEpoch mocks base method. +func (m *MockMetadata) GetLastEpoch() uint64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLastEpoch") + ret0, _ := ret[0].(uint64) + return ret0 +} + +// GetLastEpoch indicates an expected call of GetLastEpoch. +func (mr *MockMetadataMockRecorder) GetLastEpoch() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastEpoch", reflect.TypeOf((*MockMetadata)(nil).GetLastEpoch)) +} + +// GetTimestamp mocks base method. +func (m *MockMetadata) GetTimestamp() uint64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTimestamp") + ret0, _ := ret[0].(uint64) + return ret0 +} + +// GetTimestamp indicates an expected call of GetTimestamp. +func (mr *MockMetadataMockRecorder) GetTimestamp() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTimestamp", reflect.TypeOf((*MockMetadata)(nil).GetTimestamp)) +} + +// GetUpdatesetInterval mocks base method. +func (m *MockMetadata) GetUpdatesetInterval() uint64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUpdatesetInterval") + ret0, _ := ret[0].(uint64) + return ret0 +} + +// GetUpdatesetInterval indicates an expected call of GetUpdatesetInterval. +func (mr *MockMetadataMockRecorder) GetUpdatesetInterval() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUpdatesetInterval", reflect.TypeOf((*MockMetadata)(nil).GetUpdatesetInterval)) +} + +// GetUpdatesetSize mocks base method. +func (m *MockMetadata) GetUpdatesetSize() uint64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUpdatesetSize") + ret0, _ := ret[0].(uint64) + return ret0 +} + +// GetUpdatesetSize indicates an expected call of GetUpdatesetSize. +func (mr *MockMetadataMockRecorder) GetUpdatesetSize() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUpdatesetSize", reflect.TypeOf((*MockMetadata)(nil).GetUpdatesetSize)) +} + +// HasHashPatch mocks base method. +func (m *MockMetadata) HasHashPatch() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HasHashPatch") + ret0, _ := ret[0].(bool) + return ret0 +} + +// HasHashPatch indicates an expected call of HasHashPatch. +func (mr *MockMetadataMockRecorder) HasHashPatch() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasHashPatch", reflect.TypeOf((*MockMetadata)(nil).HasHashPatch)) +} + +// Merge mocks base method. +func (m *MockMetadata) Merge(arg0 Metadata) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Merge", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Merge indicates an expected call of Merge. +func (mr *MockMetadataMockRecorder) Merge(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Merge", reflect.TypeOf((*MockMetadata)(nil).Merge), arg0) +} + +// SetChainID mocks base method. +func (m *MockMetadata) SetChainID(arg0 ChainID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetChainID", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetChainID indicates an expected call of SetChainID. +func (mr *MockMetadataMockRecorder) SetChainID(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetChainID", reflect.TypeOf((*MockMetadata)(nil).SetChainID), arg0) +} + +// SetDbHash mocks base method. +func (m *MockMetadata) SetDbHash(arg0 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetDbHash", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetDbHash indicates an expected call of SetDbHash. +func (mr *MockMetadataMockRecorder) SetDbHash(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDbHash", reflect.TypeOf((*MockMetadata)(nil).SetDbHash), arg0) +} + +// SetDbType mocks base method. +func (m *MockMetadata) SetDbType(arg0 AidaDbType) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetDbType", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetDbType indicates an expected call of SetDbType. +func (mr *MockMetadataMockRecorder) SetDbType(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetDbType", reflect.TypeOf((*MockMetadata)(nil).SetDbType), arg0) +} + +// SetFirstBlock mocks base method. +func (m *MockMetadata) SetFirstBlock(arg0 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetFirstBlock", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetFirstBlock indicates an expected call of SetFirstBlock. +func (mr *MockMetadataMockRecorder) SetFirstBlock(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetFirstBlock", reflect.TypeOf((*MockMetadata)(nil).SetFirstBlock), arg0) +} + +// SetFirstEpoch mocks base method. +func (m *MockMetadata) SetFirstEpoch(arg0 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetFirstEpoch", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetFirstEpoch indicates an expected call of SetFirstEpoch. +func (mr *MockMetadataMockRecorder) SetFirstEpoch(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetFirstEpoch", reflect.TypeOf((*MockMetadata)(nil).SetFirstEpoch), arg0) +} + +// SetHasHashPatch mocks base method. +func (m *MockMetadata) SetHasHashPatch() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetHasHashPatch") + ret0, _ := ret[0].(error) + return ret0 +} + +// SetHasHashPatch indicates an expected call of SetHasHashPatch. +func (mr *MockMetadataMockRecorder) SetHasHashPatch() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetHasHashPatch", reflect.TypeOf((*MockMetadata)(nil).SetHasHashPatch)) +} + +// SetLastBlock mocks base method. +func (m *MockMetadata) SetLastBlock(arg0 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetLastBlock", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetLastBlock indicates an expected call of SetLastBlock. +func (mr *MockMetadataMockRecorder) SetLastBlock(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLastBlock", reflect.TypeOf((*MockMetadata)(nil).SetLastBlock), arg0) +} + +// SetLastEpoch mocks base method. +func (m *MockMetadata) SetLastEpoch(arg0 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetLastEpoch", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetLastEpoch indicates an expected call of SetLastEpoch. +func (mr *MockMetadataMockRecorder) SetLastEpoch(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLastEpoch", reflect.TypeOf((*MockMetadata)(nil).SetLastEpoch), arg0) +} + +// SetTimestamp mocks base method. +func (m *MockMetadata) SetTimestamp() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetTimestamp") + ret0, _ := ret[0].(error) + return ret0 +} + +// SetTimestamp indicates an expected call of SetTimestamp. +func (mr *MockMetadataMockRecorder) SetTimestamp() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetTimestamp", reflect.TypeOf((*MockMetadata)(nil).SetTimestamp)) +} + +// SetUpdatesetInterval mocks base method. +func (m *MockMetadata) SetUpdatesetInterval(arg0 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetUpdatesetInterval", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetUpdatesetInterval indicates an expected call of SetUpdatesetInterval. +func (mr *MockMetadataMockRecorder) SetUpdatesetInterval(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUpdatesetInterval", reflect.TypeOf((*MockMetadata)(nil).SetUpdatesetInterval), arg0) +} + +// SetUpdatesetSize mocks base method. +func (m *MockMetadata) SetUpdatesetSize(arg0 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetUpdatesetSize", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetUpdatesetSize indicates an expected call of SetUpdatesetSize. +func (mr *MockMetadataMockRecorder) SetUpdatesetSize(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUpdatesetSize", reflect.TypeOf((*MockMetadata)(nil).SetUpdatesetSize), arg0) +} diff --git a/utils/metadata_test.go b/utils/metadata_test.go index 0f44427c6..54e365d86 100644 --- a/utils/metadata_test.go +++ b/utils/metadata_test.go @@ -20,10 +20,10 @@ import ( "errors" "testing" + "github.com/0xsoniclabs/aida/logger" "github.com/0xsoniclabs/substate/db" "github.com/0xsoniclabs/substate/substate" "github.com/Fantom-foundation/lachesis-base/common/bigendian" - gethleveldb "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/syndtr/goleveldb/leveldb" @@ -276,17 +276,24 @@ func TestAidaDbMetadata_GetFirstBlock(t *testing.T) { defer ctrl.Finish() mockDb := db.NewMockSubstateDB(ctrl) - md := NewAidaDbMetadata(mockDb, "ERROR") + md := AidaDbMetadata{ + Db: mockDb, + log: logger.NewLogger("ERROR", "metadata-test"), + } mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(nil, errors.New("mock error")) firstBlock := md.GetFirstBlock() assert.Equal(t, uint64(0), firstBlock) - mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(nil, leveldb.ErrNotFound) + mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(bigendian.Uint64ToBytes(100), nil) firstBlock = md.GetFirstBlock() - assert.Equal(t, uint64(0), firstBlock) + assert.Equal(t, uint64(100), firstBlock) - mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(bigendian.Uint64ToBytes(100), nil) + // clear cache + md.FirstBlock = nil + // not found - get from substate + mockDb.EXPECT().Get([]byte(FirstBlockPrefix)).Return(nil, leveldb.ErrNotFound) + mockDb.EXPECT().GetFirstSubstate().Return(&substate.Substate{Block: uint64(100)}) firstBlock = md.GetFirstBlock() assert.Equal(t, uint64(100), firstBlock) @@ -300,17 +307,24 @@ func TestAidaDbMetadata_GetLastBlock(t *testing.T) { defer ctrl.Finish() mockDb := db.NewMockSubstateDB(ctrl) - md := NewAidaDbMetadata(mockDb, "ERROR") + md := AidaDbMetadata{ + Db: mockDb, + log: logger.NewLogger("ERROR", "metadata-test"), + } - mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(nil, leveldb.ErrNotFound) + mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(nil, errors.New("mock error")) lastBlock := md.GetLastBlock() assert.Equal(t, uint64(0), lastBlock) - mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(nil, errors.New("mock error")) + mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(bigendian.Uint64ToBytes(100), nil) lastBlock = md.GetLastBlock() - assert.Equal(t, uint64(0), lastBlock) + assert.Equal(t, uint64(100), lastBlock) - mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(bigendian.Uint64ToBytes(100), nil) + // clear cache + md.LastBlock = nil + // not found - get from substate + mockDb.EXPECT().Get([]byte(LastBlockPrefix)).Return(nil, leveldb.ErrNotFound) + mockDb.EXPECT().GetLastSubstate().Return(&substate.Substate{Block: uint64(100)}, nil) lastBlock = md.GetLastBlock() assert.Equal(t, uint64(100), lastBlock) @@ -434,31 +448,6 @@ func TestAidaDbMetadata_GetDbType(t *testing.T) { assert.Equal(t, AidaDbType(0), data) } -func TestHasStateHashPatch(t *testing.T) { - dir := t.TempDir() - path := dir + "/test.db" - t.Run("case not exist", func(t *testing.T) { - v, err := HasStateHashPatch(path) - assert.Nil(t, err) - assert.False(t, v) - }) - - t.Run("case exist", func(t *testing.T) { - eDb, err := gethleveldb.New(path, 1024, 100, "profiling", false) - if err != nil { - t.Fatalf("failed to open leveldb: %v", err) - } - err = eDb.Close() - if err != nil { - t.Fatalf("failed to close leveldb: %v", err) - } - v, err := HasStateHashPatch(path) - assert.Nil(t, err) - assert.False(t, v) - }) - -} - func Test_FindEpochNumber_IsSkippedForEthereumChainIDs(t *testing.T) { for chainID := range EthereumChainIDs { md := &AidaDbMetadata{ChainId: chainID} @@ -468,3 +457,15 @@ func Test_FindEpochNumber_IsSkippedForEthereumChainIDs(t *testing.T) { assert.Equal(t, md.GetLastEpoch(), uint64(0)) } } + +func TestMetadata_FindEpochNumber(t *testing.T) { + // case success + output, err := FindEpochNumber(uint64(1234), MainnetChainID) + assert.NoError(t, err) + assert.Equal(t, uint64(11), output) + + // case error + output, err = FindEpochNumber(uint64(1234), invalidChainID) + assert.Error(t, err) + assert.Equal(t, uint64(0), output) +} diff --git a/utils/rpc.go b/utils/rpc.go index 7b864a6e1..3a8e54a26 100644 --- a/utils/rpc.go +++ b/utils/rpc.go @@ -21,7 +21,6 @@ import ( "encoding/json" "fmt" "net/http" - "strconv" "github.com/ethereum/go-ethereum/common/math" ) @@ -94,15 +93,6 @@ func GetProvider(chainId ChainID) (string, error) { } } -// FindEpochNumber via RPC request GetBlockByNumber -func FindEpochNumber(blockNumber uint64, chainId ChainID) (uint64, error) { - hex := strconv.FormatUint(blockNumber, 16) - - blockStr := "0x" + hex - - return getEpochByNumber(blockStr, chainId) -} - // FindHeadEpochNumber via RPC request GetBlockByNumber func FindHeadEpochNumber(chainId ChainID) (uint64, error) { blockStr := "latest" @@ -123,7 +113,17 @@ func getEpochByNumber(blockStr string, chainId ChainID) (uint64, error) { return 0, err } - resultMap, ok := m["result"].(map[string]interface{}) + if errMsg, ok := m["error"]; ok { + return 0, fmt.Errorf("rpc error: %v", errMsg) + } + + res, ok := m["result"] + if ok { + if m["result"] == nil { + return 0, fmt.Errorf("rpc does not know this block: %x", blockStr) + } + } + resultMap, ok := res.(map[string]interface{}) if !ok { return 0, fmt.Errorf("unexpecetd answer: %v", m) } diff --git a/utils/rpc_test.go b/utils/rpc_test.go index 8780f1361..7e138f57e 100644 --- a/utils/rpc_test.go +++ b/utils/rpc_test.go @@ -106,18 +106,6 @@ func TestSendRPCRequest_InvalidChainID(t *testing.T) { } -func TestRpc_FindEpochNumber(t *testing.T) { - // case success - output, err := FindEpochNumber(uint64(1234), MainnetChainID) - assert.NoError(t, err) - assert.Equal(t, uint64(11), output) - - // case error - output, err = FindEpochNumber(uint64(1234), invalidChainID) - assert.Error(t, err) - assert.Equal(t, uint64(0), output) -} - func TestRpc_FindHeadEpochNumber(t *testing.T) { // case success output, err := FindHeadEpochNumber(MainnetChainID) diff --git a/utils/test_utils.go b/utils/test_utils.go index d6292765f..1a7800814 100644 --- a/utils/test_utils.go +++ b/utils/test_utils.go @@ -17,6 +17,7 @@ package utils import ( + "encoding/hex" "fmt" "math/big" "strconv" @@ -105,12 +106,11 @@ func CreateTestSubstateDb(t *testing.T, encoding substateDb.SubstateEncodingSche require.NoError(t, err) md := NewAidaDbMetadata(db, "CRITICAL") - // todo might cause failing tests - //dbHash, err := hex.DecodeString("a0d4f7616f3007bf8c02f816a60b2526") - //require.NoError(t, err) - + dbHash, err := hex.DecodeString("a0d4f7616f3007bf8c02f816a60b2526") + require.NoError(t, err) + require.NoError(t, md.SetDbHash(dbHash)) require.NoError(t, md.GenerateMetadata(SonicMainnetChainID)) - + require.NoError(t, md.SetDbType(GenType)) require.NoError(t, db.Close()) return ss, path From e55fe9e95fbfff6d2d49b53fd28ce7825ea5f4eb Mon Sep 17 00:00:00 2001 From: cabrador Date: Mon, 8 Sep 2025 11:26:24 +0000 Subject: [PATCH 5/5] Add metadata tests --- utils/metadata.go | 59 +--------- utils/metadata_test.go | 262 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 259 insertions(+), 62 deletions(-) diff --git a/utils/metadata.go b/utils/metadata.go index d6abece3d..d7cd1b7e9 100644 --- a/utils/metadata.go +++ b/utils/metadata.go @@ -209,12 +209,12 @@ func NewAidaDbMetadata(db db.SubstateDB, logLevel string) Metadata { func (md *AidaDbMetadata) Merge(src Metadata) error { targetChainID := md.GetChainID() srcChainID := src.GetChainID() - if targetChainID != 0 { + if targetChainID != UnknownChainID { if targetChainID != srcChainID { return fmt.Errorf("cannot merge dbs with different chainIDs; target db chainID %v, source db chainID %v", targetChainID, srcChainID) } } else { - if srcChainID == 0 { + if srcChainID == UnknownChainID { return errors.New("cannot merge dbs with no chainIDs in metadata; you can set chainID manually using the util-db insert cmd") } err := md.SetChainID(srcChainID) @@ -272,7 +272,7 @@ func (md *AidaDbMetadata) Merge(src Metadata) error { blocksOk := false // Check alignment - dbs can overlap but cannot have gaps // Target is before source - if targetLastBlock+1 >= srcFirstBlock { + if targetLastBlock < srcFirstBlock { err = md.SetLastBlock(srcLastBlock) if err != nil { return fmt.Errorf("cannot merge last block: %v", err) @@ -281,7 +281,7 @@ func (md *AidaDbMetadata) Merge(src Metadata) error { } // Target is after source - if srcLastBlock+1 >= targetFirstBlock { + if targetFirstBlock > srcLastBlock { err = md.SetFirstBlock(srcFirstBlock) if err != nil { return fmt.Errorf("cannot merge first block: %v", err) @@ -613,55 +613,6 @@ func FindEpochNumber(blockNumber uint64, chainId ChainID) (uint64, error) { return num, err } -// SetFreshMetadata for an existing AidaDb without metadata -func (md *AidaDbMetadata) SetFreshMetadata(chainID ChainID) error { - var err error - - if chainID == 0 { - return fmt.Errorf("since you have aida-db without metadata you need to specify chain-id (--%v)", ChainIDFlag.Name) - } - - // ChainID is Set by user in - if err = md.SetChainID(chainID); err != nil { - return err - } - - if err = md.findEpochs(); err != nil { - return err - } - - if err = md.SetTimestamp(); err != nil { - return err - } - - _, err = getPatchFirstBlock(md.GetLastBlock()) - if err != nil { - md.log.Warning("Uncertain AidaDbType.") - if err = md.SetDbType(NoType); err != nil { - return err - } - } else { - if err = md.SetDbType(GenType); err != nil { - return err - } - } - - return nil -} - -func (md *AidaDbMetadata) SetBlockRange(firstBlock uint64, lastBlock uint64) error { - var err error - - if err = md.SetFirstBlock(firstBlock); err != nil { - return err - } - if err = md.SetLastBlock(lastBlock); err != nil { - return err - } - - return nil -} - func (md *AidaDbMetadata) Delete() error { var finalErr error @@ -773,9 +724,7 @@ func getPatchFirstBlock(lastPatchBlock uint64) (uint64, error) { } availableLastBlocks += fmt.Sprintf("%v ", p.ToBlock) } - return 0, fmt.Errorf("cannot find find first block for requested last block; requested: %v; available: [%v]", lastPatchBlock, availableLastBlocks) - } // SetHasHashPatch marks AidaDb that it already has HashPatch merged so it will not get downloaded next update. diff --git a/utils/metadata_test.go b/utils/metadata_test.go index 54e365d86..d3eeabb4e 100644 --- a/utils/metadata_test.go +++ b/utils/metadata_test.go @@ -18,6 +18,7 @@ package utils import ( "errors" + "math" "testing" "github.com/0xsoniclabs/aida/logger" @@ -458,14 +459,261 @@ func Test_FindEpochNumber_IsSkippedForEthereumChainIDs(t *testing.T) { } } -func TestMetadata_FindEpochNumber(t *testing.T) { - // case success - output, err := FindEpochNumber(uint64(1234), MainnetChainID) +func TestMetadata_MergeOk(t *testing.T) { + // Create two real SubstateDBs in temp dirs + dir1 := t.TempDir() + dir2 := t.TempDir() + + db1, err := db.NewDefaultSubstateDB(dir1) + require.NoError(t, err) + db2, err := db.NewDefaultSubstateDB(dir2) + require.NoError(t, err) + + // Set up metadata for target + md1 := NewAidaDbMetadata(db1, "ERROR") + require.NoError(t, md1.SetChainID(SonicMainnetChainID)) + require.NoError(t, md1.SetFirstBlock(10)) + require.NoError(t, md1.SetLastBlock(20)) + require.NoError(t, md1.SetDbType(GenType)) + + // Set up metadata for source + md2 := NewAidaDbMetadata(db2, "ERROR") + require.NoError(t, md2.SetChainID(SonicMainnetChainID)) + require.NoError(t, md2.SetFirstBlock(21)) + require.NoError(t, md2.SetLastBlock(30)) + require.NoError(t, md2.SetDbType(GenType)) + + // Positive case: blocks align + err = md1.Merge(md2) assert.NoError(t, err) - assert.Equal(t, uint64(11), output) + assert.Equal(t, uint64(10), md1.GetFirstBlock()) + assert.Equal(t, uint64(30), md1.GetLastBlock()) + assert.Equal(t, GenType, md1.GetDbType()) + + // Negative case: chain IDs differ + md3 := NewAidaDbMetadata(db2, "ERROR") + require.NoError(t, md3.SetChainID(200)) + require.NoError(t, md3.SetFirstBlock(21)) + require.NoError(t, md3.SetLastBlock(30)) + require.NoError(t, md3.SetDbType(GenType)) + + err = md1.Merge(md3) + assert.Error(t, err) + assert.Contains(t, err.Error(), "cannot merge dbs with different chainIDs") +} - // case error - output, err = FindEpochNumber(uint64(1234), invalidChainID) +func TestMetadata_MergeError(t *testing.T) { + tests := []struct { + name string + targetChainID ChainID + targetFirst uint64 + targetLast uint64 + srcFirstBlock uint64 + srcLastBlock uint64 + srcChainID ChainID + errMsg string + }{ + { + name: "source is subset of target", + targetChainID: SonicMainnetChainID, + srcChainID: SonicMainnetChainID, + targetFirst: 10, + targetLast: 30, + srcFirstBlock: 15, + srcLastBlock: 20, + errMsg: "source db (15-20) is subset of target db (10-30)", + }, + { + name: "target is subset of source", + targetChainID: SonicMainnetChainID, + srcChainID: SonicMainnetChainID, + targetFirst: 15, + targetLast: 20, + srcFirstBlock: 10, + srcLastBlock: 30, + errMsg: "target db (15-20) is subset of source db (10-30)", + }, + { + name: "gap before target", + targetChainID: SonicMainnetChainID, + srcChainID: SonicMainnetChainID, + targetFirst: 20, + targetLast: 30, + srcFirstBlock: 10, + srcLastBlock: 18, + errMsg: "cannot merge dbs with gap; target db (20-30), source db (10-18)", + }, + { + name: "gap after target", + targetChainID: SonicMainnetChainID, + srcChainID: SonicMainnetChainID, + targetFirst: 10, + targetLast: 18, + srcFirstBlock: 20, + srcLastBlock: 30, + errMsg: "cannot merge dbs with gap; target db (10-18), source db (20-30)", + }, + { + name: "blocks do not align (overlap)", + targetChainID: SonicMainnetChainID, + srcChainID: SonicMainnetChainID, + targetFirst: 10, + targetLast: 20, + srcFirstBlock: 15, + srcLastBlock: 25, + errMsg: "blocks does not align; target db (10-20), source db (15-25)", + }, + { + name: "different chainIDs", + targetChainID: SonicMainnetChainID, + srcChainID: EthereumChainID, + targetFirst: 10, + targetLast: 20, + srcFirstBlock: 21, + srcLastBlock: 25, + errMsg: "cannot merge dbs with different chainIDs; target db chainID 146, source db chainID 1", + }, + { + name: "unknown chainIDs", + targetChainID: UnknownChainID, + srcChainID: UnknownChainID, + targetFirst: 10, + targetLast: 20, + srcFirstBlock: 21, + srcLastBlock: 25, + errMsg: "cannot merge dbs with no chainIDs in metadata; you can set chainID manually using the util-db insert cmd", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db1, err := db.NewDefaultSubstateDB(t.TempDir()) + require.NoError(t, err) + db2, err := db.NewDefaultSubstateDB(t.TempDir()) + require.NoError(t, err) + + md1 := NewAidaDbMetadata(db1, "ERROR") + require.NoError(t, md1.SetChainID(test.targetChainID)) + require.NoError(t, md1.SetFirstBlock(test.targetFirst)) + require.NoError(t, md1.SetLastBlock(test.targetLast)) + require.NoError(t, md1.SetDbType(GenType)) + + md2 := NewAidaDbMetadata(db2, "ERROR") + require.NoError(t, md2.SetChainID(test.srcChainID)) + require.NoError(t, md2.SetFirstBlock(test.srcFirstBlock)) + require.NoError(t, md2.SetLastBlock(test.srcLastBlock)) + require.NoError(t, md2.SetDbType(GenType)) + + err = md1.Merge(md2) + assert.ErrorContains(t, err, test.errMsg) + }) + } +} + +func TestAidaDbMetadata_SetTimestamp(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Case 1: Success + mockDb := db.NewMockSubstateDB(ctrl) + md := NewAidaDbMetadata(mockDb, "ERROR") + mockDb.EXPECT().Put([]byte(TimestampPrefix), gomock.Any()).Return(nil) + err := md.SetTimestamp() + assert.NoError(t, err) + + // Case 2: Error + mockDb = db.NewMockSubstateDB(ctrl) + md = NewAidaDbMetadata(mockDb, "ERROR") + mockDb.EXPECT().Put([]byte(TimestampPrefix), gomock.Any()).Return(errors.New("mock error")) + err = md.SetTimestamp() assert.Error(t, err) - assert.Equal(t, uint64(0), output) + assert.Contains(t, err.Error(), "mock error") +} + +func TestAidaDbMetadata_GetDb(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockDb := db.NewMockSubstateDB(ctrl) + md := NewAidaDbMetadata(mockDb, "ERROR") + assert.Equal(t, mockDb, md.GetDb()) +} + +func TestAidaDbMetadata_HasHashPatch(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockDb := db.NewMockSubstateDB(ctrl) + md := NewAidaDbMetadata(mockDb, "ERROR") + + // Case 1: Found + mockDb.EXPECT().Get([]byte(HasStateHashPatchPrefix)).Return([]byte{1}, nil) + assert.True(t, md.HasHashPatch()) + + // Case 2: Not found + mockDb.EXPECT().Get([]byte(HasStateHashPatchPrefix)).Return(nil, leveldb.ErrNotFound) + assert.False(t, md.HasHashPatch()) + + // Case 3: Error + mockDb.EXPECT().Get([]byte(HasStateHashPatchPrefix)).Return(nil, errors.New("mock error")) + assert.False(t, md.HasHashPatch()) +} + +func TestAidaDbMetadata_GetUpdatesetSize(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockDb := db.NewMockSubstateDB(ctrl) + md := NewAidaDbMetadata(mockDb, "ERROR") + + // Case 1: Error + mockDb.EXPECT().Get([]byte(db.UpdatesetSizeKey)).Return(nil, errors.New("mock error")) + size := md.GetUpdatesetSize() + assert.Equal(t, uint64(0), size) + + // Case 2: Not found + mockDb.EXPECT().Get([]byte(db.UpdatesetSizeKey)).Return(nil, leveldb.ErrNotFound) + size = md.GetUpdatesetSize() + assert.Equal(t, uint64(0), size) + + // Case 3: Success + mockDb.EXPECT().Get([]byte(db.UpdatesetSizeKey)).Return(bigendian.Uint64ToBytes(42), nil) + size = md.GetUpdatesetSize() + assert.Equal(t, uint64(42), size) + + // Case 4: Cached value, no DB call + size = md.GetUpdatesetSize() + assert.Equal(t, uint64(42), size) +} + +func TestAidaDbMetadata_GetUpdatesetInterval(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockDb := db.NewMockSubstateDB(ctrl) + md := NewAidaDbMetadata(mockDb, "ERROR") + + // Case 1: Error + mockDb.EXPECT().Get([]byte(db.UpdatesetIntervalKey)).Return(nil, errors.New("mock error")) + interval := md.GetUpdatesetInterval() + assert.Equal(t, uint64(0), interval) + + // Case 2: Not found + mockDb.EXPECT().Get([]byte(db.UpdatesetIntervalKey)).Return(nil, leveldb.ErrNotFound) + interval = md.GetUpdatesetInterval() + assert.Equal(t, uint64(0), interval) + + // Case 3: Success + mockDb.EXPECT().Get([]byte(db.UpdatesetIntervalKey)).Return(bigendian.Uint64ToBytes(99), nil) + interval = md.GetUpdatesetInterval() + assert.Equal(t, uint64(99), interval) + + // Case 4: Cached value, no DB call + interval = md.GetUpdatesetInterval() + assert.Equal(t, uint64(99), interval) +} + +func TestFindEpochNumber_UnknownBlock(t *testing.T) { + epoch, err := FindEpochNumber(math.MaxInt64, SonicMainnetChainID) + require.NoError(t, err) + require.Equal(t, uint64(0), epoch) }