[WIP] Refactor metadata

Makefile

@@ -118,8 +118,8 @@ install-dev-tools:
 	@go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.1.2
 
 format:
-	@goimports -w ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db/compact ./cmd/util-db/info ./cmd/util-db/metadata ./cmd/util-db/merge ./cmd/util-db/validate ./cmd/util-db/generate ./cmd/util-db/clone ./logger ./register
-	@gofmt -s -d -w ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db/compact ./cmd/util-db/info ./cmd/util-db/metadata ./cmd/util-db/merge ./cmd/util-db/validate ./cmd/util-db/generate ./cmd/util-db/clone ./logger ./register
+	@goimports -w ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db ./logger ./register
+	@gofmt -s -d -w ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db ./logger ./register
 
 check:
-	@golangci-lint run -c .golangci.yml ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db/compact ./cmd/util-db/info ./cmd/util-db/metadata ./cmd/util-db/merge ./cmd/util-db/validate ./cmd/util-db/generate ./cmd/util-db/clone ./logger ./register
+	@golangci-lint run -c .golangci.yml ./utils ./profile ./txcontext ./ethtest ./rpc ./stochastic ./cmd/util-db ./logger ./register

cmd/util-db/clone/clone_test.go

@@ -47,8 +47,8 @@ func TestClone(t *testing.T) {
 		dbc         string
 		wantErr     string
 	}{
-		{"NoType", utils.NoType, "", "clone failed for NoType: incorrect clone type: 0"},
-		{"GenType", utils.GenType, "", "clone failed for GenType: incorrect clone type: 1"},
+		{"NoType", utils.NoType, "", "clone failed for NoType: incorrect clone type: NoType"},
+		{"GenType", utils.GenType, "", "clone failed for GenType: incorrect clone type: Generate"},
 		{"PatchType", utils.PatchType, "", ""},
 		{"CloneType", utils.CloneType, "", ""},
 		{"CustomTypeAll", utils.CustomType, "all", ""},

cmd/util-db/clone/cloner.go

@@ -113,11 +113,15 @@ func (c *cloner) clone() error {
 	}
 
 	if c.typ != utils.CustomType {
-		sourceMD := utils.NewAidaDbMetadata(c.sourceDb, c.cfg.LogLevel)
-		chainID := sourceMD.GetChainID()
-
-		if err = utils.ProcessCloneLikeMetadata(c.cloneDb, c.typ, c.cfg.LogLevel, c.cfg.First, c.cfg.Last, chainID); err != nil {
-			return err
+		srcMd := utils.NewAidaDbMetadata(c.sourceDb, c.cfg.LogLevel)
+		md := utils.NewAidaDbMetadata(c.cloneDb, c.cfg.LogLevel)
+		err = md.SetDbType(utils.CloneType)
+		if err != nil {
+			return fmt.Errorf("cannot set db type for clone db; %v", err)
+		}
+		err = md.GenerateMetadata(srcMd.GetChainID())
+		if err != nil {
+			return fmt.Errorf("cannot generate metadata for clone db; %v", err)
 		}
 	}
 

cmd/util-db/clone/db.go

@@ -53,14 +53,15 @@ func cloneDbAction(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	defer func() {
+		utildb.MustCloseDB(aidaDb)
+		utildb.MustCloseDB(targetDb)
+	}()
 
 	err = clone(cfg, aidaDb, targetDb, utils.CloneType)
 	if err != nil {
 		return err
 	}
 
-	utildb.MustCloseDB(aidaDb)
-	utildb.MustCloseDB(targetDb)
-
-	return utildb.PrintMetadata(cfg.TargetDb)
+	return utildb.PrintMetadata(targetDb)
 }

cmd/util-db/clone/patch.go

@@ -73,15 +73,17 @@ func clonePatchAction(ctx *cli.Context) error {
 		return err
 	}
 
+	defer func() {
+		utildb.MustCloseDB(aidaDb)
+		utildb.MustCloseDB(targetDb)
+	}()
+
 	err = createPatchClone(cfg, aidaDb, targetDb, firstEpoch, lastEpoch)
 	if err != nil {
 		return err
 	}
 
-	utildb.MustCloseDB(aidaDb)
-	utildb.MustCloseDB(targetDb)
-
-	return utildb.PrintMetadata(cfg.TargetDb)
+	return utildb.PrintMetadata(targetDb)
 }
 
 // createPatchClone creates aida-db clonePatchCommand

cmd/util-db/db/update.go → cmd/util-db/db/scrape.go

@@ -17,40 +17,48 @@
 package db
 
 import (
+	"fmt"
+
 	"github.com/0xsoniclabs/aida/logger"
-	"github.com/0xsoniclabs/aida/utildb"
 	"github.com/0xsoniclabs/aida/utils"
+	"github.com/0xsoniclabs/substate/db"
 	"github.com/urfave/cli/v2"
 )
 
-// UpdateCommand downloads aida-db and new patches
-var UpdateCommand = cli.Command{
-	Action: update,
-	Name:   "update",
-	Usage:  "download aida-db patches",
+var ScrapeCommand = cli.Command{
+	Action:    scrapePrepare,
+	Name:      "scrape",
+	Usage:     "Stores state hashes into TargetDb for given range",
+	ArgsUsage: "<blockNumFirst> <blockNumLast>",
 	Flags: []cli.Flag{
-		&utils.AidaDbFlag,
+		&utils.TargetDbFlag,
 		&utils.ChainIDFlag,
+		&utils.ClientDbFlag,
 		&logger.LogLevelFlag,
-		&utils.CompactDbFlag,
-		&utils.DbTmpFlag,
-		&utils.UpdateTypeFlag,
-		&utils.SubstateEncodingFlag,
 	},
-	Description: ` 
-Updates aida-db by downloading patches from aida-db generation server.
-`,
 }
 
-// update updates aida-db by downloading patches from aida-db generation server.
-func update(ctx *cli.Context) error {
-	cfg, err := utils.NewConfig(ctx, utils.NoArgs)
+// scrapePrepare stores state hashes into Target for given range
+func scrapePrepare(ctx *cli.Context) error {
+	cfg, argErr := utils.NewConfig(ctx, utils.BlockRangeArgs)
+	if argErr != nil {
+		return argErr
+	}
+
+	log := logger.NewLogger(cfg.LogLevel, "UtilDb-Scrape")
+	log.Infof("Scraping for range %d-%d", cfg.First, cfg.Last)
+
+	database, err := db.NewDefaultBaseDB(cfg.TargetDb)
 	if err != nil {
-		return err
+		return fmt.Errorf("error opening stateHash leveldb %s: %v", cfg.TargetDb, err)
 	}
-	if err = utildb.Update(cfg); err != nil {
+	defer database.Close()
+
+	err = utils.StateAndBlockHashScraper(ctx.Context, cfg.ChainID, cfg.ClientDb, database, cfg.First, cfg.Last, log)
+	if err != nil {
 		return err
 	}
 
-	return utildb.PrintMetadata(cfg.AidaDb)
+	log.Infof("Scraping finished")
+	return nil
 }

cmd/util-db/generate/db_hash.go

@@ -28,7 +28,7 @@ func generateDbHashAction(ctx *cli.Context) error {
 		return err
 	}
 
-	aidaDb, err := db.NewDefaultBaseDB(cfg.AidaDb)
+	aidaDb, err := db.NewDefaultSubstateDB(cfg.AidaDb)
 	if err != nil {
 		return fmt.Errorf("cannot open db; %v", err)
 	}

cmd/util-db/generate/generate_test.go

@@ -85,7 +85,7 @@ func TestGenerateDbHash_Command(t *testing.T) {
 	err := app.Run(argsBuilder.Build())
 	require.NoError(t, err)
 
-	aidaDb, err := db.NewDefaultBaseDB(path)
+	aidaDb, err := db.NewDefaultSubstateDB(path)
 	require.NoError(t, err)
 	md := utils.NewAidaDbMetadata(aidaDb, "CRITICAL")
 	got := md.GetDbHash()

cmd/util-db/info/db_hash.go

@@ -41,7 +41,7 @@ var printDbHashCommand = cli.Command{
 func printDbHashAction(ctx *cli.Context) error {
 	var force = ctx.Bool(flags.ForceFlag.Name)
 
-	aidaDb, err := db.NewReadOnlyBaseDB(ctx.String(utils.AidaDbFlag.Name))
+	aidaDb, err := db.NewReadOnlySubstateDB(ctx.String(utils.AidaDbFlag.Name))
 	if err != nil {
 		return fmt.Errorf("cannot open db; %v", err)
 	}

cmd/util-db/info/info_test.go

@@ -48,28 +48,15 @@ func TestInfo_PrintCount(t *testing.T) {
 		wantErr string
 	}
 
-	aidaDbPath := t.TempDir() + "aida-db"
-
-	aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if aidaDb == nil {
-		t.Fatal("aidaDb is nil")
-	}
-	err = aidaDb.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-
+	_, aidaDbPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema)
 	tests := []testCase{
 		{
 			name: "IntegrationTest",
 			args: []string{
 				"info", "count",
 				"--aida-db", aidaDbPath,
 				"--db-component=all",
-				"1", "2",
+				"37534834", "37534834",
 			},
 			wantErr: "",
 		},

cmd/util-db/main.go

@@ -22,13 +22,13 @@ import (
 
 	"github.com/0xsoniclabs/aida/cmd/util-db/clone"
 	"github.com/0xsoniclabs/aida/cmd/util-db/compact"
-	"github.com/0xsoniclabs/aida/cmd/util-db/db"
 	"github.com/0xsoniclabs/aida/cmd/util-db/generate"
 	"github.com/0xsoniclabs/aida/cmd/util-db/info"
 	"github.com/0xsoniclabs/aida/cmd/util-db/merge"
 	"github.com/0xsoniclabs/aida/cmd/util-db/metadata"
 	"github.com/0xsoniclabs/aida/cmd/util-db/primer"
 	"github.com/0xsoniclabs/aida/cmd/util-db/scrape"
+	"github.com/0xsoniclabs/aida/cmd/util-db/update"
 	"github.com/0xsoniclabs/aida/cmd/util-db/validate"
 	"github.com/urfave/cli/v2"
 )
@@ -47,7 +47,7 @@ var UtilDbApp = cli.App{
 		&validate.Command,
 		&metadata.Command,
 		&generate.Command,
-		&db.UpdateCommand,
+		&update.Command,
 		&scrape.Command,
 
 		//Priming only

cmd/util-db/merge/merge.go

@@ -58,32 +58,27 @@ func mergeAction(ctx *cli.Context) error {
 		sourcePaths[i] = ctx.Args().Get(i)
 	}
 
-	targetDb, err := db.NewDefaultBaseDB(cfg.AidaDb)
+	targetDb, err := db.NewDefaultSubstateDB(cfg.AidaDb)
 	if err != nil {
 		return fmt.Errorf("cannot open db; %v", err)
 	}
 
-	var (
-		dbs []db.BaseDB
-		md  *utils.AidaDbMetadata
-	)
+	var dbs []db.SubstateDB
 
 	if !cfg.SkipMetadata {
 		dbs, err = utildb.OpenSourceDatabases(sourcePaths)
 		if err != nil {
 			return fmt.Errorf("cannot open source databases: %w", err)
 		}
-		md, err = utils.ProcessMergeMetadata(cfg, targetDb, dbs, sourcePaths)
-		if err != nil {
-			return err
-		}
 
-		// todo this should not be necessary - do not close aida-db in metadata
-		targetDb, err = db.NewDefaultBaseDB(cfg.AidaDb)
-		if err != nil {
-			return fmt.Errorf("cannot re-open db: %w", err)
+		// merge metadata from all source dbs
+		targetMD := utils.NewAidaDbMetadata(targetDb, cfg.LogLevel)
+		for _, db := range dbs {
+			sourceMD := utils.NewAidaDbMetadata(db, cfg.LogLevel)
+			if err := targetMD.Merge(sourceMD); err != nil {
+				return fmt.Errorf("cannot merge metadata: %w", err)
+			}
 		}
-
 		for _, database := range dbs {
 			utildb.MustCloseDB(database)
 		}
@@ -93,7 +88,7 @@ func mergeAction(ctx *cli.Context) error {
 		return err
 	}
 
-	m := utildb.NewMerger(cfg, targetDb, dbs, sourcePaths, md)
+	m := utildb.NewMerger(cfg, targetDb, dbs, sourcePaths)
 	if err = m.Merge(); err != nil {
 		return err
 	}

cmd/util-db/merge/merge_test.go

@@ -27,29 +27,38 @@ import (
 )
 
 func TestMerge_Command(t *testing.T) {
+	ss, aidaDbPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema)
+
 	path1 := t.TempDir() + "/sdb1"
 	sdb1, err := db.NewDefaultSubstateDB(path1)
 	require.NoError(t, err)
 	s1 := utils.GetTestSubstate("pb")
-	s1.Block = 10
-	s1.Transaction = 2
+	s1.Block = ss.Block + 1
+	s1.Transaction = 1
 	err = sdb1.PutSubstate(s1)
 	require.NoError(t, err)
+
+	md := utils.NewAidaDbMetadata(sdb1, "CRITICAL")
+	require.NoError(t, md.SetChainID(utils.SonicMainnetChainID))
+
 	err = sdb1.Close()
 	require.NoError(t, err)
 
 	path2 := t.TempDir() + "/sdb2"
 	sdb2, err := db.NewDefaultSubstateDB(path2)
 	require.NoError(t, err)
 	s2 := utils.GetTestSubstate("pb")
-	s2.Block = 20
-	s2.Transaction = 3
+	s2.Block = ss.Block + 2
+	s2.Transaction = 2
 	err = sdb2.PutSubstate(s2)
 	require.NoError(t, err)
+
+	md = utils.NewAidaDbMetadata(sdb2, "CRITICAL")
+	require.NoError(t, md.SetChainID(utils.SonicMainnetChainID))
+
 	err = sdb2.Close()
 	require.NoError(t, err)
 
-	_, aidaDbPath := utils.CreateTestSubstateDb(t, db.ProtobufEncodingSchema)
 	app := cli.NewApp()
 	app.Action = mergeAction
 	app.Flags = Command.Flags
@@ -69,11 +78,11 @@ func TestMerge_Command(t *testing.T) {
 	aidaDb, err := db.NewDefaultSubstateDB(aidaDbPath)
 	require.NoError(t, err)
 
-	gotS1, err := aidaDb.GetSubstate(10, 2)
+	gotS1, err := aidaDb.GetSubstate(s1.Block, s1.Transaction)
 	require.NoError(t, err)
 	require.NoError(t, gotS1.Equal(s1))
 
-	gotS2, err := aidaDb.GetSubstate(20, 3)
+	gotS2, err := aidaDb.GetSubstate(s2.Block, s2.Transaction)
 	require.NoError(t, err)
 	require.NoError(t, gotS2.Equal(s2))
 }

cmd/util-db/metadata/generate.go

@@ -41,26 +41,25 @@ func generateAction(ctx *cli.Context) error {
 		return argErr
 	}
 
-	base, err := db.NewDefaultBaseDB(cfg.AidaDb)
+	aidaDb, err := db.NewDefaultSubstateDB(cfg.AidaDb)
 	if err != nil {
 		return err
 	}
-
-	sdb := db.MakeDefaultSubstateDBFromBaseDB(base)
-	fb, lb, ok := utils.FindBlockRangeInSubstate(sdb)
+	defer utildb.MustCloseDB(aidaDb)
+	fb, lb, ok := utils.FindBlockRangeInSubstate(aidaDb)
 	if !ok {
 		return errors.New("cannot find block range in substate")
 	}
 
-	md := utils.NewAidaDbMetadata(base, "INFO")
-	md.FirstBlock = fb
-	md.LastBlock = lb
-	if err = md.SetFreshMetadata(cfg.ChainID); err != nil {
+	md := utils.NewAidaDbMetadata(aidaDb, "INFO")
+	if err = md.SetFirstBlock(fb); err != nil {
 		return err
 	}
-
-	utildb.MustCloseDB(base)
-
-	return utildb.PrintMetadata(cfg.AidaDb)
-
+	if err = md.SetLastBlock(lb); err != nil {
+		return err
+	}
+	if err = md.SetChainID(cfg.ChainID); err != nil {
+		return err
+	}
+	return utildb.PrintMetadata(aidaDb)
 }