diff --git a/.gitignore b/.gitignore index c04dc96..eb743aa 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ go.work.sum # .vscode/ .DS_Store *.db +*.db-shm +*.db-wal diff --git a/README.md b/README.md index c50fc1a..8b4836b 100644 --- a/README.md +++ b/README.md @@ -38,34 +38,39 @@ func main() { log.Fatal(err) } - plan := entities.Plan{ + plan := model.Plan{ Name: "example-plan", - Output: entities.Output{ + Output: model.Output{ DatasetName: "books", + Verbose: true, }, - Connectors: []entities.Connector{ + Connectors: []model.Connector{ { Name: "local", - Type: "INTERNAL", + Type: "internal", }, }, - Datasets: []entities.Dataset{ + Datasets: []model.Dataset{ { Name: "books", ConnectorName: "local", - Type: "JSON", + Type: "json", Data: []map[string]any{ {"id": 1, "title": "Dune"}, {"id": 2, "title": "Neuromancer"}, }, - Options: entities.JSONDatasetOptions{ - Timeout: &timeout, + Options: &model.JSONDatasetOptions{ + Timeout: 30, }, }, }, } - resp, err := rt.ExecutePlan(context.Background(), entities.ExecutePlanRequest{ + if err := plan.Validate(); err != nil { + log.Fatalf("invalid plan: %v", err) + } + + resp, err := rt.ExecutePlan(context.Background(), model.ExecutePlanRequest{ Plan: plan, }) if err != nil { @@ -147,8 +152,7 @@ type ExecutePlanResponse struct { - Enrich local datasets with external APIs - Query federated datasets using SQL - - +------------------------------------------------------------------------ ## 🎯 Philosophy diff --git a/entities/plan.go b/entities/plan.go deleted file mode 100644 index c52a6f8..0000000 --- a/entities/plan.go +++ /dev/null @@ -1,54 +0,0 @@ -package entities - -type Plan struct { - Name string - Storage Storage - Connectors []Connector - Datasets []Dataset - Output Output -} - -type Storage struct { - Type string - MetadataStorePath string - DataStorePath string -} - -type Connector struct { - Name string - Type string - Connection any - Description string -} - -type Dataset struct { - Name string - ConnectorName string - Type string - Description string - Query string - Options any - Data any - Children []DatasetChild - Dedupe []string - Columns []DataColumn - ColumnOrder []string -} - -type DatasetChild struct { - DatasetName string -} - -type Output struct { - DatasetName string - Verbose bool -} - -type DataColumn struct { - Name string - Accessor string - Alias string - Type string - Description string - IsPrimaryKey bool -} diff --git a/examples/crypto_portfolio/main.go b/examples/crypto_portfolio/main.go index 98c817e..f3c5dbf 100644 --- a/examples/crypto_portfolio/main.go +++ b/examples/crypto_portfolio/main.go @@ -5,7 +5,7 @@ import ( "fmt" "log" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/model" "github.com/kvatch-hub/kvatch-runtime/pkg/runtime" ) @@ -15,24 +15,24 @@ func main() { log.Fatalf("failed to create runtime: %v", err) } - plan := entities.Plan{ + plan := model.Plan{ Name: "crypto_portfolio", - Connectors: []entities.Connector{ + Connectors: []model.Connector{ { Name: "coin_gecko", Type: "api", - Connection: entities.APIConnectorConfig{ + Connection: model.APIConnectorConfig{ BaseURL: "https://api.coingecko.com/api/v3", DefaultHeaders: map[string]string{ "Accept": "application/json", }, - Auth: &entities.APIAuthConfig{ - Type: entities.APIAuthNone, + Auth: &model.APIAuthConfig{ + Type: model.APIAuthNone, }, - RateLimit: &entities.APIRateLimitConfig{ + RateLimit: &model.APIRateLimitConfig{ RequestsPerMinute: 60, }, - Cache: &entities.APICacheConfig{ + Cache: &model.APICacheConfig{ TTL: "1m", }, }, @@ -40,19 +40,19 @@ func main() { { Name: "holdings_sheet", Type: "googlesheet", - Connection: entities.GoogleSheetConnectorConfig{ + Connection: &model.GoogleSheetConnectorConfig{ SpreadsheetID: "1DYEHzASo9D8GHKpTCL_6ia2vDVoTaMSQiLhR2y7TGRQ", ReadRange: "holdings", }, }, }, - Datasets: []entities.Dataset{ + Datasets: []model.Dataset{ { Name: "holdings", Type: "googlesheet", ConnectorName: "holdings_sheet", Query: "holdings", - Options: entities.GoogleSheetDatasetOptions{ + Options: &model.GoogleSheetDatasetOptions{ HeaderRowNo: 1, EnableStreaming: false, }, @@ -62,13 +62,13 @@ func main() { Name: "prices", Type: "api", ConnectorName: "coin_gecko", - Options: entities.APIDatasetOptions{ + Options: &model.APIDatasetOptions{ InjectTimestamp: true, TimestampField: "fetched_at", Vars: map[string]string{ "symbols": "bitcoin,ethereum", }, - Request: entities.APIRequestSpec{ + Request: model.APIRequestSpec{ Method: "GET", Path: "/simple/price", Params: map[string]string{ @@ -78,13 +78,13 @@ func main() { Headers: map[string]string{}, Body: nil, }, - Pagination: entities.APIPaginationConfig{ - Type: entities.APIPaginationNone, + Pagination: model.APIPaginationConfig{ + Type: model.APIPaginationNone, }, - Response: entities.APIResponseSpec{ + Response: model.APIResponseSpec{ Format: "json", Extract: "$", - Normalize: entities.APINormalizeConfig{ + Normalize: model.APINormalizeConfig{ Enabled: true, KeyField: "coin", ValuePrefix: "price_", @@ -97,7 +97,7 @@ func main() { Name: "portfolio_value", Type: "sql", ConnectorName: "federated", - Options: entities.SQLDatasetOptions{ + Options: &model.SQLDatasetOptions{ Query: `SELECT h.coin, h.holding, @@ -108,19 +108,23 @@ func main() { Vars: map[string]string{}, Dedupe: []string{"coin"}, }, - Children: []entities.DatasetChild{ + Children: []model.DatasetChild{ {DatasetName: "holdings"}, {DatasetName: "prices"}, }, }, }, - Output: entities.Output{ + Output: model.Output{ DatasetName: "portfolio_value", Verbose: true, }, } - resp, err := rt.ExecutePlan(context.Background(), entities.ExecutePlanRequest{ + if err := plan.Validate(); err != nil { + log.Fatalf("invalid plan: %v", err) + } + + resp, err := rt.ExecutePlan(context.Background(), model.ExecutePlanRequest{ Plan: plan, }) if err != nil { diff --git a/examples/postgres/.env.example b/examples/postgres/.env.example new file mode 100644 index 0000000..c2bc5d9 --- /dev/null +++ b/examples/postgres/.env.example @@ -0,0 +1,13 @@ +# Books database +BOOKS_POSTGRES_HOST=localhost +BOOKS_POSTGRES_PORT=5432 +BOOKS_POSTGRES_DATABASE=books +BOOKS_POSTGRES_USERNAME=username +BOOKS_POSTGRES_PASSWORD=password + +# Authors database +AUTHORS_POSTGRES_HOST=localhost +AUTHORS_POSTGRES_PORT=5432 +AUTHORS_POSTGRES_DATABASE=authors +AUTHORS_POSTGRES_USERNAME=username +AUTHORS_POSTGRES_PASSWORD=password \ No newline at end of file diff --git a/examples/postgres/README.md b/examples/postgres/README.md new file mode 100644 index 0000000..4556b0f --- /dev/null +++ b/examples/postgres/README.md @@ -0,0 +1,110 @@ +# Postgres Federated Example + +This example demonstrates how to use Kvatch Runtime to: + +- Query data from two separate Postgres databases +- Load both datasets into the runtime +- Join them using the built-in `federated` connector +- Return a combined result set + +--- + +## What it does + +The example reads: + +- `books` from a Postgres database +- `authors` from a different Postgres database + +It then performs a federated SQL join: + +```sql +SELECT a.name, b.title as title, b.review +FROM authors a +JOIN books b ON a.slug = b.author +``` + +This shows that Kvatch can join data across completely separate data sources. + +--- + +## Prerequisites + +You need access to two Postgres databases: + +- A `books` database containing a `books` table +- An `authors` database containing an `authors` table + +--- + +## Setup + +1. Copy the example environment file: + +```bash +cp examples/postgres/.env.example .env +``` + +2. Update `.env` with your database credentials. + +Example: + +```bash +# Books database +BOOKS_POSTGRES_HOST=localhost +BOOKS_POSTGRES_PORT=5432 +BOOKS_POSTGRES_DATABASE=books +BOOKS_POSTGRES_USERNAME=root +BOOKS_POSTGRES_PASSWORD=root + +# Authors database +AUTHORS_POSTGRES_HOST=localhost +AUTHORS_POSTGRES_PORT=5432 +AUTHORS_POSTGRES_DATABASE=authors +AUTHORS_POSTGRES_USERNAME=root +AUTHORS_POSTGRES_PASSWORD=root +``` + +--- + +## Run the example + +From the root of the repository: + +```bash +go run ./examples/postgres +``` + +--- + +## Example output + +```text +Columns: +- name (TEXT) +- title (TEXT) +- review (TEXT) + +Rows: + map[name:Frank Herbert title:Dune review:Classic sci-fi] + map[name:William Gibson title:Neuromancer review:Cyberpunk essential] +``` + +--- + +## Notes + +- Credentials are loaded from environment variables using `.env` +- The `federated` connector is handled internally by the runtime +- Each database has its own independent configuration +- This example is designed for local experimentation and learning + +--- + +## Why this example matters + +This demonstrates a core Kvatch capability: + +> Query and join data across multiple independent systems using a single runtime. + +No ETL. No duplication. Just declarative data access. diff --git a/examples/postgres/config.go b/examples/postgres/config.go new file mode 100644 index 0000000..804282c --- /dev/null +++ b/examples/postgres/config.go @@ -0,0 +1,55 @@ +package main + +import ( + "log" + "os" + "strconv" +) + +type DBConfig struct { + Host string + Port int + Database string + Username string + Password string +} + +func loadConfig(prefix string) DBConfig { + return DBConfig{ + Host: envOrDefault(prefix+"_POSTGRES_HOST", "localhost"), + Port: envIntOrDefault(prefix+"_POSTGRES_PORT", 5432), + Database: requiredEnv(prefix + "_POSTGRES_DATABASE"), + Username: requiredEnv(prefix + "_POSTGRES_USERNAME"), + Password: requiredEnv(prefix + "_POSTGRES_PASSWORD"), + } +} + +func requiredEnv(key string) string { + value := os.Getenv(key) + if value == "" { + log.Fatalf("%s must be set", key) + } + return value +} + +func envOrDefault(key, fallback string) string { + value := os.Getenv(key) + if value == "" { + return fallback + } + return value +} + +func envIntOrDefault(key string, fallback int) int { + value := os.Getenv(key) + if value == "" { + return fallback + } + + parsed, err := strconv.Atoi(value) + if err != nil { + log.Fatalf("%s must be a valid integer", key) + } + + return parsed +} diff --git a/examples/postgres/main.go b/examples/postgres/main.go new file mode 100644 index 0000000..7380f79 --- /dev/null +++ b/examples/postgres/main.go @@ -0,0 +1,122 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/joho/godotenv" + "github.com/kvatch-hub/kvatch-runtime/model" + "github.com/kvatch-hub/kvatch-runtime/pkg/runtime" +) + +func main() { + if err := godotenv.Load(); err != nil { + log.Printf("warning: could not load .env: %v", err) + } + + rt, err := runtime.NewDefault() + if err != nil { + log.Fatalf("create runtime: %v", err) + } + + booksCfg := loadConfig("BOOKS") + authorsCfg := loadConfig("AUTHORS") + + plan := model.Plan{ + Name: "postgres-example", + Output: model.Output{ + DatasetName: "author_books", + Verbose: true, + }, + Connectors: []model.Connector{ + { + Name: "books_db", + Type: model.ConnectorTypePostgres, + Connection: &model.PostgresConnectorConfig{ + Host: booksCfg.Host, + Port: booksCfg.Port, + Database: booksCfg.Database, + Username: booksCfg.Username, + Password: booksCfg.Password, + SSLMode: "disable", + }, + }, + { + Name: "authors_db", + Type: model.ConnectorTypePostgres, + Connection: &model.PostgresConnectorConfig{ + Host: authorsCfg.Host, + Port: authorsCfg.Port, + Database: authorsCfg.Database, + Username: authorsCfg.Username, + Password: authorsCfg.Password, + SSLMode: "disable", + }, + }, + }, + Datasets: []model.Dataset{ + { + Name: "books", + ConnectorName: "books_db", + Type: "sql", + Options: &model.SQLDatasetOptions{ + Query: "SELECT name as title, author, review FROM books", + Timeout: 30, + }, + }, + { + Name: "authors", + ConnectorName: "authors_db", + Type: "sql", + Options: &model.SQLDatasetOptions{ + Query: "SELECT id, slug, name, bio FROM authors", + Timeout: 30, + Dedupe: []string{ + "id", + "slug", + }, + }, + }, + { + Name: "author_books", + ConnectorName: "federated", + Type: "sql", + Options: &model.SQLDatasetOptions{ + Query: "SELECT a.name, b.title as title, b.review FROM authors a JOIN books b ON a.slug = b.author", + Timeout: 30, + Dedupe: []string{ + "id", + "slug", + }, + }, + Children: []model.DatasetChild{ + { + DatasetName: "authors", + }, + { + DatasetName: "books", + }, + }, + }, + }, + } + + if err := plan.Validate(); err != nil { + log.Fatalf("invalid plan: %v", err) + } + + resp, err := rt.ExecutePlan(context.Background(), model.ExecutePlanRequest{ + UserID: "someid", + Plan: plan, + }) + if err != nil { + log.Fatalf("execute plan: %v", err) + } + + fmt.Printf("Columns: %v\n", resp.Columns) + fmt.Printf("Rows:\n") + for _, row := range resp.Data { + fmt.Printf(" %v\n", row) + } +} diff --git a/examples/quickstart/main.go b/examples/quickstart/main.go index bd5a9d6..8701fbf 100644 --- a/examples/quickstart/main.go +++ b/examples/quickstart/main.go @@ -5,47 +5,49 @@ import ( "fmt" "log" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/model" "github.com/kvatch-hub/kvatch-runtime/pkg/runtime" ) -var timeout = 30 - func main() { rt, err := runtime.NewDefault() if err != nil { log.Fatalf("create runtime: %v", err) } - plan := entities.Plan{ - Name: "example-plan", - Output: entities.Output{ + plan := model.Plan{ + Name: "quickstart", + Output: model.Output{ DatasetName: "books", Verbose: true, }, - Connectors: []entities.Connector{ + Connectors: []model.Connector{ { Name: "local", - Type: "INTERNAL", + Type: "internal", }, }, - Datasets: []entities.Dataset{ + Datasets: []model.Dataset{ { Name: "books", ConnectorName: "local", - Type: "JSON", + Type: "json", Data: []map[string]any{ {"id": 1, "title": "Dune"}, {"id": 2, "title": "Neuromancer"}, }, - Options: entities.JSONDatasetOptions{ - Timeout: &timeout, + Options: &model.JSONDatasetOptions{ + Timeout: 30, }, }, }, } - resp, err := rt.ExecutePlan(context.Background(), entities.ExecutePlanRequest{ + if err := plan.Validate(); err != nil { + log.Fatalf("invalid plan: %v", err) + } + + resp, err := rt.ExecutePlan(context.Background(), model.ExecutePlanRequest{ UserID: "someid", Plan: plan, }) diff --git a/go.mod b/go.mod index e216079..774c582 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/glebarez/sqlite v1.11.0 github.com/go-git/go-git/v5 v5.17.1 github.com/google/uuid v1.6.0 + github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.12.1 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.11.1 diff --git a/go.sum b/go.sum index 94bc3c2..244d339 100644 --- a/go.sum +++ b/go.sum @@ -121,6 +121,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= diff --git a/internal/connectors/api/connector.go b/internal/connectors/api/connector.go index 3db4559..af4edd2 100644 --- a/internal/connectors/api/connector.go +++ b/internal/connectors/api/connector.go @@ -11,14 +11,15 @@ import ( "strings" "time" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/encoders" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" ) // APIConnector implements the connectors.Connector interface for API. type APIConnector struct { - config entities.APIConnectorConfig + config model.APIConnectorConfig client *http.Client } @@ -28,8 +29,8 @@ func NewAPIConnector() connectors.Connector { } } -func (c *APIConnector) GetType() connectors.ConnectorType { - return connectors.ConnectorTypeAPI +func (c *APIConnector) GetType() model.ConnectorType { + return model.ConnectorTypeAPI } func (c *APIConnector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error { @@ -71,7 +72,7 @@ func (c *APIConnector) GetData( ctx context.Context, ds *models.EngineContextDataset, ) (io.ReadCloser, error) { - opts, err := entities.DecodeDatasetOptions[entities.APIDatasetOptions](ds.DatasetOptions) + opts, err := encoders.DecodeDatasetOptions[model.APIDatasetOptions](ds.DatasetOptions) if err != nil { return nil, fmt.Errorf("api: invalid dataset options: %w", err) } @@ -107,14 +108,14 @@ func (c *APIConnector) GetData( // // parseConfig converts various config representations into APIConnectorConfig. -func parseConfig(config interface{}) (entities.APIConnectorConfig, error) { +func parseConfig(config interface{}) (model.APIConnectorConfig, error) { switch val := config.(type) { - case entities.APIConnectorConfig: + case model.APIConnectorConfig: return val, nil - case *entities.APIConnectorConfig: + case *model.APIConnectorConfig: return *val, nil case map[string]interface{}: - var cfg entities.APIConnectorConfig + var cfg model.APIConnectorConfig buf, err := json.Marshal(val) if err != nil { return cfg, fmt.Errorf("marshal api config: %w", err) @@ -124,11 +125,11 @@ func parseConfig(config interface{}) (entities.APIConnectorConfig, error) { } return cfg, nil default: - return entities.APIConnectorConfig{}, fmt.Errorf("invalid config type for api connector: %T", config) + return model.APIConnectorConfig{}, fmt.Errorf("invalid config type for api connector: %T", config) } } -// func parseAPIOptions(raw map[string]interface{}) (*entities.APIDatasetOptions, error) { +// func parseAPIOptions(raw map[string]interface{}) (*model.APIDatasetOptions, error) { // if raw == nil { // return nil, fmt.Errorf("missing api dataset options") // } @@ -139,7 +140,7 @@ func parseConfig(config interface{}) (entities.APIConnectorConfig, error) { // return nil, err // } -// var out entities.APIDatasetOptions +// var out model.APIDatasetOptions // if err := json.Unmarshal(b, &out); err != nil { // return nil, err // } @@ -151,7 +152,7 @@ func parseConfig(config interface{}) (entities.APIConnectorConfig, error) { // template params, and optional pagination state. func (c *APIConnector) BuildRequest( ctx context.Context, - reqCfg *entities.APIRequestSpec, + reqCfg *model.APIRequestSpec, query_vars map[string]string, page *paginationState, ) (*http.Request, error) { @@ -195,7 +196,7 @@ func (c *APIConnector) BuildRequest( if page != nil { switch page.Options.Type { - case entities.APIPaginationPage: + case model.APIPaginationPage: if page.Options.PageParam != "" { q.Set(page.Options.PageParam, fmt.Sprintf("%d", page.Page)) } @@ -203,7 +204,7 @@ func (c *APIConnector) BuildRequest( q.Set(page.Options.PageSizeParam, fmt.Sprintf("%d", page.Options.PageSize)) } - case entities.APIPaginationOffset: + case model.APIPaginationOffset: if page.Options.OffsetParam != "" { q.Set(page.Options.OffsetParam, fmt.Sprintf("%d", page.Offset)) } @@ -211,7 +212,7 @@ func (c *APIConnector) BuildRequest( q.Set(page.Options.LimitParam, fmt.Sprintf("%d", page.Options.Limit)) } - case entities.APIPaginationCursor: + case model.APIPaginationCursor: if page.Options.CursorParam != "" && page.Cursor != "" { q.Set(page.Options.CursorParam, page.Cursor) } @@ -262,17 +263,17 @@ func (c *APIConnector) BuildRequest( if c.config.Auth != nil { switch c.config.Auth.Type { - case entities.APIAuthAPIKey: + case model.APIAuthAPIKey: if c.config.Auth.APIKeyHeader != "" && c.config.Auth.APIKeyValue != "" { req.Header.Set(c.config.Auth.APIKeyHeader, c.config.Auth.APIKeyValue) } - case entities.APIAuthBearer: + case model.APIAuthBearer: if c.config.Auth.BearerToken != "" { req.Header.Set("Authorization", "Bearer "+c.config.Auth.BearerToken) } - case entities.APIAuthBasic: + case model.APIAuthBasic: // future support } } diff --git a/internal/connectors/api/pagination.go b/internal/connectors/api/pagination.go index 643dd15..f79f500 100644 --- a/internal/connectors/api/pagination.go +++ b/internal/connectors/api/pagination.go @@ -4,30 +4,30 @@ import ( "encoding/json" "fmt" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/model" ) type paginationState struct { - Options entities.APIPaginationConfig + Options model.APIPaginationConfig Page int Cursor string Offset int Limit int } -func (p *paginationState) Advance(body []byte, cfg entities.APIPaginationConfig) (bool, error) { +func (p *paginationState) Advance(body []byte, cfg model.APIPaginationConfig) (bool, error) { switch cfg.Type { - case entities.APIPaginationNone: + case model.APIPaginationNone: return true, nil - case entities.APIPaginationPage: + case model.APIPaginationPage: p.Page++ if p.Page >= cfg.MaxPages && cfg.MaxPages > 0 { return true, nil } return false, nil - case entities.APIPaginationCursor: + case model.APIPaginationCursor: var parsed map[string]interface{} if err := json.Unmarshal(body, &parsed); err != nil { return false, err diff --git a/internal/connectors/aws_s3/connector.go b/internal/connectors/aws_s3/connector.go index edbb043..b7d16b2 100644 --- a/internal/connectors/aws_s3/connector.go +++ b/internal/connectors/aws_s3/connector.go @@ -10,9 +10,9 @@ import ( "path" "strings" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/connectors" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" awsCfg "github.com/aws/aws-sdk-go-v2/config" awsCreds "github.com/aws/aws-sdk-go-v2/credentials" @@ -30,8 +30,8 @@ type S3Connector struct { func NewS3Connector() *S3Connector { return &S3Connector{} } -func (c *S3Connector) GetType() connectors.ConnectorType { - return connectors.ConnectorTypeS3 +func (c *S3Connector) GetType() model.ConnectorType { + return model.ConnectorTypeS3 } func (c *S3Connector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error { @@ -42,11 +42,11 @@ func (c *S3Connector) Connect(ctx context.Context, engineConnectorCtx *models.En return fmt.Errorf("engine connector context options are missing") } - var cfg entities.S3ConnectorConfig + var cfg model.S3ConnectorConfig switch v := engineConnectorCtx.Config.(type) { - case entities.S3ConnectorConfig: + case model.S3ConnectorConfig: cfg = v - case *entities.S3ConnectorConfig: + case *model.S3ConnectorConfig: cfg = *v case map[string]interface{}: // tolerate yaml -> map[string]any @@ -219,9 +219,9 @@ func (c *S3Connector) Close() error { func (c *S3Connector) Validate(config interface{}) error { switch v := config.(type) { - case entities.S3ConnectorConfig: + case model.S3ConnectorConfig: return c.validateCfg(v) - case *entities.S3ConnectorConfig: + case *model.S3ConnectorConfig: if v == nil { return fmt.Errorf("nil s3 config") } @@ -250,7 +250,7 @@ func (c *S3Connector) Validate(config interface{}) error { } } -func (c *S3Connector) validateCfg(cfg entities.S3ConnectorConfig) error { +func (c *S3Connector) validateCfg(cfg model.S3ConnectorConfig) error { // If explicit keys are partially provided, error early if (cfg.AccessKeyID != "" && cfg.SecretAccessKey == "") || (cfg.AccessKeyID == "" && cfg.SecretAccessKey != "") { return fmt.Errorf("both access_key_id and secret_access_key must be provided together") diff --git a/internal/connectors/config_decoder.go b/internal/connectors/config_decoder.go index fb3efc7..a3c1aac 100644 --- a/internal/connectors/config_decoder.go +++ b/internal/connectors/config_decoder.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/model" ) func DecodeAndValidateConfig(connectorType string, raw json.RawMessage, mask bool) (interface{}, error) { @@ -30,41 +30,41 @@ func DecodeAndValidateConfig(connectorType string, raw json.RawMessage, mask boo return target, nil } - switch strings.ToUpper(connectorType) { - case string(ConnectorTypePostgres): - var cfg entities.PostgresConnectorConfig + switch strings.ToLower(connectorType) { + case string(model.ConnectorTypePostgres): + var cfg model.PostgresConnectorConfig return decode(&cfg, cfg.Validate) - case string(ConnectorTypeSQLite): - var cfg entities.SQLiteConnectorConfig + case string(model.ConnectorTypeSQLite): + var cfg model.SqliteConnectorConfig return decode(&cfg, nil) - case string(ConnectorTypeLocalFile): - var cfg entities.LocalFileConnectorConfig + case string(model.ConnectorTypeLocalFile): + var cfg model.LocalFileConnectorConfig return decode(&cfg, nil) - case string(ConnectorTypeLocalDirectory): - var cfg entities.DirectoryConnectorConfig + case string(model.ConnectorTypeLocalDirectory): + var cfg model.DirectoryConnectorConfig return decode(&cfg, nil) - case string(ConnectorTypeGoogleSheet): - var cfg entities.GoogleSheetConnectorConfig + case string(model.ConnectorTypeGoogleSheet): + var cfg model.GoogleSheetConnectorConfig return decode(&cfg, cfg.Validate) - case string(ConnectorTypeGoogleAPI): - var cfg entities.GoogleAPIConnectorConfig + case string(model.ConnectorTypeGoogleAPI): + var cfg model.GoogleAPIConnectorConfig return decode(&cfg, nil) - case string(ConnectorTypeGit): - var cfg entities.GitConnectorConfig + case string(model.ConnectorTypeGit): + var cfg model.GitConnectorConfig return decode(&cfg, nil) - case string(ConnectorTypeS3): - var cfg entities.S3ConnectorConfig + case string(model.ConnectorTypeS3): + var cfg model.S3ConnectorConfig return decode(&cfg, nil) - case string(ConnectorTypeAPI): - var cfg entities.APIConnectorConfig + case string(model.ConnectorTypeAPI): + var cfg model.APIConnectorConfig return decode(&cfg, func() error { return cfg.Validate() }) diff --git a/internal/connectors/git/connector.go b/internal/connectors/git/connector.go index 7b87eb6..7c6aea4 100644 --- a/internal/connectors/git/connector.go +++ b/internal/connectors/git/connector.go @@ -13,9 +13,9 @@ import ( "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/connectors" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" ) type GitConnector struct { @@ -29,8 +29,8 @@ func NewGitConnector() connectors.Connector { return &GitConnector{} } -func (c *GitConnector) GetType() connectors.ConnectorType { - return connectors.ConnectorTypeGit +func (c *GitConnector) GetType() model.ConnectorType { + return model.ConnectorTypeGit } func (c *GitConnector) Ping(ctx context.Context, config interface{}) error { @@ -207,12 +207,12 @@ func (c *GitConnector) Close() error { } func (c *GitConnector) Validate(config interface{}) error { - var cfg *entities.GitConnectorConfig + var cfg *model.GitConnectorConfig switch v := config.(type) { - case *entities.GitConnectorConfig: + case *model.GitConnectorConfig: cfg = v - case entities.GitConnectorConfig: + case model.GitConnectorConfig: cfg = &v case map[string]interface{}: _, ok := v["repo"].(string) @@ -229,7 +229,7 @@ func (c *GitConnector) Validate(config interface{}) error { return nil } -func (c *GitConnector) setPaths(cfg *entities.GitConnectorConfig, cachePath string) error { +func (c *GitConnector) setPaths(cfg *model.GitConnectorConfig, cachePath string) error { c.repoURL = cfg.Repo c.branch = cfg.Branch c.subPath = strings.Trim(cfg.Path, "/") @@ -261,14 +261,14 @@ func printDirRecursive(path string) { }) } -func parseConfig(config interface{}) (*entities.GitConnectorConfig, error) { +func parseConfig(config interface{}) (*model.GitConnectorConfig, error) { switch v := config.(type) { - case *entities.GitConnectorConfig: + case *model.GitConnectorConfig: return v, nil - case entities.GitConnectorConfig: + case model.GitConnectorConfig: return &v, nil case map[string]interface{}: - cfg := &entities.GitConnectorConfig{} + cfg := &model.GitConnectorConfig{} if repo, ok := v["repo"].(string); ok { cfg.Repo = repo } @@ -281,7 +281,7 @@ func parseConfig(config interface{}) (*entities.GitConnectorConfig, error) { return cfg, nil case string: // Interpret as repo URL only - return &entities.GitConnectorConfig{Repo: v, Branch: "main"}, nil + return &model.GitConnectorConfig{Repo: v, Branch: "main"}, nil default: return nil, fmt.Errorf("invalid config type for GitConnector: %T", config) } diff --git a/internal/connectors/googlesheets/connector.go b/internal/connectors/googlesheets/connector.go index 3d08b49..c234247 100644 --- a/internal/connectors/googlesheets/connector.go +++ b/internal/connectors/googlesheets/connector.go @@ -9,9 +9,9 @@ import ( "net/http" "strings" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/connectors" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" "google.golang.org/api/option" "google.golang.org/api/sheets/v4" ) @@ -27,8 +27,8 @@ func NewGoogleSheetConnector() connectors.Connector { return &GoogleSheetConnector{} } -func (c *GoogleSheetConnector) GetType() connectors.ConnectorType { - return connectors.ConnectorTypeGoogleSheet +func (c *GoogleSheetConnector) GetType() model.ConnectorType { + return model.ConnectorTypeGoogleSheet } // Ping checks whether the spreadsheet is reachable and readable. @@ -211,14 +211,14 @@ func (c *GoogleSheetConnector) Close() error { func (c *GoogleSheetConnector) Validate(config interface{}) error { switch v := config.(type) { - case entities.GoogleSheetConnectorConfig: + case model.GoogleSheetConnectorConfig: if v.SpreadsheetID == "" { return fmt.Errorf("missing spreadsheet_id") } // if v.ReadRange == "" { // return fmt.Errorf("missing read_range") // } - case *entities.GoogleSheetConnectorConfig: + case *model.GoogleSheetConnectorConfig: if v.SpreadsheetID == "" { return fmt.Errorf("missing spreadsheet_id") } @@ -238,14 +238,14 @@ func (c *GoogleSheetConnector) Validate(config interface{}) error { return nil } -func parseConfig(config interface{}) (*entities.GoogleSheetConnectorConfig, error) { +func parseConfig(config interface{}) (*model.GoogleSheetConnectorConfig, error) { switch v := config.(type) { - case *entities.GoogleSheetConnectorConfig: + case *model.GoogleSheetConnectorConfig: return v, nil - case entities.GoogleSheetConnectorConfig: + case model.GoogleSheetConnectorConfig: return &v, nil case map[string]interface{}: - cfg := &entities.GoogleSheetConnectorConfig{} + cfg := &model.GoogleSheetConnectorConfig{} if id, ok := v["spreadsheet_id"].(string); ok { cfg.SpreadsheetID = id } @@ -261,7 +261,7 @@ func parseConfig(config interface{}) (*entities.GoogleSheetConnectorConfig, erro return cfg, nil case string: // treat as spreadsheetID in public mode - return &entities.GoogleSheetConnectorConfig{ + return &model.GoogleSheetConnectorConfig{ SpreadsheetID: v, ReadRange: "Sheet1", // sensible default }, nil diff --git a/internal/connectors/localdirectory/connector.go b/internal/connectors/localdirectory/connector.go index 77e752c..d91bf56 100644 --- a/internal/connectors/localdirectory/connector.go +++ b/internal/connectors/localdirectory/connector.go @@ -8,9 +8,9 @@ import ( "path/filepath" "strings" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/connectors" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" ) // DirectoryConnector handles local file system access @@ -24,8 +24,8 @@ func NewDirectoryConnector() connectors.Connector { } // GetType returns the connection type -func (c *DirectoryConnector) GetType() connectors.ConnectorType { - return connectors.ConnectorTypeLocalDirectory +func (c *DirectoryConnector) GetType() model.ConnectorType { + return model.ConnectorTypeLocalDirectory } func (c *DirectoryConnector) Ping(ctx context.Context, config interface{}) error { @@ -167,11 +167,11 @@ func (c *DirectoryConnector) Close() error { // Validate validates the local file connection configuration func (c *DirectoryConnector) Validate(config interface{}) error { switch cfg := config.(type) { - case *entities.DirectoryConnectorConfig: + case *model.DirectoryConnectorConfig: if cfg.DirectoryPath == "" { return fmt.Errorf("directory connector requires directory_path") } - case entities.DirectoryConnectorConfig: + case model.DirectoryConnectorConfig: if cfg.DirectoryPath == "" { return fmt.Errorf("directory connector requires directory_path") } @@ -189,22 +189,22 @@ func (c *DirectoryConnector) Validate(config interface{}) error { return nil } -func parseConfig(config interface{}) (*entities.DirectoryConnectorConfig, error) { +func parseConfig(config interface{}) (*model.DirectoryConnectorConfig, error) { switch v := config.(type) { case string: // Allow passing just a path string - return &entities.DirectoryConnectorConfig{DirectoryPath: v}, nil - case *entities.DirectoryConnectorConfig: + return &model.DirectoryConnectorConfig{DirectoryPath: v}, nil + case *model.DirectoryConnectorConfig: return v, nil - case entities.DirectoryConnectorConfig: + case model.DirectoryConnectorConfig: return &v, nil case map[string]interface{}: // Accept either "directory_path" or "base_path" if path, ok := v["directory_path"].(string); ok && strings.TrimSpace(path) != "" { - return &entities.DirectoryConnectorConfig{DirectoryPath: path}, nil + return &model.DirectoryConnectorConfig{DirectoryPath: path}, nil } if path, ok := v["base_path"].(string); ok && strings.TrimSpace(path) != "" { - return &entities.DirectoryConnectorConfig{DirectoryPath: path}, nil + return &model.DirectoryConnectorConfig{DirectoryPath: path}, nil } return nil, fmt.Errorf("invalid local directory config: missing or empty 'directory_path'") default: diff --git a/internal/connectors/localfile/connector.go b/internal/connectors/localfile/connector.go index dd63a67..149043b 100644 --- a/internal/connectors/localfile/connector.go +++ b/internal/connectors/localfile/connector.go @@ -8,9 +8,9 @@ import ( "path/filepath" "strings" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/connectors" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" ) // LocalFileConnector handles loading a single local file. @@ -24,8 +24,8 @@ func NewLocalFileConnector() connectors.Connector { } // GetType returns the connector type. -func (c *LocalFileConnector) GetType() connectors.ConnectorType { - return connectors.ConnectorTypeLocalFile +func (c *LocalFileConnector) GetType() model.ConnectorType { + return model.ConnectorTypeLocalFile } func (c *LocalFileConnector) Ping(ctx context.Context, config interface{}) error { @@ -184,12 +184,12 @@ func (c *LocalFileConnector) Close() error { func (c *LocalFileConnector) Validate(config interface{}) error { switch cfg := config.(type) { - case *entities.LocalFileConnectorConfig: + case *model.LocalFileConnectorConfig: if strings.TrimSpace(cfg.FilePath) == "" { return fmt.Errorf("local file connection requires file_path") } - case entities.LocalFileConnectorConfig: + case model.LocalFileConnectorConfig: if strings.TrimSpace(cfg.FilePath) == "" { return fmt.Errorf("local file connection requires file_path") } @@ -212,18 +212,18 @@ func (c *LocalFileConnector) Validate(config interface{}) error { return nil } -func parseConfig(config interface{}) (*entities.LocalFileConnectorConfig, error) { +func parseConfig(config interface{}) (*model.LocalFileConnectorConfig, error) { switch v := config.(type) { case string: - return &entities.LocalFileConnectorConfig{ + return &model.LocalFileConnectorConfig{ FilePath: strings.TrimSpace(v), }, nil - case *entities.LocalFileConnectorConfig: + case *model.LocalFileConnectorConfig: return v, nil - case entities.LocalFileConnectorConfig: + case model.LocalFileConnectorConfig: return &v, nil case map[string]interface{}: @@ -231,7 +231,7 @@ func parseConfig(config interface{}) (*entities.LocalFileConnectorConfig, error) if !ok || fp == "" { return nil, fmt.Errorf("invalid localfile config: missing or empty file_path") } - return &entities.LocalFileConnectorConfig{FilePath: fp}, nil + return &model.LocalFileConnectorConfig{FilePath: fp}, nil default: return nil, fmt.Errorf("invalid config type for LocalFileConnector: %T", config) diff --git a/internal/connectors/main.go b/internal/connectors/main.go index 982e8aa..50cbac7 100644 --- a/internal/connectors/main.go +++ b/internal/connectors/main.go @@ -5,22 +5,7 @@ import ( "io" "github.com/kvatch-hub/kvatch-runtime/internal/models" -) - -// ConnectionType represents the type of connection -type ConnectorType string - -const ( - ConnectorTypePostgres ConnectorType = "POSTGRES" - ConnectorTypeSQLite ConnectorType = "SQLITE" - ConnectorTypeLocalFile ConnectorType = "LOCALFILE" - ConnectorTypeLocalDirectory ConnectorType = "LOCAL_DIRECTORY" - ConnectorTypeGoogleAPI ConnectorType = "GOOGLE_API" - ConnectorTypeGit ConnectorType = "GIT" - ConnectorTypeGoogleSheet ConnectorType = "GOOGLESHEET" - ConnectorTypeS3 ConnectorType = "S3" - ConnectorTypeAPI ConnectorType = "API" - ConnectorTypeFederated ConnectorType = "JOIN" + "github.com/kvatch-hub/kvatch-runtime/model" ) // Connector defines the interface for all connection types @@ -35,7 +20,7 @@ type Connector interface { Close() error // GetType returns the connection type - GetType() ConnectorType + GetType() model.ConnectorType // Validate validates the connection configuration Validate(config interface{}) error @@ -45,31 +30,31 @@ type Connector interface { // Registry manages available connections type Registry interface { - Register(connectionType ConnectorType, connection Connector) error - Get(connectionType ConnectorType) (Connector, error) - List() []ConnectorType + Register(connectionType model.ConnectorType, connection Connector) error + Get(connectionType model.ConnectorType) (Connector, error) + List() []model.ConnectorType } // ConnectorRegistry implements the Registry interface type ConnectorRegistry struct { - connectors map[ConnectorType]Connector + connectors map[model.ConnectorType]Connector } // NewConnectorRegistry creates a new connection registry func NewConnectorRegistry() *ConnectorRegistry { return &ConnectorRegistry{ - connectors: make(map[ConnectorType]Connector), + connectors: make(map[model.ConnectorType]Connector), } } // Register registers a new connection type -func (r *ConnectorRegistry) Register(connectionType ConnectorType, connection Connector) error { +func (r *ConnectorRegistry) Register(connectionType model.ConnectorType, connection Connector) error { r.connectors[connectionType] = connection return nil } // Get retrieves a connection by type -func (r *ConnectorRegistry) Get(connectionType ConnectorType) (Connector, error) { +func (r *ConnectorRegistry) Get(connectionType model.ConnectorType) (Connector, error) { connection, exists := r.connectors[connectionType] if !exists { return nil, &ConnectionNotFoundError{ConnectionType: connectionType} @@ -78,8 +63,8 @@ func (r *ConnectorRegistry) Get(connectionType ConnectorType) (Connector, error) } // List returns all registered connection types -func (r *ConnectorRegistry) List() []ConnectorType { - types := make([]ConnectorType, 0, len(r.connectors)) +func (r *ConnectorRegistry) List() []model.ConnectorType { + types := make([]model.ConnectorType, 0, len(r.connectors)) for t := range r.connectors { types = append(types, t) } @@ -88,7 +73,7 @@ func (r *ConnectorRegistry) List() []ConnectorType { // ConnectionNotFoundError represents an error when a connection type is not found type ConnectionNotFoundError struct { - ConnectionType ConnectorType + ConnectionType model.ConnectorType } func (e *ConnectionNotFoundError) Error() string { @@ -97,7 +82,7 @@ func (e *ConnectionNotFoundError) Error() string { // ResourceNotFoundError represents an error when a resource is not found in a connection type ResourceNotFoundError struct { - ConnectionType ConnectorType + ConnectionType model.ConnectorType ResourcePath string } diff --git a/internal/connectors/mocks/mock_interfaces.go b/internal/connectors/mocks/mock_interfaces.go index 5e594b0..0be597a 100644 --- a/internal/connectors/mocks/mock_interfaces.go +++ b/internal/connectors/mocks/mock_interfaces.go @@ -16,6 +16,7 @@ import ( connectors "github.com/kvatch-hub/kvatch-runtime/internal/connectors" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" gomock "go.uber.org/mock/gomock" ) @@ -87,10 +88,10 @@ func (mr *MockConnectorMockRecorder) GetData(ctx, engineCtx any) *gomock.Call { } // GetType mocks base method. -func (m *MockConnector) GetType() connectors.ConnectorType { +func (m *MockConnector) GetType() model.ConnectorType { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetType") - ret0, _ := ret[0].(connectors.ConnectorType) + ret0, _ := ret[0].(model.ConnectorType) return ret0 } @@ -153,7 +154,7 @@ func (m *MockRegistry) EXPECT() *MockRegistryMockRecorder { } // Get mocks base method. -func (m *MockRegistry) Get(connectionType connectors.ConnectorType) (connectors.Connector, error) { +func (m *MockRegistry) Get(connectionType model.ConnectorType) (connectors.Connector, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", connectionType) ret0, _ := ret[0].(connectors.Connector) @@ -168,10 +169,10 @@ func (mr *MockRegistryMockRecorder) Get(connectionType any) *gomock.Call { } // List mocks base method. -func (m *MockRegistry) List() []connectors.ConnectorType { +func (m *MockRegistry) List() []model.ConnectorType { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "List") - ret0, _ := ret[0].([]connectors.ConnectorType) + ret0, _ := ret[0].([]model.ConnectorType) return ret0 } @@ -182,7 +183,7 @@ func (mr *MockRegistryMockRecorder) List() *gomock.Call { } // Register mocks base method. -func (m *MockRegistry) Register(connectionType connectors.ConnectorType, connection connectors.Connector) error { +func (m *MockRegistry) Register(connectionType model.ConnectorType, connection connectors.Connector) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Register", connectionType, connection) ret0, _ := ret[0].(error) diff --git a/internal/connectors/postgres/connector.go b/internal/connectors/postgres/connector.go index 82d70c1..98fa163 100644 --- a/internal/connectors/postgres/connector.go +++ b/internal/connectors/postgres/connector.go @@ -8,10 +8,10 @@ import ( "strings" "time" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/connectors" "github.com/kvatch-hub/kvatch-runtime/internal/encoders" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" "github.com/kvatch-hub/kvatch-runtime/pkg/utils" _ "github.com/lib/pq" // PostgreSQL driver @@ -39,8 +39,8 @@ func NewPostgresConnector() connectors.Connector { } // GetType returns the connection type -func (c *PostgresConnector) GetType() connectors.ConnectorType { - return connectors.ConnectorTypePostgres +func (c *PostgresConnector) GetType() model.ConnectorType { + return model.ConnectorTypePostgres } func (c *PostgresConnector) Ping(ctx context.Context, config interface{}) error { @@ -112,7 +112,7 @@ func (c *PostgresConnector) GetData(ctx context.Context, engineCtx *models.Engin return nil, fmt.Errorf("dataset plan is missing") } - opts, err := entities.DecodeDatasetOptions[entities.SQLDatasetOptions]( + opts, err := encoders.DecodeDatasetOptions[model.SQLDatasetOptions]( engineCtx.DatasetOptions, ) if err != nil { @@ -176,11 +176,11 @@ func (c *PostgresConnector) Validate(config interface{}) error { return cfg.Validate() } -func parseConfig(config interface{}) (*entities.PostgresConnectorConfig, error) { +func parseConfig(config interface{}) (*model.PostgresConnectorConfig, error) { switch v := config.(type) { - case *entities.PostgresConnectorConfig: + case *model.PostgresConnectorConfig: return v, nil - case entities.PostgresConnectorConfig: + case model.PostgresConnectorConfig: return &v, nil case map[string]interface{}: // Map parsing is moved to a clean helper function for clarity @@ -190,8 +190,8 @@ func parseConfig(config interface{}) (*entities.PostgresConnectorConfig, error) } } -func parseConfigFromMap(v map[string]interface{}) (*entities.PostgresConnectorConfig, error) { - cfg := &entities.PostgresConnectorConfig{} +func parseConfigFromMap(v map[string]interface{}) (*model.PostgresConnectorConfig, error) { + cfg := &model.PostgresConnectorConfig{} // Use a utility function for safe type assertion/conversion (assumed utility) cfg.Host = utils.GetString(v, "host") diff --git a/internal/connectors/postgres/connector_test.go b/internal/connectors/postgres/connector_test.go index 5136e30..a56fe22 100644 --- a/internal/connectors/postgres/connector_test.go +++ b/internal/connectors/postgres/connector_test.go @@ -9,9 +9,9 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/encoders" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" "github.com/stretchr/testify/assert" ) @@ -21,7 +21,7 @@ func TestPostgresConnector_Connect_Invalid(t *testing.T) { // err := conn.Connect(context.Background(), "invalid://connstring") err := conn.Connect(context.Background(), &models.EngineContextConnector{ - Config: entities.PostgresConnectorConfig{}, + Config: model.PostgresConnectorConfig{}, }) assert.Error(t, err) } @@ -128,7 +128,7 @@ func Test_parseConfigFromMap(t *testing.T) { tests := []struct { name string input map[string]interface{} - wantCfg *entities.PostgresConnectorConfig + wantCfg *model.PostgresConnectorConfig wantErr bool }{ { @@ -141,7 +141,7 @@ func Test_parseConfigFromMap(t *testing.T) { "password": "test-pass", "ssl_mode": "disable", }, - wantCfg: &entities.PostgresConnectorConfig{ + wantCfg: &model.PostgresConnectorConfig{ Host: "test-host", Port: 5433, Database: "test-db", @@ -161,7 +161,7 @@ func Test_parseConfigFromMap(t *testing.T) { "password": "test-pass", "sslMode": "require", // CamelCase SSL }, - wantCfg: &entities.PostgresConnectorConfig{ + wantCfg: &model.PostgresConnectorConfig{ Host: "test-host", Port: 5433, Database: "test-db", @@ -181,7 +181,7 @@ func Test_parseConfigFromMap(t *testing.T) { "password": "test-pass", "sslmode": "verify-full", // Lowercase SSL }, - wantCfg: &entities.PostgresConnectorConfig{ + wantCfg: &model.PostgresConnectorConfig{ Host: "test-host", Port: 5433, Database: "test-db", @@ -212,7 +212,7 @@ func Test_parseConfigFromMap(t *testing.T) { func Test_parseConfig(t *testing.T) { // A fully defined struct for comparison - fullConfig := entities.PostgresConnectorConfig{ + fullConfig := model.PostgresConnectorConfig{ Host: "localhost", Port: 5432, Database: "testdb", @@ -241,7 +241,7 @@ func Test_parseConfig(t *testing.T) { }, { name: "Input_DSN_String", - input: entities.PostgresConnectorConfig{ + input: model.PostgresConnectorConfig{ ConnectionString: dsnString, }, wantErr: false, diff --git a/internal/connectors/sqlite/connector.go b/internal/connectors/sqlite/connector.go index e89e378..f81a459 100644 --- a/internal/connectors/sqlite/connector.go +++ b/internal/connectors/sqlite/connector.go @@ -10,10 +10,10 @@ import ( "sync" "time" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/connectors" "github.com/kvatch-hub/kvatch-runtime/internal/encoders" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" ) type ( @@ -44,8 +44,8 @@ func NewSQLiteConnector() connectors.Connector { } // GetType returns the connector type -func (c *SQLiteConnector) GetType() connectors.ConnectorType { - return connectors.ConnectorTypeSQLite +func (c *SQLiteConnector) GetType() model.ConnectorType { + return model.ConnectorTypeSQLite } // Connect establishes the connector to SQLite @@ -144,7 +144,7 @@ func (c *SQLiteConnector) GetData(ctx context.Context, engineCtx *models.EngineC return nil, fmt.Errorf("dataset plan is missing") } - opts, err := entities.DecodeDatasetOptions[entities.SQLDatasetOptions]( + opts, err := encoders.DecodeDatasetOptions[model.SQLDatasetOptions]( engineCtx.DatasetOptions, ) if err != nil { @@ -205,11 +205,11 @@ func (c *SQLiteConnector) Close() error { // Validate validates the SQLite connection configuration func (c *SQLiteConnector) Validate(config interface{}) error { switch cfg := config.(type) { - case *entities.SQLiteConnectorConfig: + case *model.SqliteConnectorConfig: if cfg.Path == "" { return fmt.Errorf("SQLite connection requires database path") } - case entities.SQLiteConnectorConfig: + case model.SqliteConnectorConfig: if cfg.Path == "" { return fmt.Errorf("SQLite connection requires database path") } @@ -227,18 +227,18 @@ func (c *SQLiteConnector) Validate(config interface{}) error { return nil } -func parseConfig(config interface{}) (*entities.SQLiteConnectorConfig, error) { +func parseConfig(config interface{}) (*model.SqliteConnectorConfig, error) { switch v := config.(type) { case string: // Allow passing just a path string - return &entities.SQLiteConnectorConfig{Path: v}, nil - case *entities.SQLiteConnectorConfig: + return &model.SqliteConnectorConfig{Path: v}, nil + case *model.SqliteConnectorConfig: return v, nil - case entities.SQLiteConnectorConfig: + case model.SqliteConnectorConfig: return &v, nil case map[string]interface{}: if path, ok := v["path"].(string); ok && strings.TrimSpace(path) != "" { - return &entities.SQLiteConnectorConfig{Path: path}, nil + return &model.SqliteConnectorConfig{Path: path}, nil } return nil, fmt.Errorf("invalid sqlite config: missing or empty 'path'") default: diff --git a/internal/connectors/sqlite/connector_test.go b/internal/connectors/sqlite/connector_test.go index 677db5d..c44211c 100644 --- a/internal/connectors/sqlite/connector_test.go +++ b/internal/connectors/sqlite/connector_test.go @@ -7,8 +7,8 @@ import ( "io" "testing" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" "github.com/stretchr/testify/assert" _ "github.com/glebarez/sqlite" @@ -72,7 +72,7 @@ func TestSQLiteConnector_GetData_NonStreaming(t *testing.T) { // conn := NewSQLiteConnector() // SetTestConnection(conn, db, ":memory:") -// plan := &entities.DatasetPlan{ +// plan := &model.DatasetPlan{ // Query: "SELECT * FROM test", // Options: map[string]interface{}{ // "streaming": true, @@ -94,9 +94,9 @@ func TestSQLiteConnector_GetData_NonStreaming(t *testing.T) { func TestSQLiteConnector_Connect_ValidConfig(t *testing.T) { conn := NewSQLiteConnector() - // err := conn.Connect(context.Background(), entities.SQLiteConnectorConfig{Path: ":memory:"}) + // err := conn.Connect(context.Background(), model.SQLiteConnectorConfig{Path: ":memory:"}) err := conn.Connect(context.Background(), &models.EngineContextConnector{ - Config: entities.SQLiteConnectorConfig{ + Config: model.SqliteConnectorConfig{ Path: ":memory:", }, }) @@ -132,7 +132,7 @@ func TestSQLiteConnector_Connect_InvalidConfig(t *testing.T) { func TestSQLiteConnector_Validate(t *testing.T) { conn := NewSQLiteConnector() - err := conn.Validate(entities.SQLiteConnectorConfig{Path: ":memory:"}) + err := conn.Validate(model.SqliteConnectorConfig{Path: ":memory:"}) assert.NoError(t, err) err = conn.Validate("missing-path") diff --git a/internal/encoders/dataset.go b/internal/encoders/dataset.go new file mode 100644 index 0000000..80ba85d --- /dev/null +++ b/internal/encoders/dataset.go @@ -0,0 +1,44 @@ +package encoders + +import ( + "encoding/json" + "fmt" +) + +func EncodeDatasetOptions(v any) map[string]interface{} { + b, _ := json.Marshal(v) + + var out map[string]interface{} + _ = json.Unmarshal(b, &out) + + return out +} + +func DecodeDatasetOptions[T any](raw any) (T, error) { + var out T + + if raw == nil { + return out, fmt.Errorf("dataset options are nil") + } + + switch v := raw.(type) { + case T: + return v, nil + case *T: + if v == nil { + return out, fmt.Errorf("dataset options are nil") + } + return *v, nil + } + + b, err := json.Marshal(raw) + if err != nil { + return out, err + } + + if err := json.Unmarshal(b, &out); err != nil { + return out, err + } + + return out, nil +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 6664d7c..ef151bb 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -3,13 +3,13 @@ package engine import ( "context" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" ) type ( EngineContextBuilder interface { - Build(ctx context.Context, plan *entities.Plan) (*models.EngineContext, error) + Build(ctx context.Context, plan *model.Plan) (*models.EngineContext, error) } DataStore interface { Find(tableName string) ([]map[string]interface{}, error) @@ -18,7 +18,7 @@ type ( ProcessPlan(ctx *models.EngineContext) error } ResponseBuilder interface { - BuildResponse(results []map[string]interface{}, columnOrder []string) (*entities.ExecutePlanResponse, error) + BuildResponse(results []map[string]interface{}, columnOrder []string) (*model.ExecutePlanResponse, error) } ) @@ -30,7 +30,7 @@ type engine struct { } type Executor interface { - ExecutePlan(ctx context.Context, req entities.ExecutePlanRequest) (*entities.ExecutePlanResponse, error) + ExecutePlan(ctx context.Context, req model.ExecutePlanRequest) (*model.ExecutePlanResponse, error) } func NewEngine(contextBuilder EngineContextBuilder, dataStore DataStore, processor Processor, responseBuilder ResponseBuilder) Executor { @@ -42,7 +42,7 @@ func NewEngine(contextBuilder EngineContextBuilder, dataStore DataStore, process } } -func (u *engine) ExecutePlan(ctx context.Context, req entities.ExecutePlanRequest) (*entities.ExecutePlanResponse, error) { +func (u *engine) ExecutePlan(ctx context.Context, req model.ExecutePlanRequest) (*model.ExecutePlanResponse, error) { engineCtx, err := u.contextBuilder.Build(ctx, &req.Plan) if err != nil { return nil, err diff --git a/internal/engine_context/engine_context.go b/internal/engine_context/engine_context.go index f2df068..2745817 100644 --- a/internal/engine_context/engine_context.go +++ b/internal/engine_context/engine_context.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" ) const federatedConnectorName = "federated" @@ -16,11 +16,11 @@ func NewBuilder() *Builder { return &Builder{} } -func (b *Builder) Build(ctx context.Context, plan *entities.Plan) (*models.EngineContext, error) { +func (b *Builder) Build(ctx context.Context, plan *model.Plan) (*models.EngineContext, error) { return buildEngineContext(ctx, plan) } -func buildEngineContext(ctx context.Context, plan *entities.Plan) (*models.EngineContext, error) { +func buildEngineContext(ctx context.Context, plan *model.Plan) (*models.EngineContext, error) { connectors := buildConnectors(plan) datasets := buildDatasets(plan) @@ -41,12 +41,12 @@ func buildEngineContext(ctx context.Context, plan *entities.Plan) (*models.Engin }, nil } -func buildConnectors(plan *entities.Plan) map[string]*models.EngineContextConnector { +func buildConnectors(plan *model.Plan) map[string]*models.EngineContextConnector { connectors := defaultConnectorMap() for _, conn := range plan.Connectors { connectors[conn.Name] = &models.EngineContextConnector{ - Type: conn.Type, + Type: string(conn.Type), Connection: conn.Connection, Config: conn.Connection, } @@ -55,14 +55,14 @@ func buildConnectors(plan *entities.Plan) map[string]*models.EngineContextConnec return connectors } -func buildDatasets(plan *entities.Plan) map[string]*models.EngineContextDataset { +func buildDatasets(plan *model.Plan) map[string]*models.EngineContextDataset { datasets := make(map[string]*models.EngineContextDataset, len(plan.Datasets)) for _, ds := range plan.Datasets { datasets[ds.Name] = &models.EngineContextDataset{ Name: ds.Name, ConnectorName: ds.ConnectorName, - Type: ds.Type, + Type: string(ds.Type), Query: ds.Query, Children: buildChildPlaceholders(ds), Dedupe: ds.Dedupe, @@ -76,7 +76,7 @@ func buildDatasets(plan *entities.Plan) map[string]*models.EngineContextDataset return datasets } -func buildChildPlaceholders(ds entities.Dataset) []*models.EngineContextDataset { +func buildChildPlaceholders(ds model.Dataset) []*models.EngineContextDataset { children := make([]*models.EngineContextDataset, 0, len(ds.Children)) for _, child := range ds.Children { @@ -88,7 +88,7 @@ func buildChildPlaceholders(ds entities.Dataset) []*models.EngineContextDataset return children } -func buildColumns(ds entities.Dataset) []models.EngineContextColumn { +func buildColumns(ds model.Dataset) []models.EngineContextColumn { columns := make([]models.EngineContextColumn, 0, len(ds.Columns)) for _, col := range ds.Columns { @@ -117,7 +117,7 @@ func resolveDatasetChildren(datasets map[string]*models.EngineContextDataset) er return nil } -func validateOutputDataset(plan *entities.Plan, datasets map[string]*models.EngineContextDataset) error { +func validateOutputDataset(plan *model.Plan, datasets map[string]*models.EngineContextDataset) error { if plan.Output.DatasetName == "" { return fmt.Errorf("must set output dataset") } @@ -132,7 +132,7 @@ func validateOutputDataset(plan *entities.Plan, datasets map[string]*models.Engi func defaultConnectorMap() map[string]*models.EngineContextConnector { return map[string]*models.EngineContextConnector{ federatedConnectorName: { - Type: "JOIN", + Type: model.ConnectorTypeFederated.String(), Connection: "", }, } diff --git a/internal/execute/execute.go b/internal/execute/execute.go index 87979c6..1c91ee6 100644 --- a/internal/execute/execute.go +++ b/internal/execute/execute.go @@ -3,15 +3,15 @@ package execute import ( "context" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/model" ) type Service interface { - Execute(ctx context.Context, req entities.ExecutePlanRequest) (*entities.ExecutePlanResponse, error) + Execute(ctx context.Context, req model.ExecutePlanRequest) (*model.ExecutePlanResponse, error) } type Engine interface { - ExecutePlan(ctx context.Context, req entities.ExecutePlanRequest) (*entities.ExecutePlanResponse, error) + ExecutePlan(ctx context.Context, req model.ExecutePlanRequest) (*model.ExecutePlanResponse, error) } type service struct { @@ -22,6 +22,6 @@ func NewService(engine Engine) Service { return &service{engine: engine} } -func (s *service) Execute(ctx context.Context, req entities.ExecutePlanRequest) (*entities.ExecutePlanResponse, error) { +func (s *service) Execute(ctx context.Context, req model.ExecutePlanRequest) (*model.ExecutePlanResponse, error) { return s.engine.ExecutePlan(ctx, req) } diff --git a/internal/plugins/api/plugin.go b/internal/plugins/api/plugin.go index 8eb7bb1..5fcb743 100644 --- a/internal/plugins/api/plugin.go +++ b/internal/plugins/api/plugin.go @@ -8,10 +8,10 @@ import ( "strings" "time" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/models" "github.com/kvatch-hub/kvatch-runtime/internal/plugins" pluginopts "github.com/kvatch-hub/kvatch-runtime/internal/plugins/options" + "github.com/kvatch-hub/kvatch-runtime/model" ) type ( @@ -29,8 +29,8 @@ func NewAPIDatasetPlugin() *APIDatasetPlugin { return &APIDatasetPlugin{} } -func (p *APIDatasetPlugin) GetType() plugins.PluginType { - return plugins.PluginType("api") // must match dataset.type +func (p *APIDatasetPlugin) GetType() model.DatasetType { + return model.DatasetType("api") // must match dataset.type } func (p *APIDatasetPlugin) ProcessData( @@ -112,10 +112,10 @@ func (p *APIDatasetPlugin) ProcessData( func parseAndExtractRows( body []byte, - respSpec entities.APIResponseSpec, + respSpec model.APIResponseSpec, ) ([]map[string]interface{}, error) { switch respSpec.Format { - case entities.APIResponseFormatJSON: + case model.APIResponseFormatJSON: var parsed interface{} if err := json.Unmarshal(body, &parsed); err != nil { return nil, fmt.Errorf("api: json decode error: %w", err) @@ -139,7 +139,7 @@ func parseAndExtractRows( } return rows, nil - case entities.APIResponseFormatText: + case model.APIResponseFormatText: // Very simple: single column "value" return []map[string]interface{}{ {"value": string(body)}, @@ -205,7 +205,7 @@ func coerceToRowArray(v interface{}) ([]map[string]interface{}, error) { } } -func injectTimestampIfNeeded(opts *entities.APIDatasetOptions, rows []map[string]interface{}) { +func injectTimestampIfNeeded(opts *model.APIDatasetOptions, rows []map[string]interface{}) { if opts == nil || !opts.InjectTimestamp { return } @@ -241,7 +241,7 @@ func injectTimestampIfNeeded(opts *entities.APIDatasetOptions, rows []map[string // key/value bag. This works well for Coingecko-like APIs. func normalizeRows( rows []map[string]interface{}, - cfg entities.APINormalizeConfig, + cfg model.APINormalizeConfig, ) ([]map[string]interface{}, error) { if !cfg.Enabled { return rows, nil diff --git a/internal/plugins/csv/plugin.go b/internal/plugins/csv/plugin.go index 384bc6b..edb454c 100644 --- a/internal/plugins/csv/plugin.go +++ b/internal/plugins/csv/plugin.go @@ -9,8 +9,9 @@ import ( "strings" "time" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/encoders" "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/model" ) // CSVDataPlugin processes CSV data from connections @@ -21,15 +22,15 @@ func NewCSVDataPlugin() *CSVDataPlugin { return &CSVDataPlugin{} } -func (p *CSVDataPlugin) GetType() plugins.PluginType { - return plugins.PluginTypeCSV +func (p *CSVDataPlugin) GetType() model.DatasetType { + return model.DatasetTypeCSV } func (p *CSVDataPlugin) ProcessData(ctx context.Context, params plugins.PluginProcessParams) (*plugins.ProcessedData, error) { const sampleSize = 100 const batchSize = 500 - opts, err := entities.DecodeDatasetOptions[entities.CSVDatasetOptions]( + opts, err := encoders.DecodeDatasetOptions[model.CSVDatasetOptions]( params.DatasetContext.DatasetOptions, ) if err != nil { diff --git a/internal/plugins/csv/plugin_test.go b/internal/plugins/csv/plugin_test.go index f1d592f..bfe546c 100644 --- a/internal/plugins/csv/plugin_test.go +++ b/internal/plugins/csv/plugin_test.go @@ -10,12 +10,12 @@ import ( "go.uber.org/mock/gomock" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/datastore" "github.com/kvatch-hub/kvatch-runtime/internal/models" "github.com/kvatch-hub/kvatch-runtime/internal/plugins" "github.com/kvatch-hub/kvatch-runtime/internal/plugins/csv" "github.com/kvatch-hub/kvatch-runtime/internal/plugins/mocks" + "github.com/kvatch-hub/kvatch-runtime/model" "github.com/stretchr/testify/assert" ) @@ -48,7 +48,7 @@ Bob,25 StorageClient: mockStorage, DatasetContext: &models.EngineContextDataset{ Name: "users", - DatasetOptions: entities.CSVDatasetOptions{ + DatasetOptions: model.CSVDatasetOptions{ HasHeaders: true, }, }, @@ -97,7 +97,7 @@ func TestCSVPlugin_StreamedBatchInsert(t *testing.T) { StorageClient: mockStorage, DatasetContext: &models.EngineContextDataset{ Name: "users", - DatasetOptions: entities.CSVDatasetOptions{ + DatasetOptions: model.CSVDatasetOptions{ HasHeaders: true, }, }, @@ -152,7 +152,7 @@ func TestCSVPlugin_InsertFailureMidBatch(t *testing.T) { StorageClient: mockStorage, DatasetContext: &models.EngineContextDataset{ Name: "users", - DatasetOptions: entities.CSVDatasetOptions{ + DatasetOptions: model.CSVDatasetOptions{ HasHeaders: true, }, }, @@ -199,7 +199,7 @@ func BenchmarkCSVPlugin_LargeFile(b *testing.B) { StorageClient: mockStorage, DatasetContext: &models.EngineContextDataset{ Name: "bench_table", - DatasetOptions: entities.CSVDatasetOptions{ + DatasetOptions: model.CSVDatasetOptions{ HasHeaders: true, }, }, diff --git a/internal/plugins/gsheet/plugin.go b/internal/plugins/gsheet/plugin.go index d867dba..f8ca33b 100644 --- a/internal/plugins/gsheet/plugin.go +++ b/internal/plugins/gsheet/plugin.go @@ -14,6 +14,7 @@ import ( "github.com/kvatch-hub/kvatch-runtime/internal/plugins" pluginopts "github.com/kvatch-hub/kvatch-runtime/internal/plugins/options" + "github.com/kvatch-hub/kvatch-runtime/model" ) type GoogleSheetsDataPlugin struct{} @@ -22,8 +23,8 @@ func NewGoogleSheetsDataPlugin() *GoogleSheetsDataPlugin { return &GoogleSheetsDataPlugin{} } -func (p *GoogleSheetsDataPlugin) GetType() plugins.PluginType { - return plugins.PluginTypeGoogleSheet +func (p *GoogleSheetsDataPlugin) GetType() model.DatasetType { + return model.DatasetTypeGoogleSheet } func (p *GoogleSheetsDataPlugin) ProcessData( diff --git a/internal/plugins/json/plugin.go b/internal/plugins/json/plugin.go index 0826f74..844c5a2 100644 --- a/internal/plugins/json/plugin.go +++ b/internal/plugins/json/plugin.go @@ -7,10 +7,10 @@ import ( "io" "strings" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/models" "github.com/kvatch-hub/kvatch-runtime/internal/plugins" pluginopts "github.com/kvatch-hub/kvatch-runtime/internal/plugins/options" + "github.com/kvatch-hub/kvatch-runtime/model" ) // JSONDataPlugin processes JSON data from connections @@ -20,8 +20,8 @@ func NewJSONDataPlugin() *JSONDataPlugin { return &JSONDataPlugin{} } -func (p *JSONDataPlugin) GetType() plugins.PluginType { - return plugins.PluginTypeJSON +func (p *JSONDataPlugin) GetType() model.DatasetType { + return model.DatasetTypeJSON } func (p *JSONDataPlugin) ProcessData(ctx context.Context, params plugins.PluginProcessParams) (*plugins.ProcessedData, error) { @@ -142,7 +142,7 @@ func extractRawFromParsed(parsed interface{}, key string) (interface{}, error) { return raw, nil } -func normalize(raw interface{}, opts entities.JSONDatasetOptions) ([]map[string]interface{}, error) { +func normalize(raw interface{}, opts model.JSONDatasetOptions) ([]map[string]interface{}, error) { switch v := raw.(type) { case []interface{}: var arr []map[string]interface{} diff --git a/internal/plugins/mocks/plugin_mocks.go b/internal/plugins/mocks/plugin_mocks.go index 1f25309..96b3676 100644 --- a/internal/plugins/mocks/plugin_mocks.go +++ b/internal/plugins/mocks/plugin_mocks.go @@ -17,6 +17,7 @@ import ( datastore "github.com/kvatch-hub/kvatch-runtime/internal/datastore" "github.com/kvatch-hub/kvatch-runtime/internal/models" plugins "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/model" gomock "go.uber.org/mock/gomock" gorm "gorm.io/gorm" ) @@ -338,10 +339,10 @@ func (m *MockPlugin) EXPECT() *MockPluginMockRecorder { } // GetType mocks base method. -func (m *MockPlugin) GetType() plugins.PluginType { +func (m *MockPlugin) GetType() model.DatasetType { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetType") - ret0, _ := ret[0].(plugins.PluginType) + ret0, _ := ret[0].(model.DatasetType) return ret0 } @@ -391,7 +392,7 @@ func (m *MockDataPluginRegistry) EXPECT() *MockDataPluginRegistryMockRecorder { } // Get mocks base method. -func (m *MockDataPluginRegistry) Get(pluginType plugins.PluginType) (plugins.Plugin, error) { +func (m *MockDataPluginRegistry) Get(pluginType model.DatasetType) (plugins.Plugin, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", pluginType) ret0, _ := ret[0].(plugins.Plugin) @@ -406,10 +407,10 @@ func (mr *MockDataPluginRegistryMockRecorder) Get(pluginType any) *gomock.Call { } // List mocks base method. -func (m *MockDataPluginRegistry) List() []plugins.PluginType { +func (m *MockDataPluginRegistry) List() []model.DatasetType { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "List") - ret0, _ := ret[0].([]plugins.PluginType) + ret0, _ := ret[0].([]model.DatasetType) return ret0 } @@ -420,7 +421,7 @@ func (mr *MockDataPluginRegistryMockRecorder) List() *gomock.Call { } // Register mocks base method. -func (m *MockDataPluginRegistry) Register(pluginType plugins.PluginType, plugin plugins.Plugin) error { +func (m *MockDataPluginRegistry) Register(pluginType model.DatasetType, plugin plugins.Plugin) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Register", pluginType, plugin) ret0, _ := ret[0].(error) diff --git a/internal/plugins/options/api.go b/internal/plugins/options/api.go index 02cae37..e2d72f2 100644 --- a/internal/plugins/options/api.go +++ b/internal/plugins/options/api.go @@ -4,18 +4,18 @@ import ( "encoding/json" "fmt" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/model" ) -func ParseDatasetOptions(in interface{}) (*entities.APIDatasetOptions, error) { +func ParseDatasetOptions(in interface{}) (*model.APIDatasetOptions, error) { if in == nil { return nil, fmt.Errorf("dataset.options is required for api datasets") } switch v := in.(type) { - case entities.APIDatasetOptions: + case model.APIDatasetOptions: return &v, nil - case *entities.APIDatasetOptions: + case *model.APIDatasetOptions: return v, nil case map[string]interface{}: // our standard pattern: map → JSON → struct @@ -23,7 +23,7 @@ func ParseDatasetOptions(in interface{}) (*entities.APIDatasetOptions, error) { if err != nil { return nil, fmt.Errorf("api: marshal options: %w", err) } - var out entities.APIDatasetOptions + var out model.APIDatasetOptions if err := json.Unmarshal(buf, &out); err != nil { return nil, fmt.Errorf("api: unmarshal options: %w", err) } diff --git a/internal/plugins/options/csv.go b/internal/plugins/options/csv.go index 47d90f0..be36292 100644 --- a/internal/plugins/options/csv.go +++ b/internal/plugins/options/csv.go @@ -1,9 +1,9 @@ package pluginopts -import "github.com/kvatch-hub/kvatch-runtime/entities" +import "github.com/kvatch-hub/kvatch-runtime/model" -func ConvertMapToCSVPluginOptions(m map[string]interface{}) entities.CSVDatasetOptions { - opts := entities.CSVDatasetOptions{} +func ConvertMapToCSVPluginOptions(m map[string]interface{}) model.CSVDatasetOptions { + opts := model.CSVDatasetOptions{} if m == nil { return opts } diff --git a/internal/plugins/options/googlesheets.go b/internal/plugins/options/googlesheets.go index af7626f..482621c 100644 --- a/internal/plugins/options/googlesheets.go +++ b/internal/plugins/options/googlesheets.go @@ -3,11 +3,11 @@ package pluginopts import ( "fmt" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/model" ) -func ParseGoogleSheetOptions(raw interface{}) (*entities.GoogleSheetDatasetOptions, error) { - var out entities.GoogleSheetDatasetOptions +func ParseGoogleSheetOptions(raw interface{}) (*model.GoogleSheetDatasetOptions, error) { + var out model.GoogleSheetDatasetOptions // Generic conversion if err := parseOptions(raw, &out); err != nil { diff --git a/internal/plugins/options/json.go b/internal/plugins/options/json.go index 3dfe9b1..1685b5d 100644 --- a/internal/plugins/options/json.go +++ b/internal/plugins/options/json.go @@ -3,11 +3,11 @@ package pluginopts import ( "fmt" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/model" ) -func ParseJSONOptions(raw interface{}) (*entities.JSONDatasetOptions, error) { - var out entities.JSONDatasetOptions +func ParseJSONOptions(raw interface{}) (*model.JSONDatasetOptions, error) { + var out model.JSONDatasetOptions // Generic conversion if err := parseOptions(raw, &out); err != nil { diff --git a/internal/plugins/plugin.go b/internal/plugins/plugin.go index 123975c..f8d8bfc 100644 --- a/internal/plugins/plugin.go +++ b/internal/plugins/plugin.go @@ -7,6 +7,7 @@ import ( "github.com/kvatch-hub/kvatch-runtime/internal/datastore" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" "gorm.io/gorm" ) @@ -34,23 +35,11 @@ type ( } ) -// PluginType represents the type of data processing plugin -type PluginType string - -const ( - PluginTypeJSON PluginType = "JSON" - PluginTypeCSV PluginType = "CSV" - PluginTypeSQL PluginType = "SQL" - PluginTypeGoogleSheet PluginType = "GOOGLESHEET" - PluginTypeYAML PluginType = "YAML" - PluginTypeAPI PluginType = "API" -) - // Plugin defines the interface for data processing plugins // These plugins know how to translate/process data but don't handle connections type Plugin interface { // GetType returns the plugin type - GetType() PluginType + GetType() model.DatasetType // ProcessData takes raw data from a connection and processes it according to the configuration ProcessData(ctx context.Context, params PluginProcessParams) (*ProcessedData, error) @@ -83,31 +72,31 @@ type ProcessedData struct { // DataPluginRegistry manages data processing plugins type DataPluginRegistry interface { - Register(pluginType PluginType, plugin Plugin) error - Get(pluginType PluginType) (Plugin, error) - List() []PluginType + Register(pluginType model.DatasetType, plugin Plugin) error + Get(pluginType model.DatasetType) (Plugin, error) + List() []model.DatasetType } // DefaultDataPluginRegistry implements DataPluginRegistry type DefaultDataPluginRegistry struct { - plugins map[PluginType]Plugin + plugins map[model.DatasetType]Plugin } // NewDataPluginRegistry creates a new data plugin registry func NewDataPluginRegistry() *DefaultDataPluginRegistry { return &DefaultDataPluginRegistry{ - plugins: make(map[PluginType]Plugin), + plugins: make(map[model.DatasetType]Plugin), } } // Register registers a new data plugin -func (r *DefaultDataPluginRegistry) Register(pluginType PluginType, plugin Plugin) error { +func (r *DefaultDataPluginRegistry) Register(pluginType model.DatasetType, plugin Plugin) error { r.plugins[pluginType] = plugin return nil } // Get retrieves a data plugin by type -func (r *DefaultDataPluginRegistry) Get(pluginType PluginType) (Plugin, error) { +func (r *DefaultDataPluginRegistry) Get(pluginType model.DatasetType) (Plugin, error) { plugin, exists := r.plugins[pluginType] if !exists { return nil, &DataPluginNotFoundError{PluginType: pluginType} @@ -116,8 +105,8 @@ func (r *DefaultDataPluginRegistry) Get(pluginType PluginType) (Plugin, error) { } // List returns all registered plugin types -func (r *DefaultDataPluginRegistry) List() []PluginType { - types := make([]PluginType, 0, len(r.plugins)) +func (r *DefaultDataPluginRegistry) List() []model.DatasetType { + types := make([]model.DatasetType, 0, len(r.plugins)) for t := range r.plugins { types = append(types, t) } @@ -126,7 +115,7 @@ func (r *DefaultDataPluginRegistry) List() []PluginType { // DataPluginNotFoundError represents an error when a plugin type is not found type DataPluginNotFoundError struct { - PluginType PluginType + PluginType model.DatasetType } func (e *DataPluginNotFoundError) Error() string { diff --git a/internal/plugins/postgres/plugin.go b/internal/plugins/postgres/plugin.go index 7bae2fe..33756b6 100644 --- a/internal/plugins/postgres/plugin.go +++ b/internal/plugins/postgres/plugin.go @@ -8,6 +8,7 @@ import ( "github.com/kvatch-hub/kvatch-runtime/internal/datastore" "github.com/kvatch-hub/kvatch-runtime/internal/models" "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/model" "github.com/kvatch-hub/kvatch-runtime/pkg/utils" ) @@ -38,8 +39,8 @@ func NewPostgresDataPlugin() *PostgresDataPlugin { } } -func (p *PostgresDataPlugin) GetType() plugins.PluginType { - return plugins.PluginTypeSQL +func (p *PostgresDataPlugin) GetType() model.DatasetType { + return model.DatasetTypeSQL } func (p *PostgresDataPlugin) ProcessData(ctx context.Context, params plugins.PluginProcessParams) (*plugins.ProcessedData, error) { diff --git a/internal/plugins/postgres/plugin_test.go b/internal/plugins/postgres/plugin_test.go index 584acba..94dd63e 100644 --- a/internal/plugins/postgres/plugin_test.go +++ b/internal/plugins/postgres/plugin_test.go @@ -150,7 +150,7 @@ func TestPostgres_ProcessData_NotJSONArray(t *testing.T) { // storage.EXPECT().Exec("CREATE").Return(&datastore.ExecResult{}) // gen.EXPECT().GenerateInsertSQL(table, gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( -// func(tableName string, data []interface{}, cols []entities.Datacolumn, dedupe []string) (string, error) { +// func(tableName string, data []interface{}, cols []model.DataColumn, dedupe []string) (string, error) { // require.Len(t, data, 1) // rowData, ok := data[0].([]interface{}) diff --git a/internal/plugins/sqlite/plugin.go b/internal/plugins/sqlite/plugin.go index fc1e20a..272eca9 100644 --- a/internal/plugins/sqlite/plugin.go +++ b/internal/plugins/sqlite/plugin.go @@ -7,6 +7,7 @@ import ( "github.com/kvatch-hub/kvatch-runtime/internal/models" "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/model" "github.com/kvatch-hub/kvatch-runtime/pkg/utils" ) @@ -28,8 +29,8 @@ func NewSQLiteDataPlugin() *SQLiteDataPlugin { } } -func (p *SQLiteDataPlugin) GetType() plugins.PluginType { - return plugins.PluginTypeSQL +func (p *SQLiteDataPlugin) GetType() model.DatasetType { + return model.DatasetTypeSQL } func (p *SQLiteDataPlugin) ProcessData( ctx context.Context, diff --git a/internal/plugins/yaml/plugin.go b/internal/plugins/yaml/plugin.go index 7836655..684f31b 100644 --- a/internal/plugins/yaml/plugin.go +++ b/internal/plugins/yaml/plugin.go @@ -8,6 +8,7 @@ import ( "github.com/kvatch-hub/kvatch-runtime/internal/models" "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/model" "gopkg.in/yaml.v3" ) @@ -17,8 +18,8 @@ func NewYAMLDataPlugin() *YAMLDataPlugin { return &YAMLDataPlugin{} } -func (p *YAMLDataPlugin) GetType() plugins.PluginType { - return plugins.PluginTypeYAML +func (p *YAMLDataPlugin) GetType() model.DatasetType { + return model.DatasetTypeYAML } func (p *YAMLDataPlugin) ProcessData(ctx context.Context, params plugins.PluginProcessParams) (*plugins.ProcessedData, error) { diff --git a/internal/processor/main.go b/internal/processor/main.go index 14cf85f..b0c91d1 100644 --- a/internal/processor/main.go +++ b/internal/processor/main.go @@ -16,6 +16,7 @@ import ( "github.com/kvatch-hub/kvatch-runtime/internal/plugins" "github.com/kvatch-hub/kvatch-runtime/internal/storage_manager" "github.com/kvatch-hub/kvatch-runtime/internal/subscription_manager" + "github.com/kvatch-hub/kvatch-runtime/model" "gorm.io/gorm" ) @@ -34,20 +35,20 @@ type ( GetGormDB() *gorm.DB } ConnectorResolver interface { - ResolveInternalJoin(ec *models.EngineContextConnector) (connectors.ConnectorType, any, error) + ResolveInternalJoin(ec *models.EngineContextConnector) (model.ConnectorType, any, error) } PluginResolver interface { - ResolvePluginType(datasetType, connectorType, originalType string) plugins.PluginType + ResolvePluginType(datasetType, connectorType, originalType string) model.DatasetType } ConnectorRegistry interface { - Register(connectionType connectors.ConnectorType, connection connectors.Connector) error - Get(connectionType connectors.ConnectorType) (connectors.Connector, error) - List() []connectors.ConnectorType + Register(connectionType model.ConnectorType, connection connectors.Connector) error + Get(connectionType model.ConnectorType) (connectors.Connector, error) + List() []model.ConnectorType } DataPluginRegistry interface { - Register(pluginType plugins.PluginType, plugin plugins.Plugin) error - Get(pluginType plugins.PluginType) (plugins.Plugin, error) - List() []plugins.PluginType + Register(pluginType model.DatasetType, plugin plugins.Plugin) error + Get(pluginType model.DatasetType) (plugins.Plugin, error) + List() []model.DatasetType } SubscriptionManager interface { GetRegisteredPlugins() subscription_manager.DataPluginRegistry @@ -286,8 +287,7 @@ func (p *Processor) processConnectorDataset( return fmt.Errorf("connector %q not found", d.ConnectorName) } - connType := connectors.ConnectorType(strings.ToUpper(ec.Type)) - + connType := model.ConnectorType(strings.ToLower(ec.Type)) conn, err := p.connectors.Get(connType) if err != nil { return fmt.Errorf("connection type not supported: %s", connType) diff --git a/internal/processor/mocks/mock_connector.go b/internal/processor/mocks/mock_connector.go index 14a1250..1bef7c9 100644 --- a/internal/processor/mocks/mock_connector.go +++ b/internal/processor/mocks/mock_connector.go @@ -14,8 +14,8 @@ import ( io "io" reflect "reflect" - connectors "github.com/kvatch-hub/kvatch-runtime/internal/connectors" "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/model" gomock "go.uber.org/mock/gomock" ) @@ -87,10 +87,10 @@ func (mr *MockConnectorMockRecorder) GetData(ctx, engineCtx any) *gomock.Call { } // GetType mocks base method. -func (m *MockConnector) GetType() connectors.ConnectorType { +func (m *MockConnector) GetType() model.ConnectorType { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetType") - ret0, _ := ret[0].(connectors.ConnectorType) + ret0, _ := ret[0].(model.ConnectorType) return ret0 } diff --git a/internal/processor/mocks/mock_interfaces.go b/internal/processor/mocks/mock_interfaces.go index 8b9bb4a..6af18e6 100644 --- a/internal/processor/mocks/mock_interfaces.go +++ b/internal/processor/mocks/mock_interfaces.go @@ -19,6 +19,7 @@ import ( "github.com/kvatch-hub/kvatch-runtime/internal/models" plugins "github.com/kvatch-hub/kvatch-runtime/internal/plugins" subscription_manager "github.com/kvatch-hub/kvatch-runtime/internal/subscription_manager" + "github.com/kvatch-hub/kvatch-runtime/model" gomock "go.uber.org/mock/gomock" gorm "gorm.io/gorm" ) @@ -234,10 +235,10 @@ func (m *MockConnectorResolver) EXPECT() *MockConnectorResolverMockRecorder { } // ResolveInternalJoin mocks base method. -func (m *MockConnectorResolver) ResolveInternalJoin(ec *models.EngineContextConnector) (connectors.ConnectorType, any, error) { +func (m *MockConnectorResolver) ResolveInternalJoin(ec *models.EngineContextConnector) (model.ConnectorType, any, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ResolveInternalJoin", ec) - ret0, _ := ret[0].(connectors.ConnectorType) + ret0, _ := ret[0].(model.ConnectorType) ret1, _ := ret[1].(any) ret2, _ := ret[2].(error) return ret0, ret1, ret2 @@ -274,10 +275,10 @@ func (m *MockPluginResolver) EXPECT() *MockPluginResolverMockRecorder { } // ResolvePluginType mocks base method. -func (m *MockPluginResolver) ResolvePluginType(datasetType, connectorType, originalType string) plugins.PluginType { +func (m *MockPluginResolver) ResolvePluginType(datasetType, connectorType, originalType string) model.DatasetType { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ResolvePluginType", datasetType, connectorType, originalType) - ret0, _ := ret[0].(plugins.PluginType) + ret0, _ := ret[0].(model.DatasetType) return ret0 } @@ -312,7 +313,7 @@ func (m *MockConnectorRegistry) EXPECT() *MockConnectorRegistryMockRecorder { } // Get mocks base method. -func (m *MockConnectorRegistry) Get(connectionType connectors.ConnectorType) (connectors.Connector, error) { +func (m *MockConnectorRegistry) Get(connectionType model.ConnectorType) (connectors.Connector, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", connectionType) ret0, _ := ret[0].(connectors.Connector) @@ -327,10 +328,10 @@ func (mr *MockConnectorRegistryMockRecorder) Get(connectionType any) *gomock.Cal } // List mocks base method. -func (m *MockConnectorRegistry) List() []connectors.ConnectorType { +func (m *MockConnectorRegistry) List() []model.ConnectorType { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "List") - ret0, _ := ret[0].([]connectors.ConnectorType) + ret0, _ := ret[0].([]model.ConnectorType) return ret0 } @@ -341,7 +342,7 @@ func (mr *MockConnectorRegistryMockRecorder) List() *gomock.Call { } // Register mocks base method. -func (m *MockConnectorRegistry) Register(connectionType connectors.ConnectorType, connection connectors.Connector) error { +func (m *MockConnectorRegistry) Register(connectionType model.ConnectorType, connection connectors.Connector) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Register", connectionType, connection) ret0, _ := ret[0].(error) @@ -379,7 +380,7 @@ func (m *MockDataPluginRegistry) EXPECT() *MockDataPluginRegistryMockRecorder { } // Get mocks base method. -func (m *MockDataPluginRegistry) Get(pluginType plugins.PluginType) (plugins.Plugin, error) { +func (m *MockDataPluginRegistry) Get(pluginType model.DatasetType) (plugins.Plugin, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", pluginType) ret0, _ := ret[0].(plugins.Plugin) @@ -394,10 +395,10 @@ func (mr *MockDataPluginRegistryMockRecorder) Get(pluginType any) *gomock.Call { } // List mocks base method. -func (m *MockDataPluginRegistry) List() []plugins.PluginType { +func (m *MockDataPluginRegistry) List() []model.DatasetType { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "List") - ret0, _ := ret[0].([]plugins.PluginType) + ret0, _ := ret[0].([]model.DatasetType) return ret0 } @@ -408,7 +409,7 @@ func (mr *MockDataPluginRegistryMockRecorder) List() *gomock.Call { } // Register mocks base method. -func (m *MockDataPluginRegistry) Register(pluginType plugins.PluginType, plugin plugins.Plugin) error { +func (m *MockDataPluginRegistry) Register(pluginType model.DatasetType, plugin plugins.Plugin) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Register", pluginType, plugin) ret0, _ := ret[0].(error) diff --git a/internal/processor/mocks/mock_plugin.go b/internal/processor/mocks/mock_plugin.go index 9bbc93c..cf9834e 100644 --- a/internal/processor/mocks/mock_plugin.go +++ b/internal/processor/mocks/mock_plugin.go @@ -14,6 +14,7 @@ import ( reflect "reflect" plugins "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/model" gomock "go.uber.org/mock/gomock" ) @@ -42,10 +43,10 @@ func (m *MockPlugin) EXPECT() *MockPluginMockRecorder { } // GetType mocks base method. -func (m *MockPlugin) GetType() plugins.PluginType { +func (m *MockPlugin) GetType() model.DatasetType { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetType") - ret0, _ := ret[0].(plugins.PluginType) + ret0, _ := ret[0].(model.DatasetType) return ret0 } diff --git a/internal/processor/resolvers.go b/internal/processor/resolvers.go index 538b2b1..a78143b 100644 --- a/internal/processor/resolvers.go +++ b/internal/processor/resolvers.go @@ -3,11 +3,9 @@ package processor import ( "strings" - "github.com/kvatch-hub/kvatch-runtime/entities" - "github.com/kvatch-hub/kvatch-runtime/internal/connectors" "github.com/kvatch-hub/kvatch-runtime/internal/datastore" "github.com/kvatch-hub/kvatch-runtime/internal/models" - "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/model" ) type JoinConnectorResolver struct { @@ -18,29 +16,29 @@ func NewJoinConnectorResolver(storage DataStoreClient) *JoinConnectorResolver { return &JoinConnectorResolver{storage: storage} } -func (r *JoinConnectorResolver) ResolveInternalJoin(ec *models.EngineContextConnector) (connectors.ConnectorType, any, error) { - // Not JOIN? Just pass through. - if strings.ToUpper(ec.Type) != string(connectors.ConnectorTypeFederated) { - return connectors.ConnectorType(strings.ToUpper(ec.Type)), ec.Config, nil +func (r *JoinConnectorResolver) ResolveInternalJoin(ec *models.EngineContextConnector) (model.ConnectorType, any, error) { + // Not JOIN/Federated? Just pass through. + if strings.ToLower(ec.Type) != string(model.ConnectorTypeFederated) { + return model.ConnectorType(strings.ToLower(ec.Type)), ec.Config, nil } // JOIN → dynamic routing based on storage backend switch r.storage.GetStorageType() { case datastore.StorageTypePOSTGRES: // Dataset federated into Postgres backing store - return connectors.ConnectorTypePostgres, ec.Config, nil + return model.ConnectorTypePostgres, ec.Config, nil case datastore.StorageTypeSQLITE: // SQLite synthesizes a config dynamically - return connectors.ConnectorTypeSQLite, - entities.SQLiteConnectorConfig{ + return model.ConnectorTypeSQLite, + model.SqliteConnectorConfig{ Path: r.storage.GetConnectionString(), }, nil default: // fallback to SQLite for any unknown storage - return connectors.ConnectorTypeSQLite, - entities.SQLiteConnectorConfig{ + return model.ConnectorTypeSQLite, + model.SqliteConnectorConfig{ Path: r.storage.GetConnectionString(), }, nil } @@ -53,13 +51,13 @@ func NewDefaultPluginResolver() PluginResolver { } func (r *DefaultPluginResolver) ResolvePluginType(datasetType, connectorType, originalType string, -) plugins.PluginType { - dsTypeUpper := strings.ToUpper(datasetType) - if dsTypeUpper == string(plugins.PluginTypeSQL) && - (strings.ToUpper(connectorType) == string(connectors.ConnectorTypePostgres) || - strings.ToUpper(originalType) == string(connectors.ConnectorTypePostgres)) { - return plugins.PluginTypeSQL +) model.DatasetType { + dsTypeLower := strings.ToLower(datasetType) + if dsTypeLower == string(model.DatasetTypeSQL) && + (strings.ToLower(connectorType) == string(model.ConnectorTypePostgres) || + strings.ToLower(originalType) == string(model.ConnectorTypePostgres)) { + return model.DatasetTypeSQL } - return plugins.PluginType(dsTypeUpper) + return model.DatasetType(dsTypeLower) } diff --git a/internal/response_builder/bulder.go b/internal/response_builder/bulder.go index 301e734..92d6062 100644 --- a/internal/response_builder/bulder.go +++ b/internal/response_builder/bulder.go @@ -4,16 +4,16 @@ import ( "fmt" "slices" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/model" ) type ResponseBuilder struct{} -func (b *ResponseBuilder) BuildResponse(results []map[string]interface{}, columnOrder []string) (*entities.ExecutePlanResponse, error) { +func (b *ResponseBuilder) BuildResponse(results []map[string]interface{}, columnOrder []string) (*model.ExecutePlanResponse, error) { if len(results) == 0 { - return &entities.ExecutePlanResponse{ + return &model.ExecutePlanResponse{ Data: []map[string]interface{}{}, - Columns: []entities.DataColumn{}, + Columns: []model.DataColumn{}, }, nil } @@ -31,9 +31,9 @@ func (b *ResponseBuilder) BuildResponse(results []map[string]interface{}, column slices.Sort(columnOrder) } - columns := make([]entities.DataColumn, 0, len(columnOrder)) + columns := make([]model.DataColumn, 0, len(columnOrder)) for _, colName := range columnOrder { - columns = append(columns, entities.DataColumn{ + columns = append(columns, model.DataColumn{ Name: colName, Type: "TEXT", Description: fmt.Sprintf("Column: %s", colName), @@ -49,7 +49,7 @@ func (b *ResponseBuilder) BuildResponse(results []map[string]interface{}, column orderedResults = append(orderedResults, orderedRow) } - return &entities.ExecutePlanResponse{ + return &model.ExecutePlanResponse{ Data: orderedResults, Columns: columns, }, nil diff --git a/internal/subscription_manager/main.go b/internal/subscription_manager/main.go index d7104a9..b755382 100644 --- a/internal/subscription_manager/main.go +++ b/internal/subscription_manager/main.go @@ -10,6 +10,7 @@ import ( postgresConnection "github.com/kvatch-hub/kvatch-runtime/internal/connectors/postgres" sqliteConnection "github.com/kvatch-hub/kvatch-runtime/internal/connectors/sqlite" "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/model" apiPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/api" csvPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/csv" @@ -22,14 +23,14 @@ import ( type ( ConnectorRegistry interface { - Register(connectionType connectors.ConnectorType, connection connectors.Connector) error - Get(connectionType connectors.ConnectorType) (connectors.Connector, error) - List() []connectors.ConnectorType + Register(connectionType model.ConnectorType, connection connectors.Connector) error + Get(connectionType model.ConnectorType) (connectors.Connector, error) + List() []model.ConnectorType } DataPluginRegistry interface { - Register(pluginType plugins.PluginType, plugin plugins.Plugin) error - Get(pluginType plugins.PluginType) (plugins.Plugin, error) - List() []plugins.PluginType + Register(pluginType model.DatasetType, plugin plugins.Plugin) error + Get(pluginType model.DatasetType) (plugins.Plugin, error) + List() []model.DatasetType } ) @@ -53,23 +54,23 @@ func NewSubscriptionManagerWithDefaults() *SubscriptionManager { } func (m *SubscriptionManager) registerDefaultPlugins() { - _ = m.pluginRegistry.Register(plugins.PluginTypeJSON, jsonPlugin.NewJSONDataPlugin()) - _ = m.pluginRegistry.Register(plugins.PluginTypeCSV, csvPlugin.NewCSVDataPlugin()) - _ = m.pluginRegistry.Register(plugins.PluginTypeSQL, sqlitePlugin.NewSQLiteDataPlugin()) - _ = m.pluginRegistry.Register(plugins.PluginTypeSQL, psqlPlugin.NewPostgresDataPlugin()) - _ = m.pluginRegistry.Register(plugins.PluginTypeYAML, yamlPlugin.NewYAMLDataPlugin()) - _ = m.pluginRegistry.Register(plugins.PluginTypeGoogleSheet, gsheetPlugin.NewGoogleSheetsDataPlugin()) - _ = m.pluginRegistry.Register(plugins.PluginTypeAPI, apiPlugin.NewAPIDatasetPlugin()) + _ = m.pluginRegistry.Register(model.DatasetTypeJSON, jsonPlugin.NewJSONDataPlugin()) + _ = m.pluginRegistry.Register(model.DatasetTypeCSV, csvPlugin.NewCSVDataPlugin()) + _ = m.pluginRegistry.Register(model.DatasetTypeSQL, sqlitePlugin.NewSQLiteDataPlugin()) + _ = m.pluginRegistry.Register(model.DatasetTypeSQL, psqlPlugin.NewPostgresDataPlugin()) + _ = m.pluginRegistry.Register(model.DatasetTypeYAML, yamlPlugin.NewYAMLDataPlugin()) + _ = m.pluginRegistry.Register(model.DatasetTypeGoogleSheet, gsheetPlugin.NewGoogleSheetsDataPlugin()) + _ = m.pluginRegistry.Register(model.DatasetTypeAPI, apiPlugin.NewAPIDatasetPlugin()) } func (m *SubscriptionManager) registerDefaultDataConnectors() { - _ = m.connectorRegistry.Register(connectors.ConnectorTypeLocalFile, localFileConnection.NewLocalFileConnector()) - _ = m.connectorRegistry.Register(connectors.ConnectorTypePostgres, postgresConnection.NewPostgresConnector()) - _ = m.connectorRegistry.Register(connectors.ConnectorTypeSQLite, sqliteConnection.NewSQLiteConnector()) - _ = m.connectorRegistry.Register(connectors.ConnectorTypeGit, gitconnector.NewGitConnector()) - _ = m.connectorRegistry.Register(connectors.ConnectorTypeLocalDirectory, localDirectoryConnection.NewDirectoryConnector()) - _ = m.connectorRegistry.Register(connectors.ConnectorTypeGoogleSheet, googlesheetConnection.NewGoogleSheetConnector()) - _ = m.connectorRegistry.Register(connectors.ConnectorTypeAPI, apiconnector.NewAPIConnector()) + _ = m.connectorRegistry.Register(model.ConnectorTypeLocalFile, localFileConnection.NewLocalFileConnector()) + _ = m.connectorRegistry.Register(model.ConnectorTypePostgres, postgresConnection.NewPostgresConnector()) + _ = m.connectorRegistry.Register(model.ConnectorTypeSQLite, sqliteConnection.NewSQLiteConnector()) + _ = m.connectorRegistry.Register(model.ConnectorTypeGit, gitconnector.NewGitConnector()) + _ = m.connectorRegistry.Register(model.ConnectorTypeLocalDirectory, localDirectoryConnection.NewDirectoryConnector()) + _ = m.connectorRegistry.Register(model.ConnectorTypeGoogleSheet, googlesheetConnection.NewGoogleSheetConnector()) + _ = m.connectorRegistry.Register(model.ConnectorTypeAPI, apiconnector.NewAPIConnector()) } func (m *SubscriptionManager) GetRegisteredConnectors() ConnectorRegistry { diff --git a/internal/subscription_manager/main_test.go b/internal/subscription_manager/main_test.go index cab042b..17300e4 100644 --- a/internal/subscription_manager/main_test.go +++ b/internal/subscription_manager/main_test.go @@ -3,9 +3,8 @@ package subscription_manager import ( "testing" - "github.com/kvatch-hub/kvatch-runtime/internal/connectors" - "github.com/kvatch-hub/kvatch-runtime/internal/plugins" "github.com/kvatch-hub/kvatch-runtime/internal/subscription_manager/mocks" + "github.com/kvatch-hub/kvatch-runtime/model" "go.uber.org/mock/gomock" ) @@ -20,13 +19,13 @@ func Test_registerDefaultPlugins_usesRegistry(t *testing.T) { // We don't assert on concrete plugin instances, just that Register is called // with the correct plugin types. gomock.Any() for the plugin value. gomock.InOrder( - mockPluginReg.EXPECT().Register(plugins.PluginTypeJSON, gomock.Any()).Return(nil), - mockPluginReg.EXPECT().Register(plugins.PluginTypeCSV, gomock.Any()).Return(nil), - mockPluginReg.EXPECT().Register(plugins.PluginTypeSQL, gomock.Any()).Return(nil), - mockPluginReg.EXPECT().Register(plugins.PluginTypeSQL, gomock.Any()).Return(nil), - mockPluginReg.EXPECT().Register(plugins.PluginTypeYAML, gomock.Any()).Return(nil), - mockPluginReg.EXPECT().Register(plugins.PluginTypeGoogleSheet, gomock.Any()).Return(nil), - mockPluginReg.EXPECT().Register(plugins.PluginTypeAPI, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(model.DatasetTypeJSON, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(model.DatasetTypeCSV, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(model.DatasetTypeSQL, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(model.DatasetTypeSQL, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(model.DatasetTypeYAML, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(model.DatasetTypeGoogleSheet, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(model.DatasetTypeAPI, gomock.Any()).Return(nil), ) // We don't care about connectors here, so a nil is fine; we won't touch it. @@ -45,13 +44,13 @@ func Test_registerDefaultDataConnectors_usesRegistry(t *testing.T) { mockConnReg := mocks.NewMockConnectorRegistry(ctrl) gomock.InOrder( - mockConnReg.EXPECT().Register(connectors.ConnectorTypeLocalFile, gomock.Any()).Return(nil), - mockConnReg.EXPECT().Register(connectors.ConnectorTypePostgres, gomock.Any()).Return(nil), - mockConnReg.EXPECT().Register(connectors.ConnectorTypeSQLite, gomock.Any()).Return(nil), - mockConnReg.EXPECT().Register(connectors.ConnectorTypeGit, gomock.Any()).Return(nil), - mockConnReg.EXPECT().Register(connectors.ConnectorTypeLocalDirectory, gomock.Any()).Return(nil), - mockConnReg.EXPECT().Register(connectors.ConnectorTypeGoogleSheet, gomock.Any()).Return(nil), - mockConnReg.EXPECT().Register(connectors.ConnectorTypeAPI, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(model.ConnectorTypeLocalFile, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(model.ConnectorTypePostgres, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(model.ConnectorTypeSQLite, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(model.ConnectorTypeGit, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(model.ConnectorTypeLocalDirectory, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(model.ConnectorTypeGoogleSheet, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(model.ConnectorTypeAPI, gomock.Any()).Return(nil), ) m := &SubscriptionManager{ @@ -66,14 +65,14 @@ func Test_NewSubscriptionManagerWithDefaults_registersEverything(t *testing.T) { m := NewSubscriptionManagerWithDefaults() // Check plugins list contains all expected types - gotPlugins := map[plugins.PluginType]bool{} + gotPlugins := map[model.DatasetType]bool{} for _, pt := range m.pluginRegistry.List() { gotPlugins[pt] = true } - wantPlugins := []plugins.PluginType{ - plugins.PluginTypeJSON, - plugins.PluginTypeCSV, - plugins.PluginTypeSQL, + wantPlugins := []model.DatasetType{ + model.DatasetTypeJSON, + model.DatasetTypeCSV, + model.DatasetTypeSQL, } for _, w := range wantPlugins { if !gotPlugins[w] { @@ -86,15 +85,15 @@ func Test_NewSubscriptionManagerWithDefaults_registersEverything(t *testing.T) { } // Check connectors list contains all expected types - gotConns := map[connectors.ConnectorType]bool{} + gotConns := map[model.ConnectorType]bool{} for _, ct := range m.connectorRegistry.List() { gotConns[ct] = true } - wantConns := []connectors.ConnectorType{ - connectors.ConnectorTypeLocalDirectory, - connectors.ConnectorTypeLocalFile, - connectors.ConnectorTypePostgres, - connectors.ConnectorTypeSQLite, + wantConns := []model.ConnectorType{ + model.ConnectorTypeLocalDirectory, + model.ConnectorTypeLocalFile, + model.ConnectorTypePostgres, + model.ConnectorTypeSQLite, } for _, w := range wantConns { if !gotConns[w] { diff --git a/internal/subscription_manager/mocks/mock_interfaces.go b/internal/subscription_manager/mocks/mock_interfaces.go index be32335..d3da5c6 100644 --- a/internal/subscription_manager/mocks/mock_interfaces.go +++ b/internal/subscription_manager/mocks/mock_interfaces.go @@ -14,6 +14,7 @@ import ( connectors "github.com/kvatch-hub/kvatch-runtime/internal/connectors" plugins "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/model" gomock "go.uber.org/mock/gomock" ) @@ -42,7 +43,7 @@ func (m *MockConnectorRegistry) EXPECT() *MockConnectorRegistryMockRecorder { } // Get mocks base method. -func (m *MockConnectorRegistry) Get(connectionType connectors.ConnectorType) (connectors.Connector, error) { +func (m *MockConnectorRegistry) Get(connectionType model.ConnectorType) (connectors.Connector, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", connectionType) ret0, _ := ret[0].(connectors.Connector) @@ -57,10 +58,10 @@ func (mr *MockConnectorRegistryMockRecorder) Get(connectionType any) *gomock.Cal } // List mocks base method. -func (m *MockConnectorRegistry) List() []connectors.ConnectorType { +func (m *MockConnectorRegistry) List() []model.ConnectorType { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "List") - ret0, _ := ret[0].([]connectors.ConnectorType) + ret0, _ := ret[0].([]model.ConnectorType) return ret0 } @@ -71,7 +72,7 @@ func (mr *MockConnectorRegistryMockRecorder) List() *gomock.Call { } // Register mocks base method. -func (m *MockConnectorRegistry) Register(connectionType connectors.ConnectorType, connection connectors.Connector) error { +func (m *MockConnectorRegistry) Register(connectionType model.ConnectorType, connection connectors.Connector) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Register", connectionType, connection) ret0, _ := ret[0].(error) @@ -109,7 +110,7 @@ func (m *MockDataPluginRegistry) EXPECT() *MockDataPluginRegistryMockRecorder { } // Get mocks base method. -func (m *MockDataPluginRegistry) Get(pluginType plugins.PluginType) (plugins.Plugin, error) { +func (m *MockDataPluginRegistry) Get(pluginType model.DatasetType) (plugins.Plugin, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", pluginType) ret0, _ := ret[0].(plugins.Plugin) @@ -124,10 +125,10 @@ func (mr *MockDataPluginRegistryMockRecorder) Get(pluginType any) *gomock.Call { } // List mocks base method. -func (m *MockDataPluginRegistry) List() []plugins.PluginType { +func (m *MockDataPluginRegistry) List() []model.DatasetType { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "List") - ret0, _ := ret[0].([]plugins.PluginType) + ret0, _ := ret[0].([]model.DatasetType) return ret0 } @@ -138,7 +139,7 @@ func (mr *MockDataPluginRegistryMockRecorder) List() *gomock.Call { } // Register mocks base method. -func (m *MockDataPluginRegistry) Register(pluginType plugins.PluginType, plugin plugins.Plugin) error { +func (m *MockDataPluginRegistry) Register(pluginType model.DatasetType, plugin plugins.Plugin) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Register", pluginType, plugin) ret0, _ := ret[0].(error) diff --git a/model/connector.go b/model/connector.go new file mode 100644 index 0000000..0918142 --- /dev/null +++ b/model/connector.go @@ -0,0 +1,47 @@ +package model + +type ConnectorType string + +const ( + ConnectorTypeAPI ConnectorType = "api" + ConnectorTypeGoogleSheet ConnectorType = "googlesheet" + ConnectorTypeInternal ConnectorType = "internal" + ConnectorTypePostgres ConnectorType = "postgres" + ConnectorTypeSQLite ConnectorType = "sqlite" + ConnectorTypeLocalFile ConnectorType = "localfile" + ConnectorTypeLocalDirectory ConnectorType = "local_directory" + ConnectorTypeGoogleAPI ConnectorType = "google_api" + ConnectorTypeGit ConnectorType = "git" + ConnectorTypeS3 ConnectorType = "s3" + ConnectorTypeFederated ConnectorType = "federated" +) + +type Connector struct { + Name string + Type ConnectorType + Connection ConnectorConfig + Description string +} + +func (t ConnectorType) String() string { + return string(t) +} + +func (t ConnectorType) IsValid() bool { + switch t { + case ConnectorTypeAPI, + ConnectorTypeGoogleSheet, + ConnectorTypeInternal, + ConnectorTypePostgres, + ConnectorTypeSQLite, + ConnectorTypeLocalFile, + ConnectorTypeLocalDirectory, + ConnectorTypeGoogleAPI, + ConnectorTypeGit, + ConnectorTypeS3, + ConnectorTypeFederated: + return true + default: + return false + } +} diff --git a/entities/connector_configs.go b/model/connector_configs.go similarity index 80% rename from entities/connector_configs.go rename to model/connector_configs.go index f0312e1..e0926f9 100644 --- a/entities/connector_configs.go +++ b/model/connector_configs.go @@ -1,34 +1,61 @@ -package entities +package model import ( - "errors" "fmt" "net/url" "strings" "time" ) +type ConnectorConfig interface { + Validate() error +} + type GitConnectorConfig struct { Repo string `yaml:"repo" json:"repo" jsonschema:"description=URL of the Git repository"` Branch string `yaml:"branch" json:"branch" jsonschema:"description=Branch to clone (default: main)"` Path string `yaml:"path" json:"path" jsonschema:"description=Optional subdirectory within the repo"` } -type SQLiteConnectorConfig struct { +func (c *GitConnectorConfig) Validate() error { + if strings.TrimSpace(c.Repo) == "" { + return fmt.Errorf("repo is required") + } + if strings.TrimSpace(c.Branch) == "" { + return fmt.Errorf("branch is required") + } + return nil +} + +type SqliteConnectorConfig struct { Path string `yaml:"path" json:"path" jsonschema:"description=Path to SQLite database file"` } -func (c *SQLiteConnectorConfig) DSN() string { +func (c *SqliteConnectorConfig) DSN() string { if strings.TrimSpace(c.Path) == "" { return ":memory:" } return c.Path } +func (c *SqliteConnectorConfig) Validate() error { + if strings.TrimSpace(c.Path) == "" { + return fmt.Errorf("path is required") + } + return nil +} + type DirectoryConnectorConfig struct { DirectoryPath string `yaml:"directory_path" json:"directoryPath" jsonschema:"description=Local path to directory containing data files"` } +func (c *DirectoryConnectorConfig) Validate() error { + if strings.TrimSpace(c.DirectoryPath) == "" { + return fmt.Errorf("directory path is required") + } + return nil +} + type LocalFileConnectorConfig struct { FilePath string `yaml:"file_path" json:"filePath" jsonschema:"description=Path to a single local data file (CSV, JSON, etc)"` } @@ -37,6 +64,13 @@ func (c *LocalFileConnectorConfig) GetFilePath() string { return strings.TrimSpace(c.FilePath) } +func (c *LocalFileConnectorConfig) Validate() error { + if strings.TrimSpace(c.FilePath) == "" { + return fmt.Errorf("file path is required") + } + return nil +} + type S3ConnectionConfig struct { Region string `yaml:"region" json:"region" jsonschema:"description=AWS region of the S3 bucket"` Bucket string `yaml:"bucket" json:"bucket" jsonschema:"description=Name of the S3 bucket"` @@ -44,11 +78,34 @@ type S3ConnectionConfig struct { SecretKey string `yaml:"secret_key" sensitive:"true" json:"secretKey" jsonschema:"description=AWS secret access key"` } +func (c *S3ConnectionConfig) Validate() error { + if strings.TrimSpace(c.Region) == "" { + return fmt.Errorf("region is required") + } + if strings.TrimSpace(c.Bucket) == "" { + return fmt.Errorf("bucket is required") + } + if strings.TrimSpace(c.AccessKeyID) == "" { + return fmt.Errorf("access key id is required") + } + if strings.TrimSpace(c.SecretKey) == "" { + return fmt.Errorf("secret key is required") + } + return nil +} + type GoogleAPIConnectorConfig struct { APIKey string `yaml:"api_key" json:"apiKey" sensitive:"true" jsonschema:"description=Google API key for authentication"` Credentials string `yaml:"credentials,omitempty" json:"credentials,omitempty" jsonschema:"description=Optional path to credentials file (e.g. service account JSON)"` } +func (c *GoogleAPIConnectorConfig) Validate() error { + if strings.TrimSpace(c.APIKey) == "" { + return fmt.Errorf("api key is required") + } + return nil +} + type PostgresConnectorConfig struct { Host string `yaml:"host" json:"host" jsonschema:"description=Database host (e.g. localhost)" validate:"required_without=ConnectionString"` Port int `yaml:"port" json:"port" jsonschema:"description=Database port (e.g. 5432)" validate:"required_without=ConnectionString,min=1"` @@ -90,19 +147,19 @@ func (c *PostgresConnectorConfig) Validate() error { return nil } if c.Host == "" { - return errors.New("postgres config: host is required") + return fmt.Errorf("postgres config: host is required") } if c.Port <= 0 { - return errors.New("postgres config: port must be > 0") + return fmt.Errorf("postgres config: port must be > 0") } if c.Database == "" { - return errors.New("postgres config: database is required") + return fmt.Errorf("postgres config: database is required") } if c.Username == "" { - return errors.New("postgres config: username is required") + return fmt.Errorf("postgres config: username is required") } if c.Password == "" { - return errors.New("postgres config: password is required") + return fmt.Errorf("postgres config: password is required") } return nil } @@ -197,7 +254,7 @@ type APIConnectorConfig struct { // Validate performs basic sanity checks on the connector configuration. func (c APIConnectorConfig) Validate() error { if strings.TrimSpace(c.BaseURL) == "" { - return errors.New("base_url is required") + return fmt.Errorf("base_url is required") } if _, err := url.ParseRequestURI(c.BaseURL); err != nil { return fmt.Errorf("invalid base_url: %w", err) @@ -208,18 +265,18 @@ func (c APIConnectorConfig) Validate() error { case APIAuthNone, "": case APIAuthAPIKey: if strings.TrimSpace(c.Auth.APIKeyHeader) == "" { - return errors.New("auth.api_key_header is required for api_key auth") + return fmt.Errorf("auth.api_key_header is required for api_key auth") } if strings.TrimSpace(c.Auth.APIKeyValue) == "" { - return errors.New("auth.api_key_value is required for api_key auth") + return fmt.Errorf("auth.api_key_value is required for api_key auth") } case APIAuthBearer: if strings.TrimSpace(c.Auth.BearerToken) == "" { - return errors.New("auth.bearer_token is required for bearer auth") + return fmt.Errorf("auth.bearer_token is required for bearer auth") } case APIAuthBasic: if strings.TrimSpace(c.Auth.BasicUsername) == "" || strings.TrimSpace(c.Auth.BasicPassword) == "" { - return errors.New("auth.basic_username and auth.basic_password are required for basic auth") + return fmt.Errorf("auth.basic_username and auth.basic_password are required for basic auth") } case APIAuthOAuth2: // Placeholder for future expansion; currently only type validation. @@ -229,7 +286,7 @@ func (c APIConnectorConfig) Validate() error { } if c.RateLimit != nil && c.RateLimit.RequestsPerMinute < 0 { - return errors.New("rate_limit.requests_per_minute must be non-negative") + return fmt.Errorf("rate_limit.requests_per_minute must be non-negative") } if c.Cache != nil && strings.TrimSpace(c.Cache.TTL) != "" { diff --git a/model/dataset.go b/model/dataset.go new file mode 100644 index 0000000..5050bc1 --- /dev/null +++ b/model/dataset.go @@ -0,0 +1,153 @@ +package model + +import ( + "fmt" + "strings" +) + +type DatasetType string + +const ( + DatasetTypeAPI DatasetType = "api" + DatasetTypeJSON DatasetType = "json" + DatasetTypeSQL DatasetType = "sql" + DatasetTypeCSV DatasetType = "csv" + DatasetTypeGoogleSheet DatasetType = "googlesheet" + DatasetTypeYAML DatasetType = "yaml" +) + +func (t DatasetType) String() string { + return string(t) +} + +func (t DatasetType) IsValid() bool { + switch t { + case DatasetTypeAPI, + DatasetTypeJSON, + DatasetTypeSQL, + DatasetTypeCSV, + DatasetTypeGoogleSheet, + DatasetTypeYAML: + return true + default: + return false + } +} + +type Dataset struct { + Name string + ConnectorName string + Type DatasetType + Description string + Query string + Options DatasetOptions + // Data []map[string]any + Data any + Children []DatasetChild + Dedupe []string + Columns []DataColumn + ColumnOrder []string +} + +type DataColumn struct { + Name string + Accessor string + Alias string + Type string + Description string + IsPrimaryKey bool +} + +type DatasetChild struct { + DatasetName string +} + +func (d *Dataset) Validate() error { + if d == nil { + return fmt.Errorf("dataset is nil") + } + + if strings.TrimSpace(d.Name) == "" { + return fmt.Errorf("dataset name is required") + } + + if !d.Type.IsValid() { + return fmt.Errorf("invalid dataset type %q", d.Type) + } + + if strings.TrimSpace(d.ConnectorName) == "" { + return fmt.Errorf("connector_name is required") + } + + if err := d.validateOptions(); err != nil { + return err + } + + return nil +} + +func (d *Dataset) validateOptions() error { + switch d.Type { + case DatasetTypeAPI: + opts, ok := d.Options.(*APIDatasetOptions) + if !ok { + return fmt.Errorf("api dataset requires APIDatasetOptions, got %T", d.Options) + } + opts.ApplyDefaults() + if err := opts.Validate(); err != nil { + return err + } + + case DatasetTypeJSON: + opts, ok := d.Options.(*JSONDatasetOptions) + if !ok { + return fmt.Errorf("json dataset requires JSONDatasetOptions, got %T", d.Options) + } + opts.ApplyDefaults() + if err := opts.Validate(); err != nil { + return err + } + + case DatasetTypeSQL: + opts, ok := d.Options.(*SQLDatasetOptions) + if !ok { + return fmt.Errorf("sql dataset requires SQLDatasetOptions, got %T", d.Options) + } + opts.ApplyDefaults() + if err := opts.Validate(); err != nil { + return err + } + + case DatasetTypeCSV: + opts, ok := d.Options.(*CSVDatasetOptions) + if !ok { + return fmt.Errorf("csv dataset requires CSVDatasetOptions, got %T", d.Options) + } + opts.ApplyDefaults() + if err := opts.Validate(); err != nil { + return err + } + + case DatasetTypeGoogleSheet: + opts, ok := d.Options.(*GoogleSheetDatasetOptions) + if !ok { + return fmt.Errorf("googlesheet dataset requires GoogleSheetDatasetOptions, got %T", d.Options) + } + opts.ApplyDefaults() + if err := opts.Validate(); err != nil { + return err + } + + case DatasetTypeYAML: + opts, ok := d.Options.(*YAMLDatasetOptions) + if !ok { + return fmt.Errorf("yaml dataset requires YAMLDatasetOptions, got %T", d.Options) + } + opts.ApplyDefaults() + if err := opts.Validate(); err != nil { + return err + } + } + + return nil +} diff --git a/entities/dataset_options.go b/model/dataset_options.go similarity index 88% rename from entities/dataset_options.go rename to model/dataset_options.go index b65ac01..fbf6f4d 100644 --- a/entities/dataset_options.go +++ b/model/dataset_options.go @@ -1,13 +1,17 @@ -package entities +package model import ( - "encoding/json" "errors" "fmt" "strings" "unicode/utf8" ) +type DatasetOptions interface { + ApplyDefaults() + Validate() error +} + const ( APIDatasetOptionsVersion = 1 ) @@ -160,7 +164,7 @@ type QueryVar struct { type JSONDatasetOptions struct { Version int `json:"version" yaml:"version"` - Timeout *int `json:"timeout,omitempty" yaml:"timeout,omitempty"` + Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` Query *string `json:"query,omitempty" yaml:"query,omitempty"` Vars map[string]string `json:"vars,omitempty" yaml:"vars,omitempty"` Dedupe []string `json:"dedupe,omitempty" yaml:"dedupe,omitempty"` @@ -186,15 +190,14 @@ func (o *JSONDatasetOptions) ApplyDefaults() { if o.Version == 0 { o.Version = 1 } - if o.Timeout == nil { - defaultTimeout := 30 - o.Timeout = &defaultTimeout + if o.Timeout == 0 { + o.Timeout = 30 } } type SQLDatasetOptions struct { Version int `json:"version" yaml:"version"` - Timeout *int `json:"timeout,omitempty" yaml:"timeout,omitempty"` + Timeout int `json:"timeout,omitempty" yaml:"timeout,omitempty"` Query string `json:"query,omitempty" yaml:"query,omitempty"` Vars map[string]string `json:"vars,omitempty" yaml:"vars,omitempty"` Dedupe []string `json:"dedupe,omitempty" yaml:"dedupe,omitempty"` @@ -214,66 +217,9 @@ func (o *SQLDatasetOptions) ApplyDefaults() { if o.Version == 0 { o.Version = 1 } - if o.Timeout == nil { - defaultTimeout := 30 - o.Timeout = &defaultTimeout - } -} - -// func DecodeDatasetOptions[T any](raw map[string]interface{}) (T, error) { -// var out T - -// if raw == nil { -// return out, fmt.Errorf("dataset options are nil") -// } - -// b, err := json.Marshal(raw) -// if err != nil { -// return out, err -// } - -// if err := json.Unmarshal(b, &out); err != nil { -// return out, err -// } - -// return out, nil -// } -func DecodeDatasetOptions[T any](raw any) (T, error) { - var out T - - if raw == nil { - return out, fmt.Errorf("dataset options are nil") - } - - switch v := raw.(type) { - case T: - return v, nil - case *T: - if v == nil { - return out, fmt.Errorf("dataset options are nil") - } - return *v, nil - } - - b, err := json.Marshal(raw) - if err != nil { - return out, err - } - - if err := json.Unmarshal(b, &out); err != nil { - return out, err + if o.Timeout == 0 { + o.Timeout = 30 } - - return out, nil -} - -func EncodeDatasetOptions(v any) map[string]interface{} { - b, _ := json.Marshal(v) - - var out map[string]interface{} - _ = json.Unmarshal(b, &out) - - return out } type GoogleSheetDatasetOptions struct { @@ -344,3 +290,8 @@ func (o *CSVDatasetOptions) ApplyDefaults() { o.Delimiter = "," } } + +type YAMLDatasetOptions struct{} + +func (o *YAMLDatasetOptions) Validate() error { return nil } +func (o *YAMLDatasetOptions) ApplyDefaults() {} diff --git a/model/plan.go b/model/plan.go new file mode 100644 index 0000000..0f0ba72 --- /dev/null +++ b/model/plan.go @@ -0,0 +1,145 @@ +package model + +import ( + "fmt" + "strings" +) + +type Plan struct { + Name string + Storage Storage + Connectors []Connector + Datasets []Dataset + Output Output +} + +type Storage struct { + Type string + MetadataStorePath string + DataStorePath string +} + +type Output struct { + DatasetName string + Verbose bool +} + +func (p *Plan) Validate() error { + if p == nil { + return fmt.Errorf("plan is nil") + } + + if strings.TrimSpace(p.Name) == "" { + return fmt.Errorf("plan name is required") + } + + if err := p.validateConnectors(); err != nil { + return err + } + + if err := p.validateDatasets(); err != nil { + return err + } + + if err := p.validateOutput(); err != nil { + return err + } + + return nil +} + +func (p *Plan) validateConnectors() error { + seen := make(map[string]struct{}, len(p.Connectors)) + + for _, c := range p.Connectors { + name := strings.TrimSpace(c.Name) + if name == "" { + return fmt.Errorf("connector name is required") + } + + if _, exists := seen[name]; exists { + return fmt.Errorf("duplicate connector name %q", name) + } + seen[name] = struct{}{} + + if !c.Type.IsValid() { + return fmt.Errorf("connector %q has invalid type %q", c.Name, c.Type) + } + + if validator, ok := c.Connection.(interface{ Validate() error }); ok { + if err := validator.Validate(); err != nil { + return fmt.Errorf("connector %q config is invalid: %w", c.Name, err) + } + } + } + + return nil +} + +func (p *Plan) validateDatasets() error { + if len(p.Datasets) == 0 { + return fmt.Errorf("plan must contain at least one dataset") + } + + connectorNames := make(map[string]struct{}, len(p.Connectors)) + for _, c := range p.Connectors { + connectorNames[c.Name] = struct{}{} + } + connectorNames[string(ConnectorTypeFederated)] = struct{}{} + + datasetNames := make(map[string]struct{}, len(p.Datasets)) + for _, d := range p.Datasets { + name := strings.TrimSpace(d.Name) + if name == "" { + return fmt.Errorf("dataset name is required") + } + + if _, exists := datasetNames[name]; exists { + return fmt.Errorf("duplicate dataset name %q", d.Name) + } + datasetNames[name] = struct{}{} + + if !d.Type.IsValid() { + return fmt.Errorf("dataset %q has invalid type %q", d.Name, d.Type) + } + + if strings.TrimSpace(d.ConnectorName) == "" { + return fmt.Errorf("dataset %q connector_name is required", d.Name) + } + + if _, ok := connectorNames[d.ConnectorName]; !ok { + return fmt.Errorf("dataset %q references unknown connector %q", d.Name, d.ConnectorName) + } + + if err := d.Validate(); err != nil { + return fmt.Errorf("dataset %q is invalid: %w", d.Name, err) + } + } + + for _, d := range p.Datasets { + for _, child := range d.Children { + if strings.TrimSpace(child.DatasetName) == "" { + return fmt.Errorf("dataset %q has child with empty dataset_name", d.Name) + } + if _, ok := datasetNames[child.DatasetName]; !ok { + return fmt.Errorf("dataset %q references unknown child dataset %q", d.Name, child.DatasetName) + } + } + } + + return nil +} + +func (p *Plan) validateOutput() error { + if strings.TrimSpace(p.Output.DatasetName) == "" { + return fmt.Errorf("output dataset_name is required") + } + + for _, d := range p.Datasets { + if d.Name == p.Output.DatasetName { + return nil + } + } + + return fmt.Errorf("output dataset %q not found in datasets", p.Output.DatasetName) +} diff --git a/entities/runtime.go b/model/runtime.go similarity index 92% rename from entities/runtime.go rename to model/runtime.go index 1599cd1..aa97b4b 100644 --- a/entities/runtime.go +++ b/model/runtime.go @@ -1,4 +1,4 @@ -package entities +package model type Source struct { Type string diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index ee5dadd..5f6477c 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/kvatch-hub/kvatch-runtime/configs" - "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/internal/datastore" "github.com/kvatch-hub/kvatch-runtime/internal/engine" "github.com/kvatch-hub/kvatch-runtime/internal/engine_context" @@ -13,10 +12,11 @@ import ( "github.com/kvatch-hub/kvatch-runtime/internal/processor" "github.com/kvatch-hub/kvatch-runtime/internal/response_builder" "github.com/kvatch-hub/kvatch-runtime/internal/subscription_manager" + "github.com/kvatch-hub/kvatch-runtime/model" ) type Runtime interface { - ExecutePlan(ctx context.Context, req entities.ExecutePlanRequest) (*entities.ExecutePlanResponse, error) + ExecutePlan(ctx context.Context, req model.ExecutePlanRequest) (*model.ExecutePlanResponse, error) } type runtimeImpl struct { @@ -47,7 +47,7 @@ func NewWithSQLite(dsn string) (Runtime, error) { }) } -func (r *runtimeImpl) ExecutePlan(ctx context.Context, req entities.ExecutePlanRequest) (*entities.ExecutePlanResponse, error) { +func (r *runtimeImpl) ExecutePlan(ctx context.Context, req model.ExecutePlanRequest) (*model.ExecutePlanResponse, error) { return r.service.Execute(ctx, req) } diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index b5a4249..9f35ef2 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/kvatch-hub/kvatch-runtime/configs" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/model" "github.com/stretchr/testify/require" ) @@ -21,18 +21,18 @@ func TestSimpleBaseCase(t *testing.T) { log.Fatalf("create runtime: %v", err) } - plan := entities.Plan{ + plan := model.Plan{ Name: "example-plan", - Output: entities.Output{ + Output: model.Output{ DatasetName: "books", }, - Connectors: []entities.Connector{ + Connectors: []model.Connector{ { Name: "local", Type: "INTERNAL", }, }, - Datasets: []entities.Dataset{ + Datasets: []model.Dataset{ { Name: "books", ConnectorName: "local", @@ -41,14 +41,14 @@ func TestSimpleBaseCase(t *testing.T) { {"id": 1, "title": "Dune"}, {"id": 2, "title": "Neuromancer"}, }, - Options: map[string]any{ - "timeout": 30, + Options: &model.JSONDatasetOptions{ + Timeout: 30, }, }, }, } - got, gotErr := rt.ExecutePlan(context.Background(), entities.ExecutePlanRequest{ + got, gotErr := rt.ExecutePlan(context.Background(), model.ExecutePlanRequest{ UserID: "someid", Plan: plan, })