diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..32d6c44 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,66 @@ +name: CI + +on: + push: + branches: ["main"] + pull_request: + +jobs: + test: + name: Test + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.25" + cache: true + + - name: Download dependencies + run: go mod download + + - name: Run vet + run: go vet ./... + + - name: Run tests + run: go test ./... -v + + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.25" + cache: true + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: v2.5.0 + + vulncheck: + name: Vulnerability Check + runs-on: ubuntu-latest + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.25" + cache: true + + - name: Run govulncheck + uses: golang/govulncheck-action@v1 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c04dc96 --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Code coverage profiles and other test artifacts +*.out +coverage.* +*.coverprofile +profile.cov + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +go.work.sum + +# env file +.env + +# Editor/IDE +# .idea/ +# .vscode/ +.DS_Store +*.db diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..bd1d54d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 James Wooltorton + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile index e69de29..e3fa83f 100644 --- a/Makefile +++ b/Makefile @@ -0,0 +1,21 @@ + +.PHONY: generate-mocks + +generate-mocks: + @echo "Generating mocks..." + mockgen -source=internal/plugins/plugin.go -destination=internal/plugins/mocks/plugin_mocks.go -package=mocks + mockgen -source internal/connectors/main.go -destination internal/connectors/mocks/mock_interfaces.go -package mocks + mockgen -source internal/subscription_manager/main.go -destination internal/subscription_manager/mocks/mock_interfaces.go -package mocks + mockgen -source internal/processor/main.go -destination internal/processor/mocks/mock_interfaces.go -package mocks + mockgen -source internal/storage_manager/main.go -destination internal/storage_manager/mocks/mock_interfaces.go -package mocks + mockgen -package=mocks -destination=internal/processor/mocks/mock_connector.go github.com/kvatch-hub/kvatch-runtime/internal/connectors Connector + mockgen -package=mocks -destination=internal/processor/mocks/mock_plugin.go github.com/kvatch-hub/kvatch-runtime/internal/plugins Plugin + + @echo "Mocks generated successfully" + +lint: + golangci-lint version && golangci-lint run --verbose -E misspell + +test: + go test -short ./... + \ No newline at end of file diff --git a/README.md b/README.md index 5495706..41e511f 100644 --- a/README.md +++ b/README.md @@ -1 +1,142 @@ -# kvatch-runtime +# 🚀 Kvatch Runtime + +![Go Version](https://img.shields.io/badge/go-1.21+-blue.svg) +![License](https://img.shields.io/badge/license-MIT-green.svg) +![Status](https://img.shields.io/badge/status-alpha-orange.svg) + +A lightweight runtime for executing federated data plans. + +> ⚠️ This project is in early development. APIs may change. + +------------------------------------------------------------------------ + +## 📦 Installation + +``` bash +go get github.com/kvatch-hub/kvatch-runtime +``` + +------------------------------------------------------------------------ + +## 🚀 Quick Start + +``` go +package main + +import ( + "context" + "fmt" + "log" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/runtime" +) + +func main() { + rt, err := runtime.NewDefault() + if err != nil { + log.Fatal(err) + } + + plan := entities.Plan{ + Datasets: []entities.Dataset{ + { + Name: "example", + Type: "static", + Data: []map[string]interface{}{ + {"id": 1, "name": "Alice"}, + {"id": 2, "name": "Bob"}, + }, + }, + }, + Output: entities.PlanOutput{ + DatasetName: "example", + }, + } + + resp, err := rt.ExecutePlan(context.Background(), entities.ExecutePlanRequest{ + Plan: plan, + }) + if err != nil { + log.Fatal(err) + } + + fmt.Println("Columns:", resp.Columns) + fmt.Println("Data:", resp.Data) +} +``` + +------------------------------------------------------------------------ + +## 🧠 What's happening + +- `runtime.NewDefault()` sets up the engine, storage, and processor +- You define a `Plan` +- `ExecutePlan` runs it and returns structured results + +------------------------------------------------------------------------ + +## ⚙️ Configuration + +### Default (recommended) + +``` go +rt, _ := runtime.NewDefault() +``` + +Uses a local SQLite database: + + ./kvatch.db + +------------------------------------------------------------------------ + +### Custom SQLite + +``` go +rt, _ := runtime.NewWithSQLite("file:my.db") +``` + +------------------------------------------------------------------------ + +### Full configuration + +``` go +cfg := configs.Config{ + Storage: configs.StorageConfig{ + Driver: "sqlite", + DSN: "file:custom.db", + }, +} + +rt, _ := runtime.New(cfg) +``` + +------------------------------------------------------------------------ + +## 📦 Output Format + +``` go +type ExecutePlanResponse struct { + Columns []DataColumn + Data []map[string]interface{} +} +``` + +- `Columns` defines order and metadata\ +- `Data` contains row values + +------------------------------------------------------------------------ + +## 🎯 Philosophy + +Kvatch Runtime is designed to be: + +- **Simple to start** → one function call\ +- **Composable internally** → clean engine architecture\ +- **Extensible** → connectors, processors, and plugins + +------------------------------------------------------------------------ + +## 📄 License + +MIT License © 2026 James Wooltorton diff --git a/entities/connector_configs.go b/entities/connector_configs.go new file mode 100644 index 0000000..f0312e1 --- /dev/null +++ b/entities/connector_configs.go @@ -0,0 +1,242 @@ +package entities + +import ( + "errors" + "fmt" + "net/url" + "strings" + "time" +) + +type GitConnectorConfig struct { + Repo string `yaml:"repo" json:"repo" jsonschema:"description=URL of the Git repository"` + Branch string `yaml:"branch" json:"branch" jsonschema:"description=Branch to clone (default: main)"` + Path string `yaml:"path" json:"path" jsonschema:"description=Optional subdirectory within the repo"` +} + +type SQLiteConnectorConfig struct { + Path string `yaml:"path" json:"path" jsonschema:"description=Path to SQLite database file"` +} + +func (c *SQLiteConnectorConfig) DSN() string { + if strings.TrimSpace(c.Path) == "" { + return ":memory:" + } + return c.Path +} + +type DirectoryConnectorConfig struct { + DirectoryPath string `yaml:"directory_path" json:"directoryPath" jsonschema:"description=Local path to directory containing data files"` +} + +type LocalFileConnectorConfig struct { + FilePath string `yaml:"file_path" json:"filePath" jsonschema:"description=Path to a single local data file (CSV, JSON, etc)"` +} + +func (c *LocalFileConnectorConfig) GetFilePath() string { + return strings.TrimSpace(c.FilePath) +} + +type S3ConnectionConfig struct { + Region string `yaml:"region" json:"region" jsonschema:"description=AWS region of the S3 bucket"` + Bucket string `yaml:"bucket" json:"bucket" jsonschema:"description=Name of the S3 bucket"` + AccessKeyID string `yaml:"access_key_id" json:"accessKeyId" jsonschema:"description=AWS access key ID"` + SecretKey string `yaml:"secret_key" sensitive:"true" json:"secretKey" jsonschema:"description=AWS secret access key"` +} + +type GoogleAPIConnectorConfig struct { + APIKey string `yaml:"api_key" json:"apiKey" sensitive:"true" jsonschema:"description=Google API key for authentication"` + Credentials string `yaml:"credentials,omitempty" json:"credentials,omitempty" jsonschema:"description=Optional path to credentials file (e.g. service account JSON)"` +} + +type PostgresConnectorConfig struct { + Host string `yaml:"host" json:"host" jsonschema:"description=Database host (e.g. localhost)" validate:"required_without=ConnectionString"` + Port int `yaml:"port" json:"port" jsonschema:"description=Database port (e.g. 5432)" validate:"required_without=ConnectionString,min=1"` + Database string `yaml:"database" json:"database" jsonschema:"description=Database name" validate:"required_without=ConnectionString"` + Username string `yaml:"username" json:"username" jsonschema:"description=Username for authentication" validate:"required_without=ConnectionString"` + Password string `yaml:"password" json:"password" sensitive:"true" jsonschema:"description=Password for authentication" validate:"required_without=ConnectionString"` + SSLMode string `yaml:"sslmode,omitempty" json:"sslMode,omitempty" jsonschema:"description=SSL mode (disable, require, verify-full, etc)"` + ConnectionString string `yaml:"connection_string,omitempty" json:"connectionString,omitempty" sensitive:"true" jsonschema:"description=If the full connection string is passed" validate:"required_without=Host"` +} + +func (c *PostgresConnectorConfig) DSN() string { + if c.ConnectionString != "" { + return c.ConnectionString + } + + host := c.Host + if host == "" { + host = "localhost" + } + port := c.Port + if port == 0 { + port = 5432 + } + sslmode := c.SSLMode + if sslmode == "" { + sslmode = "require" + } + + return fmt.Sprintf( + "host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", + host, port, c.Username, c.Password, c.Database, sslmode, + ) +} + +// Validate ensures the config is valid +func (c *PostgresConnectorConfig) Validate() error { + // Either connection string OR host/db/credentials + if c.ConnectionString != "" { + return nil + } + if c.Host == "" { + return errors.New("postgres config: host is required") + } + if c.Port <= 0 { + return errors.New("postgres config: port must be > 0") + } + if c.Database == "" { + return errors.New("postgres config: database is required") + } + if c.Username == "" { + return errors.New("postgres config: username is required") + } + if c.Password == "" { + return errors.New("postgres config: password is required") + } + return nil +} + +type GoogleSheetConnectorConfig struct { + SpreadsheetID string `yaml:"spreadsheet_id" json:"spreadsheetId" jsonschema:"description=ID of the Google Sheet" validate:"required"` + ReadRange string `yaml:"read_range" json:"readRange" jsonschema:"description=Range to read (e.g. Sheet1!A:B)" validate:"required"` + APIKey string `yaml:"api_key,omitempty" json:"apiKey,omitempty" sensitive:"true" jsonschema:"description=Google API key (required for private sheets, optional for public)"` + HeaderRowNo int `yaml:"header_row_no,omitempty" json:"headerRowNo,omitempty" jsonschema:"description=Row number of headers (default: 0)"` +} + +func (c *GoogleSheetConnectorConfig) Mode() string { + if c.APIKey != "" { + return "api" + } + return "public" +} + +func (c *GoogleSheetConnectorConfig) Validate() error { + if strings.TrimSpace(c.SpreadsheetID) == "" { + return fmt.Errorf("spreadsheet_id is required") + } + if strings.TrimSpace(c.ReadRange) == "" { + return fmt.Errorf("read_range is required") + } + // apiKey is only required for private sheets (mode "api") + if c.Mode() == "api" && strings.TrimSpace(c.APIKey) == "" { + return fmt.Errorf("api_key is required for private Google Sheets") + } + return nil +} + +type S3ConnectorConfig struct { + // One of the following auth methods: + // - Explicit keys (AccessKeyID/SecretAccessKey[/SessionToken]) + // - Shared config/profile (Profile) + // - Default env/instance role (leave empty) + Region string `yaml:"region" json:"region"` + Bucket string `yaml:"bucket" json:"bucket"` + BasePrefix string `yaml:"base_prefix,omitempty" json:"base_prefix,omitempty"` // optional; preprended to dp.Query if present + + AccessKeyID string `yaml:"access_key_id,omitempty" json:"access_key_id,omitempty"` + SecretAccessKey string `yaml:"secret_access_key,omitempty" sensitive:"true" json:"secret_access_key,omitempty"` + SessionToken string `yaml:"session_token,omitempty" json:"session_token,omitempty"` + Profile string `yaml:"profile,omitempty" json:"profile,omitempty"` + + // Optional: for MinIO/localstack or VPC endpoints + Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` + ForcePathStyle bool `yaml:"force_path_style,omitempty" json:"force_path_style,omitempty"` +} + +// APIAuthType enumerates supported authentication approaches for api connectors. +type APIAuthType string + +const ( + APIAuthNone APIAuthType = "none" + APIAuthAPIKey APIAuthType = "api_key" + APIAuthBearer APIAuthType = "bearer" + APIAuthBasic APIAuthType = "basic" + APIAuthOAuth2 APIAuthType = "oauth2" +) + +// APIRateLimitConfig configures a token bucket style limiter. +type APIRateLimitConfig struct { + RequestsPerMinute int `json:"requests_per_minute,omitempty"` +} + +// APICacheConfig allows lightweight connector-level caching controls. +type APICacheConfig struct { + TTL string `json:"ttl,omitempty"` // Go duration string +} + +// APIAuthConfig captures authentication information for API connectors. +type APIAuthConfig struct { + Type APIAuthType `json:"type"` + APIKeyHeader string `json:"api_key_header,omitempty"` + APIKeyValue string `json:"api_key_value,omitempty"` + BearerToken string `json:"bearer_token,omitempty"` + BasicUsername string `json:"basic_username,omitempty"` + BasicPassword string `json:"basic_password,omitempty"` +} + +// APIConnectorConfig describes the base connection settings for api connectors. +type APIConnectorConfig struct { + BaseURL string `json:"base_url"` + DefaultHeaders map[string]string `json:"default_headers,omitempty"` + Auth *APIAuthConfig `json:"auth,omitempty"` + RateLimit *APIRateLimitConfig `json:"rate_limit,omitempty"` + Cache *APICacheConfig `json:"cache,omitempty"` +} + +// Validate performs basic sanity checks on the connector configuration. +func (c APIConnectorConfig) Validate() error { + if strings.TrimSpace(c.BaseURL) == "" { + return errors.New("base_url is required") + } + if _, err := url.ParseRequestURI(c.BaseURL); err != nil { + return fmt.Errorf("invalid base_url: %w", err) + } + + if c.Auth != nil { + switch c.Auth.Type { + case APIAuthNone, "": + case APIAuthAPIKey: + if strings.TrimSpace(c.Auth.APIKeyHeader) == "" { + return errors.New("auth.api_key_header is required for api_key auth") + } + if strings.TrimSpace(c.Auth.APIKeyValue) == "" { + return errors.New("auth.api_key_value is required for api_key auth") + } + case APIAuthBearer: + if strings.TrimSpace(c.Auth.BearerToken) == "" { + return errors.New("auth.bearer_token is required for bearer auth") + } + case APIAuthBasic: + if strings.TrimSpace(c.Auth.BasicUsername) == "" || strings.TrimSpace(c.Auth.BasicPassword) == "" { + return errors.New("auth.basic_username and auth.basic_password are required for basic auth") + } + case APIAuthOAuth2: + // Placeholder for future expansion; currently only type validation. + default: + return fmt.Errorf("unsupported auth.type %q", c.Auth.Type) + } + } + + if c.RateLimit != nil && c.RateLimit.RequestsPerMinute < 0 { + return errors.New("rate_limit.requests_per_minute must be non-negative") + } + + if c.Cache != nil && strings.TrimSpace(c.Cache.TTL) != "" { + if _, err := time.ParseDuration(c.Cache.TTL); err != nil { + return fmt.Errorf("cache.ttl must be a valid duration: %w", err) + } + } + + return nil +} diff --git a/entities/dataset_options.go b/entities/dataset_options.go new file mode 100644 index 0000000..9efb9d9 --- /dev/null +++ b/entities/dataset_options.go @@ -0,0 +1,311 @@ +package entities + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "unicode/utf8" +) + +const ( + APIDatasetOptionsVersion = 1 +) + +type APIResponseFormat string + +const ( + APIResponseFormatJSON APIResponseFormat = "json" + APIResponseFormatCSV APIResponseFormat = "csv" + APIResponseFormatXML APIResponseFormat = "xml" + APIResponseFormatText APIResponseFormat = "text" +) + +type APINormalizeConfig struct { + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` + KeyField string `json:"key_field,omitempty" yaml:"key_field,omitempty"` + ValuePrefix string `json:"value_prefix,omitempty" yaml:"value_prefix,omitempty"` +} + +type APIResponseSpec struct { + Format APIResponseFormat `json:"format"` + Extract string `json:"extract,omitempty"` + Normalize APINormalizeConfig `json:"normalize,omitempty"` +} + +type APIDatasetOptions struct { + Version int `json:"version" yaml:"version"` + Timeout *int `json:"timeout,omitempty" yaml:"timeout,omitempty"` + Query *string `json:"query,omitempty" yaml:"query,omitempty"` + Vars map[string]string `json:"vars,omitempty" yaml:"vars,omitempty"` + Dedupe []string `json:"dedupe,omitempty" yaml:"dedupe,omitempty"` + Request APIRequestSpec `json:"request" yaml:"request"` + Pagination APIPaginationConfig `json:"pagination" yaml:"pagination"` + Response APIResponseSpec `json:"response" yaml:"response"` + InjectTimestamp bool `json:"inject_timestamp,omitempty" yaml:"inject_timestamp,omitempty"` + TimestampField string `json:"timestamp_field,omitempty" yaml:"timestamp_field,omitempty"` +} + +type APIRequestSpec struct { + Method string `json:"method"` + Path string `json:"path"` + Query map[string]string `json:"query,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + Body *string `json:"body,omitempty"` + Params map[string]string `json:"params,omitempty" yaml:"params,omitempty"` +} + +type APIPaginationType string + +const ( + APIPaginationNone APIPaginationType = "none" + APIPaginationPage APIPaginationType = "page" + APIPaginationCursor APIPaginationType = "cursor" + APIPaginationLinkHeader APIPaginationType = "link_header" + APIPaginationOffset APIPaginationType = "offset_limit" +) + +type APIPaginationConfig struct { + Type APIPaginationType `json:"type"` + + PageParam string `json:"page_param,omitempty"` + PageSizeParam string `json:"page_size_param,omitempty"` + PageSize int `json:"page_size,omitempty"` + + CursorParam string `json:"cursor_param,omitempty"` + CursorPath string `json:"cursor_path,omitempty"` + + OffsetParam string `json:"offset_param,omitempty"` + LimitParam string `json:"limit_param,omitempty"` + Limit int `json:"limit,omitempty"` + + MaxPages int `json:"max_pages,omitempty"` +} + +func (o *APIDatasetOptions) Validate() error { + if strings.TrimSpace(o.Request.Method) == "" { + return errors.New("request.method is required") + } + if strings.TrimSpace(o.Request.Path) == "" { + return errors.New("request.path is required") + } + if o.Response.Format == "" { + return errors.New("response.format is required") + } + if o.Pagination.Type == "" { + return errors.New("pagination.type is required") + } + + // if o.RequestParams != nil { + // for key, val := range o.RequestParams { + // if val == "" { + // return fmt.Errorf("params.%s must be a string value", key) + // } + // } + // } + + switch o.Pagination.Type { + case APIPaginationNone: + case APIPaginationPage: + if strings.TrimSpace(o.Pagination.PageParam) == "" { + return errors.New("pagination.page_param is required for page pagination") + } + case APIPaginationCursor: + if strings.TrimSpace(o.Pagination.CursorParam) == "" { + return errors.New("pagination.cursor_param is required for cursor pagination") + } + if strings.TrimSpace(o.Pagination.CursorPath) == "" { + return errors.New("pagination.cursor_path is required for cursor pagination") + } + case APIPaginationOffset: + if strings.TrimSpace(o.Pagination.OffsetParam) == "" || strings.TrimSpace(o.Pagination.LimitParam) == "" { + return errors.New("pagination.offset_param and pagination.limit_param are required for offset pagination") + } + case APIPaginationLinkHeader: + default: + return fmt.Errorf("unsupported pagination.type %q", o.Pagination.Type) + } + + if o.Response.Normalize.Enabled { + if strings.TrimSpace(o.Response.Normalize.KeyField) == "" { + return errors.New("response.normalize.key_field is required when normalization is enabled") + } + if strings.TrimSpace(o.Response.Normalize.ValuePrefix) == "" { + return errors.New("response.normalize.value_prefix is required when normalization is enabled") + } + } + + if o.InjectTimestamp && strings.TrimSpace(o.TimestampField) == "" { + return errors.New("timestamp_field is required when inject_timestamp is enabled") + } + + return nil +} + +func (o *APIDatasetOptions) ApplyDefaults() { + if o.Version == 0 { + o.Version = APIDatasetOptionsVersion + } + if o.Timeout == nil { + defaultTimeout := 30 + o.Timeout = &defaultTimeout + } +} + +type QueryVar struct { + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` // e.g., "string", "int", "date" + Value string `json:"value" yaml:"value"` // Stored as string, parsed based on Type +} + +type JSONDatasetOptions struct { + Version int `json:"version" yaml:"version"` + Timeout *int `json:"timeout,omitempty" yaml:"timeout,omitempty"` + Query *string `json:"query,omitempty" yaml:"query,omitempty"` + Vars map[string]string `json:"vars,omitempty" yaml:"vars,omitempty"` + Dedupe []string `json:"dedupe,omitempty" yaml:"dedupe,omitempty"` + FlattenNestedObjects bool `yaml:"flatten_nested_objects,omitempty" json:"flattenNestedObjects,omitempty"` + NormalizeNestedObjects bool `yaml:"normalize_nested_objects,omitempty" json:"normalizeNestedObjects,omitempty"` + NormalizedKeyFieldName string `yaml:"normalized_key_field_name,omitempty" json:"normalizedKeyFieldName,omitempty"` + NormalizedValuePrefix string `yaml:"normalized_value_prefix,omitempty" json:"normalizedValuePrefix,omitempty"` +} + +func (o *JSONDatasetOptions) Validate() error { + if o.NormalizeNestedObjects { + if strings.TrimSpace(o.NormalizedKeyFieldName) == "" { + return errors.New("normalized_key_field_name is required when normalization is enabled") + } + if strings.TrimSpace(o.NormalizedValuePrefix) == "" { + return errors.New("normalized_value_prefix is required when normalization is enabled") + } + } + return nil +} + +func (o *JSONDatasetOptions) ApplyDefaults() { + if o.Version == 0 { + o.Version = 1 + } + if o.Timeout == nil { + defaultTimeout := 30 + o.Timeout = &defaultTimeout + } +} + +type SQLDatasetOptions struct { + Version int `json:"version" yaml:"version"` + Timeout *int `json:"timeout,omitempty" yaml:"timeout,omitempty"` + Query *string `json:"query,omitempty" yaml:"query,omitempty"` + Vars map[string]string `json:"vars,omitempty" yaml:"vars,omitempty"` + Dedupe []string `json:"dedupe,omitempty" yaml:"dedupe,omitempty"` +} + +func (o *SQLDatasetOptions) Validate() error { + if o.Query == nil { + return errors.New("query is required for sql datasets") + } + if strings.TrimSpace(*o.Query) == "" { + return errors.New("query must not be empty for sql datasets") + } + return nil +} + +func (o *SQLDatasetOptions) ApplyDefaults() { + if o.Version == 0 { + o.Version = 1 + } + if o.Timeout == nil { + defaultTimeout := 30 + o.Timeout = &defaultTimeout + } +} + +func DecodeDatasetOptions[T any](raw map[string]interface{}) (T, error) { + var out T + + if raw == nil { + return out, fmt.Errorf("dataset options are nil") + } + + b, err := json.Marshal(raw) + if err != nil { + return out, err + } + + if err := json.Unmarshal(b, &out); err != nil { + return out, err + } + + return out, nil +} + +func EncodeDatasetOptions(v any) map[string]interface{} { + b, _ := json.Marshal(v) + + var out map[string]interface{} + _ = json.Unmarshal(b, &out) + + return out +} + +type GoogleSheetDatasetOptions struct { + Version int `json:"version" yaml:"version"` + Timeout *int `json:"timeout,omitempty" yaml:"timeout,omitempty"` + Query *string `json:"query,omitempty" yaml:"query,omitempty"` + Vars map[string]string `json:"vars,omitempty" yaml:"vars,omitempty"` + Dedupe []string `json:"dedupe,omitempty" yaml:"dedupe,omitempty"` + HeaderRowNo int `yaml:"header_row_no,omitempty" json:"headerRowNo,omitempty"` + EnableStreaming bool `yaml:"enable_streaming,omitempty" json:"enableStreaming,omitempty"` +} + +func (o *GoogleSheetDatasetOptions) Validate() error { + if o.HeaderRowNo < 0 { + return errors.New("headerRowNo cannot be negative") + } + return nil +} + +func (o *GoogleSheetDatasetOptions) ApplyDefaults() { + if o.Version == 0 { + o.Version = 1 + } + if o.Timeout == nil { + defaultTimeout := 30 + o.Timeout = &defaultTimeout + } +} + +const ( + CSVDatasetOptionsVersion = 1 +) + +type CSVDatasetOptions struct { + Version int `json:"version" yaml:"version"` + Timeout *int `json:"timeout,omitempty" yaml:"timeout,omitempty"` + Query *string `json:"query,omitempty" yaml:"query,omitempty"` + Vars map[string]string `json:"vars,omitempty" yaml:"vars,omitempty"` + Dedupe []string `json:"dedupe,omitempty" yaml:"dedupe,omitempty"` + Delimiter string `yaml:"delimiter,omitempty" json:"delimiter,omitempty"` + HasHeaders bool `yaml:"has_headers,omitempty" json:"hasHeaders,omitempty"` + SkipLines int `yaml:"skip_lines,omitempty" json:"skipLines,omitempty"` +} + +func (o *CSVDatasetOptions) Validate() error { + if o.Delimiter != "" && utf8.RuneCountInString(o.Delimiter) != 1 { + return errors.New("delimiter must be a single character when provided") + } + if o.SkipLines < 0 { + return errors.New("skipLines cannot be negative") + } + return nil +} + +func (o *CSVDatasetOptions) ApplyDefaults() { + if o.Version == 0 { + o.Version = CSVDatasetOptionsVersion + } + if o.Timeout == nil { + defaultTimeout := 30 + o.Timeout = &defaultTimeout + } +} diff --git a/entities/datastore_configs.go b/entities/datastore_configs.go new file mode 100644 index 0000000..f91dc35 --- /dev/null +++ b/entities/datastore_configs.go @@ -0,0 +1,108 @@ +package entities + +import ( + "fmt" + "net/url" + "path" + "path/filepath" + "strings" +) + +type SQLiteLocalDataStoreConfig struct { + BasePath string `json:"basePath"` + FileName string `json:"fileName"` +} + +func (cfg SQLiteLocalDataStoreConfig) Path() string { + fileName := strings.TrimSpace(cfg.FileName) + if fileName == "" { + return "" + } + + basePath := strings.TrimSpace(cfg.BasePath) + if basePath == "" { + return fileName + } + + return filepath.Join(basePath, fileName) +} + +type SQLiteS3DataStoreConfig struct { + FileName string `json:"fileName"` + Bucket string `json:"bucket"` + KeyPrefix string `json:"keyPrefix,omitempty"` + Region string `json:"region"` + Endpoint string `json:"endpoint,omitempty"` + ForcePathStyle bool `json:"forcePathStyle,omitempty"` + WorkingDir string `json:"workingDir,omitempty"` +} + +func (cfg SQLiteS3DataStoreConfig) objectKey() string { + fileName := strings.TrimSpace(cfg.FileName) + if fileName == "" { + return "" + } + + key := strings.TrimPrefix(strings.TrimSpace(cfg.KeyPrefix), "/") + if key == "" { + return fileName + } + + key = strings.TrimSuffix(key, "/") + return key + "/" + fileName +} + +func (cfg SQLiteS3DataStoreConfig) RemoteLocation() (string, bool) { + if strings.TrimSpace(cfg.Bucket) == "" { + return "", false + } + + key := cfg.objectKey() + if key == "" { + return "", false + } + + u := url.URL{ + Scheme: "s3", + Host: strings.TrimSpace(cfg.Bucket), + Path: "/" + path.Clean(key), + } + + params := url.Values{} + if region := strings.TrimSpace(cfg.Region); region != "" { + params.Set("region", region) + } + if endpoint := strings.TrimSpace(cfg.Endpoint); endpoint != "" { + params.Set("endpoint", endpoint) + } + if cfg.ForcePathStyle { + params.Set("forcePathStyle", "true") + } + if workDir := strings.TrimSpace(cfg.WorkingDir); workDir != "" { + params.Set("workDir", workDir) + } + + u.RawQuery = params.Encode() + return u.String(), true +} + +func (cfg SQLiteS3DataStoreConfig) WorkingDirectory() string { + return strings.TrimSpace(cfg.WorkingDir) +} + +type PostgresDataStoreConfig struct { + User string `json:"user"` + Password string `json:"password"` + Host string `json:"host"` + Port int `json:"port"` + Database string `json:"database"` + SSLMode string `json:"sslmode"` +} + +func (cfg PostgresDataStoreConfig) DSN() string { + return fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", + cfg.Host, cfg.Port, cfg.User, cfg.Password, cfg.Database, cfg.SSLMode) +} + +type DataStoreOptions struct { +} diff --git a/entities/plan.go b/entities/plan.go index eb40dec..639b356 100644 --- a/entities/plan.go +++ b/entities/plan.go @@ -41,6 +41,7 @@ type DatasetChild struct { type Output struct { DatasetName string + Verbose bool } type DataColumn struct { diff --git a/entities/runtime.go b/entities/runtime.go index e2a7bfc..1599cd1 100644 --- a/entities/runtime.go +++ b/entities/runtime.go @@ -11,6 +11,6 @@ type ExecutePlanRequest struct { } type ExecutePlanResponse struct { - Columns []string - Rows [][]any + Data []map[string]interface{} + Columns []DataColumn } diff --git a/examples/local_execute_plan/main.go b/examples/local_execute_plan/main.go index 6d7c0b2..65782e4 100644 --- a/examples/local_execute_plan/main.go +++ b/examples/local_execute_plan/main.go @@ -5,18 +5,12 @@ import ( "fmt" "log" - "github.com/kvatch-hub/kvatch-runtime/configs" "github.com/kvatch-hub/kvatch-runtime/entities" "github.com/kvatch-hub/kvatch-runtime/pkg/runtime" ) func main() { - rt, err := runtime.New(configs.Config{ - Storage: configs.StorageConfig{ - Driver: "sqlite", - DSN: ":memory:", - }, - }) + rt, err := runtime.NewDefault() if err != nil { log.Fatalf("create runtime: %v", err) } @@ -41,12 +35,16 @@ func main() { {"id": 1, "title": "Dune"}, {"id": 2, "title": "Neuromancer"}, }, + Options: map[string]any{ + "timeout": 30, + }, }, }, } resp, err := rt.ExecutePlan(context.Background(), entities.ExecutePlanRequest{ - Plan: plan, + UserID: "someid", + Plan: plan, }) if err != nil { log.Fatalf("execute plan: %v", err) @@ -54,7 +52,7 @@ func main() { fmt.Printf("Columns: %v\n", resp.Columns) fmt.Printf("Rows:\n") - for _, row := range resp.Rows { + for _, row := range resp.Data { fmt.Printf(" %v\n", row) } } diff --git a/go.mod b/go.mod index 71d1401..e216079 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,96 @@ module github.com/kvatch-hub/kvatch-runtime go 1.25.1 + +require ( + github.com/DATA-DOG/go-sqlmock v1.5.2 + github.com/aws/aws-sdk-go-v2 v1.41.5 + github.com/aws/aws-sdk-go-v2/config v1.32.13 + github.com/aws/aws-sdk-go-v2/credentials v1.19.13 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 + github.com/aws/smithy-go v1.24.2 + github.com/glebarez/sqlite v1.11.0 + github.com/go-git/go-git/v5 v5.17.1 + github.com/google/uuid v1.6.0 + github.com/lib/pq v1.12.1 + github.com/pkg/errors v0.9.1 + github.com/stretchr/testify v1.11.1 + go.uber.org/mock v0.6.0 + google.golang.org/api v0.273.0 + gopkg.in/yaml.v3 v3.0.1 + gorm.io/driver/postgres v1.6.0 + gorm.io/gorm v1.31.1 +) + +require ( + cloud.google.com/go/auth v0.18.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + dario.cat/mergo v1.0.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + github.com/cyphar/filepath-securejoin v0.4.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/glebarez/go-sqlite v1.21.2 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.8.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect + github.com/googleapis/gax-go/v2 v2.19.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.6.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/pjbgf/sha1cd v0.3.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.42.0 // indirect + go.opentelemetry.io/otel/metric v1.42.0 // indirect + go.opentelemetry.io/otel/trace v1.42.0 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/grpc v1.79.3 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + modernc.org/libc v1.22.5 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.5.0 // indirect + modernc.org/sqlite v1.23.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..94bc3c2 --- /dev/null +++ b/go.sum @@ -0,0 +1,245 @@ +cloud.google.com/go/auth v0.18.2 h1:+Nbt5Ev0xEqxlNjd6c+yYUeosQ5TtEUaNcN/3FozlaM= +cloud.google.com/go/auth v0.18.2/go.mod h1:xD+oY7gcahcu7G2SG2DsBerfFxgPAJz17zz2joOFF3M= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= +github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg= +github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= +github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= +github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= +github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= +github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0= +github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.17.1 h1:WnljyxIzSj9BRRUlnmAU35ohDsjRK0EKmL0evDqi5Jk= +github.com/go-git/go-git/v5 v5.17.1/go.mod h1:pW/VmeqkanRFqR6AljLcs7EA7FbZaN5MQqO7oZADXpo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.14 h1:yh8ncqsbUY4shRD5dA6RlzjJaT4hi3kII+zYw8wmLb8= +github.com/googleapis/enterprise-certificate-proxy v0.3.14/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= +github.com/googleapis/gax-go/v2 v2.19.0 h1:fYQaUOiGwll0cGj7jmHT/0nPlcrZDFPrZRhTsoCr8hE= +github.com/googleapis/gax-go/v2 v2.19.0/go.mod h1:w2ROXVdfGEVFXzmlciUU4EdjHgWvB5h2n6x/8XSTTJA= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.12.1 h1:x1nbl/338GLqeDJ/FAiILallhAsqubLzEZu/pXtHUow= +github.com/lib/pq v1.12.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= +github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= +go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= +go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= +go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= +go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= +go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= +go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= +go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= +go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= +go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.273.0 h1:r/Bcv36Xa/te1ugaN1kdJ5LoA5Wj/cL+a4gj6FiPBjQ= +google.golang.org/api v0.273.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew= +google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5 h1:JNfk58HZ8lfmXbYK2vx/UvsqIL59TzByCxPIX4TDmsE= +google.golang.org/genproto v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:x5julN69+ED4PcFk/XWayw35O0lf/nGa4aNgODCmNmw= +google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5 h1:CogIeEXn4qWYzzQU0QqvYBM8yDF9cFYzDq9ojSpv0Js= +google.golang.org/genproto/googleapis/api v0.0.0-20260316180232-0b37fe3546d5/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7 h1:ndE4FoJqsIceKP2oYSnUZqhTdYufCYYkqwtFzfrhI7w= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= +gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= +gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= +gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= +modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= +modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= +modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= diff --git a/internal/connectors/api/connector.go b/internal/connectors/api/connector.go new file mode 100644 index 0000000..937f6ca --- /dev/null +++ b/internal/connectors/api/connector.go @@ -0,0 +1,360 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +// APIConnector implements the connectors.Connector interface for API. +type APIConnector struct { + config entities.APIConnectorConfig + client *http.Client +} + +func NewAPIConnector() connectors.Connector { + return &APIConnector{ + client: &http.Client{Timeout: 30 * time.Second}, + } +} + +func (c *APIConnector) GetType() connectors.ConnectorType { + return connectors.ConnectorTypeAPI +} + +func (c *APIConnector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error { + cfg, err := parseConfig(engineConnectorCtx.Config) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + c.config = cfg + return nil +} + +func (c *APIConnector) Ping(ctx context.Context, _ interface{}) error { + // Optional: we could do a HEAD on base_url or similar. + return nil +} + +func (c *APIConnector) Close() error { + // Nothing to clean up yet; keep for future (connection pools, etc.) + return nil +} + +// Validate implements the connectors.Connector interface. It validates a +// config object without mutating receiver state. +func (c *APIConnector) Validate(config interface{}) error { + cfg, err := parseConfig(config) + if err != nil { + return err + } + return cfg.Validate() +} + +// GetData exists to satisfy the Connector interface. api normally uses the +// APIDatasetPlugin, which calls BuildRequest/Do directly, but this method +// allows legacy paths (or tests) to still fetch raw data directly. +func (c *APIConnector) GetData( + ctx context.Context, + ds *models.EngineContextDataset, +) (io.ReadCloser, error) { + opts, err := parseAPIOptions(ds.DatasetOptions) + if err != nil { + return nil, fmt.Errorf("api: invalid dataset options: %w", err) + } + + // No pagination via GetData, just single request. + req, err := c.BuildRequest(ctx, &opts.Request, opts.Vars, nil) + if err != nil { + return nil, fmt.Errorf("api: failed to build request: %w", err) + } + + resp, err := c.Do(ctx, req) + if err != nil { + return nil, fmt.Errorf("api: request failed: %w", err) + } + + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + return nil, fmt.Errorf("api: http %d: %s", resp.StatusCode, string(body)) + } + + // Caller (plugin) is responsible for closing resp.Body. + return resp.Body, nil +} + +// +// ─── INTERNAL HELPERS ──────────────────────────────────────────────── +// + +// parseConfig converts various config representations into APIConnectorConfig. +func parseConfig(config interface{}) (entities.APIConnectorConfig, error) { + switch val := config.(type) { + case entities.APIConnectorConfig: + return val, nil + case *entities.APIConnectorConfig: + return *val, nil + case map[string]interface{}: + var cfg entities.APIConnectorConfig + buf, err := json.Marshal(val) + if err != nil { + return cfg, fmt.Errorf("marshal api config: %w", err) + } + if err := json.Unmarshal(buf, &cfg); err != nil { + return cfg, fmt.Errorf("unmarshal api config: %w", err) + } + return cfg, nil + default: + return entities.APIConnectorConfig{}, fmt.Errorf("invalid config type for api connector: %T", config) + } +} + +func parseAPIOptions(raw map[string]interface{}) (*entities.APIDatasetOptions, error) { + if raw == nil { + return nil, fmt.Errorf("missing api dataset options") + } + + // Convert raw map → JSON → typed struct + b, err := json.Marshal(raw) + if err != nil { + return nil, err + } + + var out entities.APIDatasetOptions + if err := json.Unmarshal(b, &out); err != nil { + return nil, err + } + + return &out, nil +} + +// BuildRequest constructs an http.Request from the API request spec, +// template params, and optional pagination state. +func (c *APIConnector) BuildRequest( + ctx context.Context, + reqCfg *entities.APIRequestSpec, + query_vars map[string]string, + page *paginationState, +) (*http.Request, error) { + + if reqCfg == nil { + return nil, fmt.Errorf("nil request spec") + } + + // -------------------------------------------------------------------- + // 1. Manual base_url + path join (to avoid ResolveReference bugs) + // -------------------------------------------------------------------- + base := strings.TrimRight(c.config.BaseURL, "/") + path, err := renderTemplateValue(reqCfg.Path, query_vars) + if err != nil { + return nil, fmt.Errorf("api: failed to render path: %w", err) + } + path = strings.TrimLeft(path, "/") // ⬅ fix: prevents /api removal + + full := base + "/" + path + + fullURL, err := url.Parse(full) + if err != nil { + return nil, fmt.Errorf("invalid constructed URL '%s': %w", full, err) + } + + // -------------------------------------------------------------------- + // 2. Apply static + templated query params + // -------------------------------------------------------------------- + q := fullURL.Query() + for k, v := range reqCfg.Params { + rendered, err := renderTemplateValue(v, query_vars) + if err != nil { + return nil, fmt.Errorf("api: query param %s: %w", k, err) + } + q.Set(k, rendered) + } + + // -------------------------------------------------------------------- + // 3. Pagination injection + // -------------------------------------------------------------------- + if page != nil { + switch page.Options.Type { + + case entities.APIPaginationPage: + if page.Options.PageParam != "" { + q.Set(page.Options.PageParam, fmt.Sprintf("%d", page.Page)) + } + if page.Options.PageSizeParam != "" && page.Options.PageSize > 0 { + q.Set(page.Options.PageSizeParam, fmt.Sprintf("%d", page.Options.PageSize)) + } + + case entities.APIPaginationOffset: + if page.Options.OffsetParam != "" { + q.Set(page.Options.OffsetParam, fmt.Sprintf("%d", page.Offset)) + } + if page.Options.LimitParam != "" && page.Options.Limit > 0 { + q.Set(page.Options.LimitParam, fmt.Sprintf("%d", page.Options.Limit)) + } + + case entities.APIPaginationCursor: + if page.Options.CursorParam != "" && page.Cursor != "" { + q.Set(page.Options.CursorParam, page.Cursor) + } + } + } + + fullURL.RawQuery = q.Encode() + + // -------------------------------------------------------------------- + // 4. Body (templated) + // -------------------------------------------------------------------- + var body io.Reader + if reqCfg.Body != nil && *reqCfg.Body != "" { + rendered, err := renderTemplateValue(*reqCfg.Body, query_vars) + if err != nil { + return nil, fmt.Errorf("api: failed to render body: %w", err) + } + body = strings.NewReader(rendered) + } + + // -------------------------------------------------------------------- + // 5. Build request + // -------------------------------------------------------------------- + method := strings.ToUpper(strings.TrimSpace(reqCfg.Method)) + if method == "" { + method = http.MethodGet + } + + req, err := http.NewRequestWithContext(ctx, method, fullURL.String(), body) + if err != nil { + return nil, fmt.Errorf("api: failed to build request: %w", err) + } + + // -------------------------------------------------------------------- + // 6. Connector-level default headers + // -------------------------------------------------------------------- + for k, v := range c.config.DefaultHeaders { + rendered, err := renderTemplateValue(v, query_vars) + if err != nil { + return nil, fmt.Errorf("api: default header %s: %w", k, err) + } + req.Header.Set(k, rendered) + } + + // -------------------------------------------------------------------- + // 7. Authentication + // -------------------------------------------------------------------- + if c.config.Auth != nil { + switch c.config.Auth.Type { + + case entities.APIAuthAPIKey: + if c.config.Auth.APIKeyHeader != "" && c.config.Auth.APIKeyValue != "" { + req.Header.Set(c.config.Auth.APIKeyHeader, c.config.Auth.APIKeyValue) + } + + case entities.APIAuthBearer: + if c.config.Auth.BearerToken != "" { + req.Header.Set("Authorization", "Bearer "+c.config.Auth.BearerToken) + } + + case entities.APIAuthBasic: + // future support + } + } + + // -------------------------------------------------------------------- + // 8. Dataset-level headers override connector defaults + // -------------------------------------------------------------------- + for k, v := range reqCfg.Headers { + rendered, err := renderTemplateValue(v, query_vars) + if err != nil { + return nil, fmt.Errorf("api: header %s: %w", k, err) + } + req.Header.Set(k, rendered) + } + + // -------------------------------------------------------------------- + // 9. Debug logging + // -------------------------------------------------------------------- + // fmt.Printf("API DEBUG - Request:\n") + // fmt.Printf(" Method: %s\n", req.Method) + // fmt.Printf(" URL: %s\n", req.URL.String()) + // fmt.Printf(" Headers:\n") + // for k, v := range req.Header { + // fmt.Printf(" %s: %v\n", k, v) + // } + + return req, nil +} + +// Do executes the HTTP request. You can later add rate limiting, retries, logging etc. +func (c *APIConnector) Do(ctx context.Context, req *http.Request) (*http.Response, error) { + // TODO: add rate limiting, retries, logging here + return c.client.Do(req) +} + +var apiVariablePlaceholderPattern = regexp.MustCompile(`\{\{\s*([A-Za-z][A-Za-z0-9_-]*)\s*\}\}`) + +// renderTemplateValue safely renders templated strings using provided params. +// It mirrors renderDatasetQuery semantics: unknown variables or unresolved +// placeholders return errors and invalid placeholder syntax is rejected. +func renderTemplateValue(value string, params map[string]string) (string, error) { + if strings.TrimSpace(value) == "" { + return value, nil + } + + matches := apiVariablePlaceholderPattern.FindAllStringSubmatchIndex(value, -1) + if len(matches) == 0 { + if strings.Contains(value, "{{") { + return "", fmt.Errorf("invalid variable placeholder syntax") + } + return value, nil + } + + normalizedValues := make(map[string]string, len(params)) + for key, val := range params { + normalizedValues[strings.ToLower(strings.TrimSpace(key))] = val + } + + var builder strings.Builder + lastIndex := 0 + for _, match := range matches { + start, end := match[0], match[1] + nameStart, nameEnd := match[2], match[3] + + builder.WriteString(value[lastIndex:start]) + + rawName := strings.TrimSpace(value[nameStart:nameEnd]) + normalized := strings.ToLower(rawName) + val, ok := normalizedValues[normalized] + if !ok { + return "", fmt.Errorf("template references undefined variable %q", rawName) + } + + builder.WriteString(val) + lastIndex = end + } + + builder.WriteString(value[lastIndex:]) + rendered := builder.String() + + if unresolved := apiVariablePlaceholderPattern.FindStringSubmatch(rendered); unresolved != nil { + return "", fmt.Errorf("template contains unresolved variable %q", unresolved[1]) + } + + if strings.Contains(rendered, "{{") { + return "", fmt.Errorf("template contains invalid variable syntax after rendering") + } + + return rendered, nil +} diff --git a/internal/connectors/api/connector_test.go b/internal/connectors/api/connector_test.go new file mode 100644 index 0000000..803a379 --- /dev/null +++ b/internal/connectors/api/connector_test.go @@ -0,0 +1,75 @@ +package api + +import "testing" + +func TestRenderTemplateValue(t *testing.T) { + testCases := []struct { + name string + value string + params map[string]string + expectErr bool + expected string + }{ + { + name: "no placeholders", + value: "https://example.com/static", + params: map[string]string{"id": "123"}, + expected: "https://example.com/static", + }, + { + name: "json body without placeholders", + value: `{"filter":{"a":1}}`, + params: map[string]string{"id": "123"}, + expected: `{"filter":{"a":1}}`, + }, + { + name: "simple substitution", + value: "/items/{{id}}", + params: map[string]string{"id": "42"}, + expected: "/items/42", + }, + { + name: "unknown variable", + value: "/items/{{missing}}", + params: map[string]string{"id": "42"}, + expectErr: true, + }, + { + name: "invalid syntax", + value: "/items/{{id", + params: map[string]string{"id": "42"}, + expectErr: true, + }, + { + name: "unresolved placeholder after rendering", + value: "/items/{{id}}/{{nested}}", + params: map[string]string{"id": "42", "nested": "{{still}}"}, + expectErr: true, + }, + { + name: "case insensitive variables", + value: "/items/{{ ID }}", + params: map[string]string{"id": "42"}, + expected: "/items/42", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual, err := renderTemplateValue(tc.value, tc.params) + if tc.expectErr { + if err == nil { + t.Fatalf("expected error but got none (result=%s)", actual) + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if actual != tc.expected { + t.Fatalf("expected %q but got %q", tc.expected, actual) + } + }) + } +} diff --git a/internal/connectors/api/pagination.go b/internal/connectors/api/pagination.go new file mode 100644 index 0000000..643dd15 --- /dev/null +++ b/internal/connectors/api/pagination.go @@ -0,0 +1,46 @@ +package api + +import ( + "encoding/json" + "fmt" + + "github.com/kvatch-hub/kvatch-runtime/entities" +) + +type paginationState struct { + Options entities.APIPaginationConfig + Page int + Cursor string + Offset int + Limit int +} + +func (p *paginationState) Advance(body []byte, cfg entities.APIPaginationConfig) (bool, error) { + switch cfg.Type { + case entities.APIPaginationNone: + return true, nil + + case entities.APIPaginationPage: + p.Page++ + if p.Page >= cfg.MaxPages && cfg.MaxPages > 0 { + return true, nil + } + return false, nil + + case entities.APIPaginationCursor: + var parsed map[string]interface{} + if err := json.Unmarshal(body, &parsed); err != nil { + return false, err + } + next, ok := parsed[cfg.CursorPath] + if !ok || next == nil { + return true, nil + } + p.Cursor = fmt.Sprintf("%v", next) + return false, nil + + // TODO: Link header + offset-limit + } + + return true, nil +} diff --git a/internal/connectors/aws_s3/connector.go b/internal/connectors/aws_s3/connector.go new file mode 100644 index 0000000..edbb043 --- /dev/null +++ b/internal/connectors/aws_s3/connector.go @@ -0,0 +1,262 @@ +// package s3conn +package aws_s3 + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + "path" + "strings" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + + awsCfg "github.com/aws/aws-sdk-go-v2/config" + awsCreds "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" +) + +type S3Connector struct { + client *s3.Client + bucket string + basePrefix string + isReady bool +} + +func NewS3Connector() *S3Connector { return &S3Connector{} } + +func (c *S3Connector) GetType() connectors.ConnectorType { + return connectors.ConnectorTypeS3 +} + +func (c *S3Connector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error { + if engineConnectorCtx == nil { + return fmt.Errorf("engine connector context is missing") + } + if engineConnectorCtx.Config == nil { + return fmt.Errorf("engine connector context options are missing") + } + + var cfg entities.S3ConnectorConfig + switch v := engineConnectorCtx.Config.(type) { + case entities.S3ConnectorConfig: + cfg = v + case *entities.S3ConnectorConfig: + cfg = *v + case map[string]interface{}: + // tolerate yaml -> map[string]any + cfg.Region, _ = v["region"].(string) + cfg.Bucket, _ = v["bucket"].(string) + cfg.BasePrefix, _ = v["base_prefix"].(string) + cfg.AccessKeyID, _ = v["access_key_id"].(string) + cfg.SecretAccessKey, _ = v["secret_access_key"].(string) + cfg.SessionToken, _ = v["session_token"].(string) + cfg.Profile, _ = v["profile"].(string) + cfg.Endpoint, _ = v["endpoint"].(string) + if fps, ok := v["force_path_style"].(bool); ok { + cfg.ForcePathStyle = fps + } + case string: + // allow s3 url like "s3://bucket/base/prefix" + u, err := url.Parse(v) + if err != nil || u.Scheme != "s3" || u.Host == "" { + return fmt.Errorf("invalid s3 connection string: %q", v) + } + cfg.Bucket = u.Host + cfg.BasePrefix = strings.TrimPrefix(u.Path, "/") + default: + return fmt.Errorf("invalid S3 connection config type: %T", v) + } + + if err := c.Validate(cfg); err != nil { + return err + } + + // Build AWS config + var ( + lo []func(*awsCfg.LoadOptions) error + ) + if cfg.Region != "" { + lo = append(lo, awsCfg.WithRegion(cfg.Region)) + } + if cfg.Profile != "" { + lo = append(lo, awsCfg.WithSharedConfigProfile(cfg.Profile)) + } + + // If explicit keys are provided, prefer them + if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" { + lo = append(lo, awsCfg.WithCredentialsProvider( + awsCreds.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, cfg.SessionToken), + )) + } + + loaded, err := awsCfg.LoadDefaultConfig(ctx, lo...) + if err != nil { + return fmt.Errorf("failed to load AWS config: %w", err) + } + + clientOpts := []func(*s3.Options){ + func(o *s3.Options) { + if cfg.Endpoint != "" { + o.BaseEndpoint = awsStringPtr(cfg.Endpoint) + } + if cfg.ForcePathStyle { + o.UsePathStyle = true + } + }, + } + client := s3.NewFromConfig(loaded, clientOpts...) + + // Optionally, validate bucket exists / is accessible (lightweight head) + if cfg.Bucket != "" { + _, headErr := client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &cfg.Bucket}) + // Some setups (minio/localstack) might not support HeadBucket; be lenient but informative + if headErr != nil { + var re *s3types.NotFound + if errors.As(headErr, &re) { + return fmt.Errorf("s3 bucket not found: %s", cfg.Bucket) + } + // If smithy error with status 403/404, surface a clear message + var apiErr smithy.APIError + if errors.As(headErr, &apiErr) { + code := apiErr.ErrorCode() + return fmt.Errorf("unable to access bucket %q (code: %s): %v", cfg.Bucket, code, headErr) + } + // Otherwise, warn but continue (some endpoints reject HeadBucket) + } + } + + c.client = client + c.bucket = cfg.Bucket + c.basePrefix = strings.TrimPrefix(cfg.BasePrefix, "/") + c.isReady = true + return nil +} + +func (c *S3Connector) GetData(ctx context.Context, dp *models.EngineContextDataset) (io.ReadCloser, error) { + if !c.isReady || c.client == nil { + return nil, fmt.Errorf("s3 connection not established") + } + if dp == nil { + return nil, fmt.Errorf("dataset plan is missing") + } + + // dp.Query can be: + // - key like "folder/file.csv" + // - s3 uri "s3://bucket/key" + // - empty (error unless BasePrefix points exactly to a file; we avoid auto listing for now) + var bucket = c.bucket + var key string + + q := strings.TrimSpace(dp.Query) + if q == "" && c.basePrefix == "" { + return nil, fmt.Errorf("s3 dataset requires a key in dataset query or a base_prefix pointing to a file") + } + + // s3:// URL? + if strings.HasPrefix(q, "s3://") { + u, err := url.Parse(q) + if err != nil { + return nil, fmt.Errorf("invalid s3 URL in dataset query: %w", err) + } + if u.Host == "" || u.Path == "" { + return nil, fmt.Errorf("invalid s3 URL; require bucket and key: %q", q) + } + bucket = u.Host + key = strings.TrimPrefix(u.Path, "/") + } else { + key = q + } + + // If key empty, try basePrefix (treat as a single object path, not a folder) + if key == "" { + key = c.basePrefix + } else if c.basePrefix != "" && !strings.HasPrefix(key, c.basePrefix) { + key = path.Join(c.basePrefix, key) + } + + if bucket == "" { + return nil, fmt.Errorf("s3 bucket is not configured; provide in connector config or use s3:// in query") + } + if key == "" || strings.HasSuffix(key, "/") { + return nil, fmt.Errorf("s3 object key must point to a file, got %q", key) + } + + out, err := c.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + }) + if err != nil { + // map common cases + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "NoSuchKey": + return nil, &connectors.ResourceNotFoundError{ + ConnectionType: c.GetType(), + ResourcePath: fmt.Sprintf("s3://%s/%s", bucket, key), + } + } + } + return nil, fmt.Errorf("failed to get s3 object s3://%s/%s: %w", bucket, key, err) + } + + // Return the body directly; caller (plugin pipeline) will consume/close it. + return out.Body, nil +} + +func (c *S3Connector) Close() error { + // AWS S3 client has no open sockets to close; just mark not-ready + c.client = nil + c.isReady = false + return nil +} + +func (c *S3Connector) Validate(config interface{}) error { + switch v := config.(type) { + case entities.S3ConnectorConfig: + return c.validateCfg(v) + case *entities.S3ConnectorConfig: + if v == nil { + return fmt.Errorf("nil s3 config") + } + return c.validateCfg(*v) + case map[string]interface{}: + // minimal check: must have either bucket in config or be provided at dataset level via s3:// URL + _, hasBucket := v["bucket"] + _, hasRegion := v["region"] + if !hasBucket { + // allowed if dataset uses s3:// URL, but we can’t see that here; be lenient + // still, region is highly recommended when not using profile/env + if !hasRegion { + // allow missing region if relying on env/instance profile + return nil + } + } + return nil + case string: + // allow s3://bucket/prefix form + if !strings.HasPrefix(v, "s3://") { + return fmt.Errorf("string s3 config must be an s3:// URL (e.g., s3://bucket/base/prefix)") + } + return nil + default: + return fmt.Errorf("invalid S3 connection config type: %T", config) + } +} + +func (c *S3Connector) validateCfg(cfg entities.S3ConnectorConfig) error { + // If explicit keys are partially provided, error early + if (cfg.AccessKeyID != "" && cfg.SecretAccessKey == "") || (cfg.AccessKeyID == "" && cfg.SecretAccessKey != "") { + return fmt.Errorf("both access_key_id and secret_access_key must be provided together") + } + // Region: strongly recommended unless using shared config/env; keep lenient + return nil +} + +func awsStringPtr(s string) *string { return &s } diff --git a/internal/connectors/config_decoder.go b/internal/connectors/config_decoder.go new file mode 100644 index 0000000..fb3efc7 --- /dev/null +++ b/internal/connectors/config_decoder.go @@ -0,0 +1,75 @@ +package connectors + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/kvatch-hub/kvatch-runtime/entities" +) + +func DecodeAndValidateConfig(connectorType string, raw json.RawMessage, mask bool) (interface{}, error) { + if len(raw) == 0 || string(raw) == "null" { + return nil, fmt.Errorf("missing config for connector type %s", connectorType) + } + + decode := func(target any, validateFn func() error) (interface{}, error) { + if err := json.Unmarshal(raw, target); err != nil { + return nil, fmt.Errorf("invalid %s config: %w", connectorType, err) + } + + if validateFn != nil { + if err := validateFn(); err != nil { + dump, _ := json.MarshalIndent(target, "", " ") + return nil, fmt.Errorf("invalid %s config: %v\nDecoded:\n%s", connectorType, err, dump) + } + } + if mask { + return MaskSensitiveFieldsUsingTags(target), nil + } + return target, nil + } + + switch strings.ToUpper(connectorType) { + case string(ConnectorTypePostgres): + var cfg entities.PostgresConnectorConfig + return decode(&cfg, cfg.Validate) + + case string(ConnectorTypeSQLite): + var cfg entities.SQLiteConnectorConfig + return decode(&cfg, nil) + + case string(ConnectorTypeLocalFile): + var cfg entities.LocalFileConnectorConfig + return decode(&cfg, nil) + + case string(ConnectorTypeLocalDirectory): + var cfg entities.DirectoryConnectorConfig + return decode(&cfg, nil) + + case string(ConnectorTypeGoogleSheet): + var cfg entities.GoogleSheetConnectorConfig + return decode(&cfg, cfg.Validate) + + case string(ConnectorTypeGoogleAPI): + var cfg entities.GoogleAPIConnectorConfig + return decode(&cfg, nil) + + case string(ConnectorTypeGit): + var cfg entities.GitConnectorConfig + return decode(&cfg, nil) + + case string(ConnectorTypeS3): + var cfg entities.S3ConnectorConfig + return decode(&cfg, nil) + + case string(ConnectorTypeAPI): + var cfg entities.APIConnectorConfig + return decode(&cfg, func() error { + return cfg.Validate() + }) + + default: + return nil, fmt.Errorf("unsupported connector type %s", connectorType) + } +} diff --git a/internal/connectors/config_merge.go b/internal/connectors/config_merge.go new file mode 100644 index 0000000..d34dcf0 --- /dev/null +++ b/internal/connectors/config_merge.go @@ -0,0 +1,77 @@ +package connectors + +import ( + "encoding/json" + "fmt" +) + +// MergeMaskedConfig replaces masked values in the incoming connector configuration +// with their counterparts from an existing configuration map. This allows clients +// to send masked secrets without forcing the user to re-enter them on every +// update. +func MergeMaskedConfig(incoming any, existing map[string]any) (map[string]any, error) { + if incoming == nil { + return existing, nil + } + + rawIncoming, err := json.Marshal(incoming) + if err != nil { + return nil, fmt.Errorf("failed to marshal incoming config: %w", err) + } + + var decodedIncoming map[string]any + if err := json.Unmarshal(rawIncoming, &decodedIncoming); err != nil { + return nil, fmt.Errorf("failed to decode incoming config: %w", err) + } + + return mergeMap(decodedIncoming, existing), nil +} + +func mergeMap(incoming map[string]any, existing map[string]any) map[string]any { + merged := make(map[string]any, len(incoming)) + + for key, incomingVal := range incoming { + switch typedVal := incomingVal.(type) { + case string: + if typedVal == MaskedValue { + if existingVal, ok := existing[key]; ok { + merged[key] = existingVal + continue + } + } + merged[key] = incomingVal + case map[string]any: + existingMap := map[string]any{} + if rawExisting, ok := existing[key].(map[string]any); ok { + existingMap = rawExisting + } + merged[key] = mergeMap(typedVal, existingMap) + case []any: + merged[key] = mergeSlice(typedVal, existing[key]) + default: + merged[key] = incomingVal + } + } + + return merged +} + +func mergeSlice(incoming []any, existing any) []any { + existingSlice, _ := existing.([]any) + merged := make([]any, len(incoming)) + + for idx, value := range incoming { + switch typedVal := value.(type) { + case map[string]any: + var existingMap map[string]any + if idx < len(existingSlice) { + existingMap, _ = existingSlice[idx].(map[string]any) + } + merged[idx] = mergeMap(typedVal, existingMap) + default: + merged[idx] = value + } + } + + return merged +} diff --git a/internal/connectors/config_merge_test.go b/internal/connectors/config_merge_test.go new file mode 100644 index 0000000..06af219 --- /dev/null +++ b/internal/connectors/config_merge_test.go @@ -0,0 +1,81 @@ +package connectors + +import ( + "reflect" + "testing" +) + +func TestMergeMaskedConfig_ReplacesMaskedValues(t *testing.T) { + existing := map[string]any{ + "username": "admin", + "password": "secret", + } + + incoming := map[string]any{ + "username": "admin", + "password": MaskedValue, + } + + merged, err := MergeMaskedConfig(incoming, existing) + if err != nil { + t.Fatalf("MergeMaskedConfig returned error: %v", err) + } + + if merged["password"] != "secret" { + t.Fatalf("expected masked password to be replaced, got %v", merged["password"]) + } +} + +func TestMergeMaskedConfig_HandlesNestedStructures(t *testing.T) { + existing := map[string]any{ + "auth": map[string]any{ + "username": "user", + "password": "hunter2", + }, + "headers": []any{ + map[string]any{"x-api-key": "abc"}, + }, + } + + incoming := map[string]any{ + "auth": map[string]any{ + "username": "user", + "password": MaskedValue, + }, + "headers": []any{ + map[string]any{"x-api-key": MaskedValue}, + }, + } + + merged, err := MergeMaskedConfig(incoming, existing) + if err != nil { + t.Fatalf("MergeMaskedConfig returned error: %v", err) + } + + expected := map[string]any{ + "auth": map[string]any{ + "username": "user", + "password": "hunter2", + }, + "headers": []any{ + map[string]any{"x-api-key": "abc"}, + }, + } + + if !reflect.DeepEqual(expected, merged) { + t.Fatalf("unexpected merged config. got=%v expected=%v", merged, expected) + } +} + +func TestMergeMaskedConfig_AllowsNilIncoming(t *testing.T) { + existing := map[string]any{"token": "value"} + + merged, err := MergeMaskedConfig(nil, existing) + if err != nil { + t.Fatalf("MergeMaskedConfig returned error: %v", err) + } + + if !reflect.DeepEqual(existing, merged) { + t.Fatalf("expected existing config to be returned when incoming is nil") + } +} diff --git a/internal/connectors/entities.go b/internal/connectors/entities.go new file mode 100644 index 0000000..01444f8 --- /dev/null +++ b/internal/connectors/entities.go @@ -0,0 +1,3 @@ +package connectors + +const MaskedValue = "********" diff --git a/internal/connectors/git/connector.go b/internal/connectors/git/connector.go new file mode 100644 index 0000000..7b87eb6 --- /dev/null +++ b/internal/connectors/git/connector.go @@ -0,0 +1,288 @@ +package gitconnector + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +type GitConnector struct { + repoURL string + branch string + subPath string + rootPath string +} + +func NewGitConnector() connectors.Connector { + return &GitConnector{} +} + +func (c *GitConnector) GetType() connectors.ConnectorType { + return connectors.ConnectorTypeGit +} + +func (c *GitConnector) Ping(ctx context.Context, config interface{}) error { + cfg, err := parseConfig(config) + if err != nil { + return err + } + + if cfg.Repo == "" { + return fmt.Errorf("git connection requires repo") + } + if cfg.Branch == "" { + cfg.Branch = "main" + } + + // Use a temp directory for a shallow clone/fetch + cachePath := filepath.Join(os.TempDir(), "federate_git_ping", hashPath(cfg.Repo, cfg.Branch)) + + // If not already cloned, do a shallow clone + if _, err := os.Stat(cachePath); os.IsNotExist(err) { + _, err := git.PlainCloneContext(ctx, cachePath, false, &git.CloneOptions{ + URL: cfg.Repo, + ReferenceName: plumbing.NewBranchReferenceName(cfg.Branch), + SingleBranch: true, + Depth: 1, // shallow fetch is enough for Ping + }) + if err != nil { + return fmt.Errorf("failed to clone repo for ping: %w", err) + } + } else { + // Repo exists locally: just open and try fetch + repo, err := git.PlainOpen(cachePath) + if err != nil { + return fmt.Errorf("failed to open cached repo: %w", err) + } + err = repo.FetchContext(ctx, &git.FetchOptions{ + RemoteName: "origin", + Depth: 1, + }) + if err != nil && err != git.NoErrAlreadyUpToDate { + return fmt.Errorf("failed to fetch repo for ping: %w", err) + } + } + + // Validate configured path inside repo (if set) + if cfg.Path != "" { + target := filepath.Join(cachePath, cfg.Path) + info, err := os.Stat(target) + if err != nil { + return fmt.Errorf("configured path does not exist in repo: %s: %w", cfg.Path, err) + } + if !info.IsDir() { + return fmt.Errorf("configured path is not a directory: %s", cfg.Path) + } + } + + return nil +} + +func (c *GitConnector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error { + if engineConnectorCtx == nil { + return fmt.Errorf("engine connector context is missing") + } + if engineConnectorCtx.Config == nil { + return fmt.Errorf("engine connector context options are missing") + } + + cfg, err := parseConfig(engineConnectorCtx.Config) + if err != nil { + return err + } + + if cfg.Repo == "" { + return fmt.Errorf("git connection requires repo") + } + if cfg.Branch == "" { + cfg.Branch = "main" + } + + cachePath := filepath.Join(os.TempDir(), "federate_git_cache", hashPath(cfg.Repo, cfg.Branch)) + + // Clone or open + if _, err := os.Stat(cachePath); os.IsNotExist(err) { + _, err := git.PlainCloneContext(ctx, cachePath, false, &git.CloneOptions{ + URL: cfg.Repo, + ReferenceName: plumbing.NewBranchReferenceName(cfg.Branch), + SingleBranch: true, + Depth: 0, // Full history to ensure all files are fetched + }) + if err != nil { + return fmt.Errorf("failed to clone repo: %w", err) + } + } else { + repo, err := git.PlainOpen(cachePath) + if err != nil { + return fmt.Errorf("failed to open existing repo: %w", err) + } + + err = repo.FetchContext(ctx, &git.FetchOptions{ + RemoteName: "origin", + Depth: 0, + Force: true, + Tags: git.NoTags, + RefSpecs: []config.RefSpec{ + config.RefSpec(fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", cfg.Branch, cfg.Branch)), + }, + }) + if err != nil && err != git.NoErrAlreadyUpToDate { + return fmt.Errorf("failed to fetch repo: %w", err) + } + + wt, err := repo.Worktree() + if err != nil { + return fmt.Errorf("failed to get worktree: %w", err) + } + + _ = wt.Clean(&git.CleanOptions{Dir: true}) + + ref, err := repo.Reference(plumbing.NewRemoteReferenceName("origin", cfg.Branch), true) + if err != nil { + return fmt.Errorf("failed to get remote ref: %w", err) + } + + err = wt.Reset(&git.ResetOptions{ + Mode: git.HardReset, + Commit: ref.Hash(), + }) + if err != nil { + return fmt.Errorf("failed to reset worktree: %w", err) + } + } + + // Set rootPath and check files + if err := c.setPaths(cfg, cachePath); err != nil { + return err + } + + fmt.Printf("📁 Resolved repo root path: %s\n", c.rootPath) + printDirRecursive(c.rootPath) + + return nil +} + +func (c *GitConnector) GetData(ctx context.Context, dp *models.EngineContextDataset) (io.ReadCloser, error) { + if c.rootPath == "" { + return nil, fmt.Errorf("connection not established") + } + + fullPath := filepath.Join(c.rootPath, dp.Query) + + absRoot, _ := filepath.Abs(c.rootPath) + absFull, _ := filepath.Abs(fullPath) + rel, err := filepath.Rel(absRoot, absFull) + if err != nil || strings.HasPrefix(rel, "..") { + return nil, fmt.Errorf("path traversal detected: %s", dp.Query) + } + + file, err := os.Open(fullPath) + if err != nil { + if os.IsNotExist(err) { + return nil, &connectors.ResourceNotFoundError{ + ConnectionType: c.GetType(), + ResourcePath: dp.Query, + } + } + return nil, fmt.Errorf("failed to open file: %w", err) + } + return file, nil +} + +func (c *GitConnector) Close() error { + c.rootPath = "" + return nil +} + +func (c *GitConnector) Validate(config interface{}) error { + var cfg *entities.GitConnectorConfig + + switch v := config.(type) { + case *entities.GitConnectorConfig: + cfg = v + case entities.GitConnectorConfig: + cfg = &v + case map[string]interface{}: + _, ok := v["repo"].(string) + if !ok { + return fmt.Errorf("git connection requires repo") + } + default: + return fmt.Errorf("invalid git connection config type: %T", config) + } + + if cfg.Repo == "" { + return fmt.Errorf("git connection requires repo") + } + return nil +} + +func (c *GitConnector) setPaths(cfg *entities.GitConnectorConfig, cachePath string) error { + c.repoURL = cfg.Repo + c.branch = cfg.Branch + c.subPath = strings.Trim(cfg.Path, "/") + c.rootPath = filepath.Join(cachePath, c.subPath) + + stat, err := os.Stat(c.rootPath) + if err != nil || !stat.IsDir() { + return fmt.Errorf("configured path '%s' does not exist in repo", c.subPath) + } + + return nil +} + +func hashPath(repo, ref string) string { + sum := sha256.Sum256([]byte(repo + "@" + ref)) + return fmt.Sprintf("%x", sum[:])[:10] +} + +func printDirRecursive(path string) { + fmt.Printf("📦 Contents of %s:\n", path) + _ = filepath.Walk(path, func(p string, info os.FileInfo, err error) error { + if err == nil { + rel, _ := filepath.Rel(path, p) + if rel != "." { + fmt.Println(" -", rel) + } + } + return nil + }) +} + +func parseConfig(config interface{}) (*entities.GitConnectorConfig, error) { + switch v := config.(type) { + case *entities.GitConnectorConfig: + return v, nil + case entities.GitConnectorConfig: + return &v, nil + case map[string]interface{}: + cfg := &entities.GitConnectorConfig{} + if repo, ok := v["repo"].(string); ok { + cfg.Repo = repo + } + if branch, ok := v["branch"].(string); ok { + cfg.Branch = branch + } + if path, ok := v["path"].(string); ok { + cfg.Path = path + } + return cfg, nil + case string: + // Interpret as repo URL only + return &entities.GitConnectorConfig{Repo: v, Branch: "main"}, nil + default: + return nil, fmt.Errorf("invalid config type for GitConnector: %T", config) + } +} diff --git a/internal/connectors/googlesheets/connector.go b/internal/connectors/googlesheets/connector.go new file mode 100644 index 0000000..3d08b49 --- /dev/null +++ b/internal/connectors/googlesheets/connector.go @@ -0,0 +1,271 @@ +package googlesheet + +import ( + "context" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "google.golang.org/api/option" + "google.golang.org/api/sheets/v4" +) + +type GoogleSheetConnector struct { + apiKey string + spreadsheetID string + readRange string + service *sheets.Service +} + +func NewGoogleSheetConnector() connectors.Connector { + return &GoogleSheetConnector{} +} + +func (c *GoogleSheetConnector) GetType() connectors.ConnectorType { + return connectors.ConnectorTypeGoogleSheet +} + +// Ping checks whether the spreadsheet is reachable and readable. +func (c *GoogleSheetConnector) Ping(ctx context.Context, config interface{}) error { + cfg, err := parseConfig(config) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + + switch cfg.Mode() { + case "api": + // Private/authenticated sheet via API key + svc, err := sheets.NewService(ctx, option.WithAPIKey(cfg.APIKey)) + if err != nil { + return fmt.Errorf("failed to create Sheets client: %w", err) + } + // Test fetch one row + _, err = svc.Spreadsheets.Values.Get(cfg.SpreadsheetID, cfg.ReadRange).Do() + if err != nil { + return fmt.Errorf("failed to fetch sheet values via API: %w", err) + } + return nil + + case "public": + // Public sheet fallback (CSV export) + url := fmt.Sprintf( + "https://docs.google.com/spreadsheets/d/%s/gviz/tq?tqx=out:csv&sheet=%s", + cfg.SpreadsheetID, cfg.ReadRange, + ) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("failed to build request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to fetch public sheet: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("public sheet fetch returned status %d", resp.StatusCode) + } + + // Peek a few bytes to confirm data + buf := make([]byte, 64) + if _, err := resp.Body.Read(buf); err != nil && err != io.EOF { + return fmt.Errorf("failed to read public sheet: %w", err) + } + return nil + + default: + return fmt.Errorf("invalid mode: %s", cfg.Mode()) + } +} + +func (c *GoogleSheetConnector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error { + if engineConnectorCtx == nil || engineConnectorCtx.Config == nil { + return fmt.Errorf("engine connector context is missing or invalid") + } + + cfg, err := parseConfig(engineConnectorCtx.Config) + if err != nil { + return err + } + if err := cfg.Validate(); err != nil { + return err + } + + c.apiKey = cfg.APIKey + c.spreadsheetID = cfg.SpreadsheetID + c.readRange = cfg.ReadRange + + switch cfg.Mode() { + case "api": + // Only create Sheets service if API key is provided + svc, err := sheets.NewService(ctx, option.WithAPIKey(c.apiKey)) + if err != nil { + return fmt.Errorf("failed to create Sheets client: %w", err) + } + c.service = svc + + case "public": + // Nothing to initialize; just confirm basic accessibility + url := fmt.Sprintf( + "https://docs.google.com/spreadsheets/d/%s/gviz/tq?tqx=out:csv&sheet=%s", + c.spreadsheetID, c.readRange, + ) + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("failed to fetch public sheet: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("public sheet fetch returned status %d", resp.StatusCode) + } + + default: + return fmt.Errorf("invalid mode: %s", cfg.Mode()) + } + + return nil +} + +func (c *GoogleSheetConnector) GetData(ctx context.Context, dp *models.EngineContextDataset) (io.ReadCloser, error) { + if c.spreadsheetID == "" { + return nil, fmt.Errorf("spreadsheetID is required") + } + + readRange := c.readRange + if readRange == "" { + // fallback default: first sheet, all columns + readRange = "Sheet1" + } + + // Authenticated mode: use Sheets API + if c.apiKey != "" && c.service != nil { + resp, err := c.service.Spreadsheets.Values.Get(c.spreadsheetID, readRange).Do() + if err != nil { + return nil, fmt.Errorf("failed to get sheet values via API: %w", err) + } + + jsonData, err := json.Marshal(resp.Values) + if err != nil { + return nil, fmt.Errorf("failed to marshal sheet values to JSON: %w", err) + } + + return io.NopCloser(strings.NewReader(string(jsonData))), nil + } + + // Public mode: use CSV endpoint + url := fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/gviz/tq?tqx=out:csv&sheet=%s", c.spreadsheetID, readRange) + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to fetch public sheet: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("public sheet fetch returned status %d", resp.StatusCode) + } + + reader := csv.NewReader(resp.Body) + allRows, err := reader.ReadAll() + if err != nil { + return nil, fmt.Errorf("failed to parse public CSV: %w", err) + } + + var result [][]interface{} + for _, row := range allRows { + line := make([]interface{}, len(row)) + for i, col := range row { + line[i] = col + } + result = append(result, line) + } + + jsonData, err := json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("failed to marshal CSV data to JSON: %w", err) + } + + return io.NopCloser(strings.NewReader(string(jsonData))), nil +} + +func (c *GoogleSheetConnector) Close() error { + c.service = nil + return nil +} + +func (c *GoogleSheetConnector) Validate(config interface{}) error { + switch v := config.(type) { + case entities.GoogleSheetConnectorConfig: + if v.SpreadsheetID == "" { + return fmt.Errorf("missing spreadsheet_id") + } + // if v.ReadRange == "" { + // return fmt.Errorf("missing read_range") + // } + case *entities.GoogleSheetConnectorConfig: + if v.SpreadsheetID == "" { + return fmt.Errorf("missing spreadsheet_id") + } + // if v.ReadRange == "" { + // return fmt.Errorf("missing read_range") + // } + case map[string]interface{}: + if _, ok := v["spreadsheet_id"].(string); !ok { + return fmt.Errorf("missing or invalid spreadsheet_id") + } + // if _, ok := v["read_range"].(string); !ok { + // return fmt.Errorf("missing or invalid read_range") + // } + default: + return fmt.Errorf("invalid config type: %T", config) + } + return nil +} + +func parseConfig(config interface{}) (*entities.GoogleSheetConnectorConfig, error) { + switch v := config.(type) { + case *entities.GoogleSheetConnectorConfig: + return v, nil + case entities.GoogleSheetConnectorConfig: + return &v, nil + case map[string]interface{}: + cfg := &entities.GoogleSheetConnectorConfig{} + if id, ok := v["spreadsheet_id"].(string); ok { + cfg.SpreadsheetID = id + } + if rng, ok := v["read_range"].(string); ok { + cfg.ReadRange = rng + } + if apiKey, ok := v["api_key"].(string); ok { + cfg.APIKey = apiKey + } + if header, ok := v["header_row_no"].(int); ok { + cfg.HeaderRowNo = header + } + return cfg, nil + case string: + // treat as spreadsheetID in public mode + return &entities.GoogleSheetConnectorConfig{ + SpreadsheetID: v, + ReadRange: "Sheet1", // sensible default + }, nil + default: + return nil, fmt.Errorf("invalid config type for GoogleSheetConnector: %T", config) + } +} diff --git a/internal/connectors/localdirectory/connector.go b/internal/connectors/localdirectory/connector.go new file mode 100644 index 0000000..77e752c --- /dev/null +++ b/internal/connectors/localdirectory/connector.go @@ -0,0 +1,213 @@ +package localdirectory + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +// DirectoryConnector handles local file system access +type DirectoryConnector struct { + directoryPath string +} + +// NewDirectoryConnector creates a new local file connection +func NewDirectoryConnector() connectors.Connector { + return &DirectoryConnector{} +} + +// GetType returns the connection type +func (c *DirectoryConnector) GetType() connectors.ConnectorType { + return connectors.ConnectorTypeLocalDirectory +} + +func (c *DirectoryConnector) Ping(ctx context.Context, config interface{}) error { + cfg, err := parseConfig(config) + if err != nil { + return err + } + + dirPath := strings.TrimSpace(cfg.DirectoryPath) + if dirPath == "" { + return fmt.Errorf("directory connector requires directory_path") + } + + // Expand ~ to user home dir + if strings.HasPrefix(dirPath, "~") { + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("cannot resolve ~ in path: %w", err) + } + dirPath = filepath.Join(home, strings.TrimPrefix(dirPath, "~")) + } + + // Make absolute if not already + if !filepath.IsAbs(dirPath) { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("cannot get current working directory: %w", err) + } + dirPath = filepath.Join(cwd, dirPath) + } + dirPath = filepath.Clean(dirPath) + + // Try resolving symlinks + if resolved, err := filepath.EvalSymlinks(dirPath); err == nil { + dirPath = resolved + } + + info, err := os.Stat(dirPath) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("directory does not exist: %s", dirPath) + } + return fmt.Errorf("cannot stat directory: %s: %w", dirPath, err) + } + if !info.IsDir() { + return fmt.Errorf("expected a directory but got a file: %s", dirPath) + } + + // Try opening the directory + f, err := os.Open(dirPath) + if err != nil { + return fmt.Errorf("directory not accessible: %s: %w", dirPath, err) + } + _ = f.Close() + + return nil +} + +// Connect establishes the connection with the local file system +func (c *DirectoryConnector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error { + if engineConnectorCtx == nil { + return fmt.Errorf("engine connector context is missing") + } + + if engineConnectorCtx.Config == nil { + return fmt.Errorf("engine connector context options are missing") + } + + cfg, err := parseConfig(engineConnectorCtx.Config) + if err != nil { + return err + } + + if cfg.DirectoryPath == "" { + return fmt.Errorf("local file connection requires base_path") + } + + // Validate that the base path exists and is accessible + if _, err := os.Stat(cfg.DirectoryPath); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("base path does not exist: %s", cfg.DirectoryPath) + } + return fmt.Errorf("cannot access base path %s: %v", cfg.DirectoryPath, err) + } + + c.directoryPath = cfg.DirectoryPath + return nil +} + +// GetData returns a reader for the specified file +func (c *DirectoryConnector) GetData(ctx context.Context, dp *models.EngineContextDataset) (io.ReadCloser, error) { + if c.directoryPath == "" { + return nil, fmt.Errorf("connection not established") + } + + // Construct the full file path + fullPath := filepath.Join(c.directoryPath, dp.Query) + + // Security check: ensure the path doesn't escape the base directory + absBasePath, err := filepath.Abs(c.directoryPath) + if err != nil { + return nil, fmt.Errorf("failed to get absolute base path: %v", err) + } + + absFullPath, err := filepath.Abs(fullPath) + if err != nil { + return nil, fmt.Errorf("failed to get absolute file path: %v", err) + } + + absBasePath = filepath.Clean(absBasePath) + absFullPath = filepath.Clean(absFullPath) + + rel, err := filepath.Rel(absBasePath, absFullPath) + if err != nil || strings.HasPrefix(rel, "..") { + return nil, fmt.Errorf("path traversal attempt detected: %s", dp.Query) + } + + // Open the file + file, err := os.Open(fullPath) + if err != nil { + if os.IsNotExist(err) { + return nil, &connectors.ResourceNotFoundError{ + ConnectionType: c.GetType(), + ResourcePath: dp.Query, + } + } + return nil, fmt.Errorf("failed to open file %s: %v", dp.Query, err) + } + + return file, nil +} + +// Close cleans up the connection +func (c *DirectoryConnector) Close() error { + c.directoryPath = "" + return nil +} + +// Validate validates the local file connection configuration +func (c *DirectoryConnector) Validate(config interface{}) error { + switch cfg := config.(type) { + case *entities.DirectoryConnectorConfig: + if cfg.DirectoryPath == "" { + return fmt.Errorf("directory connector requires directory_path") + } + case entities.DirectoryConnectorConfig: + if cfg.DirectoryPath == "" { + return fmt.Errorf("directory connector requires directory_path") + } + case map[string]interface{}: + if _, ok := cfg["base_path"].(string); !ok { + return fmt.Errorf("directory connector requires directory_path") + } + case string: + if cfg == "" { + return fmt.Errorf("directory connector requires non-empty directory_path") + } + default: + return fmt.Errorf("invalid local directory connector config type: %T", config) + } + return nil +} + +func parseConfig(config interface{}) (*entities.DirectoryConnectorConfig, error) { + switch v := config.(type) { + case string: + // Allow passing just a path string + return &entities.DirectoryConnectorConfig{DirectoryPath: v}, nil + case *entities.DirectoryConnectorConfig: + return v, nil + case entities.DirectoryConnectorConfig: + return &v, nil + case map[string]interface{}: + // Accept either "directory_path" or "base_path" + if path, ok := v["directory_path"].(string); ok && strings.TrimSpace(path) != "" { + return &entities.DirectoryConnectorConfig{DirectoryPath: path}, nil + } + if path, ok := v["base_path"].(string); ok && strings.TrimSpace(path) != "" { + return &entities.DirectoryConnectorConfig{DirectoryPath: path}, nil + } + return nil, fmt.Errorf("invalid local directory config: missing or empty 'directory_path'") + default: + return nil, fmt.Errorf("invalid config type for DirectoryConnector: %T", config) + } +} diff --git a/internal/connectors/localfile/connector.go b/internal/connectors/localfile/connector.go new file mode 100644 index 0000000..dd63a67 --- /dev/null +++ b/internal/connectors/localfile/connector.go @@ -0,0 +1,252 @@ +package localfile + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +// LocalFileConnector handles loading a single local file. +type LocalFileConnector struct { + filePath string +} + +// NewLocalFileConnector creates a new local file connector. +func NewLocalFileConnector() connectors.Connector { + return &LocalFileConnector{} +} + +// GetType returns the connector type. +func (c *LocalFileConnector) GetType() connectors.ConnectorType { + return connectors.ConnectorTypeLocalFile +} + +func (c *LocalFileConnector) Ping(ctx context.Context, config interface{}) error { + cfg, err := parseConfig(config) + if err != nil { + return err + } + + fp := cfg.GetFilePath() + if fp == "" { + return fmt.Errorf("local file connection requires file_path") + } + + // Expand ~ to user home dir + if strings.HasPrefix(fp, "~") { + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("cannot resolve ~ in path: %w", err) + } + fp = filepath.Join(home, strings.TrimPrefix(fp, "~")) + } + + // Make absolute if not already + if !filepath.IsAbs(fp) { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("cannot get current working directory: %w", err) + } + fp = filepath.Join(cwd, fp) + } + fp = filepath.Clean(fp) + + // Try resolving symlinks + if resolved, err := filepath.EvalSymlinks(fp); err == nil { + fp = resolved + } + + // Stat the file + info, err := os.Stat(fp) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("file does not exist: %s", fp) + } + return fmt.Errorf("cannot stat file: %s: %w", fp, err) + } + if info.IsDir() { + return fmt.Errorf("expected a file but got a directory: %s", fp) + } + + // Check readability + f, err := os.Open(fp) + if err != nil { + return fmt.Errorf("file not readable: %s: %w", fp, err) + } + _ = f.Close() + + return nil +} + +// Connect prepares the connector with the given config. +func (c *LocalFileConnector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error { + if engineConnectorCtx == nil { + return fmt.Errorf("engine connector context is missing") + } + + if engineConnectorCtx.Config == nil { + return fmt.Errorf("engine connector context options are missing") + } + + cfg, err := parseConfig(engineConnectorCtx.Config) + if err != nil { + return err + } + + fp := cfg.GetFilePath() + if fp == "" { + return fmt.Errorf("local file connection requires file_path") + } + + // Expand ~ to user home dir + if strings.HasPrefix(fp, "~") { + home, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("cannot resolve ~ in path: %w", err) + } + fp = filepath.Join(home, strings.TrimPrefix(fp, "~")) + } + + // Make absolute if not already + if !filepath.IsAbs(fp) { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("cannot get current working directory: %w", err) + } + fp = filepath.Join(cwd, fp) + } + fp = filepath.Clean(fp) + + // Try resolving symlinks + if resolved, err := filepath.EvalSymlinks(fp); err == nil { + fp = resolved + } + + info, err := os.Stat(fp) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("file does not exist: %s", fp) + } + return fmt.Errorf("cannot stat file: %s: %w", fp, err) + } + if info.IsDir() { + return fmt.Errorf("expected a file but got a directory: %s", fp) + } + + // Check file is readable + f, err := os.Open(fp) + if err != nil { + return fmt.Errorf("file not readable: %s: %w", fp, err) + } + _ = f.Close() + + c.filePath = fp + return nil +} + +// GetData returns a reader for the configured file. +func (c *LocalFileConnector) GetData(ctx context.Context, engineCtx *models.EngineContextDataset) (io.ReadCloser, error) { + if c.filePath == "" { + return nil, fmt.Errorf("connection not established") + } + // if strings.TrimSpace(resourcePath) != "" && resourcePath != "." { + // return nil, fmt.Errorf("this connector loads a single configured file; resourcePath must be empty") + // } + + file, err := os.Open(c.filePath) + if err != nil { + if os.IsNotExist(err) { + return nil, &connectors.ResourceNotFoundError{ + ConnectionType: c.GetType(), + ResourcePath: c.filePath, + } + } + return nil, fmt.Errorf("failed to open file: %s: %w", c.filePath, err) + } + + return file, nil +} + +// Close resets the connector state. +func (c *LocalFileConnector) Close() error { + c.filePath = "" + return nil +} + +// Validate performs config syntax checks. +func (c *LocalFileConnector) Validate(config interface{}) error { + switch cfg := config.(type) { + + case *entities.LocalFileConnectorConfig: + if strings.TrimSpace(cfg.FilePath) == "" { + return fmt.Errorf("local file connection requires file_path") + } + + case entities.LocalFileConnectorConfig: + if strings.TrimSpace(cfg.FilePath) == "" { + return fmt.Errorf("local file connection requires file_path") + } + + case map[string]interface{}: + fp, ok := filePathFromMap(cfg) + if !ok || fp == "" { + return fmt.Errorf("local file connection requires file_path") + } + + case string: + if strings.TrimSpace(cfg) == "" { + return fmt.Errorf("local file connection requires non-empty file_path") + } + + default: + return fmt.Errorf("invalid local file connection config type: %T", config) + } + + return nil +} + +func parseConfig(config interface{}) (*entities.LocalFileConnectorConfig, error) { + switch v := config.(type) { + + case string: + return &entities.LocalFileConnectorConfig{ + FilePath: strings.TrimSpace(v), + }, nil + + case *entities.LocalFileConnectorConfig: + return v, nil + + case entities.LocalFileConnectorConfig: + return &v, nil + + case map[string]interface{}: + fp, ok := filePathFromMap(v) + if !ok || fp == "" { + return nil, fmt.Errorf("invalid localfile config: missing or empty file_path") + } + return &entities.LocalFileConnectorConfig{FilePath: fp}, nil + + default: + return nil, fmt.Errorf("invalid config type for LocalFileConnector: %T", config) + } +} + +func filePathFromMap(m map[string]interface{}) (string, bool) { + if v, ok := m["file_path"].(string); ok { + return strings.TrimSpace(v), true + } + if v, ok := m["filePath"].(string); ok { + return strings.TrimSpace(v), true + } + if v, ok := m["filepath"].(string); ok { + return strings.TrimSpace(v), true + } + return "", false +} diff --git a/internal/connectors/main.go b/internal/connectors/main.go new file mode 100644 index 0000000..982e8aa --- /dev/null +++ b/internal/connectors/main.go @@ -0,0 +1,106 @@ +package connectors + +import ( + "context" + "io" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +// ConnectionType represents the type of connection +type ConnectorType string + +const ( + ConnectorTypePostgres ConnectorType = "POSTGRES" + ConnectorTypeSQLite ConnectorType = "SQLITE" + ConnectorTypeLocalFile ConnectorType = "LOCALFILE" + ConnectorTypeLocalDirectory ConnectorType = "LOCAL_DIRECTORY" + ConnectorTypeGoogleAPI ConnectorType = "GOOGLE_API" + ConnectorTypeGit ConnectorType = "GIT" + ConnectorTypeGoogleSheet ConnectorType = "GOOGLESHEET" + ConnectorTypeS3 ConnectorType = "S3" + ConnectorTypeAPI ConnectorType = "API" + ConnectorTypeFederated ConnectorType = "JOIN" +) + +// Connector defines the interface for all connection types +type Connector interface { + // Connect establishes the connection with the given configuration + Connect(ctx context.Context, engineCtx *models.EngineContextConnector) error + + // GetData returns a reader for accessing the data + GetData(ctx context.Context, engineCtx *models.EngineContextDataset) (io.ReadCloser, error) + + // Close closes the connection and cleans up resources + Close() error + + // GetType returns the connection type + GetType() ConnectorType + + // Validate validates the connection configuration + Validate(config interface{}) error + + Ping(ctx context.Context, config interface{}) error +} + +// Registry manages available connections +type Registry interface { + Register(connectionType ConnectorType, connection Connector) error + Get(connectionType ConnectorType) (Connector, error) + List() []ConnectorType +} + +// ConnectorRegistry implements the Registry interface +type ConnectorRegistry struct { + connectors map[ConnectorType]Connector +} + +// NewConnectorRegistry creates a new connection registry +func NewConnectorRegistry() *ConnectorRegistry { + return &ConnectorRegistry{ + connectors: make(map[ConnectorType]Connector), + } +} + +// Register registers a new connection type +func (r *ConnectorRegistry) Register(connectionType ConnectorType, connection Connector) error { + r.connectors[connectionType] = connection + return nil +} + +// Get retrieves a connection by type +func (r *ConnectorRegistry) Get(connectionType ConnectorType) (Connector, error) { + connection, exists := r.connectors[connectionType] + if !exists { + return nil, &ConnectionNotFoundError{ConnectionType: connectionType} + } + return connection, nil +} + +// List returns all registered connection types +func (r *ConnectorRegistry) List() []ConnectorType { + types := make([]ConnectorType, 0, len(r.connectors)) + for t := range r.connectors { + types = append(types, t) + } + return types +} + +// ConnectionNotFoundError represents an error when a connection type is not found +type ConnectionNotFoundError struct { + ConnectionType ConnectorType +} + +func (e *ConnectionNotFoundError) Error() string { + return "connection type not found: " + string(e.ConnectionType) +} + +// ResourceNotFoundError represents an error when a resource is not found in a connection +type ResourceNotFoundError struct { + ConnectionType ConnectorType + ResourcePath string +} + +func (e *ResourceNotFoundError) Error() string { + return "resource not found in " + string(e.ConnectionType) + " connection: " + e.ResourcePath +} diff --git a/internal/connectors/masking.go b/internal/connectors/masking.go new file mode 100644 index 0000000..221b32e --- /dev/null +++ b/internal/connectors/masking.go @@ -0,0 +1,46 @@ +package connectors + +import "reflect" + +func MaskSensitiveFieldsUsingTags(cfg any) any { + if cfg == nil { + return nil + } + + v := reflect.ValueOf(cfg) + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + if v.Kind() != reflect.Struct { + return cfg + } + + t := v.Type() + copyVal := reflect.New(t).Elem() + + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + fieldType := t.Field(i) + + if fieldType.Tag.Get("sensitive") == "true" { + // Only mask if non-empty (prevents changing program logic) + if field.Kind() == reflect.String && field.String() != "" { + copyVal.Field(i).Set(reflect.ValueOf(MaskedValue)) + continue + } + // leave empty values unchanged + copyVal.Field(i).Set(field) + continue + } + + // Recurse for nested structs + if field.Kind() == reflect.Struct && field.CanInterface() { + copyVal.Field(i).Set(reflect.ValueOf(MaskSensitiveFieldsUsingTags(field.Interface()))) + continue + } + + copyVal.Field(i).Set(field) + } + + return copyVal.Interface() +} diff --git a/internal/connectors/mocks/mock_interfaces.go b/internal/connectors/mocks/mock_interfaces.go new file mode 100644 index 0000000..5e594b0 --- /dev/null +++ b/internal/connectors/mocks/mock_interfaces.go @@ -0,0 +1,196 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/connectors/main.go +// +// Generated by this command: +// +// mockgen -source internal/connectors/main.go -destination internal/connectors/mocks/mock_interfaces.go -package mocks +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + io "io" + reflect "reflect" + + connectors "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + gomock "go.uber.org/mock/gomock" +) + +// MockConnector is a mock of Connector interface. +type MockConnector struct { + ctrl *gomock.Controller + recorder *MockConnectorMockRecorder + isgomock struct{} +} + +// MockConnectorMockRecorder is the mock recorder for MockConnector. +type MockConnectorMockRecorder struct { + mock *MockConnector +} + +// NewMockConnector creates a new mock instance. +func NewMockConnector(ctrl *gomock.Controller) *MockConnector { + mock := &MockConnector{ctrl: ctrl} + mock.recorder = &MockConnectorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConnector) EXPECT() *MockConnectorMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockConnector) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockConnectorMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockConnector)(nil).Close)) +} + +// Connect mocks base method. +func (m *MockConnector) Connect(ctx context.Context, engineCtx *models.EngineContextConnector) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Connect", ctx, engineCtx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Connect indicates an expected call of Connect. +func (mr *MockConnectorMockRecorder) Connect(ctx, engineCtx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Connect", reflect.TypeOf((*MockConnector)(nil).Connect), ctx, engineCtx) +} + +// GetData mocks base method. +func (m *MockConnector) GetData(ctx context.Context, engineCtx *models.EngineContextDataset) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetData", ctx, engineCtx) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetData indicates an expected call of GetData. +func (mr *MockConnectorMockRecorder) GetData(ctx, engineCtx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetData", reflect.TypeOf((*MockConnector)(nil).GetData), ctx, engineCtx) +} + +// GetType mocks base method. +func (m *MockConnector) GetType() connectors.ConnectorType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetType") + ret0, _ := ret[0].(connectors.ConnectorType) + return ret0 +} + +// GetType indicates an expected call of GetType. +func (mr *MockConnectorMockRecorder) GetType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetType", reflect.TypeOf((*MockConnector)(nil).GetType)) +} + +// Ping mocks base method. +func (m *MockConnector) Ping(ctx context.Context, config any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Ping", ctx, config) + ret0, _ := ret[0].(error) + return ret0 +} + +// Ping indicates an expected call of Ping. +func (mr *MockConnectorMockRecorder) Ping(ctx, config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Ping", reflect.TypeOf((*MockConnector)(nil).Ping), ctx, config) +} + +// Validate mocks base method. +func (m *MockConnector) Validate(config any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Validate", config) + ret0, _ := ret[0].(error) + return ret0 +} + +// Validate indicates an expected call of Validate. +func (mr *MockConnectorMockRecorder) Validate(config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Validate", reflect.TypeOf((*MockConnector)(nil).Validate), config) +} + +// MockRegistry is a mock of Registry interface. +type MockRegistry struct { + ctrl *gomock.Controller + recorder *MockRegistryMockRecorder + isgomock struct{} +} + +// MockRegistryMockRecorder is the mock recorder for MockRegistry. +type MockRegistryMockRecorder struct { + mock *MockRegistry +} + +// NewMockRegistry creates a new mock instance. +func NewMockRegistry(ctrl *gomock.Controller) *MockRegistry { + mock := &MockRegistry{ctrl: ctrl} + mock.recorder = &MockRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRegistry) EXPECT() *MockRegistryMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockRegistry) Get(connectionType connectors.ConnectorType) (connectors.Connector, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", connectionType) + ret0, _ := ret[0].(connectors.Connector) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockRegistryMockRecorder) Get(connectionType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockRegistry)(nil).Get), connectionType) +} + +// List mocks base method. +func (m *MockRegistry) List() []connectors.ConnectorType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List") + ret0, _ := ret[0].([]connectors.ConnectorType) + return ret0 +} + +// List indicates an expected call of List. +func (mr *MockRegistryMockRecorder) List() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockRegistry)(nil).List)) +} + +// Register mocks base method. +func (m *MockRegistry) Register(connectionType connectors.ConnectorType, connection connectors.Connector) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Register", connectionType, connection) + ret0, _ := ret[0].(error) + return ret0 +} + +// Register indicates an expected call of Register. +func (mr *MockRegistryMockRecorder) Register(connectionType, connection any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Register", reflect.TypeOf((*MockRegistry)(nil).Register), connectionType, connection) +} diff --git a/internal/connectors/postgres/connector.go b/internal/connectors/postgres/connector.go new file mode 100644 index 0000000..61dd820 --- /dev/null +++ b/internal/connectors/postgres/connector.go @@ -0,0 +1,247 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "io" + "strings" + "time" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/encoders" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + + "github.com/kvatch-hub/kvatch-runtime/pkg/utils" + _ "github.com/lib/pq" // PostgreSQL driver +) + +type ( + RowEncoder interface { + EncodeAll(rows encoders.RowScanner, columnOrder []string) (io.ReadCloser, error) + } +) + +// PostgresConnector handles PostgreSQL database connections +type PostgresConnector struct { + db *sql.DB + connString string + isConnected bool + encoder RowEncoder +} + +// NewPostgresConnector creates a new PostgreSQL connection +func NewPostgresConnector() connectors.Connector { + return &PostgresConnector{ + encoder: &encoders.StreamJSONEncoderBuffered{}, + } +} + +// GetType returns the connection type +func (c *PostgresConnector) GetType() connectors.ConnectorType { + return connectors.ConnectorTypePostgres +} + +func (c *PostgresConnector) Ping(ctx context.Context, config interface{}) error { + cfg, err := parseConfig(config) + if err != nil { + return err + } + // Open database connection (lazy, pool created) + db, err := sql.Open("postgres", cfg.DSN()) + if err != nil { + return fmt.Errorf("failed to open PostgreSQL connection: %w", err) + } + defer func() { + _ = db.Close() + }() + + // Ensure we don't hang forever if no timeout was passed in + timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + if err := db.PingContext(timeoutCtx); err != nil { + return fmt.Errorf("failed to ping PostgreSQL database: %w", err) + } + + return nil +} + +// Connect establishes the connection to PostgreSQL +func (c *PostgresConnector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error { + if engineConnectorCtx == nil { + return fmt.Errorf("engine connector context is missing") + } + + if engineConnectorCtx.Config == nil { + return fmt.Errorf("engine connector context options are missing") + } + + cfg, err := parseConfig(engineConnectorCtx.Config) + if err != nil { + return err + } + + // Open database connection + db, err := sql.Open("postgres", cfg.DSN()) + if err != nil { + return fmt.Errorf("failed to open PostgreSQL connection: %v", err) + } + + // Test the connection + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + return fmt.Errorf("failed to ping PostgreSQL database: %v", err) + } + + c.db = db + c.connString = cfg.DSN() + c.isConnected = true + return nil +} + +// GetData executes a query and returns the results as a reader +// For database connections, resourcePath is typically a table name or SQL query +func (c *PostgresConnector) GetData(ctx context.Context, engineCtx *models.EngineContextDataset) (io.ReadCloser, error) { + if !c.isConnected || c.db == nil { + return nil, fmt.Errorf("postgresql connection not established") + } + + if engineCtx == nil { + return nil, fmt.Errorf("dataset plan is missing") + } + + opts, err := entities.DecodeDatasetOptions[entities.SQLDatasetOptions]( + engineCtx.DatasetOptions, + ) + if err != nil { + return nil, fmt.Errorf("invalid sql dataset options: %w", err) + } + + if err := opts.Validate(); err != nil { + return nil, err + } + + renderedQuery := engineCtx.Query + if renderedQuery == "" && (opts.Query == nil || strings.TrimSpace(*opts.Query) == "") { + return nil, fmt.Errorf("sql dataset requires query") + } + + if renderedQuery == "" { + renderedQuery = *opts.Query + } + + query := strings.TrimSpace(renderedQuery) + if !strings.Contains(strings.ToUpper(query), "SELECT") { + query = fmt.Sprintf("SELECT * FROM %s", query) + } + + rows, err := c.db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to execute query: %v", err) + } + + err = EnsureColumnsFromRows(engineCtx, rows) + if err != nil { + return nil, fmt.Errorf("failed to extract column data: %v", err) + } + + return c.encoder.EncodeAll(rows, engineCtx.ColumnOrder) +} + +// Close cleans up the connection +func (c *PostgresConnector) Close() error { + if c.db == nil || !c.isConnected { + return nil + } + + err := c.db.Close() + c.db = nil + c.isConnected = false + + if err != nil { + return fmt.Errorf("failed to close PostgreSQL connection: %w", err) + } + return nil +} + +// Validate validates the PostgreSQL connection configuration +func (c *PostgresConnector) Validate(config interface{}) error { + cfg, err := parseConfig(config) + if err != nil { + return err + } + + return cfg.Validate() +} + +func parseConfig(config interface{}) (*entities.PostgresConnectorConfig, error) { + switch v := config.(type) { + case *entities.PostgresConnectorConfig: + return v, nil + case entities.PostgresConnectorConfig: + return &v, nil + case map[string]interface{}: + // Map parsing is moved to a clean helper function for clarity + return parseConfigFromMap(v) + default: + return nil, fmt.Errorf("invalid PostgreSQL connection config type: %T", v) + } +} + +func parseConfigFromMap(v map[string]interface{}) (*entities.PostgresConnectorConfig, error) { + cfg := &entities.PostgresConnectorConfig{} + + // Use a utility function for safe type assertion/conversion (assumed utility) + cfg.Host = utils.GetString(v, "host") + cfg.Database = utils.GetString(v, "database") + cfg.Username = utils.GetString(v, "username") + cfg.Password = utils.GetString(v, "password") + cfg.ConnectionString = utils.GetString(v, "connectionsttring", "connection_string", "connectionString") + + // Handle port conversion robustly + portVal, ok := v["port"] + if ok { + portInt, err := utils.ToInt(portVal) // Assumes utils.ToInt handles int, int64, float64, string + if err != nil { + return nil, fmt.Errorf("invalid port value: %w", err) + } + cfg.Port = portInt + } + + // Normalize common SSL mode keys + cfg.SSLMode = utils.GetString(v, "ssl_mode", "sslMode", "sslmode") + + return cfg, nil +} + +func EnsureColumnsFromRows(plan *models.EngineContextDataset, rows *sql.Rows) error { + if plan == nil { + return fmt.Errorf("dataset plan is nil") + } + if rows == nil { + return fmt.Errorf("rows input is nil") + } + + columns, err := rows.Columns() + if err != nil { + return fmt.Errorf("failed to get columns: %v", err) + } + + // If ColumnOrder is not set, populate it + if len(plan.ColumnOrder) == 0 { + plan.ColumnOrder = columns + } + + // If Columns is not set, populate based on ColumnOrder + if len(plan.Columns) == 0 { + colMeta := make([]models.EngineContextColumn, 0, len(plan.ColumnOrder)) + for _, col := range plan.ColumnOrder { + colMeta = append(colMeta, models.EngineContextColumn{Name: col}) + } + plan.Columns = colMeta + } + + return nil +} diff --git a/internal/connectors/postgres/connector_benchmark_test.go b/internal/connectors/postgres/connector_benchmark_test.go new file mode 100644 index 0000000..ed9046d --- /dev/null +++ b/internal/connectors/postgres/connector_benchmark_test.go @@ -0,0 +1,124 @@ +package postgres + +import ( + "context" + "io" + "strconv" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/kvatch-hub/kvatch-runtime/internal/encoders" + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +// Helper function to create a variable set of mock rows +func setupMockRows(count int) *sqlmock.Rows { + columns := []string{"id", "data", "value"} + rows := sqlmock.NewRows(columns) + + for i := 1; i <= count; i++ { + // Use strconv.FormatInt to minimize string allocation churn, slightly cleaner than Itoa + rows.AddRow(int64(i), "data_"+strconv.FormatInt(int64(i), 10), int64(i)*10) + } + return rows +} + +// --- CONSOLIDATED AND PARAMETERIZED BENCHMARK --- + +func BenchmarkPostgresConnector_GetData(b *testing.B) { + // 1. Define the parameters for Encoder Type and Data Size + benchmarks := []struct { + name string + encoder encoders.RowEncoder + }{ + { + name: "Blocking_JSONArrayEncoder", + encoder: &encoders.JSONArrayEncoder{}, + }, + { + name: "Streaming_StreamJSONEncoder", + encoder: &encoders.StreamJSONEncoder{}, + }, + { + name: "Streaming_StreamJSONEncoderBuffered", + encoder: &encoders.StreamJSONEncoderBuffered{}, + }, + } + + dataSizes := []struct { + name string + count int + }{ + { + name: "SmallDataset", + count: 2, + }, + { + name: "MediumDataset", + count: 10000, + }, + { + name: "VeryLargeDataset", + count: 500000, + }, + } + + // 2. Loop through all combinations (Data Size x Encoder Type) + for _, size := range dataSizes { + rows := setupMockRows(size.count) // Pre-generate the rows outside the inner loop + + for _, bm := range benchmarks { + testName := size.name + "/" + bm.name + + b.Run(testName, func(b *testing.B) { + db, mock, err := sqlmock.New() + if err != nil { + b.Fatalf("failed to create sqlmock: %v", err) + } + defer func() { + _ = db.Close() + }() + + pg := &PostgresConnector{ + encoder: bm.encoder, + } + pg.SetDBForTest(db) + + b.StopTimer() // Setup outside of the time measurement + + for i := 0; i < b.N; i++ { + // Because setupMockRows is outside b.N loop, + // we must ensure go-sqlmock can handle multiple uses of the same rows definition. + // For safety, we expect the query *inside* the b.N loop + // and use a fresh set of rows for each run if the mock does not support resetting. + + // Note: For large datasets (500k rows), setupMockRows can be slow. + // To include it in the benchmark run time, call it inside the b.N loop + // and start the timer right after. + + rows = setupMockRows(size.count) // Re-generate rows for each iteration to satisfy go-sqlmock + mock.ExpectQuery("SELECT").WillReturnRows(rows) + + b.StartTimer() // START timing the actual data processing + + reader, err := pg.GetData(context.Background(), + &models.EngineContextDataset{ + Query: "SELECT id, data, value FROM data_table", + }) + if err != nil { + b.Fatalf("unexpected error: %v", err) + } + + _, _ = io.Copy(io.Discard, reader) + + b.StopTimer() // STOP timing + + err = mock.ExpectationsWereMet() + if err != nil { + b.Fatalf("mock expectations not met: %v", err) + } + } + }) + } + } +} diff --git a/internal/connectors/postgres/connector_test.go b/internal/connectors/postgres/connector_test.go new file mode 100644 index 0000000..5136e30 --- /dev/null +++ b/internal/connectors/postgres/connector_test.go @@ -0,0 +1,280 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "io" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/encoders" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + + "github.com/stretchr/testify/assert" +) + +func TestPostgresConnector_Connect_Invalid(t *testing.T) { + conn := NewPostgresConnector() + + // err := conn.Connect(context.Background(), "invalid://connstring") + err := conn.Connect(context.Background(), &models.EngineContextConnector{ + Config: entities.PostgresConnectorConfig{}, + }) + assert.Error(t, err) +} + +func TestPostgresConnector_GetData_QueryExecution(t *testing.T) { + db, mock, err := sqlmock.New() + assert.NoError(t, err) + defer func() { + _ = db.Close() + }() + + mock.ExpectQuery("SELECT id, name FROM authors"). + WillReturnRows(sqlmock.NewRows([]string{"id", "name"}). + AddRow(1, "George RR Martin"). + AddRow(2, "Conn Iggulden")) + + pg := &PostgresConnector{ + encoder: &encoders.JSONArrayEncoder{}, + } + // _ = pg.Connect(context.Background(), "dummy") // skip actual DB init + _ = pg.Connect(context.Background(), nil) // skip actual DB init + pg.SetDBForTest(db) // helper to inject mock DB + + reader, err := pg.GetData(context.Background(), + &models.EngineContextDataset{ + Query: "SELECT id, name FROM authors", + DatasetOptions: map[string]interface{}{ + "query": "SELECT id, name FROM authors", + }, + }) + assert.NoError(t, err) + + buf := new(strings.Builder) + _, err = io.Copy(buf, reader) + assert.NoError(t, err) + + assert.Contains(t, buf.String(), "George RR Martin") + assert.Contains(t, buf.String(), "Conn Iggulden") +} + +func TestPostgresConnector_Close(t *testing.T) { + db, mock, err := sqlmock.New() + assert.NoError(t, err) + + // Expect the DB to be closed + mock.ExpectClose() + + pg := &PostgresConnector{ + db: db, + isConnected: true, + } + + err = pg.Close() + assert.NoError(t, err) + + // Verify the Close was called + err = mock.ExpectationsWereMet() + assert.NoError(t, err) +} + +// Add a helper method only in test to inject sqlmock +func (c *PostgresConnector) SetDBForTest(db *sql.DB) { + c.db = db + c.isConnected = true +} + +func TestPostgresConnector_GetData_LargeDataset(t *testing.T) { + db, mock, err := sqlmock.New() + assert.NoError(t, err) + defer func() { + _ = db.Close() + }() + + // Generate 50,000 rows + rowCount := 50000 + rows := sqlmock.NewRows([]string{"id", "name"}) + for i := 0; i < rowCount; i++ { + rows.AddRow(i, fmt.Sprintf("Author %d", i)) + } + + mock.ExpectQuery("SELECT id, name FROM authors_large"). + WillReturnRows(rows) + + pg := &PostgresConnector{ + encoder: &encoders.JSONArrayEncoder{}, + } + _ = pg.Connect(context.Background(), nil) + pg.SetDBForTest(db) + + reader, err := pg.GetData(context.Background(), + &models.EngineContextDataset{ + DatasetOptions: map[string]interface{}{ + "query": "SELECT id, name FROM authors_large", + }, + }) + assert.NoError(t, err) + + n, err := io.Copy(io.Discard, reader) + assert.NoError(t, err) + assert.Greater(t, n, int64(1000000)) +} + +func Test_parseConfigFromMap(t *testing.T) { + tests := []struct { + name string + input map[string]interface{} + wantCfg *entities.PostgresConnectorConfig + wantErr bool + }{ + { + name: "Valid_Full_Config_SnakeCase", + input: map[string]interface{}{ + "host": "test-host", + "port": 5433, + "database": "test-db", + "username": "test-user", + "password": "test-pass", + "ssl_mode": "disable", + }, + wantCfg: &entities.PostgresConnectorConfig{ + Host: "test-host", + Port: 5433, + Database: "test-db", + Username: "test-user", + Password: "test-pass", + SSLMode: "disable", + }, + wantErr: false, + }, + { + name: "Valid_Port_String_and_CamelCase_SSL", + input: map[string]interface{}{ + "host": "test-host", + "port": "5433", // Port as string + "database": "test-db", + "username": "test-user", + "password": "test-pass", + "sslMode": "require", // CamelCase SSL + }, + wantCfg: &entities.PostgresConnectorConfig{ + Host: "test-host", + Port: 5433, + Database: "test-db", + Username: "test-user", + Password: "test-pass", + SSLMode: "require", + }, + wantErr: false, + }, + { + name: "Valid_Port_Float64_and_Lowercase_SSL", + input: map[string]interface{}{ + "host": "test-host", + "port": 5433.0, // Port as float64 + "database": "test-db", + "username": "test-user", + "password": "test-pass", + "sslmode": "verify-full", // Lowercase SSL + }, + wantCfg: &entities.PostgresConnectorConfig{ + Host: "test-host", + Port: 5433, + Database: "test-db", + Username: "test-user", + Password: "test-pass", + SSLMode: "verify-full", + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run("parseConfigFromMap/"+tt.name, func(t *testing.T) { + // NOTE: This test requires a mocked/stubbed version of utils.GetString and utils.ToInt + // or the real utils implementations to be available and correct. + + _, err := parseConfigFromMap(tt.input) + + if (err != nil) != tt.wantErr { + t.Fatalf("parseConfigFromMap() error status mismatch. Got error: %v, Want error: %t", err, tt.wantErr) + } + if tt.wantErr { + return + } + }) + } +} + +func Test_parseConfig(t *testing.T) { + // A fully defined struct for comparison + fullConfig := entities.PostgresConnectorConfig{ + Host: "localhost", + Port: 5432, + Database: "testdb", + Username: "testuser", + Password: "testpassword", + SSLMode: "require", + } + + // A sample DSN string + dsnString := "postgresql://myuser:mypass@127.0.0.1:5432/mydb" + + tests := []struct { + name string + input interface{} + wantErr bool + }{ + { + name: "Input_Ptr_Struct", + input: &fullConfig, + wantErr: false, + }, + { + name: "Input_Value_Struct", + input: fullConfig, + wantErr: false, + }, + { + name: "Input_DSN_String", + input: entities.PostgresConnectorConfig{ + ConnectionString: dsnString, + }, + wantErr: false, + }, + { + name: "Input_Map_Valid", + input: map[string]interface{}{ + "host": "localhost", "port": 5432, "database": "db", "username": "user", "password": "pass", + }, + wantErr: false, + }, + { + name: "Input_Unsupported_Type", + input: 12345, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run("parseConfig/"+tt.name, func(t *testing.T) { + cfg, err := parseConfig(tt.input) + + if (err != nil) != tt.wantErr { + t.Fatalf("parseConfig() error status mismatch. Got error: %v, Want error: %t", err, tt.wantErr) + } + if tt.wantErr { + return + } + + // For other successful cases, we just ensure a non-nil config was returned. + if cfg == nil { + t.Fatalf("parseConfig() returned nil config for valid input") + } + }) + } +} diff --git a/internal/connectors/sqlite/connector.go b/internal/connectors/sqlite/connector.go new file mode 100644 index 0000000..6d39012 --- /dev/null +++ b/internal/connectors/sqlite/connector.go @@ -0,0 +1,277 @@ +package sqlite + +import ( + "context" + "database/sql" + "fmt" + "io" + "log" + "strings" + "sync" + "time" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/encoders" + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +type ( + RowEncoder interface { + EncodeAll(rows encoders.RowScanner, columnOrder []string) (io.ReadCloser, error) + } +) + +// SQLiteConnector handles SQLite database connections +type SQLiteConnector struct { + db *sql.DB + dbPath string + isConnected bool + encoder RowEncoder +} + +// Global connection pool for shared in-memory DBs +var ( + connectionPool = map[string]*sql.DB{} + mu sync.Mutex +) + +// NewSQLiteConnector creates a new SQLite connector +func NewSQLiteConnector() connectors.Connector { + return &SQLiteConnector{ + encoder: &encoders.StreamJSONEncoderBuffered{}, + } +} + +// GetType returns the connector type +func (c *SQLiteConnector) GetType() connectors.ConnectorType { + return connectors.ConnectorTypeSQLite +} + +// Connect establishes the connector to SQLite +func (c *SQLiteConnector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error { + if engineConnectorCtx == nil { + return fmt.Errorf("engine connector context is missing") + } + + if engineConnectorCtx.Config == nil { + return fmt.Errorf("engine connector context options are missing") + } + + // Parse config + cfg, err := parseConfig(engineConnectorCtx.Config) + if err != nil { + return fmt.Errorf("invalid SQLite connector config: %w", err) + } + + dsn := cfg.DSN() + + mu.Lock() + defer mu.Unlock() + + // Check if connection already exists (e.g. for shared in-memory) + if existingDB, ok := connectionPool[dsn]; ok { + c.db = existingDB + c.dbPath = dsn + c.isConnected = true + return nil + } + + // Open new DB connection + db, err := sql.Open("sqlite", dsn) + if err != nil { + return fmt.Errorf("failed to open SQLite connection: %v", err) + } + + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + return fmt.Errorf("failed to ping SQLite database: %v", err) + } + + // Configure pragmatic defaults to reduce SQLITE_BUSY errors when downstream consumers + // hold read locks for longer (e.g. streaming encoders). + if _, err := db.ExecContext(ctx, "PRAGMA busy_timeout = 5000"); err != nil { + log.Printf("warning: failed to set SQLite busy_timeout: %v", err) + } + + // WAL mode allows readers and writers to operate concurrently which reduces lock + // contention when tests rapidly create and query tables. + if _, err := db.ExecContext(ctx, "PRAGMA journal_mode = WAL"); err != nil { + log.Printf("warning: failed to enable SQLite WAL mode: %v", err) + } + + // Store in connection pool + connectionPool[dsn] = db + c.db = db + c.dbPath = dsn + c.isConnected = true + return nil +} + +func (c *SQLiteConnector) Ping(ctx context.Context, config interface{}) error { + cfg, err := parseConfig(config) + if err != nil { + return err + } + + // Open database connection (lazy, pool created) + db, err := sql.Open("sqlite", cfg.DSN()) + if err != nil { + return fmt.Errorf("failed to open sqlite connection: %w", err) + } + defer func() { + _ = db.Close() + }() + + // Apply timeout wrapper to avoid hanging + timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + if err := db.PingContext(timeoutCtx); err != nil { + return fmt.Errorf("failed to ping sqlite database: %w", err) + } + + return nil +} + +// GetData executes a query and returns the results as a reader +func (c *SQLiteConnector) GetData(ctx context.Context, engineCtx *models.EngineContextDataset) (io.ReadCloser, error) { + if !c.isConnected || c.db == nil { + return nil, fmt.Errorf("SQLite connection not established") + } + + if engineCtx == nil { + return nil, fmt.Errorf("dataset plan is missing") + } + + opts, err := entities.DecodeDatasetOptions[entities.SQLDatasetOptions]( + engineCtx.DatasetOptions, + ) + if err != nil { + return nil, fmt.Errorf("invalid sql dataset options: %w", err) + } + + if err := opts.Validate(); err != nil { + return nil, err + } + + renderedQuery := engineCtx.Query + if renderedQuery == "" && (opts.Query == nil || strings.TrimSpace(*opts.Query) == "") { + return nil, fmt.Errorf("sql dataset requires query") + } + + if renderedQuery == "" { + renderedQuery = *opts.Query + } + + query := strings.TrimSpace(renderedQuery) + if !strings.Contains(strings.ToUpper(query), "SELECT") { + query = fmt.Sprintf("SELECT * FROM %s", query) + } + + rows, err := c.db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to execute query: %v", err) + } + + err = EnsureColumnsFromRows(engineCtx, rows) + if err != nil { + return nil, fmt.Errorf("failed to extract column data: %v", err) + } + + return c.encoder.EncodeAll(rows, engineCtx.ColumnOrder) +} + +// Close closes the SQLite connection if it's not from the pool +func (c *SQLiteConnector) Close() error { + mu.Lock() + defer mu.Unlock() + + if c.db != nil { + // Only close and remove if it's not shared + if poolDB, ok := connectionPool[c.dbPath]; ok && poolDB == c.db { + // Keep the shared connection alive + return nil + } + + err := c.db.Close() + c.db = nil + c.isConnected = false + return err + } + return nil +} + +// Validate validates the SQLite connection configuration +func (c *SQLiteConnector) Validate(config interface{}) error { + switch cfg := config.(type) { + case *entities.SQLiteConnectorConfig: + if cfg.Path == "" { + return fmt.Errorf("SQLite connection requires database path") + } + case entities.SQLiteConnectorConfig: + if cfg.Path == "" { + return fmt.Errorf("SQLite connection requires database path") + } + case string: + if cfg == "" { + return fmt.Errorf("SQLite connection requires non-empty database path") + } + case map[string]interface{}: + if _, ok := cfg["path"].(string); !ok { + return fmt.Errorf("SQLite connection requires path") + } + default: + return fmt.Errorf("invalid SQLite connection config type: %T", config) + } + return nil +} + +func parseConfig(config interface{}) (*entities.SQLiteConnectorConfig, error) { + switch v := config.(type) { + case string: + // Allow passing just a path string + return &entities.SQLiteConnectorConfig{Path: v}, nil + case *entities.SQLiteConnectorConfig: + return v, nil + case entities.SQLiteConnectorConfig: + return &v, nil + case map[string]interface{}: + if path, ok := v["path"].(string); ok && strings.TrimSpace(path) != "" { + return &entities.SQLiteConnectorConfig{Path: path}, nil + } + return nil, fmt.Errorf("invalid sqlite config: missing or empty 'path'") + default: + return nil, fmt.Errorf("invalid config type for SQLiteConnector: %T", config) + } +} + +func EnsureColumnsFromRows(plan *models.EngineContextDataset, rows *sql.Rows) error { + if plan == nil { + return fmt.Errorf("dataset plan is nil") + } + if rows == nil { + return fmt.Errorf("rows input is nil") + } + + columns, err := rows.Columns() + if err != nil { + return fmt.Errorf("failed to get columns: %v", err) + } + + // If ColumnOrder is not set, populate it + if len(plan.ColumnOrder) == 0 { + plan.ColumnOrder = columns + } + + // If Columns is not set, populate based on ColumnOrder + if len(plan.Columns) == 0 { + colMeta := make([]models.EngineContextColumn, 0, len(plan.ColumnOrder)) + for _, col := range plan.ColumnOrder { + colMeta = append(colMeta, models.EngineContextColumn{Name: col}) + } + plan.Columns = colMeta + } + + return nil +} diff --git a/internal/connectors/sqlite/connector_benchmark_test.go b/internal/connectors/sqlite/connector_benchmark_test.go new file mode 100644 index 0000000..7aff821 --- /dev/null +++ b/internal/connectors/sqlite/connector_benchmark_test.go @@ -0,0 +1,146 @@ +package sqlite + +import ( + "context" + "database/sql" + "io" + "strconv" + "testing" + + _ "github.com/glebarez/sqlite" + "github.com/kvatch-hub/kvatch-runtime/internal/encoders" + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +// Define the row counts for benchmarking +const ( + smallRowCount = 100 // Using the row count from your original small setup + mediumRowCount = 10000 // Using the row count from your original large setup + largeRowCount = 500000 // Adding a very large dataset test +) + +// Helper function to set up a variable-sized in-memory SQLite database +func setupTestSQLiteDB(tb testing.TB, count int) *sql.DB { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + tb.Fatalf("failed to open test DB: %v", err) + } + + create := `CREATE TABLE test_table (id INTEGER, data TEXT, value INTEGER);` + _, err = db.Exec(create) + if err != nil { + tb.Fatalf("failed to create table: %v", err) + } + + // Insert 'count' rows in a single transaction for speed + tx, err := db.Begin() + if err != nil { + tb.Fatalf("failed to begin transaction: %v", err) + } + stmt, err := tx.Prepare("INSERT INTO test_table (id, data, value) VALUES (?, ?, ?)") + if err != nil { + tb.Fatalf("failed to prepare statement: %v", err) + } + + for i := 1; i <= count; i++ { + _, err := stmt.Exec(i, "data_"+strconv.Itoa(i), i*10) + if err != nil { + tb.Fatalf("failed to insert row %d: %v", i, err) + } + } + _ = stmt.Close() + _ = tx.Commit() + + return db +} + +// --- CONSOLIDATED AND PARAMETERIZED BENCHMARK --- + +func BenchmarkSQLiteConnector_GetData(b *testing.B) { + // 1. Define the parameters for Encoder Type + benchmarks := []struct { + name string + encoder encoders.RowEncoder + }{ + { + name: "Blocking_JSONArrayEncoder", + encoder: &encoders.JSONArrayEncoder{}, + }, + { + name: "Streaming_StreamJSONEncoder", + encoder: &encoders.StreamJSONEncoder{}, + }, + { + name: "Streaming_StreamJSONEncoderBuffered", + encoder: &encoders.StreamJSONEncoderBuffered{}, + }, + } + + // 2. Define the parameters for Data Size + dataSizes := []struct { + name string + count int + }{ + { + name: "SmallDataset", + count: smallRowCount, + }, + { + name: "MediumDataset", // Original "Large" from your previous test + count: mediumRowCount, + }, + { + name: "VeryLargeDataset", // For proper stress testing (e.g., 500k rows) + count: largeRowCount, + }, + } + + // 3. Loop through all combinations (Data Size x Encoder Type) + for _, size := range dataSizes { + db := setupTestSQLiteDB(b, size.count) + defer func() { + _ = db.Close() + }() + + // Define the query based on the created table + var query string + if size.count <= smallRowCount { + query = "SELECT id, data, value FROM test_table LIMIT 100" // Match small setup query semantics + } else { + query = "SELECT id, data, value FROM test_table" + } + + engineCtx := &models.EngineContextDataset{Query: query} + ctx := context.Background() + + for _, bm := range benchmarks { + testName := size.name + "/" + bm.name + + b.Run(testName, func(b *testing.B) { + conn := &SQLiteConnector{ + encoder: bm.encoder, + } + // Pass the existing DB connection to the connector + conn.SetDBForTest(db, ":memory:") + + b.StopTimer() // Setup outside of the time measurement + + for i := 0; i < b.N; i++ { + b.StartTimer() // START timing the actual data processing + + rc, err := conn.GetData(ctx, engineCtx) + if err != nil { + b.Fatalf("GetData failed: %v", err) + } + _, err = io.Copy(io.Discard, rc) + if err != nil { + b.Fatalf("reading result failed: %v", err) + } + _ = rc.Close() + + b.StopTimer() // STOP timing + } + }) + } + } +} diff --git a/internal/connectors/sqlite/connector_test.go b/internal/connectors/sqlite/connector_test.go new file mode 100644 index 0000000..677db5d --- /dev/null +++ b/internal/connectors/sqlite/connector_test.go @@ -0,0 +1,156 @@ +package sqlite + +import ( + "context" + "database/sql" + "encoding/json" + "io" + "testing" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/stretchr/testify/assert" + + _ "github.com/glebarez/sqlite" +) + +func setupTestDB(t *testing.T) *sql.DB { + db, err := sql.Open("sqlite", ":memory:") + assert.NoError(t, err) + + _, err = db.Exec(` + CREATE TABLE test ( + id INTEGER PRIMARY KEY, + name TEXT + ); + INSERT INTO test (name) VALUES ('Alice'), ('Bob'); + `) + assert.NoError(t, err) + + return db +} + +func TestSQLiteConnector_GetData_NonStreaming(t *testing.T) { + db := setupTestDB(t) + defer func() { + _ = db.Close() + }() + + // Save test DB into global connection pool manually + conn := NewSQLiteConnector() + SetTestConnection(conn.(*SQLiteConnector), db, ":memory:") + + engineCtx := &models.EngineContextDataset{ + Query: "SELECT * FROM test", + DatasetOptions: map[string]interface{}{ + "query": "SELECT * FROM test", + }, + } + + reader, err := conn.GetData(context.Background(), engineCtx) + assert.NoError(t, err) + defer func() { + _ = reader.Close() + }() + + data, err := io.ReadAll(reader) + assert.NoError(t, err) + + var results []map[string]interface{} + err = json.Unmarshal(data, &results) + assert.NoError(t, err) + assert.Len(t, results, 2) + assert.Equal(t, "Alice", results[0]["name"]) +} + +// func TestSQLiteConnector_GetData_Streaming(t *testing.T) { +// db := setupTestDB(t) +// defer func() { +// _ = db.Close() +// }() + +// conn := NewSQLiteConnector() +// SetTestConnection(conn, db, ":memory:") + +// plan := &entities.DatasetPlan{ +// Query: "SELECT * FROM test", +// Options: map[string]interface{}{ +// "streaming": true, +// }, +// } + +// reader, err := conn.GetData(context.Background(), plan) +// assert.NoError(t, err) +// defer func() { +// _ = reader.Close() +// }() + +// decoder := json.NewDecoder(reader) +// var results []map[string]interface{} +// err = decoder.Decode(&results) +// assert.NoError(t, err) +// assert.Len(t, results, 2) +// } + +func TestSQLiteConnector_Connect_ValidConfig(t *testing.T) { + conn := NewSQLiteConnector() + // err := conn.Connect(context.Background(), entities.SQLiteConnectorConfig{Path: ":memory:"}) + err := conn.Connect(context.Background(), &models.EngineContextConnector{ + Config: entities.SQLiteConnectorConfig{ + Path: ":memory:", + }, + }) + assert.NoError(t, err) + + sqliteConn := conn.(*SQLiteConnector) + var timeout int + err = sqliteConn.db.QueryRow("PRAGMA busy_timeout;").Scan(&timeout) + assert.NoError(t, err) + assert.Equal(t, 5000, timeout) + + var journalMode string + err = sqliteConn.db.QueryRow("PRAGMA journal_mode;").Scan(&journalMode) + assert.NoError(t, err) + + // In-memory databases fall back to "memory" journal mode; file-backed databases + // should be set to WAL to reduce lock contention when streaming results. + if journalMode == "memory" { + assert.Equal(t, ":memory:", sqliteConn.dbPath) + } else { + assert.Equal(t, "wal", journalMode) + } +} + +func TestSQLiteConnector_Connect_InvalidConfig(t *testing.T) { + conn := NewSQLiteConnector() + err := conn.Connect(context.Background(), &models.EngineContextConnector{ + Config: 12334, + }) // invalid type + assert.Error(t, err) +} + +func TestSQLiteConnector_Validate(t *testing.T) { + conn := NewSQLiteConnector() + + err := conn.Validate(entities.SQLiteConnectorConfig{Path: ":memory:"}) + assert.NoError(t, err) + + err = conn.Validate("missing-path") + assert.NoError(t, err) + + err = conn.Validate(map[string]interface{}{"path": ":memory:"}) + assert.NoError(t, err) + + err = conn.Validate(nil) + assert.Error(t, err) +} + +func SetTestConnection(c *SQLiteConnector, db *sql.DB, path string) { + c.SetDBForTest(db, path) +} + +func (c *SQLiteConnector) SetDBForTest(db *sql.DB, path string) { + c.db = db + c.dbPath = path + c.isConnected = true +} diff --git a/internal/datastore/client.go b/internal/datastore/client.go new file mode 100644 index 0000000..50894b9 --- /dev/null +++ b/internal/datastore/client.go @@ -0,0 +1,49 @@ +package datastore + +import ( + "context" + "database/sql" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "gorm.io/gorm" +) + +type ExecResult struct { + Error error + RowsAffected int64 +} + +type SQLGenerator interface { + GenerateCreateTableSQL(tableName string, schema []models.EngineContextColumn) (string, error) + GenerateInsertSQL(tableName string, data []interface{}, columns []models.EngineContextColumn, dedupe []string) (string, error) +} + +type DataStoreClient interface { + GetConnectionString() string + GetTableName(id string) string + Find(tableName string) ([]map[string]interface{}, error) + GetStorageType() StorageType + Exec(stmt string) *ExecResult + IsNotFoundError(err error) bool + GetGormDB() *gorm.DB + GetSQLGenerator() SQLGenerator + Query(ctx context.Context, tableName string, opts QueryOptions) ([]map[string]interface{}, error) + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) + Close(ctx context.Context) error +} + +type QueryableDataStore interface { + GetConnectionString() string + GetTableName(id string) string + Find(tableName string) ([]map[string]interface{}, error) + Query(ctx context.Context, tableName string, opts QueryOptions) ([]map[string]interface{}, error) +} + +type QueryOptions struct { + Filters map[string]string // e.g. {"asset": "ETH", "country": "DE"} + OrderBy string // e.g. "created_at DESC" + Page int // default 1 + PageSize int // default 50 + AsOf string // optional timestamp or version string + AllowedCols []string // if nil, all columns are allowed (unsafe unless validated separately) +} diff --git a/internal/datastore/enums.go b/internal/datastore/enums.go new file mode 100644 index 0000000..b9a2235 --- /dev/null +++ b/internal/datastore/enums.go @@ -0,0 +1,14 @@ +package datastore + +import "errors" + +type StorageType string + +const ( + StorageTypeSQLITE StorageType = "sqlite" + StorageTypePOSTGRES StorageType = "postgres" +) + +var ( + ErrDBConnection = errors.New("database connection error") +) diff --git a/internal/datastore/factory.go b/internal/datastore/factory.go new file mode 100644 index 0000000..7191235 --- /dev/null +++ b/internal/datastore/factory.go @@ -0,0 +1,93 @@ +package datastore + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/google/uuid" + "github.com/kvatch-hub/kvatch-runtime/entities" +) + +type TenantSettings struct { + TenantUUID uuid.UUID + DatastoreType string + DataStoreOptions entities.DataStoreOptions + EncryptedConfig []byte +} + +type VaultSvc interface { + DecryptConfigBytes(ctx context.Context, tenantUUID uuid.UUID, enc []byte) ([]byte, error) +} + +type DataStoreFactory struct { + vaultSvc VaultSvc +} + +func NewStorageFactory(vaultSvc VaultSvc) *DataStoreFactory { + return &DataStoreFactory{ + vaultSvc, + } +} + +func (f *DataStoreFactory) BuildStorageClient(ctx context.Context, tenantUUID uuid.UUID, settings TenantSettings) (DataStoreClient, error) { + decryptedConfig, err := f.vaultSvc.DecryptConfigBytes(ctx, tenantUUID, settings.EncryptedConfig) + if err != nil { + return nil, fmt.Errorf("failed to decrypt connector config: %v", err) + } + + switch strings.TrimSpace(strings.ToLower(settings.DatastoreType)) { + case "postgres": + var cfg entities.PostgresDataStoreConfig + if err := json.Unmarshal(decryptedConfig, &cfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal sqlite config: %v", err) + } + return NewPostgresDataStoreClient(cfg.DSN(), "") + case "sqlite_local": + var cfg entities.SQLiteLocalDataStoreConfig + if err := json.Unmarshal(decryptedConfig, &cfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal sqlite_local config: %v", err) + } + path := strings.TrimSpace(cfg.Path()) + if path == "" { + return nil, fmt.Errorf("sqlite local file path is required") + } + return NewSQLiteDataStoreClientWithStorage(ctx, NewLocalSQLiteStorage(path)) + case "sqlite_s3": + var cfg entities.SQLiteS3DataStoreConfig + if err := json.Unmarshal(decryptedConfig, &cfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal sqlite_s3 config: %v", err) + } + + fileName := strings.TrimSpace(cfg.FileName) + if fileName == "" { + return nil, fmt.Errorf("sqlite s3 file name is required") + } + + key := strings.TrimPrefix(strings.TrimSpace(cfg.KeyPrefix), "/") + if key != "" { + key = strings.TrimSuffix(key, "/") + "/" + fileName + } else { + key = fileName + } + + storageCfg := SQLiteS3StorageConfig{ + Bucket: strings.TrimSpace(cfg.Bucket), + Key: key, + Region: strings.TrimSpace(cfg.Region), + Endpoint: strings.TrimSpace(cfg.Endpoint), + ForcePathStyle: cfg.ForcePathStyle, + WorkingDir: cfg.WorkingDirectory(), + } + + storage, err := NewSQLiteS3Storage(ctx, storageCfg) + if err != nil { + return nil, fmt.Errorf("failed to prepare sqlite s3 storage: %v", err) + } + + return NewSQLiteDataStoreClientWithStorage(ctx, storage) + default: + return nil, fmt.Errorf("unsupported storage type: %s", settings.DatastoreType) + } +} diff --git a/internal/datastore/postgres.go b/internal/datastore/postgres.go new file mode 100644 index 0000000..2fee249 --- /dev/null +++ b/internal/datastore/postgres.go @@ -0,0 +1,120 @@ +package datastore + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/kvatch-hub/kvatch-runtime/internal/db" + "github.com/pkg/errors" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +type PostgresDataStoreClient struct { + Db *gorm.DB + StorageType StorageType + ConnectionString string + Prefix string +} + +func NewPostgresDataStoreClient(conn, prefix string) (DataStoreClient, error) { + db, err := gorm.Open(postgres.Open(conn)) + if err != nil { + return nil, errors.Wrap(err, ErrDBConnection.Error()) + } + + return &PostgresDataStoreClient{ + Db: db, + StorageType: StorageTypePOSTGRES, + ConnectionString: conn, + Prefix: prefix, + }, nil +} + +func (c *PostgresDataStoreClient) GetTableName(id string) string { + if c.Prefix != "" { + return strings.ToLower(c.Prefix + "_" + id) + } + return id +} + +func (c *PostgresDataStoreClient) GetConnectionString() string { + return c.ConnectionString +} + +func (c *PostgresDataStoreClient) Find(tableName string) ([]map[string]interface{}, error) { + var data []map[string]interface{} + + res := c.Db.Table(tableName).Find(&data) + if res.Error != nil { + return nil, fmt.Errorf("unable to find table: %v", res.Error) + } + + // If no rows, treat as "empty dataset" – not an error, caller decides what to do + if len(data) == 0 { + return nil, fmt.Errorf("no data") + } + + return data, nil +} + +func (c *PostgresDataStoreClient) GetStorageType() StorageType { + return c.StorageType +} + +func (c *PostgresDataStoreClient) Exec(stmt string) *ExecResult { + res := c.Db.Exec(stmt) + return &ExecResult{Error: res.Error, RowsAffected: res.RowsAffected} +} + +func (s *PostgresDataStoreClient) IsNotFoundError(err error) bool { + return strings.Contains(err.Error(), "does not exist (SQLSTATE 42P01)") +} + +func (p *PostgresDataStoreClient) GetGormDB() *gorm.DB { + return p.Db +} + +func (c *PostgresDataStoreClient) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { + return c.Db.WithContext(ctx).Raw(query, args...).Rows() +} + +func (p *PostgresDataStoreClient) GetSQLGenerator() SQLGenerator { + return db.NewSQLGeneratorPostgres() +} + +func (c *PostgresDataStoreClient) Query(ctx context.Context, tableName string, opts QueryOptions) ([]map[string]interface{}, error) { + query := c.Db.Table(tableName) + + // if len(opts.AllowedCols) == 0 { + // return nil, app_errors.Validation("allowed columns not defined", nil) + // } + + query, err := ApplyQueryOptionsWithSafety(query, opts) + if err != nil { + return nil, err + } + + var results []map[string]interface{} + res := query.Find(&results) + if res.Error != nil { + return nil, fmt.Errorf("unable to fnd table: %v", res.Error) + } + + if len(results) == 0 { + return nil, fmt.Errorf("no data") + } + + return results, nil +} + +func (c *PostgresDataStoreClient) Close(ctx context.Context) error { + sqlDB, err := c.Db.DB() + if err != nil { + return err + } + + return sqlDB.Close() +} diff --git a/internal/datastore/query.go b/internal/datastore/query.go new file mode 100644 index 0000000..1389e35 --- /dev/null +++ b/internal/datastore/query.go @@ -0,0 +1,74 @@ +package datastore + +import ( + "fmt" + "strings" + + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +func ApplyQueryOptionsWithSafety(query *gorm.DB, opts QueryOptions) (*gorm.DB, error) { + // If AllowedCols is nil or empty, default to allowing all filter keys + order column + if len(opts.AllowedCols) == 0 { + // NOTE: You could return an error here instead if you prefer strictness. + for k := range opts.Filters { + opts.AllowedCols = append(opts.AllowedCols, k) + } + if orderCol := extractOrderColumn(opts.OrderBy); orderCol != "" { + opts.AllowedCols = append(opts.AllowedCols, orderCol) + } + } + + allowed := make(map[string]struct{}, len(opts.AllowedCols)) + for _, col := range opts.AllowedCols { + allowed[col] = struct{}{} + } + + // Filters: Validate and safely apply + for k, v := range opts.Filters { + if _, ok := allowed[k]; !ok { + return nil, fmt.Errorf("invalid filter column: %q", k) + } + query = query.Where(clause.Eq{Column: clause.Column{Name: k}, Value: v}) + } + + // OrderBy: Validate and safely apply + if opts.OrderBy != "" { + parts := strings.Fields(opts.OrderBy) + col := parts[0] + direction := "ASC" + if len(parts) > 1 && strings.ToUpper(parts[1]) == "DESC" { + direction = "DESC" + } + + if _, ok := allowed[col]; !ok { + return nil, fmt.Errorf("invalid order_by column: %q", col) + } + + query = query.Order(clause.OrderByColumn{ + Column: clause.Column{Name: col}, + Desc: direction == "DESC", + }) + } + + // Pagination + if opts.Page < 1 { + opts.Page = 1 + } + if opts.PageSize <= 0 || opts.PageSize > 1000 { + opts.PageSize = 50 + } + offset := (opts.Page - 1) * opts.PageSize + query = query.Offset(offset).Limit(opts.PageSize) + + return query, nil +} + +func extractOrderColumn(orderBy string) string { + parts := strings.Fields(orderBy) + if len(parts) == 0 { + return "" + } + return parts[0] +} diff --git a/internal/datastore/sqlite.go b/internal/datastore/sqlite.go new file mode 100644 index 0000000..361c0b5 --- /dev/null +++ b/internal/datastore/sqlite.go @@ -0,0 +1,195 @@ +package datastore + +import ( + "context" + "database/sql" + "fmt" + "strings" + "sync" + "time" + + // app_errors "github.com/kvatch-hub/kvatch-runtime/errors" + "github.com/kvatch-hub/kvatch-runtime/internal/db" + "github.com/pkg/errors" + + sqlite "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +type SQLiteStorageProvider interface { + Prepare(ctx context.Context) (SQLiteStorageSession, error) +} + +type SQLiteStorageSession struct { + Path string + Finalize func(context.Context) error + Cleanup func() error +} + +type SqliteDataStoreClient struct { + Db *gorm.DB + StorageType StorageType + ConnectionString string + Prefix string + session SQLiteStorageSession + closeOnce sync.Once + closeErr error +} + +func NewSQLiteDataStoreClient(fullPath string) (DataStoreClient, error) { + return NewSQLiteDataStoreClientWithStorage(context.Background(), NewLocalSQLiteStorage(fullPath)) +} + +func NewSQLiteDataStoreClientWithStorage(ctx context.Context, storage SQLiteStorageProvider) (DataStoreClient, error) { + session, err := storage.Prepare(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to prepare sqlite storage") + } + + // --- FIX: Disable GORM's default transaction wrap --- + db, err := gorm.Open(sqlite.Open(session.Path), &gorm.Config{ + // SkipDefaultTransaction: true, + }) + // ---------------------------------------------------- + if err != nil { + return nil, errors.Wrapf(err, "failed to connect to SQLite at %s", session.Path) + } + + // Optional: Also set max open connections to 1 for explicit write serialization + sqlDB, err := db.DB() + if err == nil { + // Force serialization: Only one connection open at any time + sqlDB.SetMaxOpenConns(1) + + // Crucial: Close idle connections immediately to prevent lingering locks + sqlDB.SetMaxIdleConns(0) + + // Close connections quickly to force rotation + sqlDB.SetConnMaxLifetime(5 * time.Second) + } + + return &SqliteDataStoreClient{ + Db: db, + StorageType: StorageTypeSQLITE, + ConnectionString: session.Path, + session: session, + }, nil +} + +func (c *SqliteDataStoreClient) GetTableName(id string) string { + if c.Prefix != "" { + return strings.ToLower(c.Prefix + "_" + id) + } + + return strings.ToLower(id) +} + +func (c *SqliteDataStoreClient) GetConnectionString() string { + return c.ConnectionString +} + +func (c *SqliteDataStoreClient) Find(tableName string) ([]map[string]interface{}, error) { + var data []map[string]interface{} + + res := c.Db.Table(tableName).Find(&data) + if res.Error != nil { + return nil, fmt.Errorf("unable to fnd table: %v", res.Error) + } + + if len(data) == 0 { + return nil, fmt.Errorf("no data") + } + + return data, nil +} + +func (c *SqliteDataStoreClient) GetStorageType() StorageType { + return c.StorageType +} + +func (c *SqliteDataStoreClient) Exec(stmt string) *ExecResult { + res := c.Db.Exec(stmt) + return &ExecResult{Error: res.Error, RowsAffected: res.RowsAffected} +} + +func (s *SqliteDataStoreClient) IsNotFoundError(err error) bool { + return strings.Contains(err.Error(), "no such table") +} + +func (p *SqliteDataStoreClient) GetGormDB() *gorm.DB { + return p.Db +} + +func (p *SqliteDataStoreClient) GetSQLGenerator() SQLGenerator { + return db.NewSQLGeneratorSqlite() +} + +func (c *SqliteDataStoreClient) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { + return c.Db.WithContext(ctx).Raw(query, args...).Rows() +} + +func (c *SqliteDataStoreClient) Query(ctx context.Context, tableName string, opts QueryOptions) ([]map[string]interface{}, error) { + // Defensive: Validate allowed columns exist + // if len(opts.AllowedCols) == 0 { + // return nil, app_errors.Validation("allowed columns not defined", nil) + // } + + query := c.Db.Table(tableName) + + // Apply filters, ordering, and pagination safely + query, err := ApplyQueryOptionsWithSafety(query, opts) + if err != nil { + return nil, err + } + + var results []map[string]interface{} + res := query.Find(&results) + if res.Error != nil { + return nil, fmt.Errorf("unable to fnd table: %v", res.Error) + } + + if len(results) == 0 { + return nil, fmt.Errorf("unable to fnd table") + } + + return results, nil +} + +func (c *SqliteDataStoreClient) Close(ctx context.Context) error { + c.closeOnce.Do(func() { + var err error + + // Step 1: Close the actual SQL DB to flush all WAL/journal writes. + sqlDB, dbErr := c.Db.DB() + if dbErr == nil { + if cerr := sqlDB.Close(); cerr != nil { + err = cerr + } + } else { + err = dbErr + } + + // Step 2: Finalize session (e.g., upload .db snapshot after it’s flushed) + if c.session.Finalize != nil { + if cerr := c.session.Finalize(ctx); cerr != nil && err == nil { + err = cerr + } + } + + // Step 3: Clean up temporary files, etc. + if c.session.Cleanup != nil { + if cerr := c.session.Cleanup(); cerr != nil && err == nil { + err = cerr + } + } + + // Store the result so Close() is idempotent + c.closeErr = err + }) + + return c.closeErr +} + +func (c *SqliteDataStoreClient) Transaction(fc func(txDB *gorm.DB) error) error { + return c.Db.Transaction(fc) +} diff --git a/internal/datastore/sqlite_storage_local.go b/internal/datastore/sqlite_storage_local.go new file mode 100644 index 0000000..21d9e3d --- /dev/null +++ b/internal/datastore/sqlite_storage_local.go @@ -0,0 +1,28 @@ +package datastore + +import ( + "context" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +type localSQLiteStorage struct { + path string +} + +func NewLocalSQLiteStorage(path string) SQLiteStorageProvider { + return &localSQLiteStorage{path: path} +} + +func (s *localSQLiteStorage) Prepare(ctx context.Context) (SQLiteStorageSession, error) { + dir := filepath.Dir(s.path) + if dir != "." && dir != "" { + if err := os.MkdirAll(dir, 0o755); err != nil { + return SQLiteStorageSession{}, errors.Wrapf(err, "failed to create SQLite directory: %s", dir) + } + } + + return SQLiteStorageSession{Path: s.path}, nil +} diff --git a/internal/datastore/sqlite_storage_s3.go b/internal/datastore/sqlite_storage_s3.go new file mode 100644 index 0000000..4b170a6 --- /dev/null +++ b/internal/datastore/sqlite_storage_s3.go @@ -0,0 +1,173 @@ +package datastore + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/pkg/errors" +) + +type SQLiteS3StorageConfig struct { + Bucket string + Key string + Region string + Endpoint string + ForcePathStyle bool + WorkingDir string +} + +type s3API interface { + GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) +} + +type s3SQLiteStorage struct { + client s3API + bucket string + key string + workRoot string +} + +//nolint:staticcheck // AWS SDK deprecation warnings are safe to ignore here +func NewSQLiteS3Storage(ctx context.Context, cfg SQLiteS3StorageConfig) (SQLiteStorageProvider, error) { + if strings.TrimSpace(cfg.Bucket) == "" { + return nil, errors.New("s3 bucket is required for sqlite storage") + } + if strings.TrimSpace(cfg.Key) == "" { + return nil, errors.New("s3 key is required for sqlite storage") + } + + loadOpts := []func(*config.LoadOptions) error{} + if cfg.Region != "" { + loadOpts = append(loadOpts, config.WithRegion(cfg.Region)) + } + + awsCfg, err := config.LoadDefaultConfig(ctx, loadOpts...) + if err != nil { + return nil, errors.Wrap(err, "failed to load aws config for sqlite s3 storage") + } + + var client *s3.Client + if cfg.Endpoint != "" { + // Define a per-service endpoint resolver + // TODO: Replace aws.Endpoint once s3.EndpointResolverFunc returns a non-deprecated type. + resolver := s3.EndpointResolverFunc(func(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: cfg.Endpoint, + HostnameImmutable: true, + }, nil + }) + + client = s3.NewFromConfig(awsCfg, func(o *s3.Options) { + o.Region = cfg.Region + o.UsePathStyle = cfg.ForcePathStyle + o.EndpointResolver = resolver + }) + } else { + client = s3.NewFromConfig(awsCfg, func(o *s3.Options) { + if cfg.Region != "" { + o.Region = cfg.Region + } + o.UsePathStyle = cfg.ForcePathStyle + }) + } + + return newSQLiteS3StorageWithClient(client, cfg) +} + +func newSQLiteS3StorageWithClient(client s3API, cfg SQLiteS3StorageConfig) (SQLiteStorageProvider, error) { + workDir := cfg.WorkingDir + if strings.TrimSpace(workDir) == "" { + workDir = os.TempDir() + } + + key := strings.TrimPrefix(cfg.Key, "/") + + return &s3SQLiteStorage{ + client: client, + bucket: cfg.Bucket, + key: key, + workRoot: workDir, + }, nil +} + +func (s *s3SQLiteStorage) Prepare(ctx context.Context) (SQLiteStorageSession, error) { + tmpDir, err := os.MkdirTemp(s.workRoot, "sqlite-s3-") + if err != nil { + return SQLiteStorageSession{}, errors.Wrap(err, "failed to create temporary directory for sqlite storage") + } + + localPath := filepath.Join(tmpDir, filepath.Base(s.key)) + + if err := s.download(ctx, localPath); err != nil { + return SQLiteStorageSession{}, err + } + + return SQLiteStorageSession{ + Path: localPath, + Finalize: func(ctx context.Context) error { + return s.upload(ctx, localPath) + }, + Cleanup: func() error { + return os.RemoveAll(tmpDir) + }, + }, nil +} + +func (s *s3SQLiteStorage) download(ctx context.Context, path string) error { + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(s.key)}) + if err != nil { + var nsk *s3types.NoSuchKey + var nf *s3types.NotFound + if errors.As(err, &nsk) || errors.As(err, &nf) { + return nil + } + return errors.Wrap(err, fmt.Sprintf("failed to download sqlite file from s3://%s/%s", s.bucket, s.key)) + } + defer func() { + _ = out.Body.Close() + }() + + file, err := os.Create(path) + if err != nil { + return errors.Wrap(err, "failed to create local sqlite file") + } + defer func() { + _ = file.Close() + }() + + if _, err := io.Copy(file, out.Body); err != nil { + return errors.Wrap(err, "failed to copy sqlite data from s3") + } + + return nil +} + +func (s *s3SQLiteStorage) upload(ctx context.Context, path string) error { + file, err := os.Open(path) + if err != nil { + return errors.Wrap(err, "failed to open sqlite file for upload") + } + defer func() { + _ = file.Close() + }() + + _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(s.key), + Body: file, + }) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to upload sqlite file to s3://%s/%s", s.bucket, s.key)) + } + + return nil +} diff --git a/internal/datastore/sqlite_storage_s3_test.go b/internal/datastore/sqlite_storage_s3_test.go new file mode 100644 index 0000000..0bf730d --- /dev/null +++ b/internal/datastore/sqlite_storage_s3_test.go @@ -0,0 +1,118 @@ +package datastore + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +type stubS3Client struct { + body []byte + getErr error + putErr error +} + +func (s *stubS3Client) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + if s.getErr != nil { + return nil, s.getErr + } + reader := io.NopCloser(bytes.NewReader(s.body)) + return &s3.GetObjectOutput{Body: reader}, nil +} + +func (s *stubS3Client) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + if s.putErr != nil { + return nil, s.putErr + } + buf, err := io.ReadAll(params.Body) + if err != nil { + return nil, err + } + s.body = buf + return &s3.PutObjectOutput{}, nil +} + +func TestS3StoragePrepareCreatesLocalFile(t *testing.T) { + workDir := t.TempDir() + stub := &stubS3Client{getErr: &s3types.NoSuchKey{}} + + provider, err := newSQLiteS3StorageWithClient(stub, SQLiteS3StorageConfig{ + Bucket: "bucket", + Key: "path/db.sqlite", + WorkingDir: workDir, + }) + if err != nil { + t.Fatalf("failed to create storage provider: %v", err) + } + + session, err := provider.Prepare(context.Background()) + if err != nil { + t.Fatalf("prepare failed: %v", err) + } + + if !strings.HasPrefix(session.Path, workDir) { + t.Fatalf("expected local path within work dir, got %s", session.Path) + } + + data := []byte("hello") + if err := os.WriteFile(session.Path, data, 0o600); err != nil { + t.Fatalf("failed to write local sqlite file: %v", err) + } + + if session.Finalize == nil { + t.Fatalf("expected finalize function") + } + if err := session.Finalize(context.Background()); err != nil { + t.Fatalf("finalize failed: %v", err) + } + + if !bytes.Equal(stub.body, data) { + t.Fatalf("expected uploaded data to match written data") + } + + if session.Cleanup == nil { + t.Fatalf("expected cleanup function") + } + if err := session.Cleanup(); err != nil { + t.Fatalf("cleanup failed: %v", err) + } + + if _, err := os.Stat(filepath.Dir(session.Path)); !os.IsNotExist(err) { + t.Fatalf("expected temp dir to be removed") + } +} + +func TestS3StorageDownloadsExistingFile(t *testing.T) { + workDir := t.TempDir() + stub := &stubS3Client{body: []byte("seed")} + + provider, err := newSQLiteS3StorageWithClient(stub, SQLiteS3StorageConfig{ + Bucket: "bucket", + Key: "path/db.sqlite", + WorkingDir: workDir, + }) + if err != nil { + t.Fatalf("failed to create storage provider: %v", err) + } + + session, err := provider.Prepare(context.Background()) + if err != nil { + t.Fatalf("prepare failed: %v", err) + } + + contents, err := os.ReadFile(session.Path) + if err != nil { + t.Fatalf("failed to read local sqlite file: %v", err) + } + + if !bytes.Equal(contents, []byte("seed")) { + t.Fatalf("expected downloaded contents, got %q", contents) + } +} diff --git a/internal/db/helpers.go b/internal/db/helpers.go new file mode 100644 index 0000000..78a45f4 --- /dev/null +++ b/internal/db/helpers.go @@ -0,0 +1,55 @@ +package db + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +func mapColumnNames(cols []models.EngineContextColumn) []string { + names := make([]string, len(cols)) + for i, c := range cols { + names[i] = c.Name + } + return names +} + +// func buildUpdateAssignments(cols []string) []string { +// assignments := make([]string, 0, len(cols)+4) +// for _, col := range cols { +// assignments = append(assignments, fmt.Sprintf("%s = EXCLUDED.%s", col, col)) +// } +// assignments = append(assignments, +// "_checksum = EXCLUDED._checksum", +// "_ingested_at = EXCLUDED._ingested_at", +// "_source = EXCLUDED._source", +// "_batch_id = EXCLUDED._batch_id", +// ) +// return assignments +// } + +func normalizeForHash(v interface{}) string { + switch val := v.(type) { + case nil: + return "" + case string: + return val + case bool: + if val { + return "true" + } + return "false" + case int, int32, int64, float32, float64: + return fmt.Sprintf("%v", val) + default: + return fmt.Sprintf("%v", val) + } +} + +// Compute SHA256 hex +func sha256Hex(s string) string { + h := sha256.Sum256([]byte(s)) + return hex.EncodeToString(h[:]) +} diff --git a/internal/db/postgres.go b/internal/db/postgres.go new file mode 100644 index 0000000..7f88f91 --- /dev/null +++ b/internal/db/postgres.go @@ -0,0 +1,97 @@ +package db + +import ( + "fmt" + "strings" + "time" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +type SqlGeneratorPostgres struct { + timeFunc func() time.Time +} + +func NewSQLGeneratorPostgres() *SqlGeneratorPostgres { + return &SqlGeneratorPostgres{ + timeFunc: time.Now, + } +} + +// func newSQLGeneratorPostgres(timeFunc func() time.Time) *SqlGeneratorPostgres { +// return &SqlGeneratorPostgres{timeFunc: timeFunc} +// } + +func (s *SqlGeneratorPostgres) GenerateCreateTableSQL(tableName string, columns []models.EngineContextColumn) (string, error) { + return generateCreateTableSQL(PostgresDialect, tableName, columns) +} + +func (s *SqlGeneratorPostgres) GenerateInsertSQL( + tableName string, + data []interface{}, + columns []models.EngineContextColumn, + dedupe []string, +) (string, error) { + opts := InsertDialectOptions{ + // Quote identifiers with double quotes, escape embedded quotes + QuoteIdent: func(x string) string { return `"` + strings.ReplaceAll(x, `"`, `""`) + `"` }, + + // TRUE/FALSE for Postgres + QuoteBool: func(b bool) string { + if b { + return "TRUE" + } + return "FALSE" + }, + + // Use a raw SQL expression (not a quoted string) for timestamps + // For deterministic tests, you can inject a literal like: "'2025-09-02T19:09:13Z'" + TimestampSQL: "NOW()", + + Source: "federated_plugin", + BatchID: fmt.Sprintf("batch_%d", s.timeFunc().UnixNano()), + + // Called by generateInsertSQL with QUOTED table name and QUOTED business columns + UpsertClause: func(quotedTable string, businessQuoted []string) string { + sets := make([]string, 0, len(businessQuoted)+4) + for _, c := range businessQuoted { + sets = append(sets, fmt.Sprintf("%s=EXCLUDED.%s", c, c)) + } + sets = append(sets, + `"_checksum"=EXCLUDED."_checksum"`, + `"_ingested_at"=EXCLUDED."_ingested_at"`, + `"_source"=EXCLUDED."_source"`, + `"_batch_id"=EXCLUDED."_batch_id"`, + ) + + // Only update if checksum changed (avoids pointless writes) + return " ON CONFLICT(" + `"_dedupe"` + ") DO UPDATE SET " + + strings.Join(sets, ", ") + + " WHERE " + quotedTable + `."_checksum" IS DISTINCT FROM EXCLUDED."_checksum"` + }, + } + + return generateInsertSQL(tableName, data, columns, dedupe, opts) +} + +var PostgresDialect = CreateDialect{ + Name: "postgres", + QuoteIdent: func(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` }, + NormalizeType: func(t string) string { + u := strings.ToUpper(strings.TrimSpace(t)) + if u == "" { + return "TEXT" + } + switch u { + case "REAL", "FLOAT4": + return "REAL" + case "DOUBLE", "FLOAT8": + return "DOUBLE PRECISION" + case "INT": + return "INTEGER" + default: + return u + } + }, + IngestedAtSQLType: "TIMESTAMPTZ", +} diff --git a/internal/db/shared.go b/internal/db/shared.go new file mode 100644 index 0000000..539b3a0 --- /dev/null +++ b/internal/db/shared.go @@ -0,0 +1,217 @@ +package db + +import ( + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +func castToRows(data []interface{}) ([]map[string]interface{}, error) { + rows := make([]map[string]interface{}, 0, len(data)) + for i, item := range data { + row, ok := item.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("row %d: expected map[string]interface{}, got %T", i, item) + } + rows = append(rows, row) + } + return rows, nil +} + +type InsertDialectOptions struct { + QuoteIdent func(string) string // e.g., s => `"`+escaped+`"` + QuoteBool func(bool) string // e.g., "TRUE"/"FALSE" (PG) or "1"/"0" (SQLite) + TimestampSQL string // e.g., "CURRENT_TIMESTAMP" (SQLite) or "NOW()" (PG) or "'2025-09-02T…Z'" + Source string + BatchID string + UpsertClause func(quotedTable string, quotedBusinessCols []string) string +} + +type CreateDialect struct { + Name string + QuoteIdent func(string) string + NormalizeType func(string) string + + // System types/columns + IngestedAtSQLType string // e.g. "TIMESTAMPTZ" (PG) or "TEXT" (SQLite) +} + +func generateInsertSQL( + tableName string, + data []interface{}, + columns []models.EngineContextColumn, + dedupe []string, + opts InsertDialectOptions, +) (string, error) { + if data == nil { + return "", fmt.Errorf("data is nil") + } + if len(data) == 0 { + return "", fmt.Errorf("no data to insert") + } + if opts.QuoteIdent == nil { + return "", fmt.Errorf("QuoteIdent is required") + } + if opts.QuoteBool == nil { + return "", fmt.Errorf("QuoteBool is required") + } + if strings.TrimSpace(tableName) == "" { + return "", fmt.Errorf("table name cannot be empty") + } + + mappedColumnNames := mapColumnNames(columns) // your existing helper (business cols, unquoted) + + // If no dedupe keys defined, use all business columns + if len(dedupe) == 0 { + dedupe = append([]string{}, mappedColumnNames...) + } + sort.Strings(dedupe) + + businessQuoted := make([]string, 0, len(mappedColumnNames)) + for _, c := range mappedColumnNames { + businessQuoted = append(businessQuoted, opts.QuoteIdent(c)) + } + allColsQuoted := append([]string{opts.QuoteIdent("_dedupe")}, businessQuoted...) + allColsQuoted = append(allColsQuoted, + opts.QuoteIdent("_checksum"), + opts.QuoteIdent("_ingested_at"), + opts.QuoteIdent("_source"), + opts.QuoteIdent("_batch_id"), + ) + + rows, err := castToRows(data) + if err != nil { + return "", err + } + + // Build VALUES tuples + valueRows := make([]string, 0, len(rows)) + for _, row := range rows { + // dedupe + dedupeParts := make([]string, 0, len(dedupe)) + for _, key := range dedupe { + dedupeParts = append(dedupeParts, normalizeForHash(row[key])) + } + dedupeKey := sha256Hex(strings.Join(dedupeParts, "|")) + + // checksum + checksumParts := make([]string, 0, len(mappedColumnNames)) + for _, key := range mappedColumnNames { + checksumParts = append(checksumParts, normalizeForHash(row[key])) + } + checksum := sha256Hex(strings.Join(checksumParts, "|")) + + // business values + values := []string{quoteStringLiteral(dedupeKey)} + for _, key := range mappedColumnNames { + if val, ok := row[key]; ok && val != nil { + switch v := val.(type) { + case string: + values = append(values, quoteStringLiteral(v)) + case float32, float64, int, int32, int64, uint, uint32, uint64: + values = append(values, fmt.Sprintf("%v", v)) + case bool: + values = append(values, opts.QuoteBool(v)) // no quotes + case time.Time: + // store as RFC3339 string by default + values = append(values, quoteStringLiteral(v.UTC().Format(time.RFC3339Nano))) + default: + // fallback to string + values = append(values, quoteStringLiteral(fmt.Sprintf("%v", v))) + } + } else { + values = append(values, "NULL") + } + } + + // system columns (checksum, timestamp expr, source, batch) + values = append(values, + quoteStringLiteral(checksum), + rawOrQuoted(opts.TimestampSQL), + quoteStringLiteral(opts.Source), + quoteStringLiteral(opts.BatchID), + ) + + valueRows = append(valueRows, fmt.Sprintf("(%s)", strings.Join(values, ", "))) + } + + // INSERT ... VALUES ... + quotedTable := opts.QuoteIdent(tableName) + insert := fmt.Sprintf( + "INSERT INTO %s (%s) VALUES %s", + quotedTable, + strings.Join(allColsQuoted, ", "), + strings.Join(valueRows, ", "), + ) + + if opts.UpsertClause != nil { + insert += opts.UpsertClause(quotedTable, businessQuoted) + } + + return insert, nil +} + +func generateCreateTableSQL( + d CreateDialect, + tableName string, + columns []models.EngineContextColumn, +) (string, error) { + if len(columns) == 0 { + return "", errors.New("generated table has no columns") + } + if strings.TrimSpace(tableName) == "" { + return "", errors.New("table name cannot be empty") + } + + defs := make([]string, 0, len(columns)+5) + for _, c := range columns { + name := strings.TrimSpace(c.Name) + if name == "" { + return "", fmt.Errorf("empty column name detected") + } + + // Skip any user-provided columns starting with "_" + if strings.HasPrefix(name, "_") { + continue + } + + defs = append(defs, fmt.Sprintf("%s %s", d.QuoteIdent(name), d.NormalizeType(c.Type))) + } + + defs = append(defs, + fmt.Sprintf("%s TEXT PRIMARY KEY", d.QuoteIdent("_dedupe")), + fmt.Sprintf("%s TEXT", d.QuoteIdent("_checksum")), + fmt.Sprintf("%s %s", d.QuoteIdent("_ingested_at"), d.IngestedAtSQLType), + fmt.Sprintf("%s TEXT", d.QuoteIdent("_source")), + fmt.Sprintf("%s TEXT", d.QuoteIdent("_batch_id")), + ) + + return fmt.Sprintf( + "CREATE TABLE IF NOT EXISTS %s (%s)", + d.QuoteIdent(tableName), + strings.Join(defs, ", "), + ), nil +} + +func quoteStringLiteral(s string) string { + // single-quote + escape internal single-quotes + return "'" + strings.ReplaceAll(s, "'", "''") + "'" +} + +func rawOrQuoted(ts string) string { + if ts == "" { + // default to a literal ISO timestamp if none provided + return quoteStringLiteral(time.Now().UTC().Format(time.RFC3339Nano)) + } + // Heuristic: if it's already quoted like '2025-09-02...', just return + trimmed := strings.TrimSpace(ts) + if strings.HasPrefix(trimmed, "'") && strings.HasSuffix(trimmed, "'") { + return trimmed + } + // Otherwise treat as RAW SQL (e.g., CURRENT_TIMESTAMP, NOW()) + return trimmed +} diff --git a/internal/db/sqlite.go b/internal/db/sqlite.go new file mode 100644 index 0000000..9a3e2ad --- /dev/null +++ b/internal/db/sqlite.go @@ -0,0 +1,100 @@ +package db + +import ( + "fmt" + "strings" + "time" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +type SqlGeneratorSqlite struct { + timeFunc func() time.Time +} + +func NewSQLGeneratorSqlite() *SqlGeneratorSqlite { + return &SqlGeneratorSqlite{ + timeFunc: time.Now, + } +} + +// func newSQLGeneratorSqlite(timeFunc func() time.Time) *SqlGeneratorSqlite { +// if timeFunc == nil { +// timeFunc = time.Now +// } +// return &SqlGeneratorSqlite{timeFunc: timeFunc} +// } + +func (s *SqlGeneratorSqlite) GenerateCreateTableSQL(tableName string, columns []models.EngineContextColumn) (string, error) { + return generateCreateTableSQL(SQLiteDialect, tableName, columns) +} + +func (s *SqlGeneratorSqlite) GenerateInsertSQL( + tableName string, + data []interface{}, + columns []models.EngineContextColumn, + dedupe []string, +) (string, error) { + opts := InsertDialectOptions{ + // Double-quote identifiers (escape embedded quotes) + QuoteIdent: func(x string) string { return `"` + strings.ReplaceAll(x, `"`, `""`) + `"` }, + + // SQLite booleans as 1/0 + QuoteBool: func(b bool) string { + if b { + return "1" + } + return "0" + }, + + // Raw SQL expression (not quoted) + TimestampSQL: "CURRENT_TIMESTAMP", + + Source: "federated_plugin", + BatchID: fmt.Sprintf("batch_%d", s.timeFunc().UnixNano()), + + // Use proper UPSERT rather than INSERT OR REPLACE (requires SQLite ≥ 3.24.0) + UpsertClause: func(quotedTable string, businessQuoted []string) string { + sets := make([]string, 0, len(businessQuoted)+4) + for _, c := range businessQuoted { + sets = append(sets, fmt.Sprintf("%s=excluded.%s", c, c)) + } + sets = append(sets, + `"_checksum"=excluded."_checksum"`, + `"_ingested_at"=excluded."_ingested_at"`, + `"_source"=excluded."_source"`, + `"_batch_id"=excluded."_batch_id"`, + ) + // If you only want to update when data changed, add a WHERE with checksum compare. + // SQLite supports IS DISTINCT FROM since 3.37 via newer expression; commonly emulate with <> OR IS NULL checks. + return " ON CONFLICT(" + `"_dedupe"` + ") DO UPDATE SET " + strings.Join(sets, ", ") + }, + } + + return generateInsertSQL(tableName, data, columns, dedupe, opts) +} + +var SQLiteDialect = CreateDialect{ + Name: "sqlite", + QuoteIdent: func(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` }, + NormalizeType: func(t string) string { + u := strings.ToUpper(strings.TrimSpace(t)) + if u == "" { + return "TEXT" + } + switch u { + case "DOUBLE", "DOUBLE PRECISION", "FLOAT", "REAL": + return "REAL" + case "INT", "INTEGER", "BIGINT", "SMALLINT": + return "INTEGER" + case "BOOL", "BOOLEAN": + return "NUMERIC" // 0/1 + case "DECIMAL", "NUMERIC": + return "NUMERIC" + default: + return u + } + }, + // Store ISO8601 strings (or use NUMERIC if you prefer epoch) + IngestedAtSQLType: "TEXT", +} diff --git a/internal/encoders/encoders.go b/internal/encoders/encoders.go new file mode 100644 index 0000000..1ed393b --- /dev/null +++ b/internal/encoders/encoders.go @@ -0,0 +1,22 @@ +package encoders + +import ( + "io" +) + +type ( + RowScanner interface { + Next() bool + Scan(...interface{}) error + Columns() ([]string, error) + Close() error + Err() error + } + RowEncoder interface { + EncodeAll(rows RowScanner, columnOrder []string) (io.ReadCloser, error) + } +) + +func DefaultEncoder() RowEncoder { + return &StreamJSONEncoderBuffered{} +} diff --git a/internal/encoders/json_decoder.go b/internal/encoders/json_decoder.go new file mode 100644 index 0000000..948c846 --- /dev/null +++ b/internal/encoders/json_decoder.go @@ -0,0 +1,40 @@ +package encoders + +import ( + "encoding/json" + "fmt" + "io" +) + +func DecodeJSONArray(r io.Reader) ([]map[string]interface{}, error) { + dec := json.NewDecoder(r) + + // Read the opening bracket of the array + tok, err := dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read opening token: %w", err) + } + if delim, ok := tok.(json.Delim); !ok || delim != '[' { + return nil, fmt.Errorf("expected opening '[' for JSON array") + } + + var results []map[string]interface{} + for dec.More() { + var obj map[string]interface{} + if err := dec.Decode(&obj); err != nil { + return nil, fmt.Errorf("failed to decode array element: %w", err) + } + results = append(results, obj) + } + + // Read the closing bracket of the array + tok, err = dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read closing token: %w", err) + } + if delim, ok := tok.(json.Delim); !ok || delim != ']' { + return nil, fmt.Errorf("expected closing ']' for JSON array") + } + + return results, nil +} diff --git a/internal/encoders/json_encoder.go b/internal/encoders/json_encoder.go new file mode 100644 index 0000000..60f9170 --- /dev/null +++ b/internal/encoders/json_encoder.go @@ -0,0 +1,85 @@ +package encoders + +import ( + "encoding/json" + "fmt" + "io" + "strings" +) + +type JSONArrayEncoder struct{} + +func (e *JSONArrayEncoder) EncodeAll(rows RowScanner, columnOrder []string) (io.ReadCloser, error) { + if rows == nil { + return nil, fmt.Errorf("nil rows received") + } + defer func() { + _ = rows.Close() + }() + + var ( + columns []string + colIndex map[string]int + result []map[string]interface{} + first = true + ) + + for rows.Next() { + + // Load metadata lazily (required to let sqlmock RowError work) + if first { + var err error + columns, err = rows.Columns() + if err != nil { + return nil, err + } + + colIndex = make(map[string]int) + for i, col := range columns { + colIndex[col] = i + } + + result = make([]map[string]interface{}, 0) + first = false + } + + // Prepare scan targets + values := make([]interface{}, len(columns)) + ptrs := make([]interface{}, len(columns)) + for i := range values { + ptrs[i] = &values[i] + } + + // This is where sqlmock RowError triggers + if err := rows.Scan(ptrs...); err != nil { + return nil, err + } + + row := make(map[string]interface{}) + for _, col := range columnOrder { + idx, ok := colIndex[col] + if !ok { + continue + } + if b, ok := values[idx].([]byte); ok { + row[col] = string(b) + } else { + row[col] = values[idx] + } + } + + result = append(result, row) + } + + // If no rows, return empty JSON array + if len(result) == 0 { + return io.NopCloser(strings.NewReader("[]")), nil + } + + jsonBytes, err := json.Marshal(result) + if err != nil { + return nil, err + } + + return io.NopCloser(strings.NewReader(string(jsonBytes))), nil +} diff --git a/internal/encoders/json_encoder_test.go b/internal/encoders/json_encoder_test.go new file mode 100644 index 0000000..a78ef30 --- /dev/null +++ b/internal/encoders/json_encoder_test.go @@ -0,0 +1,209 @@ +package encoders + +import ( + "database/sql" + "encoding/json" + "fmt" + "io" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJSONArrayEncoder_EncodeAll_WithColumnOrder(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + rows := sqlmock.NewRows([]string{"id", "name", "email"}). + AddRow(1, "Alice", "alice@example.com"). + AddRow(2, "Bob", "bob@example.com") + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + + sqlRows, _ := db.Query("SELECT id, name, email FROM users") + + enc := &JSONArrayEncoder{} + rc, err := enc.EncodeAll(sqlRows, []string{"name", "email", "id"}) + assert.NoError(t, err) + + data, _ := io.ReadAll(rc) + _ = rc.Close() + + var out []map[string]any + err = json.Unmarshal(data, &out) + assert.NoError(t, err) + + assert.Equal(t, "Alice", out[0]["name"]) + assert.Equal(t, "alice@example.com", out[0]["email"]) + assert.Equal(t, float64(1), out[0]["id"]) +} + +func TestJSONArrayEncoder_EncodeAll_MissingColumns(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + rows := sqlmock.NewRows([]string{"id", "name"}). + AddRow(10, "Test") + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + + sqlRows, _ := db.Query("SELECT id, name FROM test") + + enc := &JSONArrayEncoder{} + rc, err := enc.EncodeAll(sqlRows, []string{"name", "nonexistent", "id"}) + assert.NoError(t, err) + + data, _ := io.ReadAll(rc) + _ = rc.Close() + + var out []map[string]any + _ = json.Unmarshal(data, &out) + + assert.Len(t, out, 1) + assert.Contains(t, out[0], "name") + assert.Contains(t, out[0], "id") + assert.NotContains(t, out[0], "nonexistent") +} + +func TestJSONArrayEncoder_EncodeAll_EmptyRows(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + rows := sqlmock.NewRows([]string{"id", "name"}) // no rows added + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + + sqlRows, _ := db.Query("SELECT id, name FROM empty") + + enc := &JSONArrayEncoder{} + rc, err := enc.EncodeAll(sqlRows, []string{"id", "name"}) + assert.NoError(t, err) + + data, _ := io.ReadAll(rc) + func() { + _ = rc.Close() + }() + + assert.Equal(t, "[]", strings.TrimSpace(string(data))) +} + +func TestJSONArrayEncoder_EncodeAll_ByteSlices(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + rows := sqlmock.NewRows([]string{"data"}). + AddRow([]byte("hello")) + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + + sqlRows, _ := db.Query("SELECT data FROM table") + + enc := &JSONArrayEncoder{} + rc, err := enc.EncodeAll(sqlRows, []string{"data"}) + assert.NoError(t, err) + + data, _ := io.ReadAll(rc) + _ = rc.Close() + + var out []map[string]any + _ = json.Unmarshal(data, &out) + + assert.Equal(t, "hello", out[0]["data"]) +} + +func TestJSONArrayEncoder_EncodeAll_Nulls(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + rows := sqlmock.NewRows([]string{"id", "value"}). + AddRow(1, nil) + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + + sqlRows, _ := db.Query("SELECT id, value FROM t") + + enc := &JSONArrayEncoder{} + rc, err := enc.EncodeAll(sqlRows, []string{"id", "value"}) + assert.NoError(t, err) + + data, _ := io.ReadAll(rc) + defer func() { + _ = db.Close() + }() + + var out []map[string]any + _ = json.Unmarshal(data, &out) + + assert.Nil(t, out[0]["value"]) +} + +func TestJSONArrayEncoder_EncodeAll_NoColumnOrderUsesOriginalOrder(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + rows := sqlmock.NewRows([]string{"a", "b", "c"}). + AddRow("A", "B", "C") + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + + sqlRows, _ := db.Query("SELECT a, b, c FROM t") + + enc := &JSONArrayEncoder{} + rc, err := enc.EncodeAll(sqlRows, []string{"a", "b", "c"}) + assert.NoError(t, err) + + data, _ := io.ReadAll(rc) + _ = rc.Close() + + var out []map[string]any + _ = json.Unmarshal(data, &out) + + assert.Equal(t, []any{"A", "B", "C"}, []any{out[0]["a"], out[0]["b"], out[0]["c"]}) +} + +type failingRows struct { + *sql.Rows +} + +func (f *failingRows) Scan(dest ...interface{}) error { + return fmt.Errorf("forced scan failure") +} +func TestJSONArrayEncoder_EncodeAll_ScanError(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + // A real row (we will override Scan) + rows := sqlmock.NewRows([]string{"id"}). + AddRow(1) + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + + sqlRows, _ := db.Query("SELECT id FROM faulty") + + // Wrap to force Scan error + wrapped := &failingRows{Rows: sqlRows} + + enc := &JSONArrayEncoder{} + rc, err := enc.EncodeAll(wrapped, []string{"id"}) + + require.Error(t, err) + assert.Contains(t, err.Error(), "forced scan failure") + assert.Nil(t, rc) +} diff --git a/internal/encoders/stream_json_buffered_encoder.go b/internal/encoders/stream_json_buffered_encoder.go new file mode 100644 index 0000000..e38988b --- /dev/null +++ b/internal/encoders/stream_json_buffered_encoder.go @@ -0,0 +1,162 @@ +package encoders + +import ( + "bufio" + "encoding/json" + "fmt" + "io" +) + +type newlineStripWriter struct { + w io.Writer +} + +func (n *newlineStripWriter) Write(p []byte) (int, error) { + // Check if the last byte is a newline + if len(p) > 0 && p[len(p)-1] == '\n' { + // Write all but the last byte to the underlying writer + // This is where the magic happens: we strip the \n + if len(p) > 1 { + if _, err := n.w.Write(p[:len(p)-1]); err != nil { + return 0, err + } + } + // Report that the full slice (including the stripped newline) was processed successfully + return len(p), nil + } + + // No newline at the end, write the whole slice + return n.w.Write(p) +} + +type StreamJSONEncoderBuffered struct{} + +func (e *StreamJSONEncoderBuffered) EncodeAll(rows RowScanner, columnOrder []string) (io.ReadCloser, error) { + if rows == nil { + return nil, fmt.Errorf("nil rows received") + } + + // Standard Pipe setup + pr, pw := io.Pipe() + + // The entire encoding process runs in a separate goroutine + go func() { + // CRITICAL 1: Wrap the PipeWriter with a BufferedWriter + // Using a default buffer size (like 4KB or 8KB) is often sufficient. + // We use bufio.NewWriterSize for control. os.Getpagesize() is a good starting point. + bw := bufio.NewWriterSize(pw, 8192) // 8KB buffer (adjust as needed) + jsonEnc := json.NewEncoder(&newlineStripWriter{w: bw}) + + // --- 1. Control Flags --- + var arrayStarted bool + + // --- 2. Deferred Cleanup --- + defer func() { + // Flush the buffer before closing. If this fails, the error will be passed to the pipe. + if err := bw.Flush(); err != nil { + // If flushing fails, close the pipe with that error. + pw.CloseWithError(fmt.Errorf("buffered writer flush failed: %w", err)) + } + + // CRITICAL: Ensure rows are closed. + _ = rows.Close() + + // Handle closing the pipe writer (pw) based on the iteration outcome. + if err := rows.Err(); err != nil { + // Close with error if rows encountered an issue + pw.CloseWithError(fmt.Errorf("rows iteration failed: %w", err)) + } else { + // Successfully finished iteration. + if !arrayStarted { + // Case 1: No rows were processed. Write "[]" directly to the underlying pipe. + // Note: We MUST call Flush() before this if we wrote anything to bw, but here we know we didn't. + if _, err := pw.Write([]byte("[]")); err != nil { + pw.CloseWithError(fmt.Errorf("failed to write empty array: %w", err)) + } + } + _ = pw.Close() + } + }() + + // --- 3. Metadata Initialization (Identical to V1) --- + columns, err := rows.Columns() + if err != nil { + pw.CloseWithError(fmt.Errorf("could not get columns: %w", err)) + return + } + + colIndex := make(map[string]int, len(columns)) + for i, col := range columns { + colIndex[col] = i + } + + // Reusable buffers for Scan (Identical to V1) + values := make([]interface{}, len(columns)) + ptrs := make([]interface{}, len(columns)) + for i := range values { + ptrs[i] = &values[i] + } + + // --- 4. Stream Loop --- + for rows.Next() { + // --- CRITICAL Optimization 1: Array Opener Logic --- + if !arrayStarted { + // Write the opening bracket ONLY on the first successful row iteration. + if _, err := bw.Write([]byte{'['}); err != nil { + pw.CloseWithError(fmt.Errorf("failed to write start bracket: %w", err)) + return + } + arrayStarted = true + } else { + // Write the comma and newline separator. This write is now buffered! + if _, err := bw.Write([]byte{',', '\n'}); err != nil { + pw.CloseWithError(fmt.Errorf("failed to write separator: %w", err)) + return + } + } + + // Scan the row data (Identical to V1) + if err := rows.Scan(ptrs...); err != nil { + pw.CloseWithError(fmt.Errorf("row scan failed: %w", err)) + return + } + + // Build the row map (Identical to V1) + row := make(map[string]interface{}) + for _, col := range columnOrder { + idx, ok := colIndex[col] + if !ok { + continue + } + + if b, ok := values[idx].([]byte); ok { + row[col] = string(b) + } else { + row[col] = values[idx] + } + } + + // --- CRITICAL Optimization 2: Use json.Encoder instead of json.Marshal --- + // This avoids allocating and copying a new byte slice for every row. + if err := jsonEnc.Encode(row); err != nil { + pw.CloseWithError(fmt.Errorf("failed to encode row: %w", err)) + return + } + + // Note: Unlike V1, we don't need a final ']' here because json.Encoder.Encode + // naturally includes a newline, and the defer ensures the final ']' is written. + } + + // If the array was opened, write the final closing bracket (buffered). + if arrayStarted { + if _, err := bw.Write([]byte{'\n', ']'}); err != nil { + pw.CloseWithError(fmt.Errorf("failed to write closing bracket: %w", err)) + return + } + } + + // The defer will handle the final bw.Flush() and pw.Close(). + }() // End goroutine + + return pr, nil +} diff --git a/internal/encoders/stream_json_encoder.go b/internal/encoders/stream_json_encoder.go new file mode 100644 index 0000000..37b19ae --- /dev/null +++ b/internal/encoders/stream_json_encoder.go @@ -0,0 +1,133 @@ +package encoders + +import ( + "encoding/json" + "fmt" + "io" +) + +type StreamJSONEncoder struct{} + +func (e *StreamJSONEncoder) EncodeAll(rows RowScanner, columnOrder []string) (io.ReadCloser, error) { + if rows == nil { + return nil, fmt.Errorf("nil rows received") + } + + pr, pw := io.Pipe() + + // log.Printf("[STREAM] Goroutine starting to process rows.") + + // The entire encoding process runs in a separate goroutine + go func() { + // --- 1. Control Flags --- + var arrayStarted bool + var rowProcessed bool + + // --- 2. Deferred Cleanup --- + defer func() { + // CRITICAL: Ensure rows are closed to release the database connection/lock (e.g., SQLite read lock). + _ = rows.Close() + // log.Printf("[STREAM] ROWS CLOSED (Read Lock Released).") + + // CRITICAL: Handle closing the pipe writer (pw). + if err := rows.Err(); err != nil { + // Close with error if rows encountered an issue during iteration + pw.CloseWithError(fmt.Errorf("rows iteration failed: %w", err)) + } else { + // Ensure the array structure is closed correctly on successful iteration. + if !arrayStarted { + // Case 1: No rows were processed. Write "[]". + if _, err := pw.Write([]byte("[]")); err != nil { + pw.CloseWithError(fmt.Errorf("failed to write empty array: %w", err)) + } + } else { + // Case 2: Array was opened. Write the final ']'. + // We add a newline for structure. + if _, err := pw.Write([]byte("\n]")); err != nil { + pw.CloseWithError(fmt.Errorf("failed to write closing bracket: %w", err)) + } + } + _ = pw.Close() + } + // log.Printf("[STREAM] PIPE CLOSED (Stream Ended).") + }() + + // --- 3. Metadata Initialization --- + columns, err := rows.Columns() + if err != nil { + pw.CloseWithError(fmt.Errorf("could not get columns: %w", err)) + return + } + + colIndex := make(map[string]int, len(columns)) + for i, col := range columns { + colIndex[col] = i + } + + // Reusable buffers for Scan (Defined inside the goroutine) + values := make([]interface{}, len(columns)) + ptrs := make([]interface{}, len(columns)) + for i := range values { + ptrs[i] = &values[i] + } + + // --- 4. Stream Loop --- + for rows.Next() { + // --- Comma and Array Opener Logic --- + if !arrayStarted { + // Write the opening bracket ONLY on the first successful row iteration. + if _, err := pw.Write([]byte("[")); err != nil { + pw.CloseWithError(fmt.Errorf("failed to write start bracket: %w", err)) + return + } + arrayStarted = true + } else if rowProcessed { + // Write the comma ONLY if it's NOT the first element. + if _, err := pw.Write([]byte(",\n")); err != nil { + pw.CloseWithError(fmt.Errorf("failed to write separator: %w", err)) + return + } + } + + // Scan the row data + if err := rows.Scan(ptrs...); err != nil { + pw.CloseWithError(fmt.Errorf("row scan failed: %w", err)) + return + } + + // Build the row map + row := make(map[string]interface{}) + for _, col := range columnOrder { + idx, ok := colIndex[col] + if !ok { + continue + } + + // Handle []byte specifically (common for database text/binary types) + // This is a known performance area for future optimization. + if b, ok := values[idx].([]byte); ok { + row[col] = string(b) + } else { + row[col] = values[idx] + } + } + + // Marshal and write the single row object + jsonBytes, err := json.Marshal(row) + if err != nil { + pw.CloseWithError(fmt.Errorf("failed to marshal row: %w", err)) + return + } + if _, err := pw.Write(jsonBytes); err != nil { + pw.CloseWithError(fmt.Errorf("failed to write row bytes: %w", err)) + return + } + + rowProcessed = true + } + + // The final defer will handle rows.Err() and closing the pipe writer (pw) + }() // End goroutine + + return pr, nil +} diff --git a/internal/encoders/stream_json_encoder_test.go b/internal/encoders/stream_json_encoder_test.go new file mode 100644 index 0000000..47e2850 --- /dev/null +++ b/internal/encoders/stream_json_encoder_test.go @@ -0,0 +1,191 @@ +package encoders + +import ( + "database/sql" + "encoding/json" + "fmt" + "io" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// +// Helpers +// + +// Wraps sql.Rows and forces Scan to fail. +type failingScanRows struct { + *sql.Rows +} + +func (f *failingScanRows) Scan(dest ...interface{}) error { + return fmt.Errorf("forced scan failure") +} + +// +// TESTS +// + +func TestStreamJSONEncoder_EncodeAll_Normal(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + rows := sqlmock.NewRows([]string{"id", "name"}). + AddRow(1, "Alice"). + AddRow(2, "Bob") + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + + sqlRows, _ := db.Query("SELECT id, name FROM x") + + enc := &StreamJSONEncoder{} + + // NOTE: first argument "s" was in your signature but unused — keeping "" for now + rc, err := enc.EncodeAll(sqlRows, []string{"name", "id"}) + require.NoError(t, err) + require.NotNil(t, rc) + + data, err := io.ReadAll(rc) + require.NoError(t, err) + _ = rc.Close() + + var out []map[string]any + err = json.Unmarshal(data, &out) + require.NoError(t, err) + + require.Len(t, out, 2) + assert.Equal(t, "Alice", out[0]["name"]) + assert.Equal(t, float64(1), out[0]["id"]) +} + +func TestStreamJSONEncoder_EncodeAll_EmptyRows(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + rows := sqlmock.NewRows([]string{"id", "name"}) // no rows added + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + sqlRows, _ := db.Query("SELECT id, name FROM empty") + + enc := &StreamJSONEncoder{} + rc, err := enc.EncodeAll(sqlRows, []string{"id", "name"}) + require.NoError(t, err) + + data, err := io.ReadAll(rc) + require.NoError(t, err) + _ = rc.Close() + + assert.Equal(t, "[]", strings.TrimSpace(string(data))) +} + +func TestStreamJSONEncoder_EncodeAll_MissingColumns(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + rows := sqlmock.NewRows([]string{"id"}). + AddRow(100) + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + sqlRows, _ := db.Query("SELECT id FROM test") + + enc := &StreamJSONEncoder{} + rc, err := enc.EncodeAll(sqlRows, []string{"id", "missing"}) + require.NoError(t, err) + + data, _ := io.ReadAll(rc) + _ = rc.Close() + + var out []map[string]any + _ = json.Unmarshal(data, &out) + + require.Len(t, out, 1) + assert.Contains(t, out[0], "id") + assert.NotContains(t, out[0], "missing") +} + +func TestStreamJSONEncoder_EncodeAll_BytesConverted(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + rows := sqlmock.NewRows([]string{"blob"}). + AddRow([]byte("xyz")) + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + sqlRows, _ := db.Query("SELECT blob FROM test") + + enc := &StreamJSONEncoder{} + rc, err := enc.EncodeAll(sqlRows, []string{"blob"}) + require.NoError(t, err) + + data, _ := io.ReadAll(rc) + _ = rc.Close() + + var out []map[string]any + _ = json.Unmarshal(data, &out) + + assert.Equal(t, "xyz", out[0]["blob"]) +} + +func TestStreamJSONEncoder_EncodeAll_ScanError(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + rows := sqlmock.NewRows([]string{"id"}).AddRow(1) + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + + sqlRows, _ := db.Query("SELECT id FROM failing") + + // Wrap to FORCE scan errors + wrapped := &failingScanRows{Rows: sqlRows} + + enc := &StreamJSONEncoder{} + rc, err := enc.EncodeAll(wrapped, []string{"id"}) + // streaming encoder returns reader + nil here; the error is async + require.NoError(t, err) + require.NotNil(t, rc) + + data, readErr := io.ReadAll(rc) + _ = rc.Close() + + require.Error(t, readErr) + assert.Contains(t, readErr.Error(), "row scan failed") + // optional: we don't really care about partial JSON, but we can at least assert it's not a valid full array + _ = data +} + +func TestStreamJSONEncoder_EncodeAll_StreamStartsImmediately(t *testing.T) { + db, mock, _ := sqlmock.New() + defer func() { + _ = db.Close() + }() + + // Single row is enough — we test that rc is returned BEFORE the goroutine finishes + rows := sqlmock.NewRows([]string{"v"}).AddRow("hello") + + mock.ExpectQuery("SELECT .*").WillReturnRows(rows) + sqlRows, _ := db.Query("SELECT v FROM t") + + enc := &StreamJSONEncoder{} + + rc, err := enc.EncodeAll(sqlRows, []string{"v"}) + require.NoError(t, err) + require.NotNil(t, rc) + + // If this blocks, streaming isn't working + partial := make([]byte, 1) + _, err = rc.Read(partial) + require.NoError(t, err) +} diff --git a/internal/encoders/stream_ndjson_encoder.go b/internal/encoders/stream_ndjson_encoder.go new file mode 100644 index 0000000..91c4c15 --- /dev/null +++ b/internal/encoders/stream_ndjson_encoder.go @@ -0,0 +1,168 @@ +package encoders + +import ( + "encoding/json" + "fmt" + "io" +) + +// Experimental - not prod ready, will require tweaks to the engine + +type NDJSONEncoder struct{} + +func (e *NDJSONEncoder) EncodeAll(rows RowScanner, columnOrder []string) (io.ReadCloser, error) { + pr, pw := io.Pipe() + + go func() { + defer func() { + _ = rows.Close() + _ = pw.Close() + }() + + columns, err := rows.Columns() + if err != nil { + pw.CloseWithError(fmt.Errorf("failed to get columns: %w", err)) + return + } + + colIndex := make(map[string]int, len(columns)) + for i, col := range columns { + colIndex[col] = i + } + + values := make([]interface{}, len(columns)) + ptrs := make([]interface{}, len(columns)) + for i := range values { + ptrs[i] = &values[i] + } + + enc := json.NewEncoder(pw) + + for rows.Next() { + if err := rows.Scan(ptrs...); err != nil { + pw.CloseWithError(fmt.Errorf("row scan failed: %w", err)) + return + } + + row := map[string]interface{}{} + for _, col := range columnOrder { + idx := colIndex[col] + if b, ok := values[idx].([]byte); ok { + row[col] = string(b) + } else { + row[col] = values[idx] + } + } + + // Encode as standalone NDJSON object + if err := enc.Encode(row); err != nil { + pw.CloseWithError(fmt.Errorf("encode error: %w", err)) + return + } + } + }() + + return pr, nil +} + +// type NDJSONEncoder struct{} + +// func (e NDJSONEncoder) EncodeAll(rows RowScanner, columnOrder []string) (io.ReadCloser, error) { +// if rows == nil { +// return nil, fmt.Errorf("nil rows received") +// } + +// pr, pw := io.Pipe() + +// go func() { +// defer func() { +// _ = rows.Close() +// pw.Close() +// }() + +// columns, err := rows.Columns() +// if err != nil { +// pw.CloseWithError(fmt.Errorf("failed to get columns: %w", err)) +// return +// } + +// colIndex := make(map[string]int, len(columns)) +// for i, col := range columns { +// colIndex[col] = i +// } + +// values := make([]interface{}, len(columns)) +// ptrs := make([]interface{}, len(columns)) +// for i := range values { +// ptrs[i] = &values[i] +// } + +// bw := bufio.NewWriterSize(pw, 64*1024) // 64KB buffer is optimal for streaming +// enc := json.NewEncoder(bw) + +// for rows.Next() { +// if err := rows.Scan(ptrs...); err != nil { +// pw.CloseWithError(fmt.Errorf("scan failed: %w", err)) +// return +// } + +// row := make(map[string]interface{}, len(columnOrder)) +// for _, col := range columnOrder { +// idx := colIndex[col] +// if b, ok := values[idx].([]byte); ok { +// row[col] = string(b) +// } else { +// row[col] = values[idx] +// } +// } + +// if err := enc.Encode(row); err != nil { +// pw.CloseWithError(fmt.Errorf("encode failed: %w", err)) +// return +// } +// } + +// if err := rows.Err(); err != nil { +// pw.CloseWithError(fmt.Errorf("rows iteration error: %w", err)) +// return +// } + +// bw.Flush() +// }() + +// return pr, nil +// } + +// type NDJSONToJSONArrayEncoder struct { +// Inner RowEncoder +// } + +// func (e *NDJSONToJSONArrayEncoder) EncodeAll(rows RowScanner, order []string) (io.ReadCloser, error) { +// ndjson, err := e.Inner.EncodeAll(rows, order) +// if err != nil { +// return nil, err +// } + +// pr, pw := io.Pipe() + +// go func() { +// defer pw.Close() +// defer ndjson.Close() + +// scanner := bufio.NewScanner(ndjson) +// pw.Write([]byte("[")) + +// first := true +// for scanner.Scan() { +// if !first { +// pw.Write([]byte(",")) +// } +// first = false +// pw.Write(scanner.Bytes()) +// } + +// pw.Write([]byte("]")) +// }() + +// return pr, nil +// } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 17762c5..6664d7c 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -4,25 +4,58 @@ import ( "context" "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/models" ) -type Storage interface{} +type ( + EngineContextBuilder interface { + Build(ctx context.Context, plan *entities.Plan) (*models.EngineContext, error) + } + DataStore interface { + Find(tableName string) ([]map[string]interface{}, error) + } + Processor interface { + ProcessPlan(ctx *models.EngineContext) error + } + ResponseBuilder interface { + BuildResponse(results []map[string]interface{}, columnOrder []string) (*entities.ExecutePlanResponse, error) + } +) type engine struct { - storage Storage + contextBuilder EngineContextBuilder + dataStore DataStore + processor Processor + responseBuilder ResponseBuilder +} + +type Executor interface { + ExecutePlan(ctx context.Context, req entities.ExecutePlanRequest) (*entities.ExecutePlanResponse, error) } -func NewEngine(storage Storage) *engine { +func NewEngine(contextBuilder EngineContextBuilder, dataStore DataStore, processor Processor, responseBuilder ResponseBuilder) Executor { return &engine{ - storage: storage, + contextBuilder: contextBuilder, + dataStore: dataStore, + processor: processor, + responseBuilder: responseBuilder, } } -func (e *engine) ExecutePlan(ctx context.Context, req entities.ExecutePlanRequest) (*entities.ExecutePlanResponse, error) { - return &entities.ExecutePlanResponse{ - Columns: []string{"status", "plan"}, - Rows: [][]any{ - {"ok", req.Plan.Name}, - }, - }, nil +func (u *engine) ExecutePlan(ctx context.Context, req entities.ExecutePlanRequest) (*entities.ExecutePlanResponse, error) { + engineCtx, err := u.contextBuilder.Build(ctx, &req.Plan) + if err != nil { + return nil, err + } + + if err := u.processor.ProcessPlan(engineCtx); err != nil { + return nil, err + } + + results, err := u.dataStore.Find(engineCtx.OutputDatasetName) + if err != nil { + return nil, err + } + + return u.responseBuilder.BuildResponse(results, engineCtx.GetResultsOrderedColumns()) } diff --git a/internal/engine_context/engine_context.go b/internal/engine_context/engine_context.go new file mode 100644 index 0000000..f2df068 --- /dev/null +++ b/internal/engine_context/engine_context.go @@ -0,0 +1,139 @@ +package engine_context + +import ( + "context" + "fmt" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +const federatedConnectorName = "federated" + +type Builder struct{} + +func NewBuilder() *Builder { + return &Builder{} +} + +func (b *Builder) Build(ctx context.Context, plan *entities.Plan) (*models.EngineContext, error) { + return buildEngineContext(ctx, plan) +} + +func buildEngineContext(ctx context.Context, plan *entities.Plan) (*models.EngineContext, error) { + connectors := buildConnectors(plan) + datasets := buildDatasets(plan) + + if err := resolveDatasetChildren(datasets); err != nil { + return nil, err + } + + if err := validateOutputDataset(plan, datasets); err != nil { + return nil, err + } + + return &models.EngineContext{ + Ctx: ctx, + Datasets: datasets, + Connectors: connectors, + EngineOptions: models.EngineContextOptions{Verbose: plan.Output.Verbose}, + OutputDatasetName: plan.Output.DatasetName, + }, nil +} + +func buildConnectors(plan *entities.Plan) map[string]*models.EngineContextConnector { + connectors := defaultConnectorMap() + + for _, conn := range plan.Connectors { + connectors[conn.Name] = &models.EngineContextConnector{ + Type: conn.Type, + Connection: conn.Connection, + Config: conn.Connection, + } + } + + return connectors +} + +func buildDatasets(plan *entities.Plan) map[string]*models.EngineContextDataset { + datasets := make(map[string]*models.EngineContextDataset, len(plan.Datasets)) + + for _, ds := range plan.Datasets { + datasets[ds.Name] = &models.EngineContextDataset{ + Name: ds.Name, + ConnectorName: ds.ConnectorName, + Type: ds.Type, + Query: ds.Query, + Children: buildChildPlaceholders(ds), + Dedupe: ds.Dedupe, + Columns: buildColumns(ds), + ColumnOrder: ds.ColumnOrder, + DatasetOptions: ds.Options, + Data: ds.Data, + } + } + + return datasets +} + +func buildChildPlaceholders(ds entities.Dataset) []*models.EngineContextDataset { + children := make([]*models.EngineContextDataset, 0, len(ds.Children)) + + for _, child := range ds.Children { + children = append(children, &models.EngineContextDataset{ + Name: child.DatasetName, + }) + } + + return children +} + +func buildColumns(ds entities.Dataset) []models.EngineContextColumn { + columns := make([]models.EngineContextColumn, 0, len(ds.Columns)) + + for _, col := range ds.Columns { + columns = append(columns, models.EngineContextColumn{ + Name: col.Name, + Type: col.Type, + Description: col.Description, + }) + } + + return columns +} + +func resolveDatasetChildren(datasets map[string]*models.EngineContextDataset) error { + for _, ds := range datasets { + for i, child := range ds.Children { + resolvedChild, ok := datasets[child.Name] + if !ok { + return fmt.Errorf("dataset %s references unknown child dataset: %s", ds.Name, child.Name) + } + + ds.Children[i] = resolvedChild + } + } + + return nil +} + +func validateOutputDataset(plan *entities.Plan, datasets map[string]*models.EngineContextDataset) error { + if plan.Output.DatasetName == "" { + return fmt.Errorf("must set output dataset") + } + + if _, ok := datasets[plan.Output.DatasetName]; !ok { + return fmt.Errorf("output dataset must be in defined datasets") + } + + return nil +} + +func defaultConnectorMap() map[string]*models.EngineContextConnector { + return map[string]*models.EngineContextConnector{ + federatedConnectorName: { + Type: "JOIN", + Connection: "", + }, + } +} diff --git a/internal/models/engine_context.go b/internal/models/engine_context.go new file mode 100644 index 0000000..b228bae --- /dev/null +++ b/internal/models/engine_context.go @@ -0,0 +1,45 @@ +package models + +import "context" + +type EngineContext struct { + Ctx context.Context + Datasets map[string]*EngineContextDataset + Connectors map[string]*EngineContextConnector + EngineOptions EngineContextOptions + OutputDatasetName string +} + +func (c *EngineContext) GetResultsOrderedColumns() []string { + return c.Datasets[c.OutputDatasetName].ColumnOrder +} + +type EngineContextColumn struct { + Name string + Type string + Description string +} + +type EngineContextDataset struct { + Name string + ConnectorName string + Type string + Query string + RawQuery string + Children []*EngineContextDataset + Dedupe []string + Columns []EngineContextColumn + ColumnOrder []string + DatasetOptions map[string]interface{} + Data any +} + +type EngineContextOptions struct { + Verbose bool +} + +type EngineContextConnector struct { + Type string + Connection interface{} + Config interface{} +} diff --git a/internal/plugins/api/plugin.go b/internal/plugins/api/plugin.go new file mode 100644 index 0000000..8eb7bb1 --- /dev/null +++ b/internal/plugins/api/plugin.go @@ -0,0 +1,321 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strings" + "time" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + pluginopts "github.com/kvatch-hub/kvatch-runtime/internal/plugins/options" +) + +type ( + StorageManager interface { + EnsureTable(ctx context.Context, table string, columns []models.EngineContextColumn, dedupe []string) error + Insert(ctx context.Context, table string, data []map[string]interface{}, columns []models.EngineContextColumn, dedupe []string) error + } +) + +type APIDatasetPlugin struct { + Manager StorageManager +} + +func NewAPIDatasetPlugin() *APIDatasetPlugin { + return &APIDatasetPlugin{} +} + +func (p *APIDatasetPlugin) GetType() plugins.PluginType { + return plugins.PluginType("api") // must match dataset.type +} + +func (p *APIDatasetPlugin) ProcessData( + ctx context.Context, + params plugins.PluginProcessParams, +) (*plugins.ProcessedData, error) { + + rawOpts := params.DatasetContext.DatasetOptions + if rawOpts == nil { + return nil, fmt.Errorf("api: missing options") + } + + opts, err := pluginopts.ParseDatasetOptions(rawOpts) + if err != nil { + return nil, fmt.Errorf("api: invalid api options: %w", err) + } + + data, err := io.ReadAll(params.DataReader) + if err != nil { + return nil, fmt.Errorf("api: failed to read response body: %w", err) + } + defer func() { _ = params.DataReader.Close() }() + + if len(data) == 0 { + return emptyAPITableResult(params), nil + } + + rows, err := parseAndExtractRows(data, opts.Response) + if err != nil { + return nil, fmt.Errorf("api: failed to parse response: %w", err) + } + + if len(rows) == 0 { + return emptyAPITableResult(params), nil + } + + injectTimestampIfNeeded(opts, rows) + + // ───────────────────────────────────────────── + // StorageManager path starts here + // ───────────────────────────────────────────── + + mgr := params.Manager + tableName := params.StorageClient.GetTableName(params.DatasetContext.Name) + + // Infer columns once + columns := inferColumnsFromRows(rows) + params.DatasetContext.Columns = columns + + if err := mgr.EnsureTable( + ctx, + tableName, + columns, + params.DatasetContext.Dedupe, + ); err != nil { + return nil, err + } + + if err := mgr.Insert( + ctx, + tableName, + rows, + columns, + params.DatasetContext.Dedupe, + ); err != nil { + return nil, err + } + + return &plugins.ProcessedData{ + TableName: tableName, + Columns: columns, + RowCount: int64(len(rows)), + }, nil +} + +// +// ─── RESPONSE PARSING & EXTRACTION ────────────────────────────────── +// + +func parseAndExtractRows( + body []byte, + respSpec entities.APIResponseSpec, +) ([]map[string]interface{}, error) { + switch respSpec.Format { + case entities.APIResponseFormatJSON: + var parsed interface{} + if err := json.Unmarshal(body, &parsed); err != nil { + return nil, fmt.Errorf("api: json decode error: %w", err) + } + + // Extract sub-tree using simple JSONPath-like syntax + extracted, err := extractJSONPath(parsed, respSpec.Extract) + if err != nil { + return nil, err + } + + // Coerce extracted value into []map[string]interface{} + rows, err := coerceToRowArray(extracted) + if err != nil { + return nil, err + } + + // Apply normalization if requested + if respSpec.Normalize.Enabled { + return normalizeRows(rows, respSpec.Normalize) + } + return rows, nil + + case entities.APIResponseFormatText: + // Very simple: single column "value" + return []map[string]interface{}{ + {"value": string(body)}, + }, nil + + default: + return nil, fmt.Errorf("api: unsupported response format: %s", respSpec.Format) + } +} + +func extractJSONPath(parsed interface{}, path string) (interface{}, error) { + path = strings.TrimSpace(path) + if path == "" || path == "$" { + return parsed, nil + } + + // Strip leading "$." or "$" + if strings.HasPrefix(path, "$.") { + path = path[2:] + } else if strings.HasPrefix(path, "$") { + path = path[1:] + } + + segments := strings.Split(path, ".") + cur := parsed + + for _, seg := range segments { + m, ok := cur.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("api: path '%s' segment '%s' not on an object", path, seg) + } + next, found := m[seg] + if !found { + return nil, fmt.Errorf("api: key '%s' not found in path '%s'", seg, path) + } + cur = next + } + + return cur, nil +} + +// coerceToRowArray converts arbitrary parsed JSON into a row slice: +// - []object → rows +// - object → single-row slice +func coerceToRowArray(v interface{}) ([]map[string]interface{}, error) { + switch t := v.(type) { + case []interface{}: + out := make([]map[string]interface{}, 0, len(t)) + for _, item := range t { + obj, ok := item.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("api: expected object in array, got %T", item) + } + out = append(out, obj) + } + return out, nil + + case map[string]interface{}: + return []map[string]interface{}{t}, nil + + default: + return nil, fmt.Errorf("api: cannot coerce %T to row array", v) + } +} + +func injectTimestampIfNeeded(opts *entities.APIDatasetOptions, rows []map[string]interface{}) { + if opts == nil || !opts.InjectTimestamp { + return + } + + field := strings.TrimSpace(opts.TimestampField) + if field == "" { + field = "fetched_at" + } + + now := time.Now().UTC().Format(time.RFC3339) + + for _, row := range rows { + if _, exists := row[field]; !exists { + row[field] = now + } + } +} + +// +// ─── NORMALISATION ───────────────────────────────────────────────── +// + +// normalizeRows turns objects like: +// +// { "bitcoin": { "usd": 123 }, "ethereum": { "usd": 456 } } +// +// into: +// +// { coin: "bitcoin", price_usd: 123 } +// { coin: "ethereum", price_usd: 456 } +// +// When Normalize.Enabled is true, we interpret the *row itself* as a +// key/value bag. This works well for Coingecko-like APIs. +func normalizeRows( + rows []map[string]interface{}, + cfg entities.APINormalizeConfig, +) ([]map[string]interface{}, error) { + if !cfg.Enabled { + return rows, nil + } + + keyField := cfg.KeyField + if keyField == "" { + keyField = "key" + } + prefix := cfg.ValuePrefix + if prefix == "" { + prefix = "value_" + } + + var out []map[string]interface{} + for _, row := range rows { + for k, v := range row { + switch inner := v.(type) { + case map[string]interface{}: + flat := map[string]interface{}{keyField: k} + for innerK, innerV := range inner { + flat[prefix+innerK] = innerV + } + out = append(out, flat) + default: + flat := map[string]interface{}{ + keyField: k, + prefix + "value": inner, + } + out = append(out, flat) + } + } + } + return out, nil +} + +// +// ─── TABLE/SCHEMA HELPERS ────────────────────────────────────────── +// + +func inferColumnsFromRows(rows []map[string]interface{}) []models.EngineContextColumn { + first := rows[0] + cols := make([]models.EngineContextColumn, 0, len(first)) + + for k, v := range first { + cols = append(cols, models.EngineContextColumn{ + Name: k, + Type: inferJSONType(v), + Description: fmt.Sprintf("API field: %s", k), + }) + } + return cols +} + +func inferJSONType(value interface{}) string { + switch value.(type) { + case string: + return "TEXT" + case float64, int, int64: + return "NUMERIC" + case bool: + return "BOOLEAN" + case nil: + return "TEXT" + default: + return "TEXT" + } +} + +func emptyAPITableResult(params plugins.PluginProcessParams) *plugins.ProcessedData { + tableName := params.StorageClient.GetTableName(params.DatasetContext.Name) + return &plugins.ProcessedData{ + TableName: tableName, + Columns: []models.EngineContextColumn{}, + RowCount: 0, + } +} diff --git a/internal/plugins/api/plugin_test.go b/internal/plugins/api/plugin_test.go new file mode 100644 index 0000000..53daf16 --- /dev/null +++ b/internal/plugins/api/plugin_test.go @@ -0,0 +1,269 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "testing" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins/mocks" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestAPI_ProcessData_SimpleSuccess(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockManager := mocks.NewMockStorageManager(ctrl) + + payload := []map[string]any{ + {"id": 1, "name": "Alice"}, + {"id": 2, "name": "Bob"}, + } + + reader := io.NopCloser(bytes.NewReader(mustJSONTB(t, payload))) + + ds := "users" + table := "api_users" + + mockStorageClient.EXPECT(). + GetTableName(ds). + Return(table) + + mockManager.EXPECT(). + EnsureTable( + gomock.Any(), + table, + gomock.Any(), // inferred columns + nil, + ). + Return(nil) + + mockManager.EXPECT(). + Insert( + gomock.Any(), + table, + gomock.AssignableToTypeOf([]map[string]interface{}{}), + gomock.Any(), + nil, + ). + Return(nil) + + plugin := NewAPIDatasetPlugin() + + out, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + Manager: mockManager, + DatasetContext: &models.EngineContextDataset{ + Name: ds, + DatasetOptions: map[string]interface{}{ + "response": map[string]interface{}{ + "format": "json", + }, + }, + }, + }) + + require.NoError(t, err) + require.Equal(t, table, out.TableName) + require.Equal(t, int64(2), out.RowCount) + require.NotEmpty(t, out.Columns) +} + +func TestAPI_ProcessData_EmptyArray(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockManager := mocks.NewMockStorageManager(ctrl) + + mockStorageClient.EXPECT(). + GetTableName("empty"). + Return("api_empty") + + plugin := NewAPIDatasetPlugin() + + out, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: io.NopCloser(bytes.NewReader([]byte(`[]`))), + StorageClient: mockStorageClient, + Manager: mockManager, + DatasetContext: &models.EngineContextDataset{ + Name: "empty", + DatasetOptions: map[string]interface{}{ + "response": map[string]interface{}{ + "format": "json", + }, + }, + }, + }) + + require.NoError(t, err) + require.Equal(t, "api_empty", out.TableName) + require.Equal(t, int64(0), out.RowCount) + require.Empty(t, out.Columns) +} + +func TestAPI_ProcessData_EnsureTableFails(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockManager := mocks.NewMockStorageManager(ctrl) + + reader := io.NopCloser(bytes.NewReader([]byte(`[{"id":1}]`))) + + mockStorageClient.EXPECT(). + GetTableName("fail"). + Return("api_fail") + + mockManager.EXPECT(). + EnsureTable(gomock.Any(), "api_fail", gomock.Any(), nil). + Return(fmt.Errorf("boom")) + + plugin := NewAPIDatasetPlugin() + + out, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + Manager: mockManager, + DatasetContext: &models.EngineContextDataset{ + Name: "fail", + DatasetOptions: map[string]interface{}{ + "response": map[string]interface{}{ + "format": "json", + }, + }, + }, + }) + + require.Nil(t, out) + require.Error(t, err) + require.Contains(t, err.Error(), "boom") +} + +func TestAPI_ProcessData_InsertFails(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockManager := mocks.NewMockStorageManager(ctrl) + + reader := io.NopCloser(bytes.NewReader([]byte(`[{"id":1}]`))) + + mockStorageClient.EXPECT(). + GetTableName("fail_insert"). + Return("api_fail_insert") + + mockManager.EXPECT(). + EnsureTable(gomock.Any(), "api_fail_insert", gomock.Any(), nil). + Return(nil) + + mockManager.EXPECT(). + Insert(gomock.Any(), "api_fail_insert", gomock.Any(), gomock.Any(), nil). + Return(fmt.Errorf("insert boom")) + + plugin := NewAPIDatasetPlugin() + + out, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + Manager: mockManager, + DatasetContext: &models.EngineContextDataset{ + Name: "fail_insert", + DatasetOptions: map[string]interface{}{ + "response": map[string]interface{}{ + "format": "json", + }, + }, + }, + }) + + require.Nil(t, out) + require.Error(t, err) + require.Contains(t, err.Error(), "insert") +} + +func TestAPI_ProcessData_LargeBatch(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockManager := mocks.NewMockStorageManager(ctrl) + + plugin := NewAPIDatasetPlugin() + + rows := make([]map[string]interface{}, 1200) + for i := 0; i < len(rows); i++ { + rows[i] = map[string]interface{}{ + "id": i, + "value": fmt.Sprintf("v-%d", i), + } + } + + reader := io.NopCloser(bytes.NewReader(mustJSONTB(t, rows))) + + ds := "api_big" + table := "api_big_table" + + mockStorageClient.EXPECT(). + GetTableName(ds). + Return(table). + Times(1) + + mockManager.EXPECT(). + EnsureTable( + gomock.Any(), + table, + gomock.Any(), + nil, + ). + Return(nil). + Times(1) + + // API plugin inserts once with all rows + mockManager.EXPECT(). + Insert( + gomock.Any(), + table, + gomock.AssignableToTypeOf([]map[string]interface{}{}), + gomock.Any(), + nil, + ). + Return(nil). + Times(1) + + out, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + Manager: mockManager, + DatasetContext: &models.EngineContextDataset{ + Name: ds, + DatasetOptions: map[string]interface{}{ + "response": map[string]interface{}{ + "format": "json", + }, + }, + }, + }) + + require.NoError(t, err) + require.Equal(t, table, out.TableName) + require.Equal(t, int64(1200), out.RowCount) + require.NotEmpty(t, out.Columns) +} + +// /* ---------- Helpers ---------- */ + +func mustJSONTB(t testing.TB, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + require.NoError(t, err) + return b +} diff --git a/internal/plugins/csv/plugin.go b/internal/plugins/csv/plugin.go new file mode 100644 index 0000000..294b484 --- /dev/null +++ b/internal/plugins/csv/plugin.go @@ -0,0 +1,232 @@ +package csv + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + pluginopts "github.com/kvatch-hub/kvatch-runtime/internal/plugins/options" +) + +// CSVDataPlugin processes CSV data from connections +type CSVDataPlugin struct{} + +// NewCSVDataPlugin creates a new CSV data plugin +func NewCSVDataPlugin() *CSVDataPlugin { + return &CSVDataPlugin{} +} + +func (p *CSVDataPlugin) GetType() plugins.PluginType { + return plugins.PluginTypeCSV +} + +func (p *CSVDataPlugin) ProcessData(ctx context.Context, params plugins.PluginProcessParams) (*plugins.ProcessedData, error) { + const sampleSize = 100 + const batchSize = 500 + + opts := pluginopts.ConvertMapToCSVPluginOptions(params.DatasetContext.DatasetOptions) + + reader := csv.NewReader(params.DataReader) + reader.Comma = rune(opts.Delimiter[0]) + defer func() { _ = params.DataReader.Close() }() + + // Skip lines if requested + for i := 0; i < opts.SkipLines; i++ { + _, err := reader.Read() + if err != nil { + if err == io.EOF { + return nil, fmt.Errorf("CSV file has fewer lines than skip_lines (%d)", opts.SkipLines) + } + return nil, fmt.Errorf("failed to skip line %d: %v", i+1, err) + } + } + + // ----------------------------------------------------- + // State + // ----------------------------------------------------- + var headers []string + var sampleRows [][]string + var rowCount int64 + + tableName := params.StorageClient.GetTableName(params.DatasetContext.Name) + sqlGen := params.StorageClient.GetSQLGenerator() + + // ----------------------------------------------------- + // Insert batch helper + // ----------------------------------------------------- + insertBatch := func(rows [][]string) error { + if len(rows) == 0 { + return nil + } + + dataArray := make([]interface{}, len(rows)) + for i, row := range rows { + rowMap := make(map[string]interface{}, len(headers)) + for j, val := range row { + if j < len(headers) { + rowMap[headers[j]] = val + } + } + dataArray[i] = rowMap + } + + insertStmt, err := sqlGen.GenerateInsertSQL( + tableName, + dataArray, + params.DatasetContext.Columns, + opts.Dedupe, + ) + if err != nil { + return fmt.Errorf("failed to generate insert statement: %v", err) + } + if result := params.StorageClient.Exec(insertStmt); result.Error != nil { + return fmt.Errorf("failed to insert data: %v", result.Error) + } + + rowCount += int64(len(rows)) + return nil + } + + // ----------------------------------------------------- + // Create table helper + // ----------------------------------------------------- + tableCreated := false + createTableIfNeeded := func(sampleRowForTypes []string) error { + if tableCreated { + return nil + } + + plugins.HandleCSVColumns(params.DatasetContext, headers, sampleRowForTypes, InferCSVType) + + createStmt, err := sqlGen.GenerateCreateTableSQL(tableName, params.DatasetContext.Columns) + if err != nil { + return fmt.Errorf("failed to generate create table statement: %v", err) + } + if result := params.StorageClient.Exec(createStmt); result.Error != nil { + return fmt.Errorf("failed to create table: %v", result.Error) + } + + tableCreated = true + return nil + } + + // ----------------------------------------------------- + // 2. Read CSV rows + // ----------------------------------------------------- + batch := make([][]string, 0, batchSize) + lineNum := 0 + + for { + row, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("error reading CSV at line %d: %w", lineNum+1, err) + } + + // ----------------------------- + // Header handling (FIXED) + // ----------------------------- + if len(headers) == 0 { + if opts.HasHeaders { + headers = append([]string(nil), row...) + lineNum++ + continue // skip header row + } + + // No headers → synthesize + for i := range row { + headers = append(headers, fmt.Sprintf("col_%d", i+1)) + } + // DO NOT continue — first row is data + } + + // ----------------------------- + // Collect samples (first N rows) + // ----------------------------- + if !tableCreated && len(sampleRows) < sampleSize { + sampleRows = append(sampleRows, row) + + if len(sampleRows) == 1 { + if err := createTableIfNeeded(sampleRows[0]); err != nil { + return nil, err + } + batch = append(batch, sampleRows...) + sampleRows = nil + } + + lineNum++ + continue + } + + // ----------------------------- + // After table exists — batching + // ----------------------------- + batch = append(batch, row) + if len(batch) >= batchSize { + if err := insertBatch(batch); err != nil { + return nil, err + } + batch = batch[:0] + } + + lineNum++ + } + + // ----------------------------------------------------- + // End of file: handle table creation if needed + // ----------------------------------------------------- + if !tableCreated { + if len(sampleRows) == 0 { + return nil, fmt.Errorf("CSV file has no data rows") + } + if err := createTableIfNeeded(sampleRows[0]); err != nil { + return nil, err + } + batch = append(batch, sampleRows...) + } + + // ----------------------------------------------------- + // Flush remaining batch + // ----------------------------------------------------- + if len(batch) > 0 { + if err := insertBatch(batch); err != nil { + return nil, err + } + } + + // ----------------------------------------------------- + // Return result + // ----------------------------------------------------- + return &plugins.ProcessedData{ + TableName: tableName, + Columns: params.DatasetContext.Columns, + RowCount: rowCount, + }, nil +} + +func InferCSVType(cell string) string { + if cell == "" { + return "TEXT" + } + switch strings.ToLower(cell) { + case "true", "false": + return "BOOLEAN" + } + if _, err := strconv.ParseInt(cell, 10, 64); err == nil { + return "BIGINT" + } + if _, err := strconv.ParseFloat(cell, 64); err == nil { + return "DOUBLE PRECISION" + } + if _, err := time.Parse(time.RFC3339, cell); err == nil { + return "TIMESTAMPTZ" + } + return "TEXT" +} diff --git a/internal/plugins/csv/plugin_test.go b/internal/plugins/csv/plugin_test.go new file mode 100644 index 0000000..7b7c2ef --- /dev/null +++ b/internal/plugins/csv/plugin_test.go @@ -0,0 +1,214 @@ +package csv_test + +import ( + "context" + "errors" + "io" + "strconv" + "strings" + "testing" + + "go.uber.org/mock/gomock" + + "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins/csv" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins/mocks" + "github.com/stretchr/testify/assert" +) + +func TestCSVPlugin_ValidWithHeaders(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockSQLGen := mocks.NewMockSQLGenerator(ctrl) + mockStorage := mocks.NewMockStorageClient(ctrl) + mockSQLGen.EXPECT(). + GenerateCreateTableSQL("test_table", gomock.Any()). + Return("CREATE TABLE test_table", nil) + + mockSQLGen.EXPECT(). + GenerateInsertSQL("test_table", gomock.Any(), gomock.Any(), gomock.Any()). + Return("INSERT INTO test_table", nil) + + mockStorage.EXPECT().GetSQLGenerator().Return(mockSQLGen).AnyTimes() + mockStorage.EXPECT().GetTableName("users").Return("test_table") + mockStorage.EXPECT().Exec(gomock.Any()).Return(&datastore.ExecResult{}).Times(2) + + data := `name,age +Alice,30 +Bob,25 +` + plugin := csv.NewCSVDataPlugin() + + result, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: ioNopCloser(strings.NewReader(data)), + StorageClient: mockStorage, + DatasetContext: &models.EngineContextDataset{ + Name: "users", + DatasetOptions: map[string]interface{}{ + "has_headers": true, + }, + }, + }) + + assert.NoError(t, err) + assert.Equal(t, int64(2), result.RowCount) + assert.Equal(t, "test_table", result.TableName) + assert.Equal(t, "name", result.Columns[0].Name) + assert.Equal(t, "age", result.Columns[1].Name) +} + +func TestCSVPlugin_StreamedBatchInsert(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockSQLGen := mocks.NewMockSQLGenerator(ctrl) + mockStorage := mocks.NewMockStorageClient(ctrl) + + mockSQLGen.EXPECT(). + GenerateCreateTableSQL("test_table", gomock.Any()). + Return("CREATE TABLE test_table", nil) + + mockSQLGen.EXPECT(). + GenerateInsertSQL("test_table", gomock.Any(), gomock.Any(), gomock.Any()). + Return("INSERT INTO test_table", nil). + AnyTimes() + + mockStorage.EXPECT().GetSQLGenerator().Return(mockSQLGen).AnyTimes() + mockStorage.EXPECT().GetTableName("users").Return("test_table").AnyTimes() + mockStorage.EXPECT().Exec(gomock.Any()).Return(&datastore.ExecResult{}).AnyTimes() + + builder := &strings.Builder{} + builder.WriteString("name,age\n") + for i := 0; i < 1500; i++ { + builder.WriteString("User") + builder.WriteString(strconv.Itoa(i)) + builder.WriteString(",") + builder.WriteString(strconv.Itoa(20 + (i % 30))) + builder.WriteString("\n") + } + + plugin := csv.NewCSVDataPlugin() + result, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: ioNopCloser(strings.NewReader(builder.String())), + StorageClient: mockStorage, + DatasetContext: &models.EngineContextDataset{ + Name: "users", + DatasetOptions: map[string]interface{}{ + "has_headers": true, + }, + }, + }) + + assert.NoError(t, err) + assert.Equal(t, int64(1500), result.RowCount) + assert.Equal(t, "test_table", result.TableName) + assert.Len(t, result.Columns, 2) + assert.Equal(t, "name", result.Columns[0].Name) + assert.Equal(t, "age", result.Columns[1].Name) +} + +func TestCSVPlugin_InsertFailureMidBatch(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockSQLGen := mocks.NewMockSQLGenerator(ctrl) + mockStorage := mocks.NewMockStorageClient(ctrl) + + mockSQLGen.EXPECT(). + GenerateCreateTableSQL("test_table", gomock.Any()). + Return("CREATE TABLE test_table", nil) + + mockSQLGen.EXPECT(). + GenerateInsertSQL("test_table", gomock.Any(), gomock.Any(), gomock.Any()). + Return("INSERT INTO test_table", nil). + Times(2) + + mockStorage.EXPECT().GetSQLGenerator().Return(mockSQLGen).AnyTimes() + mockStorage.EXPECT().GetTableName("users").Return("test_table").AnyTimes() + + firstCall := mockStorage.EXPECT().Exec("CREATE TABLE test_table").Return(&datastore.ExecResult{}) + secondCall := mockStorage.EXPECT().Exec("INSERT INTO test_table").Return(&datastore.ExecResult{}) + thirdCall := mockStorage.EXPECT().Exec("INSERT INTO test_table").Return(&datastore.ExecResult{Error: errors.New("boom")}) + + gomock.InOrder(firstCall, secondCall, thirdCall) + + builder := &strings.Builder{} + builder.WriteString("name,age\n") + for i := 0; i < 700; i++ { + builder.WriteString("User") + builder.WriteString(strconv.Itoa(i)) + builder.WriteString(",") + builder.WriteString(strconv.Itoa(30)) + builder.WriteString("\n") + } + + plugin := csv.NewCSVDataPlugin() + _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: ioNopCloser(strings.NewReader(builder.String())), + StorageClient: mockStorage, + DatasetContext: &models.EngineContextDataset{ + Name: "users", + DatasetOptions: map[string]interface{}{ + "has_headers": true, + }, + }, + }) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "insert") +} + +func BenchmarkCSVPlugin_LargeFile(b *testing.B) { + ctrl := gomock.NewController(b) + defer ctrl.Finish() + + mockSQLGen := mocks.NewMockSQLGenerator(ctrl) + mockStorage := mocks.NewMockStorageClient(ctrl) + + mockSQLGen.EXPECT(). + GenerateCreateTableSQL("bench_table", gomock.Any()). + Return("CREATE TABLE bench_table", nil).AnyTimes() + + mockSQLGen.EXPECT(). + GenerateInsertSQL("bench_table", gomock.Any(), gomock.Any(), gomock.Any()). + Return("INSERT INTO bench_table", nil).AnyTimes() + + mockStorage.EXPECT().GetSQLGenerator().Return(mockSQLGen).AnyTimes() + mockStorage.EXPECT().GetTableName("bench_table").Return("bench_table").AnyTimes() + mockStorage.EXPECT().Exec(gomock.Any()).Return(&datastore.ExecResult{}).AnyTimes() + + builder := &strings.Builder{} + builder.WriteString("name,age\n") + for i := 0; i < 10000; i++ { + builder.WriteString("User") + builder.WriteString(strconv.Itoa(i)) + builder.WriteString(",") + builder.WriteString(strconv.Itoa(25)) + builder.WriteString("\n") + } + + plugin := csv.NewCSVDataPlugin() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: ioNopCloser(strings.NewReader(builder.String())), + StorageClient: mockStorage, + DatasetContext: &models.EngineContextDataset{ + Name: "bench_table", + DatasetOptions: map[string]interface{}{ + "has_headers": true, + }, + }, + }) + if err != nil { + b.Fatal(err) + } + } +} + +func ioNopCloser(r *strings.Reader) io.ReadCloser { + return io.NopCloser(r) +} diff --git a/internal/plugins/gsheet/plugin.go b/internal/plugins/gsheet/plugin.go new file mode 100644 index 0000000..d867dba --- /dev/null +++ b/internal/plugins/gsheet/plugin.go @@ -0,0 +1,300 @@ +// Package google_sheets provides a plugin to process tabular data retrieved from a Google Sheet. +// The plugin operates on data in [][]interface{} JSON format, typically produced from Sheets API responses. +// It supports header sanitization, dynamic type inference, table creation, and row insertion. +package google_sheets + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + pluginopts "github.com/kvatch-hub/kvatch-runtime/internal/plugins/options" +) + +type GoogleSheetsDataPlugin struct{} + +func NewGoogleSheetsDataPlugin() *GoogleSheetsDataPlugin { + return &GoogleSheetsDataPlugin{} +} + +func (p *GoogleSheetsDataPlugin) GetType() plugins.PluginType { + return plugins.PluginTypeGoogleSheet +} + +func (p *GoogleSheetsDataPlugin) ProcessData( + ctx context.Context, + params plugins.PluginProcessParams, +) (*plugins.ProcessedData, error) { + opts, err := pluginopts.ParseGoogleSheetOptions(params.DatasetContext.DatasetOptions) + if err != nil { + return nil, fmt.Errorf("failed to parse Google Sheets options: %w", err) + } + + params.DatasetContext.Dedupe = opts.Dedupe + + // Process using the correct mode + if opts.EnableStreaming { + return processStreaming(params, opts.HeaderRowNo) + } + + return processInMemory(params, opts.HeaderRowNo) +} + +func processInMemory(params plugins.PluginProcessParams, headerRowNo int) (*plugins.ProcessedData, error) { + raw, err := io.ReadAll(params.DataReader) + if err != nil { + return nil, fmt.Errorf("failed to read Google Sheets data: %w", err) + } + defer func() { _ = params.DataReader.Close() }() + + var values [][]interface{} + if err := json.Unmarshal(raw, &values); err != nil { + return nil, fmt.Errorf("failed to parse Google Sheets data: %w", err) + } + return processParsedData(params, headerRowNo, values) +} + +func processStreaming(params plugins.PluginProcessParams, headerRowNo int) (*plugins.ProcessedData, error) { + defer func() { + _ = params.DataReader.Close() + }() + + decoder := json.NewDecoder(params.DataReader) + + var values [][]interface{} + if err := decoder.Decode(&values); err != nil { + return nil, fmt.Errorf("failed to parse Google Sheets data: %w", err) + } + + if len(values) == 0 { + return nil, fmt.Errorf("no rows returned from Google Sheet") + } + + headerIdx := headerRowNo - 1 + if headerIdx >= len(values) { + return nil, fmt.Errorf("header_row_no %d exceeds number of rows", headerRowNo) + } + + rawHeaders := rowToStrings(values[headerIdx]) + headers := sanitizeHeaders(rawHeaders) + + types := make([]string, len(headers)) + for i := range types { + types[i] = "TEXT" + } + + var dataRows []interface{} + for i := headerIdx + 1; i < len(values); i++ { + row := rowToStrings(values[i]) + if isAllEmpty(row) { + continue + } + rowMap := make(map[string]interface{}) + for j := range headers { + val := "" + if j < len(row) { + val = strings.TrimSpace(row[j]) + } + types[j] = tightenTypeGuess(types[j], val) + rowMap[headers[j]] = val + } + dataRows = append(dataRows, rowMap) + } + + if len(dataRows) == 0 { + return nil, fmt.Errorf("no non-empty rows to insert") + } + + plugins.HandleGoogleSheetColumns(params.DatasetContext, headers, types) + + tableName := params.StorageClient.GetTableName(params.DatasetContext.Name) + gen := params.StorageClient.GetSQLGenerator() + + createStmt, err := gen.GenerateCreateTableSQL(tableName, params.DatasetContext.Columns) + if err != nil { + return nil, fmt.Errorf("failed to generate CREATE TABLE: %w", err) + } + if result := params.StorageClient.Exec(createStmt); result.Error != nil { + return nil, fmt.Errorf("failed to exec CREATE: %w", result.Error) + } + + insertStmt, err := gen.GenerateInsertSQL(tableName, dataRows, params.DatasetContext.Columns, params.DatasetContext.Dedupe) + if err != nil { + return nil, fmt.Errorf("failed to generate INSERT: %w", err) + } + if result := params.StorageClient.Exec(insertStmt); result.Error != nil { + return nil, fmt.Errorf("failed to exec INSERT: %w", result.Error) + } + + return &plugins.ProcessedData{ + TableName: tableName, + Columns: params.DatasetContext.Columns, + RowCount: int64(len(dataRows)), + }, nil +} + +func processParsedData(params plugins.PluginProcessParams, headerRowNo int, values [][]interface{}) (*plugins.ProcessedData, error) { + if len(values) == 0 { + return nil, fmt.Errorf("no rows returned from Google Sheet") + } + headerIdx := headerRowNo - 1 + if headerIdx >= len(values) { + return nil, fmt.Errorf("header_row_no %d exceeds number of rows", headerRowNo) + } + + rawHeaders := rowToStrings(values[headerIdx]) + headers := sanitizeHeaders(rawHeaders) + + types := make([]string, len(headers)) + for i := range types { + types[i] = "TEXT" + } + + var dataRows []interface{} + for i := headerIdx + 1; i < len(values); i++ { + row := rowToStrings(values[i]) + if isAllEmpty(row) { + continue + } + rowMap := make(map[string]interface{}) + for j := range headers { + val := "" + if j < len(row) { + val = strings.TrimSpace(row[j]) + } + types[j] = tightenTypeGuess(types[j], val) + rowMap[headers[j]] = val + } + dataRows = append(dataRows, rowMap) + } + + if len(dataRows) == 0 { + return nil, fmt.Errorf("no non-empty rows to insert") + } + + plugins.HandleGoogleSheetColumns(params.DatasetContext, headers, types) + + tableName := params.StorageClient.GetTableName(params.DatasetContext.Name) + gen := params.StorageClient.GetSQLGenerator() + + createStmt, err := gen.GenerateCreateTableSQL(tableName, params.DatasetContext.Columns) + if err != nil { + return nil, fmt.Errorf("failed to generate CREATE TABLE: %w", err) + } + if result := params.StorageClient.Exec(createStmt); result.Error != nil { + return nil, fmt.Errorf("failed to exec CREATE: %w", result.Error) + } + + insertStmt, err := gen.GenerateInsertSQL(tableName, dataRows, params.DatasetContext.Columns, params.DatasetContext.Dedupe) + if err != nil { + return nil, fmt.Errorf("failed to generate INSERT: %w", err) + } + if result := params.StorageClient.Exec(insertStmt); result.Error != nil { + return nil, fmt.Errorf("failed to exec INSERT: %w", result.Error) + } + + return &plugins.ProcessedData{ + TableName: tableName, + Columns: params.DatasetContext.Columns, + RowCount: int64(len(dataRows)), + }, nil +} + +func sanitizeHeaders(headers []string) []string { + clean := make([]string, len(headers)) + seen := map[string]int{} + for i, h := range headers { + h = strings.ToLower(strings.TrimSpace(h)) + h = strings.ReplaceAll(h, " ", "_") + if h == "" { + h = fmt.Sprintf("col_%d", i+1) + } + base := h + for seen[h] > 0 { + seen[base]++ + h = fmt.Sprintf("%s_%d", base, seen[base]) + } + seen[h]++ + clean[i] = h + } + return clean +} + +func tightenTypeGuess(current, val string) string { + if val == "" { + return current + } + if _, err := strconv.ParseInt(val, 10, 64); err == nil { + return mergeType(current, "INTEGER") + } + if _, err := strconv.ParseFloat(val, 64); err == nil { + return mergeType(current, "REAL") + } + if isBool(val) { + return mergeType(current, "BOOLEAN") + } + if isTimestamp(val) { + return mergeType(current, "TIMESTAMP") + } + return "TEXT" +} + +func mergeType(a, b string) string { + if a == b { + return a + } + switch { + case a == "TEXT" || b == "TEXT": + return "TEXT" + case a == "REAL" || b == "REAL": + return "REAL" + default: + return "TEXT" + } +} + +func isBool(s string) bool { + s = strings.ToLower(s) + return s == "true" || s == "false" || s == "yes" || s == "no" || s == "1" || s == "0" +} + +func isTimestamp(s string) bool { + formats := []string{ + time.RFC3339, + "2006-01-02", + "2006-01-02 15:04:05", + "02/01/2006", + } + for _, f := range formats { + if _, err := time.Parse(f, s); err == nil { + return true + } + } + return false +} + +func rowToStrings(row []interface{}) []string { + out := make([]string, len(row)) + for i, v := range row { + if v == nil { + out[i] = "" + continue + } + out[i] = fmt.Sprint(v) + } + return out +} + +func isAllEmpty(row []string) bool { + for _, v := range row { + if strings.TrimSpace(v) != "" { + return false + } + } + return true +} diff --git a/internal/plugins/gsheet/plugin_test.go b/internal/plugins/gsheet/plugin_test.go new file mode 100644 index 0000000..a2c6d17 --- /dev/null +++ b/internal/plugins/gsheet/plugin_test.go @@ -0,0 +1,621 @@ +package google_sheets + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "strings" + "testing" + "time" + + "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestGoogleSheets_ProcessData_DefaultHeaderRow(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + testCases := []struct { + name string + enableStreaming bool + }{ + {"non-streaming", false}, + {"streaming", true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + storage := mocks.NewMockStorageClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + values := [][]interface{}{ + {"ID", "Name", "Amount"}, + {1.0, "Alice", "10.5"}, + {2.0, "Bob", "7"}, + } + reader := io.NopCloser(bytes.NewReader(mustJSON(t, values))) + + tableName := "gs_table" + dataset := "dataset" + dedupe := []string{"id"} + + storage.EXPECT().GetTableName(dataset).Return(tableName) + storage.EXPECT().GetSQLGenerator().AnyTimes().Return(gen) + + gen.EXPECT(). + GenerateCreateTableSQL(tableName, gomock.Any()). + DoAndReturn(func(_ string, cols []models.EngineContextColumn) (string, error) { + var got []string + for _, c := range cols { + got = append(got, c.Name+"|"+c.Type) + } + + joined := strings.Join(got, ",") + + require.Contains(t, joined, "id|TEXT") + require.Contains(t, joined, "name|TEXT") + require.Contains(t, joined, "amount|TEXT") + + return "CREATE", nil + }) + + storage.EXPECT().Exec("CREATE").Return(&datastore.ExecResult{}) + + gen.EXPECT(). + GenerateInsertSQL(tableName, gomock.Any(), gomock.Any(), dedupe). + DoAndReturn(func(_ string, rows []interface{}, _ []models.EngineContextColumn, _ []string) (string, error) { + require.Equal(t, 2, len(rows)) + r0 := rows[0].(map[string]interface{}) + require.Equal(t, "Alice", r0["name"]) + return "INSERT", nil + }) + + storage.EXPECT().Exec("INSERT").Return(&datastore.ExecResult{}) + + p := NewGoogleSheetsDataPlugin() + res, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: dataset, + DatasetOptions: map[string]interface{}{ + "enable_streaming": tc.enableStreaming, + "dedupe": ToInterfaceSlice(dedupe), + }, + }, + }) + + require.NoError(t, err) + require.Equal(t, tableName, res.TableName) + require.Equal(t, int64(2), res.RowCount) + require.NotEmpty(t, res.Columns) + }) + } +} + +func TestGoogleSheets_ProcessData_CustomHeaderRow(t *testing.T) { + testCases := []struct { + name string + enableStreaming bool + }{ + {"non-streaming", false}, + {"streaming", true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + storage := mocks.NewMockStorageClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + values := [][]interface{}{ + {"skip", "me"}, + {"ID", "When", "Flag"}, + {1, "2024-07-01", "true"}, + } + reader := io.NopCloser(bytes.NewReader(mustJSON(t, values))) + + tableName := "gs_table2" + dataset := "ds2" + + storage.EXPECT().GetTableName(dataset).Return(tableName) + storage.EXPECT().GetSQLGenerator().Return(gen) + + gen.EXPECT().GenerateCreateTableSQL(tableName, gomock.Any()).DoAndReturn( + func(_ string, cols []models.EngineContextColumn) (string, error) { + m := map[string]string{} + for _, c := range cols { + m[c.Name] = c.Type + } + t.Logf("Inferred types: %#v", m) + + assert.Contains(t, []string{"INTEGER", "REAL", "TEXT"}, m["id"], "id column should be numeric or fallback to text") + assert.Contains(t, []string{"TIMESTAMP", "TEXT"}, m["when"], "when column should be timestamp or text") + assert.Contains(t, []string{"BOOLEAN", "TEXT"}, m["flag"], "flag column should be boolean or text") + return "CREATE", nil + }) + + storage.EXPECT().Exec("CREATE").Return(&datastore.ExecResult{}) + gen.EXPECT().GenerateInsertSQL(tableName, gomock.Any(), gomock.Any(), gomock.Any()).Return("INSERT", nil) + storage.EXPECT().Exec("INSERT").Return(&datastore.ExecResult{}) + + p := NewGoogleSheetsDataPlugin() + res, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: dataset, + DatasetOptions: map[string]interface{}{ + "headerRowNo": 2, + "enableStreaming": tc.enableStreaming, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, int64(1), res.RowCount) + }) + } +} + +func TestGoogleSheets_ProcessData_HeaderSanitizationAndDupes(t *testing.T) { + testCases := []struct { + name string + enableStreaming bool + }{ + {"non-streaming", false}, + {"streaming", true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + storage := mocks.NewMockStorageClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + values := [][]interface{}{ + {" Full Name ", "", "full-name", "FULL NAME"}, + {"Alice", "x", "y", "z"}, + } + reader := io.NopCloser(bytes.NewReader(mustJSON(t, values))) + + table := "gs_table3" + ds := "ds3" + + storage.EXPECT().GetTableName(ds).Return(table) + storage.EXPECT().GetSQLGenerator().Return(gen) + + gen.EXPECT(). + GenerateCreateTableSQL(table, gomock.Any()). + DoAndReturn(func(_ string, cols []models.EngineContextColumn) (string, error) { + var names []string + for _, c := range cols { + names = append(names, c.Name) + } + // Expected after sanitize + de-dupe: + // "full_name", "col_2", "full-name", "full_name_2" + require.Contains(t, names, "full_name") + require.Contains(t, names, "col_2") + require.Contains(t, names, "full-name") + require.Contains(t, names, "full_name_2") + return "CREATE", nil + }) + + storage.EXPECT().Exec("CREATE").Return(&datastore.ExecResult{}) + + gen.EXPECT(). + GenerateInsertSQL(table, gomock.Any(), gomock.Any(), gomock.Any()). + Return("INSERT", nil) + storage.EXPECT().Exec("INSERT").Return(&datastore.ExecResult{}) + + p := NewGoogleSheetsDataPlugin() + _, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: ds, + DatasetOptions: map[string]interface{}{ + "enableStreaming": tc.enableStreaming, + }, + }, + }) + require.NoError(t, err) + }) + } +} + +func TestGoogleSheets_ProcessData_SkipsAllEmptyRows(t *testing.T) { + testCases := []struct { + name string + enableStreaming bool + }{ + {"non-streaming", false}, + {"streaming", true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + storage := mocks.NewMockStorageClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + values := [][]interface{}{ + {"id", "val"}, + {"", " "}, // empty row + {nil, nil}, // empty row + {1.0, "ok"}, // one valid + } + reader := io.NopCloser(bytes.NewReader(mustJSON(t, values))) + + table := "gs_table4" + ds := "ds4" + + storage.EXPECT().GetTableName(ds).Return(table) + storage.EXPECT().GetSQLGenerator().Return(gen) + gen.EXPECT().GenerateCreateTableSQL(table, gomock.Any()).Return("CREATE", nil) + storage.EXPECT().Exec("CREATE").Return(&datastore.ExecResult{}) + gen.EXPECT().GenerateInsertSQL(table, gomock.Any(), gomock.Any(), gomock.Any()).Return("INSERT", nil) + storage.EXPECT().Exec("INSERT").Return(&datastore.ExecResult{}) + + p := NewGoogleSheetsDataPlugin() + res, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: ds, + DatasetOptions: map[string]interface{}{ + "enableStreaming": tc.enableStreaming, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, int64(1), res.RowCount) + }) + } +} + +func TestGoogleSheets_ProcessData_Errors(t *testing.T) { + streamModes := []struct { + name string + enableStreaming bool + }{ + {"non-streaming", false}, + {"streaming", true}, + } + + for _, mode := range streamModes { + t.Run(mode.name+"/bad JSON", func(t *testing.T) { + p := NewGoogleSheetsDataPlugin() + reader := io.NopCloser(strings.NewReader(`not-json`)) + _, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + DatasetContext: &models.EngineContextDataset{ + Name: "x", + DatasetOptions: map[string]interface{}{ + "enableStreaming": mode.enableStreaming, + }, + }, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to parse Google Sheets data") + }) + + t.Run(mode.name+"/empty sheet", func(t *testing.T) { + p := NewGoogleSheetsDataPlugin() + reader := io.NopCloser(bytes.NewReader(mustJSON(t, [][]interface{}{}))) + _, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + DatasetContext: &models.EngineContextDataset{ + Name: "x", + DatasetOptions: map[string]interface{}{ + "enableStreaming": mode.enableStreaming, + }, + }, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "no rows returned") + }) + + t.Run(mode.name+"/header row out of range", func(t *testing.T) { + p := NewGoogleSheetsDataPlugin() + reader := io.NopCloser(bytes.NewReader(mustJSON(t, [][]interface{}{{"a"}}))) + _, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + DatasetContext: &models.EngineContextDataset{ + Name: "x", + DatasetOptions: map[string]interface{}{ + "enableStreaming": mode.enableStreaming, + "headerRowNo": 3, + }, + }, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "exceeds number of rows") + }) + + t.Run(mode.name+"/create table generator error", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + storage := mocks.NewMockStorageClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + values := [][]interface{}{ + {"a"}, + {"b"}, + } + reader := io.NopCloser(bytes.NewReader(mustJSON(t, values))) + + storage.EXPECT().GetTableName("ds").Return("t") + storage.EXPECT().GetSQLGenerator().Return(gen) + gen.EXPECT().GenerateCreateTableSQL("t", gomock.Any()).Return("", fmt.Errorf("boom")) + + p := NewGoogleSheetsDataPlugin() + _, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: "ds", + DatasetOptions: map[string]interface{}{ + "enableStreaming": mode.enableStreaming, + }, + }, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to generate CREATE TABLE") + }) + + t.Run(mode.name+"/create table exec error", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + storage := mocks.NewMockStorageClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + values := [][]interface{}{ + {"a"}, + {"b"}, + } + reader := io.NopCloser(bytes.NewReader(mustJSON(t, values))) + + storage.EXPECT().GetTableName("ds").Return("t") + storage.EXPECT().GetSQLGenerator().Return(gen) + gen.EXPECT().GenerateCreateTableSQL("t", gomock.Any()).Return("CREATE", nil) + storage.EXPECT().Exec("CREATE").Return(&datastore.ExecResult{Error: fmt.Errorf("db fail")}) + + p := NewGoogleSheetsDataPlugin() + _, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: "ds", + DatasetOptions: map[string]interface{}{ + "enableStreaming": mode.enableStreaming, + }, + }, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to exec CREATE") + }) + + t.Run(mode.name+"/insert generator error", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + storage := mocks.NewMockStorageClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + values := [][]interface{}{ + {"a"}, + {"b"}, + } + reader := io.NopCloser(bytes.NewReader(mustJSON(t, values))) + + storage.EXPECT().GetTableName("ds").Return("t") + storage.EXPECT().GetSQLGenerator().Return(gen) + gen.EXPECT().GenerateCreateTableSQL("t", gomock.Any()).Return("CREATE", nil) + storage.EXPECT().Exec("CREATE").Return(&datastore.ExecResult{}) + gen.EXPECT().GenerateInsertSQL("t", gomock.Any(), gomock.Any(), gomock.Any()).Return("", fmt.Errorf("boom")) + + p := NewGoogleSheetsDataPlugin() + _, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: "ds", + DatasetOptions: map[string]interface{}{ + "googlesheet": map[string]interface{}{ + "enableStreaming": mode.enableStreaming, + }, + }, + }, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to generate INSERT") + }) + + t.Run(mode.name+"/insert exec error", func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + storage := mocks.NewMockStorageClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + values := [][]interface{}{ + {"a"}, + {"b"}, + } + reader := io.NopCloser(bytes.NewReader(mustJSON(t, values))) + + storage.EXPECT().GetTableName("ds").Return("t") + storage.EXPECT().GetSQLGenerator().Return(gen) + gen.EXPECT().GenerateCreateTableSQL("t", gomock.Any()).Return("CREATE", nil) + storage.EXPECT().Exec("CREATE").Return(&datastore.ExecResult{}) + gen.EXPECT().GenerateInsertSQL("t", gomock.Any(), gomock.Any(), gomock.Any()).Return("INSERT", nil) + storage.EXPECT().Exec("INSERT").Return(&datastore.ExecResult{Error: fmt.Errorf("db fail")}) + + p := NewGoogleSheetsDataPlugin() + _, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: "ds", + DatasetOptions: map[string]interface{}{ + "enableStreaming": mode.enableStreaming, + }, + }, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to exec INSERT") + }) + } +} + +func Test_tightenTypeGuess(t *testing.T) { + // empty keeps current + require.Equal(t, "TEXT", tightenTypeGuess("TEXT", "")) + // ints + require.Equal(t, "TEXT", tightenTypeGuess("TEXT", "42")) + // floats + require.Equal(t, "REAL", tightenTypeGuess("INTEGER", "3.14")) + // booleans + require.Equal(t, "BOOLEAN", tightenTypeGuess("BOOLEAN", "TRUE")) + require.Equal(t, "BOOLEAN", tightenTypeGuess("BOOLEAN", "no")) + // timestamps + require.True(t, isTimestamp("2006-01-02")) + require.True(t, isTimestamp(time.Now().Format(time.RFC3339))) + // mixing types gravitates to TEXT/REAL rules via mergeType + require.Equal(t, "TEXT", mergeType("BOOLEAN", "TEXT")) + require.Equal(t, "REAL", mergeType("INTEGER", "REAL")) + require.Equal(t, "TEXT", mergeType("TIMESTAMP", "TEXT")) +} + +func BenchmarkGoogleSheets_NonStreaming(b *testing.B) { + ctrl := gomock.NewController(b) + defer ctrl.Finish() + storage := mocks.NewMockStorageClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + table := "bench_sheet" + storage.EXPECT().GetTableName(gomock.Any()).AnyTimes().Return(table) + storage.EXPECT().GetSQLGenerator().AnyTimes().Return(gen) + gen.EXPECT().GenerateCreateTableSQL(gomock.Any(), gomock.Any()).AnyTimes().Return("CREATE", nil) + storage.EXPECT().Exec("CREATE").AnyTimes().Return(&datastore.ExecResult{}) + gen.EXPECT().GenerateInsertSQL(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return("INSERT", nil) + storage.EXPECT().Exec("INSERT").AnyTimes().Return(&datastore.ExecResult{}) + + data := generateLargeSheet(b) + plugin := NewGoogleSheetsDataPlugin() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + reader := io.NopCloser(bytes.NewReader(data)) + _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: "bench", + DatasetOptions: map[string]interface{}{ + "enableStreaming": false, + }, + Dedupe: []string{"col_1"}, + }, + }) + if err != nil { + b.Fatalf("failed: %v", err) + } + } +} + +func BenchmarkGoogleSheets_Streaming(b *testing.B) { + ctrl := gomock.NewController(b) + defer ctrl.Finish() + + storage := mocks.NewMockStorageClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + table := "bench_sheet" + storage.EXPECT().GetTableName(gomock.Any()).AnyTimes().Return(table) + storage.EXPECT().GetSQLGenerator().AnyTimes().Return(gen) + gen.EXPECT().GenerateCreateTableSQL(gomock.Any(), gomock.Any()).AnyTimes().Return("CREATE", nil) + storage.EXPECT().Exec("CREATE").AnyTimes().Return(&datastore.ExecResult{}) + gen.EXPECT().GenerateInsertSQL(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return("INSERT", nil) + storage.EXPECT().Exec("INSERT").AnyTimes().Return(&datastore.ExecResult{}) + + data := generateLargeSheet(b) + plugin := NewGoogleSheetsDataPlugin() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + reader := io.NopCloser(bytes.NewReader(data)) + _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: "bench", + DatasetOptions: map[string]interface{}{ + "enableStreaming": false, + }, + Dedupe: []string{"col_1"}, + }, + }) + if err != nil { + b.Fatalf("streaming failed: %v", err) + } + } +} + +func generateLargeSheet(tb testing.TB) []byte { + const rows, cols = 5000, 10 // you can change these for different benchmark sizes + + values := make([][]interface{}, rows+1) // +1 for header + header := make([]interface{}, cols) + for j := 0; j < cols; j++ { + header[j] = fmt.Sprintf("Col %d", j+1) + } + values[0] = header + + for i := 1; i <= rows; i++ { + r := make([]interface{}, cols) + for j := 0; j < cols; j++ { + switch j % 4 { + case 0: + r[j] = i // numeric (float64) + case 1: + r[j] = fmt.Sprintf("val-%d", i) // string + case 2: + r[j] = (i%2 == 0) // boolean + case 3: + r[j] = "2006-01-02" // date string + } + } + values[i] = r + } + + data, err := json.Marshal(values) + require.NoError(tb, err) + return data +} + +func mustJSON(t testing.TB, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + require.NoError(t, err) + return b +} + +func ToInterfaceSlice(ss []string) []interface{} { + out := make([]interface{}, len(ss)) + for i, v := range ss { + out[i] = v + } + return out +} diff --git a/internal/plugins/json/plugin.go b/internal/plugins/json/plugin.go new file mode 100644 index 0000000..0826f74 --- /dev/null +++ b/internal/plugins/json/plugin.go @@ -0,0 +1,293 @@ +package json + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strings" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + pluginopts "github.com/kvatch-hub/kvatch-runtime/internal/plugins/options" +) + +// JSONDataPlugin processes JSON data from connections +type JSONDataPlugin struct{} + +func NewJSONDataPlugin() *JSONDataPlugin { + return &JSONDataPlugin{} +} + +func (p *JSONDataPlugin) GetType() plugins.PluginType { + return plugins.PluginTypeJSON +} + +func (p *JSONDataPlugin) ProcessData(ctx context.Context, params plugins.PluginProcessParams) (*plugins.ProcessedData, error) { + opts, err := pluginopts.ParseJSONOptions(params.DatasetContext.DatasetOptions) + if err != nil { + return nil, fmt.Errorf("failed to parse json options: %w", err) + } + + params.DatasetContext.Dedupe = opts.Dedupe + + data, err := io.ReadAll(params.DataReader) + if err != nil { + return nil, fmt.Errorf("failed to read JSON data: %v", err) + } + defer func() { + _ = params.DataReader.Close() + }() + + var parsed interface{} + if err := json.Unmarshal(data, &parsed); err != nil { + return nil, fmt.Errorf("failed to decode JSON: %v", err) + } + + query := "" + if opts.Query != nil { + query = *opts.Query + } + + key := extractKeyFromQuery(query) + if key == "" { + key = "$" + } + + raw, err := extractRawFromParsed(parsed, key) + if err != nil { + return nil, err + } + + dataArray, err := normalize(raw, *opts) + if err != nil { + return nil, err + } + + if len(dataArray) == 0 { + return &plugins.ProcessedData{ + TableName: params.StorageClient.GetTableName(params.DatasetContext.Name), + Columns: []models.EngineContextColumn{}, + RowCount: 0, + }, nil + } + + var flattenedData []map[string]interface{} + if opts.FlattenNestedObjects { + flattenedData = make([]map[string]interface{}, len(dataArray)) + for i, row := range dataArray { + flattenedData[i] = flattenMap(row, "") + } + } else { + flattenedData = dataArray + } + + firstRow := flattenedData[0] + var columns []models.EngineContextColumn + for k, v := range firstRow { + columns = append(columns, models.EngineContextColumn{ + Name: k, + Type: inferJSONType(v), + Description: fmt.Sprintf("JSON field: %s", k), + }) + } + + tableName := params.StorageClient.GetTableName(params.DatasetContext.Name) + sqlGen := params.StorageClient.GetSQLGenerator() + + createStmt, err := sqlGen.GenerateCreateTableSQL(tableName, columns) + if err != nil { + return nil, fmt.Errorf("failed to generate CREATE TABLE: %v", err) + } + if result := params.StorageClient.Exec(createStmt); result.Error != nil { + return nil, fmt.Errorf("failed to create table: %v", result.Error) + } + + insertStmt, err := sqlGen.GenerateInsertSQL(tableName, dataToGeneric(flattenedData), columns, params.DatasetContext.Dedupe) + if err != nil { + return nil, fmt.Errorf("failed to generate INSERT: %v", err) + } + if result := params.StorageClient.Exec(insertStmt); result.Error != nil { + return nil, fmt.Errorf("failed to insert data: %v", result.Error) + } + + return &plugins.ProcessedData{ + TableName: tableName, + Columns: columns, + RowCount: int64(len(dataArray)), + }, nil +} + +func dataToGeneric(arr []map[string]interface{}) []interface{} { + out := make([]interface{}, len(arr)) + for i, row := range arr { + out[i] = row + } + return out +} + +func extractRawFromParsed(parsed interface{}, key string) (interface{}, error) { + if key == "$" { + return parsed, nil + } + rootMap, ok := parsed.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("JSON root must be an object to extract key '%s'", key) + } + raw, found := rootMap[key] + if !found { + return nil, fmt.Errorf("key '%s' not found in JSON root", key) + } + return raw, nil +} + +func normalize(raw interface{}, opts entities.JSONDatasetOptions) ([]map[string]interface{}, error) { + switch v := raw.(type) { + case []interface{}: + var arr []map[string]interface{} + for _, item := range v { + obj, ok := item.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("array item is not an object: %T", item) + } + arr = append(arr, obj) + } + return arr, nil + case map[string]interface{}: + // Normalize if enabled and inner values are objects or primitives + if opts.NormalizeNestedObjects { + keyField := opts.NormalizedKeyFieldName + if keyField == "" { + keyField = "key" + } + prefix := opts.NormalizedValuePrefix + if prefix == "" { + prefix = "value_" + } + + var arr []map[string]interface{} + for k, val := range v { + switch inner := val.(type) { + case map[string]interface{}: + // Normal object case + row := map[string]interface{}{keyField: k} + for innerK, innerV := range inner { + row[prefix+innerK] = innerV + } + arr = append(arr, row) + default: + // Primitive value (string, float, int, etc.) + row := map[string]interface{}{ + keyField: k, + prefix + "value": inner, + } + arr = append(arr, row) + } + } + return arr, nil + } + return []map[string]interface{}{v}, nil + // case map[string]interface{}: + // // Normalize if enabled and inner values are objects + // if opts.NormalizeNestedObjects { + // keyField := opts.NormalizedKeyFieldName + // if keyField == "" { + // keyField = "key" + // } + // prefix := opts.NormalizedValuePrefix + + // var arr []map[string]interface{} + // for k, val := range v { + // inner, ok := val.(map[string]interface{}) + // if !ok { + // return nil, fmt.Errorf("expected map[string]interface{} for '%s', got %T", k, val) + // } + // row := map[string]interface{}{keyField: k} + // for innerK, innerV := range inner { + // row[prefix+innerK] = innerV + // } + // arr = append(arr, row) + // } + // return arr, nil + // } + // return []map[string]interface{}{v}, nil + default: + return nil, fmt.Errorf("unexpected type: %T", raw) + } +} + +func inferJSONType(value interface{}) string { + switch value.(type) { + case string: + return "TEXT" + case float64, int, int64: + return "NUMERIC" + case bool: + return "BOOLEAN" + case nil: + return "TEXT" + default: + return "TEXT" + } +} + +// func extractKeyFromQuery(query string) string { +// q := strings.TrimSpace(strings.ToLower(query)) +// q = strings.TrimSuffix(q, ";") + +// // Handle special "$" syntax for root-level document +// if q == "$" { +// return "$" +// } + +// prefix := "select * from " +// if strings.HasPrefix(q, prefix) { +// return strings.TrimSpace(q[len(prefix):]) +// } + +// return "" +// } +func extractKeyFromQuery(query string) string { + q := strings.TrimSpace(strings.ToLower(query)) + q = strings.TrimSuffix(q, ";") + + // Handle special "$" syntax for root-level document + if q == "" || q == "$" { + return "$" + } + + // Handle JSONPath style queries like "$.rates" or "$.foo.bar" + if strings.HasPrefix(query, "$.") { + return query + } + + // Handle SELECT * FROM + prefix := "select * from " + if strings.HasPrefix(q, prefix) { + return strings.TrimSpace(query[len(prefix):]) + } + + return "" +} + +func flattenMap(data map[string]interface{}, prefix string) map[string]interface{} { + flat := make(map[string]interface{}) + for k, v := range data { + key := k + if prefix != "" { + key = prefix + "." + k + } + + switch val := v.(type) { + case map[string]interface{}: + nested := flattenMap(val, key) + for nk, nv := range nested { + flat[nk] = nv + } + default: + flat[key] = val + } + } + return flat +} diff --git a/internal/plugins/json/plugin_test.go b/internal/plugins/json/plugin_test.go new file mode 100644 index 0000000..3386855 --- /dev/null +++ b/internal/plugins/json/plugin_test.go @@ -0,0 +1,544 @@ +// Let's create the test and benchmark scaffold for your JSON plugin. + +package json + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins/mocks" +) + +func TestJSONPlugin_ProcessData(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + storage := mocks.NewMockStorageClient(ctrl) + generator := mocks.NewMockSQLGenerator(ctrl) + + jsonStr := `{ + "data": [ + {"id": 1, "name": "Alice"}, + {"id": 2, "name": "Bob"} + ] + }` + + reader := io.NopCloser(bytes.NewReader([]byte(jsonStr))) + tableName := "test_table" + dedupe := []string{"id"} + + storage.EXPECT().GetTableName("test_dataset").Return(tableName) + storage.EXPECT().GetSQLGenerator().Return(generator) + + generator.EXPECT().GenerateCreateTableSQL(tableName, gomock.Any()).Return("CREATE TABLE", nil) + storage.EXPECT().Exec("CREATE TABLE").Return(&datastore.ExecResult{}) + + generator.EXPECT().GenerateInsertSQL(tableName, gomock.Any(), gomock.Any(), dedupe).Return("INSERT", nil) + storage.EXPECT().Exec("INSERT").Return(&datastore.ExecResult{}) + + plugin := NewJSONDataPlugin() + res, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: "test_dataset", + DatasetOptions: map[string]interface{}{ + "query": "SELECT * FROM data", + "dedupe": ToInterfaceSlice(dedupe), + }, + }, + }) + + require.NoError(t, err) + require.Equal(t, tableName, res.TableName) + require.Equal(t, int64(2), res.RowCount) + require.Len(t, res.Columns, 2) +} + +func TestJSONPlugin_ProcessData_Errors(t *testing.T) { + tests := []struct { + name string + jsonInput string + query string + wantErrMsg string + }{ + { + name: "invalid JSON syntax", + jsonInput: `{"data": [ { "id": 1, } ]}`, // trailing comma + query: "SELECT * FROM data", + wantErrMsg: "failed to decode JSON", + }, + { + name: "missing key in root", + jsonInput: `{"users": []}`, + query: "SELECT * FROM data", + wantErrMsg: "key 'data' not found", + }, + { + name: "root is array, not object", + jsonInput: `[{"id":1}]`, + query: "SELECT * FROM data", + wantErrMsg: "JSON root must be an object", + }, + { + name: "array item not object", + jsonInput: `{"data": [1, 2, 3]}`, + query: "SELECT * FROM data", + wantErrMsg: "array item is not an object", + }, + { + name: "value is string not array or object", + jsonInput: `{"data": "not a struct"}`, + query: "SELECT * FROM data", + wantErrMsg: "unexpected type: string", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + plugin := NewJSONDataPlugin() + reader := io.NopCloser(bytes.NewReader([]byte(tc.jsonInput))) + + _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: nil, // won't be used for these failures + DatasetContext: &models.EngineContextDataset{ + Name: "test_dataset", + DatasetOptions: map[string]interface{}{ + "query": tc.query, + "dedupe": nil, + }, + }, + }) + + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErrMsg) + }) + } +} + +func TestJSONPlugin_ProcessData_NestedObjectFlattening(t *testing.T) { + cases := []struct { + name string + jsonStr string + expectCols []string + expectRow map[string]interface{} + }{ + { + name: "nested object", + jsonStr: `{ + "data": [ + { + "id": 1, + "name": { "first": "Alice", "last": "Smith" }, + "contact": { "email": "alice@example.com", "phone": "1234567890" } + } + ] + }`, + expectCols: []string{"id", "name.first", "name.last", "contact.email", "contact.phone"}, + expectRow: map[string]interface{}{ + "id": float64(1), + "name.first": "Alice", + "name.last": "Smith", + "contact.email": "alice@example.com", + "contact.phone": "1234567890", + }, + }, + { + name: "deeply nested", + jsonStr: `{ + "data": [ + { + "id": 2, + "profile": { + "name": { "first": "Bob", "last": "Jones" }, + "location": { + "city": "Berlin", + "address": { "street": "Main St", "number": 123 } + } + } + } + ] + }`, + expectCols: []string{"id", "profile.name.first", "profile.name.last", "profile.location.city", "profile.location.address.street", "profile.location.address.number"}, + expectRow: map[string]interface{}{ + "id": float64(2), + "profile.name.first": "Bob", + "profile.name.last": "Jones", + "profile.location.city": "Berlin", + "profile.location.address.street": "Main St", + "profile.location.address.number": float64(123), + }, + }, + { + name: "empty nested", + jsonStr: `{ + "data": [ + { + "id": 3, + "meta": {} + } + ] + }`, + expectCols: []string{"id"}, + expectRow: map[string]interface{}{ + "id": float64(3), + }, + }, + { + name: "null values", + jsonStr: `{ + "data": [ + { + "id": 4, + "details": { "note": null, "tags": null } + } + ] + }`, + expectCols: []string{"id", "details.note", "details.tags"}, + expectRow: map[string]interface{}{ + "id": float64(4), + "details.note": nil, + "details.tags": nil, + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + storage := mocks.NewMockStorageClient(ctrl) + generator := mocks.NewMockSQLGenerator(ctrl) + plugin := NewJSONDataPlugin() + + dedupe := []string{"id"} + datasetName := "test_dataset" + tableName := "test_table" + + reader := io.NopCloser(strings.NewReader(c.jsonStr)) + + storage.EXPECT().GetTableName(datasetName).Return(tableName) + storage.EXPECT().GetSQLGenerator().Return(generator) + + generator.EXPECT().GenerateCreateTableSQL(tableName, gomock.Any()). + DoAndReturn(func(_ string, cols []models.EngineContextColumn) (string, error) { + var colNames []string + for _, col := range cols { + colNames = append(colNames, col.Name) + } + for _, expectedCol := range c.expectCols { + require.Contains(t, colNames, expectedCol) + } + return "CREATE TABLE", nil + }) + + storage.EXPECT().Exec("CREATE TABLE").Return(&datastore.ExecResult{}) + + generator.EXPECT().GenerateInsertSQL(tableName, gomock.Any(), gomock.Any(), dedupe). + DoAndReturn(func(_ string, rows []interface{}, _ []models.EngineContextColumn, _ []string) (string, error) { + require.Len(t, rows, 1) + row := rows[0].(map[string]interface{}) + for k, v := range c.expectRow { + require.Contains(t, row, k) + require.Equal(t, v, row[k]) + } + return "INSERT", nil + }) + + storage.EXPECT().Exec("INSERT").Return(&datastore.ExecResult{}) + + res, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: datasetName, + DatasetOptions: map[string]interface{}{ + "flatten_nested_objects": true, + "query": "SELECT * FROM data", + "dedupe": ToInterfaceSlice(dedupe), + }, + }, + }) + + require.NoError(t, err) + require.NotNil(t, res) + require.Equal(t, tableName, res.TableName) + require.Equal(t, int64(1), res.RowCount) + }) + } +} + +func ToInterfaceSlice(ss []string) []interface{} { + out := make([]interface{}, len(ss)) + for i, v := range ss { + out[i] = v + } + return out +} + +func BenchmarkJSONPlugin_LargeInput(b *testing.B) { + ctrl := gomock.NewController(b) + defer ctrl.Finish() + + storage := mocks.NewMockStorageClient(ctrl) + generator := mocks.NewMockSQLGenerator(ctrl) + + tableName := "bench_table" + storage.EXPECT().GetTableName(gomock.Any()).AnyTimes().Return(tableName) + storage.EXPECT().GetSQLGenerator().AnyTimes().Return(generator) + generator.EXPECT().GenerateCreateTableSQL(gomock.Any(), gomock.Any()).AnyTimes().Return("CREATE", nil) + storage.EXPECT().Exec("CREATE").AnyTimes().Return(&datastore.ExecResult{}) + generator.EXPECT().GenerateInsertSQL(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return("INSERT", nil) + storage.EXPECT().Exec("INSERT").AnyTimes().Return(&datastore.ExecResult{}) + + var bld strings.Builder + bld.WriteString(`{"data": [`) + for i := 0; i < 10000; i++ { + fmt.Fprintf(&bld, `{"id": %d, "name": "Name%d"}`, i, i) + if i < 9999 { + bld.WriteString(",") + } + } + bld.WriteString(`]}`) + + data := []byte(bld.String()) + plugin := NewJSONDataPlugin() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + reader := io.NopCloser(bytes.NewReader(data)) + _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Name: "bench_dataset", + DatasetOptions: map[string]interface{}{ + "flatten_nested_objects": true, + "query": "SELECT * FROM data", + "dedupe": ToInterfaceSlice([]string{"id"}), + }, + }, + }) + if err != nil { + b.Fatalf("failed: %v", err) + } + } +} + +func BenchmarkJSONPlugin_LargeInput_WithFlattening(b *testing.B) { + ctrl := gomock.NewController(b) + defer ctrl.Finish() + + storage := mocks.NewMockStorageClient(ctrl) + generator := mocks.NewMockSQLGenerator(ctrl) + + tableName := "bench_table_flattened" + storage.EXPECT().GetTableName(gomock.Any()).AnyTimes().Return(tableName) + storage.EXPECT().GetSQLGenerator().AnyTimes().Return(generator) + generator.EXPECT().GenerateCreateTableSQL(gomock.Any(), gomock.Any()).AnyTimes().Return("CREATE", nil) + storage.EXPECT().Exec("CREATE").AnyTimes().Return(&datastore.ExecResult{}) + generator.EXPECT().GenerateInsertSQL(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return("INSERT", nil) + storage.EXPECT().Exec("INSERT").AnyTimes().Return(&datastore.ExecResult{}) + + var bld strings.Builder + bld.WriteString(`{"data": [`) + for i := 0; i < 10000; i++ { + // Add nested fields + fmt.Fprintf(&bld, `{ + "id": %d, + "name": { "first": "Name%d", "last": "Surname%d" }, + "contact": { "email": "user%d@example.com", "phone": "1234567890" } + }`, i, i, i, i) + if i < 9999 { + bld.WriteString(",") + } + } + bld.WriteString(`]}`) + + data := []byte(bld.String()) + plugin := NewJSONDataPlugin() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + reader := io.NopCloser(bytes.NewReader(data)) + _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Dedupe: []string{"id"}, + Query: "SELECT * FROM data", + Name: "bench_dataset_flattened", + DatasetOptions: map[string]interface{}{ + "json": map[string]interface{}{ + "flatten_nested_objects": true, + }, + }, + }, + }) + if err != nil { + b.Fatalf("failed: %v", err) + } + } +} + +// func TestJSONPlugin_ProcessData_FlatteningToggle(t *testing.T) { +// ctrl := gomock.NewController(t) +// defer ctrl.Finish() + +// jsonStr := `{ +// "data": [ +// { +// "id": 1, +// "user": { +// "name": "Alice", +// "contact": { +// "email": "alice@example.com" +// } +// } +// } +// ] +// }` + +// dedupe := []string{"id"} +// dataset := "test_dataset" +// tableName := "toggle_test_table" + +// testCases := []struct { +// name string +// flatten bool +// expectedCols []string +// expectedRow map[string]interface{} +// }{ +// { +// name: "no flattening", +// flatten: false, +// expectedCols: []string{"id", "user"}, +// expectedRow: map[string]interface{}{ +// "id": float64(1), +// "user": map[string]interface{}{ +// "name": "Alice", +// "contact": map[string]interface{}{ +// "email": "alice@example.com", +// }, +// }, +// }, +// }, +// { +// name: "with flattening", +// flatten: true, +// expectedCols: []string{"id", "user.name", "user.contact.email"}, +// expectedRow: map[string]interface{}{ +// "id": float64(1), +// "user.name": "Alice", +// "user.contact.email": "alice@example.com", +// }, +// }, +// } + +// for _, tc := range testCases { +// t.Run(tc.name, func(t *testing.T) { +// storage := mocks.NewMockStorageClient(ctrl) +// generator := mocks.NewMockSQLGenerator(ctrl) +// reader := io.NopCloser(strings.NewReader(jsonStr)) + +// storage.EXPECT().GetTableName(dataset).Return(tableName) +// storage.EXPECT().GetSQLGenerator().Return(generator) + +// generator.EXPECT().GenerateCreateTableSQL(tableName, gomock.Any()). +// DoAndReturn(func(_ string, cols []models.EngineContextColumn) (string, error) { +// var colNames []string +// for _, col := range cols { +// colNames = append(colNames, col.Name) +// } +// for _, expected := range tc.expectedCols { +// require.Contains(t, colNames, expected) +// } +// return "CREATE TABLE", nil +// }) + +// storage.EXPECT().Exec("CREATE TABLE").Return(&datastore.ExecResult{}) + +// generator.EXPECT().GenerateInsertSQL(tableName, gomock.Any(), gomock.Any(), dedupe). +// DoAndReturn(func(_ string, rows []interface{}, _ []models.EngineContextColumn, _ []string) (string, error) { +// require.Len(t, rows, 1) +// row := rows[0].(map[string]interface{}) +// for k, v := range tc.expectedRow { +// require.Contains(t, row, k) +// require.Equal(t, v, row[k]) +// } +// return "INSERT", nil +// }) + +// storage.EXPECT().Exec("INSERT").Return(&datastore.ExecResult{}) + +// plugin := NewJSONDataPlugin() +// _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ +// DataReader: reader, +// StorageClient: storage, +// DatasetContext: &models.EngineContextDataset{ +// dedupe: dedupe, +// Query: "SELECT * FROM data", +// Name: dataset, +// DatasetOptions: &entities.JSONPluginOptions{ +// FlattenNestedObjects: true, +// }, +// }, +// }) +// require.NoError(t, err) +// }) +// } +// } + +func BenchmarkFlattenMap(b *testing.B) { + row := map[string]interface{}{ + "id": 1, + "name": map[string]interface{}{ + "first": "Alice", + "last": "Smith", + }, + "contact": map[string]interface{}{ + "email": "alice@example.com", + "phone": "1234567890", + }, + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = flattenMap(row, "") + } +} + +func BenchmarkFlattenMap_DeepStructure(b *testing.B) { + sample := map[string]interface{}{ + "id": 123, + "name": map[string]interface{}{ + "first": "Alice", + "last": "Smith", + }, + "contact": map[string]interface{}{ + "email": "alice@example.com", + "phone": "1234567890", + "address": map[string]interface{}{ + "street": "Main St", + "city": "Metropolis", + }, + }, + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = flattenMap(sample, "") + } +} diff --git a/internal/plugins/mocks/plugin_mocks.go b/internal/plugins/mocks/plugin_mocks.go new file mode 100644 index 0000000..1f25309 --- /dev/null +++ b/internal/plugins/mocks/plugin_mocks.go @@ -0,0 +1,434 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/plugins/plugin.go +// +// Generated by this command: +// +// mockgen -source=internal/plugins/plugin.go -destination=internal/plugins/mocks/plugin_mocks.go -package=mocks +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + sql "database/sql" + reflect "reflect" + + datastore "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + plugins "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + gomock "go.uber.org/mock/gomock" + gorm "gorm.io/gorm" +) + +// MockStorageClient is a mock of StorageClient interface. +type MockStorageClient struct { + ctrl *gomock.Controller + recorder *MockStorageClientMockRecorder + isgomock struct{} +} + +// MockStorageClientMockRecorder is the mock recorder for MockStorageClient. +type MockStorageClientMockRecorder struct { + mock *MockStorageClient +} + +// NewMockStorageClient creates a new mock instance. +func NewMockStorageClient(ctrl *gomock.Controller) *MockStorageClient { + mock := &MockStorageClient{ctrl: ctrl} + mock.recorder = &MockStorageClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStorageClient) EXPECT() *MockStorageClientMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockStorageClient) Close(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockStorageClientMockRecorder) Close(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockStorageClient)(nil).Close), ctx) +} + +// Exec mocks base method. +func (m *MockStorageClient) Exec(query string) *datastore.ExecResult { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exec", query) + ret0, _ := ret[0].(*datastore.ExecResult) + return ret0 +} + +// Exec indicates an expected call of Exec. +func (mr *MockStorageClientMockRecorder) Exec(query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exec", reflect.TypeOf((*MockStorageClient)(nil).Exec), query) +} + +// Find mocks base method. +func (m *MockStorageClient) Find(tableName string) ([]map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Find", tableName) + ret0, _ := ret[0].([]map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Find indicates an expected call of Find. +func (mr *MockStorageClientMockRecorder) Find(tableName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Find", reflect.TypeOf((*MockStorageClient)(nil).Find), tableName) +} + +// GetConnectionString mocks base method. +func (m *MockStorageClient) GetConnectionString() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConnectionString") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetConnectionString indicates an expected call of GetConnectionString. +func (mr *MockStorageClientMockRecorder) GetConnectionString() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConnectionString", reflect.TypeOf((*MockStorageClient)(nil).GetConnectionString)) +} + +// GetGormDB mocks base method. +func (m *MockStorageClient) GetGormDB() *gorm.DB { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetGormDB") + ret0, _ := ret[0].(*gorm.DB) + return ret0 +} + +// GetGormDB indicates an expected call of GetGormDB. +func (mr *MockStorageClientMockRecorder) GetGormDB() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGormDB", reflect.TypeOf((*MockStorageClient)(nil).GetGormDB)) +} + +// GetSQLGenerator mocks base method. +func (m *MockStorageClient) GetSQLGenerator() datastore.SQLGenerator { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSQLGenerator") + ret0, _ := ret[0].(datastore.SQLGenerator) + return ret0 +} + +// GetSQLGenerator indicates an expected call of GetSQLGenerator. +func (mr *MockStorageClientMockRecorder) GetSQLGenerator() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSQLGenerator", reflect.TypeOf((*MockStorageClient)(nil).GetSQLGenerator)) +} + +// GetStorageType mocks base method. +func (m *MockStorageClient) GetStorageType() datastore.StorageType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetStorageType") + ret0, _ := ret[0].(datastore.StorageType) + return ret0 +} + +// GetStorageType indicates an expected call of GetStorageType. +func (mr *MockStorageClientMockRecorder) GetStorageType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStorageType", reflect.TypeOf((*MockStorageClient)(nil).GetStorageType)) +} + +// GetTableName mocks base method. +func (m *MockStorageClient) GetTableName(datasetID string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTableName", datasetID) + ret0, _ := ret[0].(string) + return ret0 +} + +// GetTableName indicates an expected call of GetTableName. +func (mr *MockStorageClientMockRecorder) GetTableName(datasetID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTableName", reflect.TypeOf((*MockStorageClient)(nil).GetTableName), datasetID) +} + +// IsNotFoundError mocks base method. +func (m *MockStorageClient) IsNotFoundError(err error) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsNotFoundError", err) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsNotFoundError indicates an expected call of IsNotFoundError. +func (mr *MockStorageClientMockRecorder) IsNotFoundError(err any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsNotFoundError", reflect.TypeOf((*MockStorageClient)(nil).IsNotFoundError), err) +} + +// Query mocks base method. +func (m *MockStorageClient) Query(ctx context.Context, tableName string, opts datastore.QueryOptions) ([]map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Query", ctx, tableName, opts) + ret0, _ := ret[0].([]map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Query indicates an expected call of Query. +func (mr *MockStorageClientMockRecorder) Query(ctx, tableName, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Query", reflect.TypeOf((*MockStorageClient)(nil).Query), ctx, tableName, opts) +} + +// QueryContext mocks base method. +func (m *MockStorageClient) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, query} + for _, a := range args { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "QueryContext", varargs...) + ret0, _ := ret[0].(*sql.Rows) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryContext indicates an expected call of QueryContext. +func (mr *MockStorageClientMockRecorder) QueryContext(ctx, query any, args ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, query}, args...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryContext", reflect.TypeOf((*MockStorageClient)(nil).QueryContext), varargs...) +} + +// MockSQLGenerator is a mock of SQLGenerator interface. +type MockSQLGenerator struct { + ctrl *gomock.Controller + recorder *MockSQLGeneratorMockRecorder + isgomock struct{} +} + +// MockSQLGeneratorMockRecorder is the mock recorder for MockSQLGenerator. +type MockSQLGeneratorMockRecorder struct { + mock *MockSQLGenerator +} + +// NewMockSQLGenerator creates a new mock instance. +func NewMockSQLGenerator(ctrl *gomock.Controller) *MockSQLGenerator { + mock := &MockSQLGenerator{ctrl: ctrl} + mock.recorder = &MockSQLGeneratorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSQLGenerator) EXPECT() *MockSQLGeneratorMockRecorder { + return m.recorder +} + +// GenerateCreateTableSQL mocks base method. +func (m *MockSQLGenerator) GenerateCreateTableSQL(tableName string, schema []models.EngineContextColumn) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateCreateTableSQL", tableName, schema) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GenerateCreateTableSQL indicates an expected call of GenerateCreateTableSQL. +func (mr *MockSQLGeneratorMockRecorder) GenerateCreateTableSQL(tableName, schema any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateCreateTableSQL", reflect.TypeOf((*MockSQLGenerator)(nil).GenerateCreateTableSQL), tableName, schema) +} + +// GenerateInsertSQL mocks base method. +func (m *MockSQLGenerator) GenerateInsertSQL(tableName string, data []any, columns []models.EngineContextColumn, dedupe []string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateInsertSQL", tableName, data, columns, dedupe) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GenerateInsertSQL indicates an expected call of GenerateInsertSQL. +func (mr *MockSQLGeneratorMockRecorder) GenerateInsertSQL(tableName, data, columns, dedupe any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateInsertSQL", reflect.TypeOf((*MockSQLGenerator)(nil).GenerateInsertSQL), tableName, data, columns, dedupe) +} + +// MockStorageManager is a mock of StorageManager interface. +type MockStorageManager struct { + ctrl *gomock.Controller + recorder *MockStorageManagerMockRecorder + isgomock struct{} +} + +// MockStorageManagerMockRecorder is the mock recorder for MockStorageManager. +type MockStorageManagerMockRecorder struct { + mock *MockStorageManager +} + +// NewMockStorageManager creates a new mock instance. +func NewMockStorageManager(ctrl *gomock.Controller) *MockStorageManager { + mock := &MockStorageManager{ctrl: ctrl} + mock.recorder = &MockStorageManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStorageManager) EXPECT() *MockStorageManagerMockRecorder { + return m.recorder +} + +// EnsureTable mocks base method. +func (m *MockStorageManager) EnsureTable(ctx context.Context, table string, columns []models.EngineContextColumn, dedupe []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureTable", ctx, table, columns, dedupe) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureTable indicates an expected call of EnsureTable. +func (mr *MockStorageManagerMockRecorder) EnsureTable(ctx, table, columns, dedupe any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureTable", reflect.TypeOf((*MockStorageManager)(nil).EnsureTable), ctx, table, columns, dedupe) +} + +// Insert mocks base method. +func (m *MockStorageManager) Insert(ctx context.Context, table string, data []map[string]any, columns []models.EngineContextColumn, dedupe []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Insert", ctx, table, data, columns, dedupe) + ret0, _ := ret[0].(error) + return ret0 +} + +// Insert indicates an expected call of Insert. +func (mr *MockStorageManagerMockRecorder) Insert(ctx, table, data, columns, dedupe any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockStorageManager)(nil).Insert), ctx, table, data, columns, dedupe) +} + +// MockPlugin is a mock of Plugin interface. +type MockPlugin struct { + ctrl *gomock.Controller + recorder *MockPluginMockRecorder + isgomock struct{} +} + +// MockPluginMockRecorder is the mock recorder for MockPlugin. +type MockPluginMockRecorder struct { + mock *MockPlugin +} + +// NewMockPlugin creates a new mock instance. +func NewMockPlugin(ctrl *gomock.Controller) *MockPlugin { + mock := &MockPlugin{ctrl: ctrl} + mock.recorder = &MockPluginMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPlugin) EXPECT() *MockPluginMockRecorder { + return m.recorder +} + +// GetType mocks base method. +func (m *MockPlugin) GetType() plugins.PluginType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetType") + ret0, _ := ret[0].(plugins.PluginType) + return ret0 +} + +// GetType indicates an expected call of GetType. +func (mr *MockPluginMockRecorder) GetType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetType", reflect.TypeOf((*MockPlugin)(nil).GetType)) +} + +// ProcessData mocks base method. +func (m *MockPlugin) ProcessData(ctx context.Context, params plugins.PluginProcessParams) (*plugins.ProcessedData, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ProcessData", ctx, params) + ret0, _ := ret[0].(*plugins.ProcessedData) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ProcessData indicates an expected call of ProcessData. +func (mr *MockPluginMockRecorder) ProcessData(ctx, params any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessData", reflect.TypeOf((*MockPlugin)(nil).ProcessData), ctx, params) +} + +// MockDataPluginRegistry is a mock of DataPluginRegistry interface. +type MockDataPluginRegistry struct { + ctrl *gomock.Controller + recorder *MockDataPluginRegistryMockRecorder + isgomock struct{} +} + +// MockDataPluginRegistryMockRecorder is the mock recorder for MockDataPluginRegistry. +type MockDataPluginRegistryMockRecorder struct { + mock *MockDataPluginRegistry +} + +// NewMockDataPluginRegistry creates a new mock instance. +func NewMockDataPluginRegistry(ctrl *gomock.Controller) *MockDataPluginRegistry { + mock := &MockDataPluginRegistry{ctrl: ctrl} + mock.recorder = &MockDataPluginRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDataPluginRegistry) EXPECT() *MockDataPluginRegistryMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockDataPluginRegistry) Get(pluginType plugins.PluginType) (plugins.Plugin, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", pluginType) + ret0, _ := ret[0].(plugins.Plugin) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockDataPluginRegistryMockRecorder) Get(pluginType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDataPluginRegistry)(nil).Get), pluginType) +} + +// List mocks base method. +func (m *MockDataPluginRegistry) List() []plugins.PluginType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List") + ret0, _ := ret[0].([]plugins.PluginType) + return ret0 +} + +// List indicates an expected call of List. +func (mr *MockDataPluginRegistryMockRecorder) List() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockDataPluginRegistry)(nil).List)) +} + +// Register mocks base method. +func (m *MockDataPluginRegistry) Register(pluginType plugins.PluginType, plugin plugins.Plugin) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Register", pluginType, plugin) + ret0, _ := ret[0].(error) + return ret0 +} + +// Register indicates an expected call of Register. +func (mr *MockDataPluginRegistryMockRecorder) Register(pluginType, plugin any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Register", reflect.TypeOf((*MockDataPluginRegistry)(nil).Register), pluginType, plugin) +} diff --git a/internal/plugins/options/api.go b/internal/plugins/options/api.go new file mode 100644 index 0000000..02cae37 --- /dev/null +++ b/internal/plugins/options/api.go @@ -0,0 +1,34 @@ +package pluginopts + +import ( + "encoding/json" + "fmt" + + "github.com/kvatch-hub/kvatch-runtime/entities" +) + +func ParseDatasetOptions(in interface{}) (*entities.APIDatasetOptions, error) { + if in == nil { + return nil, fmt.Errorf("dataset.options is required for api datasets") + } + + switch v := in.(type) { + case entities.APIDatasetOptions: + return &v, nil + case *entities.APIDatasetOptions: + return v, nil + case map[string]interface{}: + // our standard pattern: map → JSON → struct + buf, err := json.Marshal(v) + if err != nil { + return nil, fmt.Errorf("api: marshal options: %w", err) + } + var out entities.APIDatasetOptions + if err := json.Unmarshal(buf, &out); err != nil { + return nil, fmt.Errorf("api: unmarshal options: %w", err) + } + return &out, nil + default: + return nil, fmt.Errorf("api: unexpected options type %T", in) + } +} diff --git a/internal/plugins/options/csv.go b/internal/plugins/options/csv.go new file mode 100644 index 0000000..cea6c8b --- /dev/null +++ b/internal/plugins/options/csv.go @@ -0,0 +1,31 @@ +package pluginopts + +import ( + "github.com/kvatch-hub/kvatch-runtime/entities" +) + +func ConvertMapToCSVPluginOptions(m map[string]interface{}) entities.CSVDatasetOptions { + opts := entities.CSVDatasetOptions{} + if m == nil { + return opts + } + + if v, ok := Bool(m, "has_headers", "hasHeaders"); ok { + opts.HasHeaders = v + } + if v, ok := IntFromFloat(m, "skip_lines", "skipLines"); ok { + opts.SkipLines = v + } + if v, ok := String(m, "delimiter"); ok { + opts.Delimiter = v + } + if v, ok := StringSlice(m, "dedupe", "dedupe_key", "dedupeKey"); ok { + opts.Dedupe = v + } + + if opts.Delimiter == "" { + opts.Delimiter = "," + } + + return opts +} diff --git a/internal/plugins/options/extract.go b/internal/plugins/options/extract.go new file mode 100644 index 0000000..50501b0 --- /dev/null +++ b/internal/plugins/options/extract.go @@ -0,0 +1,52 @@ +package pluginopts + +func Bool(m map[string]interface{}, keys ...string) (bool, bool) { + for _, k := range keys { + if v, ok := m[k].(bool); ok { + return v, true + } + } + return false, false +} + +func String(m map[string]interface{}, keys ...string) (string, bool) { + for _, k := range keys { + if v, ok := m[k].(string); ok { + return v, true + } + } + return "", false +} + +func IntFromFloat(m map[string]interface{}, keys ...string) (int, bool) { + for _, k := range keys { + if v, ok := m[k].(float64); ok { + return int(v), true + } + } + return 0, false +} + +func Int(m map[string]interface{}, keys ...string) (int, bool) { + for _, k := range keys { + if v, ok := m[k].(int); ok { + return int(v), true + } + } + return 0, false +} + +func StringSlice(m map[string]interface{}, keys ...string) ([]string, bool) { + for _, k := range keys { + if arr, ok := m[k].([]interface{}); ok { + out := make([]string, 0, len(arr)) + for _, v := range arr { + if s, ok := v.(string); ok { + out = append(out, s) + } + } + return out, true + } + } + return nil, false +} diff --git a/internal/plugins/options/googlesheets.go b/internal/plugins/options/googlesheets.go new file mode 100644 index 0000000..af7626f --- /dev/null +++ b/internal/plugins/options/googlesheets.go @@ -0,0 +1,23 @@ +package pluginopts + +import ( + "fmt" + + "github.com/kvatch-hub/kvatch-runtime/entities" +) + +func ParseGoogleSheetOptions(raw interface{}) (*entities.GoogleSheetDatasetOptions, error) { + var out entities.GoogleSheetDatasetOptions + + // Generic conversion + if err := parseOptions(raw, &out); err != nil { + return nil, fmt.Errorf("google_sheet: %w", err) + } + + // Apply safe defaults + if out.HeaderRowNo <= 0 { + out.HeaderRowNo = 1 + } + + return &out, nil +} diff --git a/internal/plugins/options/json.go b/internal/plugins/options/json.go new file mode 100644 index 0000000..3dfe9b1 --- /dev/null +++ b/internal/plugins/options/json.go @@ -0,0 +1,18 @@ +package pluginopts + +import ( + "fmt" + + "github.com/kvatch-hub/kvatch-runtime/entities" +) + +func ParseJSONOptions(raw interface{}) (*entities.JSONDatasetOptions, error) { + var out entities.JSONDatasetOptions + + // Generic conversion + if err := parseOptions(raw, &out); err != nil { + return nil, fmt.Errorf("google_sheet: %w", err) + } + + return &out, nil +} diff --git a/internal/plugins/options/main.go b/internal/plugins/options/main.go new file mode 100644 index 0000000..e6ba04f --- /dev/null +++ b/internal/plugins/options/main.go @@ -0,0 +1,73 @@ +package pluginopts + +import ( + "encoding/json" + "fmt" + "strings" +) + +func parseOptions[T any](raw interface{}, out *T) error { + if raw == nil { + return fmt.Errorf("options are required") + } + + switch v := raw.(type) { + case T: + *out = v + return nil + case *T: + *out = *v + return nil + case map[string]interface{}: + // Normalize keys (crucial) + nv := normalizeKeys(v) + + // Re-map normalized keys into actual JSON expected shape + fixed := map[string]interface{}{} + + // Example mapping rules: + for key, val := range nv { + switch key { + case "dedupekey", "dedupe": + fixed["dedupe"] = val + case "headerrowno", "headerrow", "rowno", "headerrownum": + fixed["headerRowNo"] = val + case "enablestreaming", "streaming": + fixed["enableStreaming"] = val + case "flattennestedobjects": + fixed["flattenNestedObjects"] = val + case "normalizemestedobjects": + fixed["normalizeNestedObjects"] = val + case "normalizedkeyfieldname": + fixed["normalizedKeyFieldName"] = val + case "skiplines": + fixed["skipLines"] = val + case "hasheaders": + fixed["hasHeaders"] = val + default: + fixed[key] = val + } + } + + // JSON unmarshal into struct + buf, _ := json.Marshal(fixed) + if err := json.Unmarshal(buf, out); err != nil { + return fmt.Errorf("unmarshal into struct: %w", err) + } + + return nil + + default: + return fmt.Errorf("unexpected options type: %T", raw) + } +} + +func normalizeKeys(in map[string]interface{}) map[string]interface{} { + out := make(map[string]interface{}, len(in)) + for k, v := range in { + nk := strings.ToLower(strings.ReplaceAll(k, "_", "")) + nk = strings.ToLower(strings.ReplaceAll(nk, "-", "")) + out[nk] = v + } + return out +} diff --git a/internal/plugins/plugin.go b/internal/plugins/plugin.go new file mode 100644 index 0000000..123975c --- /dev/null +++ b/internal/plugins/plugin.go @@ -0,0 +1,142 @@ +package plugins + +import ( + "context" + "database/sql" + "io" + + "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "gorm.io/gorm" +) + +type ( + StorageClient interface { + GetTableName(datasetID string) string + Exec(query string) *datastore.ExecResult + GetSQLGenerator() datastore.SQLGenerator + Close(ctx context.Context) error + Find(tableName string) ([]map[string]interface{}, error) + GetConnectionString() string + IsNotFoundError(err error) bool + GetGormDB() *gorm.DB + Query(ctx context.Context, tableName string, opts datastore.QueryOptions) ([]map[string]interface{}, error) + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) + GetStorageType() datastore.StorageType + } + SQLGenerator interface { + GenerateCreateTableSQL(tableName string, schema []models.EngineContextColumn) (string, error) + GenerateInsertSQL(tableName string, data []interface{}, columns []models.EngineContextColumn, dedupe []string) (string, error) + } + StorageManager interface { + EnsureTable(ctx context.Context, table string, columns []models.EngineContextColumn, dedupe []string) error + Insert(ctx context.Context, table string, data []map[string]interface{}, columns []models.EngineContextColumn, dedupe []string) error + } +) + +// PluginType represents the type of data processing plugin +type PluginType string + +const ( + PluginTypeJSON PluginType = "JSON" + PluginTypeCSV PluginType = "CSV" + PluginTypeSQL PluginType = "SQL" + PluginTypeGoogleSheet PluginType = "GOOGLESHEET" + PluginTypeYAML PluginType = "YAML" + PluginTypeAPI PluginType = "API" +) + +// Plugin defines the interface for data processing plugins +// These plugins know how to translate/process data but don't handle connections +type Plugin interface { + // GetType returns the plugin type + GetType() PluginType + + // ProcessData takes raw data from a connection and processes it according to the configuration + ProcessData(ctx context.Context, params PluginProcessParams) (*ProcessedData, error) +} + +// PluginProcessParams contains the parameters needed for data processing +type PluginProcessParams struct { + DataReader io.ReadCloser + + StorageClient StorageClient + + DatasetContext *models.EngineContextDataset + + Ctx context.Context + + Manager StorageManager +} + +// ProcessedData represents the result of data processing +type ProcessedData struct { + // TableName is the name of the table where data was stored + TableName string + + // Columns describes the structure of the processed data + Columns []models.EngineContextColumn + + // RowCount indicates how many rows were processed + RowCount int64 +} + +// DataPluginRegistry manages data processing plugins +type DataPluginRegistry interface { + Register(pluginType PluginType, plugin Plugin) error + Get(pluginType PluginType) (Plugin, error) + List() []PluginType +} + +// DefaultDataPluginRegistry implements DataPluginRegistry +type DefaultDataPluginRegistry struct { + plugins map[PluginType]Plugin +} + +// NewDataPluginRegistry creates a new data plugin registry +func NewDataPluginRegistry() *DefaultDataPluginRegistry { + return &DefaultDataPluginRegistry{ + plugins: make(map[PluginType]Plugin), + } +} + +// Register registers a new data plugin +func (r *DefaultDataPluginRegistry) Register(pluginType PluginType, plugin Plugin) error { + r.plugins[pluginType] = plugin + return nil +} + +// Get retrieves a data plugin by type +func (r *DefaultDataPluginRegistry) Get(pluginType PluginType) (Plugin, error) { + plugin, exists := r.plugins[pluginType] + if !exists { + return nil, &DataPluginNotFoundError{PluginType: pluginType} + } + return plugin, nil +} + +// List returns all registered plugin types +func (r *DefaultDataPluginRegistry) List() []PluginType { + types := make([]PluginType, 0, len(r.plugins)) + for t := range r.plugins { + types = append(types, t) + } + return types +} + +// DataPluginNotFoundError represents an error when a plugin type is not found +type DataPluginNotFoundError struct { + PluginType PluginType +} + +func (e *DataPluginNotFoundError) Error() string { + return "data plugin type not found: " + string(e.PluginType) +} + +type DataStructureRequestConfig struct { + Ctx context.Context + Query string + StorageClient interface { + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) + } +} diff --git a/internal/plugins/postgres/README.md b/internal/plugins/postgres/README.md new file mode 100644 index 0000000..36ef17a --- /dev/null +++ b/internal/plugins/postgres/README.md @@ -0,0 +1,55 @@ + +# Postgres Data Plugin + +This plugin handles processing SQL query results (returned as JSON arrays) and efficiently importing them into Postgres. It supports large result sets with streaming JSON decoding and batched inserts. + +## ✅ Features + +- **Streaming JSON decoding** – avoids loading full result into memory. +- **Batch inserts** – default batch size is 500 rows. +- **Automatic schema inference** – detects column types from the first row. +- **Column ordering preservation** – ensures table column order matches input. +- **Data type detection**: + - `BOOLEAN`, `BIGINT`, `DOUBLE PRECISION` + - `TIMESTAMPTZ` via robust `time.Parse` detection + - Fallback to `TEXT` for unknowns +- **Dedupe support** – configurable deduplication via primary key-like columns. + +## ⚙️ Configuration + +Currently no required config. Plugin auto-detects structure and operates on the input JSON. + +## 🧪 Testing + +- Test coverage includes: + - Empty input + - Invalid JSON + - Column order enforcement + - Insert batching + - Duplicate key detection + - Type inference edge cases + +## 🚀 Performance + +Benchmarks show: + +- Streaming reduces memory pressure for large inputs. +- Avoids full JSON unmarshal to slice (which can be costly). +- Efficient detection of timestamp formats without regex. + +## 🛠️ Code Quality Notes + +- Column ordering is preserved by maintaining a `columnOrder []string` slice. +- Type inference has been refined for accuracy and performance. +- Removed use of regex in favor of `time.Parse` for timestamp detection. +- Mock SQL generators are injected once per test to reduce test flakiness. + +## 🧭 Future Improvements + +- [ ] Add support for configurable batch size at runtime. +- [ ] Add plugin-level config for overriding type inference. +- [ ] Support flattening of nested JSON (like the JSON plugin). +- [ ] Optional validation for required dedupe keys. +- [ ] Improve handling of inconsistent schemas (e.g., missing fields in later rows). +- [ ] Support more timestamp formats with localization/timezone awareness. +- [ ] Log insert failures with row context (for partial failure debugging). diff --git a/internal/plugins/postgres/plugin.go b/internal/plugins/postgres/plugin.go new file mode 100644 index 0000000..7bae2fe --- /dev/null +++ b/internal/plugins/postgres/plugin.go @@ -0,0 +1,148 @@ +package postgres + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/pkg/utils" +) + +type ( + StorageManager interface { + EnsureTable(ctx context.Context, table string, columns []models.EngineContextColumn, dedupe []string) error + Insert(ctx context.Context, table string, data []map[string]interface{}, columns []models.EngineContextColumn, dedupe []string) error + } + StorageClient interface { + Exec(sql string) *datastore.ExecResult + GetSQLGenerator() SQLGenerator + GetTableName(datasetName string) string + } + SQLGenerator interface { + GenerateCreateTableSQL(tableName string, schema []models.EngineContextColumn) (string, error) + GenerateInsertSQL(tableName string, data []interface{}, columns []models.EngineContextColumn, dedupe []string) (string, error) + } +) + +type PostgresDataPlugin struct { + batchSize int + Manager StorageManager +} + +func NewPostgresDataPlugin() *PostgresDataPlugin { + return &PostgresDataPlugin{ + batchSize: 500, + } +} + +func (p *PostgresDataPlugin) GetType() plugins.PluginType { + return plugins.PluginTypeSQL +} + +func (p *PostgresDataPlugin) ProcessData(ctx context.Context, params plugins.PluginProcessParams) (*plugins.ProcessedData, error) { + dec := json.NewDecoder(params.DataReader) + defer func() { + _ = params.DataReader.Close() + }() + + // Must begin with '[' + t, err := dec.Token() + if err != nil { + return nil, fmt.Errorf("invalid JSON array input: %w", err) + } + if t != json.Delim('[') { + return nil, fmt.Errorf("invalid JSON array input: expected array start") + } + + mgr := params.Manager + tableName := params.StorageClient.GetTableName(params.DatasetContext.Name) + + var batch []map[string]interface{} + rowCount := 0 + + for dec.More() { + var row map[string]interface{} + if err := dec.Decode(&row); err != nil { + return nil, fmt.Errorf("failed to decode row: %w", err) + } + + if rowCount == 0 { + plugins.HandleColumns(params.DatasetContext, row, p.inferSQLType) + + if err := mgr.EnsureTable(ctx, + tableName, + params.DatasetContext.Columns, + params.DatasetContext.Dedupe, + ); err != nil { + return nil, err + } + } + + batch = append(batch, row) + rowCount++ + + if len(batch) >= p.batchSize { + if err := mgr.Insert(ctx, + tableName, + batch, + params.DatasetContext.Columns, + params.DatasetContext.Dedupe, + ); err != nil { + return nil, err + } + batch = batch[:0] + } + } + + // Handle any remaining rows + if len(batch) > 0 { + if err := mgr.Insert(ctx, + tableName, + batch, + params.DatasetContext.Columns, + params.DatasetContext.Dedupe, + ); err != nil { + return nil, err + } + } + + if rowCount == 0 { + return nil, fmt.Errorf("no rows found in input") + } + + return &plugins.ProcessedData{ + TableName: tableName, + Columns: params.DatasetContext.Columns, + RowCount: int64(rowCount), + }, nil +} + +func (p *PostgresDataPlugin) inferSQLType(value interface{}) string { + switch v := value.(type) { + case bool: + return "BOOLEAN" + case float64: + if utils.IsWholeNumber(v) { + return "BIGINT" + } + return "DOUBLE PRECISION" + case string: + if utils.IsLikelyTimestamp(v) { + return "TIMESTAMPTZ" + } + if utils.IsLikelyNumber(v) { + if utils.IsWholeNumberString(v) { + return "BIGINT" + } + return "DOUBLE PRECISION" + } + return "TEXT" + case nil: + return "TEXT" + default: + return "TEXT" + } +} diff --git a/internal/plugins/postgres/plugin_test.go b/internal/plugins/postgres/plugin_test.go new file mode 100644 index 0000000..584acba --- /dev/null +++ b/internal/plugins/postgres/plugin_test.go @@ -0,0 +1,392 @@ +package postgres + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins/mocks" +) + +func TestPostgres_ProcessData_SimpleSuccess(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockStorageManager := mocks.NewMockStorageManager(ctrl) + + // Two rows typical of a SQL result set serialized as JSON array + payload := []map[string]any{ + {"id": 1, "name": "Alice", "active": true}, + {"id": 2, "name": "Bob", "active": false}, + } + reader := io.NopCloser(bytes.NewReader(mustJSONTB(t, payload))) + + table := "pg_table" + ds := "dataset" + dedupe := []string{"id"} + + // Expectations + mockStorageClient.EXPECT().GetTableName(ds).Return(table) + + mockStorageManager.EXPECT().EnsureTable(gomock.Any(), table, gomock.Any(), gomock.Any()).Return(nil) + + mockStorageManager.EXPECT(). + Insert( + gomock.Any(), + table, + gomock.AssignableToTypeOf([]map[string]interface{}{}), + gomock.Any(), + dedupe, + ). + Return(nil) + + p := NewPostgresDataPlugin() + out, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + DatasetContext: &models.EngineContextDataset{ + Name: ds, + Dedupe: dedupe, + }, + Manager: mockStorageManager, + }) + require.NoError(t, err) + require.Equal(t, table, out.TableName) + require.Equal(t, int64(2), out.RowCount) + require.NotEmpty(t, out.Columns) +} + +func TestPostgres_ProcessData_EmptyArray(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockStorageManager := mocks.NewMockStorageManager(ctrl) + + reader := io.NopCloser(bytes.NewReader([]byte(`[]`))) + + dsName := "empty_array" + tableName := "pg_empty_array" + + // GetTableName is still called by plugin + mockStorageClient.EXPECT().GetTableName(dsName).Return(tableName) + + p := NewPostgresDataPlugin() + + out, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + DatasetContext: &models.EngineContextDataset{Name: dsName}, + Manager: mockStorageManager, + }) + + require.Nil(t, out) + require.Error(t, err) + require.Contains(t, err.Error(), "no rows found") +} + +/* ---------- Error paths ---------- */ + +func TestPostgres_ProcessData_InvalidJSON(t *testing.T) { + p := NewPostgresDataPlugin() + reader := io.NopCloser(bytes.NewBufferString(`not-json`)) + _, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: nil, // not reached + DatasetContext: &models.EngineContextDataset{ + Name: "x", + }, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid JSON array input") +} + +func TestPostgres_ProcessData_NotJSONArray(t *testing.T) { + p := NewPostgresDataPlugin() + reader := io.NopCloser(bytes.NewBufferString(`{"id": 1}`)) // valid JSON, but not an array + _, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: nil, // not reached + DatasetContext: &models.EngineContextDataset{ + Name: "x", + }, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid JSON array input") +} + +// Need to resolve issues with ordering in the connector ordering before restoring this test +// func TestPostgres_ProcessData_EnforcesColumnOrder(t *testing.T) { +// ctrl := gomock.NewController(t) +// defer ctrl.Finish() +// storage := mocks.NewMockStorageClient(ctrl) +// gen := mocks.NewMockSQLGenerator(ctrl) + +// // Simulate an unordered map row +// row := map[string]interface{}{ +// "b": "second", +// "a": "first", +// } + +// // JSON encoding does not preserve map order +// jsonData := mustJSONTB(t, []map[string]interface{}{row}) +// reader := io.NopCloser(bytes.NewReader(jsonData)) + +// table := "pg_test" +// storage.EXPECT().GetTableName("ds1").Return(table) +// storage.EXPECT().GetSQLGenerator().Return(gen) +// gen.EXPECT().GenerateCreateTableSQL(table, gomock.Any()).Return("CREATE", nil) +// storage.EXPECT().Exec("CREATE").Return(&datastore.ExecResult{}) + +// gen.EXPECT().GenerateInsertSQL(table, gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( +// func(tableName string, data []interface{}, cols []entities.Datacolumn, dedupe []string) (string, error) { +// require.Len(t, data, 1) + +// rowData, ok := data[0].([]interface{}) +// require.True(t, ok, "row should be transformed into []interface{}") + +// // Check enforced order is ["a", "b"] +// require.Equal(t, "first", rowData[0]) +// require.Equal(t, "second", rowData[1]) + +// return "INSERT", nil +// }, +// ) + +// storage.EXPECT().Exec("INSERT").Return(&datastore.ExecResult{}) + +// plugin := NewPostgresDataPlugin() +// res, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ +// DatasetName: "ds1", +// DataReader: reader, +// StorageClient: storage, +// }) +// require.NoError(t, err) +// require.Equal(t, int64(1), res.RowCount) +// } + +func TestPostgres_ProcessData_CreateTableGenError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockStorageManager := mocks.NewMockStorageManager(ctrl) + + ds := &models.EngineContextDataset{Name: "authors"} + + reader := io.NopCloser(bytes.NewReader([]byte(`[ + {"id":1, "name":"Alice"} + ]`))) + + table := "tbl_authors" + + mockStorageClient.EXPECT().GetTableName("authors").Return(table) + + mockStorageManager.EXPECT(). + EnsureTable(gomock.Any(), table, gomock.Any(), gomock.Any()). + Return(fmt.Errorf("boom")) // fails + + // Insert should NOT be called + mockStorageManager.EXPECT().Insert(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + p := NewPostgresDataPlugin() + + out, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + DatasetContext: ds, + Manager: mockStorageManager, + }) + + require.Nil(t, out) + require.Error(t, err) + require.Contains(t, err.Error(), "boom") +} + +func TestPostgres_ProcessData_InsertFails(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockStorageManager := mocks.NewMockStorageManager(ctrl) + + reader := io.NopCloser(bytes.NewReader([]byte(`[ + {"id":1, "name":"Alice"}, + {"id":2, "name":"Bob"} + ]`))) + + ds := &models.EngineContextDataset{Name: "ds"} + table := "tbl_ds" + + mockStorageClient.EXPECT().GetTableName("ds").Return(table) + + mockStorageManager.EXPECT(). + EnsureTable(gomock.Any(), table, gomock.Any(), gomock.Any()). + Return(nil) + + mockStorageManager.EXPECT(). + Insert(gomock.Any(), table, gomock.Any(), gomock.Any(), gomock.Any()). + Return(fmt.Errorf("insert error")) + + p := NewPostgresDataPlugin() + + out, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + DatasetContext: ds, + Manager: mockStorageManager, + }) + + require.Nil(t, out) + require.Error(t, err) + require.Contains(t, err.Error(), "insert error") +} + +func TestPostgres_ProcessData_LargeBatch(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockStorageManager := mocks.NewMockStorageManager(ctrl) + + p := NewPostgresDataPlugin() + p.batchSize = 500 // required for predictable batching + + dsName := "big_dataset" + tableName := "pg_big_table" + + // Build 1005 JSON rows + var input []map[string]interface{} + for i := 0; i < 1005; i++ { + input = append(input, map[string]interface{}{ + "id": i, + "value": fmt.Sprintf("row-%d", i), + }) + } + + reader := io.NopCloser(bytes.NewReader(mustJSONTB(t, input))) + + // Expectation: plugin calls GetTableName once + mockStorageClient.EXPECT(). + GetTableName(dsName). + Return(tableName). + Times(1) + + // Expectation: EnsureTable called once (first row) + mockStorageManager.EXPECT(). + EnsureTable( + gomock.Any(), + tableName, + gomock.Any(), // inferred columns + gomock.Any(), // dedupe keys + ). + Return(nil). + Times(1) + + // Expect: Insert called 3 times: + // - batch 1: 500 rows + // - batch 2: 500 rows + // - batch 3: 5 rows + mockStorageManager.EXPECT(). + Insert( + gomock.Any(), + tableName, + gomock.Any(), // batch slice + gomock.Any(), + gomock.Any(), + ). + Return(nil). + Times(3) + + out, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + DatasetContext: &models.EngineContextDataset{ + Name: dsName, + Dedupe: []string{"id"}, + }, + Manager: mockStorageManager, + }) + + require.NoError(t, err) + require.Equal(t, tableName, out.TableName) + require.Equal(t, int64(1005), out.RowCount) + require.NotEmpty(t, out.Columns) +} + +/* ---------- Benchmarks ---------- */ + +func BenchmarkPostgres_LargeResult(b *testing.B) { + ctrl := gomock.NewController(b) + defer ctrl.Finish() + + storage := mocks.NewMockStorageClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + table := "bench_pg" + storage.EXPECT().GetTableName(gomock.Any()).AnyTimes().Return(table) + storage.EXPECT().GetSQLGenerator().AnyTimes().Return(gen) + gen.EXPECT().GenerateCreateTableSQL(gomock.Any(), gomock.Any()).AnyTimes().Return("CREATE", nil) + storage.EXPECT().Exec("CREATE").AnyTimes().Return(&datastore.ExecResult{}) + gen.EXPECT().GenerateInsertSQL(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return("INSERT", nil) + storage.EXPECT().Exec("INSERT").AnyTimes().Return(&datastore.ExecResult{}) + + // Generate 10k rows x ~8 columns + data := generateLargePostgresResult(b, 10000, 8) + plugin := NewPostgresDataPlugin() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + reader := io.NopCloser(bytes.NewReader(data)) + _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: storage, + DatasetContext: &models.EngineContextDataset{ + Dedupe: []string{"id"}, + Name: "bench", + }, + }) + if err != nil { + b.Fatalf("failed: %v", err) + } + } +} + +func generateLargePostgresResult(t testing.TB, rows, cols int) []byte { + t.Helper() + // Build rows with a mix of types + out := make([]map[string]any, rows) + for i := 0; i < rows; i++ { + row := make(map[string]any, cols) + row["id"] = i + 1 + row["name"] = fmt.Sprintf("name-%d", i+1) + row["active"] = (i%2 == 0) + row["score"] = float64(i%100) + 0.5 + row["note"] = "" + row["ts"] = "2006-01-02" // stays TEXT in plugin + row["grp"] = i % 10 + row["misc"] = nil + out[i] = row + } + return mustJSONTB(t, out) +} + +/* ---------- Helpers ---------- */ + +func mustJSONTB(t testing.TB, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + require.NoError(t, err) + return b +} diff --git a/internal/plugins/sqlite/plugin.go b/internal/plugins/sqlite/plugin.go new file mode 100644 index 0000000..fc1e20a --- /dev/null +++ b/internal/plugins/sqlite/plugin.go @@ -0,0 +1,148 @@ +package sqliteplugin + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/pkg/utils" +) + +type ( + StorageManager interface { + EnsureTable(ctx context.Context, table string, columns []models.EngineContextColumn, dedupe []string) error + Insert(ctx context.Context, table string, data []map[string]interface{}, columns []models.EngineContextColumn, dedupe []string) error + } +) + +type SQLiteDataPlugin struct { + batchSize int + Manager StorageManager +} + +func NewSQLiteDataPlugin() *SQLiteDataPlugin { + return &SQLiteDataPlugin{ + batchSize: 500, + } +} + +func (p *SQLiteDataPlugin) GetType() plugins.PluginType { + return plugins.PluginTypeSQL +} +func (p *SQLiteDataPlugin) ProcessData( + ctx context.Context, + params plugins.PluginProcessParams, +) (*plugins.ProcessedData, error) { + + dec := json.NewDecoder(params.DataReader) + defer func() { _ = params.DataReader.Close() }() + + // Must begin with '[' + tok, err := dec.Token() + if err != nil { + return nil, fmt.Errorf("invalid JSON array input: %w", err) + } + if tok != json.Delim('[') { + return nil, fmt.Errorf("invalid JSON array input: expected array start") + } + + mgr := params.Manager + tableName := params.StorageClient.GetTableName(params.DatasetContext.Name) + + var ( + batch []map[string]interface{} + rowCount int + ) + + for dec.More() { + var row map[string]interface{} + if err := dec.Decode(&row); err != nil { + return nil, fmt.Errorf("failed to decode row: %w", err) + } + + if rowCount == 0 { + plugins.HandleColumns( + params.DatasetContext, + row, + p.inferSQLiteType, + ) + + if err := mgr.EnsureTable( + ctx, + tableName, + params.DatasetContext.Columns, + params.DatasetContext.Dedupe, + ); err != nil { + return nil, err + } + } + + batch = append(batch, row) + rowCount++ + + if len(batch) >= p.batchSize { + if err := mgr.Insert( + ctx, + tableName, + batch, + params.DatasetContext.Columns, + params.DatasetContext.Dedupe, + ); err != nil { + return nil, err + } + batch = batch[:0] + } + } + + // Flush remainder + if len(batch) > 0 { + if err := mgr.Insert( + ctx, + tableName, + batch, + params.DatasetContext.Columns, + params.DatasetContext.Dedupe, + ); err != nil { + return nil, err + } + } + + if rowCount == 0 { + return nil, fmt.Errorf("no rows found in input") + } + + return &plugins.ProcessedData{ + TableName: tableName, + Columns: params.DatasetContext.Columns, + RowCount: int64(rowCount), + }, nil +} + +func (p *SQLiteDataPlugin) inferSQLiteType(value interface{}) string { + switch v := value.(type) { + case bool: + return "INTEGER" + case float64: + if utils.IsWholeNumber(v) { + return "INTEGER" + } + return "REAL" + case string: + if utils.IsLikelyTimestamp(v) { + return "TEXT" + } + if utils.IsLikelyNumber(v) { + if utils.IsWholeNumberString(v) { + return "INTEGER" + } + return "REAL" + } + return "TEXT" + case nil: + return "TEXT" + default: + return "TEXT" + } +} diff --git a/internal/plugins/sqlite/plugin_test.go b/internal/plugins/sqlite/plugin_test.go new file mode 100644 index 0000000..9ae529e --- /dev/null +++ b/internal/plugins/sqlite/plugin_test.go @@ -0,0 +1,461 @@ +package sqliteplugin + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "testing" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins/mocks" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestSQLite_ProcessData_SimpleSuccess(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockStorageManager := mocks.NewMockStorageManager(ctrl) + + // Two rows typical of a SQL result set serialized as JSON array + payload := []map[string]any{ + {"id": 1, "name": "Alice", "active": true}, + {"id": 2, "name": "Bob", "active": false}, + } + reader := io.NopCloser(bytes.NewReader(mustJSONTB(t, payload))) + + table := "pg_table" + ds := "dataset" + dedupe := []string{"id"} + + // Expectations + mockStorageClient.EXPECT().GetTableName(ds).Return(table) + + mockStorageManager.EXPECT().EnsureTable(gomock.Any(), table, gomock.Any(), gomock.Any()).Return(nil) + + mockStorageManager.EXPECT(). + Insert( + gomock.Any(), + table, + gomock.AssignableToTypeOf([]map[string]interface{}{}), + gomock.Any(), + dedupe, + ). + Return(nil) + + p := NewSQLiteDataPlugin() + out, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + DatasetContext: &models.EngineContextDataset{ + Name: ds, + Dedupe: dedupe, + }, + Manager: mockStorageManager, + }) + require.NoError(t, err) + require.Equal(t, table, out.TableName) + require.Equal(t, int64(2), out.RowCount) + require.NotEmpty(t, out.Columns) +} + +func TestSQLite_ProcessData_EmptyArray(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockStorageManager := mocks.NewMockStorageManager(ctrl) + + reader := io.NopCloser(bytes.NewReader([]byte(`[]`))) + + dsName := "empty_array" + tableName := "pg_empty_array" + + // GetTableName is still called by plugin + mockStorageClient.EXPECT().GetTableName(dsName).Return(tableName) + + p := NewSQLiteDataPlugin() + + out, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + DatasetContext: &models.EngineContextDataset{Name: dsName}, + Manager: mockStorageManager, + }) + + require.Nil(t, out) + require.Error(t, err) + require.Contains(t, err.Error(), "no rows found") +} + +func TestSQLite_ProcessData_EnsureTableFails(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockStorageManager := mocks.NewMockStorageManager(ctrl) + + ds := &models.EngineContextDataset{Name: "authors"} + + reader := io.NopCloser(bytes.NewReader([]byte(`[ + {"id":1, "name":"Alice"} + ]`))) + + table := "tbl_authors" + + mockStorageClient.EXPECT().GetTableName("authors").Return(table) + + mockStorageManager.EXPECT(). + EnsureTable(gomock.Any(), table, gomock.Any(), gomock.Any()). + Return(fmt.Errorf("boom")) // fails + + // Insert should NOT be called + mockStorageManager.EXPECT().Insert(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Times(0) + + p := NewSQLiteDataPlugin() + + out, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + DatasetContext: ds, + Manager: mockStorageManager, + }) + + require.Nil(t, out) + require.Error(t, err) + require.Contains(t, err.Error(), "boom") +} + +func TestSQLite_ProcessData_InsertFails(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockStorageManager := mocks.NewMockStorageManager(ctrl) + + reader := io.NopCloser(bytes.NewReader([]byte(`[ + {"id":1, "name":"Alice"}, + {"id":2, "name":"Bob"} + ]`))) + + ds := &models.EngineContextDataset{Name: "ds"} + table := "tbl_ds" + + mockStorageClient.EXPECT().GetTableName("ds").Return(table) + + mockStorageManager.EXPECT(). + EnsureTable(gomock.Any(), table, gomock.Any(), gomock.Any()). + Return(nil) + + mockStorageManager.EXPECT(). + Insert(gomock.Any(), table, gomock.Any(), gomock.Any(), gomock.Any()). + Return(fmt.Errorf("insert error")) + + p := NewSQLiteDataPlugin() + + out, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + DatasetContext: ds, + Manager: mockStorageManager, + }) + + require.Nil(t, out) + require.Error(t, err) + require.Contains(t, err.Error(), "insert error") +} + +func TestSQLite_ProcessData_LargeBatch(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStorageClient := mocks.NewMockStorageClient(ctrl) + mockStorageManager := mocks.NewMockStorageManager(ctrl) + + p := NewSQLiteDataPlugin() + p.batchSize = 500 // required for predictable batching + + dsName := "big_dataset" + tableName := "pg_big_table" + + // Build 1005 JSON rows + var input []map[string]interface{} + for i := 0; i < 1005; i++ { + input = append(input, map[string]interface{}{ + "id": i, + "value": fmt.Sprintf("row-%d", i), + }) + } + + reader := io.NopCloser(bytes.NewReader(mustJSONTB(t, input))) + + // Expectation: plugin calls GetTableName once + mockStorageClient.EXPECT(). + GetTableName(dsName). + Return(tableName). + Times(1) + + // Expectation: EnsureTable called once (first row) + mockStorageManager.EXPECT(). + EnsureTable( + gomock.Any(), + tableName, + gomock.Any(), // inferred columns + gomock.Any(), // dedupe keys + ). + Return(nil). + Times(1) + + // Expect: Insert called 3 times: + // - batch 1: 500 rows + // - batch 2: 500 rows + // - batch 3: 5 rows + mockStorageManager.EXPECT(). + Insert( + gomock.Any(), + tableName, + gomock.Any(), // batch slice + gomock.Any(), + gomock.Any(), + ). + Return(nil). + Times(3) + + out, err := p.ProcessData(context.Background(), plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: mockStorageClient, + DatasetContext: &models.EngineContextDataset{ + Name: dsName, + Dedupe: []string{"id"}, + }, + Manager: mockStorageManager, + }) + + require.NoError(t, err) + require.Equal(t, tableName, out.TableName) + require.Equal(t, int64(1005), out.RowCount) + require.NotEmpty(t, out.Columns) +} + +// /* ---------- Helpers ---------- */ + +func mustJSONTB(t testing.TB, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + require.NoError(t, err) + return b +} + +// func TestSQLDataPlugin_ProcessData(t *testing.T) { +// ctrl := gomock.NewController(t) +// defer ctrl.Finish() + +// mockStorage := mocks.NewMockStorageClient(ctrl) +// mockGenerator := mocks.NewMockSQLGenerator(ctrl) + +// mockStorage.EXPECT().GetTableName("test_dataset").Return("table_test") +// mockStorage.EXPECT().GetSQLGenerator().Return(mockGenerator) + +// mockGenerator.EXPECT(). +// GenerateCreateTableSQL("table_test", gomock.Any()). +// Return("CREATE TABLE stmt", nil) + +// mockStorage.EXPECT().Exec("CREATE TABLE stmt"). +// Return(&datastore.ExecResult{Error: nil}) + +// mockGenerator.EXPECT(). +// GenerateInsertSQL("table_test", gomock.Any(), gomock.Any(), nil). +// Return("INSERT stmt", nil) + +// mockStorage.EXPECT().Exec("INSERT stmt"). +// Return(&datastore.ExecResult{Error: nil}) + +// // JSON input +// data := []map[string]interface{}{ +// {"id": 1, "name": "Alice"}, +// {"id": 2, "name": "Bob"}, +// } +// buf, _ := json.Marshal(data) + +// plugin := NewSQLDataPlugin() + +// result, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ +// DataReader: io.NopCloser(bytes.NewReader(buf)), +// StorageClient: mockStorage, +// DatasetContext: &models.EngineContextDataset{ +// Name: "test_dataset", +// }, +// }) + +// assert.NoError(t, err) +// assert.Equal(t, "table_test", result.TableName) +// assert.Equal(t, int64(2), result.RowCount) +// assert.Len(t, result.Columns, 2) +// } + +// func TestSQLDataPlugin_ProcessData_Empty(t *testing.T) { +// ctrl := gomock.NewController(t) +// defer ctrl.Finish() + +// mockStorage := mocks.NewMockStorageClient(ctrl) +// mockStorage.EXPECT().GetTableName("test_dataset").Return("empty_table") + +// plugin := NewSQLDataPlugin() + +// reader := io.NopCloser(bytes.NewReader([]byte(`[]`))) + +// result, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ +// DataReader: reader, +// StorageClient: mockStorage, +// DatasetContext: &models.EngineContextDataset{ +// Name: "test_dataset", +// }, +// }) + +// assert.NoError(t, err) +// assert.Equal(t, "empty_table", result.TableName) +// assert.Equal(t, int64(0), result.RowCount) +// assert.Len(t, result.Columns, 0) +// } + +// func TestSQLDataPlugin_ProcessData_BadJSON(t *testing.T) { +// plugin := NewSQLDataPlugin() + +// reader := io.NopCloser(bytes.NewReader([]byte(`not-json`))) + +// _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ +// DataReader: reader, +// StorageClient: nil, // will fail before usage +// DatasetContext: &models.EngineContextDataset{ +// Name: "fail_dataset", +// }, +// }) + +// assert.Error(t, err) +// assert.Contains(t, err.Error(), "failed to decode SQLite result JSON array") +// } + +// func TestSQLDataPlugin_ProcessData_CreateFails(t *testing.T) { +// ctrl := gomock.NewController(t) +// defer ctrl.Finish() + +// mockStorage := mocks.NewMockStorageClient(ctrl) +// mockGenerator := mocks.NewMockSQLGenerator(ctrl) + +// mockStorage.EXPECT().GetTableName("fail_dataset").Return("fail_table") +// mockStorage.EXPECT().GetSQLGenerator().Return(mockGenerator) +// mockGenerator.EXPECT().GenerateCreateTableSQL("fail_table", gomock.Any()).Return("bad create", nil) +// mockStorage.EXPECT().Exec("bad create").Return(&datastore.ExecResult{Error: errors.New("boom")}) + +// data := []map[string]interface{}{ +// {"id": 1}, +// } +// buf, _ := json.Marshal(data) + +// plugin := NewSQLDataPlugin() + +// _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ +// DataReader: io.NopCloser(bytes.NewReader(buf)), +// StorageClient: mockStorage, +// DatasetContext: &models.EngineContextDataset{ +// Name: "fail_dataset", +// }, +// }) + +// assert.Error(t, err) +// assert.Contains(t, err.Error(), "failed to create table") +// } + +// func TestSQLDataPlugin_ProcessData_InsertFails(t *testing.T) { +// ctrl := gomock.NewController(t) +// defer ctrl.Finish() + +// mockStorage := mocks.NewMockStorageClient(ctrl) +// mockGenerator := mocks.NewMockSQLGenerator(ctrl) + +// mockStorage.EXPECT().GetTableName("fail_dataset").Return("fail_table") +// mockStorage.EXPECT().GetSQLGenerator().Return(mockGenerator) +// mockGenerator.EXPECT().GenerateCreateTableSQL("fail_table", gomock.Any()).Return("create stmt", nil) +// mockStorage.EXPECT().Exec("create stmt").Return(&datastore.ExecResult{Error: nil}) +// mockGenerator.EXPECT().GenerateInsertSQL("fail_table", gomock.Any(), gomock.Any(), nil).Return("bad insert", nil) +// mockStorage.EXPECT().Exec("bad insert").Return(&datastore.ExecResult{Error: errors.New("boom insert")}) + +// data := []map[string]interface{}{ +// {"id": 1}, +// } +// buf, _ := json.Marshal(data) + +// plugin := NewSQLDataPlugin() + +// _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ +// DataReader: io.NopCloser(bytes.NewReader(buf)), +// StorageClient: mockStorage, +// DatasetContext: &models.EngineContextDataset{ +// Name: "fail_dataset", +// }, +// }) + +// assert.Error(t, err) +// assert.Contains(t, err.Error(), "failed to insert") +// } + +// func BenchmarkSQLDataPlugin_ProcessData(b *testing.B) { +// ctrl := gomock.NewController(b) +// defer ctrl.Finish() + +// mockStorage := mocks.NewMockStorageClient(ctrl) +// mockGenerator := mocks.NewMockSQLGenerator(ctrl) + +// mockStorage.EXPECT().GetSQLGenerator().Return(mockGenerator).AnyTimes() +// mockStorage.EXPECT().GetTableName(gomock.Any()).Return("test_table").AnyTimes() + +// mockGenerator.EXPECT(). +// GenerateCreateTableSQL(gomock.Any(), gomock.Any()). +// Return("CREATE TABLE test (...)", nil). +// AnyTimes() + +// mockStorage.EXPECT(). +// Exec(gomock.Any()). +// Return(&datastore.ExecResult{Error: nil}). +// AnyTimes() + +// mockGenerator.EXPECT(). +// GenerateInsertSQL(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). +// Return("INSERT INTO test ...", nil). +// AnyTimes() + +// // Generate synthetic data +// data := make([]map[string]interface{}, 1000) +// for i := 0; i < len(data); i++ { +// data[i] = map[string]interface{}{ +// "id": i, +// "name": "row", +// "value": float64(i) * 0.1, +// } +// } +// jsonBytes, _ := json.Marshal(data) + +// plugin := NewSQLDataPlugin() + +// b.ReportAllocs() +// b.ResetTimer() + +// for i := 0; i < b.N; i++ { +// _, err := plugin.ProcessData(context.Background(), plugins.PluginProcessParams{ +// DataReader: io.NopCloser(bytes.NewReader(jsonBytes)), +// StorageClient: mockStorage, +// DatasetContext: &models.EngineContextDataset{ +// Name: "test_dataset", +// DedupeKeys: []string{}, +// }, +// }) +// if err != nil { +// b.Fatalf("failed to process data: %v", err) +// } +// } +// } diff --git a/internal/plugins/utils.go b/internal/plugins/utils.go new file mode 100644 index 0000000..f020ae8 --- /dev/null +++ b/internal/plugins/utils.go @@ -0,0 +1,80 @@ +package plugins + +import ( + "fmt" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +func HandleColumns(ctx *models.EngineContextDataset, row map[string]interface{}, inferType func(val interface{}) string) { + var columns []models.EngineContextColumn + + if ctx.Columns == nil { + var columnOrder []string + + for key, val := range row { + columns = append(columns, models.EngineContextColumn{ + Name: key, + Type: inferType(val), + Description: fmt.Sprintf("SQL result column: %s", key), + }) + columnOrder = append(columnOrder, key) + } + ctx.Columns = columns + + if ctx.ColumnOrder == nil { + ctx.ColumnOrder = columnOrder + } + } + + if ctx.ColumnOrder == nil { + var columnOrder []string + for _, col := range ctx.Columns { + columnOrder = append(columnOrder, col.Name) + } + ctx.ColumnOrder = columnOrder + } +} + +func HandleGoogleSheetColumns(ctx *models.EngineContextDataset, headers []string, types []string) { + if ctx.Columns != nil && ctx.ColumnOrder != nil { + return + } + + if len(ctx.Columns) == 0 { + columns := make([]models.EngineContextColumn, len(headers)) + for i := range headers { + columns[i] = models.EngineContextColumn{Name: headers[i], Type: types[i]} + } + ctx.Columns = columns + } + + if len(ctx.ColumnOrder) == 0 { + var columnOrder []string + for _, col := range ctx.Columns { + columnOrder = append(columnOrder, col.Name) + } + ctx.ColumnOrder = columnOrder + } +} + +func HandleCSVColumns(ctx *models.EngineContextDataset, hdrs []string, sampleRow []string, inferType func(cell string) string) { + if len(ctx.ColumnOrder) == 0 { + ctx.ColumnOrder = append([]string(nil), hdrs...) + } + if len(ctx.Columns) == 0 { + cols := make([]models.EngineContextColumn, len(hdrs)) + for i, name := range hdrs { + typ := "TEXT" + if i < len(sampleRow) { + typ = inferType(sampleRow[i]) + } + cols[i] = models.EngineContextColumn{ + Name: name, + Type: typ, + Description: fmt.Sprintf("CSV column: %s", name), + } + } + ctx.Columns = cols + } +} diff --git a/internal/plugins/utils_test.go b/internal/plugins/utils_test.go new file mode 100644 index 0000000..61aaa54 --- /dev/null +++ b/internal/plugins/utils_test.go @@ -0,0 +1,176 @@ +package plugins + +import ( + "reflect" + "sort" + "testing" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +// helper: compare two string slices ignoring order +func equalStringSets(a, b []string) bool { + if len(a) != len(b) { + return false + } + aa := append([]string(nil), a...) + bb := append([]string(nil), b...) + sort.Strings(aa) + sort.Strings(bb) + return reflect.DeepEqual(aa, bb) +} + +// helper: make a simple inferType used by tests +func testInferType(v interface{}) string { + switch v.(type) { + case int, int64, int32, uint, uint64, uint32: + return "BIGINT" + case float32, float64: + return "DOUBLE" + case bool: + return "BOOLEAN" + case string: + return "TEXT" + default: + return "TEXT" + } +} + +func TestHandleColumns_BuildsColumnsAndOrderFromRow(t *testing.T) { + params := &models.EngineContextDataset{ + Columns: nil, + ColumnOrder: nil, + } + + row := map[string]interface{}{ + "id": 1, + "name": "Alice", + "active": true, + } + + HandleColumns(params, row, testInferType) + + // Columns built? + if params.Columns == nil || len(params.Columns) != 3 { + t.Fatalf("expected 3 columns built from row, got %d", len(params.Columns)) + } + + // ColumnOrder built? + if params.ColumnOrder == nil || len(params.ColumnOrder) != 3 { + t.Fatalf("expected ColumnOrder of length 3, got %v", params.ColumnOrder) + } + + // Check names as a set (map iteration is unordered) + var names []string + types := map[string]string{} + for _, c := range params.Columns { + names = append(names, c.Name) + types[c.Name] = c.Type + } + + if !equalStringSets(names, []string{"id", "name", "active"}) { + t.Fatalf("expected column names to match row keys, got %v", names) + } + if !equalStringSets(params.ColumnOrder, []string{"id", "name", "active"}) { + t.Fatalf("expected ColumnOrder to match row keys (order-agnostic), got %v", params.ColumnOrder) + } + + // Check inferType was applied + if types["id"] != "BIGINT" { + t.Fatalf("expected type BIGINT for id, got %s", types["id"]) + } + if types["name"] != "TEXT" { + t.Fatalf("expected type TEXT for name, got %s", types["name"]) + } + if types["active"] != "BOOLEAN" { + t.Fatalf("expected type BOOLEAN for active, got %s", types["active"]) + } +} + +func TestHandleColumns_FlattensExistingColumnsIntoOrder(t *testing.T) { + params := &models.EngineContextDataset{ + Columns: []models.EngineContextColumn{ + {Name: "id", Type: "BIGINT"}, + {Name: "email", Type: "TEXT"}, + {Name: "name", Type: "TEXT"}, + }, + ColumnOrder: nil, + } + + // Row is irrelevant when Columns already exist + row := map[string]interface{}{} + + HandleColumns(params, row, testInferType) + + want := []string{"id", "email", "name"} + if !reflect.DeepEqual(params.ColumnOrder, want) { + t.Fatalf("expected ColumnOrder %v, got %v", want, params.ColumnOrder) + } +} + +func TestHandleColumns_DoesNotOverrideExistingOrder(t *testing.T) { + params := &models.EngineContextDataset{ + Columns: []models.EngineContextColumn{ + {Name: "id", Type: "BIGINT"}, + {Name: "email", Type: "TEXT"}, + {Name: "name", Type: "TEXT"}, + }, + ColumnOrder: []string{"name", "id", "email"}, // preset order + } + + row := map[string]interface{}{ + "id": 1, + "name": "Alice", + "email": "alice@example.com", + } + + HandleColumns(params, row, testInferType) + + want := []string{"name", "id", "email"} + if !reflect.DeepEqual(params.ColumnOrder, want) { + t.Fatalf("expected existing ColumnOrder to remain %v, got %v", want, params.ColumnOrder) + } +} + +func TestHandleColumns_EmptyRowAndNoColumns(t *testing.T) { + params := &models.EngineContextDataset{ + Columns: nil, + ColumnOrder: nil, + } + + row := map[string]interface{}{} // empty + + HandleColumns(params, row, testInferType) + + if len(params.Columns) != 0 { + t.Fatalf("expected 0 columns for empty row, got %d", len(params.Columns)) + } + if len(params.ColumnOrder) != 0 { + t.Fatalf("expected empty ColumnOrder for empty row, got %v", params.ColumnOrder) + } +} + +// Optional: ensure that when Columns are preset, we don't rebuild them from the row. +func TestHandleColumns_DoesNotRebuildPresetColumns(t *testing.T) { + params := &models.EngineContextDataset{ + Columns: []models.EngineContextColumn{ + {Name: "x", Type: "TEXT"}, + }, + ColumnOrder: []string{"x"}, + } + + row := map[string]interface{}{ + "id": 1, + "name": "Alice", + } + + HandleColumns(params, row, testInferType) + + // Columns should remain as preset + if len(params.Columns) != 1 || params.Columns[0].Name != "x" { + t.Fatalf("expected preset Columns to remain unchanged, got %v", params.Columns) + } + if !reflect.DeepEqual(params.ColumnOrder, []string{"x"}) { + t.Fatalf("expected preset ColumnOrder to remain unchanged, got %v", params.ColumnOrder) + } +} diff --git a/internal/plugins/yaml/plugin.go b/internal/plugins/yaml/plugin.go new file mode 100644 index 0000000..7836655 --- /dev/null +++ b/internal/plugins/yaml/plugin.go @@ -0,0 +1,155 @@ +package yaml + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "gopkg.in/yaml.v3" +) + +type YAMLDataPlugin struct{} + +func NewYAMLDataPlugin() *YAMLDataPlugin { + return &YAMLDataPlugin{} +} + +func (p *YAMLDataPlugin) GetType() plugins.PluginType { + return plugins.PluginTypeYAML +} + +func (p *YAMLDataPlugin) ProcessData(ctx context.Context, params plugins.PluginProcessParams) (*plugins.ProcessedData, error) { + data, err := io.ReadAll(params.DataReader) + if err != nil { + return nil, fmt.Errorf("failed to read YAML data: %v", err) + } + defer func() { + _ = params.DataReader.Close() + }() + + var parsed interface{} + if err := yaml.Unmarshal(data, &parsed); err != nil { + return nil, fmt.Errorf("failed to parse YAML: %v", err) + } + parsed = convertYAML(parsed) + + // Extract key from query + key := extractKeyFromQuery(params.DatasetContext.Query) + if key == "" { + return nil, fmt.Errorf("query must be of form 'SELECT * FROM '") + } + + rootMap, ok := parsed.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("YAML root must be a map to extract key '%s'", key) + } + + raw, ok := rootMap[key] + if !ok { + return nil, fmt.Errorf("key '%s' not found in YAML root", key) + } + + // Normalize into []interface{} + var dataArray []interface{} + switch v := raw.(type) { + case []interface{}: + for _, item := range v { + row, ok := item.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("array item is not an object: %T", item) + } + dataArray = append(dataArray, row) + } + case map[string]interface{}: + dataArray = []interface{}{v} + default: + return nil, fmt.Errorf("value at key '%s' must be an object or array of objects, got %T", key, v) + } + + if len(dataArray) == 0 { + return nil, fmt.Errorf("no data rows found under '%s'", key) + } + + // Infer columns from the first row + var columns []models.EngineContextColumn + firstRow := dataArray[0].(map[string]interface{}) + for colName, val := range firstRow { + columns = append(columns, models.EngineContextColumn{ + Name: colName, + Type: inferYAMLType(val), + Description: fmt.Sprintf("YAML field: %s", colName), + }) + } + + tableName := params.StorageClient.GetTableName(params.DatasetContext.Name) + generator := params.StorageClient.GetSQLGenerator() + + // Generate CREATE TABLE + createStmt, err := generator.GenerateCreateTableSQL(tableName, columns) + if err != nil { + return nil, fmt.Errorf("failed to generate CREATE TABLE: %v", err) + } + if result := params.StorageClient.Exec(createStmt); result.Error != nil { + return nil, fmt.Errorf("failed to create table: %v", result.Error) + } + + // Generate INSERT + insertStmt, err := generator.GenerateInsertSQL(tableName, dataArray, columns, params.DatasetContext.Dedupe) + if err != nil { + return nil, fmt.Errorf("failed to generate INSERT: %v", err) + } + if result := params.StorageClient.Exec(insertStmt); result.Error != nil { + return nil, fmt.Errorf("failed to insert data: %v", result.Error) + } + + return &plugins.ProcessedData{ + TableName: tableName, + Columns: columns, + RowCount: int64(len(dataArray)), + }, nil +} + +// extractKeyFromQuery pulls the key from "SELECT * FROM key" +func extractKeyFromQuery(query string) string { + q := strings.TrimSpace(strings.ToLower(query)) + prefix := "select * from " + if strings.HasPrefix(q, prefix) { + return strings.TrimSpace(query[len(prefix):]) + } + return "" +} + +// convertYAML ensures all map keys are strings +func convertYAML(v interface{}) interface{} { + switch val := v.(type) { + case map[interface{}]interface{}: + m := make(map[string]interface{}) + for k, v := range val { + m[fmt.Sprint(k)] = convertYAML(v) + } + return m + case []interface{}: + for i, u := range val { + val[i] = convertYAML(u) + } + } + return v +} + +func inferYAMLType(value interface{}) string { + switch value.(type) { + case string: + return "TEXT" + case int, int64, float64: + return "NUMERIC" + case bool: + return "BOOLEAN" + case nil: + return "TEXT" + default: + return "TEXT" + } +} diff --git a/internal/processor/graph.go b/internal/processor/graph.go new file mode 100644 index 0000000..4cbfa9f --- /dev/null +++ b/internal/processor/graph.go @@ -0,0 +1,88 @@ +package processor + +import ( + "fmt" + "maps" + "sort" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +func buildGraph(datasets map[string]*models.EngineContextDataset, strictDeps bool) (*dependencyGraph, error) { + nodes := maps.Clone(datasets) + edges := make(map[string][]string) + indegree := make(map[string]int, len(nodes)) + for name := range nodes { + indegree[name] = 0 + } + + for _, d := range datasets { + for _, child := range d.Children { + if _, ok := nodes[child.Name]; !ok { + if strictDeps { + return nil, fmt.Errorf("dataset %q depends on missing dataset %q", d.Name, child.Name) + } + continue + } + // reverse edge direction: child → parent + edges[child.Name] = append(edges[child.Name], d.Name) + indegree[d.Name]++ + } + } + for k := range edges { + sort.Strings(edges[k]) + } + return &dependencyGraph{nodes: nodes, edges: edges, indegree: indegree}, nil +} + +func topoOrder(g *dependencyGraph) ([]*models.EngineContextDataset, error) { + indeg := maps.Clone(g.indegree) + var zero []string + for name, deg := range indeg { + if deg == 0 { + zero = append(zero, name) + } + } + sort.Strings(zero) + + var ordered []*models.EngineContextDataset + for len(zero) > 0 { + name := zero[0] + zero = zero[1:] + ordered = append(ordered, g.nodes[name]) + for _, child := range g.edges[name] { + indeg[child]-- + if indeg[child] == 0 { + zero = append(zero, child) + } + } + sort.Strings(zero) + } + + if len(ordered) != len(g.nodes) { + return nil, fmt.Errorf("circular dependency detected") + } + return ordered, nil +} + +func flattenDatasets(root map[string]*models.EngineContextDataset) map[string]*models.EngineContextDataset { + out := make(map[string]*models.EngineContextDataset) + for name, ds := range root { + flattenDatasetRecursive(ds, out) + out[name] = ds + } + return out +} + +func flattenDatasetRecursive(ds *models.EngineContextDataset, out map[string]*models.EngineContextDataset) { + if ds == nil { + return + } + if _, exists := out[ds.Name]; exists { + return + } + out[ds.Name] = ds + for _, child := range ds.Children { + flattenDatasetRecursive(child, out) + } +} diff --git a/internal/processor/main.go b/internal/processor/main.go new file mode 100644 index 0000000..14cf85f --- /dev/null +++ b/internal/processor/main.go @@ -0,0 +1,327 @@ +package processor + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "log/slog" + "strings" + + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/internal/storage_manager" + "github.com/kvatch-hub/kvatch-runtime/internal/subscription_manager" + "gorm.io/gorm" +) + +type ( + DataStoreClient interface { + GetConnectionString() string + GetTableName(datasetID string) string + Find(tableName string) ([]map[string]interface{}, error) + Exec(query string) *datastore.ExecResult + IsNotFoundError(err error) bool + Query(ctx context.Context, tableName string, opts datastore.QueryOptions) ([]map[string]interface{}, error) + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + GetSQLGenerator() datastore.SQLGenerator + GetStorageType() datastore.StorageType + Close(ctx context.Context) error + GetGormDB() *gorm.DB + } + ConnectorResolver interface { + ResolveInternalJoin(ec *models.EngineContextConnector) (connectors.ConnectorType, any, error) + } + PluginResolver interface { + ResolvePluginType(datasetType, connectorType, originalType string) plugins.PluginType + } + ConnectorRegistry interface { + Register(connectionType connectors.ConnectorType, connection connectors.Connector) error + Get(connectionType connectors.ConnectorType) (connectors.Connector, error) + List() []connectors.ConnectorType + } + DataPluginRegistry interface { + Register(pluginType plugins.PluginType, plugin plugins.Plugin) error + Get(pluginType plugins.PluginType) (plugins.Plugin, error) + List() []plugins.PluginType + } + SubscriptionManager interface { + GetRegisteredPlugins() subscription_manager.DataPluginRegistry + GetRegisteredConnectors() subscription_manager.ConnectorRegistry + } + StorageManager interface { + EnsureTable(ctx context.Context, table string, columns []models.EngineContextColumn, dedupe []string) error + Insert(ctx context.Context, table string, data []map[string]interface{}, columns []models.EngineContextColumn, dedupe []string) error + } +) + +type dependencyGraph struct { + nodes map[string]*models.EngineContextDataset + edges map[string][]string + indegree map[string]int +} + +type Processor struct { + connectors ConnectorRegistry + plugins DataPluginRegistry + dataStore DataStoreClient + verbose bool + strictDeps bool + connectorResolver ConnectorResolver + pluginResolver PluginResolver +} + +func NewProcessor( + dataStore DataStoreClient, + connectors ConnectorRegistry, + plugins DataPluginRegistry, + opts ...func(*Processor), +) *Processor { + p := &Processor{ + dataStore: dataStore, + connectors: connectors, + plugins: plugins, + connectorResolver: NewJoinConnectorResolver(dataStore), + pluginResolver: NewDefaultPluginResolver(), + } + + for _, o := range opts { + o(p) + } + return p +} + +func WithVerbose(v bool) func(*Processor) { + return func(p *Processor) { p.verbose = v } +} +func WithStrictDeps(v bool) func(*Processor) { + return func(p *Processor) { p.strictDeps = v } +} +func WithConnectorResolver(r ConnectorResolver) func(*Processor) { + return func(p *Processor) { p.connectorResolver = r } +} +func WithPluginResolver(r PluginResolver) func(*Processor) { + return func(p *Processor) { p.pluginResolver = r } +} + +func (p *Processor) ProcessPlan(ctx *models.EngineContext) error { + p.verbose = ctx.EngineOptions.Verbose + + allDatasets := flattenDatasets(ctx.Datasets) + graph, err := buildGraph(allDatasets, p.strictDeps) + if err != nil { + return err + } + + order, err := topoOrder(graph) + if err != nil { + return fmt.Errorf("failed to determine processing order: %w", err) + } + + if p.verbose { + slog.Info("processing datasets", "count", len(order)) + } + + processed := make(map[string]struct{}) + + for i, d := range order { + if _, ok := processed[d.Name]; ok { + continue + } + if p.verbose { + slog.Info("processing dataset tree", "index", i+1, "root", d.Name) + } + + if err := p.processDatasetRecursive(ctx.Ctx, d, ctx.Connectors, processed); err != nil { + return err + } + } + + return nil +} + +func (p *Processor) processDatasetRecursive( + ctx context.Context, + dataset *models.EngineContextDataset, + engineConnectors map[string]*models.EngineContextConnector, + processed map[string]struct{}, +) error { + if _, ok := processed[dataset.Name]; ok { + return nil + } + + for _, child := range dataset.Children { + if err := p.processDatasetRecursive(ctx, child, engineConnectors, processed); err != nil { + return err + } + } + + if p.verbose { + slog.Info("processing dataset", "name", dataset.Name, "type", dataset.Type, "connector", dataset.ConnectorName) + } + + if err := p.processDataset(ctx, dataset, engineConnectors, false); err != nil { + return fmt.Errorf("failed to process dataset %q: %w", dataset.Name, err) + } + + processed[dataset.Name] = struct{}{} + return nil +} + +func (p *Processor) processDataset( + ctx context.Context, + d *models.EngineContextDataset, + engineConnectors map[string]*models.EngineContextConnector, + _ bool, +) error { + if d.Data != nil { + return p.processEmbeddedDataset(ctx, d) + } + + if strings.EqualFold(d.ConnectorName, "federated") { + return p.processFederatedDataset(ctx, d, engineConnectors) + } + + return p.processConnectorDataset(ctx, d, engineConnectors) +} + +func (p *Processor) processEmbeddedDataset( + ctx context.Context, + d *models.EngineContextDataset, +) error { + if d.Data == nil { + return fmt.Errorf("dataset %q has no embedded data", d.Name) + } + + raw, err := json.Marshal(d.Data) + if err != nil { + return fmt.Errorf("failed to marshal embedded data for dataset %q: %w", d.Name, err) + } + + reader := io.NopCloser(bytes.NewReader(raw)) + defer func() { _ = reader.Close() }() + + pluginType := p.pluginResolver.ResolvePluginType(d.Type, "", "") + pl, err := p.plugins.Get(pluginType) + if err != nil { + return fmt.Errorf("plugin not registered %s: %w", pluginType, err) + } + + if _, err := pl.ProcessData(ctx, plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: p.dataStore, + DatasetContext: d, + Manager: storage_manager.NewStorageManager(p.dataStore), + }); err != nil { + return fmt.Errorf("issue processing embedded data for dataset %s: %w", d.Name, err) + } + + return nil +} + +func (p *Processor) processFederatedDataset( + ctx context.Context, + d *models.EngineContextDataset, + engineConnectors map[string]*models.EngineContextConnector, +) error { + ec, ok := engineConnectors[d.ConnectorName] + if !ok { + return fmt.Errorf("connector %q not found for federated dataset %q", d.ConnectorName, d.Name) + } + + connType, connCfg, err := p.connectorResolver.ResolveInternalJoin(ec) + if err != nil { + return fmt.Errorf("failed to resolve federated dataset %q: %w", d.Name, err) + } + + conn, err := p.connectors.Get(connType) + if err != nil { + return fmt.Errorf("connection type not supported for federated dataset %q: %s", d.Name, connType) + } + + if err := conn.Connect(ctx, &models.EngineContextConnector{ + Type: string(connType), + Connection: ec.Connection, + Config: connCfg, + }); err != nil { + return fmt.Errorf("failed to connect federated dataset %q using %s: %w", d.Name, connType, err) + } + defer func() { _ = conn.Close() }() + + reader, err := conn.GetData(ctx, d) + if err != nil { + return fmt.Errorf("failed to get federated data for dataset %q: %w", d.Name, err) + } + defer func() { _ = reader.Close() }() + + pluginType := p.pluginResolver.ResolvePluginType(d.Type, string(connType), ec.Type) + pl, err := p.plugins.Get(pluginType) + if err != nil { + return fmt.Errorf("plugin not registered %s: %w", pluginType, err) + } + + if _, err := pl.ProcessData(ctx, plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: p.dataStore, + DatasetContext: d, + Manager: storage_manager.NewStorageManager(p.dataStore), + }); err != nil { + return fmt.Errorf("issue processing federated dataset %s: %w", d.Name, err) + } + + return nil +} + +func (p *Processor) processConnectorDataset( + ctx context.Context, + d *models.EngineContextDataset, + engineConnectors map[string]*models.EngineContextConnector, +) error { + ec, ok := engineConnectors[d.ConnectorName] + if !ok { + return fmt.Errorf("connector %q not found", d.ConnectorName) + } + + connType := connectors.ConnectorType(strings.ToUpper(ec.Type)) + + conn, err := p.connectors.Get(connType) + if err != nil { + return fmt.Errorf("connection type not supported: %s", connType) + } + + if err := conn.Connect(ctx, &models.EngineContextConnector{ + Type: string(connType), + Connection: ec.Connection, + Config: ec.Config, + }); err != nil { + return fmt.Errorf("failed to connect to %s: %w", connType, err) + } + defer func() { _ = conn.Close() }() + + reader, err := conn.GetData(ctx, d) + if err != nil { + return fmt.Errorf("failed to get data from connection: %w", err) + } + defer func() { _ = reader.Close() }() + + pluginType := p.pluginResolver.ResolvePluginType(d.Type, string(connType), ec.Type) + pl, err := p.plugins.Get(pluginType) + if err != nil { + return fmt.Errorf("plugin not registered %s: %w", pluginType, err) + } + + if _, err := pl.ProcessData(ctx, plugins.PluginProcessParams{ + DataReader: reader, + StorageClient: p.dataStore, + DatasetContext: d, + Manager: storage_manager.NewStorageManager(p.dataStore), + }); err != nil { + return fmt.Errorf("issue processing the data %s: %w", d.Name, err) + } + + return nil +} diff --git a/internal/processor/mocks/mock_connector.go b/internal/processor/mocks/mock_connector.go new file mode 100644 index 0000000..14a1250 --- /dev/null +++ b/internal/processor/mocks/mock_connector.go @@ -0,0 +1,129 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kvatch-hub/kvatch-runtime/internal/connectors (interfaces: Connector) +// +// Generated by this command: +// +// mockgen -package=mocks -destination=internal/processor/mocks/mock_connector.go github.com/kvatch-hub/kvatch-runtime/internal/connectors Connector +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + io "io" + reflect "reflect" + + connectors "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + gomock "go.uber.org/mock/gomock" +) + +// MockConnector is a mock of Connector interface. +type MockConnector struct { + ctrl *gomock.Controller + recorder *MockConnectorMockRecorder + isgomock struct{} +} + +// MockConnectorMockRecorder is the mock recorder for MockConnector. +type MockConnectorMockRecorder struct { + mock *MockConnector +} + +// NewMockConnector creates a new mock instance. +func NewMockConnector(ctrl *gomock.Controller) *MockConnector { + mock := &MockConnector{ctrl: ctrl} + mock.recorder = &MockConnectorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConnector) EXPECT() *MockConnectorMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockConnector) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockConnectorMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockConnector)(nil).Close)) +} + +// Connect mocks base method. +func (m *MockConnector) Connect(ctx context.Context, engineCtx *models.EngineContextConnector) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Connect", ctx, engineCtx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Connect indicates an expected call of Connect. +func (mr *MockConnectorMockRecorder) Connect(ctx, engineCtx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Connect", reflect.TypeOf((*MockConnector)(nil).Connect), ctx, engineCtx) +} + +// GetData mocks base method. +func (m *MockConnector) GetData(ctx context.Context, engineCtx *models.EngineContextDataset) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetData", ctx, engineCtx) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetData indicates an expected call of GetData. +func (mr *MockConnectorMockRecorder) GetData(ctx, engineCtx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetData", reflect.TypeOf((*MockConnector)(nil).GetData), ctx, engineCtx) +} + +// GetType mocks base method. +func (m *MockConnector) GetType() connectors.ConnectorType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetType") + ret0, _ := ret[0].(connectors.ConnectorType) + return ret0 +} + +// GetType indicates an expected call of GetType. +func (mr *MockConnectorMockRecorder) GetType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetType", reflect.TypeOf((*MockConnector)(nil).GetType)) +} + +// Ping mocks base method. +func (m *MockConnector) Ping(ctx context.Context, config any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Ping", ctx, config) + ret0, _ := ret[0].(error) + return ret0 +} + +// Ping indicates an expected call of Ping. +func (mr *MockConnectorMockRecorder) Ping(ctx, config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Ping", reflect.TypeOf((*MockConnector)(nil).Ping), ctx, config) +} + +// Validate mocks base method. +func (m *MockConnector) Validate(config any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Validate", config) + ret0, _ := ret[0].(error) + return ret0 +} + +// Validate indicates an expected call of Validate. +func (mr *MockConnectorMockRecorder) Validate(config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Validate", reflect.TypeOf((*MockConnector)(nil).Validate), config) +} diff --git a/internal/processor/mocks/mock_interfaces.go b/internal/processor/mocks/mock_interfaces.go new file mode 100644 index 0000000..8b9bb4a --- /dev/null +++ b/internal/processor/mocks/mock_interfaces.go @@ -0,0 +1,526 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/processor/main.go +// +// Generated by this command: +// +// mockgen -source internal/processor/main.go -destination internal/processor/mocks/mock_interfaces.go -package mocks +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + sql "database/sql" + reflect "reflect" + + connectors "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + datastore "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + plugins "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + subscription_manager "github.com/kvatch-hub/kvatch-runtime/internal/subscription_manager" + gomock "go.uber.org/mock/gomock" + gorm "gorm.io/gorm" +) + +// MockDataStoreClient is a mock of DataStoreClient interface. +type MockDataStoreClient struct { + ctrl *gomock.Controller + recorder *MockDataStoreClientMockRecorder + isgomock struct{} +} + +// MockDataStoreClientMockRecorder is the mock recorder for MockDataStoreClient. +type MockDataStoreClientMockRecorder struct { + mock *MockDataStoreClient +} + +// NewMockDataStoreClient creates a new mock instance. +func NewMockDataStoreClient(ctrl *gomock.Controller) *MockDataStoreClient { + mock := &MockDataStoreClient{ctrl: ctrl} + mock.recorder = &MockDataStoreClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDataStoreClient) EXPECT() *MockDataStoreClientMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockDataStoreClient) Close(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockDataStoreClientMockRecorder) Close(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockDataStoreClient)(nil).Close), ctx) +} + +// Exec mocks base method. +func (m *MockDataStoreClient) Exec(query string) *datastore.ExecResult { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Exec", query) + ret0, _ := ret[0].(*datastore.ExecResult) + return ret0 +} + +// Exec indicates an expected call of Exec. +func (mr *MockDataStoreClientMockRecorder) Exec(query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exec", reflect.TypeOf((*MockDataStoreClient)(nil).Exec), query) +} + +// Find mocks base method. +func (m *MockDataStoreClient) Find(tableName string) ([]map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Find", tableName) + ret0, _ := ret[0].([]map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Find indicates an expected call of Find. +func (mr *MockDataStoreClientMockRecorder) Find(tableName any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Find", reflect.TypeOf((*MockDataStoreClient)(nil).Find), tableName) +} + +// GetConnectionString mocks base method. +func (m *MockDataStoreClient) GetConnectionString() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetConnectionString") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetConnectionString indicates an expected call of GetConnectionString. +func (mr *MockDataStoreClientMockRecorder) GetConnectionString() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConnectionString", reflect.TypeOf((*MockDataStoreClient)(nil).GetConnectionString)) +} + +// GetGormDB mocks base method. +func (m *MockDataStoreClient) GetGormDB() *gorm.DB { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetGormDB") + ret0, _ := ret[0].(*gorm.DB) + return ret0 +} + +// GetGormDB indicates an expected call of GetGormDB. +func (mr *MockDataStoreClientMockRecorder) GetGormDB() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGormDB", reflect.TypeOf((*MockDataStoreClient)(nil).GetGormDB)) +} + +// GetSQLGenerator mocks base method. +func (m *MockDataStoreClient) GetSQLGenerator() datastore.SQLGenerator { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetSQLGenerator") + ret0, _ := ret[0].(datastore.SQLGenerator) + return ret0 +} + +// GetSQLGenerator indicates an expected call of GetSQLGenerator. +func (mr *MockDataStoreClientMockRecorder) GetSQLGenerator() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSQLGenerator", reflect.TypeOf((*MockDataStoreClient)(nil).GetSQLGenerator)) +} + +// GetStorageType mocks base method. +func (m *MockDataStoreClient) GetStorageType() datastore.StorageType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetStorageType") + ret0, _ := ret[0].(datastore.StorageType) + return ret0 +} + +// GetStorageType indicates an expected call of GetStorageType. +func (mr *MockDataStoreClientMockRecorder) GetStorageType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetStorageType", reflect.TypeOf((*MockDataStoreClient)(nil).GetStorageType)) +} + +// GetTableName mocks base method. +func (m *MockDataStoreClient) GetTableName(datasetID string) string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTableName", datasetID) + ret0, _ := ret[0].(string) + return ret0 +} + +// GetTableName indicates an expected call of GetTableName. +func (mr *MockDataStoreClientMockRecorder) GetTableName(datasetID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTableName", reflect.TypeOf((*MockDataStoreClient)(nil).GetTableName), datasetID) +} + +// IsNotFoundError mocks base method. +func (m *MockDataStoreClient) IsNotFoundError(err error) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsNotFoundError", err) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsNotFoundError indicates an expected call of IsNotFoundError. +func (mr *MockDataStoreClientMockRecorder) IsNotFoundError(err any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsNotFoundError", reflect.TypeOf((*MockDataStoreClient)(nil).IsNotFoundError), err) +} + +// Query mocks base method. +func (m *MockDataStoreClient) Query(ctx context.Context, tableName string, opts datastore.QueryOptions) ([]map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Query", ctx, tableName, opts) + ret0, _ := ret[0].([]map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Query indicates an expected call of Query. +func (mr *MockDataStoreClientMockRecorder) Query(ctx, tableName, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Query", reflect.TypeOf((*MockDataStoreClient)(nil).Query), ctx, tableName, opts) +} + +// QueryContext mocks base method. +func (m *MockDataStoreClient) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, query} + for _, a := range args { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "QueryContext", varargs...) + ret0, _ := ret[0].(*sql.Rows) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryContext indicates an expected call of QueryContext. +func (mr *MockDataStoreClientMockRecorder) QueryContext(ctx, query any, args ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, query}, args...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryContext", reflect.TypeOf((*MockDataStoreClient)(nil).QueryContext), varargs...) +} + +// MockConnectorResolver is a mock of ConnectorResolver interface. +type MockConnectorResolver struct { + ctrl *gomock.Controller + recorder *MockConnectorResolverMockRecorder + isgomock struct{} +} + +// MockConnectorResolverMockRecorder is the mock recorder for MockConnectorResolver. +type MockConnectorResolverMockRecorder struct { + mock *MockConnectorResolver +} + +// NewMockConnectorResolver creates a new mock instance. +func NewMockConnectorResolver(ctrl *gomock.Controller) *MockConnectorResolver { + mock := &MockConnectorResolver{ctrl: ctrl} + mock.recorder = &MockConnectorResolverMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConnectorResolver) EXPECT() *MockConnectorResolverMockRecorder { + return m.recorder +} + +// ResolveInternalJoin mocks base method. +func (m *MockConnectorResolver) ResolveInternalJoin(ec *models.EngineContextConnector) (connectors.ConnectorType, any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveInternalJoin", ec) + ret0, _ := ret[0].(connectors.ConnectorType) + ret1, _ := ret[1].(any) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ResolveInternalJoin indicates an expected call of ResolveInternalJoin. +func (mr *MockConnectorResolverMockRecorder) ResolveInternalJoin(ec any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveInternalJoin", reflect.TypeOf((*MockConnectorResolver)(nil).ResolveInternalJoin), ec) +} + +// MockPluginResolver is a mock of PluginResolver interface. +type MockPluginResolver struct { + ctrl *gomock.Controller + recorder *MockPluginResolverMockRecorder + isgomock struct{} +} + +// MockPluginResolverMockRecorder is the mock recorder for MockPluginResolver. +type MockPluginResolverMockRecorder struct { + mock *MockPluginResolver +} + +// NewMockPluginResolver creates a new mock instance. +func NewMockPluginResolver(ctrl *gomock.Controller) *MockPluginResolver { + mock := &MockPluginResolver{ctrl: ctrl} + mock.recorder = &MockPluginResolverMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPluginResolver) EXPECT() *MockPluginResolverMockRecorder { + return m.recorder +} + +// ResolvePluginType mocks base method. +func (m *MockPluginResolver) ResolvePluginType(datasetType, connectorType, originalType string) plugins.PluginType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolvePluginType", datasetType, connectorType, originalType) + ret0, _ := ret[0].(plugins.PluginType) + return ret0 +} + +// ResolvePluginType indicates an expected call of ResolvePluginType. +func (mr *MockPluginResolverMockRecorder) ResolvePluginType(datasetType, connectorType, originalType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolvePluginType", reflect.TypeOf((*MockPluginResolver)(nil).ResolvePluginType), datasetType, connectorType, originalType) +} + +// MockConnectorRegistry is a mock of ConnectorRegistry interface. +type MockConnectorRegistry struct { + ctrl *gomock.Controller + recorder *MockConnectorRegistryMockRecorder + isgomock struct{} +} + +// MockConnectorRegistryMockRecorder is the mock recorder for MockConnectorRegistry. +type MockConnectorRegistryMockRecorder struct { + mock *MockConnectorRegistry +} + +// NewMockConnectorRegistry creates a new mock instance. +func NewMockConnectorRegistry(ctrl *gomock.Controller) *MockConnectorRegistry { + mock := &MockConnectorRegistry{ctrl: ctrl} + mock.recorder = &MockConnectorRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConnectorRegistry) EXPECT() *MockConnectorRegistryMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockConnectorRegistry) Get(connectionType connectors.ConnectorType) (connectors.Connector, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", connectionType) + ret0, _ := ret[0].(connectors.Connector) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockConnectorRegistryMockRecorder) Get(connectionType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockConnectorRegistry)(nil).Get), connectionType) +} + +// List mocks base method. +func (m *MockConnectorRegistry) List() []connectors.ConnectorType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List") + ret0, _ := ret[0].([]connectors.ConnectorType) + return ret0 +} + +// List indicates an expected call of List. +func (mr *MockConnectorRegistryMockRecorder) List() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockConnectorRegistry)(nil).List)) +} + +// Register mocks base method. +func (m *MockConnectorRegistry) Register(connectionType connectors.ConnectorType, connection connectors.Connector) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Register", connectionType, connection) + ret0, _ := ret[0].(error) + return ret0 +} + +// Register indicates an expected call of Register. +func (mr *MockConnectorRegistryMockRecorder) Register(connectionType, connection any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Register", reflect.TypeOf((*MockConnectorRegistry)(nil).Register), connectionType, connection) +} + +// MockDataPluginRegistry is a mock of DataPluginRegistry interface. +type MockDataPluginRegistry struct { + ctrl *gomock.Controller + recorder *MockDataPluginRegistryMockRecorder + isgomock struct{} +} + +// MockDataPluginRegistryMockRecorder is the mock recorder for MockDataPluginRegistry. +type MockDataPluginRegistryMockRecorder struct { + mock *MockDataPluginRegistry +} + +// NewMockDataPluginRegistry creates a new mock instance. +func NewMockDataPluginRegistry(ctrl *gomock.Controller) *MockDataPluginRegistry { + mock := &MockDataPluginRegistry{ctrl: ctrl} + mock.recorder = &MockDataPluginRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDataPluginRegistry) EXPECT() *MockDataPluginRegistryMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockDataPluginRegistry) Get(pluginType plugins.PluginType) (plugins.Plugin, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", pluginType) + ret0, _ := ret[0].(plugins.Plugin) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockDataPluginRegistryMockRecorder) Get(pluginType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDataPluginRegistry)(nil).Get), pluginType) +} + +// List mocks base method. +func (m *MockDataPluginRegistry) List() []plugins.PluginType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List") + ret0, _ := ret[0].([]plugins.PluginType) + return ret0 +} + +// List indicates an expected call of List. +func (mr *MockDataPluginRegistryMockRecorder) List() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockDataPluginRegistry)(nil).List)) +} + +// Register mocks base method. +func (m *MockDataPluginRegistry) Register(pluginType plugins.PluginType, plugin plugins.Plugin) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Register", pluginType, plugin) + ret0, _ := ret[0].(error) + return ret0 +} + +// Register indicates an expected call of Register. +func (mr *MockDataPluginRegistryMockRecorder) Register(pluginType, plugin any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Register", reflect.TypeOf((*MockDataPluginRegistry)(nil).Register), pluginType, plugin) +} + +// MockSubscriptionManager is a mock of SubscriptionManager interface. +type MockSubscriptionManager struct { + ctrl *gomock.Controller + recorder *MockSubscriptionManagerMockRecorder + isgomock struct{} +} + +// MockSubscriptionManagerMockRecorder is the mock recorder for MockSubscriptionManager. +type MockSubscriptionManagerMockRecorder struct { + mock *MockSubscriptionManager +} + +// NewMockSubscriptionManager creates a new mock instance. +func NewMockSubscriptionManager(ctrl *gomock.Controller) *MockSubscriptionManager { + mock := &MockSubscriptionManager{ctrl: ctrl} + mock.recorder = &MockSubscriptionManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSubscriptionManager) EXPECT() *MockSubscriptionManagerMockRecorder { + return m.recorder +} + +// GetRegisteredConnectors mocks base method. +func (m *MockSubscriptionManager) GetRegisteredConnectors() subscription_manager.ConnectorRegistry { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRegisteredConnectors") + ret0, _ := ret[0].(subscription_manager.ConnectorRegistry) + return ret0 +} + +// GetRegisteredConnectors indicates an expected call of GetRegisteredConnectors. +func (mr *MockSubscriptionManagerMockRecorder) GetRegisteredConnectors() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRegisteredConnectors", reflect.TypeOf((*MockSubscriptionManager)(nil).GetRegisteredConnectors)) +} + +// GetRegisteredPlugins mocks base method. +func (m *MockSubscriptionManager) GetRegisteredPlugins() subscription_manager.DataPluginRegistry { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRegisteredPlugins") + ret0, _ := ret[0].(subscription_manager.DataPluginRegistry) + return ret0 +} + +// GetRegisteredPlugins indicates an expected call of GetRegisteredPlugins. +func (mr *MockSubscriptionManagerMockRecorder) GetRegisteredPlugins() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRegisteredPlugins", reflect.TypeOf((*MockSubscriptionManager)(nil).GetRegisteredPlugins)) +} + +// MockStorageManager is a mock of StorageManager interface. +type MockStorageManager struct { + ctrl *gomock.Controller + recorder *MockStorageManagerMockRecorder + isgomock struct{} +} + +// MockStorageManagerMockRecorder is the mock recorder for MockStorageManager. +type MockStorageManagerMockRecorder struct { + mock *MockStorageManager +} + +// NewMockStorageManager creates a new mock instance. +func NewMockStorageManager(ctrl *gomock.Controller) *MockStorageManager { + mock := &MockStorageManager{ctrl: ctrl} + mock.recorder = &MockStorageManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStorageManager) EXPECT() *MockStorageManagerMockRecorder { + return m.recorder +} + +// EnsureTable mocks base method. +func (m *MockStorageManager) EnsureTable(ctx context.Context, table string, columns []models.EngineContextColumn, dedupe []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureTable", ctx, table, columns, dedupe) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureTable indicates an expected call of EnsureTable. +func (mr *MockStorageManagerMockRecorder) EnsureTable(ctx, table, columns, dedupe any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureTable", reflect.TypeOf((*MockStorageManager)(nil).EnsureTable), ctx, table, columns, dedupe) +} + +// Insert mocks base method. +func (m *MockStorageManager) Insert(ctx context.Context, table string, data []map[string]any, columns []models.EngineContextColumn, dedupe []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Insert", ctx, table, data, columns, dedupe) + ret0, _ := ret[0].(error) + return ret0 +} + +// Insert indicates an expected call of Insert. +func (mr *MockStorageManagerMockRecorder) Insert(ctx, table, data, columns, dedupe any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockStorageManager)(nil).Insert), ctx, table, data, columns, dedupe) +} diff --git a/internal/processor/mocks/mock_plugin.go b/internal/processor/mocks/mock_plugin.go new file mode 100644 index 0000000..9bbc93c --- /dev/null +++ b/internal/processor/mocks/mock_plugin.go @@ -0,0 +1,71 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kvatch-hub/kvatch-runtime/internal/plugins (interfaces: Plugin) +// +// Generated by this command: +// +// mockgen -package=mocks -destination=internal/processor/mocks/mock_plugin.go github.com/kvatch-hub/kvatch-runtime/internal/plugins Plugin +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + plugins "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + gomock "go.uber.org/mock/gomock" +) + +// MockPlugin is a mock of Plugin interface. +type MockPlugin struct { + ctrl *gomock.Controller + recorder *MockPluginMockRecorder + isgomock struct{} +} + +// MockPluginMockRecorder is the mock recorder for MockPlugin. +type MockPluginMockRecorder struct { + mock *MockPlugin +} + +// NewMockPlugin creates a new mock instance. +func NewMockPlugin(ctrl *gomock.Controller) *MockPlugin { + mock := &MockPlugin{ctrl: ctrl} + mock.recorder = &MockPluginMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPlugin) EXPECT() *MockPluginMockRecorder { + return m.recorder +} + +// GetType mocks base method. +func (m *MockPlugin) GetType() plugins.PluginType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetType") + ret0, _ := ret[0].(plugins.PluginType) + return ret0 +} + +// GetType indicates an expected call of GetType. +func (mr *MockPluginMockRecorder) GetType() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetType", reflect.TypeOf((*MockPlugin)(nil).GetType)) +} + +// ProcessData mocks base method. +func (m *MockPlugin) ProcessData(ctx context.Context, params plugins.PluginProcessParams) (*plugins.ProcessedData, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ProcessData", ctx, params) + ret0, _ := ret[0].(*plugins.ProcessedData) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ProcessData indicates an expected call of ProcessData. +func (mr *MockPluginMockRecorder) ProcessData(ctx, params any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessData", reflect.TypeOf((*MockPlugin)(nil).ProcessData), ctx, params) +} diff --git a/internal/processor/resolvers.go b/internal/processor/resolvers.go new file mode 100644 index 0000000..538b2b1 --- /dev/null +++ b/internal/processor/resolvers.go @@ -0,0 +1,65 @@ +package processor + +import ( + "strings" + + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" +) + +type JoinConnectorResolver struct { + storage DataStoreClient +} + +func NewJoinConnectorResolver(storage DataStoreClient) *JoinConnectorResolver { + return &JoinConnectorResolver{storage: storage} +} + +func (r *JoinConnectorResolver) ResolveInternalJoin(ec *models.EngineContextConnector) (connectors.ConnectorType, any, error) { + // Not JOIN? Just pass through. + if strings.ToUpper(ec.Type) != string(connectors.ConnectorTypeFederated) { + return connectors.ConnectorType(strings.ToUpper(ec.Type)), ec.Config, nil + } + + // JOIN → dynamic routing based on storage backend + switch r.storage.GetStorageType() { + case datastore.StorageTypePOSTGRES: + // Dataset federated into Postgres backing store + return connectors.ConnectorTypePostgres, ec.Config, nil + + case datastore.StorageTypeSQLITE: + // SQLite synthesizes a config dynamically + return connectors.ConnectorTypeSQLite, + entities.SQLiteConnectorConfig{ + Path: r.storage.GetConnectionString(), + }, nil + + default: + // fallback to SQLite for any unknown storage + return connectors.ConnectorTypeSQLite, + entities.SQLiteConnectorConfig{ + Path: r.storage.GetConnectionString(), + }, nil + } +} + +type DefaultPluginResolver struct{} + +func NewDefaultPluginResolver() PluginResolver { + return &DefaultPluginResolver{} +} + +func (r *DefaultPluginResolver) ResolvePluginType(datasetType, connectorType, originalType string, +) plugins.PluginType { + dsTypeUpper := strings.ToUpper(datasetType) + if dsTypeUpper == string(plugins.PluginTypeSQL) && + (strings.ToUpper(connectorType) == string(connectors.ConnectorTypePostgres) || + strings.ToUpper(originalType) == string(connectors.ConnectorTypePostgres)) { + return plugins.PluginTypeSQL + } + + return plugins.PluginType(dsTypeUpper) +} diff --git a/internal/response_builder/bulder.go b/internal/response_builder/bulder.go new file mode 100644 index 0000000..301e734 --- /dev/null +++ b/internal/response_builder/bulder.go @@ -0,0 +1,56 @@ +package response_builder + +import ( + "fmt" + "slices" + + "github.com/kvatch-hub/kvatch-runtime/entities" +) + +type ResponseBuilder struct{} + +func (b *ResponseBuilder) BuildResponse(results []map[string]interface{}, columnOrder []string) (*entities.ExecutePlanResponse, error) { + if len(results) == 0 { + return &entities.ExecutePlanResponse{ + Data: []map[string]interface{}{}, + Columns: []entities.DataColumn{}, + }, nil + } + + if len(columnOrder) == 0 { + seen := make(map[string]struct{}) + for _, row := range results { + for colName := range row { + seen[colName] = struct{}{} + } + } + + for colName := range seen { + columnOrder = append(columnOrder, colName) + } + slices.Sort(columnOrder) + } + + columns := make([]entities.DataColumn, 0, len(columnOrder)) + for _, colName := range columnOrder { + columns = append(columns, entities.DataColumn{ + Name: colName, + Type: "TEXT", + Description: fmt.Sprintf("Column: %s", colName), + }) + } + + orderedResults := make([]map[string]interface{}, 0, len(results)) + for _, row := range results { + orderedRow := make(map[string]interface{}, len(columnOrder)) + for _, colName := range columnOrder { + orderedRow[colName] = row[colName] + } + orderedResults = append(orderedResults, orderedRow) + } + + return &entities.ExecutePlanResponse{ + Data: orderedResults, + Columns: columns, + }, nil +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go deleted file mode 100644 index cb3ec6e..0000000 --- a/internal/storage/storage.go +++ /dev/null @@ -1,15 +0,0 @@ -package storage - -import "github.com/kvatch-hub/kvatch-runtime/configs" - -type Storage interface{} - -type sqliteStorage struct { - DSN string -} - -func NewStorage(cfg configs.StorageConfig) (Storage, error) { - return &sqliteStorage{ - DSN: cfg.DSN, - }, nil -} diff --git a/internal/storage_manager/main.go b/internal/storage_manager/main.go new file mode 100644 index 0000000..8995cf8 --- /dev/null +++ b/internal/storage_manager/main.go @@ -0,0 +1,89 @@ +package storage_manager + +import ( + "context" + "fmt" + + "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" +) + +type ( + StorageManager interface { + EnsureTable(ctx context.Context, table string, columns []models.EngineContextColumn, dedupe []string) error + Insert(ctx context.Context, table string, data []map[string]interface{}, columns []models.EngineContextColumn, dedupe []string) error + } + SQLGenerator interface { + GenerateCreateTableSQL(tableName string, schema []models.EngineContextColumn) (string, error) + GenerateInsertSQL(tableName string, data []interface{}, columns []models.EngineContextColumn, dedupeKeys []string) (string, error) + } +) + +type storageManager struct { + storage datastore.DataStoreClient +} + +func NewStorageManager(sc datastore.DataStoreClient) StorageManager { + return &storageManager{storage: sc} +} + +func (m *storageManager) EnsureTable( + ctx context.Context, + table string, + columns []models.EngineContextColumn, + dedupe []string, +) error { + + generator := m.storage.GetSQLGenerator() + if generator == nil { + return fmt.Errorf("storage client has no SQL generator") + } + + createSQL, err := generator.GenerateCreateTableSQL(table, columns) + if err != nil { + return fmt.Errorf("failed to generate create-table SQL: %w", err) + } + + res := m.storage.Exec(createSQL) + if res.Error != nil { + return fmt.Errorf("failed to create table %s: %w", table, res.Error) + } + + return nil +} + +func (m *storageManager) Insert( + ctx context.Context, + table string, + data []map[string]interface{}, + columns []models.EngineContextColumn, + dedupe []string, +) error { + + if len(data) == 0 { + return nil + } + + generator := m.storage.GetSQLGenerator() + if generator == nil { + return fmt.Errorf("storage client has no SQL generator") + } + + // Convert []map[string]interface{} → []interface{} + genericRows := make([]interface{}, len(data)) + for i := range data { + genericRows[i] = data[i] + } + + insertSQL, err := generator.GenerateInsertSQL(table, genericRows, columns, dedupe) + if err != nil { + return fmt.Errorf("failed to generate insert SQL: %w", err) + } + + res := m.storage.Exec(insertSQL) + if res.Error != nil { + return fmt.Errorf("failed to insert batch into %s: %w", table, res.Error) + } + + return nil +} diff --git a/internal/storage_manager/main_test.go b/internal/storage_manager/main_test.go new file mode 100644 index 0000000..a09d111 --- /dev/null +++ b/internal/storage_manager/main_test.go @@ -0,0 +1,160 @@ +package storage_manager + +import ( + "context" + "errors" + "testing" + + "github.com/kvatch-hub/kvatch-runtime/internal/datastore" + "github.com/kvatch-hub/kvatch-runtime/internal/models" + processorMocks "github.com/kvatch-hub/kvatch-runtime/internal/processor/mocks" + mocks "github.com/kvatch-hub/kvatch-runtime/internal/storage_manager/mocks" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestEnsureTable_NoGenerator(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + store := processorMocks.NewMockDataStoreClient(ctrl) + store.EXPECT(). + GetSQLGenerator(). + Return(nil) + + mgr := NewStorageManager(store) + + err := mgr.EnsureTable(context.Background(), "test", nil, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "no SQL generator") +} + +func TestEnsureTable_Success(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + store := processorMocks.NewMockDataStoreClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + cols := []models.EngineContextColumn{ + {Name: "id", Type: "INTEGER"}, + } + + store.EXPECT(). + GetSQLGenerator(). + Return(gen) + + gen.EXPECT(). + GenerateCreateTableSQL("users", cols). + Return("CREATE TABLE users (...)", nil) + + store.EXPECT(). + Exec("CREATE TABLE users (...)"). + Return(&datastore.ExecResult{}) + + mgr := NewStorageManager(store) + + err := mgr.EnsureTable(context.Background(), "users", cols, nil) + require.NoError(t, err) +} + +func TestEnsureTable_ExecError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + store := processorMocks.NewMockDataStoreClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + store.EXPECT().GetSQLGenerator().Return(gen) + gen.EXPECT(). + GenerateCreateTableSQL(gomock.Any(), gomock.Any()). + Return("SQL", nil) + + store.EXPECT(). + Exec("SQL"). + Return(&datastore.ExecResult{Error: errors.New("boom")}) + + mgr := NewStorageManager(store) + + err := mgr.EnsureTable(context.Background(), "t", nil, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "boom") +} + +func TestInsert_EmptyData(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + store := processorMocks.NewMockDataStoreClient(ctrl) + + mgr := NewStorageManager(store) + + err := mgr.Insert(context.Background(), "table", nil, nil, nil) + require.NoError(t, err) +} + +func TestInsert_Success(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + store := processorMocks.NewMockDataStoreClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + data := []map[string]interface{}{ + {"id": 1, "name": "Alice"}, + } + + cols := []models.EngineContextColumn{ + {Name: "id"}, + {Name: "name"}, + } + + store.EXPECT(). + GetSQLGenerator(). + Return(gen) + + gen.EXPECT(). + GenerateInsertSQL( + "users", + gomock.Len(1), + cols, + []string{"id"}, + ). + Return("INSERT SQL", nil) + + store.EXPECT(). + Exec("INSERT SQL"). + Return(&datastore.ExecResult{}) + + mgr := NewStorageManager(store) + + err := mgr.Insert(context.Background(), "users", data, cols, []string{"id"}) + require.NoError(t, err) +} + +func TestInsert_ExecError(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + store := processorMocks.NewMockDataStoreClient(ctrl) + gen := mocks.NewMockSQLGenerator(ctrl) + + data := []map[string]interface{}{ + {"id": 1}, + } + + store.EXPECT().GetSQLGenerator().Return(gen) + gen.EXPECT(). + GenerateInsertSQL(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return("SQL", nil) + + store.EXPECT(). + Exec("SQL"). + Return(&datastore.ExecResult{Error: errors.New("insert failed")}) + + mgr := NewStorageManager(store) + + err := mgr.Insert(context.Background(), "t", data, nil, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "insert failed") +} diff --git a/internal/storage_manager/mocks/mock_interfaces.go b/internal/storage_manager/mocks/mock_interfaces.go new file mode 100644 index 0000000..91f52af --- /dev/null +++ b/internal/storage_manager/mocks/mock_interfaces.go @@ -0,0 +1,124 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/storage_manager/main.go +// +// Generated by this command: +// +// mockgen -source internal/storage_manager/main.go -destination internal/storage_manager/mocks/mock_interfaces.go -package mocks +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + "github.com/kvatch-hub/kvatch-runtime/internal/models" + gomock "go.uber.org/mock/gomock" +) + +// MockStorageManager is a mock of StorageManager interface. +type MockStorageManager struct { + ctrl *gomock.Controller + recorder *MockStorageManagerMockRecorder + isgomock struct{} +} + +// MockStorageManagerMockRecorder is the mock recorder for MockStorageManager. +type MockStorageManagerMockRecorder struct { + mock *MockStorageManager +} + +// NewMockStorageManager creates a new mock instance. +func NewMockStorageManager(ctrl *gomock.Controller) *MockStorageManager { + mock := &MockStorageManager{ctrl: ctrl} + mock.recorder = &MockStorageManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStorageManager) EXPECT() *MockStorageManagerMockRecorder { + return m.recorder +} + +// EnsureTable mocks base method. +func (m *MockStorageManager) EnsureTable(ctx context.Context, table string, columns []models.EngineContextColumn, dedupe []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureTable", ctx, table, columns, dedupe) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureTable indicates an expected call of EnsureTable. +func (mr *MockStorageManagerMockRecorder) EnsureTable(ctx, table, columns, dedupe any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureTable", reflect.TypeOf((*MockStorageManager)(nil).EnsureTable), ctx, table, columns, dedupe) +} + +// Insert mocks base method. +func (m *MockStorageManager) Insert(ctx context.Context, table string, data []map[string]any, columns []models.EngineContextColumn, dedupe []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Insert", ctx, table, data, columns, dedupe) + ret0, _ := ret[0].(error) + return ret0 +} + +// Insert indicates an expected call of Insert. +func (mr *MockStorageManagerMockRecorder) Insert(ctx, table, data, columns, dedupe any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*MockStorageManager)(nil).Insert), ctx, table, data, columns, dedupe) +} + +// MockSQLGenerator is a mock of SQLGenerator interface. +type MockSQLGenerator struct { + ctrl *gomock.Controller + recorder *MockSQLGeneratorMockRecorder + isgomock struct{} +} + +// MockSQLGeneratorMockRecorder is the mock recorder for MockSQLGenerator. +type MockSQLGeneratorMockRecorder struct { + mock *MockSQLGenerator +} + +// NewMockSQLGenerator creates a new mock instance. +func NewMockSQLGenerator(ctrl *gomock.Controller) *MockSQLGenerator { + mock := &MockSQLGenerator{ctrl: ctrl} + mock.recorder = &MockSQLGeneratorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSQLGenerator) EXPECT() *MockSQLGeneratorMockRecorder { + return m.recorder +} + +// GenerateCreateTableSQL mocks base method. +func (m *MockSQLGenerator) GenerateCreateTableSQL(tableName string, schema []models.EngineContextColumn) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateCreateTableSQL", tableName, schema) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GenerateCreateTableSQL indicates an expected call of GenerateCreateTableSQL. +func (mr *MockSQLGeneratorMockRecorder) GenerateCreateTableSQL(tableName, schema any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateCreateTableSQL", reflect.TypeOf((*MockSQLGenerator)(nil).GenerateCreateTableSQL), tableName, schema) +} + +// GenerateInsertSQL mocks base method. +func (m *MockSQLGenerator) GenerateInsertSQL(tableName string, data []any, columns []models.EngineContextColumn, dedupeKeys []string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateInsertSQL", tableName, data, columns, dedupeKeys) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GenerateInsertSQL indicates an expected call of GenerateInsertSQL. +func (mr *MockSQLGeneratorMockRecorder) GenerateInsertSQL(tableName, data, columns, dedupeKeys any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateInsertSQL", reflect.TypeOf((*MockSQLGenerator)(nil).GenerateInsertSQL), tableName, data, columns, dedupeKeys) +} diff --git a/internal/subscription_manager/main.go b/internal/subscription_manager/main.go new file mode 100644 index 0000000..d7104a9 --- /dev/null +++ b/internal/subscription_manager/main.go @@ -0,0 +1,81 @@ +package subscription_manager + +import ( + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + apiconnector "github.com/kvatch-hub/kvatch-runtime/internal/connectors/api" + gitconnector "github.com/kvatch-hub/kvatch-runtime/internal/connectors/git" + googlesheetConnection "github.com/kvatch-hub/kvatch-runtime/internal/connectors/googlesheets" + localDirectoryConnection "github.com/kvatch-hub/kvatch-runtime/internal/connectors/localdirectory" + localFileConnection "github.com/kvatch-hub/kvatch-runtime/internal/connectors/localfile" + postgresConnection "github.com/kvatch-hub/kvatch-runtime/internal/connectors/postgres" + sqliteConnection "github.com/kvatch-hub/kvatch-runtime/internal/connectors/sqlite" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + + apiPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/api" + csvPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/csv" + gsheetPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/gsheet" + jsonPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/json" + psqlPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/postgres" + sqlitePlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/sqlite" + yamlPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/yaml" +) + +type ( + ConnectorRegistry interface { + Register(connectionType connectors.ConnectorType, connection connectors.Connector) error + Get(connectionType connectors.ConnectorType) (connectors.Connector, error) + List() []connectors.ConnectorType + } + DataPluginRegistry interface { + Register(pluginType plugins.PluginType, plugin plugins.Plugin) error + Get(pluginType plugins.PluginType) (plugins.Plugin, error) + List() []plugins.PluginType + } +) + +type SubscriptionManager struct { + connectorRegistry ConnectorRegistry + pluginRegistry DataPluginRegistry +} + +func NewSubscriptionManagerWithDefaults() *SubscriptionManager { + manager := &SubscriptionManager{ + pluginRegistry: plugins.NewDataPluginRegistry(), + connectorRegistry: connectors.NewConnectorRegistry(), + } + + // Register default plugins + manager.registerDefaultPlugins() + + // // Register default connections + manager.registerDefaultDataConnectors() + return manager +} + +func (m *SubscriptionManager) registerDefaultPlugins() { + _ = m.pluginRegistry.Register(plugins.PluginTypeJSON, jsonPlugin.NewJSONDataPlugin()) + _ = m.pluginRegistry.Register(plugins.PluginTypeCSV, csvPlugin.NewCSVDataPlugin()) + _ = m.pluginRegistry.Register(plugins.PluginTypeSQL, sqlitePlugin.NewSQLiteDataPlugin()) + _ = m.pluginRegistry.Register(plugins.PluginTypeSQL, psqlPlugin.NewPostgresDataPlugin()) + _ = m.pluginRegistry.Register(plugins.PluginTypeYAML, yamlPlugin.NewYAMLDataPlugin()) + _ = m.pluginRegistry.Register(plugins.PluginTypeGoogleSheet, gsheetPlugin.NewGoogleSheetsDataPlugin()) + _ = m.pluginRegistry.Register(plugins.PluginTypeAPI, apiPlugin.NewAPIDatasetPlugin()) +} + +func (m *SubscriptionManager) registerDefaultDataConnectors() { + _ = m.connectorRegistry.Register(connectors.ConnectorTypeLocalFile, localFileConnection.NewLocalFileConnector()) + _ = m.connectorRegistry.Register(connectors.ConnectorTypePostgres, postgresConnection.NewPostgresConnector()) + _ = m.connectorRegistry.Register(connectors.ConnectorTypeSQLite, sqliteConnection.NewSQLiteConnector()) + _ = m.connectorRegistry.Register(connectors.ConnectorTypeGit, gitconnector.NewGitConnector()) + _ = m.connectorRegistry.Register(connectors.ConnectorTypeLocalDirectory, localDirectoryConnection.NewDirectoryConnector()) + _ = m.connectorRegistry.Register(connectors.ConnectorTypeGoogleSheet, googlesheetConnection.NewGoogleSheetConnector()) + _ = m.connectorRegistry.Register(connectors.ConnectorTypeAPI, apiconnector.NewAPIConnector()) +} + +func (m *SubscriptionManager) GetRegisteredConnectors() ConnectorRegistry { + return m.connectorRegistry +} + +func (m *SubscriptionManager) GetRegisteredPlugins() DataPluginRegistry { + return m.pluginRegistry +} diff --git a/internal/subscription_manager/main_test.go b/internal/subscription_manager/main_test.go new file mode 100644 index 0000000..cab042b --- /dev/null +++ b/internal/subscription_manager/main_test.go @@ -0,0 +1,119 @@ +package subscription_manager + +import ( + "testing" + + "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + "github.com/kvatch-hub/kvatch-runtime/internal/subscription_manager/mocks" + "go.uber.org/mock/gomock" +) + +//go:generate mockgen -destination=./mocks_test.go -package=subscription . ConnectorRegistry,DataPluginRegistry + +func Test_registerDefaultPlugins_usesRegistry(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockPluginReg := mocks.NewMockDataPluginRegistry(ctrl) + + // We don't assert on concrete plugin instances, just that Register is called + // with the correct plugin types. gomock.Any() for the plugin value. + gomock.InOrder( + mockPluginReg.EXPECT().Register(plugins.PluginTypeJSON, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(plugins.PluginTypeCSV, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(plugins.PluginTypeSQL, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(plugins.PluginTypeSQL, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(plugins.PluginTypeYAML, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(plugins.PluginTypeGoogleSheet, gomock.Any()).Return(nil), + mockPluginReg.EXPECT().Register(plugins.PluginTypeAPI, gomock.Any()).Return(nil), + ) + + // We don't care about connectors here, so a nil is fine; we won't touch it. + m := &SubscriptionManager{ + connectorRegistry: nil, + pluginRegistry: mockPluginReg, + } + + m.registerDefaultPlugins() +} + +func Test_registerDefaultDataConnectors_usesRegistry(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockConnReg := mocks.NewMockConnectorRegistry(ctrl) + + gomock.InOrder( + mockConnReg.EXPECT().Register(connectors.ConnectorTypeLocalFile, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(connectors.ConnectorTypePostgres, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(connectors.ConnectorTypeSQLite, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(connectors.ConnectorTypeGit, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(connectors.ConnectorTypeLocalDirectory, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(connectors.ConnectorTypeGoogleSheet, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(connectors.ConnectorTypeAPI, gomock.Any()).Return(nil), + ) + + m := &SubscriptionManager{ + connectorRegistry: mockConnReg, + pluginRegistry: nil, + } + + m.registerDefaultDataConnectors() +} + +func Test_NewSubscriptionManagerWithDefaults_registersEverything(t *testing.T) { + m := NewSubscriptionManagerWithDefaults() + + // Check plugins list contains all expected types + gotPlugins := map[plugins.PluginType]bool{} + for _, pt := range m.pluginRegistry.List() { + gotPlugins[pt] = true + } + wantPlugins := []plugins.PluginType{ + plugins.PluginTypeJSON, + plugins.PluginTypeCSV, + plugins.PluginTypeSQL, + } + for _, w := range wantPlugins { + if !gotPlugins[w] { + t.Fatalf("expected plugin type %v to be registered", w) + } + // Smoke test Get(...) returns something without error + if _, err := m.pluginRegistry.Get(w); err != nil { + t.Fatalf("expected plugin %v to be retrievable: %v", w, err) + } + } + + // Check connectors list contains all expected types + gotConns := map[connectors.ConnectorType]bool{} + for _, ct := range m.connectorRegistry.List() { + gotConns[ct] = true + } + wantConns := []connectors.ConnectorType{ + connectors.ConnectorTypeLocalDirectory, + connectors.ConnectorTypeLocalFile, + connectors.ConnectorTypePostgres, + connectors.ConnectorTypeSQLite, + } + for _, w := range wantConns { + if !gotConns[w] { + t.Fatalf("expected connection type %v to be registered", w) + } + if _, err := m.connectorRegistry.Get(w); err != nil { + t.Fatalf("expected connector %v to be retrievable: %v", w, err) + } + } +} + +func Test_Getters_returnSameRegistries(t *testing.T) { + // Using the real manager here is fine; just verify getters wire through. + m := NewSubscriptionManagerWithDefaults() + + if m.GetRegisteredConnectors() == nil { + t.Fatal("GetRegisteredConnectors returned nil") + } + if m.GetRegisteredPlugins() == nil { + t.Fatal("GetRegisteredPlugins returned nil") + } +} diff --git a/internal/subscription_manager/mocks/mock_interfaces.go b/internal/subscription_manager/mocks/mock_interfaces.go new file mode 100644 index 0000000..be32335 --- /dev/null +++ b/internal/subscription_manager/mocks/mock_interfaces.go @@ -0,0 +1,152 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: internal/subscription_manager/main.go +// +// Generated by this command: +// +// mockgen -source internal/subscription_manager/main.go -destination internal/subscription_manager/mocks/mock_interfaces.go -package mocks +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + connectors "github.com/kvatch-hub/kvatch-runtime/internal/connectors" + plugins "github.com/kvatch-hub/kvatch-runtime/internal/plugins" + gomock "go.uber.org/mock/gomock" +) + +// MockConnectorRegistry is a mock of ConnectorRegistry interface. +type MockConnectorRegistry struct { + ctrl *gomock.Controller + recorder *MockConnectorRegistryMockRecorder + isgomock struct{} +} + +// MockConnectorRegistryMockRecorder is the mock recorder for MockConnectorRegistry. +type MockConnectorRegistryMockRecorder struct { + mock *MockConnectorRegistry +} + +// NewMockConnectorRegistry creates a new mock instance. +func NewMockConnectorRegistry(ctrl *gomock.Controller) *MockConnectorRegistry { + mock := &MockConnectorRegistry{ctrl: ctrl} + mock.recorder = &MockConnectorRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConnectorRegistry) EXPECT() *MockConnectorRegistryMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockConnectorRegistry) Get(connectionType connectors.ConnectorType) (connectors.Connector, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", connectionType) + ret0, _ := ret[0].(connectors.Connector) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockConnectorRegistryMockRecorder) Get(connectionType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockConnectorRegistry)(nil).Get), connectionType) +} + +// List mocks base method. +func (m *MockConnectorRegistry) List() []connectors.ConnectorType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List") + ret0, _ := ret[0].([]connectors.ConnectorType) + return ret0 +} + +// List indicates an expected call of List. +func (mr *MockConnectorRegistryMockRecorder) List() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockConnectorRegistry)(nil).List)) +} + +// Register mocks base method. +func (m *MockConnectorRegistry) Register(connectionType connectors.ConnectorType, connection connectors.Connector) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Register", connectionType, connection) + ret0, _ := ret[0].(error) + return ret0 +} + +// Register indicates an expected call of Register. +func (mr *MockConnectorRegistryMockRecorder) Register(connectionType, connection any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Register", reflect.TypeOf((*MockConnectorRegistry)(nil).Register), connectionType, connection) +} + +// MockDataPluginRegistry is a mock of DataPluginRegistry interface. +type MockDataPluginRegistry struct { + ctrl *gomock.Controller + recorder *MockDataPluginRegistryMockRecorder + isgomock struct{} +} + +// MockDataPluginRegistryMockRecorder is the mock recorder for MockDataPluginRegistry. +type MockDataPluginRegistryMockRecorder struct { + mock *MockDataPluginRegistry +} + +// NewMockDataPluginRegistry creates a new mock instance. +func NewMockDataPluginRegistry(ctrl *gomock.Controller) *MockDataPluginRegistry { + mock := &MockDataPluginRegistry{ctrl: ctrl} + mock.recorder = &MockDataPluginRegistryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDataPluginRegistry) EXPECT() *MockDataPluginRegistryMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockDataPluginRegistry) Get(pluginType plugins.PluginType) (plugins.Plugin, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", pluginType) + ret0, _ := ret[0].(plugins.Plugin) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockDataPluginRegistryMockRecorder) Get(pluginType any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockDataPluginRegistry)(nil).Get), pluginType) +} + +// List mocks base method. +func (m *MockDataPluginRegistry) List() []plugins.PluginType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List") + ret0, _ := ret[0].([]plugins.PluginType) + return ret0 +} + +// List indicates an expected call of List. +func (mr *MockDataPluginRegistryMockRecorder) List() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockDataPluginRegistry)(nil).List)) +} + +// Register mocks base method. +func (m *MockDataPluginRegistry) Register(pluginType plugins.PluginType, plugin plugins.Plugin) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Register", pluginType, plugin) + ret0, _ := ret[0].(error) + return ret0 +} + +// Register indicates an expected call of Register. +func (mr *MockDataPluginRegistryMockRecorder) Register(pluginType, plugin any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Register", reflect.TypeOf((*MockDataPluginRegistry)(nil).Register), pluginType, plugin) +} diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 7bda510..ee5dadd 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -2,12 +2,17 @@ package runtime import ( "context" + "fmt" "github.com/kvatch-hub/kvatch-runtime/configs" "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/internal/datastore" "github.com/kvatch-hub/kvatch-runtime/internal/engine" + "github.com/kvatch-hub/kvatch-runtime/internal/engine_context" "github.com/kvatch-hub/kvatch-runtime/internal/execute" - "github.com/kvatch-hub/kvatch-runtime/internal/storage" + "github.com/kvatch-hub/kvatch-runtime/internal/processor" + "github.com/kvatch-hub/kvatch-runtime/internal/response_builder" + "github.com/kvatch-hub/kvatch-runtime/internal/subscription_manager" ) type Runtime interface { @@ -22,17 +27,48 @@ func New(cfg configs.Config) (Runtime, error) { return newRuntime(cfg) } +// NewDefault creates a Runtime with sensible defaults using a local SQLite database. +// This is intended for quick start, testing, and local development. +func NewDefault() (Runtime, error) { + return newRuntime(configs.Config{ + Storage: configs.StorageConfig{ + Driver: "sqlite", + DSN: "./kvatch.db", + }, + }) +} + +func NewWithSQLite(dsn string) (Runtime, error) { + return newRuntime(configs.Config{ + Storage: configs.StorageConfig{ + Driver: "sqlite", + DSN: dsn, + }, + }) +} + func (r *runtimeImpl) ExecutePlan(ctx context.Context, req entities.ExecutePlanRequest) (*entities.ExecutePlanResponse, error) { return r.service.Execute(ctx, req) } func newRuntime(cfg configs.Config) (Runtime, error) { - st, err := storage.NewStorage(cfg.Storage) + store, err := datastore.NewSQLiteDataStoreClient(cfg.Storage.DSN) if err != nil { return nil, err } - eng := engine.NewEngine(st) + subSvc := subscription_manager.NewSubscriptionManagerWithDefaults() + if subSvc == nil { + return nil, fmt.Errorf("sub svc not registered") + } + + processor := processor.NewProcessor( + store, + subSvc.GetRegisteredConnectors(), + subSvc.GetRegisteredPlugins(), + processor.WithVerbose(true), + ) + eng := engine.NewEngine(&engine_context.Builder{}, store, processor, &response_builder.ResponseBuilder{}) svc := execute.NewService(eng) return &runtimeImpl{ diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go new file mode 100644 index 0000000..b5a4249 --- /dev/null +++ b/pkg/runtime/runtime_test.go @@ -0,0 +1,64 @@ +package runtime + +import ( + "context" + "log" + "testing" + + "github.com/kvatch-hub/kvatch-runtime/configs" + "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/stretchr/testify/require" +) + +func TestSimpleBaseCase(t *testing.T) { + rt, err := New(configs.Config{ + Storage: configs.StorageConfig{ + Driver: "sqlite", + DSN: "./test.db", + }, + }) + if err != nil { + log.Fatalf("create runtime: %v", err) + } + + plan := entities.Plan{ + Name: "example-plan", + Output: entities.Output{ + DatasetName: "books", + }, + Connectors: []entities.Connector{ + { + Name: "local", + Type: "INTERNAL", + }, + }, + Datasets: []entities.Dataset{ + { + Name: "books", + ConnectorName: "local", + Type: "JSON", + Data: []map[string]any{ + {"id": 1, "title": "Dune"}, + {"id": 2, "title": "Neuromancer"}, + }, + Options: map[string]any{ + "timeout": 30, + }, + }, + }, + } + + got, gotErr := rt.ExecutePlan(context.Background(), entities.ExecutePlanRequest{ + UserID: "someid", + Plan: plan, + }) + if gotErr != nil { + log.Fatalf("execute plan: %v", err) + } + + expectedColumns := 7 + expectedRows := 2 + + require.Equalf(t, expectedColumns, len(got.Columns), "expected %d columns, got %d", expectedColumns, len(got.Columns)) + require.Equalf(t, expectedRows, len(got.Data), "expected %d rows, got %d", expectedRows, len(got.Data)) +} diff --git a/pkg/runtime/types.go b/pkg/runtime/types.go deleted file mode 100644 index a84d221..0000000 --- a/pkg/runtime/types.go +++ /dev/null @@ -1,68 +0,0 @@ -package runtime - -// type Source struct { -// Type string -// Config map[string]any -// } - -// type ExecutePlanRequest struct { -// UserID string -// Plan Plan -// } - -// type ExecutePlanResponse struct { -// Columns []string -// Rows [][]any -// } - -// type Plan struct { -// Name string -// Storage Storage -// Connectors []Connector -// Datasets []Dataset -// Output Output -// } - -// type Storage struct { -// Type string -// MetadataStorePath string -// DataStorePath string -// } - -// type Connector struct { -// Name string -// Type string -// Connection any -// Description string -// } - -// type Dataset struct { -// Name string -// ConnectorName string -// Type string -// Description string -// Query string -// Options map[string]any -// Data any -// Children []DatasetChild -// Dedupe []string -// Columns []DataColumn -// ColumnOrder []string -// } - -// type DatasetChild struct { -// DatasetName string -// } - -// type Output struct { -// DatasetName string -// } - -// type DataColumn struct { -// Name string -// Accessor string -// Alias string -// Type string -// Description string -// IsPrimaryKey bool -// } diff --git a/pkg/utils/conversion.go b/pkg/utils/conversion.go new file mode 100644 index 0000000..6903c58 --- /dev/null +++ b/pkg/utils/conversion.go @@ -0,0 +1,41 @@ +package utils + +import ( + "fmt" + "strconv" +) + +func GetString(data map[string]interface{}, keys ...string) string { + for _, key := range keys { // Loop through all provided keys + if val, ok := data[key]; ok { // Check if the key exists + if strVal, ok := val.(string); ok { // Check if the value is a string + return strVal // Success: return the string and stop + } + } + } + return "" // Failure: return an empty string after checking all keys +} + +func ToInt(val interface{}) (int, error) { + if val == nil { + return 0, fmt.Errorf("value is nil") + } + + switch v := val.(type) { + case int: + return v, nil + case int64: + return int(v), nil + case float64: + // Commonly used for numbers in JSON/YAML parsing + return int(v), nil + case string: + // Handle ports or numbers passed as strings + if parsed, err := strconv.Atoi(v); err == nil { + return parsed, nil + } + return 0, fmt.Errorf("cannot convert string '%s' to int", v) + default: + return 0, fmt.Errorf("unsupported type for int conversion: %T", v) + } +} diff --git a/pkg/utils/conversion_test.go b/pkg/utils/conversion_test.go new file mode 100644 index 0000000..f2b3eed --- /dev/null +++ b/pkg/utils/conversion_test.go @@ -0,0 +1,147 @@ +package utils + +import ( + "testing" +) + +func TestGetString(t *testing.T) { + testMap := map[string]interface{}{ + "snake_case_key": "snake_value", + "camelCaseKey": "camel_value", + "int_key": 123, + "empty_key": "", + } + + tests := []struct { + name string + keys []string + expected string + }{ + { + name: "Single_Key_Found_Snake", + keys: []string{"snake_case_key"}, + expected: "snake_value", + }, + { + name: "Single_Key_Found_Camel", + keys: []string{"camelCaseKey"}, + expected: "camel_value", + }, + { + name: "Key_Not_Found", + keys: []string{"missing_key"}, + expected: "", + }, + { + name: "Value_Not_A_String", + keys: []string{"int_key"}, + expected: "", // Should return "" because the value is an int, not a string + }, + { + name: "Key_Found_With_Empty_String", + keys: []string{"empty_key"}, + expected: "", // Correctly handles an empty string value + }, + { + name: "Multi_Key_First_Match", + keys: []string{"snake_case_key", "camelCaseKey"}, + expected: "snake_value", // Should find the first key + }, + { + name: "Multi_Key_Second_Match", + keys: []string{"missing_1", "camelCaseKey", "missing_2"}, + expected: "camel_value", // Should skip missing keys and find the second + }, + { + name: "Multi_Key_No_Match", + keys: []string{"missing_1", "missing_2", "missing_3"}, + expected: "", + }, + { + name: "Multi_Key_Wrong_Type_Fallback", + keys: []string{"int_key", "camelCaseKey"}, + expected: "camel_value", // Should skip "int_key" (wrong type) and find "camelCaseKey" + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := GetString(testMap, tt.keys...) + if result != tt.expected { + t.Errorf("GetString(keys: %v) got %q, want %q", tt.keys, result, tt.expected) + } + }) + } +} + +func TestToInt(t *testing.T) { + tests := []struct { + name string + input interface{} + expected int + wantErr bool + }{ + { + name: "Input_int", + input: 123, + expected: 123, + wantErr: false, + }, + { + name: "Input_int64", + input: int64(987654), + expected: 987654, + wantErr: false, + }, + { + name: "Input_float64_integer", + input: 3.0, + expected: 3, + wantErr: false, + }, + { + name: "Input_float64_with_decimal", + input: 3.14159, + expected: 3, // Floating point conversion to int truncates + wantErr: false, + }, + { + name: "Input_string_valid", + input: "456", + expected: 456, + wantErr: false, + }, + { + name: "Input_string_invalid", + input: "abc", + expected: 0, + wantErr: true, + }, + { + name: "Input_nil", + input: nil, + expected: 0, + wantErr: true, + }, + { + name: "Input_unsupported_type_bool", + input: true, + expected: 0, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := ToInt(tt.input) + + if (err != nil) != tt.wantErr { + t.Fatalf("ToInt(%v) error status mismatch. Got error: %v, Want error: %t", tt.input, err, tt.wantErr) + } + + if !tt.wantErr && result != tt.expected { + t.Errorf("ToInt(%v) got %d, want %d", tt.input, result, tt.expected) + } + }) + } +} diff --git a/pkg/utils/inference.go b/pkg/utils/inference.go new file mode 100644 index 0000000..24b9a9a --- /dev/null +++ b/pkg/utils/inference.go @@ -0,0 +1,38 @@ +package utils + +import ( + "strconv" + "time" +) + +func IsWholeNumber(f float64) bool { + return f == float64(int64(f)) +} + +func IsWholeNumberString(s string) bool { + _, err := strconv.ParseInt(s, 10, 64) + return err == nil +} + +func IsLikelyNumber(s string) bool { + _, err := strconv.ParseFloat(s, 64) + return err == nil +} + +var CommonTimestampLayouts = []string{ + time.RFC3339, + "2006-01-02 15:04:05", + "2006-01-02", + "2006-01-02 15:04:05.999999", + "2006-01-02 15:04:05Z07:00", + "2006-01-02T15:04:05.999999Z07:00", +} + +func IsLikelyTimestamp(s string) bool { + for _, layout := range CommonTimestampLayouts { + if _, err := time.Parse(layout, s); err == nil { + return true + } + } + return false +}