From 1fe7caf93be94a1ed0132bb45c8e4a940b729107 Mon Sep 17 00:00:00 2001 From: Jamie Wooltorton Date: Tue, 7 Apr 2026 14:01:42 +0200 Subject: [PATCH 1/3] fix: address core public release readiness gaps --- RELEASE_READINESS_REVIEW.md | 19 +++++++++++++++++++ internal/connectors/aws_s3/connector.go | 7 +++++++ internal/connectors/git/connector.go | 16 ---------------- internal/subscription_manager/main.go | 5 +++-- internal/subscription_manager/main_test.go | 5 ++++- pkg/runtime/runtime.go | 18 +++++++++++++++++- pkg/runtime/runtime_test.go | 21 +++++++++++++++++++++ 7 files changed, 71 insertions(+), 20 deletions(-) create mode 100644 RELEASE_READINESS_REVIEW.md diff --git a/RELEASE_READINESS_REVIEW.md b/RELEASE_READINESS_REVIEW.md new file mode 100644 index 0000000..8574980 --- /dev/null +++ b/RELEASE_READINESS_REVIEW.md @@ -0,0 +1,19 @@ +# Public Release Readiness Review (April 7, 2026) + +Status: **Not ready** for a clean first open-source release without targeted cleanup. + +## Highest-priority concerns + +1. Public API/config mismatch (`runtime.New` ignores storage driver and always creates SQLite client). +2. Declared connector types are broader than registered defaults (e.g. S3 declared but not registered). +3. Internal-only operational details leak into runtime output (git connector prints filesystem paths and tree contents). +4. Public package surface and naming are inconsistent (`model` + `configs` + `entities`, typos, mixed naming styles). +5. Documentation lacks clear stability guarantees and API boundaries for external contributors. + +## Release-focused recommendations + +- Tighten public API surface to only the packages intended for users (`pkg/runtime`, `model`, and stable config package). +- Ensure declared connector/plugin types and runtime registrations are aligned. +- Remove debug prints and commented-out blocks from runtime-facing paths. +- Publish an explicit support matrix (connectors, dataset types, auth modes, and production readiness). +- Add tests for public constructors and capability registration consistency. diff --git a/internal/connectors/aws_s3/connector.go b/internal/connectors/aws_s3/connector.go index b7d16b2..499df28 100644 --- a/internal/connectors/aws_s3/connector.go +++ b/internal/connectors/aws_s3/connector.go @@ -34,6 +34,13 @@ func (c *S3Connector) GetType() model.ConnectorType { return model.ConnectorTypeS3 } +func (c *S3Connector) Ping(ctx context.Context, cfg interface{}) error { + if err := c.Connect(ctx, &models.EngineContextConnector{Config: cfg}); err != nil { + return err + } + return c.Close() +} + func (c *S3Connector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error { if engineConnectorCtx == nil { return fmt.Errorf("engine connector context is missing") diff --git a/internal/connectors/git/connector.go b/internal/connectors/git/connector.go index 7c6aea4..29df705 100644 --- a/internal/connectors/git/connector.go +++ b/internal/connectors/git/connector.go @@ -168,9 +168,6 @@ func (c *GitConnector) Connect(ctx context.Context, engineConnectorCtx *models.E return err } - fmt.Printf("📁 Resolved repo root path: %s\n", c.rootPath) - printDirRecursive(c.rootPath) - return nil } @@ -248,19 +245,6 @@ func hashPath(repo, ref string) string { return fmt.Sprintf("%x", sum[:])[:10] } -func printDirRecursive(path string) { - fmt.Printf("📦 Contents of %s:\n", path) - _ = filepath.Walk(path, func(p string, info os.FileInfo, err error) error { - if err == nil { - rel, _ := filepath.Rel(path, p) - if rel != "." { - fmt.Println(" -", rel) - } - } - return nil - }) -} - func parseConfig(config interface{}) (*model.GitConnectorConfig, error) { switch v := config.(type) { case *model.GitConnectorConfig: diff --git a/internal/subscription_manager/main.go b/internal/subscription_manager/main.go index b755382..411dccd 100644 --- a/internal/subscription_manager/main.go +++ b/internal/subscription_manager/main.go @@ -3,6 +3,7 @@ package subscription_manager import ( "github.com/kvatch-hub/kvatch-runtime/internal/connectors" apiconnector "github.com/kvatch-hub/kvatch-runtime/internal/connectors/api" + s3connector "github.com/kvatch-hub/kvatch-runtime/internal/connectors/aws_s3" gitconnector "github.com/kvatch-hub/kvatch-runtime/internal/connectors/git" googlesheetConnection "github.com/kvatch-hub/kvatch-runtime/internal/connectors/googlesheets" localDirectoryConnection "github.com/kvatch-hub/kvatch-runtime/internal/connectors/localdirectory" @@ -16,7 +17,6 @@ import ( csvPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/csv" gsheetPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/gsheet" jsonPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/json" - psqlPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/postgres" sqlitePlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/sqlite" yamlPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/yaml" ) @@ -57,7 +57,6 @@ func (m *SubscriptionManager) registerDefaultPlugins() { _ = m.pluginRegistry.Register(model.DatasetTypeJSON, jsonPlugin.NewJSONDataPlugin()) _ = m.pluginRegistry.Register(model.DatasetTypeCSV, csvPlugin.NewCSVDataPlugin()) _ = m.pluginRegistry.Register(model.DatasetTypeSQL, sqlitePlugin.NewSQLiteDataPlugin()) - _ = m.pluginRegistry.Register(model.DatasetTypeSQL, psqlPlugin.NewPostgresDataPlugin()) _ = m.pluginRegistry.Register(model.DatasetTypeYAML, yamlPlugin.NewYAMLDataPlugin()) _ = m.pluginRegistry.Register(model.DatasetTypeGoogleSheet, gsheetPlugin.NewGoogleSheetsDataPlugin()) _ = m.pluginRegistry.Register(model.DatasetTypeAPI, apiPlugin.NewAPIDatasetPlugin()) @@ -68,9 +67,11 @@ func (m *SubscriptionManager) registerDefaultDataConnectors() { _ = m.connectorRegistry.Register(model.ConnectorTypePostgres, postgresConnection.NewPostgresConnector()) _ = m.connectorRegistry.Register(model.ConnectorTypeSQLite, sqliteConnection.NewSQLiteConnector()) _ = m.connectorRegistry.Register(model.ConnectorTypeGit, gitconnector.NewGitConnector()) + _ = m.connectorRegistry.Register(model.ConnectorTypeS3, s3connector.NewS3Connector()) _ = m.connectorRegistry.Register(model.ConnectorTypeLocalDirectory, localDirectoryConnection.NewDirectoryConnector()) _ = m.connectorRegistry.Register(model.ConnectorTypeGoogleSheet, googlesheetConnection.NewGoogleSheetConnector()) _ = m.connectorRegistry.Register(model.ConnectorTypeAPI, apiconnector.NewAPIConnector()) + _ = m.connectorRegistry.Register(model.ConnectorTypeGoogleAPI, apiconnector.NewAPIConnector()) } func (m *SubscriptionManager) GetRegisteredConnectors() ConnectorRegistry { diff --git a/internal/subscription_manager/main_test.go b/internal/subscription_manager/main_test.go index 17300e4..92b0cc3 100644 --- a/internal/subscription_manager/main_test.go +++ b/internal/subscription_manager/main_test.go @@ -22,7 +22,6 @@ func Test_registerDefaultPlugins_usesRegistry(t *testing.T) { mockPluginReg.EXPECT().Register(model.DatasetTypeJSON, gomock.Any()).Return(nil), mockPluginReg.EXPECT().Register(model.DatasetTypeCSV, gomock.Any()).Return(nil), mockPluginReg.EXPECT().Register(model.DatasetTypeSQL, gomock.Any()).Return(nil), - mockPluginReg.EXPECT().Register(model.DatasetTypeSQL, gomock.Any()).Return(nil), mockPluginReg.EXPECT().Register(model.DatasetTypeYAML, gomock.Any()).Return(nil), mockPluginReg.EXPECT().Register(model.DatasetTypeGoogleSheet, gomock.Any()).Return(nil), mockPluginReg.EXPECT().Register(model.DatasetTypeAPI, gomock.Any()).Return(nil), @@ -48,9 +47,11 @@ func Test_registerDefaultDataConnectors_usesRegistry(t *testing.T) { mockConnReg.EXPECT().Register(model.ConnectorTypePostgres, gomock.Any()).Return(nil), mockConnReg.EXPECT().Register(model.ConnectorTypeSQLite, gomock.Any()).Return(nil), mockConnReg.EXPECT().Register(model.ConnectorTypeGit, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(model.ConnectorTypeS3, gomock.Any()).Return(nil), mockConnReg.EXPECT().Register(model.ConnectorTypeLocalDirectory, gomock.Any()).Return(nil), mockConnReg.EXPECT().Register(model.ConnectorTypeGoogleSheet, gomock.Any()).Return(nil), mockConnReg.EXPECT().Register(model.ConnectorTypeAPI, gomock.Any()).Return(nil), + mockConnReg.EXPECT().Register(model.ConnectorTypeGoogleAPI, gomock.Any()).Return(nil), ) m := &SubscriptionManager{ @@ -94,6 +95,8 @@ func Test_NewSubscriptionManagerWithDefaults_registersEverything(t *testing.T) { model.ConnectorTypeLocalFile, model.ConnectorTypePostgres, model.ConnectorTypeSQLite, + model.ConnectorTypeS3, + model.ConnectorTypeGoogleAPI, } for _, w := range wantConns { if !gotConns[w] { diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 5f6477c..69e4df5 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -3,6 +3,7 @@ package runtime import ( "context" "fmt" + "strings" "github.com/kvatch-hub/kvatch-runtime/configs" "github.com/kvatch-hub/kvatch-runtime/internal/datastore" @@ -52,7 +53,7 @@ func (r *runtimeImpl) ExecutePlan(ctx context.Context, req model.ExecutePlanRequ } func newRuntime(cfg configs.Config) (Runtime, error) { - store, err := datastore.NewSQLiteDataStoreClient(cfg.Storage.DSN) + store, err := newDataStoreClient(cfg) if err != nil { return nil, err } @@ -75,3 +76,18 @@ func newRuntime(cfg configs.Config) (Runtime, error) { service: svc, }, nil } + +func newDataStoreClient(cfg configs.Config) (datastore.DataStoreClient, error) { + driver := strings.TrimSpace(strings.ToLower(cfg.Storage.Driver)) + switch driver { + case "", "sqlite": + return datastore.NewSQLiteDataStoreClient(cfg.Storage.DSN) + case "postgres": + if strings.TrimSpace(cfg.Storage.DSN) == "" { + return nil, fmt.Errorf("storage.dsn is required when storage.driver is postgres") + } + return datastore.NewPostgresDataStoreClient(cfg.Storage.DSN, "") + default: + return nil, fmt.Errorf("unsupported storage driver %q", cfg.Storage.Driver) + } +} diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index 9f35ef2..2464183 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -62,3 +62,24 @@ func TestSimpleBaseCase(t *testing.T) { require.Equalf(t, expectedColumns, len(got.Columns), "expected %d columns, got %d", expectedColumns, len(got.Columns)) require.Equalf(t, expectedRows, len(got.Data), "expected %d rows, got %d", expectedRows, len(got.Data)) } + +func TestNew_UnsupportedDriver(t *testing.T) { + _, err := New(configs.Config{ + Storage: configs.StorageConfig{ + Driver: "mongodb", + DSN: "ignored", + }, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported storage driver") +} + +func TestNew_PostgresWithoutDSN(t *testing.T) { + _, err := New(configs.Config{ + Storage: configs.StorageConfig{ + Driver: "postgres", + }, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "storage.dsn is required") +} From 821c8f06be40bd8008c80206be11d8c7a6c8d179 Mon Sep 17 00:00:00 2001 From: Jamie Wooltorton Date: Tue, 7 Apr 2026 14:11:28 +0200 Subject: [PATCH 2/3] chore: apply post-release-readiness cleanup updates --- README.md | 24 ++++++++++- examples/quickstart/main.go | 4 +- internal/connectors/api/connector.go | 42 +++---------------- .../{bulder.go => builder.go} | 0 4 files changed, 31 insertions(+), 39 deletions(-) rename internal/response_builder/{bulder.go => builder.go} (100%) diff --git a/README.md b/README.md index c6b0e96..2c385e2 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ No ETL. No pipelines. Just declarative data access. --- -## 🚀 Quickstart +## ▶️ Run the Postgres Example ```bash git clone https://github.com/kvatch-hub/kvatch-runtime @@ -165,6 +165,28 @@ go run ./examples/postgres --- +## 🧭 Public API Boundary + +The packages intended for external use are: + +- `pkg/runtime` +- `model` +- `configs` + +Everything under `internal/` is implementation detail and may change without notice. + +--- + +## 🔌 Connector Support (current) + +- Stable/primary: `postgres`, `sqlite`, `api`, `localfile`, `local_directory` +- Available: `googlesheet`, `git`, `s3`, `google_api` +- Special runtime-only connector: `federated` + +For early releases, treat non-primary connectors as evolving APIs. + +--- + ## 💡 Examples - Track a crypto portfolio → `examples/crypto_portfolio` diff --git a/examples/quickstart/main.go b/examples/quickstart/main.go index 8701fbf..0851f32 100644 --- a/examples/quickstart/main.go +++ b/examples/quickstart/main.go @@ -24,14 +24,14 @@ func main() { Connectors: []model.Connector{ { Name: "local", - Type: "internal", + Type: model.ConnectorTypeInternal, }, }, Datasets: []model.Dataset{ { Name: "books", ConnectorName: "local", - Type: "json", + Type: model.DatasetTypeJSON, Data: []map[string]any{ {"id": 1, "title": "Dune"}, {"id": 2, "title": "Neuromancer"}, diff --git a/internal/connectors/api/connector.go b/internal/connectors/api/connector.go index af4edd2..51aba48 100644 --- a/internal/connectors/api/connector.go +++ b/internal/connectors/api/connector.go @@ -129,31 +129,12 @@ func parseConfig(config interface{}) (model.APIConnectorConfig, error) { } } -// func parseAPIOptions(raw map[string]interface{}) (*model.APIDatasetOptions, error) { -// if raw == nil { -// return nil, fmt.Errorf("missing api dataset options") -// } - -// // Convert raw map → JSON → typed struct -// b, err := json.Marshal(raw) -// if err != nil { -// return nil, err -// } - -// var out model.APIDatasetOptions -// if err := json.Unmarshal(b, &out); err != nil { -// return nil, err -// } - -// return &out, nil -// } - // BuildRequest constructs an http.Request from the API request spec, // template params, and optional pagination state. func (c *APIConnector) BuildRequest( ctx context.Context, reqCfg *model.APIRequestSpec, - query_vars map[string]string, + queryVars map[string]string, page *paginationState, ) (*http.Request, error) { @@ -165,7 +146,7 @@ func (c *APIConnector) BuildRequest( // 1. Manual base_url + path join (to avoid ResolveReference bugs) // -------------------------------------------------------------------- base := strings.TrimRight(c.config.BaseURL, "/") - path, err := renderTemplateValue(reqCfg.Path, query_vars) + path, err := renderTemplateValue(reqCfg.Path, queryVars) if err != nil { return nil, fmt.Errorf("api: failed to render path: %w", err) } @@ -183,7 +164,7 @@ func (c *APIConnector) BuildRequest( // -------------------------------------------------------------------- q := fullURL.Query() for k, v := range reqCfg.Params { - rendered, err := renderTemplateValue(v, query_vars) + rendered, err := renderTemplateValue(v, queryVars) if err != nil { return nil, fmt.Errorf("api: query param %s: %w", k, err) } @@ -226,7 +207,7 @@ func (c *APIConnector) BuildRequest( // -------------------------------------------------------------------- var body io.Reader if reqCfg.Body != nil && *reqCfg.Body != "" { - rendered, err := renderTemplateValue(*reqCfg.Body, query_vars) + rendered, err := renderTemplateValue(*reqCfg.Body, queryVars) if err != nil { return nil, fmt.Errorf("api: failed to render body: %w", err) } @@ -250,7 +231,7 @@ func (c *APIConnector) BuildRequest( // 6. Connector-level default headers // -------------------------------------------------------------------- for k, v := range c.config.DefaultHeaders { - rendered, err := renderTemplateValue(v, query_vars) + rendered, err := renderTemplateValue(v, queryVars) if err != nil { return nil, fmt.Errorf("api: default header %s: %w", k, err) } @@ -282,24 +263,13 @@ func (c *APIConnector) BuildRequest( // 8. Dataset-level headers override connector defaults // -------------------------------------------------------------------- for k, v := range reqCfg.Headers { - rendered, err := renderTemplateValue(v, query_vars) + rendered, err := renderTemplateValue(v, queryVars) if err != nil { return nil, fmt.Errorf("api: header %s: %w", k, err) } req.Header.Set(k, rendered) } - // -------------------------------------------------------------------- - // 9. Debug logging - // -------------------------------------------------------------------- - // fmt.Printf("API DEBUG - Request:\n") - // fmt.Printf(" Method: %s\n", req.Method) - // fmt.Printf(" URL: %s\n", req.URL.String()) - // fmt.Printf(" Headers:\n") - // for k, v := range req.Header { - // fmt.Printf(" %s: %v\n", k, v) - // } - return req, nil } diff --git a/internal/response_builder/bulder.go b/internal/response_builder/builder.go similarity index 100% rename from internal/response_builder/bulder.go rename to internal/response_builder/builder.go From a480a72a4f10ab213e3fe849fdf0246d81de2edc Mon Sep 17 00:00:00 2001 From: squeakycheese75 Date: Tue, 7 Apr 2026 14:25:30 +0200 Subject: [PATCH 3/3] move items to models --- internal/datastore/factory.go | 10 +++++----- {entities => model}/datastore_configs.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) rename {entities => model}/datastore_configs.go (99%) diff --git a/internal/datastore/factory.go b/internal/datastore/factory.go index 58b0110..d92e0c0 100644 --- a/internal/datastore/factory.go +++ b/internal/datastore/factory.go @@ -7,13 +7,13 @@ import ( "strings" "github.com/google/uuid" - "github.com/kvatch-hub/kvatch-runtime/entities" + "github.com/kvatch-hub/kvatch-runtime/model" ) type TenantSettings struct { TenantUUID uuid.UUID DatastoreType string - DataStoreOptions entities.DataStoreOptions + DataStoreOptions model.DataStoreOptions EncryptedConfig []byte } @@ -39,13 +39,13 @@ func (f *DataStoreFactory) BuildStorageClient(ctx context.Context, tenantUUID uu switch strings.TrimSpace(strings.ToLower(settings.DatastoreType)) { case "postgres": - var cfg entities.PostgresDataStoreConfig + var cfg model.PostgresDataStoreConfig if err := json.Unmarshal(decryptedConfig, &cfg); err != nil { return nil, fmt.Errorf("failed to unmarshal postgres config: %w", err) } return NewPostgresDataStoreClient(cfg.DSN(), "") case "sqlite_local": - var cfg entities.SQLiteLocalDataStoreConfig + var cfg model.SQLiteLocalDataStoreConfig if err := json.Unmarshal(decryptedConfig, &cfg); err != nil { return nil, fmt.Errorf("failed to unmarshal sqlite_local config: %w", err) } @@ -55,7 +55,7 @@ func (f *DataStoreFactory) BuildStorageClient(ctx context.Context, tenantUUID uu } return NewSQLiteDataStoreClientWithStorage(ctx, NewLocalSQLiteStorage(path)) case "sqlite_s3": - var cfg entities.SQLiteS3DataStoreConfig + var cfg model.SQLiteS3DataStoreConfig if err := json.Unmarshal(decryptedConfig, &cfg); err != nil { return nil, fmt.Errorf("failed to unmarshal sqlite_s3 config: %w", err) } diff --git a/entities/datastore_configs.go b/model/datastore_configs.go similarity index 99% rename from entities/datastore_configs.go rename to model/datastore_configs.go index f91dc35..fd87174 100644 --- a/entities/datastore_configs.go +++ b/model/datastore_configs.go @@ -1,4 +1,4 @@ -package entities +package model import ( "fmt"