Support configurable storage driver, register S3/Google connectors, remove debug prints, and add release readiness review

README.md

@@ -143,7 +143,7 @@ No ETL. No pipelines. Just declarative data access.
 
 ---
 
-## 🚀 Quickstart
+## ▶️ Run the Postgres Example
 
 ```bash
 git clone https://github.com/kvatch-hub/kvatch-runtime
@@ -165,6 +165,28 @@ go run ./examples/postgres
 
 ---
 
+## 🧭 Public API Boundary
+
+The packages intended for external use are:
+
+- `pkg/runtime`
+- `model`
+- `configs`
+
+Everything under `internal/` is implementation detail and may change without notice.
+
+---
+
+## 🔌 Connector Support (current)
+
+- Stable/primary: `postgres`, `sqlite`, `api`, `localfile`, `local_directory`
+- Available: `googlesheet`, `git`, `s3`, `google_api`
+- Special runtime-only connector: `federated`
+
+For early releases, treat non-primary connectors as evolving APIs.
+
+---
+
 ## 💡 Examples
 
 - Track a crypto portfolio → `examples/crypto_portfolio`

RELEASE_READINESS_REVIEW.md

@@ -0,0 +1,19 @@
+# Public Release Readiness Review (April 7, 2026)
+
+Status: **Not ready** for a clean first open-source release without targeted cleanup.
+
+## Highest-priority concerns
+
+1. Public API/config mismatch (`runtime.New` ignores storage driver and always creates SQLite client).
+2. Declared connector types are broader than registered defaults (e.g. S3 declared but not registered).
+3. Internal-only operational details leak into runtime output (git connector prints filesystem paths and tree contents).
+4. Public package surface and naming are inconsistent (`model` + `configs` + `entities`, typos, mixed naming styles).
+5. Documentation lacks clear stability guarantees and API boundaries for external contributors.
+
+## Release-focused recommendations
+
+- Tighten public API surface to only the packages intended for users (`pkg/runtime`, `model`, and stable config package).
+- Ensure declared connector/plugin types and runtime registrations are aligned.
+- Remove debug prints and commented-out blocks from runtime-facing paths.
+- Publish an explicit support matrix (connectors, dataset types, auth modes, and production readiness).
+- Add tests for public constructors and capability registration consistency.

examples/quickstart/main.go

@@ -24,14 +24,14 @@ func main() {
 		Connectors: []model.Connector{
 			{
 				Name: "local",
-				Type: "internal",
+				Type: model.ConnectorTypeInternal,
 			},
 		},
 		Datasets: []model.Dataset{
 			{
 				Name:          "books",
 				ConnectorName: "local",
-				Type:          "json",
+				Type:          model.DatasetTypeJSON,
 				Data: []map[string]any{
 					{"id": 1, "title": "Dune"},
 					{"id": 2, "title": "Neuromancer"},

internal/connectors/api/connector.go

@@ -129,31 +129,12 @@ func parseConfig(config interface{}) (model.APIConnectorConfig, error) {
 	}
 }
 
-// func parseAPIOptions(raw map[string]interface{}) (*model.APIDatasetOptions, error) {
-// 	if raw == nil {
-// 		return nil, fmt.Errorf("missing api dataset options")
-// 	}
-
-// 	// Convert raw map → JSON → typed struct
-// 	b, err := json.Marshal(raw)
-// 	if err != nil {
-// 		return nil, err
-// 	}
-
-// 	var out model.APIDatasetOptions
-// 	if err := json.Unmarshal(b, &out); err != nil {
-// 		return nil, err
-// 	}
-
-// 	return &out, nil
-// }
-
 // BuildRequest constructs an http.Request from the API request spec,
 // template params, and optional pagination state.
 func (c *APIConnector) BuildRequest(
 	ctx context.Context,
 	reqCfg *model.APIRequestSpec,
-	query_vars map[string]string,
+	queryVars map[string]string,
 	page *paginationState,
 ) (*http.Request, error) {
 
@@ -165,7 +146,7 @@ func (c *APIConnector) BuildRequest(
 	// 1. Manual base_url + path join (to avoid ResolveReference bugs)
 	// --------------------------------------------------------------------
 	base := strings.TrimRight(c.config.BaseURL, "/")
-	path, err := renderTemplateValue(reqCfg.Path, query_vars)
+	path, err := renderTemplateValue(reqCfg.Path, queryVars)
 	if err != nil {
 		return nil, fmt.Errorf("api: failed to render path: %w", err)
 	}
@@ -183,7 +164,7 @@ func (c *APIConnector) BuildRequest(
 	// --------------------------------------------------------------------
 	q := fullURL.Query()
 	for k, v := range reqCfg.Params {
-		rendered, err := renderTemplateValue(v, query_vars)
+		rendered, err := renderTemplateValue(v, queryVars)
 		if err != nil {
 			return nil, fmt.Errorf("api: query param %s: %w", k, err)
 		}
@@ -226,7 +207,7 @@ func (c *APIConnector) BuildRequest(
 	// --------------------------------------------------------------------
 	var body io.Reader
 	if reqCfg.Body != nil && *reqCfg.Body != "" {
-		rendered, err := renderTemplateValue(*reqCfg.Body, query_vars)
+		rendered, err := renderTemplateValue(*reqCfg.Body, queryVars)
 		if err != nil {
 			return nil, fmt.Errorf("api: failed to render body: %w", err)
 		}
@@ -250,7 +231,7 @@ func (c *APIConnector) BuildRequest(
 	// 6. Connector-level default headers
 	// --------------------------------------------------------------------
 	for k, v := range c.config.DefaultHeaders {
-		rendered, err := renderTemplateValue(v, query_vars)
+		rendered, err := renderTemplateValue(v, queryVars)
 		if err != nil {
 			return nil, fmt.Errorf("api: default header %s: %w", k, err)
 		}
@@ -282,24 +263,13 @@ func (c *APIConnector) BuildRequest(
 	// 8. Dataset-level headers override connector defaults
 	// --------------------------------------------------------------------
 	for k, v := range reqCfg.Headers {
-		rendered, err := renderTemplateValue(v, query_vars)
+		rendered, err := renderTemplateValue(v, queryVars)
 		if err != nil {
 			return nil, fmt.Errorf("api: header %s: %w", k, err)
 		}
 		req.Header.Set(k, rendered)
 	}
 
-	// --------------------------------------------------------------------
-	// 9. Debug logging
-	// --------------------------------------------------------------------
-	// fmt.Printf("API DEBUG - Request:\n")
-	// fmt.Printf("  Method: %s\n", req.Method)
-	// fmt.Printf("  URL:    %s\n", req.URL.String())
-	// fmt.Printf("  Headers:\n")
-	// for k, v := range req.Header {
-	// 	fmt.Printf("    %s: %v\n", k, v)
-	// }
-
 	return req, nil
 }
 

internal/connectors/aws_s3/connector.go

@@ -34,6 +34,13 @@ func (c *S3Connector) GetType() model.ConnectorType {
 	return model.ConnectorTypeS3
 }
 
+func (c *S3Connector) Ping(ctx context.Context, cfg interface{}) error {
+	if err := c.Connect(ctx, &models.EngineContextConnector{Config: cfg}); err != nil {
+		return err
+	}
+	return c.Close()
+}
+
 func (c *S3Connector) Connect(ctx context.Context, engineConnectorCtx *models.EngineContextConnector) error {
 	if engineConnectorCtx == nil {
 		return fmt.Errorf("engine connector context is missing")

internal/connectors/git/connector.go

@@ -168,9 +168,6 @@ func (c *GitConnector) Connect(ctx context.Context, engineConnectorCtx *models.E
 		return err
 	}
 
-	fmt.Printf("📁 Resolved repo root path: %s\n", c.rootPath)
-	printDirRecursive(c.rootPath)
-
 	return nil
 }
 
@@ -248,19 +245,6 @@ func hashPath(repo, ref string) string {
 	return fmt.Sprintf("%x", sum[:])[:10]
 }
 
-func printDirRecursive(path string) {
-	fmt.Printf("📦 Contents of %s:\n", path)
-	_ = filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
-		if err == nil {
-			rel, _ := filepath.Rel(path, p)
-			if rel != "." {
-				fmt.Println("  -", rel)
-			}
-		}
-		return nil
-	})
-}
-
 func parseConfig(config interface{}) (*model.GitConnectorConfig, error) {
 	switch v := config.(type) {
 	case *model.GitConnectorConfig:

internal/datastore/factory.go

@@ -7,13 +7,13 @@ import (
 	"strings"
 
 	"github.com/google/uuid"
-	"github.com/kvatch-hub/kvatch-runtime/entities"
+	"github.com/kvatch-hub/kvatch-runtime/model"
 )
 
 type TenantSettings struct {
 	TenantUUID       uuid.UUID
 	DatastoreType    string
-	DataStoreOptions entities.DataStoreOptions
+	DataStoreOptions model.DataStoreOptions
 	EncryptedConfig  []byte
 }
 
@@ -39,13 +39,13 @@ func (f *DataStoreFactory) BuildStorageClient(ctx context.Context, tenantUUID uu
 
 	switch strings.TrimSpace(strings.ToLower(settings.DatastoreType)) {
 	case "postgres":
-		var cfg entities.PostgresDataStoreConfig
+		var cfg model.PostgresDataStoreConfig
 		if err := json.Unmarshal(decryptedConfig, &cfg); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal postgres config: %w", err)
 		}
 		return NewPostgresDataStoreClient(cfg.DSN(), "")
 	case "sqlite_local":
-		var cfg entities.SQLiteLocalDataStoreConfig
+		var cfg model.SQLiteLocalDataStoreConfig
 		if err := json.Unmarshal(decryptedConfig, &cfg); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal sqlite_local config: %w", err)
 		}
@@ -55,7 +55,7 @@ func (f *DataStoreFactory) BuildStorageClient(ctx context.Context, tenantUUID uu
 		}
 		return NewSQLiteDataStoreClientWithStorage(ctx, NewLocalSQLiteStorage(path))
 	case "sqlite_s3":
-		var cfg entities.SQLiteS3DataStoreConfig
+		var cfg model.SQLiteS3DataStoreConfig
 		if err := json.Unmarshal(decryptedConfig, &cfg); err != nil {
 			return nil, fmt.Errorf("failed to unmarshal sqlite_s3 config: %w", err)
 		}

internal/subscription_manager/main.go

@@ -3,6 +3,7 @@ package subscription_manager
 import (
 	"github.com/kvatch-hub/kvatch-runtime/internal/connectors"
 	apiconnector "github.com/kvatch-hub/kvatch-runtime/internal/connectors/api"
+	s3connector "github.com/kvatch-hub/kvatch-runtime/internal/connectors/aws_s3"
 	gitconnector "github.com/kvatch-hub/kvatch-runtime/internal/connectors/git"
 	googlesheetConnection "github.com/kvatch-hub/kvatch-runtime/internal/connectors/googlesheets"
 	localDirectoryConnection "github.com/kvatch-hub/kvatch-runtime/internal/connectors/localdirectory"
@@ -16,7 +17,6 @@ import (
 	csvPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/csv"
 	gsheetPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/gsheet"
 	jsonPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/json"
-	psqlPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/postgres"
 	sqlitePlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/sqlite"
 	yamlPlugin "github.com/kvatch-hub/kvatch-runtime/internal/plugins/yaml"
 )
@@ -57,7 +57,6 @@ func (m *SubscriptionManager) registerDefaultPlugins() {
 	_ = m.pluginRegistry.Register(model.DatasetTypeJSON, jsonPlugin.NewJSONDataPlugin())
 	_ = m.pluginRegistry.Register(model.DatasetTypeCSV, csvPlugin.NewCSVDataPlugin())
 	_ = m.pluginRegistry.Register(model.DatasetTypeSQL, sqlitePlugin.NewSQLiteDataPlugin())
-	_ = m.pluginRegistry.Register(model.DatasetTypeSQL, psqlPlugin.NewPostgresDataPlugin())
 	_ = m.pluginRegistry.Register(model.DatasetTypeYAML, yamlPlugin.NewYAMLDataPlugin())
 	_ = m.pluginRegistry.Register(model.DatasetTypeGoogleSheet, gsheetPlugin.NewGoogleSheetsDataPlugin())
 	_ = m.pluginRegistry.Register(model.DatasetTypeAPI, apiPlugin.NewAPIDatasetPlugin())
@@ -68,9 +67,11 @@ func (m *SubscriptionManager) registerDefaultDataConnectors() {
 	_ = m.connectorRegistry.Register(model.ConnectorTypePostgres, postgresConnection.NewPostgresConnector())
 	_ = m.connectorRegistry.Register(model.ConnectorTypeSQLite, sqliteConnection.NewSQLiteConnector())
 	_ = m.connectorRegistry.Register(model.ConnectorTypeGit, gitconnector.NewGitConnector())
+	_ = m.connectorRegistry.Register(model.ConnectorTypeS3, s3connector.NewS3Connector())
 	_ = m.connectorRegistry.Register(model.ConnectorTypeLocalDirectory, localDirectoryConnection.NewDirectoryConnector())
 	_ = m.connectorRegistry.Register(model.ConnectorTypeGoogleSheet, googlesheetConnection.NewGoogleSheetConnector())
 	_ = m.connectorRegistry.Register(model.ConnectorTypeAPI, apiconnector.NewAPIConnector())
+	_ = m.connectorRegistry.Register(model.ConnectorTypeGoogleAPI, apiconnector.NewAPIConnector())
 }
 
 func (m *SubscriptionManager) GetRegisteredConnectors() ConnectorRegistry {

internal/subscription_manager/main_test.go

@@ -22,7 +22,6 @@ func Test_registerDefaultPlugins_usesRegistry(t *testing.T) {
 		mockPluginReg.EXPECT().Register(model.DatasetTypeJSON, gomock.Any()).Return(nil),
 		mockPluginReg.EXPECT().Register(model.DatasetTypeCSV, gomock.Any()).Return(nil),
 		mockPluginReg.EXPECT().Register(model.DatasetTypeSQL, gomock.Any()).Return(nil),
-		mockPluginReg.EXPECT().Register(model.DatasetTypeSQL, gomock.Any()).Return(nil),
 		mockPluginReg.EXPECT().Register(model.DatasetTypeYAML, gomock.Any()).Return(nil),
 		mockPluginReg.EXPECT().Register(model.DatasetTypeGoogleSheet, gomock.Any()).Return(nil),
 		mockPluginReg.EXPECT().Register(model.DatasetTypeAPI, gomock.Any()).Return(nil),
@@ -48,9 +47,11 @@ func Test_registerDefaultDataConnectors_usesRegistry(t *testing.T) {
 		mockConnReg.EXPECT().Register(model.ConnectorTypePostgres, gomock.Any()).Return(nil),
 		mockConnReg.EXPECT().Register(model.ConnectorTypeSQLite, gomock.Any()).Return(nil),
 		mockConnReg.EXPECT().Register(model.ConnectorTypeGit, gomock.Any()).Return(nil),
+		mockConnReg.EXPECT().Register(model.ConnectorTypeS3, gomock.Any()).Return(nil),
 		mockConnReg.EXPECT().Register(model.ConnectorTypeLocalDirectory, gomock.Any()).Return(nil),
 		mockConnReg.EXPECT().Register(model.ConnectorTypeGoogleSheet, gomock.Any()).Return(nil),
 		mockConnReg.EXPECT().Register(model.ConnectorTypeAPI, gomock.Any()).Return(nil),
+		mockConnReg.EXPECT().Register(model.ConnectorTypeGoogleAPI, gomock.Any()).Return(nil),
 	)
 
 	m := &SubscriptionManager{
@@ -94,6 +95,8 @@ func Test_NewSubscriptionManagerWithDefaults_registersEverything(t *testing.T) {
 		model.ConnectorTypeLocalFile,
 		model.ConnectorTypePostgres,
 		model.ConnectorTypeSQLite,
+		model.ConnectorTypeS3,
+		model.ConnectorTypeGoogleAPI,
 	}
 	for _, w := range wantConns {
 		if !gotConns[w] {

entities/datastore_configs.go → model/datastore_configs.go

@@ -1,4 +1,4 @@
-package entities
+package model
 
 import (
 	"fmt"

pkg/runtime/runtime.go

@@ -3,6 +3,7 @@ package runtime
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/kvatch-hub/kvatch-runtime/configs"
 	"github.com/kvatch-hub/kvatch-runtime/internal/datastore"
@@ -52,7 +53,7 @@ func (r *runtimeImpl) ExecutePlan(ctx context.Context, req model.ExecutePlanRequ
 }
 
 func newRuntime(cfg configs.Config) (Runtime, error) {
-	store, err := datastore.NewSQLiteDataStoreClient(cfg.Storage.DSN)
+	store, err := newDataStoreClient(cfg)
 	if err != nil {
 		return nil, err
 	}
@@ -75,3 +76,18 @@ func newRuntime(cfg configs.Config) (Runtime, error) {
 		service: svc,
 	}, nil
 }
+
+func newDataStoreClient(cfg configs.Config) (datastore.DataStoreClient, error) {
+	driver := strings.TrimSpace(strings.ToLower(cfg.Storage.Driver))
+	switch driver {
+	case "", "sqlite":
+		return datastore.NewSQLiteDataStoreClient(cfg.Storage.DSN)
+	case "postgres":
+		if strings.TrimSpace(cfg.Storage.DSN) == "" {
+			return nil, fmt.Errorf("storage.dsn is required when storage.driver is postgres")
+		}
+		return datastore.NewPostgresDataStoreClient(cfg.Storage.DSN, "")
+	default:
+		return nil, fmt.Errorf("unsupported storage driver %q", cfg.Storage.Driver)
+	}
+}

pkg/runtime/runtime_test.go

@@ -62,3 +62,24 @@ func TestSimpleBaseCase(t *testing.T) {
 	require.Equalf(t, expectedColumns, len(got.Columns), "expected %d columns, got %d", expectedColumns, len(got.Columns))
 	require.Equalf(t, expectedRows, len(got.Data), "expected %d rows, got %d", expectedRows, len(got.Data))
 }
+
+func TestNew_UnsupportedDriver(t *testing.T) {
+	_, err := New(configs.Config{
+		Storage: configs.StorageConfig{
+			Driver: "mongodb",
+			DSN:    "ignored",
+		},
+	})
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "unsupported storage driver")
+}
+
+func TestNew_PostgresWithoutDSN(t *testing.T) {
+	_, err := New(configs.Config{
+		Storage: configs.StorageConfig{
+			Driver: "postgres",
+		},
+	})
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "storage.dsn is required")
+}