From 046b9e434ff9752e099520f957db6901dba3949f Mon Sep 17 00:00:00 2001 From: Sam Mortenson Date: Thu, 19 Mar 2026 00:35:56 -0700 Subject: [PATCH 1/6] Initial work on plugin system --- README.md | 8 ++ cmd/helloplugin/helloplugin.go | 80 +++++++++++++++++ db.go | 22 +++-- export_test.go | 2 + plugins.go | 130 ++++++++++++++++++++++++++++ ripoff_file.go | 15 +++- testdata/import/plugins/plugins.yml | 27 ++++++ testdata/import/plugins/schema.sql | 5 ++ 8 files changed, 281 insertions(+), 8 deletions(-) create mode 100644 cmd/helloplugin/helloplugin.go create mode 100644 plugins.go create mode 100644 testdata/import/plugins/plugins.yml create mode 100644 testdata/import/plugins/schema.sql diff --git a/README.md b/README.md index 8a839a0..f10aa8d 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,14 @@ The latter format is especially useful if you have generated columns on every ta In the future, additional flags may be added to allow you to include tables, add arbitrary `WHERE` conditions, modify the row id/key, export multiple files, or use existing templates. +## Plugins + +### Re-build example plugin + +```bash +GOOS=wasip1 GOARCH=wasm go build -o testdata/plugins/hellowasm/hellowasm.wasm testdata/plugins/hellowasm/hellowasm.go +``` + ## Installation 1. Run `go install github.com/mortenson/ripoff/cmd/ripoff-export@latest` diff --git a/cmd/helloplugin/helloplugin.go b/cmd/helloplugin/helloplugin.go new file mode 100644 index 0000000..8785504 --- /dev/null +++ b/cmd/helloplugin/helloplugin.go @@ -0,0 +1,80 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "log" + "net" + "os" +) + +func main() { + listener, err := net.Listen("tcp", ":6767") + if err != nil { + log.Fatal("Error listening:", err) + } + defer listener.Close() + fmt.Println("READY") + + for { + conn, err := listener.Accept() + if err != nil { + log.Println("Error accepting connection:", err) + continue + } + go handleConnection(conn) + } +} + +type Request struct { + Type string `json:"type"` + ValueFunc string `json:"valueFunc"` + Args []string `json:"args"` +} + +type Response struct { + Value string `json:"value"` +} + +func handleConnection(conn net.Conn) { + defer conn.Close() + + scanner := bufio.NewScanner(conn) + for scanner.Scan() { + message := scanner.Bytes() + r := Request{} + err := json.Unmarshal(message, &r) + if err != nil { + log.Println("Error parsing body:", err) + return + } + if r.Type == "exit" { + os.Exit(0) + return + } + if r.ValueFunc != "sayHello" { + log.Println("Unknown value func:", r.ValueFunc) + return + } + if len(r.Args) == 0 { + log.Println("No args provided") + return + } + resp, err := json.Marshal(Response{ + Value: fmt.Sprintf("Hello %s", r.Args[0]), + }) + if err != nil { + log.Println("Could not marshal message:", r) + return + } + _, err = conn.Write(append(resp, '\n')) + if err != nil { + log.Println("Could not send message:", r.ValueFunc) + return + } + } + if err := scanner.Err(); err != nil { + log.Println("Scanner error:", err) + } +} diff --git a/db.go b/db.go index 54e47b2..387313b 100644 --- a/db.go +++ b/db.go @@ -21,12 +21,18 @@ import ( // Runs ripoff from start to finish, without committing the transaction. func RunRipoff(ctx context.Context, tx pgx.Tx, totalRipoff RipoffFile) error { + manager, err := NewPluginManager(totalRipoff.Plugins) + if err != nil { + return err + } + defer manager.Close() + primaryKeys, err := getPrimaryKeys(ctx, tx) if err != nil { return err } - queries, err := buildQueriesForRipoff(primaryKeys, totalRipoff) + queries, err := buildQueriesForRipoff(manager, primaryKeys, totalRipoff) if err != nil { return err } @@ -111,7 +117,7 @@ func GetEnumValues(ctx context.Context, tx pgx.Tx) (EnumValuesResult, error) { var valueFuncRegex = regexp.MustCompile(`([a-zA-Z]+)\((.*)\)$`) var referenceRegex = regexp.MustCompile(`^[a-zA-Z0-9_]+:[a-zA-Z]+\(`) -func prepareValue(rawValue string) (string, error) { +func prepareValue(manager *PluginManager, rawValue string) (string, error) { valueFuncMatches := valueFuncRegex.FindStringSubmatch(rawValue) if len(valueFuncMatches) != 3 { return rawValue, nil @@ -120,6 +126,10 @@ func prepareValue(rawValue string) (string, error) { value := valueFuncMatches[2] valueParts := strings.Split(strings.ReplaceAll(" ", "", valueFuncMatches[2]), ",") + if manager.Supports(methodName) { + return manager.Call(methodName, valueParts...) + } + // Create a new random seed based on a sha256 hash of the value. h := sha256.New() h.Write([]byte(value)) @@ -153,7 +163,7 @@ func prepareValue(rawValue string) (string, error) { return fakerResult, nil } -func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, dependencyGraph map[string][]string) (string, error) { +func buildQueryForRow(manager *PluginManager, primaryKeys PrimaryKeysResult, rowId string, row Row, dependencyGraph map[string][]string) (string, error) { parts := strings.Split(rowId, ":") if len(parts) < 2 { return "", fmt.Errorf("invalid id: %s", rowId) @@ -225,7 +235,7 @@ func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, depe } columns = append(columns, pq.QuoteIdentifier(column)) - valuePrepared, err := prepareValue(value) + valuePrepared, err := prepareValue(manager, value) if err != nil { return "", err } @@ -257,7 +267,7 @@ func buildQueryForRow(primaryKeys PrimaryKeysResult, rowId string, row Row, depe } // Returns a sorted array of queries to run based on a given ripoff file. -func buildQueriesForRipoff(primaryKeys PrimaryKeysResult, totalRipoff RipoffFile) ([]string, error) { +func buildQueriesForRipoff(manager *PluginManager, primaryKeys PrimaryKeysResult, totalRipoff RipoffFile) ([]string, error) { dependencyGraph := map[string][]string{} queries := map[string]string{} @@ -268,7 +278,7 @@ func buildQueriesForRipoff(primaryKeys PrimaryKeysResult, totalRipoff RipoffFile // Build queries. for rowId, row := range totalRipoff.Rows { - query, err := buildQueryForRow(primaryKeys, rowId, row, dependencyGraph) + query, err := buildQueryForRow(manager, primaryKeys, rowId, row, dependencyGraph) if err != nil { return []string{}, err } diff --git a/export_test.go b/export_test.go index b0b5d8b..9e4b998 100644 --- a/export_test.go +++ b/export_test.go @@ -38,6 +38,8 @@ func runExportTestData(t *testing.T, ctx context.Context, tx pgx.Tx, testDir str expectedRipoffFile := &RipoffFile{} err = yaml.Unmarshal(expectedRipoffYaml, expectedRipoffFile) require.NoError(t, err) + expectedRipoffFile.Plugins = nil + newRipoffFile.Plugins = nil require.Equal(t, expectedRipoffFile, newRipoffFile) // Wipe database. diff --git a/plugins.go b/plugins.go new file mode 100644 index 0000000..7dfa2f1 --- /dev/null +++ b/plugins.go @@ -0,0 +1,130 @@ +package ripoff + +import ( + "bufio" + "encoding/json" + "fmt" + "net" + "os/exec" + "strings" + "time" +) + +type PluginManager struct { + valueFuncMap map[string]RipoffPlugin + spawnedCommands []*exec.Cmd + connections map[string]net.Conn +} + +func (m *PluginManager) Close() { + // Ignore errors + for _, conn := range m.connections { + message, _ := json.Marshal(Request{ + Type: "exit", + }) + conn.Write(append(message, '\n')) + conn.Close() + } + for _, cmd := range m.spawnedCommands { + cmd.Process.Kill() + } +} + +func (m *PluginManager) Supports(valueFunc string) bool { + _, ok := m.valueFuncMap[valueFunc] + return ok +} + +type Request struct { + Type string `json:"type"` + ValueFunc string `json:"valueFunc"` + Args []string `json:"args"` +} + +type Response struct { + Value string `json:"value"` +} + +func (m *PluginManager) Call(valueFunc string, args ...string) (string, error) { + plugin, hasPlugin := m.valueFuncMap[valueFunc] + if !hasPlugin { + return "", fmt.Errorf("Plugin for valueFunc %s is not defined", valueFunc) + } + conn, ok := m.connections[valueFunc] + // Attempt to start process and wait for port to open + if !ok { + commandArgs := []string{} + if len(plugin.Command) > 1 { + commandArgs = plugin.Command[1:] + } + cmd := exec.Command(plugin.Command[0], commandArgs...) + stdout, err := cmd.StdoutPipe() + if err != nil { + return "", err + } + err = cmd.Start() + if err != nil { + return "", err + } + m.spawnedCommands = append(m.spawnedCommands, cmd) + // Wait for plugin to be ready + scanner := bufio.NewScanner(stdout) + scanner.Scan() + line := scanner.Text() + // Note that there's no timeout here, which isn't great + if !strings.Contains(string(line), "READY") { + return "", fmt.Errorf("Plugin command '%s' failed to output READY. Got: '%s' instead", strings.Join(plugin.Command, " "), line) + } + conn, err = net.Dial("tcp", plugin.Address) + if err != nil { + return "", err + } + m.connections[valueFunc] = conn + } + // Send message to open TCP socket + conn.SetReadDeadline(time.Now().Add(time.Second)) + scanner := bufio.NewScanner(conn) + message, err := json.Marshal(Request{ + Type: "valueFunc", + ValueFunc: valueFunc, + Args: args, + }) + if err != nil { + return "", err + } + _, err = conn.Write(append(message, '\n')) + if err != nil { + return "", err + } + if !scanner.Scan() { + return "", fmt.Errorf("Plugin command '%s' failed to response to TCP message: %v", strings.Join(plugin.Command, " "), scanner.Err()) + } + line := scanner.Bytes() + response := Response{} + err = json.Unmarshal(line, &response) + if err != nil { + return "", err + } + return response.Value, nil +} + +func NewPluginManager(plugins map[string]RipoffPlugin) (*PluginManager, error) { + m := &PluginManager{ + valueFuncMap: map[string]RipoffPlugin{}, + connections: map[string]net.Conn{}, + spawnedCommands: []*exec.Cmd{}, + } + for pluginName, plugin := range plugins { + if len(plugin.Command) == 0 { + return nil, fmt.Errorf("Cannot create new PluginManager - the plugin %s does not define a command.", pluginName) + } + for _, valueFunc := range plugin.ValueFuncs { + _, alreadySet := m.valueFuncMap[valueFunc] + if alreadySet { + return nil, fmt.Errorf("Cannot create new PluginManager - the valueFunc %s is set in more than one plugin.", valueFunc) + } + m.valueFuncMap[valueFunc] = plugin + } + } + return m, nil +} diff --git a/ripoff_file.go b/ripoff_file.go index 4f9d7c9..eec7bfc 100644 --- a/ripoff_file.go +++ b/ripoff_file.go @@ -14,8 +14,15 @@ import ( type Row map[string]interface{} +type RipoffPlugin struct { + Command []string `yaml:"command"` + Address string `yaml:"address"` + ValueFuncs []string `yaml:"valueFuncs"` +} + type RipoffFile struct { - Rows map[string]Row `yaml:"rows"` + Plugins map[string]RipoffPlugin `yaml:"plugins"` + Rows map[string]Row `yaml:"rows"` } var funcMap = template.FuncMap{ @@ -110,10 +117,14 @@ func RipoffFromDirectory(dir string, enums EnumValuesResult) (RipoffFile, error) } totalRipoff := RipoffFile{ - Rows: map[string]Row{}, + Rows: map[string]Row{}, + Plugins: map[string]RipoffPlugin{}, } for _, ripoff := range allRipoffs { + for k, v := range ripoff.Plugins { + totalRipoff.Plugins[k] = v + } err = concatRows(templates, totalRipoff.Rows, ripoff.Rows, enums) if err != nil { return RipoffFile{}, err diff --git a/testdata/import/plugins/plugins.yml b/testdata/import/plugins/plugins.yml new file mode 100644 index 0000000..511b55c --- /dev/null +++ b/testdata/import/plugins/plugins.yml @@ -0,0 +1,27 @@ +plugins: + helloplugin: + command: [go, run, cmd/helloplugin/helloplugin.go] + address: localhost:6767 + valueFuncs: [sayHello] +rows: + uuid_users:uuid(fooBar): + email: foobar@example.com + name: sayHello(World) + uuid_users:uuid(smorty): + email: smorty@example.com + name: sayHello(Foo) + uuid_users:uuid(smorty2): + email: smorty2@example.com + name: sayHello(Foo) + uuid_users:uuid(smorty3): + email: smorty3@example.com + name: sayHello(Foo) + uuid_users:uuid(smorty4): + email: smorty4@example.com + name: sayHello(Foo) + uuid_users:uuid(smorty5): + email: smorty5@example.com + name: sayHello(Foo) + uuid_users:uuid(smorty6): + email: smorty6@example.com + name: sayHello(Foo) diff --git a/testdata/import/plugins/schema.sql b/testdata/import/plugins/schema.sql new file mode 100644 index 0000000..53cafbb --- /dev/null +++ b/testdata/import/plugins/schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE uuid_users ( + id UUID NOT NULL PRIMARY KEY, + email TEXT NOT NULL, + name TEXT NOT NULL +); From 359f3725b80987c32344b0fcd4a2bc3431db8934 Mon Sep 17 00:00:00 2001 From: Sam Mortenson Date: Thu, 19 Mar 2026 11:51:32 -0700 Subject: [PATCH 2/6] Fixed nasty gofakeit bug --- db.go | 2 +- db_test.go | 4 +- plugins.go | 63 +++++++++++++++++++--------- testdata/import/enums/validate.sql | 2 +- testdata/import/faker/validate.sql | 2 +- testdata/import/plugins/plugins.yml | 19 +-------- testdata/import/plugins/schema.sql | 2 +- testdata/import/plugins/validate.sql | 7 ++++ 8 files changed, 58 insertions(+), 43 deletions(-) create mode 100644 testdata/import/plugins/validate.sql diff --git a/db.go b/db.go index 387313b..0e70741 100644 --- a/db.go +++ b/db.go @@ -124,7 +124,7 @@ func prepareValue(manager *PluginManager, rawValue string) (string, error) { } methodName := valueFuncMatches[1] value := valueFuncMatches[2] - valueParts := strings.Split(strings.ReplaceAll(" ", "", valueFuncMatches[2]), ",") + valueParts := strings.Split(strings.ReplaceAll(valueFuncMatches[2], " ", ""), ",") if manager.Supports(methodName) { return manager.Call(methodName, valueParts...) diff --git a/db_test.go b/db_test.go index 781b905..fa2a09b 100644 --- a/db_test.go +++ b/db_test.go @@ -47,11 +47,11 @@ func runTestData(t *testing.T, ctx context.Context, tx pgx.Tx, testDir string) { validationFile, err := os.ReadFile(path.Join(testDir, "validate.sql")) if err == nil { row := tx.QueryRow(ctx, string(validationFile)) - var success int + var success bool var debug string err := row.Scan(&success, &debug) require.NoError(t, err) - if success != 1 { + if !success { t.Fatalf("Validation failed with debug content: %s", debug) } } diff --git a/plugins.go b/plugins.go index 7dfa2f1..aa28e29 100644 --- a/plugins.go +++ b/plugins.go @@ -7,6 +7,7 @@ import ( "net" "os/exec" "strings" + "syscall" "time" ) @@ -26,7 +27,7 @@ func (m *PluginManager) Close() { conn.Close() } for _, cmd := range m.spawnedCommands { - cmd.Process.Kill() + syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) } } @@ -45,6 +46,45 @@ type Response struct { Value string `json:"value"` } +func spawn(command []string) (*exec.Cmd, error) { + commandArgs := []string{} + if len(command) > 1 { + commandArgs = command[1:] + } + cmd := exec.Command(command[0], commandArgs...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + err = cmd.Start() + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(stdout) + scanner.Scan() + line := scanner.Text() + // Set deadline for outputting READY message + timer := time.AfterFunc(5*time.Second, func() { + syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + }) + if !strings.Contains(string(line), "READY") { + syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + return nil, fmt.Errorf("Plugin command '%s' failed to output READY. Got: '%s' instead", strings.Join(command, " "), line) + } + // Stop the timeout kill + timer.Stop() + return cmd, nil +} + +func connect(address string) (net.Conn, error) { + conn, err := net.Dial("tcp", address) + if err != nil { + return nil, err + } + return conn, nil +} + func (m *PluginManager) Call(valueFunc string, args ...string) (string, error) { plugin, hasPlugin := m.valueFuncMap[valueFunc] if !hasPlugin { @@ -53,29 +93,12 @@ func (m *PluginManager) Call(valueFunc string, args ...string) (string, error) { conn, ok := m.connections[valueFunc] // Attempt to start process and wait for port to open if !ok { - commandArgs := []string{} - if len(plugin.Command) > 1 { - commandArgs = plugin.Command[1:] - } - cmd := exec.Command(plugin.Command[0], commandArgs...) - stdout, err := cmd.StdoutPipe() - if err != nil { - return "", err - } - err = cmd.Start() + cmd, err := spawn(plugin.Command) if err != nil { return "", err } m.spawnedCommands = append(m.spawnedCommands, cmd) - // Wait for plugin to be ready - scanner := bufio.NewScanner(stdout) - scanner.Scan() - line := scanner.Text() - // Note that there's no timeout here, which isn't great - if !strings.Contains(string(line), "READY") { - return "", fmt.Errorf("Plugin command '%s' failed to output READY. Got: '%s' instead", strings.Join(plugin.Command, " "), line) - } - conn, err = net.Dial("tcp", plugin.Address) + conn, err = connect(plugin.Address) if err != nil { return "", err } diff --git a/testdata/import/enums/validate.sql b/testdata/import/enums/validate.sql index 8d9c688..0f20d54 100644 --- a/testdata/import/enums/validate.sql +++ b/testdata/import/enums/validate.sql @@ -4,5 +4,5 @@ WITH test AS ( -- db_test.go will automatically determine that the correct number of rows -- were inserted, but in this case we want to make sure every users row also -- has a distinct user role. -SELECT case when array_length(roles, 1) = 4 then 1 else 0 end,roles +SELECT array_length(roles, 1) = 4,roles FROM test; diff --git a/testdata/import/faker/validate.sql b/testdata/import/faker/validate.sql index 0066c6b..c5a5aba 100644 --- a/testdata/import/faker/validate.sql +++ b/testdata/import/faker/validate.sql @@ -4,5 +4,5 @@ WITH test AS ( AND id = '6b30cfb0-a35b-4584-a035-1334515f846b' AND date_trunc('day', last_logged_in_at) = date_trunc('day', now() - interval '10 days') ) -SELECT (select count from test),'email: ' || users.email || ' id: ' || users.id || ' last_logged_in_at: ' || date_trunc('day', last_logged_in_at) || ' 10_days_ago: ' || date_trunc('day', now() - interval '10 days') +SELECT (select count from test) = 1,'email: ' || users.email || ' id: ' || users.id || ' last_logged_in_at: ' || date_trunc('day', last_logged_in_at) || ' 10_days_ago: ' || date_trunc('day', now() - interval '10 days') FROM users; diff --git a/testdata/import/plugins/plugins.yml b/testdata/import/plugins/plugins.yml index 511b55c..9b08a58 100644 --- a/testdata/import/plugins/plugins.yml +++ b/testdata/import/plugins/plugins.yml @@ -4,24 +4,9 @@ plugins: address: localhost:6767 valueFuncs: [sayHello] rows: - uuid_users:uuid(fooBar): + users:uuid(fooBar): email: foobar@example.com name: sayHello(World) - uuid_users:uuid(smorty): + users:uuid(smorty): email: smorty@example.com name: sayHello(Foo) - uuid_users:uuid(smorty2): - email: smorty2@example.com - name: sayHello(Foo) - uuid_users:uuid(smorty3): - email: smorty3@example.com - name: sayHello(Foo) - uuid_users:uuid(smorty4): - email: smorty4@example.com - name: sayHello(Foo) - uuid_users:uuid(smorty5): - email: smorty5@example.com - name: sayHello(Foo) - uuid_users:uuid(smorty6): - email: smorty6@example.com - name: sayHello(Foo) diff --git a/testdata/import/plugins/schema.sql b/testdata/import/plugins/schema.sql index 53cafbb..50f3760 100644 --- a/testdata/import/plugins/schema.sql +++ b/testdata/import/plugins/schema.sql @@ -1,4 +1,4 @@ -CREATE TABLE uuid_users ( +CREATE TABLE users ( id UUID NOT NULL PRIMARY KEY, email TEXT NOT NULL, name TEXT NOT NULL diff --git a/testdata/import/plugins/validate.sql b/testdata/import/plugins/validate.sql new file mode 100644 index 0000000..ca9108c --- /dev/null +++ b/testdata/import/plugins/validate.sql @@ -0,0 +1,7 @@ +WITH test AS ( + SELECT count(*) as count FROM users + WHERE (email = 'foobar@example.com' AND name = 'Hello World') + OR (email = 'smorty@example.com' AND name = 'Hello Foo') +) +SELECT (select count from test) = 2,'email: ' || string_agg(users.email, ',') || ' name: ' || string_agg(users.name, ',') +FROM users; From eceff8bb68749ce78232c621975c4c3a631f3188 Mon Sep 17 00:00:00 2001 From: Sam Mortenson Date: Thu, 19 Mar 2026 12:14:23 -0700 Subject: [PATCH 3/6] Docs --- README.md | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f10aa8d..635ca0f 100644 --- a/README.md +++ b/README.md @@ -127,10 +127,73 @@ In the future, additional flags may be added to allow you to include tables, add ## Plugins -### Re-build example plugin +If you would like to implement your own `valueFuncs`, you can do so by writing a ripoff plugin. -```bash -GOOS=wasip1 GOARCH=wasm go build -o testdata/plugins/hellowasm/hellowasm.wasm testdata/plugins/hellowasm/hellowasm.go +Plugins are local unauthenticated TCP servers that consume and emit newline-separated JSON messages from ripoff. + +### Writing a plugin + +Plugins must listen to a local TCP port and provide a TCP stream (loop of receiving and sending messages) to clients. + +On startup, plugins must output the string `READY` in its first line of output to indicate to ripoff that it is ready to receive TCP messges. + +Each incoming message will be a single line of JSON in the following types: + +#### Return a value + +Your plugin must process an arbitrary `valueFunc` and return a string value. You can decide how to handle functions you do not expect/provide, by either returning an empty value or disconnecting the client. + +Message from ripoff: + +```json +{"type": "valueFunc", "valueFunc": "someFuncName", "args": ["some", "argument", "list"]} +``` + +Response from your TCP server: + +```json +{"value": "someString"} +``` + +#### Exit your process + +Ripoff will send a kill signal to your process, but if you'd like to clean up before that an exit message will be sent beforehand. + +Request message: + +```json +{"type": "exit"} +``` + +#### Example + +An example plugin can be found at `cmd/helloplugin/helloplugin.go`. although TCP servers in other languages may be much easier to implement. + +### Using a plugin + +Plugins are defined in your ripoff files, which instruct ripoff to spawn a process to start your TCP server, then later connect to it with a single TCP stream. Here's an example from ripoff's tests: + +```yml +# A list of plugins to register with ripoff. +plugins: + # An arbitrary name for the plugin. Used only to handle merging ripoff files + # that may define duplicate plugins, which would otherwise conflict. + helloplugin: + # An arbitrary command to execute, which should start your TCP server. + # The command must output READY in its first line of stdout. + command: [go, run, cmd/helloplugin/helloplugin.go] + # A TCP address to connect to after your command is ready. Note that a + # single connection is used to avoid a handshake per valueFunc call. + address: localhost:6767 + # The list of valueFuncs this plugin provides. If ripoff encounters these + # it will call out to your plugin. Note that these take precedence over + # built in valueFuncs, so you can override ripoff's defaults (like uuid()). + valueFuncs: [sayHello] +rows: + users:uuid(fooBar): + # In your ripoff files, you can now call your plugin's registered + # valueFuncs the same as any other valueFunc. + name: sayHello(World) ``` ## Installation From 98e133614ece32325e4c8edbdad73eeb675f806a Mon Sep 17 00:00:00 2001 From: Sam Mortenson Date: Thu, 19 Mar 2026 12:24:34 -0700 Subject: [PATCH 4/6] Bit weird --- go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 72e1212..a4e8dcf 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/mortenson/ripoff -go 1.22 +go 1.26 -toolchain go1.22.4 +toolchain go1.26.0 require ( github.com/brianvoe/gofakeit/v7 v7.0.4 From 494839ac4e102b2a2a682f691b08e1228ed51d37 Mon Sep 17 00:00:00 2001 From: Sam Mortenson Date: Thu, 19 Mar 2026 14:59:07 -0700 Subject: [PATCH 5/6] Lint fixes --- .github/workflows/lint.yml | 2 +- cmd/helloplugin/helloplugin.go | 14 ++++++++-- cmd/ripoff-export/ripoff_export.go | 7 ++++- cmd/ripoff/ripoff.go | 7 ++++- db.go | 2 +- db_test.go | 7 ++--- export_test.go | 21 ++++++++------- plugins.go | 42 +++++++++++++++++++++--------- 8 files changed, 72 insertions(+), 30 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index eadfb53..89ab8bd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,4 +20,4 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.64.5 + version: v2.11.3 diff --git a/cmd/helloplugin/helloplugin.go b/cmd/helloplugin/helloplugin.go index 8785504..f828c0c 100644 --- a/cmd/helloplugin/helloplugin.go +++ b/cmd/helloplugin/helloplugin.go @@ -14,7 +14,12 @@ func main() { if err != nil { log.Fatal("Error listening:", err) } - defer listener.Close() + defer func() { + err := listener.Close() + if err != nil { + log.Println("Error closing listener", err) + } + }() fmt.Println("READY") for { @@ -38,7 +43,12 @@ type Response struct { } func handleConnection(conn net.Conn) { - defer conn.Close() + defer func() { + err := conn.Close() + if err != nil { + log.Println("Error closing connection", err) + } + }() scanner := bufio.NewScanner(conn) for scanner.Scan() { diff --git a/cmd/ripoff-export/ripoff_export.go b/cmd/ripoff-export/ripoff_export.go index 8039f8d..f967541 100644 --- a/cmd/ripoff-export/ripoff_export.go +++ b/cmd/ripoff-export/ripoff_export.go @@ -49,7 +49,12 @@ func main() { slog.Error("Could not connect to database", errAttr(err)) os.Exit(1) } - defer conn.Close(ctx) + defer func() { + err := conn.Close(ctx) + if err != nil { + slog.Error("Could not close database connection", errAttr(err)) + } + }() exportDirectory := path.Clean(args[0]) dirInfo, err := os.Stat(exportDirectory) diff --git a/cmd/ripoff/ripoff.go b/cmd/ripoff/ripoff.go index 4099b73..aedbc47 100644 --- a/cmd/ripoff/ripoff.go +++ b/cmd/ripoff/ripoff.go @@ -44,7 +44,12 @@ func main() { slog.Error("Could not connect to database", errAttr(err)) os.Exit(1) } - defer conn.Close(ctx) + defer func() { + err := conn.Close(ctx) + if err != nil { + slog.Error("Could not close database connection", errAttr(err)) + } + }() tx, err := conn.Begin(ctx) if err != nil { diff --git a/db.go b/db.go index 0e70741..24257e5 100644 --- a/db.go +++ b/db.go @@ -410,7 +410,7 @@ func topologicalSort(digraph map[string][]string) ([]string, error) { visit = func(u string) { if temporaryMark[u] { acyclic = false - } else if !(temporaryMark[u] || permanentMark[u]) { + } else if !temporaryMark[u] && !permanentMark[u] { temporaryMark[u] = true for _, v := range digraph[u] { visit(v) diff --git a/db_test.go b/db_test.go index fa2a09b..07010f5 100644 --- a/db_test.go +++ b/db_test.go @@ -64,10 +64,11 @@ func TestRipoff(t *testing.T) { } ctx := context.Background() conn, err := pgx.Connect(ctx, envUrl) - if err != nil { + require.NoError(t, err) + defer func() { + err := conn.Close(ctx) require.NoError(t, err) - } - defer conn.Close(ctx) + }() _, filename, _, _ := runtime.Caller(0) dir := path.Join(path.Dir(filename), "testdata", "import") diff --git a/export_test.go b/export_test.go index 9e4b998..8955071 100644 --- a/export_test.go +++ b/export_test.go @@ -74,10 +74,11 @@ func TestRipoffExport(t *testing.T) { } ctx := context.Background() conn, err := pgx.Connect(ctx, envUrl) - if err != nil { + require.NoError(t, err) + defer func() { + err := conn.Close(ctx) require.NoError(t, err) - } - defer conn.Close(ctx) + }() _, filename, _, _ := runtime.Caller(0) dir := path.Join(path.Dir(filename), "testdata", "export") @@ -104,10 +105,11 @@ func TestExcludeFlag(t *testing.T) { } ctx := context.Background() conn, err := pgx.Connect(ctx, envUrl) - if err != nil { + require.NoError(t, err) + defer func() { + err := conn.Close(ctx) require.NoError(t, err) - } - defer conn.Close(ctx) + }() // Start a transaction that we'll roll back at the end tx, err := conn.Begin(ctx) @@ -267,10 +269,11 @@ func TestExcludeColumnsFlag(t *testing.T) { } ctx := context.Background() conn, err := pgx.Connect(ctx, envUrl) - if err != nil { + require.NoError(t, err) + defer func() { + err := conn.Close(ctx) require.NoError(t, err) - } - defer conn.Close(ctx) + }() // Start a transaction that we'll roll back at the end tx, err := conn.Begin(ctx) diff --git a/plugins.go b/plugins.go index aa28e29..fbf7486 100644 --- a/plugins.go +++ b/plugins.go @@ -4,6 +4,7 @@ import ( "bufio" "encoding/json" "fmt" + "log/slog" "net" "os/exec" "strings" @@ -18,16 +19,24 @@ type PluginManager struct { } func (m *PluginManager) Close() { - // Ignore errors for _, conn := range m.connections { message, _ := json.Marshal(Request{ Type: "exit", }) - conn.Write(append(message, '\n')) - conn.Close() + _, err := conn.Write(append(message, '\n')) + if err != nil { + slog.Error("Could not write exit message to plugn connection", slog.Any("error", err)) + } + err = conn.Close() + if err != nil { + slog.Error("Could not close plugin connection", slog.Any("error", err)) + } } for _, cmd := range m.spawnedCommands { - syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if err != nil { + slog.Error("Could not kill process", slog.Any("command", cmd), slog.Any("error", err)) + } } } @@ -66,11 +75,17 @@ func spawn(command []string) (*exec.Cmd, error) { line := scanner.Text() // Set deadline for outputting READY message timer := time.AfterFunc(5*time.Second, func() { - syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if err != nil { + slog.Error("Could not kill plugin after READY timeout", slog.Any("error", err)) + } }) if !strings.Contains(string(line), "READY") { - syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) - return nil, fmt.Errorf("Plugin command '%s' failed to output READY. Got: '%s' instead", strings.Join(command, " "), line) + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if err != nil { + slog.Error("Could not kill plugin after seeing it did not output READY", slog.Any("error", err)) + } + return nil, fmt.Errorf("plugin command '%s' failed to output READY. Got: '%s' instead", strings.Join(command, " "), line) } // Stop the timeout kill timer.Stop() @@ -88,7 +103,7 @@ func connect(address string) (net.Conn, error) { func (m *PluginManager) Call(valueFunc string, args ...string) (string, error) { plugin, hasPlugin := m.valueFuncMap[valueFunc] if !hasPlugin { - return "", fmt.Errorf("Plugin for valueFunc %s is not defined", valueFunc) + return "", fmt.Errorf("plugin for valueFunc %s is not defined", valueFunc) } conn, ok := m.connections[valueFunc] // Attempt to start process and wait for port to open @@ -105,7 +120,10 @@ func (m *PluginManager) Call(valueFunc string, args ...string) (string, error) { m.connections[valueFunc] = conn } // Send message to open TCP socket - conn.SetReadDeadline(time.Now().Add(time.Second)) + err := conn.SetReadDeadline(time.Now().Add(time.Second)) + if err != nil { + slog.Error("Could not set read deadline for plugin connection", slog.Any("error", err)) + } scanner := bufio.NewScanner(conn) message, err := json.Marshal(Request{ Type: "valueFunc", @@ -120,7 +138,7 @@ func (m *PluginManager) Call(valueFunc string, args ...string) (string, error) { return "", err } if !scanner.Scan() { - return "", fmt.Errorf("Plugin command '%s' failed to response to TCP message: %v", strings.Join(plugin.Command, " "), scanner.Err()) + return "", fmt.Errorf("plugin command '%s' failed to response to TCP message: %v", strings.Join(plugin.Command, " "), scanner.Err()) } line := scanner.Bytes() response := Response{} @@ -139,12 +157,12 @@ func NewPluginManager(plugins map[string]RipoffPlugin) (*PluginManager, error) { } for pluginName, plugin := range plugins { if len(plugin.Command) == 0 { - return nil, fmt.Errorf("Cannot create new PluginManager - the plugin %s does not define a command.", pluginName) + return nil, fmt.Errorf("cannot create new PluginManager - the plugin %s does not define a command", pluginName) } for _, valueFunc := range plugin.ValueFuncs { _, alreadySet := m.valueFuncMap[valueFunc] if alreadySet { - return nil, fmt.Errorf("Cannot create new PluginManager - the valueFunc %s is set in more than one plugin.", valueFunc) + return nil, fmt.Errorf("cannot create new PluginManager - the valueFunc %s is set in more than one plugin", valueFunc) } m.valueFuncMap[valueFunc] = plugin } From b39a566da98d3840ffcad1a6e9c39615b953858a Mon Sep 17 00:00:00 2001 From: Sam Mortenson Date: Thu, 19 Mar 2026 15:02:12 -0700 Subject: [PATCH 6/6] Bump golangci/golangci-lint-action --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 89ab8bd..ba9e679 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,6 +18,6 @@ jobs: with: go-version: stable - name: golangci-lint - uses: golangci/golangci-lint-action@v6 + uses: golangci/golangci-lint-action@v7 with: version: v2.11.3