From 81cbcfa5efd4de7d8102e1ae80b2d8b293ac4674 Mon Sep 17 00:00:00 2001 From: Jordan Oroshiba Date: Fri, 18 Jul 2025 17:53:11 -0700 Subject: [PATCH 1/4] feat: grpc support --- README.md | 81 +++++++++ cmd/root.go | 8 + cmd/web.go | 4 +- config-example.yaml | 51 ++++++ config-grpc-example.yaml | 45 +++++ config.yaml | 14 ++ config/config.go | 20 ++- go.mod | 12 +- go.sum | 27 ++- justfile | 8 +- mock/grpc_test.go | 73 ++++++++ mock/mock_engine.go | 168 +++++++++++++++++- proxy/grpc_handler.go | 239 ++++++++++++++++++++++++++ proxy/grpc_raw_proxy.go | 320 +++++++++++++++++++++++++++++++++++ proxy/grpc_test.go | 158 +++++++++++++++++ proxy/grpc_utils.go | 94 ++++++++++ proxy/proxy_engine.go | 68 +++++++- server/multi_proxy_server.go | 146 ++++++++++++---- web/server.go | 54 +++--- web/static/app.js | 114 +++++++++++-- web/static/style.css | 26 +++ 21 files changed, 1617 insertions(+), 113 deletions(-) create mode 100644 config-example.yaml create mode 100644 config-grpc-example.yaml create mode 100644 mock/grpc_test.go create mode 100644 proxy/grpc_handler.go create mode 100644 proxy/grpc_raw_proxy.go create mode 100644 proxy/grpc_test.go create mode 100644 proxy/grpc_utils.go diff --git a/README.md b/README.md index 04d40fc..f9bb303 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,87 @@ export: compress: false ``` +## gRPC Support + +Mimic now provides full gRPC proxy functionality for recording and replaying gRPC interactions. This includes support for unary and streaming RPCs with automatic protobuf message handling. + +### gRPC Configuration + +Configure gRPC proxies by setting the protocol to "grpc": + +```yaml +proxies: + grpc-api: + mode: "record" # or "mock" + protocol: "grpc" # Set to grpc for gRPC support + target_host: "api.grpc-service.com" + target_port: 9090 + session_name: "grpc-session" + +grpc: + proto_paths: # Paths to .proto files (optional) + - "./protos" + - "/usr/local/include" + reflection_enabled: true # Enable gRPC reflection (default: true) +``` + +### gRPC Recording + +Record gRPC interactions by running mimic in record mode with a gRPC-configured proxy: + +```bash +# Start gRPC recording +mimic --config config-grpc.yaml + +# Your gRPC client should connect to localhost:9080 (HTTP port + 1000) instead of the original server +# Mimic will forward the calls and record all interactions +# Example: if HTTP server runs on :8080, gRPC proxy will be on :9080 +``` + +### gRPC Mocking + +Replay recorded gRPC interactions: + +```bash +# Switch to mock mode in your config or use command line +mimic --mode mock --config config-grpc.yaml +``` + +### gRPC Features + +- **Unary RPCs**: Full support for request/response recording and replay +- **Streaming RPCs**: Support for server streaming, client streaming, and bidirectional streaming +- **Metadata Handling**: Records and replays gRPC metadata (headers) +- **Protobuf Messages**: Automatically converts protobuf messages to JSON for storage +- **Service Reflection**: Supports gRPC server reflection for dynamic service discovery +- **Error Handling**: Records and replays gRPC status codes and error messages + +### Example gRPC Workflow + +1. **Record gRPC calls**: + ```bash + # Start mimic with gRPC proxy configuration + mimic --config config-grpc-example.yaml + + # Your gRPC client should connect to the gRPC proxy port (HTTP port + 1000) + # If HTTP server is on :8080, gRPC proxy will be on :9080 + buf curl --schema buf.build/your/api --protocol grpc localhost:9080/your.service/Method + ``` + +2. **Export recorded session**: + ```bash + mimic export --session "grpc-session" --output "grpc-mocks.json" + ``` + +3. **Use in tests**: + ```bash + # Import the recorded session + mimic import --input "grpc-mocks.json" --session "test-grpc" + + # Start mock server (gRPC mock will be available on port 9080) + mimic --mode mock --config config-grpc-example.yaml + ``` + ## Usage ### Record Mode diff --git a/cmd/root.go b/cmd/root.go index e7d18c2..76db80e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,6 +21,7 @@ var ( outputFile string inputFile string mergeStrategy string + debugMode bool ) var rootCmd = &cobra.Command{ @@ -39,6 +40,7 @@ func Execute() error { func init() { rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is config.yaml)") + rootCmd.PersistentFlags().BoolVar(&debugMode, "debug", false, "enable debug logging") rootCmd.AddCommand(exportCmd) rootCmd.AddCommand(importCmd) @@ -48,6 +50,12 @@ func init() { } func runProxy() { + // Set up debug logging if requested + if debugMode { + log.SetFlags(log.LstdFlags | log.Lshortfile) + log.Println("Debug mode enabled") + } + cfg, err := config.LoadConfig(cfgFile) if err != nil { log.Fatal("Failed to load config:", err) diff --git a/cmd/web.go b/cmd/web.go index 91d7907..80ed91e 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -18,8 +18,6 @@ var webCmd = &cobra.Command{ }, } - - func runWebServer() { cfg, err := config.LoadConfig(cfgFile) if err != nil { @@ -40,4 +38,4 @@ func runWebServer() { if err := webServer.Start(); err != nil { log.Fatal("Web server failed:", err) } -} \ No newline at end of file +} diff --git a/config-example.yaml b/config-example.yaml new file mode 100644 index 0000000..4396487 --- /dev/null +++ b/config-example.yaml @@ -0,0 +1,51 @@ +server: + listen_host: "0.0.0.0" + listen_port: 8080 + +proxies: + anthropic: + mode: "record" + target_host: "api.anthropic.com" + target_port: 443 + protocol: "https" + session_name: "anthropic-session" + openai: + mode: "record" + target_host: "api.openai.com" + target_port: 443 + protocol: "https" + session_name: "openai-session" + local-mock: + mode: "mock" + protocol: "http" + session_name: "local-test-session" + +database: + path: "~/.mimic/recordings.db" + connection_pool_size: 10 + +recording: + session_name: "default" + capture_headers: true + capture_body: true + redact_patterns: + - "Authorization: Bearer .*" + - "X-Api-Key: .*" + +mock: + matching_strategy: "exact" # exact | pattern | fuzzy + sequence_mode: "ordered" # ordered | random + not_found_response: + status: 404 + body: + error: "Recording not found" + +grpc: + proto_paths: + - "./protos" + reflection_enabled: true + +export: + format: "json" + pretty_print: true + compress: false diff --git a/config-grpc-example.yaml b/config-grpc-example.yaml new file mode 100644 index 0000000..3598928 --- /dev/null +++ b/config-grpc-example.yaml @@ -0,0 +1,45 @@ +server: + listen_host: "0.0.0.0" + listen_port: 8080 + +proxies: + grpc-example: + mode: "record" + protocol: "grpc" + target_host: "localhost" + target_port: 9090 + session_name: "grpc-session" + + grpc-mock: + mode: "mock" + protocol: "grpc" + session_name: "grpc-session" + +database: + path: "~/.mimic/recordings.db" + connection_pool_size: 10 + +recording: + session_name: "default" + capture_headers: true + capture_body: true + redact_patterns: [] + +mock: + matching_strategy: "exact" + sequence_mode: "ordered" + not_found_response: + status: 404 + body: + error: "Recording not found" + +grpc: + proto_paths: + - "./protos" + - "/usr/local/include" + reflection_enabled: true + +export: + format: "json" + pretty_print: true + compress: false \ No newline at end of file diff --git a/config.yaml b/config.yaml index cec23cc..c76b183 100644 --- a/config.yaml +++ b/config.yaml @@ -10,6 +10,20 @@ proxies: protocol: "https" session_name: "anthropic-test" + astria-grpc: + mode: "mock" + protocol: "grpc" + target_host: "grpc.astria.org" + target_port: 443 + session_name: "grpc-mock-session" + + # test-grpc: + # mode: "record" + # protocol: "grpc" + # target_host: "localhost" + # target_port: 9191 + # session_name: "test-grpc-session" + database: path: "~/.mimic/recordings.db" connection_pool_size: 10 diff --git a/config/config.go b/config/config.go index 5ef0f7e..bb95c9f 100644 --- a/config/config.go +++ b/config/config.go @@ -9,13 +9,13 @@ import ( ) type Config struct { - Server ServerConfig `mapstructure:"server"` - Proxies map[string]ProxyConfig `mapstructure:"proxies"` - Database DatabaseConfig `mapstructure:"database"` - Recording RecordingConfig `mapstructure:"recording"` - Mock MockConfig `mapstructure:"mock"` - GRPC GRPCConfig `mapstructure:"grpc"` - Export ExportConfig `mapstructure:"export"` + Server ServerConfig `mapstructure:"server"` + Proxies map[string]ProxyConfig `mapstructure:"proxies"` + Database DatabaseConfig `mapstructure:"database"` + Recording RecordingConfig `mapstructure:"recording"` + Mock MockConfig `mapstructure:"mock"` + GRPC GRPCConfig `mapstructure:"grpc"` + Export ExportConfig `mapstructure:"export"` } type ServerConfig struct { @@ -57,6 +57,8 @@ type NotFoundResponseConfig struct { type GRPCConfig struct { ProtoPaths []string `mapstructure:"proto_paths"` ReflectionEnabled bool `mapstructure:"reflection_enabled"` + MaxMessageSize int `mapstructure:"max_message_size"` // Max message size in bytes + MaxHeaderSize int `mapstructure:"max_header_size"` // Max header list size in bytes } type ExportConfig struct { @@ -133,6 +135,8 @@ func setDefaults() { }) viper.SetDefault("grpc.reflection_enabled", true) + viper.SetDefault("grpc.max_message_size", 64*1024*1024) // 64MB + viper.SetDefault("grpc.max_header_size", 64*1024*1024) // 64MB viper.SetDefault("export.format", "json") viper.SetDefault("export.pretty_print", true) @@ -176,6 +180,8 @@ func getDefaultConfig() *Config { GRPC: GRPCConfig{ ProtoPaths: []string{}, ReflectionEnabled: true, + MaxMessageSize: 64 * 1024 * 1024, // 64MB + MaxHeaderSize: 64 * 1024 * 1024, // 64MB }, Export: ExportConfig{ Format: "json", diff --git a/go.mod b/go.mod index f59ffc8..91f2696 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,19 @@ go 1.21 require ( github.com/google/uuid v1.6.0 + github.com/gorilla/websocket v1.5.3 github.com/mattn/go-sqlite3 v1.14.19 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 + golang.org/x/net v0.19.0 + google.golang.org/grpc v1.59.0 + google.golang.org/protobuf v1.31.0 ) require ( github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/gorilla/websocket v1.5.3 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -27,8 +32,9 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 885260c..cb24cb1 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,12 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= @@ -67,10 +71,21 @@ go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/justfile b/justfile index 78e08f1..5e9fd6e 100644 --- a/justfile +++ b/justfile @@ -23,14 +23,14 @@ install: run *args: go run . {{args}} -# Run with default config -run: - go run . --config config.yaml - # Run tests test: go test ./... +# Run integration tests +integration-test: + ./integration_test.sh + # Run tests with coverage test-coverage: go test -coverprofile=coverage.out ./... diff --git a/mock/grpc_test.go b/mock/grpc_test.go new file mode 100644 index 0000000..6f27a03 --- /dev/null +++ b/mock/grpc_test.go @@ -0,0 +1,73 @@ +package mock + +import ( + "testing" + + "mimic/config" + "mimic/storage" +) + +func TestMockEngineWithGRPC(t *testing.T) { + // Create temporary database + db, err := storage.NewDatabase(":memory:") + if err != nil { + t.Fatalf("Failed to create database: %v", err) + } + defer db.Close() + + // Create gRPC mock config + proxyConfig := config.ProxyConfig{ + Mode: "mock", + Protocol: "grpc", + SessionName: "test-session", + } + + // Create mock engine + engine, err := NewMockEngine(proxyConfig, db) + if err != nil { + t.Fatalf("Failed to create mock engine: %v", err) + } + defer engine.Stop() + + // Verify gRPC components are initialized + if engine.grpcHandler == nil { + t.Error("Expected gRPC handler to be initialized") + } + + if engine.grpcServer == nil { + t.Error("Expected gRPC server to be initialized for gRPC protocol") + } +} + +func TestMockEngineWithHTTP(t *testing.T) { + // Create temporary database + db, err := storage.NewDatabase(":memory:") + if err != nil { + t.Fatalf("Failed to create database: %v", err) + } + defer db.Close() + + // Create HTTP mock config + proxyConfig := config.ProxyConfig{ + Mode: "mock", + Protocol: "http", + SessionName: "test-session", + } + + // Create mock engine + engine, err := NewMockEngine(proxyConfig, db) + if err != nil { + t.Fatalf("Failed to create mock engine: %v", err) + } + defer engine.Stop() + + // Verify HTTP components are initialized + if engine.restHandler == nil { + t.Error("Expected REST handler to be initialized") + } + + // Verify gRPC components are NOT initialized for HTTP protocol + if engine.grpcServer != nil { + t.Error("Expected gRPC server to be nil for HTTP protocol") + } +} diff --git a/mock/mock_engine.go b/mock/mock_engine.go index 51c9818..82ff314 100644 --- a/mock/mock_engine.go +++ b/mock/mock_engine.go @@ -6,19 +6,61 @@ import ( "fmt" "io" "log" + "net" "net/http" "strings" "sync" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" "mimic/config" "mimic/proxy" "mimic/storage" ) +// mockRawMessage for handling raw gRPC message data in mock responses +type mockRawMessage struct { + Data []byte +} + +// Reset implements proto.Message +func (m *mockRawMessage) Reset() { + m.Data = nil +} + +// String implements proto.Message +func (m *mockRawMessage) String() string { + return fmt.Sprintf("mockRawMessage{%d bytes}", len(m.Data)) +} + +// ProtoMessage implements proto.Message +func (m *mockRawMessage) ProtoMessage() {} + +// Marshal implements protobuf marshaling +func (m *mockRawMessage) Marshal() ([]byte, error) { + return m.Data, nil +} + +// Unmarshal implements protobuf unmarshaling +func (m *mockRawMessage) Unmarshal(data []byte) error { + m.Data = make([]byte, len(data)) + copy(m.Data, data) + return nil +} + +// Size returns the size of the message +func (m *mockRawMessage) Size() int { + return len(m.Data) +} + type MockEngine struct { proxyConfig *config.ProxyConfig database *storage.Database restHandler *proxy.RESTHandler + grpcHandler *proxy.GRPCHandler + grpcServer *grpc.Server session *storage.Session sequenceState map[string]int sequenceMutex sync.RWMutex @@ -41,37 +83,77 @@ func NewMockEngineWithBroadcaster(proxyConfig config.ProxyConfig, db *storage.Da } restHandler := proxy.NewRESTHandler([]string{}) // Use empty redact patterns for now + grpcHandler := proxy.NewGRPCHandler([]string{}) // Use empty redact patterns for now + + var grpcServer *grpc.Server + if proxyConfig.Protocol == "grpc" { + grpcServer = grpc.NewServer( + grpc.MaxRecvMsgSize(64*1024*1024), // 64MB max receive message size + grpc.MaxSendMsgSize(64*1024*1024), // 64MB max send message size + grpc.MaxHeaderListSize(64*1024*1024), // 64MB max header list size + grpc.InitialWindowSize(64*1024*1024), // 64MB initial window + grpc.InitialConnWindowSize(64*1024*1024), // 64MB connection window + grpc.UnknownServiceHandler(func(srv interface{}, stream grpc.ServerStream) error { + return handleGRPCMockRequest(stream, db, session, grpcHandler) + }), + ) + } return &MockEngine{ proxyConfig: &proxyConfig, database: db, restHandler: restHandler, + grpcHandler: grpcHandler, + grpcServer: grpcServer, session: session, sequenceState: make(map[string]int), webServer: webServer, }, nil } +func (m *MockEngine) Start() error { + address := "0.0.0.0:8080" // This method shouldn't be used in multi-proxy mode + if m.proxyConfig.Protocol == "grpc" { + return m.startGRPCMockServer(address) + } else { + return m.startHTTPMockServer(address) + } +} -func (m *MockEngine) Start() error { +func (m *MockEngine) startHTTPMockServer(address string) error { mux := http.NewServeMux() - + // Register web UI routes if webServer is available if webServer, ok := m.webServer.(interface{ RegisterRoutes(*http.ServeMux) }); ok { webServer.RegisterRoutes(mux) } - + // All other requests go to mock handler mux.HandleFunc("/", m.handleRequest) - address := "0.0.0.0:8080" // This method shouldn't be used in multi-proxy mode - log.Printf("Starting mock server in %s mode on %s", m.proxyConfig.Mode, address) + log.Printf("Starting HTTP mock server in %s mode on %s", m.proxyConfig.Mode, address) log.Printf("Serving mocked responses for session: %s", m.session.SessionName) return http.ListenAndServe(address, mux) } +func (m *MockEngine) startGRPCMockServer(address string) error { + if m.grpcServer == nil { + return fmt.Errorf("gRPC mock server not initialized") + } + + lis, err := net.Listen("tcp", address) + if err != nil { + return fmt.Errorf("failed to listen on %s: %w", address, err) + } + + log.Printf("Starting gRPC mock server in %s mode on %s", m.proxyConfig.Mode, address) + log.Printf("Serving mocked responses for session: %s", m.session.SessionName) + + return m.grpcServer.Serve(lis) +} + // HandleRequest implements the ProxyHandler interface func (m *MockEngine) HandleRequest(w http.ResponseWriter, r *http.Request) { m.handleRequest(w, r) @@ -86,7 +168,7 @@ func (m *MockEngine) handleRequest(w http.ResponseWriter, r *http.Request) { for key, values := range r.Header { requestHeaders[key] = strings.Join(values, ", ") } - + var requestBody string if r.Body != nil { bodyBytes, err := io.ReadAll(r.Body) @@ -95,7 +177,7 @@ func (m *MockEngine) handleRequest(w http.ResponseWriter, r *http.Request) { r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) } } - + m.webServer.BroadcastRequest(r.Method, r.URL.Path, m.session.SessionName, r.RemoteAddr, "", requestHeaders, requestBody) } @@ -349,9 +431,16 @@ func (m *MockEngine) sendNotFoundResponse(w http.ResponseWriter) { } func (m *MockEngine) Stop() error { + if m.grpcServer != nil { + m.grpcServer.GracefulStop() + } return nil } +func (m *MockEngine) GetGRPCServer() *grpc.Server { + return m.grpcServer +} + func (m *MockEngine) ResetSequenceState() { m.sequenceMutex.Lock() defer m.sequenceMutex.Unlock() @@ -371,3 +460,68 @@ func (m *MockEngine) GetSequenceState() map[string]int { return state } + +// handleGRPCMockRequest handles gRPC mock requests +func handleGRPCMockRequest(stream grpc.ServerStream, db *storage.Database, session *storage.Session, grpcHandler *proxy.GRPCHandler) error { + fullMethodName, ok := grpc.MethodFromServerStream(stream) + if !ok { + return status.Errorf(codes.Internal, "failed to get method from stream") + } + + log.Printf("[GRPC MOCK] %s", fullMethodName) + + // Find matching gRPC interactions + interactions, err := db.FindMatchingInteractions(session.ID, fullMethodName, fullMethodName) + if err != nil { + log.Printf("Error finding matching gRPC interactions: %v", err) + return status.Errorf(codes.Internal, "failed to find matching interactions") + } + + if len(interactions) == 0 { + log.Printf("No matching gRPC interactions found for %s", fullMethodName) + return status.Errorf(codes.NotFound, "no recorded interaction found for method %s", fullMethodName) + } + + // For simplicity, use the first matching interaction + // In a more sophisticated implementation, we could add sequence support for gRPC + selectedInteraction := &interactions[0] + + // Create a mock gRPC response + // Note: This is a simplified implementation + // In practice, we would need to handle protobuf message types dynamically + + // Send response headers/metadata if present + if selectedInteraction.ResponseHeaders != "" { + var metadataMap map[string][]string + if err := json.Unmarshal([]byte(selectedInteraction.ResponseHeaders), &metadataMap); err == nil { + md := metadata.New(nil) + for key, values := range metadataMap { + md.Set(key, values...) + } + if err := stream.SendHeader(md); err != nil { + return err + } + } + } + + // First, receive the request message from the client (required for unary calls) + var requestMsg mockRawMessage + if err := stream.RecvMsg(&requestMsg); err != nil { + log.Printf("Error receiving request message: %v", err) + return status.Errorf(codes.Internal, "failed to receive request: %v", err) + } + + // Send the recorded response body if available + if len(selectedInteraction.ResponseBody) > 0 { + // Create a raw message with the recorded response data + responseMsg := mockRawMessage{Data: selectedInteraction.ResponseBody} + if err := stream.SendMsg(&responseMsg); err != nil { + return status.Errorf(codes.Internal, "failed to send response: %v", err) + } + log.Printf("Served gRPC mock response: %s -> %d (%d bytes)", fullMethodName, selectedInteraction.ResponseStatus, len(selectedInteraction.ResponseBody)) + } else { + log.Printf("Served gRPC mock response: %s -> %d (empty response)", fullMethodName, selectedInteraction.ResponseStatus) + } + + return nil +} diff --git a/proxy/grpc_handler.go b/proxy/grpc_handler.go new file mode 100644 index 0000000..33bb77a --- /dev/null +++ b/proxy/grpc_handler.go @@ -0,0 +1,239 @@ +package proxy + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/dynamicpb" + + "github.com/google/uuid" + "mimic/storage" +) + +type GRPCHandler struct { + redactPatterns []*regexp.Regexp +} + +func NewGRPCHandler(redactPatterns []string) *GRPCHandler { + patterns := make([]*regexp.Regexp, len(redactPatterns)) + for i, pattern := range redactPatterns { + if compiled, err := regexp.Compile(pattern); err == nil { + patterns[i] = compiled + } + } + return &GRPCHandler{redactPatterns: patterns} +} + +type GRPCRequest struct { + Method string + Metadata metadata.MD + Message proto.Message +} + +type GRPCResponse struct { + Status *status.Status + Metadata metadata.MD + Message proto.Message +} + +func (h *GRPCHandler) ExtractGRPCRequest(method string, md metadata.MD, req proto.Message) (*storage.Interaction, error) { + requestID := uuid.New().String() + + // Convert metadata to JSON + metadataMap := make(map[string][]string) + for key, values := range md { + metadataMap[key] = values + } + + headersJSON, err := json.Marshal(metadataMap) + if err != nil { + return nil, fmt.Errorf("failed to marshal gRPC metadata: %w", err) + } + + headersStr := string(headersJSON) + headersStr = h.redactSensitiveData(headersStr) + + // Convert protobuf message to JSON for storage + var body []byte + if req != nil { + jsonBytes, err := protojson.Marshal(req) + if err != nil { + return nil, fmt.Errorf("failed to marshal protobuf message to JSON: %w", err) + } + body = jsonBytes + } + + return &storage.Interaction{ + RequestID: requestID, + Protocol: "gRPC", + Method: method, + Endpoint: method, // For gRPC, method is the endpoint + RequestHeaders: headersStr, + RequestBody: body, + Timestamp: time.Now(), + }, nil +} + +func (h *GRPCHandler) ExtractGRPCResponse(st *status.Status, md metadata.MD, resp proto.Message) (int, string, []byte, error) { + // Convert status to HTTP-like status code for storage + statusCode := int(st.Code()) + + // Convert metadata to JSON + metadataMap := make(map[string][]string) + for key, values := range md { + metadataMap[key] = values + } + + headersJSON, err := json.Marshal(metadataMap) + if err != nil { + return 0, "", nil, fmt.Errorf("failed to marshal gRPC response metadata: %w", err) + } + + headersStr := string(headersJSON) + headersStr = h.redactSensitiveData(headersStr) + + // Convert protobuf message to JSON for storage + var body []byte + if resp != nil { + jsonBytes, err := protojson.Marshal(resp) + if err != nil { + return 0, "", nil, fmt.Errorf("failed to marshal protobuf response to JSON: %w", err) + } + body = jsonBytes + } + + return statusCode, headersStr, body, nil +} + +func (h *GRPCHandler) CreateGRPCResponse(interaction *storage.Interaction, messageType protoreflect.MessageType) (*GRPCResponse, error) { + // Parse stored metadata + var metadataMap map[string][]string + if interaction.ResponseHeaders != "" { + if err := json.Unmarshal([]byte(interaction.ResponseHeaders), &metadataMap); err != nil { + return nil, fmt.Errorf("failed to unmarshal response metadata: %w", err) + } + } + + md := metadata.New(nil) + for key, values := range metadataMap { + md.Set(key, values...) + } + + // Create status from stored status code + st := status.New(codes.Code(interaction.ResponseStatus), "") + + // Parse stored message + var message proto.Message + if len(interaction.ResponseBody) > 0 && messageType != nil { + message = dynamicpb.NewMessage(messageType.Descriptor()) + if err := protojson.Unmarshal(interaction.ResponseBody, message); err != nil { + return nil, fmt.Errorf("failed to unmarshal response message: %w", err) + } + } + + return &GRPCResponse{ + Status: st, + Metadata: md, + Message: message, + }, nil +} + +func (h *GRPCHandler) MatchGRPCRequest(method string, md metadata.MD, interaction *storage.Interaction, strategy string) bool { + switch strategy { + case "exact": + return h.exactGRPCMatch(method, md, interaction) + case "pattern": + return h.patternGRPCMatch(method, interaction) + case "fuzzy": + return h.fuzzyGRPCMatch(method, interaction) + default: + return h.exactGRPCMatch(method, md, interaction) + } +} + +func (h *GRPCHandler) exactGRPCMatch(method string, md metadata.MD, interaction *storage.Interaction) bool { + return method == interaction.Method +} + +func (h *GRPCHandler) patternGRPCMatch(method string, interaction *storage.Interaction) bool { + pattern, err := regexp.Compile(interaction.Method) + if err != nil { + return false + } + return pattern.MatchString(method) +} + +func (h *GRPCHandler) fuzzyGRPCMatch(method string, interaction *storage.Interaction) bool { + // Simple fuzzy matching for gRPC methods + // This could be enhanced with more sophisticated matching logic + return method == interaction.Method +} + +func (h *GRPCHandler) redactSensitiveData(data string) string { + result := data + for _, pattern := range h.redactPatterns { + result = pattern.ReplaceAllString(result, "[REDACTED]") + } + return result +} + +func (h *GRPCHandler) GetRedactPatterns() []*regexp.Regexp { + return h.redactPatterns +} + +// GRPCInterceptor creates a grpc.UnaryServerInterceptor for recording +func (h *GRPCHandler) GRPCInterceptor(db *storage.Database, session *storage.Session) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + md, _ := metadata.FromIncomingContext(ctx) + + // Extract request + protoReq, ok := req.(proto.Message) + if !ok { + return handler(ctx, req) + } + + interaction, err := h.ExtractGRPCRequest(info.FullMethod, md, protoReq) + if err != nil { + return handler(ctx, req) + } + + interaction.SessionID = session.ID + + // Call the actual handler + resp, err := handler(ctx, req) + + // Extract response + var protoResp proto.Message + if resp != nil { + if pr, ok := resp.(proto.Message); ok { + protoResp = pr + } + } + + st, _ := status.FromError(err) + outMd, _ := metadata.FromOutgoingContext(ctx) + + statusCode, headers, body, extractErr := h.ExtractGRPCResponse(st, outMd, protoResp) + if extractErr == nil { + interaction.ResponseStatus = statusCode + interaction.ResponseHeaders = headers + interaction.ResponseBody = body + + if recordErr := db.RecordInteraction(interaction); recordErr != nil { + // Log error but don't fail the request + } + } + + return resp, err + } +} diff --git a/proxy/grpc_raw_proxy.go b/proxy/grpc_raw_proxy.go new file mode 100644 index 0000000..1332d0b --- /dev/null +++ b/proxy/grpc_raw_proxy.go @@ -0,0 +1,320 @@ +package proxy + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "mimic/config" + "mimic/storage" +) + +// RawGRPCProxy implements raw byte-level gRPC proxying +type RawGRPCProxy struct { + config *config.ProxyConfig + database *storage.Database + session *storage.Session + handler *GRPCHandler + webServer WebBroadcaster +} + + + +func NewRawGRPCProxy(proxyConfig *config.ProxyConfig, db *storage.Database, session *storage.Session, grpcHandler *GRPCHandler) *RawGRPCProxy { + return &RawGRPCProxy{ + config: proxyConfig, + database: db, + session: session, + handler: grpcHandler, + webServer: nil, // Will be set by proxy engine + } +} + +func (p *RawGRPCProxy) SetWebBroadcaster(wb WebBroadcaster) { + p.webServer = wb +} + + + +// GetUnknownServiceHandler returns a handler that can proxy any gRPC service using raw bytes +func (p *RawGRPCProxy) GetUnknownServiceHandler() grpc.StreamHandler { + // Register our raw codec + RegisterRawCodec() + + return func(srv interface{}, stream grpc.ServerStream) error { + fullMethodName, ok := grpc.MethodFromServerStream(stream) + if !ok { + return status.Errorf(codes.Internal, "failed to get method from stream") + } + + log.Printf("Raw proxy handling: %s", fullMethodName) + + // Create connection to target + targetAddr := fmt.Sprintf("%s:%d", p.config.TargetHost, p.config.TargetPort) + ctx := stream.Context() + + // Determine if we should use TLS based on port + var creds credentials.TransportCredentials + if p.config.TargetPort == 443 || p.config.Protocol == "https" { + creds = credentials.NewTLS(nil) // Use system root CAs + } else { + creds = insecure.NewCredentials() + } + + conn, err := grpc.DialContext(ctx, targetAddr, + grpc.WithTransportCredentials(creds), + grpc.WithInitialWindowSize(64*1024*1024), // 64MB initial window + grpc.WithInitialConnWindowSize(64*1024*1024), // 64MB connection window + grpc.WithReadBufferSize(1024*1024), // 1MB read buffer + grpc.WithWriteBufferSize(1024*1024), // 1MB write buffer + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(64*1024*1024), + grpc.MaxCallSendMsgSize(64*1024*1024), + ), + ) + if err != nil { + return status.Errorf(codes.Unavailable, "failed to connect to backend %s: %v", targetAddr, err) + } + defer conn.Close() + + // Determine if this is a unary vs streaming call + if p.isLikelyUnaryCall(fullMethodName) { + return p.handleUnaryCall(ctx, conn, stream, fullMethodName) + } + // Create client stream using raw codec + clientStream, err := conn.NewStream( + ctx, + &grpc.StreamDesc{ + StreamName: fullMethodName, + ServerStreams: true, + ClientStreams: true, + }, + fullMethodName, + grpc.ForceCodec(GetRawCodec()), + ) + if err != nil { + return status.Errorf(codes.Internal, "failed to create client stream for %s: %v", fullMethodName, err) + } + + return p.proxyRawStream(stream, clientStream, fullMethodName) + } +} + +// proxyRawStream proxies using raw message handling +func (p *RawGRPCProxy) proxyRawStream(serverStream grpc.ServerStream, clientStream grpc.ClientStream, method string) error { + errCh := make(chan error, 2) + + // Proxy client->server (requests) + go func() { + defer func() { + clientStream.CloseSend() + }() + + for { + var msg RawMessage + if err := serverStream.RecvMsg(&msg); err != nil { + if err == io.EOF { + errCh <- nil + return + } + errCh <- fmt.Errorf("server recv error: %w", err) + return + } + + if p.config.Mode == "record" { + log.Printf("→ %s: %d bytes", method, len(msg.Data)) + } + + if err := clientStream.SendMsg(msg); err != nil { + errCh <- fmt.Errorf("client send error: %w", err) + return + } + } + }() + + // Proxy server->client (responses) + go func() { + for { + var msg RawMessage + if err := clientStream.RecvMsg(&msg); err != nil { + if err == io.EOF { + errCh <- nil + return + } + errCh <- fmt.Errorf("client recv error: %w", err) + return + } + + if p.config.Mode == "record" { + log.Printf("← %s: %d bytes", method, len(msg.Data)) + } + + if err := serverStream.SendMsg(msg); err != nil { + errCh <- fmt.Errorf("server send error: %w", err) + return + } + } + }() + + return <-errCh +} + + + +func (p *RawGRPCProxy) metadataToJSON(md metadata.MD) string { + metadataMap := make(map[string][]string) + for key, values := range md { + metadataMap[key] = values + } + jsonBytes, err := json.Marshal(metadataMap) + if err != nil { + return "{}" + } + return string(jsonBytes) +} + +func (p *RawGRPCProxy) metadataToMap(md metadata.MD) map[string]interface{} { + result := make(map[string]interface{}) + for key, values := range md { + if len(values) == 1 { + result[key] = values[0] + } else { + result[key] = values + } + } + return result +} + +// isLikelyUnaryCall heuristically determines if a method is likely a unary call +func (p *RawGRPCProxy) isLikelyUnaryCall(method string) bool { + // Methods with streaming patterns are definitely streaming + streamingPatterns := []string{ + "Stream", "Watch", "Subscribe", "Listen", "Monitor", "Observe", + } + + for _, pattern := range streamingPatterns { + if strings.Contains(method, pattern) { + return false + } + } + + // Common patterns for unary calls + unaryPatterns := []string{ + "Get", "Create", "Update", "Delete", "Check", "Validate", + "Info", "Status", "Health", "Ping", "Version", "List", + } + + for _, pattern := range unaryPatterns { + if strings.Contains(method, pattern) { + return true + } + } + + // Default to unary for unknown patterns + return true +} + +// handleUnaryCall handles unary gRPC calls +func (p *RawGRPCProxy) handleUnaryCall(ctx context.Context, conn *grpc.ClientConn, stream grpc.ServerStream, method string) error { + // Receive the request from client + var requestMsg RawMessage + if err := stream.RecvMsg(&requestMsg); err != nil { + return status.Errorf(codes.Internal, "failed to receive request: %v", err) + } + + if p.config.Mode == "record" { + log.Printf("→ %s: %d bytes (unary)", method, len(requestMsg.Data)) + } + + // Extract and forward metadata + md, _ := metadata.FromIncomingContext(stream.Context()) + outCtx := metadata.NewOutgoingContext(ctx, md) + + // Create interaction record for database storage + var interaction *storage.Interaction + if p.config.Mode == "record" { + interaction = &storage.Interaction{ + RequestID: GenerateRequestID(), + SessionID: p.session.ID, + Protocol: "gRPC", + Method: method, + Endpoint: method, + RequestHeaders: p.metadataToJSON(md), + RequestBody: requestMsg.Data, + Timestamp: time.Now(), + } + + // Broadcast request event to web UI + if p.webServer != nil { + log.Printf("[DEBUG] Broadcasting gRPC request to web UI: %s", method) + headers := p.metadataToMap(md) + body := fmt.Sprintf("gRPC raw message (%d bytes)", len(requestMsg.Data)) + p.webServer.BroadcastRequest(method, method, p.session.SessionName, "grpc-client", interaction.RequestID, headers, body) + } else { + log.Printf("[DEBUG] No webServer available for broadcasting gRPC request") + } + } + + // Forward the unary call to target server + var responseMsg RawMessage + err := conn.Invoke(outCtx, method, &requestMsg, &responseMsg, grpc.ForceCodec(GetRawCodec())) + + // Handle recording and response + if p.config.Mode == "record" { + statusCode := 0 + if err != nil { + if st, ok := status.FromError(err); ok { + statusCode = int(st.Code()) + } else { + statusCode = int(codes.Unknown) + } + } else { + statusCode = int(codes.OK) + } + + log.Printf("← %s: %d bytes (unary)", method, len(responseMsg.Data)) + + // Complete the interaction record + interaction.ResponseStatus = statusCode + interaction.ResponseHeaders = "{}" // Empty metadata for now + interaction.ResponseBody = responseMsg.Data + + // Save to database + if recordErr := p.database.RecordInteraction(interaction); recordErr != nil { + log.Printf("Error recording gRPC interaction: %v", recordErr) + } else { + log.Printf("Recorded gRPC interaction: %s -> %d", method, statusCode) + } + + // Broadcast response event to web UI + if p.webServer != nil { + log.Printf("[DEBUG] Broadcasting gRPC response to web UI: %s", method) + responseHeaders := make(map[string]interface{}) + responseBody := fmt.Sprintf("gRPC raw message (%d bytes)", len(responseMsg.Data)) + p.webServer.BroadcastResponse(method, method, p.session.SessionName, "grpc-client", interaction.RequestID, statusCode, responseHeaders, responseBody) + } else { + log.Printf("[DEBUG] No webServer available for broadcasting gRPC response") + } + } + + if err != nil { + return err + } + + // Send response back to client + if err := stream.SendMsg(&responseMsg); err != nil { + return status.Errorf(codes.Internal, "failed to send response: %v", err) + } + + return nil +} \ No newline at end of file diff --git a/proxy/grpc_test.go b/proxy/grpc_test.go new file mode 100644 index 0000000..72cde50 --- /dev/null +++ b/proxy/grpc_test.go @@ -0,0 +1,158 @@ +package proxy + +import ( + "testing" + + "mimic/config" + "mimic/storage" +) + +func TestNewGRPCHandler(t *testing.T) { + redactPatterns := []string{"password", "token"} + handler := NewGRPCHandler(redactPatterns) + + if handler == nil { + t.Fatal("Expected non-nil gRPC handler") + } + + if len(handler.redactPatterns) != 2 { + t.Errorf("Expected 2 redact patterns, got %d", len(handler.redactPatterns)) + } +} + +func TestGRPCHandlerRedactSensitiveData(t *testing.T) { + redactPatterns := []string{"password"} + handler := NewGRPCHandler(redactPatterns) + + data := `{"username": "john", "password": "secret123"}` + redacted := handler.redactSensitiveData(data) + + if redacted == data { + t.Error("Expected data to be redacted") + } + + if !contains(redacted, "[REDACTED]") { + t.Error("Expected data to contain [REDACTED]") + } +} + +func TestRawGRPCProxyUnaryCallDetection(t *testing.T) { + db, err := storage.NewDatabase(":memory:") + if err != nil { + t.Fatalf("Failed to create database: %v", err) + } + defer db.Close() + + proxyConfig := config.ProxyConfig{ + Mode: "record", + Protocol: "grpc", + TargetHost: "localhost", + TargetPort: 9090, + SessionName: "test-session", + } + + grpcHandler := NewGRPCHandler([]string{}) + session, _ := db.GetOrCreateSession("test", "test") + + rawProxy := NewRawGRPCProxy(&proxyConfig, db, session, grpcHandler) + + // Test unary call detection + testCases := []struct { + method string + expected bool + }{ + {"/service/GetInfo", true}, + {"/service/CreateUser", true}, + {"/service/UpdateData", true}, + {"/service/DeleteItem", true}, + {"/service/StreamData", false}, + {"/service/WatchEvents", false}, + {"/astria.sequencerblock.v1.SequencerService/GetUpgradesInfo", true}, + } + + for _, tc := range testCases { + result := rawProxy.isLikelyUnaryCall(tc.method) + if result != tc.expected { + t.Errorf("Method %s: expected %v, got %v", tc.method, tc.expected, result) + } + } +} + +func TestProxyEngineWithGRPC(t *testing.T) { + // Create temporary database + db, err := storage.NewDatabase(":memory:") + if err != nil { + t.Fatalf("Failed to create database: %v", err) + } + defer db.Close() + + // Create gRPC proxy config + proxyConfig := config.ProxyConfig{ + Mode: "record", + Protocol: "grpc", + TargetHost: "localhost", + TargetPort: 9090, + SessionName: "test-session", + } + + // Create proxy engine + engine, err := NewProxyEngine(proxyConfig, db) + if err != nil { + t.Fatalf("Failed to create proxy engine: %v", err) + } + defer engine.Stop() + + // Verify gRPC components are initialized + if engine.grpcHandler == nil { + t.Error("Expected gRPC handler to be initialized") + } + + if engine.grpcServer == nil { + t.Error("Expected gRPC server to be initialized for gRPC protocol") + } +} + +func TestProxyEngineWithHTTP(t *testing.T) { + // Create temporary database + db, err := storage.NewDatabase(":memory:") + if err != nil { + t.Fatalf("Failed to create database: %v", err) + } + defer db.Close() + + // Create HTTP proxy config + proxyConfig := config.ProxyConfig{ + Mode: "record", + Protocol: "http", + TargetHost: "localhost", + TargetPort: 8080, + SessionName: "test-session", + } + + // Create proxy engine + engine, err := NewProxyEngine(proxyConfig, db) + if err != nil { + t.Fatalf("Failed to create proxy engine: %v", err) + } + defer engine.Stop() + + // Verify HTTP components are initialized + if engine.restHandler == nil { + t.Error("Expected REST handler to be initialized") + } + + // Verify gRPC components are NOT initialized for HTTP protocol + if engine.grpcServer != nil { + t.Error("Expected gRPC server to be nil for HTTP protocol") + } +} + +// Helper function +func contains(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} diff --git a/proxy/grpc_utils.go b/proxy/grpc_utils.go new file mode 100644 index 0000000..d61fde5 --- /dev/null +++ b/proxy/grpc_utils.go @@ -0,0 +1,94 @@ +package proxy + +import ( + "fmt" + "time" + + "google.golang.org/grpc/encoding" +) + +// RawMessage is a message that holds raw bytes for gRPC proxying +type RawMessage struct { + Data []byte +} + +// Reset implements proto.Message +func (m *RawMessage) Reset() { m.Data = nil } + +// String implements proto.Message +func (m *RawMessage) String() string { return fmt.Sprintf("RawMessage{%d bytes}", len(m.Data)) } + +// ProtoMessage implements proto.Message +func (m *RawMessage) ProtoMessage() {} + +// XXX_WellKnownType is used by gRPC-Go to determine if this message is well-known +func (m *RawMessage) XXX_WellKnownType() string { return "" } + +// Marshal implements proto.Marshal +func (m *RawMessage) Marshal() ([]byte, error) { + return m.Data, nil +} + +// Unmarshal implements proto.Unmarshal +func (m *RawMessage) Unmarshal(data []byte) error { + m.Data = make([]byte, len(data)) + copy(m.Data, data) + return nil +} + +// Size returns the size of the marshaled message +func (m *RawMessage) Size() int { + return len(m.Data) +} + +// XXX_MessageName returns message name +func (m *RawMessage) XXX_MessageName() string { + return "mimic.RawMessage" +} + +// rawCodec implements encoding.Codec for raw byte handling +type rawCodec struct{} + +func (rawCodec) Marshal(v interface{}) ([]byte, error) { + switch m := v.(type) { + case *RawMessage: + return m.Data, nil + case RawMessage: + return m.Data, nil + case []byte: + return m, nil + default: + return nil, fmt.Errorf("unsupported message type: %T", v) + } +} + +func (rawCodec) Unmarshal(data []byte, v interface{}) error { + switch m := v.(type) { + case *RawMessage: + m.Data = make([]byte, len(data)) + copy(m.Data, data) + return nil + case *[]byte: + *m = data + return nil + default: + return fmt.Errorf("unsupported message type: %T", v) + } +} + +func (rawCodec) Name() string { return "raw" } + +// GetRawCodec returns a raw codec instance for gRPC +func GetRawCodec() encoding.Codec { + return rawCodec{} +} + +// RegisterRawCodec registers the raw codec for gRPC usage +func RegisterRawCodec() { + encoding.RegisterCodec(rawCodec{}) +} + +// GenerateRequestID generates a unique request ID for gRPC interactions +func GenerateRequestID() string { + return fmt.Sprintf("grpc-%d", time.Now().UnixNano()) +} \ No newline at end of file diff --git a/proxy/proxy_engine.go b/proxy/proxy_engine.go index a7afc0a..a8d6c1e 100644 --- a/proxy/proxy_engine.go +++ b/proxy/proxy_engine.go @@ -4,9 +4,11 @@ import ( "encoding/json" "fmt" "log" + "net" "net/http" "time" + "google.golang.org/grpc" "mimic/config" "mimic/storage" ) @@ -15,8 +17,10 @@ type ProxyEngine struct { proxyConfig *config.ProxyConfig database *storage.Database restHandler *RESTHandler + grpcHandler *GRPCHandler session *storage.Session client *http.Client + grpcServer *grpc.Server webServer WebBroadcaster } @@ -36,6 +40,7 @@ func NewProxyEngineWithBroadcaster(proxyConfig config.ProxyConfig, db *storage.D } restHandler := NewRESTHandler([]string{}) // Use empty redact patterns for now + grpcHandler := NewGRPCHandler([]string{}) // Use empty redact patterns for now client := &http.Client{ Timeout: 30 * time.Second, @@ -47,36 +52,82 @@ func NewProxyEngineWithBroadcaster(proxyConfig config.ProxyConfig, db *storage.D }, } + var grpcServer *grpc.Server + + if proxyConfig.Protocol == "grpc" { + // Use raw proxy for better compatibility + rawProxy := NewRawGRPCProxy(&proxyConfig, db, session, grpcHandler) + + // Set web broadcaster if available + if webServer != nil { + rawProxy.SetWebBroadcaster(webServer) + } + + grpcServer = grpc.NewServer( + grpc.MaxRecvMsgSize(64*1024*1024), // 64MB max receive message size + grpc.MaxSendMsgSize(64*1024*1024), // 64MB max send message size + grpc.MaxHeaderListSize(64*1024*1024), // 64MB max header list size + grpc.InitialWindowSize(64*1024*1024), // 64MB initial window + grpc.InitialConnWindowSize(64*1024*1024), // 64MB connection window + grpc.UnknownServiceHandler(rawProxy.GetUnknownServiceHandler()), + ) + } + return &ProxyEngine{ proxyConfig: &proxyConfig, database: db, restHandler: restHandler, + grpcHandler: grpcHandler, session: session, client: client, + grpcServer: grpcServer, webServer: webServer, }, nil } +func (p *ProxyEngine) Start() error { + address := "0.0.0.0:8080" // This method shouldn't be used in multi-proxy mode + if p.proxyConfig.Protocol == "grpc" { + return p.startGRPCServer(address) + } else { + return p.startHTTPServer(address) + } +} -func (p *ProxyEngine) Start() error { +func (p *ProxyEngine) startHTTPServer(address string) error { mux := http.NewServeMux() - + // Register web UI routes if webServer is available if webServer, ok := p.webServer.(interface{ RegisterRoutes(*http.ServeMux) }); ok { webServer.RegisterRoutes(mux) } - + // All other requests go to proxy handler mux.HandleFunc("/", p.handleRequest) - address := "0.0.0.0:8080" // This method shouldn't be used in multi-proxy mode - log.Printf("Starting proxy server in %s mode on %s", p.proxyConfig.Mode, address) + log.Printf("Starting HTTP proxy server in %s mode on %s", p.proxyConfig.Mode, address) log.Printf("Proxying to %s://%s:%d", p.proxyConfig.Protocol, p.proxyConfig.TargetHost, p.proxyConfig.TargetPort) return http.ListenAndServe(address, mux) } +func (p *ProxyEngine) startGRPCServer(address string) error { + if p.grpcServer == nil { + return fmt.Errorf("gRPC server not initialized") + } + + lis, err := net.Listen("tcp", address) + if err != nil { + return fmt.Errorf("failed to listen on %s: %w", address, err) + } + + log.Printf("Starting gRPC proxy server in %s mode on %s", p.proxyConfig.Mode, address) + log.Printf("Proxying to %s://%s:%d", p.proxyConfig.Protocol, p.proxyConfig.TargetHost, p.proxyConfig.TargetPort) + + return p.grpcServer.Serve(lis) +} + // HandleRequest implements the ProxyHandler interface func (p *ProxyEngine) HandleRequest(w http.ResponseWriter, r *http.Request) { p.handleRequest(w, r) @@ -154,5 +205,12 @@ func (p *ProxyEngine) handleRequest(w http.ResponseWriter, r *http.Request) { } func (p *ProxyEngine) Stop() error { + if p.grpcServer != nil { + p.grpcServer.GracefulStop() + } return nil } + +func (p *ProxyEngine) GetGRPCServer() *grpc.Server { + return p.grpcServer +} diff --git a/server/multi_proxy_server.go b/server/multi_proxy_server.go index 31f3856..f118012 100644 --- a/server/multi_proxy_server.go +++ b/server/multi_proxy_server.go @@ -3,9 +3,11 @@ package server import ( "fmt" "log" + "net" "net/http" "strings" + "google.golang.org/grpc" "mimic/config" "mimic/mock" "mimic/proxy" @@ -14,10 +16,11 @@ import ( ) type MultiProxyServer struct { - config *config.Config - database *storage.Database - webServer *web.Server - proxies map[string]ProxyHandler + config *config.Config + database *storage.Database + webServer *web.Server + proxies map[string]ProxyHandler + grpcServers map[string]*grpc.Server } type ProxyHandler interface { @@ -26,67 +29,142 @@ type ProxyHandler interface { func NewMultiProxyServer(cfg *config.Config, db *storage.Database) (*MultiProxyServer, error) { webServer := web.NewServer(cfg, db) - + server := &MultiProxyServer{ - config: cfg, - database: db, - webServer: webServer, - proxies: make(map[string]ProxyHandler), + config: cfg, + database: db, + webServer: webServer, + proxies: make(map[string]ProxyHandler), + grpcServers: make(map[string]*grpc.Server), } // Initialize all configured proxies for name, proxyConfig := range cfg.Proxies { var handler ProxyHandler - var err error switch proxyConfig.Mode { case "record": - handler, err = proxy.NewProxyEngineWithBroadcaster(proxyConfig, db, webServer) + if proxyEngine, err := proxy.NewProxyEngineWithBroadcaster(proxyConfig, db, webServer); err == nil { + handler = proxyEngine + // Extract gRPC server if this is a gRPC proxy + if proxyConfig.Protocol == "grpc" { + server.grpcServers[name] = proxyEngine.GetGRPCServer() + } + } else { + return nil, fmt.Errorf("failed to create proxy engine for '%s': %w", name, err) + } case "mock": - handler, err = mock.NewMockEngineWithBroadcaster(proxyConfig, db, webServer) + if mockEngine, err := mock.NewMockEngineWithBroadcaster(proxyConfig, db, webServer); err == nil { + handler = mockEngine + // Extract gRPC server if this is a gRPC mock + if proxyConfig.Protocol == "grpc" { + server.grpcServers[name] = mockEngine.GetGRPCServer() + } + } else { + return nil, fmt.Errorf("failed to create mock engine for '%s': %w", name, err) + } default: return nil, fmt.Errorf("invalid proxy mode for '%s': %s", name, proxyConfig.Mode) } - if err != nil { - return nil, fmt.Errorf("failed to create proxy handler for '%s': %w", name, err) - } - server.proxies[name] = handler - log.Printf("Initialized proxy '%s' in %s mode", name, proxyConfig.Mode) + log.Printf("Initialized proxy '%s' in %s mode (protocol: %s)", name, proxyConfig.Mode, proxyConfig.Protocol) } return server, nil } func (s *MultiProxyServer) Start() error { + // Start gRPC servers on separate ports if any exist + grpcPort := s.config.Server.ListenPort + 1000 // Use a different port range for gRPC + grpcPortMapping := make(map[string]int) // Track which proxy gets which port + + for name, grpcServer := range s.grpcServers { + if grpcServer != nil { + currentPort := grpcPort + grpcPortMapping[name] = currentPort + + go func(serverName string, server *grpc.Server, port int) { + address := fmt.Sprintf("%s:%d", s.config.Server.ListenHost, port) + lis, err := net.Listen("tcp", address) + if err != nil { + log.Printf("Failed to start gRPC server for '%s' on %s: %v", serverName, address, err) + return + } + log.Printf("gRPC proxy '%s' listening on %s", serverName, address) + if err := server.Serve(lis); err != nil { + log.Printf("gRPC server for '%s' failed: %v", serverName, err) + } + }(name, grpcServer, currentPort) + grpcPort++ + } + } + + // Set up HTTP server for web UI and HTTP proxies mux := http.NewServeMux() // Register web UI routes at top level s.webServer.RegisterRoutes(mux) // Register proxy routes at /proxy// + httpProxyCount := 0 + grpcProxyCount := 0 + baseGRPCPort := s.config.Server.ListenPort + 1000 + for name, proxyHandler := range s.proxies { proxyPath := fmt.Sprintf("/proxy/%s/", name) - mux.HandleFunc(proxyPath, func(w http.ResponseWriter, r *http.Request) { - // Strip the proxy path prefix and forward to the proxy handler - originalPath := r.URL.Path - r.URL.Path = strings.TrimPrefix(originalPath, fmt.Sprintf("/proxy/%s", name)) - if r.URL.Path == "" { - r.URL.Path = "/" - } - proxyHandler.HandleRequest(w, r) - }) - log.Printf("Registered proxy '%s' at path %s", name, proxyPath) + + // Check if this is a gRPC proxy + if _, isGRPC := s.grpcServers[name]; isGRPC { + // For gRPC proxies, provide an informational HTTP endpoint + currentGRPCPort := grpcPortMapping[name] // Use the correct mapped port + mux.HandleFunc(proxyPath, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, `{ + "message": "This is a gRPC proxy endpoint", + "protocol": "grpc", + "grpc_address": "%s:%d", + "usage": "Connect your gRPC client to %s:%d", + "example": "buf curl --schema --protocol grpc %s:%d/your.service/Method" +}`, s.config.Server.ListenHost, currentGRPCPort, s.config.Server.ListenHost, currentGRPCPort, s.config.Server.ListenHost, currentGRPCPort) + }) + log.Printf("Registered gRPC proxy info '%s' at HTTP path %s", name, proxyPath) + grpcProxyCount++ + } else { + // Regular HTTP proxy + mux.HandleFunc(proxyPath, func(w http.ResponseWriter, r *http.Request) { + // Strip the proxy path prefix and forward to the proxy handler + originalPath := r.URL.Path + r.URL.Path = strings.TrimPrefix(originalPath, fmt.Sprintf("/proxy/%s", name)) + if r.URL.Path == "" { + r.URL.Path = "/" + } + proxyHandler.HandleRequest(w, r) + }) + log.Printf("Registered HTTP proxy '%s' at path %s", name, proxyPath) + httpProxyCount++ + } } - address := fmt.Sprintf("%s:%d", s.config.Server.ListenHost, s.config.Server.ListenPort) - log.Printf("Starting multi-proxy server on %s", address) - log.Printf("Web UI available at http://%s/", address) + httpAddress := fmt.Sprintf("%s:%d", s.config.Server.ListenHost, s.config.Server.ListenPort) - for name := range s.proxies { - log.Printf("Proxy '%s' available at http://%s/proxy/%s/", name, address, name) + log.Printf("Starting multi-proxy server") + log.Printf("Web UI available at http://%s/", httpAddress) + + if httpProxyCount > 0 { + log.Printf("HTTP proxies (%d) available at http://%s/proxy//", httpProxyCount, httpAddress) + } + + if grpcProxyCount > 0 { + log.Printf("gRPC proxies (%d) available on ports %d-%d", grpcProxyCount, baseGRPCPort, baseGRPCPort+grpcProxyCount-1) + for name := range s.grpcServers { + if s.grpcServers[name] != nil { + currentPort := grpcPortMapping[name] + log.Printf(" → gRPC proxy '%s' at %s:%d", name, s.config.Server.ListenHost, currentPort) + } + } } - return http.ListenAndServe(address, mux) -} \ No newline at end of file + return http.ListenAndServe(httpAddress, mux) +} diff --git a/web/server.go b/web/server.go index 8101588..bf8178d 100644 --- a/web/server.go +++ b/web/server.go @@ -15,12 +15,12 @@ import ( ) type Server struct { - config *config.Config - database *storage.Database - upgrader websocket.Upgrader - clients map[*websocket.Conn]bool - clientsMux sync.RWMutex - broadcast chan []byte + config *config.Config + database *storage.Database + upgrader websocket.Upgrader + clients map[*websocket.Conn]bool + clientsMux sync.RWMutex + broadcast chan []byte } type Message struct { @@ -30,15 +30,15 @@ type Message struct { } type RequestResponseEvent struct { - Type string `json:"type"` // "request" or "response" - Method string `json:"method"` - Endpoint string `json:"endpoint"` - Headers map[string]interface{} `json:"headers"` - Body string `json:"body"` - Status int `json:"status,omitempty"` - SessionName string `json:"session_name"` - RemoteAddr string `json:"remote_addr"` - RequestID string `json:"request_id"` + Type string `json:"type"` // "request" or "response" + Method string `json:"method"` + Endpoint string `json:"endpoint"` + Headers map[string]interface{} `json:"headers"` + Body string `json:"body"` + Status int `json:"status,omitempty"` + SessionName string `json:"session_name"` + RemoteAddr string `json:"remote_addr"` + RequestID string `json:"request_id"` } func NewServer(cfg *config.Config, db *storage.Database) *Server { @@ -61,22 +61,22 @@ func (s *Server) Start() error { // Create a new HTTP multiplexer for the web server mux := http.NewServeMux() - + // Static files mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("web/static/")))) - + // Main page mux.HandleFunc("/", s.handleHome) - + // WebSocket endpoint mux.HandleFunc("/ws", s.handleWebSocket) - + // API endpoints mux.HandleFunc("/api/sessions", s.handleSessions) mux.HandleFunc("/api/sessions/", s.handleSessionDetail) mux.HandleFunc("/api/interactions/", s.handleInteractions) mux.HandleFunc("/api/clear", s.handleClear) - + address := fmt.Sprintf("%s:%d", s.config.Server.ListenHost, s.config.Server.ListenPort) // Use same port as server log.Printf("Starting web UI on http://%s", address) return http.ListenAndServe(address, mux) @@ -86,22 +86,22 @@ func (s *Server) Start() error { func (s *Server) RegisterRoutes(mux *http.ServeMux) { // Start the broadcast handler go s.handleBroadcast() - + // Static files at /static/ mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("web/static/")))) - + // Main page at / mux.HandleFunc("/", s.handleHome) - + // WebSocket endpoint at /ws mux.HandleFunc("/ws", s.handleWebSocket) - + // API endpoints at /api/ mux.HandleFunc("/api/sessions", s.handleSessions) mux.HandleFunc("/api/sessions/", s.handleSessionDetail) mux.HandleFunc("/api/interactions/", s.handleInteractions) mux.HandleFunc("/api/clear", s.handleClear) - + log.Printf("Web UI registered at top level") } @@ -185,7 +185,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) { ` - + w.Header().Set("Content-Type", "text/html") w.Write([]byte(html)) } @@ -351,4 +351,4 @@ func (s *Server) BroadcastResponse(method, endpoint, sessionName, remoteAddr, re RequestID: requestID, } s.BroadcastEvent("response", event) -} \ No newline at end of file +} diff --git a/web/static/app.js b/web/static/app.js index 0a2836d..39078a0 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -2,6 +2,7 @@ class MimicUI { constructor() { this.ws = null; this.events = []; + this.requestPairs = new Map(); // Track request/response pairs this.maxEvents = 1000; this.autoScroll = true; this.currentSession = null; @@ -72,18 +73,40 @@ class MimicUI { } addEvent(message) { - this.events.unshift({ + const eventData = { id: Date.now() + Math.random(), timestamp: new Date(message.timestamp), + type: message.type, ...message.data - }); + }; - // Limit the number of events - if (this.events.length > this.maxEvents) { - this.events = this.events.slice(0, this.maxEvents); + if (message.type === 'request') { + // Store the request and create initial event + this.requestPairs.set(eventData.request_id, { + request: eventData, + response: null, + timestamp: eventData.timestamp, + isComplete: false + }); + } else if (message.type === 'response') { + // Update existing request with response + const pair = this.requestPairs.get(eventData.request_id); + if (pair) { + pair.response = eventData; + pair.isComplete = true; + pair.duration = eventData.timestamp - pair.request.timestamp; + } else { + // Response without matching request (shouldn't happen but handle gracefully) + this.requestPairs.set(eventData.request_id, { + request: null, + response: eventData, + timestamp: eventData.timestamp, + isComplete: true + }); + } } - this.updateEventsList(); + this.updateEventsFromPairs(); this.updateEventCount(); // Auto-scroll if enabled @@ -93,6 +116,15 @@ class MimicUI { } } + updateEventsFromPairs() { + // Convert request/response pairs to display events + this.events = Array.from(this.requestPairs.values()) + .sort((a, b) => b.timestamp - a.timestamp) + .slice(0, this.maxEvents); + + this.updateEventsList(); + } + updateEventsList() { const eventsList = document.getElementById('events-list'); @@ -105,32 +137,51 @@ class MimicUI { eventsList.innerHTML = eventsHtml; } - renderEvent(event) { - const timestamp = event.timestamp.toLocaleTimeString(); - const statusClass = event.status ? `status-${Math.floor(event.status / 100)}xx` : ''; + renderEvent(pair) { + const timestamp = pair.timestamp.toLocaleTimeString(); + const request = pair.request; + const response = pair.response; + + if (!request && !response) return ''; + + const event = request || response; + const statusClass = response?.status ? `status-${Math.floor(response.status / 100)}xx` : ''; const methodClass = `method-${event.method}`; + // Determine if this is a mock response + const isMock = this.isMockResponse(pair); + const mockBadge = isMock ? 'MOCK' : ''; + + // Create duration display + const durationText = pair.duration ? `${pair.duration}ms` : pair.isComplete ? '0ms' : 'pending...'; + let bodyHtml = ''; - if (event.body && event.body.trim()) { - const displayBody = event.body.length > 200 ? - event.body.substring(0, 200) + '...' : event.body; - bodyHtml = `
${this.escapeHtml(displayBody)}
`; + if (request?.body && request.body.trim()) { + const displayBody = request.body.length > 100 ? + request.body.substring(0, 100) + '...' : request.body; + bodyHtml = `
Request: ${this.escapeHtml(displayBody)}
`; + } + if (response?.body && response.body.trim()) { + const displayBody = response.body.length > 100 ? + response.body.substring(0, 100) + '...' : response.body; + bodyHtml += `
Response: ${this.escapeHtml(displayBody)}
`; } return ` -
+
${event.method} ${event.endpoint} - ${event.status ? `${event.status}` : ''} + ${response?.status ? `${response.status}` : 'PENDING'} + ${mockBadge}
${timestamp}
Session: ${event.session_name} - From: ${event.remote_addr} - Type: ${event.type} + From: ${event.remote_addr || 'unknown'} + Duration: ${durationText} ${event.request_id ? `ID: ${event.request_id.substring(0, 8)}...` : ''}
${bodyHtml} @@ -138,6 +189,33 @@ class MimicUI { `; } + isMockResponse(pair) { + // Mock detection logic: + // 1. Check if response came very quickly (< 50ms) indicating local mock + // 2. Check for mock-specific session names + // 3. Check for mock indicators in headers/metadata + + if (!pair.response) return false; + + // Fast response time suggests mock + if (pair.duration !== undefined && pair.duration < 50) { + return true; + } + + // Check session name for mock indicators + const sessionName = pair.request?.session_name || pair.response?.session_name || ''; + if (sessionName.includes('mock') || sessionName.includes('test')) { + return true; + } + + // Check for gRPC mock responses (they come from local mock engine) + if (pair.request?.remote_addr === 'grpc-client') { + return true; + } + + return false; + } + updateEventCount() { document.getElementById('event-count').textContent = this.events.length; } @@ -158,6 +236,7 @@ class MimicUI { // Clear events button document.getElementById('clear-events').addEventListener('click', () => { this.events = []; + this.requestPairs.clear(); this.updateEventsList(); this.updateEventCount(); }); @@ -410,6 +489,7 @@ class MimicUI { this.loadSessions(); this.loadInteractions(); this.events = []; + this.requestPairs.clear(); this.updateEventsList(); this.updateEventCount(); } diff --git a/web/static/style.css b/web/static/style.css index f896779..2bedd4c 100644 --- a/web/static/style.css +++ b/web/static/style.css @@ -252,6 +252,8 @@ header p { .method-PUT { background: #e74c3c; } .method-DELETE { background: #8e44ad; } .method-PATCH { background: #16a085; } +.method-/astria.sequencerblock.v1.SequencerService/GetUpgradesInfo, +.method-grpc { background: #9b59b6; } .event-endpoint { font-family: 'Courier New', monospace; @@ -276,6 +278,30 @@ header p { .status-3xx { background: #f39c12; } .status-4xx { background: #e74c3c; } .status-5xx { background: #8e44ad; } +.status-pending { background: #95a5a6; } + +/* Mock badge styling */ +.mock-badge { + background: #e67e22; + color: white; + font-size: 10px; + padding: 2px 6px; + border-radius: 3px; + margin-left: 8px; + font-weight: bold; + animation: pulse 2s infinite; +} + +.mock-event { + border-left: 3px solid #e67e22; + background: rgba(230, 126, 34, 0.05); +} + +@keyframes pulse { + 0% { opacity: 1; } + 50% { opacity: 0.7; } + 100% { opacity: 1; } +} .event-meta { display: flex; From 3441a4ff01ab49e936bf18f093dfce603656f7c8 Mon Sep 17 00:00:00 2001 From: Jordan Oroshiba Date: Fri, 18 Jul 2025 20:13:27 -0700 Subject: [PATCH 2/4] update --- README.md | 9 +- cmd/root.go | 7 + config.yaml | 8 +- config/config.go | 44 +++++-- mock/grpc_test.go | 2 - mock/mock_engine.go | 4 +- proxy/grpc_raw_proxy.go | 18 ++- proxy/grpc_test.go | 5 +- proxy/proxy_engine.go | 6 +- server/multi_proxy_server.go | 242 +++++++++++++++++++++++------------ 10 files changed, 222 insertions(+), 123 deletions(-) diff --git a/README.md b/README.md index f9bb303..402ccc1 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ Mimic uses a configuration file located at `~/.mimic/config.yaml` by default. Th server: listen_host: "0.0.0.0" listen_port: 8080 + grpc_port: 9080 # Optional: defaults to listen_port + 1000 proxies: api1: @@ -155,9 +156,9 @@ Record gRPC interactions by running mimic in record mode with a gRPC-configured # Start gRPC recording mimic --config config-grpc.yaml -# Your gRPC client should connect to localhost:9080 (HTTP port + 1000) instead of the original server +# Your gRPC client should connect to localhost:9080 (configurable grpc_port) instead of the original server # Mimic will forward the calls and record all interactions -# Example: if HTTP server runs on :8080, gRPC proxy will be on :9080 +# Example: if HTTP server runs on :8080, gRPC proxy will be on :9080 by default ``` ### gRPC Mocking @@ -185,8 +186,8 @@ mimic --mode mock --config config-grpc.yaml # Start mimic with gRPC proxy configuration mimic --config config-grpc-example.yaml - # Your gRPC client should connect to the gRPC proxy port (HTTP port + 1000) - # If HTTP server is on :8080, gRPC proxy will be on :9080 + # Your gRPC client should connect to the gRPC proxy port (configurable grpc_port) + # If HTTP server is on :8080, gRPC proxy will be on :9080 by default buf curl --schema buf.build/your/api --protocol grpc localhost:9080/your.service/Method ``` diff --git a/cmd/root.go b/cmd/root.go index 76db80e..1e04311 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -22,6 +22,7 @@ var ( inputFile string mergeStrategy string debugMode bool + modeFlag string ) var rootCmd = &cobra.Command{ @@ -41,6 +42,7 @@ func Execute() error { func init() { rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is config.yaml)") rootCmd.PersistentFlags().BoolVar(&debugMode, "debug", false, "enable debug logging") + rootCmd.PersistentFlags().StringVar(&modeFlag, "mode", "", "operation mode (record or mock) - overrides config file setting") rootCmd.AddCommand(exportCmd) rootCmd.AddCommand(importCmd) @@ -61,6 +63,11 @@ func runProxy() { log.Fatal("Failed to load config:", err) } + // Override mode from CLI flag if provided + if modeFlag != "" { + cfg.Mode = modeFlag + } + if err := cfg.Validate(); err != nil { log.Fatal("Invalid configuration:", err) } diff --git a/config.yaml b/config.yaml index c76b183..ecc5e29 100644 --- a/config.yaml +++ b/config.yaml @@ -1,21 +1,23 @@ +mode: mock + server: listen_host: "0.0.0.0" listen_port: 8080 + grpc_port: 9090 proxies: anthropic: - mode: "mock" + protocol: "https" target_host: "api.anthropic.com" target_port: 443 - protocol: "https" session_name: "anthropic-test" astria-grpc: - mode: "mock" protocol: "grpc" target_host: "grpc.astria.org" target_port: 443 session_name: "grpc-mock-session" + service_pattern: "astria\\..*" # test-grpc: # mode: "record" diff --git a/config/config.go b/config/config.go index bb95c9f..d2cbc2e 100644 --- a/config/config.go +++ b/config/config.go @@ -9,6 +9,7 @@ import ( ) type Config struct { + Mode string `mapstructure:"mode"` // Global mode: "record" or "mock" Server ServerConfig `mapstructure:"server"` Proxies map[string]ProxyConfig `mapstructure:"proxies"` Database DatabaseConfig `mapstructure:"database"` @@ -21,14 +22,18 @@ type Config struct { type ServerConfig struct { ListenHost string `mapstructure:"listen_host"` ListenPort int `mapstructure:"listen_port"` + GRPCPort int `mapstructure:"grpc_port"` // Port for gRPC server (defaults to listen_port + 1000) } type ProxyConfig struct { - Mode string `mapstructure:"mode"` - TargetHost string `mapstructure:"target_host"` - TargetPort int `mapstructure:"target_port"` - Protocol string `mapstructure:"protocol"` - SessionName string `mapstructure:"session_name"` + TargetHost string `mapstructure:"target_host"` + TargetPort int `mapstructure:"target_port"` + Protocol string `mapstructure:"protocol"` + SessionName string `mapstructure:"session_name"` + // gRPC routing patterns (optional) + ServicePattern string `mapstructure:"service_pattern"` // Regex pattern for service names + MethodPattern string `mapstructure:"method_pattern"` // Regex pattern for method names + IsDefault bool `mapstructure:"is_default"` // Whether this is the default/fallback route } type DatabaseConfig struct { @@ -117,8 +122,11 @@ func setDefaults() { homeDir, _ := os.UserHomeDir() defaultDBPath := filepath.Join(homeDir, ".mimic", "recordings.db") + viper.SetDefault("mode", "record") + viper.SetDefault("server.listen_host", "0.0.0.0") viper.SetDefault("server.listen_port", 8080) + viper.SetDefault("server.grpc_port", 9080) // Default to 9080 viper.SetDefault("database.path", defaultDBPath) viper.SetDefault("database.connection_pool_size", 10) @@ -148,13 +156,14 @@ func getDefaultConfig() *Config { defaultDBPath := filepath.Join(homeDir, ".mimic", "recordings.db") return &Config{ + Mode: "record", Server: ServerConfig{ ListenHost: "0.0.0.0", ListenPort: 8080, + GRPCPort: 9080, }, Proxies: map[string]ProxyConfig{ "default": { - Mode: "record", Protocol: "http", SessionName: "default", }, @@ -192,20 +201,32 @@ func getDefaultConfig() *Config { } func (c *Config) Validate() error { + // Validate global mode + if c.Mode != "record" && c.Mode != "mock" { + return fmt.Errorf("invalid mode: %s (must be 'record' or 'mock')", c.Mode) + } + + // Validate server config if c.Server.ListenPort <= 0 || c.Server.ListenPort > 65535 { return fmt.Errorf("invalid server listen_port: %d", c.Server.ListenPort) } + // Set default gRPC port if not configured + if c.Server.GRPCPort == 0 { + c.Server.GRPCPort = c.Server.ListenPort + 1000 + } + + if c.Server.GRPCPort <= 0 || c.Server.GRPCPort > 65535 { + return fmt.Errorf("invalid server grpc_port: %d", c.Server.GRPCPort) + } + if len(c.Proxies) == 0 { return fmt.Errorf("at least one proxy must be configured") } + // Validate proxy configs for name, proxy := range c.Proxies { - if proxy.Mode != "record" && proxy.Mode != "mock" { - return fmt.Errorf("invalid proxy mode for '%s': %s (must be 'record' or 'mock')", name, proxy.Mode) - } - - if proxy.Mode == "record" && (proxy.TargetHost == "" || proxy.TargetPort == 0) { + if c.Mode == "record" && (proxy.TargetHost == "" || proxy.TargetPort == 0) { return fmt.Errorf("target_host and target_port are required in record mode for proxy '%s'", name) } @@ -222,6 +243,7 @@ func (c *Config) Validate() error { } func SaveConfig(config *Config, path string) error { + viper.Set("mode", config.Mode) viper.Set("server", config.Server) viper.Set("proxies", config.Proxies) viper.Set("database", config.Database) diff --git a/mock/grpc_test.go b/mock/grpc_test.go index 6f27a03..e65ab76 100644 --- a/mock/grpc_test.go +++ b/mock/grpc_test.go @@ -17,7 +17,6 @@ func TestMockEngineWithGRPC(t *testing.T) { // Create gRPC mock config proxyConfig := config.ProxyConfig{ - Mode: "mock", Protocol: "grpc", SessionName: "test-session", } @@ -49,7 +48,6 @@ func TestMockEngineWithHTTP(t *testing.T) { // Create HTTP mock config proxyConfig := config.ProxyConfig{ - Mode: "mock", Protocol: "http", SessionName: "test-session", } diff --git a/mock/mock_engine.go b/mock/mock_engine.go index 82ff314..2e79b4f 100644 --- a/mock/mock_engine.go +++ b/mock/mock_engine.go @@ -132,7 +132,7 @@ func (m *MockEngine) startHTTPMockServer(address string) error { // All other requests go to mock handler mux.HandleFunc("/", m.handleRequest) - log.Printf("Starting HTTP mock server in %s mode on %s", m.proxyConfig.Mode, address) + log.Printf("Starting HTTP mock server on %s", address) log.Printf("Serving mocked responses for session: %s", m.session.SessionName) return http.ListenAndServe(address, mux) @@ -148,7 +148,7 @@ func (m *MockEngine) startGRPCMockServer(address string) error { return fmt.Errorf("failed to listen on %s: %w", address, err) } - log.Printf("Starting gRPC mock server in %s mode on %s", m.proxyConfig.Mode, address) + log.Printf("Starting gRPC mock server on %s", address) log.Printf("Serving mocked responses for session: %s", m.session.SessionName) return m.grpcServer.Serve(lis) diff --git a/proxy/grpc_raw_proxy.go b/proxy/grpc_raw_proxy.go index 1332d0b..068bf4e 100644 --- a/proxy/grpc_raw_proxy.go +++ b/proxy/grpc_raw_proxy.go @@ -22,6 +22,7 @@ import ( // RawGRPCProxy implements raw byte-level gRPC proxying type RawGRPCProxy struct { config *config.ProxyConfig + mode string // Global mode: "record" or "mock" database *storage.Database session *storage.Session handler *GRPCHandler @@ -30,9 +31,10 @@ type RawGRPCProxy struct { -func NewRawGRPCProxy(proxyConfig *config.ProxyConfig, db *storage.Database, session *storage.Session, grpcHandler *GRPCHandler) *RawGRPCProxy { +func NewRawGRPCProxy(proxyConfig *config.ProxyConfig, mode string, db *storage.Database, session *storage.Session, grpcHandler *GRPCHandler) *RawGRPCProxy { return &RawGRPCProxy{ config: proxyConfig, + mode: mode, database: db, session: session, handler: grpcHandler, @@ -131,9 +133,7 @@ func (p *RawGRPCProxy) proxyRawStream(serverStream grpc.ServerStream, clientStre return } - if p.config.Mode == "record" { - log.Printf("→ %s: %d bytes", method, len(msg.Data)) - } + log.Printf("→ %s: %d bytes", method, len(msg.Data)) if err := clientStream.SendMsg(msg); err != nil { errCh <- fmt.Errorf("client send error: %w", err) @@ -155,9 +155,7 @@ func (p *RawGRPCProxy) proxyRawStream(serverStream grpc.ServerStream, clientStre return } - if p.config.Mode == "record" { - log.Printf("← %s: %d bytes", method, len(msg.Data)) - } + log.Printf("← %s: %d bytes", method, len(msg.Data)) if err := serverStream.SendMsg(msg); err != nil { errCh <- fmt.Errorf("server send error: %w", err) @@ -232,7 +230,7 @@ func (p *RawGRPCProxy) handleUnaryCall(ctx context.Context, conn *grpc.ClientCon return status.Errorf(codes.Internal, "failed to receive request: %v", err) } - if p.config.Mode == "record" { + if p.mode == "record" { log.Printf("→ %s: %d bytes (unary)", method, len(requestMsg.Data)) } @@ -242,7 +240,7 @@ func (p *RawGRPCProxy) handleUnaryCall(ctx context.Context, conn *grpc.ClientCon // Create interaction record for database storage var interaction *storage.Interaction - if p.config.Mode == "record" { + if p.mode == "record" { interaction = &storage.Interaction{ RequestID: GenerateRequestID(), SessionID: p.session.ID, @@ -270,7 +268,7 @@ func (p *RawGRPCProxy) handleUnaryCall(ctx context.Context, conn *grpc.ClientCon err := conn.Invoke(outCtx, method, &requestMsg, &responseMsg, grpc.ForceCodec(GetRawCodec())) // Handle recording and response - if p.config.Mode == "record" { + if p.mode == "record" { statusCode := 0 if err != nil { if st, ok := status.FromError(err); ok { diff --git a/proxy/grpc_test.go b/proxy/grpc_test.go index 72cde50..439cecb 100644 --- a/proxy/grpc_test.go +++ b/proxy/grpc_test.go @@ -44,7 +44,6 @@ func TestRawGRPCProxyUnaryCallDetection(t *testing.T) { defer db.Close() proxyConfig := config.ProxyConfig{ - Mode: "record", Protocol: "grpc", TargetHost: "localhost", TargetPort: 9090, @@ -54,7 +53,7 @@ func TestRawGRPCProxyUnaryCallDetection(t *testing.T) { grpcHandler := NewGRPCHandler([]string{}) session, _ := db.GetOrCreateSession("test", "test") - rawProxy := NewRawGRPCProxy(&proxyConfig, db, session, grpcHandler) + rawProxy := NewRawGRPCProxy(&proxyConfig, "record", db, session, grpcHandler) // Test unary call detection testCases := []struct { @@ -88,7 +87,6 @@ func TestProxyEngineWithGRPC(t *testing.T) { // Create gRPC proxy config proxyConfig := config.ProxyConfig{ - Mode: "record", Protocol: "grpc", TargetHost: "localhost", TargetPort: 9090, @@ -122,7 +120,6 @@ func TestProxyEngineWithHTTP(t *testing.T) { // Create HTTP proxy config proxyConfig := config.ProxyConfig{ - Mode: "record", Protocol: "http", TargetHost: "localhost", TargetPort: 8080, diff --git a/proxy/proxy_engine.go b/proxy/proxy_engine.go index a8d6c1e..2c01e91 100644 --- a/proxy/proxy_engine.go +++ b/proxy/proxy_engine.go @@ -56,7 +56,7 @@ func NewProxyEngineWithBroadcaster(proxyConfig config.ProxyConfig, db *storage.D if proxyConfig.Protocol == "grpc" { // Use raw proxy for better compatibility - rawProxy := NewRawGRPCProxy(&proxyConfig, db, session, grpcHandler) + rawProxy := NewRawGRPCProxy(&proxyConfig, "record", db, session, grpcHandler) // Set web broadcaster if available if webServer != nil { @@ -106,7 +106,7 @@ func (p *ProxyEngine) startHTTPServer(address string) error { // All other requests go to proxy handler mux.HandleFunc("/", p.handleRequest) - log.Printf("Starting HTTP proxy server in %s mode on %s", p.proxyConfig.Mode, address) + log.Printf("Starting HTTP proxy server on %s", address) log.Printf("Proxying to %s://%s:%d", p.proxyConfig.Protocol, p.proxyConfig.TargetHost, p.proxyConfig.TargetPort) return http.ListenAndServe(address, mux) @@ -122,7 +122,7 @@ func (p *ProxyEngine) startGRPCServer(address string) error { return fmt.Errorf("failed to listen on %s: %w", address, err) } - log.Printf("Starting gRPC proxy server in %s mode on %s", p.proxyConfig.Mode, address) + log.Printf("Starting gRPC proxy server on %s", address) log.Printf("Proxying to %s://%s:%d", p.proxyConfig.Protocol, p.proxyConfig.TargetHost, p.proxyConfig.TargetPort) return p.grpcServer.Serve(lis) diff --git a/server/multi_proxy_server.go b/server/multi_proxy_server.go index f118012..9a2052c 100644 --- a/server/multi_proxy_server.go +++ b/server/multi_proxy_server.go @@ -16,11 +16,13 @@ import ( ) type MultiProxyServer struct { - config *config.Config - database *storage.Database - webServer *web.Server - proxies map[string]ProxyHandler - grpcServers map[string]*grpc.Server + config *config.Config + database *storage.Database + webServer *web.Server + proxies map[string]ProxyHandler + grpcServer *grpc.Server // Single gRPC server with routing + grpcRouter *proxy.GRPCRouter // For gRPC record proxies + grpcMockRouter *mock.GRPCMockRouter // For gRPC mock proxies } type ProxyHandler interface { @@ -31,73 +33,149 @@ func NewMultiProxyServer(cfg *config.Config, db *storage.Database) (*MultiProxyS webServer := web.NewServer(cfg, db) server := &MultiProxyServer{ - config: cfg, - database: db, - webServer: webServer, - proxies: make(map[string]ProxyHandler), - grpcServers: make(map[string]*grpc.Server), + config: cfg, + database: db, + webServer: webServer, + proxies: make(map[string]ProxyHandler), } - // Initialize all configured proxies + // Separate HTTP and gRPC proxies + httpProxies := make(map[string]config.ProxyConfig) + grpcProxies := make(map[string]config.ProxyConfig) + for name, proxyConfig := range cfg.Proxies { + if proxyConfig.Protocol == "grpc" { + grpcProxies[name] = proxyConfig + } else { + // HTTP/HTTPS proxies - handle individually + httpProxies[name] = proxyConfig + } + } + + // Initialize HTTP proxies (existing logic) + for name, proxyConfig := range httpProxies { var handler ProxyHandler - switch proxyConfig.Mode { + switch cfg.Mode { case "record": if proxyEngine, err := proxy.NewProxyEngineWithBroadcaster(proxyConfig, db, webServer); err == nil { handler = proxyEngine - // Extract gRPC server if this is a gRPC proxy - if proxyConfig.Protocol == "grpc" { - server.grpcServers[name] = proxyEngine.GetGRPCServer() - } } else { return nil, fmt.Errorf("failed to create proxy engine for '%s': %w", name, err) } case "mock": if mockEngine, err := mock.NewMockEngineWithBroadcaster(proxyConfig, db, webServer); err == nil { handler = mockEngine - // Extract gRPC server if this is a gRPC mock - if proxyConfig.Protocol == "grpc" { - server.grpcServers[name] = mockEngine.GetGRPCServer() - } } else { return nil, fmt.Errorf("failed to create mock engine for '%s': %w", name, err) } default: - return nil, fmt.Errorf("invalid proxy mode for '%s': %s", name, proxyConfig.Mode) + return nil, fmt.Errorf("invalid global mode: %s", cfg.Mode) } server.proxies[name] = handler - log.Printf("Initialized proxy '%s' in %s mode (protocol: %s)", name, proxyConfig.Mode, proxyConfig.Protocol) + log.Printf("Initialized HTTP proxy '%s' in %s mode", name, cfg.Mode) + } + + // Initialize single gRPC server with routing (if any gRPC proxies exist) + if len(grpcProxies) > 0 { + var unknownServiceHandler grpc.StreamHandler + + // Create gRPC router for record mode + if cfg.Mode == "record" { + router, err := proxy.NewGRPCRouter(grpcProxies, cfg.Mode, db, webServer) + if err != nil { + return nil, fmt.Errorf("failed to create gRPC router: %w", err) + } + server.grpcRouter = router + unknownServiceHandler = router.GetUnknownServiceHandler() + + } + + // Create gRPC mock router for mock mode + if cfg.Mode == "mock" { + mockRouter, err := mock.NewGRPCMockRouter(grpcProxies, db, webServer) + if err != nil { + return nil, fmt.Errorf("failed to create gRPC mock router: %w", err) + } + server.grpcMockRouter = mockRouter + // If we don't have a record router, use the mock router as the handler + if unknownServiceHandler == nil { + unknownServiceHandler = mockRouter.GetUnknownServiceHandler() + } + } + log.Printf("Initialized gRPC router with %d %s routes", len(grpcProxies), cfg.Mode) + + // Create single gRPC server with routing + server.grpcServer = grpc.NewServer( + grpc.MaxRecvMsgSize(64*1024*1024), // 64MB max receive message size + grpc.MaxSendMsgSize(64*1024*1024), // 64MB max send message size + grpc.MaxHeaderListSize(64*1024*1024), // 64MB max header list size + grpc.InitialWindowSize(64*1024*1024), // 64MB initial window + grpc.InitialConnWindowSize(64*1024*1024), // 64MB connection window + grpc.UnknownServiceHandler(unknownServiceHandler), + ) + + log.Printf("Created single gRPC server with routing") } return server, nil } func (s *MultiProxyServer) Start() error { - // Start gRPC servers on separate ports if any exist - grpcPort := s.config.Server.ListenPort + 1000 // Use a different port range for gRPC - grpcPortMapping := make(map[string]int) // Track which proxy gets which port - - for name, grpcServer := range s.grpcServers { - if grpcServer != nil { - currentPort := grpcPort - grpcPortMapping[name] = currentPort + // Start single gRPC server with routing if any gRPC proxies exist + var grpcAddress string + if s.grpcServer != nil { + grpcAddress = fmt.Sprintf("%s:%d", s.config.Server.ListenHost, s.config.Server.GRPCPort) + + go func() { + lis, err := net.Listen("tcp", grpcAddress) + if err != nil { + log.Printf("Failed to start gRPC router server on %s: %v", grpcAddress, err) + return + } + + log.Printf("gRPC router server listening on %s", grpcAddress) - go func(serverName string, server *grpc.Server, port int) { - address := fmt.Sprintf("%s:%d", s.config.Server.ListenHost, port) - lis, err := net.Listen("tcp", address) - if err != nil { - log.Printf("Failed to start gRPC server for '%s' on %s: %v", serverName, address, err) - return + // Log route information + if s.grpcRouter != nil { + routes := s.grpcRouter.GetRoutes() + for _, route := range routes { + if route.Config.ServicePattern != "" { + log.Printf(" → Route '%s': %s -> %s:%d", + route.Name, + route.Config.ServicePattern, + route.Config.TargetHost, + route.Config.TargetPort) + } else { + log.Printf(" → Route '%s': (default) -> %s:%d", + route.Name, + route.Config.TargetHost, + route.Config.TargetPort) + } } - log.Printf("gRPC proxy '%s' listening on %s", serverName, address) - if err := server.Serve(lis); err != nil { - log.Printf("gRPC server for '%s' failed: %v", serverName, err) + } + + if s.grpcMockRouter != nil { + routes := s.grpcMockRouter.GetRoutes() + for _, route := range routes { + if route.Config.ServicePattern != "" { + log.Printf(" → Mock Route '%s': %s -> session '%s'", + route.Name, + route.Config.ServicePattern, + route.Session.SessionName) + } else { + log.Printf(" → Mock Route '%s': (default) -> session '%s'", + route.Name, + route.Session.SessionName) + } } - }(name, grpcServer, currentPort) - grpcPort++ - } + } + + if err := s.grpcServer.Serve(lis); err != nil { + log.Printf("gRPC router server failed: %v", err) + } + }() } // Set up HTTP server for web UI and HTTP proxies @@ -106,45 +184,38 @@ func (s *MultiProxyServer) Start() error { // Register web UI routes at top level s.webServer.RegisterRoutes(mux) - // Register proxy routes at /proxy// + // Register HTTP proxy routes at /proxy// httpProxyCount := 0 - grpcProxyCount := 0 - baseGRPCPort := s.config.Server.ListenPort + 1000 for name, proxyHandler := range s.proxies { proxyPath := fmt.Sprintf("/proxy/%s/", name) - // Check if this is a gRPC proxy - if _, isGRPC := s.grpcServers[name]; isGRPC { - // For gRPC proxies, provide an informational HTTP endpoint - currentGRPCPort := grpcPortMapping[name] // Use the correct mapped port - mux.HandleFunc(proxyPath, func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - fmt.Fprintf(w, `{ - "message": "This is a gRPC proxy endpoint", - "protocol": "grpc", - "grpc_address": "%s:%d", - "usage": "Connect your gRPC client to %s:%d", - "example": "buf curl --schema --protocol grpc %s:%d/your.service/Method" -}`, s.config.Server.ListenHost, currentGRPCPort, s.config.Server.ListenHost, currentGRPCPort, s.config.Server.ListenHost, currentGRPCPort) - }) - log.Printf("Registered gRPC proxy info '%s' at HTTP path %s", name, proxyPath) - grpcProxyCount++ - } else { - // Regular HTTP proxy - mux.HandleFunc(proxyPath, func(w http.ResponseWriter, r *http.Request) { - // Strip the proxy path prefix and forward to the proxy handler - originalPath := r.URL.Path - r.URL.Path = strings.TrimPrefix(originalPath, fmt.Sprintf("/proxy/%s", name)) - if r.URL.Path == "" { - r.URL.Path = "/" - } - proxyHandler.HandleRequest(w, r) - }) - log.Printf("Registered HTTP proxy '%s' at path %s", name, proxyPath) - httpProxyCount++ - } + // Regular HTTP proxy + mux.HandleFunc(proxyPath, func(w http.ResponseWriter, r *http.Request) { + // Strip the proxy path prefix and forward to the proxy handler + originalPath := r.URL.Path + r.URL.Path = strings.TrimPrefix(originalPath, fmt.Sprintf("/proxy/%s", name)) + if r.URL.Path == "" { + r.URL.Path = "/" + } + proxyHandler.HandleRequest(w, r) + }) + log.Printf("Registered HTTP proxy '%s' at path %s", name, proxyPath) + httpProxyCount++ + } + + // Add gRPC info endpoint if gRPC server exists + if s.grpcServer != nil { + mux.HandleFunc("/grpc/info", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, `{ + "message": "gRPC router server with path-based routing", + "grpc_address": "%s", + "usage": "Connect your gRPC client to %s", + "example": "grpcurl -plaintext %s your.service/Method" +}`, grpcAddress, grpcAddress, grpcAddress) + }) } httpAddress := fmt.Sprintf("%s:%d", s.config.Server.ListenHost, s.config.Server.ListenPort) @@ -156,15 +227,18 @@ func (s *MultiProxyServer) Start() error { log.Printf("HTTP proxies (%d) available at http://%s/proxy//", httpProxyCount, httpAddress) } - if grpcProxyCount > 0 { - log.Printf("gRPC proxies (%d) available on ports %d-%d", grpcProxyCount, baseGRPCPort, baseGRPCPort+grpcProxyCount-1) - for name := range s.grpcServers { - if s.grpcServers[name] != nil { - currentPort := grpcPortMapping[name] - log.Printf(" → gRPC proxy '%s' at %s:%d", name, s.config.Server.ListenHost, currentPort) - } - } + if s.grpcServer != nil { + log.Printf("gRPC router server available at %s", grpcAddress) + log.Printf("gRPC info available at http://%s/grpc/info", httpAddress) } return http.ListenAndServe(httpAddress, mux) } + +// Stop gracefully stops the server +func (s *MultiProxyServer) Stop() error { + if s.grpcServer != nil { + s.grpcServer.GracefulStop() + } + return nil +} From 3f71657b476770611bdf066abf15ebcad789a284 Mon Sep 17 00:00:00 2001 From: Jordan Oroshiba Date: Fri, 18 Jul 2025 21:00:10 -0700 Subject: [PATCH 3/4] feat: replay --- GRPC_ROUTING.md | 172 ++++++++++ README.md | 98 +++++- cmd/replay.go | 148 +++++++++ cmd/root.go | 2 +- config-grpc-replay-example.yaml | 55 ++++ config-grpc-replay-production.yaml | 55 ++++ config-grpc-routing-example.yaml | 88 ++++++ config-grpc-simple.yaml | 42 +++ config-replay-example.yaml | 50 +++ config.yaml | 2 +- config/config.go | 91 +++++- mock/grpc_mock_router.go | 175 +++++++++++ mock/mock_engine.go | 39 ++- proxy/grpc_raw_proxy.go | 24 +- proxy/grpc_router.go | 180 +++++++++++ proxy/grpc_test.go | 2 +- proxy/grpc_utils.go | 2 +- proxy/proxy_engine.go | 10 +- replay/replay_engine.go | 485 +++++++++++++++++++++++++++++ server/multi_proxy_server.go | 59 ++-- server/replay_handler.go | 140 +++++++++ 21 files changed, 1848 insertions(+), 71 deletions(-) create mode 100644 GRPC_ROUTING.md create mode 100644 cmd/replay.go create mode 100644 config-grpc-replay-example.yaml create mode 100644 config-grpc-replay-production.yaml create mode 100644 config-grpc-routing-example.yaml create mode 100644 config-grpc-simple.yaml create mode 100644 config-replay-example.yaml create mode 100644 mock/grpc_mock_router.go create mode 100644 proxy/grpc_router.go create mode 100644 replay/replay_engine.go create mode 100644 server/replay_handler.go diff --git a/GRPC_ROUTING.md b/GRPC_ROUTING.md new file mode 100644 index 0000000..ceec57d --- /dev/null +++ b/GRPC_ROUTING.md @@ -0,0 +1,172 @@ +# gRPC Path-Based Routing + +Mimic uses a single gRPC server with intelligent path-based routing to handle multiple backend services. This approach is more efficient and easier to manage than running multiple gRPC server instances. + +## How gRPC Routing Works +```yaml +# config-grpc-routing.yaml +proxies: + user-service: + mode: "record" + protocol: "grpc" + target_host: "user-api.internal.com" + target_port: 9090 + session_name: "user-session" + service_pattern: "com\\.example\\.userservice\\..*" # Routes based on service name + + order-service: + mode: "record" + protocol: "grpc" + target_host: "order-api.internal.com" + target_port: 9091 + session_name: "order-session" + service_pattern: "com\\.example\\.orderservice\\..*" + + default-backend: + mode: "record" + protocol: "grpc" + target_host: "default-api.internal.com" + target_port: 9090 + session_name: "default-session" + is_default: true # Catches unmatched requests +``` + +**Result**: Creates 1 gRPC server on configurable port (defaults to 9080) that routes based on service patterns + +## How It Works + +### Service/Method Routing + +gRPC method names follow the format: `/package.ServiceName/MethodName` + +Examples: +- `/com.example.userservice.UserService/GetUser` → routed to user-api.internal.com:9090 +- `/com.example.orderservice.OrderService/CreateOrder` → routed to order-api.internal.com:9091 +- `/grpc.health.v1.Health/Check` → routed to default-api.internal.com:9090 + +### Pattern Types + +1. **Service Pattern**: Routes based on service name + ```yaml + service_pattern: "com\\.example\\.userservice\\..*" + ``` + +2. **Method Pattern**: Routes based on method name + ```yaml + method_pattern: "Get.*|List.*" # Only GET and LIST methods + ``` + +3. **Combined Patterns**: Both service and method must match + ```yaml + service_pattern: "com\\.example\\..*" + method_pattern: "Health.*" + ``` + +4. **Default Route**: Catches everything that doesn't match other patterns + ```yaml + is_default: true + ``` + +## Usage Examples + +### Start the Server with gRPC Routing +```bash +# Start mimic with gRPC routing (routing is automatic for gRPC proxies) +mimic --config config-grpc-routing.yaml + +# Output: +# Starting multi-proxy server +# Web UI available at http://0.0.0.0:8080/ +# gRPC router server listening on 0.0.0.0:9080 +# → Route 'user-service': com\.example\.userservice\..* -> user-api.internal.com:9090 +# → Route 'order-service': com\.example\.orderservice\..* -> order-api.internal.com:9091 +# → Route 'default-backend': (default) -> default-api.internal.com:9090 +# gRPC router server available at 0.0.0.0:9080 +``` + +### Test Routing +```bash +# This goes to user-api.internal.com:9090 +grpcurl -plaintext localhost:9080 com.example.userservice.UserService/GetUser + +# This goes to order-api.internal.com:9091 +grpcurl -plaintext localhost:9080 com.example.orderservice.OrderService/CreateOrder + +# This goes to default-api.internal.com:9090 (default route) +grpcurl -plaintext localhost:9080 some.other.service.SomeService/SomeMethod +``` + +## Benefits of gRPC Routing + +1. **Resource Efficiency**: Single gRPC server handles all routes +2. **Simplified Deployment**: One configurable gRPC port to manage +3. **Flexible Routing**: Route by service name, method patterns, or any combination +4. **Session Isolation**: Different sessions per route for organized recording +5. **Fallback Support**: Default routes for unmatched requests +6. **Automatic**: No special configuration needed - just define gRPC proxies with patterns + +## Advanced Examples + +### Method-Specific Routing +```yaml +# Route only health checks to a dedicated service +health-service: + mode: "record" + protocol: "grpc" + target_host: "health.internal.com" + target_port: 9093 + session_name: "health-session" + method_pattern: "Check|Watch" # Only health check methods +``` + +### Environment-Based Routing +```yaml +# Route staging services to staging backends +staging-services: + mode: "record" + protocol: "grpc" + target_host: "staging-api.internal.com" + target_port: 9090 + session_name: "staging-session" + service_pattern: ".*\\.staging\\..*" + +# Route production services to production backends +prod-services: + mode: "record" + protocol: "grpc" + target_host: "prod-api.internal.com" + target_port: 9090 + session_name: "prod-session" + service_pattern: ".*\\.prod\\..*" +``` + +### Mock Mode with Routing +```yaml +# Same routing works for mock mode +proxies: + user-service-mock: + mode: "mock" # Switch to mock mode + protocol: "grpc" + session_name: "user-session" # Use recorded session + service_pattern: "com\\.example\\.userservice\\..*" + + order-service-mock: + mode: "mock" + protocol: "grpc" + session_name: "order-session" + service_pattern: "com\\.example\\.orderservice\\..*" +``` + +## Implementation Details + +The routing is implemented using: + +1. **GRPCRouter**: Manages multiple routes and pattern matching +2. **GRPCRoute**: Individual route with target configuration and patterns +3. **UnknownServiceHandler**: Intercepts all gRPC calls and routes them +4. **Pattern Matching**: Uses regex patterns to match service/method names +5. **MultiProxyServer**: Automatically creates routing when gRPC proxies are configured + +The router extracts the service and method from the full gRPC method name (`/package.ServiceName/MethodName`) and matches against configured patterns to determine the target backend. + +This approach provides the flexibility of multiple gRPC proxies while maintaining the simplicity of a single server endpoint. Routing is enabled automatically when you configure gRPC proxies - no special commands or setup required! \ No newline at end of file diff --git a/README.md b/README.md index 402ccc1..80411e3 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ A transparent proxy for intercepting, recording, and mocking API calls. Supports - **Transparent Proxy Mode**: Intercepts and records API requests/responses - **Mock Server Mode**: Replays recorded interactions +- **Replay Mode**: Tests recorded interactions against live servers with timing and validation - **Protocol Support**: REST (HTTP/HTTPS) and gRPC - **SQLite Storage**: Reliable local storage with ordering preservation - **JSON Export/Import**: Version control integration and data portability @@ -234,6 +235,85 @@ Start the proxy in mock mode to serve recorded responses: mimic --mode mock ``` +### Replay Mode + +Replay recorded interactions against a live server for testing and validation: + +```bash +# Replay a session against a target server +mimic replay --session "my-session" --target-host api.example.com --target-port 443 + +# Replay with different validation strategies +mimic replay --session "api-tests" --target-host staging.api.com --matching-strategy fuzzy + +# Replay with concurrent requests (faster execution) +mimic replay --session "load-test" --target-host localhost --target-port 8080 --concurrency 5 + +# Replay ignoring original timing (fire all requests immediately) +mimic replay --session "quick-test" --target-host api.test.com --ignore-timestamps + +# Replay with fail-fast mode (exit on first mismatch) +mimic replay --session "critical-test" --target-host prod.api.com --fail-fast + +# Replay gRPC interactions +mimic replay --session "grpc-session" --target-host localhost --target-port 9090 --protocol grpc + +# Replay gRPC interactions with larger message sizes (for production servers) +mimic replay --session "grpc-session" --target-host grpc.production.com --protocol grpc --grpc-max-message-size 268435456 + +# Replay gRPC interactions with insecure connection (for testing) +mimic replay --session "grpc-session" --target-host localhost --target-port 9090 --protocol grpc --grpc-insecure +``` + +#### Replay Configuration Options + +- `--session`: Session name to replay (required) +- `--target-host`: Target server hostname (required) +- `--target-port`: Target server port (default: 443) +- `--protocol`: Protocol to use - http, https, or grpc (default: https) +- `--matching-strategy`: Response validation strategy: + - `exact`: Responses must match exactly (default) + - `fuzzy`: Allow minor differences in JSON structure + - `status_code`: Only validate HTTP status codes +- `--fail-fast`: Exit on first validation failure (default: false) +- `--timeout`: Request timeout in seconds (default: 30) +- `--concurrency`: Max concurrent requests (default: 0 for sequential) +- `--ignore-timestamps`: Skip timing-based replay (default: false) +- `--insecure-skip-verify`: Skip TLS verification for HTTPS/gRPC (default: false) +- `--grpc-max-message-size`: Max gRPC message size in bytes (default: 256MB) +- `--grpc-insecure`: Use insecure gRPC connection without TLS (default: false) + +#### Replay via Server Mode + +You can also run replay through the HTTP server interface: + +```bash +# Start server in replay mode +mimic --mode replay --config config-replay.yaml + +# Trigger replay via HTTP +curl -X POST "http://localhost:8080/proxy/replay-api/?session=my-session&target_host=api.example.com" +``` + +#### gRPC Replay + +gRPC replay works by: +1. Establishing a gRPC client connection to the target server +2. Replaying recorded gRPC calls using the stored raw protobuf message data +3. Comparing responses based on the selected matching strategy +4. Supporting both unary and streaming RPCs (though streaming is sequential only) + +**gRPC Replay Considerations:** +- gRPC replay uses TLS by default for production servers; use `--grpc-insecure` for local testing +- Raw protobuf message data is replayed exactly as recorded +- Status codes are compared using gRPC status codes (0 = OK, etc.) +- Metadata (gRPC headers) from the original requests is preserved and replayed +- Default message size limit is 256MB; increase with `--grpc-max-message-size` for larger messages +- For exact matching, both status code and response message must match +- For fuzzy matching, only status codes are compared +- Concurrent replay is supported for unary calls but not recommended for order-sensitive services +- Use `--insecure-skip-verify` to skip TLS certificate verification for testing environments + ### Export Session Export recorded session data to JSON: @@ -353,7 +433,7 @@ mimic clear --session "my-session" ### Proxy Settings -- `mode`: Operation mode (`record` or `mock`) +- `mode`: Operation mode (`record`, `mock`, or `replay`) - `target_host`: Target server hostname (required in record mode) - `target_port`: Target server port (required in record mode) - `listen_host`: Proxy listen address (default: `0.0.0.0`) @@ -373,6 +453,22 @@ mimic clear --session "my-session" - `sequence_mode`: Response selection mode (`ordered`, `random`) - `not_found_response`: Default response for unmatched requests +### Replay Settings + +- `target_host`: Target server hostname for replay +- `target_port`: Target server port for replay +- `protocol`: Target server protocol (`http`, `https`, or `grpc`) +- `session_name`: Session to replay +- `matching_strategy`: Response validation strategy (`exact`, `fuzzy`, `status_code`) +- `fail_fast`: Exit on first mismatch (boolean) +- `timeout_seconds`: Request timeout in seconds +- `max_concurrency`: Maximum concurrent requests (0 for sequential) +- `ignore_timestamps`: Skip timing-based replay (boolean) +- `insecure_skip_verify`: Skip TLS verification (boolean) +- `grpc_max_message_size`: Max gRPC message size in bytes +- `grpc_max_header_size`: Max gRPC header size in bytes +- `grpc_insecure`: Use insecure gRPC connection (boolean) + ### Export Settings - `format`: Export format (currently only `json`) diff --git a/cmd/replay.go b/cmd/replay.go new file mode 100644 index 0000000..47a3d75 --- /dev/null +++ b/cmd/replay.go @@ -0,0 +1,148 @@ +package cmd + +import ( + "fmt" + "log" + "os" + + "mimic/config" + "mimic/replay" + "mimic/storage" + + "github.com/spf13/cobra" +) + +var ( + replaySessionName string + replayTargetHost string + replayTargetPort int + replayProtocol string + replayMatchingStrategy string + replayFailFast bool + replayTimeoutSeconds int + replayMaxConcurrency int + replayIgnoreTimestamps bool + replayInsecureSkipVerify bool + replayGRPCMaxMessageSize int + replayGRPCInsecure bool +) + +var replayCmd = &cobra.Command{ + Use: "replay", + Short: "Replay recorded interactions against a target server", + Long: `Replay recorded interactions from a session against a target server for testing purposes. +This validates that the target server returns the same responses as were originally recorded.`, + Run: func(cmd *cobra.Command, args []string) { + runReplay() + }, +} + +func init() { + replayCmd.Flags().StringVar(&replaySessionName, "session", "", "session name to replay (required)") + replayCmd.Flags().StringVar(&replayTargetHost, "target-host", "", "target server hostname (required)") + replayCmd.Flags().IntVar(&replayTargetPort, "target-port", 443, "target server port") + replayCmd.Flags().StringVar(&replayProtocol, "protocol", "https", "target server protocol (http, https, or grpc)") + replayCmd.Flags().StringVar(&replayMatchingStrategy, "matching-strategy", "exact", "response matching strategy (exact, fuzzy, or status_code)") + replayCmd.Flags().BoolVar(&replayFailFast, "fail-fast", false, "exit on first mismatch (otherwise collect all errors)") + replayCmd.Flags().IntVar(&replayTimeoutSeconds, "timeout", 30, "request timeout in seconds") + replayCmd.Flags().IntVar(&replayMaxConcurrency, "concurrency", 0, "max concurrent requests (0 for sequential)") + replayCmd.Flags().BoolVar(&replayIgnoreTimestamps, "ignore-timestamps", false, "ignore original timing and fire all requests immediately") + replayCmd.Flags().BoolVar(&replayInsecureSkipVerify, "insecure-skip-verify", false, "skip TLS verification for HTTPS/gRPC") + replayCmd.Flags().IntVar(&replayGRPCMaxMessageSize, "grpc-max-message-size", 256*1024*1024, "max gRPC message size in bytes") + replayCmd.Flags().BoolVar(&replayGRPCInsecure, "grpc-insecure", false, "use insecure gRPC connection (no TLS)") + + replayCmd.MarkFlagRequired("session") + replayCmd.MarkFlagRequired("target-host") + + rootCmd.AddCommand(replayCmd) +} + +func runReplay() { + cfg, err := config.LoadConfig(cfgFile) + if err != nil { + log.Fatal("Failed to load config:", err) + } + + db, err := storage.NewDatabase(cfg.Database.Path) + if err != nil { + log.Fatal("Failed to initialize database:", err) + } + defer db.Close() + + // Build replay config from CLI flags + replayConfig := &config.ReplayConfig{ + TargetHost: replayTargetHost, + TargetPort: replayTargetPort, + Protocol: replayProtocol, + SessionName: replaySessionName, + MatchingStrategy: replayMatchingStrategy, + FailFast: replayFailFast, + TimeoutSeconds: replayTimeoutSeconds, + MaxConcurrency: replayMaxConcurrency, + IgnoreTimestamps: replayIgnoreTimestamps, + } + + // Validate the replay config + if replayConfig.TargetHost == "" { + log.Fatal("target-host is required") + } + if replayConfig.SessionName == "" { + log.Fatal("session is required") + } + if replayConfig.Protocol != "http" && replayConfig.Protocol != "https" && replayConfig.Protocol != "grpc" { + log.Fatal("protocol must be 'http', 'https', or 'grpc'") + } + if replayConfig.MatchingStrategy != "exact" && replayConfig.MatchingStrategy != "fuzzy" && replayConfig.MatchingStrategy != "status_code" { + log.Fatal("matching-strategy must be 'exact', 'fuzzy', or 'status_code'") + } + + // Create and run the replay engine + engine, err := replay.NewReplayEngine(replayConfig, db) + if err != nil { + log.Fatal("Failed to create replay engine:", err) + } + + fmt.Printf("Starting replay of session '%s' against %s://%s:%d\n", + replayConfig.SessionName, replayConfig.Protocol, replayConfig.TargetHost, replayConfig.TargetPort) + + replaySession, err := engine.Replay() + if err != nil { + if replayConfig.FailFast { + log.Fatal("Replay failed:", err) + } else { + log.Printf("Replay completed with errors: %v", err) + } + } + + // Print summary + fmt.Printf("\nReplay Summary:\n") + fmt.Printf("Session: %s\n", replaySession.SessionName) + fmt.Printf("Total Requests: %d\n", replaySession.TotalRequests) + fmt.Printf("Successful: %d\n", replaySession.SuccessCount) + fmt.Printf("Failed: %d\n", replaySession.FailureCount) + fmt.Printf("Duration: %v\n", replaySession.Duration) + + // Print detailed results if there were failures + if replaySession.FailureCount > 0 { + fmt.Printf("\nFailure Details:\n") + for i, result := range replaySession.Results { + if !result.Success { + fmt.Printf("%d. %s %s\n", i+1, result.Interaction.Method, result.Interaction.Endpoint) + if result.Error != nil { + fmt.Printf(" Error: %v\n", result.Error) + } + if result.ValidationError != "" { + fmt.Printf(" Validation: %s\n", result.ValidationError) + } + fmt.Printf(" Expected Status: %d, Actual Status: %d\n", result.ExpectedStatus, result.ActualStatus) + fmt.Printf(" Response Time: %v\n", result.ResponseTime) + fmt.Printf("\n") + } + } + } + + // Exit with error code if there were failures + if replaySession.FailureCount > 0 { + os.Exit(1) + } +} diff --git a/cmd/root.go b/cmd/root.go index 1e04311..f659121 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -42,7 +42,7 @@ func Execute() error { func init() { rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is config.yaml)") rootCmd.PersistentFlags().BoolVar(&debugMode, "debug", false, "enable debug logging") - rootCmd.PersistentFlags().StringVar(&modeFlag, "mode", "", "operation mode (record or mock) - overrides config file setting") + rootCmd.PersistentFlags().StringVar(&modeFlag, "mode", "", "operation mode (record, mock, or replay) - overrides config file setting") rootCmd.AddCommand(exportCmd) rootCmd.AddCommand(importCmd) diff --git a/config-grpc-replay-example.yaml b/config-grpc-replay-example.yaml new file mode 100644 index 0000000..ae52eb7 --- /dev/null +++ b/config-grpc-replay-example.yaml @@ -0,0 +1,55 @@ +mode: replay + +server: + listen_host: "0.0.0.0" + listen_port: 8080 + grpc_port: 9080 + +proxies: + grpc-replay: + protocol: "grpc" + session_name: "grpc-session" + +replay: + target_host: "localhost" + target_port: 9090 + protocol: "grpc" + session_name: "grpc-session" + matching_strategy: "exact" # exact, fuzzy, or status_code + fail_fast: false # true to exit on first failure, false to collect all errors + timeout_seconds: 30 + max_concurrency: 0 # 0 for sequential, >0 for concurrent replay + ignore_timestamps: false # true to ignore original timing + insecure_skip_verify: false # skip TLS verification + grpc_max_message_size: 268435456 # 256MB + grpc_max_header_size: 16777216 # 16MB + grpc_insecure: false # use insecure gRPC connection + +database: + path: "~/.mimic/recordings.db" + +recording: + session_name: "grpc-session" + capture_headers: true + capture_body: true + redact_patterns: + - "authorization: Bearer .*" + - "x-api-key: .*" + +mock: + matching_strategy: "exact" + sequence_mode: "ordered" + not_found_response: + status: 404 + body: + error: "Recording not found" + +grpc: + reflection_enabled: true + max_message_size: 67108864 # 64MB + max_header_size: 67108864 # 64MB + +export: + format: "json" + pretty_print: true + compress: false \ No newline at end of file diff --git a/config-grpc-replay-production.yaml b/config-grpc-replay-production.yaml new file mode 100644 index 0000000..1f5d7c6 --- /dev/null +++ b/config-grpc-replay-production.yaml @@ -0,0 +1,55 @@ +mode: replay + +server: + listen_host: "0.0.0.0" + listen_port: 8080 + grpc_port: 9080 + +proxies: + grpc-replay: + protocol: "grpc" + session_name: "test-replay" + +replay: + target_host: "grpc.astria.org" + target_port: 443 + protocol: "grpc" + session_name: "test-replay" + matching_strategy: "exact" # exact, fuzzy, or status_code + fail_fast: false # true to exit on first failure, false to collect all errors + timeout_seconds: 30 + max_concurrency: 0 # 0 for sequential, >0 for concurrent replay + ignore_timestamps: false # true to ignore original timing + insecure_skip_verify: false # skip TLS verification + grpc_max_message_size: 268435456 # 256MB + grpc_max_header_size: 16777216 # 16MB + grpc_insecure: false # use TLS (production server) + +database: + path: "~/.mimic/recordings.db" + +recording: + session_name: "test-replay" + capture_headers: true + capture_body: true + redact_patterns: + - "authorization: Bearer .*" + - "x-api-key: .*" + +mock: + matching_strategy: "exact" + sequence_mode: "ordered" + not_found_response: + status: 404 + body: + error: "Recording not found" + +grpc: + reflection_enabled: true + max_message_size: 268435456 # 256MB + max_header_size: 16777216 # 16MB + +export: + format: "json" + pretty_print: true + compress: false \ No newline at end of file diff --git a/config-grpc-routing-example.yaml b/config-grpc-routing-example.yaml new file mode 100644 index 0000000..16736bc --- /dev/null +++ b/config-grpc-routing-example.yaml @@ -0,0 +1,88 @@ +# Example configuration for gRPC path-based routing +# Single gRPC server handling multiple services with different backends + +server: + port: 8080 + +proxies: + # User service - routes all com.example.userservice.* calls + user-service: + mode: "record" + protocol: "grpc" + target_host: "user-api.internal.com" + target_port: 9090 + session_name: "user-service-session" + service_pattern: "com\\.example\\.userservice\\..*" # Regex: com.example.userservice.* + + # Order service - routes all com.example.orderservice.* calls + order-service: + mode: "record" + protocol: "grpc" + target_host: "order-api.internal.com" + target_port: 9091 + session_name: "order-service-session" + service_pattern: "com\\.example\\.orderservice\\..*" # Regex: com.example.orderservice.* + + # Payment service - routes all payment related calls + payment-service: + mode: "record" + protocol: "grpc" + target_host: "payment-api.internal.com" + target_port: 9092 + session_name: "payment-service-session" + service_pattern: "com\\.example\\.paymentservice\\..*" # Regex: com.example.paymentservice.* + + # Method-specific routing - route only specific methods + health-checks: + mode: "record" + protocol: "grpc" + target_host: "health.internal.com" + target_port: 9093 + session_name: "health-session" + method_pattern: "Check|Watch" # Only route Check and Watch methods + + # Default/fallback route - catches everything else + default-backend: + mode: "record" + protocol: "grpc" + target_host: "default-api.internal.com" + target_port: 9090 + session_name: "default-session" + is_default: true # This route catches unmatched requests + +database: + path: "./mimic.db" + connection_pool_size: 10 + +grpc: + reflection_enabled: true + +# Usage Examples: +# +# 1. Start mimic with routing config: +# mimic --config config-grpc-routing-example.yaml +# +# 2. gRPC calls will be routed based on service patterns to localhost:9080 (HTTP port + 1000): +# +# # This goes to user-api.internal.com:9090 +# grpcurl -plaintext localhost:9080 com.example.userservice.UserService/GetUser +# +# # This goes to order-api.internal.com:9091 +# grpcurl -plaintext localhost:9080 com.example.orderservice.OrderService/CreateOrder +# +# # This goes to payment-api.internal.com:9092 +# grpcurl -plaintext localhost:9080 com.example.paymentservice.PaymentService/ProcessPayment +# +# # Health checks go to health.internal.com:9093 +# grpcurl -plaintext localhost:9080 grpc.health.v1.Health/Check +# +# # Anything else goes to default-api.internal.com:9090 +# grpcurl -plaintext localhost:9080 some.other.service.SomeService/SomeMethod +# +# 3. Web UI and gRPC info available at: +# # Web UI: http://localhost:8080/ +# # gRPC info: http://localhost:8080/grpc/info + +export: + format: "json" + pretty_print: true \ No newline at end of file diff --git a/config-grpc-simple.yaml b/config-grpc-simple.yaml new file mode 100644 index 0000000..5d159c6 --- /dev/null +++ b/config-grpc-simple.yaml @@ -0,0 +1,42 @@ +# Simple gRPC routing example +# Shows how multiple gRPC services route to different backends automatically + +server: + port: 8080 + +proxies: + # Route user service calls to user backend + user-service: + mode: "record" + protocol: "grpc" + target_host: "user-api.example.com" + target_port: 9090 + session_name: "user-session" + service_pattern: ".*\\.userservice\\..*" # Matches any package with userservice + + # Route order service calls to order backend + order-service: + mode: "record" + protocol: "grpc" + target_host: "order-api.example.com" + target_port: 9091 + session_name: "order-session" + service_pattern: ".*\\.orderservice\\..*" # Matches any package with orderservice + + # Default route for everything else + default: + mode: "record" + protocol: "grpc" + target_host: "api.example.com" + target_port: 9090 + session_name: "default-session" + is_default: true + +database: + path: "./mimic.db" + +# Usage: +# 1. Start: mimic --config config-grpc-simple.yaml +# 2. gRPC server runs on localhost:9080 (port 8080 + 1000) +# 3. Web UI runs on http://localhost:8080/ +# 4. Test with: grpcurl -plaintext localhost:9080 your.userservice.UserService/GetUser \ No newline at end of file diff --git a/config-replay-example.yaml b/config-replay-example.yaml new file mode 100644 index 0000000..223bb87 --- /dev/null +++ b/config-replay-example.yaml @@ -0,0 +1,50 @@ +mode: replay + +server: + listen_host: "0.0.0.0" + listen_port: 8080 + +proxies: + replay-api: + protocol: "https" + session_name: "test-session" + +replay: + target_host: "httpbin.org" + target_port: 443 + protocol: "https" + session_name: "test-session" + matching_strategy: "exact" # exact, fuzzy, or status_code + fail_fast: false # true to exit on first failure, false to collect all errors + timeout_seconds: 30 + max_concurrency: 0 # 0 for sequential, >0 for concurrent replay + ignore_timestamps: false # true to ignore original timing + +database: + path: "~/.mimic/recordings.db" + +recording: + session_name: "test-session" + capture_headers: true + capture_body: true + redact_patterns: + - "Authorization: Bearer .*" + - "X-API-Key: .*" + +mock: + matching_strategy: "exact" + sequence_mode: "ordered" + not_found_response: + status: 404 + body: + error: "Recording not found" + +grpc: + reflection_enabled: true + max_message_size: 67108864 # 64MB + max_header_size: 67108864 # 64MB + +export: + format: "json" + pretty_print: true + compress: false \ No newline at end of file diff --git a/config.yaml b/config.yaml index ecc5e29..bb6d4f5 100644 --- a/config.yaml +++ b/config.yaml @@ -16,7 +16,7 @@ proxies: protocol: "grpc" target_host: "grpc.astria.org" target_port: 443 - session_name: "grpc-mock-session" + session_name: "test-replay" service_pattern: "astria\\..*" # test-grpc: diff --git a/config/config.go b/config/config.go index d2cbc2e..752d642 100644 --- a/config/config.go +++ b/config/config.go @@ -9,12 +9,13 @@ import ( ) type Config struct { - Mode string `mapstructure:"mode"` // Global mode: "record" or "mock" + Mode string `mapstructure:"mode"` // Global mode: "record", "mock", or "replay" Server ServerConfig `mapstructure:"server"` Proxies map[string]ProxyConfig `mapstructure:"proxies"` Database DatabaseConfig `mapstructure:"database"` Recording RecordingConfig `mapstructure:"recording"` Mock MockConfig `mapstructure:"mock"` + Replay ReplayConfig `mapstructure:"replay"` GRPC GRPCConfig `mapstructure:"grpc"` Export ExportConfig `mapstructure:"export"` } @@ -22,17 +23,17 @@ type Config struct { type ServerConfig struct { ListenHost string `mapstructure:"listen_host"` ListenPort int `mapstructure:"listen_port"` - GRPCPort int `mapstructure:"grpc_port"` // Port for gRPC server (defaults to listen_port + 1000) + GRPCPort int `mapstructure:"grpc_port"` // Port for gRPC server (defaults to listen_port + 1000) } type ProxyConfig struct { - TargetHost string `mapstructure:"target_host"` - TargetPort int `mapstructure:"target_port"` - Protocol string `mapstructure:"protocol"` - SessionName string `mapstructure:"session_name"` + TargetHost string `mapstructure:"target_host"` + TargetPort int `mapstructure:"target_port"` + Protocol string `mapstructure:"protocol"` + SessionName string `mapstructure:"session_name"` // gRPC routing patterns (optional) ServicePattern string `mapstructure:"service_pattern"` // Regex pattern for service names - MethodPattern string `mapstructure:"method_pattern"` // Regex pattern for method names + MethodPattern string `mapstructure:"method_pattern"` // Regex pattern for method names IsDefault bool `mapstructure:"is_default"` // Whether this is the default/fallback route } @@ -59,11 +60,28 @@ type NotFoundResponseConfig struct { Body map[string]interface{} `mapstructure:"body"` } +type ReplayConfig struct { + TargetHost string `mapstructure:"target_host"` // Target server to replay against + TargetPort int `mapstructure:"target_port"` // Target server port + Protocol string `mapstructure:"protocol"` // http, https, or grpc + SessionName string `mapstructure:"session_name"` // Session to replay + MatchingStrategy string `mapstructure:"matching_strategy"` // How to compare responses: exact, fuzzy, status_code + FailFast bool `mapstructure:"fail_fast"` // Exit on first mismatch or collect all errors + TimeoutSeconds int `mapstructure:"timeout_seconds"` // Request timeout in seconds + MaxConcurrency int `mapstructure:"max_concurrency"` // Max concurrent requests (0 = sequential) + IgnoreTimestamps bool `mapstructure:"ignore_timestamps"` // Skip timing-based replay, fire all at once + InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"` // Skip TLS verification for HTTPS/gRPC + // gRPC-specific settings + GRPCMaxMessageSize int `mapstructure:"grpc_max_message_size"` // Max gRPC message size in bytes + GRPCMaxHeaderSize int `mapstructure:"grpc_max_header_size"` // Max gRPC header size in bytes + GRPCInsecure bool `mapstructure:"grpc_insecure"` // Use insecure gRPC connection +} + type GRPCConfig struct { ProtoPaths []string `mapstructure:"proto_paths"` ReflectionEnabled bool `mapstructure:"reflection_enabled"` - MaxMessageSize int `mapstructure:"max_message_size"` // Max message size in bytes - MaxHeaderSize int `mapstructure:"max_header_size"` // Max header list size in bytes + MaxMessageSize int `mapstructure:"max_message_size"` // Max message size in bytes + MaxHeaderSize int `mapstructure:"max_header_size"` // Max header list size in bytes } type ExportConfig struct { @@ -123,7 +141,7 @@ func setDefaults() { defaultDBPath := filepath.Join(homeDir, ".mimic", "recordings.db") viper.SetDefault("mode", "record") - + viper.SetDefault("server.listen_host", "0.0.0.0") viper.SetDefault("server.listen_port", 8080) viper.SetDefault("server.grpc_port", 9080) // Default to 9080 @@ -142,6 +160,17 @@ func setDefaults() { "error": "Recording not found", }) + viper.SetDefault("replay.protocol", "https") + viper.SetDefault("replay.matching_strategy", "exact") + viper.SetDefault("replay.fail_fast", false) + viper.SetDefault("replay.timeout_seconds", 30) + viper.SetDefault("replay.max_concurrency", 0) + viper.SetDefault("replay.ignore_timestamps", false) + viper.SetDefault("replay.insecure_skip_verify", false) + viper.SetDefault("replay.grpc_max_message_size", 256*1024*1024) // 256MB + viper.SetDefault("replay.grpc_max_header_size", 16*1024*1024) // 16MB + viper.SetDefault("replay.grpc_insecure", false) + viper.SetDefault("grpc.reflection_enabled", true) viper.SetDefault("grpc.max_message_size", 64*1024*1024) // 64MB viper.SetDefault("grpc.max_header_size", 64*1024*1024) // 64MB @@ -186,6 +215,18 @@ func getDefaultConfig() *Config { Body: map[string]interface{}{"error": "Recording not found"}, }, }, + Replay: ReplayConfig{ + Protocol: "https", + MatchingStrategy: "exact", + FailFast: false, + TimeoutSeconds: 30, + MaxConcurrency: 0, + IgnoreTimestamps: false, + InsecureSkipVerify: false, + GRPCMaxMessageSize: 256 * 1024 * 1024, // 256MB + GRPCMaxHeaderSize: 16 * 1024 * 1024, // 16MB + GRPCInsecure: false, + }, GRPC: GRPCConfig{ ProtoPaths: []string{}, ReflectionEnabled: true, @@ -202,8 +243,8 @@ func getDefaultConfig() *Config { func (c *Config) Validate() error { // Validate global mode - if c.Mode != "record" && c.Mode != "mock" { - return fmt.Errorf("invalid mode: %s (must be 'record' or 'mock')", c.Mode) + if c.Mode != "record" && c.Mode != "mock" && c.Mode != "replay" { + return fmt.Errorf("invalid mode: %s (must be 'record', 'mock', or 'replay')", c.Mode) } // Validate server config @@ -235,6 +276,31 @@ func (c *Config) Validate() error { } } + // Validate replay config + if c.Mode == "replay" { + if c.Replay.TargetHost == "" { + return fmt.Errorf("target_host is required in replay mode") + } + if c.Replay.TargetPort == 0 { + return fmt.Errorf("target_port is required in replay mode") + } + if c.Replay.SessionName == "" { + return fmt.Errorf("session_name is required in replay mode") + } + if c.Replay.Protocol != "http" && c.Replay.Protocol != "https" && c.Replay.Protocol != "grpc" { + return fmt.Errorf("invalid replay protocol: %s (must be 'http', 'https', or 'grpc')", c.Replay.Protocol) + } + if c.Replay.GRPCMaxMessageSize <= 0 { + c.Replay.GRPCMaxMessageSize = 256 * 1024 * 1024 // 256MB default + } + if c.Replay.GRPCMaxHeaderSize <= 0 { + c.Replay.GRPCMaxHeaderSize = 16 * 1024 * 1024 // 16MB default + } + if c.Replay.MatchingStrategy != "exact" && c.Replay.MatchingStrategy != "fuzzy" && c.Replay.MatchingStrategy != "status_code" { + return fmt.Errorf("invalid replay matching strategy: %s (must be 'exact', 'fuzzy', or 'status_code')", c.Replay.MatchingStrategy) + } + } + if c.Database.Path == "" { return fmt.Errorf("database path cannot be empty") } @@ -249,6 +315,7 @@ func SaveConfig(config *Config, path string) error { viper.Set("database", config.Database) viper.Set("recording", config.Recording) viper.Set("mock", config.Mock) + viper.Set("replay", config.Replay) viper.Set("grpc", config.GRPC) viper.Set("export", config.Export) diff --git a/mock/grpc_mock_router.go b/mock/grpc_mock_router.go new file mode 100644 index 0000000..50f40c8 --- /dev/null +++ b/mock/grpc_mock_router.go @@ -0,0 +1,175 @@ +package mock + +import ( + "fmt" + "log" + "regexp" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "mimic/config" + "mimic/proxy" + "mimic/storage" +) + +// GRPCMockRoute represents a mock routing rule for gRPC services +type GRPCMockRoute struct { + Name string // Route name for identification + ServicePattern *regexp.Regexp // Pattern to match service names + MethodPattern *regexp.Regexp // Pattern to match method names + Config *config.ProxyConfig // Configuration for this route + Session *storage.Session // Session for this route +} + +// GRPCMockRouter handles routing gRPC mock calls based on service/method patterns +type GRPCMockRouter struct { + routes []*GRPCMockRoute + database *storage.Database + grpcHandler *proxy.GRPCHandler + webServer proxy.WebBroadcaster + defaultRoute *GRPCMockRoute // Fallback route if no patterns match +} + +// NewGRPCMockRouter creates a new gRPC mock router with multiple routes +func NewGRPCMockRouter(routeConfigs map[string]config.ProxyConfig, db *storage.Database, webServer proxy.WebBroadcaster) (*GRPCMockRouter, error) { + router := &GRPCMockRouter{ + routes: make([]*GRPCMockRoute, 0), + database: db, + grpcHandler: proxy.NewGRPCHandler([]string{}), // Use empty redact patterns for now + webServer: webServer, + } + + for name, proxyConfig := range routeConfigs { + session, err := db.GetOrCreateSession(proxyConfig.SessionName, fmt.Sprintf("Mock session for %s", name)) + if err != nil { + return nil, fmt.Errorf("failed to create session for mock route %s: %w", name, err) + } + + route := &GRPCMockRoute{ + Name: name, + Config: &proxyConfig, + Session: session, + } + + // Parse service and method patterns from config + if servicePattern := proxyConfig.ServicePattern; servicePattern != "" { + if pattern, err := regexp.Compile(servicePattern); err == nil { + route.ServicePattern = pattern + } else { + log.Printf("Invalid service pattern for mock route %s: %v", name, err) + } + } + + if methodPattern := proxyConfig.MethodPattern; methodPattern != "" { + if pattern, err := regexp.Compile(methodPattern); err == nil { + route.MethodPattern = pattern + } else { + log.Printf("Invalid method pattern for mock route %s: %v", name, err) + } + } + + // Set as default route if specified + if proxyConfig.IsDefault { + router.defaultRoute = route + } else { + router.routes = append(router.routes, route) + } + + log.Printf("Added gRPC mock route '%s' for session '%s' (service: %s, method: %s)", + name, session.SessionName, proxyConfig.ServicePattern, proxyConfig.MethodPattern) + } + + return router, nil +} + +// GetUnknownServiceHandler returns a handler that routes gRPC mock calls based on service/method patterns +func (r *GRPCMockRouter) GetUnknownServiceHandler() grpc.StreamHandler { + proxy.RegisterRawCodec() + + return func(srv interface{}, stream grpc.ServerStream) error { + fullMethodName, ok := grpc.MethodFromServerStream(stream) + if !ok { + return status.Errorf(codes.Internal, "failed to get method from stream") + } + + log.Printf("gRPC Mock Router: routing %s", fullMethodName) + + // Extract service and method from full method name + // Format: /package.ServiceName/MethodName + parts := strings.Split(strings.TrimPrefix(fullMethodName, "/"), "/") + if len(parts) != 2 { + return status.Errorf(codes.InvalidArgument, "invalid method name format: %s", fullMethodName) + } + + serviceName := parts[0] + methodName := parts[1] + + // Find matching route + route := r.findRoute(serviceName, methodName, fullMethodName) + if route == nil { + return status.Errorf(codes.Unimplemented, "no mock route found for service %s method %s", serviceName, methodName) + } + + log.Printf("gRPC Mock Router: matched route '%s' for %s", route.Name, fullMethodName) + + // Handle the mock request using the found route's session + return handleGRPCMockRequest(stream, r.database, route.Session, r.grpcHandler, r.webServer) + } +} + +// findRoute finds the best matching route for a service/method combination +func (r *GRPCMockRouter) findRoute(serviceName, methodName, fullMethodName string) *GRPCMockRoute { + // Try to find exact pattern matches first + for _, route := range r.routes { + if r.routeMatches(route, serviceName, methodName, fullMethodName) { + return route + } + } + + // Fall back to default route if available + if r.defaultRoute != nil { + log.Printf("gRPC Mock Router: using default route '%s' for %s", r.defaultRoute.Name, fullMethodName) + return r.defaultRoute + } + + return nil +} + +// routeMatches checks if a route matches the given service/method +func (r *GRPCMockRouter) routeMatches(route *GRPCMockRoute, serviceName, methodName, fullMethodName string) bool { + // Check service pattern + if route.ServicePattern != nil { + if !route.ServicePattern.MatchString(serviceName) { + return false + } + } + + // Check method pattern + if route.MethodPattern != nil { + if !route.MethodPattern.MatchString(methodName) { + return false + } + } + + // If no patterns are specified, this route matches everything (shouldn't happen with proper config) + if route.ServicePattern == nil && route.MethodPattern == nil { + log.Printf("Warning: mock route '%s' has no patterns - matches all", route.Name) + return true + } + + return true +} + +// GetRoutes returns all configured routes for debugging/monitoring +func (r *GRPCMockRouter) GetRoutes() []*GRPCMockRoute { + routes := make([]*GRPCMockRoute, len(r.routes)) + copy(routes, r.routes) + + if r.defaultRoute != nil { + routes = append(routes, r.defaultRoute) + } + + return routes +} diff --git a/mock/mock_engine.go b/mock/mock_engine.go index 2e79b4f..8f33aee 100644 --- a/mock/mock_engine.go +++ b/mock/mock_engine.go @@ -20,7 +20,7 @@ import ( "mimic/storage" ) -// mockRawMessage for handling raw gRPC message data in mock responses +// mockRawMessage for handling raw gRPC message data in mock responses type mockRawMessage struct { Data []byte } @@ -43,7 +43,7 @@ func (m *mockRawMessage) Marshal() ([]byte, error) { return m.Data, nil } -// Unmarshal implements protobuf unmarshaling +// Unmarshal implements protobuf unmarshaling func (m *mockRawMessage) Unmarshal(data []byte) error { m.Data = make([]byte, len(data)) copy(m.Data, data) @@ -88,13 +88,13 @@ func NewMockEngineWithBroadcaster(proxyConfig config.ProxyConfig, db *storage.Da var grpcServer *grpc.Server if proxyConfig.Protocol == "grpc" { grpcServer = grpc.NewServer( - grpc.MaxRecvMsgSize(64*1024*1024), // 64MB max receive message size - grpc.MaxSendMsgSize(64*1024*1024), // 64MB max send message size - grpc.MaxHeaderListSize(64*1024*1024), // 64MB max header list size - grpc.InitialWindowSize(64*1024*1024), // 64MB initial window + grpc.MaxRecvMsgSize(64*1024*1024), // 64MB max receive message size + grpc.MaxSendMsgSize(64*1024*1024), // 64MB max send message size + grpc.MaxHeaderListSize(64*1024*1024), // 64MB max header list size + grpc.InitialWindowSize(64*1024*1024), // 64MB initial window grpc.InitialConnWindowSize(64*1024*1024), // 64MB connection window grpc.UnknownServiceHandler(func(srv interface{}, stream grpc.ServerStream) error { - return handleGRPCMockRequest(stream, db, session, grpcHandler) + return handleGRPCMockRequest(stream, db, session, grpcHandler, webServer) }), ) } @@ -462,7 +462,7 @@ func (m *MockEngine) GetSequenceState() map[string]int { } // handleGRPCMockRequest handles gRPC mock requests -func handleGRPCMockRequest(stream grpc.ServerStream, db *storage.Database, session *storage.Session, grpcHandler *proxy.GRPCHandler) error { +func handleGRPCMockRequest(stream grpc.ServerStream, db *storage.Database, session *storage.Session, grpcHandler *proxy.GRPCHandler, webServer WebBroadcaster) error { fullMethodName, ok := grpc.MethodFromServerStream(stream) if !ok { return status.Errorf(codes.Internal, "failed to get method from stream") @@ -511,6 +511,19 @@ func handleGRPCMockRequest(stream grpc.ServerStream, db *storage.Database, sessi return status.Errorf(codes.Internal, "failed to receive request: %v", err) } + // Generate request ID for tracking + requestID := proxy.GenerateRequestID() + + // Broadcast request event to web UI + if webServer != nil { + log.Printf("[DEBUG] Broadcasting gRPC mock request to web UI: %s", fullMethodName) + headers := make(map[string]interface{}) + body := fmt.Sprintf("gRPC mock request (%d bytes)", len(requestMsg.Data)) + webServer.BroadcastRequest(fullMethodName, fullMethodName, session.SessionName, "grpc-mock-client", requestID, headers, body) + } else { + log.Printf("[DEBUG] No webServer available for broadcasting gRPC mock request") + } + // Send the recorded response body if available if len(selectedInteraction.ResponseBody) > 0 { // Create a raw message with the recorded response data @@ -523,5 +536,15 @@ func handleGRPCMockRequest(stream grpc.ServerStream, db *storage.Database, sessi log.Printf("Served gRPC mock response: %s -> %d (empty response)", fullMethodName, selectedInteraction.ResponseStatus) } + // Broadcast response event to web UI + if webServer != nil { + log.Printf("[DEBUG] Broadcasting gRPC mock response to web UI: %s", fullMethodName) + responseHeaders := make(map[string]interface{}) + responseBody := fmt.Sprintf("gRPC mock response (%d bytes)", len(selectedInteraction.ResponseBody)) + webServer.BroadcastResponse(fullMethodName, fullMethodName, session.SessionName, "grpc-mock-client", requestID, selectedInteraction.ResponseStatus, responseHeaders, responseBody) + } else { + log.Printf("[DEBUG] No webServer available for broadcasting gRPC mock response") + } + return nil } diff --git a/proxy/grpc_raw_proxy.go b/proxy/grpc_raw_proxy.go index 068bf4e..c24ee86 100644 --- a/proxy/grpc_raw_proxy.go +++ b/proxy/grpc_raw_proxy.go @@ -29,8 +29,6 @@ type RawGRPCProxy struct { webServer WebBroadcaster } - - func NewRawGRPCProxy(proxyConfig *config.ProxyConfig, mode string, db *storage.Database, session *storage.Session, grpcHandler *GRPCHandler) *RawGRPCProxy { return &RawGRPCProxy{ config: proxyConfig, @@ -46,8 +44,6 @@ func (p *RawGRPCProxy) SetWebBroadcaster(wb WebBroadcaster) { p.webServer = wb } - - // GetUnknownServiceHandler returns a handler that can proxy any gRPC service using raw bytes func (p *RawGRPCProxy) GetUnknownServiceHandler() grpc.StreamHandler { // Register our raw codec @@ -167,8 +163,6 @@ func (p *RawGRPCProxy) proxyRawStream(serverStream grpc.ServerStream, clientStre return <-errCh } - - func (p *RawGRPCProxy) metadataToJSON(md metadata.MD) string { metadataMap := make(map[string][]string) for key, values := range md { @@ -199,25 +193,25 @@ func (p *RawGRPCProxy) isLikelyUnaryCall(method string) bool { streamingPatterns := []string{ "Stream", "Watch", "Subscribe", "Listen", "Monitor", "Observe", } - + for _, pattern := range streamingPatterns { if strings.Contains(method, pattern) { return false } } - + // Common patterns for unary calls unaryPatterns := []string{ - "Get", "Create", "Update", "Delete", "Check", "Validate", + "Get", "Create", "Update", "Delete", "Check", "Validate", "Info", "Status", "Health", "Ping", "Version", "List", } - + for _, pattern := range unaryPatterns { if strings.Contains(method, pattern) { return true } } - + // Default to unary for unknown patterns return true } @@ -237,7 +231,7 @@ func (p *RawGRPCProxy) handleUnaryCall(ctx context.Context, conn *grpc.ClientCon // Extract and forward metadata md, _ := metadata.FromIncomingContext(stream.Context()) outCtx := metadata.NewOutgoingContext(ctx, md) - + // Create interaction record for database storage var interaction *storage.Interaction if p.mode == "record" { @@ -262,11 +256,11 @@ func (p *RawGRPCProxy) handleUnaryCall(ctx context.Context, conn *grpc.ClientCon log.Printf("[DEBUG] No webServer available for broadcasting gRPC request") } } - + // Forward the unary call to target server var responseMsg RawMessage err := conn.Invoke(outCtx, method, &requestMsg, &responseMsg, grpc.ForceCodec(GetRawCodec())) - + // Handle recording and response if p.mode == "record" { statusCode := 0 @@ -315,4 +309,4 @@ func (p *RawGRPCProxy) handleUnaryCall(ctx context.Context, conn *grpc.ClientCon } return nil -} \ No newline at end of file +} diff --git a/proxy/grpc_router.go b/proxy/grpc_router.go new file mode 100644 index 0000000..156e3a6 --- /dev/null +++ b/proxy/grpc_router.go @@ -0,0 +1,180 @@ +package proxy + +import ( + "fmt" + "log" + "regexp" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "mimic/config" + "mimic/storage" +) + +// GRPCRoute represents a routing rule for gRPC services +type GRPCRoute struct { + Name string // Route name for identification + ServicePattern *regexp.Regexp // Pattern to match service names + MethodPattern *regexp.Regexp // Pattern to match method names + Config *config.ProxyConfig // Target configuration + Proxy *RawGRPCProxy // Proxy instance for this route +} + +// GRPCRouter handles routing gRPC calls to different backends based on service/method patterns +type GRPCRouter struct { + routes []*GRPCRoute + database *storage.Database + webServer WebBroadcaster + defaultRoute *GRPCRoute // Fallback route if no patterns match +} + +// NewGRPCRouter creates a new gRPC router with multiple routes +func NewGRPCRouter(routeConfigs map[string]config.ProxyConfig, mode string, db *storage.Database, webServer WebBroadcaster) (*GRPCRouter, error) { + router := &GRPCRouter{ + routes: make([]*GRPCRoute, 0), + database: db, + webServer: webServer, + } + + for name, proxyConfig := range routeConfigs { + session, err := db.GetOrCreateSession(proxyConfig.SessionName, fmt.Sprintf("Proxy session for %s", name)) + if err != nil { + return nil, fmt.Errorf("failed to create session for route %s: %w", name, err) + } + + grpcHandler := NewGRPCHandler([]string{}) // Use empty redact patterns for now + rawProxy := NewRawGRPCProxy(&proxyConfig, mode, db, session, grpcHandler) + + if webServer != nil { + rawProxy.SetWebBroadcaster(webServer) + } + + route := &GRPCRoute{ + Name: name, + Config: &proxyConfig, + Proxy: rawProxy, + } + + // Parse service and method patterns from config + if servicePattern := proxyConfig.ServicePattern; servicePattern != "" { + if pattern, err := regexp.Compile(servicePattern); err == nil { + route.ServicePattern = pattern + } else { + log.Printf("Invalid service pattern for route %s: %v", name, err) + } + } + + if methodPattern := proxyConfig.MethodPattern; methodPattern != "" { + if pattern, err := regexp.Compile(methodPattern); err == nil { + route.MethodPattern = pattern + } else { + log.Printf("Invalid method pattern for route %s: %v", name, err) + } + } + + // Set as default route if specified + if proxyConfig.IsDefault { + router.defaultRoute = route + } else { + router.routes = append(router.routes, route) + } + + log.Printf("Added gRPC route '%s' -> %s:%d (service: %s, method: %s)", + name, proxyConfig.TargetHost, proxyConfig.TargetPort, + proxyConfig.ServicePattern, proxyConfig.MethodPattern) + } + + return router, nil +} + +// GetUnknownServiceHandler returns a handler that routes gRPC calls based on service/method patterns +func (r *GRPCRouter) GetUnknownServiceHandler() grpc.StreamHandler { + RegisterRawCodec() + + return func(srv interface{}, stream grpc.ServerStream) error { + fullMethodName, ok := grpc.MethodFromServerStream(stream) + if !ok { + return status.Errorf(codes.Internal, "failed to get method from stream") + } + + log.Printf("gRPC Router: routing %s", fullMethodName) + + // Extract service and method from full method name + // Format: /package.ServiceName/MethodName + parts := strings.Split(strings.TrimPrefix(fullMethodName, "/"), "/") + if len(parts) != 2 { + return status.Errorf(codes.InvalidArgument, "invalid method name format: %s", fullMethodName) + } + + serviceName := parts[0] + methodName := parts[1] + + // Find matching route + route := r.findRoute(serviceName, methodName, fullMethodName) + if route == nil { + return status.Errorf(codes.Unimplemented, "no route found for service %s method %s", serviceName, methodName) + } + + log.Printf("gRPC Router: matched route '%s' for %s", route.Name, fullMethodName) + + // Delegate to the route's proxy handler + return route.Proxy.GetUnknownServiceHandler()(srv, stream) + } +} + +// findRoute finds the best matching route for a service/method combination +func (r *GRPCRouter) findRoute(serviceName, methodName, fullMethodName string) *GRPCRoute { + // Try to find exact pattern matches first + for _, route := range r.routes { + if r.routeMatches(route, serviceName, methodName, fullMethodName) { + return route + } + } + + // Fall back to default route if available + if r.defaultRoute != nil { + log.Printf("gRPC Router: using default route '%s' for %s", r.defaultRoute.Name, fullMethodName) + return r.defaultRoute + } + + return nil +} + +// routeMatches checks if a route matches the given service/method +func (r *GRPCRouter) routeMatches(route *GRPCRoute, serviceName, methodName, fullMethodName string) bool { + // Check service pattern + if route.ServicePattern != nil { + if !route.ServicePattern.MatchString(serviceName) { + return false + } + } + + // Check method pattern + if route.MethodPattern != nil { + if !route.MethodPattern.MatchString(methodName) { + return false + } + } + + // If no patterns are specified, this route matches everything (shouldn't happen with proper config) + if route.ServicePattern == nil && route.MethodPattern == nil { + log.Printf("Warning: route '%s' has no patterns - matches all", route.Name) + return true + } + + return true +} + +// GetRoutes returns all configured routes for debugging/monitoring +func (r *GRPCRouter) GetRoutes() []*GRPCRoute { + routes := make([]*GRPCRoute, len(r.routes)) + copy(routes, r.routes) + + if r.defaultRoute != nil { + routes = append(routes, r.defaultRoute) + } + + return routes +} diff --git a/proxy/grpc_test.go b/proxy/grpc_test.go index 439cecb..1718ffc 100644 --- a/proxy/grpc_test.go +++ b/proxy/grpc_test.go @@ -52,7 +52,7 @@ func TestRawGRPCProxyUnaryCallDetection(t *testing.T) { grpcHandler := NewGRPCHandler([]string{}) session, _ := db.GetOrCreateSession("test", "test") - + rawProxy := NewRawGRPCProxy(&proxyConfig, "record", db, session, grpcHandler) // Test unary call detection diff --git a/proxy/grpc_utils.go b/proxy/grpc_utils.go index d61fde5..3c07d75 100644 --- a/proxy/grpc_utils.go +++ b/proxy/grpc_utils.go @@ -91,4 +91,4 @@ func RegisterRawCodec() { // GenerateRequestID generates a unique request ID for gRPC interactions func GenerateRequestID() string { return fmt.Sprintf("grpc-%d", time.Now().UnixNano()) -} \ No newline at end of file +} diff --git a/proxy/proxy_engine.go b/proxy/proxy_engine.go index 2c01e91..76dfa65 100644 --- a/proxy/proxy_engine.go +++ b/proxy/proxy_engine.go @@ -57,17 +57,17 @@ func NewProxyEngineWithBroadcaster(proxyConfig config.ProxyConfig, db *storage.D if proxyConfig.Protocol == "grpc" { // Use raw proxy for better compatibility rawProxy := NewRawGRPCProxy(&proxyConfig, "record", db, session, grpcHandler) - + // Set web broadcaster if available if webServer != nil { rawProxy.SetWebBroadcaster(webServer) } grpcServer = grpc.NewServer( - grpc.MaxRecvMsgSize(64*1024*1024), // 64MB max receive message size - grpc.MaxSendMsgSize(64*1024*1024), // 64MB max send message size - grpc.MaxHeaderListSize(64*1024*1024), // 64MB max header list size - grpc.InitialWindowSize(64*1024*1024), // 64MB initial window + grpc.MaxRecvMsgSize(64*1024*1024), // 64MB max receive message size + grpc.MaxSendMsgSize(64*1024*1024), // 64MB max send message size + grpc.MaxHeaderListSize(64*1024*1024), // 64MB max header list size + grpc.InitialWindowSize(64*1024*1024), // 64MB initial window grpc.InitialConnWindowSize(64*1024*1024), // 64MB connection window grpc.UnknownServiceHandler(rawProxy.GetUnknownServiceHandler()), ) diff --git a/replay/replay_engine.go b/replay/replay_engine.go new file mode 100644 index 0000000..8bb8827 --- /dev/null +++ b/replay/replay_engine.go @@ -0,0 +1,485 @@ +package replay + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "sort" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "mimic/config" + "mimic/proxy" + "mimic/storage" +) + +// ReplayResult represents the result of replaying a single interaction +type ReplayResult struct { + Interaction *storage.Interaction `json:"interaction"` + Success bool `json:"success"` + ExpectedStatus int `json:"expected_status"` + ActualStatus int `json:"actual_status"` + ExpectedBody []byte `json:"expected_body"` + ActualBody []byte `json:"actual_body"` + ResponseTime time.Duration `json:"response_time"` + Error error `json:"error,omitempty"` + ValidationError string `json:"validation_error,omitempty"` +} + +// ReplaySession represents the overall replay session results +type ReplaySession struct { + SessionName string `json:"session_name"` + TotalRequests int `json:"total_requests"` + SuccessCount int `json:"success_count"` + FailureCount int `json:"failure_count"` + Results []*ReplayResult `json:"results"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + Duration time.Duration `json:"duration"` +} + +// ReplayEngine handles replaying recorded interactions against a target server +type ReplayEngine struct { + config *config.ReplayConfig + database *storage.Database + session *storage.Session + client *http.Client + grpcConn *grpc.ClientConn + results []*ReplayResult + mutex sync.RWMutex +} + +// NewReplayEngine creates a new replay engine +func NewReplayEngine(replayConfig *config.ReplayConfig, db *storage.Database) (*ReplayEngine, error) { + session, err := db.GetSession(replayConfig.SessionName) + if err != nil { + return nil, fmt.Errorf("failed to get session '%s': %w", replayConfig.SessionName, err) + } + + httpClient := &http.Client{ + Timeout: time.Duration(replayConfig.TimeoutSeconds) * time.Second, + } + + var grpcConn *grpc.ClientConn + // Check if we have any gRPC interactions in the session or if protocol is gRPC + hasGRPCInteractions := replayConfig.Protocol == "grpc" + if !hasGRPCInteractions { + // Quick check to see if we have gRPC interactions + interactions, err := db.GetInteractionsBySession(session.ID) + if err == nil { + for _, interaction := range interactions { + if interaction.Protocol == "gRPC" { + hasGRPCInteractions = true + break + } + } + } + } + + if hasGRPCInteractions { + // Register raw codec for gRPC + proxy.RegisterRawCodec() + + var creds credentials.TransportCredentials + if replayConfig.GRPCInsecure { + creds = insecure.NewCredentials() + } else { + // Use TLS credentials for secure connections + creds = credentials.NewTLS(nil) + } + + target := fmt.Sprintf("%s:%d", replayConfig.TargetHost, replayConfig.TargetPort) + + // Create dial options with larger limits to handle big gRPC messages + maxSize := replayConfig.GRPCMaxMessageSize + if maxSize < 64*1024*1024 { + maxSize = 64 * 1024 * 1024 // Minimum 64MB + } + + dialOpts := []grpc.DialOption{ + grpc.WithTransportCredentials(creds), + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(maxSize), + grpc.MaxCallSendMsgSize(maxSize), + ), + // Set very large HTTP/2 window sizes to handle large frames + grpc.WithInitialWindowSize(int32(maxSize)), + grpc.WithInitialConnWindowSize(int32(maxSize)), + // Add buffer sizes for large messages + grpc.WithReadBufferSize(maxSize), + grpc.WithWriteBufferSize(maxSize), + } + + conn, err := grpc.Dial(target, dialOpts...) + if err != nil { + return nil, fmt.Errorf("failed to create gRPC connection: %w", err) + } + grpcConn = conn + } + + return &ReplayEngine{ + config: replayConfig, + database: db, + session: session, + client: httpClient, + grpcConn: grpcConn, + results: make([]*ReplayResult, 0), + }, nil +} + +// Replay replays all interactions from the session against the target server +func (r *ReplayEngine) Replay() (*ReplaySession, error) { + log.Printf("Starting replay of session '%s' against %s://%s:%d", + r.config.SessionName, r.config.Protocol, r.config.TargetHost, r.config.TargetPort) + + interactions, err := r.database.GetInteractionsBySession(r.session.ID) + if err != nil { + return nil, fmt.Errorf("failed to get interactions: %w", err) + } + + if len(interactions) == 0 { + return nil, fmt.Errorf("no interactions found in session '%s'", r.config.SessionName) + } + + // Sort interactions by timestamp to maintain original order + sort.Slice(interactions, func(i, j int) bool { + return interactions[i].Timestamp.Before(interactions[j].Timestamp) + }) + + replaySession := &ReplaySession{ + SessionName: r.config.SessionName, + TotalRequests: len(interactions), + Results: make([]*ReplayResult, 0), + StartTime: time.Now(), + } + + if r.config.MaxConcurrency > 0 { + err = r.replayConcurrent(interactions, replaySession) + } else { + err = r.replaySequential(interactions, replaySession) + } + + replaySession.EndTime = time.Now() + replaySession.Duration = replaySession.EndTime.Sub(replaySession.StartTime) + replaySession.Results = r.results + replaySession.SuccessCount = r.countSuccesses() + replaySession.FailureCount = replaySession.TotalRequests - replaySession.SuccessCount + + // Close any open connections + if r.grpcConn != nil { + if closeErr := r.grpcConn.Close(); closeErr != nil { + log.Printf("Warning: failed to close gRPC connection: %v", closeErr) + } + } + + if err != nil { + return replaySession, err + } + + log.Printf("Replay completed: %d/%d successful, %d failed", + replaySession.SuccessCount, replaySession.TotalRequests, replaySession.FailureCount) + + return replaySession, nil +} + +// replaySequential replays interactions one by one, respecting original timing +func (r *ReplayEngine) replaySequential(interactions []storage.Interaction, replaySession *ReplaySession) error { + var baseTime *time.Time + + for i, interaction := range interactions { + // Calculate delay based on original timestamps + if !r.config.IgnoreTimestamps && baseTime != nil { + delay := interaction.Timestamp.Sub(*baseTime) + if delay > 0 { + log.Printf("Waiting %v before next request (original timing)", delay) + time.Sleep(delay) + } + } else if baseTime == nil { + baseTime = &interaction.Timestamp + } + + result := r.replayInteraction(&interaction) + r.addResult(result) + + if !result.Success && r.config.FailFast { + return fmt.Errorf("replay failed at interaction %d: %s", i+1, result.ValidationError) + } + + // Update base time for next iteration + baseTime = &interaction.Timestamp + } + + return nil +} + +// replayConcurrent replays interactions concurrently with a semaphore for max concurrency +func (r *ReplayEngine) replayConcurrent(interactions []storage.Interaction, replaySession *ReplaySession) error { + semaphore := make(chan struct{}, r.config.MaxConcurrency) + var wg sync.WaitGroup + var firstError error + var errorMutex sync.Mutex + + for i, interaction := range interactions { + wg.Add(1) + go func(idx int, inter storage.Interaction) { + defer wg.Done() + + semaphore <- struct{}{} // Acquire semaphore + defer func() { <-semaphore }() // Release semaphore + + result := r.replayInteraction(&inter) + r.addResult(result) + + if !result.Success && r.config.FailFast { + errorMutex.Lock() + if firstError == nil { + firstError = fmt.Errorf("replay failed at interaction %d: %s", idx+1, result.ValidationError) + } + errorMutex.Unlock() + } + }(i, interaction) + } + + wg.Wait() + + return firstError +} + +// replayInteraction replays a single interaction and validates the response +func (r *ReplayEngine) replayInteraction(interaction *storage.Interaction) *ReplayResult { + result := &ReplayResult{ + Interaction: interaction, + ExpectedStatus: interaction.ResponseStatus, + ExpectedBody: interaction.ResponseBody, + } + + startTime := time.Now() + + // Handle gRPC interactions differently from HTTP + if interaction.Protocol == "gRPC" { + return r.replayGRPCInteraction(interaction, result, startTime) + } else { + return r.replayHTTPInteraction(interaction, result, startTime) + } +} + +// replayHTTPInteraction handles HTTP/HTTPS replay +func (r *ReplayEngine) replayHTTPInteraction(interaction *storage.Interaction, result *ReplayResult, startTime time.Time) *ReplayResult { + // Construct the request URL + url := fmt.Sprintf("%s://%s:%d%s", r.config.Protocol, r.config.TargetHost, r.config.TargetPort, interaction.Endpoint) + + // Create the HTTP request + req, err := http.NewRequest(interaction.Method, url, bytes.NewBuffer(interaction.RequestBody)) + if err != nil { + result.Error = fmt.Errorf("failed to create request: %w", err) + return result + } + + // Add headers from the recorded interaction + if interaction.RequestHeaders != "" { + headers := make(map[string]string) + if err := json.Unmarshal([]byte(interaction.RequestHeaders), &headers); err == nil { + for key, value := range headers { + req.Header.Set(key, value) + } + } + } + + // Execute the request + resp, err := r.client.Do(req) + if err != nil { + result.Error = fmt.Errorf("request failed: %w", err) + result.ResponseTime = time.Since(startTime) + return result + } + defer resp.Body.Close() + + result.ResponseTime = time.Since(startTime) + result.ActualStatus = resp.StatusCode + + // Read response body + var actualBody bytes.Buffer + if _, err := actualBody.ReadFrom(resp.Body); err != nil { + result.Error = fmt.Errorf("failed to read response body: %w", err) + return result + } + result.ActualBody = actualBody.Bytes() + + // Validate the response based on matching strategy + result.Success, result.ValidationError = r.validateResponse(result) + + return result +} + +// replayGRPCInteraction handles gRPC replay +func (r *ReplayEngine) replayGRPCInteraction(interaction *storage.Interaction, result *ReplayResult, startTime time.Time) *ReplayResult { + if r.grpcConn == nil { + result.Error = fmt.Errorf("gRPC connection not available") + result.ResponseTime = time.Since(startTime) + return result + } + + // Parse metadata from the recorded interaction + var metadataMap map[string][]string + ctx := context.Background() + if interaction.RequestHeaders != "" { + if err := json.Unmarshal([]byte(interaction.RequestHeaders), &metadataMap); err == nil { + md := metadata.New(nil) + for key, values := range metadataMap { + md.Set(key, values...) + } + ctx = metadata.NewOutgoingContext(ctx, md) + } + } + + // Add timeout to context + ctx, cancel := context.WithTimeout(ctx, time.Duration(r.config.TimeoutSeconds)*time.Second) + defer cancel() + + // Create request message with recorded raw data + requestMsg := &proxy.RawMessage{ + Data: interaction.RequestBody, + } + + // Create response message + responseMsg := &proxy.RawMessage{} + + // Invoke the gRPC method + err := r.grpcConn.Invoke(ctx, interaction.Method, requestMsg, responseMsg, grpc.ForceCodec(proxy.GetRawCodec())) + + result.ResponseTime = time.Since(startTime) + + if err != nil { + // Handle gRPC errors + if st, ok := status.FromError(err); ok { + result.ActualStatus = int(st.Code()) + } else { + result.ActualStatus = int(codes.Unknown) + } + + // For gRPC, some errors might be expected (like validation errors) + // So we don't always treat errors as failures + if result.ExpectedStatus != 0 && result.ActualStatus == result.ExpectedStatus { + // If we expected this error status, it's not a failure + result.Success, result.ValidationError = r.validateResponse(result) + } else { + result.Error = fmt.Errorf("gRPC call failed: %w", err) + } + return result + } + + // Success response + result.ActualStatus = int(codes.OK) + result.ActualBody = responseMsg.Data + + // Validate the response based on matching strategy + result.Success, result.ValidationError = r.validateResponse(result) + + return result +} + +// validateResponse validates the actual response against the expected response +func (r *ReplayEngine) validateResponse(result *ReplayResult) (bool, string) { + switch r.config.MatchingStrategy { + case "exact": + return r.exactMatch(result) + case "fuzzy": + return r.fuzzyMatch(result) + case "status_code": + return r.statusCodeMatch(result) + default: + return r.exactMatch(result) + } +} + +// exactMatch validates that the response matches exactly +func (r *ReplayEngine) exactMatch(result *ReplayResult) (bool, string) { + if result.ActualStatus != result.ExpectedStatus { + return false, fmt.Sprintf("status mismatch: expected %d, got %d", result.ExpectedStatus, result.ActualStatus) + } + + if !bytes.Equal(result.ActualBody, result.ExpectedBody) { + return false, fmt.Sprintf("body mismatch: expected %d bytes, got %d bytes", len(result.ExpectedBody), len(result.ActualBody)) + } + + return true, "" +} + +// fuzzyMatch validates with some tolerance for differences +func (r *ReplayEngine) fuzzyMatch(result *ReplayResult) (bool, string) { + // For fuzzy matching, we only check status code and basic structure + if result.ActualStatus != result.ExpectedStatus { + return false, fmt.Sprintf("status mismatch: expected %d, got %d", result.ExpectedStatus, result.ActualStatus) + } + + // For JSON responses, compare structure rather than exact content + if r.isJSON(result.ExpectedBody) && r.isJSON(result.ActualBody) { + var expectedJSON, actualJSON interface{} + if json.Unmarshal(result.ExpectedBody, &expectedJSON) == nil && + json.Unmarshal(result.ActualBody, &actualJSON) == nil { + // For fuzzy matching, we just verify both are valid JSON with similar structure + expectedType := fmt.Sprintf("%T", expectedJSON) + actualType := fmt.Sprintf("%T", actualJSON) + if expectedType != actualType { + return false, fmt.Sprintf("JSON structure mismatch: expected %s, got %s", expectedType, actualType) + } + } + } + + return true, "" +} + +// statusCodeMatch only validates the status code +func (r *ReplayEngine) statusCodeMatch(result *ReplayResult) (bool, string) { + if result.ActualStatus != result.ExpectedStatus { + return false, fmt.Sprintf("status mismatch: expected %d, got %d", result.ExpectedStatus, result.ActualStatus) + } + return true, "" +} + +// isJSON checks if the given bytes represent valid JSON +func (r *ReplayEngine) isJSON(data []byte) bool { + var js interface{} + return json.Unmarshal(data, &js) == nil +} + +// addResult adds a result to the engine's results slice (thread-safe) +func (r *ReplayEngine) addResult(result *ReplayResult) { + r.mutex.Lock() + defer r.mutex.Unlock() + r.results = append(r.results, result) +} + +// countSuccesses counts the number of successful replays +func (r *ReplayEngine) countSuccesses() int { + r.mutex.RLock() + defer r.mutex.RUnlock() + + count := 0 + for _, result := range r.results { + if result.Success { + count++ + } + } + return count +} + +// GetResults returns a copy of all replay results +func (r *ReplayEngine) GetResults() []*ReplayResult { + r.mutex.RLock() + defer r.mutex.RUnlock() + + results := make([]*ReplayResult, len(r.results)) + copy(results, r.results) + return results +} diff --git a/server/multi_proxy_server.go b/server/multi_proxy_server.go index 9a2052c..5502e00 100644 --- a/server/multi_proxy_server.go +++ b/server/multi_proxy_server.go @@ -20,8 +20,8 @@ type MultiProxyServer struct { database *storage.Database webServer *web.Server proxies map[string]ProxyHandler - grpcServer *grpc.Server // Single gRPC server with routing - grpcRouter *proxy.GRPCRouter // For gRPC record proxies + grpcServer *grpc.Server // Single gRPC server with routing + grpcRouter *proxy.GRPCRouter // For gRPC record proxies grpcMockRouter *mock.GRPCMockRouter // For gRPC mock proxies } @@ -69,6 +69,13 @@ func NewMultiProxyServer(cfg *config.Config, db *storage.Database) (*MultiProxyS } else { return nil, fmt.Errorf("failed to create mock engine for '%s': %w", name, err) } + case "replay": + // For replay mode, we create a special handler that provides replay endpoints + if replayHandler, err := NewReplayHandler(&cfg.Replay, db, webServer); err == nil { + handler = replayHandler + } else { + return nil, fmt.Errorf("failed to create replay handler for '%s': %w", name, err) + } default: return nil, fmt.Errorf("invalid global mode: %s", cfg.Mode) } @@ -77,8 +84,8 @@ func NewMultiProxyServer(cfg *config.Config, db *storage.Database) (*MultiProxyS log.Printf("Initialized HTTP proxy '%s' in %s mode", name, cfg.Mode) } - // Initialize single gRPC server with routing (if any gRPC proxies exist) - if len(grpcProxies) > 0 { + // Initialize single gRPC server with routing (if any gRPC proxies exist and not in replay mode) + if len(grpcProxies) > 0 && cfg.Mode != "replay" { var unknownServiceHandler grpc.StreamHandler // Create gRPC router for record mode @@ -89,7 +96,7 @@ func NewMultiProxyServer(cfg *config.Config, db *storage.Database) (*MultiProxyS } server.grpcRouter = router unknownServiceHandler = router.GetUnknownServiceHandler() - + } // Create gRPC mock router for mock mode @@ -127,51 +134,51 @@ func (s *MultiProxyServer) Start() error { var grpcAddress string if s.grpcServer != nil { grpcAddress = fmt.Sprintf("%s:%d", s.config.Server.ListenHost, s.config.Server.GRPCPort) - + go func() { lis, err := net.Listen("tcp", grpcAddress) if err != nil { log.Printf("Failed to start gRPC router server on %s: %v", grpcAddress, err) return } - + log.Printf("gRPC router server listening on %s", grpcAddress) - + // Log route information if s.grpcRouter != nil { routes := s.grpcRouter.GetRoutes() for _, route := range routes { if route.Config.ServicePattern != "" { - log.Printf(" → Route '%s': %s -> %s:%d", - route.Name, - route.Config.ServicePattern, - route.Config.TargetHost, + log.Printf(" → Route '%s': %s -> %s:%d", + route.Name, + route.Config.ServicePattern, + route.Config.TargetHost, route.Config.TargetPort) } else { - log.Printf(" → Route '%s': (default) -> %s:%d", - route.Name, - route.Config.TargetHost, + log.Printf(" → Route '%s': (default) -> %s:%d", + route.Name, + route.Config.TargetHost, route.Config.TargetPort) } } } - + if s.grpcMockRouter != nil { routes := s.grpcMockRouter.GetRoutes() for _, route := range routes { if route.Config.ServicePattern != "" { - log.Printf(" → Mock Route '%s': %s -> session '%s'", - route.Name, - route.Config.ServicePattern, + log.Printf(" → Mock Route '%s': %s -> session '%s'", + route.Name, + route.Config.ServicePattern, route.Session.SessionName) } else { - log.Printf(" → Mock Route '%s': (default) -> session '%s'", - route.Name, + log.Printf(" → Mock Route '%s': (default) -> session '%s'", + route.Name, route.Session.SessionName) } } } - + if err := s.grpcServer.Serve(lis); err != nil { log.Printf("gRPC router server failed: %v", err) } @@ -189,7 +196,7 @@ func (s *MultiProxyServer) Start() error { for name, proxyHandler := range s.proxies { proxyPath := fmt.Sprintf("/proxy/%s/", name) - + // Regular HTTP proxy mux.HandleFunc(proxyPath, func(w http.ResponseWriter, r *http.Request) { // Strip the proxy path prefix and forward to the proxy handler @@ -219,14 +226,14 @@ func (s *MultiProxyServer) Start() error { } httpAddress := fmt.Sprintf("%s:%d", s.config.Server.ListenHost, s.config.Server.ListenPort) - + log.Printf("Starting multi-proxy server") log.Printf("Web UI available at http://%s/", httpAddress) - + if httpProxyCount > 0 { log.Printf("HTTP proxies (%d) available at http://%s/proxy//", httpProxyCount, httpAddress) } - + if s.grpcServer != nil { log.Printf("gRPC router server available at %s", grpcAddress) log.Printf("gRPC info available at http://%s/grpc/info", httpAddress) diff --git a/server/replay_handler.go b/server/replay_handler.go new file mode 100644 index 0000000..93e4f89 --- /dev/null +++ b/server/replay_handler.go @@ -0,0 +1,140 @@ +package server + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "strconv" + + "mimic/config" + "mimic/replay" + "mimic/storage" + "mimic/web" +) + +// ReplayHandler handles HTTP requests for replay functionality +type ReplayHandler struct { + config *config.ReplayConfig + database *storage.Database + webServer *web.Server +} + +// NewReplayHandler creates a new replay handler +func NewReplayHandler(replayConfig *config.ReplayConfig, db *storage.Database, webServer *web.Server) (*ReplayHandler, error) { + return &ReplayHandler{ + config: replayConfig, + database: db, + webServer: webServer, + }, nil +} + +// HandleRequest implements the ProxyHandler interface for replay functionality +func (h *ReplayHandler) HandleRequest(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "GET": + h.handleStatus(w, r) + case "POST": + h.handleReplay(w, r) + default: + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } +} + +// handleStatus returns replay configuration and available sessions +func (h *ReplayHandler) handleStatus(w http.ResponseWriter, r *http.Request) { + sessions, err := h.database.ListSessions() + if err != nil { + http.Error(w, fmt.Sprintf("Failed to list sessions: %v", err), http.StatusInternalServerError) + return + } + + status := map[string]interface{}{ + "mode": "replay", + "target_host": h.config.TargetHost, + "target_port": h.config.TargetPort, + "protocol": h.config.Protocol, + "session_name": h.config.SessionName, + "matching_strategy": h.config.MatchingStrategy, + "fail_fast": h.config.FailFast, + "timeout_seconds": h.config.TimeoutSeconds, + "max_concurrency": h.config.MaxConcurrency, + "ignore_timestamps": h.config.IgnoreTimestamps, + "available_sessions": sessions, + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(status) +} + +// handleReplay executes a replay based on request parameters +func (h *ReplayHandler) handleReplay(w http.ResponseWriter, r *http.Request) { + // Parse query parameters to override config + replayConfig := *h.config // Copy the config + + if sessionName := r.URL.Query().Get("session"); sessionName != "" { + replayConfig.SessionName = sessionName + } + + if targetHost := r.URL.Query().Get("target_host"); targetHost != "" { + replayConfig.TargetHost = targetHost + } + + if targetPortStr := r.URL.Query().Get("target_port"); targetPortStr != "" { + if targetPort, err := strconv.Atoi(targetPortStr); err == nil { + replayConfig.TargetPort = targetPort + } + } + + if protocol := r.URL.Query().Get("protocol"); protocol != "" { + replayConfig.Protocol = protocol + } + + if matchingStrategy := r.URL.Query().Get("matching_strategy"); matchingStrategy != "" { + replayConfig.MatchingStrategy = matchingStrategy + } + + if failFastStr := r.URL.Query().Get("fail_fast"); failFastStr != "" { + if failFast, err := strconv.ParseBool(failFastStr); err == nil { + replayConfig.FailFast = failFast + } + } + + if ignoreTimestampsStr := r.URL.Query().Get("ignore_timestamps"); ignoreTimestampsStr != "" { + if ignoreTimestamps, err := strconv.ParseBool(ignoreTimestampsStr); err == nil { + replayConfig.IgnoreTimestamps = ignoreTimestamps + } + } + + // Create replay engine and execute replay + engine, err := replay.NewReplayEngine(&replayConfig, h.database) + if err != nil { + http.Error(w, fmt.Sprintf("Failed to create replay engine: %v", err), http.StatusBadRequest) + return + } + + log.Printf("Starting replay of session '%s' against %s://%s:%d", + replayConfig.SessionName, replayConfig.Protocol, replayConfig.TargetHost, replayConfig.TargetPort) + + // Execute the replay + replaySession, err := engine.Replay() + if err != nil { + // Even if there were failures, we want to return the results + log.Printf("Replay completed with errors: %v", err) + } + + // Broadcast replay results to web clients if available + if h.webServer != nil { + h.webServer.BroadcastEvent("replay_completed", replaySession) + } + + // Return the replay results + w.Header().Set("Content-Type", "application/json") + if replaySession.FailureCount > 0 { + w.WriteHeader(http.StatusExpectationFailed) // 417 to indicate test failures + } + json.NewEncoder(w).Encode(replaySession) + + log.Printf("Replay completed: %d/%d successful, %d failed, duration: %v", + replaySession.SuccessCount, replaySession.TotalRequests, replaySession.FailureCount, replaySession.Duration) +} From 557c8aa0212dfef317a95f1d6e638ce5078fa139 Mon Sep 17 00:00:00 2001 From: Jordan Oroshiba Date: Fri, 18 Jul 2025 21:23:18 -0700 Subject: [PATCH 4/4] add docker and helm chart --- .dockerignore | 37 +++ .gitignore | 3 +- Dockerfile | 83 +++++++ Dockerfile.multiarch | 83 +++++++ KUBERNETES.md | 150 +++++++++++ README-K8S.md | 304 +++++++++++++++++++++++ config.yaml | 2 +- examples/k8s-deployment.yaml | 200 +++++++++++++++ examples/quick-start.sh | 82 ++++++ helm/mimic/.helmignore | 23 ++ helm/mimic/Chart.yaml | 22 ++ helm/mimic/templates/NOTES.txt | 50 ++++ helm/mimic/templates/_helpers.tpl | 127 ++++++++++ helm/mimic/templates/configmap.yaml | 9 + helm/mimic/templates/deployment.yaml | 116 +++++++++ helm/mimic/templates/hpa.yaml | 32 +++ helm/mimic/templates/ingress.yaml | 59 +++++ helm/mimic/templates/pdb.yaml | 18 ++ helm/mimic/templates/podmonitor.yaml | 20 ++ helm/mimic/templates/pvc.yaml | 25 ++ helm/mimic/templates/service.yaml | 23 ++ helm/mimic/templates/serviceaccount.yaml | 13 + helm/mimic/templates/servicemonitor.yaml | 20 ++ helm/mimic/values.yaml | 252 +++++++++++++++++++ nginx.conf | 47 ++++ 25 files changed, 1798 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 Dockerfile.multiarch create mode 100644 KUBERNETES.md create mode 100644 README-K8S.md create mode 100644 examples/k8s-deployment.yaml create mode 100755 examples/quick-start.sh create mode 100644 helm/mimic/.helmignore create mode 100644 helm/mimic/Chart.yaml create mode 100644 helm/mimic/templates/NOTES.txt create mode 100644 helm/mimic/templates/_helpers.tpl create mode 100644 helm/mimic/templates/configmap.yaml create mode 100644 helm/mimic/templates/deployment.yaml create mode 100644 helm/mimic/templates/hpa.yaml create mode 100644 helm/mimic/templates/ingress.yaml create mode 100644 helm/mimic/templates/pdb.yaml create mode 100644 helm/mimic/templates/podmonitor.yaml create mode 100644 helm/mimic/templates/pvc.yaml create mode 100644 helm/mimic/templates/service.yaml create mode 100644 helm/mimic/templates/serviceaccount.yaml create mode 100644 helm/mimic/templates/servicemonitor.yaml create mode 100644 helm/mimic/values.yaml create mode 100644 nginx.conf diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..126a54e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,37 @@ +# Ignore build artifacts +mimic +*.db +*.log + +# Ignore development files +.git +.gitignore +README.md +*.md +justfile + +# Ignore test files +*_test.go +test/ + +# Ignore IDE files +.vscode/ +.idea/ +*.swp +*.swo + +# Ignore OS files +.DS_Store +Thumbs.db + +# Ignore temporary files +tmp/ +temp/ +*.tmp + +# Ignore helm chart (built separately) +helm/ + +# Ignore docker files +Dockerfile* +.dockerignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index dc8cb70..af507ff 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,10 @@ *.dll *.so *.dylib -mimic mimic-* +./mimic + # Test binary, built with `go test -c` *.test diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..42b22ec --- /dev/null +++ b/Dockerfile @@ -0,0 +1,83 @@ +# Multi-architecture Dockerfile for Mimic +# Supports both amd64 and arm64 architectures + +# Build stage +FROM --platform=$BUILDPLATFORM golang:1.21-alpine AS builder + +# Build arguments for cross-compilation +ARG TARGETOS +ARG TARGETARCH +ARG BUILDPLATFORM + +# Install build dependencies including gcc for CGO (required for SQLite) +RUN apk update && apk add --no-cache \ + git \ + ca-certificates \ + tzdata \ + gcc \ + musl-dev \ + sqlite-dev \ + && update-ca-certificates + +# Create appuser for security +RUN adduser -D -g '' appuser + +# Set working directory +WORKDIR /app + +# Copy go mod files first for better caching +COPY go.mod go.sum ./ + +# Download dependencies +RUN go mod download +RUN go mod verify + +# Copy source code +COPY . . + +# Build the binary with optimizations for target architecture +RUN CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build \ + -ldflags='-w -s' \ + -o mimic . + +# Final stage - minimal runtime image +FROM --platform=$TARGETPLATFORM alpine:latest + +# Install runtime dependencies +RUN apk --no-cache add \ + ca-certificates \ + sqlite \ + musl \ + wget + +# Create appuser +RUN addgroup -g 1001 appuser && adduser -u 1001 -G appuser -s /bin/sh -D appuser + +# Create directories +RUN mkdir -p /app/web/static /app/protos /data && \ + chown -R appuser:appuser /app /data + +WORKDIR /app + +# Copy static files for web UI +COPY --from=builder --chown=appuser:appuser /app/web/static ./web/static + +# Copy the binary +COPY --from=builder --chown=appuser:appuser /app/mimic . + +# Copy example configurations +COPY --from=builder --chown=appuser:appuser /app/config*.yaml ./ + +# Use non-root user +USER appuser + +# Expose ports +EXPOSE 8080 9090 + +# Health check +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:8080/ || exit 1 + +# Set entrypoint +ENTRYPOINT ["./mimic"] +CMD ["--config", "/app/config/config.yaml"] diff --git a/Dockerfile.multiarch b/Dockerfile.multiarch new file mode 100644 index 0000000..3138f89 --- /dev/null +++ b/Dockerfile.multiarch @@ -0,0 +1,83 @@ +# Multi-architecture Dockerfile for Mimic +# Supports both amd64 and arm64 architectures + +# Build stage +FROM --platform=$BUILDPLATFORM golang:1.21-alpine AS builder + +# Build arguments for cross-compilation +ARG TARGETOS +ARG TARGETARCH +ARG BUILDPLATFORM + +# Install build dependencies including gcc for CGO (required for SQLite) +RUN apk update && apk add --no-cache \ + git \ + ca-certificates \ + tzdata \ + gcc \ + musl-dev \ + sqlite-dev \ + && update-ca-certificates + +# Create appuser for security +RUN adduser -D -g '' appuser + +# Set working directory +WORKDIR /app + +# Copy go mod files first for better caching +COPY go.mod go.sum ./ + +# Download dependencies +RUN go mod download +RUN go mod verify + +# Copy source code +COPY . . + +# Build the binary with optimizations for target architecture +RUN CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH go build \ + -ldflags='-w -s' \ + -o mimic . + +# Final stage - minimal runtime image +FROM --platform=$TARGETPLATFORM alpine:latest + +# Install runtime dependencies +RUN apk --no-cache add \ + ca-certificates \ + sqlite \ + musl \ + wget + +# Create appuser +RUN addgroup -g 1001 appuser && adduser -u 1001 -G appuser -s /bin/sh -D appuser + +# Create directories +RUN mkdir -p /app/web/static /app/protos /data && \ + chown -R appuser:appuser /app /data + +WORKDIR /app + +# Copy static files for web UI +COPY --from=builder --chown=appuser:appuser /app/web/static ./web/static + +# Copy the binary +COPY --from=builder --chown=appuser:appuser /app/mimic . + +# Copy example configurations +COPY --from=builder --chown=appuser:appuser /app/config*.yaml ./ + +# Use non-root user +USER appuser + +# Expose ports +EXPOSE 8080 9090 + +# Health check +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD wget --no-verbose --tries=1 --spider http://localhost:8080/ || exit 1 + +# Set entrypoint +ENTRYPOINT ["./mimic"] +CMD ["--config", "/app/config/config.yaml"] \ No newline at end of file diff --git a/KUBERNETES.md b/KUBERNETES.md new file mode 100644 index 0000000..6f49974 --- /dev/null +++ b/KUBERNETES.md @@ -0,0 +1,150 @@ +# Kubernetes Deployment for Mimic + +This document provides everything needed to deploy Mimic in a Kubernetes cluster using Docker and Helm. + +## 📁 Files Created + +### Docker + +- `Dockerfile` - Multi-architecture build supporting amd64 and arm64 +- `.dockerignore` - Excludes unnecessary files from build context + +### Helm Chart (`helm/mimic/`) + +- `Chart.yaml` - Chart metadata +- `values.yaml` - Default configuration values +- `templates/` - Kubernetes resource templates + - `deployment.yaml` - Main application deployment + - `service.yaml` - Service for HTTP and gRPC endpoints + - `configmap.yaml` - Configuration file + - `serviceaccount.yaml` - Service account + - `pvc.yaml` - Persistent volume claim for database + - `ingress.yaml` - Ingress configuration + - `hpa.yaml` - Horizontal Pod Autoscaler + - `pdb.yaml` - Pod Disruption Budget + - `servicemonitor.yaml` - Prometheus ServiceMonitor + - `podmonitor.yaml` - Prometheus PodMonitor + - `_helpers.tpl` - Template helpers + +### Examples +- `examples/k8s-deployment.yaml` - Example configurations for dev/test/prod +- `examples/quick-start.sh` - Quick start script +- `README-K8S.md` - Detailed deployment guide + +## 🚀 Quick Start + +1. **Build and deploy:** + ```bash + ./examples/quick-start.sh + ``` + +2. **Access the application:** + ```bash + # Web UI + http://localhost:8080 + + # Test proxy + curl http://localhost:8080/proxy/httpbin/get + ``` + +## 🔧 Configuration Highlights + +### Multi-Environment Support +- **Development**: Recording mode with minimal resources +- **Testing**: Mock mode with ingress for integration tests +- **Production**: High availability with autoscaling and monitoring + +### Security Features +- Non-root container execution +- Read-only root filesystem option +- Security contexts and pod security standards +- RBAC with minimal permissions + +### Scalability +- Horizontal Pod Autoscaler support +- Pod Disruption Budgets for availability +- Resource limits and requests +- Anti-affinity rules for distribution + +### Persistence +- SQLite database stored in persistent volumes +- Configurable storage classes and sizes +- Backup-friendly volume configurations + +### Monitoring +- Prometheus ServiceMonitor and PodMonitor +- Health checks (liveness, readiness, startup) +- Structured logging + +## 📋 Key Configuration Options + +| Feature | Values Path | Description | +|---------|-------------|-------------| +| Mode | `config.mode` | record/mock/replay | +| Proxies | `config.proxies` | Target service configurations | +| Persistence | `persistence.enabled` | Enable database persistence | +| Scaling | `autoscaling.enabled` | Enable horizontal scaling | +| Monitoring | `serviceMonitor.enabled` | Enable Prometheus monitoring | +| Security | `podSecurityContext` | Pod security settings | + +## 🏗️ Architecture + +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ Applications │────│ Mimic │────│ Target Services │ +│ │ │ │ │ │ +│ - HTTP clients │ │ - Record/Mock │ │ - APIs │ +│ - gRPC clients │ │ - Web UI │ │ - gRPC services │ +│ │ │ - Database │ │ │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ + ▼ + ┌─────────────────┐ + │ Persistent │ + │ Volume │ + │ (SQLite DB) │ + └─────────────────┘ +``` + +## 🔍 Use Cases + +### 1. Development Environment +```bash +helm install mimic-dev ./helm/mimic \ + --set config.mode=record \ + --set config.proxies.api.target_host=api.example.com +``` + +### 2. Testing Environment +```bash +helm install mimic-test ./helm/mimic \ + --set config.mode=mock \ + --set ingress.enabled=true +``` + +### 3. Production Mock Server +```bash +helm install mimic-prod ./helm/mimic \ + -f examples/values-prod.yaml +``` + +## ✅ Validation + +The Helm chart has been validated and passes linting: +```bash +$ helm lint helm/mimic +==> Linting mimic +[INFO] Chart.yaml: icon is recommended + +1 chart(s) linted, 0 chart(s) failed +``` + +## 📚 Next Steps + +1. **Customize** the values.yaml for your environment +2. **Build** the Docker image and push to your registry +3. **Deploy** using Helm with your custom configuration +4. **Configure** your applications to use Mimic as a proxy +5. **Monitor** using the web UI and Prometheus metrics + +For detailed instructions, see `README-K8S.md`. diff --git a/README-K8S.md b/README-K8S.md new file mode 100644 index 0000000..ca1d2d4 --- /dev/null +++ b/README-K8S.md @@ -0,0 +1,304 @@ +# Kubernetes Deployment Guide for Mimic + +This guide covers deploying Mimic in Kubernetes using Docker and Helm. + +## Prerequisites + +- Kubernetes cluster (1.19+) +- Helm 3.x +- Docker (for building images) +- kubectl configured for your cluster + +## Building the Docker Image + +1. **Build the image locally:** + ```bash + ./scripts/build-local.sh + ``` + + Or manually: + ```bash + docker build -t mimic:latest . + ``` + +2. **For production, build multi-architecture and push to your registry:** + ```bash + # Multi-architecture build (requires Docker buildx) + REGISTRY=your-registry.com TAG=v1.0.0 ./scripts/build-multiarch.sh + + # Or single architecture + docker tag mimic:latest your-registry.com/mimic:v1.0.0 + docker push your-registry.com/mimic:v1.0.0 + ``` + +## Quick Start with Helm + +1. **Install Mimic with default values:** + ```bash + helm install mimic ./helm/mimic + ``` + +2. **Install with custom values:** + ```bash + helm install mimic ./helm/mimic -f custom-values.yaml + ``` + +3. **Access the application:** + ```bash + kubectl port-forward svc/mimic 8080:8080 + # Open http://localhost:8080 in your browser + ``` + +## Configuration Examples + +### Basic Recording Setup + +```yaml +# values.yaml +config: + mode: "record" + proxies: + api-service: + protocol: "https" + target_host: "api.example.com" + target_port: 443 + session_name: "api-recording" + + grpc-service: + protocol: "grpc" + target_host: "grpc.example.com" + target_port: 443 + session_name: "grpc-recording" + service_pattern: "example\\..*" + +# Install +helm install mimic ./helm/mimic -f values.yaml +``` + +### Mock Server Setup + +```yaml +# values.yaml +config: + mode: "mock" + mock: + matching_strategy: "exact" + sequence_mode: "ordered" + proxies: + api-service: + protocol: "https" + session_name: "api-recordings" + +ingress: + enabled: true + hosts: + - host: mimic.example.com + paths: + - path: / + pathType: Prefix + +# Install +helm install mimic ./helm/mimic -f values.yaml +``` + +### Production Setup with Persistence + +```yaml +# values.yaml +replicaCount: 2 + +resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 200m + memory: 256Mi + +persistence: + enabled: true + storageClass: "gp2" + size: 5Gi + +podDisruptionBudget: + enabled: true + minAvailable: 1 + +autoscaling: + enabled: true + minReplicas: 2 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + +service: + type: LoadBalancer + +# Install +helm install mimic ./helm/mimic -f values.yaml +``` + +## Deployment Scenarios + +### 1. API Testing in Development + +Deploy Mimic as a recording proxy for capturing API interactions: + +```yaml +config: + mode: "record" + proxies: + external-api: + protocol: "https" + target_host: "external-api.com" + target_port: 443 + session_name: "dev-testing" + +# Your application should point to: +# http://mimic.default.svc.cluster.local:8080/proxy/external-api/ +``` + +### 2. Integration Testing + +Use recorded sessions for integration tests: + +```yaml +config: + mode: "mock" + proxies: + test-api: + session_name: "integration-tests" + +# Tests can hit: http://mimic.test.svc.cluster.local:8080/proxy/test-api/ +``` + +### 3. gRPC Service Mocking + +Mock gRPC services for testing: + +```yaml +config: + mode: "mock" + proxies: + user-service: + protocol: "grpc" + session_name: "user-service-mocks" + service_pattern: "user\\..*" + +# gRPC clients should connect to: mimic.default.svc.cluster.local:9090 +``` + +## Configuration Reference + +### Core Settings + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `config.mode` | Operation mode (record/mock/replay) | `"record"` | +| `config.server.listen_port` | HTTP server port | `8080` | +| `config.server.grpc_port` | gRPC server port | `9090` | + +### Persistence + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `persistence.enabled` | Enable persistent storage | `true` | +| `persistence.size` | Storage size | `"1Gi"` | +| `persistence.storageClass` | Storage class | `""` | + +### Security + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `podSecurityContext.runAsUser` | User ID | `1001` | +| `podSecurityContext.runAsNonRoot` | Run as non-root | `true` | +| `securityContext.readOnlyRootFilesystem` | Read-only root filesystem | `false` | + +## Monitoring + +### Prometheus Integration + +Enable ServiceMonitor for Prometheus Operator: + +```yaml +serviceMonitor: + enabled: true + interval: 30s + labels: + prometheus: default +``` + +### Health Checks + +The deployment includes: +- **Liveness Probe**: Checks if the application is running +- **Readiness Probe**: Checks if the application is ready to serve traffic +- **Startup Probe**: Handles application startup + +## Troubleshooting + +### Check Pod Status +```bash +kubectl get pods -l app.kubernetes.io/name=mimic +kubectl describe pod +``` + +### View Logs +```bash +kubectl logs -f deployment/mimic +``` + +### Debug Configuration +```bash +kubectl get configmap mimic-config -o yaml +``` + +### Test Connectivity +```bash +# Test HTTP endpoint +kubectl run debug --image=curlimages/curl -it --rm -- curl http://mimic:8080/ + +# Test gRPC endpoint (if grpcurl is available) +kubectl run debug --image=fullstorydev/grpcurl -it --rm -- grpcurl -plaintext mimic:9090 list +``` + +### Common Issues + +1. **Database permission issues**: Ensure persistence is enabled and storage class exists +2. **Image pull errors**: Verify image name and registry credentials +3. **Service connectivity**: Check service and ingress configuration +4. **Config parsing errors**: Validate YAML syntax in values.yaml + +## Upgrading + +```bash +# Upgrade to new version +helm upgrade mimic ./helm/mimic --set image.tag=v1.1.0 + +# Upgrade with new configuration +helm upgrade mimic ./helm/mimic -f new-values.yaml +``` + +## Uninstalling + +```bash +# Remove the deployment (keeps PVC by default) +helm uninstall mimic + +# Remove PVC as well (this will delete all recorded data) +kubectl delete pvc mimic-data +``` + +## Security Considerations + +1. **Network Policies**: Implement network policies to restrict traffic +2. **RBAC**: The service account has minimal permissions by default +3. **Secrets**: Use Kubernetes secrets for sensitive configuration +4. **TLS**: Enable TLS for production deployments via ingress +5. **Image Security**: Scan images for vulnerabilities before deployment + +## Performance Tuning + +1. **Resource Limits**: Set appropriate CPU/memory limits based on traffic +2. **Replica Count**: Scale horizontally for high availability +3. **Storage**: Use SSD storage classes for better database performance +4. **Autoscaling**: Enable HPA based on CPU/memory metrics \ No newline at end of file diff --git a/config.yaml b/config.yaml index bb6d4f5..5657f96 100644 --- a/config.yaml +++ b/config.yaml @@ -16,7 +16,7 @@ proxies: protocol: "grpc" target_host: "grpc.astria.org" target_port: 443 - session_name: "test-replay" + session_name: "test-replay-2" service_pattern: "astria\\..*" # test-grpc: diff --git a/examples/k8s-deployment.yaml b/examples/k8s-deployment.yaml new file mode 100644 index 0000000..2fa4336 --- /dev/null +++ b/examples/k8s-deployment.yaml @@ -0,0 +1,200 @@ +# Example Kubernetes deployment configurations for different use cases + +--- +# Development environment - Recording mode +apiVersion: v1 +kind: Namespace +metadata: + name: mimic-dev +--- +# values-dev.yaml for: helm install mimic-dev ./helm/mimic -f examples/values-dev.yaml -n mimic-dev +# Development recording setup +config: + mode: "record" + server: + listen_host: "0.0.0.0" + listen_port: 8080 + grpc_port: 9090 + proxies: + external-api: + protocol: "https" + target_host: "jsonplaceholder.typicode.com" + target_port: 443 + session_name: "dev-api-recording" + grpc-demo: + protocol: "grpc" + target_host: "grpc-demo.example.com" + target_port: 443 + session_name: "grpc-demo-recording" + service_pattern: "demo\\..*" + +persistence: + enabled: true + size: "500Mi" + +resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 50m + memory: 64Mi + +--- +# Testing environment - Mock mode +apiVersion: v1 +kind: Namespace +metadata: + name: mimic-test +--- +# values-test.yaml for: helm install mimic-test ./helm/mimic -f examples/values-test.yaml -n mimic-test +# Testing mock setup +config: + mode: "mock" + mock: + matching_strategy: "exact" + sequence_mode: "ordered" + proxies: + user-service: + session_name: "user-service-mocks" + payment-service: + session_name: "payment-service-mocks" + notification-grpc: + protocol: "grpc" + session_name: "notification-mocks" + service_pattern: "notification\\..*" + +service: + type: ClusterIP + +ingress: + enabled: true + className: "nginx" + hosts: + - host: mimic-test.local + paths: + - path: / + pathType: Prefix + +persistence: + enabled: true + size: "1Gi" + +--- +# Production environment - High availability +apiVersion: v1 +kind: Namespace +metadata: + name: mimic-prod +--- +# values-prod.yaml for: helm install mimic-prod ./helm/mimic -f examples/values-prod.yaml -n mimic-prod +# Production setup +replicaCount: 3 + +config: + mode: "mock" + mock: + matching_strategy: "exact" + sequence_mode: "random" + recording: + redact_patterns: + - "Authorization: Bearer .*" + - "X-Api-Key: .*" + - "X-Auth-Token: .*" + proxies: + api-gateway: + session_name: "production-api-mocks" + user-grpc: + protocol: "grpc" + session_name: "user-service-prod" + service_pattern: "user\\.v1\\..*" + order-grpc: + protocol: "grpc" + session_name: "order-service-prod" + service_pattern: "order\\.v1\\..*" + +image: + tag: "v1.0.0" + pullPolicy: Always + +resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 200m + memory: 256Mi + +podSecurityContext: + fsGroup: 1001 + runAsUser: 1001 + runAsNonRoot: true + +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1001 + capabilities: + drop: + - ALL + +persistence: + enabled: true + storageClass: "gp2" + size: "10Gi" + +service: + type: LoadBalancer + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: nlb + +ingress: + enabled: true + className: "alb" + annotations: + kubernetes.io/ingress.class: alb + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip + hosts: + - host: mimic.yourdomain.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: mimic-tls + hosts: + - mimic.yourdomain.com + +podDisruptionBudget: + enabled: true + minAvailable: 2 + +autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 10 + targetCPUUtilizationPercentage: 70 + targetMemoryUtilizationPercentage: 80 + +nodeSelector: + kubernetes.io/arch: amd64 + +tolerations: + - key: "mimic-nodes" + operator: "Equal" + value: "true" + effect: "NoSchedule" + +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - mimic + topologyKey: kubernetes.io/hostname \ No newline at end of file diff --git a/examples/quick-start.sh b/examples/quick-start.sh new file mode 100755 index 0000000..3f92de6 --- /dev/null +++ b/examples/quick-start.sh @@ -0,0 +1,82 @@ +#!/bin/bash +set -e + +# Mimic Kubernetes Quick Start Script +echo "🚀 Mimic Kubernetes Quick Start" + +# Check prerequisites +echo "📋 Checking prerequisites..." + +if ! command -v kubectl &> /dev/null; then + echo "❌ kubectl is required but not installed" + exit 1 +fi + +if ! command -v helm &> /dev/null; then + echo "❌ helm is required but not installed" + exit 1 +fi + +if ! command -v docker &> /dev/null; then + echo "❌ docker is required but not installed" + exit 1 +fi + +# Check if kubectl is configured +if ! kubectl cluster-info &> /dev/null; then + echo "❌ kubectl is not configured or cluster is not accessible" + exit 1 +fi + +echo "✅ All prerequisites met" + +# Build Docker image +echo "🔨 Building Mimic Docker image..." +./scripts/build-local.sh || { + echo "❌ Failed to build Docker image" + exit 1 +} + +# If using kind, load image into cluster +if kubectl config current-context | grep -q kind; then + echo "📦 Loading image into kind cluster..." + kind load docker-image mimic:latest +fi + +# Create namespace +echo "🏠 Creating mimic-demo namespace..." +kubectl create namespace mimic-demo --dry-run=client -o yaml | kubectl apply -f - + +# Deploy with Helm +echo "🚢 Deploying Mimic with Helm..." +helm upgrade --install mimic-demo ./helm/mimic \ + --namespace mimic-demo \ + --set image.tag=latest \ + --set image.pullPolicy=Never \ + --set config.mode=record \ + --set config.proxies.httpbin.protocol=https \ + --set config.proxies.httpbin.target_host=httpbin.org \ + --set config.proxies.httpbin.target_port=443 \ + --set config.proxies.httpbin.session_name=httpbin-demo \ + --wait + +# Wait for deployment +echo "⏳ Waiting for deployment to be ready..." +kubectl wait --for=condition=available --timeout=300s deployment/mimic-demo -n mimic-demo + +# Get status +echo "📊 Deployment status:" +kubectl get pods,svc,ingress -n mimic-demo + +# Port forward for access +echo "🌐 Setting up port forwarding..." +echo "Access Mimic Web UI at: http://localhost:8080" +echo "HTTP Proxy endpoint: http://localhost:8080/proxy/httpbin/" +echo "gRPC endpoint: localhost:9090" +echo "" +echo "To test the proxy:" +echo " curl http://localhost:8080/proxy/httpbin/get" +echo "" +echo "To stop port forwarding, press Ctrl+C" + +kubectl port-forward svc/mimic-demo 8080:8080 9090:9090 -n mimic-demo \ No newline at end of file diff --git a/helm/mimic/.helmignore b/helm/mimic/.helmignore new file mode 100644 index 0000000..691fa13 --- /dev/null +++ b/helm/mimic/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ \ No newline at end of file diff --git a/helm/mimic/Chart.yaml b/helm/mimic/Chart.yaml new file mode 100644 index 0000000..63d2b8d --- /dev/null +++ b/helm/mimic/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +name: mimic +description: A Helm chart for Mimic - API Record and Replay Tool +type: application +version: 0.1.0 +appVersion: "1.0.0" +keywords: + - proxy + - api + - testing + - mock + - recording + - grpc + - http +home: https://github.com/your-org/mimic +sources: + - https://github.com/your-org/mimic +maintainers: + - name: Mimic Team + email: team@mimic.dev +annotations: + category: Testing \ No newline at end of file diff --git a/helm/mimic/templates/NOTES.txt b/helm/mimic/templates/NOTES.txt new file mode 100644 index 0000000..3b6efde --- /dev/null +++ b/helm/mimic/templates/NOTES.txt @@ -0,0 +1,50 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "mimic.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "mimic.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "mimic.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.http.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "mimic.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} + +2. Access the Web UI: + The Mimic web interface will be available at the above URL for monitoring live requests and managing sessions. + +3. Configure your applications to use Mimic as a proxy: + - HTTP/HTTPS traffic: Point to http://{{ include "mimic.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.http.port }} + - gRPC traffic: Point to {{ include "mimic.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.grpc.port }} + +4. Current mode: {{ .Values.config.mode }} + {{- if eq .Values.config.mode "record" }} + - Mimic is in RECORD mode: It will proxy requests to target services and record interactions. + {{- else if eq .Values.config.mode "mock" }} + - Mimic is in MOCK mode: It will serve recorded responses without forwarding to target services. + {{- else if eq .Values.config.mode "replay" }} + - Mimic is in REPLAY mode: It will replay recorded interactions against target services for testing. + {{- end }} + +5. To change the mode or configuration: + helm upgrade {{ .Release.Name }} {{ .Chart.Name }} --set config.mode=mock + +6. To view logs: + kubectl logs -f deployment/{{ include "mimic.fullname" . }} -n {{ .Release.Namespace }} + +{{- if .Values.persistence.enabled }} +7. Persistence is enabled. Database will be stored in a persistent volume. +{{- else }} +8. WARNING: Persistence is disabled. Database will be lost when pods restart. +{{- end }} \ No newline at end of file diff --git a/helm/mimic/templates/_helpers.tpl b/helm/mimic/templates/_helpers.tpl new file mode 100644 index 0000000..d400ac1 --- /dev/null +++ b/helm/mimic/templates/_helpers.tpl @@ -0,0 +1,127 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "mimic.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "mimic.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "mimic.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "mimic.labels" -}} +helm.sh/chart: {{ include "mimic.chart" . }} +{{ include "mimic.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "mimic.selectorLabels" -}} +app.kubernetes.io/name: {{ include "mimic.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "mimic.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "mimic.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the image name +*/}} +{{- define "mimic.image" -}} +{{- if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) }} +{{- else }} +{{- printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) }} +{{- end }} +{{- end }} + +{{/* +Generate config file content +*/}} +{{- define "mimic.config" -}} +mode: {{ .Values.config.mode | quote }} + +server: + listen_host: {{ .Values.config.server.listen_host | quote }} + listen_port: {{ .Values.config.server.listen_port }} + grpc_port: {{ .Values.config.server.grpc_port }} + +database: + path: {{ .Values.config.database.path | quote }} + connection_pool_size: {{ .Values.config.database.connection_pool_size }} + +recording: + session_name: {{ .Values.config.recording.session_name | quote }} + capture_headers: {{ .Values.config.recording.capture_headers }} + capture_body: {{ .Values.config.recording.capture_body }} + {{- if .Values.config.recording.redact_patterns }} + redact_patterns: + {{- range .Values.config.recording.redact_patterns }} + - {{ . | quote }} + {{- end }} + {{- end }} + +mock: + matching_strategy: {{ .Values.config.mock.matching_strategy | quote }} + sequence_mode: {{ .Values.config.mock.sequence_mode | quote }} + not_found_response: + status: {{ .Values.config.mock.not_found_response.status }} + body: + {{- toYaml .Values.config.mock.not_found_response.body | nindent 6 }} + +grpc: + {{- if .Values.config.grpc.proto_paths }} + proto_paths: + {{- range .Values.config.grpc.proto_paths }} + - {{ . | quote }} + {{- end }} + {{- end }} + reflection_enabled: {{ .Values.config.grpc.reflection_enabled }} + +export: + format: {{ .Values.config.export.format | quote }} + pretty_print: {{ .Values.config.export.pretty_print }} + compress: {{ .Values.config.export.compress }} + +{{- if .Values.config.proxies }} +proxies: + {{- toYaml .Values.config.proxies | nindent 2 }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/mimic/templates/configmap.yaml b/helm/mimic/templates/configmap.yaml new file mode 100644 index 0000000..23d0c28 --- /dev/null +++ b/helm/mimic/templates/configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "mimic.fullname" . }}-config + labels: + {{- include "mimic.labels" . | nindent 4 }} +data: + config.yaml: | + {{- include "mimic.config" . | nindent 4 }} \ No newline at end of file diff --git a/helm/mimic/templates/deployment.yaml b/helm/mimic/templates/deployment.yaml new file mode 100644 index 0000000..77ed23a --- /dev/null +++ b/helm/mimic/templates/deployment.yaml @@ -0,0 +1,116 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mimic.fullname" . }} + labels: + {{- include "mimic.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "mimic.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "mimic.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "mimic.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: {{ include "mimic.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - ./mimic + args: + - --config + - /app/config/config.yaml + ports: + - name: http + containerPort: {{ .Values.config.server.listen_port }} + protocol: TCP + - name: grpc + containerPort: {{ .Values.config.server.grpc_port }} + protocol: TCP + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + {{- omit .Values.livenessProbe "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + {{- omit .Values.readinessProbe "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- if .Values.startupProbe.enabled }} + startupProbe: + {{- omit .Values.startupProbe "enabled" | toYaml | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + env: + {{- range .Values.env }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} + {{- with .Values.envFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: config + mountPath: /app/config + readOnly: true + {{- if .Values.persistence.enabled }} + - name: data + mountPath: /data + {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ include "mimic.fullname" . }}-config + {{- if .Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ include "mimic.fullname" . }}-data + {{- end }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/helm/mimic/templates/hpa.yaml b/helm/mimic/templates/hpa.yaml new file mode 100644 index 0000000..0b23ac1 --- /dev/null +++ b/helm/mimic/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "mimic.fullname" . }} + labels: + {{- include "mimic.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "mimic.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/mimic/templates/ingress.yaml b/helm/mimic/templates/ingress.yaml new file mode 100644 index 0000000..8b2bea5 --- /dev/null +++ b/helm/mimic/templates/ingress.yaml @@ -0,0 +1,59 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "mimic.fullname" . -}} +{{- $svcPort := .Values.service.http.port -}} +{{- if and .Values.ingress.className (not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class")) }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "mimic.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/mimic/templates/pdb.yaml b/helm/mimic/templates/pdb.yaml new file mode 100644 index 0000000..9eac79e --- /dev/null +++ b/helm/mimic/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "mimic.fullname" . }} + labels: + {{- include "mimic.labels" . | nindent 4 }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + {{- include "mimic.selectorLabels" . | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/helm/mimic/templates/podmonitor.yaml b/helm/mimic/templates/podmonitor.yaml new file mode 100644 index 0000000..df8e0e3 --- /dev/null +++ b/helm/mimic/templates/podmonitor.yaml @@ -0,0 +1,20 @@ +{{- if .Values.podMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "mimic.fullname" . }} + labels: + {{- include "mimic.labels" . | nindent 4 }} + {{- with .Values.podMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "mimic.selectorLabels" . | nindent 6 }} + podMetricsEndpoints: + - port: http + interval: {{ .Values.podMonitor.interval }} + scrapeTimeout: {{ .Values.podMonitor.scrapeTimeout }} + path: /metrics +{{- end }} \ No newline at end of file diff --git a/helm/mimic/templates/pvc.yaml b/helm/mimic/templates/pvc.yaml new file mode 100644 index 0000000..301407a --- /dev/null +++ b/helm/mimic/templates/pvc.yaml @@ -0,0 +1,25 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "mimic.fullname" . }}-data + labels: + {{- include "mimic.labels" . | nindent 4 }} + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClass }} + {{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: {{ .Values.persistence.storageClass }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/mimic/templates/service.yaml b/helm/mimic/templates/service.yaml new file mode 100644 index 0000000..733f171 --- /dev/null +++ b/helm/mimic/templates/service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimic.fullname" . }} + labels: + {{- include "mimic.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.http.port }} + targetPort: {{ .Values.service.http.targetPort }} + protocol: TCP + name: {{ .Values.service.http.name }} + - port: {{ .Values.service.grpc.port }} + targetPort: {{ .Values.service.grpc.targetPort }} + protocol: TCP + name: {{ .Values.service.grpc.name }} + selector: + {{- include "mimic.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/helm/mimic/templates/serviceaccount.yaml b/helm/mimic/templates/serviceaccount.yaml new file mode 100644 index 0000000..32493b7 --- /dev/null +++ b/helm/mimic/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "mimic.serviceAccountName" . }} + labels: + {{- include "mimic.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: false +{{- end }} \ No newline at end of file diff --git a/helm/mimic/templates/servicemonitor.yaml b/helm/mimic/templates/servicemonitor.yaml new file mode 100644 index 0000000..cb03c9d --- /dev/null +++ b/helm/mimic/templates/servicemonitor.yaml @@ -0,0 +1,20 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "mimic.fullname" . }} + labels: + {{- include "mimic.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "mimic.selectorLabels" . | nindent 6 }} + endpoints: + - port: http + interval: {{ .Values.serviceMonitor.interval }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} + path: /metrics +{{- end }} \ No newline at end of file diff --git a/helm/mimic/values.yaml b/helm/mimic/values.yaml new file mode 100644 index 0000000..9181205 --- /dev/null +++ b/helm/mimic/values.yaml @@ -0,0 +1,252 @@ +# Default values for mimic +# This is a YAML-formatted file. + +# Global settings +global: + imageRegistry: "" + imagePullSecrets: [] + +# Image configuration +image: + repository: mimic + tag: "latest" + pullPolicy: IfNotPresent + +# Service account +serviceAccount: + create: true + name: "" + annotations: {} + +# Pod configuration +replicaCount: 1 + +# Pod disruption budget +podDisruptionBudget: + enabled: false + minAvailable: 1 + +# Pod security context +podSecurityContext: + fsGroup: 1001 + runAsUser: 1001 + runAsGroup: 1001 + runAsNonRoot: true + +# Container security context +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 1001 + capabilities: + drop: + - ALL + +# Resource limits +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 128Mi + +# Node selection +nodeSelector: {} +tolerations: [] +affinity: {} + +# Pod annotations and labels +podAnnotations: {} +podLabels: {} + +# Service configuration +service: + type: ClusterIP + http: + port: 8080 + targetPort: 8080 + name: http + grpc: + port: 9090 + targetPort: 9090 + name: grpc + annotations: {} + +# Ingress configuration +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: mimic.local + paths: + - path: / + pathType: Prefix + tls: [] + # - secretName: mimic-tls + # hosts: + # - mimic.local + +# Persistence for SQLite database +persistence: + enabled: true + storageClass: "" + accessMode: ReadWriteOnce + size: 1Gi + annotations: {} + +# Mimic application configuration +config: + # Global mode: record | mock | replay + mode: "record" + + # Server configuration + server: + listen_host: "0.0.0.0" + listen_port: 8080 + grpc_port: 9090 + + # Database configuration + database: + path: "/data/recordings.db" + connection_pool_size: 10 + + # Recording configuration + recording: + session_name: "default" + capture_headers: true + capture_body: true + redact_patterns: + - "Authorization: Bearer .*" + - "X-Api-Key: .*" + + # Mock configuration + mock: + matching_strategy: "exact" + sequence_mode: "ordered" + not_found_response: + status: 404 + body: + error: "Recording not found" + + # gRPC configuration + grpc: + proto_paths: + - "/app/protos" + reflection_enabled: true + + # Export configuration + export: + format: "json" + pretty_print: true + compress: false + + # Proxy configurations + proxies: {} + # Example proxy configuration: + # api-example: + # mode: "record" + # protocol: "https" + # target_host: "api.example.com" + # target_port: 443 + # session_name: "api-example-session" + # + # grpc-example: + # mode: "record" + # protocol: "grpc" + # target_host: "grpc.example.com" + # target_port: 443 + # session_name: "grpc-example-session" + # service_pattern: "example\\..*" + +# Environment variables +env: [] + # - name: MIMIC_LOG_LEVEL + # value: "debug" + +# Extra environment variables from secrets or configmaps +envFrom: [] + # - secretRef: + # name: mimic-secrets + # - configMapRef: + # name: mimic-config + +# Liveness probe +livenessProbe: + enabled: true + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + +# Readiness probe +readinessProbe: + enabled: true + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + successThreshold: 1 + +# Startup probe +startupProbe: + enabled: true + httpGet: + path: / + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 30 + successThreshold: 1 + +# Horizontal Pod Autoscaler +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 + +# Pod monitor for Prometheus (if prometheus-operator is installed) +podMonitor: + enabled: false + interval: 30s + scrapeTimeout: 10s + labels: {} + +# Service monitor for Prometheus (if prometheus-operator is installed) +serviceMonitor: + enabled: false + interval: 30s + scrapeTimeout: 10s + labels: {} + +# Extra volumes +extraVolumes: [] + # - name: proto-files + # configMap: + # name: proto-files + +# Extra volume mounts +extraVolumeMounts: [] + # - name: proto-files + # mountPath: /app/protos + # readOnly: true + +# Init containers +initContainers: [] + +# Sidecar containers +sidecars: [] \ No newline at end of file diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..74ced2e --- /dev/null +++ b/nginx.conf @@ -0,0 +1,47 @@ +events { + worker_connections 1024; +} + +http { + upstream mimic { + server mimic:8080; + } + + upstream mimic-grpc { + server mimic:9090; + } + + # HTTP proxy + server { + listen 80; + server_name localhost; + + # Web UI and HTTP API + location / { + proxy_pass http://mimic; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + } + + # gRPC proxy (if needed) + server { + listen 81 http2; + server_name localhost; + + location / { + grpc_pass grpc://mimic-grpc; + grpc_set_header Host $host; + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + grpc_set_header X-Forwarded-Proto $scheme; + } + } +} \ No newline at end of file