From 85fdf837f54e1afc2b3f3592886d77de3c1f195a Mon Sep 17 00:00:00 2001 From: Umang Goyal Date: Wed, 7 May 2025 12:07:31 +0530 Subject: [PATCH 1/6] WIP: Add traceID to backend --- internal/service/client.go | 19 ++++--------------- internal/service/environment.go | 7 +++++++ internal/service/service.go | 4 ++-- .../od/environment/v1/environment.proto | 7 +++++++ 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/internal/service/client.go b/internal/service/client.go index ab68fc2b..cd8fa4e5 100644 --- a/internal/service/client.go +++ b/internal/service/client.go @@ -8,7 +8,6 @@ import ( "time" "github.com/dream11/odin/pkg/config" - "github.com/dream11/odin/pkg/constant" "github.com/dream11/odin/pkg/util" log "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -21,19 +20,10 @@ import ( func grpcClient(ctx *context.Context) (*grpc.ClientConn, *context.Context, error) { appConfig := config.GetConfig() - var traceID string - var contextWithTrace context.Context - if (*ctx).Value(constant.TraceIDKey) == nil { - traceID = util.GenerateTraceID() - contextWithTrace = context.WithValue(*ctx, constant.TraceIDKey, traceID) - } else { - traceID = (*ctx).Value(constant.TraceIDKey).(string) - contextWithTrace = *ctx - } - if appConfig.BackendAddress == "" { log.Fatal("Cannot create grpc client: Backend address is empty in config! Run `odin configure` to set backend address") } + var opts []grpc.DialOption if appConfig.Insecure { if util.IsIPAddress(strings.Split(appConfig.BackendAddress, ":")[0]) { @@ -45,7 +35,6 @@ func grpcClient(ctx *context.Context) (*grpc.ClientConn, *context.Context, error tlsConf.InsecureSkipVerify = true opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(&tlsConf))) } - } else { cred := credentials.NewTLS(&tls.Config{}) opts = append(opts, grpc.WithTransportCredentials(cred)) @@ -60,11 +49,11 @@ func grpcClient(ctx *context.Context) (*grpc.ClientConn, *context.Context, error )) conn, err := grpc.NewClient(appConfig.BackendAddress, opts...) - if err != nil { return nil, nil, err } - // Enrich context with authorisation metadata - requestCtx := metadata.AppendToOutgoingContext(contextWithTrace, "Authorization", fmt.Sprintf("Bearer %s", appConfig.AccessToken), string(constant.TraceIDKey), traceID) + + // Enrich context with authorization metadata only + requestCtx := metadata.AppendToOutgoingContext(*ctx, "Authorization", fmt.Sprintf("Bearer %s", appConfig.AccessToken)) return conn, &requestCtx, nil } diff --git a/internal/service/environment.go b/internal/service/environment.go index b3fa41bf..4a282184 100644 --- a/internal/service/environment.go +++ b/internal/service/environment.go @@ -51,6 +51,7 @@ func (e *Environment) CreateEnvironment(ctx *context.Context, request *environme return err } var message string + var traceID string for { response, err := stream.Recv() spinnerInstance.Stop() @@ -61,6 +62,12 @@ func (e *Environment) CreateEnvironment(ctx *context.Context, request *environme return err } if response != nil { + // Print Trace ID only once, when it first arrives + if traceID == "" && response.TraceId != "" { + traceID = response.TraceId + log.Infof("Generated Trace Id for env creation: %s", traceID) + } + message = response.Message spinnerInstance.Prefix = fmt.Sprintf(" %s ", response.Message) spinnerInstance.Start() diff --git a/internal/service/service.go b/internal/service/service.go index c97ce5b6..b3244ef5 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -406,7 +406,7 @@ func (e *Service) GetConflictingServices(ctx *context.Context, request *serviceP func streamLogs(streamCtx context.Context, ctx *context.Context, serviceName string) { var err error lastLogTime := int64(0) - traceID := (*ctx).Value(constant.TraceIDKey).(string) + //traceID := (*ctx).Value(constant.TraceIDKey).(string) follow := true // Start the spinner in a background goroutine go func() { @@ -429,7 +429,7 @@ func streamLogs(streamCtx context.Context, ctx *context.Context, serviceName str default: // Get logs with retry on error lastLogTime, err = logsClient.GetLogs(ctx, &logs.GetLogsRequest{ - TraceId: &traceID, + //TraceId: &traceID, Follow: &follow, ServiceName: &serviceName, StartTime: &lastLogTime, diff --git a/proto/dream11/od/environment/v1/environment.proto b/proto/dream11/od/environment/v1/environment.proto index d137a9b7..f2a0acca 100644 --- a/proto/dream11/od/environment/v1/environment.proto +++ b/proto/dream11/od/environment/v1/environment.proto @@ -69,18 +69,25 @@ message CreateEnvironmentRequest { string env_name = 1; repeated string accounts = 2; string provisioning_type = 3; + optional string trace_id = 4; + } message CreateEnvironmentResponse { string message = 1; + string trace_id = 2; + } message DeleteEnvironmentRequest { string env_name = 1; + optional string trace_id = 2; } message DeleteEnvironmentResponse { string message = 1; + string trace_id = 2; + } message IsStrictEnvironmentRequest { From d01f3569fd04a9fab4c58a52d8785ddca8785f6f Mon Sep 17 00:00:00 2001 From: Umang Goyal Date: Thu, 8 May 2025 13:03:31 +0530 Subject: [PATCH 2/6] update environment --- internal/service/environment.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/service/environment.go b/internal/service/environment.go index 4a282184..8c4eea64 100644 --- a/internal/service/environment.go +++ b/internal/service/environment.go @@ -98,6 +98,7 @@ func (e *Environment) DeleteEnvironment(ctx *context.Context, request *environme return err } var message string + var traceID string for { response, err := stream.Recv() spinnerInstance.Stop() @@ -108,6 +109,11 @@ func (e *Environment) DeleteEnvironment(ctx *context.Context, request *environme return err } if response != nil { + // Print Trace ID only once, when it first arrives + if traceID == "" && response.TraceId != "" { + traceID = response.TraceId + log.Infof("Generated Trace Id for env creation: %s", traceID) + } message = response.Message spinnerInstance.Prefix = fmt.Sprintf(" %s ", response.Message) spinnerInstance.Start() From 281e92ee293d05301f119589b6c4f29e0eb87329 Mon Sep 17 00:00:00 2001 From: Umang Goyal Date: Tue, 13 May 2025 14:25:25 +0530 Subject: [PATCH 3/6] update --- internal/service/client.go | 4 ---- proto/dream11/od/service/v1/service.proto | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/service/client.go b/internal/service/client.go index 3511ec74..01772ea7 100644 --- a/internal/service/client.go +++ b/internal/service/client.go @@ -9,7 +9,6 @@ import ( "time" "github.com/dream11/odin/pkg/config" - "github.com/dream11/odin/pkg/util" log "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -38,8 +37,6 @@ func getTLSOpts(appConfig *configuration.Configuration) grpc.DialOption { func grpcClient(ctx *context.Context) (*grpc.ClientConn, *context.Context, error) { appConfig := config.GetConfig() - - if appConfig.BackendAddress == "" { log.Fatal("Cannot create grpc client: Backend address is empty in config! Run `odin configure` to set backend address") } @@ -52,7 +49,6 @@ func grpcClient(ctx *context.Context) (*grpc.ClientConn, *context.Context, error PermitWithoutStream: true, }), getTLSOpts(appConfig), - } conn, err := grpc.NewClient(appConfig.BackendAddress, opts...) if err != nil { diff --git a/proto/dream11/od/service/v1/service.proto b/proto/dream11/od/service/v1/service.proto index ecf14f09..6b270f83 100644 --- a/proto/dream11/od/service/v1/service.proto +++ b/proto/dream11/od/service/v1/service.proto @@ -23,6 +23,7 @@ message DeployServiceRequest { string env_name = 1; dto.v1.ServiceDefinition service_definition = 2; dto.v1.ProvisioningConfig provisioning_config = 3; + optional string trace_id = 4; } message ServiceResponse { @@ -66,6 +67,7 @@ message DeployServiceSetServiceResponse { message DeployServiceResponse { ServiceResponse service_response = 1; + string trace_id = 4; } message ServiceStatus { From 946d256669cfcb4fd83cb7c509a5b165c764163a Mon Sep 17 00:00:00 2001 From: Umang Goyal Date: Tue, 13 May 2025 15:57:06 +0530 Subject: [PATCH 4/6] update service client and proto files --- internal/service/client.go | 10 ++-------- internal/service/component.go | 2 +- internal/service/service.go | 12 ++++++------ proto/dream11/od/logs/v1/logs.proto | 4 ++-- proto/dream11/od/service/v1/service.proto | 2 +- 5 files changed, 12 insertions(+), 18 deletions(-) diff --git a/internal/service/client.go b/internal/service/client.go index 01772ea7..d616b6a7 100644 --- a/internal/service/client.go +++ b/internal/service/client.go @@ -21,14 +21,8 @@ func getTLSOpts(appConfig *configuration.Configuration) grpc.DialOption { tlsConf := tls.Config{ ServerName: strings.Split(appConfig.BackendAddress, ":")[0], } - if appConfig.Plaintext { - // Disable TLS - return grpc.WithTransportCredentials(insecure.NewCredentials()) - } - if appConfig.Insecure { - // Perform TLS handshake but skip certificate verification - tlsConf.InsecureSkipVerify = true - } + // Disable TLS + return grpc.WithTransportCredentials(insecure.NewCredentials()) return grpc.WithTransportCredentials(credentials.NewTLS(&tlsConf)) diff --git a/internal/service/component.go b/internal/service/component.go index 5a606009..cc83a5db 100644 --- a/internal/service/component.go +++ b/internal/service/component.go @@ -24,7 +24,7 @@ func (e *Component) OperateComponent(ctx *context.Context, request *serviceProto defer cancelFunction() // Start log streaming in background - go streamLogs(streamCtx, ctx, request.GetServiceName()) + go streamLogs(streamCtx, ctx, request.GetServiceName(), request.GetEnvName()) // Attempt operation with retries return retry.Do( diff --git a/internal/service/service.go b/internal/service/service.go index 513436cb..cad2886b 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -57,7 +57,7 @@ func (e *Service) DeployService(ctx *context.Context, request *serviceProto.Depl defer cancelFunction() // Start log streaming in background - go streamLogs(streamCtx, ctx, request.GetServiceDefinition().GetName()) + go streamLogs(streamCtx, ctx, request.GetServiceDefinition().GetName(), request.GetEnvName()) // Attempt deployment with retries return retry.Do( @@ -162,7 +162,7 @@ func (e *Service) DeployReleasedService(ctx *context.Context, request *servicePr defer cancelFunction() // Start log streaming in background - go streamLogs(streamCtx, ctx, request.GetServiceIdentifier().GetServiceName()) + go streamLogs(streamCtx, ctx, request.GetServiceIdentifier().GetServiceName(), request.GetEnvName()) // Attempt deployment with retries return retry.Do( @@ -210,7 +210,7 @@ func (e *Service) UndeployService(ctx *context.Context, request *serviceProto.Un defer cancelFunction() // Start log streaming in background - go streamLogs(streamCtx, &contextWithTrace, request.GetServiceName()) + go streamLogs(streamCtx, &contextWithTrace, request.GetServiceName(), request.GetEnvName()) conn, requestCtx, err := grpcClient(&contextWithTrace) if err != nil { @@ -261,7 +261,7 @@ func (e *Service) OperateService(ctx *context.Context, request *serviceProto.Ope defer cancelFunction() // Start log streaming in background - go streamLogs(streamCtx, ctx, request.GetServiceName()) + go streamLogs(streamCtx, ctx, request.GetServiceName(), request.GetEnvName()) // Attempt operation with retries return retry.Do( @@ -403,7 +403,7 @@ func (e *Service) GetConflictingServices(ctx *context.Context, request *serviceP } // streamLogs streams logs for a service -func streamLogs(streamCtx context.Context, ctx *context.Context, serviceName string) { +func streamLogs(streamCtx context.Context, ctx *context.Context, serviceName string, envName string) { var err error lastLogTime := int64(0) //traceID := (*ctx).Value(constant.TraceIDKey).(string) @@ -430,10 +430,10 @@ func streamLogs(streamCtx context.Context, ctx *context.Context, serviceName str default: // Get logs with retry on error lastLogTime, err = logsClient.GetLogs(ctx, &logs.GetLogsRequest{ - //TraceId: &traceID, Follow: &follow, ServiceName: &serviceName, StartTime: &lastLogTime, + EnvName: &envName, }) if err != nil { diff --git a/proto/dream11/od/logs/v1/logs.proto b/proto/dream11/od/logs/v1/logs.proto index a3a5f212..bec0377a 100644 --- a/proto/dream11/od/logs/v1/logs.proto +++ b/proto/dream11/od/logs/v1/logs.proto @@ -8,11 +8,11 @@ service LogsService { } message GetLogsRequest { - string trace_id = 1; optional bool follow = 2; optional string service_name = 3; optional string component_name = 4; - optional int64 start_time = 5; + optional string env_name = 5; + optional int64 start_time = 6; } message GetLogsResponse { diff --git a/proto/dream11/od/service/v1/service.proto b/proto/dream11/od/service/v1/service.proto index 6b270f83..3e02ecef 100644 --- a/proto/dream11/od/service/v1/service.proto +++ b/proto/dream11/od/service/v1/service.proto @@ -67,7 +67,7 @@ message DeployServiceSetServiceResponse { message DeployServiceResponse { ServiceResponse service_response = 1; - string trace_id = 4; + string trace_id = 2; } message ServiceStatus { From 3f6f1fd3813359ddb8dac568809f28b50e9fd642 Mon Sep 17 00:00:00 2001 From: Umang Goyal Date: Tue, 13 May 2025 16:57:51 +0530 Subject: [PATCH 5/6] update service client and proto files --- cmd/deploy/service.go | 11 +++----- internal/service/component.go | 5 +++- internal/service/service.go | 33 ++++++++++++++++++++--- proto/dream11/od/service/v1/service.proto | 4 +++ 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/cmd/deploy/service.go b/cmd/deploy/service.go index c84ff69c..7c057a90 100644 --- a/cmd/deploy/service.go +++ b/cmd/deploy/service.go @@ -53,23 +53,20 @@ func init() { func execute(cmd *cobra.Command) { env = config.EnsureEnvPresent(env) ctx := cmd.Context() - traceID := util.GenerateTraceID() - contextWithTrace := context.WithValue(ctx, constant.TraceIDKey, traceID) - if isStrictEnvironment(contextWithTrace, env) { + if isStrictEnvironment(ctx, env) { consentMessage := fmt.Sprintf(constant.ConsentMessageTemplate, env) util.AskForConfirmation(env, consentMessage) } - if (serviceName == "" && serviceVersion == "" && labels == "") && (definitionFile != "" && provisioningFile != "") { - deployUsingFiles(contextWithTrace) + deployUsingFiles(ctx) } else if (serviceName != "" && serviceVersion != "" && labels == "") && (definitionFile == "" && provisioningFile == "") { - deployUsingServiceNameAndVersion(contextWithTrace) + deployUsingServiceNameAndVersion(ctx) } else if (serviceName != "" && labels != "" && serviceVersion == "") && (definitionFile == "" && provisioningFile == "") { if err := validateLabels(labels); err != nil { log.Fatal("Invalid labels format: ", err) } - deployUsingServiceNameAndLabels(contextWithTrace) + deployUsingServiceNameAndLabels(ctx) } else { log.Fatal("Invalid combination of flags. Use either (service name and version) or (definitionFile and provisioningFile).") } diff --git a/internal/service/component.go b/internal/service/component.go index cc83a5db..35cf8d06 100644 --- a/internal/service/component.go +++ b/internal/service/component.go @@ -52,8 +52,11 @@ func (e *Component) OperateComponent(ctx *context.Context, request *serviceProto return response.GetServiceResponse().GetServiceStatus().GetServiceStatus(), response.GetServiceResponse().GetServiceStatus().GetServiceAction() } + getTraceId := func(response *serviceProto.OperateServiceResponse) string { + return response.GetTraceId() + } - return handleResponse(stream, cancelFunction, getMessage, getStatus) + return handleResponse(stream, cancelFunction, getMessage, getStatus, getTraceId) }, retry.Delay(constant.Delay), retry.RetryIf(isRetryableError), diff --git a/internal/service/service.go b/internal/service/service.go index cad2886b..d5f83bd1 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -48,6 +48,8 @@ type getStatus[R any] func(response R) (serviceAction, serviceStatus string) type getMessage[R any] func(response R) string +type getTraceId[R any] func(response R) string + // DeployService deploys service func (e *Service) DeployService(ctx *context.Context, request *serviceProto.DeployServiceRequest) error { log.Info("Deploying Service...") @@ -86,7 +88,11 @@ func (e *Service) DeployService(ctx *context.Context, request *serviceProto.Depl response.GetServiceResponse().GetServiceStatus().GetServiceAction() } - return handleResponse(stream, cancelFunction, getMessage, getStatus) + getTraceId := func(response *serviceProto.DeployServiceResponse) string { + return response.GetTraceId() + } + + return handleResponse(stream, cancelFunction, getMessage, getStatus, getTraceId) }, retry.Delay(constant.Delay), retry.RetryIf(isRetryableError), @@ -192,7 +198,11 @@ func (e *Service) DeployReleasedService(ctx *context.Context, request *servicePr response.GetServiceResponse().GetServiceStatus().GetServiceAction() } - return handleResponse(stream, cancelFunction, getMessage, getStatus) + getTraceId := func(response *serviceProto.DeployReleasedServiceResponse) string { + return response.GetTraceId() + } + + return handleResponse(stream, cancelFunction, getMessage, getStatus, getTraceId) }, retry.Delay(constant.Delay), retry.RetryIf(isRetryableError), @@ -290,7 +300,11 @@ func (e *Service) OperateService(ctx *context.Context, request *serviceProto.Ope response.GetServiceResponse().GetServiceStatus().GetServiceAction() } - return handleResponse(stream, cancelFunction, getMessage, getStatus) + getTraceId := func(response *serviceProto.OperateServiceResponse) string { + return response.GetTraceId() + } + + return handleResponse(stream, cancelFunction, getMessage, getStatus, getTraceId) }, retry.Delay(constant.Delay), retry.RetryIf(isRetryableError), @@ -445,10 +459,21 @@ func streamLogs(streamCtx context.Context, ctx *context.Context, serviceName str } // handleResponse streams the service deploy response and call cancel on action termination -func handleResponse[S StreamReceiverInterface[R], R any](stream S, cancelFunc context.CancelFunc, getMessage getMessage[R], getStatus getStatus[R]) error { +func handleResponse[S StreamReceiverInterface[R], R any](stream S, cancelFunc context.CancelFunc, getMessage getMessage[R], getStatus getStatus[R], getTraceId getTraceId[R]) error { var serviceAction, serviceStatus string + var traceID string for { response, err := stream.Recv() + // Log the raw response for debugging + log.Infof("Received deploy service response: %+v", response) + + // Extract and log trace ID if available and not already set + trace := getTraceId(response) + if traceID == "" && trace != "" { + traceID = trace + log.Infof("Generated Trace Id for deploy service: %s", traceID) + } + if err != nil { if isActionCompleted(serviceAction, serviceStatus) { cancelFunc() diff --git a/proto/dream11/od/service/v1/service.proto b/proto/dream11/od/service/v1/service.proto index 3e02ecef..148f3613 100644 --- a/proto/dream11/od/service/v1/service.proto +++ b/proto/dream11/od/service/v1/service.proto @@ -36,6 +36,7 @@ message ServiceResponse { message DeployReleasedServiceResponse { ServiceResponse service_response = 1; + string trace_id = 2; } message ServiceIdentifier { @@ -48,6 +49,7 @@ message ServiceIdentifier { message DeployReleasedServiceRequest { ServiceIdentifier service_identifier = 1; string env_name = 3; + optional string trace_id = 4; } message DeployServiceSetRequest { @@ -102,10 +104,12 @@ message OperateServiceRequest { bool is_component_operation = 4; string operation = 5; optional google.protobuf.Struct config = 6; + optional string trace_id = 7; } message OperateServiceResponse { ServiceResponse service_response = 1; + string trace_id = 2; } message UndeployServiceRequest { From 13b6461bca9d72c278f60320a57e819c64da4c24 Mon Sep 17 00:00:00 2001 From: Umang Goyal Date: Thu, 15 May 2025 18:25:49 +0530 Subject: [PATCH 6/6] Log traceid in deploy service --- internal/service/service.go | 43 ++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/internal/service/service.go b/internal/service/service.go index d5f83bd1..b4146c64 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -423,20 +423,20 @@ func streamLogs(streamCtx context.Context, ctx *context.Context, serviceName str //traceID := (*ctx).Value(constant.TraceIDKey).(string) follow := true // Start the spinner in a background goroutine - go func() { - spinnerInstance := spinner.New(spinner.CharSets[constant.SpinnerType], constant.SpinnerDelay) - err = spinnerInstance.Color(constant.SpinnerColor, constant.SpinnerStyle) - if err != nil { - spinnerInstance.Stop() - } - spinnerInstance.Prefix = fmt.Sprintf("Fetching live logs for service: %s ", serviceName) - spinnerInstance.Suffix = "\n" - spinnerInstance.Start() - time.Sleep(30 * time.Second) - spinnerInstance.Stop() - fmt.Printf("Fetching live logs for service: %s \n", serviceName) - }() - + //go func() { + //spinnerInstance := spinner.New(spinner.CharSets[constant.SpinnerType], constant.SpinnerDelay) + //err = spinnerInstance.Color(constant.SpinnerColor, constant.SpinnerStyle) + //if err != nil { + // spinnerInstance.Stop() + //} + //spinnerInstance.Prefix = fmt.Sprintf("Fetching live logs for service: %s ", serviceName) + //spinnerInstance.Suffix = "\n" + //spinnerInstance.Start() + //time.Sleep(30 * time.Second) + //spinnerInstance.Stop() + + //}() + fmt.Printf("Fetching live logs for service: %s \n", serviceName) for { select { case <-streamCtx.Done(): @@ -462,16 +462,15 @@ func streamLogs(streamCtx context.Context, ctx *context.Context, serviceName str func handleResponse[S StreamReceiverInterface[R], R any](stream S, cancelFunc context.CancelFunc, getMessage getMessage[R], getStatus getStatus[R], getTraceId getTraceId[R]) error { var serviceAction, serviceStatus string var traceID string + traceIdLogged := false for { response, err := stream.Recv() - // Log the raw response for debugging - log.Infof("Received deploy service response: %+v", response) - - // Extract and log trace ID if available and not already set - trace := getTraceId(response) - if traceID == "" && trace != "" { - traceID = trace - log.Infof("Generated Trace Id for deploy service: %s", traceID) + traceID = getTraceId(response) + if !traceIdLogged { + if traceID != "" { + log.Info("Generated Trace Id for deploy service: ", traceID) + traceIdLogged = true + } } if err != nil {