From 45bbdd2e1847bd03ac48d804f42212083c605b17 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 24 May 2026 16:04:21 +0800 Subject: [PATCH 1/7] feat: add access_token to security --- core/config/system.go | 12 + .../security/access_token/authentication.go | 559 ++++++++++++++++++ .../security/access_token/authorization.go | 39 ++ modules/security/http_filters/permission.go | 1 + modules/security/init.go | 6 + modules/security/rbac/role.go | 5 + 6 files changed, 622 insertions(+) create mode 100644 modules/security/access_token/authentication.go create mode 100644 modules/security/access_token/authorization.go diff --git a/core/config/system.go b/core/config/system.go index b34a5ce47..ead7360bd 100755 --- a/core/config/system.go +++ b/core/config/system.go @@ -330,10 +330,22 @@ type RealmConfig struct { type AuthenticationConfig struct { Native RealmConfig `config:"native"` + AccessToken AccessTokenConfig `config:"access_token"` HTTPBasicAuthProvider HTTPBasicAuthProvider `config:"http_basic"` OAuth map[string]OAuthConfig `config:"oauth"` } +// AccessTokenConfig controls API access-token management. +// +// When Native is true (default when the native realm is enabled) tokens are +// persisted via ORM in addition to KV. When Native is false the module runs +// in KV-only mode — suitable for agent-style deployments without an ORM +// backend. +type AccessTokenConfig struct { + Enabled bool `config:"enabled"` + Native bool `config:"native"` +} + type HTTPBasicAuthProvider struct { Enabled bool `config:"enabled"` Endpoint string `config:"endpoint" json:"endpoint,omitempty"` diff --git a/modules/security/access_token/authentication.go b/modules/security/access_token/authentication.go new file mode 100644 index 000000000..dddd31476 --- /dev/null +++ b/modules/security/access_token/authentication.go @@ -0,0 +1,559 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package access_token + +import ( + "fmt" + "infini.sh/framework/core/log" + "net/http" + "sync" + "time" + + "github.com/emirpasic/gods/sets/hashset" + + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/global" + "infini.sh/framework/core/kv" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/security" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/security/http_filters" +) + +const ProviderName = "access_token" + +const ( + // KVAccessTokenBucket stores token_string -> AccessToken JSON. Used by + // byAPITokenHeader to authenticate inbound requests in both modes. + KVAccessTokenBucket = ProviderName + + // kvAccessTokenIndexBucket is used only in non-native mode. + // - key "__ids__" -> JSON []string of all known token IDs + // - key "" -> token_string (for delete/update by ID) + kvAccessTokenIndexBucket = "access_token_index" + kvIndexListKey = "__ids__" + + HeaderAPIToken = "X-API-TOKEN" +) + +var ( + indexLock sync.Mutex + initOnce sync.Once + + // Permission catalog. Registered unconditionally in init() so that roles + // referencing these permissions remain valid even when the access_token + // HTTP endpoints are disabled. + createTokenPermission security.PermissionKey + updateTokenPermission security.PermissionKey + deleteTokenPermission security.PermissionKey + searchTokenPermission security.PermissionKey +) + +func init() { + createTokenPermission = security.GetOrInitPermission("generic", "security:auth:api-token", security.Create) + updateTokenPermission = security.GetOrInitPermission("generic", "security:auth:api-token", security.Update) + deleteTokenPermission = security.GetOrInitPermission("generic", "security:auth:api-token", security.Delete) + searchTokenPermission = security.GetOrInitPermission("generic", "security:auth:api-token", security.Search) + + // The auth filter provider is registered unconditionally so that inbound + // requests carrying X-API-TOKEN can be authenticated even before Init() is + // called (e.g. in embedded scenarios that never call Init explicitly). + security.RegisterHTTPAuthFilterProvider("api_token", byAPITokenHeader) +} + +// Init registers the HTTP management endpoints for access tokens. Safe to +// call multiple times; only the first call has effect. +func Init() { + initOnce.Do(func() { + api.HandleUIMethod(api.POST, "/auth/access_token", RequestAccessToken, api.RequirePermission(createTokenPermission)) + api.HandleUIMethod(api.GET, "/auth/access_token/_search", SearchAccessToken, api.RequirePermission(searchTokenPermission), api.Feature(http_filters.FeatureMaskSensitiveField)) + api.HandleUIMethod(api.DELETE, "/auth/access_token/:token_id", DeleteAccessToken, api.RequirePermission(deleteTokenPermission)) + api.HandleUIMethod(api.PUT, "/auth/access_token/:token_id", UpdateAccessToken, api.RequirePermission(updateTokenPermission)) + }) +} + +// isNative reports whether the access_token module should persist tokens via +// ORM. When false the module operates in KV-only mode (e.g. for agent-style +// deployments without an ORM backend). +func isNative() bool { + return global.Env().SystemConfig.WebAppConfig.Security.Authentication.AccessToken.Native +} + +func byAPITokenHeader(w http.ResponseWriter, r *http.Request) (claims *security.UserClaims, err error) { + apiToken := r.Header.Get(HeaderAPIToken) + + accessToken, permissions, err := getTokenPermissions(apiToken) + if err!=nil { + return nil, err + } + + claims = security.NewUserClaims() + claims.SetUserID(accessToken.GetOwnerID()) + claims.Provider = ProviderName + claims.Login = apiToken + claims.Permissions = permissions + claims.Data = accessToken.CloneData() + + return claims, nil +} + +func getTokenPermissions(apiToken string) (*security.AccessToken, []security.PermissionKey,error) { + if apiToken == "" { + return nil,nil, errors.Error("api token not found") + } + + bytes, err := kv.GetValue(KVAccessTokenBucket, []byte(apiToken)) + if err != nil { + return nil, nil, err + } + + if len(bytes) == 0 { + return nil, nil, errors.Errorf("invalid %s", HeaderAPIToken) + } + + accessToken := security.AccessToken{} + util.MustFromJSONBytes(bytes, &accessToken) + + if global.Env().IsDebug { + log.Debug("get AccessToken from store:", string(bytes)) + } + + //-1 means never expire + if accessToken.ExpireIn>0 { + expireAtTime := time.Unix(accessToken.ExpireIn, 0) + if time.Now().After(expireAtTime) { + return nil, nil, errors.Error("token expired") + } + } + + permissions := accessToken.Permissions + + //user may be revoked some permission after issued the api token + if isNative() { + // Effective permissions = token permissions ∩ owning user's current permissions. + // Requires a user/role store. + apiTokenLevelPermission := security.ConvertPermissionKeysToHashSet(accessToken.Permissions) + + userSessionInfo := security.UserSessionInfo{} + userSessionInfo.Provider = "native" + userSessionInfo.SetUserID(accessToken.GetOwnerID()) + + userLevelTokenLevelPermission := security.ConvertPermissionKeysToHashSet(security.GetAllPermissionsForUser(&userSessionInfo)) + intersectedPermission := security.IntersectSetsFast(apiTokenLevelPermission, userLevelTokenLevelPermission) + if global.Env().IsDebug { + log.Trace("apiTokenLevelPermission:", apiTokenLevelPermission.Values()) + log.Trace("userLevelTokenLevelPermission:", userLevelTokenLevelPermission.Values()) + log.Trace("intersectedPermission:", intersectedPermission.Values()) + } + + permissions = security.ConvertPermissionHashSetToKeys(intersectedPermission) + } + return &accessToken, permissions, nil +} + +func GetPermissionHashSet(u *security.UserSessionInfo) *hashset.Set { + keys := security.GetAllPermissionsForUser(u) + set := security.ConvertPermissionKeysToHashSet(keys) + return set +} + +func RequestAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + + // user already login + reqUser, err := security.GetUserFromContext(req.Context()) + if reqUser == nil || err != nil { + panic(err) + } + + reqBody := struct { + Name string `json:"name"` + Permissions []security.PermissionKey `json:"permissions,omitempty"` + }{} + err = api.DecodeJSON(req, &reqBody) + if err != nil { + panic(err) + } + if reqBody.Name == "" { + reqBody.Name = GenerateApiTokenName("") + } + + var permissions []security.PermissionKey + if isNative() { + permissions = security.GetAllPermissionsForUser(reqUser) + if len(reqBody.Permissions) > 0 { + // requested permissions must be within the caller's own scope + if !util.IsSuperset(security.ConvertPermissionKeysToHashSet(permissions), security.ConvertPermissionKeysToHashSet(reqBody.Permissions)) { + panic("invalid permissions") + } + permissions = reqBody.Permissions + } + } else { + // no user/role store: trust whatever the caller asked for, falling + // back to the caller's session permissions when none are specified. + if len(reqBody.Permissions) > 0 { + permissions = reqBody.Permissions + } else { + permissions = append(permissions, reqUser.Permissions...) + } + } + + expiredAT := time.Now().Add(365 * 24 * time.Hour).Unix() + res, err := CreateAPIToken(reqUser, reqBody.Name, "general",expiredAT, permissions) + if err != nil { + panic(err) + } + + api.WriteJSON(w, res, 200) +} + +func CreateAPIToken(user *security.UserSessionInfo, tokenName, typeName string,expiredAT int64 , permissions []security.PermissionKey) (util.MapStr, error) { + + if tokenName == "" { + tokenName = GenerateApiTokenName("") + } + + accessTokenStr := util.GetUUID() + util.GenerateRandomString(64) + + accessToken := security.AccessToken{} + tokenID := util.GetUUID() + accessToken.ID = tokenID + accessToken.AccessToken = accessTokenStr + if user != nil { + user.Roles = nil + accessToken.SetOwnerID(user.MustGetUserID()) + accessToken.Data = user.CloneData() + } + + accessToken.Type = typeName + accessToken.Permissions = permissions + accessToken.ExpireIn = expiredAT + accessToken.Name = tokenName + + if isNative() { + ctx := orm.NewContext() + ctx.DirectAccess() + ctx.Refresh = orm.WaitForRefresh + ctx.PermissionScope(security.PermissionScopePlatform) + + if err := orm.Create(ctx, &accessToken); err != nil { + return nil, err + } + } else { + if err := addTokenToIndex(tokenID, accessTokenStr); err != nil { + return nil, err + } + } + + // persist token for fast lookup by token string (used by auth filter) + if err := kv.AddValue(KVAccessTokenBucket, []byte(accessTokenStr), util.MustToJSONBytes(&accessToken)); err != nil { + return nil, err + } + + res := util.MapStr{ + "_id": tokenID, + "access_token": accessTokenStr, + "expire_in": expiredAT, + } + return res, nil +} + +func SearchAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + if isNative() { + builder, err := orm.NewQueryBuilderFromRequest(req, "name") + if err != nil { + panic(err) + } + + ctx := orm.NewContextWithParent(req.Context()) + orm.WithModel(ctx, &security.AccessToken{}) + + res, err := orm.SearchV2(ctx, builder) + if err != nil { + panic(err) + } + + if _, err = api.Write(w, res.Payload.([]byte)); err != nil { + api.Error(w, err) + } + return + } + + tokens, err := listAccessTokensFromKV() + if err != nil { + api.Error(w, err) + return + } + + api.WriteJSON(w, util.MapStr{ + "hits": util.MapStr{ + "total": util.MapStr{"value": len(tokens), "relation": "eq"}, + "max_score": 0, + "hits": tokens, + }, + }, 200) +} + +func DeleteAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + reqUser, err := security.GetUserFromContext(req.Context()) + if reqUser == nil || err != nil { + panic(err) + } + tokenID := ps.ByName("token_id") + + var tokenString string + + if isNative() { + ctx := orm.NewContextWithParent(req.Context()) + + // Load first so we know the token string (needed to clean up the KV + // lookup entry below). Without this, orm.Delete only knows the ID and + // leaves the KV row orphaned. + token := security.AccessToken{} + token.ID = tokenID + exists, err := orm.GetV2(ctx, &token) + if err != nil { + panic(err) + } + if !exists { + api.WriteError(w, "access token not found", 404) + return + } + tokenString = token.AccessToken + + ctx.Refresh = orm.WaitForRefresh + if err = orm.Delete(ctx, &token); err != nil { + panic(err) + } + } else { + s, err := removeTokenFromIndex(tokenID) + if err != nil { + api.Error(w, err) + return + } + tokenString = s + } + + if tokenString != "" { + if err = kv.DeleteKey(KVAccessTokenBucket, []byte(tokenString)); err != nil { + panic(err) + } + } + + // Invalidate the per-user permission cache so any request that authenticated + // with this token (or any other token belonging to the same owner) re-derives + // its UserAssignedPermission on the next call instead of using a stale entry. + security.IncreasePermissionVersion() + + api.WriteDeletedOKJSON(w, tokenID) +} + +func GetToken(token string) (*security.AccessToken, error) { + tokenBytes, err := kv.GetValue(KVAccessTokenBucket, []byte(token)) + if err != nil { + return nil, err + } + if len(tokenBytes) == 0 { + return nil, errors.Errorf("token not found") + } + accessToken := security.AccessToken{} + if err = util.FromJSONBytes(tokenBytes, &accessToken); err != nil { + return nil, err + } + return &accessToken, nil +} + +func UpdateAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + reqUser, err := security.GetUserFromContext(req.Context()) + if reqUser == nil || err != nil { + panic(err) + } + reqBody := struct { + Name string `json:"name,omitempty"` + Permissions []security.PermissionKey `json:"permissions,omitempty"` + }{} + err = api.DecodeJSON(req, &reqBody) + if err != nil { + panic(err) + } + if reqBody.Name == "" { + api.WriteError(w, "name is required", 400) + return + } + tokenID := ps.ByName("token_id") + + var token *security.AccessToken + + if isNative() { + ctx := orm.NewContextWithParent(req.Context()) + token = &security.AccessToken{} + token.ID = tokenID + exists, err := orm.GetV2(ctx, token) + if err != nil { + panic(err) + } + if !exists { + api.WriteError(w, "access token not found", 404) + return + } + } else { + t, err := getAccessTokenByIDFromKV(tokenID) + if err != nil { + api.WriteError(w, err.Error(), 404) + return + } + token = t + } + + if reqBody.Name != "" { + token.Name = reqBody.Name + } + if len(reqBody.Permissions) > 0 { + if isNative() { + // The NEW permissions must be a subset of the caller's own permissions. + requested := security.ConvertPermissionKeysToHashSet(reqBody.Permissions) + if !util.IsSuperset(GetPermissionHashSet(reqUser), requested) { + panic("invalid permissions") + } + } + token.Permissions = reqBody.Permissions + } + + if isNative() { + ctx := orm.NewContextWithParent(req.Context()) + ctx.Refresh = orm.WaitForRefresh + if err = orm.Save(ctx, token); err != nil { + panic(err) + } + } + + if err = kv.AddValue(KVAccessTokenBucket, []byte(token.AccessToken), util.MustToJSONBytes(token)); err != nil { + panic(err) + } + + // Force the next request that uses this (or any) token to re-derive its + // permissions, so the freshly-saved permission set takes effect immediately + // instead of waiting for the 30-minute permissionCache TTL. + security.IncreasePermissionVersion() + + api.WriteUpdatedOKJSON(w, tokenID) +} + +// GenerateApiTokenName generates a unique API token name +func GenerateApiTokenName(prefix string) string { + if prefix == "" { + prefix = "token" + } + timestamp := time.Now().UnixMilli() + randomStr := util.GenerateRandomString(8) + return fmt.Sprintf("%s_%d_%s", prefix, timestamp, randomStr) +} + +// --- KV-side index helpers (non-native mode only) ------------------------- + +func loadTokenIDs() ([]string, error) { + bytes, err := kv.GetValue(kvAccessTokenIndexBucket, []byte(kvIndexListKey)) + if err != nil { + return nil, err + } + if len(bytes) == 0 { + return nil, nil + } + ids := []string{} + if err := util.FromJSONBytes(bytes, &ids); err != nil { + return nil, err + } + return ids, nil +} + +func saveTokenIDs(ids []string) error { + return kv.AddValue(kvAccessTokenIndexBucket, []byte(kvIndexListKey), util.MustToJSONBytes(ids)) +} + +func addTokenToIndex(tokenID, tokenString string) error { + indexLock.Lock() + defer indexLock.Unlock() + + if err := kv.AddValue(kvAccessTokenIndexBucket, []byte(tokenID), []byte(tokenString)); err != nil { + return err + } + + ids, err := loadTokenIDs() + if err != nil { + return err + } + for _, id := range ids { + if id == tokenID { + return nil + } + } + ids = append(ids, tokenID) + return saveTokenIDs(ids) +} + +func removeTokenFromIndex(tokenID string) (string, error) { + indexLock.Lock() + defer indexLock.Unlock() + + tokenStringBytes, err := kv.GetValue(kvAccessTokenIndexBucket, []byte(tokenID)) + if err != nil { + return "", err + } + if len(tokenStringBytes) == 0 { + return "", errors.Errorf("access token not found: %s", tokenID) + } + + if err := kv.DeleteKey(kvAccessTokenIndexBucket, []byte(tokenID)); err != nil { + return "", err + } + + ids, err := loadTokenIDs() + if err != nil { + return string(tokenStringBytes), err + } + out := ids[:0] + for _, id := range ids { + if id != tokenID { + out = append(out, id) + } + } + if err := saveTokenIDs(out); err != nil { + return string(tokenStringBytes), err + } + + return string(tokenStringBytes), nil +} + +func getAccessTokenByIDFromKV(tokenID string) (*security.AccessToken, error) { + tokenStringBytes, err := kv.GetValue(kvAccessTokenIndexBucket, []byte(tokenID)) + if err != nil { + return nil, err + } + if len(tokenStringBytes) == 0 { + return nil, errors.Errorf("access token not found: %s", tokenID) + } + return GetToken(string(tokenStringBytes)) +} + +func listAccessTokensFromKV() ([]util.MapStr, error) { + ids, err := loadTokenIDs() + if err != nil { + return nil, err + } + out := make([]util.MapStr, 0, len(ids)) + for _, id := range ids { + t, err := getAccessTokenByIDFromKV(id) + if err != nil { + log.Warnf("load access token [%s] failed: %v", id, err) + continue + } + out = append(out, util.MapStr{ + "_id": t.ID, + "_source": t, + }) + } + return out, nil +} diff --git a/modules/security/access_token/authorization.go b/modules/security/access_token/authorization.go new file mode 100644 index 000000000..25a0306fe --- /dev/null +++ b/modules/security/access_token/authorization.go @@ -0,0 +1,39 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package access_token + +import ( + "context" + "infini.sh/framework/core/log" + "infini.sh/framework/core/security" +) + +func init() { + provider := SecurityBackendProvider{} + security.RegisterAuthorizationProvider(ProviderName, &provider) +} + +type SecurityBackendProvider struct { +} + +func (provider *SecurityBackendProvider) GetPermissionKeysByUserID(ctx1 context.Context, providerID, userID string) []security.PermissionKey { + var allowedPermissions = []security.PermissionKey{} + + if providerID==ProviderName{ + _, permissions, err := getTokenPermissions(userID) + if err!=nil { + log.Error(err) + }else{ + return permissions + } + } + + return allowedPermissions +} + +func (provider *SecurityBackendProvider) GetPermissionKeysByRoles(ctx context.Context, roles []string) []security.PermissionKey { + return []security.PermissionKey{} +} + diff --git a/modules/security/http_filters/permission.go b/modules/security/http_filters/permission.go index a152e2847..692b5b428 100644 --- a/modules/security/http_filters/permission.go +++ b/modules/security/http_filters/permission.go @@ -55,6 +55,7 @@ func (f *PermissionFilter) ApplyFilter( return } + //for API Token based session, there is maybe only a subset of user's permission if reqUser.UserAssignedPermission == nil || reqUser.UserAssignedPermission.NeedRefresh() { reqUser.UserAssignedPermission = security.GetUserPermissions(reqUser) } diff --git a/modules/security/init.go b/modules/security/init.go index 004ead1b7..96ff4e121 100644 --- a/modules/security/init.go +++ b/modules/security/init.go @@ -25,12 +25,14 @@ package security import ( "fmt" + "infini.sh/framework/core/api" "infini.sh/framework/core/config" "infini.sh/framework/core/env" "infini.sh/framework/core/global" "infini.sh/framework/core/module" "infini.sh/framework/core/util" + "infini.sh/framework/modules/security/access_token" _ "infini.sh/framework/modules/security/account" _ "infini.sh/framework/modules/security/http_filters" _ "infini.sh/framework/modules/security/oauth_client" @@ -72,6 +74,10 @@ func (module *Module) Setup() { rbac.Init() } + if module.cfg.Authentication.AccessToken.Enabled { + access_token.Init() + } + oauthSettings := util.MapStr{} for k, v := range module.cfg.Authentication.OAuth { if v.Enabled { diff --git a/modules/security/rbac/role.go b/modules/security/rbac/role.go index 2bda923ad..a38e21e28 100644 --- a/modules/security/rbac/role.go +++ b/modules/security/rbac/role.go @@ -195,6 +195,11 @@ type SecurityBackendProvider struct { } func (provider *SecurityBackendProvider) GetPermissionKeysByUserID(ctx1 context.Context, providerID, userID string) []security.PermissionKey { + + if providerID!=security.DefaultNativeAuthBackend{ + return nil + } + var allowedPermissions = []security.PermissionKey{} //bypass managed mode From f325def96a36d45cc7308e6802d92e3b2b6ee921 Mon Sep 17 00:00:00 2001 From: medcl Date: Sun, 24 May 2026 16:07:20 +0800 Subject: [PATCH 2/7] chore: update release notes --- docs/content.en/docs/release-notes/_index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index 24380fb7b..fd3aeae75 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -23,6 +23,7 @@ Information about release notes of INFINI Framework is provided here. - feat(cookie): prevent aggressive session cookie expiration #284 - feat(client): support token-based authorization #288 - feat: add pluggable sink to host metrics collectors #288 +- feat: add access_token to security #359 ### 🐛 Bug fix ### ✈️ Improvements From 7fd160ca5425ceb97c93befd09f06eb50e379c53 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 24 May 2026 21:47:30 +0800 Subject: [PATCH 3/7] chore: add post-register hooks so managed clients can run follow-up steps like token exchange immediately after a successful register --- core/config/system.go | 5 +- core/credential/access_token_test.go | 29 ++++++ core/credential/credential.go | 25 ++++- core/credential/domain.go | 77 +++++++++++++++- core/model/instance.go | 8 +- docs/content.en/docs/release-notes/_index.md | 4 + modules/configs/client/client.go | 92 ++++++++++++++----- modules/configs/client/client_test.go | 28 ++++++ modules/configs/common/domain.go | 1 + .../security/access_token/authentication.go | 12 +-- 10 files changed, 249 insertions(+), 32 deletions(-) create mode 100644 core/credential/access_token_test.go create mode 100644 modules/configs/client/client_test.go diff --git a/core/config/system.go b/core/config/system.go index ead7360bd..d08aa97ee 100755 --- a/core/config/system.go +++ b/core/config/system.go @@ -289,8 +289,9 @@ type ConfigsConfig struct { ValidConfigsExtensions []string `config:"valid_config_extensions"` TLSConfig TLSConfig `config:"tls"` //server or client's certs ManagerConfig struct { - LocalConfigsRepoPath string `config:"local_configs_repo_path"` - BasicAuth BasicAuth `config:"basic_auth"` + LocalConfigsRepoPath string `config:"local_configs_repo_path"` + BasicAuth BasicAuth `config:"basic_auth"` + AccessToken ucfg.SecretString `config:"access_token" yaml:"access_token"` } `config:"manager"` AlwaysRegisterAfterRestart bool `config:"always_register_after_restart"` AllowGeneratedMetricsTasks bool `config:"allow_generated_metrics_tasks"` diff --git a/core/credential/access_token_test.go b/core/credential/access_token_test.go new file mode 100644 index 000000000..92a3f8734 --- /dev/null +++ b/core/credential/access_token_test.go @@ -0,0 +1,29 @@ +package credential + +import "testing" + +func TestEncodeDecodeAccessToken(t *testing.T) { + cred := &Credential{ + Name: "agent-token", + Type: AccessToken, + Payload: map[string]interface{}{ + AccessToken: map[string]interface{}{ + "access_token": "mock-token", + }, + }, + } + cred.SetSecret([]byte("12345678901234567890123456789012")) + + if err := cred.Encode(); err != nil { + t.Fatalf("Encode() returned error: %v", err) + } + + token, err := cred.DecodeAccessToken() + if err != nil { + t.Fatalf("DecodeAccessToken() returned error: %v", err) + } + + if got := token.AccessToken.Get(); got != "mock-token" { + t.Fatalf("expected access token %q, got %q", "mock-token", got) + } +} diff --git a/core/credential/credential.go b/core/credential/credential.go index 62e17a4e6..1f7b1a692 100644 --- a/core/credential/credential.go +++ b/core/credential/credential.go @@ -31,6 +31,7 @@ import ( "fmt" "infini.sh/framework/core/model" "infini.sh/framework/core/orm" + "infini.sh/framework/lib/go-ucfg" ) type Credential struct { @@ -48,6 +49,10 @@ type Credential struct { Invalid bool `json:"invalid" elastic_mapping:"invalid:{type:boolean}"` } +type AccessTokenPayload struct { + AccessToken ucfg.SecretString `json:"access_token,omitempty" config:"access_token" yaml:"access_token"` +} + func (cred *Credential) SetSecret(secret []byte) { cred.secret = secret } @@ -69,6 +74,8 @@ func (cred *Credential) Encode() error { switch cred.Type { case BasicAuth: return encodeBasicAuth(cred) + case AccessToken: + return encodeAccessToken(cred) default: return fmt.Errorf("unkonow credential type [%s]", cred.Type) } @@ -86,15 +93,31 @@ func (cred *Credential) DecodeBasicAuth() (*model.BasicAuth, error) { return nil, fmt.Errorf("unkonow credential type [%s]", cred.Type) } +func (cred *Credential) DecodeAccessToken() (*AccessTokenPayload, error) { + dv, err := cred.Decode() + if err != nil { + return nil, err + } + + if token, ok := dv.(AccessTokenPayload); ok { + return &token, nil + } + + return nil, fmt.Errorf("unkonow credential type [%s]", cred.Type) +} + func (cred *Credential) Decode() (interface{}, error) { switch cred.Type { case BasicAuth: return decodeBasicAuth(cred) + case AccessToken: + return decodeAccessToken(cred) default: return nil, fmt.Errorf("unkonow credential type [%s]", cred.Type) } } const ( - BasicAuth string = "basic_auth" + BasicAuth string = "basic_auth" + AccessToken string = "access_token" ) diff --git a/core/credential/domain.go b/core/credential/domain.go index 2dbb05f4c..eac1b1cc5 100644 --- a/core/credential/domain.go +++ b/core/credential/domain.go @@ -85,6 +85,13 @@ func InitSecret(ks keystore2.Keystore, secret []byte) error { return nil } +func getCredentialSecret(cred *Credential) ([]byte, error) { + if cred != nil && cred.secret != nil { + return cred.secret, nil + } + return GetOrInitSecret() +} + func encodeBasicAuth(cred *Credential) error { var ( params map[string]interface{} @@ -100,7 +107,7 @@ func encodeBasicAuth(cred *Credential) error { if pwd == "" { return fmt.Errorf("credential parameters password can not be empty") } - secret, err := GetOrInitSecret() + secret, err := getCredentialSecret(cred) if err != nil { return err } @@ -117,6 +124,38 @@ func encodeBasicAuth(cred *Credential) error { return nil } +func encodeAccessToken(cred *Credential) error { + var ( + params map[string]interface{} + ok bool + token string + ) + if params, ok = cred.Payload[cred.Type].(map[string]interface{}); !ok { + return fmt.Errorf("wrong credential parameters for type [%s], expect a map", cred.Type) + } + if token, ok = params["access_token"].(string); !ok { + return fmt.Errorf("wrong credential parameters access_token for type [%s], expect a string", cred.Type) + } + if token == "" { + return fmt.Errorf("credential parameters access_token can not be empty") + } + secret, err := getCredentialSecret(cred) + if err != nil { + return err + } + encodeBytes, salt, err := util.AesGcmEncrypt([]byte(token), secret) + if err != nil { + return fmt.Errorf("encrypt access token error: %w", err) + } + cred.Encrypt.Type = "AES" + cred.Encrypt.Params = map[string]interface{}{ + "salt": string(salt), + } + params["access_token"] = string(encodeBytes) + cred.Payload[cred.Type] = params + return nil +} + func decodeBasicAuth(cred *Credential) (basicAuth model.BasicAuth, err error) { var ( params map[string]interface{} @@ -157,6 +196,42 @@ func decodeBasicAuth(cred *Credential) (basicAuth model.BasicAuth, err error) { return } +func decodeAccessToken(cred *Credential) (tokenPayload AccessTokenPayload, err error) { + var ( + params map[string]interface{} + ok bool + token string + salt string + ) + if params, ok = cred.Payload[cred.Type].(map[string]interface{}); !ok { + err = fmt.Errorf("wrong credential parameters for type [%s], expect a map", cred.Type) + return + } + if token, ok = params["access_token"].(string); !ok { + err = fmt.Errorf("wrong credential parameters access_token for type [%s], expect a string", cred.Type) + return + } + if token == "" { + err = fmt.Errorf("credential parameters access_token can not be empty") + return + } + if salt, ok = cred.Encrypt.Params["salt"].(string); !ok { + err = fmt.Errorf("credential encrypt parameters salt can not be empty") + return + } + secret, err := getCredentialSecret(cred) + if err != nil { + return tokenPayload, err + } + + plaintext, err := util.AesGcmDecrypt([]byte(token), secret, []byte(salt)) + if err != nil { + return tokenPayload, err + } + tokenPayload.AccessToken = ucfg.SecretString(plaintext) + return +} + type ChangeEvent func(credentials *Credential) var changeEvents []ChangeEvent diff --git a/core/model/instance.go b/core/model/instance.go index f9c60b44b..f2f04c4e3 100644 --- a/core/model/instance.go +++ b/core/model/instance.go @@ -56,6 +56,8 @@ type Instance struct { BasicAuth *BasicAuth `config:"basic_auth" json:"basic_auth,omitempty" elastic_mapping:"basic_auth:{type:object}"` + CredentialID string `json:"credential_id,omitempty" elastic_mapping:"credential_id:{type:keyword}"` + Labels map[string]string `json:"labels,omitempty" elastic_mapping:"labels:{type:object}"` Tags []string `json:"tags,omitempty"` @@ -137,7 +139,11 @@ func GetInstanceInfo() Instance { _, publicIP, _, _ := util.GetPublishNetworkDeviceInfo(global.Env().SystemConfig.NodeConfig.MajorIpPattern) - instance.Endpoint = global.Env().SystemConfig.APIConfig.GetEndpoint() + if !global.Env().SystemConfig.APIConfig.Enabled && global.Env().SystemConfig.WebAppConfig.Enabled { + instance.Endpoint = global.Env().SystemConfig.WebAppConfig.GetEndpoint() + } else { + instance.Endpoint = global.Env().SystemConfig.APIConfig.GetEndpoint() + } ips := util.GetLocalIPs() if len(ips) > 0 { diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index fd3aeae75..9c0f9990a 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -16,6 +16,10 @@ Information about release notes of INFINI Framework is provided here. ### 🚀 Features - feat: support team-based scope for sharing services #258 - feat: add semantic, hybrid, and nested query support #265 +### 🐛 Bug fix +- fix: use the web endpoint in instance info when the API listener is disabled, and keep managed config sync requests authenticated after registration +### ✈️ Improvements +- chore: add post-register hooks so managed clients can run follow-up steps like token exchange immediately after a successful register - feat: extract BuildFuzzinessQueryClauses as public API #266 - feat(keystore): support large stdin secrets (>1024 bytes) and multiline #271 - feat(cors): add X-SERVICE-ID to allowed CORS headers #275 diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index f863434b2..cb5e28811 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -53,12 +53,43 @@ import ( const bucketName = "instance_registered" const configRegisterEnvKey = "CONFIG_MANAGED_SUCCESS" +var postRegisterHooks []func() error +var postRegisterHooksLock sync.RWMutex + +func AddPostRegisterHook(hook func() error) { + if hook == nil { + return + } + postRegisterHooksLock.Lock() + defer postRegisterHooksLock.Unlock() + postRegisterHooks = append(postRegisterHooks, hook) +} + +func execPostRegisterHooks() error { + postRegisterHooksLock.RLock() + defer postRegisterHooksLock.RUnlock() + for _, hook := range postRegisterHooks { + if hook == nil { + continue + } + if err := hook(); err != nil { + return err + } + } + return nil +} + func ConnectToManager() error { if !global.Env().SystemConfig.Configs.Managed { return nil } + //register to config manager + if global.Env().SystemConfig.Configs.Servers == nil || len(global.Env().SystemConfig.Configs.Servers) == 0 { + return errors.Errorf("no config manager was found") + } + // k8s env setting always_register_after_restart and pod after restart the ip will change so need register again if !global.Env().SystemConfig.Configs.AlwaysRegisterAfterRestart { if exists, err := kv.ExistsKey(bucketName, []byte(global.Env().SystemConfig.NodeConfig.ID)); exists && err == nil { @@ -71,11 +102,6 @@ func ConnectToManager() error { log.Info("register new instance to config manager") - //register to config manager - if global.Env().SystemConfig.Configs.Servers == nil || len(global.Env().SystemConfig.Configs.Servers) == 0 { - return errors.Errorf("no config manager was found") - } - info := model.GetInstanceInfo() req := util.Request{Method: util.Verb_POST} @@ -83,35 +109,51 @@ func ConnectToManager() error { req.Path = common.REGISTER_API req.Body = util.MustToJSONBytes(info) - server, res, err := submitRequestToManager(&req) + server, res, err := DoManagerRequest(&req) if err == nil && server != "" { if res.StatusCode == 200 || util.ContainStr(string(res.Body), "exists") { + if err := execPostRegisterHooks(); err != nil { + return err + } log.Infof("success register to config manager: %v", string(server)) err := kv.AddValue(bucketName, []byte(global.Env().SystemConfig.NodeConfig.ID), []byte(util.GetLowPrecisionCurrentTime().String())) if err != nil { panic(err) } global.Register(configRegisterEnvKey, true) + return nil } + err = errors.Errorf("failed to register to config manager, status: %d, body: %s", res.StatusCode, string(res.Body)) } else { log.Error("failed to register to config manager,", err, ",", server) } return err } -func submitRequestToManager(req *util.Request) (string, *util.Result, error) { +func applyManagerRequestAuth(req *util.Request, username, password, token string) { + if username != "" { + req.SetBasicAuth(username, password) + } + if token != "" { + req.AddHeader(common.API_TOKEN, token) + } +} + +func DoManagerRequest(req *util.Request) (string, *util.Result, error) { var err error var res *util.Result cfg := global.Env().SystemConfig.Configs - if cfg.ManagerConfig.BasicAuth.Username != "" { - req.SetBasicAuth(cfg.ManagerConfig.BasicAuth.Username, cfg.ManagerConfig.BasicAuth.Password.Get()) + applyManagerRequestAuth(req, cfg.ManagerConfig.BasicAuth.Username, cfg.ManagerConfig.BasicAuth.Password.Get(), cfg.ManagerConfig.AccessToken.Get()) + httpClient, err := getManagerHTTPClient() + if err != nil { + return "", nil, err } for _, server := range cfg.Servers { req.Url, err = url.JoinPath(server, req.Path) if err != nil { continue } - res, err = util.ExecuteRequestWithCatchFlag(mTLSClient, req, true) + res, err = util.ExecuteRequestWithCatchFlag(httpClient, req, true) if err != nil { continue } @@ -120,21 +162,29 @@ func submitRequestToManager(req *util.Request) (string, *util.Result, error) { return "", nil, err } -var clientInitLock = sync.Once{} -var mTLSClient *http.Client +var managerHTTPClientOnce sync.Once +var managerHTTPClient *http.Client +var managerHTTPClientErr error -func ListenConfigChanges() error { +func getManagerHTTPClient() (*http.Client, error) { + managerHTTPClientOnce.Do(func() { + cfg := global.Env().GetHTTPClientConfig("configs", "") + if cfg == nil { + return + } + managerHTTPClient, managerHTTPClientErr = api.NewHTTPClient(cfg) + }) + return managerHTTPClient, managerHTTPClientErr +} + +var clientInitLock sync.Once +func ListenConfigChanges() error { clientInitLock.Do(func() { if global.Env().SystemConfig.Configs.Managed { - cfg := global.Env().GetHTTPClientConfig("configs", "") - if cfg != nil { - hClient, err := api.NewHTTPClient(cfg) - if err != nil { - panic(err) - } - mTLSClient = hClient + if _, err := getManagerHTTPClient(); err != nil { + panic(err) } //init config sync listening @@ -160,7 +210,7 @@ func ListenConfigChanges() error { log.Debug("config sync request: ", string(util.MustToJSONBytes(req))) } - _, res, err := submitRequestToManager(&request) + _, res, err := DoManagerRequest(&request) if err != nil { log.Error("failed to submit request to config manager,", err) return diff --git a/modules/configs/client/client_test.go b/modules/configs/client/client_test.go new file mode 100644 index 000000000..0ee3c560a --- /dev/null +++ b/modules/configs/client/client_test.go @@ -0,0 +1,28 @@ +package client + +import ( + "testing" + + "infini.sh/framework/core/util" + "infini.sh/framework/modules/configs/common" +) + +func TestApplyManagerRequestAuthAddsRegisterTokenHeader(t *testing.T) { + req := util.Request{Path: common.REGISTER_API} + + applyManagerRequestAuth(&req, "", "", "token-1") + + if got := req.AllHeaders()[common.API_TOKEN]; got != "token-1" { + t.Fatalf("expected register token header %q, got %q", "token-1", got) + } +} + +func TestApplyManagerRequestAuthAddsTokenHeaderForSync(t *testing.T) { + req := util.Request{Path: common.SYNC_API} + + applyManagerRequestAuth(&req, "", "", "token-1") + + if got := req.AllHeaders()[common.API_TOKEN]; got != "token-1" { + t.Fatalf("expected sync token header %q, got %q", "token-1", got) + } +} diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go index 232c1fc52..7e51c38bf 100644 --- a/modules/configs/common/domain.go +++ b/modules/configs/common/domain.go @@ -31,6 +31,7 @@ import "infini.sh/framework/core/model" const REGISTER_API = "/instance/_register" const SYNC_API = "/configs/_sync" +const API_TOKEN = "X-API-TOKEN" type ConfigFile struct { Name string `json:"name,omitempty"` diff --git a/modules/security/access_token/authentication.go b/modules/security/access_token/authentication.go index dddd31476..a267aa713 100644 --- a/modules/security/access_token/authentication.go +++ b/modules/security/access_token/authentication.go @@ -87,7 +87,7 @@ func byAPITokenHeader(w http.ResponseWriter, r *http.Request) (claims *security. apiToken := r.Header.Get(HeaderAPIToken) accessToken, permissions, err := getTokenPermissions(apiToken) - if err!=nil { + if err != nil { return nil, err } @@ -101,9 +101,9 @@ func byAPITokenHeader(w http.ResponseWriter, r *http.Request) (claims *security. return claims, nil } -func getTokenPermissions(apiToken string) (*security.AccessToken, []security.PermissionKey,error) { +func getTokenPermissions(apiToken string) (*security.AccessToken, []security.PermissionKey, error) { if apiToken == "" { - return nil,nil, errors.Error("api token not found") + return nil, nil, errors.Error("api token not found") } bytes, err := kv.GetValue(KVAccessTokenBucket, []byte(apiToken)) @@ -123,7 +123,7 @@ func getTokenPermissions(apiToken string) (*security.AccessToken, []security.Per } //-1 means never expire - if accessToken.ExpireIn>0 { + if accessToken.ExpireIn > 0 { expireAtTime := time.Unix(accessToken.ExpireIn, 0) if time.Now().After(expireAtTime) { return nil, nil, errors.Error("token expired") @@ -202,7 +202,7 @@ func RequestAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter. } expiredAT := time.Now().Add(365 * 24 * time.Hour).Unix() - res, err := CreateAPIToken(reqUser, reqBody.Name, "general",expiredAT, permissions) + res, err := CreateAPIToken(reqUser, reqBody.Name, "general", expiredAT, permissions) if err != nil { panic(err) } @@ -210,7 +210,7 @@ func RequestAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter. api.WriteJSON(w, res, 200) } -func CreateAPIToken(user *security.UserSessionInfo, tokenName, typeName string,expiredAT int64 , permissions []security.PermissionKey) (util.MapStr, error) { +func CreateAPIToken(user *security.UserSessionInfo, tokenName, typeName string, expiredAT int64, permissions []security.PermissionKey) (util.MapStr, error) { if tokenName == "" { tokenName = GenerateApiTokenName("") From c8b12796b2008d903e1556e349caf2cc517253c5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 25 May 2026 07:44:33 +0000 Subject: [PATCH 4/7] Merge origin/main into pr/framework-managed-token-flow-20260524 Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/61a03f37-2d99-48ff-b97e-934d39469c33 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> --- .github/workflows/commit-message-check.yml | 1 + Makefile | 8 +- app.go | 2 +- cmd/vfs/main.go | 2 +- core/host/sys_info.go | 10 +- core/pipeline/config.go | 81 ++- core/pipeline/context.go | 26 +- core/pipeline/processor.go | 3 + docs/content.en/docs/references/api_web.md | 119 ++++ docs/content.en/docs/release-notes/_index.md | 5 +- go.mod | 24 +- go.sum | 86 +-- lib/status/fs.go | 2 +- modules/metrics/host/cpu/cpu.go | 4 +- modules/metrics/host/disk/disk.go | 2 +- modules/metrics/host/memory/memory.go | 2 +- modules/metrics/host/network/network.go | 2 +- .../metrics/host/network/sockstat_linux.go | 2 +- .../metrics/host/network/sockstat_other.go | 2 +- .../metrics/host/overall/netspeed_darwin.go | 89 +++ .../metrics/host/overall/netspeed_linux.go | 75 +++ .../metrics/host/overall/netspeed_windows.go | 142 ++++ modules/metrics/host/overall/overall.go | 636 ++++++++++++++++++ modules/metrics/metrics.go | 24 + modules/pipeline/model.go | 2 +- modules/pipeline/module.go | 609 +++++++++++++++++ modules/pipeline/pipeline.go | 630 +++-------------- modules/pipeline/proto.go | 4 +- modules/pipeline/{api.go => tasks.go} | 41 +- .../security/access_token/authentication.go | 6 +- .../security/access_token/authorization.go | 7 +- modules/security/rbac/role.go | 2 +- modules/stats/simple.go | 2 +- 33 files changed, 1994 insertions(+), 658 deletions(-) create mode 100644 modules/metrics/host/overall/netspeed_darwin.go create mode 100644 modules/metrics/host/overall/netspeed_linux.go create mode 100644 modules/metrics/host/overall/netspeed_windows.go create mode 100644 modules/metrics/host/overall/overall.go create mode 100755 modules/pipeline/module.go mode change 100755 => 100644 modules/pipeline/pipeline.go rename modules/pipeline/{api.go => tasks.go} (74%) diff --git a/.github/workflows/commit-message-check.yml b/.github/workflows/commit-message-check.yml index 45af0b0f2..331ae5ce7 100644 --- a/.github/workflows/commit-message-check.yml +++ b/.github/workflows/commit-message-check.yml @@ -1,6 +1,7 @@ name: 'commit-message-check' on: pull_request: + types: [opened, synchronize, reopened, edited] jobs: check-commit-message: diff --git a/Makefile b/Makefile index 90c8bdb2f..a1ae31188 100755 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ APP_UI_FOLDER ?= ui APP_PLUGIN_FOLDER ?= plugins APP_PLUGIN_PKG ?= $(APP_PLUGIN_FOLDER) APP_NEED_CGO ?= 0 +GOLANGCI_LINT_VERSION ?= v2.12.2 # Get release version from environment ifneq '$(VERSION)' '' @@ -242,7 +243,10 @@ cross-build-all-platform: clean config build-bsd build-linux build-darwin build- format: @echo "formatting code" - $(GO) fmt $$($(GO) list ./...) + find . -type f -name '*.go' \ + -not -path './vendor/*' \ + -not -path './.git/*' \ + -exec gofmt -w {} + test: config $(GOTEST) -v $(GOFLAGS) -timeout 30m ./... @@ -257,7 +261,7 @@ tidy: lint: config @if ! command -v golangci-lint >/dev/null 2>&1; then \ echo "Installing golangci-lint..."; \ - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v2.1.6; \ + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(shell go env GOPATH)/bin $(GOLANGCI_LINT_VERSION); \ fi golangci-lint run @$(MAKE) restore-generated-file diff --git a/app.go b/app.go index af18713b2..30e73c761 100755 --- a/app.go +++ b/app.go @@ -40,7 +40,7 @@ import ( "time" "github.com/fsnotify/fsnotify" - "github.com/shirou/gopsutil/v3/process" + "github.com/shirou/gopsutil/v4/process" "infini.sh/framework/core/task" "infini.sh/framework/core/wrapper/taskset" "infini.sh/framework/modules/configs/client" diff --git a/cmd/vfs/main.go b/cmd/vfs/main.go index 3a27f2ba0..dfe5f3593 100755 --- a/cmd/vfs/main.go +++ b/cmd/vfs/main.go @@ -232,7 +232,7 @@ import ( "bytes" "infini.sh/framework/core/util/zstd" "encoding/base64" - log "github.com/cihub/seelog" + "infini.sh/framework/core/log" "infini.sh/framework/core/errors" "infini.sh/framework/core/util" "infini.sh/framework/core/vfs" diff --git a/core/host/sys_info.go b/core/host/sys_info.go index 13317ffe3..f59b533c8 100644 --- a/core/host/sys_info.go +++ b/core/host/sys_info.go @@ -26,11 +26,11 @@ package host import ( "fmt" log "github.com/cihub/seelog" - "github.com/shirou/gopsutil/v3/cpu" - "github.com/shirou/gopsutil/v3/disk" - "github.com/shirou/gopsutil/v3/host" - "github.com/shirou/gopsutil/v3/mem" - "github.com/shirou/gopsutil/v3/net" + "github.com/shirou/gopsutil/v4/cpu" + "github.com/shirou/gopsutil/v4/disk" + "github.com/shirou/gopsutil/v4/host" + "github.com/shirou/gopsutil/v4/mem" + "github.com/shirou/gopsutil/v4/net" "infini.sh/framework/core/errors" "runtime" "time" diff --git a/core/pipeline/config.go b/core/pipeline/config.go index 0d1dfac08..ceec5f42d 100644 --- a/core/pipeline/config.go +++ b/core/pipeline/config.go @@ -26,24 +26,87 @@ package pipeline import ( "infini.sh/framework/core/config" "infini.sh/framework/core/errors" + "infini.sh/framework/core/orm" "infini.sh/framework/core/util" "infini.sh/framework/lib/go-ucfg" ) +// PipelineConfig is the declarative definition of a pipeline: a named, +// ordered chain of processors plus the policies that govern how the framework +// runs it. +// +// It is pure configuration. Once loaded into memory and instantiated, a +// PipelineConfigV2 becomes a PipelineTask — the live runtime form, consisting +// of a compiled processor chain, an execution context, and a managed +// goroutine. type PipelineConfigV2 struct { - Name string `config:"name" json:"name,omitempty"` - Enabled *bool `config:"enabled" json:"enabled,omitempty"` - Singleton bool `config:"singleton" json:"singleton"` - AutoStart bool `config:"auto_start" json:"auto_start"` - KeepRunning bool `config:"keep_running" json:"keep_running"` - RetryDelayInMs int `config:"retry_delay_in_ms" json:"retry_delay_in_ms"` - MaxRunningInMs int64 `config:"max_running_in_ms" json:"max_running_in_ms"` - Logging struct { + orm.ORMObjectBase + + // Human-readable identifier for this pipeline. + // Required when creating a pipeline via the API. + Name string `config:"name" json:"name,omitempty"` + + // A disabled pipeline is treated as if it does not exist. Nil means + // "use the module default". + Enabled *bool `config:"enabled" json:"enabled,omitempty"` + + // Singleton ensures that at most one instance of this pipeline runs + // across the entire cluster (not just within a single node). It is + // enforced jointly by two things: + // 1. Setting this field to true. + // 2. MaxRunningInMs — used as the TTL of the distributed lock that + // guards the singleton slot. The lock prevents other nodes from + // starting a competing instance while one is already running. + Singleton bool `config:"singleton" json:"singleton"` + + // AutoStart controls the initial running state once the pipeline is + // loaded into memory. When true the pipeline begins executing + // immediately; when false it stays idle until an explicit start request + // arrives. + AutoStart bool `config:"auto_start" json:"auto_start"` + + // KeepRunning marks the pipeline as long-lived: after the processor + // chain finishes one pass it loops and runs again. When false the + // pipeline executes once and exits. + KeepRunning bool `config:"keep_running" json:"keep_running"` + + // RetryDelayInMs is the pause between successive runs of the processor + // chain in the keep-running loop. Applies whether the previous run + // finished successfully or failed. + // + // If it is smaller than or equal to 0, use the default value 1000. + RetryDelayInMs int `config:"retry_delay_in_ms" json:"retry_delay_in_ms"` + + // MaxRunningInMs is the TTL of the cross-node singleton lock: the + // maximum time this node is allowed to hold the lock for one run. Once + // the TTL elapses the lock is considered released and another node may + // claim the singleton slot. The running goroutine itself is unaffected + // — it is NOT canceled or killed when the TTL expires. + // + // Only consulted when Singleton is true. When the value is zero or + // negative it falls back to a built-in default of 60000 ms (1 minute). + // + // NOTE: the name of this field is misleading — it sounds like a run + // timeout but is actually a lock TTL. It is kept for backwards + // compatibility. + MaxRunningInMs int64 `config:"max_running_in_ms" json:"max_running_in_ms"` + + Logging struct { Enabled bool `config:"enabled" json:"enabled"` } `config:"logging" json:"logging"` + + // Processors is the ordered chain of processor definitions that makes + // up the actual work the pipeline performs. Each entry is a single + // processor config keyed by processor type. Processors []map[string]interface{} `config:"processor" json:"processor"` - Labels map[string]interface{} `config:"labels" json:"labels"` + // Labels are arbitrary metadata attached to the pipeline. + Labels map[string]interface{} `config:"labels" json:"labels"` + + // Transient, if false, marks that this pipeline config shpuld be persisted. + // + // In the current implementation, only the pipeline configs created with the + // `POST /pipeline` API will set this to true. Transient bool `config:"-" json:"transient"` } diff --git a/core/pipeline/context.go b/core/pipeline/context.go index 62008306a..97fe18af8 100755 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -93,12 +93,21 @@ type Context struct { id string steps int64 - cancelFunc context.CancelFunc - isPaused bool - pause sync.WaitGroup - isQuit bool - stateLock sync.Mutex - released bool + // cancelFunc closes the Done channel of the embedded context.Context, + // signaling processors to stop early. + // + // This is a cooperative mechanism: it only takes effect if the processor's process() + // implementation explicitly checks IsCanceled() and returns when it is true. + cancelFunc context.CancelFunc + // True means the goroutine has been paused/suspended. + isPaused bool + pause sync.WaitGroup + // Set this to true if you want to stop the pipeline, and then, pause (suspend) the goroutine. + isQuit bool + stateLock sync.Mutex + // Set this to true if you want to let the goroutine exit, i.e., the kill signal. + released bool + // True means the goroutine already exited. loopReleased bool } @@ -107,6 +116,7 @@ func AcquireContext(config PipelineConfigV2) *Context { ctx.ResetContext() ctx.id = util.GetUUID() ctx.createTime = time.Now() + // Placeholder state; the pipeline task execution loop will overwrite this. ctx.runningState = FINISHED ctx.Config = config return &ctx @@ -289,7 +299,7 @@ func (ctx *Context) Errors() []error { return ctx.processErrs } -// Pause will pause the pipeline running loop until Resume called +// Pause suspends the goroutine that is running this pipeline. func (ctx *Context) Pause() { ctx.stateLock.Lock() if ctx.isPaused { @@ -303,7 +313,7 @@ func (ctx *Context) Pause() { ctx.pause.Wait() } -// Resume recovers pipeline from Pause +// Resume wakes up the goroutine that was suspended by Pause. func (ctx *Context) Resume() { ctx.stateLock.Lock() if !ctx.isPaused { diff --git a/core/pipeline/processor.go b/core/pipeline/processor.go index 802f89f9e..2fd4f2284 100755 --- a/core/pipeline/processor.go +++ b/core/pipeline/processor.go @@ -62,6 +62,9 @@ type Releaser interface { Release() error } +// Processors is an implementation of the Processor interface that holds an +// ordered list of child Processors and runs them in sequence when its Process +// method is called. type Processors struct { SkipCatchError bool // skip catch internal error List []Processor diff --git a/docs/content.en/docs/references/api_web.md b/docs/content.en/docs/references/api_web.md index f3808a26c..2b686e4ad 100644 --- a/docs/content.en/docs/references/api_web.md +++ b/docs/content.en/docs/references/api_web.md @@ -362,6 +362,125 @@ api.HandleUIMethod(api.GET, "/health", handler.healthCheck, api.AllowPublicAccess()) ``` +### Access Token Authentication + +The framework provides a built-in access token (API token) module for programmatic authentication. Access tokens allow services and automation tools to authenticate API requests without interactive login sessions. + +#### How It Works + +Requests carrying an `X-API-TOKEN` header are automatically authenticated by the framework's HTTP auth filter pipeline. The token is validated against the KV store, checked for expiration, and the caller is granted the intersection of the token's permissions and its owner's current permissions (in native mode). + +#### Configuration + +Enable access token authentication in the web server security configuration: + +```yaml +web: + security: + enabled: true + authentication: + access_token: + native: true # Use ORM-backed persistence (requires a backend store) +``` + +When `native` is `false`, the module operates in KV-only mode suitable for lightweight/agent-style deployments without a full ORM backend. + +#### API Endpoints + +| Method | Path | Description | Permission | +|--------|------|-------------|------------| +| `POST` | `/auth/access_token` | Create a new access token | `security:auth:api-token:create` | +| `GET` | `/auth/access_token/_search` | Search/list access tokens | `security:auth:api-token:search` | +| `PUT` | `/auth/access_token/:token_id` | Update an access token | `security:auth:api-token:update` | +| `DELETE` | `/auth/access_token/:token_id` | Delete an access token | `security:auth:api-token:delete` | + +#### Creating an Access Token + +Send a `POST` request to `/auth/access_token` (requires an authenticated session): + +**Request:** + +```json +{ + "name": "my-ci-token", + "permissions": ["category:resource:read", "category:resource:search"] +} +``` + +- `name` (optional): A human-readable name for the token. Auto-generated if omitted. +- `permissions` (optional): A subset of the caller's own permissions to assign to the token. If omitted, the token inherits all of the caller's permissions. + +**Response:** + +```json +{ + "_id": "token-uuid", + "access_token": "generated-token-string", + "expire_in": 1748102400 +} +``` + +The `access_token` value is the secret string used in subsequent API calls. Store it securely — it cannot be retrieved again after creation. + +#### Using an Access Token + +Include the token in the `X-API-TOKEN` HTTP header: + +```bash +curl -H "X-API-TOKEN: " https://localhost:9200/_api/resource/_search +``` + +#### Updating an Access Token + +Send a `PUT` request to `/auth/access_token/:token_id`: + +```json +{ + "name": "renamed-token", + "permissions": ["category:resource:read"] +} +``` + +- `name` (required): The new name for the token. +- `permissions` (optional): Updated permissions. Must be a subset of the caller's current permissions. + +#### Deleting an Access Token + +Send a `DELETE` request to `/auth/access_token/:token_id`. The token is immediately invalidated and can no longer be used for authentication. + +#### Token Expiration + +Tokens are created with a default expiration of 1 year. A token with `expire_in` set to `-1` never expires. Expired tokens are rejected at authentication time. + +#### Permission Model + +In **native mode**, the effective permissions of a token are computed as: + +$$\text{effective} = \text{token\_permissions} \cap \text{owner\_current\_permissions}$$ + +This ensures that if a user's permissions are revoked after a token is issued, the token's effective permissions are automatically narrowed. Permission changes take effect immediately (the permission cache version is incremented on every token update/delete). + +In **non-native (KV-only) mode**, the token's stored permissions are used directly without intersection. + +#### Programmatic Token Creation + +Tokens can also be created programmatically from Go code: + +```go +import "infini.sh/framework/modules/security/access_token" + +// Create a token for a user with specific permissions +result, err := access_token.CreateAPIToken( + userSession, // *security.UserSessionInfo - token owner + "service-token", // token name + "general", // token type + expireUnix, // expiration as Unix timestamp + permissions, // []security.PermissionKey +) +// result["access_token"] contains the token string +// result["_id"] contains the token ID +``` + ### Label and Metadata ```go diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index 9c0f9990a..fdeae5174 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -24,6 +24,9 @@ Information about release notes of INFINI Framework is provided here. - feat(keystore): support large stdin secrets (>1024 bytes) and multiline #271 - feat(cors): add X-SERVICE-ID to allowed CORS headers #275 - feat: output HTTP access logs to file +- feat(metrics): monitor each disk and network interface independently for bottleneck detection +- feat(metrics): auto-detect network interface bandwidth per device (Linux, macOS, Windows) +- feat(metrics): identify specific bottleneck device (e.g., `disk_io:nvme0n1`, `network:eth0`) - feat(cookie): prevent aggressive session cookie expiration #284 - feat(client): support token-based authorization #288 - feat: add pluggable sink to host metrics collectors #288 @@ -49,7 +52,7 @@ Information about release notes of INFINI Framework is provided here. - chore: permissions refactoring - chore: security configuration structure enhanced - chore: remove unused grpc and cuckoo filter" - +- chore: update seelog for vfs #363 ## 1.4.0 (2025-12-19) diff --git a/go.mod b/go.mod index 43dc0a6be..bc3714b89 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/andybalholm/brotli v1.1.1 github.com/arl/statsviz v0.6.0 github.com/bkaradzic/go-lz4 v1.0.0 - github.com/buger/jsonparser v1.1.1 + github.com/buger/jsonparser v1.1.2 github.com/caddyserver/certmagic v0.25.3 github.com/cihub/seelog v0.0.0-00010101000000-000000000000 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc @@ -20,7 +20,7 @@ require ( github.com/fsnotify/fsnotify v1.9.0 github.com/go-ldap/ldap/v3 v3.4.13 github.com/go-redis/redis/v8 v8.11.5 - github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/golang-jwt/jwt/v4 v4.5.2 github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f github.com/google/go-cmp v0.7.0 github.com/google/go-github v17.0.0+incompatible @@ -33,6 +33,7 @@ require ( github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/kardianos/service v1.2.2 github.com/klauspost/compress v1.18.0 + github.com/libdns/libdns v1.1.1 github.com/magiconair/properties v1.8.10 github.com/mailru/easyjson v0.9.0 github.com/minio/minio-go/v7 v7.0.90 @@ -45,11 +46,10 @@ require ( github.com/ryanuber/go-glob v1.0.0 github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287 github.com/segmentio/encoding v0.4.1 - github.com/seiflotfy/cuckoofilter v0.0.0-20240715131351-a2f2c23f1771 github.com/shaj13/go-guardian/v2 v2.11.6 - github.com/shirou/gopsutil/v3 v3.24.5 + github.com/shirou/gopsutil/v4 v4.26.3 github.com/spf13/viper v1.20.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/twmb/franz-go v1.18.1 github.com/twmb/franz-go/pkg/kadm v1.16.0 github.com/twmb/franz-go/pkg/kmsg v1.11.2 @@ -63,6 +63,7 @@ require ( golang.org/x/text v0.36.0 golang.org/x/time v0.11.0 golang.org/x/tools v0.44.0 + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/hjson/hjson-go.v3 v3.3.0 gopkg.in/square/go-jose.v2 v2.6.0 @@ -77,9 +78,9 @@ require ( github.com/caddyserver/zerossl v0.1.5 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect - github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.0 // indirect github.com/fatih/color v1.18.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect @@ -90,7 +91,6 @@ require ( github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/flatbuffers v25.2.10+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -102,7 +102,6 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect - github.com/libdns/libdns v1.1.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -116,19 +115,18 @@ require ( github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/segmentio/asm v1.1.3 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.12.0 // indirect github.com/spf13/cast v1.7.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect github.com/x448/float16 v0.8.4 // indirect @@ -144,9 +142,7 @@ require ( golang.org/x/sync v0.20.0 // indirect golang.org/x/term v0.42.0 // indirect google.golang.org/appengine v1.6.6 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect google.golang.org/protobuf v1.36.6 // indirect - gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect diff --git a/go.sum b/go.sum index 3d3f64898..e1f48ebba 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +code.pfad.fr/check v1.1.0 h1:GWvjdzhSEgHvEHe2uJujDcpmZoySKuHQNrZMfzfO0bE= +code.pfad.fr/check v1.1.0/go.mod h1:NiUH13DtYsb7xp5wll0U4SXx7KhXQVCtRgdC96IPfoM= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= -github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A= github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -12,9 +12,8 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= -github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= -github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI= +github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/arl/statsviz v0.6.0 h1:jbW1QJkEYQkufd//4NDYRSNBpwJNrdzPahF7ZmoGdyE= @@ -24,8 +23,8 @@ github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6 github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/caddyserver/certmagic v0.25.3 h1:mGf5ba8F7xA4c5jfDZZbK2buY1VEkbnwpMDixaju94A= github.com/caddyserver/certmagic v0.25.3/go.mod h1:YVs43D5+H/Dckt4bTga1KSO/xYfFBfVZainGDywYPAA= github.com/caddyserver/zerossl v0.1.5 h1:dkvOjBAEEtY6LIGAHei7sw2UgqSD6TrWweXpV7lvEvE= @@ -44,13 +43,13 @@ github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINA github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 h1:BS21ZUJ/B5X2UVUbczfmdWH7GapPWAhxcMsDnjJTU1E= -github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -73,9 +72,9 @@ github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-ldap/ldap/v3 v3.2.4/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg= -github.com/go-ldap/ldap/v3 v3.4.11 h1:4k0Yxweg+a3OyBLjdYn5OKglv18JNvfDykSoI8bW0gU= -github.com/go-ldap/ldap/v3 v3.4.11/go.mod h1:bY7t0FLK8OAVpp/vV6sSlpz3EQDGcQwc8pF0ujLgKvM= github.com/go-ldap/ldap/v3 v3.4.13 h1:+x1nG9h+MZN7h/lUi5Q3UZ0fJ1GyDQYbPvbuH38baDQ= github.com/go-ldap/ldap/v3 v3.4.13/go.mod h1:LxsGZV6vbaK0sIvYfsv47rfh4ca0JXokCoKjZxsszv0= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= @@ -100,8 +99,6 @@ github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PU github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f h1:16RtHeWGkJMc80Etb8RPCcKevXGldr57+LOyZt8zOlg= @@ -192,8 +189,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -204,6 +199,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/letsencrypt/challtestsrv v1.4.2 h1:0ON3ldMhZyWlfVNYYpFuWRTmZNnyfiL9Hh5YzC3JVwU= +github.com/letsencrypt/challtestsrv v1.4.2/go.mod h1:GhqMqcSoeGpYd5zX5TgwA6er/1MbWzx/o7yuuVya+Wk= +github.com/letsencrypt/pebble/v2 v2.10.0 h1:Wq6gYXlsY6ubqI3hhxsTzdyotvfdjFBxuwYqCLCnj/U= +github.com/letsencrypt/pebble/v2 v2.10.0/go.mod h1:Sk8cmUIPcIdv2nINo+9PB4L+ZBhzY+F9A1a/h/xmWiQ= github.com/libdns/libdns v1.1.1 h1:wPrHrXILoSHKWJKGd0EiAVmiJbFShguILTg9leS/P/U= github.com/libdns/libdns v1.1.1/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -223,12 +222,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mholt/acmez/v3 v3.1.2 h1:auob8J/0FhmdClQicvJvuDavgd5ezwLBfKuYmynhYzc= -github.com/mholt/acmez/v3 v3.1.2/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= github.com/mholt/acmez/v3 v3.1.6 h1:eGVQNObP0pBN4sxqrXeg7MYqTOWyoiYpQqITVWlrevk= github.com/mholt/acmez/v3 v3.1.6/go.mod h1:5nTPosTGosLxF3+LU4ygbgMRFDhbAVpqMI4+a4aHLBY= -github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY= -github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= @@ -275,8 +270,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/r3labs/diff/v2 v2.15.1 h1:EOrVqPUzi+njlumoqJwiS/TgGgmZo83619FNDB9xQUg= github.com/r3labs/diff/v2 v2.15.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= @@ -297,17 +292,11 @@ github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.4.1 h1:KLGaLSW0jrmhB58Nn4+98spfvPvmo4Ci1P/WIQ9wn7w= github.com/segmentio/encoding v0.4.1/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrWPgtJHYZSnI= -github.com/seiflotfy/cuckoofilter v0.0.0-20240715131351-a2f2c23f1771 h1:emzAzMZ1L9iaKCTxdy3Em8Wv4ChIAGnfiz18Cda70g4= -github.com/seiflotfy/cuckoofilter v0.0.0-20240715131351-a2f2c23f1771/go.mod h1:bR6DqgcAl1zTcOX8/pE2Qkj9XO00eCNqmKb7lXP8EAg= github.com/shaj13/go-guardian/v2 v2.11.6 h1:N0UgnL+AI0IH59eii0H0QnQEesyPPmGFB1h9g1MkZ8g= github.com/shaj13/go-guardian/v2 v2.11.6/go.mod h1:rSe5VLuWu9EyUT68Xi6qxb/DJc+ajiqPAq+VKhEUKkE= github.com/shaj13/libcache v1.0.0/go.mod h1:YCq92Zosqj4erhlLdm2Mu1cX2FDAxjfFOxTphzN7S9U= -github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= -github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -333,14 +322,14 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/twmb/franz-go v1.18.1 h1:D75xxCDyvTqBSiImFx2lkPduE39jz1vaD7+FNc+vMkc= github.com/twmb/franz-go v1.18.1/go.mod h1:Uzo77TarcLTUZeLuGq+9lNpSkfZI+JErv7YJhlDjs9M= github.com/twmb/franz-go/pkg/kadm v1.16.0 h1:STMs1t5lYR5mR974PSiwNzE5TvsosByTp+rKXLOhAjE= @@ -375,18 +364,12 @@ go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= @@ -395,14 +378,10 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -413,8 +392,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -425,8 +402,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -439,22 +414,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= -golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -466,8 +433,6 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= -golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -479,8 +444,6 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20170918111702-1e559d0a00ee/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= @@ -495,8 +458,6 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= -gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= -gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/hjson/hjson-go.v3 v3.3.0 h1:F/aKL7cJ3rTfjQIxdetssl+ryKBz0V1mLJnrBs6ljFg= gopkg.in/hjson/hjson-go.v3 v3.3.0/go.mod h1:X6zrTSVeImfwfZLfgQdInl9mWjqPqgH90jom9nym/lw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -511,7 +472,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.18.8/go.mod h1:d/CXqwWv+Z2XEG1LgceeDmHQwpUJhROPx16SlxJgERY= diff --git a/lib/status/fs.go b/lib/status/fs.go index 95910343a..ee0d4370f 100644 --- a/lib/status/fs.go +++ b/lib/status/fs.go @@ -4,7 +4,7 @@ package status import ( log "github.com/cihub/seelog" - disk2 "github.com/shirou/gopsutil/v3/disk" + disk2 "github.com/shirou/gopsutil/v4/disk" ) type DiskStatus struct { diff --git a/modules/metrics/host/cpu/cpu.go b/modules/metrics/host/cpu/cpu.go index acd3f69ad..4e2ff1c24 100644 --- a/modules/metrics/host/cpu/cpu.go +++ b/modules/metrics/host/cpu/cpu.go @@ -31,8 +31,8 @@ import ( "strconv" log "github.com/cihub/seelog" - "github.com/shirou/gopsutil/v3/cpu" - "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v4/cpu" + "github.com/shirou/gopsutil/v4/load" "infini.sh/framework/core/config" "infini.sh/framework/core/event" "infini.sh/framework/core/util" diff --git a/modules/metrics/host/disk/disk.go b/modules/metrics/host/disk/disk.go index b994fba10..28f2d7fbf 100644 --- a/modules/metrics/host/disk/disk.go +++ b/modules/metrics/host/disk/disk.go @@ -34,7 +34,7 @@ import ( "strings" log "github.com/cihub/seelog" - "github.com/shirou/gopsutil/v3/disk" + "github.com/shirou/gopsutil/v4/disk" "infini.sh/framework/core/config" "infini.sh/framework/core/event" "infini.sh/framework/core/util" diff --git a/modules/metrics/host/memory/memory.go b/modules/metrics/host/memory/memory.go index 53ea9a9cf..1b8cb0413 100644 --- a/modules/metrics/host/memory/memory.go +++ b/modules/metrics/host/memory/memory.go @@ -32,7 +32,7 @@ import ( "strings" log "github.com/cihub/seelog" - "github.com/shirou/gopsutil/v3/mem" + "github.com/shirou/gopsutil/v4/mem" "infini.sh/framework/core/config" "infini.sh/framework/core/errors" "infini.sh/framework/core/event" diff --git a/modules/metrics/host/network/network.go b/modules/metrics/host/network/network.go index e867753dc..fda0bc965 100644 --- a/modules/metrics/host/network/network.go +++ b/modules/metrics/host/network/network.go @@ -31,7 +31,7 @@ import ( "syscall" log "github.com/cihub/seelog" - "github.com/shirou/gopsutil/v3/net" + "github.com/shirou/gopsutil/v4/net" "infini.sh/framework/core/config" "infini.sh/framework/core/errors" "infini.sh/framework/core/event" diff --git a/modules/metrics/host/network/sockstat_linux.go b/modules/metrics/host/network/sockstat_linux.go index ccdf3c3c6..f64047004 100644 --- a/modules/metrics/host/network/sockstat_linux.go +++ b/modules/metrics/host/network/sockstat_linux.go @@ -47,7 +47,7 @@ import ( "bufio" "fmt" "github.com/pkg/errors" - "github.com/shirou/gopsutil/v3/net" + "github.com/shirou/gopsutil/v4/net" "infini.sh/framework/core/util" "os" ) diff --git a/modules/metrics/host/network/sockstat_other.go b/modules/metrics/host/network/sockstat_other.go index b9c878407..cce297723 100644 --- a/modules/metrics/host/network/sockstat_other.go +++ b/modules/metrics/host/network/sockstat_other.go @@ -44,7 +44,7 @@ package network import ( - "github.com/shirou/gopsutil/v3/net" + "github.com/shirou/gopsutil/v4/net" "infini.sh/framework/core/util" ) diff --git a/modules/metrics/host/overall/netspeed_darwin.go b/modules/metrics/host/overall/netspeed_darwin.go new file mode 100644 index 000000000..482038cf2 --- /dev/null +++ b/modules/metrics/host/overall/netspeed_darwin.go @@ -0,0 +1,89 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build darwin + +package overall + +import ( + "os/exec" + "strings" + + log "github.com/cihub/seelog" + "github.com/shirou/gopsutil/v4/net" +) + +// detectNetworkBandwidthPerInterface detects network interface speeds in Mbps for each interface. +// On macOS, it uses ifconfig to get network interface speeds. +// Returns a map of interface name to bandwidth in Mbps. +func detectNetworkBandwidthPerInterface() map[string]float64 { + result := make(map[string]float64) + + interfaces, err := net.IOCounters(true) + if err != nil { + log.Debugf("overall: failed to get network interfaces: %v", err) + return result + } + + for _, iface := range interfaces { + // Skip loopback and virtual interfaces + if isVirtualInterface(iface.Name) { + continue + } + + speed := detectDarwinInterfaceSpeed(iface.Name) + if speed > 0 { + log.Debugf("overall: detected interface %s speed: %.0f Mbps", iface.Name, speed) + result[iface.Name] = speed + } + } + + return result +} + +// detectDarwinInterfaceSpeed attempts to detect the speed of a single interface on macOS +func detectDarwinInterfaceSpeed(ifaceName string) float64 { + // Try ifconfig for link speed + out, err := exec.Command("ifconfig", ifaceName).Output() + if err != nil { + return 0 + } + + output := string(out) + + // Look for "media: autoselect (1000baseT )" + if strings.Contains(output, "10Gbase") || strings.Contains(output, "10GBASE") { + return 10000 + } + if strings.Contains(output, "1000baseT") || strings.Contains(output, "1000BASE-T") { + return 1000 + } + if strings.Contains(output, "100baseT") || strings.Contains(output, "100BASE-T") { + return 100 + } + if strings.Contains(output, "10baseT") || strings.Contains(output, "10BASE-T") { + return 10 + } + + return 0 +} diff --git a/modules/metrics/host/overall/netspeed_linux.go b/modules/metrics/host/overall/netspeed_linux.go new file mode 100644 index 000000000..ea5649d16 --- /dev/null +++ b/modules/metrics/host/overall/netspeed_linux.go @@ -0,0 +1,75 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build linux + +package overall + +import ( + "fmt" + "os" + "strconv" + "strings" + + log "github.com/cihub/seelog" + "github.com/shirou/gopsutil/v4/net" +) + +// detectNetworkBandwidthPerInterface detects network interface speeds in Mbps for each interface. +// On Linux, it reads from /sys/class/net//speed for each interface. +// Returns a map of interface name to bandwidth in Mbps. +func detectNetworkBandwidthPerInterface() map[string]float64 { + result := make(map[string]float64) + + interfaces, err := net.IOCounters(true) + if err != nil { + log.Debugf("overall: failed to get network interfaces: %v", err) + return result + } + + for _, iface := range interfaces { + // Skip loopback and virtual interfaces + if isVirtualInterface(iface.Name) { + continue + } + + speedPath := fmt.Sprintf("/sys/class/net/%s/speed", iface.Name) + data, err := os.ReadFile(speedPath) + if err != nil { + log.Debugf("overall: failed to read speed for %s: %v", iface.Name, err) + continue + } + + speedStr := strings.TrimSpace(string(data)) + speed, err := strconv.ParseFloat(speedStr, 64) + if err != nil || speed <= 0 { + // Speed might be -1 if link is down or unknown + continue + } + + log.Debugf("overall: detected interface %s speed: %.0f Mbps", iface.Name, speed) + result[iface.Name] = speed + } + + return result +} diff --git a/modules/metrics/host/overall/netspeed_windows.go b/modules/metrics/host/overall/netspeed_windows.go new file mode 100644 index 000000000..4b8bcb2b6 --- /dev/null +++ b/modules/metrics/host/overall/netspeed_windows.go @@ -0,0 +1,142 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build windows + +package overall + +import ( + "os/exec" + "regexp" + "strconv" + "strings" + + log "github.com/cihub/seelog" +) + +// detectNetworkBandwidthPerInterface detects network interface speeds in Mbps for each interface. +// On Windows, it uses PowerShell/WMI to query network adapter speeds. +// Returns a map of interface name to bandwidth in Mbps. +func detectNetworkBandwidthPerInterface() map[string]float64 { + result := make(map[string]float64) + + // Use PowerShell to get network adapter speeds with names + cmd := exec.Command("powershell", "-Command", + "Get-NetAdapter | Where-Object {$_.Status -eq 'Up'} | Select-Object Name,LinkSpeed | ForEach-Object { $_.Name + '|' + $_.LinkSpeed }") + out, err := cmd.Output() + if err != nil { + log.Debugf("overall: failed to get network adapter speed via PowerShell: %v", err) + return tryWMICPerInterface() + } + + lines := strings.Split(string(out), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + parts := strings.SplitN(line, "|", 2) + if len(parts) != 2 { + continue + } + + name := strings.TrimSpace(parts[0]) + linkSpeed := strings.TrimSpace(parts[1]) + speed := parseWindowsLinkSpeed(linkSpeed) + if speed > 0 { + log.Debugf("overall: detected interface %s speed: %.0f Mbps", name, speed) + result[name] = speed + } + } + + return result +} + +// tryWMICPerInterface tries to get network speed using wmic (fallback for older Windows) +func tryWMICPerInterface() map[string]float64 { + result := make(map[string]float64) + + cmd := exec.Command("wmic", "nic", "where", "NetEnabled=true", "get", "Name,Speed") + out, err := cmd.Output() + if err != nil { + log.Debugf("overall: failed to get network adapter speed via wmic: %v", err) + return result + } + + lines := strings.Split(string(out), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "Name") { + continue + } + + // WMIC output is space-separated, speed is the last field + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + + speedStr := fields[len(fields)-1] + name := strings.Join(fields[:len(fields)-1], " ") + + // Speed from wmic is in bits per second + bps, err := strconv.ParseFloat(speedStr, 64) + if err != nil || bps <= 0 { + continue + } + mbps := bps / 1000000.0 + log.Debugf("overall: detected interface %s speed: %.0f Mbps", name, mbps) + result[name] = mbps + } + + return result +} + +// parseWindowsLinkSpeed parses Windows link speed strings like "1 Gbps", "100 Mbps" +func parseWindowsLinkSpeed(linkSpeed string) float64 { + linkSpeed = strings.TrimSpace(linkSpeed) + + // Match patterns like "1 Gbps", "100 Mbps", "10 Gbps" + gbpsRegex := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*[Gg]bps`) + mbpsRegex := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*[Mm]bps`) + kbpsRegex := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*[Kk]bps`) + + if matches := gbpsRegex.FindStringSubmatch(linkSpeed); len(matches) > 1 { + if speed, err := strconv.ParseFloat(matches[1], 64); err == nil { + return speed * 1000 // Convert Gbps to Mbps + } + } + if matches := mbpsRegex.FindStringSubmatch(linkSpeed); len(matches) > 1 { + if speed, err := strconv.ParseFloat(matches[1], 64); err == nil { + return speed + } + } + if matches := kbpsRegex.FindStringSubmatch(linkSpeed); len(matches) > 1 { + if speed, err := strconv.ParseFloat(matches[1], 64); err == nil { + return speed / 1000 // Convert Kbps to Mbps + } + } + + return 0 +} diff --git a/modules/metrics/host/overall/overall.go b/modules/metrics/host/overall/overall.go new file mode 100644 index 000000000..81fed49c0 --- /dev/null +++ b/modules/metrics/host/overall/overall.go @@ -0,0 +1,636 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package overall + +import ( + "runtime" + "strings" + "sync" + + log "github.com/cihub/seelog" + "github.com/shirou/gopsutil/v4/cpu" + "github.com/shirou/gopsutil/v4/disk" + "github.com/shirou/gopsutil/v4/mem" + "github.com/shirou/gopsutil/v4/net" + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/util" +) + +// DefaultBandwidthMbps is the default network bandwidth in Mbps when auto-detection fails +const DefaultBandwidthMbps = 1000 + +// Metric collects overall system utilization percentages for CPU, memory, disk, disk I/O and network. +// Each disk and network interface is monitored independently to identify specific bottlenecks. +type Metric struct { + Enabled bool `config:"enabled"` + IntervalSeconds float64 `config:"interval_seconds"` + YellowThreshold float64 `config:"yellow_threshold"` + RedThreshold float64 `config:"red_threshold"` + + mu sync.Mutex + + // Per-disk I/O tracking: map[deviceName] -> snapshot + prevDiskIO map[string]*diskIOSnapshot + + // Per-network interface tracking: map[ifaceName] -> snapshot + prevNetIO map[string]*netIOSnapshot + + // Per-network interface bandwidth (auto-detected): map[ifaceName] -> Mbps + netBandwidth map[string]float64 + + // Previous aggregate CPU times, used to compute the steal-time percentage + // between collections. nil before the first sample is taken. + prevCPUTimes *cpu.TimesStat + + // Previous TCP RetransSegs / OutSegs counters, used to compute both the + // retransmits-per-second rate and the retransmit ratio (retrans/out, a + // capacity-like 0-100% signal). hasPrevTCPRetrans guards against emitting + // bogus values on the first call (when the counters are unknown) and + // after a counter reset. + prevTCPRetrans int64 + prevTCPOutSegs int64 + hasPrevTCPRetrans bool +} + +// diskIOSnapshot stores previous I/O counters for a disk device. +// weightedIO is gopsutil's WeightedIO field (in milliseconds), used to derive +// the average queue depth (iostat "aqu-sz") across the collection interval. +type diskIOSnapshot struct { + readTime uint64 + writeTime uint64 + weightedIO uint64 +} + +// netIOSnapshot stores previous I/O counters for a network interface +type netIOSnapshot struct { + bytesRecv uint64 + bytesSent uint64 +} + +// deviceUtilization represents utilization info for a single device. +// queueDepth is only populated for disk I/O devices (avg outstanding requests +// over the collection interval, derived from WeightedIO). +type deviceUtilization struct { + name string + usedPercent float64 + queueDepth float64 +} + +func New(cfg *config.Config) (*Metric, error) { + me := &Metric{ + Enabled: true, + IntervalSeconds: 10, + YellowThreshold: 70, + RedThreshold: 90, + prevDiskIO: make(map[string]*diskIOSnapshot), + prevNetIO: make(map[string]*netIOSnapshot), + netBandwidth: make(map[string]float64), + } + + err := cfg.Unpack(&me) + if err != nil { + panic(err) + } + + // Initialize network bandwidth detection for all interfaces + me.initNetworkBandwidth() + + log.Debugf("overall utilization metric enabled") + return me, nil +} + +// initNetworkBandwidth detects and stores bandwidth for each network interface +func (m *Metric) initNetworkBandwidth() { + bandwidths := detectNetworkBandwidthPerInterface() + for name, bw := range bandwidths { + m.netBandwidth[name] = bw + log.Debugf("overall: interface %s bandwidth: %.0f Mbps", name, bw) + } +} + +// Collect gathers CPU, memory, disk, disk I/O and network utilization +// and emits a "host/overall" event with raw values for the front layer to interpret. +// Each disk and network interface is monitored independently. +func (m *Metric) Collect() error { + if !m.Enabled { + return nil + } + + fields := util.MapStr{} + + // Collect all metrics + cpuPercent, cpuStealPercent := m.collectCPU() + memPercent := m.collectMemory() + diskPercent, diskInodePercent := m.collectDiskUsage() + diskIODevices := m.collectDiskIO() + netDevices := m.collectNetwork() + tcpRetransPerSec, tcpRetransPercent, hasTCPRetrans := m.collectTCPRetrans() + + // --- CPU utilization --- + fields["cpu.used_percent"] = cpuPercent + fields["cpu.steal_percent"] = cpuStealPercent + + // --- Memory utilization --- + fields["memory.used_percent"] = memPercent + + // --- Disk capacity utilization --- + fields["disk.used_percent"] = diskPercent + fields["disk.inodes_used_percent"] = diskInodePercent + + // --- Per-disk I/O utilization --- + diskIOMap := util.MapStr{} + var maxDiskIO deviceUtilization + for _, dev := range diskIODevices { + diskIOMap[dev.name] = util.MapStr{ + "used_percent": dev.usedPercent, + "queue_depth": dev.queueDepth, + } + if dev.usedPercent > maxDiskIO.usedPercent { + maxDiskIO = dev + } + } + if len(diskIOMap) > 0 { + fields["disk_io.devices"] = diskIOMap + fields["disk_io.used_percent"] = maxDiskIO.usedPercent + fields["disk_io.queue_depth"] = maxDiskIO.queueDepth + fields["disk_io.bottleneck_device"] = maxDiskIO.name + } + + // --- Per-network interface utilization --- + netMap := util.MapStr{} + var maxNet deviceUtilization + for _, dev := range netDevices { + bw := m.netBandwidth[dev.name] + if bw <= 0 { + bw = DefaultBandwidthMbps // Default if unknown + } + netMap[dev.name] = util.MapStr{ + "used_percent": dev.usedPercent, + "bandwidth_mbps": bw, + } + if dev.usedPercent > maxNet.usedPercent { + maxNet = dev + } + } + if len(netMap) > 0 { + fields["network.devices"] = netMap + fields["network.used_percent"] = maxNet.usedPercent + fields["network.bottleneck_device"] = maxNet.name + } + + // --- TCP retransmits --- + // tcp_retrans_per_sec is a raw throughput-style signal. tcp_retrans_percent + // is the retransmit ratio (RetransSegs/OutSegs over the interval), a true + // 0-100% capacity-like signal that is directly comparable to the other + // utilization percentages and therefore folded into status/bottleneck + // below as part of the network subsystem. Only emit once we have a valid + // delta (skips first call and counter resets). + if hasTCPRetrans { + fields["network.tcp_retrans_per_sec"] = tcpRetransPerSec + fields["network.tcp_retrans_percent"] = tcpRetransPercent + } + + // --- Calculate overall status and bottleneck --- + // CPU stress can come from either user/system load or hypervisor steal; + // disk pressure can come from either capacity or inode exhaustion; network + // pressure can come from either link saturation or a high TCP retransmit + // ratio. Pick the worst signal in each subsystem so the bottleneck + // reflects reality. + cpuStatus := cpuPercent + if cpuStealPercent > cpuStatus { + cpuStatus = cpuStealPercent + } + diskStatus := diskPercent + if diskInodePercent > diskStatus { + diskStatus = diskInodePercent + } + netStatus := maxNet + if hasTCPRetrans && tcpRetransPercent > netStatus.usedPercent { + // Surface as bottleneck="network:tcp_retrans" via existing naming. + netStatus = deviceUtilization{name: "tcp_retrans", usedPercent: tcpRetransPercent} + } + status, bottleneck := m.calculateStatus(cpuStatus, memPercent, diskStatus, maxDiskIO, netStatus) + fields["status"] = status + fields["bottleneck"] = bottleneck + + return event.Save(&event.Event{ + Metadata: event.EventMetadata{ + Category: "host", + Name: "overall", + Datatype: "gauge", + }, + Fields: util.MapStr{ + "host": util.MapStr{ + "overall": fields, + }, + }, + }) +} + +// collectCPU returns the current overall CPU utilization percentage and the +// hypervisor steal-time percentage (both 0-100). The steal percentage is +// derived from the delta of cpu.Times() across collections; it is reported as +// 0 on the first call (no baseline) and on platforms where Steal is unavailable. +func (m *Metric) collectCPU() (usedPercent, stealPercent float64) { + percents, err := cpu.Percent(0, false) + if err != nil { + log.Errorf("overall: failed to get cpu percent: %v", err) + } + if len(percents) > 0 { + usedPercent = percents[0] + } + + times, err := cpu.Times(false) + if err != nil || len(times) == 0 { + if err != nil { + log.Debugf("overall: failed to get cpu times: %v", err) + } + return usedPercent, 0 + } + cur := times[0] + + m.mu.Lock() + prev := m.prevCPUTimes + m.prevCPUTimes = &cur + m.mu.Unlock() + + if prev == nil { + // First sample; no delta available yet. + return usedPercent, 0 + } + + deltaTotal := cur.Total() - prev.Total() + deltaSteal := cur.Steal - prev.Steal + if deltaTotal <= 0 || deltaSteal < 0 { + // Counter reset or non-monotonic reading; skip this sample. + return usedPercent, 0 + } + stealPercent = deltaSteal / deltaTotal * 100.0 + if stealPercent > 100.0 { + stealPercent = 100.0 + } + return usedPercent, stealPercent +} + +// collectMemory returns the current memory utilization percentage (0-100). +func (m *Metric) collectMemory() float64 { + v, err := mem.VirtualMemory() + if err != nil { + log.Errorf("overall: failed to get memory info: %v", err) + return 0 + } + if v == nil { + return 0 + } + return v.UsedPercent +} + +// collectDiskUsage returns disk capacity and inode utilization percentages +// (both 0-100). Inode usage is aggregated across all partitions (sum of used +// inodes over sum of total inodes); filesystems without inodes (e.g. some +// pseudo-filesystems, certain Windows volumes) are skipped. +func (m *Metric) collectDiskUsage() (usedPercent, inodesUsedPercent float64) { + if runtime.GOOS == "darwin" { + v, err := disk.Usage("/") + if err != nil { + log.Errorf("overall: failed to get disk usage: %v", err) + return 0, 0 + } + return v.UsedPercent, v.InodesUsedPercent + } + + partitions, err := disk.Partitions(false) + if err != nil || len(partitions) == 0 { + log.Errorf("overall: failed to get disk partitions: %v", err) + return 0, 0 + } + var total, used, inodesTotal, inodesUsed uint64 + for _, p := range partitions { + if p.Device == "" { + continue + } + v, err := disk.Usage(p.Mountpoint) + if err != nil { + continue + } + total += v.Total + used += v.Used + inodesTotal += v.InodesTotal + inodesUsed += v.InodesUsed + } + if total > 0 { + usedPercent = float64(used) / float64(total) * 100.0 + } + if inodesTotal > 0 { + inodesUsedPercent = float64(inodesUsed) / float64(inodesTotal) * 100.0 + } + return usedPercent, inodesUsedPercent +} + +// collectDiskIO returns per-disk I/O utilization percentages (0-100) based on io time deltas. +// Returns empty slice if data is not yet available (first call). +func (m *Metric) collectDiskIO() []deviceUtilization { + ret, err := disk.IOCounters() + if err != nil { + log.Debugf("overall: failed to get disk io counters: %v", err) + return nil + } + if len(ret) == 0 { + return nil + } + + m.mu.Lock() + defer m.mu.Unlock() + + var results []deviceUtilization + + for name, io := range ret { + // Skip certain device types + if strings.HasPrefix(name, "loop") || strings.HasPrefix(name, "ram") { + continue + } + + prev, exists := m.prevDiskIO[name] + if !exists { + // First time seeing this device, store initial values + m.prevDiskIO[name] = &diskIOSnapshot{ + readTime: io.ReadTime, + writeTime: io.WriteTime, + weightedIO: io.WeightedIO, + } + continue + } + + // Calculate IO busy time delta + deltaRead := io.ReadTime - prev.readTime + deltaWrite := io.WriteTime - prev.writeTime + deltaIO := deltaRead + deltaWrite + deltaWeighted := io.WeightedIO - prev.weightedIO + + // Update stored values + prev.readTime = io.ReadTime + prev.writeTime = io.WriteTime + prev.weightedIO = io.WeightedIO + + // IO busy time delta in ms over the collection interval + intervalMs := m.IntervalSeconds * 1000.0 + busy := float64(deltaIO) / intervalMs * 100.0 + if busy > 100.0 { + busy = 100.0 + } + if busy < 0 { + busy = 0 + } + + // Average queue depth over the interval (iostat "aqu-sz"): + // WeightedIO is the cumulative weighted time spent doing I/Os in ms, + // so dividing the delta by the interval in ms yields the average + // number of in-flight requests. WeightedIO is always 0 on platforms + // that don't populate it (e.g. macOS), which correctly yields 0. + var queueDepth float64 + if intervalMs > 0 { + queueDepth = float64(deltaWeighted) / intervalMs + } + if queueDepth < 0 { + queueDepth = 0 + } + + results = append(results, deviceUtilization{ + name: name, + usedPercent: busy, + queueDepth: queueDepth, + }) + } + + return results +} + +// collectTCPRetrans returns the TCP retransmits-per-second rate and the +// retransmit ratio (RetransSegs/OutSegs over the interval, 0-100%) computed +// from the delta of the kernel's TCP counters. The boolean return is false +// on the first call (no baseline), on platforms where ProtoCounters is not +// supported (currently non-Linux), or when the counters are unavailable / have +// been reset (negative delta). +func (m *Metric) collectTCPRetrans() (perSec, percent float64, ok bool) { + if m.IntervalSeconds <= 0 { + return 0, 0, false + } + counters, err := net.ProtoCounters([]string{"tcp"}) + if err != nil || len(counters) == 0 { + // Platforms without ProtoCounters support (darwin, windows, ...) end + // up here; log at debug level to avoid spamming production logs. + if err != nil { + log.Debugf("overall: failed to get tcp proto counters: %v", err) + } + return 0, 0, false + } + stats := counters[0].Stats + curRetrans, hasRetrans := stats["RetransSegs"] + curOut, hasOut := stats["OutSegs"] + if !hasRetrans || !hasOut { + return 0, 0, false + } + + m.mu.Lock() + prevRetrans := m.prevTCPRetrans + prevOut := m.prevTCPOutSegs + hadPrev := m.hasPrevTCPRetrans + m.prevTCPRetrans = curRetrans + m.prevTCPOutSegs = curOut + m.hasPrevTCPRetrans = true + m.mu.Unlock() + + if !hadPrev { + return 0, 0, false + } + deltaRetrans := curRetrans - prevRetrans + deltaOut := curOut - prevOut + if deltaRetrans < 0 || deltaOut < 0 { + // Counter reset (e.g. kernel restart, namespace change); skip. + return 0, 0, false + } + perSec = float64(deltaRetrans) / m.IntervalSeconds + if deltaOut > 0 { + percent = float64(deltaRetrans) / float64(deltaOut) * 100.0 + if percent > 100.0 { + percent = 100.0 + } + } + return perSec, percent, true +} + +// collectNetwork returns per-interface network utilization percentages (0-100) +// based on throughput relative to each interface's detected bandwidth. +// Returns empty slice if data is not yet available (first call). +func (m *Metric) collectNetwork() []deviceUtilization { + stats, err := net.IOCounters(true) // true = per-interface + if err != nil { + log.Debugf("overall: failed to get network io counters: %v", err) + return nil + } + + m.mu.Lock() + defer m.mu.Unlock() + + var results []deviceUtilization + + for _, stat := range stats { + name := stat.Name + + // Skip loopback and virtual interfaces + if isVirtualInterface(name) { + continue + } + + prev, exists := m.prevNetIO[name] + if !exists { + // First time seeing this interface, store initial values + m.prevNetIO[name] = &netIOSnapshot{ + bytesRecv: stat.BytesRecv, + bytesSent: stat.BytesSent, + } + continue + } + + // Calculate deltas + deltaRecv := stat.BytesRecv - prev.bytesRecv + deltaSent := stat.BytesSent - prev.bytesSent + + // Update stored values + prev.bytesRecv = stat.BytesRecv + prev.bytesSent = stat.BytesSent + + // Use the higher of in/out throughput for utilization + deltaMax := deltaRecv + if deltaSent > deltaMax { + deltaMax = deltaSent + } + + // Get bandwidth for this interface + bandwidth := m.netBandwidth[name] + if bandwidth <= 0 { + bandwidth = DefaultBandwidthMbps // Default if unknown + } + + // Convert bandwidth from Mbps to bytes/sec: Mbps * 1_000_000 / 8 + bandwidthBytesPerSec := bandwidth * 1000000.0 / 8.0 + + throughputBytesPerSec := float64(deltaMax) / m.IntervalSeconds + percent := throughputBytesPerSec / bandwidthBytesPerSec * 100.0 + if percent > 100.0 { + percent = 100.0 + } + if percent < 0 { + percent = 0 + } + + results = append(results, deviceUtilization{ + name: name, + usedPercent: percent, + }) + } + + return results +} + +// isVirtualInterface returns true if the interface name looks like a virtual/loopback interface +func isVirtualInterface(name string) bool { + // Common virtual interface prefixes across platforms + virtualPrefixes := []string{ + "lo", "lo0", // Loopback + "veth", "docker", "br-", // Docker/containers + "virbr", "vnet", // Libvirt/KVM + "utun", "awdl", "bridge", "llw", "ap", "XHC", // macOS virtual + "vmnet", // VMware + "Loopback", // Windows loopback + } + + for _, prefix := range virtualPrefixes { + if strings.HasPrefix(name, prefix) || name == prefix { + return true + } + } + return false +} + +// calculateStatus determines the overall system status (green/yellow/red) and +// identifies the bottleneck subsystem (if any) based on configured thresholds. +// For disk_io and network, it includes the specific device name in the bottleneck. +func (m *Metric) calculateStatus(cpuPct, memPct, diskPct float64, maxDiskIO, maxNet deviceUtilization) (status, bottleneck string) { + status = "green" + bottleneck = "" + + // Subsystems to check with their utilization percentages + type subsystem struct { + name string + percent float64 + device string // Optional device name for disk_io and network + } + + subsystems := []subsystem{ + {"cpu", cpuPct, ""}, + {"memory", memPct, ""}, + {"disk", diskPct, ""}, + } + + // Add disk_io if we have data + if maxDiskIO.name != "" { + subsystems = append(subsystems, subsystem{"disk_io", maxDiskIO.usedPercent, maxDiskIO.name}) + } + + // Add network if we have data + if maxNet.name != "" { + subsystems = append(subsystems, subsystem{"network", maxNet.usedPercent, maxNet.name}) + } + + // Find the highest utilization and determine status + var maxPercent float64 + var maxSubsystem subsystem + for _, s := range subsystems { + if s.percent > maxPercent { + maxPercent = s.percent + maxSubsystem = s + } + } + + // Determine status based on thresholds + if maxPercent >= m.RedThreshold { + status = "red" + if maxSubsystem.device != "" { + bottleneck = maxSubsystem.name + ":" + maxSubsystem.device + } else { + bottleneck = maxSubsystem.name + } + } else if maxPercent >= m.YellowThreshold { + status = "yellow" + if maxSubsystem.device != "" { + bottleneck = maxSubsystem.name + ":" + maxSubsystem.device + } else { + bottleneck = maxSubsystem.name + } + } + + return status, bottleneck +} diff --git a/modules/metrics/metrics.go b/modules/metrics/metrics.go index 1e2444b9a..78ec1e05d 100755 --- a/modules/metrics/metrics.go +++ b/modules/metrics/metrics.go @@ -39,6 +39,7 @@ import ( "infini.sh/framework/modules/metrics/host/disk" "infini.sh/framework/modules/metrics/host/memory" "infini.sh/framework/modules/metrics/host/network" + "infini.sh/framework/modules/metrics/host/overall" agent2 "infini.sh/framework/modules/metrics/instance" ) @@ -55,6 +56,7 @@ type MetricConfig struct { DiskConfig *Config `config:"disk"` CPUConfig *Config `config:"cpu"` MemoryConfig *Config `config:"memory"` + OverallConfig *Config `config:"overall"` ElasticsearchConfig *Config `config:"elasticsearch"` Tags []string `config:"tags"` @@ -274,6 +276,28 @@ func (module *MetricsModule) CollectHostMetric() { } task.RegisterScheduleTask(memTask) } + + if module.config.OverallConfig != nil { + overallM, err := overall.New(module.config.OverallConfig) + if err != nil { + panic(err) + } + if overallM.Enabled { + taskId := util.GetUUID() + module.taskIDs = append(module.taskIDs, taskId) + var overallTask = task.ScheduleTask{ + ID: taskId, + Description: "fetch overall utilization metrics", + Type: "interval", + Interval: "10s", + Task: func(ctx context.Context) { + log.Debug("collecting overall utilization metrics") + overallM.Collect() + }, + } + task.RegisterScheduleTask(overallTask) + } + } } func (module *MetricsModule) Start() error { diff --git a/modules/pipeline/model.go b/modules/pipeline/model.go index dcb79679d..10c4b52f5 100644 --- a/modules/pipeline/model.go +++ b/modules/pipeline/model.go @@ -30,7 +30,7 @@ import ( "infini.sh/framework/core/util" ) -type PipelineStatus struct { +type PipelineTaskStatus struct { State pipeline.RunningState `json:"state"` CreateTime time.Time `json:"create_time"` StartTime *time.Time `json:"start_time"` diff --git a/modules/pipeline/module.go b/modules/pipeline/module.go new file mode 100755 index 000000000..575a0da85 --- /dev/null +++ b/modules/pipeline/module.go @@ -0,0 +1,609 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package pipeline + +import ( + "context" + "fmt" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/fsnotify/fsnotify" + "infini.sh/framework/core/locker" + "infini.sh/framework/core/security" + "infini.sh/framework/core/task" + + log "github.com/cihub/seelog" + "infini.sh/framework/core/api" + "infini.sh/framework/core/config" + "infini.sh/framework/core/env" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/global" + "infini.sh/framework/core/pipeline" + "infini.sh/framework/core/rate" + "infini.sh/framework/core/util" +) + +type PipeModule struct { + api.Handler + closed atomic.Bool + + /* + * States of pipeline tasks. keyed by pipeline ID. + */ + pipelines sync.Map + configs sync.Map + contexts sync.Map +} + +func (module *PipeModule) Name() string { + return "pipeline" +} + +var moduleCfg = struct { + PipelineEnabledByDefault bool `config:"pipeline_enabled_by_default"` +}{PipelineEnabledByDefault: true} + +func (module *PipeModule) Setup() { + if global.Env().IsDebug { + log.Debug("pipeline framework config: ", moduleCfg) + } + + ok, err := env.ParseConfig("preference", &moduleCfg) + if ok && err != nil && global.Env().SystemConfig.Configs.PanicOnConfigError { + panic(err) + } + + module.pipelines = sync.Map{} + module.contexts = sync.Map{} + module.configs = sync.Map{} + + pipeline.RegisterProcessorPlugin("dag", pipeline.NewDAGProcessor) + pipeline.RegisterProcessorPlugin("echo", NewEchoProcessor) + + //TODO remove + api.HandleAPIMethod(api.GET, "/pipeline/tasks/", module.getRunningPipelineTasksHandler) + api.HandleAPIMethod(api.POST, "/pipeline/tasks/_search", module.searchPipelineTasksHandler) + api.HandleAPIMethod(api.POST, "/pipeline/tasks/", module.createPipelineTaskHandler) + api.HandleAPIMethod(api.GET, "/pipeline/task/:id", module.getPipelineTaskHandler) + api.HandleAPIMethod(api.DELETE, "/pipeline/task/:id", module.deletePipelineTaskHandler) + api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_start", module.startPipelineTaskHandler) + api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_stop", module.stopPipelineTaskHandler) + + //use pipelines to avoid naming conflicts + api.HandleUIMethod(api.POST, "/pipelines/_search", module.searchPipelineHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Search))) + api.HandleUIMethod(api.POST, "/pipelines/", module.createPipelineHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Create))) + api.HandleUIMethod(api.GET, "/pipelines/:id", module.getPipelineHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Read))) + api.HandleUIMethod(api.PUT, "/pipelines/:id", module.updatePipelineHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Update))) + api.HandleUIMethod(api.DELETE, "/pipelines/:id", module.deletePipelineHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Delete))) + + api.HandleUIMethod(api.GET, "/pipelines/_running", module.getRunningPipelineTasksHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Admin))) + api.HandleUIMethod(api.POST, "/pipelines/:id/_start", module.startPipelineTaskHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Admin))) + api.HandleUIMethod(api.POST, "/pipelines/:id/_stop", module.stopPipelineTaskHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Admin))) + //api.HandleUIMethod(api.POST, "/pipeline/task/:id/logging/_search", module.getPipelineLoggingsHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Admin))) + //api.HandleUIMethod(api.POST, "/pipeline/task/:id/metrics/_search", module.getPipelineMetricsHandler, api.RequirePermission(security.GetOrInitPermission("generic", "pipeline", security.Admin))) +} + +func (module *PipeModule) startTask(taskID string) (exists bool) { + if module.closed.Load() { + return false + } + + ctx, ok := module.contexts.Load(taskID) + if !ok { + return + } + v1, ok := ctx.(*pipeline.Context) + if !ok { + return + } + + exists = true + + // Mark exited pipeline to start again + if v1.IsExit() { + v1.Restart() + } + // Resume pipeline loop + if v1.IsPause() { + // Mark pipeline status as starting + v1.Starting() + v1.Resume() + } + + return +} + +// stopTask will cancel the current pipeline context, abort the pipeline execution. +func (module *PipeModule) stopTask(taskID string) (exists bool) { + ctx, ok := module.contexts.Load(taskID) + if !ok { + return + } + v1, ok := ctx.(*pipeline.Context) + if !ok { + return + } + + exists = true + + if global.Env().IsDebug { + if rate.GetRateLimiterPerSecond("pipeline", "shutdown "+taskID+string(v1.GetRunningState()), 1).Allow() { + log.Trace("start shutting down pipeline:", taskID, ",state:", v1.GetRunningState()) + } + } + + // Mark pipeline as exited + v1.Exit() + // Mark pipeline as STOPPING as needed + v1.Stopping() + // call cancelFunc(), will mark IsCanceled asynchronously + v1.CancelTask() + + return +} + +// deleteTask will clean all in-memory states and release the pipeline context +func (module *PipeModule) deleteTask(taskID string) { + module.pipelines.Delete(taskID) + module.configs.Delete(taskID) + module.releaseContext(taskID) + module.contexts.Delete(taskID) +} + +// releaseContext will release the task context +func (module *PipeModule) releaseContext(taskID string) { + ctx, ok := module.contexts.Load(taskID) + if ok { + v1, ok := ctx.(*pipeline.Context) + if ok { + pipeline.ReleaseContext(v1) + if v1.IsPause() { + // release loop + v1.Resume() + } + } + } +} + +func getPipelineConfig() ([]pipeline.PipelineConfigV2, error) { + configFile := global.Env().GetConfigFile() + configDir := global.Env().GetConfigDir() + parentCfg, err := config.LoadFile(configFile) + if err != nil { + return nil, fmt.Errorf("failed to load config file: %v, path: %s", err, configFile) + } + childCfg, err := config.LoadPath(configDir) + if err != nil { + return nil, fmt.Errorf("failed to load config dir: %v, path: %s", err, configDir) + } + err = parentCfg.Merge(childCfg) + if err != nil { + return nil, fmt.Errorf("failed to merge configs: %v", err) + } + + pipelineCfg := []pipeline.PipelineConfigV2{} + + if ok := parentCfg.HasField("pipeline"); ok { + parentCfg, err = parentCfg.Child("pipeline", -1) + if err != nil { + return nil, err + } + err = parentCfg.Unpack(&pipelineCfg) + if err != nil { + return nil, err + } + // YAML pipeline configs rarely set an explicit id field; fall back to + // name so the in-memory task map key is stable and predictable. + for i := range pipelineCfg { + if pipelineCfg[i].ID == "" { + pipelineCfg[i].ID = pipelineCfg[i].Name + } + } + return pipelineCfg, nil + } + return pipelineCfg, nil +} + +func (module *PipeModule) Start() error { + var ( + pipelines []pipeline.PipelineConfigV2 + err error + ) + pipelines, err = getPipelineConfig() + if err != nil && global.Env().SystemConfig.Configs.PanicOnConfigError { + panic(err) + } + for _, v := range pipelines { + err := module.createPipelineTask(v, false) + if err != nil { + log.Errorf("error on running pipeline: %v(%v), err: %v", v.Name, v.ID, err) + continue + } + } + + //listen on changes + config.NotifyOnConfigChange(func(ev fsnotify.Event) { + if module.closed.Load() || global.ShuttingDown() { + log.Warn("module closed, skip reloading pipelines") + return + } + + log.Infof("config changed, checking for new pipeline configs, %v, %v", ev.Op, ev.Name) + + newConfig := []pipeline.PipelineConfigV2{} + newConfig, err = getPipelineConfig() + + if err != nil { + log.Error(err) + return + } + + defer func() { + if !global.Env().IsDebug { + if r := recover(); r != nil { + var v string + switch r.(type) { + case error: + v = r.(error).Error() + case runtime.Error: + v = r.(runtime.Error).Error() + case string: + v = r.(string) + } + log.Error("error on apply pipeline change,", v) + } + } + }() + + needStopAndClean := []string{} + newPipelines := map[string]pipeline.PipelineConfigV2{} + + newPipelineNames := []string{} + for _, v := range newConfig { + newPipelines[v.ID] = v + newPipelineNames = append(newPipelineNames, v.ID) + } + + log.Debugf("we now have %v new pipelines: %v", len(newPipelineNames), newPipelineNames) + + module.configs.Range(func(k, v any) bool { + oldC, ok := v.(pipeline.PipelineConfigV2) + if !ok { + log.Warnf("impossible value from configs: %v", v) + return true + } + + // Don't stop transient pipelines + if oldC.Transient { + log.Debugf("transient pipeline %v should not be reloaded", oldC.Name) + return true + } + newC, ok := newPipelines[oldC.ID] + isSame := newC.Equals(oldC) + // Skip condition: (old pipeline is present in the new pipeline configs, config is the same, new config is also enabled) + if ok && isSame && isPipelineEnabled(newC.Enabled) { + log.Debugf("pipeline %v config not changed, skip reloading", oldC.Name) + return true + } + + log.Debug("pipeline config changed, stop and clean:", oldC.Name, ",", oldC, ",", ok, ",", isSame, ",", isPipelineEnabled(newC.Enabled)) + + needStopAndClean = append(needStopAndClean, oldC.ID) + return true + }) + + log.Debugf("we now have %v old pipelines need to clean: %v", len(needStopAndClean), needStopAndClean) + + if len(needStopAndClean) > 0 { + log.Trace("stop and wait for pipelines to release: ", needStopAndClean) + module.stopAndWaitForRelease(needStopAndClean, time.Minute) + log.Debug("old pipelines released") + + for _, taskID := range needStopAndClean { + log.Infof("removing pipeline [%s]", taskID) + module.deleteTask(taskID) + } + } + + log.Debugf("starting %v pipelines", len(newPipelineNames)) + for k, v := range newPipelines { + err := module.createPipelineTask(v, false) + if err != nil { + log.Errorf("failed to create pipeline: %v, err: %v", k, err) + } + } + }) + + return nil +} + +func (module *PipeModule) Stop() error { + if module.closed.Load() { + return nil + } + module.closed.Store(true) + + total := util.GetSyncMapSize(&module.contexts) + if total <= 0 { + return nil + } + + log.Info("shutting down pipelines") + + var taskIDs []string + module.contexts.Range(func(key, value any) bool { + taskID, ok := key.(string) + if !ok { + return false + } + taskIDs = append(taskIDs, taskID) + return true + }) + + module.stopAndWaitForRelease(taskIDs, time.Minute*5) + + log.Info("finished shut down pipelines") + return nil +} + +func (module *PipeModule) stopAndWaitForRelease(taskIDs []string, timeout time.Duration) { + start := time.Now() + + for { + if time.Now().Sub(start) > timeout { + log.Error("waitForStop timed out") + break + } + + // Send stop signal to all contexts + for _, taskID := range taskIDs { + // cancel & stop + module.stopTask(taskID) + // release loop + module.releaseContext(taskID) + // don't delete context yet + } + + needRetry := false + for _, taskID := range taskIDs { + v, ok := module.contexts.Load(taskID) + if !ok { + continue + } + + ctx, ok := v.(*pipeline.Context) + if !ok { + log.Errorf("impossible value from contexts: %t", v) + continue + } + + if !ctx.IsLoopReleased() { + if rate.GetRateLimiterPerSecond("pipeline", "shutdown"+taskID+string(ctx.GetRunningState()), 1).Allow() { + log.Debug("pipeline still running: ", taskID, ",state: ", ctx.GetRunningState()) + } + needRetry = true + break + } + } + + if !needRetry { + break + } + } +} + +const pipelineSingleton = "pipeline_singleton" + +var creatingLocker = sync.Mutex{} + +// Helper function to run a pipeline, i.e., create a pipeline task. +func (module *PipeModule) createPipelineTask(v pipeline.PipelineConfigV2, transient bool) error { + if module.closed.Load() { + return errors.New("module closed") + } + + if !isPipelineEnabled(v.Enabled) { + // pipeline config explicitly disabled + return nil + } + + creatingLocker.Lock() + defer creatingLocker.Unlock() + + v.Transient = transient + + // NOTE: hold the slot before creating pipeline loops + if _, ok := module.configs.LoadOrStore(v.ID, v); ok { + log.Tracef("pipeline [%v] is already created, skip", v.Name) + return nil + } + + if v.Singleton { + log.Info("creating pipeline: " + v.Name + ", singleton") + } else { + log.Info("creating pipeline: " + v.Name) + } + + task.RunWithContext("pipeline:"+v.Name, func(taskCtx context.Context) error { + + cfgV := taskCtx.Value("cfg") + cfg, ok := cfgV.(pipeline.PipelineConfigV2) + if !ok { + return errors.New("invalid pipeline config") + } + processorConfigs, err := cfg.GetProcessorsConfig() + if err != nil { + return errors.Errorf("failed to get processor config, %v", err) + } + + processors, err := pipeline.NewPipeline(processorConfigs) + if err != nil { + return err + } + + ctx := pipeline.AcquireContext(v) + module.pipelines.Store(v.ID, processors) + module.contexts.Store(v.ID, ctx) + + defer func() { + if !global.Env().IsDebug { + if r := recover(); r != nil { + var err string + switch r.(type) { + case error: + err = r.(error).Error() + case runtime.Error: + err = r.(runtime.Error).Error() + case string: + err = r.(string) + } + log.Errorf("error on pipeline: %v, retry delay: %vms", cfg.Name, err) + } + } + + ctx.SetLoopReleased() + processors.Release() + }() + + if !cfg.AutoStart { + // Mark pipeline as exited, don't run automatically + ctx.Exit() + } else { + ctx.Starting() + } + + log.Debug("processing pipeline_v2: ", cfg.Name) + + retryDelayInMs := 1000 + if cfg.RetryDelayInMs > 0 { + retryDelayInMs = cfg.RetryDelayInMs + } + + started := false + + for { + if global.ShuttingDown() { + log.Debugf("system is shutting down, pipeline [%v] will be stopped", cfg.Name) + break + } + + if ctx.IsReleased() { + break + } + if module.closed.Load() { + break + } + if ctx.IsExit() { + ctx.Stopped() + } + + // NOTE: state must be checked as the last step + state := ctx.GetRunningState() + if global.Env().IsDebug { + log.Tracef("pipeline [%v], state: %v", cfg.Name, state) + time.Sleep(500 * time.Millisecond) + } + switch state { + case pipeline.STARTING: + + //check + if v.Singleton { + if v.MaxRunningInMs <= 0 { + v.MaxRunningInMs = 60000 + } + ok, err := locker.Hold(pipelineSingleton, v.ID, global.Env().SystemConfig.NodeConfig.ID, time.Duration(v.MaxRunningInMs)*time.Millisecond, true) + if !ok { + log.Debugf("pipeline [%v] is already running somewhere, %v", cfg.Name, err) + ctx.Finished() + continue + } + } + + // Pipeline needs to run + if started { + log.Errorf("pipeline [%v] started twice, should not happen", cfg.Name) + } + started = true + ctx.Started() + ctx.ResetContext() + + err = processors.Process(ctx) + + if err != nil { + log.Errorf("error on pipeline:%v, %v", cfg.Name, err) + ctx.Failed(err) + } else { + if global.Env().IsDebug { + log.Debugf("pipeline [%v] end running", cfg.Name) + } + ctx.Finished() + } + started = false + case pipeline.STARTED, pipeline.STOPPING: + log.Errorf("pipeline [%v] loop should not detect %s", cfg.Name, state) + case pipeline.FINISHED, pipeline.FAILED: + // Pipeline ended, pause or start next round + // keep_running: true & not stopped manually by Exit() + // For IsExit, don't pause here, wait for STOPPED state, or we could Pause twice for STOPPED & IsExit. + if cfg.KeepRunning { + if global.Env().IsDebug { + log.Tracef("pipeline [%v] end running, restart again, retry in [%v]ms", cfg.Name, retryDelayInMs) + } + + if retryDelayInMs > 0 { + log.Tracef("start sleep [%v]ms", retryDelayInMs) + time.Sleep(time.Duration(retryDelayInMs) * time.Millisecond) + log.Tracef("end sleep [%v]ms", retryDelayInMs) + } + + // restart after delay. + ctx.Starting() + } else { + ctx.Stopped() + ctx.Pause() + } + case pipeline.STOPPED: + // Pipeline manually stopped, pause + ctx.Pause() + } + } + + log.Debugf("pipeline [%v] loop exited with state [%v]", cfg.Name, ctx.GetRunningState()) + + return nil + }, context.WithValue(context.Background(), "cfg", v)) + + return nil +} + +func isPipelineEnabled(enabled *bool) bool { + // if not configured `enabled: true`, by default true + if enabled == nil { + return moduleCfg.PipelineEnabledByDefault + } + return *enabled +} diff --git a/modules/pipeline/pipeline.go b/modules/pipeline/pipeline.go old mode 100755 new mode 100644 index 25772b200..7bedfeccc --- a/modules/pipeline/pipeline.go +++ b/modules/pipeline/pipeline.go @@ -1,580 +1,174 @@ -// Copyright (C) INFINI Labs & INFINI LIMITED. -// -// The INFINI Framework is offered under the GNU Affero General Public License v3.0 -// and as commercial software. -// -// For commercial licensing, contact us at: -// - Website: infinilabs.com -// - Email: hello@infini.ltd -// -// Open Source licensed under AGPL V3: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ package pipeline import ( - "context" - "fmt" - "github.com/fsnotify/fsnotify" - "infini.sh/framework/core/locker" - "infini.sh/framework/core/task" - "runtime" - "sync" - "sync/atomic" + "net/http" "time" - log "github.com/cihub/seelog" - "infini.sh/framework/core/api" - "infini.sh/framework/core/config" - "infini.sh/framework/core/env" - "infini.sh/framework/core/errors" - "infini.sh/framework/core/global" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" "infini.sh/framework/core/pipeline" - "infini.sh/framework/core/rate" - "infini.sh/framework/core/util" ) -type PipeModule struct { - api.Handler - closed atomic.Bool - - pipelines sync.Map - configs sync.Map - contexts sync.Map -} - -func (module *PipeModule) Name() string { - return "pipeline" -} - -var moduleCfg = struct { - PipelineEnabledByDefault bool `config:"pipeline_enabled_by_default"` -}{PipelineEnabledByDefault: true} - -func (module *PipeModule) Setup() { - if global.Env().IsDebug { - log.Debug("pipeline framework config: ", moduleCfg) +func (h *PipeModule) createPipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var obj = &pipeline.PipelineConfigV2{} + err := h.DecodeJSON(req, obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusBadRequest) + return } - ok, err := env.ParseConfig("preference", &moduleCfg) - if ok && err != nil && global.Env().SystemConfig.Configs.PanicOnConfigError { - panic(err) + if obj.Name == "" { + h.WriteError(w, "name is required", http.StatusBadRequest) + return } - module.pipelines = sync.Map{} - module.contexts = sync.Map{} - module.configs = sync.Map{} + ctx := orm.NewContextWithParent(req.Context()) + ctx.Refresh = orm.WaitForRefresh - pipeline.RegisterProcessorPlugin("dag", pipeline.NewDAGProcessor) - pipeline.RegisterProcessorPlugin("echo", NewEchoProcessor) - - api.HandleAPIMethod(api.GET, "/pipeline/tasks/", module.getPipelinesHandler) - api.HandleAPIMethod(api.POST, "/pipeline/tasks/_search", module.searchPipelinesHandler) - api.HandleAPIMethod(api.POST, "/pipeline/tasks/", module.createPipelineHandler) - api.HandleAPIMethod(api.GET, "/pipeline/task/:id", module.getPipelineHandler) - api.HandleAPIMethod(api.DELETE, "/pipeline/task/:id", module.deletePipelineHandler) - api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_start", module.startTaskHandler) - api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_stop", module.stopTaskHandler) + err = orm.Create(ctx, obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + h.WriteCreatedOKJSON(w, obj.ID) } -func (module *PipeModule) startTask(taskID string) (exists bool) { - if module.closed.Load() { - return false - } +func (h *PipeModule) getPipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + + obj := pipeline.PipelineConfigV2{} + obj.ID = id - ctx, ok := module.contexts.Load(taskID) - if !ok { + ctx := orm.NewContextWithParent(req.Context()) + exists, err := orm.GetV2(ctx, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) return } - v1, ok := ctx.(*pipeline.Context) - if !ok { + if !exists { + h.WriteOpRecordNotFoundJSON(w, id) return } - exists = true - - // Mark exited pipeline to start again - if v1.IsExit() { - v1.Restart() - } - // Resume pipeline loop - if v1.IsPause() { - // Mark pipeline status as starting - v1.Starting() - v1.Resume() - } - - return + h.WriteGetOKJSON(w, id, obj) } -// stopTask will cancel the current pipeline context, abort the pipeline execution. -func (module *PipeModule) stopTask(taskID string) (exists bool) { - ctx, ok := module.contexts.Load(taskID) - if !ok { +func (h *PipeModule) updatePipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") + replace := h.GetBoolOrDefault(req, "replace", false) + var newConfig pipeline.PipelineConfigV2 + err := h.DecodeJSON(req, &newConfig) + if err != nil { + h.WriteError(w, err.Error(), http.StatusBadRequest) return } - v1, ok := ctx.(*pipeline.Context) - if !ok { + + if newConfig.ID != "" && newConfig.ID != id { + h.WriteError(w, "id in request body does not match id in URL", http.StatusBadRequest) return } - exists = true - - if global.Env().IsDebug { - if rate.GetRateLimiterPerSecond("pipeline", "shutdown "+taskID+string(v1.GetRunningState()), 1).Allow() { - log.Trace("start shutting down pipeline:", taskID, ",state:", v1.GetRunningState()) - } + if newConfig.Name == "" { + h.WriteError(w, "name is required", http.StatusBadRequest) + return } - // Mark pipeline as exited - v1.Exit() - // Mark pipeline as STOPPING as needed - v1.Stopping() - // call cancelFunc(), will mark IsCanceled asynchronously - v1.CancelTask() - - return -} - -// deleteTask will clean all in-memory states and release the pipeline context -func (module *PipeModule) deleteTask(taskID string) { - module.pipelines.Delete(taskID) - module.configs.Delete(taskID) - module.releaseContext(taskID) - module.contexts.Delete(taskID) -} - -// releaseContext will release the task context -func (module *PipeModule) releaseContext(taskID string) { - ctx, ok := module.contexts.Load(taskID) - if ok { - v1, ok := ctx.(*pipeline.Context) - if ok { - pipeline.ReleaseContext(v1) - if v1.IsPause() { - // release loop - v1.Resume() - } - } - } -} + ctx := orm.NewContextWithParent(req.Context()) -func getPipelineConfig() ([]pipeline.PipelineConfigV2, error) { - configFile := global.Env().GetConfigFile() - configDir := global.Env().GetConfigDir() - parentCfg, err := config.LoadFile(configFile) - if err != nil { - return nil, fmt.Errorf("failed to load config file: %v, path: %s", err, configFile) - } - childCfg, err := config.LoadPath(configDir) + var oldConfig pipeline.PipelineConfigV2 + oldConfig.ID = id + exists, err := orm.GetWithSystemFields(ctx, &oldConfig) if err != nil { - return nil, fmt.Errorf("failed to load config dir: %v, path: %s", err, configDir) + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return } - err = parentCfg.Merge(childCfg) - if err != nil { - return nil, fmt.Errorf("failed to merge configs: %v", err) + if !exists { + h.WriteOpRecordNotFoundJSON(w, id) + return } - pipelineCfg := []pipeline.PipelineConfigV2{} - - if ok := parentCfg.HasField("pipeline"); ok { - parentCfg, err = parentCfg.Child("pipeline", -1) - if err != nil { - return nil, err - } - err = parentCfg.Unpack(&pipelineCfg) - return pipelineCfg, err + if !replace { + newConfig.Created = oldConfig.Created + } else { + t := time.Now() + newConfig.Created = &t } - return pipelineCfg, nil -} -func (module *PipeModule) Start() error { - var ( - pipelines []pipeline.PipelineConfigV2 - err error - ) - pipelines, err = getPipelineConfig() - if err != nil && global.Env().SystemConfig.Configs.PanicOnConfigError { - panic(err) - } - for _, v := range pipelines { - err := module.createPipeline(v, false) - if err != nil { - log.Errorf("error on running pipeline: %v, err: %v", v.Name, err) - continue - } - } + // Ensure ID is set even if the client omitted it from the request body + newConfig.ID = id - //listen on changes - config.NotifyOnConfigChange(func(ev fsnotify.Event) { - if module.closed.Load() || global.ShuttingDown() { - log.Warn("module closed, skip reloading pipelines") - return - } - - log.Infof("config changed, checking for new pipeline configs, %v, %v", ev.Op, ev.Name) - - newConfig := []pipeline.PipelineConfigV2{} - newConfig, err = getPipelineConfig() - - if err != nil { - log.Error(err) - return - } - - defer func() { - if !global.Env().IsDebug { - if r := recover(); r != nil { - var v string - switch r.(type) { - case error: - v = r.(error).Error() - case runtime.Error: - v = r.(runtime.Error).Error() - case string: - v = r.(string) - } - log.Error("error on apply pipeline change,", v) - } - } - }() - - needStopAndClean := []string{} - newPipelines := map[string]pipeline.PipelineConfigV2{} - - newPipelineNames := []string{} - for _, v := range newConfig { - newPipelines[v.Name] = v - newPipelineNames = append(newPipelineNames, v.Name) - } - - log.Debugf("we now have %v new pipelines: %v", len(newPipelineNames), newPipelineNames) - - module.configs.Range(func(k, v any) bool { - oldC, ok := v.(pipeline.PipelineConfigV2) - if !ok { - log.Warnf("impossible value from configs: %v", v) - return true - } - - // Don't stop transient pipelines - if oldC.Transient { - log.Debugf("transient pipeline %v should not be reloaded", oldC.Name) - return true - } - newC, ok := newPipelines[oldC.Name] - isSame := newC.Equals(oldC) - // Skip condition: (old pipeline is present in the new pipeline configs, config is the same, new config is also enabled) - if ok && isSame && isPipelineEnabled(newC.Enabled) { - log.Debugf("pipeline %v config not changed, skip reloading", oldC.Name) - return true - } - - log.Debug("pipeline config changed, stop and clean:", oldC.Name, ",", oldC, ",", ok, ",", isSame, ",", isPipelineEnabled(newC.Enabled)) - - needStopAndClean = append(needStopAndClean, oldC.Name) - return true - }) - - log.Debugf("we now have %v old pipelines need to clean: %v", len(needStopAndClean), needStopAndClean) - - if len(needStopAndClean) > 0 { - log.Trace("stop and wait for pipelines to release: ", needStopAndClean) - module.stopAndWaitForRelease(needStopAndClean, time.Minute) - log.Debug("old pipelines released") - - for _, taskID := range needStopAndClean { - log.Infof("removing pipeline [%s]", taskID) - module.deleteTask(taskID) - } - } - - log.Debugf("starting %v pipelines", len(newPipelineNames)) - for k, v := range newPipelines { - err := module.createPipeline(v, false) - if err != nil { - log.Errorf("failed to create pipeline: %v, err: %v", k, err) - } - } - }) - - return nil -} - -func (module *PipeModule) Stop() error { - if module.closed.Load() { - return nil + ctx.Refresh = orm.WaitForRefresh + err = orm.Save(ctx, &newConfig) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return } - module.closed.Store(true) - total := util.GetSyncMapSize(&module.contexts) - if total <= 0 { - return nil - } + h.WriteUpdatedOKJSON(w, newConfig.ID) +} - log.Info("shutting down pipelines") +func (h *PipeModule) deletePipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("id") - var taskIDs []string - module.contexts.Range(func(key, value any) bool { - taskID, ok := key.(string) - if !ok { - return false - } - taskIDs = append(taskIDs, taskID) - return true - }) + obj := pipeline.PipelineConfigV2{} + obj.ID = id - module.stopAndWaitForRelease(taskIDs, time.Minute*5) + ctx := orm.NewContextWithParent(req.Context()) - log.Info("finished shut down pipelines") - return nil -} + exists, err := orm.GetV2(ctx, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !exists { + h.WriteOpRecordNotFoundJSON(w, id) + return + } -func (module *PipeModule) stopAndWaitForRelease(taskIDs []string, timeout time.Duration) { - start := time.Now() - - for { - if time.Now().Sub(start) > timeout { - log.Error("waitForStop timed out") - break - } - - // Send stop signal to all contexts - for _, taskID := range taskIDs { - // cancel & stop - module.stopTask(taskID) - // release loop - module.releaseContext(taskID) - // don't delete context yet - } - - needRetry := false - for _, taskID := range taskIDs { - v, ok := module.contexts.Load(taskID) - if !ok { - continue - } - - ctx, ok := v.(*pipeline.Context) - if !ok { - log.Errorf("impossible value from contexts: %t", v) - continue - } - - if !ctx.IsLoopReleased() { - if rate.GetRateLimiterPerSecond("pipeline", "shutdown"+taskID+string(ctx.GetRunningState()), 1).Allow() { - log.Debug("pipeline still running: ", taskID, ",state: ", ctx.GetRunningState()) - } - needRetry = true - break - } - } - - if !needRetry { - break - } + ctx.Refresh = orm.WaitForRefresh + err = orm.Delete(ctx, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return } -} -const pipelineSingleton = "pipeline_singleton" + h.WriteDeletedOKJSON(w, id) +} -var creatingLocker = sync.Mutex{} +func (h *PipeModule) searchPipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { -func (module *PipeModule) createPipeline(v pipeline.PipelineConfigV2, transient bool) error { - if module.closed.Load() { - return errors.New("module closed") + var err error + builder, err := orm.NewQueryBuilderFromRequest(req, "name", "combined_fulltext") + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return } - if !isPipelineEnabled(v.Enabled) { - // pipeline config explicitly disabled - return nil + if len(builder.Sorts()) == 0 { + builder.SortBy(orm.Sort{Field: "created", SortType: orm.DESC}) } - creatingLocker.Lock() - defer creatingLocker.Unlock() + builder.EnableBodyBytes() - v.Transient = transient + ctx := orm.NewContextWithParent(req.Context()) + orm.WithModel(ctx, &pipeline.PipelineConfigV2{}) - // NOTE: hold the slot before creating pipeline loops - if _, ok := module.configs.LoadOrStore(v.Name, v); ok { - log.Tracef("pipeline [%v] is already created, skip", v.Name) - return nil - } + var objs []pipeline.PipelineConfigV2 - if v.Singleton { - log.Info("creating pipeline: " + v.Name + ", singleton") - } else { - log.Info("creating pipeline: " + v.Name) + err, res := elastic.SearchV2WithResultItemMapper(ctx, &objs, builder, nil) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return } - task.RunWithContext("pipeline:"+v.Name, func(taskCtx context.Context) error { - - cfgV := taskCtx.Value("cfg") - cfg, ok := cfgV.(pipeline.PipelineConfigV2) - if !ok { - return errors.New("invalid pipeline config") - } - processors, err := cfg.GetProcessorsConfig() - if err != nil { - return errors.Errorf("failed to get processor config, %v", err) - } - - processor, err := pipeline.NewPipeline(processors) - if err != nil { - return err - } - - ctx := pipeline.AcquireContext(v) - module.pipelines.Store(v.Name, processor) - module.contexts.Store(v.Name, ctx) - - defer func() { - if !global.Env().IsDebug { - if r := recover(); r != nil { - var err string - switch r.(type) { - case error: - err = r.(error).Error() - case runtime.Error: - err = r.(runtime.Error).Error() - case string: - err = r.(string) - } - log.Errorf("error on pipeline: %v, retry delay: %vms", cfg.Name, err) - } - } - - ctx.SetLoopReleased() - processor.Release() - }() - - if !cfg.AutoStart { - // Mark pipeline as exited, don't run automatically - ctx.Exit() - } else { - ctx.Starting() - } - - log.Debug("processing pipeline_v2: ", cfg.Name) - - retryDelayInMs := 1000 - if cfg.RetryDelayInMs > 0 { - retryDelayInMs = cfg.RetryDelayInMs - } - - started := false - - for { - if global.ShuttingDown() { - log.Debugf("system is shutting down, pipeline [%v] will be stopped", cfg.Name) - break - } - - if ctx.IsReleased() { - break - } - if module.closed.Load() { - break - } - if ctx.IsExit() { - ctx.Stopped() - } - - // NOTE: state must be checked as the last step - state := ctx.GetRunningState() - if global.Env().IsDebug { - log.Tracef("pipeline [%v], state: %v", cfg.Name, state) - time.Sleep(500 * time.Millisecond) - } - switch state { - case pipeline.STARTING: - - //check - if v.Singleton { - if v.MaxRunningInMs <= 0 { - v.MaxRunningInMs = 60000 - } - ok, err := locker.Hold(pipelineSingleton, v.Name, global.Env().SystemConfig.NodeConfig.ID, time.Duration(v.MaxRunningInMs)*time.Millisecond, true) - if !ok { - log.Debugf("pipeline [%v] is already running somewhere, %v", cfg.Name, err) - ctx.Finished() - continue - } - } - - // Pipeline needs to run - if started { - log.Errorf("pipeline [%v] started twice, should not happen", cfg.Name) - } - started = true - ctx.Started() - ctx.ResetContext() - - err = processor.Process(ctx) - - if err != nil { - log.Errorf("error on pipeline:%v, %v", cfg.Name, err) - ctx.Failed(err) - } else { - if global.Env().IsDebug { - log.Debugf("pipeline [%v] end running", cfg.Name) - } - ctx.Finished() - } - started = false - case pipeline.STARTED, pipeline.STOPPING: - log.Errorf("pipeline [%v] loop should not detect %s", cfg.Name, state) - case pipeline.FINISHED, pipeline.FAILED: - // Pipeline ended, pause or start next round - // keep_running: true & not stopped manually by Exit() - // For IsExit, don't pause here, wait for STOPPED state, or we could Pause twice for STOPPED & IsExit. - if cfg.KeepRunning { - if global.Env().IsDebug { - log.Tracef("pipeline [%v] end running, restart again, retry in [%v]ms", cfg.Name, retryDelayInMs) - } - - if retryDelayInMs > 0 { - log.Tracef("start sleep [%v]ms", retryDelayInMs) - time.Sleep(time.Duration(retryDelayInMs) * time.Millisecond) - log.Tracef("end sleep [%v]ms", retryDelayInMs) - } - - // restart after delay. - ctx.Starting() - } else { - ctx.Stopped() - ctx.Pause() - } - case pipeline.STOPPED: - // Pipeline manually stopped, pause - ctx.Pause() - } - } - - log.Debugf("pipeline [%v] loop exited with state [%v]", cfg.Name, ctx.GetRunningState()) - - return nil - }, context.WithValue(context.Background(), "cfg", v)) - - return nil -} - -func isPipelineEnabled(enabled *bool) bool { - // if not configured `enabled: true`, by default true - if enabled == nil { - return moduleCfg.PipelineEnabledByDefault + _, err = h.Write(w, res.Raw) + if err != nil { + h.Error(w, err) } - return *enabled } diff --git a/modules/pipeline/proto.go b/modules/pipeline/proto.go index cfabbb9d0..922bb27cc 100644 --- a/modules/pipeline/proto.go +++ b/modules/pipeline/proto.go @@ -25,13 +25,13 @@ package pipeline import "infini.sh/framework/core/pipeline" -type GetPipelinesResponse map[string]*PipelineStatus +type GetPipelineTasksResponse map[string]*PipelineTaskStatus type CreatePipelineRequest struct { pipeline.PipelineConfigV2 Processors []map[string]interface{} `json:"processor"` } -type SearchPipelinesRequest struct { +type SearchPipelineTasksRequest struct { Ids []string `json:"ids"` } diff --git a/modules/pipeline/api.go b/modules/pipeline/tasks.go similarity index 74% rename from modules/pipeline/api.go rename to modules/pipeline/tasks.go index d34346ec0..eb7bc6662 100644 --- a/modules/pipeline/api.go +++ b/modules/pipeline/tasks.go @@ -32,16 +32,16 @@ import ( "infini.sh/framework/core/util" ) -func (module *PipeModule) getPipelinesHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +func (module *PipeModule) getRunningPipelineTasksHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { config := module.Get(req, "config", "false") processor := module.Get(req, "processor", "false") - resp := GetPipelinesResponse{} + resp := GetPipelineTasksResponse{} module.configs.Range(func(key, value any) bool { id, ok := key.(string) if !ok { return true } - status := module.getPipelineStatus(id, config, processor) + status := module.getPipelineTaskStatus(id, config, processor) if status != nil { resp[id] = status } @@ -50,19 +50,19 @@ func (module *PipeModule) getPipelinesHandler(w http.ResponseWriter, req *http.R module.WriteJSON(w, resp, 200) } -func (module *PipeModule) searchPipelinesHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +func (module *PipeModule) searchPipelineTasksHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { config := module.Get(req, "config", "false") processor := module.Get(req, "processor", "false") - var obj = SearchPipelinesRequest{} + var obj = SearchPipelineTasksRequest{} err := module.DecodeJSON(req, &obj) if err != nil { module.WriteError(w, err.Error(), http.StatusBadRequest) _ = log.Error("failed to parse request: ", err) return } - resp := GetPipelinesResponse{} + resp := GetPipelineTasksResponse{} for _, id := range obj.Ids { - status := module.getPipelineStatus(id, config, processor) + status := module.getPipelineTaskStatus(id, config, processor) if status != nil { resp[id] = status } @@ -70,7 +70,7 @@ func (module *PipeModule) searchPipelinesHandler(w http.ResponseWriter, req *htt module.WriteJSON(w, resp, 200) } -func (module *PipeModule) getPipelineStatus(id string, config string, processor string) *PipelineStatus { +func (module *PipeModule) getPipelineTaskStatus(id string, config string, processor string) *PipelineTaskStatus { c, ok := module.contexts.Load(id) if !ok { return nil @@ -79,7 +79,7 @@ func (module *PipeModule) getPipelineStatus(id string, config string, processor if !ok { return nil } - ret := &PipelineStatus{ + ret := &PipelineTaskStatus{ State: c1.GetRunningState(), CreateTime: c1.GetCreateTime(), StartTime: c1.GetStartTime(), @@ -117,11 +117,11 @@ func (module *PipeModule) getPipelineStatus(id string, config string, processor return ret } -func (module *PipeModule) getPipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +func (module *PipeModule) getPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.ByName("id") config := module.Get(req, "config", "false") processor := module.Get(req, "processor", "false") - status := module.getPipelineStatus(id, config, processor) + status := module.getPipelineTaskStatus(id, config, processor) if status == nil { module.WriteError(w, "pipeline not found", http.StatusNotFound) return @@ -129,8 +129,10 @@ func (module *PipeModule) getPipelineHandler(w http.ResponseWriter, req *http.Re module.WriteJSON(w, status, 200) } +// For the task pipeline config, if it is enabled, create a task to run it. +// // eg: curl -XPOST http://localhost:2900/pipeline/tasks/ -d'{"name":"echo-test","enabled":true,"auto_start":true,"keep_running":true,"processor":[{"echo":{"message":"hello world"}}]}' -func (module *PipeModule) createPipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +func (module *PipeModule) createPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { var obj = pipeline.PipelineConfigV2{} err := module.DecodeJSON(req, &obj) if err != nil { @@ -138,7 +140,14 @@ func (module *PipeModule) createPipelineHandler(w http.ResponseWriter, req *http _ = log.Error("failed to parse pipeline config: ", err) return } - err = module.createPipeline(obj, true) + if obj.Name == "" { + module.WriteError(w, "name is required", http.StatusBadRequest) + return + } + if obj.ID == "" { + obj.ID = obj.Name + } + err = module.createPipelineTask(obj, true) if err != nil { module.WriteError(w, err.Error(), http.StatusBadRequest) _ = log.Error("failed to start pipeline: ", err) @@ -148,7 +157,7 @@ func (module *PipeModule) createPipelineHandler(w http.ResponseWriter, req *http } // curl -XDELETE http://localhost:2900/pipeline/task/echo-test -func (module *PipeModule) deletePipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +func (module *PipeModule) deletePipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.ByName("id") _, exists := module.contexts.Load(id) if exists { @@ -162,7 +171,7 @@ func (module *PipeModule) deletePipelineHandler(w http.ResponseWriter, req *http } // curl -XPOST http://localhost:2900/pipeline/task/echo-test/_start -func (module *PipeModule) startTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +func (module *PipeModule) startPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.ByName("id") exists := module.startTask(id) if exists { @@ -175,7 +184,7 @@ func (module *PipeModule) startTaskHandler(w http.ResponseWriter, req *http.Requ } // curl -XPOST http://localhost:2900/pipeline/task/echo-test/_stop -func (module *PipeModule) stopTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { +func (module *PipeModule) stopPipelineTaskHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.ByName("id") exists := module.stopTask(id) if exists { diff --git a/modules/security/access_token/authentication.go b/modules/security/access_token/authentication.go index a267aa713..5ae6b8348 100644 --- a/modules/security/access_token/authentication.go +++ b/modules/security/access_token/authentication.go @@ -103,7 +103,7 @@ func byAPITokenHeader(w http.ResponseWriter, r *http.Request) (claims *security. func getTokenPermissions(apiToken string) (*security.AccessToken, []security.PermissionKey, error) { if apiToken == "" { - return nil, nil, errors.Error("api token not found") + return nil, nil, errors.Error("API token not provided") } bytes, err := kv.GetValue(KVAccessTokenBucket, []byte(apiToken)) @@ -175,7 +175,7 @@ func RequestAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter. }{} err = api.DecodeJSON(req, &reqBody) if err != nil { - panic(err) + panic(errors.NewWithHTTPCode(400, "invalid token")) } if reqBody.Name == "" { reqBody.Name = GenerateApiTokenName("") @@ -187,7 +187,7 @@ func RequestAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter. if len(reqBody.Permissions) > 0 { // requested permissions must be within the caller's own scope if !util.IsSuperset(security.ConvertPermissionKeysToHashSet(permissions), security.ConvertPermissionKeysToHashSet(reqBody.Permissions)) { - panic("invalid permissions") + panic(errors.NewWithHTTPCode(403, "invalid permissions")) } permissions = reqBody.Permissions } diff --git a/modules/security/access_token/authorization.go b/modules/security/access_token/authorization.go index 25a0306fe..d0906ae56 100644 --- a/modules/security/access_token/authorization.go +++ b/modules/security/access_token/authorization.go @@ -21,11 +21,11 @@ type SecurityBackendProvider struct { func (provider *SecurityBackendProvider) GetPermissionKeysByUserID(ctx1 context.Context, providerID, userID string) []security.PermissionKey { var allowedPermissions = []security.PermissionKey{} - if providerID==ProviderName{ + if providerID == ProviderName { _, permissions, err := getTokenPermissions(userID) - if err!=nil { + if err != nil { log.Error(err) - }else{ + } else { return permissions } } @@ -36,4 +36,3 @@ func (provider *SecurityBackendProvider) GetPermissionKeysByUserID(ctx1 context. func (provider *SecurityBackendProvider) GetPermissionKeysByRoles(ctx context.Context, roles []string) []security.PermissionKey { return []security.PermissionKey{} } - diff --git a/modules/security/rbac/role.go b/modules/security/rbac/role.go index a38e21e28..a611d881a 100644 --- a/modules/security/rbac/role.go +++ b/modules/security/rbac/role.go @@ -196,7 +196,7 @@ type SecurityBackendProvider struct { func (provider *SecurityBackendProvider) GetPermissionKeysByUserID(ctx1 context.Context, providerID, userID string) []security.PermissionKey { - if providerID!=security.DefaultNativeAuthBackend{ + if providerID != security.DefaultNativeAuthBackend { return nil } diff --git a/modules/stats/simple.go b/modules/stats/simple.go index c02dcc19d..3ee895883 100755 --- a/modules/stats/simple.go +++ b/modules/stats/simple.go @@ -40,7 +40,7 @@ import ( log "github.com/cihub/seelog" "github.com/segmentio/encoding/json" - "github.com/shirou/gopsutil/v3/process" + "github.com/shirou/gopsutil/v4/process" "infini.sh/framework/core/api" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/env" From 1dfdbc1fc2fe26dc92137a7fe26a3abe06f550a3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 25 May 2026 07:48:31 +0000 Subject: [PATCH 5/7] Merge main and reconcile security conflicts for managed token bootstrap flow Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/61a03f37-2d99-48ff-b97e-934d39469c33 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> --- config/generated.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/generated.go b/config/generated.go index baf497913..d81d01e80 100644 --- a/config/generated.go +++ b/config/generated.go @@ -1,11 +1,11 @@ package config -const LastCommitLog = "N/A" +const LastCommitLog = "c8b12796b2008d903e1556e349caf2cc517253c5" -const BuildDate = "N/A" +const BuildDate = "2026-05-25T07:45:53Z" -const EOLDate = "N/A" +const EOLDate = "2023-12-31T10:10:10Z" -const Version = "0.0.1-SNAPSHOT" +const Version = "1.0.0_SNAPSHOT" -const BuildNumber = "001" +const BuildNumber = "001" From ef778e951bf4cf23b05a2c9573a2e906d2dba5a2 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 25 May 2026 16:46:04 +0800 Subject: [PATCH 6/7] chore: drop generated config diff Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- config/generated.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/generated.go b/config/generated.go index d81d01e80..baf497913 100644 --- a/config/generated.go +++ b/config/generated.go @@ -1,11 +1,11 @@ package config -const LastCommitLog = "c8b12796b2008d903e1556e349caf2cc517253c5" +const LastCommitLog = "N/A" -const BuildDate = "2026-05-25T07:45:53Z" +const BuildDate = "N/A" -const EOLDate = "2023-12-31T10:10:10Z" +const EOLDate = "N/A" -const Version = "1.0.0_SNAPSHOT" +const Version = "0.0.1-SNAPSHOT" -const BuildNumber = "001" +const BuildNumber = "001" From 944ef36fcfd8df93610aa96ab17c5d3519bcd908 Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 26 May 2026 16:06:59 +0800 Subject: [PATCH 7/7] refactor: refactoring instance access_token --- core/credential/credential.go | 14 ++++++++------ core/model/const.go | 8 ++++++++ core/model/instance.go | 4 +--- modules/configs/client/client.go | 2 +- modules/configs/client/client_test.go | 5 +++-- modules/configs/common/domain.go | 1 - 6 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 core/model/const.go diff --git a/core/credential/credential.go b/core/credential/credential.go index 1f7b1a692..6573e13a0 100644 --- a/core/credential/credential.go +++ b/core/credential/credential.go @@ -36,10 +36,10 @@ import ( type Credential struct { orm.ORMObjectBase - Name string `json:"name" elastic_mapping:"name:{type:keyword,copy_to:search_text}"` - Type string `json:"type" elastic_mapping:"type:{type:keyword}"` - Tags []string `json:"tags" elastic_mapping:"category:{type:keyword,copy_to:search_text}"` - Payload map[string]interface{} `json:"payload" elastic_mapping:"payload:{type:object,enabled:false}"` + Name string `json:"name" elastic_mapping:"name:{type:keyword,copy_to:search_text}"` + Type CredentialType `json:"type" elastic_mapping:"type:{type:keyword}"` + Tags []string `json:"tags" elastic_mapping:"category:{type:keyword,copy_to:search_text}"` + Payload map[CredentialType]interface{} `json:"payload" elastic_mapping:"payload:{type:object,enabled:false}"` Encrypt struct { Type string `json:"type"` Params map[string]interface{} `json:"params"` @@ -117,7 +117,9 @@ func (cred *Credential) Decode() (interface{}, error) { } } +type CredentialType string + const ( - BasicAuth string = "basic_auth" - AccessToken string = "access_token" + BasicAuth CredentialType = "basic_auth" + AccessToken CredentialType = "access_token" ) diff --git a/core/model/const.go b/core/model/const.go new file mode 100644 index 000000000..425a867f1 --- /dev/null +++ b/core/model/const.go @@ -0,0 +1,8 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package model + +const API_TOKEN = "X-API-TOKEN" +const CredentialIDSystemKey = "credential_id" diff --git a/core/model/instance.go b/core/model/instance.go index f2f04c4e3..18d79fea8 100644 --- a/core/model/instance.go +++ b/core/model/instance.go @@ -56,7 +56,7 @@ type Instance struct { BasicAuth *BasicAuth `config:"basic_auth" json:"basic_auth,omitempty" elastic_mapping:"basic_auth:{type:object}"` - CredentialID string `json:"credential_id,omitempty" elastic_mapping:"credential_id:{type:keyword}"` + AccessToken string `json:"access_token,omitempty" elastic_mapping:"access_token:{type:keyword}"` Labels map[string]string `json:"labels,omitempty" elastic_mapping:"labels:{type:object}"` Tags []string `json:"tags,omitempty"` @@ -71,8 +71,6 @@ type Instance struct { Network NetworkInfo `json:"network,omitempty" elastic_mapping:"network: { type: object }"` Services []ServiceInfo `json:"services,omitempty" elastic_mapping:"services: { type: object }"` Status string `json:"status,omitempty" elastic_mapping:"status: { type: keyword, copy_to:search_text }"` - - //SearchText string `json:"search_text,omitempty" elastic_mapping:"search_text:{type:text,index_prefixes:{},index_phrases:true, analyzer:suggest_text_search }"` } type ServiceInfo struct { diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index cb5e28811..ca47df32a 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -135,7 +135,7 @@ func applyManagerRequestAuth(req *util.Request, username, password, token string req.SetBasicAuth(username, password) } if token != "" { - req.AddHeader(common.API_TOKEN, token) + req.AddHeader(model.API_TOKEN, token) } } diff --git a/modules/configs/client/client_test.go b/modules/configs/client/client_test.go index 0ee3c560a..e613ded21 100644 --- a/modules/configs/client/client_test.go +++ b/modules/configs/client/client_test.go @@ -1,6 +1,7 @@ package client import ( + "infini.sh/framework/core/model" "testing" "infini.sh/framework/core/util" @@ -12,7 +13,7 @@ func TestApplyManagerRequestAuthAddsRegisterTokenHeader(t *testing.T) { applyManagerRequestAuth(&req, "", "", "token-1") - if got := req.AllHeaders()[common.API_TOKEN]; got != "token-1" { + if got := req.AllHeaders()[model.API_TOKEN]; got != "token-1" { t.Fatalf("expected register token header %q, got %q", "token-1", got) } } @@ -22,7 +23,7 @@ func TestApplyManagerRequestAuthAddsTokenHeaderForSync(t *testing.T) { applyManagerRequestAuth(&req, "", "", "token-1") - if got := req.AllHeaders()[common.API_TOKEN]; got != "token-1" { + if got := req.AllHeaders()[model.API_TOKEN]; got != "token-1" { t.Fatalf("expected sync token header %q, got %q", "token-1", got) } } diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go index 7e51c38bf..232c1fc52 100644 --- a/modules/configs/common/domain.go +++ b/modules/configs/common/domain.go @@ -31,7 +31,6 @@ import "infini.sh/framework/core/model" const REGISTER_API = "/instance/_register" const SYNC_API = "/configs/_sync" -const API_TOKEN = "X-API-TOKEN" type ConfigFile struct { Name string `json:"name,omitempty"`