feat: implements git gc cron job & add cronjob ssh command

cmd/soft/serve/server.go

@@ -59,7 +59,16 @@ func NewServer(ctx context.Context) (*Server, error) {
 	// Add cron jobs.
 	sched := cron.NewScheduler(ctx)
 	for n, j := range jobs.List() {
-		id, err := sched.AddFunc(j.Runner.Spec(ctx), j.Runner.Func(ctx))
+		jobCfg, err := j.Runner.Config(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("parse cronjob [%s] config: %w", n, err)
+		}
+		spec := jobCfg.Spec()
+		if spec == "" {
+			continue
+		}
+
+		id, err := sched.AddFunc(spec, j.Runner.Func(ctx, jobCfg))
 		if err != nil {
 			logger.Warn("error adding cron job", "job", n, "err", err)
 		}
@@ -183,7 +192,10 @@ func (s *Server) Shutdown(ctx context.Context) error {
 	})
 	errg.Go(func() error {
 		for _, j := range jobs.List() {
-			s.Cron.Remove(j.ID)
+			// jobID from github.com/robfig/cron/v2 starts from 1
+			if j.ID != 0 {
+				s.Cron.Remove(j.ID)
+			}
 		}
 		s.Cron.Stop()
 		return nil

git/repo.go

@@ -1,7 +1,11 @@
 package git
 
 import (
+	"crypto/sha1"
+	"crypto/sha256"
+	"hash"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/aymanbagabas/git-module"
@@ -16,11 +20,20 @@ var (
 	DiffMaxLineChars = 1000
 )
 
+// Objbectformat is repository hash format
+type ObjectFormat func() hash.Hash
+
+var (
+	ObjectFormatSha1   ObjectFormat = sha1.New
+	ObjectFormatSha256 ObjectFormat = sha256.New
+)
+
 // Repository is a wrapper around git.Repository with helper methods.
 type Repository struct {
 	*git.Repository
-	Path   string
-	IsBare bool
+	Path        string
+	IsBare      bool
+	ObjctFormat ObjectFormat
 }
 
 // Clone clones a repository.
@@ -188,3 +201,32 @@ func (r *Repository) SymbolicRef(name string, ref string, opts ...git.SymbolicRe
 	opt.Ref = ref
 	return r.Repository.SymbolicRef(opt)
 }
+
+func (r *Repository) ComputeObjectHash(t git.ObjectType, content []byte) []byte {
+	hasher := r.Hasher()
+	_, _ = hasher.Write([]byte(t))
+	_, _ = hasher.Write([]byte(" "))
+	_, _ = hasher.Write([]byte(strconv.Itoa(len(content))))
+	_, _ = hasher.Write([]byte{0})
+	_, _ = hasher.Write(content)
+	return hasher.Sum(nil)
+}
+
+// Hasher returns the object format algorithem factory of repository
+func (r *Repository) Hasher() hash.Hash {
+	if r.ObjctFormat != nil {
+		return r.ObjctFormat()
+	}
+
+	out, _ := NewCommand("rev-parse", "--show-object-format").RunInDir(r.Path)
+	switch strings.TrimSpace(string(out)) {
+	case "sha1":
+		r.ObjctFormat = ObjectFormatSha1
+	case "sha256":
+		r.ObjctFormat = ObjectFormatSha256
+	default:
+		panic("unknown object format " + string(out))
+	}
+
+	return r.ObjctFormat()
+}

pkg/backend/lfs.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/charmbracelet/soft-serve/pkg/config"
 	"github.com/charmbracelet/soft-serve/pkg/db"
+	"github.com/charmbracelet/soft-serve/pkg/db/models"
 	"github.com/charmbracelet/soft-serve/pkg/lfs"
 	"github.com/charmbracelet/soft-serve/pkg/proto"
 	"github.com/charmbracelet/soft-serve/pkg/storage"
@@ -86,3 +87,46 @@ func StoreRepoMissingLFSObjects(ctx context.Context, repo proto.Repository, dbx
 
 	return nil
 }
+
+// GetUnreachableLFSObjects get lfs objects in database but not referenced within git repository.
+func (b *Backend) GetUnreachableLFSObjects(ctx context.Context, name string) ([]models.LFSObject, error) {
+	repo, err := b.Repository(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+	r, err := repo.Open()
+	if err != nil {
+		return nil, err
+	}
+
+	objs, err := b.store.GetLFSObjects(ctx, b.db, repo.ID())
+	if err != nil {
+		return nil, err
+	}
+	pointerChan := make(chan lfs.Pointer)
+	chkChan, stop := lfs.CheckPointerExist(ctx, r, pointerChan)
+	defer func() {
+		close(pointerChan)
+		stop()
+	}()
+
+	unreachObj := []models.LFSObject{}
+	for _, obj := range objs {
+		select {
+		case pointerChan <- lfs.Pointer{Oid: obj.Oid, Size: obj.Size}:
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+
+		select {
+		case exits := <-chkChan:
+			if !exits {
+				unreachObj = append(unreachObj, obj)
+			}
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+
+	return unreachObj, nil
+}

pkg/config/config.go

@@ -137,6 +137,7 @@ type LFSConfig struct {
 // JobsConfig is the configuration for cron jobs.
 type JobsConfig struct {
 	MirrorPull string `env:"MIRROR_PULL" yaml:"mirror_pull"`
+	GitGC      string `env:"GIT_GC" yaml:"git_gc"`
 }
 
 // Config is the configuration for Soft Serve.
@@ -220,6 +221,7 @@ func (c *Config) Environ() []string {
 		fmt.Sprintf("SOFT_SERVE_LFS_ENABLED=%t", c.LFS.Enabled),
 		fmt.Sprintf("SOFT_SERVE_LFS_SSH_ENABLED=%t", c.LFS.SSHEnabled),
 		fmt.Sprintf("SOFT_SERVE_JOBS_MIRROR_PULL=%s", c.Jobs.MirrorPull),
+		fmt.Sprintf("SOFT_SERVE_JOBS_GIT_GC=%s", c.Jobs.GitGC),
 	}...)
 
 	return envs

pkg/config/file.go

@@ -143,6 +143,7 @@ lfs:
 # Cron job configuration
 jobs:
   mirror_pull: "{{ .Jobs.MirrorPull }}"
+  git_gc: "{{ .Jobs.GitGC }}"
 
 # Additional admin keys.
 #initial_admin_keys:

pkg/jobs/garbage_collector.go

@@ -0,0 +1,185 @@
+package jobs
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"time"
+
+	"charm.land/log/v2"
+	"github.com/charmbracelet/soft-serve/git"
+	"github.com/charmbracelet/soft-serve/pkg/backend"
+	"github.com/charmbracelet/soft-serve/pkg/config"
+	"github.com/charmbracelet/soft-serve/pkg/db"
+	"github.com/charmbracelet/soft-serve/pkg/lfs"
+	"github.com/charmbracelet/soft-serve/pkg/proto"
+	"github.com/charmbracelet/soft-serve/pkg/storage"
+	"github.com/charmbracelet/soft-serve/pkg/store"
+	"github.com/charmbracelet/soft-serve/pkg/sync"
+	"github.com/spf13/pflag"
+)
+
+func init() {
+	Register("git-gc", gitGC{})
+}
+
+type (
+	gitGC       struct{}
+	gitGCConfig struct {
+		baseRunnerConfig
+
+		RepoConfig     map[string]string
+		LFSPruneExpire time.Duration
+		Aggressive     bool
+	}
+)
+
+// Description return the description of garbage collector job task and implements Runner.
+func (m gitGC) Description() string {
+	return "clean up the garbage in repositories"
+}
+
+// Config returns the garbage collector job task configuration and implements Runner.
+func (m gitGC) Config(ctx context.Context) (RunnerConfig, error) {
+	cfg := gitGCConfig{
+		baseRunnerConfig: baseRunnerConfig{CronSpec: ""},
+		Aggressive:       false,
+		RepoConfig:       make(map[string]string),
+		LFSPruneExpire:   time.Hour * 24 * 7,
+	}
+
+	if spec := config.FromContext(ctx).Jobs.GitGC; spec != "" {
+		cfg.CronSpec = spec
+	}
+
+	return &cfg, nil
+}
+
+// cleanup return garbage collection work function on a repository added to sync.WorkPool
+func (g gitGC) cleanup(ctx context.Context, jobcfg *gitGCConfig, repo proto.Repository) func() {
+	repoName := repo.Name()
+
+	b := backend.FromContext(ctx)
+	datastore := store.FromContext(ctx)
+	dbx := db.FromContext(ctx)
+	cfg := config.FromContext(ctx)
+
+	logger := log.FromContext(ctx).WithPrefix("jobs.gitgc")
+	logger = logger.With("repo", repoName)
+
+	return func() {
+		r, err := repo.Open()
+		if err != nil {
+			logger.Error("error opening repository", "err", err)
+			fmt.Fprintf(jobcfg.Error(), "[%s] error opening repository: %v\n", repoName, err)
+			return
+		}
+
+		// buffer and write to stdout/stderr in one go,
+		// avoiding output confusion through parallel writing.
+		var (
+			stdout = bytes.NewBuffer(nil)
+			stderr = bytes.NewBuffer(nil)
+		)
+		defer func() {
+			jobcfg.Output().Write(stdout.Bytes())
+			jobcfg.Error().Write(stderr.Bytes())
+		}()
+
+		logger.Debug("start git garbage collection")
+
+		var cmdArgs []string = nil
+		for key, val := range jobcfg.RepoConfig {
+			cmdArgs = append(cmdArgs, "-c", key+"="+val)
+		}
+
+		cmdArgs = append(cmdArgs, "gc")
+
+		if jobcfg.Aggressive {
+			cmdArgs = append(cmdArgs, "--aggressive")
+		}
+
+		// `git gc` would not output anything if no tty
+		cmd := git.NewCommand(cmdArgs...).WithContext(ctx)
+		if _, err := cmd.RunInDir(r.Path); err != nil {
+			logger.Error("error running git remote update", "err", err)
+			fmt.Fprintf(stderr, "[%s] git gc failed: %v\n", repoName, err)
+			return
+		}
+
+		// clean up unreachable lfs objects
+		if cfg.LFS.Enabled {
+			logger.Debug("start lfs objects garbage collection")
+			pruneBefore := time.Now().Add(-jobcfg.LFSPruneExpire)
+			repoID := strconv.FormatInt(repo.ID(), 10)
+			strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs", repoID))
+
+			objs, err := b.GetUnreachableLFSObjects(ctx, repoName)
+			if err != nil {
+				logger.Error("error get unreachable lfs objects", "err", err)
+				fmt.Fprintf(stderr, "[%s] get unreachable lfs objects: %v\n", repoName, err)
+				return
+			}
+
+			for _, obj := range objs {
+				if obj.UpdatedAt.Before(pruneBefore) {
+					if err := dbx.TransactionContext(ctx, func(tx *db.Tx) error {
+						if err := datastore.DeleteLFSObjectByOid(ctx, dbx, repo.ID(), obj.Oid); err != nil {
+							return err
+						}
+						p := lfs.Pointer{Oid: obj.Oid}
+						return strg.Delete(filepath.Join("objects", p.RelativePath()))
+					}); err != nil {
+						logger.Error("error clear lfs objects", "err", err, "oid", obj.Oid)
+						fmt.Fprintf(stderr, "[%s] error delete lfs object %s: %v\n", repoName, obj.Oid, err)
+					} else {
+						logger.Info("removed unreachable lfs object", "repo", repoName, "oid", obj.Oid)
+						fmt.Fprintf(stdout, "[%s] removed lfs object %s\n", repoName, obj.Oid)
+					}
+				}
+			}
+		}
+		fmt.Fprintf(stdout, "[%s] git gc successful\n", repoName)
+	}
+}
+
+// Func runs the garbage collector job task and implements Runner.
+func (g gitGC) Func(ctx context.Context, cronCfg RunnerConfig) func() {
+	b := backend.FromContext(ctx)
+	logger := log.FromContext(ctx).WithPrefix("jobs.gitgc")
+	jobcfg := cronCfg.(*gitGCConfig)
+
+	return func() {
+		repos, err := b.Repositories(ctx)
+		if err != nil {
+			logger.Error("error getting repositories", "err", err)
+			fmt.Fprintf(jobcfg.Error(), "error getting repositories: %v\n", err)
+			return
+		}
+
+		wq := sync.NewWorkPool(ctx, runtime.GOMAXPROCS(0),
+			sync.WithWorkPoolLogger(logger.Errorf),
+		)
+
+		for _, repo := range repos {
+			name := repo.Name()
+			wq.Add(name, g.cleanup(ctx, jobcfg, repo))
+		}
+
+		wq.Run()
+	}
+}
+
+// FlagSet returns the flag set that can modify configuration values and implements RunnerConfig
+func (cfg *gitGCConfig) FlagSet() *pflag.FlagSet {
+	flags := pflag.NewFlagSet("git-gc", pflag.ContinueOnError)
+	flags.StringToStringVarP(&cfg.RepoConfig, "config", "c", cfg.RepoConfig, "Override values from git repository configuration files")
+
+	flags.BoolVar(&cfg.Aggressive, "aggressive", cfg.Aggressive, "Optimize the repository more aggressively, see git-gc(1) for more details")
+	flags.DurationVar(&cfg.LFSPruneExpire, "lfsprune-expire", cfg.LFSPruneExpire, "Expire duration of unreachable LFS objects following the latest update")
+
+	return flags
+}