diff --git a/cmd/soft/serve/server.go b/cmd/soft/serve/server.go index fda090050..25162bf62 100644 --- a/cmd/soft/serve/server.go +++ b/cmd/soft/serve/server.go @@ -59,7 +59,16 @@ func NewServer(ctx context.Context) (*Server, error) { // Add cron jobs. sched := cron.NewScheduler(ctx) for n, j := range jobs.List() { - id, err := sched.AddFunc(j.Runner.Spec(ctx), j.Runner.Func(ctx)) + jobCfg, err := j.Runner.Config(ctx) + if err != nil { + return nil, fmt.Errorf("parse cronjob [%s] config: %w", n, err) + } + spec := jobCfg.Spec() + if spec == "" { + continue + } + + id, err := sched.AddFunc(spec, j.Runner.Func(ctx, jobCfg)) if err != nil { logger.Warn("error adding cron job", "job", n, "err", err) } @@ -183,7 +192,10 @@ func (s *Server) Shutdown(ctx context.Context) error { }) errg.Go(func() error { for _, j := range jobs.List() { - s.Cron.Remove(j.ID) + // jobID from github.com/robfig/cron/v2 starts from 1 + if j.ID != 0 { + s.Cron.Remove(j.ID) + } } s.Cron.Stop() return nil diff --git a/git/repo.go b/git/repo.go index 382f07ee0..4e01ebe32 100644 --- a/git/repo.go +++ b/git/repo.go @@ -1,7 +1,11 @@ package git import ( + "crypto/sha1" + "crypto/sha256" + "hash" "path/filepath" + "strconv" "strings" "github.com/aymanbagabas/git-module" @@ -16,11 +20,20 @@ var ( DiffMaxLineChars = 1000 ) +// Objbectformat is repository hash format +type ObjectFormat func() hash.Hash + +var ( + ObjectFormatSha1 ObjectFormat = sha1.New + ObjectFormatSha256 ObjectFormat = sha256.New +) + // Repository is a wrapper around git.Repository with helper methods. type Repository struct { *git.Repository - Path string - IsBare bool + Path string + IsBare bool + ObjctFormat ObjectFormat } // Clone clones a repository. @@ -188,3 +201,32 @@ func (r *Repository) SymbolicRef(name string, ref string, opts ...git.SymbolicRe opt.Ref = ref return r.Repository.SymbolicRef(opt) } + +func (r *Repository) ComputeObjectHash(t git.ObjectType, content []byte) []byte { + hasher := r.Hasher() + _, _ = hasher.Write([]byte(t)) + _, _ = hasher.Write([]byte(" ")) + _, _ = hasher.Write([]byte(strconv.Itoa(len(content)))) + _, _ = hasher.Write([]byte{0}) + _, _ = hasher.Write(content) + return hasher.Sum(nil) +} + +// Hasher returns the object format algorithem factory of repository +func (r *Repository) Hasher() hash.Hash { + if r.ObjctFormat != nil { + return r.ObjctFormat() + } + + out, _ := NewCommand("rev-parse", "--show-object-format").RunInDir(r.Path) + switch strings.TrimSpace(string(out)) { + case "sha1": + r.ObjctFormat = ObjectFormatSha1 + case "sha256": + r.ObjctFormat = ObjectFormatSha256 + default: + panic("unknown object format " + string(out)) + } + + return r.ObjctFormat() +} diff --git a/pkg/backend/lfs.go b/pkg/backend/lfs.go index 16569094a..c6710cdf4 100644 --- a/pkg/backend/lfs.go +++ b/pkg/backend/lfs.go @@ -10,6 +10,7 @@ import ( "github.com/charmbracelet/soft-serve/pkg/config" "github.com/charmbracelet/soft-serve/pkg/db" + "github.com/charmbracelet/soft-serve/pkg/db/models" "github.com/charmbracelet/soft-serve/pkg/lfs" "github.com/charmbracelet/soft-serve/pkg/proto" "github.com/charmbracelet/soft-serve/pkg/storage" @@ -86,3 +87,46 @@ func StoreRepoMissingLFSObjects(ctx context.Context, repo proto.Repository, dbx return nil } + +// GetUnreachableLFSObjects get lfs objects in database but not referenced within git repository. +func (b *Backend) GetUnreachableLFSObjects(ctx context.Context, name string) ([]models.LFSObject, error) { + repo, err := b.Repository(ctx, name) + if err != nil { + return nil, err + } + r, err := repo.Open() + if err != nil { + return nil, err + } + + objs, err := b.store.GetLFSObjects(ctx, b.db, repo.ID()) + if err != nil { + return nil, err + } + pointerChan := make(chan lfs.Pointer) + chkChan, stop := lfs.CheckPointerExist(ctx, r, pointerChan) + defer func() { + close(pointerChan) + stop() + }() + + unreachObj := []models.LFSObject{} + for _, obj := range objs { + select { + case pointerChan <- lfs.Pointer{Oid: obj.Oid, Size: obj.Size}: + case <-ctx.Done(): + return nil, ctx.Err() + } + + select { + case exits := <-chkChan: + if !exits { + unreachObj = append(unreachObj, obj) + } + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + return unreachObj, nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 97a5dc43d..b7e0aa95f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -137,6 +137,7 @@ type LFSConfig struct { // JobsConfig is the configuration for cron jobs. type JobsConfig struct { MirrorPull string `env:"MIRROR_PULL" yaml:"mirror_pull"` + GitGC string `env:"GIT_GC" yaml:"git_gc"` } // Config is the configuration for Soft Serve. @@ -220,6 +221,7 @@ func (c *Config) Environ() []string { fmt.Sprintf("SOFT_SERVE_LFS_ENABLED=%t", c.LFS.Enabled), fmt.Sprintf("SOFT_SERVE_LFS_SSH_ENABLED=%t", c.LFS.SSHEnabled), fmt.Sprintf("SOFT_SERVE_JOBS_MIRROR_PULL=%s", c.Jobs.MirrorPull), + fmt.Sprintf("SOFT_SERVE_JOBS_GIT_GC=%s", c.Jobs.GitGC), }...) return envs diff --git a/pkg/config/file.go b/pkg/config/file.go index 8170493c3..bd9e58037 100644 --- a/pkg/config/file.go +++ b/pkg/config/file.go @@ -143,6 +143,7 @@ lfs: # Cron job configuration jobs: mirror_pull: "{{ .Jobs.MirrorPull }}" + git_gc: "{{ .Jobs.GitGC }}" # Additional admin keys. #initial_admin_keys: diff --git a/pkg/jobs/garbage_collector.go b/pkg/jobs/garbage_collector.go new file mode 100644 index 000000000..8dea9a949 --- /dev/null +++ b/pkg/jobs/garbage_collector.go @@ -0,0 +1,185 @@ +package jobs + +import ( + "bytes" + "context" + "fmt" + "path/filepath" + "runtime" + "strconv" + "time" + + "charm.land/log/v2" + "github.com/charmbracelet/soft-serve/git" + "github.com/charmbracelet/soft-serve/pkg/backend" + "github.com/charmbracelet/soft-serve/pkg/config" + "github.com/charmbracelet/soft-serve/pkg/db" + "github.com/charmbracelet/soft-serve/pkg/lfs" + "github.com/charmbracelet/soft-serve/pkg/proto" + "github.com/charmbracelet/soft-serve/pkg/storage" + "github.com/charmbracelet/soft-serve/pkg/store" + "github.com/charmbracelet/soft-serve/pkg/sync" + "github.com/spf13/pflag" +) + +func init() { + Register("git-gc", gitGC{}) +} + +type ( + gitGC struct{} + gitGCConfig struct { + baseRunnerConfig + + RepoConfig map[string]string + LFSPruneExpire time.Duration + Aggressive bool + } +) + +// Description return the description of garbage collector job task and implements Runner. +func (m gitGC) Description() string { + return "clean up the garbage in repositories" +} + +// Config returns the garbage collector job task configuration and implements Runner. +func (m gitGC) Config(ctx context.Context) (RunnerConfig, error) { + cfg := gitGCConfig{ + baseRunnerConfig: baseRunnerConfig{CronSpec: ""}, + Aggressive: false, + RepoConfig: make(map[string]string), + LFSPruneExpire: time.Hour * 24 * 7, + } + + if spec := config.FromContext(ctx).Jobs.GitGC; spec != "" { + cfg.CronSpec = spec + } + + return &cfg, nil +} + +// cleanup return garbage collection work function on a repository added to sync.WorkPool +func (g gitGC) cleanup(ctx context.Context, jobcfg *gitGCConfig, repo proto.Repository) func() { + repoName := repo.Name() + + b := backend.FromContext(ctx) + datastore := store.FromContext(ctx) + dbx := db.FromContext(ctx) + cfg := config.FromContext(ctx) + + logger := log.FromContext(ctx).WithPrefix("jobs.gitgc") + logger = logger.With("repo", repoName) + + return func() { + r, err := repo.Open() + if err != nil { + logger.Error("error opening repository", "err", err) + fmt.Fprintf(jobcfg.Error(), "[%s] error opening repository: %v\n", repoName, err) + return + } + + // buffer and write to stdout/stderr in one go, + // avoiding output confusion through parallel writing. + var ( + stdout = bytes.NewBuffer(nil) + stderr = bytes.NewBuffer(nil) + ) + defer func() { + jobcfg.Output().Write(stdout.Bytes()) + jobcfg.Error().Write(stderr.Bytes()) + }() + + logger.Debug("start git garbage collection") + + var cmdArgs []string = nil + for key, val := range jobcfg.RepoConfig { + cmdArgs = append(cmdArgs, "-c", key+"="+val) + } + + cmdArgs = append(cmdArgs, "gc") + + if jobcfg.Aggressive { + cmdArgs = append(cmdArgs, "--aggressive") + } + + // `git gc` would not output anything if no tty + cmd := git.NewCommand(cmdArgs...).WithContext(ctx) + if _, err := cmd.RunInDir(r.Path); err != nil { + logger.Error("error running git remote update", "err", err) + fmt.Fprintf(stderr, "[%s] git gc failed: %v\n", repoName, err) + return + } + + // clean up unreachable lfs objects + if cfg.LFS.Enabled { + logger.Debug("start lfs objects garbage collection") + pruneBefore := time.Now().Add(-jobcfg.LFSPruneExpire) + repoID := strconv.FormatInt(repo.ID(), 10) + strg := storage.NewLocalStorage(filepath.Join(cfg.DataPath, "lfs", repoID)) + + objs, err := b.GetUnreachableLFSObjects(ctx, repoName) + if err != nil { + logger.Error("error get unreachable lfs objects", "err", err) + fmt.Fprintf(stderr, "[%s] get unreachable lfs objects: %v\n", repoName, err) + return + } + + for _, obj := range objs { + if obj.UpdatedAt.Before(pruneBefore) { + if err := dbx.TransactionContext(ctx, func(tx *db.Tx) error { + if err := datastore.DeleteLFSObjectByOid(ctx, dbx, repo.ID(), obj.Oid); err != nil { + return err + } + p := lfs.Pointer{Oid: obj.Oid} + return strg.Delete(filepath.Join("objects", p.RelativePath())) + }); err != nil { + logger.Error("error clear lfs objects", "err", err, "oid", obj.Oid) + fmt.Fprintf(stderr, "[%s] error delete lfs object %s: %v\n", repoName, obj.Oid, err) + } else { + logger.Info("removed unreachable lfs object", "repo", repoName, "oid", obj.Oid) + fmt.Fprintf(stdout, "[%s] removed lfs object %s\n", repoName, obj.Oid) + } + } + } + } + fmt.Fprintf(stdout, "[%s] git gc successful\n", repoName) + } +} + +// Func runs the garbage collector job task and implements Runner. +func (g gitGC) Func(ctx context.Context, cronCfg RunnerConfig) func() { + b := backend.FromContext(ctx) + logger := log.FromContext(ctx).WithPrefix("jobs.gitgc") + jobcfg := cronCfg.(*gitGCConfig) + + return func() { + repos, err := b.Repositories(ctx) + if err != nil { + logger.Error("error getting repositories", "err", err) + fmt.Fprintf(jobcfg.Error(), "error getting repositories: %v\n", err) + return + } + + wq := sync.NewWorkPool(ctx, runtime.GOMAXPROCS(0), + sync.WithWorkPoolLogger(logger.Errorf), + ) + + for _, repo := range repos { + name := repo.Name() + wq.Add(name, g.cleanup(ctx, jobcfg, repo)) + } + + wq.Run() + } +} + +// FlagSet returns the flag set that can modify configuration values and implements RunnerConfig +func (cfg *gitGCConfig) FlagSet() *pflag.FlagSet { + flags := pflag.NewFlagSet("git-gc", pflag.ContinueOnError) + flags.StringToStringVarP(&cfg.RepoConfig, "config", "c", cfg.RepoConfig, "Override values from git repository configuration files") + + flags.BoolVar(&cfg.Aggressive, "aggressive", cfg.Aggressive, "Optimize the repository more aggressively, see git-gc(1) for more details") + flags.DurationVar(&cfg.LFSPruneExpire, "lfsprune-expire", cfg.LFSPruneExpire, "Expire duration of unreachable LFS objects following the latest update") + + return flags +} diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go index 505863ea0..1ecd57217 100644 --- a/pkg/jobs/jobs.go +++ b/pkg/jobs/jobs.go @@ -2,7 +2,10 @@ package jobs import ( "context" + "io" "sync" + + "github.com/spf13/pflag" ) // Job is a job that can be registered with the scheduler. @@ -13,8 +16,56 @@ type Job struct { // Runner is a job runner. type Runner interface { - Spec(context.Context) string - Func(context.Context) func() + Description() string + Config(context.Context) (RunnerConfig, error) + + Func(context.Context, RunnerConfig) func() +} + +// JobConfig is the configuration for Runner, passed to Runner.Func +type RunnerConfig interface { + FlagSet() *pflag.FlagSet + Spec() string + + SetOut(out io.Writer) + Output() io.Writer + SetErr(err io.Writer) + Error() io.Writer +} + +// baseRunnerConfig implements the common part of job tasks' RunnerConfig +type baseRunnerConfig struct { + CronSpec string `yaml:"spec"` + + output io.Writer + error io.Writer +} + +// SetOut sets the stdout of cron job +func (cfg *baseRunnerConfig) SetOut(out io.Writer) { cfg.output = out } + +// Output return the stdout of cron job +func (cfg *baseRunnerConfig) Output() io.Writer { + if cfg.output == nil { + return io.Discard + } + return cfg.output +} + +// SetErr sets the stderr of cron job +func (cfg *baseRunnerConfig) SetErr(err io.Writer) { cfg.error = err } + +// Error return the stderr of cron job +func (cfg *baseRunnerConfig) Error() io.Writer { + if cfg.error == nil { + return io.Discard + } + return cfg.error +} + +// Spec derives the spec for built-in job scheduler and implements RunnerConfig. +func (cfg *baseRunnerConfig) Spec() string { + return cfg.CronSpec } var ( diff --git a/pkg/jobs/mirror.go b/pkg/jobs/mirror.go index 4642e72b1..43b968df9 100644 --- a/pkg/jobs/mirror.go +++ b/pkg/jobs/mirror.go @@ -1,6 +1,7 @@ package jobs import ( + "bytes" "context" "fmt" "path/filepath" @@ -15,30 +16,47 @@ import ( "github.com/charmbracelet/soft-serve/pkg/lfs" "github.com/charmbracelet/soft-serve/pkg/store" "github.com/charmbracelet/soft-serve/pkg/sync" + "github.com/spf13/pflag" ) func init() { Register("mirror-pull", mirrorPull{}) } -type mirrorPull struct{} +type ( + mirrorPull struct{} + mirrorPullConfig struct { + baseRunnerConfig + RepoConfig map[string]string + } +) -// Spec derives the spec used for pull mirrors and implements Runner. -func (m mirrorPull) Spec(ctx context.Context) string { - cfg := config.FromContext(ctx) - if cfg.Jobs.MirrorPull != "" { - return cfg.Jobs.MirrorPull +// Description return the description of (pull) mirror job task and implements Runner. +func (m mirrorPull) Description() string { + return "fetch upstream for mirror repositories" +} + +// Config returns the (pull) mirror cronjob configuration and implements Runner. +func (m mirrorPull) Config(ctx context.Context) (RunnerConfig, error) { + cfg := mirrorPullConfig{ + baseRunnerConfig: baseRunnerConfig{CronSpec: "@every 10m"}, + RepoConfig: make(map[string]string), + } + if spec := config.FromContext(ctx).Jobs.MirrorPull; spec != "" { + cfg.CronSpec = spec } - return "@every 10m" + + return &cfg, nil } // Func runs the (pull) mirror job task and implements Runner. -func (m mirrorPull) Func(ctx context.Context) func() { +func (m mirrorPull) Func(ctx context.Context, cronCfg RunnerConfig) func() { cfg := config.FromContext(ctx) logger := log.FromContext(ctx).WithPrefix("jobs.mirror") b := backend.FromContext(ctx) dbx := db.FromContext(ctx) datastore := store.FromContext(ctx) + jobcfg := cronCfg.(*mirrorPullConfig) return func() { repos, err := b.Repositories(ctx) if err != nil { @@ -64,13 +82,31 @@ func (m mirrorPull) Func(ctx context.Context) func() { wq.Add(name, func() { repo := repo + // buffer and write to stdout/stderr in one go, + // avoiding output confusion through parallel writing. + var ( + stdout = bytes.NewBuffer(nil) + stderr = bytes.NewBuffer(nil) + ) + defer func() { + jobcfg.Output().Write(stdout.Bytes()) + jobcfg.Error().Write(stderr.Bytes()) + }() + cmds := []string{ "fetch --prune", // fetch prune before updating remote "remote update --prune", // update remote and prune remote refs } + gitFlags := []string{} + for key, val := range jobcfg.RepoConfig { + gitFlags = append(gitFlags, "-c", key+"="+val) + } + for _, c := range cmds { args := strings.Split(c, " ") + args = append(gitFlags, args...) + cmd := git.NewCommand(args...).WithContext(ctx) cmd.AddEnvs( fmt.Sprintf(`GIT_SSH_COMMAND=ssh -o UserKnownHostsFile="%s" -o StrictHostKeyChecking=no -i "%s"`, @@ -80,6 +116,7 @@ func (m mirrorPull) Func(ctx context.Context) func() { ) if _, err := cmd.RunInDir(r.Path); err != nil { + fmt.Fprintf(stderr, "[%s]: error running git remote update: %v\n", name, err) logger.Error("error running git remote update", "repo", name, "err", err) } } @@ -88,6 +125,7 @@ func (m mirrorPull) Func(ctx context.Context) func() { rcfg, err := r.Config() if err != nil { logger.Error("error getting git config", "repo", name, "err", err) + fmt.Fprintf(stderr, "[%s]: lfs pull: error getting git config: %v", name, err) return } @@ -101,20 +139,26 @@ func (m mirrorPull) Func(ctx context.Context) func() { ep, err := lfs.NewEndpoint(lfsEndpoint) if err != nil { logger.Error("error creating LFS endpoint", "repo", name, "err", err) + fmt.Fprintf(stderr, "[%s]: lfs pull: creating LFS endpoint: %v", name, err) return } client := lfs.NewClient(ep) if client == nil { + fmt.Fprintf(stderr, + "[%s]: lfs pull: failed to create lfs client: unsupported endpoint %s", + name, lfsEndpoint) logger.Errorf("failed to create lfs client: unsupported endpoint %s", lfsEndpoint) return } if err := backend.StoreRepoMissingLFSObjects(ctx, repo, dbx, datastore, client); err != nil { + fmt.Fprintf(stderr, "[%s]: lfs pull: failed to store missing lfs objects: %v", name, err) logger.Error("failed to store missing lfs objects", "err", err, "path", r.Path) return } } + fmt.Fprintf(stdout, "[%s]: mirror pull succeed\n", name) }) } } @@ -122,3 +166,11 @@ func (m mirrorPull) Func(ctx context.Context) func() { wq.Run() } } + +// FlagSet returns the flag set that can modify configuration values and implements RunnerConfig +func (cfg *mirrorPullConfig) FlagSet() *pflag.FlagSet { + flags := pflag.NewFlagSet("mirror-pull", pflag.ContinueOnError) + flags.StringToStringVarP(&cfg.RepoConfig, "config", "c", cfg.RepoConfig, "Override values from git repository configuration files") + + return flags +} diff --git a/pkg/lfs/scanner.go b/pkg/lfs/scanner.go index 6688e4225..a5ada02af 100644 --- a/pkg/lfs/scanner.go +++ b/pkg/lfs/scanner.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "encoding/hex" "fmt" "io" "strconv" @@ -49,6 +50,30 @@ func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan c close(errChan) } +// CheckPointerExist scans lfs Pointers from pointerChan and outputs through the returned channel whether they exist +func CheckPointerExist(ctx context.Context, repo *git.Repository, pointerChan <-chan Pointer) (<-chan bool, func()) { + ctx, cancel := context.WithCancel(ctx) + resultChan := make(chan bool) + wg := sync.WaitGroup{} + wg.Add(3) + + stop := func() { + cancel() + wg.Wait() + } + + shasToCheckReader, shasToCheckWriter := io.Pipe() + catFileCheckReader, catFileCheckWriter := io.Pipe() + + go catFileBatchLineExist(catFileCheckReader, resultChan, &wg) + + go catFileBatchCheck(ctx, shasToCheckReader, catFileCheckWriter, &wg, repo.Path) + + go pointerBlobHash(repo, pointerChan, shasToCheckWriter, &wg) + + return resultChan, stop +} + func createPointerResultsFromCatFileBatch(ctx context.Context, catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- PointerBlob) { defer wg.Done() defer catFileBatchReader.Close() //nolint: errcheck @@ -214,3 +239,54 @@ func revListAllObjects(ctx context.Context, revListWriter *io.PipeWriter, wg *sy errChan <- fmt.Errorf("git rev-list [%s]: %w - %s", basePath, err, errbuf.String()) } } + +// pointerBlobHash calculates the hash value of each LFS pointer +func pointerBlobHash(r *git.Repository, pointerChan <-chan Pointer, shastoCheckWriter *io.PipeWriter, wg *sync.WaitGroup) { + defer wg.Done() + defer shastoCheckWriter.Close() + + for pointer := range pointerChan { + pointerSha := hex.EncodeToString(r.ComputeObjectHash(gitm.ObjectBlob, []byte(pointer.String()))) + shastoCheckWriter.Write([]byte(pointerSha + "\n")) + } +} + +// catFileBatchLineExist reads the git cat-file batch check results line by line from the reader, +// and outputs the existence status through resultChan. +func catFileBatchLineExist(catFileCheckReader *io.PipeReader, resultChan chan<- bool, wg *sync.WaitGroup) { + defer wg.Done() + defer catFileCheckReader.Close() + scanner := bufio.NewScanner(catFileCheckReader) + defer func() { + close(resultChan) + }() + + for scanner.Scan() { + typ := scanner.Text() + if len(typ) == 0 { + continue + } + // typ is: + // SP SP LF + // or SP missing|excluded|ambiguous|submodule LF + + idx := strings.IndexByte(typ, ' ') + if idx < 0 { + resultChan <- false + continue + } + typ = typ[idx+1:] // remove + + idx = strings.IndexByte(typ, ' ') + if idx < 0 { + resultChan <- false + continue + } + + sizeStr := typ[idx+1 : len(typ)-1] // remove + typ = typ[:idx] + + _, err := strconv.ParseInt(sizeStr, 10, 64) // # validate size + resultChan <- (err == nil) + } +} diff --git a/pkg/ssh/cmd/cronjob.go b/pkg/ssh/cmd/cronjob.go new file mode 100644 index 000000000..76e596d89 --- /dev/null +++ b/pkg/ssh/cmd/cronjob.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "context" + + "github.com/charmbracelet/soft-serve/pkg/jobs" + "github.com/spf13/cobra" +) + +// CronJobCommand returns a command for manually triggering cronjobs +func CronJobCommand(ctx context.Context) (*cobra.Command, error) { + cmd := &cobra.Command{ + Use: "cronjob", + Short: "Run cron job task", + PersistentPreRunE: checkIfAdmin, + } + + for name, j := range jobs.List() { + cfg, err := j.Runner.Config(ctx) + if err != nil { + return nil, err + } + + cronCmd := &cobra.Command{ + Use: name, + Short: j.Runner.Description(), + Run: func(cmd *cobra.Command, args []string) { + cfg.SetOut(cmd.OutOrStdout()) + cfg.SetErr(cmd.OutOrStderr()) + + j.Runner.Func(cmd.Context(), cfg)() + }, + } + cronCmd.Flags().AddFlagSet(cfg.FlagSet()) + + cmd.AddCommand(cronCmd) + } + + return cmd, nil +} diff --git a/pkg/ssh/middleware.go b/pkg/ssh/middleware.go index 25a00b1a2..34b652022 100644 --- a/pkg/ssh/middleware.go +++ b/pkg/ssh/middleware.go @@ -145,6 +145,14 @@ func CommandMiddleware(sh ssh.Handler) ssh.Handler { } } + jobCmd, err := cmd.CronJobCommand(ctx) + if err != nil { + fmt.Fprintf(s.Stderr(), "error create cronjob sub command: %v\n", err) + s.Exit(1) + return + } + rootCmd.AddCommand(jobCmd) + rootCmd.SetArgs(args) if len(args) == 0 { // otherwise it'll default to os.Args, which is not what we want. diff --git a/testscript/script_test.go b/testscript/script_test.go index 550a3b3e0..2904a6b17 100644 --- a/testscript/script_test.go +++ b/testscript/script_test.go @@ -313,6 +313,8 @@ Host * ServerAliveInterval 60 ` +const GIT_DATE = "2026-03-29T00:00:00Z" + func cmdGit(key string) func(ts *testscript.TestScript, neg bool, args []string) { return func(ts *testscript.TestScript, neg bool, args []string) { ts.Check(os.WriteFile( @@ -330,6 +332,11 @@ func cmdGit(key string) func(ts *testscript.TestScript, neg bool, args []string) ) // Disable git prompting for credentials. ts.Setenv("GIT_TERMINAL_PROMPT", "0") + + // Keep the same git commit hash in an empty repository + ts.Setenv("GIT_AUTHOR_DATE", GIT_DATE) + ts.Setenv("GIT_COMMITTER_DATE", GIT_DATE) + args = append([]string{ "-c", "user.email=john@example.com", "-c", "user.name=John Doe", diff --git a/testscript/testdata/cronjob-git-gc.txtar b/testscript/testdata/cronjob-git-gc.txtar new file mode 100644 index 000000000..cc2839584 --- /dev/null +++ b/testscript/testdata/cronjob-git-gc.txtar @@ -0,0 +1,98 @@ +# vi: set ft=conf + +# convert crlf to lf on windows +[windows] dos2unix first_commit.txt + +# start soft serve +exec soft serve & +# wait for SSH server to start +ensureserverrunning SSH_PORT + +# create a repo +soft repo create repo1 + +# clone repo +git clone ssh://localhost:$SSH_PORT/repo1 repo1 + +# create some files, commits, lfs objects +mkfile ./repo1/README.md '# Hello in first commit' +mkfile ./repo1/foo.png 'foo' +mkfile ./repo1/bar.png 'bar' +git -C repo1 lfs install --local +git -C repo1 lfs track '*.png' +git -C repo1 add -A +git -C repo1 commit -m 'first commit' +git -C repo1 push origin HEAD +soft repo commit repo1 7088ebc +cmp stdout first_commit.txt + +# create an empty repo1/foo.png file, overwrite the first commit +mkfile ./repo1/README.md '# Hello in second commit' +mkfile ./repo1/foo.png '' +git -C repo1 add -A +git -C repo1 commit --amend -m 'second commit' +git -C repo1 push --force origin HEAD +# the first commit still exists +soft repo commit repo1 7088ebc +cmp stdout first_commit.txt + +# clean up +soft cronjob git-gc --config gc.reflogExpire=now --config gc.pruneExpire=now --lfsprune-expire 0 +# original repo1/foo.png cleaned +stdout 'removed lfs object 2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae' +# repo1/bar.png not cleaned +! stdout 'removed lfs object fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9' +stdout 'git gc successful' +! soft repo commit repo1 7088ebc +stderr 'Error: revision does not exist' + +# stop the server +[windows] stopserver +[windows] ! stderr . + +-- first_commit.txt -- +commit 7088ebc +Author: John Doe +Date: Sun Mar 29 00:00:00 UTC 2026 +first commit + + +.gitattributes | 1 + +README.md | 1 + +bar.png | 3 +++ +foo.png | 3 +++ +4 files changed, 8 insertions(+) + +diff --git a/.gitattributes b/.gitattributes +new file mode 100644 +index 0000000000000000000000000000000000000000..24a8e87939aa53cdd833f6be7610cb4972e063ad +--- /dev/null ++++ b/.gitattributes +@@ -0,0 +1 @@ ++*.png filter=lfs diff=lfs merge=lfs -text +diff --git a/README.md b/README.md +new file mode 100644 +index 0000000000000000000000000000000000000000..e01a0969efececc7f96e77f41cf754aab78f7669 +--- /dev/null ++++ b/README.md +@@ -0,0 +1 @@ ++# Hello in first commit +diff --git a/bar.png b/bar.png +new file mode 100644 +index 0000000000000000000000000000000000000000..a226a70db3e220dec6656979d64fd0f20f1fec1d +--- /dev/null ++++ b/bar.png +@@ -0,0 +1,3 @@ ++version https://git-lfs.github.com/spec/v1 ++oid sha256:fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9 ++size 3 +diff --git a/foo.png b/foo.png +new file mode 100644 +index 0000000000000000000000000000000000000000..90c4f8e4241b291f2c005db06e8840d83945dbcd +--- /dev/null ++++ b/foo.png +@@ -0,0 +1,3 @@ ++version https://git-lfs.github.com/spec/v1 ++oid sha256:2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae ++size 3 + diff --git a/testscript/testdata/help.txtar b/testscript/testdata/help.txtar index 299e29500..510999b79 100644 --- a/testscript/testdata/help.txtar +++ b/testscript/testdata/help.txtar @@ -20,6 +20,7 @@ Usage: ssh -p $SSH_PORT localhost [command] Available Commands: + cronjob Run cron job task help Help about any command info Show your info jwt Generate a JSON Web Token