fix: fail startup when a background process exits

docs/sdk.md

@@ -114,7 +114,9 @@ Typical sequences:
 
 - **blocking/once step:** `Running → Exited` (or `Failed`)
 - **primary:** `Running`, then on each reload `Killed → Running`, and `Killed` at shutdown
-- **background:** `Running` once, `Killed` at shutdown
+- **background:** `Running` once, `Killed` at shutdown. If it exits while the
+  initial process sequence is still running, startup fails and the active
+  `once`/`blocking` step is cancelled.
 
 > `OnProcessEvent` is called **synchronously from the engine's goroutine — do not
 > block in it.** If you need to do real work, hand the event to your own channel

process/process.go

@@ -3,6 +3,7 @@ package process
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"log/slog"
 	"os"
@@ -204,17 +205,25 @@ func (pm *ProcessManager) Start(ctx context.Context) error {
 	if len(pm.Processes) == 0 {
 		return errors.New("no processes configured")
 	}
-	return pm.runCycle(ctx, true)
+
+	startupFailures := make(chan error, 1)
+	return pm.runCycle(ctx, true, startupFailures)
 }
 
 // Reload re-runs blocking steps and restarts the primary process. Background and
 // once processes started during Start are left running.
 func (pm *ProcessManager) Reload(ctx context.Context) error {
-	return pm.runCycle(ctx, false)
+	return pm.runCycle(ctx, false, nil)
 }
 
-func (pm *ProcessManager) runCycle(ctx context.Context, firstRun bool) error {
+func (pm *ProcessManager) runCycle(ctx context.Context, firstRun bool, startupFailures chan error) error {
 	for _, p := range pm.Processes {
+		select {
+		case err := <-startupFailures:
+			return err
+		default:
+		}
+
 		// Markers used by the ExecList config form; no-ops in the struct form.
 		if p.Exec == KILL_EXEC || p.Exec == REFRESH_EXEC {
 			continue
@@ -224,20 +233,27 @@ func (pm *ProcessManager) runCycle(ctx context.Context, firstRun bool) error {
 			if !firstRun {
 				continue
 			}
-			if err := pm.startAsync(ctx, p); err != nil {
+
+			onExit := func(err error) {
+				select {
+				case startupFailures <- err:
+				default:
+				}
+			}
+			if err := pm.startAsync(ctx, p, onExit); err != nil {
 				slog.Error("starting background process", "exec", p.Exec, "err", err)
 				return err
 			}
 		case Once:
 			if !firstRun {
 				continue
 			}
-			if err := pm.runBlocking(ctx, p); err != nil {
+			if err := pm.runBlocking(ctx, p, startupFailures); err != nil {
 				slog.Error("once process failed", "exec", p.Exec, "err", err)
 				return err
 			}
 		case Blocking:
-			if err := pm.runBlocking(ctx, p); err != nil {
+			if err := pm.runBlocking(ctx, p, startupFailures); err != nil {
 				// On reload a failed blocking step (typically a build error)
 				// aborts the cycle and leaves the current primary running, so a
 				// broken build doesn't take down the last good process.
@@ -246,7 +262,7 @@ func (pm *ProcessManager) runCycle(ctx context.Context, firstRun bool) error {
 			}
 		case Primary:
 			pm.stopProcess(p) // kill the previous instance (no-op on first run)
-			if err := pm.startAsync(ctx, p); err != nil {
+			if err := pm.startAsync(ctx, p, nil); err != nil {
 				slog.Error("starting primary process", "exec", p.Exec, "err", err)
 				return err
 			}
@@ -283,7 +299,7 @@ func (pm *ProcessManager) delayNext(ctx context.Context, p *Process) bool {
 // process group and tracks it so it can be terminated on the next cycle or
 // shutdown. The command is started in a fresh process group so the whole tree
 // can be signalled, not just the direct child.
-func (pm *ProcessManager) startAsync(ctx context.Context, p *Process) error {
+func (pm *ProcessManager) startAsync(ctx context.Context, p *Process, onUnexpectedExit func(error)) error {
 	procCtx, cancel := context.WithCancel(ctx)
 	cmd := generateExec(p.Exec)
 	cmd.Dir = pm.resolveDir(p.Dir)
@@ -322,6 +338,13 @@ func (pm *ProcessManager) startAsync(ctx context.Context, p *Process) error {
 			} else {
 				pm.transition(p, StateExited, 0, 0, nil)
 			}
+			if onUnexpectedExit != nil {
+				if err != nil {
+					onUnexpectedExit(fmt.Errorf("background process %q exited during startup: %w", p.Exec, err))
+				} else {
+					onUnexpectedExit(fmt.Errorf("background process %q exited during startup", p.Exec))
+				}
+			}
 		}
 	}()
 	return nil
@@ -331,7 +354,7 @@ func (pm *ProcessManager) startAsync(ctx context.Context, p *Process) error {
 // group and is bound to ctx, so a shutdown while it is running force-kills the
 // whole tree (not just the direct child, which is all CommandContext would
 // reach) and unblocks the wait.
-func (pm *ProcessManager) runBlocking(ctx context.Context, p *Process) error {
+func (pm *ProcessManager) runBlocking(ctx context.Context, p *Process, startupFailures <-chan error) error {
 	cmd := generateExec(p.Exec)
 	cmd.Dir = pm.resolveDir(p.Dir)
 	cmd.Stdout = pm.stdio(p, "stdout", os.Stdout)
@@ -350,12 +373,9 @@ func (pm *ProcessManager) runBlocking(ctx context.Context, p *Process) error {
 	var err error
 	select {
 	case <-ctx.Done():
-		if kerr := killProcessTree(cmd); kerr != nil {
-			slog.Debug("killing blocking process tree", "exec", p.Exec, "err", kerr)
-		}
-		<-waitErr // reap after the kill
-		pm.transition(p, StateKilled, 0, noExitYet, nil)
-		return ctx.Err()
+		return pm.stopBlocking(p, cmd, waitErr, ctx.Err())
+	case startupErr := <-startupFailures:
+		return pm.stopBlocking(p, cmd, waitErr, startupErr)
 	case err = <-waitErr:
 	}
 	if err != nil {
@@ -366,6 +386,16 @@ func (pm *ProcessManager) runBlocking(ctx context.Context, p *Process) error {
 	return nil
 }
 
+func (pm *ProcessManager) stopBlocking(p *Process, cmd *exec.Cmd, waitErr <-chan error, cause error) error {
+	if err := killProcessTree(cmd); err != nil {
+		slog.Debug("killing blocking process tree", "exec", p.Exec, "err", err)
+	}
+	<-waitErr
+	pm.transition(p, StateKilled, 0, noExitYet, nil)
+
+	return cause
+}
+
 // exitCodeOf extracts the process exit code from a completed command, falling
 // back to -1 when the failure wasn't a normal non-zero exit (e.g. a signal).
 func exitCodeOf(cmd *exec.Cmd, err error) int {

process/process_test.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"testing"
 	"time"
@@ -222,3 +223,71 @@ func TestBlockingFailureAbortsCycle(t *testing.T) {
 	}
 	pm.Shutdown()
 }
+
+func TestBackgroundFailureCancelsBlockingStartupStep(t *testing.T) {
+	root := t.TempDir()
+	pm := NewProcessManager()
+	if err := pm.SetRootDirectory(root); err != nil {
+		t.Fatal(err)
+	}
+	backgroundCmd := "while [ ! -f readiness-started ]; do sleep 0.01; done; exit 23"
+	if err := pm.AddProcessSpec(Execute{Name: "frontend", Cmd: backgroundCmd, Type: Background}); err != nil {
+		t.Fatal(err)
+	}
+	if err := pm.AddProcessSpec(Execute{Name: "readiness", Cmd: "touch readiness-started; sleep 30", Type: Once}); err != nil {
+		t.Fatal(err)
+	}
+	if err := pm.AddProcessSpec(Execute{Name: "application", Cmd: "sleep 30", Type: Primary}); err != nil {
+		t.Fatal(err)
+	}
+	defer pm.Shutdown()
+
+	started := time.Now()
+	err := pm.Start(context.Background())
+	if err == nil {
+		t.Fatal("expected Start to fail when the background process exits")
+	}
+	want := `background process "` + backgroundCmd + `" exited during startup`
+	if !strings.Contains(err.Error(), want) {
+		t.Fatalf("Start error = %q, want background startup failure", err)
+	}
+	if elapsed := time.Since(started); elapsed > 5*time.Second {
+		t.Fatalf("Start took %s; readiness step was not cancelled promptly", elapsed)
+	}
+	if primary := pm.Processes[2]; primary.cmd != nil {
+		t.Error("primary should not have started after the background failure")
+	}
+}
+
+func TestBackgroundFailureAfterStartupDoesNotStopPrimary(t *testing.T) {
+	root := t.TempDir()
+	pm := NewProcessManager()
+	if err := pm.SetRootDirectory(root); err != nil {
+		t.Fatal(err)
+	}
+	if err := pm.AddProcessSpec(Execute{Name: "frontend", Cmd: "while [ ! -f stop-background ]; do sleep 0.01; done; exit 23", Type: Background}); err != nil {
+		t.Fatal(err)
+	}
+	if err := pm.AddProcessSpec(Execute{Name: "application", Cmd: "sleep 30", Type: Primary}); err != nil {
+		t.Fatal(err)
+	}
+
+	defer pm.Shutdown()
+	if err := pm.Start(context.Background()); err != nil {
+		t.Fatalf("Start: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "stop-background"), nil, 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	if !waitFor(func() bool {
+		return pm.Snapshot()[0].State == StateFailed
+	}) {
+		t.Fatal("background process did not exit")
+	}
+
+	primary := pm.Processes[1]
+	if primary.cmd == nil || !alive(primary.cmd.Process.Pid) {
+		t.Error("primary stopped after a post-startup background failure")
+	}
+}