diff --git a/cmd/y-cluster/echo.go b/cmd/y-cluster/echo.go new file mode 100644 index 0000000..f0e644d --- /dev/null +++ b/cmd/y-cluster/echo.go @@ -0,0 +1,89 @@ +package main + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/Yolean/y-cluster/pkg/cluster" + "github.com/Yolean/y-cluster/pkg/echo" +) + +// echoCmd groups the appliance-test workload subcommands. Today +// only `deploy` is implemented; `undeploy` and `probe` are +// candidates if/when call sites grow. The deploy logic itself +// lives in pkg/echo so other Go consumers (e2e tests, future +// appliance build/verify subcommands) can call it directly. +func echoCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "echo", + Short: "Manage the appliance-test echo workload (Gateway + Deployment + Service + HTTPRoute)", + } + cmd.AddCommand(echoDeployCmd()) + cmd.AddCommand(echoRenderCmd()) + return cmd +} + +// echoRenderCmd prints the rendered manifest to stdout without +// touching a cluster. The Packer-based Hetzner appliance build +// uses this to drop the workload into k3s's auto-apply manifests +// dir at snapshot time, so first cold boot bootstraps clean +// rather than carrying a stale node registration in the +// datastore. +func echoRenderCmd() *cobra.Command { + var ( + namespace string + gatewayClass string + ) + cmd := &cobra.Command{ + Use: "render", + Short: "Print the rendered echo manifest YAML to stdout (no cluster contact)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + body, err := echo.Render(echo.Options{ + Namespace: namespace, + GatewayClass: gatewayClass, + }) + if err != nil { + return err + } + _, err = cmd.OutOrStdout().Write(body) + return err + }, + } + cmd.Flags().StringVarP(&namespace, "namespace", "n", echo.DefaultNamespace, "namespace to render into") + cmd.Flags().StringVar(&gatewayClass, "gateway-class", echo.DefaultGatewayClass, + "GatewayClass the rendered Gateway resource references") + return cmd +} + +func echoDeployCmd() *cobra.Command { + var ( + contextName string + namespace string + gatewayClass string + ) + cmd := &cobra.Command{ + Use: "deploy", + Short: "Apply the standard echo workload (Namespace, Gateway, Deployment, Service, HTTPRoute)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if err := echo.Deploy(cmd.Context(), echo.Options{ + ContextName: contextName, + Namespace: namespace, + GatewayClass: gatewayClass, + }); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), + "echo deployed to namespace %q on context %q\nprobe: curl http://%s\n", + namespace, contextName, echo.PathPrefix) + return nil + }, + } + cmd.Flags().StringVar(&contextName, "context", cluster.DefaultContext, "kubeconfig context name") + cmd.Flags().StringVarP(&namespace, "namespace", "n", echo.DefaultNamespace, "namespace to deploy into") + cmd.Flags().StringVar(&gatewayClass, "gateway-class", echo.DefaultGatewayClass, + "GatewayClass the created Gateway resource references; matches the y-cluster default") + return cmd +} diff --git a/cmd/y-cluster/lifecycle.go b/cmd/y-cluster/lifecycle.go new file mode 100644 index 0000000..736aa40 --- /dev/null +++ b/cmd/y-cluster/lifecycle.go @@ -0,0 +1,264 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + "go.uber.org/zap" + + "github.com/Yolean/y-cluster/pkg/cluster" + "github.com/Yolean/y-cluster/pkg/provision/docker" + "github.com/Yolean/y-cluster/pkg/provision/multipass" + "github.com/Yolean/y-cluster/pkg/provision/qemu" +) + +// pauseCmd, resumeCmd, stopCmd, startCmd are the cluster lifecycle +// subcommands -- a provisioner-neutral surface that today only +// implements the qemu backend. docker and multipass return a "not +// yet implemented for " error so the surface is stable +// while the per-backend semantics are still being worked out. +// +// All four resolve the cluster via the kubeconfig context (the +// same convention detect / ctr / crictl use) -- no -c +// required. Pause / Resume / Stop find the running cluster via +// cluster.Lookup; Start can't (the cluster is stopped) and +// instead reads the cluster name from kubeconfig and rehydrates +// the qemu launch parameters from the sidecar Provision wrote. + +func pauseCmd() *cobra.Command { + return signalCmd("pause", "Pause the cluster VM (SIGSTOP); resume to unfreeze", qemu.Pause) +} + +func resumeCmd() *cobra.Command { + return signalCmd("resume", "Resume a paused cluster VM (SIGCONT)", qemu.Resume) +} + +// stopCmd dispatches per-backend rather than going through +// signalCmd: qemu.Stop's signature takes (cacheDir, name); docker +// and multipass take (ctx, name); a uniform signalCmd helper +// would need a wrapper per backend anyway, so the explicit switch +// is clearer. +func stopCmd() *cobra.Command { + var contextName string + cmd := &cobra.Command{ + Use: "stop", + Short: "Gracefully shut down the cluster; disk preserved for `y-cluster start`", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + ctx := cmd.Context() + logger := loggerFromContext(ctx) + lr, err := cluster.Lookup(ctx, "", contextName) + if err != nil { + return err + } + switch lr.Backend { + case cluster.BackendQEMU: + return qemu.Stop(qemuCacheDir(), lr.ClusterName, logger) + case cluster.BackendDocker: + return docker.Stop(ctx, lr.ClusterName, logger) + case cluster.BackendMultipass: + return multipass.Stop(ctx, lr.ClusterName, logger) + default: + return fmt.Errorf("stop: not yet implemented for %s", lr.Backend) + } + }, + } + cmd.Flags().StringVar(&contextName, "context", cluster.DefaultContext, "kubeconfig context name") + return cmd +} + +// signalCmd is the shared shape for pause / resume / stop. Each +// looks up the running cluster, dispatches by backend, and calls +// the qemu lifecycle function. Non-qemu backends return a "not +// implemented" error. +func signalCmd(name, short string, run func(cacheDir, name string, logger *zap.Logger) error) *cobra.Command { + var contextName string + cmd := &cobra.Command{ + Use: name, + Short: short, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + logger := loggerFromContext(cmd.Context()) + lr, err := cluster.Lookup(cmd.Context(), "", contextName) + if err != nil { + return err + } + switch lr.Backend { + case cluster.BackendQEMU: + return run(qemuCacheDir(), lr.ClusterName, logger) + default: + return fmt.Errorf("%s: not yet implemented for %s", name, lr.Backend) + } + }, + } + cmd.Flags().StringVar(&contextName, "context", cluster.DefaultContext, "kubeconfig context name") + return cmd +} + +// prepareExportCmd runs libguestfs's virt-sysprep against the +// stopped appliance disk so it boots cleanly on a different +// hypervisor. qemu-only today; other backends would need their own +// disk-export hook so we don't stub them with a misleading "not +// implemented" -- the surface stays narrow on purpose. +// +// Cluster must be stopped first; virt-sysprep operates on the +// offline qcow2 (libguestfs mounts it directly, no boot involved). +// We resolve the cluster name from the kubeconfig context the same +// way startCmd does, since cluster.Lookup needs a running cluster +// to probe. +func prepareExportCmd() *cobra.Command { + var contextName string + cmd := &cobra.Command{ + Use: "prepare-export", + Short: "Prepare a stopped qemu appliance for export to a different hypervisor", + Long: `Clears host-specific state baked into the appliance disk so the +same disk boots cleanly when imported elsewhere (VMware, KVM, +cloud providers, Hetzner). Wipes machine-id, SSH host keys, +udev persistent net rules, MAC-bound netplan, and the cloud-init +state cache. Registers a firstboot ssh-keygen so the imported +instance regenerates its own host keys. + +Cluster must be stopped first (virt-sysprep needs an offline +disk): + + y-cluster stop + y-cluster prepare-export + +Idempotent. A prepared appliance is no longer a usable dev +cluster locally: the next start runs cloud-init re-init and +regenerates identity. Re-provision for a fresh dev loop. + +Requires libguestfs-tools (sudo apt install libguestfs-tools).`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + logger := loggerFromContext(cmd.Context()) + clusterName, err := cluster.ResolveClusterName("", contextName) + if err != nil { + return err + } + if clusterName == "" { + return fmt.Errorf("kubeconfig context %q has no associated cluster; nothing to prepare", contextName) + } + return qemu.PrepareExport(cmd.Context(), qemuCacheDir(), clusterName, logger) + }, + } + cmd.Flags().StringVar(&contextName, "context", cluster.DefaultContext, "kubeconfig context name") + return cmd +} + +// exportCmd writes a customer-handoff bundle. The cluster must +// be stopped (and ideally prepare-export'd first) so the disk is +// quiesced and portable. Bundle layout + boot instructions live +// in pkg/provision/qemu.Export. +// +// --format is required (qcow2 or raw) until we know which one +// becomes the canonical handoff. Customers on different +// hypervisors will prefer different things; better to pick +// explicitly than have the default surprise someone. +func exportCmd() *cobra.Command { + var ( + contextName string + format string + vmdkSubformat string + ) + cmd := &cobra.Command{ + Use: "export ", + Short: "Write a customer-installable appliance bundle to ", + Long: `Writes a self-contained bundle directory containing the +appliance disk (flattened, no backing-file dependency on the +build host), the SSH keypair, and a README with boot + ssh +instructions for the customer. + +Run AFTER y-cluster stop (and ideally after y-cluster +prepare-export). Refuses to overwrite a non-empty bundle +directory; remove it first if you really mean to re-export. + +Per-customer keypair: each provision generates a fresh keypair +which is bundled here. Re-running provision (after teardown) +produces a different keypair, so re-exporting an old cluster +versus a new provision yields a distinct customer handoff. + +When --format=vmdk, --vmdk-subformat picks the qemu-img VMDK +shape. Default is streamOptimized (ESXi-friendly); pass +monolithicSparse for VirtualBox.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + logger := loggerFromContext(cmd.Context()) + clusterName, err := cluster.ResolveClusterName("", contextName) + if err != nil { + return err + } + if clusterName == "" { + return fmt.Errorf("kubeconfig context %q has no associated cluster; nothing to export", contextName) + } + if vmdkSubformat != "" && format != string(qemu.FormatVMDK) { + return fmt.Errorf("--vmdk-subformat is only valid with --format=vmdk") + } + return qemu.Export(cmd.Context(), qemu.ExportOptions{ + CacheDir: qemuCacheDir(), + Name: clusterName, + BundleDir: args[0], + Format: qemu.Format(format), + VMDKSubformat: vmdkSubformat, + Logger: logger, + }) + }, + } + cmd.Flags().StringVar(&contextName, "context", cluster.DefaultContext, "kubeconfig context name") + cmd.Flags().StringVar(&format, "format", "", + fmt.Sprintf("disk format (one of: %v); required until a default is chosen", qemu.AllFormats())) + cmd.Flags().StringVar(&vmdkSubformat, "vmdk-subformat", "", + fmt.Sprintf("VMDK subformat (one of: %v); only valid with --format=vmdk; default %q", qemu.AllVMDKSubformats(), qemu.VMDKSubformatDefault)) + if err := cmd.MarkFlagRequired("format"); err != nil { + panic(err) + } + return cmd +} + +// startCmd reverses stopCmd. The cluster is stopped, so +// cluster.Lookup can't find it; we resolve the cluster name +// straight off the kubeconfig context and ask qemu.Start to +// rehydrate from the saved sidecar in the default cache dir. +func startCmd() *cobra.Command { + var contextName string + cmd := &cobra.Command{ + Use: "start", + Short: "Start a previously-stopped cluster VM (qemu only)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + logger := loggerFromContext(cmd.Context()) + clusterName, err := cluster.ResolveClusterName("", contextName) + if err != nil { + return err + } + if clusterName == "" { + return fmt.Errorf("kubeconfig context %q has no associated cluster; nothing to start", contextName) + } + c, err := qemu.Start(cmd.Context(), qemuCacheDir(), clusterName, logger) + if err != nil { + return err + } + logger.Info("cluster started", + zap.String("context", c.Context()), + ) + return nil + }, + } + cmd.Flags().StringVar(&contextName, "context", cluster.DefaultContext, "kubeconfig context name") + return cmd +} + +// qemuCacheDir returns the same cache dir the qemu provisioner +// uses by default: $Y_CLUSTER_QEMU_CACHE_DIR when set, else +// ~/.cache/y-cluster-qemu. Centralised so the lifecycle +// subcommands and pkg/cluster's qemuRunning probe agree on where +// to look. +func qemuCacheDir() string { + if env := os.Getenv("Y_CLUSTER_QEMU_CACHE_DIR"); env != "" { + return env + } + home, _ := os.UserHomeDir() + return filepath.Join(home, ".cache", "y-cluster-qemu") +} diff --git a/cmd/y-cluster/localstorage.go b/cmd/y-cluster/localstorage.go new file mode 100644 index 0000000..d383cda --- /dev/null +++ b/cmd/y-cluster/localstorage.go @@ -0,0 +1,77 @@ +package main + +import ( + "github.com/spf13/cobra" + + "github.com/Yolean/y-cluster/pkg/provision/config" + "github.com/Yolean/y-cluster/pkg/provision/localstorage" +) + +// localstorageCmd groups the bundled-local-path-provisioner +// subcommands. Today only `render` is implemented; `apply` +// could land later if a build tool wants to skip the y-cluster +// provision flow but still install the same StorageClass. +// +// The flag defaults match CommonConfig.Storage so a render with +// no flags produces the exact manifest the Go-side provisioners +// install (path /data/yolean, predictable PVC namespace_name +// pattern, Retain reclaim). +func localstorageCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "localstorage", + Short: "Manage the bundled local-path-provisioner install", + } + cmd.AddCommand(localstorageRenderCmd()) + return cmd +} + +// localstorageRenderCmd prints the rendered manifest to stdout +// with the configured Path / Pattern / ReclaimPolicy. Used by +// the Hetzner Packer driver (scripts/e2e-appliance-hetzner.sh) +// to render once on the build host and ship a single yaml file +// to the build VM, avoiding kustomize / template machinery on +// the build VM side. +func localstorageRenderCmd() *cobra.Command { + defaults := defaultStorage() + var ( + path string + pattern string + reclaim string + ) + cmd := &cobra.Command{ + Use: "render", + Short: "Print the rendered local-path-provisioner manifest YAML to stdout (no cluster contact)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + body, err := localstorage.Render(localstorage.Options{ + Path: path, + Pattern: pattern, + ReclaimPolicy: reclaim, + }) + if err != nil { + return err + } + _, err = cmd.OutOrStdout().Write(body) + return err + }, + } + cmd.Flags().StringVar(&path, "path", defaults.Path, + "nodePathMap default path; where one subdirectory per PV lands on the node") + cmd.Flags().StringVar(&pattern, "pattern", defaults.PathPattern, + "per-PV directory pattern (Go text/template against local-path-provisioner helper-pod vars)") + cmd.Flags().StringVar(&reclaim, "reclaim-policy", defaults.ReclaimPolicy, + "StorageClass reclaim policy (Retain or Delete)") + return cmd +} + +// defaultStorage returns the same StorageConfig +// applyCommonDefaults installs on a freshly loaded +// y-cluster-provision.yaml. Wired off the QEMU defaulter (any +// provider would yield the same Storage block; QEMU is just +// the most-used one) so the CLI flag defaults track the +// runtime defaults -- one source of truth. +func defaultStorage() config.StorageConfig { + c := &config.QEMUConfig{CommonConfig: config.CommonConfig{Provider: config.ProviderQEMU}} + c.ApplyDefaults() + return c.Storage +} diff --git a/cmd/y-cluster/main.go b/cmd/y-cluster/main.go index 2e24aad..8409072 100644 --- a/cmd/y-cluster/main.go +++ b/cmd/y-cluster/main.go @@ -19,7 +19,6 @@ import ( "github.com/Yolean/y-cluster/pkg/provision/docker" "github.com/Yolean/y-cluster/pkg/provision/multipass" "github.com/Yolean/y-cluster/pkg/provision/qemu" - "github.com/Yolean/y-cluster/pkg/serve" "github.com/Yolean/y-cluster/pkg/yconverge" ) @@ -87,13 +86,6 @@ func versionString() string { } func main() { - // Surface the binary's version to pkg/serve so the daemon - // startup log reports it. The daemon process is a re-exec - // of THIS binary, so main() runs again on the daemon side - // and this assignment is what makes `serve logs` answer - // "which y-cluster build is running". - serve.DaemonVersion = versionString() - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer cancel() @@ -132,14 +124,22 @@ func rootCmd() *cobra.Command { root.AddCommand(yconvergeCmd()) root.AddCommand(provisionCmd()) root.AddCommand(teardownCmd()) + root.AddCommand(pauseCmd()) + root.AddCommand(resumeCmd()) + root.AddCommand(stopCmd()) + root.AddCommand(startCmd()) + root.AddCommand(prepareExportCmd()) root.AddCommand(exportCmd()) root.AddCommand(importCmd()) root.AddCommand(serveCmd()) root.AddCommand(imagesCmd()) + root.AddCommand(manifestsCmd()) root.AddCommand(detectCmd()) root.AddCommand(ctrCmd()) root.AddCommand(crictlCmd()) root.AddCommand(cacheCmd()) + root.AddCommand(echoCmd()) + root.AddCommand(localstorageCmd()) return root } @@ -456,31 +456,6 @@ func (k *dockerNamedTeardown) run() error { return dockerexec.Remove(context.Background(), cli, k.name) } -func exportCmd() *cobra.Command { - var configDir string - cmd := &cobra.Command{ - Use: "export ", - Short: "Export the cluster disk as a VMware appliance", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - loaded, err := loadProvision(configDir) - if err != nil { - return err - } - q, err := asQEMU(loaded) - if err != nil { - return err - } - rc := qemu.FromConfig(q) - return qemu.ExportVMDK(filepath.Join(rc.CacheDir, rc.Name+".qcow2"), args[0]) - }, - } - cmd.Flags().StringVarP(&configDir, "config", "c", "", "directory containing y-cluster-provision.yaml") - if err := cmd.MarkFlagRequired("config"); err != nil { - panic(err) - } - return cmd -} func importCmd() *cobra.Command { var configDir string diff --git a/cmd/y-cluster/manifests.go b/cmd/y-cluster/manifests.go new file mode 100644 index 0000000..e2c6caa --- /dev/null +++ b/cmd/y-cluster/manifests.go @@ -0,0 +1,133 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "regexp" + "strings" + + "github.com/spf13/cobra" + + "github.com/Yolean/y-cluster/pkg/cluster" +) + +// manifestsCmd is the umbrella for build-time manifest staging on the +// cluster's appliance disk. Today: just `add`. Future subcommands +// (`list`, `remove`, `get`) read or mutate the same staging dir. +func manifestsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "manifests", + Short: "Stage Kubernetes manifests on the cluster's appliance for first-customer-boot apply", + Long: `Manifests written via this command land in +` + "`/var/lib/y-cluster/manifests-staging/.yaml`" + ` on the +cluster node, NOT on the apiserver. They are NOT applied during +build. + +prepare-export moves the staging directory's contents to +` + "`/var/lib/rancher/k3s/server/manifests/`" + ` on the appliance +disk, where k3s auto-applies them on every cluster start. The +customer's first boot of the appliance therefore runs all the +staged manifests against THEIR cluster (with their data), not +against the build cluster. + +Typical use: ship a migration Job that runs once on the +customer's first boot of a new appliance version. See +specs/y-cluster/APPLIANCE_UPGRADES.md for the recommended Job +shape and idempotency conventions.`, + } + cmd.AddCommand(manifestsAddCmd()) + return cmd +} + +// manifestNameRE constrains to a portable filename: leading +// alphanumeric (no leading dot or dash), then alphanumerics + dot + +// dash + underscore. Reject path separators, ".." traversal, empty, +// and shell-metacharacters in one regex. The same regex would accept +// a kubectl resource name, which is convenient since the manifest's +// filename and metadata.name typically match. +var manifestNameRE = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*$`) + +func manifestsAddCmd() *cobra.Command { + var contextName string + + cmd := &cobra.Command{ + Use: "add ", + Short: "Stage a manifest for first-customer-boot apply", + Long: `Reads the YAML at (or stdin when is "-"), then +writes it to ` + "`/var/lib/y-cluster/manifests-staging/.yaml`" + ` +on the cluster node. Bails if a manifest with the same is +already staged. + +Example: + + y-cluster manifests add migrate-v0.5.0-userdb \ + ./migrations/v0.5.0-userdb.yaml + + cat my-job.yaml | y-cluster manifests add migrate-v0.5.0-userdb - + +Naming convention: include the source-target version pair so the +filename is unique across appliance builds. Identical names across +two builds = idempotent re-apply (no-op on the customer's cluster +since k3s remembers the prior apply).`, + Args: cobra.ExactArgs(2), + RunE: func(c *cobra.Command, args []string) error { + name := args[0] + input := args[1] + + if !manifestNameRE.MatchString(name) { + return fmt.Errorf("invalid manifest name %q: must match %s (no slashes, no .., must start with alphanumeric)", name, manifestNameRE) + } + + r, closer, err := openYAMLInput(input, c.InOrStdin()) + if err != nil { + return err + } + defer closer() + + data, err := io.ReadAll(r) + if err != nil { + return fmt.Errorf("read manifest: %w", err) + } + if len(strings.TrimSpace(string(data))) == 0 { + return fmt.Errorf("manifest is empty") + } + + lr, err := cluster.Lookup(c.Context(), "", contextName) + if err != nil { + return err + } + + target := "/var/lib/y-cluster/manifests-staging/" + name + ".yaml" + + // Stage in two RunShell calls: first probe for an + // existing entry (refuse if found), then atomically + // write the new one. We don't use a single + // `cat > ` redirect because that'd overwrite + // silently. install -m 0644 -T also creates the + // parent dir's permissions cleanly. + if cluster.RunShell(c.Context(), lr, + "test ! -e "+target, nil, nil, nil) != nil { + return fmt.Errorf("manifest %q already staged at %s; remove it first or pick a different name", name, target) + } + + // Use install(1) to create the parent dir and write + // the file atomically with a known mode. /dev/stdin + // is the standard way to feed install(1) bytes from + // the pipe. + writeCmd := "install -d -m 0755 /var/lib/y-cluster/manifests-staging && " + + "install -m 0644 /dev/stdin " + target + var stderr bytes.Buffer + if err := cluster.RunShell(c.Context(), lr, writeCmd, + bytes.NewReader(data), nil, &stderr); err != nil { + return fmt.Errorf("write manifest: %s: %w", stderr.String(), err) + } + + fmt.Fprintf(c.OutOrStdout(), "staged manifest %q -> %s (%d bytes)\n", name, target, len(data)) + return nil + }, + } + cmd.Flags().StringVar(&contextName, "context", cluster.DefaultContext, "kubeconfig context name") + return cmd +} + diff --git a/cmd/y-cluster/manifests_test.go b/cmd/y-cluster/manifests_test.go new file mode 100644 index 0000000..5fe79c8 --- /dev/null +++ b/cmd/y-cluster/manifests_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "strings" + "testing" +) + +// TestManifestNameRE_Valid pins the names that the validation +// regex accepts. Anything in this list MUST round-trip from +// `y-cluster manifests add ...` to a file at +// /var/lib/y-cluster/manifests-staging/.yaml on the cluster +// node. The shape of the regex deliberately matches what kubectl +// accepts as a resource name so the manifest's filename and +// metadata.name typically line up. +func TestManifestNameRE_Valid(t *testing.T) { + cases := []string{ + "a", + "abc", + "migrate-v0.5.0-userdb", + "migrate-v0.5.0-userdb-add-tenants", + "01-bootstrap", + "x.y.z", + "a_b", + } + for _, in := range cases { + if !manifestNameRE.MatchString(in) { + t.Errorf("manifestNameRE rejected valid name %q", in) + } + } +} + +// TestManifestNameRE_Invalid pins the rejection set. The regex is +// the only safety belt before we write to the cluster node's +// filesystem -- a slip here turns into a path-traversal that +// writes outside the staging dir. +func TestManifestNameRE_Invalid(t *testing.T) { + cases := []string{ + "", // empty + ".hidden", // leading dot + "-leading-dash", // leading dash + "path/with/slash", // path separator + "..", // path traversal + "../escape", // path traversal + "name with space", // whitespace + "name\twith\ttab", // whitespace + "name;rm -rf /", // shell metacharacter + "name`whoami`", // shell metacharacter + "name$HOME", // shell metacharacter + "name*", // glob + "name?", // glob + } + for _, in := range cases { + if manifestNameRE.MatchString(in) { + t.Errorf("manifestNameRE accepted invalid name %q", in) + } + } +} + +// TestManifestsAddCmd_Wired smokes that the cobra subcommand +// graph rejects the obvious bad-input paths before any cluster +// I/O happens. Doesn't hit a real cluster -- the important shape +// here is that bad trips the regex check, so a typo +// can't reach RunShell. +func TestManifestsAddCmd_RejectsInvalidName(t *testing.T) { + cmd := manifestsAddCmd() + cmd.SetArgs([]string{"../escape", "/dev/null"}) + cmd.SetOut(&strings.Builder{}) + cmd.SetErr(&strings.Builder{}) + err := cmd.Execute() + if err == nil { + t.Fatal("expected error for path-traversal name, got nil") + } + if !strings.Contains(err.Error(), "invalid manifest name") { + t.Errorf("expected 'invalid manifest name' in error, got: %v", err) + } +} diff --git a/cmd/y-cluster/serve.go b/cmd/y-cluster/serve.go index 68c541c..66edbd9 100644 --- a/cmd/y-cluster/serve.go +++ b/cmd/y-cluster/serve.go @@ -101,13 +101,10 @@ func serveEnsureCmd() *cobra.Command { // Success status to stdout: makes the line scriptable // (`if y-cluster serve ensure ... | grep -q started`) // and keeps stderr free for warnings the daemon may - // emit later. The short SHA tells the operator WHICH - // config the running daemon is on, which matters most - // when the answer is `noop` and they suspected the - // daemon was on stale state. + // emit later. fmt.Fprintf(cmd.OutOrStdout(), - "y-cluster serve %s on %s (sha %s)\n", - res.Action, formatPorts(res.Ports), shortSHA(res.Digest)) + "y-cluster serve %s on %s\n", + res.Action, formatPorts(res.Ports)) return nil }, } @@ -122,20 +119,6 @@ func serveEnsureCmd() *cobra.Command { // formatPorts renders ports as ":N" for one port or ":N, :M, ..." // for several. Single-port is the common case so we optimise the // reading. -// shortSHA truncates a hex sha256 digest to 12 chars (git -// short-rev convention) for the ensure output. Empty input -// returns "unknown" so the line stays parseable when a -// future Ensure path forgets to populate Digest. -func shortSHA(s string) string { - if s == "" { - return "unknown" - } - if len(s) > 12 { - return s[:12] - } - return s -} - func formatPorts(ports []int) string { if len(ports) == 1 { return fmt.Sprintf(":%d", ports[0]) diff --git a/e2e/docker_test.go b/e2e/docker_test.go index 056847e..3a126d6 100644 --- a/e2e/docker_test.go +++ b/e2e/docker_test.go @@ -9,6 +9,7 @@ import ( "context" "os" "os/exec" + "path/filepath" "strings" "testing" "time" @@ -17,6 +18,7 @@ import ( "github.com/Yolean/y-cluster/pkg/provision/config" "github.com/Yolean/y-cluster/pkg/provision/docker" + "github.com/Yolean/y-cluster/pkg/yconverge" ) // e2eDockerConfig builds a defaults-applied DockerConfig. The @@ -178,3 +180,140 @@ func TestDocker_ProvisionTeardown(t *testing.T) { } } } + +// TestDocker_Stop is the regression guard for pkg/provision/docker.Stop. +// Stop is documented to terminate the container while preserving it +// (no remove) so a follow-up `docker container start` could resume the +// same instance -- the docker half of the appliance lifecycle that +// qemu's TestQemu_StopStart already exercises. +// +// The Stop function shipped in PR #19 with no e2e gate. This proves: +// - the call returns nil for a normally-running container +// - the container exists with State.Status == "exited" afterwards +// (NOT removed; that's Teardown's job) +// - kubectl through the merged context fails to reach the apiserver +// because the host port forward is gone +func TestDocker_Stop(t *testing.T) { + skipIfNoDocker(t) + + logger, _ := zap.NewDevelopment() + cfg := e2eDockerConfig("y-cluster-e2e-stop", "36444", "y-cluster-e2e-stop") + + kcfgPath := os.Getenv("KUBECONFIG") + if kcfgPath == "" { + t.Skip("KUBECONFIG must be set") + } + + _ = exec.Command("docker", "rm", "-f", cfg.Name).Run() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + cluster, err := docker.Provision(ctx, *cfg, logger) + if err != nil { + t.Fatal(err) + } + // Teardown removes the container even after Stop -- ContainerRemove + // is force=true. Cleanup is unconditional so a Stop-without-Teardown + // regression doesn't leak a stopped container into the next test. + t.Cleanup(func() { _ = cluster.Teardown(false) }) + + // Stop the running container. docker.Stop preserves the container + // for a follow-up start; it should NOT remove it. + if err := docker.Stop(ctx, cfg.Name, logger); err != nil { + t.Fatalf("docker.Stop: %v", err) + } + + // Container exists and is exited (not removed). + statusOut, err := exec.CommandContext(ctx, "docker", "inspect", + "--format", "{{.State.Status}}", cfg.Name).CombinedOutput() + if err != nil { + t.Fatalf("docker inspect after Stop: %s: %v", statusOut, err) + } + if got := strings.TrimSpace(string(statusOut)); got != "exited" { + t.Fatalf("State.Status after Stop: got %q, want %q (container should be preserved-but-stopped)", got, "exited") + } + + // kubectl through the merged context cannot reach the apiserver -- + // the host port forward dies with the container. We accept any + // non-zero exit; the message is daemon-version-specific. + kc := exec.CommandContext(ctx, "kubectl", "--context="+cluster.Context(), + "--request-timeout=3s", "get", "nodes") + kc.Env = append(os.Environ(), "KUBECONFIG="+kcfgPath) + if out, err := kc.CombinedOutput(); err == nil { + t.Fatalf("kubectl get nodes succeeded after Stop; expected failure.\nOutput:\n%s", out) + } +} + +// TestDocker_ApplianceStateful converges the testdata/appliance-stateful +// fixture against a real docker-provisioned cluster and asserts the +// StatefulSet rolls out and its PVC binds. Closes the localstorage gap +// in PR #19: install_test.go covers the install function; nothing +// otherwise verifies that converging a stateful workload through +// kubectl-yconverge actually produces a Bound PVC and a Ready pod +// against k3s's bundled local-path provisioner. +// +// Doubles as the only consumer of testdata/appliance-stateful/, which +// the PR ships referenced only by an export.go comment otherwise. +func TestDocker_ApplianceStateful(t *testing.T) { + skipIfNoDocker(t) + + logger, _ := zap.NewDevelopment() + cfg := e2eDockerConfig("y-cluster-e2e-appstate", "36445", "y-cluster-e2e-appstate") + + kcfgPath := os.Getenv("KUBECONFIG") + if kcfgPath == "" { + t.Skip("KUBECONFIG must be set") + } + + _ = exec.Command("docker", "rm", "-f", cfg.Name).Run() + + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute) + defer cancel() + + cluster, err := docker.Provision(ctx, *cfg, logger) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cluster.Teardown(false) }) + + // Resolve testdata path relative to this test file. + wd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + kustomizeDir := filepath.Join(wd, "..", "testdata", "appliance-stateful", "base") + if _, err := os.Stat(kustomizeDir); err != nil { + t.Fatalf("testdata path %s: %v", kustomizeDir, err) + } + + // yconverge.Run resolves the namespace dep first (per the cue + // file's _dep_ns import), applies the base, and runs the cue + // rollout check on the StatefulSet (180s timeout in the fixture). + // A non-nil error here means either dep ordering broke, kustomize + // rendering broke, or the StatefulSet didn't roll out within 180s. + if _, err := yconverge.Run(ctx, yconverge.Options{ + Context: cluster.Context(), + KustomizeDir: kustomizeDir, + }, logger); err != nil { + t.Fatalf("yconverge.Run: %v", err) + } + + // The fixture's cue file checks rollout but not PVC binding. k3s's + // local-path provisioner is dynamic, so an unbound PVC would block + // pod scheduling and rollout would already have failed -- but + // asserting Bound directly catches a regression where, say, a + // future provisioner change leaves PVCs Pending while pods are + // somehow Ready against an emptyDir fallback. Belt-and-braces. + pvc := exec.CommandContext(ctx, "kubectl", "--context="+cluster.Context(), + "-n", "appliance-stateful", "get", "pvc", "data-versitygw-0", + "-o", "jsonpath={.status.phase}") + pvc.Env = append(os.Environ(), "KUBECONFIG="+kcfgPath) + pvcOut, err := pvc.CombinedOutput() + if err != nil { + t.Fatalf("kubectl get pvc: %s: %v", pvcOut, err) + } + if got := strings.TrimSpace(string(pvcOut)); got != "Bound" { + t.Fatalf("PVC data-versitygw-0 phase: got %q, want %q", got, "Bound") + } +} diff --git a/e2e/qemu_test.go b/e2e/qemu_test.go index d52dffb..8fbded5 100644 --- a/e2e/qemu_test.go +++ b/e2e/qemu_test.go @@ -279,3 +279,102 @@ func TestQemu_ExportImport(t *testing.T) { t.Fatal(err) } } + +// TestQemu_StopStart provisions, stops the VM, asserts disk + +// state sidecar are preserved while the pidfile is gone, then +// starts the VM and asserts kubectl still works against the +// merged context. +// +// This is the regression guard for the appliance lifecycle: a +// stop/start round-trip must produce an indistinguishable cluster +// (same kubeconfig context, same workloads in etcd, same node +// IP from the host's perspective). +func TestQemu_StopStart(t *testing.T) { + if _, err := os.Stat("/dev/kvm"); err != nil { + t.Skip("QEMU tests require /dev/kvm") + } + if err := qemu.CheckPrerequisites(); err != nil { + t.Skip(err) + } + + logger, _ := zap.NewDevelopment() + cfg := e2eQEMURuntime() + cfg.Name = "y-cluster-e2e-stopstart" + cfg.Context = "y-cluster-e2e-stopstart" + cfg.CacheDir = t.TempDir() + cfg.Memory = "4096" + cfg.CPUs = "2" + cfg.SSHPort = "2226" + cfg.PortForwards = e2eUniqueForwards("26446") + cfg.Kubeconfig = os.Getenv("KUBECONFIG") + if cfg.Kubeconfig == "" { + t.Skip("KUBECONFIG must be set") + } + t.Setenv("Y_CLUSTER_QEMU_CACHE_DIR", cfg.CacheDir) + + ctx := context.Background() + + cluster, err := qemu.Provision(ctx, cfg, logger) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cluster.Teardown(false) }) + + // Sanity: kubectl works against the freshly-provisioned cluster. + assertNodeReady(t, cfg.Context, cfg.Kubeconfig) + + // Sidecar landed at provision time -- start needs it. + if _, err := os.Stat(filepath.Join(cfg.CacheDir, cfg.Name+".json")); err != nil { + t.Fatalf("state sidecar missing after Provision: %v", err) + } + + // Stop. Pidfile should be gone, disk + sidecar preserved. + vmPid := readPid(t, cfg.CacheDir, cfg.Name) + if err := qemu.Stop(cfg.CacheDir, cfg.Name, logger); err != nil { + t.Fatalf("Stop: %v", err) + } + if _, err := os.Stat(filepath.Join(cfg.CacheDir, cfg.Name+".pid")); !os.IsNotExist(err) { + t.Fatalf("pidfile should be gone after Stop; stat err=%v", err) + } + if _, err := os.Stat(cluster.DiskPath()); err != nil { + t.Fatalf("disk should be preserved after Stop: %v", err) + } + if _, err := os.Stat(filepath.Join(cfg.CacheDir, cfg.Name+".json")); err != nil { + t.Fatalf("state sidecar should be preserved after Stop: %v", err) + } + assertPidGone(t, vmPid) + + // Start from the saved sidecar. Should not need cfg passed in + // -- everything required is on disk under cfg.CacheDir. + cluster2, err := qemu.Start(ctx, cfg.CacheDir, cfg.Name, logger) + if err != nil { + t.Fatalf("Start: %v", err) + } + + // kubectl works again. The cluster came back with all workloads + // intact (etcd is on the qcow2 disk, not in RAM). + assertNodeReady(t, cfg.Context, cfg.Kubeconfig) + + // SSH still works against the freshly-resumed VM. + if out, err := cluster2.SSH(ctx, "hostname"); err != nil { + t.Fatalf("SSH after Start: %s: %v", out, err) + } +} + +// assertNodeReady polls `kubectl get nodes` against ctx until at +// least one Ready node is reported, up to 2 minutes. Shared by +// the lifecycle e2e legs. +func assertNodeReady(t *testing.T, ctxName, kcfgPath string) { + t.Helper() + deadline := time.Now().Add(2 * time.Minute) + for time.Now().Before(deadline) { + kc := exec.Command("kubectl", "--context="+ctxName, "get", "nodes", "--no-headers") + kc.Env = append(os.Environ(), "KUBECONFIG="+kcfgPath) + out, err := kc.CombinedOutput() + if err == nil && strings.Contains(string(out), "Ready") { + return + } + time.Sleep(2 * time.Second) + } + t.Fatal("node never reached Ready within 2 minutes") +} diff --git a/pkg/cluster/exec.go b/pkg/cluster/exec.go index a1b5f36..bace663 100644 --- a/pkg/cluster/exec.go +++ b/pkg/cluster/exec.go @@ -76,6 +76,50 @@ func buildVMNodeRemote(binary string, args []string) string { return "sudo k3s " + binary + shellQuoteJoin(args) } +// RunShell executes an arbitrary shell command (parsed by `sh -c`) +// on the cluster node. Used by callers that need filesystem writes +// or other ad-hoc operations the ctr/crictl wrappers don't cover -- +// the canonical example is `y-cluster manifests add` writing to +// /var/lib/y-cluster/manifests-staging/. The command runs as root +// on the node (sudo on qemu/multipass; the k3s container's user +// is already root for docker). +// +// stdin/stdout/stderr are passthrough so callers can pipe arbitrary +// bytes (manifest YAML on stdin, command output to stdout). +// +// Routing per backend mirrors RunCtr. +func RunShell(ctx context.Context, lr *LookupResult, cmd string, stdin io.Reader, stdout, stderr io.Writer) error { + switch lr.Backend { + case BackendDocker: + cli, err := dockerexec.New() + if err != nil { + return fmt.Errorf("docker client: %w", err) + } + defer func() { _ = cli.Close() }() + return dockerexec.Exec(ctx, cli, lr.ContainerName, + []string{"sh", "-c", cmd}, + stdin, stdout, stderr) + case BackendQEMU: + return sshexec.ExecStream(ctx, sshexec.Target{ + Host: lr.SSHHost, Port: lr.SSHPort, + User: lr.SSHUser, KeyPath: lr.SSHKey, + }, "sudo sh -c "+singleQuote(cmd), stdin, stdout, stderr) + case BackendMultipass: + return multipassexec.ExecStream(ctx, lr.MultipassName, + "sudo sh -c "+singleQuote(cmd), stdin, stdout, stderr) + default: + return fmt.Errorf("unsupported backend %q", lr.Backend) + } +} + +// singleQuote wraps a string in POSIX single quotes for safe inclusion +// in a remote shell command. Single quotes inside the string are +// escaped via the standard '\'' trick. Used to pass an entire `sh -c` +// command line through ssh / multipass-exec without re-parsing. +func singleQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + // shellQuoteJoin shell-quotes each arg with single quotes (POSIX- // safe) and joins with leading spaces. Empty `args` returns "". // Single quotes inside an arg become `'\''` per the standard diff --git a/pkg/cluster/exec_test.go b/pkg/cluster/exec_test.go index 8c221f8..114d6a4 100644 --- a/pkg/cluster/exec_test.go +++ b/pkg/cluster/exec_test.go @@ -25,6 +25,32 @@ func TestBuildQemuRemote_NoArgs(t *testing.T) { } } +// TestSingleQuote covers the helper RunShell uses to wrap an +// entire `sh -c ` string for ssh / multipass-exec -- the +// caller's command must survive a second round of shell parsing +// without losing semantics. Embedded single quotes go through +// the standard `'\''` trick. +func TestSingleQuote(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"", "''"}, + {"hello", "'hello'"}, + {"hello world", "'hello world'"}, + {"it's", `'it'\''s'`}, + {"'leading", `''\''leading'`}, + {"trailing'", `'trailing'\'''`}, + {"a'b'c", `'a'\''b'\''c'`}, + {"echo $HOME", "'echo $HOME'"}, + } + for _, c := range cases { + if got := singleQuote(c.in); got != c.want { + t.Errorf("singleQuote(%q) = %q, want %q", c.in, got, c.want) + } + } +} + func TestShellQuoteJoin(t *testing.T) { cases := []struct { in []string diff --git a/pkg/cluster/lookup.go b/pkg/cluster/lookup.go index 2abf686..6e7286d 100644 --- a/pkg/cluster/lookup.go +++ b/pkg/cluster/lookup.go @@ -183,6 +183,18 @@ func multipassRunning(ctx context.Context, name string) bool { return running } +// ResolveClusterName resolves contextName to its cluster entry +// name in the kubeconfig. Empty kubeconfigPath uses the +// kubectl-style search ($KUBECONFIG, then ~/.kube/config). Returns +// "" (no error) when the context does not exist. +// +// Used by Lookup, but also exported for `y-cluster start` which +// needs the cluster name without first finding a running cluster +// (the cluster is, by definition, stopped at start time). +func ResolveClusterName(kubeconfigPath, contextName string) (string, error) { + return readClusterName(context.TODO(), kubeconfigPath, contextName) +} + // readClusterName resolves contextName to its cluster entry name // in the kubeconfig. Empty kubeconfigPath uses the kubectl-style // search ($KUBECONFIG, then ~/.kube/config). Returns "" (no error) diff --git a/pkg/dockerexec/dockerexec.go b/pkg/dockerexec/dockerexec.go index a764537..2c42a12 100644 --- a/pkg/dockerexec/dockerexec.go +++ b/pkg/dockerexec/dockerexec.go @@ -47,6 +47,24 @@ func Remove(ctx context.Context, cli *client.Client, name string) error { return nil } +// Stop sends SIGTERM to the named container's PID 1 and waits +// up to timeoutSecs for it to exit; after that the daemon +// escalates to SIGKILL. NotFound (container missing or already +// stopped) is treated as success. +// +// Docker's default timeout is 10s, which is too short for a +// k3s-in-container to flush its containerd snapshot writes; +// callers should pass a more generous value (60s+) when the +// container hosts a stateful workload. +func Stop(ctx context.Context, cli *client.Client, name string, timeoutSecs int) error { + t := timeoutSecs + _, err := cli.ContainerStop(ctx, name, client.ContainerStopOptions{Timeout: &t}) + if err != nil && !cerrdefs.IsNotFound(err) { + return fmt.Errorf("stop %s: %w", name, err) + } + return nil +} + // PullIfMissing fetches the named image from its registry when the // daemon doesn't already have it. ImageInspect returns a NotFound // error when the image is absent; that's the only case we pull, diff --git a/pkg/echo/echo.go b/pkg/echo/echo.go new file mode 100644 index 0000000..7bf4a28 --- /dev/null +++ b/pkg/echo/echo.go @@ -0,0 +1,164 @@ +// Package echo deploys the standard appliance-test workload: a +// Gateway listening on :80, an envoy-echo Deployment, a Service, +// and an HTTPRoute matching /q/envoy/echo on any hostname. +// +// The point is a self-contained workload the appliance e2e can +// curl through Envoy Gateway after a provision (and, eventually, +// after a stop/start or import round-trip) to prove the cluster +// still serves traffic. The response body dumps every request +// header plus pod info, so tests can assert routing AND identity +// in one curl. +// +// Sibling to pkg/provision/envoygateway: that package owns the +// Gateway controller install at provision time; this one owns +// the test workload that lands on top of it. Both shell out to +// kubectl with --server-side --field-manager=y-cluster so +// re-applies converge. +package echo + +import ( + "bytes" + "context" + _ "embed" + "fmt" + "os" + "os/exec" + "text/template" +) + +// DefaultImage is the Yolean envoy echo image: distroless envoy +// plus a Rust dynamic-modules .so that intercepts requests at +// `/q/envoy/echo` and replies with a JSON dump of headers, request +// info, and pod identity. The bundled /etc/envoy/envoy.yaml wires +// the dynamic-modules filter in front of the router, so the +// container is a drop-in upstream service for any HTTPRoute. +// +// Replaces the older registry.k8s.io/echoserver:1.10 (nginx-lua +// behind a self-signed wrapper) -- the envoy variant matches what +// production traffic flows through (real Envoy data plane), the +// response is structured JSON instead of plain-text, and the +// bundled config requires no init scaffolding (no /var/lib/nginx +// emptyDir, no /run shim, no /certs writability). +// +// Pinned by digest so a re-tag of echo-v1.38.0 upstream can't +// change the bits we ship inside the appliance. The tag stays in +// the ref for human readability of `kubectl get pod -o yaml`. +// Multi-arch (linux/amd64 + linux/arm64); pulling on either host +// arch resolves to the right manifest. +const DefaultImage = "ghcr.io/yolean/envoy:echo-v1.38.0@sha256:e86a32467f2583e3c57b76e69535fc36fb0ca3af524f4c43d19b193b70f9dd60" + +// DefaultNamespace is what `y-cluster echo deploy` defaults to +// when -n is omitted. Matches the GatewayClass name convention. +const DefaultNamespace = "y-cluster" + +// DefaultGatewayClass is the GatewayClass name the bundled Envoy +// Gateway install creates (pkg/provision/config/common.go's +// GatewayConfig.ClassName). Override on Options when the cluster +// pinned a different class via gateway.className. +const DefaultGatewayClass = "y-cluster" + +// PathPrefix is the HTTPRoute path the standard workload responds +// to. Tracks the bundled envoy.yaml in the upstream +// ghcr.io/yolean/envoy:echo-* image, which intercepts at +// /q/envoy/echo by default. Override on the HTTPRoute side AND +// the filter's path_prefix in a custom envoy.yaml if you need a +// different prefix; both must agree. +const PathPrefix = "/q/envoy/echo" + +//go:embed template.yaml +var manifestTemplate string + +// Options controls Deploy. ContextName is required; everything +// else has a documented default. +type Options struct { + ContextName string // required: kubectl context to apply against + Namespace string // empty -> DefaultNamespace + GatewayClass string // empty -> DefaultGatewayClass + Image string // empty -> DefaultImage +} + +// Deploy applies the standard echo workload. Idempotent (server- +// side apply with the y-cluster field manager) so re-running +// converges -- safe to call from a Provision that already ran it. +// +// What lands: +// +// - Namespace (default y-cluster) +// - Gateway "y-cluster" listening :80, attached to the configured +// GatewayClass, accepting routes from any namespace and any +// hostname. +// - Deployment "echo" with downward-API env vars so the response +// body's "Pod Information" section is populated. +// - Service "echo" (ClusterIP) :80 -> :8080. +// - HTTPRoute "echo" matching PathPrefix /q/envoy/echo on +// any hostname. +// +// Probe (after rollout): curl http:///q/envoy/echo +// where is whatever port the provisioner forwarded +// to guest 80. +func Deploy(ctx context.Context, opts Options) error { + if opts.ContextName == "" { + return fmt.Errorf("echo.Deploy: ContextName is required") + } + if opts.Namespace == "" { + opts.Namespace = DefaultNamespace + } + if opts.GatewayClass == "" { + opts.GatewayClass = DefaultGatewayClass + } + if opts.Image == "" { + opts.Image = DefaultImage + } + manifest, err := Render(opts) + if err != nil { + return err + } + return kubectlApplyServerSide(ctx, opts.ContextName, manifest) +} + +// Render produces the YAML manifest Deploy would apply, without +// touching a cluster. Pure function -- tests pin the rendered +// shape against expected substrings, and a future `y-cluster echo +// manifest` subcommand could surface this for inspection. +func Render(opts Options) ([]byte, error) { + if opts.Namespace == "" { + opts.Namespace = DefaultNamespace + } + if opts.GatewayClass == "" { + opts.GatewayClass = DefaultGatewayClass + } + if opts.Image == "" { + opts.Image = DefaultImage + } + tpl, err := template.New("echo").Parse(manifestTemplate) + if err != nil { + return nil, fmt.Errorf("parse echo template: %w", err) + } + var buf bytes.Buffer + if err := tpl.Execute(&buf, opts); err != nil { + return nil, fmt.Errorf("render echo manifest: %w", err) + } + return buf.Bytes(), nil +} + +// kubectlApplyServerSide pipes manifest through `kubectl apply +// --server-side --force-conflicts --field-manager=y-cluster -f -`. +// Field manager matches what pkg/yconverge and pkg/provision/ +// envoygateway use, so re-applies under any path don't fight +// over field ownership. +func kubectlApplyServerSide(ctx context.Context, contextName string, manifest []byte) error { + cmd := exec.CommandContext(ctx, "kubectl", + "--context="+contextName, + "apply", + "--server-side", "--force-conflicts", + "--field-manager=y-cluster", + "-f", "-", + ) + cmd.Stdin = bytes.NewReader(manifest) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("kubectl apply: %w", err) + } + return nil +} diff --git a/pkg/echo/echo_test.go b/pkg/echo/echo_test.go new file mode 100644 index 0000000..fe26c25 --- /dev/null +++ b/pkg/echo/echo_test.go @@ -0,0 +1,80 @@ +package echo + +import ( + "strings" + "testing" +) + +func TestRender_Defaults(t *testing.T) { + got, err := Render(Options{}) + if err != nil { + t.Fatal(err) + } + body := string(got) + + for _, want := range []string{ + "kind: Namespace", + "kind: Gateway", + "kind: Deployment", + "kind: Service", + "kind: HTTPRoute", + "namespace: y-cluster", + "gatewayClassName: y-cluster", + "image: ghcr.io/yolean/envoy:echo-v1.38.0@sha256:", + "value: /q/envoy/echo", + "type: PathPrefix", + "name: POD_NAME", + "name: NODE_NAME", + } { + if !strings.Contains(body, want) { + t.Errorf("rendered manifest missing %q\n%s", want, body) + } + } + + // HTTPRoute must NOT pin a hostname -- the request was "any + // hostname", which Gateway API expresses as the absence of + // `hostnames:` on the HTTPRoute. + if strings.Contains(body, "hostnames:") { + t.Errorf("HTTPRoute should not pin hostnames; body:\n%s", body) + } +} + +// TestRender_RespectsOverrides pins the namespace and +// gateway-class overrides so a custom appliance config (e.g. +// gateway.className: eg) renders manifests that line up with +// the cluster's actual GatewayClass. +func TestRender_RespectsOverrides(t *testing.T) { + got, err := Render(Options{ + Namespace: "test-ns", + GatewayClass: "eg", + Image: "ghcr.io/yolean/echo:test", + }) + if err != nil { + t.Fatal(err) + } + body := string(got) + + for _, want := range []string{ + "namespace: test-ns", + "name: test-ns", // the Namespace resource itself + "gatewayClassName: eg", + "image: ghcr.io/yolean/echo:test", + } { + if !strings.Contains(body, want) { + t.Errorf("rendered manifest missing %q\n%s", want, body) + } + } + if strings.Contains(body, "namespace: y-cluster") { + t.Errorf("default namespace leaked into customised render:\n%s", body) + } +} + +func TestDeploy_RequiresContext(t *testing.T) { + err := Deploy(t.Context(), Options{}) + if err == nil { + t.Fatal("Deploy with empty ContextName should error") + } + if !strings.Contains(err.Error(), "ContextName is required") { + t.Fatalf("error should name the missing field: %v", err) + } +} diff --git a/pkg/echo/template.yaml b/pkg/echo/template.yaml new file mode 100644 index 0000000..3a19392 --- /dev/null +++ b/pkg/echo/template.yaml @@ -0,0 +1,135 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{.Namespace}} +--- +# A Gateway listening on :80, accepting routes from any namespace +# and any hostname. The bundled GatewayClass (default name +# "y-cluster") wires it to Envoy Gateway. +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: y-cluster + namespace: {{.Namespace}} +spec: + gatewayClassName: {{.GatewayClass}} + listeners: + - name: http + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: All +--- +# Yolean envoy echo image: distroless envoyproxy/envoy with a Rust +# dynamic-modules .so loaded by the bundled /etc/envoy/envoy.yaml. +# The filter intercepts /q/envoy/echo and answers with a JSON dump +# of request info, headers, and the upstream envoy version. No +# extra config or scaffolding required -- the image is a complete +# upstream service ready to sit behind an HTTPRoute. +# +# Replaces the older registry.k8s.io/echoserver:1.10 (nginx + Lua). +# Distroless envoy is read-only-root-filesystem-friendly and +# matches the data plane that production traffic flows through, so +# the appliance test workload exercises the same code path the +# customer's apps will land on. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: echo + namespace: {{.Namespace}} + labels: + app.kubernetes.io/name: echo + app.kubernetes.io/managed-by: y-cluster +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: echo + template: + metadata: + labels: + app.kubernetes.io/name: echo + spec: + # Distroless envoy ships without a USER directive but doesn't + # need root for an unprivileged :8080 listener. We pin the + # security context so the pod fails fast on a node configured + # to require runAsRoot, and so a future image bump that drops + # the implicit non-root assumption is caught here. + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + containers: + - name: echo + image: {{.Image}} + ports: + - name: http + containerPort: 8080 + - name: admin + containerPort: 9901 + env: + # Downward-API vars for symmetry with the older + # echoserver template -- the bundled echo filter passes + # request headers through, so a custom envoy.yaml could + # surface these via header.add. Default config doesn't + # use them; harmless to set. + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName +--- +apiVersion: v1 +kind: Service +metadata: + name: echo + namespace: {{.Namespace}} + labels: + app.kubernetes.io/name: echo + app.kubernetes.io/managed-by: y-cluster +spec: + selector: + app.kubernetes.io/name: echo + ports: + - name: http + port: 80 + targetPort: http +--- +# HTTPRoute matches /q/envoy/echo on ANY hostname (no hostnames +# field on the HTTPRoute). Path type is PathPrefix so /q/envoy/echo +# AND /q/envoy/echo/anything-deeper both route here. The path +# tracks the upstream image's bundled envoy.yaml filter +# `path_prefix`; both sides must agree or the upstream filter +# 404s the request despite a successful gateway match. +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: echo + namespace: {{.Namespace}} + labels: + app.kubernetes.io/name: echo + app.kubernetes.io/managed-by: y-cluster +spec: + parentRefs: + - name: y-cluster + rules: + - matches: + - path: + type: PathPrefix + value: /q/envoy/echo + backendRefs: + - name: echo + port: 80 diff --git a/pkg/provision/config/common.go b/pkg/provision/config/common.go index de4d40a..4c1d8b6 100644 --- a/pkg/provision/config/common.go +++ b/pkg/provision/config/common.go @@ -63,6 +63,51 @@ type CommonConfig struct { PortForwards []PortForward `yaml:"portForwards,omitempty" json:"portForwards,omitempty" jsonschema:"description=Host->guest TCP port forwards. Defaults to 6443/80/443 when omitted. Must include a guest:6443 entry so the host's kubectl can reach the API server."` Registries Registries `yaml:"registries,omitempty" json:"registries,omitempty" jsonschema:"description=k3s registries.yaml content. Written to /etc/rancher/k3s/registries.yaml on the node before k3s starts. ${VAR} substitution is supported on credential and endpoint fields."` Gateway GatewayConfig `yaml:"gateway,omitempty" json:"gateway,omitempty" jsonschema:"description=Bundled Envoy Gateway install. Skip the install entirely (no CRDs, controller, or GatewayClass) by setting skip:true; rename the default GatewayClass via name."` + Storage StorageConfig `yaml:"storage,omitempty" json:"storage,omitempty" jsonschema:"description=Bundled local-path-provisioner install. Defaults give a predictable on-disk layout (/data/yolean/_) and Retain reclaim so PV content survives PVC delete and an appliance upgrade rebinds the same directory by name."` +} + +// StorageConfig controls the local-path-provisioner install that +// y-cluster ships in place of k3s's bundled local-storage. Three +// knobs, all with defaults that suit the per-customer appliance +// model: +// +// - Path (default /data/yolean): root directory for every PV +// directory. Predictable and namespace-friendly so customers +// can mount a separate disk at this path on import without +// digging through /var/lib/rancher/k3s/storage/. Override +// when a customer wants a different mountpoint convention. +// - PathPattern (default `{{ .PVC.Namespace }}_{{ .PVC.Name }}`): +// directory name for each PV beneath Path. The default omits +// the upstream `pvc-_` prefix so the path is reachable +// by namespace+name alone -- which lets an appliance upgrade +// (a fresh appliance disk) rebind to the same data directory +// under the operator's mounted data disk just by re-creating +// the PVC with the same namespace+name. Combined with +// ReclaimPolicy=Retain, this is the appliance-upgrade +// migration story. +// - ReclaimPolicy (default Retain): PV reclaim policy on the +// y-cluster StorageClass. Retain keeps the directory on PVC +// delete; the customer's data outlives a stray +// `kubectl delete pvc` and a fresh appliance install picks +// up the same data on the next bind. +// +// All three knobs flow into pkg/provision/localstorage.Install +// via the runtime Config translation in each provisioner. +type StorageConfig struct { + // Path is the local-path-provisioner nodePathMap default + // path -- the directory that holds one subdirectory per PV. + Path string `yaml:"path,omitempty" json:"path,omitempty" jsonschema:"default=/data/yolean,description=Root directory under which local-path-provisioner allocates one subdirectory per PV. Customers who attach a separate data disk should mount it here."` + + // PathPattern is the per-PV subdirectory naming template + // (Go text/template against the local-path-provisioner + // helper-pod variables: .PVName, .PVC.Namespace, .PVC.Name, + // .PVC.UID). + PathPattern string `yaml:"pathPattern,omitempty" json:"pathPattern,omitempty" jsonschema:"default={{ .PVC.Namespace }}_{{ .PVC.Name }},description=Per-PV subdirectory template (Go text/template; vars: .PVName, .PVC.Namespace, .PVC.Name, .PVC.UID). Drop .PVName for predictable upgrade rebinding; keep it if you need uniqueness across PVC delete+recreate cycles under Retain."` + + // ReclaimPolicy is applied to the y-cluster StorageClass. + // Retain (default) preserves directories on PVC delete; + // Delete wipes them. + ReclaimPolicy string `yaml:"reclaimPolicy,omitempty" json:"reclaimPolicy,omitempty" jsonschema:"enum=Retain,enum=Delete,default=Retain,description=Reclaim policy on the y-cluster StorageClass. Retain preserves PV directories on PVC delete; Delete (the upstream k3s default) wipes them."` } // GatewayConfig controls the bundled Envoy Gateway install @@ -207,6 +252,23 @@ func (c *CommonConfig) applyCommonDefaults() { } } c.applyGatewayDefaults() + c.applyStorageDefaults() +} + +// applyStorageDefaults fills the bundled-local-storage knobs. +// All three values are independent: a user who only overrides +// Path (e.g. customer-side disk mounted at /mnt/customer-data) +// keeps the default pattern + reclaim policy. +func (c *CommonConfig) applyStorageDefaults() { + if c.Storage.Path == "" { + c.Storage.Path = "/data/yolean" + } + if c.Storage.PathPattern == "" { + c.Storage.PathPattern = "{{ .PVC.Namespace }}_{{ .PVC.Name }}" + } + if c.Storage.ReclaimPolicy == "" { + c.Storage.ReclaimPolicy = "Retain" + } } // validateCommon checks invariants every provider relies on. The diff --git a/pkg/provision/config/k3s.yaml b/pkg/provision/config/k3s.yaml index acab405..d1282f3 100644 --- a/pkg/provision/config/k3s.yaml +++ b/pkg/provision/config/k3s.yaml @@ -16,7 +16,7 @@ # letting fresh k3s versions be tested before the mirror runs. # # Format note: k3s GitHub releases use `+k3sN` as the build-metadata -# separator (e.g. v1.35.4-rc3+k3s1). Docker tag syntax forbids `+`, +# separator (e.g. v1.35.4+k3s1). Docker tag syntax forbids `+`, # so the mirrored image's tag substitutes `+` with `-`. We pin the # GitHub-release form here because that's what the install script # and binary download URL need; the workflow and the DockerTag @@ -30,7 +30,7 @@ # 4. CI's generate-drift job re-runs schemagen and fails until the # regenerated schemas are committed alongside the bump. -version: v1.35.4-rc3+k3s1 +version: v1.35.4+k3s1 mirror: upstream: docker.io/rancher/k3s diff --git a/pkg/provision/config/storage_test.go b/pkg/provision/config/storage_test.go new file mode 100644 index 0000000..7f41c45 --- /dev/null +++ b/pkg/provision/config/storage_test.go @@ -0,0 +1,59 @@ +package config + +import "testing" + +// TestStorage_Defaults pins the appliance-shape defaults: an +// opinionated path that customers can mount a separate disk at, +// a predictable per-PV directory pattern that lets a fresh +// appliance disk rebind to the same data by namespace+name, and +// Retain reclaim so a stray `kubectl delete pvc` doesn't wipe +// customer data. +func TestStorage_Defaults(t *testing.T) { + c := &CommonConfig{} + c.applyCommonDefaults() + if c.Storage.Path != "/data/yolean" { + t.Errorf("Storage.Path: got %q, want /data/yolean", c.Storage.Path) + } + if c.Storage.PathPattern != "{{ .PVC.Namespace }}_{{ .PVC.Name }}" { + t.Errorf("Storage.PathPattern: got %q, want {{ .PVC.Namespace }}_{{ .PVC.Name }}", c.Storage.PathPattern) + } + if c.Storage.ReclaimPolicy != "Retain" { + t.Errorf("Storage.ReclaimPolicy: got %q, want Retain", c.Storage.ReclaimPolicy) + } +} + +// TestStorage_PreservesExplicitOverrides: each of the three knobs +// is independently overridable. Customers attaching a disk at a +// non-default mountpoint, or using the upstream UUID-suffixed +// pattern, or wanting Delete reclaim, must each survive the +// defaulter unchanged. +func TestStorage_PreservesExplicitOverrides(t *testing.T) { + cases := []struct { + name string + in StorageConfig + }{ + {"path only", StorageConfig{Path: "/mnt/customer-data"}}, + {"pattern only", StorageConfig{PathPattern: "{{ .PVC.Namespace }}/{{ .PVC.Name }}-{{ .PVName }}"}}, + {"reclaim only", StorageConfig{ReclaimPolicy: "Delete"}}, + {"all three", StorageConfig{ + Path: "/srv/data", + PathPattern: "{{ .PVName }}", + ReclaimPolicy: "Delete", + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := &CommonConfig{Storage: tc.in} + c.applyCommonDefaults() + if tc.in.Path != "" && c.Storage.Path != tc.in.Path { + t.Errorf("Path: got %q, want %q", c.Storage.Path, tc.in.Path) + } + if tc.in.PathPattern != "" && c.Storage.PathPattern != tc.in.PathPattern { + t.Errorf("PathPattern: got %q, want %q", c.Storage.PathPattern, tc.in.PathPattern) + } + if tc.in.ReclaimPolicy != "" && c.Storage.ReclaimPolicy != tc.in.ReclaimPolicy { + t.Errorf("ReclaimPolicy: got %q, want %q", c.Storage.ReclaimPolicy, tc.in.ReclaimPolicy) + } + }) + } +} diff --git a/pkg/provision/docker/docker.go b/pkg/provision/docker/docker.go index 26cc6a9..07a5860 100644 --- a/pkg/provision/docker/docker.go +++ b/pkg/provision/docker/docker.go @@ -27,6 +27,7 @@ import ( "github.com/Yolean/y-cluster/pkg/provision" "github.com/Yolean/y-cluster/pkg/provision/config" "github.com/Yolean/y-cluster/pkg/provision/envoygateway" + "github.com/Yolean/y-cluster/pkg/provision/localstorage" "github.com/Yolean/y-cluster/pkg/provision/registries" ) @@ -108,6 +109,22 @@ func Provision(ctx context.Context, cfg config.DockerConfig, logger *zap.Logger) if logger == nil { logger = zap.NewNop() } + + // Cross-provisioner preflight: host ports + kubeconfig context. + // Container reuse via dockerexec.Remove is idempotent so we + // don't preflight the container name; ports are the bit that + // fails in a non-obvious way (the docker daemon error message + // names the port but not what to change in y-cluster's config). + pf := provision.Preflight{ + HostPorts: dockerHostPorts(cfg), + ContextName: cfg.Context, + ContextCluster: cfg.Name, + KubeconfigPath: os.Getenv("KUBECONFIG"), + } + if err := pf.Run(); err != nil { + return nil, err + } + kubecfg, err := kubeconfig.New(cfg.Context, cfg.Name, logger) if err != nil { return nil, err @@ -170,7 +187,17 @@ func Provision(ctx context.Context, cfg config.DockerConfig, logger *zap.Logger) // --disable=traefik because y-cluster bundles Envoy // Gateway as the ingress controller; two of them // would fight over host:80/:443. - Cmd: []string{"server", "--tls-san=127.0.0.1", "--disable=traefik"}, + // --disable=local-storage because y-cluster ships + // its own local-path-provisioner via + // pkg/provision/localstorage with the appliance- + // shape defaults (path /data/yolean, PVC + // namespace_name pattern, Retain reclaim). + Cmd: []string{ + "server", + "--tls-san=127.0.0.1", + "--disable=traefik", + "--disable=local-storage", + }, // ExposedPorts must list every guest port carried by // HostConfig.PortBindings. The Docker CLI auto-fills // this when you `-p`; the moby SDK does not. Engine @@ -232,6 +259,19 @@ func Provision(ctx context.Context, cfg config.DockerConfig, logger *zap.Logger) logger.Info("k3s ready", zap.String("context", cfg.Context)) + // Install the bundled local-path-provisioner (replaces k3s's + // disabled local-storage addon) before any workload install + // so the StorageClass exists when consumer PVCs land. + if err := localstorage.Install(ctx, localstorage.Options{ + ContextName: cfg.Context, + Path: cfg.Storage.Path, + Pattern: cfg.Storage.PathPattern, + ReclaimPolicy: cfg.Storage.ReclaimPolicy, + Logger: logger, + }); err != nil { + return nil, fmt.Errorf("install local-path-provisioner: %w", err) + } + // Install the bundled Envoy Gateway (CRDs + controller + // default GatewayClass). Replaces Traefik, which we disabled // in the k3s server cmd above. Skipped wholesale when @@ -284,6 +324,19 @@ func writeRegistriesToContainer(ctx context.Context, cli *client.Client, contain return nil } +// dockerHostPorts gathers every host port docker.Provision will +// bind. Empty Host entries are skipped (docker daemon picks a +// free port); the preflight only checks what the user pinned. +func dockerHostPorts(cfg config.DockerConfig) []string { + ports := make([]string, 0, len(cfg.PortForwards)) + for _, pf := range cfg.PortForwards { + if pf.Host != "" { + ports = append(ports, pf.Host) + } + } + return ports +} + // buildHostConfig translates the YAML-shaped DockerConfig into // the moby HostConfig + the matching Config.ExposedPorts set. // Both are required for the daemon to publish bindings: @@ -353,6 +406,38 @@ func buildHostConfig(cfg config.DockerConfig) (*container.HostConfig, network.Po return hc, exposed, nil } +// Stop gracefully shuts down the docker container. The Docker +// daemon sends SIGTERM to PID 1 (k3s) and waits up to 60s for +// the guest to exit before escalating to SIGKILL. 60s vs the +// CLI default of 10s because k3s + containerd's overlayfs +// snapshot writes can take longer to flush than the default +// allows; on a faster timeout we'd see the same "exec format +// error" crash loops on next start that we hit on qemu when +// SIGTERM exited the qemu process in 200ms. +// +// Container is preserved for a follow-up `docker container +// start` (the y-cluster equivalent isn't wired yet for the +// docker backend; this is the lower half of that lifecycle). +// +// NotFound is treated as success. +func Stop(ctx context.Context, name string, logger *zap.Logger) error { + if logger == nil { + logger = zap.NewNop() + } + cli, err := dockerexec.New() + if err != nil { + return fmt.Errorf("docker daemon: %w", err) + } + defer func() { _ = cli.Close() }() + logger.Info("stopping docker container", zap.String("name", name)) + return dockerexec.Stop(ctx, cli, name, dockerStopTimeoutSecs) +} + +// dockerStopTimeoutSecs is the SIGTERM-to-SIGKILL grace the +// Stop function passes to the Docker daemon. Tunable via the +// var so tests can shorten it. +var dockerStopTimeoutSecs = 60 + // Teardown removes the container. keepDisk is ignored -- k3s state // lives entirely inside the container, so there is no persistent // disk to keep across teardowns. diff --git a/pkg/provision/localstorage/install.go b/pkg/provision/localstorage/install.go new file mode 100644 index 0000000..30a9417 --- /dev/null +++ b/pkg/provision/localstorage/install.go @@ -0,0 +1,179 @@ +// Package localstorage installs y-cluster's bundled +// local-path-provisioner. It replaces k3s's stock local-storage +// addon (provisioners pass --disable=local-storage to the k3s +// install) so y-cluster owns the StorageClass + ConfigMap + +// Deployment and can apply the appliance-shape defaults from +// CommonConfig.Storage: +// +// - Path (default /data/yolean): nodePathMap default, where +// PV directories land on the node. +// - PathPattern (default {{ .PVC.Namespace }}_{{ .PVC.Name }}): +// the per-PV subdirectory naming template, predictable +// enough that an appliance upgrade rebinds to the same +// directory by namespace+name alone. +// - ReclaimPolicy (default Retain): on the StorageClass, so +// a stray `kubectl delete pvc` doesn't wipe customer data. +// +// Sibling to pkg/provision/envoygateway: that package owns the +// Gateway controller; this one owns the StorageClass. Both use +// the same kubectl-shellout-with-server-side-apply shape. +package localstorage + +import ( + "bytes" + "context" + _ "embed" + "fmt" + "os" + "os/exec" + "text/template" + "time" + + "go.uber.org/zap" +) + +// Namespace is where the local-path-provisioner Deployment runs. +const Namespace = "local-path-storage" + +// DeploymentName is what Install waits on with `kubectl rollout +// status` after applying the manifest. +const DeploymentName = "local-path-provisioner" + +// StorageClassName is the y-cluster default StorageClass. Named +// "local-path" to match the upstream / k3s convention so consumer +// PVCs that hardcode the name (or rely on the storageclass.kubernetes.io/ +// is-default-class annotation) keep working without changes. +const StorageClassName = "local-path" + +// DefaultReadyTimeout caps how long Install waits for the +// provisioner Deployment to roll out. 90s is generous for a +// fresh image pull on a slow cluster (the image is ~20 MiB). +const DefaultReadyTimeout = 90 * time.Second + +//go:embed install.yaml +var manifestTemplate string + +// Options controls Install. Path / Pattern / ReclaimPolicy come +// from CommonConfig.Storage; ContextName is the kubeconfig +// context to apply against. +type Options struct { + ContextName string + Path string + Pattern string + ReclaimPolicy string + Logger *zap.Logger + // ReadyTimeout overrides DefaultReadyTimeout for the wait + // step. A negative value skips the wait entirely (used by + // kwok-based tests where the controller never actually + // rolls out). + ReadyTimeout time.Duration +} + +// Render produces the manifest YAML Install would apply, without +// touching a cluster. Pure function -- tests pin the rendered +// shape against expected substrings. +func Render(opts Options) ([]byte, error) { + if opts.Path == "" { + return nil, fmt.Errorf("localstorage.Render: Path is required") + } + if opts.Pattern == "" { + return nil, fmt.Errorf("localstorage.Render: Pattern is required") + } + if opts.ReclaimPolicy == "" { + return nil, fmt.Errorf("localstorage.Render: ReclaimPolicy is required") + } + tpl, err := template.New("localstorage").Parse(manifestTemplate) + if err != nil { + return nil, fmt.Errorf("parse install.yaml template: %w", err) + } + var buf bytes.Buffer + if err := tpl.Execute(&buf, opts); err != nil { + return nil, fmt.Errorf("render install.yaml: %w", err) + } + return buf.Bytes(), nil +} + +// Install renders the manifest and applies it to the cluster +// pointed at by opts.ContextName, then waits for the provisioner +// Deployment to roll out. +// +// Idempotent: re-running on a cluster that already has the +// y-cluster local-path install reconciles via SSA. Field manager +// `y-cluster` matches what envoygateway and yconverge use, so +// re-applies under any path don't fight over field ownership. +func Install(ctx context.Context, opts Options) error { + if opts.ContextName == "" { + return fmt.Errorf("localstorage.Install: ContextName is required") + } + logger := opts.Logger + if logger == nil { + logger = zap.NewNop() + } + manifest, err := Render(opts) + if err != nil { + return err + } + + logger.Info("applying local-path-provisioner", + zap.String("path", opts.Path), + zap.String("pattern", opts.Pattern), + zap.String("reclaimPolicy", opts.ReclaimPolicy), + ) + if err := kubectlApplyStdin(ctx, opts.ContextName, manifest); err != nil { + return fmt.Errorf("apply local-path manifest: %w", err) + } + + if opts.ReadyTimeout >= 0 { + timeout := opts.ReadyTimeout + if timeout == 0 { + timeout = DefaultReadyTimeout + } + logger.Info("waiting for local-path-provisioner rollout", + zap.String("namespace", Namespace), + zap.String("deployment", DeploymentName), + zap.Duration("timeout", timeout), + ) + if err := kubectlRolloutStatus(ctx, opts.ContextName, DeploymentName, Namespace, timeout); err != nil { + return fmt.Errorf("wait for %s/%s rollout: %w", Namespace, DeploymentName, err) + } + } + return nil +} + +// kubectlApplyStdin pipes the manifest into `kubectl apply +// --server-side --force-conflicts --field-manager=y-cluster`. +// Mirrors envoygateway's helper of the same name. +func kubectlApplyStdin(ctx context.Context, contextName string, manifest []byte) error { + cmd := exec.CommandContext(ctx, "kubectl", + "--context="+contextName, + "apply", + "--server-side", "--force-conflicts", + "--field-manager=y-cluster", + "-f", "-", + ) + cmd.Stdin = bytes.NewReader(manifest) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("kubectl apply --server-side: %w", err) + } + return nil +} + +// kubectlRolloutStatus runs `kubectl rollout status deployment/ +// -n --timeout=`. +func kubectlRolloutStatus(ctx context.Context, contextName, deployment, namespace string, timeout time.Duration) error { + cmd := exec.CommandContext(ctx, "kubectl", + "--context="+contextName, + "rollout", "status", + "deployment/"+deployment, + "-n", namespace, + "--timeout="+timeout.String(), + ) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("kubectl rollout status: %w", err) + } + return nil +} diff --git a/pkg/provision/localstorage/install.yaml b/pkg/provision/localstorage/install.yaml new file mode 100644 index 0000000..52af8ff --- /dev/null +++ b/pkg/provision/localstorage/install.yaml @@ -0,0 +1,208 @@ +# y-cluster bundled local-path-provisioner. +# +# Replaces k3s's stock local-storage addon (we pass +# --disable=local-storage to k3s install) so y-cluster owns the +# StorageClass, the ConfigMap, and the Deployment. The reasons +# k3s's bundled version isn't enough on its own: +# +# - k3s's deploy controller reconciles the local-path-config +# ConfigMap back to its template on every restart, so our +# pathPattern override would silently revert. +# - k3s's StorageClass uses reclaimPolicy: Delete; the appliance +# wants Retain so a stray PVC delete doesn't wipe customer +# data. +# - k3s's nodePathMap default is /var/lib/rancher/k3s/storage; +# the appliance wants /data/yolean (or whatever the operator +# mounts a separate disk at). +# +# Rendered by pkg/provision/localstorage.Render. Three Options +# fields land via Go text/template: +# +# .Path default /data/yolean +# .Pattern default is the PVC namespace_name template +# (the local-path-provisioner re-evaluates the +# string verbatim at runtime, so it stays a +# literal in the rendered ConfigMap) +# .ReclaimPolicy default Retain +--- +apiVersion: v1 +kind: Namespace +metadata: + name: local-path-storage +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: local-path-provisioner-role + namespace: local-path-storage +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: local-path-provisioner-role +rules: + - apiGroups: [""] + resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: local-path-provisioner-bind + namespace: local-path-storage +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: local-path-provisioner-role +subjects: + - kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-path-provisioner-bind +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: local-path-provisioner-role +subjects: + - kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: local-path-provisioner + namespace: local-path-storage + labels: + app.kubernetes.io/name: local-path-provisioner + app.kubernetes.io/managed-by: y-cluster +spec: + replicas: 1 + selector: + matchLabels: + app: local-path-provisioner + template: + metadata: + labels: + app: local-path-provisioner + spec: + serviceAccountName: local-path-provisioner-service-account + containers: + - name: local-path-provisioner + image: docker.io/rancher/local-path-provisioner:v0.0.35@sha256:34ff0847cc47ebf69656ba44a3de9324596d0036b66ffd323b21614dd8221530 + imagePullPolicy: IfNotPresent + command: + - local-path-provisioner + - start + - --config + - /etc/config/config.json + volumeMounts: + - name: config-volume + mountPath: /etc/config/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONFIG_MOUNT_PATH + value: /etc/config/ + volumes: + - name: config-volume + configMap: + name: local-path-config +--- +# StorageClass marked default so PVCs without an explicit +# storageClassName resolve here. ReclaimPolicy is rendered from +# CommonConfig.Storage.ReclaimPolicy (default Retain). +# +# pathPattern is a StorageClass parameter (NOT a ConfigMap +# field, despite some online examples to the contrary). The +# upstream provisioner enforces a "must start with +# namespace/name/" safety check on any pattern that doesn't +# carry allowUnsafePathPattern=true; we set the latter so the +# appliance default underscore form (namespace_name) is +# accepted. The escape hatch is "unsafe" only against the +# namespace-name safety guard upstream codifies; per-PV +# directory isolation is preserved as long as the rendered +# pattern includes both +# namespace and name (the appliance defaults do). +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-path + annotations: + storageclass.kubernetes.io/is-default-class: "true" + labels: + app.kubernetes.io/managed-by: y-cluster +provisioner: rancher.io/local-path +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: {{.ReclaimPolicy}} +parameters: + pathPattern: "{{.Pattern}}" + allowUnsafePathPattern: "true" +--- +# ConfigMap config.json -- the local-path-provisioner reads this +# at startup AND watches it for changes (no Deployment restart +# required to repoint the path). pathPattern lives on the +# StorageClass parameters above, NOT here. +kind: ConfigMap +apiVersion: v1 +metadata: + name: local-path-config + namespace: local-path-storage +data: + config.json: |- + { + "nodePathMap": [ + { + "node": "DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths": ["{{.Path}}"] + } + ] + } + setup: |- + #!/bin/sh + set -eu + mkdir -m 0777 -p "$VOL_DIR" + teardown: |- + #!/bin/sh + set -eu + rm -rf "$VOL_DIR" + helperPod.yaml: |- + apiVersion: v1 + kind: Pod + metadata: + name: helper-pod + spec: + priorityClassName: system-node-critical + tolerations: + - key: node.kubernetes.io/disk-pressure + operator: Exists + effect: NoSchedule + containers: + - name: helper-pod + image: busybox + imagePullPolicy: IfNotPresent diff --git a/pkg/provision/localstorage/install_test.go b/pkg/provision/localstorage/install_test.go new file mode 100644 index 0000000..5c95337 --- /dev/null +++ b/pkg/provision/localstorage/install_test.go @@ -0,0 +1,120 @@ +package localstorage + +import ( + "strings" + "testing" +) + +// TestRender_Defaults pins the rendered shape against the +// appliance-default values: path /data/yolean, predictable +// pattern, Retain reclaim. These are the values that ship in +// every customer build unless they explicitly override. +func TestRender_Defaults(t *testing.T) { + body := mustRender(t, Options{ + Path: "/data/yolean", + Pattern: "{{ .PVC.Namespace }}_{{ .PVC.Name }}", + ReclaimPolicy: "Retain", + }) + + for _, want := range []string{ + // Path lands in the nodePathMap. + `"paths": ["/data/yolean"]`, + // Pattern lands on the StorageClass parameters -- + // upstream local-path-provisioner reads pathPattern + // from there, NOT from the ConfigMap. + `pathPattern: "{{ .PVC.Namespace }}_{{ .PVC.Name }}"`, + // allowUnsafePathPattern: "true" lets the underscore + // form pass the upstream "must start with ns/name/" + // safety check. + `allowUnsafePathPattern: "true"`, + // ReclaimPolicy: Retain is on the StorageClass, not + // the upstream Delete. + "reclaimPolicy: Retain", + // StorageClass name + default-class annotation pinned. + "name: local-path", + `storageclass.kubernetes.io/is-default-class: "true"`, + // Image pinned by digest (regression guard against an + // accidental `:latest` retag). + "@sha256:34ff0847cc47ebf69656ba44a3de9324596d0036b66ffd323b21614dd8221530", + } { + if !strings.Contains(body, want) { + t.Errorf("rendered manifest missing %q", want) + } + } +} + +// TestRender_RespectsOverrides covers the three knobs being +// independent: a customer who only overrides Path keeps the +// pattern + reclaim defaults, etc. +func TestRender_RespectsOverrides(t *testing.T) { + body := mustRender(t, Options{ + Path: "/mnt/customer-data", + Pattern: "{{ .PVName }}", + ReclaimPolicy: "Delete", + }) + + for _, want := range []string{ + `"paths": ["/mnt/customer-data"]`, + `pathPattern: "{{ .PVName }}"`, + "reclaimPolicy: Delete", + } { + if !strings.Contains(body, want) { + t.Errorf("rendered manifest missing %q\n\n---rendered---\n%s", want, body) + } + } +} + +// TestRender_RequiresAllFields surfaces friendly errors when +// callers forget to fill a knob (a defaults bug would land here +// before any kubectl invocation, not silently as an empty value +// in the ConfigMap). +func TestRender_RequiresAllFields(t *testing.T) { + cases := []struct { + name string + opts Options + want string + }{ + {"path", Options{Pattern: "p", ReclaimPolicy: "Retain"}, "Path is required"}, + {"pattern", Options{Path: "/p", ReclaimPolicy: "Retain"}, "Pattern is required"}, + {"reclaim", Options{Path: "/p", Pattern: "p"}, "ReclaimPolicy is required"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := Render(tc.opts) + if err == nil { + t.Fatalf("expected error mentioning %q", tc.want) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error %q should mention %q", err.Error(), tc.want) + } + }) + } +} + +// TestRender_NoTemplateLeakage: the .Pattern field contains +// `{{ ... }}` text/template syntax that the local-path-provisioner +// re-parses at runtime. Our Go-side render must NOT try to +// interpret it -- a leak would manifest as the pattern resolving +// to "" in the rendered ConfigMap. +func TestRender_NoTemplateLeakage(t *testing.T) { + body := mustRender(t, Options{ + Path: "/data/yolean", + Pattern: "{{ .PVC.Namespace }}_{{ .PVC.Name }}", + ReclaimPolicy: "Retain", + }) + if strings.Contains(body, "") { + t.Errorf("template leakage; rendered manifest contains :\n%s", body) + } + if !strings.Contains(body, `{{ .PVC.Namespace }}_{{ .PVC.Name }}`) { + t.Errorf("pattern not preserved verbatim:\n%s", body) + } +} + +func mustRender(t *testing.T, opts Options) string { + t.Helper() + body, err := Render(opts) + if err != nil { + t.Fatal(err) + } + return string(body) +} diff --git a/pkg/provision/multipass/k3s.go b/pkg/provision/multipass/k3s.go index 8450a93..3948566 100644 --- a/pkg/provision/multipass/k3s.go +++ b/pkg/provision/multipass/k3s.go @@ -109,8 +109,13 @@ func (c *Cluster) installK3sAirgap(ctx context.Context) error { // strategies. tls-san pinning is what makes the host's kubectl // trust the cert when it dials https://:6443 instead of the // in-VM 127.0.0.1. +// +// --disable=local-storage hands the StorageClass over to +// pkg/provision/localstorage, which bundles a y-cluster-shaped +// local-path-provisioner (path /data/yolean, predictable PVC +// namespace_name pattern, Retain reclaim). func (c *Cluster) k3sExecArgs() string { - return "--write-kubeconfig-mode=644 --disable=traefik --tls-san=" + c.vmIP + return "--write-kubeconfig-mode=644 --disable=traefik --disable=local-storage --tls-san=" + c.vmIP } // cacheK3sAirgap downloads (or reuses cached) k3s binary + airgap diff --git a/pkg/provision/multipass/multipass.go b/pkg/provision/multipass/multipass.go index ee6e366..c247cd5 100644 --- a/pkg/provision/multipass/multipass.go +++ b/pkg/provision/multipass/multipass.go @@ -26,6 +26,7 @@ import ( "github.com/Yolean/y-cluster/pkg/provision" "github.com/Yolean/y-cluster/pkg/provision/config" "github.com/Yolean/y-cluster/pkg/provision/envoygateway" + "github.com/Yolean/y-cluster/pkg/provision/localstorage" "github.com/Yolean/y-cluster/pkg/provision/registries" ) @@ -44,6 +45,7 @@ type Config struct { K3s K3s Registries config.Registries Gateway config.GatewayConfig + Storage config.StorageConfig } // diskSize is the multipass --disk argument. Hardcoded rather than @@ -80,6 +82,7 @@ func FromConfig(c *config.MultipassConfig) Config { }, Registries: c.Registries, Gateway: c.Gateway, + Storage: c.Storage, } } @@ -114,6 +117,18 @@ func Provision(ctx context.Context, cfg Config, logger *zap.Logger) (*Cluster, e return nil, err } + // Cross-provisioner preflight: only the kubeconfig context + // matters here. Multipass binds on the hypervisor bridge, not + // on host ports, so HostPorts is empty. + pf := provision.Preflight{ + ContextName: cfg.Context, + ContextCluster: cfg.Name, + KubeconfigPath: os.Getenv("KUBECONFIG"), + } + if err := pf.Run(); err != nil { + return nil, err + } + kubecfg, err := kubeconfig.New(cfg.Context, cfg.Name, logger) if err != nil { return nil, err @@ -165,6 +180,19 @@ func Provision(ctx context.Context, cfg Config, logger *zap.Logger) (*Cluster, e } logger.Info("k3s ready", zap.String("context", cfg.Context)) + // Install the bundled local-path-provisioner (replaces k3s's + // disabled local-storage addon) before any workload install + // so the StorageClass exists when consumer PVCs land. + if err := localstorage.Install(ctx, localstorage.Options{ + ContextName: cfg.Context, + Path: cfg.Storage.Path, + Pattern: cfg.Storage.PathPattern, + ReclaimPolicy: cfg.Storage.ReclaimPolicy, + Logger: logger, + }); err != nil { + return nil, fmt.Errorf("install local-path-provisioner: %w", err) + } + if cfg.Gateway.Skip { logger.Info("envoy gateway install skipped (gateway.skip)") } else { @@ -192,6 +220,27 @@ func Provision(ctx context.Context, cfg Config, logger *zap.Logger) (*Cluster, e return c, nil } +// Stop gracefully shuts down the multipass VM via `multipass +// stop`, which sends ACPI poweroff to the guest and waits for +// shutdown to complete. multipass's behaviour is already +// graceful by design (no SIGTERM-too-fast issue analogous to +// what bit qemu); this wrapper just plumbs `multipass stop` +// into the y-cluster stopCmd path so docker / qemu / multipass +// are equivalent at that surface. +// +// VM disk is preserved for a follow-up `multipass start`. +// NotFound is treated as success. +func Stop(ctx context.Context, name string, logger *zap.Logger) error { + if logger == nil { + logger = zap.NewNop() + } + logger.Info("stopping multipass VM", zap.String("name", name)) + if err := multipassexec.Stop(ctx, name); err != nil && !errors.Is(err, multipassexec.ErrNotFound) { + return err + } + return nil +} + // Teardown stops and (unless keepDisk) deletes/purges the VM, then // removes the kubeconfig context entry. keepDisk=true means the VM // is stopped but not deleted, so a follow-up `multipass start` diff --git a/pkg/provision/preflight.go b/pkg/provision/preflight.go new file mode 100644 index 0000000..254d2fb --- /dev/null +++ b/pkg/provision/preflight.go @@ -0,0 +1,122 @@ +package provision + +import ( + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strings" + + "github.com/Yolean/y-cluster/pkg/kubeconfig" +) + +// Preflight runs cross-provisioner checks BEFORE any state-mutating +// step in Provision. The point is to fail fast with an actionable +// message ("host port 6443 already bound; change portForwards in +// the config") rather than letting the user discover the conflict +// halfway through a partial provision. +// +// Two classes of check: +// +// - HostPorts: every entry must currently be free. Empty values +// skip (provider auto-assigns). +// - KubeconfigContext: the context name must be either absent or +// already pointing at clusterName. A second cluster that +// reuses an existing context name would clobber the first +// cluster's user/cert and silently break kubectl for it. +// +// Provider-specific checks (qemu's "is the named VM already +// running") layer on top in the per-provider Provision; they're +// not generalisable. +type Preflight struct { + HostPorts []string + ContextName string + ContextCluster string + KubeconfigPath string // empty -> kubectl-style env+default search +} + +// Run executes every check, accumulating errors so the caller +// sees the full list of conflicts in one go (typical case: the +// developer copy-pasted the existing config and forgot to change +// any of the host-bound identifiers). +func (p Preflight) Run() error { + var problems []string + for _, port := range p.HostPorts { + if err := checkHostPort(port); err != nil { + problems = append(problems, err.Error()) + } + } + if p.ContextName != "" { + if err := checkKubeconfigContext(p.KubeconfigPath, p.ContextName, p.ContextCluster); err != nil { + problems = append(problems, err.Error()) + } + } + if len(problems) == 0 { + return nil + } + return fmt.Errorf("preflight checks failed:\n - %s", strings.Join(problems, "\n - ")) +} + +// checkHostPort verifies port (a string for cobra-friendliness) is +// not currently bound on 127.0.0.1. Probes by binding briefly and +// closing immediately. Race window is negligible for human-driven +// provisions. +func checkHostPort(port string) error { + if port == "" { + return nil // provider auto-assigns + } + addr := net.JoinHostPort("127.0.0.1", port) + l, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("host port %s in use (likely another cluster); change the binding in the config", port) + } + _ = l.Close() + return nil +} + +// checkKubeconfigContext returns nil when the context is absent +// or already points at expectedCluster. Otherwise the context +// belongs to a different cluster and re-using it for a new +// provision would orphan that other cluster's kubectl access. +func checkKubeconfigContext(kubeconfigPath, contextName, expectedCluster string) error { + resolved := resolveKubeconfigPath(kubeconfigPath) + if resolved == "" { + return nil // no kubeconfig to read; nothing to clobber + } + cfg, err := kubeconfig.Load(resolved) + if err != nil { + // File not present is fine; means there's nothing to clobber. + if errors.Is(err, os.ErrNotExist) { + return nil + } + return fmt.Errorf("load kubeconfig %s: %w", resolved, err) + } + existing := cfg.ContextCluster(contextName) + if existing == "" || existing == expectedCluster { + return nil + } + return fmt.Errorf( + "kubeconfig context %q already points at cluster %q; "+ + "this provision wants cluster %q. Change `context:` in the config "+ + "to a unique name (convention: equal to `name:`) so kubectl access "+ + "to the existing cluster isn't clobbered", + contextName, existing, expectedCluster) +} + +// resolveKubeconfigPath mirrors the kubectl-style search used by +// pkg/cluster.readClusterName: explicit arg, then $KUBECONFIG +// (first entry of a colon-list), then ~/.kube/config. +func resolveKubeconfigPath(explicit string) string { + if explicit != "" { + return explicit + } + if env := os.Getenv("KUBECONFIG"); env != "" { + return strings.SplitN(env, string(os.PathListSeparator), 2)[0] + } + home, err := os.UserHomeDir() + if err != nil { + return "" + } + return filepath.Join(home, ".kube", "config") +} diff --git a/pkg/provision/preflight_test.go b/pkg/provision/preflight_test.go new file mode 100644 index 0000000..3f037dc --- /dev/null +++ b/pkg/provision/preflight_test.go @@ -0,0 +1,176 @@ +package provision + +import ( + "net" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestPreflight_PortFree exercises the happy path: a port that +// nobody is listening on passes the check. +func TestPreflight_PortFree(t *testing.T) { + port := pickFreePort(t) + if err := checkHostPort(port); err != nil { + t.Fatalf("free port %s: %v", port, err) + } +} + +// TestPreflight_PortInUse: bind a port for the duration of the +// test so the check sees it as taken. +func TestPreflight_PortInUse(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { _ = l.Close() }() + port := portFromAddr(l.Addr().String()) + err = checkHostPort(port) + if err == nil { + t.Fatalf("port %s should report in-use", port) + } + if !strings.Contains(err.Error(), port) || !strings.Contains(err.Error(), "in use") { + t.Fatalf("error should name the port and 'in use': %v", err) + } +} + +// TestPreflight_PortEmpty: an empty Host (provider auto-assigns) +// must not error -- there's nothing to check. +func TestPreflight_PortEmpty(t *testing.T) { + if err := checkHostPort(""); err != nil { + t.Fatalf("empty port should pass: %v", err) + } +} + +// TestPreflight_ContextAbsent: a context name that doesn't exist +// in kubeconfig is fine -- there's nothing to clobber. +func TestPreflight_ContextAbsent(t *testing.T) { + path := writeKubeconfig(t, "") + if err := checkKubeconfigContext(path, "absent", "y-cluster"); err != nil { + t.Fatalf("absent context should pass: %v", err) + } +} + +// TestPreflight_ContextSameCluster: re-provisioning is the common +// case -- the context already exists pointing at the same cluster +// the new Provision will produce. Pass. +func TestPreflight_ContextSameCluster(t *testing.T) { + path := writeKubeconfig(t, ` +apiVersion: v1 +kind: Config +clusters: +- name: y-cluster + cluster: + server: https://127.0.0.1:26443 +contexts: +- name: local + context: + cluster: y-cluster + user: ystack +`) + if err := checkKubeconfigContext(path, "local", "y-cluster"); err != nil { + t.Fatalf("re-provision (same cluster) should pass: %v", err) + } +} + +// TestPreflight_ContextDifferentCluster: this is the regression +// guard -- a second cluster that re-uses an existing context name +// would clobber the first cluster's kubectl access. +func TestPreflight_ContextDifferentCluster(t *testing.T) { + path := writeKubeconfig(t, ` +apiVersion: v1 +kind: Config +clusters: +- name: y-cluster + cluster: + server: https://127.0.0.1:26443 +contexts: +- name: local + context: + cluster: y-cluster + user: ystack +`) + err := checkKubeconfigContext(path, "local", "y-cluster-tiny") + if err == nil { + t.Fatal("context pointing at a different cluster should error") + } + for _, want := range []string{"local", "y-cluster", "y-cluster-tiny"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("error should name %q; got %v", want, err) + } + } +} + +// TestPreflight_RunAccumulatesProblems: exercising Run as a +// whole. The point is a single error listing every conflict the +// caller has to fix, not a fail-fast that surfaces them one at a +// time. +func TestPreflight_RunAccumulatesProblems(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { _ = l.Close() }() + boundPort := portFromAddr(l.Addr().String()) + path := writeKubeconfig(t, ` +apiVersion: v1 +kind: Config +clusters: +- name: y-cluster + cluster: + server: https://127.0.0.1:26443 +contexts: +- name: local + context: + cluster: y-cluster + user: ystack +`) + pf := Preflight{ + HostPorts: []string{boundPort}, + ContextName: "local", + ContextCluster: "y-cluster-tiny", + KubeconfigPath: path, + } + err = pf.Run() + if err == nil { + t.Fatal("want aggregated error") + } + if !strings.Contains(err.Error(), "preflight checks failed") { + t.Fatalf("missing summary header: %v", err) + } + if !strings.Contains(err.Error(), boundPort) { + t.Fatalf("missing port in error: %v", err) + } + if !strings.Contains(err.Error(), "local") { + t.Fatalf("missing context name in error: %v", err) + } +} + +// pickFreePort obtains a port number that's currently free by +// asking the kernel (Listen on :0, capture, close). +func pickFreePort(t *testing.T) string { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := portFromAddr(l.Addr().String()) + _ = l.Close() + return port +} + +func portFromAddr(addr string) string { + _, port, _ := net.SplitHostPort(addr) + return port +} + +func writeKubeconfig(t *testing.T, body string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "kubeconfig") + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return path +} diff --git a/pkg/provision/qemu/data_seed.go b/pkg/provision/qemu/data_seed.go new file mode 100644 index 0000000..c666c6c --- /dev/null +++ b/pkg/provision/qemu/data_seed.go @@ -0,0 +1,253 @@ +package qemu + +import ( + "context" + "crypto/sha256" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "time" + + "go.uber.org/zap" +) + +// Embedded assets that travel with the appliance disk and run +// on the customer's first boot. See +// specs/y-cluster/APPLIANCE_UPGRADES.md for the design. + +//go:embed data_seed.service +var dataSeedUnit string + +//go:embed data_seed_check.sh +var dataSeedCheckScript string + +//go:embed seed_status.sh +var seedStatusScript string + +//go:embed k3s_data_seed.conf +var k3sDataSeedDropin string + +// SeedMetaSchemaVersion is the on-disk schema version for the seed +// meta JSON. Bump when the schema changes incompatibly. See +// dataSeedMeta below. +const SeedMetaSchemaVersion = 1 + +// dataSeedMeta is the JSON written to /var/lib/y-cluster/data-seed.meta.json +// on the appliance disk AND copied verbatim to /data/yolean/.y-cluster-seeded +// on the customer's first boot. +// +// The customer-side marker is the source of truth for "this volume +// has been seeded". Future appliance versions can compare the +// customer's marker against the new seed's sha256 to decide whether +// migrations are needed. +type dataSeedMeta struct { + SchemaVersion int `json:"schemaVersion"` + SeededAt string `json:"seeded_at"` + SeededBy string `json:"seeded_by"` + ApplianceName string `json:"appliance_name"` + SeedSHA256 string `json:"seed_sha256"` +} + +// extractDataYolean reads /data/yolean from an offline qcow2 via +// libguestfs's virt-tar-out and writes a zstd-compressed tar at +// outPath. Returns the SHA-256 of the resulting file. +// +// virt-tar-out emits an uncompressed tar on stdout; we pipe it +// through zstd at default level (3) for a good speed/ratio balance. +// The compressed file lives ON the appliance boot disk (under +// /var/lib/y-cluster/) so it travels with the appliance and is +// available even when the customer mounts an empty drive at +// /data/yolean (which obscures the boot disk's copy). +func extractDataYolean(ctx context.Context, qcow2Path, outPath string) (string, error) { + out, err := os.Create(outPath) + if err != nil { + return "", fmt.Errorf("create seed file: %w", err) + } + defer out.Close() + + hasher := sha256.New() + mw := io.MultiWriter(out, hasher) + + // virt-tar-out -a /data/yolean - -> stdout + tarOut := exec.CommandContext(ctx, "virt-tar-out", + "-a", qcow2Path, "/data/yolean", "-") + tarOut.Stderr = os.Stderr + tarPipe, err := tarOut.StdoutPipe() + if err != nil { + return "", fmt.Errorf("virt-tar-out pipe: %w", err) + } + + // zstd -q - (compress stdin to stdout) + zstd := exec.CommandContext(ctx, "zstd", "-q", "-") + zstd.Stdin = tarPipe + zstd.Stdout = mw + zstd.Stderr = os.Stderr + + if err := zstd.Start(); err != nil { + return "", fmt.Errorf("start zstd: %w", err) + } + if err := tarOut.Run(); err != nil { + _ = zstd.Wait() + return "", fmt.Errorf("virt-tar-out: %w", err) + } + if err := zstd.Wait(); err != nil { + return "", fmt.Errorf("zstd: %w", err) + } + + if err := out.Sync(); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(hasher.Sum(nil)), nil +} + +// writeSeedMeta serialises dataSeedMeta to outPath. Side-channel +// used to inject the same metadata into both +// /var/lib/y-cluster/data-seed.meta.json on the appliance AND +// /data/yolean/.y-cluster-seeded on the customer's first boot. +func writeSeedMeta(outPath string, meta dataSeedMeta) error { + data, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return err + } + return os.WriteFile(outPath, data, 0o644) +} + +// SeedAssets bundles the host-side artefacts that need to land on +// the appliance disk for the data-seed feature to work at customer +// boot. PrepareExport materialises all of these and feeds them into +// virt-customize via --copy-in / --upload. +type SeedAssets struct { + SeedTarPath string // /data-seed.tar.zst -- the snapshot + SeedMetaPath string // /data-seed.meta.json -- metadata + SeedCheckPath string // /y-cluster-data-seed-check -- the boot-time logic (chmod 0755) + SeedStatusPath string // /y-cluster-seed-status -- the customer-facing helper (chmod 0755) + UnitPath string // /y-cluster-data-seed.service + K3sDropinPath string // /k3s.service.d/y-cluster-data-seed.conf + TmpDir string // parent of the above; caller os.RemoveAll's at the end +} + +// BuildSeedAssets snapshots /data/yolean from the offline qcow2, +// generates the meta JSON, and stages every asset PrepareExport +// will copy into the appliance via virt-customize. The caller MUST +// `os.RemoveAll(s.TmpDir)` to clean up. +// +// Returns (nil, nil) if the qcow2 doesn't have a /data/yolean dir +// at all -- e.g., a build cluster that never ran any workload that +// uses the bundled local-path. In that case PrepareExport SHOULD +// proceed without seed assets; the appliance will simply have no +// data-seed.tar.zst, the systemd unit's ConditionPathExists fires, +// and the unit no-ops at boot. +func BuildSeedAssets(ctx context.Context, qcow2Path, applianceName string) (*SeedAssets, error) { + tmpDir, err := os.MkdirTemp("", "y-cluster-seed-") + if err != nil { + return nil, fmt.Errorf("mkdir seed tmp: %w", err) + } + + tarPath := filepath.Join(tmpDir, "data-seed.tar.zst") + sha, err := extractDataYolean(ctx, qcow2Path, tarPath) + if err != nil { + os.RemoveAll(tmpDir) + // virt-tar-out fails when the source path doesn't exist + // in the guest. We treat that as "no /data/yolean to + // seed", not an error worth aborting prepare-export over. + // PrepareExport's logger surfaces a Warn so the operator + // sees we skipped seed creation. + return nil, fmt.Errorf("extract /data/yolean: %w", err) + } + + metaPath := filepath.Join(tmpDir, "data-seed.meta.json") + meta := dataSeedMeta{ + SchemaVersion: SeedMetaSchemaVersion, + SeededAt: time.Now().UTC().Format(time.RFC3339), + SeededBy: "y-cluster prepare-export", + ApplianceName: applianceName, + SeedSHA256: sha, + } + if err := writeSeedMeta(metaPath, meta); err != nil { + os.RemoveAll(tmpDir) + return nil, fmt.Errorf("write seed meta: %w", err) + } + + checkPath := filepath.Join(tmpDir, "y-cluster-data-seed-check") + if err := os.WriteFile(checkPath, []byte(dataSeedCheckScript), 0o755); err != nil { + os.RemoveAll(tmpDir) + return nil, fmt.Errorf("write seed-check: %w", err) + } + statusPath := filepath.Join(tmpDir, "y-cluster-seed-status") + if err := os.WriteFile(statusPath, []byte(seedStatusScript), 0o755); err != nil { + os.RemoveAll(tmpDir) + return nil, fmt.Errorf("write seed-status: %w", err) + } + unitPath := filepath.Join(tmpDir, "y-cluster-data-seed.service") + if err := os.WriteFile(unitPath, []byte(dataSeedUnit), 0o644); err != nil { + os.RemoveAll(tmpDir) + return nil, fmt.Errorf("write seed unit: %w", err) + } + dropinDir := filepath.Join(tmpDir, "k3s.service.d") + if err := os.MkdirAll(dropinDir, 0o755); err != nil { + os.RemoveAll(tmpDir) + return nil, fmt.Errorf("mkdir k3s dropin: %w", err) + } + dropinPath := filepath.Join(dropinDir, "y-cluster-data-seed.conf") + if err := os.WriteFile(dropinPath, []byte(k3sDataSeedDropin), 0o644); err != nil { + os.RemoveAll(tmpDir) + return nil, fmt.Errorf("write k3s dropin: %w", err) + } + + return &SeedAssets{ + SeedTarPath: tarPath, + SeedMetaPath: metaPath, + SeedCheckPath: checkPath, + SeedStatusPath: statusPath, + UnitPath: unitPath, + K3sDropinPath: dropinPath, + TmpDir: tmpDir, + }, nil +} + +// virtCustomizeArgsForSeed returns the additional --upload arguments +// PrepareExport appends to its virt-customize invocation when seed +// assets are present. --upload places a host file at an absolute +// path inside the guest filesystem; --copy-in places it at a +// directory. We use --upload throughout so each destination path is +// explicit and the caller doesn't have to think about basename +// collisions. +func virtCustomizeArgsForSeed(s *SeedAssets) []string { + if s == nil { + return nil + } + return []string{ + "--mkdir", "/var/lib/y-cluster", + "--upload", s.SeedTarPath + ":/var/lib/y-cluster/data-seed.tar.zst", + "--upload", s.SeedMetaPath + ":/var/lib/y-cluster/data-seed.meta.json", + "--upload", s.SeedCheckPath + ":/usr/local/bin/y-cluster-data-seed-check", + "--upload", s.SeedStatusPath + ":/usr/local/bin/y-cluster-seed-status", + "--upload", s.UnitPath + ":/etc/systemd/system/y-cluster-data-seed.service", + "--mkdir", "/etc/systemd/system/k3s.service.d", + "--upload", s.K3sDropinPath + ":/etc/systemd/system/k3s.service.d/y-cluster-data-seed.conf", + "--chmod", "0755:/usr/local/bin/y-cluster-data-seed-check", + "--chmod", "0755:/usr/local/bin/y-cluster-seed-status", + // The unit gets enabled (creates the wantedby symlink) + // AFTER prepare_inguest.sh runs -- order matters because + // systemctl enable requires /etc/systemd/system to be + // writable, which it is throughout virt-customize. + "--run-command", "systemctl enable y-cluster-data-seed.service", + } +} + +// applianceNameFromConfig is a small adapter so PrepareExport doesn't +// hard-code the field. Keeps the dataSeedMeta struct decoupled from +// Config's own evolution. +func applianceNameFromConfig(cfg Config) string { + return cfg.Name +} + +// silenceUnused references the logger import so a future build that +// drops the seed feature's only Warn doesn't fail with "imported and +// not used". Tiny cost, makes the import survive intermediate edits. +var _ = zap.NewNop diff --git a/pkg/provision/qemu/data_seed.service b/pkg/provision/qemu/data_seed.service new file mode 100644 index 0000000..7b16d95 --- /dev/null +++ b/pkg/provision/qemu/data_seed.service @@ -0,0 +1,17 @@ +[Unit] +Description=y-cluster appliance data-dir first-boot seed +DefaultDependencies=no +RequiresMountsFor=/data/yolean +After=local-fs.target systemd-tmpfiles-setup.service +Before=k3s.service +ConditionPathExists=/var/lib/y-cluster/data-seed.tar.zst + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/bin/y-cluster-data-seed-check +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/pkg/provision/qemu/data_seed_check.sh b/pkg/provision/qemu/data_seed_check.sh new file mode 100644 index 0000000..03507c5 --- /dev/null +++ b/pkg/provision/qemu/data_seed_check.sh @@ -0,0 +1,102 @@ +#!/bin/sh +# y-cluster-data-seed-check: extract the build-time /data/yolean +# snapshot onto a freshly-attached customer disk, or no-op when +# the appliance disk's own /data/yolean is in use, or refuse to +# clobber unrecognised data. +# +# Run by y-cluster-data-seed.service (oneshot) Before=k3s.service. +# k3s.service has a Requires= drop-in pointing here so a failure +# blocks the cluster from coming up -- the customer SSHes in, +# reads the journal, fixes the situation, and either restarts +# this unit or starts k3s manually. +# +# Decision table: +# /data/yolean is NOT a separate mount -> no-op +# marker present -> no-op +# no marker, dir empty (excl. lost+found) -> extract seed +# no marker, dir non-empty -> fail (conflict) +# +# See specs/y-cluster/APPLIANCE_UPGRADES.md for the full design. + +set -eu + +MOUNT=/data/yolean +SEED=/var/lib/y-cluster/data-seed.tar.zst +META=/var/lib/y-cluster/data-seed.meta.json +MARKER="$MOUNT/.y-cluster-seeded" + +# 1. Not a separate mount -> data lives on the boot disk, already +# populated by the appliance build. No customer drive attached; +# nothing to seed. We let k3s come up against the boot-disk data. +if ! mountpoint -q "$MOUNT" 2>/dev/null; then + echo "y-cluster-data-seed: $MOUNT is not a separate mount; using appliance boot-disk data, no seed needed." + exit 0 +fi + +# 2. Marker present -> the customer's drive has been seeded before +# (by us, on a previous boot) OR the customer placed the marker +# manually after restoring data themselves. Either way we respect +# what's there and do not touch it. +if [ -e "$MARKER" ]; then + echo "y-cluster-data-seed: marker present at $MARKER; respecting existing data." + cat "$MARKER" 2>/dev/null || true + exit 0 +fi + +# 3. No marker. Inspect contents excluding lost+found (created by +# the kernel on every fresh ext4 mount; not a sign of customer data). +ENTRIES=$(find "$MOUNT" -mindepth 1 -maxdepth 1 ! -name 'lost+found' 2>/dev/null | head -20) +if [ -n "$ENTRIES" ]; then + cat >&2 </dev/null + sudo systemctl restart y-cluster-data-seed.service + (b) The data is junk (e.g., from a previous wrong boot) and you + want a fresh seed (DESTRUCTIVE): + sudo rm -rf $MOUNT/* $MOUNT/.[!.]* + sudo systemctl restart y-cluster-data-seed.service + (c) The drive is the wrong drive entirely: + poweroff, attach the right drive, boot again. +EOF + exit 1 +fi + +# 4. Empty mountpoint, no marker. Extract the seed. +if [ ! -f "$SEED" ]; then + echo "y-cluster-data-seed: no seed at $SEED; this appliance was built without one." >&2 + echo "y-cluster-data-seed: cannot proceed -- mark $MARKER manually if the empty mount is intentional." >&2 + exit 1 +fi + +if [ ! -f "$META" ]; then + echo "y-cluster-data-seed: no metadata at $META; appliance build is incomplete." >&2 + exit 1 +fi + +echo "y-cluster-data-seed: extracting $SEED to $MOUNT" +zstdcat "$SEED" | tar -C "$MOUNT" -xpf - +echo "y-cluster-data-seed: extracted." + +# 5. Write the marker LAST. A crashed extract leaves no marker, so +# the next boot detects "non-empty without marker" -> conflict mode +# (case 3) and surfaces the problem instead of silently retrying. +# We copy the build-time meta verbatim; it carries the seed sha, +# build version, and timestamp. +cp "$META" "$MARKER" +chmod 0644 "$MARKER" + +echo "y-cluster-data-seed: seeded $MOUNT successfully." +cat "$MARKER" diff --git a/pkg/provision/qemu/data_seed_test.go b/pkg/provision/qemu/data_seed_test.go new file mode 100644 index 0000000..9e24bec --- /dev/null +++ b/pkg/provision/qemu/data_seed_test.go @@ -0,0 +1,275 @@ +package qemu + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// runSeedCheck executes the embedded data_seed_check.sh against a +// fake $MOUNT and $SEED setup the caller wires up under tmp. +// +// The script is hardcoded to /data/yolean / /var/lib/y-cluster paths. +// We override them by writing a tiny wrapper that exports the same +// names as shell environment variables and then sources the original +// script with sed-substituted constants. Cheaper than refactoring the +// boot-time script to take env-var paths -- the boot-time script is +// what runs on a real customer machine and shouldn't grow knobs we +// don't need there. +func runSeedCheck(t *testing.T, mount, seed, meta string) (stdout, stderr string, exit int) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("seed-check is /bin/sh-only") + } + if _, err := exec.LookPath("zstd"); err != nil { + t.Skip("zstd not on PATH") + } + + // Materialise the script with path substitutions. + src := dataSeedCheckScript + src = strings.Replace(src, "MOUNT=/data/yolean", "MOUNT="+mount, 1) + src = strings.Replace(src, "SEED=/var/lib/y-cluster/data-seed.tar.zst", "SEED="+seed, 1) + src = strings.Replace(src, "META=/var/lib/y-cluster/data-seed.meta.json", "META="+meta, 1) + // The script's mountpoint check uses `mountpoint -q` against + // MOUNT. tmp dirs are not mountpoints, so the test would + // always exit early at "not a separate mount". Replace the + // guard with a TEST_FORCE_MOUNT env-var override so we can + // exercise the real branches. + src = strings.Replace(src, + `if ! mountpoint -q "$MOUNT" 2>/dev/null; then`, + `if [ -z "${TEST_FORCE_MOUNT:-}" ] && ! mountpoint -q "$MOUNT" 2>/dev/null; then`, + 1) + + scriptPath := filepath.Join(t.TempDir(), "seed-check.sh") + if err := os.WriteFile(scriptPath, []byte(src), 0o755); err != nil { + t.Fatal(err) + } + cmd := exec.Command("/bin/sh", scriptPath) + cmd.Env = append(os.Environ(), "TEST_FORCE_MOUNT=1") + var sob, seb strings.Builder + cmd.Stdout = &sob + cmd.Stderr = &seb + err := cmd.Run() + if exitErr, ok := err.(*exec.ExitError); ok { + exit = exitErr.ExitCode() + } else if err != nil { + t.Fatalf("script run: %v", err) + } + return sob.String(), seb.String(), exit +} + +// makeSeedTar writes a small tar.zst at seedPath whose contents are +// the entries (path -> body) given. The sha256 of the resulting +// tarball is returned for verification. +func makeSeedTar(t *testing.T, seedPath string, entries map[string]string) { + t.Helper() + if _, err := exec.LookPath("zstd"); err != nil { + t.Skip("zstd not on PATH") + } + contentDir := filepath.Join(filepath.Dir(seedPath), "seed-src") + if err := os.MkdirAll(contentDir, 0o755); err != nil { + t.Fatal(err) + } + for name, body := range entries { + full := filepath.Join(contentDir, name) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + // tar -C contentDir -cf - . | zstd > seedPath + tarCmd := exec.Command("tar", "-C", contentDir, "-cf", "-", ".") + tarOut, err := tarCmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + zstdCmd := exec.Command("zstd", "-q", "-") + zstdCmd.Stdin = tarOut + out, err := os.Create(seedPath) + if err != nil { + t.Fatal(err) + } + defer out.Close() + zstdCmd.Stdout = out + if err := zstdCmd.Start(); err != nil { + t.Fatal(err) + } + if err := tarCmd.Run(); err != nil { + t.Fatal(err) + } + if err := zstdCmd.Wait(); err != nil { + t.Fatal(err) + } +} + +// TestSeedCheck_MarkerPresent_NoOp pins the upgrade fast path: the +// customer's drive already has a marker, we respect it and exit 0. +func TestSeedCheck_MarkerPresent_NoOp(t *testing.T) { + dir := t.TempDir() + mount := filepath.Join(dir, "mount") + if err := os.MkdirAll(mount, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mount, ".y-cluster-seeded"), + []byte(`{"schemaVersion":1,"existing":"marker"}`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mount, "existing-data.txt"), + []byte("customer's data"), 0o644); err != nil { + t.Fatal(err) + } + + stdout, _, exit := runSeedCheck(t, mount, "/nonexistent/seed", "/nonexistent/meta") + if exit != 0 { + t.Errorf("exit: got %d, want 0; stdout=%q", exit, stdout) + } + if !strings.Contains(stdout, "marker present") { + t.Errorf("expected 'marker present' in stdout, got: %s", stdout) + } + // Existing data must be untouched. + body, _ := os.ReadFile(filepath.Join(mount, "existing-data.txt")) + if string(body) != "customer's data" { + t.Errorf("existing data mutated: %q", body) + } +} + +// TestSeedCheck_EmptyMount_Seeds covers the green path: customer +// attached an empty drive, we extract the seed and write the marker. +func TestSeedCheck_EmptyMount_Seeds(t *testing.T) { + dir := t.TempDir() + mount := filepath.Join(dir, "mount") + if err := os.MkdirAll(mount, 0o755); err != nil { + t.Fatal(err) + } + // lost+found is allowed and must be ignored. + if err := os.MkdirAll(filepath.Join(mount, "lost+found"), 0o755); err != nil { + t.Fatal(err) + } + + seed := filepath.Join(dir, "seed.tar.zst") + meta := filepath.Join(dir, "seed.meta.json") + makeSeedTar(t, seed, map[string]string{ + "workload-data/db.txt": "schema=v0.4.0", + }) + if err := os.WriteFile(meta, + []byte(`{"schemaVersion":1,"seed_sha256":"sha256:fake"}`), 0o644); err != nil { + t.Fatal(err) + } + + stdout, stderr, exit := runSeedCheck(t, mount, seed, meta) + if exit != 0 { + t.Fatalf("exit: got %d, want 0; stdout=%q stderr=%q", exit, stdout, stderr) + } + body, err := os.ReadFile(filepath.Join(mount, "workload-data/db.txt")) + if err != nil { + t.Fatalf("seed file should be extracted: %v", err) + } + if string(body) != "schema=v0.4.0" { + t.Errorf("extracted body: got %q, want schema=v0.4.0", body) + } + markerBody, err := os.ReadFile(filepath.Join(mount, ".y-cluster-seeded")) + if err != nil { + t.Fatalf("marker should be written: %v", err) + } + if !strings.Contains(string(markerBody), "seed_sha256") { + t.Errorf("marker should contain seed metadata: %s", markerBody) + } +} + +// TestSeedCheck_NonEmptyNoMarker_Conflict pins the safety belt: +// customer drive has unrelated data, no marker -> we refuse and +// exit non-zero so the k3s drop-in blocks startup. +func TestSeedCheck_NonEmptyNoMarker_Conflict(t *testing.T) { + dir := t.TempDir() + mount := filepath.Join(dir, "mount") + if err := os.MkdirAll(mount, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(mount, "customer-stuff.txt"), + []byte("not ours"), 0o644); err != nil { + t.Fatal(err) + } + + seed := filepath.Join(dir, "seed.tar.zst") + meta := filepath.Join(dir, "seed.meta.json") + makeSeedTar(t, seed, map[string]string{"x": "y"}) + if err := os.WriteFile(meta, []byte(`{"schemaVersion":1}`), 0o644); err != nil { + t.Fatal(err) + } + + stdout, stderr, exit := runSeedCheck(t, mount, seed, meta) + if exit == 0 { + t.Errorf("exit: got 0, want non-zero (conflict); stdout=%q", stdout) + } + if !strings.Contains(stderr, "refusing to seed") { + t.Errorf("stderr should mention refusal: %s", stderr) + } + if !strings.Contains(stderr, "Resolution") { + t.Errorf("stderr should include recovery recipes: %s", stderr) + } + // Customer file must be untouched. + body, _ := os.ReadFile(filepath.Join(mount, "customer-stuff.txt")) + if string(body) != "not ours" { + t.Errorf("customer file mutated: %q", body) + } + // Marker must NOT have been written. + if _, err := os.Stat(filepath.Join(mount, ".y-cluster-seeded")); err == nil { + t.Errorf("marker should not exist after conflict") + } +} + +// TestSeedCheck_LostFoundIgnored covers the freshly-formatted ext4 +// case: lost+found exists but isn't customer data. +func TestSeedCheck_LostFoundIgnored(t *testing.T) { + dir := t.TempDir() + mount := filepath.Join(dir, "mount") + if err := os.MkdirAll(filepath.Join(mount, "lost+found"), 0o755); err != nil { + t.Fatal(err) + } + seed := filepath.Join(dir, "seed.tar.zst") + meta := filepath.Join(dir, "seed.meta.json") + makeSeedTar(t, seed, map[string]string{"hello.txt": "world"}) + if err := os.WriteFile(meta, []byte(`{"schemaVersion":1}`), 0o644); err != nil { + t.Fatal(err) + } + + _, _, exit := runSeedCheck(t, mount, seed, meta) + if exit != 0 { + t.Errorf("lost+found should be ignored; exit=%d", exit) + } +} + +// TestWriteSeedMeta_RoundTrip pins the JSON shape since it's the +// on-disk schema the customer's marker carries forward. +func TestWriteSeedMeta_RoundTrip(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "meta.json") + if err := writeSeedMeta(path, dataSeedMeta{ + SchemaVersion: SeedMetaSchemaVersion, + SeededAt: "2026-05-04T12:30:00Z", + SeededBy: "y-cluster v0.4.0 (abc1234)", + ApplianceName: "appliance-test", + SeedSHA256: "sha256:c7e3", + }); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `"schemaVersion": 1`, + `"seed_sha256": "sha256:c7e3"`, + `"appliance_name": "appliance-test"`, + `"seeded_by": "y-cluster v0.4.0 (abc1234)"`, + } { + if !strings.Contains(string(data), want) { + t.Errorf("meta missing %q:\n%s", want, data) + } + } +} diff --git a/pkg/provision/qemu/export.go b/pkg/provision/qemu/export.go new file mode 100644 index 0000000..74681c1 --- /dev/null +++ b/pkg/provision/qemu/export.go @@ -0,0 +1,514 @@ +package qemu + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + + "go.uber.org/zap" +) + +// Format is the on-disk format the customer's bundle ships in. +// The format is required (no default) until we know which one +// becomes the canonical handoff -- different customers will +// import on different hypervisors. +type Format string + +const ( + // FormatQcow2 produces a self-contained qcow2 (no backing + // file). Recommended for QEMU/KVM/libvirt/Proxmox/oVirt. + FormatQcow2 Format = "qcow2" + // FormatRaw produces a raw disk image. Universal: dd-able to + // bare metal, importable by VirtualBox / VMware / most cloud + // providers' "import disk" flows. + FormatRaw Format = "raw" + // FormatVMDK produces a VMDK. The default subformat is + // streamOptimized, which ESXi accepts natively via the + // datastore browser and VMware Workstation imports via + // "Open" / attach-existing-disk. Operators targeting + // VirtualBox should pass --vmdk-subformat=monolithicSparse + // (VirtualBox's "Use Existing Virtual Hard Disk File" is + // happier with monolithicSparse than streamOptimized). + // All qemu-img VMDK subformats are valid: + // monolithicSparse, monolithicFlat, twoGbMaxExtentSparse, + // twoGbMaxExtentFlat, streamOptimized. + FormatVMDK Format = "vmdk" + // FormatOVA produces a single .ova file: an uncompressed + // tar containing an OVF descriptor + a streamOptimized + // VMDK. VirtualBox accepts it via File -> Import Appliance + // (which only takes OVF/OVA, not raw VMDK). VMware + // Workstation / Fusion / ESXi (via vSphere Client) accept + // it through the same import path. The customer drops the + // .ova in, picks CPU/RAM/network on the import wizard, and + // the bundled OVF carries sensible defaults for them. + FormatOVA Format = "ova" + // FormatGCPTar produces a gzip-compressed tar containing + // exactly one member named `disk.raw` -- the on-the-wire + // shape Google Compute Engine accepts as a custom image + // source. Upload to GCS, then `gcloud compute images + // create --source-uri=gs://bucket/.tar.gz` ingests + // it directly with no managed conversion job. The single + // member name is mandated by GCE; deviating breaks the + // import. + FormatGCPTar Format = "gcp-tar" +) + +// VMDKSubformatDefault is the subformat used when --vmdk-subformat +// is not set. streamOptimized is what VMware ESXi expects out of +// the box; we keep it as the default so the historical "y-cluster +// export --format=vmdk" shape still produces an ESXi-importable +// disk. +const VMDKSubformatDefault = "streamOptimized" + +// AllVMDKSubformats lists every VMDK subformat qemu-img accepts. +// Used to validate the --vmdk-subformat flag. +func AllVMDKSubformats() []string { + return []string{ + "streamOptimized", + "monolithicSparse", + "monolithicFlat", + "twoGbMaxExtentSparse", + "twoGbMaxExtentFlat", + } +} + +// AllFormats lists every Format the export subcommand accepts. +// Used by cobra's flag validation and the "unknown format" error +// message so help output and error stay in sync. +func AllFormats() []string { + return []string{string(FormatQcow2), string(FormatRaw), string(FormatVMDK), string(FormatOVA), string(FormatGCPTar)} +} + +// extensionFor returns the on-disk filename extension for the +// given format. Convention: .qcow2 for qcow2, .img for raw (the +// most widely-recognised raw extension), .vmdk for vmdk, .ova +// for ova. +func extensionFor(f Format) (string, error) { + switch f { + case FormatQcow2: + return ".qcow2", nil + case FormatRaw: + return ".img", nil + case FormatVMDK: + return ".vmdk", nil + case FormatOVA: + return ".ova", nil + case FormatGCPTar: + return ".tar.gz", nil + } + return "", fmt.Errorf("unsupported format %q (want one of: %v)", f, AllFormats()) +} + +// qemuImgConvertArgs returns the format-specific args to pass +// after `-f qcow2` and before the source/dest paths. VMDK gets a +// `-o subformat=...` so the resulting file matches the target +// hypervisor's import flow (streamOptimized for ESXi by default, +// monolithicSparse for VirtualBox). vmdkSubformat is ignored for +// non-VMDK formats; an empty value falls back to +// VMDKSubformatDefault. +func qemuImgConvertArgs(f Format, vmdkSubformat string) []string { + switch f { + case FormatVMDK: + if vmdkSubformat == "" { + vmdkSubformat = VMDKSubformatDefault + } + return []string{"-O", "vmdk", "-o", "subformat=" + vmdkSubformat} + default: + return []string{"-O", string(f)} + } +} + +// ExportOptions controls Export. Required: CacheDir, Name, +// BundleDir, Format. VMDKSubformat applies only when +// Format=FormatVMDK; empty falls back to VMDKSubformatDefault. +// Logger optional. +type ExportOptions struct { + CacheDir string + Name string + BundleDir string + Format Format + VMDKSubformat string + Logger *zap.Logger +} + +// Export writes a customer-handoff bundle to opts.BundleDir. +// Layout: +// +// / +// .qcow2 (or .img) -- self-contained disk, no backing file +// -ssh -- SSH private key, mode 0600 +// -ssh.pub -- SSH public key, mode 0644 +// README.md -- boot + ssh instructions +// +// The disk is flattened via `qemu-img convert`, so the bundle has +// no reference to y-cluster's local cloud-image cache. The +// customer can scp the directory to their host and follow +// README.md from there. +// +// Refuses to overwrite a non-empty bundle directory: the customer +// handoff is precious, force the operator to remove the dir +// first if they really mean to re-export. The cluster must be +// stopped (export needs a quiesced disk; running qemu locks the +// qcow2 anyway). +func Export(ctx context.Context, opts ExportOptions) error { + logger := opts.Logger + if logger == nil { + logger = zap.NewNop() + } + + ext, err := extensionFor(opts.Format) + if err != nil { + return err + } + if opts.Format == FormatVMDK && opts.VMDKSubformat != "" { + valid := false + for _, s := range AllVMDKSubformats() { + if s == opts.VMDKSubformat { + valid = true + break + } + } + if !valid { + return fmt.Errorf("unsupported vmdk subformat %q (want one of: %v)", opts.VMDKSubformat, AllVMDKSubformats()) + } + } + + cfg, err := loadState(opts.CacheDir, opts.Name) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("no saved state for %q in %s; run `y-cluster provision` first", opts.Name, opts.CacheDir) + } + return fmt.Errorf("load state: %w", err) + } + if running, _ := cfg.IsRunning(); running { + return fmt.Errorf("VM %q is running; run `y-cluster stop` first (export needs a quiesced disk)", opts.Name) + } + + if entries, err := os.ReadDir(opts.BundleDir); err == nil && len(entries) > 0 { + return fmt.Errorf("bundle directory %s already has contents; remove it before re-exporting", opts.BundleDir) + } + if err := os.MkdirAll(opts.BundleDir, 0o755); err != nil { + return fmt.Errorf("create bundle dir: %w", err) + } + + diskSrc := filepath.Join(cfg.CacheDir, cfg.Name+".qcow2") + if _, err := os.Stat(diskSrc); err != nil { + return fmt.Errorf("source disk %s not found: %w", diskSrc, err) + } + diskDst := filepath.Join(opts.BundleDir, opts.Name+ext) + + logger.Info("converting disk", + zap.String("format", string(opts.Format)), + zap.String("src", diskSrc), + zap.String("dst", diskDst), + ) + switch opts.Format { + case FormatOVA: + // writeOVA owns its own qemu-img convert (always + // streamOptimized -- the only VMDK subformat the OVF + // disk-format URI references) plus the OVF descriptor + // and the ordered tar. Skip the generic convert path + // below; the .ova IS the disk artefact. + if err := writeOVA(ctx, diskSrc, diskDst, cfg); err != nil { + return fmt.Errorf("write ova: %w", err) + } + case FormatGCPTar: + // writeGCPTar pipes qemu-img convert -O raw straight + // into a gzip writer wrapping a tar writer that emits + // a single member literally named `disk.raw`. The + // pipe avoids materialising the full 20 GiB raw + // expansion on disk. + if err := writeGCPTar(ctx, diskSrc, diskDst); err != nil { + return fmt.Errorf("write gcp-tar: %w", err) + } + default: + convertArgs := append([]string{"convert", "-f", "qcow2"}, qemuImgConvertArgs(opts.Format, opts.VMDKSubformat)...) + convertArgs = append(convertArgs, diskSrc, diskDst) + cmd := exec.CommandContext(ctx, "qemu-img", convertArgs...) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("qemu-img convert: %s: %w", out, err) + } + } + + keySrc := filepath.Join(cfg.CacheDir, cfg.Name+"-ssh") + if err := copyKeyPair(keySrc, filepath.Join(opts.BundleDir, opts.Name+"-ssh")); err != nil { + return fmt.Errorf("copy keypair: %w", err) + } + + vmdkSub := opts.VMDKSubformat + if vmdkSub == "" { + vmdkSub = VMDKSubformatDefault + } + readme := renderBundleReadme(cfg, opts.Format, ext, vmdkSub) + if err := os.WriteFile(filepath.Join(opts.BundleDir, "README.md"), []byte(readme), 0o644); err != nil { + return fmt.Errorf("write README: %w", err) + } + + logger.Info("export complete", zap.String("bundle", opts.BundleDir)) + return nil +} + +// copyKeyPair copies -> at 0600 and .pub -> +// .pub at 0644. +func copyKeyPair(src, dst string) error { + if err := copyFile(src, dst, 0o600); err != nil { + return fmt.Errorf("private key: %w", err) + } + if err := copyFile(src+".pub", dst+".pub", 0o644); err != nil { + return fmt.Errorf("public key: %w", err) + } + return nil +} + +// copyFile copies src -> dst with the given mode. Truncates dst +// if it exists. The mode is reapplied via Chmod after the write +// because OpenFile's mode arg is masked by umask. +func copyFile(src, dst string, mode os.FileMode) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return err + } + defer out.Close() + if _, err := io.Copy(out, in); err != nil { + return err + } + return out.Chmod(mode) +} + +// renderBundleReadme renders the customer-facing README. Pure +// function so unit tests can pin the rendered shape (boot +// command per format, ssh user, port forwards) without writing +// to disk. +func renderBundleReadme(cfg Config, format Format, ext, vmdkSubformat string) string { + diskFile := cfg.Name + ext + keyFile := cfg.Name + "-ssh" + + var bootSection string + switch format { + case FormatQcow2: + bootSection = fmt.Sprintf(`### Boot via QEMU/KVM + + qemu-system-x86_64 \ + -name %s \ + -machine accel=kvm -cpu host \ + -smp %s -m %s \ + -drive file=%s,format=qcow2,if=virtio \ + -netdev user,id=n0,hostfwd=tcp::8080-:80,hostfwd=tcp::2222-:22 \ + -device virtio-net-pci,netdev=n0 \ + -display none -daemonize \ + -pidfile %s.pid + +Or import the qcow2 into your hypervisor (libvirt / Proxmox / +oVirt all accept qcow2 directly).`, + cfg.Name, cfg.CPUs, cfg.Memory, diskFile, cfg.Name) + case FormatVMDK: + bootSection = fmt.Sprintf(`### Bundled VMDK subformat: %s + +(Re-export with `+"`y-cluster export --format=vmdk --vmdk-subformat=...`"+` +to pick a different one. streamOptimized is the ESXi default; +monolithicSparse is the VirtualBox-friendly choice.) + +### Import into VMware ESXi + +Upload %s into a datastore via the vSphere Client / ESXi Host +Client (Browse Datastore -> Upload), then create a new VM and +attach it as the existing virtual disk. + +### Import into VMware Workstation / Fusion + +File -> Open -> select %s. Workstation will create a wrapper +VMX around the disk; adjust CPU / memory to match the +original (%s vCPU, %s MiB RAM) and add a NAT or bridged NIC. + +### Import into VirtualBox + +VirtualBox accepts streamOptimized VMDK via Tools -> Import +Appliance, but in many cases a plain monolithicSparse VMDK +imports more cleanly. Convert if needed: + + qemu-img convert -f vmdk -O vmdk -o subformat=monolithicSparse \ + %s %s + +The bundled VMDK is %s.`, + vmdkSubformat, + diskFile, diskFile, cfg.CPUs, cfg.Memory, + diskFile, cfg.Name+"-monolithic.vmdk", + vmdkSubformat) + case FormatOVA: + bootSection = fmt.Sprintf(`### Import into VirtualBox + +File -> Import Appliance -> select %s. VirtualBox reads the +embedded OVF descriptor (CPU=%s, RAM=%s MiB, NAT NIC, single +SATA disk) and creates a new VM around the bundled +streamOptimized VMDK. + +After import, edit the VM's Network -> Adapter 1 -> Advanced +-> Port Forwarding to add: + Name=ssh Protocol=TCP Host Port=2222 Guest Port=22 + Name=http Protocol=TCP Host Port=8080 Guest Port=80 + Name=https Protocol=TCP Host Port=8443 Guest Port=443 + +### Import into VMware Workstation / Fusion + +File -> Open -> select %s. Workstation parses the OVF and +materialises a wrapper VMX. Adjust port forwarding via the +NAT settings in Edit -> Virtual Network Editor. + +### Import into VMware ESXi + +vSphere Client -> Deploy OVF Template -> select %s. ESXi +honours the OVF descriptor's CPU / memory hints and the +streamOptimized VMDK is the canonical disk shape for this +import path. + +### Inspect / convert + +The .ova is a plain (uncompressed) tar of two files; if you +want to crack it open: + + tar tvf %s + tar xvf %s # extracts .ovf and .vmdk`, + diskFile, cfg.CPUs, cfg.Memory, + diskFile, + diskFile, + diskFile, diskFile) + case FormatRaw: + bootSection = fmt.Sprintf(`### Boot via QEMU/KVM (universal raw) + + qemu-system-x86_64 \ + -name %s \ + -machine accel=kvm -cpu host \ + -smp %s -m %s \ + -drive file=%s,format=raw,if=virtio \ + -netdev user,id=n0,hostfwd=tcp::8080-:80,hostfwd=tcp::2222-:22 \ + -device virtio-net-pci,netdev=n0 \ + -display none -daemonize \ + -pidfile %s.pid + +### Or dd to a block device (bare-metal install) + + sudo dd if=%s of=/dev/sdX bs=4M status=progress conv=fsync + +(Replace /dev/sdX with the target disk -- DESTRUCTIVE.) + +### Or import as a virtual disk + +VirtualBox / VMware Workstation / Proxmox accept .img files via +their disk-attach UI; some prefer the .raw extension -- rename +if your hypervisor doesn't recognise .img.`, + cfg.Name, cfg.CPUs, cfg.Memory, diskFile, cfg.Name, diskFile) + } + + return fmt.Sprintf(`# y-cluster appliance bundle + +Source cluster: %s + +## Contents + +| File | Purpose | +| ---- | ------- | +| %s | Disk image (%s, self-contained, no external backing file) | +| %s | SSH private key (mode 0600) | +| %s.pub | SSH public key | +| README.md | This file | + +## Boot + +%s + +After boot the VM hosts a single-node Kubernetes cluster (k3s) +with the application preinstalled. The application starts +automatically. + +## SSH access + + ssh -i %s -p 2222 ystack@127.0.0.1 + +(Adjust -p to whatever port you forwarded to guest 22.) The +ystack user has passwordless sudo. + +## kubectl access (optional, for inspection) + + ssh -i %s -p 2222 ystack@127.0.0.1 sudo cat /etc/rancher/k3s/k3s.yaml \ + | sed 's|server: .*|server: https://127.0.0.1:6443|' > k3s.yaml + KUBECONFIG=k3s.yaml kubectl get nodes + +The default boot command above does not forward 6443 -- add +` + "`hostfwd=tcp::6443-:6443`" + ` to the netdev to expose the +apiserver to the host. + +## Persistent storage + +The appliance ships y-cluster's bundled local-path-provisioner +(replaces k3s's stock local-storage). Stateful workloads with +PersistentVolumeClaims against the default StorageClass +` + "`local-path`" + ` end up under ` + "`/data/yolean/`" + ` on +the appliance disk, named ` + "`_`" + ` +(e.g. ` + "`/data/yolean/appliance-stateful_data-versitygw-0/`" + `). +The reclaim policy is ` + "`Retain`" + ` -- a stray +` + "`kubectl delete pvc`" + ` does NOT wipe the data; the +directory persists and the next PVC of the same +namespace+name picks it back up. + +This is the appliance-upgrade story: ship a new appliance +disk, the customer's data stays at /data/yolean (whether on +the appliance disk or on a separately-mounted disk; see +below), and re-creating the same PVC names binds the same +data automatically. + +### Optional: separate data disk + +For workloads that need substantially more storage, or +separation between the OS and data devices, attach a second +virtual disk to the VM and mount it at /data/yolean: + +1. Shut the VM down (your hypervisor's ACPI poweroff). +2. Attach a second virtual disk via your hypervisor. +3. Boot the VM. Format the new device (DESTRUCTIVE): + + sudo systemctl stop k3s + sudo mkfs.ext4 /dev/vdb + sudo rsync -aAX /data/yolean/ /mnt/new/ # if PVs already exist + echo '/dev/vdb /data/yolean ext4 defaults,nofail 0 2' \ + | sudo tee -a /etc/fstab + sudo mount /data/yolean + sudo systemctl start k3s + +4. Existing PVs (named ` + "`_`" + `) are now on + the new disk; PV bindings are unchanged. New PVCs land on + the same path on the new disk. + +### Custom storage path or pattern + +The y-cluster-provision.yaml ` + "`storage`" + ` block overrides +all three knobs (path / pathPattern / reclaimPolicy): + + storage: + path: /mnt/customer-data + pathPattern: "{{ .PVC.Namespace }}/{{ .PVC.Name }}-{{ .PVName }}" + reclaimPolicy: Delete + +(.PVName expands to ` + "`pvc-`" + ` -- the upstream +local-path-provisioner shape, useful when you want unique +per-PV directories that survive PVC delete+recreate without +inheriting the previous PV's data.) + +Defaults are picked for the per-customer appliance: predictable +namespace_name path so an upgrade rebinds by name, and Retain +so accidental deletes don't lose data. +`, + cfg.Name, + diskFile, format, + keyFile, keyFile, + bootSection, + keyFile, keyFile, + ) +} diff --git a/pkg/provision/qemu/export_test.go b/pkg/provision/qemu/export_test.go new file mode 100644 index 0000000..99456d4 --- /dev/null +++ b/pkg/provision/qemu/export_test.go @@ -0,0 +1,344 @@ +package qemu + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestAllFormats pins the formats the export subcommand accepts. +// Adding qcow2 / raw to AllFormats but forgetting to wire them +// through extensionFor would break Export silently otherwise. +func TestAllFormats(t *testing.T) { + formats := AllFormats() + for _, f := range formats { + if _, err := extensionFor(Format(f)); err != nil { + t.Errorf("AllFormats lists %q but extensionFor rejects it: %v", f, err) + } + } +} + +// TestExtensionFor pins the on-disk filename extensions per +// format. +func TestExtensionFor(t *testing.T) { + cases := []struct { + f Format + want string + }{ + {FormatQcow2, ".qcow2"}, + {FormatRaw, ".img"}, + {FormatVMDK, ".vmdk"}, + {FormatOVA, ".ova"}, + {FormatGCPTar, ".tar.gz"}, + } + for _, c := range cases { + got, err := extensionFor(c.f) + if err != nil { + t.Errorf("extensionFor(%q): %v", c.f, err) + continue + } + if got != c.want { + t.Errorf("extensionFor(%q): got %q, want %q", c.f, got, c.want) + } + } + if _, err := extensionFor(Format("vhdx")); err == nil { + t.Errorf("extensionFor(vhdx) should error until VHDX is wired up") + } +} + +// TestQemuImgConvertArgs pins the per-format qemu-img convert +// shape. VMDK defaults to `subformat=streamOptimized` (ESXi); +// an explicit subformat overrides it. Raw / qcow2 take no +// per-format options. Drift here would silently produce a VMDK +// the target hypervisor can't import. +func TestQemuImgConvertArgs(t *testing.T) { + cases := []struct { + name string + f Format + subformat string + want []string + }{ + {"qcow2", FormatQcow2, "", []string{"-O", "qcow2"}}, + {"raw", FormatRaw, "", []string{"-O", "raw"}}, + {"vmdk default", FormatVMDK, "", []string{"-O", "vmdk", "-o", "subformat=streamOptimized"}}, + {"vmdk monolithicSparse", FormatVMDK, "monolithicSparse", []string{"-O", "vmdk", "-o", "subformat=monolithicSparse"}}, + {"vmdk subformat ignored for non-vmdk", FormatRaw, "monolithicSparse", []string{"-O", "raw"}}, + } + for _, c := range cases { + got := qemuImgConvertArgs(c.f, c.subformat) + if len(got) != len(c.want) { + t.Errorf("%s: got %v, want %v", c.name, got, c.want) + continue + } + for i := range c.want { + if got[i] != c.want[i] { + t.Errorf("%s [%d]: got %q, want %q", c.name, i, got[i], c.want[i]) + } + } + } +} + +// TestExport_RejectsUnknownVMDKSubformat surfaces a friendly +// error for typos. Only fires for FormatVMDK; other formats +// ignore the field entirely. +func TestExport_RejectsUnknownVMDKSubformat(t *testing.T) { + cacheDir := t.TempDir() + cfg := defaultedRuntimeConfig(t) + cfg.CacheDir = cacheDir + if err := saveState(cfg); err != nil { + t.Fatal(err) + } + err := Export(context.Background(), ExportOptions{ + CacheDir: cacheDir, + Name: cfg.Name, + BundleDir: filepath.Join(t.TempDir(), "bundle"), + Format: FormatVMDK, + VMDKSubformat: "monolithic-sparse", + }) + if err == nil { + t.Fatal("expected error for unsupported vmdk subformat") + } + if !strings.Contains(err.Error(), "unsupported vmdk subformat") { + t.Errorf("error should mention unsupported subformat: %v", err) + } +} + +// TestExport_RejectsUnknownFormat surfaces the friendly error. +// Uses "vhdx" deliberately -- it's a format we plan to add but +// haven't, so it stays "unknown" until that work lands and is a +// natural canary for "drop the unknown-format error before +// extensionFor knows the new format". +func TestExport_RejectsUnknownFormat(t *testing.T) { + cacheDir := t.TempDir() + cfg := defaultedRuntimeConfig(t) + cfg.CacheDir = cacheDir + if err := saveState(cfg); err != nil { + t.Fatal(err) + } + err := Export(context.Background(), ExportOptions{ + CacheDir: cacheDir, + Name: cfg.Name, + BundleDir: filepath.Join(t.TempDir(), "bundle"), + Format: Format("vhdx"), + }) + if err == nil { + t.Fatal("expected error for unsupported format") + } + if !strings.Contains(err.Error(), "unsupported format") { + t.Errorf("error should mention unsupported format: %v", err) + } +} + +// TestExport_RejectsMissingState exercises the "no saved state" +// branch. Operator should be told to run provision, not get a +// cryptic os.IsNotExist. +func TestExport_RejectsMissingState(t *testing.T) { + err := Export(context.Background(), ExportOptions{ + CacheDir: t.TempDir(), + Name: "missing", + BundleDir: filepath.Join(t.TempDir(), "bundle"), + Format: FormatQcow2, + }) + if err == nil { + t.Fatal("expected error when no saved state exists") + } + if !strings.Contains(err.Error(), "y-cluster provision") { + t.Errorf("error should hint at provision: %v", err) + } +} + +// TestExport_RejectsRunningCluster exercises the IsRunning check. +// Exporting while qemu is alive would race the qcow2 against +// in-flight writes. +func TestExport_RejectsRunningCluster(t *testing.T) { + cacheDir := t.TempDir() + cfg := defaultedRuntimeConfig(t) + cfg.CacheDir = cacheDir + if err := saveState(cfg); err != nil { + t.Fatal(err) + } + pidFile := filepath.Join(cacheDir, cfg.Name+".pid") + if err := os.WriteFile(pidFile, []byte("1\n"), 0o644); err != nil { + t.Fatal(err) + } + err := Export(context.Background(), ExportOptions{ + CacheDir: cacheDir, + Name: cfg.Name, + BundleDir: filepath.Join(t.TempDir(), "bundle"), + Format: FormatQcow2, + }) + if err == nil { + t.Fatal("expected error when VM still running") + } + if !strings.Contains(err.Error(), "y-cluster stop") { + t.Errorf("error should hint at stop: %v", err) + } +} + +// TestExport_RejectsNonEmptyBundleDir is the precious-handoff +// guard. We never overwrite a customer's bundle silently; force +// the operator to remove the dir first. +func TestExport_RejectsNonEmptyBundleDir(t *testing.T) { + cacheDir := t.TempDir() + cfg := defaultedRuntimeConfig(t) + cfg.CacheDir = cacheDir + if err := saveState(cfg); err != nil { + t.Fatal(err) + } + bundleDir := t.TempDir() + if err := os.WriteFile(filepath.Join(bundleDir, "leftover"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + err := Export(context.Background(), ExportOptions{ + CacheDir: cacheDir, + Name: cfg.Name, + BundleDir: bundleDir, + Format: FormatQcow2, + }) + if err == nil { + t.Fatal("expected error when bundle dir already has contents") + } + if !strings.Contains(err.Error(), "remove it") { + t.Errorf("error should hint at removing the dir: %v", err) + } +} + +// TestRenderBundleReadme_Qcow2 pins the README's qcow2 boot +// shape: name, file, port forwards, ssh command. Drift here = +// drift in what we tell the customer. +func TestRenderBundleReadme_Qcow2(t *testing.T) { + cfg := Config{Name: "acme", CPUs: "4", Memory: "8192"} + body := renderBundleReadme(cfg, FormatQcow2, ".qcow2", "") + for _, want := range []string{ + "y-cluster appliance bundle", + "Source cluster: acme", + "acme.qcow2", + "acme-ssh", + "format=qcow2", + "-smp 4", + "-m 8192", + "hostfwd=tcp::8080-:80", + "hostfwd=tcp::2222-:22", + "ssh -i acme-ssh -p 2222 ystack@127.0.0.1", + } { + if !strings.Contains(body, want) { + t.Errorf("README missing %q:\n%s", want, body) + } + } +} + +// TestRenderBundleReadme_Raw pins the raw-format bonus sections +// (dd to /dev/sdX, hypervisor-import notes). +func TestRenderBundleReadme_Raw(t *testing.T) { + cfg := Config{Name: "acme", CPUs: "2", Memory: "4096"} + body := renderBundleReadme(cfg, FormatRaw, ".img", "") + for _, want := range []string{ + "acme.img", + "format=raw", + "dd if=acme.img", + "of=/dev/sdX", + "DESTRUCTIVE", + } { + if !strings.Contains(body, want) { + t.Errorf("raw README missing %q:\n%s", want, body) + } + } +} + +// TestRenderBundleReadme_VMDK pins the VMDK-specific guidance +// (ESXi datastore upload, VirtualBox subformat conversion) for +// the default streamOptimized subformat. +func TestRenderBundleReadme_VMDK(t *testing.T) { + cfg := Config{Name: "acme", CPUs: "2", Memory: "4096"} + body := renderBundleReadme(cfg, FormatVMDK, ".vmdk", "streamOptimized") + for _, want := range []string{ + "acme.vmdk", + "VMware ESXi", + "datastore", + "VMware Workstation", + "VirtualBox", + "streamOptimized", + "monolithicSparse", + "Bundled VMDK subformat: streamOptimized", + } { + if !strings.Contains(body, want) { + t.Errorf("vmdk README missing %q:\n%s", want, body) + } + } +} + +// TestRenderBundleReadme_VMDKMonolithicSparse pins that the +// README reflects the actual bundled subformat, not just the +// ESXi default. +func TestRenderBundleReadme_VMDKMonolithicSparse(t *testing.T) { + cfg := Config{Name: "acme", CPUs: "2", Memory: "4096"} + body := renderBundleReadme(cfg, FormatVMDK, ".vmdk", "monolithicSparse") + if !strings.Contains(body, "Bundled VMDK subformat: monolithicSparse") { + t.Errorf("README should report bundled subformat=monolithicSparse:\n%s", body) + } +} + +// TestRenderBundleReadme_OVA pins the OVA-format guidance: +// File -> Import Appliance is the only path that works +// (VirtualBox refuses raw VMDK via that wizard) and the +// per-hypervisor sections must mention CPU/RAM hints baked +// into the OVF descriptor. +func TestRenderBundleReadme_OVA(t *testing.T) { + cfg := Config{Name: "acme", CPUs: "2", Memory: "4096"} + body := renderBundleReadme(cfg, FormatOVA, ".ova", "") + for _, want := range []string{ + "acme.ova", + "VirtualBox", + "File -> Import Appliance", + "Port Forwarding", + "VMware Workstation", + "VMware ESXi", + "OVF", + } { + if !strings.Contains(body, want) { + t.Errorf("ova README missing %q:\n%s", want, body) + } + } +} + +// TestRenderOVF pins the OVF descriptor shape: descriptor +// references the .vmdk by name, capacity matches the qcow2 +// virtual size, CPU/RAM come from the cluster config, and the +// VirtualSystemType is virtualbox-2.2 (the value VirtualBox +// honours; VMware ignores it). Drift here = customer can't +// import the OVA cleanly. +func TestRenderOVF(t *testing.T) { + cfg := Config{Name: "acme", CPUs: "2", Memory: "4096"} + body := renderOVF(cfg, 21474836480, 1500000000) + for _, want := range []string{ + `ovf:href="acme.vmdk"`, + `ovf:size="1500000000"`, + `ovf:capacity="21474836480"`, + `http://www.vmware.com/interfaces/specifications/vmdk.html#streamOptimized`, + `acme`, + `virtualbox-2.2`, + `2`, // CPU + `4096`, // memory + } { + if !strings.Contains(body, want) { + t.Errorf("OVF missing %q:\n%s", want, body) + } + } +} + +// TestRenderOVF_EscapesName guards against an operator-supplied +// cluster name with XML-special chars breaking the descriptor +// (the cobra layer rejects most of these but defense in depth +// is cheap and matches what html.EscapeString gives us). +func TestRenderOVF_EscapesName(t *testing.T) { + cfg := Config{Name: "a&b", CPUs: "2", Memory: "4096"} + body := renderOVF(cfg, 1, 1) + if !strings.Contains(body, "a&b<c>") { + t.Errorf("OVF should XML-escape cluster name, got:\n%s", body) + } + if strings.Contains(body, "a&b") { + t.Errorf("OVF still contains raw special chars:\n%s", body) + } +} diff --git a/pkg/provision/qemu/gcp_tar.go b/pkg/provision/qemu/gcp_tar.go new file mode 100644 index 0000000..ded608e --- /dev/null +++ b/pkg/provision/qemu/gcp_tar.go @@ -0,0 +1,107 @@ +package qemu + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" +) + +// writeGCPTar produces a gzip-compressed tar at outPath +// containing exactly one member named `disk.raw`. This is the +// on-the-wire shape Google Compute Engine accepts as a custom +// image source: upload to GCS, then `gcloud compute images +// create --source-uri=gs://bucket/.tar.gz` ingests it +// directly. The single member name is mandated by GCE -- any +// other layout makes the image-create call fail with an +// opaque "no disk.raw found in tarball" error. +// +// Implementation detail: we'd love to pipe `qemu-img convert +// -O raw - /dev/stdout` straight into the tar/gzip stream, +// but qemu-img's raw output driver calls ftruncate() at the +// end to seal the size. ftruncate fails on a pipe, so we +// materialise the raw expansion in a temp dir alongside the +// final .tar.gz and stream from there. Tmpdir is on the +// bundle's chosen output volume (NOT /tmp), since the raw is +// the full virtual size (~20 GiB for our appliance) and /tmp +// is tmpfs on most distros. +func writeGCPTar(ctx context.Context, qcow2Src, outPath string) error { + tmpDir, err := os.MkdirTemp(filepath.Dir(outPath), ".yc-gcp-tar-") + if err != nil { + return fmt.Errorf("gcp-tar tmp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + rawPath := filepath.Join(tmpDir, "disk.raw") + convert := exec.CommandContext(ctx, "qemu-img", "convert", + "-f", "qcow2", "-O", "raw", + qcow2Src, rawPath, + ) + if out, err := convert.CombinedOutput(); err != nil { + return fmt.Errorf("qemu-img convert (gcp-tar): %s: %w", out, err) + } + + rawInfo, err := os.Stat(rawPath) + if err != nil { + return fmt.Errorf("stat raw: %w", err) + } + + rawFile, err := os.Open(rawPath) + if err != nil { + return fmt.Errorf("open raw: %w", err) + } + defer rawFile.Close() + + out, err := os.Create(outPath) + if err != nil { + return fmt.Errorf("create gcp tar: %w", err) + } + defer out.Close() + + gzw, err := gzip.NewWriterLevel(out, gzip.BestSpeed) + if err != nil { + return fmt.Errorf("gzip writer: %w", err) + } + tw := tar.NewWriter(gzw) + + // Force GNU tar format. For files >8 GiB (our 20 GiB + // raw is well above this), the default tar format in + // Go's archive/tar emits a PAX extended-header tarball + // member ahead of the actual file. GCE's image parser + // expects a SINGLE member literally named `disk.raw` + // and treats the PaxHeaders entry as "not disk.raw", + // rejecting the upload with the unhelpful "The tar + // archive is not a valid image." GNU format encodes + // large sizes inline in the header (base-256 numeric + // fields) -- no separate member, no rejection. + if err := tw.WriteHeader(&tar.Header{ + Name: "disk.raw", + Mode: 0o644, + Size: rawInfo.Size(), + Typeflag: tar.TypeReg, + Format: tar.FormatGNU, + }); err != nil { + _ = tw.Close() + _ = gzw.Close() + return fmt.Errorf("tar header: %w", err) + } + + if _, err := io.Copy(tw, rawFile); err != nil { + _ = tw.Close() + _ = gzw.Close() + return fmt.Errorf("stream raw -> tar: %w", err) + } + + if err := tw.Close(); err != nil { + _ = gzw.Close() + return fmt.Errorf("close tar: %w", err) + } + if err := gzw.Close(); err != nil { + return fmt.Errorf("close gzip: %w", err) + } + return nil +} diff --git a/pkg/provision/qemu/gcp_tar_test.go b/pkg/provision/qemu/gcp_tar_test.go new file mode 100644 index 0000000..d6d352e --- /dev/null +++ b/pkg/provision/qemu/gcp_tar_test.go @@ -0,0 +1,103 @@ +package qemu + +import ( + "archive/tar" + "compress/gzip" + "context" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// TestWriteGCPTar exercises the qcow2 -> tar+gzip(disk.raw) +// pipeline against a real qemu-img-built source. Tarball must +// contain exactly one member named `disk.raw`, and that +// member's size must equal the qcow2's virtual size (qemu-img +// always pads to virtual size on raw conversion). Drift here +// breaks GCE's image-create which silently fails or imports +// a truncated boot disk. +func TestWriteGCPTar(t *testing.T) { + if _, err := exec.LookPath("qemu-img"); err != nil { + t.Skip("qemu-img not installed") + } + + // Tiny 16 MiB qcow2 source -- big enough to exercise the + // pipe and gzip frame, small enough to run in a unit + // test without flooding /tmp. + dir := t.TempDir() + qcow2 := filepath.Join(dir, "src.qcow2") + const virtSize = 16 * 1024 * 1024 + if out, err := exec.Command("qemu-img", "create", "-f", "qcow2", qcow2, "16M").CombinedOutput(); err != nil { + t.Fatalf("qemu-img create: %s: %v", out, err) + } + + tarGz := filepath.Join(dir, "out.tar.gz") + if err := writeGCPTar(context.Background(), qcow2, tarGz); err != nil { + t.Fatalf("writeGCPTar: %v", err) + } + + // Inspect the tarball. + f, err := os.Open(tarGz) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gzr, err := gzip.NewReader(f) + if err != nil { + t.Fatalf("gzip open: %v", err) + } + tr := tar.NewReader(gzr) + + hdr, err := tr.Next() + if err != nil { + t.Fatalf("tar.Next: %v", err) + } + if hdr.Name != "disk.raw" { + t.Errorf("first member name: got %q, want disk.raw", hdr.Name) + } + if hdr.Size != virtSize { + t.Errorf("disk.raw size: got %d, want %d (virtual size)", hdr.Size, int64(virtSize)) + } + + // Drain the body to ensure no corruption / truncation in + // the pipe-stream path. + n, err := io.Copy(io.Discard, tr) + if err != nil { + t.Fatalf("read disk.raw body: %v", err) + } + if n != virtSize { + t.Errorf("disk.raw body length: got %d, want %d", n, int64(virtSize)) + } + + // No second member: GCE rejects tarballs with anything + // other than disk.raw. + if _, err := tr.Next(); err != io.EOF { + t.Errorf("expected EOF after disk.raw, got %v", err) + } +} + +// TestGCPTar_FormatGNU_NoExtraMembers guards against the +// regression that surfaced as GCE "The tar archive is not a +// valid image": for files >8 GiB the default tar format (PAX) +// emits an extra PaxHeaders member ahead of disk.raw, making +// the tarball two-member from GCE's perspective. The fix is +// pinning Header.Format = tar.FormatGNU, which encodes large +// sizes inline. +// +// We can't easily synthesize a 9 GiB body in a unit test, so +// we test the inverse: a PAX-formatted small file emits two +// members IF the size triggers it, and a GNU-formatted file +// always emits one. Reading the writeGCPTar source directly +// is good enough to confirm we use FormatGNU. +func TestGCPTar_FormatGNU_NoExtraMembers(t *testing.T) { + src, err := os.ReadFile("gcp_tar.go") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(src), "tar.FormatGNU") { + t.Errorf("gcp_tar.go must pin tar.FormatGNU; PAX emits a separate header member that GCE rejects with 'tar archive is not a valid image' for disks >8 GiB") + } +} diff --git a/pkg/provision/qemu/k3s.go b/pkg/provision/qemu/k3s.go index 1b010e4..e82e7ac 100644 --- a/pkg/provision/qemu/k3s.go +++ b/pkg/provision/qemu/k3s.go @@ -55,11 +55,18 @@ func (c *Cluster) installK3s(ctx context.Context) error { // Gateway as the cluster ingress. Running both controllers // would have two consumers fighting over the host:80/:443 // forwards. +// +// `--disable=local-storage` is added because y-cluster ships +// its own local-path-provisioner (pkg/provision/localstorage) +// with appliance-shape defaults (path /data/yolean, +// PVC namespace_name pattern, Retain reclaim). k3s's deploy +// controller would otherwise reconcile our config back to the +// upstream defaults on every restart. func (c *Cluster) installK3sScript(ctx context.Context) error { cmd := fmt.Sprintf( "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s INSTALL_K3S_EXEC=%s sudo -E sh -", shellQuote(c.cfg.K3s.Version), - shellQuote("--write-kubeconfig-mode=644 --disable=traefik"), + shellQuote("--write-kubeconfig-mode=644 --disable=traefik --disable=local-storage"), ) out, err := c.SSH(ctx, cmd) if err != nil { @@ -93,7 +100,7 @@ func (c *Cluster) installK3sAirgap(ctx context.Context) error { fmt.Sprintf( "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_EXEC=%s sudo -E sh -", shellQuote(c.cfg.K3s.Version), - shellQuote("--write-kubeconfig-mode=644 --disable=traefik"), + shellQuote("--write-kubeconfig-mode=644 --disable=traefik --disable=local-storage"), ), } { out, err := c.SSH(ctx, step) diff --git a/pkg/provision/qemu/k3s_data_seed.conf b/pkg/provision/qemu/k3s_data_seed.conf new file mode 100644 index 0000000..2f6efa6 --- /dev/null +++ b/pkg/provision/qemu/k3s_data_seed.conf @@ -0,0 +1,8 @@ +[Unit] +# Block k3s on the data-seed unit. If the seed unit fails (conflict +# mode -- /data/yolean has unmarked customer data) k3s does NOT +# start; the customer SSHes in, runs y-cluster-seed-status, applies +# the recommended recovery, then `systemctl start k3s.service`. +# See specs/y-cluster/APPLIANCE_UPGRADES.md. +Requires=y-cluster-data-seed.service +After=y-cluster-data-seed.service diff --git a/pkg/provision/qemu/lifecycle.go b/pkg/provision/qemu/lifecycle.go new file mode 100644 index 0000000..0557eb8 --- /dev/null +++ b/pkg/provision/qemu/lifecycle.go @@ -0,0 +1,248 @@ +package qemu + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "go.uber.org/zap" + + "github.com/Yolean/y-cluster/pkg/kubeconfig" + "github.com/Yolean/y-cluster/pkg/sshexec" +) + +// Pause freezes the running qemu process via SIGSTOP. The VM's +// guest is paused mid-clock; no in-flight syscalls complete until +// Resume sends SIGCONT. Use for "I want CPU back for a minute" +// rather than "I'm done"; Stop is the latter. +// +// Pidfile-driven so callers don't need a *Cluster handle from the +// original Provision call -- the y-cluster CLI's `pause` subcommand +// finds the cluster via cluster.Lookup and hands its cacheDir/name +// to this function. +func Pause(cacheDir, name string, logger *zap.Logger) error { + if logger == nil { + logger = zap.NewNop() + } + pid, err := readPidFile(pidFilePath(cacheDir, name)) + if err != nil { + return err + } + logger.Info("pausing qemu VM", zap.String("name", name), zap.Int("pid", pid)) + return pidSignal(pid, syscall.SIGSTOP) +} + +// Resume sends SIGCONT to a paused qemu process. No-op when the +// process isn't paused. +func Resume(cacheDir, name string, logger *zap.Logger) error { + if logger == nil { + logger = zap.NewNop() + } + pid, err := readPidFile(pidFilePath(cacheDir, name)) + if err != nil { + return err + } + logger.Info("resuming qemu VM", zap.String("name", name), zap.Int("pid", pid)) + return pidSignal(pid, syscall.SIGCONT) +} + +// Stop gracefully shuts down a running qemu VM. Order: +// +// 1. Try to issue `sudo sync; sudo poweroff` over SSH so the +// guest's systemd-shutdown sequence flushes k3s/containerd +// state cleanly. This was the root cause of "exec format +// error" crash loops on the imported side of the appliance +// round-trip: qemu's SIGTERM exit (~200ms) drops the guest +// pagecache mid-write and containerd's overlayfs snapshot +// files end up zero-byte. +// 2. Wait up to gracefulShutdownGrace for qemu to exit on its +// own (the guest's poweroff propagates back through qemu). +// 3. Fall back to stopVM's existing SIGTERM -> SIGKILL ladder +// for the cases where SSH is unreachable (sshd not up yet, +// network broken, key changed) or the guest hangs. +// +// Disk and state sidecar are preserved so Start can resume. +// The kubeconfig context is left intact -- consumers who want +// to "permanently" stop should use Teardown. +func Stop(cacheDir, name string, logger *zap.Logger) error { + if logger == nil { + logger = zap.NewNop() + } + logger.Info("stopping qemu VM", zap.String("name", name)) + + pidFile := pidFilePath(cacheDir, name) + pid, err := readPidFile(pidFile) + if err != nil { + // No live cluster -- nothing to do. stopVM is idempotent + // and handles a missing/stale pidfile by returning nil. + return stopVM(pidFile, logger) + } + + // Best-effort graceful guest shutdown via ssh. Failures here + // are logged but not fatal; we always fall through to the + // signal ladder below. + if err := guestPoweroff(cacheDir, name, pid, logger); err != nil { + logger.Warn("graceful guest shutdown failed; falling back to qemu signals", + zap.Error(err)) + } else if !pidAlive(pid) { + _ = os.Remove(pidFile) + return nil + } + + return stopVM(pidFile, logger) +} + +// gracefulShutdownGrace caps how long Stop waits for the guest's +// poweroff to propagate to qemu exit. k3s + containerd flush +// state during the systemd shutdown sequence; 60s is generous on +// a healthy cluster, well below the 10s + 5s SIGTERM/SIGKILL +// fallback that an unhealthy cluster would hit. +var gracefulShutdownGrace = 60 * time.Second + +// guestPoweroff runs `sudo sync; sudo poweroff` over the qemu's +// host-port-forwarded ssh and then polls until the qemu pid +// exits. Errors are typed: +// - load/state errors: caller logs and falls back to signals. +// - ssh dial failures: same. +// - waitForExit timeout: caller logs and falls back to signals. +// +// The ssh command itself returns in seconds (poweroff signals +// systemd-shutdown and returns); the longer wait is for the +// guest to actually finish unmounting filesystems and qemu to +// notice the guest powered off. +func guestPoweroff(cacheDir, name string, pid int, logger *zap.Logger) error { + cfg, err := loadState(cacheDir, name) + if err != nil { + return fmt.Errorf("load state: %w", err) + } + target := sshexec.Target{ + Host: "127.0.0.1", + Port: cfg.SSHPort, + User: "ystack", + KeyPath: filepath.Join(cfg.CacheDir, cfg.Name+"-ssh"), + } + // sync first so any pending writes hit disk before systemd + // kills off the writers. poweroff is async; the command + // returns immediately and shutdown propagates. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + logger.Info("requesting graceful guest shutdown via ssh poweroff", + zap.String("name", name), zap.Int("pid", pid)) + if _, err := sshexec.Exec(ctx, target, "sudo sync; sudo poweroff", nil); err != nil { + // Some sshd configs drop the connection during the + // poweroff exec; treat that as expected-not-fatal here + // and let the wait loop decide based on pid liveness. + logger.Debug("ssh poweroff returned an error (may be expected on disconnect)", + zap.Error(err)) + } + if !waitForExit(pid, gracefulShutdownGrace) { + return fmt.Errorf("qemu pid %d still alive after %s of graceful shutdown", + pid, gracefulShutdownGrace) + } + logger.Info("qemu exited cleanly via guest poweroff", zap.Int("pid", pid)) + return nil +} + +// Start re-launches a previously-stopped qemu VM. Reads the state +// sidecar Provision wrote, invokes startVM against the existing +// qcow2 (no cloud-init re-run -- the disk already has the user +// and SSH key from first boot), waits for k3s to come back up, +// then re-imports the kubeconfig so the host-side context is +// fresh even if it was cleaned while the cluster was down. +func Start(ctx context.Context, cacheDir, name string, logger *zap.Logger) (*Cluster, error) { + if logger == nil { + logger = zap.NewNop() + } + cfg, err := loadState(cacheDir, name) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("no saved state for %q in %s; run `y-cluster provision` first", name, cacheDir) + } + return nil, fmt.Errorf("load state: %w", err) + } + + if running, pid := cfg.IsRunning(); running { + return nil, fmt.Errorf("VM %q already running (pid %d)", name, pid) + } + + diskPath := filepath.Join(cfg.CacheDir, cfg.Name+".qcow2") + if _, err := os.Stat(diskPath); err != nil { + return nil, fmt.Errorf("disk %s not found; re-provision", diskPath) + } + + kubecfg, err := kubeconfig.New(cfg.Context, clusterName(cfg.Name), logger) + if err != nil { + return nil, err + } + + c := &Cluster{ + cfg: cfg, + sshKey: filepath.Join(cfg.CacheDir, cfg.Name+"-ssh"), + pidFile: pidFilePath(cfg.CacheDir, cfg.Name), + logger: logger, + Kubeconfig: kubecfg, + } + + if err := c.startVM(ctx, diskPath, ""); err != nil { + return nil, fmt.Errorf("start VM: %w", err) + } + if err := c.waitForSSH(ctx); err != nil { + return nil, fmt.Errorf("wait for SSH: %w", err) + } + logger.Info("VM up; waiting for k3s") + if err := c.waitForK3sReady(ctx); err != nil { + return nil, fmt.Errorf("wait for k3s: %w", err) + } + + rawKubeconfig, err := c.extractKubeconfig(ctx) + if err != nil { + return nil, fmt.Errorf("extract kubeconfig: %w", err) + } + if err := kubecfg.Import(rawKubeconfig); err != nil { + return nil, fmt.Errorf("merge kubeconfig: %w", err) + } + logger.Info("k3s ready", zap.String("context", cfg.Context)) + return c, nil +} + +// pidFilePath is the canonical pidfile path used by Provision / +// Teardown / lifecycle. Mirrors the same join pattern; centralised +// so a layout change touches one spot. +func pidFilePath(cacheDir, name string) string { + return filepath.Join(cacheDir, name+".pid") +} + +// readPidFile parses the qemu pid out of pidFile and verifies the +// process is alive. Returns os.ErrNotExist when the file isn't +// there so callers can branch on errors.Is. +func readPidFile(pidFile string) (int, error) { + data, err := os.ReadFile(pidFile) + if err != nil { + return 0, err + } + var pid int + if _, err := fmt.Sscanf(strings.TrimSpace(string(data)), "%d", &pid); err != nil { + return 0, fmt.Errorf("parse %s: %w", pidFile, err) + } + if !pidAlive(pid) { + return 0, fmt.Errorf("%s: pid %d not alive (cluster stopped?)", pidFile, pid) + } + return pid, nil +} + +// pidSignal sends sig to pid. Wraps the *os.Process boilerplate +// for the SIGSTOP/SIGCONT cases Pause/Resume need. +func pidSignal(pid int, sig syscall.Signal) error { + proc, err := os.FindProcess(pid) + if err != nil { + return fmt.Errorf("find pid %d: %w", pid, err) + } + if err := proc.Signal(sig); err != nil { + return fmt.Errorf("signal pid %d: %w", pid, err) + } + return nil +} diff --git a/pkg/provision/qemu/ova.go b/pkg/provision/qemu/ova.go new file mode 100644 index 0000000..1e9c251 --- /dev/null +++ b/pkg/provision/qemu/ova.go @@ -0,0 +1,239 @@ +package qemu + +import ( + "archive/tar" + "context" + "encoding/json" + "fmt" + "html" + "io" + "os" + "os/exec" + "path/filepath" +) + +// writeOVA produces an OVA at ovaPath that VirtualBox / VMware +// Workstation accept via File -> Import Appliance. The OVA is +// an uncompressed tar containing two members, in this order: +// +// .ovf - XML descriptor (must come first per OVF 1.0 +// streaming-import contract) +// .vmdk - streamOptimized VMDK (the only subformat +// blessed by the OVF disk-format URI) +// +// The intermediate VMDK is written to a temp dir alongside the +// final .ova (NOT under /tmp) and removed after the tar +// finishes. /tmp is tmpfs on most Linux distros and a multi-GB +// streamOptimized VMDK exhausts the typical 16 GB tmpfs in +// minutes; the bundle dir lives on the operator's chosen +// output disk where space matches the .ova size. +func writeOVA(ctx context.Context, qcow2Src, ovaPath string, cfg Config) error { + tmpDir, err := os.MkdirTemp(filepath.Dir(ovaPath), ".yc-ova-") + if err != nil { + return fmt.Errorf("ova tmp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + vmdkPath := filepath.Join(tmpDir, cfg.Name+".vmdk") + convert := exec.CommandContext(ctx, "qemu-img", "convert", + "-f", "qcow2", "-O", "vmdk", + "-o", "subformat=streamOptimized", + qcow2Src, vmdkPath, + ) + if out, err := convert.CombinedOutput(); err != nil { + return fmt.Errorf("qemu-img convert (ova): %s: %w", out, err) + } + + capacity, err := qemuImgVirtualSize(ctx, qcow2Src) + if err != nil { + return err + } + + vmdkInfo, err := os.Stat(vmdkPath) + if err != nil { + return fmt.Errorf("stat ova vmdk: %w", err) + } + + ovfBytes := []byte(renderOVF(cfg, capacity, vmdkInfo.Size())) + ovfPath := filepath.Join(tmpDir, cfg.Name+".ovf") + if err := os.WriteFile(ovfPath, ovfBytes, 0o644); err != nil { + return fmt.Errorf("write ovf: %w", err) + } + + out, err := os.Create(ovaPath) + if err != nil { + return fmt.Errorf("create ova: %w", err) + } + defer out.Close() + + tw := tar.NewWriter(out) + // Order matters: OVF MUST come first so streaming OVA + // readers can parse the descriptor before the disk bytes + // land. Hence we write the .ovf entry, then the .vmdk. + for _, name := range []string{cfg.Name + ".ovf", cfg.Name + ".vmdk"} { + if err := tarAppendFile(tw, filepath.Join(tmpDir, name), name); err != nil { + _ = tw.Close() + return err + } + } + if err := tw.Close(); err != nil { + return fmt.Errorf("close ova tar: %w", err) + } + return nil +} + +// tarAppendFile streams srcPath into tw under archiveName. +func tarAppendFile(tw *tar.Writer, srcPath, archiveName string) error { + f, err := os.Open(srcPath) + if err != nil { + return err + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return err + } + hdr := &tar.Header{ + Name: archiveName, + Mode: 0o644, + Size: st.Size(), + ModTime: st.ModTime(), + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("tar header for %s: %w", archiveName, err) + } + if _, err := io.Copy(tw, f); err != nil { + return fmt.Errorf("tar copy %s: %w", archiveName, err) + } + return nil +} + +// qemuImgVirtualSize returns the qcow2's virtual disk size in +// bytes, used as ovf:capacity in the OVF descriptor. We use the +// SOURCE qcow2's virtual size rather than re-statting the +// streamOptimized vmdk because qemu-img preserves virtual +// geometry across the convert and the qcow2 is what we already +// trust. ovf:capacity is the size guests see, NOT the on-disk +// file size; those are only the same for raw / monolithicFlat. +func qemuImgVirtualSize(ctx context.Context, qcow2Src string) (int64, error) { + cmd := exec.CommandContext(ctx, "qemu-img", "info", "--output=json", qcow2Src) + out, err := cmd.Output() + if err != nil { + return 0, fmt.Errorf("qemu-img info: %w", err) + } + var info struct { + VirtualSize int64 `json:"virtual-size"` + } + if err := json.Unmarshal(out, &info); err != nil { + return 0, fmt.Errorf("parse qemu-img info: %w", err) + } + if info.VirtualSize == 0 { + return 0, fmt.Errorf("qemu-img info reported zero virtual-size for %s", qcow2Src) + } + return info.VirtualSize, nil +} + +// renderOVF returns a minimal OVF 1.0 descriptor that VirtualBox +// and VMware Workstation accept. CPU + memory are taken from +// cfg; capacity is the qcow2's virtual disk size in bytes; +// vmdkBytes is the on-disk size of the streamOptimized VMDK +// shipped in the same tar (referenced from ). +// +// VirtualSystemType=virtualbox-2.2 makes VirtualBox happy +// without locking VMware out -- VMware ignores the value and +// applies its own defaults during import. +func renderOVF(cfg Config, capacity, vmdkBytes int64) string { + name := html.EscapeString(cfg.Name) + cpus := html.EscapeString(cfg.CPUs) + mem := html.EscapeString(cfg.Memory) + return fmt.Sprintf(` + + + + + + List of the virtual disks + + + + The list of logical networks + + NAT network + + + + A y-cluster appliance + %s + + The kind of installed guest operating system + Ubuntu Linux (64-bit) + + + Virtual hardware requirements + + Virtual Hardware Family + 0 + %s + virtualbox-2.2 + + + %s virtual CPU + Number of virtual CPUs + %s virtual CPU + 1 + 3 + %s + + + MegaBytes + %s MB of memory + Memory Size + %s MB of memory + 2 + 4 + %s + + + 0 + sataController0 + SATA Controller + sataController0 + 3 + AHCI + 20 + + + 0 + disk1 + Disk Image + disk1 + /disk/vmdisk1 + 4 + 3 + 17 + + + true + Ethernet adapter on 'NAT' + NAT + Ethernet adapter on 'NAT' + 5 + E1000 + 10 + + + + +`, + name, vmdkBytes, + capacity, + name, name, name, + cpus, cpus, cpus, + mem, mem, mem, + ) +} diff --git a/pkg/provision/qemu/prepare_export.go b/pkg/provision/qemu/prepare_export.go new file mode 100644 index 0000000..45b4d02 --- /dev/null +++ b/pkg/provision/qemu/prepare_export.go @@ -0,0 +1,175 @@ +package qemu + +import ( + "context" + _ "embed" + "fmt" + "os" + "os/exec" + "path/filepath" + + "go.uber.org/zap" +) + +// prepareInguestScript is the shared identity-reset script. The +// SAME script runs in two contexts: +// +// - qemu PrepareExport runs it against an offline qcow2 via +// `virt-customize --run`, which mounts the disk with +// libguestfs and chroots into it. +// - The Hetzner Packer build runs it inline via Packer's shell +// provisioner against the live build VM's filesystem just +// before snapshot. +// +// One source of truth means the local and Hetzner appliance +// builds end up with the same on-disk state for cloud-init, +// netplan, machine-id, ssh host keys, and friends. See the +// script header for the full list of what's wiped vs kept and +// the reasoning behind each choice. +// +//go:embed prepare_inguest.sh +var prepareInguestScript string + +// PrepareExport strips host-specific identity from the offline +// disk image so the same disk boots cleanly when imported on a +// different hypervisor (VMware, KVM, cloud providers). It uses +// libguestfs's virt-customize to mount the qcow2 (no boot, no +// SSH, no host kernel involvement) and run the embedded +// prepare-inguest.sh script inside the chrooted filesystem. +// +// The same script also runs on the Hetzner Packer build path +// (inline, in a live VM); see prepareInguestScript above. +// +// VM must be stopped first; virt-customize refuses to operate +// on a disk in use by a running qemu. Run order: +// +// y-cluster provision +// y-cluster stop +// y-cluster prepare-export +// +// Idempotent. A prepared appliance is no longer a usable dev +// cluster locally; the next start runs cloud-init re-init and +// regenerates identity bits. Re-provision (teardown + provision) +// for a fresh dev cluster. +func PrepareExport(ctx context.Context, cacheDir, name string, logger *zap.Logger) error { + if logger == nil { + logger = zap.NewNop() + } + cfg, err := loadState(cacheDir, name) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("no saved state for %q in %s; run `y-cluster provision` first", name, cacheDir) + } + return fmt.Errorf("load state: %w", err) + } + if running, _ := cfg.IsRunning(); running { + return fmt.Errorf("VM %q is running; run `y-cluster stop` first (virt-customize needs an offline disk)", name) + } + if _, err := exec.LookPath("virt-customize"); err != nil { + return fmt.Errorf("virt-customize not found in PATH; install with: sudo apt install libguestfs-tools") + } + diskPath := filepath.Join(cfg.CacheDir, cfg.Name+".qcow2") + if _, err := os.Stat(diskPath); err != nil { + return fmt.Errorf("disk image not found at %s: %w", diskPath, err) + } + + scriptPath, err := WritePrepareInguestScript("") + if err != nil { + return fmt.Errorf("write prepare script: %w", err) + } + defer os.Remove(scriptPath) + + // Build the seed assets. virt-tar-out is part of the same + // libguestfs-tools package as virt-customize, so its presence + // is implied. If the guest has no /data/yolean dir at all + // (e.g., a build cluster that never ran a workload using the + // bundled local-path), we WARN and skip the seed step. The + // systemd unit's ConditionPathExists fires at customer boot + // and the unit no-ops -- no spurious failures. + var seed *SeedAssets + seed, err = BuildSeedAssets(ctx, diskPath, applianceNameFromConfig(cfg)) + if err != nil { + logger.Warn("seed assets not built; appliance will ship without first-boot seed", + zap.Error(err)) + seed = nil + } else { + logger.Info("data-seed staged", + zap.String("seed", seed.SeedTarPath), + zap.String("meta", seed.SeedMetaPath)) + defer os.RemoveAll(seed.TmpDir) + } + + args := prepareExportArgs(diskPath, scriptPath, seed) + logger.Info("running virt-customize", zap.String("disk", diskPath), zap.String("script", scriptPath)) + cmd := exec.CommandContext(ctx, "virt-customize", args...) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("virt-customize: %s: %w", out, err) + } + return nil +} + +// prepareExportArgs is the canonical virt-customize argv for +// PrepareExport. Pulled out so unit tests can pin its shape +// without spinning up libguestfs. +// +// Order: +// +// 1. --upload / --mkdir / --chmod for seed assets (idempotent +// filesystem ops; no side effects on the running guest's state) +// 2. --run prepare-inguest.sh (identity reset + manifest staging +// -> auto-apply move + systemctl enable on the seed unit and +// timesyncd) +// +// The seed args go FIRST so prepare-inguest can `systemctl enable` +// against units we've already uploaded. virt-customize processes +// flags strictly in order. +func prepareExportArgs(diskPath, scriptPath string, seed *SeedAssets) []string { + args := []string{"-a", diskPath} + args = append(args, virtCustomizeArgsForSeed(seed)...) + args = append(args, "--run", scriptPath) + return args +} + +// WritePrepareInguestScript writes the embedded prepare-inguest +// shell script to a temp file (or a caller-supplied directory if +// dir is non-empty), marks it executable, and returns the path. +// Caller is responsible for removing the file. +// +// Exposed publicly so the Hetzner Packer driver script can also +// emit the script to disk and upload it via Packer's file +// provisioner -- one source of truth across both build paths. +func WritePrepareInguestScript(dir string) (string, error) { + pattern := "y-cluster-prepare-*.sh" + var f *os.File + var err error + if dir == "" { + f, err = os.CreateTemp("", pattern) + } else { + f, err = os.CreateTemp(dir, pattern) + } + if err != nil { + return "", err + } + if _, err := f.WriteString(prepareInguestScript); err != nil { + f.Close() + os.Remove(f.Name()) + return "", err + } + if err := f.Chmod(0o755); err != nil { + f.Close() + os.Remove(f.Name()) + return "", err + } + if err := f.Close(); err != nil { + os.Remove(f.Name()) + return "", err + } + return f.Name(), nil +} + +// PrepareInguestScript returns the embedded script source. Used +// by tests and by the y-cluster `prepare-script` subcommand +// (consumed by the Hetzner Packer driver). +func PrepareInguestScript() string { + return prepareInguestScript +} diff --git a/pkg/provision/qemu/prepare_export_test.go b/pkg/provision/qemu/prepare_export_test.go new file mode 100644 index 0000000..14648fd --- /dev/null +++ b/pkg/provision/qemu/prepare_export_test.go @@ -0,0 +1,254 @@ +package qemu + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestPrepareInguestScript_NoMACPinning is the regression guard +// for the original Hetzner failure: nothing in the embedded +// netplan body anchors a specific MAC. The script lands the +// netplan into /etc/netplan/50-cloud-init.yaml on the imported +// VM; if it pins a build-host MAC, DHCP fails for any new MAC. +func TestPrepareInguestScript_NoMACPinning(t *testing.T) { + body := PrepareInguestScript() + if strings.Contains(body, "macaddress") { + t.Errorf("prepare-inguest script must not pin macaddress:\n%s", body) + } + if strings.Contains(body, "52:54:") { + t.Errorf("prepare-inguest script must not contain qemu SLIRP MAC:\n%s", body) + } +} + +// TestPrepareInguestScript_GenericNetplan pins the netplan match +// shape so a future edit can't accidentally narrow it (e.g. +// matching only "en*", which would miss Hetzner's eth0). +func TestPrepareInguestScript_GenericNetplan(t *testing.T) { + body := PrepareInguestScript() + for _, want := range []string{ + `/etc/netplan/50-cloud-init.yaml`, + `name: "e*"`, + `dhcp4: true`, + } { + if !strings.Contains(body, want) { + t.Errorf("prepare-inguest script missing %q:\n%s", want, body) + } + } +} + +// TestPrepareInguestScript_CloudInitClean pins the cloud-init +// reset. We do NOT pass --machine-id (would defeat the +// keep-machine-id stance and break Ubuntu 24.04 DHCP). +func TestPrepareInguestScript_CloudInitClean(t *testing.T) { + body := PrepareInguestScript() + if !strings.Contains(body, "cloud-init clean --logs --seed") { + t.Errorf("prepare-inguest script must clean cloud-init state (--logs --seed):\n%s", body) + } + for _, line := range strings.Split(body, "\n") { + trimmed := strings.TrimSpace(line) + // Skip comment lines so the header's "Do NOT pass --machine-id" + // note doesn't trip the assertion. + if strings.HasPrefix(trimmed, "#") { + continue + } + if strings.Contains(trimmed, "cloud-init clean") && strings.Contains(trimmed, "--machine-id") { + t.Errorf("prepare-inguest script must NOT pass --machine-id to cloud-init clean: %q", trimmed) + } + } +} + +// TestPrepareInguestScript_DisablesCloudInitNetworkConfig pins +// the cfg drop that prevents cloud-init from regenerating +// /etc/netplan/50-cloud-init.yaml on the imported VM's first +// boot pinned to whatever NIC's MAC. +func TestPrepareInguestScript_DisablesCloudInitNetworkConfig(t *testing.T) { + body := PrepareInguestScript() + for _, want := range []string{ + "/etc/cloud/cloud.cfg.d/99-y-cluster-no-network-config.cfg", + "network: {config: disabled}", + } { + if !strings.Contains(body, want) { + t.Errorf("prepare-inguest script missing %q:\n%s", want, body) + } + } +} + +// TestPrepareInguestScript_KeepsHostIdentity documents the +// "keep these" stance: machine-id and ssh host keys + user dir +// must NOT be wiped (would break Ubuntu DHCP and customer ssh). +// Ensures the script can't be edited to add e.g. +// `> /etc/machine-id` or `rm -rf ~/.ssh` without a test failure. +func TestPrepareInguestScript_KeepsHostIdentity(t *testing.T) { + body := PrepareInguestScript() + mustNotMatch := []string{ + "> /etc/machine-id", + "rm -f /etc/machine-id", + "rm /etc/machine-id", + "truncate -s 0 /etc/machine-id", + "rm -f /etc/ssh/ssh_host_", + "rm /etc/ssh/ssh_host_", + "rm -rf /home/*/.ssh", + "rm -rf /root/.ssh", + } + for _, bad := range mustNotMatch { + if strings.Contains(body, bad) { + t.Errorf("prepare-inguest script must not contain %q (host identity must survive prepare):\n%s", bad, body) + } + } +} + +// TestPrepareExportArgs pins the virt-customize argv shape: the +// disk under -a, the script under --run. With no seed assets the +// argv is the minimal shape; with seed assets the seed --upload / +// --mkdir / --chmod / --run-command flags are inserted BEFORE the +// --run script (virt-customize processes flags in order). Drift +// here means PrepareExport silently runs different libguestfs +// operations. +func TestPrepareExportArgs(t *testing.T) { + args := prepareExportArgs("/tmp/x.qcow2", "/tmp/script.sh", nil) + want := []string{"-a", "/tmp/x.qcow2", "--run", "/tmp/script.sh"} + if len(args) != len(want) { + t.Fatalf("got %d args, want %d: %v", len(args), len(want), args) + } + for i, a := range args { + if a != want[i] { + t.Errorf("arg[%d]: got %q, want %q", i, a, want[i]) + } + } +} + +// TestPrepareExportArgs_WithSeed pins that seed assets land BEFORE +// the --run script. Customer-side correctness depends on the +// systemd unit being present on the disk before prepare_inguest's +// `systemctl enable y-cluster-data-seed.service` runs. +func TestPrepareExportArgs_WithSeed(t *testing.T) { + seed := &SeedAssets{ + SeedTarPath: "/tmp/seed.tar.zst", + SeedMetaPath: "/tmp/seed.meta.json", + SeedCheckPath: "/tmp/seed-check", + SeedStatusPath: "/tmp/seed-status", + UnitPath: "/tmp/seed.service", + K3sDropinPath: "/tmp/k3s-dropin.conf", + } + args := prepareExportArgs("/tmp/x.qcow2", "/tmp/script.sh", seed) + + if args[0] != "-a" || args[1] != "/tmp/x.qcow2" { + t.Fatalf("first two args should be -a , got %v", args[:2]) + } + // --run must appear once and at the end. + runIdx := -1 + for i, a := range args { + if a == "--run" { + if runIdx >= 0 { + t.Fatalf("--run appears more than once: %v", args) + } + runIdx = i + } + } + if runIdx == -1 || runIdx != len(args)-2 || args[runIdx+1] != "/tmp/script.sh" { + t.Fatalf("--run must be the last flag: %v", args) + } + // The script unit upload MUST be before --run. + wantUnitArg := "/tmp/seed.service:/etc/systemd/system/y-cluster-data-seed.service" + found := false + for i, a := range args { + if a == wantUnitArg { + if i > runIdx { + t.Errorf("seed unit upload appears AFTER --run: index %d > %d", i, runIdx) + } + found = true + } + } + if !found { + t.Errorf("seed unit upload not found in args: %v", args) + } +} + +// TestWritePrepareInguestScript checks that the embedded script +// round-trips through the temp-file helper unchanged and is +// executable. +func TestWritePrepareInguestScript(t *testing.T) { + path, err := WritePrepareInguestScript(t.TempDir()) + if err != nil { + t.Fatal(err) + } + defer os.Remove(path) + + st, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if st.Mode().Perm() != 0o755 { + t.Errorf("script mode: got %v, want 0755", st.Mode().Perm()) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(body) != PrepareInguestScript() { + t.Errorf("written content does not match embedded script") + } +} + +// TestPrepareExport_NoSavedState exercises the "no saved state" +// branch: the error must point the user at `y-cluster provision`, +// not bubble up an opaque os.IsNotExist. +func TestPrepareExport_NoSavedState(t *testing.T) { + err := PrepareExport(context.Background(), t.TempDir(), "missing", nil) + if err == nil { + t.Fatal("expected error when no saved state exists") + } + if !strings.Contains(err.Error(), "y-cluster provision") { + t.Errorf("error should hint at provision: %v", err) + } +} + +// TestPrepareExport_VMRunning exercises the IsRunning guard: a +// stale-but-live pidfile (we write our own pid into it) must +// trigger the "run y-cluster stop first" error rather than +// blindly invoking virt-customize on a busy disk. +func TestPrepareExport_VMRunning(t *testing.T) { + cacheDir := t.TempDir() + cfg := defaultedRuntimeConfig(t) + cfg.CacheDir = cacheDir + if err := saveState(cfg); err != nil { + t.Fatal(err) + } + pidFile := filepath.Join(cacheDir, cfg.Name+".pid") + if err := os.WriteFile(pidFile, []byte("1\n"), 0o644); err != nil { + t.Fatal(err) + } + + err := PrepareExport(context.Background(), cacheDir, cfg.Name, nil) + if err == nil { + t.Fatal("expected error when VM still running") + } + if !strings.Contains(err.Error(), "y-cluster stop") { + t.Errorf("error should hint at stop: %v", err) + } +} + +// TestPrepareExport_MissingVirtCustomize exercises the LookPath +// guard. We empty $PATH so virt-customize can't be found, then +// confirm the apt-install hint fires before we ever try to +// invoke it. +func TestPrepareExport_MissingVirtCustomize(t *testing.T) { + cacheDir := t.TempDir() + cfg := defaultedRuntimeConfig(t) + cfg.CacheDir = cacheDir + if err := saveState(cfg); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", "") + + err := PrepareExport(context.Background(), cacheDir, cfg.Name, nil) + if err == nil { + t.Fatal("expected error when virt-customize is missing from PATH") + } + if !strings.Contains(err.Error(), "libguestfs-tools") { + t.Errorf("error should hint at apt install libguestfs-tools: %v", err) + } +} diff --git a/pkg/provision/qemu/prepare_inguest.sh b/pkg/provision/qemu/prepare_inguest.sh new file mode 100644 index 0000000..77ab1dd --- /dev/null +++ b/pkg/provision/qemu/prepare_inguest.sh @@ -0,0 +1,158 @@ +#!/bin/sh +# y-cluster appliance prepare-script: in-guest cleanup that strips +# host-specific identity to make the disk portable. +# +# Run as root, either: +# - via virt-customize against an offline qcow2 (qemu prepare-export) +# - inline in the build VM before snapshot (Hetzner Packer) +# +# Idempotent. Does NOT power off; the caller decides when to stop +# k3s and shut down. +# +# Per-customer build model: keep the things that would otherwise +# break first boot or block ssh access, wipe the things that +# leak host identity or would mis-match a different NIC. +# +# Kept on purpose: +# - /etc/machine-id -- removing it breaks systemd-networkd DHCP +# on Ubuntu 24.04 ("No such file or +# directory" on init). +# - /etc/ssh/ssh_host_*_key -- baked-in keys are part of the +# per-customer handoff bundle. Hetzner's +# cc_ssh module regenerates them on first +# cloned boot when instance-id changes, +# which is fine; keeping them here costs +# nothing and avoids racing ssh.service +# on the qemu side. +# - ~/.ssh/authorized_keys -- the operator's per-customer keypair +# lives here; wiping it would break ssh +# access on the imported boot. + +# POSIX sh because virt-customize's --run executes via /bin/sh +# (dash on Ubuntu) regardless of shebang. dash doesn't accept +# `-o pipefail`, and we don't pipe anywhere in this script, so +# `-eux` covers the cases we care about: stop on error, expand +# unset variables verbatim (we have none), and trace each line. +set -eux + +# 1. cloud-init: wipe cached state so first boot of the imported +# VM re-runs first-boot logic against whatever datasource the +# target host provides. Do NOT pass --machine-id; we keep +# /etc/machine-id intact (see header). +cloud-init clean --logs --seed + +# 2. cloud-init: disable network-config regeneration. Without this, +# cloud-init re-creates /etc/netplan/50-cloud-init.yaml on first +# boot pinned to the current NIC's MAC, clobbering our generic +# netplan written below. +install -d -m 0755 /etc/cloud/cloud.cfg.d +cat > /etc/cloud/cloud.cfg.d/99-y-cluster-no-network-config.cfg <<'CFG' +# y-cluster prepare-export: keep cloud-init from regenerating +# /etc/netplan/50-cloud-init.yaml on the imported host. Without +# this, cloud-init recreates the file pinned to the build host's +# NIC MAC and DHCP fails for any new MAC. +network: {config: disabled} +CFG +chmod 0644 /etc/cloud/cloud.cfg.d/99-y-cluster-no-network-config.cfg + +# 3. Generic netplan. Matches any en* / eth* NIC -- the two +# common kernel NIC name schemes (predictable interface names +# for newer images, classic eth0 on hosts that disable that). +# Both stanzas use DHCP, which every cloud / hypervisor target +# supports out of the box. +install -d -m 0755 /etc/netplan +cat > /etc/netplan/50-cloud-init.yaml <<'NETPLAN' +network: + version: 2 + renderer: networkd + ethernets: + any-iface: + match: + name: "e*" + dhcp4: true + dhcp6: true +NETPLAN +chmod 0600 /etc/netplan/50-cloud-init.yaml + +# 4. Hygienic wipes for shipping a clean appliance. None of these +# are load-bearing for portability -- they just keep the bundle +# small and free of build-time stamps the customer would see. +rm -f /root/.bash_history +# POSIX-safe glob: when no /home/*/.bash_history exists, the +# loop variable is the literal pattern, which `[ -f ]` rejects. +for h in /home/*/.bash_history; do + [ -f "$h" ] && rm -f "$h" +done +rm -f /etc/udev/rules.d/70-persistent-net.rules +rm -f /var/lib/systemd/random-seed +rm -f /var/lib/dhcp/dhclient.leases +apt-get clean + +# 5. Enable wall-clock sync at first boot. Without this, an +# imported VM whose RTC was set by the host clock can boot +# minutes-to-hours away from real UTC, and k3s's TLS certs +# (NotBefore = build time) read as "not yet valid", which +# manifests as a healthy-looking k3s-server with every pod +# stuck in Completed and authentication.go x509 errors in +# the journal. systemd-timesyncd is in Ubuntu's default +# install; just flip its enable bit so first boot syncs +# before k3s starts. The k3s service is `Wants=` not +# `Requires=` time-sync, so an offline appliance still boots, +# just with whatever clock the hypervisor handed over. +systemctl enable systemd-timesyncd.service + +# 6. Move build-time-staged manifests into k3s's auto-apply +# directory. y-cluster's `manifests add` command writes manifests +# the appliance builder wants applied at customer boot to +# /var/lib/y-cluster/manifests-staging/. k3s does NOT scan that +# directory, so the manifests sit dormant during build (the build +# cluster doesn't react). At export time we move them into +# /var/lib/rancher/k3s/server/manifests/ -- which IS auto-applied +# by k3s on every cluster start. The customer's first boot of the +# appliance therefore runs every staged manifest against THEIR +# cluster (e.g., migration Jobs). +# +# See specs/y-cluster/APPLIANCE_UPGRADES.md. +if [ -d /var/lib/y-cluster/manifests-staging ] \ + && [ "$(ls -A /var/lib/y-cluster/manifests-staging 2>/dev/null)" ]; then + mkdir -p /var/lib/rancher/k3s/server/manifests + mv /var/lib/y-cluster/manifests-staging/* \ + /var/lib/rancher/k3s/server/manifests/ +fi +# Remove the (now-empty) staging dir so it's clear the customer's +# boot has nothing left to do at this layer. +rmdir /var/lib/y-cluster/manifests-staging 2>/dev/null || true + +# 7. fstrim every mounted filesystem. ext4 marks deleted blocks +# free in its bitmap but doesn't zero them on disk; the next +# `qemu-img convert -O raw` carries the old content into the +# raw output and gzip can't squash non-zero bytes. Trimming +# tells the underlying device "these blocks are discardable", +# which on qemu's qcow2 + file-backed disk means they get +# zero-filled in the convert step. Effect: the gzipped tar +# we ship to GCS / VirtualBox / a customer drops by 10-30% +# on a heavily-used appliance, with no impact on what +# actually runs at first boot. +# +# We do NOT prune unused container images here -- the +# appliance contract is "ship every image the customer might +# need pre-loaded, including ones not currently referenced by +# a running pod" (e.g., upgrade images, fallback images). +# fstrim reclaims FREED blocks; it leaves every byte that's +# actually in use untouched. That is the right scope. +# +# `|| true`: a filesystem that doesn't support TRIM (tmpfs, +# read-only mounts, anything weird) returns nonzero. We don't +# care -- trimming what we can is a strict win and any +# unsupported mount fails locally without affecting the +# rest of the script. +fstrim -av || true + +# 7. Truncate (don't delete) log files so service file descriptors +# stay valid if any service is still writing. Next boot starts +# with empty logs. +for f in /var/log/syslog /var/log/auth.log /var/log/cloud-init.log /var/log/cloud-init-output.log; do + if [ -f "$f" ]; then + truncate -s 0 "$f" + fi +done diff --git a/pkg/provision/qemu/qemu.go b/pkg/provision/qemu/qemu.go index 39b1174..976a2f5 100644 --- a/pkg/provision/qemu/qemu.go +++ b/pkg/provision/qemu/qemu.go @@ -23,6 +23,7 @@ import ( "github.com/Yolean/y-cluster/pkg/provision" "github.com/Yolean/y-cluster/pkg/provision/config" "github.com/Yolean/y-cluster/pkg/provision/envoygateway" + "github.com/Yolean/y-cluster/pkg/provision/localstorage" "github.com/Yolean/y-cluster/pkg/provision/registries" "github.com/Yolean/y-cluster/pkg/sshexec" ) @@ -57,6 +58,7 @@ type Config struct { K3s K3s Registries config.Registries Gateway config.GatewayConfig + Storage config.StorageConfig } // K3s carries the runtime view of K3sConfig: which version to @@ -70,6 +72,22 @@ type K3s struct { Install string } +// preflightHostPorts gathers every host port the qemu Provision +// will bind: the SSH forward plus every PortForward.Host entry. +// Empty Host entries are skipped (qemu picks via SLIRP). +func preflightHostPorts(c Config) []string { + ports := make([]string, 0, 1+len(c.PortForwards)) + if c.SSHPort != "" { + ports = append(ports, c.SSHPort) + } + for _, pf := range c.PortForwards { + if pf.Host != "" { + ports = append(ports, pf.Host) + } + } + return ports +} + // hostAPIPort scans the configured port forwards and returns the // host-side port that maps to guest 6443. Empty string means no // such forward is configured -- in which case Provision can't @@ -139,6 +157,7 @@ func FromConfig(c *config.QEMUConfig) Config { }, Registries: c.Registries, Gateway: c.Gateway, + Storage: c.Storage, } } @@ -183,6 +202,21 @@ func (c Config) IsRunning() (bool, int) { // Provision creates and starts a QEMU VM with k3s installed. func Provision(ctx context.Context, cfg Config, logger *zap.Logger) (*Cluster, error) { + // Cross-provisioner preflight: every host port we'll bind is + // free, and the kubeconfig context isn't pointing at a + // different cluster (which a second-cluster mistake would + // silently clobber). Fail with the full list of conflicts so + // the user fixes them in one config edit, not three. + pf := provision.Preflight{ + HostPorts: preflightHostPorts(cfg), + ContextName: cfg.Context, + ContextCluster: clusterName(cfg.Name), + KubeconfigPath: cfg.Kubeconfig, + } + if err := pf.Run(); err != nil { + return nil, err + } + // Initialize kubeconfig manager early — validates KUBECONFIG env kubecfg, err := kubeconfig.New(cfg.Context, clusterName(cfg.Name), logger) if err != nil { @@ -214,26 +248,25 @@ func Provision(ctx context.Context, cfg Config, logger *zap.Logger) (*Cluster, e return nil, err } - // Create disk from cloud image (or reuse existing) + // Create disk from cloud image. ensureDisk errors when the + // disk already exists -- per-customer Provision is always + // fresh; the operator runs teardown first or `start` to + // resume. diskPath := filepath.Join(cfg.CacheDir, cfg.Name+".qcow2") - diskReused := diskExisted(diskPath) if err := c.ensureDisk(ctx, cloudImg, diskPath); err != nil { return nil, err } - // Generate SSH key + // Generate a fresh SSH keypair (always; never reused across + // provisions). The public half lands in the disk via the + // cloud-init seed below. if err := c.ensureSSHKey(); err != nil { return nil, err } - // Create cloud-init seed (only needed for first boot) - var seedPath string - if !diskReused { - var err error - seedPath, err = c.createCloudInitSeed() - if err != nil { - return nil, err - } + seedPath, err := c.createCloudInitSeed() + if err != nil { + return nil, err } // Start VM @@ -241,6 +274,14 @@ func Provision(ctx context.Context, cfg Config, logger *zap.Logger) (*Cluster, e return nil, err } + // Persist launch parameters so `y-cluster start` can re-launch + // the same shape after a stop. Best-effort: a failed write + // here doesn't unwind a working Provision; the operator can + // teardown if they care that the sidecar is missing. + if err := saveState(cfg); err != nil { + logger.Warn("could not save state sidecar (start will not work without it)", zap.Error(err)) + } + // Wait for SSH if err := c.waitForSSH(ctx); err != nil { return nil, err @@ -276,6 +317,19 @@ func Provision(ctx context.Context, cfg Config, logger *zap.Logger) (*Cluster, e } logger.Info("k3s ready", zap.String("context", cfg.Context)) + // Install the bundled local-path-provisioner (replaces k3s's + // disabled local-storage addon). Runs before any workload + // install so the StorageClass exists when consumer PVCs land. + if err := localstorage.Install(ctx, localstorage.Options{ + ContextName: cfg.Context, + Path: cfg.Storage.Path, + Pattern: cfg.Storage.PathPattern, + ReclaimPolicy: cfg.Storage.ReclaimPolicy, + Logger: logger, + }); err != nil { + return nil, fmt.Errorf("install local-path-provisioner: %w", err) + } + // Install the bundled Envoy Gateway (CRDs + controller + // default GatewayClass). Replaces the Traefik k3s would // otherwise have run; --disable=traefik passed to k3s above @@ -331,17 +385,43 @@ func TeardownConfig(cfg Config, keepDisk bool, logger *zap.Logger) error { kubecfg.CleanupTeardown() } - // Handle disk + // Handle per-VM artefacts. keepDisk preserves everything (for + // export, where the operator wants the qcow2 plus the keypair + // that authenticates against it). The default removes + // everything: disk, state sidecar, ssh keypair, cloud-init seed, + // rendered cloud-init.yaml, and console log. Wiping the keypair + // is load-bearing -- the next provision generates a fresh one, + // which is the contract for per-customer appliance delivery (no + // key reuse across provision runs). diskPath := filepath.Join(cfg.CacheDir, cfg.Name+".qcow2") if keepDisk { logger.Info("teardown complete, disk preserved", zap.String("disk", diskPath)) - } else { - os.Remove(diskPath) - logger.Info("teardown complete, disk deleted") + return nil + } + for _, p := range perVMArtefacts(cfg.CacheDir, cfg.Name) { + _ = os.Remove(p) } + _ = removeState(cfg.CacheDir, cfg.Name) + logger.Info("teardown complete, disk and keypair deleted") return nil } +// perVMArtefacts returns every path Provision creates for a given +// cluster. Used by TeardownConfig to leave the cache dir clean for +// the next provision -- the keypair in particular must go so the +// per-customer "no key reuse" contract holds. +func perVMArtefacts(cacheDir, name string) []string { + prefix := filepath.Join(cacheDir, name) + return []string{ + prefix + ".qcow2", + prefix + "-ssh", + prefix + "-ssh.pub", + prefix + "-seed.img", + prefix + "-cloud-init.yaml", + prefix + "-console.log", + } +} + // stopVM ends the qemu process whose pid lives in pidFile, // then removes the pidfile. The sequence: // @@ -473,21 +553,6 @@ func (c *Cluster) DiskPath() string { return filepath.Join(c.cfg.CacheDir, c.cfg.Name+".qcow2") } -// ExportVMDK converts the disk image to a streamOptimized VMDK. -func ExportVMDK(diskPath, outputPath string) error { - if _, err := os.Stat(diskPath); err != nil { - return fmt.Errorf("disk not found: %s", diskPath) - } - cmd := exec.Command("qemu-img", "convert", - "-f", "qcow2", "-O", "vmdk", - "-o", "subformat=streamOptimized", - diskPath, outputPath) - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("qemu-img convert: %s: %w", out, err) - } - return nil -} - // ImportVMDK converts a VMDK to qcow2 for use as a VM disk. func ImportVMDK(vmdkPath, diskPath string) error { if _, err := os.Stat(vmdkPath); err != nil { @@ -512,11 +577,6 @@ func clusterName(vmName string) string { return vmName } -func diskExisted(path string) bool { - _, err := os.Stat(path) - return err == nil -} - func (c *Cluster) ensureCloudImage(ctx context.Context) (string, error) { imgPath := filepath.Join(c.cfg.CacheDir, fmt.Sprintf("ubuntu-%s-server-cloudimg-amd64.img", ubuntuVersion)) if _, err := os.Stat(imgPath); err == nil { @@ -530,10 +590,19 @@ func (c *Cluster) ensureCloudImage(ctx context.Context) (string, error) { return imgPath, nil } +// ensureDisk creates the per-VM qcow2. Errors when the disk +// already exists -- the per-customer appliance model treats every +// `y-cluster provision` as a fresh build (fresh disk, fresh +// keypair, fresh machine-id once the guest boots). Reusing a +// disk under a new keypair would also leave the old pubkey in +// the disk's authorized_keys, which we have no way to update +// from the host. The operator runs `y-cluster teardown` first +// (or `y-cluster start` to resume an existing one). func (c *Cluster) ensureDisk(ctx context.Context, cloudImg, diskPath string) error { if _, err := os.Stat(diskPath); err == nil { - c.logger.Info("reusing existing disk", zap.String("path", diskPath)) - return nil + return fmt.Errorf( + "disk %s already exists; run `y-cluster teardown -c %s` to start fresh, or `y-cluster start --context=%s` to resume", + diskPath, c.cfg.CacheDir, c.cfg.Context) } c.logger.Info("creating disk", zap.String("size", c.cfg.DiskSize)) cmd := exec.CommandContext(ctx, "qemu-img", "create", @@ -545,20 +614,35 @@ func (c *Cluster) ensureDisk(ctx context.Context, cloudImg, diskPath string) err return nil } +// ensureSSHKey generates a fresh SSH keypair on every provision. +// y-cluster ships the keypair as part of the per-customer appliance +// handoff; reusing keys across provisions would compromise the +// contract that each shipped appliance authenticates with its own +// keypair. We delete any leftover key first (a clean teardown +// already removed it; this is belt-and-braces against a half-done +// teardown) and regenerate. func (c *Cluster) ensureSSHKey() error { - if _, err := os.Stat(c.sshKey); err == nil { - return nil - } + _ = os.Remove(c.sshKey) + _ = os.Remove(c.sshKey + ".pub") return sshexec.GenerateKey(c.sshKey) } -func (c *Cluster) createCloudInitSeed() (string, error) { - pubKey, err := os.ReadFile(c.sshKey + ".pub") - if err != nil { - return "", fmt.Errorf("read SSH public key: %w", err) - } - - cloudInit := fmt.Sprintf(`#cloud-config +// renderCloudInitUserData returns the #cloud-config user-data the +// qemu provisioner writes into the seed image. Pulled out as a +// pure function so unit tests can pin the shape without spinning +// up cloud-localds. +// +// The write_files entry drops a cloud-init config snippet onto the +// disk during first boot. It pins datasource_list so that when this +// disk is later exported and re-imported (Hetzner snapshot, VMware +// OVA, dd to bare metal), cloud-init only probes NoCloud (the qemu +// seed shape) and then falls through to None instead of spending +// minutes hammering EC2 IMDS / GCE metadata / OpenStack ConfigDrive +// on hosts that don't provide them. The "no SSH banner" failure +// mode on Hetzner was cloud-init blocking sshd's network ordering; +// this pin prevents the recurrence. +func renderCloudInitUserData(hostname, sshPubKey string) string { + return fmt.Sprintf(`#cloud-config hostname: %s users: - name: ystack @@ -567,9 +651,31 @@ users: ssh_authorized_keys: - %s package_update: false -`, c.cfg.Name, strings.TrimSpace(string(pubKey))) +write_files: + - path: /etc/cloud/cloud.cfg.d/99-y-cluster-pin.cfg + permissions: '0644' + content: | + # y-cluster: bound cloud-init datasource discovery so a re-imported + # disk does not stall on EC2 IMDS / GCE metadata probing on hosts + # that don't provide them. NoCloud covers the qemu seed; None lets + # cloud-init proceed when no NoCloud source is present. + datasource_list: [NoCloud, None] +`, hostname, strings.TrimSpace(sshPubKey)) +} + +func (c *Cluster) createCloudInitSeed() (string, error) { + pubKey, err := os.ReadFile(c.sshKey + ".pub") + if err != nil { + return "", fmt.Errorf("read SSH public key: %w", err) + } + + cloudInit := renderCloudInitUserData(c.cfg.Name, string(pubKey)) - cloudInitPath := filepath.Join(c.cfg.CacheDir, "cloud-init.yaml") + // Name-prefix the cloud-init source so two concurrent provisions + // in the same cacheDir don't race on the file. Per-VM artifacts + // elsewhere (qcow2, pidfile, ssh key, seed image, console log, + // state sidecar) all follow the same -prefixed convention. + cloudInitPath := filepath.Join(c.cfg.CacheDir, c.cfg.Name+"-cloud-init.yaml") if err := os.WriteFile(cloudInitPath, []byte(cloudInit), 0o644); err != nil { return "", err } diff --git a/pkg/provision/qemu/qemu_test.go b/pkg/provision/qemu/qemu_test.go index dd126af..627733d 100644 --- a/pkg/provision/qemu/qemu_test.go +++ b/pkg/provision/qemu/qemu_test.go @@ -3,6 +3,7 @@ package qemu import ( "os" "path/filepath" + "strings" "testing" "github.com/Yolean/y-cluster/pkg/provision/config" @@ -81,12 +82,6 @@ func TestIsRunning_StalePidFile(t *testing.T) { } } -func TestExportVMDK_MissingDisk(t *testing.T) { - if err := ExportVMDK("/nonexistent/disk.qcow2", "/tmp/out.vmdk"); err == nil { - t.Fatal("expected error for missing disk") - } -} - func TestImportVMDK_MissingVMDK(t *testing.T) { if err := ImportVMDK("/nonexistent/disk.vmdk", "/tmp/out.qcow2"); err == nil { t.Fatal("expected error for missing VMDK") @@ -118,6 +113,42 @@ func TestTeardownConfig_KeepDisk(t *testing.T) { } } +// TestRenderCloudInitUserData_DatasourceListPin guards the +// portability fix: the seed must drop a cloud-init config +// snippet that pins datasource_list to NoCloud + None so a +// re-imported disk doesn't stall on EC2 IMDS probing. +func TestRenderCloudInitUserData_DatasourceListPin(t *testing.T) { + body := renderCloudInitUserData("foo", "ssh-ed25519 AAAA test@host\n") + if !strings.Contains(body, "/etc/cloud/cloud.cfg.d/99-y-cluster-pin.cfg") { + t.Errorf("user-data must drop pin file under /etc/cloud/cloud.cfg.d/:\n%s", body) + } + if !strings.Contains(body, "datasource_list: [NoCloud, None]") { + t.Errorf("user-data must pin datasource_list to [NoCloud, None]:\n%s", body) + } +} + +// TestRenderCloudInitUserData_KeepsCoreShape pins the rest of the +// user-data so the pin addition didn't accidentally drop the +// hostname / user / sshkey wiring the qemu provisioner relies on. +func TestRenderCloudInitUserData_KeepsCoreShape(t *testing.T) { + body := renderCloudInitUserData("my-cluster", "ssh-ed25519 KEY user@h\n") + for _, want := range []string{ + "hostname: my-cluster", + "name: ystack", + "sudo: ALL=(ALL) NOPASSWD:ALL", + "ssh-ed25519 KEY user@h", + } { + if !strings.Contains(body, want) { + t.Errorf("user-data missing %q:\n%s", want, body) + } + } + // Pubkey must be trimmed -- a trailing newline inside the + // YAML list item produces a malformed block. + if strings.Contains(body, "user@h\n - ") { + t.Errorf("trailing newline on ssh key not trimmed:\n%s", body) + } +} + func TestTeardownConfig_DeleteDisk(t *testing.T) { cfg := defaultedRuntimeConfig(t) cfg.CacheDir = t.TempDir() @@ -133,3 +164,83 @@ func TestTeardownConfig_DeleteDisk(t *testing.T) { t.Fatal("disk should be deleted with keepDisk=false") } } + +// TestTeardownConfig_DeletesKeypair pins the no-key-reuse contract: +// teardown must remove the SSH keypair (and the other per-VM +// artefacts) so the next provision generates a fresh one. Reusing +// keys across customer builds would compromise the per-customer +// appliance handoff. +func TestTeardownConfig_DeletesKeypair(t *testing.T) { + cfg := defaultedRuntimeConfig(t) + cfg.CacheDir = t.TempDir() + cfg.Kubeconfig = "" + for _, name := range []string{ + cfg.Name + ".qcow2", + cfg.Name + "-ssh", + cfg.Name + "-ssh.pub", + cfg.Name + "-seed.img", + cfg.Name + "-cloud-init.yaml", + cfg.Name + "-console.log", + } { + if err := os.WriteFile(filepath.Join(cfg.CacheDir, name), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + } + if err := TeardownConfig(cfg, false, nil); err != nil { + t.Fatal(err) + } + for _, name := range []string{ + cfg.Name + ".qcow2", + cfg.Name + "-ssh", + cfg.Name + "-ssh.pub", + cfg.Name + "-seed.img", + cfg.Name + "-cloud-init.yaml", + cfg.Name + "-console.log", + } { + if _, err := os.Stat(filepath.Join(cfg.CacheDir, name)); err == nil { + t.Errorf("teardown should remove %s", name) + } + } +} + +// TestTeardownConfig_KeepDiskKeepsKeypair documents that keepDisk +// also preserves the keypair. Export workflows want both: the +// qcow2 to ship and the keypair that authenticates against it. +func TestTeardownConfig_KeepDiskKeepsKeypair(t *testing.T) { + cfg := defaultedRuntimeConfig(t) + cfg.CacheDir = t.TempDir() + cfg.Kubeconfig = "" + keyPath := filepath.Join(cfg.CacheDir, cfg.Name+"-ssh") + if err := os.WriteFile(keyPath, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if err := TeardownConfig(cfg, true, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(keyPath); err != nil { + t.Errorf("keepDisk=true must preserve the keypair: %v", err) + } +} + +// TestPerVMArtefacts pins the path layout. Provision creates these +// files; teardown removes them. A drift between the two leaves +// stale state that breaks the no-key-reuse contract. +func TestPerVMArtefacts(t *testing.T) { + got := perVMArtefacts("/c", "n") + want := []string{ + "/c/n.qcow2", + "/c/n-ssh", + "/c/n-ssh.pub", + "/c/n-seed.img", + "/c/n-cloud-init.yaml", + "/c/n-console.log", + } + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("artefact[%d]: got %q, want %q", i, got[i], want[i]) + } + } +} diff --git a/pkg/provision/qemu/seed_status.sh b/pkg/provision/qemu/seed_status.sh new file mode 100644 index 0000000..e0704a1 --- /dev/null +++ b/pkg/provision/qemu/seed_status.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# y-cluster-seed-status: customer-facing troubleshooting helper. +# Prints the seed metadata, marker contents, k3s state, and any +# conflict listing in a single readable block. +# +# Suggested when something looks wrong with the appliance's +# first-boot setup. Pointed at by the bundle README. + +set -u + +MOUNT=/data/yolean +SEED=/var/lib/y-cluster/data-seed.tar.zst +META=/var/lib/y-cluster/data-seed.meta.json +MARKER="$MOUNT/.y-cluster-seeded" + +echo "=== y-cluster appliance seed status ===" +echo + +echo "--- /data/yolean mount" +if mountpoint -q "$MOUNT" 2>/dev/null; then + findmnt -no SOURCE,TARGET,FSTYPE,SIZE "$MOUNT" 2>/dev/null \ + || mount | grep " $MOUNT " || true + echo + echo "Used / free:" + df -h "$MOUNT" 2>/dev/null | tail -1 +else + echo "NOT a separate mount -- using boot-disk /data/yolean." +fi +echo + +echo "--- seed assets on appliance disk" +ls -lh "$SEED" "$META" 2>&1 | head -5 +echo + +echo "--- seed metadata (build-time)" +if [ -f "$META" ]; then + cat "$META" +else + echo "(no $META)" +fi +echo + +echo "--- marker on /data/yolean" +if [ -e "$MARKER" ]; then + echo "PRESENT:" + cat "$MARKER" +else + echo "ABSENT." +fi +echo + +echo "--- /data/yolean entries (first 20, excluding lost+found)" +if [ -d "$MOUNT" ]; then + find "$MOUNT" -mindepth 1 -maxdepth 1 ! -name 'lost+found' 2>/dev/null | head -20 +fi +echo + +echo "--- y-cluster-data-seed.service" +systemctl status y-cluster-data-seed.service --no-pager --lines=20 2>&1 || true +echo + +echo "--- k3s.service" +systemctl is-active k3s.service 2>/dev/null || echo "k3s.service is not active" +echo + +echo "Recovery recipes (if conflict mode):" +echo " - mark existing data as seeded: echo '{\"schemaVersion\":1,\"manuallyMarked\":true}' | sudo tee $MARKER" +echo " - wipe and re-seed (DESTRUCTIVE): sudo rm -rf $MOUNT/* $MOUNT/.[!.]* && sudo systemctl restart y-cluster-data-seed.service" +echo "Then: sudo systemctl start k3s.service" diff --git a/pkg/provision/qemu/state.go b/pkg/provision/qemu/state.go new file mode 100644 index 0000000..784c166 --- /dev/null +++ b/pkg/provision/qemu/state.go @@ -0,0 +1,115 @@ +package qemu + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +// stateVersion guards forward-compat: a newer y-cluster reading an +// older sidecar bails out with a clear error rather than guessing. +// Bump when the schema changes incompatibly. +const stateVersion = 1 + +// savedState is the JSON sidecar written at /.json +// at provision time. Read by Start when the user runs +// `y-cluster start` without a -c : enough to re-invoke startVM +// against the existing qcow2 without re-running cloud-init (the +// disk already has the cluster-init user and SSH keys). +// +// Decoupled from Config: Config is a runtime struct, this is the +// on-disk shape. Kubeconfig (env-derived) and Registries / Gateway +// (cluster-state, not launch-state) are deliberately omitted -- +// they live inside the cluster on the qcow2 disk now. +type savedState struct { + Version int `json:"version"` + Name string `json:"name"` + DiskSize string `json:"diskSize"` + Memory string `json:"memory"` + CPUs string `json:"cpus"` + SSHPort string `json:"sshPort"` + PortForwards []PortForward `json:"portForwards"` + Context string `json:"context"` + CacheDir string `json:"cacheDir"` + K3s K3s `json:"k3s"` +} + +// statePath returns the sidecar path for (cacheDir, name). +func statePath(cacheDir, name string) string { + return filepath.Join(cacheDir, name+".json") +} + +// saveState writes the launch-relevant subset of cfg to the +// sidecar. Atomic via a .tmp+rename so a crash mid-write doesn't +// leave a half-written file Start would later fail to parse. +func saveState(cfg Config) error { + s := savedState{ + Version: stateVersion, + Name: cfg.Name, + DiskSize: cfg.DiskSize, + Memory: cfg.Memory, + CPUs: cfg.CPUs, + SSHPort: cfg.SSHPort, + PortForwards: cfg.PortForwards, + Context: cfg.Context, + CacheDir: cfg.CacheDir, + K3s: cfg.K3s, + } + data, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + path := statePath(cfg.CacheDir, cfg.Name) + tmp := path + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return fmt.Errorf("write %s: %w", tmp, err) + } + if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) + return fmt.Errorf("rename %s -> %s: %w", tmp, path, err) + } + return nil +} + +// loadState reads /.json and rehydrates a runtime +// Config. Kubeconfig is re-resolved from $KUBECONFIG at call time +// rather than persisted -- it's an environmental concern that +// shouldn't bake into the sidecar. +func loadState(cacheDir, name string) (Config, error) { + path := statePath(cacheDir, name) + data, err := os.ReadFile(path) + if err != nil { + return Config{}, err + } + var s savedState + if err := json.Unmarshal(data, &s); err != nil { + return Config{}, fmt.Errorf("parse %s: %w", path, err) + } + if s.Version != stateVersion { + return Config{}, fmt.Errorf( + "%s: unsupported state version %d (want %d); re-provision to refresh", + path, s.Version, stateVersion) + } + return Config{ + Name: s.Name, + DiskSize: s.DiskSize, + Memory: s.Memory, + CPUs: s.CPUs, + SSHPort: s.SSHPort, + PortForwards: s.PortForwards, + Context: s.Context, + CacheDir: s.CacheDir, + Kubeconfig: os.Getenv("KUBECONFIG"), + K3s: s.K3s, + }, nil +} + +// removeState deletes the sidecar and ignores "not present" +// errors so teardown is idempotent. +func removeState(cacheDir, name string) error { + if err := os.Remove(statePath(cacheDir, name)); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} diff --git a/pkg/provision/qemu/state_test.go b/pkg/provision/qemu/state_test.go new file mode 100644 index 0000000..1476ffa --- /dev/null +++ b/pkg/provision/qemu/state_test.go @@ -0,0 +1,112 @@ +package qemu + +import ( + "os" + "path/filepath" + "testing" +) + +func TestSaveLoadState_Roundtrip(t *testing.T) { + dir := t.TempDir() + t.Setenv("KUBECONFIG", "/path/to/kubeconfig") + + cfg := Config{ + Name: "y-cluster-test", + DiskSize: "20G", + Memory: "8192", + CPUs: "4", + SSHPort: "2222", + PortForwards: []PortForward{ + {Host: "26443", Guest: "6443"}, + {Host: "8080", Guest: "80"}, + }, + Context: "local", + CacheDir: dir, + Kubeconfig: "/path/to/kubeconfig", + K3s: K3s{Version: "v1.35.4+k3s1", Install: "airgap"}, + } + if err := saveState(cfg); err != nil { + t.Fatalf("saveState: %v", err) + } + + got, err := loadState(dir, cfg.Name) + if err != nil { + t.Fatalf("loadState: %v", err) + } + for _, c := range []struct{ name, got, want string }{ + {"Name", got.Name, cfg.Name}, + {"DiskSize", got.DiskSize, cfg.DiskSize}, + {"Memory", got.Memory, cfg.Memory}, + {"CPUs", got.CPUs, cfg.CPUs}, + {"SSHPort", got.SSHPort, cfg.SSHPort}, + {"Context", got.Context, cfg.Context}, + {"CacheDir", got.CacheDir, cfg.CacheDir}, + {"K3s.Version", got.K3s.Version, cfg.K3s.Version}, + {"K3s.Install", got.K3s.Install, cfg.K3s.Install}, + {"Kubeconfig", got.Kubeconfig, "/path/to/kubeconfig"}, + } { + if c.got != c.want { + t.Errorf("%s: got %q, want %q", c.name, c.got, c.want) + } + } + if len(got.PortForwards) != 2 { + t.Fatalf("PortForwards length: got %d, want 2", len(got.PortForwards)) + } + if got.PortForwards[0] != cfg.PortForwards[0] || got.PortForwards[1] != cfg.PortForwards[1] { + t.Fatalf("PortForwards mismatch: got %v, want %v", got.PortForwards, cfg.PortForwards) + } +} + +// TestLoadState_VersionMismatch covers the forward-compat guard: +// a sidecar with an unknown schema version errors loud rather +// than letting the caller proceed with a half-deserialized Config. +func TestLoadState_VersionMismatch(t *testing.T) { + dir := t.TempDir() + stale := []byte(`{"version":99,"name":"y-cluster-test","cacheDir":"/x"}`) + if err := os.WriteFile(filepath.Join(dir, "y-cluster-test.json"), stale, 0o644); err != nil { + t.Fatal(err) + } + _, err := loadState(dir, "y-cluster-test") + if err == nil { + t.Fatal("want error for stale version") + } + if !contains(err.Error(), "unsupported state version 99") { + t.Fatalf("want version error, got %v", err) + } +} + +// TestLoadState_NotFound surfaces the os.ErrNotExist expected by +// `y-cluster start` so it can produce a friendly message ("no +// stopped cluster to start; run `y-cluster provision`"). +func TestLoadState_NotFound(t *testing.T) { + dir := t.TempDir() + _, err := loadState(dir, "missing") + if !os.IsNotExist(err) { + t.Fatalf("want IsNotExist, got %v", err) + } +} + +func TestRemoveState_Idempotent(t *testing.T) { + dir := t.TempDir() + if err := removeState(dir, "missing"); err != nil { + t.Fatalf("removeState on missing should be no-op: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "x.json"), []byte("{}"), 0o644); err != nil { + t.Fatal(err) + } + if err := removeState(dir, "x"); err != nil { + t.Fatalf("removeState: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "x.json")); !os.IsNotExist(err) { + t.Fatal("file should have been removed") + } +} + +func contains(s, sub string) bool { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return true + } + } + return false +} diff --git a/pkg/provision/qemu/stopvm_test.go b/pkg/provision/qemu/stopvm_test.go index 2fad925..60785bd 100644 --- a/pkg/provision/qemu/stopvm_test.go +++ b/pkg/provision/qemu/stopvm_test.go @@ -97,6 +97,52 @@ func TestStopVM_TerminatesOnSIGTERM(t *testing.T) { } } +// TestStop_FallsBackToSignalsWhenSSHUnreachable covers the +// graceful-shutdown path's failure mode: when sshd isn't +// reachable (no real qemu, port closed) Stop must still kill +// the recorded pid via the SIGTERM/SIGKILL ladder. We simulate +// this with a sleep process whose pidfile is what Stop reads, +// and an unused SSH port so sshexec.Exec dial fails fast. +func TestStop_FallsBackToSignalsWhenSSHUnreachable(t *testing.T) { + withGraceTimeouts(t, 2*time.Second, 1*time.Second) + + // Shrink the graceful budget so the test doesn't sit on the + // production 60s. + prevGrace := gracefulShutdownGrace + gracefulShutdownGrace = 500 * time.Millisecond + t.Cleanup(func() { gracefulShutdownGrace = prevGrace }) + + cacheDir := t.TempDir() + cfg := defaultedRuntimeConfig(t) + cfg.CacheDir = cacheDir + cfg.SSHPort = "1" // privileged port, nothing listens; ssh dial fails fast + if err := saveState(cfg); err != nil { + t.Fatal(err) + } + + cmd := startReapableChild(t, "sleep", "60") + pidFile := pidFilePath(cacheDir, cfg.Name) + if err := os.WriteFile(pidFile, []byte(fmt.Sprintf("%d\n", cmd.Process.Pid)), 0o644); err != nil { + t.Fatal(err) + } + // Leave a placeholder ssh key file so guestPoweroff doesn't + // fail at the file-read step (the dial is the failure we + // want to exercise). + if err := os.WriteFile(filepath.Join(cacheDir, cfg.Name+"-ssh"), []byte("not-a-key"), 0o600); err != nil { + t.Fatal(err) + } + + if err := Stop(cacheDir, cfg.Name, nil); err != nil { + t.Fatalf("Stop: %v", err) + } + if pidAlive(cmd.Process.Pid) { + t.Fatal("process should be dead after Stop fallback") + } + if _, err := os.Stat(pidFile); !os.IsNotExist(err) { + t.Fatalf("pidfile should be removed; stat err=%v", err) + } +} + // TestStopVM_EscalatesToSIGKILL covers the regression we're fixing: // a process that ignores SIGTERM must still be killed (and the // pidfile cleaned) by stopVM. The downstream agent saw qemu surviving diff --git a/pkg/provision/schema/common.schema.json b/pkg/provision/schema/common.schema.json index 277096b..f6c4511 100644 --- a/pkg/provision/schema/common.schema.json +++ b/pkg/provision/schema/common.schema.json @@ -50,6 +50,10 @@ "registries": { "$ref": "#/$defs/Registries", "description": "k3s registries.yaml content. Written to /etc/rancher/k3s/registries.yaml on the node before k3s starts. ${VAR} substitution is supported on credential and endpoint fields." + }, + "storage": { + "$ref": "#/$defs/StorageConfig", + "description": "Bundled local-path-provisioner install. Defaults give a predictable on-disk layout (/data/yolean/\u003cns\u003e_\u003cpvc\u003e) and Retain reclaim so PV content survives PVC delete and an appliance upgrade rebinds the same directory by name." } }, "type": "object" @@ -82,7 +86,7 @@ "type": "string" }, "version": { - "default": "v1.35.4-rc3+k3s1", + "default": "v1.35.4+k3s1", "description": "k3s release version e.g. vX.Y.Z+k3sN.", "type": "string" } @@ -204,6 +208,31 @@ } }, "type": "object" + }, + "StorageConfig": { + "additionalProperties": false, + "properties": { + "path": { + "default": "/data/yolean", + "description": "Root directory under which local-path-provisioner allocates one subdirectory per PV. Customers who attach a separate data disk should mount it here.", + "type": "string" + }, + "pathPattern": { + "default": "{{ .PVC.Namespace }}_{{ .PVC.Name }}", + "description": "Per-PV subdirectory template (Go text/template; vars: .PVName", + "type": "string" + }, + "reclaimPolicy": { + "default": "Retain", + "description": "Reclaim policy on the y-cluster StorageClass. Retain preserves PV directories on PVC delete; Delete (the upstream k3s default) wipes them.", + "enum": [ + "Retain", + "Delete" + ], + "type": "string" + } + }, + "type": "object" } }, "$id": "https://github.com/Yolean/y-cluster/pkg/provision/config/common-config", diff --git a/pkg/provision/schema/docker.schema.json b/pkg/provision/schema/docker.schema.json index 322a24f..700d8c8 100644 --- a/pkg/provision/schema/docker.schema.json +++ b/pkg/provision/schema/docker.schema.json @@ -46,6 +46,10 @@ "registries": { "$ref": "#/$defs/Registries", "description": "k3s registries.yaml content. Written to /etc/rancher/k3s/registries.yaml on the node before k3s starts. ${VAR} substitution is supported on credential and endpoint fields." + }, + "storage": { + "$ref": "#/$defs/StorageConfig", + "description": "Bundled local-path-provisioner install. Defaults give a predictable on-disk layout (/data/yolean/\u003cns\u003e_\u003cpvc\u003e) and Retain reclaim so PV content survives PVC delete and an appliance upgrade rebinds the same directory by name." } }, "required": [ @@ -81,7 +85,7 @@ "type": "string" }, "version": { - "default": "v1.35.4-rc3+k3s1", + "default": "v1.35.4+k3s1", "description": "k3s release version e.g. vX.Y.Z+k3sN.", "type": "string" } @@ -203,6 +207,31 @@ } }, "type": "object" + }, + "StorageConfig": { + "additionalProperties": false, + "properties": { + "path": { + "default": "/data/yolean", + "description": "Root directory under which local-path-provisioner allocates one subdirectory per PV. Customers who attach a separate data disk should mount it here.", + "type": "string" + }, + "pathPattern": { + "default": "{{ .PVC.Namespace }}_{{ .PVC.Name }}", + "description": "Per-PV subdirectory template (Go text/template; vars: .PVName", + "type": "string" + }, + "reclaimPolicy": { + "default": "Retain", + "description": "Reclaim policy on the y-cluster StorageClass. Retain preserves PV directories on PVC delete; Delete (the upstream k3s default) wipes them.", + "enum": [ + "Retain", + "Delete" + ], + "type": "string" + } + }, + "type": "object" } }, "$id": "https://github.com/Yolean/y-cluster/pkg/provision/config/docker-config", diff --git a/pkg/provision/schema/multipass.schema.json b/pkg/provision/schema/multipass.schema.json index f7437bc..61c3a72 100644 --- a/pkg/provision/schema/multipass.schema.json +++ b/pkg/provision/schema/multipass.schema.json @@ -28,7 +28,7 @@ "type": "string" }, "version": { - "default": "v1.35.4-rc3+k3s1", + "default": "v1.35.4+k3s1", "description": "k3s release version e.g. vX.Y.Z+k3sN.", "type": "string" } @@ -86,6 +86,10 @@ "registries": { "$ref": "#/$defs/Registries", "description": "k3s registries.yaml content. Written to /etc/rancher/k3s/registries.yaml on the node before k3s starts. ${VAR} substitution is supported on credential and endpoint fields." + }, + "storage": { + "$ref": "#/$defs/StorageConfig", + "description": "Bundled local-path-provisioner install. Defaults give a predictable on-disk layout (/data/yolean/\u003cns\u003e_\u003cpvc\u003e) and Retain reclaim so PV content survives PVC delete and an appliance upgrade rebinds the same directory by name." } }, "required": [ @@ -208,6 +212,31 @@ } }, "type": "object" + }, + "StorageConfig": { + "additionalProperties": false, + "properties": { + "path": { + "default": "/data/yolean", + "description": "Root directory under which local-path-provisioner allocates one subdirectory per PV. Customers who attach a separate data disk should mount it here.", + "type": "string" + }, + "pathPattern": { + "default": "{{ .PVC.Namespace }}_{{ .PVC.Name }}", + "description": "Per-PV subdirectory template (Go text/template; vars: .PVName", + "type": "string" + }, + "reclaimPolicy": { + "default": "Retain", + "description": "Reclaim policy on the y-cluster StorageClass. Retain preserves PV directories on PVC delete; Delete (the upstream k3s default) wipes them.", + "enum": [ + "Retain", + "Delete" + ], + "type": "string" + } + }, + "type": "object" } }, "$id": "https://github.com/Yolean/y-cluster/pkg/provision/config/multipass-config", diff --git a/pkg/provision/schema/qemu.schema.json b/pkg/provision/schema/qemu.schema.json index eef163b..909db33 100644 --- a/pkg/provision/schema/qemu.schema.json +++ b/pkg/provision/schema/qemu.schema.json @@ -28,7 +28,7 @@ "type": "string" }, "version": { - "default": "v1.35.4-rc3+k3s1", + "default": "v1.35.4+k3s1", "description": "k3s release version e.g. vX.Y.Z+k3sN.", "type": "string" } @@ -113,6 +113,10 @@ "default": "2222", "description": "Host port forwarded to the VM's SSH server. Added on top of CommonConfig.PortForwards.", "type": "string" + }, + "storage": { + "$ref": "#/$defs/StorageConfig", + "description": "Bundled local-path-provisioner install. Defaults give a predictable on-disk layout (/data/yolean/\u003cns\u003e_\u003cpvc\u003e) and Retain reclaim so PV content survives PVC delete and an appliance upgrade rebinds the same directory by name." } }, "required": [ @@ -217,6 +221,31 @@ } }, "type": "object" + }, + "StorageConfig": { + "additionalProperties": false, + "properties": { + "path": { + "default": "/data/yolean", + "description": "Root directory under which local-path-provisioner allocates one subdirectory per PV. Customers who attach a separate data disk should mount it here.", + "type": "string" + }, + "pathPattern": { + "default": "{{ .PVC.Namespace }}_{{ .PVC.Name }}", + "description": "Per-PV subdirectory template (Go text/template; vars: .PVName", + "type": "string" + }, + "reclaimPolicy": { + "default": "Retain", + "description": "Reclaim policy on the y-cluster StorageClass. Retain preserves PV directories on PVC delete; Delete (the upstream k3s default) wipes them.", + "enum": [ + "Retain", + "Delete" + ], + "type": "string" + } + }, + "type": "object" } }, "$id": "https://github.com/Yolean/y-cluster/pkg/provision/config/qemu-config", diff --git a/pkg/serve/serve.go b/pkg/serve/serve.go index aa4ed2e..dd3fe06 100644 --- a/pkg/serve/serve.go +++ b/pkg/serve/serve.go @@ -67,14 +67,6 @@ func Run(ctx context.Context, opts Options) error { return startBackground(ctx, cfgs, paths, opts) } -// DaemonVersion is logged at daemon startup so `serve logs` can -// report which y-cluster build is running. Set by main.go from -// versionString() before any serve subcommand executes; pkg/serve -// itself can't compute the user-facing release tag because that -// lives in cmd/y-cluster's `var version` (overridable via ldflags -// at release time). -var DaemonVersion = "unknown" - // EnsureAction describes what Ensure had to do. type EnsureAction int @@ -110,8 +102,7 @@ func (a EnsureAction) String() string { // actually changed. type EnsureResult struct { Action EnsureAction - Ports []int // every port the daemon now listens on - Digest string // sha256 hex of the running daemon's normalized config set + Ports []int // every port the daemon now listens on } // Ensure launches or restarts the daemon so that the configured @@ -137,13 +128,7 @@ func Ensure(ctx context.Context, opts Options) (EnsureResult, error) { want := Digest(cfgs) have, healthy := inspectRunning(paths, cfgs) if healthy && have == want { - // Surface the digest so an operator who suspects the - // running daemon is on stale config can compare against - // what they expect (the digest of the current `-c` - // directories' YAML state). `noop` was the right answer; - // the digest tells them WHY -- the new request was - // byte-identical to the running config. - return EnsureResult{Action: EnsureNoop, Ports: ports, Digest: want}, nil + return EnsureResult{Action: EnsureNoop, Ports: ports}, nil } // Anything we have to clean up before launching counts as a // restart from the operator's view. That includes a stale @@ -160,7 +145,7 @@ func Ensure(ctx context.Context, opts Options) (EnsureResult, error) { if err := startBackground(ctx, cfgs, paths, opts); err != nil { return EnsureResult{}, err } - return EnsureResult{Action: action, Ports: ports, Digest: want}, nil + return EnsureResult{Action: action, Ports: ports}, nil } // pidfilePresent is the "is there state on disk to clean up?" @@ -373,24 +358,6 @@ func runAsDaemon(parent context.Context, cfgs []*Config, paths StatePaths) (retE logger := newJSONLogger() defer func() { _ = logger.Sync() }() - digest := Digest(cfgs) - configDirs := make([]string, len(cfgs)) - for i, c := range cfgs { - configDirs[i] = c.Dir - } - // Log the binary version + config dirs + digest at startup - // so `y-cluster serve logs` answers "what version is running" - // and "which -c paths is it serving" without the operator - // having to grep state files. These are the questions we - // hit when an unexpected `noop` made us doubt the daemon - // was on the config we thought it was on. - logger.Info("y-cluster serve daemon start", - zap.String("version", DaemonVersion), - zap.Int("pid", os.Getpid()), - zap.Strings("configDirs", configDirs), - zap.String("digest", digest), - ) - if err := WritePidfile(paths.Pid, os.Getpid()); err != nil { logger.Error("write pidfile", zap.Error(err)) return err @@ -399,7 +366,7 @@ func runAsDaemon(parent context.Context, cfgs []*Config, paths StatePaths) (retE _ = os.Remove(paths.Pid) }() - snap := map[string]string{"digest": digest} + snap := map[string]string{"digest": Digest(cfgs)} data, _ := json.Marshal(snap) if err := os.WriteFile(paths.Config, data, 0o600); err != nil { logger.Error("write config snapshot", zap.Error(err)) diff --git a/pkg/serve/serve_test.go b/pkg/serve/serve_test.go index e4aac86..2dd64b1 100644 --- a/pkg/serve/serve_test.go +++ b/pkg/serve/serve_test.go @@ -233,13 +233,6 @@ func TestEnsure_FirstStartAndNoop(t *testing.T) { if len(res.Ports) != 1 || res.Ports[0] != port { t.Fatalf("ports=%v want [%d]", res.Ports, port) } - // Digest is the sha256 hex of the normalized config -- 64 - // chars. The CLI truncates to 12 for display, but the - // EnsureResult carries the full string so callers that want - // to compare programmatically can. - if len(res.Digest) != 64 { - t.Errorf("digest length: got %d, want 64 (full sha256 hex)", len(res.Digest)) - } res2, err := Ensure(context.Background(), Options{ ConfigDirs: []string{cfgDir}, @@ -252,12 +245,6 @@ func TestEnsure_FirstStartAndNoop(t *testing.T) { if res2.Action != EnsureNoop { t.Fatalf("second ensure: action=%s want noop", res2.Action) } - // Same config, same digest -- the noop branch must - // surface the same value the started branch did, so an - // operator who compares the two outputs sees a match. - if res2.Digest != res.Digest { - t.Errorf("digest noop=%q started=%q; should be identical for same config", res2.Digest, res.Digest) - } } // TestEnsure_RestartWhenStaleStatePresent covers the "daemon already diff --git a/testdata/appliance-stateful/base/httproute.yaml b/testdata/appliance-stateful/base/httproute.yaml new file mode 100644 index 0000000..958cc6e --- /dev/null +++ b/testdata/appliance-stateful/base/httproute.yaml @@ -0,0 +1,43 @@ +# HTTPRoute exposes versitygw via the y-cluster Gateway under a +# /s3 prefix; URLRewrite strips the prefix so requests reach +# versitygw at its native paths (/health, //, ...). +# +# Why a prefix instead of a separate Gateway listener: keeps the +# appliance to one inbound port (80) for the customer's external +# proxy / load balancer, and lets the e2e smoketest use the same +# host:port pair the echo workload already exposes +# (curl http:///s3/health vs http:///q/envoy/echo). +# +# parentRefs targets the y-cluster Gateway in the y-cluster +# namespace -- created by `y-cluster echo deploy` (or any consumer +# kustomization that ships the same Gateway), and reachable +# cross-namespace via the listener's `allowedRoutes.namespaces.from: +# All` config. +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: versitygw + namespace: appliance-stateful + labels: + app.kubernetes.io/name: versitygw + app.kubernetes.io/part-of: appliance-stateful +spec: + parentRefs: + - name: y-cluster + namespace: y-cluster + rules: + - matches: + - path: + type: PathPrefix + value: /s3 + filters: + - type: URLRewrite + urlRewrite: + path: + type: ReplacePrefixMatch + # Strip the /s3 prefix so versitygw sees /health + # (or / etc.) at its native root. + replacePrefixMatch: / + backendRefs: + - name: versitygw + port: 7070 diff --git a/testdata/appliance-stateful/base/kustomization.yaml b/testdata/appliance-stateful/base/kustomization.yaml new file mode 100644 index 0000000..5e8f02c --- /dev/null +++ b/testdata/appliance-stateful/base/kustomization.yaml @@ -0,0 +1,28 @@ +# Stateful workload fixture used by the appliance e2e: +# scripts/e2e-appliance-export-import.sh applies this base via +# kubectl after the echo workload is up. The same base also lands +# in the Hetzner appliance snapshot via the Packer template. +# +# The fixture exercises the persistent-volume path: a one-replica +# StatefulSet with a 1Gi volumeClaimTemplate against k3s's default +# `local-path` StorageClass. Local-path-provisioner allocates the +# volume under /var/lib/rancher/k3s/storage on the node, so the +# data survives `y-cluster stop` / `y-cluster start` (no host-side +# cleanup) AND survives a qcow2 export / import (the directory is +# part of the appliance disk). +# +# Workload: VersityGW (https://github.com/versity/versitygw), +# an S3 API server with a posix-filesystem backend. Exercises a +# real stateful scenario: bucket / object writes hit the PV. +# +# This fixture is e2e-test scaffolding, NOT a customer-shipped +# appliance default. The credentials below are deliberately +# obvious test-only strings; customers writing real workloads +# follow the same shape with their own Secret-managed values. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: appliance-stateful +resources: + - statefulset.yaml + - service.yaml + - httproute.yaml diff --git a/testdata/appliance-stateful/base/service.yaml b/testdata/appliance-stateful/base/service.yaml new file mode 100644 index 0000000..4b07946 --- /dev/null +++ b/testdata/appliance-stateful/base/service.yaml @@ -0,0 +1,20 @@ +# Headless service so StatefulSet pod DNS works +# (versitygw-0.versitygw.appliance-stateful), plus a stable +# port mapping for the HTTPRoute. +apiVersion: v1 +kind: Service +metadata: + name: versitygw + namespace: appliance-stateful + labels: + app.kubernetes.io/name: versitygw + app.kubernetes.io/part-of: appliance-stateful +spec: + selector: + app.kubernetes.io/name: versitygw + ports: + - name: s3 + port: 7070 + targetPort: s3 + # Headless: required by StatefulSet for stable pod DNS. + clusterIP: None diff --git a/testdata/appliance-stateful/base/statefulset.yaml b/testdata/appliance-stateful/base/statefulset.yaml new file mode 100644 index 0000000..43d5dce --- /dev/null +++ b/testdata/appliance-stateful/base/statefulset.yaml @@ -0,0 +1,137 @@ +# VersityGW StatefulSet: one-replica S3 API server with a posix +# backend rooted in a PV. Pinned by digest so the fixture can't +# silently drift on a `:latest` retag. v1.4.1 is the upstream +# stable release as of 2026-04-22. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: versitygw + namespace: appliance-stateful + labels: + app.kubernetes.io/name: versitygw + app.kubernetes.io/part-of: appliance-stateful + app.kubernetes.io/managed-by: y-cluster +spec: + serviceName: versitygw + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: versitygw + template: + metadata: + labels: + app.kubernetes.io/name: versitygw + app.kubernetes.io/part-of: appliance-stateful + spec: + # Upstream image runs as root by default; the upstream Helm + # chart explicitly drops to UID/GID 1000 with caps cleared + # and a read-only root filesystem. We mirror that posture so + # the fixture demonstrates the production-shape security + # context an appliance customer would use. + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + containers: + - name: versitygw + image: ghcr.io/versity/versitygw:v1.4.1@sha256:0400cb59f59da0f1cf9f7fd49505191abc348dfadf54509bf1988caaff4eb96f + ports: + - name: s3 + containerPort: 7070 + env: + # ROOT_ACCESS_KEY / ROOT_SECRET_KEY are mandatory -- + # versitygw refuses to start without them. These are + # TEST-ONLY values; a real customer install supplies + # their own via a Secret. The fixture's smoketest + # (curl /s3/health) doesn't need them, but the + # process refuses to boot without something here. + - name: ROOT_ACCESS_KEY + value: appliance-test + - name: ROOT_SECRET_KEY + value: appliance-test-secret-not-for-prod + # Posix backend rooted in the PV. The mount path below + # must match the volumeMounts entry, and the gateway + # never writes outside it. + - name: VGW_BACKEND + value: posix + - name: VGW_BACKEND_ARG + value: /data + # Listener port. Default is :7070; we set it explicitly + # so the Service / HTTPRoute / smoketest line up + # without anyone hunting for the upstream default. + - name: VGW_PORT + value: ":7070" + # Unauth GET endpoint at /health. Without this set, the + # gateway answers /s3/health with an S3 auth error + # (which is fine as a TCP liveness signal but not as a + # readiness probe -- HTTP 200 is much more useful for + # the e2e curl loop). + - name: VGW_HEALTH + value: /health + # IAM dir lives in /tmp/vgw because the root filesystem + # is read-only. The path is mounted directly via the + # vgw-iam emptyDir below -- versitygw's binary doesn't + # auto-mkdir its iam-dir, so an emptyDir mount at + # /tmp/vgw is the simplest way to have the directory + # exist on container start. + - name: VGW_ARGS + value: --iam-dir /tmp/vgw + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: data + mountPath: /data + - name: tmp + mountPath: /tmp + - name: vgw-iam + mountPath: /tmp/vgw + # Liveness + readiness against the unauth /health + # endpoint. The probe runs from inside the pod (kubelet + # talks to :7070), so it bypasses the gateway entirely + # and only reflects the VGW process's own health. + readinessProbe: + httpGet: + path: /health + port: s3 + initialDelaySeconds: 2 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /health + port: s3 + initialDelaySeconds: 30 + periodSeconds: 30 + volumes: + - name: tmp + emptyDir: {} + # Dedicated emptyDir for VGW's IAM dir. Mounted at + # /tmp/vgw (the path passed via --iam-dir) so the + # directory exists on container start; versitygw's binary + # writes /tmp/vgw/users.json on first boot when no IAM + # service is configured. Lives on emptyDir (not the PV) + # because IAM data is per-instance state, not bucket + # content -- and a fresh appliance ought to start with a + # clean IAM seed every time the StatefulSet pod (re) + # creates. + - name: vgw-iam + emptyDir: {} + volumeClaimTemplates: + # 1Gi against k3s's default `local-path` StorageClass. The + # provisioner creates /var/lib/rancher/k3s/storage/_ + # appliance-stateful_data-versitygw-0 on the node and bind- + # mounts it into the pod. Survives stop/start and the qcow2 + # export -> import (the directory is on the appliance disk). + - metadata: + name: data + labels: + app.kubernetes.io/part-of: appliance-stateful + spec: + accessModes: [ ReadWriteOnce ] + resources: + requests: + storage: 1Gi diff --git a/testdata/appliance-stateful/base/yconverge.cue b/testdata/appliance-stateful/base/yconverge.cue new file mode 100644 index 0000000..8568690 --- /dev/null +++ b/testdata/appliance-stateful/base/yconverge.cue @@ -0,0 +1,25 @@ +package applianceStateful + +import ( + "yolean.se/ystack/yconverge/verify" + "yolean.se/ystack/appliance-stateful/namespace:applianceStatefulNs" +) + +// Declare the namespace module as a dependency. yconverge +// converges _dep_ns FIRST (apply + checks), then this base. +// The "_dep_" prefix is convention; the field name is +// arbitrary, what matters is that we reference the imported +// step so cue's import-tracker pulls it into the graph. +_dep_ns: applianceStatefulNs.step + +step: verify.#Step & { + checks: [ + { + kind: "rollout" + resource: "statefulset/versitygw" + namespace: "appliance-stateful" + timeout: "180s" + description: "versitygw StatefulSet ready" + }, + ] +} diff --git a/testdata/appliance-stateful/namespace/kustomization.yaml b/testdata/appliance-stateful/namespace/kustomization.yaml new file mode 100644 index 0000000..5fe330c --- /dev/null +++ b/testdata/appliance-stateful/namespace/kustomization.yaml @@ -0,0 +1,12 @@ +# Namespace module for the appliance-stateful fixture. +# Split out from base/ so yconverge can apply + verify the +# namespace before the StatefulSet that lives in it. Without +# this dep edge, kubectl-apply-with-mixed-resources races: +# the StatefulSet validates against an apiserver that hasn't +# observed the namespace yet, and the rollout-status check +# can fall through "namespace not found" before the controller +# catches up. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml diff --git a/testdata/appliance-stateful/namespace/namespace.yaml b/testdata/appliance-stateful/namespace/namespace.yaml new file mode 100644 index 0000000..cbcd1fd --- /dev/null +++ b/testdata/appliance-stateful/namespace/namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: appliance-stateful + labels: + # Marker label so cluster-wide cleanup in tests can sweep the + # whole fixture in one selector ("delete ns -l app.kubernetes.io/ + # part-of=appliance-stateful"). + app.kubernetes.io/part-of: appliance-stateful diff --git a/testdata/appliance-stateful/namespace/yconverge.cue b/testdata/appliance-stateful/namespace/yconverge.cue new file mode 100644 index 0000000..06513b4 --- /dev/null +++ b/testdata/appliance-stateful/namespace/yconverge.cue @@ -0,0 +1,19 @@ +package applianceStatefulNs + +import "yolean.se/ystack/yconverge/verify" + +// Apply this base, then wait until the apiserver reports the +// namespace as Active. Importing modules can rely on the +// namespace existing in their own checks because yconverge +// runs deps' apply + checks BEFORE the importer's apply. +step: verify.#Step & { + checks: [ + { + kind: "wait" + resource: "namespace/appliance-stateful" + for: "jsonpath={.status.phase}=Active" + timeout: "30s" + description: "appliance-stateful namespace Active" + }, + ] +}