diff --git a/internal/api/types.go b/internal/api/types.go index f5c28dc..ad36221 100644 --- a/internal/api/types.go +++ b/internal/api/types.go @@ -263,6 +263,16 @@ type ImageRegisterParams struct { Docker bool `json:"docker,omitempty"` } +type ImagePullParams struct { + Ref string `json:"ref"` + Name string `json:"name,omitempty"` + KernelPath string `json:"kernel_path,omitempty"` + InitrdPath string `json:"initrd_path,omitempty"` + ModulesDir string `json:"modules_dir,omitempty"` + KernelRef string `json:"kernel_ref,omitempty"` + SizeBytes int64 `json:"size_bytes,omitempty"` +} + type ImageRefParams struct { IDOrName string `json:"id_or_name"` } diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 3af71e2..811de65 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -50,6 +50,7 @@ type Daemon struct { vmDNS *vmdns.Server vmCaps []vmCapability imageBuild func(context.Context, imageBuildSpec) error + pullAndFlatten func(ctx context.Context, ref, cacheDir, destDir string) error requestHandler func(context.Context, rpc.Request) rpc.Response guestWaitForSSH func(context.Context, string, string, time.Duration) error guestDial func(context.Context, string, string) (guestSSHClient, error) @@ -527,6 +528,13 @@ func (d *Daemon) dispatch(ctx context.Context, req rpc.Request) rpc.Response { } image, err := d.DeleteImage(ctx, params.IDOrName) return marshalResultOrError(api.ImageShowResult{Image: image}, err) + case "image.pull": + params, err := rpc.DecodeParams[api.ImagePullParams](req) + if err != nil { + return rpc.NewError("bad_request", err.Error()) + } + image, err := d.PullImage(ctx, params) + return marshalResultOrError(api.ImageShowResult{Image: image}, err) case "kernel.list": return marshalResultOrError(d.KernelList(ctx)) case "kernel.show": diff --git a/internal/daemon/imagemgr/paths.go b/internal/daemon/imagemgr/paths.go index 12996d6..a0f826d 100644 --- a/internal/daemon/imagemgr/paths.go +++ b/internal/daemon/imagemgr/paths.go @@ -21,17 +21,29 @@ import ( func ValidateRegisterPaths(rootfsPath, workSeedPath, kernelPath, initrdPath, modulesDir string) error { checks := system.NewPreflight() checks.RequireFile(rootfsPath, "rootfs image", `pass --rootfs `) - checks.RequireFile(kernelPath, "kernel image", `pass --kernel `) if workSeedPath != "" { checks.RequireFile(workSeedPath, "work-seed image", `pass --work-seed or rebuild the image with a work seed`) } + addKernelChecks(checks, kernelPath, initrdPath, modulesDir) + return checks.Err("image register failed") +} + +// ValidateKernelPaths checks the kernel triple alone, used by flows +// (e.g. image pull) that produce the rootfs themselves. +func ValidateKernelPaths(kernelPath, initrdPath, modulesDir string) error { + checks := system.NewPreflight() + addKernelChecks(checks, kernelPath, initrdPath, modulesDir) + return checks.Err("kernel preflight failed") +} + +func addKernelChecks(checks *system.Preflight, kernelPath, initrdPath, modulesDir string) { + checks.RequireFile(kernelPath, "kernel image", `pass --kernel `) if initrdPath != "" { checks.RequireFile(initrdPath, "initrd image", `pass --initrd `) } if modulesDir != "" { checks.RequireDir(modulesDir, "kernel modules dir", `pass --modules `) } - return checks.Err("image register failed") } // ValidatePromotePaths checks that an existing registered image's artifacts diff --git a/internal/daemon/images.go b/internal/daemon/images.go index bfc8448..2cdb3dc 100644 --- a/internal/daemon/images.go +++ b/internal/daemon/images.go @@ -179,29 +179,9 @@ func (d *Daemon) RegisterImage(ctx context.Context, params api.ImageRegisterPara } } } - kernelPath := strings.TrimSpace(params.KernelPath) - initrdPath := strings.TrimSpace(params.InitrdPath) - modulesDir := strings.TrimSpace(params.ModulesDir) - kernelRef := strings.TrimSpace(params.KernelRef) - - if kernelRef != "" { - if kernelPath != "" || initrdPath != "" || modulesDir != "" { - return model.Image{}, fmt.Errorf("--kernel-ref is mutually exclusive with --kernel/--initrd/--modules") - } - entry, err := kernelcat.ReadLocal(d.layout.KernelsDir, kernelRef) - if err != nil { - if os.IsNotExist(err) { - return model.Image{}, fmt.Errorf("kernel %q not found in catalog; run 'banger kernel list' to see available entries", kernelRef) - } - return model.Image{}, fmt.Errorf("resolve kernel %q: %w", kernelRef, err) - } - kernelPath = entry.KernelPath - initrdPath = entry.InitrdPath - modulesDir = entry.ModulesDir - } - - if kernelPath == "" { - return model.Image{}, fmt.Errorf("kernel path is required (pass --kernel or --kernel-ref )") + kernelPath, initrdPath, modulesDir, err := d.resolveKernelInputs(params.KernelRef, params.KernelPath, params.InitrdPath, params.ModulesDir) + if err != nil { + return model.Image{}, err } if err := imagemgr.ValidateRegisterPaths(rootfsPath, workSeedPath, kernelPath, initrdPath, modulesDir); err != nil { @@ -391,3 +371,31 @@ func firstNonEmpty(values ...string) string { } return "" } + +// resolveKernelInputs canonicalises user-supplied kernel info: either direct +// paths or a kernel-catalog ref. Shared by RegisterImage and PullImage. +func (d *Daemon) resolveKernelInputs(kernelRef, kernelPath, initrdPath, modulesDir string) (string, string, string, error) { + kernelRef = strings.TrimSpace(kernelRef) + kernelPath = strings.TrimSpace(kernelPath) + initrdPath = strings.TrimSpace(initrdPath) + modulesDir = strings.TrimSpace(modulesDir) + + if kernelRef != "" { + if kernelPath != "" || initrdPath != "" || modulesDir != "" { + return "", "", "", fmt.Errorf("--kernel-ref is mutually exclusive with --kernel/--initrd/--modules") + } + entry, err := kernelcat.ReadLocal(d.layout.KernelsDir, kernelRef) + if err != nil { + if os.IsNotExist(err) { + return "", "", "", fmt.Errorf("kernel %q not found in catalog; run 'banger kernel list' to see available entries", kernelRef) + } + return "", "", "", fmt.Errorf("resolve kernel %q: %w", kernelRef, err) + } + return entry.KernelPath, entry.InitrdPath, entry.ModulesDir, nil + } + + if kernelPath == "" { + return "", "", "", fmt.Errorf("kernel path is required (pass --kernel or --kernel-ref )") + } + return kernelPath, initrdPath, modulesDir, nil +} diff --git a/internal/daemon/images_pull.go b/internal/daemon/images_pull.go new file mode 100644 index 0000000..7204c42 --- /dev/null +++ b/internal/daemon/images_pull.go @@ -0,0 +1,213 @@ +package daemon + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + + "banger/internal/api" + "banger/internal/daemon/imagemgr" + "banger/internal/imagepull" + "banger/internal/model" + + "github.com/google/go-containerregistry/pkg/name" +) + +// minPullExt4Size keeps the floor consistent with imagepull.MinExt4Size +// when the caller doesn't override --size and the OCI tree is tiny. +const minPullExt4Size int64 = 1 << 30 // 1 GiB + +// PullImage downloads an OCI image, flattens it into an ext4 rootfs, and +// registers it as a managed banger image. Kernel info comes via --kernel-ref +// or direct paths, mirroring RegisterImage. +// +// The pulled rootfs's file ownership is the runner's uid/gid (Phase A v1 +// limitation; see internal/imagepull). The image is suitable as input to +// `image build --from-image` but is not directly bootable until a future +// fixup pass lands. +func (d *Daemon) PullImage(ctx context.Context, params api.ImagePullParams) (image model.Image, err error) { + d.imageOpsMu.Lock() + defer d.imageOpsMu.Unlock() + + ref := strings.TrimSpace(params.Ref) + if ref == "" { + return model.Image{}, errors.New("oci reference is required") + } + parsed, err := name.ParseReference(ref) + if err != nil { + return model.Image{}, fmt.Errorf("parse oci ref %q: %w", ref, err) + } + + imgName := strings.TrimSpace(params.Name) + if imgName == "" { + imgName = defaultImageNameFromRef(parsed) + if imgName == "" { + return model.Image{}, errors.New("could not derive image name from ref; pass --name") + } + } + if existing, lookupErr := d.store.GetImageByName(ctx, imgName); lookupErr == nil { + return model.Image{}, fmt.Errorf("image %q already exists (id=%s); pick a different --name or delete it first", imgName, existing.ID) + } + + kernelPath, initrdPath, modulesDir, err := d.resolveKernelInputs(params.KernelRef, params.KernelPath, params.InitrdPath, params.ModulesDir) + if err != nil { + return model.Image{}, err + } + if err := imagemgr.ValidateKernelPaths(kernelPath, initrdPath, modulesDir); err != nil { + return model.Image{}, err + } + + id, err := model.NewID() + if err != nil { + return model.Image{}, err + } + finalDir := filepath.Join(d.layout.ImagesDir, id) + stagingDir := finalDir + ".staging" + if err := os.MkdirAll(stagingDir, 0o755); err != nil { + return model.Image{}, err + } + cleanupStaging := true + defer func() { + if cleanupStaging { + _ = os.RemoveAll(stagingDir) + } + }() + + // Extract OCI layers into a working tree under TempDir so the + // state filesystem doesn't temporarily double in size. + rootfsTree, err := os.MkdirTemp("", "banger-pull-") + if err != nil { + return model.Image{}, err + } + defer os.RemoveAll(rootfsTree) + + if err := d.runPullAndFlatten(ctx, ref, d.layout.OCICacheDir, rootfsTree); err != nil { + return model.Image{}, fmt.Errorf("pull oci image: %w", err) + } + + sizeBytes := params.SizeBytes + if sizeBytes <= 0 { + treeSize, err := dirSizeBytes(rootfsTree) + if err != nil { + return model.Image{}, fmt.Errorf("size oci tree: %w", err) + } + sizeBytes = treeSize + treeSize/4 // +25% headroom + if sizeBytes < minPullExt4Size { + sizeBytes = minPullExt4Size + } + } + + rootfsExt4 := filepath.Join(stagingDir, "rootfs.ext4") + if err := imagepull.BuildExt4(ctx, d.runner, rootfsTree, rootfsExt4, sizeBytes); err != nil { + return model.Image{}, fmt.Errorf("build rootfs ext4: %w", err) + } + + stagedKernel, stagedInitrd, stagedModules, err := imagemgr.StageBootArtifacts(ctx, d.runner, stagingDir, kernelPath, initrdPath, modulesDir) + if err != nil { + return model.Image{}, fmt.Errorf("stage boot artifacts: %w", err) + } + + if err := os.Rename(stagingDir, finalDir); err != nil { + return model.Image{}, fmt.Errorf("publish artifact dir: %w", err) + } + cleanupStaging = false + + now := model.Now() + image = model.Image{ + ID: id, + Name: imgName, + Managed: true, + ArtifactDir: finalDir, + RootfsPath: filepath.Join(finalDir, filepath.Base(rootfsExt4)), + KernelPath: rebaseUnder(stagedKernel, stagingDir, finalDir), + InitrdPath: rebaseUnder(stagedInitrd, stagingDir, finalDir), + ModulesDir: rebaseUnder(stagedModules, stagingDir, finalDir), + CreatedAt: now, + UpdatedAt: now, + } + if err := d.store.UpsertImage(ctx, image); err != nil { + _ = os.RemoveAll(finalDir) + return model.Image{}, err + } + return image, nil +} + +// runPullAndFlatten is the seam tests substitute. nil → real implementation. +func (d *Daemon) runPullAndFlatten(ctx context.Context, ref, cacheDir, destDir string) error { + if d.pullAndFlatten != nil { + return d.pullAndFlatten(ctx, ref, cacheDir, destDir) + } + pulled, err := imagepull.Pull(ctx, ref, cacheDir) + if err != nil { + return err + } + return imagepull.Flatten(ctx, pulled, destDir) +} + +// nameSanitize keeps lowercase alphanumerics + hyphens, collapses runs. +var nameSanitizeRE = regexp.MustCompile(`[^a-z0-9]+`) + +// defaultImageNameFromRef derives a friendly name like "debian-bookworm" +// from "docker.io/library/debian:bookworm". Returns "" if it can't. +func defaultImageNameFromRef(ref name.Reference) string { + repo := ref.Context().RepositoryStr() // e.g. library/debian + parts := strings.Split(repo, "/") + base := parts[len(parts)-1] + + suffix := "" + switch r := ref.(type) { + case name.Tag: + if t := r.TagStr(); t != "" && t != "latest" { + suffix = "-" + t + } + case name.Digest: + // take the first 12 hex chars after sha256: + d := r.DigestStr() + if i := strings.Index(d, ":"); i >= 0 && len(d) >= i+13 { + suffix = "-" + d[i+1:i+13] + } + } + + out := nameSanitizeRE.ReplaceAllString(strings.ToLower(base+suffix), "-") + out = strings.Trim(out, "-") + return out +} + +// rebaseUnder rewrites a path that points inside oldRoot to point inside +// newRoot. Empty input returns empty (kept by StageBootArtifacts when an +// optional artifact is absent). +func rebaseUnder(path, oldRoot, newRoot string) string { + if path == "" { + return "" + } + if rel, err := filepath.Rel(oldRoot, path); err == nil && !strings.HasPrefix(rel, "..") { + return filepath.Join(newRoot, rel) + } + return path +} + +// dirSizeBytes returns the sum of regular-file sizes under root, following +// no symlinks (lstat). Suitable for sizing an ext4 image. +func dirSizeBytes(root string) (int64, error) { + var total int64 + err := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.Type().IsRegular() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + total += info.Size() + return nil + }) + return total, err +} diff --git a/internal/daemon/images_pull_test.go b/internal/daemon/images_pull_test.go new file mode 100644 index 0000000..ac3af8a --- /dev/null +++ b/internal/daemon/images_pull_test.go @@ -0,0 +1,191 @@ +package daemon + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "banger/internal/api" + "banger/internal/model" + "banger/internal/paths" + "banger/internal/system" + + "github.com/google/go-containerregistry/pkg/name" +) + +func writeFakeKernelTriple(t *testing.T) (kernelPath, initrdPath, modulesDir string) { + t.Helper() + dir := t.TempDir() + kernelPath = filepath.Join(dir, "vmlinux") + if err := os.WriteFile(kernelPath, []byte("kernel"), 0o644); err != nil { + t.Fatal(err) + } + initrdPath = filepath.Join(dir, "initrd.img") + if err := os.WriteFile(initrdPath, []byte("initrd"), 0o644); err != nil { + t.Fatal(err) + } + modulesDir = filepath.Join(dir, "modules") + if err := os.MkdirAll(modulesDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(modulesDir, "modules.dep"), []byte(""), 0o644); err != nil { + t.Fatal(err) + } + return +} + +// stubPullAndFlatten writes a fixed file tree into destDir, simulating a +// successful OCI pull without the network or tarball machinery. +func stubPullAndFlatten(_ context.Context, _ string, _ string, destDir string) error { + if err := os.MkdirAll(filepath.Join(destDir, "etc"), 0o755); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(destDir, "etc", "hello"), []byte("world"), 0o644); err != nil { + return err + } + return os.WriteFile(filepath.Join(destDir, "marker"), []byte("ok"), 0o644) +} + +func TestPullImageHappyPath(t *testing.T) { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + t.Skip("mkfs.ext4 not available; skipping") + } + imagesDir := t.TempDir() + cacheDir := t.TempDir() + kernel, initrd, modules := writeFakeKernelTriple(t) + + d := &Daemon{ + layout: paths.Layout{ImagesDir: imagesDir, OCICacheDir: cacheDir}, + store: openDaemonStore(t), + runner: system.NewRunner(), + pullAndFlatten: stubPullAndFlatten, + } + + image, err := d.PullImage(context.Background(), api.ImagePullParams{ + Ref: "docker.io/library/debian:bookworm", + KernelPath: kernel, + InitrdPath: initrd, + ModulesDir: modules, + }) + if err != nil { + t.Fatalf("PullImage: %v", err) + } + + if image.Name != "debian-bookworm" { + t.Errorf("Name = %q, want debian-bookworm", image.Name) + } + if !image.Managed { + t.Errorf("expected Managed=true") + } + if image.ArtifactDir == "" || !strings.HasPrefix(image.ArtifactDir, imagesDir) { + t.Errorf("ArtifactDir = %q, want under %q", image.ArtifactDir, imagesDir) + } + + for _, rel := range []string{"rootfs.ext4", "kernel", "initrd.img", "modules"} { + if _, err := os.Stat(filepath.Join(image.ArtifactDir, rel)); err != nil { + t.Errorf("missing artifact %s: %v", rel, err) + } + } + + // Staging dir should be gone after publish. + stagings, _ := filepath.Glob(filepath.Join(imagesDir, "*.staging")) + if len(stagings) != 0 { + t.Errorf("staging dirs left behind: %v", stagings) + } +} + +func TestPullImageRejectsExistingName(t *testing.T) { + imagesDir := t.TempDir() + kernel, _, _ := writeFakeKernelTriple(t) + + d := &Daemon{ + layout: paths.Layout{ImagesDir: imagesDir, OCICacheDir: t.TempDir()}, + store: openDaemonStore(t), + runner: system.NewRunner(), + pullAndFlatten: stubPullAndFlatten, + } + // Seed a preexisting image with the would-be derived name. + id, _ := model.NewID() + if err := d.store.UpsertImage(context.Background(), model.Image{ + ID: id, + Name: "debian-bookworm", + CreatedAt: model.Now(), + UpdatedAt: model.Now(), + }); err != nil { + t.Fatal(err) + } + + _, err := d.PullImage(context.Background(), api.ImagePullParams{ + Ref: "docker.io/library/debian:bookworm", + KernelPath: kernel, + }) + if err == nil || !strings.Contains(err.Error(), "already exists") { + t.Fatalf("expected already-exists error, got %v", err) + } +} + +func TestPullImageRequiresKernel(t *testing.T) { + d := &Daemon{ + layout: paths.Layout{ImagesDir: t.TempDir(), OCICacheDir: t.TempDir()}, + store: openDaemonStore(t), + runner: system.NewRunner(), + pullAndFlatten: stubPullAndFlatten, + } + _, err := d.PullImage(context.Background(), api.ImagePullParams{ + Ref: "docker.io/library/debian:bookworm", + }) + if err == nil || !strings.Contains(err.Error(), "kernel") { + t.Fatalf("expected kernel-required error, got %v", err) + } +} + +func TestPullImageCleansStagingOnFailure(t *testing.T) { + imagesDir := t.TempDir() + kernel, _, _ := writeFakeKernelTriple(t) + failureSeam := func(_ context.Context, _ string, _ string, _ string) error { + return errors.New("network borked") + } + + d := &Daemon{ + layout: paths.Layout{ImagesDir: imagesDir, OCICacheDir: t.TempDir()}, + store: openDaemonStore(t), + runner: system.NewRunner(), + pullAndFlatten: failureSeam, + } + _, err := d.PullImage(context.Background(), api.ImagePullParams{ + Ref: "docker.io/library/debian:bookworm", + KernelPath: kernel, + }) + if err == nil || !strings.Contains(err.Error(), "network borked") { + t.Fatalf("expected propagated pull error, got %v", err) + } + stagings, _ := filepath.Glob(filepath.Join(imagesDir, "*.staging")) + if len(stagings) != 0 { + t.Errorf("staging dir left behind on failure: %v", stagings) + } +} + +func TestDefaultImageNameFromRef(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"docker.io/library/debian:bookworm", "debian-bookworm"}, + {"alpine:3.20", "alpine-3-20"}, + {"docker.io/library/debian", "debian"}, + {"ghcr.io/some/org/my-image:v2.1", "my-image-v2-1"}, + } + for _, tc := range cases { + ref, err := name.ParseReference(tc.in) + if err != nil { + t.Fatalf("parse %s: %v", tc.in, err) + } + if got := defaultImageNameFromRef(ref); got != tc.want { + t.Errorf("defaultImageNameFromRef(%s) = %q, want %q", tc.in, got, tc.want) + } + } +}