diff --git a/internal/cli/banger.go b/internal/cli/banger.go index 53edbd6..78b38c2 100644 --- a/internal/cli/banger.go +++ b/internal/cli/banger.go @@ -1,12 +1,16 @@ package cli import ( + "archive/tar" "bytes" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" "io" + "io/fs" "net" "net/url" "os" @@ -25,7 +29,9 @@ import ( "banger/internal/daemon" "banger/internal/guest" "banger/internal/hostnat" + "banger/internal/imagecat" "banger/internal/imagepreset" + "banger/internal/imagepull" "banger/internal/model" "banger/internal/paths" "banger/internal/rpc" @@ -35,6 +41,7 @@ import ( "banger/internal/vmdns" "banger/internal/vsockagent" + "github.com/klauspost/compress/zstd" "github.com/spf13/cobra" ) @@ -213,6 +220,7 @@ func newInternalCommand() *cobra.Command { newInternalFirecrackerPathCommand(), newInternalVSockAgentPathCommand(), newInternalPackagesCommand(), + newInternalMakeBundleCommand(), ) return cmd } @@ -309,6 +317,265 @@ func newInternalPackagesCommand() *cobra.Command { return cmd } +func newInternalMakeBundleCommand() *cobra.Command { + var ( + rootfsTarPath string + name string + distro string + arch string + kernelRef string + description string + sizeSpec string + outPath string + ) + cmd := &cobra.Command{ + Use: "make-bundle", + Hidden: true, + Short: "Build a banger image bundle (.tar.zst) from a flat rootfs tar", + Args: noArgsUsage("usage: banger internal make-bundle --rootfs-tar --name --out "), + RunE: func(cmd *cobra.Command, args []string) error { + return runInternalMakeBundle(cmd, internalMakeBundleOpts{ + rootfsTarPath: rootfsTarPath, + name: name, + distro: distro, + arch: arch, + kernelRef: kernelRef, + description: description, + sizeSpec: sizeSpec, + outPath: outPath, + }) + }, + } + cmd.Flags().StringVar(&rootfsTarPath, "rootfs-tar", "", "flat rootfs tar file, or '-' for stdin") + cmd.Flags().StringVar(&name, "name", "", "bundle name (filesystem-safe identifier)") + cmd.Flags().StringVar(&distro, "distro", "", "distro label (e.g. debian)") + cmd.Flags().StringVar(&arch, "arch", "x86_64", "architecture label") + cmd.Flags().StringVar(&kernelRef, "kernel-ref", "", "kernelcat entry name this image pairs with") + cmd.Flags().StringVar(&description, "description", "", "short description") + cmd.Flags().StringVar(&sizeSpec, "size", "", "rootfs ext4 size (e.g. 4G); defaults to tree size + 25%") + cmd.Flags().StringVar(&outPath, "out", "", "output bundle path (.tar.zst)") + return cmd +} + +type internalMakeBundleOpts struct { + rootfsTarPath string + name string + distro string + arch string + kernelRef string + description string + sizeSpec string + outPath string +} + +func runInternalMakeBundle(cmd *cobra.Command, opts internalMakeBundleOpts) error { + if err := imagecat.ValidateName(opts.name); err != nil { + return err + } + if strings.TrimSpace(opts.rootfsTarPath) == "" { + return errors.New("--rootfs-tar is required") + } + if strings.TrimSpace(opts.outPath) == "" { + return errors.New("--out is required") + } + if strings.TrimSpace(opts.arch) == "" { + opts.arch = "x86_64" + } + + var sizeBytes int64 + if s := strings.TrimSpace(opts.sizeSpec); s != "" { + n, err := model.ParseSize(s) + if err != nil { + return fmt.Errorf("parse --size: %w", err) + } + sizeBytes = n + } + + ctx := cmd.Context() + stagingRoot, err := os.MkdirTemp("", "banger-mkbundle-") + if err != nil { + return err + } + defer os.RemoveAll(stagingRoot) + rootfsTree := filepath.Join(stagingRoot, "rootfs") + if err := os.MkdirAll(rootfsTree, 0o755); err != nil { + return err + } + + // Open tar input (file or stdin). + var tarReader io.Reader + if opts.rootfsTarPath == "-" { + tarReader = cmd.InOrStdin() + } else { + f, err := os.Open(opts.rootfsTarPath) + if err != nil { + return fmt.Errorf("open rootfs tar: %w", err) + } + defer f.Close() + tarReader = f + } + + fmt.Fprintln(cmd.ErrOrStderr(), "[make-bundle] extracting rootfs") + meta, err := imagepull.FlattenTar(ctx, tarReader, rootfsTree) + if err != nil { + return fmt.Errorf("flatten rootfs: %w", err) + } + + if sizeBytes <= 0 { + treeSize, err := dirSize(rootfsTree) + if err != nil { + return fmt.Errorf("size rootfs tree: %w", err) + } + sizeBytes = treeSize + treeSize/4 + if sizeBytes < imagepull.MinExt4Size { + sizeBytes = imagepull.MinExt4Size + } + } + + ext4Path := filepath.Join(stagingRoot, imagecat.RootfsFilename) + runner := system.NewRunner() + fmt.Fprintf(cmd.ErrOrStderr(), "[make-bundle] building rootfs.ext4 (%d bytes)\n", sizeBytes) + if err := imagepull.BuildExt4(ctx, runner, rootfsTree, ext4Path, sizeBytes); err != nil { + return fmt.Errorf("build ext4: %w", err) + } + fmt.Fprintln(cmd.ErrOrStderr(), "[make-bundle] applying ownership fixup") + if err := imagepull.ApplyOwnership(ctx, runner, ext4Path, meta); err != nil { + return fmt.Errorf("apply ownership: %w", err) + } + fmt.Fprintln(cmd.ErrOrStderr(), "[make-bundle] injecting guest agents") + vsockBin, err := paths.CompanionBinaryPath("banger-vsock-agent") + if err != nil { + return fmt.Errorf("locate vsock agent: %w", err) + } + if err := imagepull.InjectGuestAgents(ctx, runner, ext4Path, imagepull.GuestAgentAssets{VsockAgentBin: vsockBin}); err != nil { + return fmt.Errorf("inject guest agents: %w", err) + } + + // Write manifest.json. + manifest := imagecat.Manifest{ + Name: opts.name, + Distro: strings.TrimSpace(opts.distro), + Arch: opts.arch, + KernelRef: strings.TrimSpace(opts.kernelRef), + Description: strings.TrimSpace(opts.description), + } + manifestPath := filepath.Join(stagingRoot, imagecat.ManifestFilename) + manifestData, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(manifestPath, append(manifestData, '\n'), 0o644); err != nil { + return err + } + + fmt.Fprintln(cmd.ErrOrStderr(), "[make-bundle] packaging bundle") + if err := writeBundleTarZst(opts.outPath, ext4Path, manifestPath); err != nil { + return fmt.Errorf("write bundle: %w", err) + } + + sum, err := sha256HexFile(opts.outPath) + if err != nil { + return err + } + stat, err := os.Stat(opts.outPath) + if err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "bundle: %s\nsha256: %s\nsize: %d\n", opts.outPath, sum, stat.Size()) + return nil +} + +// dirSize returns the sum of regular-file sizes under root (no symlink follow). +func dirSize(root string) (int64, error) { + var total int64 + err := filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.Type().IsRegular() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + total += info.Size() + return nil + }) + return total, err +} + +// writeBundleTarZst packages rootfs.ext4 + manifest.json into outPath as tar+zstd. +func writeBundleTarZst(outPath, rootfsPath, manifestPath string) error { + if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil { + return err + } + out, err := os.OpenFile(outPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer out.Close() + zw, err := zstd.NewWriter(out, zstd.WithEncoderLevel(zstd.SpeedBestCompression)) + if err != nil { + return err + } + tw := tar.NewWriter(zw) + for _, src := range []struct{ path, name string }{ + {rootfsPath, imagecat.RootfsFilename}, + {manifestPath, imagecat.ManifestFilename}, + } { + if err := writeBundleFile(tw, src.path, src.name); err != nil { + _ = tw.Close() + _ = zw.Close() + return err + } + } + if err := tw.Close(); err != nil { + _ = zw.Close() + return err + } + if err := zw.Close(); err != nil { + return err + } + return out.Close() +} + +func writeBundleFile(tw *tar.Writer, src, name string) error { + f, err := os.Open(src) + if err != nil { + return err + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + return err + } + if err := tw.WriteHeader(&tar.Header{ + Name: name, + Size: fi.Size(), + Mode: 0o644, + Typeflag: tar.TypeReg, + ModTime: fi.ModTime(), + }); err != nil { + return err + } + _, err = io.Copy(tw, f) + return err +} + +func sha256HexFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + func newInternalWorkSeedCommand() *cobra.Command { var rootfsPath string var outPath string diff --git a/internal/cli/make_bundle_test.go b/internal/cli/make_bundle_test.go new file mode 100644 index 0000000..fdce359 --- /dev/null +++ b/internal/cli/make_bundle_test.go @@ -0,0 +1,320 @@ +package cli + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "banger/internal/imagecat" + + "github.com/klauspost/compress/zstd" +) + +func TestInternalMakeBundleFlagsExist(t *testing.T) { + root := NewBangerCommand() + internal, _, err := root.Find([]string{"internal"}) + if err != nil { + t.Fatalf("find internal: %v", err) + } + mk, _, err := internal.Find([]string{"make-bundle"}) + if err != nil { + t.Fatalf("find make-bundle: %v", err) + } + for _, name := range []string{"rootfs-tar", "name", "distro", "arch", "kernel-ref", "description", "size", "out"} { + if mk.Flags().Lookup(name) == nil { + t.Errorf("missing flag %q", name) + } + } +} + +func TestMakeBundleRequiresName(t *testing.T) { + cmd := NewBangerCommand() + cmd.SetArgs([]string{"internal", "make-bundle", "--rootfs-tar", "some.tar", "--out", "out.tar.zst"}) + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&bytes.Buffer{}) + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), "image name is required") { + t.Fatalf("execute error = %v, want image-name-required", err) + } +} + +func TestMakeBundleRequiresRootfsTar(t *testing.T) { + cmd := NewBangerCommand() + cmd.SetArgs([]string{"internal", "make-bundle", "--name", "x", "--out", "out.tar.zst"}) + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&bytes.Buffer{}) + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), "--rootfs-tar is required") { + t.Fatalf("execute error = %v, want --rootfs-tar required", err) + } +} + +func TestMakeBundleRequiresOut(t *testing.T) { + cmd := NewBangerCommand() + cmd.SetArgs([]string{"internal", "make-bundle", "--name", "x", "--rootfs-tar", "-"}) + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&bytes.Buffer{}) + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), "--out is required") { + t.Fatalf("execute error = %v, want --out required", err) + } +} + +func TestWriteBundleTarZstRoundTrip(t *testing.T) { + stage := t.TempDir() + rootfsContent := []byte("fake-rootfs-bytes") + rootfsPath := filepath.Join(stage, "rootfs.ext4") + if err := os.WriteFile(rootfsPath, rootfsContent, 0o644); err != nil { + t.Fatal(err) + } + manifest := imagecat.Manifest{Name: "debian-bookworm", Distro: "debian"} + manifestJSON, _ := json.Marshal(manifest) + manifestPath := filepath.Join(stage, "manifest.json") + if err := os.WriteFile(manifestPath, manifestJSON, 0o644); err != nil { + t.Fatal(err) + } + + bundlePath := filepath.Join(stage, "bundle.tar.zst") + if err := writeBundleTarZst(bundlePath, rootfsPath, manifestPath); err != nil { + t.Fatalf("writeBundleTarZst: %v", err) + } + + // Decode and verify. + raw, err := os.Open(bundlePath) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { raw.Close() }) + zr, err := zstd.NewReader(raw) + if err != nil { + t.Fatal(err) + } + tr := tar.NewReader(zr) + got := map[string][]byte{} + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(tr) + got[hdr.Name] = b + } + if !bytes.Equal(got[imagecat.RootfsFilename], rootfsContent) { + t.Errorf("rootfs mismatch: got %q want %q", got[imagecat.RootfsFilename], rootfsContent) + } + if !bytes.Equal(got[imagecat.ManifestFilename], manifestJSON) { + t.Errorf("manifest mismatch: got %q want %q", got[imagecat.ManifestFilename], manifestJSON) + } +} + +func TestSha256HexFile(t *testing.T) { + dir := t.TempDir() + content := []byte("hello world") + p := filepath.Join(dir, "f") + if err := os.WriteFile(p, content, 0o644); err != nil { + t.Fatal(err) + } + got, err := sha256HexFile(p) + if err != nil { + t.Fatal(err) + } + expected := sha256.Sum256(content) + if got != hex.EncodeToString(expected[:]) { + t.Fatalf("sha256 = %q, want %q", got, hex.EncodeToString(expected[:])) + } +} + +func TestDirSize(t *testing.T) { + dir := t.TempDir() + _ = os.MkdirAll(filepath.Join(dir, "sub"), 0o755) + _ = os.WriteFile(filepath.Join(dir, "a"), []byte("abc"), 0o644) // 3 + _ = os.WriteFile(filepath.Join(dir, "sub", "b"), []byte("defgh"), 0o644) // 5 + // Symlink must not be counted. + _ = os.Symlink(filepath.Join(dir, "a"), filepath.Join(dir, "link")) + n, err := dirSize(dir) + if err != nil { + t.Fatal(err) + } + if n != 8 { + t.Fatalf("dirSize = %d, want 8", n) + } +} + +// TestMakeBundleEndToEnd exercises the full pipeline against a tiny +// synthesized rootfs tar. Skips if any external tool (mkfs.ext4 / +// debugfs) or the companion banger-vsock-agent binary is unavailable. +func TestMakeBundleEndToEnd(t *testing.T) { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + t.Skip("mkfs.ext4 not installed") + } + if _, err := exec.LookPath("debugfs"); err != nil { + t.Skip("debugfs not installed") + } + // Build companion binary if the build tree doesn't already have one. + buildDir := findBuildBinDir(t) + if buildDir == "" { + t.Skip("build/bin not found; run `make build` to enable this test") + } + if _, err := os.Stat(filepath.Join(buildDir, "banger-vsock-agent")); err != nil { + t.Skip("banger-vsock-agent not in build/bin; run `make build`") + } + // Ensure the banger binary also exists so CompanionBinaryPath + // resolves (it looks alongside the banger binary). + if _, err := os.Stat(filepath.Join(buildDir, "banger")); err != nil { + t.Skip("banger not in build/bin; run `make build`") + } + + // Build a minimal rootfs tar: just /etc/os-release and /tmp (a dir). + dir := t.TempDir() + tarPath := filepath.Join(dir, "rootfs.tar") + if err := writeMinimalTar(tarPath); err != nil { + t.Fatal(err) + } + outPath := filepath.Join(dir, "bundle.tar.zst") + + // Invoke via the cobra command to cover arg handling too. + cmd := NewBangerCommand() + cmd.SetArgs([]string{ + "internal", "make-bundle", + "--rootfs-tar", tarPath, + "--name", "test-bundle", + "--distro", "debian", + "--arch", "x86_64", + "--kernel-ref", "generic-6.12", + "--size", "64M", + "--out", outPath, + }) + var stderr bytes.Buffer + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&stderr) + // paths.CompanionBinaryPath looks alongside the banger binary, but + // the test binary lives elsewhere. Use the env override instead. + t.Setenv("BANGER_VSOCK_AGENT_BIN", filepath.Join(buildDir, "banger-vsock-agent")) + cmd.SetContext(context.Background()) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v\nstderr:\n%s", err, stderr.String()) + } + + if stat, err := os.Stat(outPath); err != nil { + t.Fatalf("output not written: %v", err) + } else if stat.Size() < 1024 { + t.Fatalf("output suspiciously small: %d bytes", stat.Size()) + } + + // Verify we can fetch-reparse it (mirror of imagecat.Fetch logic, + // but reading straight from disk instead of HTTP). + extractDir := t.TempDir() + verifyBundle(t, outPath, extractDir) +} + +// findBuildBinDir returns the absolute path to the project's build/bin, +// or "" if it can't be located. Walks up from CWD to find go.mod. +func findBuildBinDir(t *testing.T) string { + t.Helper() + cwd, err := os.Getwd() + if err != nil { + return "" + } + for d := cwd; d != "/" && d != "."; d = filepath.Dir(d) { + if _, err := os.Stat(filepath.Join(d, "go.mod")); err == nil { + return filepath.Join(d, "build", "bin") + } + } + return "" +} + +func writeMinimalTar(path string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + tw := tar.NewWriter(f) + defer tw.Close() + + // /etc dir + if err := tw.WriteHeader(&tar.Header{ + Name: "etc/", Typeflag: tar.TypeDir, Mode: 0o755, Uid: 0, Gid: 0, + }); err != nil { + return err + } + // /etc/os-release + body := []byte(`ID=debian` + "\n" + `PRETTY_NAME="banger test"` + "\n") + if err := tw.WriteHeader(&tar.Header{ + Name: "etc/os-release", Typeflag: tar.TypeReg, Mode: 0o644, + Size: int64(len(body)), Uid: 0, Gid: 0, + }); err != nil { + return err + } + if _, err := tw.Write(body); err != nil { + return err + } + // /tmp dir + return tw.WriteHeader(&tar.Header{ + Name: "tmp/", Typeflag: tar.TypeDir, Mode: 0o1777, Uid: 0, Gid: 0, + }) +} + +func verifyBundle(t *testing.T, bundlePath, extractDir string) { + t.Helper() + f, err := os.Open(bundlePath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + zr, err := zstd.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer zr.Close() + tr := tar.NewReader(zr) + seen := map[string]bool{} + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + dst := filepath.Join(extractDir, hdr.Name) + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + t.Fatal(err) + } + out, err := os.Create(dst) + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(out, tr); err != nil { + t.Fatal(err) + } + out.Close() + seen[hdr.Name] = true + } + if !seen[imagecat.RootfsFilename] || !seen[imagecat.ManifestFilename] { + t.Fatalf("bundle missing expected files: seen=%v", seen) + } + manifestData, err := os.ReadFile(filepath.Join(extractDir, imagecat.ManifestFilename)) + if err != nil { + t.Fatal(err) + } + var m imagecat.Manifest + if err := json.Unmarshal(manifestData, &m); err != nil { + t.Fatal(err) + } + if m.Name != "test-bundle" || m.KernelRef != "generic-6.12" || m.Distro != "debian" { + t.Fatalf("manifest = %+v", m) + } +} diff --git a/internal/imagepull/flatten.go b/internal/imagepull/flatten.go index 865dbe6..0582564 100644 --- a/internal/imagepull/flatten.go +++ b/internal/imagepull/flatten.go @@ -41,6 +41,42 @@ func newMetadata() Metadata { return Metadata{Entries: make(map[string]FileMeta)} } +// FlattenTar reads a single flat tar stream (e.g. the output of +// `docker export`) into destDir, returning per-file metadata. Unlike +// Flatten this does NOT treat the input as OCI-layered — there are no +// whiteouts, no previous layers. Whiteout markers, if they somehow +// appear, are still handled by applyEntry but should never be present +// in a docker-export stream. +// +// destDir must exist. Path-traversal members and symlink targets that +// escape destDir are rejected. +func FlattenTar(ctx context.Context, r io.Reader, destDir string) (Metadata, error) { + meta := newMetadata() + absDest, err := filepath.Abs(destDir) + if err != nil { + return meta, err + } + if err := ctx.Err(); err != nil { + return meta, err + } + tr := tar.NewReader(r) + for { + if err := ctx.Err(); err != nil { + return meta, err + } + hdr, err := tr.Next() + if err == io.EOF { + return meta, nil + } + if err != nil { + return meta, fmt.Errorf("read tar entry: %w", err) + } + if err := applyEntry(tr, hdr, absDest, &meta); err != nil { + return meta, err + } + } +} + // Flatten replays the image's layers in oldest-first order into destDir // and returns a Metadata record of each surviving file's tar-header // ownership/mode. destDir must exist and ideally be empty. Path-traversal