thaloco/banger
1
Fork 0
Code Pull requests Releases Packages Activity Actions

Speed up VM create with work seeds

Browse source 
Beat VM create wall time without changing VM semantics.

Generate a work-seed ext4 sidecar during image builds and rootfs rebuilds, then clone and resize that seed for each new VM instead of rebuilding /root from scratch. Plumb the new seed artifact through config, runtime metadata, store state, runtime-bundle defaults, doctor checks, and default-image reconciliation so older images still fall back cleanly.

Add a daemon TAP pool to keep idle bridge-attached devices warm, expose stage timing in lifecycle logs, add a create/SSH benchmark script plus Make target, and teach verify.sh that tap-pool-* devices are reusable capacity rather than cleanup leaks.

Validated with go test ./..., make build, ./verify.sh, and make bench-create ARGS="--runs 2".
This commit is contained in:
Thales Maciel 2026-03-18 21:22:12 -03:00
parent a14a80fd6b
commit c8d9a122f9
No known key found for this signature in database
GPG key ID: 33112E6833C34679
24 changed files with 695 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

29
internal/cli/banger.go
View file

@ -87,7 +87,34 @@ func newInternalCommand() *cobra.Command {
Hidden: true,
RunE: helpNoArgs,
}
cmd.AddCommand(newInternalNATCommand())
cmd.AddCommand(newInternalNATCommand(), newInternalWorkSeedCommand())
return cmd
}
func newInternalWorkSeedCommand() *cobra.Command {
var rootfsPath string
var outPath string
cmd := &cobra.Command{
Use: "work-seed",
Hidden: true,
Args: noArgsUsage("usage: banger internal work-seed --rootfs <path> [--out <path>]"),
RunE: func(cmd *cobra.Command, args []string) error {
rootfsPath = strings.TrimSpace(rootfsPath)
outPath = strings.TrimSpace(outPath)
if rootfsPath == "" {
return errors.New("rootfs path is required")
}
if outPath == "" {
outPath = system.WorkSeedPath(rootfsPath)
}
if err := system.EnsureSudo(cmd.Context()); err != nil {
return err
}
return system.BuildWorkSeedImage(cmd.Context(), system.NewRunner(), rootfsPath, outPath)
},
}
cmd.Flags().StringVar(&rootfsPath, "rootfs", "", "rootfs image path")
cmd.Flags().StringVar(&outPath, "out", "", "output work-seed image path")
return cmd
}

26
internal/config/config.go
View file

@ -4,6 +4,7 @@ import (
"errors"
"os"
"path/filepath"
"strings"
"time"
toml "github.com/pelletier/go-toml"
@ -22,6 +23,7 @@ type fileConfig struct {
NamegenPath string `toml:"namegen_path"`
CustomizeScript string `toml:"customize_script"`
VSockPingHelper string `toml:"vsock_ping_helper_path"`
DefaultWorkSeed string `toml:"default_work_seed"`
DefaultImageName string `toml:"default_image_name"`
DefaultRootfs string `toml:"default_rootfs"`
DefaultBaseRootfs string `toml:"default_base_rootfs"`
@ -35,6 +37,7 @@ type fileConfig struct {
BridgeName string `toml:"bridge_name"`
BridgeIP string `toml:"bridge_ip"`
CIDR string `toml:"cidr"`
TapPoolSize int `toml:"tap_pool_size"`
DefaultDNS string `toml:"default_dns"`
}
@ -47,6 +50,7 @@ func Load(layout paths.Layout) (model.DaemonConfig, error) {
BridgeName: model.DefaultBridgeName,
BridgeIP: model.DefaultBridgeIP,
CIDR: model.DefaultCIDR,
TapPoolSize: 4,
DefaultDNS: model.DefaultDNS,
DefaultImageName: "default",
}
@ -91,6 +95,9 @@ func Load(layout paths.Layout) (model.DaemonConfig, error) {
if file.VSockPingHelper != "" {
cfg.VSockPingHelperPath = file.VSockPingHelper
}
if file.DefaultWorkSeed != "" {
cfg.DefaultWorkSeed = file.DefaultWorkSeed
}
if file.DefaultImageName != "" {
cfg.DefaultImageName = file.DefaultImageName
}
@ -121,6 +128,9 @@ func Load(layout paths.Layout) (model.DaemonConfig, error) {
if file.CIDR != "" {
cfg.CIDR = file.CIDR
}
if file.TapPoolSize > 0 {
cfg.TapPoolSize = file.TapPoolSize
}
if file.DefaultDNS != "" {
cfg.DefaultDNS = file.DefaultDNS
}
@ -176,6 +186,9 @@ func applyRuntimeDefaults(cfg *model.DaemonConfig) error {
cfg.DefaultRootfs,
)
}
if cfg.DefaultWorkSeed == "" && cfg.DefaultRootfs != "" {
cfg.DefaultWorkSeed = firstExistingRuntimePath(associatedWorkSeedPath(cfg.DefaultRootfs))
}
return nil
}
@ -185,6 +198,7 @@ func applyBundleMetadataDefaults(cfg *model.DaemonConfig, runtimeDir string, met
cfg.NamegenPath = defaultRuntimePath(cfg.NamegenPath, runtimeDir, meta.NamegenPath)
cfg.CustomizeScript = defaultRuntimePath(cfg.CustomizeScript, runtimeDir, meta.CustomizeScript)
cfg.VSockPingHelperPath = defaultRuntimePath(cfg.VSockPingHelperPath, runtimeDir, meta.VSockPingHelperPath)
cfg.DefaultWorkSeed = defaultRuntimePath(cfg.DefaultWorkSeed, runtimeDir, meta.DefaultWorkSeed)
cfg.DefaultKernel = defaultRuntimePath(cfg.DefaultKernel, runtimeDir, meta.DefaultKernel)
cfg.DefaultInitrd = defaultRuntimePath(cfg.DefaultInitrd, runtimeDir, meta.DefaultInitrd)
cfg.DefaultModulesDir = defaultRuntimePath(cfg.DefaultModulesDir, runtimeDir, meta.DefaultModulesDir)
@ -199,6 +213,7 @@ func applyLegacyRuntimeDefaults(cfg *model.DaemonConfig) {
cfg.NamegenPath = defaultRuntimePath(cfg.NamegenPath, cfg.RuntimeDir, "namegen")
cfg.CustomizeScript = defaultRuntimePath(cfg.CustomizeScript, cfg.RuntimeDir, "customize.sh")
cfg.VSockPingHelperPath = defaultRuntimePath(cfg.VSockPingHelperPath, cfg.RuntimeDir, "banger-vsock-pingd")
cfg.DefaultWorkSeed = defaultRuntimePath(cfg.DefaultWorkSeed, cfg.RuntimeDir, "rootfs-docker.work-seed.ext4")
cfg.DefaultKernel = defaultRuntimePath(cfg.DefaultKernel, cfg.RuntimeDir, "wtf/root/boot/vmlinux-6.8.0-94-generic")
cfg.DefaultInitrd = defaultRuntimePath(cfg.DefaultInitrd, cfg.RuntimeDir, "wtf/root/boot/initrd.img-6.8.0-94-generic")
cfg.DefaultModulesDir = defaultRuntimePath(cfg.DefaultModulesDir, cfg.RuntimeDir, "wtf/root/lib/modules/6.8.0-94-generic")
@ -223,3 +238,14 @@ func firstExistingRuntimePath(paths ...string) string {
}
return ""
}
func associatedWorkSeedPath(rootfsPath string) string {
rootfsPath = strings.TrimSpace(rootfsPath)
if rootfsPath == "" {
return ""
}
if strings.HasSuffix(rootfsPath, ".ext4") {
return strings.TrimSuffix(rootfsPath, ".ext4") + ".work-seed.ext4"
}
return rootfsPath + ".work-seed"
}

9
internal/config/config_test.go
View file

@ -20,6 +20,7 @@ func TestLoadDerivesArtifactPathsFromRuntimeDir(t *testing.T) {
VSockPingHelperPath: "bin/banger-vsock-pingd",
DefaultPackages: "config/packages.apt",
DefaultRootfs: "images/rootfs-docker.ext4",
DefaultWorkSeed: "images/rootfs-docker.work-seed.ext4",
DefaultKernel: "kernels/vmlinux",
DefaultInitrd: "kernels/initrd.img",
DefaultModulesDir: "modules/current",
@ -32,6 +33,7 @@ func TestLoadDerivesArtifactPathsFromRuntimeDir(t *testing.T) {
meta.VSockPingHelperPath,
meta.DefaultPackages,
meta.DefaultRootfs,
meta.DefaultWorkSeed,
meta.DefaultKernel,
meta.DefaultInitrd,
filepath.Join(meta.DefaultModulesDir, "modules.dep"),
@ -79,6 +81,9 @@ func TestLoadDerivesArtifactPathsFromRuntimeDir(t *testing.T) {
if cfg.DefaultRootfs != filepath.Join(runtimeDir, meta.DefaultRootfs) {
t.Fatalf("DefaultRootfs = %q", cfg.DefaultRootfs)
}
if cfg.DefaultWorkSeed != filepath.Join(runtimeDir, meta.DefaultWorkSeed) {
t.Fatalf("DefaultWorkSeed = %q", cfg.DefaultWorkSeed)
}
if cfg.DefaultBaseRootfs != filepath.Join(runtimeDir, meta.DefaultRootfs) {
t.Fatalf("DefaultBaseRootfs = %q", cfg.DefaultBaseRootfs)
}
@ -106,6 +111,7 @@ func TestLoadFallsBackToLegacyRuntimeLayoutWithoutBundleMetadata(t *testing.T) {
"banger-vsock-pingd",
"packages.apt",
"rootfs-docker.ext4",
"rootfs-docker.work-seed.ext4",
"wtf/root/boot/vmlinux-6.8.0-94-generic",
"wtf/root/boot/initrd.img-6.8.0-94-generic",
"wtf/root/lib/modules/6.8.0-94-generic/modules.dep",
@ -131,6 +137,9 @@ func TestLoadFallsBackToLegacyRuntimeLayoutWithoutBundleMetadata(t *testing.T) {
if cfg.VSockPingHelperPath != filepath.Join(runtimeDir, "banger-vsock-pingd") {
t.Fatalf("VSockPingHelperPath = %q", cfg.VSockPingHelperPath)
}
if cfg.DefaultWorkSeed != filepath.Join(runtimeDir, "rootfs-docker.work-seed.ext4") {
t.Fatalf("DefaultWorkSeed = %q", cfg.DefaultWorkSeed)
}
if cfg.DefaultKernel != filepath.Join(runtimeDir, "wtf/root/boot/vmlinux-6.8.0-94-generic") {
t.Fatalf("DefaultKernel = %q", cfg.DefaultKernel)
}

29
internal/daemon/capabilities.go
View file

@ -4,6 +4,7 @@ import (
"context"
"errors"
"net"
"os"
"strings"
"banger/internal/firecracker"
@ -150,10 +151,21 @@ type workDiskCapability struct{}
func (workDiskCapability) Name() string { return "work-disk" }
func (workDiskCapability) AddStartPreflight(_ context.Context, _ *Daemon, checks *system.Preflight, vm model.VMRecord, _ model.Image) {
func (workDiskCapability) AddStartPreflight(_ context.Context, _ *Daemon, checks *system.Preflight, vm model.VMRecord, image model.Image) {
if exists(vm.Runtime.WorkDiskPath) {
return
}
imageSeed := ""
if image.RootfsPath != "" {
imageSeed = image.WorkSeedPath
}
if exists(imageSeed) {
if info, err := os.Stat(imageSeed); err == nil && vm.Spec.WorkDiskSizeBytes > info.Size() {
checks.RequireCommand("e2fsck", toolHint("e2fsck"))
checks.RequireCommand("resize2fs", toolHint("resize2fs"))
}
return
}
for _, command := range []string{"mkfs.ext4", "mount", "umount", "cp"} {
checks.RequireCommand(command, toolHint(command))
}
@ -178,16 +190,23 @@ func (workDiskCapability) ContributeMachine(cfg *firecracker.MachineConfig, vm m
})
}
func (workDiskCapability) PrepareHost(ctx context.Context, d *Daemon, vm *model.VMRecord, _ model.Image) error {
return d.ensureWorkDisk(ctx, vm)
func (workDiskCapability) PrepareHost(ctx context.Context, d *Daemon, vm *model.VMRecord, image model.Image) error {
return d.ensureWorkDisk(ctx, vm, image)
}
func (workDiskCapability) AddDoctorChecks(_ context.Context, _ *Daemon, report *system.Report) {
func (workDiskCapability) AddDoctorChecks(_ context.Context, d *Daemon, report *system.Report) {
if strings.TrimSpace(d.config.DefaultWorkSeed) != "" && exists(d.config.DefaultWorkSeed) {
checks := system.NewPreflight()
checks.RequireFile(d.config.DefaultWorkSeed, "default work seed image", `rebuild the default runtime rootfs to regenerate the /root seed`)
report.AddPreflight("feature /root work disk", checks, "seeded /root work disk artifact available")
return
}
checks := system.NewPreflight()
for _, command := range []string{"mkfs.ext4", "mount", "umount", "cp"} {
checks.RequireCommand(command, toolHint(command))
}
report.AddPreflight("feature /root work disk", checks, "guest /root work disk tooling available")
report.AddPreflight("feature /root work disk", checks, "fallback /root work disk tooling available")
report.AddWarn("feature /root work disk", "default image has no work-seed artifact; new VM creates will be slower until the image is rebuilt")
}
type dnsCapability struct{}

12
internal/daemon/daemon.go
View file

@ -34,6 +34,9 @@ type Daemon struct {
mu sync.Mutex
vmLocksMu sync.Mutex
vmLocks map[string]*sync.Mutex
tapPoolMu sync.Mutex
tapPool []string
tapPoolNext int
closing chan struct{}
once sync.Once
pid int
@ -92,6 +95,11 @@ func Open(ctx context.Context) (d *Daemon, err error) {
d.logger.Error("daemon open failed", "stage", "reconcile", "error", err.Error())
return nil, err
}
if err = d.initializeTapPool(ctx); err != nil {
d.logger.Error("daemon open failed", "stage", "initialize_tap_pool", "error", err.Error())
return nil, err
}
go d.ensureTapPool(context.Background())
return d, nil
}
@ -436,7 +444,7 @@ func (d *Daemon) ensureDefaultImage(ctx context.Context) error {
return err
}
if d.logger != nil {
d.logger.Info("default image reconciled", append(imageLogAttrs(updated), "previous_rootfs_path", image.RootfsPath, "previous_kernel_path", image.KernelPath)...)
d.logger.Info("default image reconciled", append(imageLogAttrs(updated), "previous_rootfs_path", image.RootfsPath, "previous_work_seed_path", image.WorkSeedPath, "previous_kernel_path", image.KernelPath)...)
}
return nil
case errors.Is(err, sql.ErrNoRows):
@ -471,6 +479,7 @@ func (d *Daemon) desiredDefaultImage() (model.Image, bool) {
Managed: false,
ArtifactDir: "",
RootfsPath: rootfs,
WorkSeedPath: d.config.DefaultWorkSeed,
KernelPath: kernel,
InitrdPath: d.config.DefaultInitrd,
ModulesDir: d.config.DefaultModulesDir,
@ -484,6 +493,7 @@ func defaultImageMatches(current, desired model.Image) bool {
current.Managed == desired.Managed &&
current.ArtifactDir == desired.ArtifactDir &&
current.RootfsPath == desired.RootfsPath &&
current.WorkSeedPath == desired.WorkSeedPath &&
current.KernelPath == desired.KernelPath &&
current.InitrdPath == desired.InitrdPath &&
current.ModulesDir == desired.ModulesDir &&

92
internal/daemon/fastpath_test.go Normal file
View file

@ -0,0 +1,92 @@
package daemon
import (
"context"
"errors"
"os"
"path/filepath"
"strconv"
"testing"
"banger/internal/model"
)
func TestEnsureWorkDiskClonesSeedImageAndResizes(t *testing.T) {
t.Parallel()
vmDir := t.TempDir()
seedPath := filepath.Join(t.TempDir(), "root.work-seed.ext4")
if err := os.WriteFile(seedPath, []byte("seed-data"), 0o644); err != nil {
t.Fatalf("WriteFile(seed): %v", err)
}
workDiskPath := filepath.Join(vmDir, "root.ext4")
runner := &scriptedRunner{
t: t,
steps: []runnerStep{
{call: runnerCall{name: "e2fsck", args: []string{"-p", "-f", workDiskPath}}},
{call: runnerCall{name: "resize2fs", args: []string{workDiskPath}}},
},
}
d := &Daemon{runner: runner}
vm := testVM("seeded", "image-seeded", "172.16.0.60")
vm.Runtime.WorkDiskPath = workDiskPath
vm.Spec.WorkDiskSizeBytes = 2 * 1024 * 1024
image := testImage("image-seeded")
image.WorkSeedPath = seedPath
if err := d.ensureWorkDisk(context.Background(), &vm, image); err != nil {
t.Fatalf("ensureWorkDisk: %v", err)
}
runner.assertExhausted()
info, err := os.Stat(workDiskPath)
if err != nil {
t.Fatalf("Stat(work disk): %v", err)
}
if info.Size() != vm.Spec.WorkDiskSizeBytes {
t.Fatalf("work disk size = %d, want %d", info.Size(), vm.Spec.WorkDiskSizeBytes)
}
}
func TestTapPoolWarmsAndReusesIdleTap(t *testing.T) {
t.Parallel()
runner := &scriptedRunner{
t: t,
steps: []runnerStep{
{call: runnerCall{name: "ip", args: []string{"link", "show", "tap-pool-0"}}, err: errors.New("exit status 1")},
sudoStep("", nil, "ip", "tuntap", "add", "dev", "tap-pool-0", "mode", "tap", "user", strconv.Itoa(os.Getuid()), "group", strconv.Itoa(os.Getgid())),
sudoStep("", nil, "ip", "link", "set", "tap-pool-0", "master", model.DefaultBridgeName),
sudoStep("", nil, "ip", "link", "set", "tap-pool-0", "up"),
sudoStep("", nil, "ip", "link", "set", model.DefaultBridgeName, "up"),
},
}
d := &Daemon{
runner: runner,
config: model.DaemonConfig{
BridgeName: model.DefaultBridgeName,
TapPoolSize: 1,
},
closing: make(chan struct{}),
}
d.ensureTapPool(context.Background())
tapName, err := d.acquireTap(context.Background(), "tap-fallback")
if err != nil {
t.Fatalf("acquireTap: %v", err)
}
if tapName != "tap-pool-0" {
t.Fatalf("tapName = %q, want tap-pool-0", tapName)
}
if err := d.releaseTap(context.Background(), tapName); err != nil {
t.Fatalf("releaseTap: %v", err)
}
tapName, err = d.acquireTap(context.Background(), "tap-fallback")
if err != nil {
t.Fatalf("acquireTap second time: %v", err)
}
if tapName != "tap-pool-0" {
t.Fatalf("tapName second = %q, want tap-pool-0", tapName)
}
runner.assertExhausted()
}

10
internal/daemon/images.go
View file

@ -60,6 +60,7 @@ func (d *Daemon) BuildImage(ctx context.Context, params api.ImageBuildParams) (i
}
defer logFile.Close()
rootfsPath := filepath.Join(artifactDir, "rootfs.ext4")
workSeedPath := filepath.Join(artifactDir, "work-seed.ext4")
kernelPath := params.KernelPath
if kernelPath == "" {
kernelPath = d.config.DefaultKernel
@ -90,10 +91,17 @@ func (d *Daemon) BuildImage(ctx context.Context, params api.ImageBuildParams) (i
}
op.stage("launch_builder", "build_log_path", buildLogPath, "artifact_dir", artifactDir)
if err := d.runImageBuild(ctx, spec); err != nil {
_ = logFile.Sync()
_ = os.RemoveAll(artifactDir)
return model.Image{}, err
}
if err := system.BuildWorkSeedImage(ctx, d.runner, rootfsPath, workSeedPath); err != nil {
_ = logFile.Sync()
_ = os.RemoveAll(artifactDir)
return model.Image{}, err
}
if err := writePackagesMetadata(rootfsPath, d.config.DefaultPackagesFile); err != nil {
_ = logFile.Sync()
_ = os.RemoveAll(artifactDir)
return model.Image{}, err
}
@ -103,6 +111,7 @@ func (d *Daemon) BuildImage(ctx context.Context, params api.ImageBuildParams) (i
Managed: true,
ArtifactDir: artifactDir,
RootfsPath: rootfsPath,
WorkSeedPath: workSeedPath,
KernelPath: kernelPath,
InitrdPath: initrdPath,
ModulesDir: modulesDir,
@ -119,6 +128,7 @@ func (d *Daemon) BuildImage(ctx context.Context, params api.ImageBuildParams) (i
if d.logger != nil {
d.logger.Info("image build log preserved", append(imageLogAttrs(image), "build_log_path", buildLogPath)...)
}
_ = logFile.Sync()
return image, nil
}

36
internal/daemon/logger.go
View file

@ -35,14 +35,16 @@ func parseLogLevel(raw string) (slog.Level, string, error) {
}
}
func (d *Daemon) beginOperation(name string, attrs ...any) operationLog {
func (d *Daemon) beginOperation(name string, attrs ...any) *operationLog {
if d.logger != nil {
d.logger.Info("operation started", append([]any{"operation", name}, attrs...)...)
}
return operationLog{
now := time.Now()
return &operationLog{
logger: d.logger,
name: name,
started: time.Now(),
started: now,
last: now,
attrs: append([]any(nil), attrs...),
}
}
@ -51,22 +53,35 @@ type operationLog struct {
logger *slog.Logger
name string
started time.Time
last time.Time
attrs []any
}
func (o operationLog) stage(stage string, attrs ...any) {
o.log(slog.LevelInfo, "operation stage", append([]any{"stage", stage}, attrs...)...)
func (o *operationLog) stage(stage string, attrs ...any) {
now := time.Now()
o.log(slog.LevelInfo, "operation stage", append([]any{
"stage", stage,
"since_start_ms", now.Sub(o.started).Milliseconds(),
"since_last_stage_ms", now.Sub(o.last).Milliseconds(),
}, attrs...)...)
o.last = now
}
func (o operationLog) debugStage(stage string, attrs ...any) {
o.log(slog.LevelDebug, "operation stage", append([]any{"stage", stage}, attrs...)...)
func (o *operationLog) debugStage(stage string, attrs ...any) {
now := time.Now()
o.log(slog.LevelDebug, "operation stage", append([]any{
"stage", stage,
"since_start_ms", now.Sub(o.started).Milliseconds(),
"since_last_stage_ms", now.Sub(o.last).Milliseconds(),
}, attrs...)...)
o.last = now
}
func (o operationLog) done(attrs ...any) {
func (o *operationLog) done(attrs ...any) {
o.log(slog.LevelInfo, "operation completed", append([]any{"duration_ms", time.Since(o.started).Milliseconds()}, attrs...)...)
}
func (o operationLog) fail(err error, attrs ...any) error {
func (o *operationLog) fail(err error, attrs ...any) error {
if err == nil {
return nil
}
@ -118,6 +133,9 @@ func imageLogAttrs(image model.Image) []any {
if image.RootfsPath != "" {
attrs = append(attrs, "rootfs_path", image.RootfsPath)
}
if image.WorkSeedPath != "" {
attrs = append(attrs, "work_seed_path", image.WorkSeedPath)
}
return attrs
}

2
internal/daemon/logger_test.go
View file

@ -141,7 +141,7 @@ func TestBuildImagePreservesBuildLogOnFailure(t *testing.T) {
}
binDir := t.TempDir()
for _, name := range []string{"sudo", "ip", "pgrep", "chown", "chmod", "kill", "iptables", "sysctl", "e2fsck", "resize2fs"} {
for _, name := range []string{"sudo", "ip", "pgrep", "chown", "chmod", "kill", "iptables", "sysctl", "e2fsck", "resize2fs", "mkfs.ext4", "mount", "umount", "cp"} {
writeFakeExecutable(t, filepath.Join(binDir, name))
}
t.Setenv("PATH", binDir)

3
internal/daemon/preflight.go
View file

@ -75,6 +75,9 @@ func (d *Daemon) addImageBuildPrereqs(ctx context.Context, checks *system.Prefli
for _, command := range []string{"sudo", "ip", "pgrep", "chown", "chmod", "kill"} {
checks.RequireCommand(command, toolHint(command))
}
for _, command := range []string{"mkfs.ext4", "mount", "umount", "cp"} {
checks.RequireCommand(command, toolHint(command))
}
checks.RequireExecutable(d.config.FirecrackerBin, "firecracker binary", hint)
checks.RequireFile(d.config.SSHKeyPath, "ssh private key", `set "ssh_key_path" or refresh the runtime bundle`)
checks.RequireExecutable(d.config.VSockPingHelperPath, "vsock ping helper", `run 'make build' or refresh the runtime bundle`)

121
internal/daemon/tap_pool.go Normal file
View file

@ -0,0 +1,121 @@
package daemon
import (
"context"
"fmt"
"strconv"
"strings"
)
const tapPoolPrefix = "tap-pool-"
func (d *Daemon) initializeTapPool(ctx context.Context) error {
if d.config.TapPoolSize <= 0 || d.store == nil {
return nil
}
vms, err := d.store.ListVMs(ctx)
if err != nil {
return err
}
next := 0
for _, vm := range vms {
if index, ok := parseTapPoolIndex(vm.Runtime.TapDevice); ok && index >= next {
next = index + 1
}
}
d.tapPoolMu.Lock()
d.tapPoolNext = next
d.tapPoolMu.Unlock()
return nil
}
func (d *Daemon) ensureTapPool(ctx context.Context) {
if d.config.TapPoolSize <= 0 {
return
}
for {
select {
case <-ctx.Done():
return
case <-d.closing:
return
default:
}
d.tapPoolMu.Lock()
if len(d.tapPool) >= d.config.TapPoolSize {
d.tapPoolMu.Unlock()
return
}
tapName := fmt.Sprintf("%s%d", tapPoolPrefix, d.tapPoolNext)
d.tapPoolNext++
d.tapPoolMu.Unlock()
if err := d.createTap(ctx, tapName); err != nil {
if d.logger != nil {
d.logger.Warn("tap pool warmup failed", "tap_device", tapName, "error", err.Error())
}
return
}
d.tapPoolMu.Lock()
d.tapPool = append(d.tapPool, tapName)
d.tapPoolMu.Unlock()
if d.logger != nil {
d.logger.Debug("tap added to idle pool", "tap_device", tapName)
}
}
}
func (d *Daemon) acquireTap(ctx context.Context, fallbackName string) (string, error) {
d.tapPoolMu.Lock()
if n := len(d.tapPool); n > 0 {
tapName := d.tapPool[n-1]
d.tapPool = d.tapPool[:n-1]
d.tapPoolMu.Unlock()
return tapName, nil
}
d.tapPoolMu.Unlock()
if err := d.createTap(ctx, fallbackName); err != nil {
return "", err
}
return fallbackName, nil
}
func (d *Daemon) releaseTap(ctx context.Context, tapName string) error {
tapName = strings.TrimSpace(tapName)
if tapName == "" {
return nil
}
if isTapPoolName(tapName) {
d.tapPoolMu.Lock()
if len(d.tapPool) < d.config.TapPoolSize {
d.tapPool = append(d.tapPool, tapName)
d.tapPoolMu.Unlock()
return nil
}
d.tapPoolMu.Unlock()
}
_, err := d.runner.RunSudo(ctx, "ip", "link", "del", tapName)
if err == nil {
go d.ensureTapPool(context.Background())
}
return err
}
func isTapPoolName(tapName string) bool {
return strings.HasPrefix(strings.TrimSpace(tapName), tapPoolPrefix)
}
func parseTapPoolIndex(tapName string) (int, bool) {
if !isTapPoolName(tapName) {
return 0, false
}
value, err := strconv.Atoi(strings.TrimPrefix(strings.TrimSpace(tapName), tapPoolPrefix))
if err != nil {
return 0, false
}
return value, true
}

54
internal/daemon/vm.go
View file

@ -188,8 +188,8 @@ func (d *Daemon) startVMLocked(ctx context.Context, vm model.VMRecord, image mod
shortID := system.ShortID(vm.ID)
apiSock := filepath.Join(d.layout.RuntimeDir, "fc-"+shortID+".sock")
tap := "tap-fc-" + shortID
dmName := "fc-rootfs-" + shortID
tapName := "tap-fc-" + shortID
if strings.TrimSpace(vm.Runtime.VSockPath) == "" {
vm.Runtime.VSockPath = defaultVSockPath(d.layout.RuntimeDir, vm.ID)
}
@ -221,7 +221,6 @@ func (d *Daemon) startVMLocked(ctx context.Context, vm model.VMRecord, image mod
vm.Runtime.DMName = handles.DMName
vm.Runtime.DMDev = handles.DMDev
vm.Runtime.APISockPath = apiSock
vm.Runtime.TapDevice = tap
vm.Runtime.State = model.VMStateRunning
vm.State = model.VMStateRunning
vm.Runtime.LastError = ""
@ -247,10 +246,12 @@ func (d *Daemon) startVMLocked(ctx context.Context, vm model.VMRecord, image mod
if err := d.prepareCapabilityHosts(ctx, &vm, image); err != nil {
return cleanupOnErr(err)
}
op.stage("tap", "tap_device", tap)
if err := d.createTap(ctx, tap); err != nil {
op.stage("tap")
tap, err := d.acquireTap(ctx, tapName)
if err != nil {
return cleanupOnErr(err)
}
vm.Runtime.TapDevice = tap
op.stage("metrics_file", "metrics_path", vm.Runtime.MetricsPath)
if err := os.WriteFile(vm.Runtime.MetricsPath, nil, 0o644); err != nil {
return cleanupOnErr(err)
@ -766,10 +767,28 @@ func (d *Daemon) patchRootOverlay(ctx context.Context, vm model.VMRecord, image
return nil
}
func (d *Daemon) ensureWorkDisk(ctx context.Context, vm *model.VMRecord) error {
func (d *Daemon) ensureWorkDisk(ctx context.Context, vm *model.VMRecord, image model.Image) error {
if exists(vm.Runtime.WorkDiskPath) {
return nil
}
if exists(image.WorkSeedPath) {
if err := system.CopyFilePreferClone(image.WorkSeedPath, vm.Runtime.WorkDiskPath); err != nil {
return err
}
seedInfo, err := os.Stat(image.WorkSeedPath)
if err != nil {
return err
}
if vm.Spec.WorkDiskSizeBytes < seedInfo.Size() {
return fmt.Errorf("requested work disk size %d is smaller than seed image %d", vm.Spec.WorkDiskSizeBytes, seedInfo.Size())
}
if vm.Spec.WorkDiskSizeBytes > seedInfo.Size() {
if err := system.ResizeExt4Image(ctx, d.runner, vm.Runtime.WorkDiskPath, vm.Spec.WorkDiskSizeBytes); err != nil {
return err
}
}
return nil
}
if _, err := d.runner.Run(ctx, "truncate", "-s", strconv.FormatInt(vm.Spec.WorkDiskSizeBytes, 10), vm.Runtime.WorkDiskPath); err != nil {
return err
}
@ -936,15 +955,6 @@ func (d *Daemon) cleanupRuntime(ctx context.Context, vm model.VMRecord, preserve
return err
}
}
if vm.Runtime.TapDevice != "" {
_, _ = d.runner.RunSudo(ctx, "ip", "link", "del", vm.Runtime.TapDevice)
}
if vm.Runtime.APISockPath != "" {
_ = os.Remove(vm.Runtime.APISockPath)
}
if vm.Runtime.VSockPath != "" {
_ = os.Remove(vm.Runtime.VSockPath)
}
snapshotErr := d.cleanupDMSnapshot(ctx, dmSnapshotHandles{
BaseLoop: vm.Runtime.BaseLoop,
COWLoop: vm.Runtime.COWLoop,
@ -952,10 +962,20 @@ func (d *Daemon) cleanupRuntime(ctx context.Context, vm model.VMRecord, preserve
DMDev: vm.Runtime.DMDev,
})
featureErr := d.cleanupCapabilityState(ctx, vm)
if !preserveDisks && vm.Runtime.VMDir != "" {
return errors.Join(snapshotErr, featureErr, os.RemoveAll(vm.Runtime.VMDir))
var tapErr error
if vm.Runtime.TapDevice != "" {
tapErr = d.releaseTap(ctx, vm.Runtime.TapDevice)
}
return errors.Join(snapshotErr, featureErr)
if vm.Runtime.APISockPath != "" {
_ = os.Remove(vm.Runtime.APISockPath)
}
if vm.Runtime.VSockPath != "" {
_ = os.Remove(vm.Runtime.VSockPath)
}
if !preserveDisks && vm.Runtime.VMDir != "" {
return errors.Join(snapshotErr, featureErr, tapErr, os.RemoveAll(vm.Runtime.VMDir))
}
return errors.Join(snapshotErr, featureErr, tapErr)
}
func clearRuntimeHandles(vm *model.VMRecord) {

3
internal/model/types.go
View file

@ -42,12 +42,14 @@ type DaemonConfig struct {
NamegenPath string
CustomizeScript string
VSockPingHelperPath string
DefaultWorkSeed string
AutoStopStaleAfter time.Duration
StatsPollInterval time.Duration
MetricsPollInterval time.Duration
BridgeName string
BridgeIP string
CIDR string
TapPoolSize int
DefaultDNS string
DefaultImageName string
DefaultRootfs string
@ -64,6 +66,7 @@ type Image struct {
Managed bool `json:"managed"`
ArtifactDir string `json:"artifact_dir,omitempty"`
RootfsPath string `json:"rootfs_path"`
WorkSeedPath string `json:"work_seed_path,omitempty"`
KernelPath string `json:"kernel_path"`
InitrdPath string `json:"initrd_path,omitempty"`
ModulesDir string `json:"modules_dir,omitempty"`

4
internal/runtimebundle/bundle.go
View file

@ -38,6 +38,7 @@ type BundleMetadata struct {
DefaultPackages string `json:"default_packages_file" toml:"default_packages_file"`
DefaultRootfs string `json:"default_rootfs" toml:"default_rootfs"`
DefaultBaseRootfs string `json:"default_base_rootfs,omitempty" toml:"default_base_rootfs"`
DefaultWorkSeed string `json:"default_work_seed,omitempty" toml:"default_work_seed"`
DefaultKernel string `json:"default_kernel" toml:"default_kernel"`
DefaultInitrd string `json:"default_initrd,omitempty" toml:"default_initrd"`
DefaultModulesDir string `json:"default_modules_dir,omitempty" toml:"default_modules_dir"`
@ -233,6 +234,7 @@ func validateBundleMetadata(runtimeDir string, meta BundleMetadata) error {
{meta.DefaultPackages, "default_packages_file", true},
{meta.DefaultRootfs, "default_rootfs", true},
{meta.DefaultBaseRootfs, "default_base_rootfs", false},
{meta.DefaultWorkSeed, "default_work_seed", false},
{meta.DefaultKernel, "default_kernel", true},
{meta.DefaultInitrd, "default_initrd", false},
{meta.DefaultModulesDir, "default_modules_dir", false},
@ -271,6 +273,7 @@ func metadataArchiveBytes(runtimeDir string, meta BundleMetadata) ([]byte, error
strings.TrimSpace(meta.DefaultPackages) == "" &&
strings.TrimSpace(meta.DefaultRootfs) == "" &&
strings.TrimSpace(meta.DefaultBaseRootfs) == "" &&
strings.TrimSpace(meta.DefaultWorkSeed) == "" &&
strings.TrimSpace(meta.DefaultKernel) == "" &&
strings.TrimSpace(meta.DefaultInitrd) == "" &&
strings.TrimSpace(meta.DefaultModulesDir) == "" {
@ -291,6 +294,7 @@ func normalizeBundleMetadata(meta BundleMetadata) BundleMetadata {
meta.DefaultPackages = strings.TrimSpace(meta.DefaultPackages)
meta.DefaultRootfs = strings.TrimSpace(meta.DefaultRootfs)
meta.DefaultBaseRootfs = strings.TrimSpace(meta.DefaultBaseRootfs)
meta.DefaultWorkSeed = strings.TrimSpace(meta.DefaultWorkSeed)
meta.DefaultKernel = strings.TrimSpace(meta.DefaultKernel)
meta.DefaultInitrd = strings.TrimSpace(meta.DefaultInitrd)
meta.DefaultModulesDir = strings.TrimSpace(meta.DefaultModulesDir)

48
internal/store/store.go
View file

@ -74,6 +74,7 @@ func (s *Store) migrate() error {
managed INTEGER NOT NULL DEFAULT 0,
artifact_dir TEXT,
rootfs_path TEXT NOT NULL,
work_seed_path TEXT,
kernel_path TEXT NOT NULL,
initrd_path TEXT,
modules_dir TEXT,
@ -103,6 +104,9 @@ func (s *Store) migrate() error {
return err
}
}
if err := ensureColumnExists(s.db, "images", "work_seed_path", "TEXT"); err != nil {
return err
}
return nil
}
@ -111,14 +115,15 @@ func (s *Store) UpsertImage(ctx context.Context, image model.Image) error {
defer s.writeMu.Unlock()
const query = `
INSERT INTO images (
id, name, managed, artifact_dir, rootfs_path, kernel_path, initrd_path,
id, name, managed, artifact_dir, rootfs_path, work_seed_path, kernel_path, initrd_path,
modules_dir, packages_path, build_size, docker, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET
name=excluded.name,
managed=excluded.managed,
artifact_dir=excluded.artifact_dir,
rootfs_path=excluded.rootfs_path,
work_seed_path=excluded.work_seed_path,
kernel_path=excluded.kernel_path,
initrd_path=excluded.initrd_path,
modules_dir=excluded.modules_dir,
@ -132,6 +137,7 @@ func (s *Store) UpsertImage(ctx context.Context, image model.Image) error {
boolToInt(image.Managed),
image.ArtifactDir,
image.RootfsPath,
image.WorkSeedPath,
image.KernelPath,
image.InitrdPath,
image.ModulesDir,
@ -145,15 +151,15 @@ func (s *Store) UpsertImage(ctx context.Context, image model.Image) error {
}
func (s *Store) GetImageByName(ctx context.Context, name string) (model.Image, error) {
return s.getImage(ctx, "SELECT id, name, managed, artifact_dir, rootfs_path, kernel_path, initrd_path, modules_dir, packages_path, build_size, docker, created_at, updated_at FROM images WHERE name = ?", name)
return s.getImage(ctx, "SELECT id, name, managed, artifact_dir, rootfs_path, work_seed_path, kernel_path, initrd_path, modules_dir, packages_path, build_size, docker, created_at, updated_at FROM images WHERE name = ?", name)
}
func (s *Store) GetImageByID(ctx context.Context, id string) (model.Image, error) {
return s.getImage(ctx, "SELECT id, name, managed, artifact_dir, rootfs_path, kernel_path, initrd_path, modules_dir, packages_path, build_size, docker, created_at, updated_at FROM images WHERE id = ?", id)
return s.getImage(ctx, "SELECT id, name, managed, artifact_dir, rootfs_path, work_seed_path, kernel_path, initrd_path, modules_dir, packages_path, build_size, docker, created_at, updated_at FROM images WHERE id = ?", id)
}
func (s *Store) ListImages(ctx context.Context) ([]model.Image, error) {
rows, err := s.db.QueryContext(ctx, "SELECT id, name, managed, artifact_dir, rootfs_path, kernel_path, initrd_path, modules_dir, packages_path, build_size, docker, created_at, updated_at FROM images ORDER BY created_at ASC")
rows, err := s.db.QueryContext(ctx, "SELECT id, name, managed, artifact_dir, rootfs_path, work_seed_path, kernel_path, initrd_path, modules_dir, packages_path, build_size, docker, created_at, updated_at FROM images ORDER BY created_at ASC")
if err != nil {
return nil, err
}
@ -330,6 +336,7 @@ type scanner interface {
func scanImageRow(row scanner) (model.Image, error) {
var image model.Image
var managed, docker int
var workSeedPath sql.NullString
var createdAt, updatedAt string
err := row.Scan(
&image.ID,
@ -337,6 +344,7 @@ func scanImageRow(row scanner) (model.Image, error) {
&managed,
&image.ArtifactDir,
&image.RootfsPath,
&workSeedPath,
&image.KernelPath,
&image.InitrdPath,
&image.ModulesDir,
@ -351,6 +359,7 @@ func scanImageRow(row scanner) (model.Image, error) {
}
image.Managed = managed == 1
image.Docker = docker == 1
image.WorkSeedPath = workSeedPath.String
image.CreatedAt, err = time.Parse(time.RFC3339, createdAt)
if err != nil {
return image, err
@ -417,6 +426,35 @@ func scanVMInto(row scanner) (model.VMRecord, error) {
return vm, nil
}
func ensureColumnExists(db *sql.DB, table, column, columnType string) error {
rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table))
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var (
cid int
name string
valueType string
notNull int
defaultV sql.NullString
pk int
)
if err := rows.Scan(&cid, &name, &valueType, &notNull, &defaultV, &pk); err != nil {
return err
}
if name == column {
return nil
}
}
if err := rows.Err(); err != nil {
return err
}
_, err = db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", table, column, columnType))
return err
}
func boolToInt(value bool) int {
if value {
return 1

1
internal/store/store_test.go
View file

@ -340,6 +340,7 @@ func sampleImage(name string) model.Image {
Managed: true,
ArtifactDir: "/artifacts/" + name,
RootfsPath: "/images/" + name + ".ext4",
WorkSeedPath: "/images/" + name + ".work-seed.ext4",
KernelPath: "/kernels/" + name,
InitrdPath: "/initrd/" + name,
ModulesDir: "/modules/" + name,

81
internal/system/files.go
View file

@ -1,14 +1,22 @@
package system
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"golang.org/x/sys/unix"
)
const (
minWorkSeedBytes int64 = 512 * 1024 * 1024
workSeedSlackBytes int64 = 256 * 1024 * 1024
workSeedRoundBytes int64 = 64 * 1024 * 1024
)
func CopyFilePreferClone(sourcePath, targetPath string) error {
source, err := os.Open(sourcePath)
if err != nil {
@ -48,6 +56,79 @@ func CopyFilePreferClone(sourcePath, targetPath string) error {
return nil
}
func WorkSeedPath(rootfsPath string) string {
rootfsPath = strings.TrimSpace(rootfsPath)
if rootfsPath == "" {
return ""
}
if strings.HasSuffix(rootfsPath, ".ext4") {
return strings.TrimSuffix(rootfsPath, ".ext4") + ".work-seed.ext4"
}
return rootfsPath + ".work-seed"
}
func BuildWorkSeedImage(ctx context.Context, runner CommandRunner, rootfsPath, outPath string) error {
rootMount, cleanupRoot, err := MountTempDir(ctx, runner, rootfsPath, true)
if err != nil {
return err
}
defer cleanupRoot()
rootHome := filepath.Join(rootMount, "root")
sizeBytes, err := estimateWorkSeedSize(rootHome)
if err != nil {
return err
}
if err := os.RemoveAll(outPath); err != nil && !os.IsNotExist(err) {
return err
}
file, err := os.OpenFile(outPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
return err
}
if err := file.Close(); err != nil {
return err
}
if err := os.Truncate(outPath, sizeBytes); err != nil {
return err
}
if _, err := runner.Run(ctx, "mkfs.ext4", "-F", outPath); err != nil {
return err
}
workMount, cleanupWork, err := MountTempDir(ctx, runner, outPath, false)
if err != nil {
return err
}
defer cleanupWork()
return CopyDirContents(ctx, runner, rootHome, workMount, true)
}
func estimateWorkSeedSize(rootHome string) (int64, error) {
var usedBytes int64
err := filepath.Walk(rootHome, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode().IsRegular() {
usedBytes += info.Size()
}
return nil
})
if err != nil {
return 0, err
}
sizeBytes := usedBytes*2 + workSeedSlackBytes
if sizeBytes < minWorkSeedBytes {
sizeBytes = minWorkSeedBytes
}
if rem := sizeBytes % workSeedRoundBytes; rem != 0 {
sizeBytes += workSeedRoundBytes - rem
}
return sizeBytes, nil
}
func ReadNormalizedLines(path string) ([]string, error) {
data, err := os.ReadFile(path)
if err != nil {