From 14d8563f3c5d375cdbbf9616f8f6ecabc530e561 Mon Sep 17 00:00:00 2001 From: Thales Maciel Date: Sat, 21 Mar 2026 21:54:18 -0300 Subject: [PATCH] Stop using kernel IP autoconfig for runtime VMs Avoid the Alpine boot stall caused by kernel ip= autoconfig running before virtio_net is available. Split runtime and image-build boot args so managed VMs boot without kernel network autoconfig, inject a static guest network config plus bootstrap script into the runtime overlay, and keep image builds on the old path for compatibility with existing base images. Preserve executable bits when patching guest files into ext4 images and add coverage for the new boot-arg split and guest network config generation. Validated with go test ./..., a rebuilt Alpine image, and a fresh alp-fast create/ssh check that brought vm.start down to about 2.7s. --- internal/daemon/imagebuild.go | 2 +- internal/daemon/vm.go | 11 ++++- internal/guestnet/assets/bootstrap.sh | 69 +++++++++++++++++++-------- internal/guestnet/guestnet.go | 24 +++++++++- internal/guestnet/guestnet_test.go | 24 ++++++++++ internal/system/system.go | 28 +++++++++-- internal/system/system_test.go | 56 ++++++++++++++++++++-- 7 files changed, 186 insertions(+), 28 deletions(-) create mode 100644 internal/guestnet/guestnet_test.go diff --git a/internal/daemon/imagebuild.go b/internal/daemon/imagebuild.go index f22f040..fbff27b 100644 --- a/internal/daemon/imagebuild.go +++ b/internal/daemon/imagebuild.go @@ -201,7 +201,7 @@ func (d *Daemon) startImageBuildVM(ctx context.Context, spec imageBuildSpec) (im MetricsPath: filepath.Join(filepath.Dir(spec.RootfsPath), "metrics.json"), KernelImagePath: spec.KernelPath, InitrdPath: spec.InitrdPath, - KernelArgs: system.BuildBootArgs(vm.Name, vm.GuestIP, d.config.BridgeIP, d.config.DefaultDNS), + KernelArgs: system.BuildBootArgsWithKernelIP(vm.Name, vm.GuestIP, d.config.BridgeIP, d.config.DefaultDNS), Drives: []firecracker.DriveConfig{{ ID: "rootfs", Path: spec.RootfsPath, diff --git a/internal/daemon/vm.go b/internal/daemon/vm.go index 3585db2..9ed25cd 100644 --- a/internal/daemon/vm.go +++ b/internal/daemon/vm.go @@ -16,6 +16,7 @@ import ( "banger/internal/firecracker" "banger/internal/guest" "banger/internal/guestconfig" + "banger/internal/guestnet" "banger/internal/model" "banger/internal/namegen" "banger/internal/system" @@ -287,7 +288,7 @@ func (d *Daemon) startVMLocked(ctx context.Context, vm model.VMRecord, image mod MetricsPath: vm.Runtime.MetricsPath, KernelImagePath: image.KernelPath, InitrdPath: image.InitrdPath, - KernelArgs: system.BuildBootArgs(vm.Name, vm.Runtime.GuestIP, d.config.BridgeIP, d.config.DefaultDNS), + KernelArgs: system.BuildBootArgs(vm.Name), Drives: []firecracker.DriveConfig{{ ID: "rootfs", Path: vm.Runtime.DMDev, @@ -765,6 +766,8 @@ func (d *Daemon) patchRootOverlay(ctx context.Context, vm model.VMRecord, image builder.WriteFile("/etc/resolv.conf", resolv) builder.WriteFile("/etc/hostname", hostname) builder.WriteFile("/etc/hosts", hosts) + builder.WriteFile(guestnet.ConfigPath, guestnet.ConfigFile(vm.Runtime.GuestIP, d.config.BridgeIP, d.config.DefaultDNS)) + builder.WriteFile(guestnet.GuestScriptPath, []byte(guestnet.BootstrapScript())) builder.WriteFile("/etc/ssh/sshd_config.d/99-banger.conf", sshdConfig) builder.DropMountTarget("/home") builder.DropMountTarget("/var") @@ -789,6 +792,12 @@ func (d *Daemon) patchRootOverlay(ctx context.Context, vm model.VMRecord, image files := builder.Files() for _, guestPath := range builder.FilePaths() { data := files[guestPath] + if guestPath == guestnet.GuestScriptPath { + if err := system.WriteExt4FileMode(ctx, d.runner, vm.Runtime.DMDev, guestPath, 0o755, data); err != nil { + return err + } + continue + } if err := system.WriteExt4File(ctx, d.runner, vm.Runtime.DMDev, guestPath, data); err != nil { return err } diff --git a/internal/guestnet/assets/bootstrap.sh b/internal/guestnet/assets/bootstrap.sh index 38a75ec..d35657f 100644 --- a/internal/guestnet/assets/bootstrap.sh +++ b/internal/guestnet/assets/bootstrap.sh @@ -2,25 +2,59 @@ set -eu PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +CONFIG_PATH="/etc/banger-network.conf" if ! command -v ip >/dev/null 2>&1; then exit 0 fi -cmdline="$(cat /proc/cmdline 2>/dev/null || true)" -ip_arg="" -for arg in $cmdline; do - case "$arg" in - ip=*) - ip_arg="${arg#ip=}" - break - ;; - esac -done +guest_ip="" +gateway_ip="" +netmask="" +iface_hint="" +dns1="" +dns2="" -if [ -z "$ip_arg" ]; then - exit 0 -fi +load_file_config() { + if [ ! -r "$CONFIG_PATH" ]; then + return 1 + fi + + # shellcheck disable=SC1090 + . "$CONFIG_PATH" + guest_ip="${GUEST_IP:-}" + gateway_ip="${GATEWAY_IP:-}" + netmask="${NETMASK:-}" + iface_hint="${INTERFACE:-}" + dns1="${DNS1:-}" + dns2="${DNS2:-}" + [ -n "$guest_ip" ] +} + +load_cmdline_config() { + cmdline="$(cat /proc/cmdline 2>/dev/null || true)" + ip_arg="" + for arg in $cmdline; do + case "$arg" in + ip=*) + ip_arg="${arg#ip=}" + break + ;; + esac + done + + if [ -z "$ip_arg" ]; then + return 1 + fi + + guest_ip="$(field 1)" + gateway_ip="$(field 3)" + netmask="$(field 4)" + iface_hint="$(field 6)" + dns1="$(field 8)" + dns2="$(field 9)" + [ -n "$guest_ip" ] +} field() { printf '%s' "$ip_arg" | cut -d: -f"$1" @@ -81,12 +115,9 @@ find_iface() { return 1 } -guest_ip="$(field 1)" -gateway_ip="$(field 3)" -netmask="$(field 4)" -iface_hint="$(field 6)" -dns1="$(field 8)" -dns2="$(field 9)" +if ! load_file_config; then + load_cmdline_config || exit 0 +fi if [ -z "$guest_ip" ]; then exit 0 diff --git a/internal/guestnet/guestnet.go b/internal/guestnet/guestnet.go index d4dfc6f..7f1d50e 100644 --- a/internal/guestnet/guestnet.go +++ b/internal/guestnet/guestnet.go @@ -1,11 +1,17 @@ package guestnet -import _ "embed" +import ( + _ "embed" + "strings" +) const ( GuestScriptPath = "/usr/local/libexec/banger-network-bootstrap" + ConfigPath = "/etc/banger-network.conf" SystemdServiceName = "banger-network.service" VoidCoreServicePath = "/etc/runit/core-services/20-banger-network.sh" + DefaultInterface = "eth0" + DefaultNetmask = "255.255.255.0" ) var ( @@ -21,6 +27,18 @@ func BootstrapScript() string { return bootstrapScript } +func ConfigFile(guestIP, gatewayIP, dns string) []byte { + lines := []string{ + "GUEST_IP=" + shellQuote(guestIP), + "GATEWAY_IP=" + shellQuote(gatewayIP), + "NETMASK=" + shellQuote(DefaultNetmask), + "INTERFACE=" + shellQuote(DefaultInterface), + "DNS1=" + shellQuote(dns), + "DNS2=''", + } + return []byte(strings.Join(lines, "\n") + "\n") +} + func SystemdServiceUnit() string { return systemdService } @@ -28,3 +46,7 @@ func SystemdServiceUnit() string { func VoidCoreService() string { return voidCoreService } + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", `'"'"'`) + "'" +} diff --git a/internal/guestnet/guestnet_test.go b/internal/guestnet/guestnet_test.go new file mode 100644 index 0000000..4ab53ca --- /dev/null +++ b/internal/guestnet/guestnet_test.go @@ -0,0 +1,24 @@ +package guestnet + +import ( + "strings" + "testing" +) + +func TestConfigFileIncludesStaticGuestNetworking(t *testing.T) { + t.Parallel() + + got := string(ConfigFile("172.16.0.2", "172.16.0.1", "1.1.1.1")) + for _, want := range []string{ + "GUEST_IP='172.16.0.2'", + "GATEWAY_IP='172.16.0.1'", + "NETMASK='255.255.255.0'", + "INTERFACE='eth0'", + "DNS1='1.1.1.1'", + "DNS2=''", + } { + if !strings.Contains(got, want) { + t.Fatalf("ConfigFile() missing %q\nconfig:\n%s", want, got) + } + } +} diff --git a/internal/system/system.go b/internal/system/system.go index 0559d8b..6368ed6 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -318,6 +318,10 @@ func ReadDebugFSText(ctx context.Context, runner CommandRunner, imagePath, guest } func WriteExt4File(ctx context.Context, runner CommandRunner, imagePath, guestPath string, data []byte) error { + return WriteExt4FileMode(ctx, runner, imagePath, guestPath, 0o600, data) +} + +func WriteExt4FileMode(ctx context.Context, runner CommandRunner, imagePath, guestPath string, mode os.FileMode, data []byte) error { tmp, err := os.CreateTemp("", "banger-ext4-*") if err != nil { return err @@ -330,9 +334,20 @@ func WriteExt4File(ctx context.Context, runner CommandRunner, imagePath, guestPa if err := tmp.Close(); err != nil { return err } + if err := os.Chmod(tmp.Name(), mode); err != nil { + return err + } _, _ = runner.RunSudo(ctx, "e2rm", imagePath+":"+guestPath) - _, err = runner.RunSudo(ctx, "e2cp", tmp.Name(), imagePath+":"+guestPath) - return err + if _, err := runner.RunSudo(ctx, "e2cp", tmp.Name(), imagePath+":"+guestPath); err != nil { + return err + } + if mode.Perm()&0o111 != 0 { + modeValue := fmt.Sprintf("%#o", uint32(0o100000|mode.Perm())) + if _, err := runner.RunSudo(ctx, "debugfs", "-w", "-R", "sif "+guestPath+" mode "+modeValue, imagePath); err != nil { + return err + } + } + return nil } func MountTempDir(ctx context.Context, runner CommandRunner, source string, readOnly bool) (string, func() error, error) { @@ -415,7 +430,14 @@ func UpdateFSTab(existing string) string { return strings.Join(out, "\n") + "\n" } -func BuildBootArgs(vmName, guestIP, bridgeIP, dns string) string { +func BuildBootArgs(vmName string) string { + return fmt.Sprintf( + "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rootfstype=ext4 rw hostname=%s systemd.mask=home.mount systemd.mask=var.mount", + vmName, + ) +} + +func BuildBootArgsWithKernelIP(vmName, guestIP, bridgeIP, dns string) string { return fmt.Sprintf( "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rootfstype=ext4 rw ip=%s::%s:255.255.255.0:%s:eth0:off:%s hostname=%s systemd.mask=home.mount systemd.mask=var.mount", guestIP, diff --git a/internal/system/system_test.go b/internal/system/system_test.go index 4da8ba7..5f69a53 100644 --- a/internal/system/system_test.go +++ b/internal/system/system_test.go @@ -167,16 +167,26 @@ func TestReadNormalizedLines(t *testing.T) { } } -func TestBuildBootArgsIncludesHostnameInIPField(t *testing.T) { +func TestBuildBootArgsOmitsKernelIPAutoconfig(t *testing.T) { t.Parallel() - got := BuildBootArgs("devbox", "172.16.0.2", "172.16.0.1", "1.1.1.1") - want := "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rootfstype=ext4 rw ip=172.16.0.2::172.16.0.1:255.255.255.0:devbox:eth0:off:1.1.1.1 hostname=devbox systemd.mask=home.mount systemd.mask=var.mount" + got := BuildBootArgs("devbox") + want := "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rootfstype=ext4 rw hostname=devbox systemd.mask=home.mount systemd.mask=var.mount" if got != want { t.Fatalf("BuildBootArgs() = %q, want %q", got, want) } } +func TestBuildBootArgsWithKernelIPIncludesHostnameInIPField(t *testing.T) { + t.Parallel() + + got := BuildBootArgsWithKernelIP("devbox", "172.16.0.2", "172.16.0.1", "1.1.1.1") + want := "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rootfstype=ext4 rw ip=172.16.0.2::172.16.0.1:255.255.255.0:devbox:eth0:off:1.1.1.1 hostname=devbox systemd.mask=home.mount systemd.mask=var.mount" + if got != want { + t.Fatalf("BuildBootArgsWithKernelIP() = %q, want %q", got, want) + } +} + func TestWriteExt4FileRemovesTempFileAndReturnsCopyError(t *testing.T) { t.Parallel() @@ -212,6 +222,46 @@ func TestWriteExt4FileRemovesTempFileAndReturnsCopyError(t *testing.T) { } } +func TestWriteExt4FileModeUsesRequestedPermissions(t *testing.T) { + t.Parallel() + + debugfsCalled := false + runner := funcRunner{ + runSudo: func(ctx context.Context, args ...string) ([]byte, error) { + switch args[0] { + case "e2rm": + return nil, nil + case "e2cp": + info, err := os.Stat(args[1]) + if err != nil { + t.Fatalf("Stat(temp file): %v", err) + } + if got := info.Mode().Perm(); got != 0o755 { + t.Fatalf("temp file mode = %o, want 755", got) + } + return nil, nil + case "debugfs": + debugfsCalled = true + want := []string{"debugfs", "-w", "-R", "sif /usr/local/libexec/banger-network-bootstrap mode 0100755", "/tmp/root.ext4"} + if !reflect.DeepEqual(args, want) { + t.Fatalf("debugfs args = %v, want %v", args, want) + } + return nil, nil + default: + t.Fatalf("unexpected sudo call: %v", args) + return nil, nil + } + }, + } + + if err := WriteExt4FileMode(context.Background(), runner, "/tmp/root.ext4", "/usr/local/libexec/banger-network-bootstrap", 0o755, []byte("#!/bin/sh\n")); err != nil { + t.Fatalf("WriteExt4FileMode() error = %v", err) + } + if !debugfsCalled { + t.Fatal("expected debugfs mode fixup to run") + } +} + func TestMountTempDirUsesLoopForRegularFilesAndCleanupUsesBackgroundContext(t *testing.T) { t.Parallel()