firecracker: adopt firecracker-jailer for VM launch (Phase B)
Each VM's firecracker now runs inside a per-VM chroot dropped to the registered owner UID via firecracker-jailer. Closes the broad ambient- sudo escalation surface that survived Phase A: the helper still needs caps for tap/bridge/dm/loop/iptables, but the VMM itself no longer runs as root in the host root filesystem. The host helper stages each chroot up front: hard-links the kernel and (optional) initrd, mknods block-device drives + /dev/vhost-vsock, copies in the firecracker binary (jailer opens it O_RDWR so a ro bind fails with EROFS), and bind-mounts /usr/lib + /lib trees read-only so the dynamic linker can resolve. Self-binds the chroot first so the findmnt-guarded cleanup can recurse safely. AF_UNIX sun_path is 108 bytes; the chroot path easily blows past that. Daemon-side launch pre-symlinks the short request socket path to the long chroot socket before Machine.Start so the SDK's poll/connect sees the short path while the kernel resolves to the chroot socket. --new-pid-ns is intentionally disabled — jailer's PID-namespace fork makes the SDK see the parent exit and tear the API socket down too early. CapabilityBoundingSet for the helper expands to add CAP_FOWNER, CAP_KILL, CAP_MKNOD, CAP_SETGID, CAP_SETUID, CAP_SYS_CHROOT alongside the existing CAP_CHOWN/CAP_DAC_OVERRIDE/CAP_NET_ADMIN/CAP_NET_RAW/ CAP_SYS_ADMIN. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
d73efe6fbc
commit
6b543cb17f
12 changed files with 864 additions and 56 deletions
|
|
@ -9,6 +9,8 @@ import (
|
|||
"log/slog"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
|
@ -88,6 +90,14 @@ func OpenSystem(ctx context.Context) (*Daemon, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// config.Load fills JailerChrootBase from the layout it sees. In
|
||||
// system mode that's the owner's layout (no privileged StateDir) so
|
||||
// the value lands under the owner home — wrong for the helper, which
|
||||
// validates paths against the system StateDir. Override unconditionally
|
||||
// here so both daemon and helper see /var/lib/banger/jail.
|
||||
if strings.TrimSpace(cfg.JailerChrootBase) == "" || !filepath.IsAbs(cfg.JailerChrootBase) || strings.HasPrefix(cfg.JailerChrootBase, ownerLayout.StateDir) {
|
||||
cfg.JailerChrootBase = filepath.Join(layout.StateDir, "jail")
|
||||
}
|
||||
helper := newHelperPrivilegedOps(roothelper.NewClient(installmeta.DefaultRootHelperSocketPath), cfg, layout)
|
||||
return openWithConfig(ctx, layout, ownerLayout, cfg, -1, -1, false, helper)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,11 +11,15 @@ import (
|
|||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"banger/internal/firecracker"
|
||||
"banger/internal/system"
|
||||
)
|
||||
|
|
@ -271,6 +275,306 @@ func (m *Manager) Kill(ctx context.Context, pid int) error {
|
|||
return err
|
||||
}
|
||||
|
||||
// ChrootDriveSpec describes how a single drive should appear inside the
|
||||
// jailer chroot. HostPath is the host-side source (a regular file or a
|
||||
// /dev/mapper/* block device); ChrootName is the bare filename it should
|
||||
// be reachable as inside the chroot (e.g. "rootfs"). The DM block device
|
||||
// case is detected via os.Stat (S_IFBLK) — the helper mknods a matching
|
||||
// node; everything else is hard-linked.
|
||||
type ChrootDriveSpec struct {
|
||||
ChrootName string
|
||||
HostPath string
|
||||
}
|
||||
|
||||
// PrepareJailerChroot stages the chroot tree at chrootRoot for the jailer
|
||||
// to take over on launch. After this call:
|
||||
//
|
||||
// - chrootRoot exists, mode 0700, owned by uid:gid.
|
||||
// - chrootRoot/<kernel-name> is a hard link of kernelHostPath, owned uid:gid.
|
||||
// - chrootRoot/<initrd-name> is a hard link of initrdHostPath if set.
|
||||
// - For each drive: a hard link (regular file source) or a freshly
|
||||
// mknod'd block device with the source's major/minor (DM source).
|
||||
// - If wantVSock, /dev/vhost-vsock is mknod'd into the chroot so
|
||||
// firecracker can open it after chroot.
|
||||
//
|
||||
// All filesystem mutations go through runner.RunSudo when the caller isn't
|
||||
// root, so this works in dev (sudo) and system (root helper) modes alike.
|
||||
// Path components are validated by the caller (roothelper) — this helper
|
||||
// trusts them.
|
||||
func (m *Manager) PrepareJailerChroot(ctx context.Context, chrootRoot string, uid, gid int, firecrackerHostPath, kernelHostPath, kernelName, initrdHostPath, initrdName string, drives []ChrootDriveSpec, wantVSock bool) error {
|
||||
if strings.TrimSpace(chrootRoot) == "" {
|
||||
return fmt.Errorf("chroot root is required")
|
||||
}
|
||||
if err := m.sudo(ctx, "mkdir", "-p", chrootRoot); err != nil {
|
||||
return fmt.Errorf("create chroot root: %w", err)
|
||||
}
|
||||
if err := m.sudo(ctx, "chmod", "0700", chrootRoot); err != nil {
|
||||
return fmt.Errorf("chmod chroot root: %w", err)
|
||||
}
|
||||
if err := m.chown(ctx, chrootRoot, uid, gid); err != nil {
|
||||
return fmt.Errorf("chown chroot root: %w", err)
|
||||
}
|
||||
// The daemon (uid) needs to traverse the intermediate directories to reach
|
||||
// the sockets firecracker creates inside the chroot. The per-VM dir
|
||||
// (<base>/firecracker/<vmid>/) is chowned to uid so the daemon can reach
|
||||
// <vmid>/root/. The <base>/firecracker/ base and <base>/jail/ dirs get
|
||||
// world-execute (--x) so any UID can traverse through them without listing
|
||||
// their contents (the per-VM dirs are still protected by their own mode).
|
||||
vmDir := filepath.Dir(chrootRoot)
|
||||
if err := m.chown(ctx, vmDir, uid, gid); err != nil {
|
||||
return fmt.Errorf("chown vm dir: %w", err)
|
||||
}
|
||||
fcBaseDir := filepath.Dir(vmDir)
|
||||
if err := m.sudo(ctx, "chmod", "0711", fcBaseDir); err != nil {
|
||||
return fmt.Errorf("chmod firecracker base dir: %w", err)
|
||||
}
|
||||
jailBaseDir := filepath.Dir(fcBaseDir)
|
||||
if err := m.sudo(ctx, "chmod", "0711", jailBaseDir); err != nil {
|
||||
return fmt.Errorf("chmod jail base dir: %w", err)
|
||||
}
|
||||
// Order matters: hard-link the kernel + file-backed drives BEFORE
|
||||
// the self-bind below. link(2) refuses to cross mount points even
|
||||
// when the underlying superblock is the same — once chrootRoot is a
|
||||
// mount point, `ln /var/lib/.../kernel <chroot>/vmlinux` returns
|
||||
// EXDEV.
|
||||
if err := m.linkInto(ctx, chrootRoot, kernelHostPath, kernelName, uid, gid); err != nil {
|
||||
return fmt.Errorf("link kernel: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(initrdHostPath) != "" {
|
||||
if err := m.linkInto(ctx, chrootRoot, initrdHostPath, initrdName, uid, gid); err != nil {
|
||||
return fmt.Errorf("link initrd: %w", err)
|
||||
}
|
||||
}
|
||||
for _, d := range drives {
|
||||
if err := m.stageDrive(ctx, chrootRoot, d, uid, gid); err != nil {
|
||||
return fmt.Errorf("stage drive %s: %w", d.ChrootName, err)
|
||||
}
|
||||
}
|
||||
if wantVSock {
|
||||
// The jailer creates /dev inside the chroot, but /dev/vhost-vsock must
|
||||
// be pre-staged so firecracker can open it after the jailer chroots.
|
||||
devDir := chrootRoot + "/dev"
|
||||
if err := m.sudo(ctx, "mkdir", "-p", devDir); err != nil {
|
||||
return fmt.Errorf("create chroot/dev: %w", err)
|
||||
}
|
||||
if err := m.chown(ctx, devDir, uid, gid); err != nil {
|
||||
return fmt.Errorf("chown chroot/dev: %w", err)
|
||||
}
|
||||
if err := m.stageDevice(ctx, chrootRoot, "dev/vhost-vsock", "/dev/vhost-vsock", uid, gid); err != nil {
|
||||
return fmt.Errorf("stage vhost-vsock: %w", err)
|
||||
}
|
||||
}
|
||||
// Bind firecracker + the host libdirs into the chroot read-only.
|
||||
// firecracker is dynamically linked (interpreter /lib64/ld-linux-*,
|
||||
// libc, libgcc), and inside the chroot ENOENT on those is reported
|
||||
// as "Failed to exec into Firecracker: No such file or directory" —
|
||||
// the kernel's misleading ENOENT-for-missing-interpreter error.
|
||||
//
|
||||
// Done last so the link/mknod steps above don't have to cross the
|
||||
// self-bind mount boundary (link(2) returns EXDEV at mount edges).
|
||||
// Self-bind first so CleanupJailerChroot's `umount -lR` can recurse
|
||||
// from chrootRoot itself; --make-private blocks propagation back to
|
||||
// the host mount namespace.
|
||||
// firecracker is copied (not bind-mounted) because jailer opens the
|
||||
// binary O_RDWR — apparently to seal it or rewrite something — and
|
||||
// fails with EROFS on a ro-bind.
|
||||
chrootFC := chrootRoot + "/" + filepath.Base(firecrackerHostPath)
|
||||
if err := m.sudo(ctx, "cp", "-f", firecrackerHostPath, chrootFC); err != nil {
|
||||
return fmt.Errorf("copy firecracker into chroot: %w", err)
|
||||
}
|
||||
if err := m.sudo(ctx, "chmod", "0755", chrootFC); err != nil {
|
||||
return fmt.Errorf("chmod firecracker in chroot: %w", err)
|
||||
}
|
||||
if err := m.chown(ctx, chrootFC, uid, gid); err != nil {
|
||||
return fmt.Errorf("chown firecracker in chroot: %w", err)
|
||||
}
|
||||
if err := m.sudo(ctx, "mount", "--bind", chrootRoot, chrootRoot); err != nil {
|
||||
return fmt.Errorf("self-bind chroot: %w", err)
|
||||
}
|
||||
// Remount without nosuid: the helper unit's ReadWritePaths binding marks
|
||||
// /var/lib/banger nosuid, and bind mounts inherit that flag. The jailer
|
||||
// needs to exec /firecracker as UID 1000, which the kernel denies on a
|
||||
// nosuid mount when NoNewPrivileges is set on the unit.
|
||||
if err := m.sudo(ctx, "mount", "-o", "remount,bind,suid", chrootRoot, chrootRoot); err != nil {
|
||||
return fmt.Errorf("remount chroot suid: %w", err)
|
||||
}
|
||||
if err := m.sudo(ctx, "mount", "--make-private", chrootRoot); err != nil {
|
||||
return fmt.Errorf("make-private chroot: %w", err)
|
||||
}
|
||||
// Pre-create /usr with world-traversable permissions. UMask=0077 on the
|
||||
// helper unit causes plain mkdir to produce 0700 dirs; UID 1000 must be
|
||||
// able to traverse /usr/ to reach the dynamic linker via lib64 → usr/lib.
|
||||
if err := m.sudo(ctx, "install", "-d", "-m", "0755", chrootRoot+"/usr"); err != nil {
|
||||
return fmt.Errorf("create chroot/usr: %w", err)
|
||||
}
|
||||
// Bind real libdirs and replicate the host's compat symlinks
|
||||
// (/lib64 → /usr/lib, etc) inside the chroot so firecracker's
|
||||
// PT_INTERP path (/lib64/ld-linux-*) resolves to the bound libs.
|
||||
for _, libDir := range []string{"/usr/lib", "/usr/lib64", "/lib", "/lib64"} {
|
||||
info, err := os.Lstat(libDir)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
target := chrootRoot + libDir
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
link, err := os.Readlink(libDir)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if err := m.sudo(ctx, "ln", "-sfn", link, target); err != nil {
|
||||
return fmt.Errorf("symlink %s -> %s: %w", target, link, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !info.IsDir() {
|
||||
continue
|
||||
}
|
||||
if err := m.bindDir(ctx, libDir, target, true); err != nil {
|
||||
return fmt.Errorf("bind %s: %w", libDir, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanupJailerChroot tears down a chroot built by PrepareJailerChroot:
|
||||
// lazy-recursive umount of every mount under (or at) chrootRoot, then a
|
||||
// findmnt-guarded `rm -rf`. The guard is load-bearing: if any bind mount
|
||||
// remained, `rm -rf` would descend into the bind source (e.g. /usr/lib)
|
||||
// and start deleting host files. The umount runs `-l` (lazy) so an in-use
|
||||
// bind point still gets detached from the namespace; the guarded check
|
||||
// then catches the rare case where detachment didn't happen.
|
||||
func (m *Manager) CleanupJailerChroot(ctx context.Context, chrootRoot string) error {
|
||||
if strings.TrimSpace(chrootRoot) == "" {
|
||||
return nil
|
||||
}
|
||||
if _, err := os.Stat(chrootRoot); os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
// Best-effort umount: for chroots that were never bind-mounted (a
|
||||
// stale install pre-bind-mount work, say) this fails — that's fine,
|
||||
// the findmnt guard below is what enforces safety.
|
||||
_ = m.sudoIgnore(ctx, "umount", "--recursive", "--lazy", chrootRoot)
|
||||
if mounts, err := m.mountsUnder(ctx, chrootRoot); err != nil {
|
||||
return fmt.Errorf("inspect chroot mounts: %w", err)
|
||||
} else if len(mounts) > 0 {
|
||||
return fmt.Errorf("refusing to rm -rf %q: still has %d mount(s): %v", chrootRoot, len(mounts), mounts)
|
||||
}
|
||||
return m.sudo(ctx, "rm", "-rf", "--", chrootRoot)
|
||||
}
|
||||
|
||||
func (m *Manager) sudoIgnore(ctx context.Context, name string, args ...string) error {
|
||||
err := m.sudo(ctx, name, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (m *Manager) bindFile(ctx context.Context, source, target string, readOnly bool) error {
|
||||
if err := m.sudo(ctx, "install", "-D", "-m", "0644", "/dev/null", target); err != nil {
|
||||
return fmt.Errorf("create bind target file: %w", err)
|
||||
}
|
||||
return m.bindMount(ctx, source, target, readOnly)
|
||||
}
|
||||
|
||||
func (m *Manager) bindDir(ctx context.Context, source, target string, readOnly bool) error {
|
||||
if err := m.sudo(ctx, "mkdir", "-p", target); err != nil {
|
||||
return fmt.Errorf("create bind target dir: %w", err)
|
||||
}
|
||||
return m.bindMount(ctx, source, target, readOnly)
|
||||
}
|
||||
|
||||
func (m *Manager) bindMount(ctx context.Context, source, target string, readOnly bool) error {
|
||||
if err := m.sudo(ctx, "mount", "--bind", source, target); err != nil {
|
||||
return err
|
||||
}
|
||||
if !readOnly {
|
||||
return nil
|
||||
}
|
||||
// Single-step ro bind isn't honored by all kernels — the bind happens
|
||||
// rw and the ro flag is silently ignored. Remount makes it stick.
|
||||
return m.sudo(ctx, "mount", "-o", "remount,bind,ro", target)
|
||||
}
|
||||
|
||||
// mountsUnder returns the list of mount targets at or under chrootRoot.
|
||||
// findmnt's output is one path per line; an empty list means no leftovers.
|
||||
func (m *Manager) mountsUnder(ctx context.Context, chrootRoot string) ([]string, error) {
|
||||
out, err := m.runner.Run(ctx, "findmnt", "--output", "TARGET", "--list", "--noheadings")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var mounts []string
|
||||
prefix := chrootRoot + string(os.PathSeparator)
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
t := strings.TrimSpace(line)
|
||||
if t == chrootRoot || strings.HasPrefix(t, prefix) {
|
||||
mounts = append(mounts, t)
|
||||
}
|
||||
}
|
||||
return mounts, nil
|
||||
}
|
||||
|
||||
func (m *Manager) stageDrive(ctx context.Context, chrootRoot string, d ChrootDriveSpec, uid, gid int) error {
|
||||
info, err := os.Stat(d.HostPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeDevice != 0 {
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return fmt.Errorf("stat %s: cannot read device numbers", d.HostPath)
|
||||
}
|
||||
major := unix.Major(stat.Rdev)
|
||||
minor := unix.Minor(stat.Rdev)
|
||||
return m.mknodBlock(ctx, chrootRoot, d.ChrootName, major, minor, uid, gid)
|
||||
}
|
||||
return m.linkInto(ctx, chrootRoot, d.HostPath, d.ChrootName, uid, gid)
|
||||
}
|
||||
|
||||
func (m *Manager) stageDevice(ctx context.Context, chrootRoot, chrootName, hostDevice string, uid, gid int) error {
|
||||
info, err := os.Stat(hostDevice)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return fmt.Errorf("stat %s: cannot read device numbers", hostDevice)
|
||||
}
|
||||
major := unix.Major(stat.Rdev)
|
||||
minor := unix.Minor(stat.Rdev)
|
||||
target := chrootRoot + "/" + chrootName
|
||||
if err := m.sudo(ctx, "mknod", "-m", "0660", target, "c", strconv.FormatUint(uint64(major), 10), strconv.FormatUint(uint64(minor), 10)); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.chown(ctx, target, uid, gid)
|
||||
}
|
||||
|
||||
func (m *Manager) mknodBlock(ctx context.Context, chrootRoot, name string, major, minor uint32, uid, gid int) error {
|
||||
target := chrootRoot + "/" + name
|
||||
if err := m.sudo(ctx, "mknod", "-m", "0660", target, "b", strconv.FormatUint(uint64(major), 10), strconv.FormatUint(uint64(minor), 10)); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.chown(ctx, target, uid, gid)
|
||||
}
|
||||
|
||||
func (m *Manager) linkInto(ctx context.Context, chrootRoot, source, name string, uid, gid int) error {
|
||||
target := chrootRoot + "/" + name
|
||||
if err := m.sudo(ctx, "ln", "-f", source, target); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.chown(ctx, target, uid, gid)
|
||||
}
|
||||
|
||||
func (m *Manager) chown(ctx context.Context, target string, uid, gid int) error {
|
||||
return m.sudo(ctx, "chown", fmt.Sprintf("%d:%d", uid, gid), target)
|
||||
}
|
||||
|
||||
func (m *Manager) sudo(ctx context.Context, name string, args ...string) error {
|
||||
if os.Geteuid() == 0 {
|
||||
_, err := m.runner.Run(ctx, name, args...)
|
||||
return err
|
||||
}
|
||||
_, err := m.runner.RunSudo(ctx, append([]string{name}, args...)...)
|
||||
return err
|
||||
}
|
||||
|
||||
func waitForPath(ctx context.Context, path string, timeout time.Duration, label string) error {
|
||||
return pollPath(ctx, path, timeout, 100*time.Millisecond, label)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@ package daemon
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
|
@ -39,6 +41,7 @@ type privilegedOps interface {
|
|||
KillProcess(context.Context, int) error
|
||||
SignalProcess(context.Context, int, string) error
|
||||
ProcessRunning(context.Context, int, string) (bool, error)
|
||||
CleanupJailerChroot(context.Context, string) error
|
||||
}
|
||||
|
||||
type localPrivilegedOps struct {
|
||||
|
|
@ -170,7 +173,77 @@ func (o *localPrivilegedOps) ResolveFirecrackerBinary(_ context.Context, request
|
|||
}
|
||||
|
||||
func (o *localPrivilegedOps) LaunchFirecracker(ctx context.Context, req roothelper.FirecrackerLaunchRequest) (int, error) {
|
||||
machine, err := firecracker.NewMachine(ctx, firecracker.MachineConfig{
|
||||
mc, err := o.buildLaunchMachineConfig(ctx, req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Symlink before Start: with jailer the actual API socket lives at
|
||||
// `<chroot>/firecracker.socket` (~120+ bytes — over the AF_UNIX
|
||||
// sun_path limit of 108). The SDK's waitForSocket and connect(2)
|
||||
// would EINVAL on the long path. Pre-creating the symlink at the
|
||||
// short req.SocketPath lets the SDK poll/connect via the short
|
||||
// path; the kernel only enforces sun_path on the path you pass,
|
||||
// not on the resolved target.
|
||||
if err := o.exposeJailerSockets(req); err != nil {
|
||||
return 0, fmt.Errorf("expose jailer sockets: %w", err)
|
||||
}
|
||||
machine, err := firecracker.NewMachine(ctx, mc)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
chownDone := o.maybeChownSockets(ctx, req, mc)
|
||||
startErr := machine.Start(ctx)
|
||||
chownErr := <-chownDone
|
||||
if startErr != nil {
|
||||
if pid := o.fc().ResolvePID(context.Background(), machine, mc.SocketPath); pid > 0 {
|
||||
_ = o.KillProcess(context.Background(), pid)
|
||||
}
|
||||
return 0, startErr
|
||||
}
|
||||
if chownErr != nil {
|
||||
return 0, chownErr
|
||||
}
|
||||
if req.Jailer == nil {
|
||||
// Belt-and-suspenders for the legacy direct-firecracker path.
|
||||
// The jailer path doesn't need this — firecracker drops to the
|
||||
// configured uid before creating the socket.
|
||||
if err := o.EnsureSocketAccess(ctx, mc.SocketPath, "firecracker api socket"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if strings.TrimSpace(mc.VSockPath) != "" {
|
||||
if err := o.EnsureSocketAccess(ctx, mc.VSockPath, "firecracker vsock socket"); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
pid := o.fc().ResolvePID(context.Background(), machine, mc.SocketPath)
|
||||
if pid <= 0 {
|
||||
return 0, errors.New("firecracker started but pid could not be resolved")
|
||||
}
|
||||
return pid, nil
|
||||
}
|
||||
|
||||
// maybeChownSockets runs the post-Start sudo-chown race only on the legacy
|
||||
// direct-firecracker path. With the jailer the firecracker process is
|
||||
// already running as the configured uid before it creates the socket, so
|
||||
// no chown is needed (and chown on the symlink would tweak the symlink's
|
||||
// metadata — not the target's — anyway).
|
||||
func (o *localPrivilegedOps) maybeChownSockets(ctx context.Context, req roothelper.FirecrackerLaunchRequest, mc firecracker.MachineConfig) <-chan error {
|
||||
if req.Jailer != nil {
|
||||
ch := make(chan error, 1)
|
||||
ch <- nil
|
||||
close(ch)
|
||||
return ch
|
||||
}
|
||||
return o.fc().EnsureSocketAccessForAsync(ctx, []string{mc.SocketPath, mc.VSockPath}, o.clientUID, o.clientGID)
|
||||
}
|
||||
|
||||
// buildLaunchMachineConfig mirrors the helper-side equivalent: when jailer
|
||||
// is enabled, stage the chroot tree and rewrite the path fields to their
|
||||
// chroot-translated form (host-visible for sockets, chroot-internal for
|
||||
// kernel/drives — see firecracker.MachineConfig.Jailer doc).
|
||||
func (o *localPrivilegedOps) buildLaunchMachineConfig(ctx context.Context, req roothelper.FirecrackerLaunchRequest) (firecracker.MachineConfig, error) {
|
||||
mc := firecracker.MachineConfig{
|
||||
BinaryPath: req.BinaryPath,
|
||||
VMID: req.VMID,
|
||||
SocketPath: req.SocketPath,
|
||||
|
|
@ -186,40 +259,101 @@ func (o *localPrivilegedOps) LaunchFirecracker(ctx context.Context, req roothelp
|
|||
VCPUCount: req.VCPUCount,
|
||||
MemoryMiB: req.MemoryMiB,
|
||||
Logger: o.logger,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Race the chown against the SDK's HTTP probe inside Start: when the
|
||||
// daemon is non-root, firecracker is launched under sudo and the API
|
||||
// socket appears root-owned. Without a concurrent chown the SDK's
|
||||
// connect(2) gets EACCES and Start times out before our post-Start
|
||||
// EnsureSocketAccess can ever run.
|
||||
chownDone := o.fc().EnsureSocketAccessForAsync(ctx, []string{req.SocketPath, req.VSockPath}, o.clientUID, o.clientGID)
|
||||
startErr := machine.Start(ctx)
|
||||
chownErr := <-chownDone
|
||||
if startErr != nil {
|
||||
if pid := o.fc().ResolvePID(context.Background(), machine, req.SocketPath); pid > 0 {
|
||||
_ = o.KillProcess(context.Background(), pid)
|
||||
}
|
||||
return 0, startErr
|
||||
if req.Jailer == nil {
|
||||
return mc, nil
|
||||
}
|
||||
if chownErr != nil {
|
||||
return 0, chownErr
|
||||
chrootRoot := firecracker.JailerChrootRoot(req.Jailer.ChrootBaseDir, req.VMID)
|
||||
driveSpecs := make([]fcproc.ChrootDriveSpec, 0, len(req.Drives))
|
||||
chrootDrives := make([]firecracker.DriveConfig, 0, len(req.Drives))
|
||||
for _, d := range req.Drives {
|
||||
name := chrootDriveName(d)
|
||||
driveSpecs = append(driveSpecs, fcproc.ChrootDriveSpec{ChrootName: name, HostPath: d.Path})
|
||||
chrootDrives = append(chrootDrives, firecracker.DriveConfig{
|
||||
ID: d.ID,
|
||||
Path: "/" + name,
|
||||
ReadOnly: d.ReadOnly,
|
||||
IsRoot: d.IsRoot,
|
||||
})
|
||||
}
|
||||
if err := o.EnsureSocketAccess(ctx, req.SocketPath, "firecracker api socket"); err != nil {
|
||||
return 0, err
|
||||
wantVSock := strings.TrimSpace(req.VSockPath) != ""
|
||||
if err := o.fc().PrepareJailerChroot(ctx, chrootRoot,
|
||||
req.Jailer.UID, req.Jailer.GID,
|
||||
req.BinaryPath,
|
||||
req.KernelImagePath, "vmlinux",
|
||||
req.InitrdPath, "initrd",
|
||||
driveSpecs, wantVSock,
|
||||
); err != nil {
|
||||
return firecracker.MachineConfig{}, fmt.Errorf("prepare jailer chroot: %w", err)
|
||||
}
|
||||
// SocketPath stays the short request path: the SDK polls/connects
|
||||
// to it via os.Stat / net.Dial("unix", ...), and AF_UNIX sun_path
|
||||
// is hard-capped at 108 bytes — the actual chroot path is well over
|
||||
// that. exposeJailerSockets pre-creates the req.SocketPath as a
|
||||
// symlink whose target is the long chroot socket; the kernel only
|
||||
// enforces sun_path on the path you hand to connect, not on the
|
||||
// resolved target.
|
||||
//
|
||||
// VSockPath, by contrast, is sent to firecracker via the API and
|
||||
// resolved from inside the chroot, so it must be the chroot-internal
|
||||
// path. The host-visible vsock socket is reachable via a symlink
|
||||
// at req.VSockPath, also installed by exposeJailerSockets.
|
||||
_ = chrootRoot
|
||||
if wantVSock {
|
||||
mc.VSockPath = firecracker.JailerVSockName
|
||||
}
|
||||
mc.KernelImagePath = "/vmlinux"
|
||||
if strings.TrimSpace(req.InitrdPath) != "" {
|
||||
mc.InitrdPath = "/initrd"
|
||||
} else {
|
||||
mc.InitrdPath = ""
|
||||
}
|
||||
mc.Drives = chrootDrives
|
||||
// LogPath stays set so buildProcessRunner's openLogFile captures firecracker
|
||||
// stderr via cmd.Stderr. buildConfig clears sdk.Config.LogPath for jailer
|
||||
// mode to avoid PUT /logger with a host path firecracker can't open.
|
||||
mc.MetricsPath = ""
|
||||
mc.Jailer = &firecracker.JailerOpts{
|
||||
Binary: req.Jailer.Binary,
|
||||
ChrootBaseDir: req.Jailer.ChrootBaseDir,
|
||||
UID: req.Jailer.UID,
|
||||
GID: req.Jailer.GID,
|
||||
}
|
||||
return mc, nil
|
||||
}
|
||||
|
||||
func (o *localPrivilegedOps) exposeJailerSockets(req roothelper.FirecrackerLaunchRequest) error {
|
||||
if req.Jailer == nil {
|
||||
return nil
|
||||
}
|
||||
chrootRoot := firecracker.JailerChrootRoot(req.Jailer.ChrootBaseDir, req.VMID)
|
||||
hostAPI := filepath.Join(chrootRoot, strings.TrimPrefix(firecracker.JailerSocketName, "/"))
|
||||
if err := atomicSymlink(hostAPI, req.SocketPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(req.VSockPath) != "" {
|
||||
if err := o.EnsureSocketAccess(ctx, req.VSockPath, "firecracker vsock socket"); err != nil {
|
||||
return 0, err
|
||||
hostVSock := filepath.Join(chrootRoot, strings.TrimPrefix(firecracker.JailerVSockName, "/"))
|
||||
if err := atomicSymlink(hostVSock, req.VSockPath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
pid := o.fc().ResolvePID(context.Background(), machine, req.SocketPath)
|
||||
if pid <= 0 {
|
||||
return 0, errors.New("firecracker started but pid could not be resolved")
|
||||
return nil
|
||||
}
|
||||
|
||||
// chrootDriveName mirrors the helper-side helper of the same name; kept as
|
||||
// a free function so both paths produce identical chroot layouts.
|
||||
func chrootDriveName(d firecracker.DriveConfig) string {
|
||||
if id := strings.TrimSpace(d.ID); id != "" {
|
||||
return id
|
||||
}
|
||||
return pid, nil
|
||||
return filepath.Base(d.Path)
|
||||
}
|
||||
|
||||
func atomicSymlink(target, link string) error {
|
||||
if err := os.Remove(link); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
return os.Symlink(target, link)
|
||||
}
|
||||
|
||||
func (o *localPrivilegedOps) EnsureSocketAccess(ctx context.Context, socketPath, label string) error {
|
||||
|
|
@ -246,6 +380,10 @@ func (o *localPrivilegedOps) ProcessRunning(_ context.Context, pid int, apiSock
|
|||
return system.ProcessRunning(pid, apiSock), nil
|
||||
}
|
||||
|
||||
func (o *localPrivilegedOps) CleanupJailerChroot(ctx context.Context, chrootRoot string) error {
|
||||
return o.fc().CleanupJailerChroot(ctx, chrootRoot)
|
||||
}
|
||||
|
||||
func (o *localPrivilegedOps) fc() *fcproc.Manager {
|
||||
return fcproc.New(o.runner, fcproc.Config{
|
||||
FirecrackerBin: normalizeFirecrackerBinary("", o.config.FirecrackerBin),
|
||||
|
|
@ -320,7 +458,27 @@ func (o *helperPrivilegedOps) ResolveFirecrackerBinary(ctx context.Context, requ
|
|||
|
||||
func (o *helperPrivilegedOps) LaunchFirecracker(ctx context.Context, req roothelper.FirecrackerLaunchRequest) (int, error) {
|
||||
req.Network = o.networkConfig()
|
||||
return o.client.LaunchFirecracker(ctx, req)
|
||||
pid, err := o.client.LaunchFirecracker(ctx, req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// The root helper runs with PrivateMounts=yes, so symlinks it creates
|
||||
// (exposeJailerSockets) are invisible to the daemon's namespace. Re-create
|
||||
// them here so the daemon can reach the API and vsock sockets.
|
||||
if req.Jailer != nil {
|
||||
chrootRoot := firecracker.JailerChrootRoot(req.Jailer.ChrootBaseDir, req.VMID)
|
||||
hostAPI := filepath.Join(chrootRoot, strings.TrimPrefix(firecracker.JailerSocketName, "/"))
|
||||
if err := atomicSymlink(hostAPI, req.SocketPath); err != nil {
|
||||
return 0, fmt.Errorf("api socket symlink: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(req.VSockPath) != "" {
|
||||
hostVSock := filepath.Join(chrootRoot, strings.TrimPrefix(firecracker.JailerVSockName, "/"))
|
||||
if err := atomicSymlink(hostVSock, req.VSockPath); err != nil {
|
||||
return 0, fmt.Errorf("vsock symlink: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return pid, nil
|
||||
}
|
||||
|
||||
func (o *helperPrivilegedOps) EnsureSocketAccess(ctx context.Context, socketPath, label string) error {
|
||||
|
|
@ -348,6 +506,10 @@ func (o *helperPrivilegedOps) ProcessRunning(ctx context.Context, pid int, apiSo
|
|||
return o.client.ProcessRunning(ctx, pid, apiSock)
|
||||
}
|
||||
|
||||
func (o *helperPrivilegedOps) CleanupJailerChroot(ctx context.Context, chrootRoot string) error {
|
||||
return o.client.CleanupJailerChroot(ctx, chrootRoot)
|
||||
}
|
||||
|
||||
func (o *helperPrivilegedOps) networkConfig() roothelper.NetworkConfig {
|
||||
return roothelper.NetworkConfig{
|
||||
BridgeName: o.config.BridgeName,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"time"
|
||||
|
||||
"banger/internal/daemon/fcproc"
|
||||
"banger/internal/firecracker"
|
||||
"banger/internal/model"
|
||||
"banger/internal/namegen"
|
||||
"banger/internal/system"
|
||||
|
|
@ -149,14 +150,35 @@ func (s *VMService) cleanupRuntime(ctx context.Context, vm model.VMRecord, prese
|
|||
if vm.Runtime.VSockPath != "" {
|
||||
_ = os.Remove(vm.Runtime.VSockPath)
|
||||
}
|
||||
// Remove the jailer chroot tree (kernel hard-links, mknod'd device
|
||||
// nodes, the chroot root itself). Skipped silently when the jailer
|
||||
// is disabled or the chroot was never created. We intentionally
|
||||
// don't gate on JailerEnabled today — old VMs created before the
|
||||
// flag flipped on still need their chroots removed if any exist.
|
||||
jailerErr := s.cleanupJailerChroot(ctx, vm)
|
||||
// The handles are only meaningful while the kernel objects exist;
|
||||
// dropping them here keeps the cache in sync with reality even
|
||||
// when the caller forgets to call clearVMHandles explicitly.
|
||||
s.clearVMHandles(vm)
|
||||
if !preserveDisks && vm.Runtime.VMDir != "" {
|
||||
return errors.Join(waitErr, snapshotErr, featureErr, tapErr, os.RemoveAll(vm.Runtime.VMDir))
|
||||
return errors.Join(waitErr, snapshotErr, featureErr, tapErr, jailerErr, os.RemoveAll(vm.Runtime.VMDir))
|
||||
}
|
||||
return errors.Join(waitErr, snapshotErr, featureErr, tapErr)
|
||||
return errors.Join(waitErr, snapshotErr, featureErr, tapErr, jailerErr)
|
||||
}
|
||||
|
||||
// cleanupJailerChroot removes the per-VM chroot tree if it exists. Returns
|
||||
// nil silently when the jailer was never enabled or the chroot path can't
|
||||
// be computed (no JailerChrootBase configured).
|
||||
func (s *VMService) cleanupJailerChroot(ctx context.Context, vm model.VMRecord) error {
|
||||
base := strings.TrimSpace(s.config.JailerChrootBase)
|
||||
if base == "" {
|
||||
return nil
|
||||
}
|
||||
chrootRoot := firecracker.JailerChrootRoot(base, vm.ID)
|
||||
if _, err := os.Stat(chrootRoot); os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return s.privOps().CleanupJailerChroot(ctx, chrootRoot)
|
||||
}
|
||||
|
||||
func (s *VMService) generateName(ctx context.Context) (string, error) {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,24 @@ import (
|
|||
"banger/internal/system"
|
||||
)
|
||||
|
||||
// jailerOpts returns the jailer launch options to bundle in the firecracker
|
||||
// launch request, or nil when the jailer is disabled or misconfigured.
|
||||
// nil makes the launch fall back to the legacy direct-firecracker path.
|
||||
func (s *VMService) jailerOpts() *roothelper.JailerLaunchOpts {
|
||||
if !s.config.JailerEnabled {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(s.config.JailerBin) == "" || strings.TrimSpace(s.config.JailerChrootBase) == "" {
|
||||
return nil
|
||||
}
|
||||
return &roothelper.JailerLaunchOpts{
|
||||
Binary: s.config.JailerBin,
|
||||
ChrootBaseDir: s.config.JailerChrootBase,
|
||||
UID: os.Getuid(),
|
||||
GID: os.Getgid(),
|
||||
}
|
||||
}
|
||||
|
||||
// buildKernelArgs assembles the kernel command line for a start.
|
||||
// Direct-boot images (no initrd) get kernel-level IP config so the
|
||||
// network is up before init, plus init= pointing at the universal
|
||||
|
|
@ -344,6 +362,7 @@ func (s *VMService) buildStartSteps(op *operationLog, sc *startContext) []startS
|
|||
VSockCID: sc.vm.Runtime.VSockCID,
|
||||
VCPUCount: sc.vm.Spec.VCPUCount,
|
||||
MemoryMiB: sc.vm.Spec.MemoryMiB,
|
||||
Jailer: s.jailerOpts(),
|
||||
}
|
||||
machineConfig := firecracker.MachineConfig{Drives: launchReq.Drives}
|
||||
s.capHooks.contributeMachine(&machineConfig, *sc.vm, sc.image)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue