Thales Maciel
7b7f7e676c
Stop and delete could fail with device-mapper busy errors when the persisted Firecracker PID was stale or the kernel needed longer to release the root snapshot. Rediscover a live Firecracker process by API socket during cleanup, kill and wait on that PID instead of trusting only the stored runtime PID, and extend dm snapshot removal retries for transient busy handles. Add daemon regressions for stale-runtime reconcile, rediscovered process cleanup, and repeated busy dm removal. Validate with go test ./..., make build, and a live ./banger vm stop debug-ssh run that now exits cleanly.
953 lines
28 KiB
Go
953 lines
28 KiB
Go
package daemon
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"banger/internal/api"
|
|
"banger/internal/firecracker"
|
|
"banger/internal/model"
|
|
"banger/internal/paths"
|
|
"banger/internal/system"
|
|
"banger/internal/vmdns"
|
|
)
|
|
|
|
func (d *Daemon) CreateVM(ctx context.Context, params api.VMCreateParams) (vm model.VMRecord, err error) {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
op := d.beginOperation("vm.create")
|
|
defer func() {
|
|
if err != nil {
|
|
op.fail(err)
|
|
return
|
|
}
|
|
op.done(vmLogAttrs(vm)...)
|
|
}()
|
|
if err := validateOptionalPositiveSetting("vcpu", params.VCPUCount); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if err := validateOptionalPositiveSetting("memory", params.MemoryMiB); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
|
|
imageName := params.ImageName
|
|
if imageName == "" {
|
|
imageName = d.config.DefaultImageName
|
|
}
|
|
image, err := d.FindImage(ctx, imageName)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
op.stage("image_resolved", imageLogAttrs(image)...)
|
|
name := strings.TrimSpace(params.Name)
|
|
if name == "" {
|
|
name, err = d.generateName(ctx)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
}
|
|
if _, err := d.FindVM(ctx, name); err == nil {
|
|
return model.VMRecord{}, fmt.Errorf("vm name already exists: %s", name)
|
|
}
|
|
id, err := model.NewID()
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
guestIP, err := d.store.NextGuestIP(ctx, bridgePrefix(d.config.BridgeIP))
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
vmDir := filepath.Join(d.layout.VMsDir, id)
|
|
if err := os.MkdirAll(vmDir, 0o755); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
systemOverlaySize := int64(model.DefaultSystemOverlaySize)
|
|
if params.SystemOverlaySize != "" {
|
|
systemOverlaySize, err = model.ParseSize(params.SystemOverlaySize)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
}
|
|
workDiskSize := int64(model.DefaultWorkDiskSize)
|
|
if params.WorkDiskSize != "" {
|
|
workDiskSize, err = model.ParseSize(params.WorkDiskSize)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
}
|
|
now := model.Now()
|
|
spec := model.VMSpec{
|
|
VCPUCount: optionalIntOrDefault(params.VCPUCount, model.DefaultVCPUCount),
|
|
MemoryMiB: optionalIntOrDefault(params.MemoryMiB, model.DefaultMemoryMiB),
|
|
SystemOverlaySizeByte: systemOverlaySize,
|
|
WorkDiskSizeBytes: workDiskSize,
|
|
NATEnabled: params.NATEnabled,
|
|
}
|
|
vm = model.VMRecord{
|
|
ID: id,
|
|
Name: name,
|
|
ImageID: image.ID,
|
|
State: model.VMStateCreated,
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
LastTouchedAt: now,
|
|
Spec: spec,
|
|
Runtime: model.VMRuntime{
|
|
State: model.VMStateCreated,
|
|
GuestIP: guestIP,
|
|
DNSName: vmdns.RecordName(name),
|
|
VMDir: vmDir,
|
|
SystemOverlay: filepath.Join(vmDir, "system.cow"),
|
|
WorkDiskPath: filepath.Join(vmDir, "root.ext4"),
|
|
LogPath: filepath.Join(vmDir, "firecracker.log"),
|
|
MetricsPath: filepath.Join(vmDir, "metrics.json"),
|
|
},
|
|
}
|
|
if err := d.store.UpsertVM(ctx, vm); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
op.stage("persisted", vmLogAttrs(vm)...)
|
|
if params.NoStart {
|
|
vm.State = model.VMStateStopped
|
|
vm.Runtime.State = model.VMStateStopped
|
|
if err := d.store.UpsertVM(ctx, vm); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
return vm, nil
|
|
}
|
|
return d.startVMLocked(ctx, vm, image)
|
|
}
|
|
|
|
func (d *Daemon) StartVM(ctx context.Context, idOrName string) (model.VMRecord, error) {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
vm, err := d.FindVM(ctx, idOrName)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
image, err := d.store.GetImageByID(ctx, vm.ImageID)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if vm.State == model.VMStateRunning && system.ProcessRunning(vm.Runtime.PID, vm.Runtime.APISockPath) {
|
|
if d.logger != nil {
|
|
d.logger.Info("vm already running", vmLogAttrs(vm)...)
|
|
}
|
|
return vm, nil
|
|
}
|
|
return d.startVMLocked(ctx, vm, image)
|
|
}
|
|
|
|
func (d *Daemon) startVMLocked(ctx context.Context, vm model.VMRecord, image model.Image) (_ model.VMRecord, err error) {
|
|
op := d.beginOperation("vm.start", append(vmLogAttrs(vm), imageLogAttrs(image)...)...)
|
|
defer func() {
|
|
if err != nil {
|
|
err = annotateLogPath(err, vm.Runtime.LogPath)
|
|
op.fail(err, vmLogAttrs(vm)...)
|
|
return
|
|
}
|
|
op.done(vmLogAttrs(vm)...)
|
|
}()
|
|
op.stage("preflight")
|
|
if err := d.validateStartPrereqs(ctx, vm, image); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if err := os.MkdirAll(vm.Runtime.VMDir, 0o755); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
op.stage("cleanup_runtime")
|
|
if err := d.cleanupRuntime(ctx, vm, true); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
clearRuntimeHandles(&vm)
|
|
op.stage("bridge")
|
|
if err := d.ensureBridge(ctx); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
op.stage("socket_dir")
|
|
if err := d.ensureSocketDir(); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
|
|
shortID := system.ShortID(vm.ID)
|
|
apiSock := filepath.Join(d.layout.RuntimeDir, "fc-"+shortID+".sock")
|
|
tap := "tap-fc-" + shortID
|
|
dmName := "fc-rootfs-" + shortID
|
|
if err := os.RemoveAll(apiSock); err != nil && !os.IsNotExist(err) {
|
|
return model.VMRecord{}, err
|
|
}
|
|
|
|
op.stage("system_overlay", "overlay_path", vm.Runtime.SystemOverlay)
|
|
if err := d.ensureSystemOverlay(ctx, &vm); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
|
|
op.stage("dm_snapshot", "dm_name", dmName)
|
|
handles, err := d.createDMSnapshot(ctx, image.RootfsPath, vm.Runtime.SystemOverlay, dmName)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
vm.Runtime.BaseLoop = handles.BaseLoop
|
|
vm.Runtime.COWLoop = handles.COWLoop
|
|
vm.Runtime.DMName = handles.DMName
|
|
vm.Runtime.DMDev = handles.DMDev
|
|
vm.Runtime.APISockPath = apiSock
|
|
vm.Runtime.TapDevice = tap
|
|
vm.Runtime.State = model.VMStateRunning
|
|
vm.State = model.VMStateRunning
|
|
vm.Runtime.LastError = ""
|
|
|
|
cleanupOnErr := func(err error) (model.VMRecord, error) {
|
|
vm.State = model.VMStateError
|
|
vm.Runtime.State = model.VMStateError
|
|
vm.Runtime.LastError = err.Error()
|
|
op.stage("cleanup_after_failure", "error", err.Error())
|
|
if cleanupErr := d.cleanupRuntime(context.Background(), vm, true); cleanupErr != nil {
|
|
err = errors.Join(err, cleanupErr)
|
|
}
|
|
clearRuntimeHandles(&vm)
|
|
_ = d.store.UpsertVM(context.Background(), vm)
|
|
return model.VMRecord{}, err
|
|
}
|
|
|
|
op.stage("patch_root_overlay")
|
|
if err := d.patchRootOverlay(ctx, vm, image); err != nil {
|
|
return cleanupOnErr(err)
|
|
}
|
|
op.stage("work_disk", "work_disk_path", vm.Runtime.WorkDiskPath)
|
|
if err := d.ensureWorkDisk(ctx, &vm); err != nil {
|
|
return cleanupOnErr(err)
|
|
}
|
|
op.stage("tap", "tap_device", tap)
|
|
if err := d.createTap(ctx, tap); err != nil {
|
|
return cleanupOnErr(err)
|
|
}
|
|
op.stage("metrics_file", "metrics_path", vm.Runtime.MetricsPath)
|
|
if err := os.WriteFile(vm.Runtime.MetricsPath, nil, 0o644); err != nil {
|
|
return cleanupOnErr(err)
|
|
}
|
|
|
|
op.stage("firecracker_binary")
|
|
fcPath, err := d.firecrackerBinary()
|
|
if err != nil {
|
|
return cleanupOnErr(err)
|
|
}
|
|
op.stage("firecracker_launch", "log_path", vm.Runtime.LogPath, "metrics_path", vm.Runtime.MetricsPath)
|
|
firecrackerCtx := context.Background()
|
|
machine, err := firecracker.NewMachine(firecrackerCtx, firecracker.MachineConfig{
|
|
BinaryPath: fcPath,
|
|
VMID: vm.ID,
|
|
SocketPath: apiSock,
|
|
LogPath: vm.Runtime.LogPath,
|
|
MetricsPath: vm.Runtime.MetricsPath,
|
|
KernelImagePath: image.KernelPath,
|
|
InitrdPath: image.InitrdPath,
|
|
KernelArgs: system.BuildBootArgs(vm.Name, vm.Runtime.GuestIP, d.config.BridgeIP, d.config.DefaultDNS),
|
|
RootDrivePath: vm.Runtime.DMDev,
|
|
WorkDrivePath: vm.Runtime.WorkDiskPath,
|
|
TapDevice: tap,
|
|
VCPUCount: vm.Spec.VCPUCount,
|
|
MemoryMiB: vm.Spec.MemoryMiB,
|
|
Logger: d.logger,
|
|
})
|
|
if err != nil {
|
|
return cleanupOnErr(err)
|
|
}
|
|
if err := machine.Start(firecrackerCtx); err != nil {
|
|
vm.Runtime.PID = d.resolveFirecrackerPID(firecrackerCtx, machine, apiSock)
|
|
return cleanupOnErr(err)
|
|
}
|
|
vm.Runtime.PID = d.resolveFirecrackerPID(firecrackerCtx, machine, apiSock)
|
|
op.debugStage("firecracker_started", "pid", vm.Runtime.PID)
|
|
op.stage("socket_access", "api_socket", apiSock)
|
|
if err := d.ensureSocketAccess(ctx, apiSock); err != nil {
|
|
return cleanupOnErr(err)
|
|
}
|
|
op.stage("dns", "dns_name", vm.Runtime.DNSName)
|
|
if err := d.setDNS(ctx, vm.Name, vm.Runtime.GuestIP); err != nil {
|
|
return cleanupOnErr(err)
|
|
}
|
|
if vm.Spec.NATEnabled {
|
|
op.stage("nat")
|
|
if err := d.ensureNAT(ctx, vm, true); err != nil {
|
|
return cleanupOnErr(err)
|
|
}
|
|
}
|
|
system.TouchNow(&vm)
|
|
op.stage("persist")
|
|
if err := d.store.UpsertVM(ctx, vm); err != nil {
|
|
return cleanupOnErr(err)
|
|
}
|
|
return vm, nil
|
|
}
|
|
|
|
func (d *Daemon) StopVM(ctx context.Context, idOrName string) (vm model.VMRecord, err error) {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
op := d.beginOperation("vm.stop", "vm_ref", idOrName)
|
|
defer func() {
|
|
if err != nil {
|
|
op.fail(err, vmLogAttrs(vm)...)
|
|
return
|
|
}
|
|
op.done(vmLogAttrs(vm)...)
|
|
}()
|
|
vm, err = d.FindVM(ctx, idOrName)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if vm.State != model.VMStateRunning || !system.ProcessRunning(vm.Runtime.PID, vm.Runtime.APISockPath) {
|
|
op.stage("cleanup_stale_runtime")
|
|
if err := d.cleanupRuntime(ctx, vm, true); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
vm.State = model.VMStateStopped
|
|
vm.Runtime.State = model.VMStateStopped
|
|
clearRuntimeHandles(&vm)
|
|
if err := d.store.UpsertVM(ctx, vm); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
return vm, nil
|
|
}
|
|
op.stage("graceful_shutdown")
|
|
if err := d.sendCtrlAltDel(ctx, vm); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
op.stage("wait_for_exit", "pid", vm.Runtime.PID)
|
|
if err := d.waitForExit(ctx, vm.Runtime.PID, vm.Runtime.APISockPath, 30*time.Second); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
op.stage("cleanup_runtime")
|
|
if err := d.cleanupRuntime(ctx, vm, true); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
vm.State = model.VMStateStopped
|
|
vm.Runtime.State = model.VMStateStopped
|
|
clearRuntimeHandles(&vm)
|
|
system.TouchNow(&vm)
|
|
if err := d.store.UpsertVM(ctx, vm); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
return vm, nil
|
|
}
|
|
|
|
func (d *Daemon) KillVM(ctx context.Context, params api.VMKillParams) (vm model.VMRecord, err error) {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
op := d.beginOperation("vm.kill", "vm_ref", params.IDOrName, "signal", params.Signal)
|
|
defer func() {
|
|
if err != nil {
|
|
op.fail(err, vmLogAttrs(vm)...)
|
|
return
|
|
}
|
|
op.done(vmLogAttrs(vm)...)
|
|
}()
|
|
|
|
vm, err = d.FindVM(ctx, params.IDOrName)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if vm.State != model.VMStateRunning || !system.ProcessRunning(vm.Runtime.PID, vm.Runtime.APISockPath) {
|
|
op.stage("cleanup_stale_runtime")
|
|
if err := d.cleanupRuntime(ctx, vm, true); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
vm.State = model.VMStateStopped
|
|
vm.Runtime.State = model.VMStateStopped
|
|
clearRuntimeHandles(&vm)
|
|
if err := d.store.UpsertVM(ctx, vm); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
return vm, nil
|
|
}
|
|
|
|
signal := strings.TrimSpace(params.Signal)
|
|
if signal == "" {
|
|
signal = "TERM"
|
|
}
|
|
op.stage("send_signal", "pid", vm.Runtime.PID, "signal", signal)
|
|
if _, err := d.runner.RunSudo(ctx, "kill", "-"+signal, strconv.Itoa(vm.Runtime.PID)); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
op.stage("wait_for_exit", "pid", vm.Runtime.PID)
|
|
if err := d.waitForExit(ctx, vm.Runtime.PID, vm.Runtime.APISockPath, 30*time.Second); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
op.stage("cleanup_runtime")
|
|
if err := d.cleanupRuntime(ctx, vm, true); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
vm.State = model.VMStateStopped
|
|
vm.Runtime.State = model.VMStateStopped
|
|
clearRuntimeHandles(&vm)
|
|
system.TouchNow(&vm)
|
|
if err := d.store.UpsertVM(ctx, vm); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
return vm, nil
|
|
}
|
|
|
|
func (d *Daemon) RestartVM(ctx context.Context, idOrName string) (vm model.VMRecord, err error) {
|
|
op := d.beginOperation("vm.restart", "vm_ref", idOrName)
|
|
defer func() {
|
|
if err != nil {
|
|
op.fail(err, vmLogAttrs(vm)...)
|
|
return
|
|
}
|
|
op.done(vmLogAttrs(vm)...)
|
|
}()
|
|
op.stage("stop")
|
|
vm, err = d.StopVM(ctx, idOrName)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
op.stage("start", vmLogAttrs(vm)...)
|
|
return d.StartVM(ctx, vm.ID)
|
|
}
|
|
|
|
func (d *Daemon) DeleteVM(ctx context.Context, idOrName string) (vm model.VMRecord, err error) {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
op := d.beginOperation("vm.delete", "vm_ref", idOrName)
|
|
defer func() {
|
|
if err != nil {
|
|
op.fail(err, vmLogAttrs(vm)...)
|
|
return
|
|
}
|
|
op.done(vmLogAttrs(vm)...)
|
|
}()
|
|
vm, err = d.FindVM(ctx, idOrName)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if vm.State == model.VMStateRunning && system.ProcessRunning(vm.Runtime.PID, vm.Runtime.APISockPath) {
|
|
op.stage("kill_running_vm", "pid", vm.Runtime.PID)
|
|
_ = d.killVMProcess(ctx, vm.Runtime.PID)
|
|
}
|
|
op.stage("cleanup_runtime")
|
|
if err := d.cleanupRuntime(ctx, vm, false); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if vm.Spec.NATEnabled {
|
|
op.debugStage("disable_nat")
|
|
_ = d.ensureNAT(ctx, vm, false)
|
|
}
|
|
op.debugStage("remove_dns", "dns_name", vm.Runtime.DNSName)
|
|
_ = d.removeDNS(ctx, vm.Runtime.DNSName)
|
|
op.stage("delete_store_record")
|
|
if err := d.store.DeleteVM(ctx, vm.ID); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if vm.Runtime.VMDir != "" {
|
|
op.stage("delete_vm_dir", "vm_dir", vm.Runtime.VMDir)
|
|
if err := os.RemoveAll(vm.Runtime.VMDir); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
}
|
|
return vm, nil
|
|
}
|
|
|
|
func (d *Daemon) SetVM(ctx context.Context, params api.VMSetParams) (vm model.VMRecord, err error) {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
op := d.beginOperation("vm.set", "vm_ref", params.IDOrName)
|
|
defer func() {
|
|
if err != nil {
|
|
op.fail(err, vmLogAttrs(vm)...)
|
|
return
|
|
}
|
|
op.done(vmLogAttrs(vm)...)
|
|
}()
|
|
vm, err = d.FindVM(ctx, params.IDOrName)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
running := vm.State == model.VMStateRunning && system.ProcessRunning(vm.Runtime.PID, vm.Runtime.APISockPath)
|
|
if params.VCPUCount != nil {
|
|
if err := validateOptionalPositiveSetting("vcpu", params.VCPUCount); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if running {
|
|
return model.VMRecord{}, errors.New("vcpu changes require the VM to be stopped")
|
|
}
|
|
op.stage("update_vcpu", "vcpu_count", *params.VCPUCount)
|
|
vm.Spec.VCPUCount = *params.VCPUCount
|
|
}
|
|
if params.MemoryMiB != nil {
|
|
if err := validateOptionalPositiveSetting("memory", params.MemoryMiB); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if running {
|
|
return model.VMRecord{}, errors.New("memory changes require the VM to be stopped")
|
|
}
|
|
op.stage("update_memory", "memory_mib", *params.MemoryMiB)
|
|
vm.Spec.MemoryMiB = *params.MemoryMiB
|
|
}
|
|
if params.WorkDiskSize != "" {
|
|
size, err := model.ParseSize(params.WorkDiskSize)
|
|
if err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if running {
|
|
return model.VMRecord{}, errors.New("disk changes require the VM to be stopped")
|
|
}
|
|
if size < vm.Spec.WorkDiskSizeBytes {
|
|
return model.VMRecord{}, errors.New("disk size can only grow")
|
|
}
|
|
if size > vm.Spec.WorkDiskSizeBytes {
|
|
if exists(vm.Runtime.WorkDiskPath) {
|
|
op.stage("resize_work_disk", "from_bytes", vm.Spec.WorkDiskSizeBytes, "to_bytes", size)
|
|
if err := d.validateWorkDiskResizePrereqs(); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
if err := system.ResizeExt4Image(ctx, d.runner, vm.Runtime.WorkDiskPath, size); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
}
|
|
vm.Spec.WorkDiskSizeBytes = size
|
|
}
|
|
}
|
|
if params.NATEnabled != nil {
|
|
op.stage("update_nat", "nat_enabled", *params.NATEnabled)
|
|
vm.Spec.NATEnabled = *params.NATEnabled
|
|
if running {
|
|
if err := d.ensureNAT(ctx, vm, *params.NATEnabled); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
}
|
|
}
|
|
system.TouchNow(&vm)
|
|
if err := d.store.UpsertVM(ctx, vm); err != nil {
|
|
return model.VMRecord{}, err
|
|
}
|
|
return vm, nil
|
|
}
|
|
|
|
func (d *Daemon) GetVMStats(ctx context.Context, idOrName string) (model.VMRecord, model.VMStats, error) {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
vm, err := d.FindVM(ctx, idOrName)
|
|
if err != nil {
|
|
return model.VMRecord{}, model.VMStats{}, err
|
|
}
|
|
stats, err := d.collectStats(ctx, vm)
|
|
if err == nil {
|
|
vm.Stats = stats
|
|
vm.UpdatedAt = model.Now()
|
|
_ = d.store.UpsertVM(ctx, vm)
|
|
if d.logger != nil {
|
|
d.logger.Debug("vm stats collected", append(vmLogAttrs(vm), "rss_bytes", stats.RSSBytes, "vsz_bytes", stats.VSZBytes, "cpu_percent", stats.CPUPercent)...)
|
|
}
|
|
}
|
|
return vm, vm.Stats, nil
|
|
}
|
|
|
|
func (d *Daemon) pollStats(ctx context.Context) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
vms, err := d.store.ListVMs(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, vm := range vms {
|
|
if vm.State != model.VMStateRunning || !system.ProcessRunning(vm.Runtime.PID, vm.Runtime.APISockPath) {
|
|
continue
|
|
}
|
|
stats, err := d.collectStats(ctx, vm)
|
|
if err != nil {
|
|
if d.logger != nil {
|
|
d.logger.Debug("vm stats collection failed", append(vmLogAttrs(vm), "error", err.Error())...)
|
|
}
|
|
continue
|
|
}
|
|
vm.Stats = stats
|
|
vm.UpdatedAt = model.Now()
|
|
_ = d.store.UpsertVM(ctx, vm)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *Daemon) stopStaleVMs(ctx context.Context) (err error) {
|
|
if d.config.AutoStopStaleAfter <= 0 {
|
|
return nil
|
|
}
|
|
op := d.beginOperation("vm.stop_stale")
|
|
defer func() {
|
|
if err != nil {
|
|
op.fail(err)
|
|
return
|
|
}
|
|
op.done()
|
|
}()
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
vms, err := d.store.ListVMs(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now := model.Now()
|
|
for _, vm := range vms {
|
|
if vm.State != model.VMStateRunning || !system.ProcessRunning(vm.Runtime.PID, vm.Runtime.APISockPath) {
|
|
continue
|
|
}
|
|
if now.Sub(vm.LastTouchedAt) < d.config.AutoStopStaleAfter {
|
|
continue
|
|
}
|
|
op.stage("stopping_vm", vmLogAttrs(vm)...)
|
|
_ = d.sendCtrlAltDel(ctx, vm)
|
|
_ = d.waitForExit(ctx, vm.Runtime.PID, vm.Runtime.APISockPath, 10*time.Second)
|
|
_ = d.cleanupRuntime(ctx, vm, true)
|
|
vm.State = model.VMStateStopped
|
|
vm.Runtime.State = model.VMStateStopped
|
|
clearRuntimeHandles(&vm)
|
|
vm.UpdatedAt = model.Now()
|
|
_ = d.store.UpsertVM(ctx, vm)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *Daemon) collectStats(ctx context.Context, vm model.VMRecord) (model.VMStats, error) {
|
|
stats := model.VMStats{
|
|
CollectedAt: model.Now(),
|
|
SystemOverlayBytes: system.AllocatedBytes(vm.Runtime.SystemOverlay),
|
|
WorkDiskBytes: system.AllocatedBytes(vm.Runtime.WorkDiskPath),
|
|
MetricsRaw: system.ParseMetricsFile(vm.Runtime.MetricsPath),
|
|
}
|
|
if vm.Runtime.PID > 0 && system.ProcessRunning(vm.Runtime.PID, vm.Runtime.APISockPath) {
|
|
ps, err := system.ReadProcessStats(ctx, vm.Runtime.PID)
|
|
if err == nil {
|
|
stats.CPUPercent = ps.CPUPercent
|
|
stats.RSSBytes = ps.RSSBytes
|
|
stats.VSZBytes = ps.VSZBytes
|
|
}
|
|
}
|
|
return stats, nil
|
|
}
|
|
|
|
func (d *Daemon) ensureSystemOverlay(ctx context.Context, vm *model.VMRecord) error {
|
|
if exists(vm.Runtime.SystemOverlay) {
|
|
return nil
|
|
}
|
|
_, err := d.runner.Run(ctx, "truncate", "-s", strconv.FormatInt(vm.Spec.SystemOverlaySizeByte, 10), vm.Runtime.SystemOverlay)
|
|
return err
|
|
}
|
|
|
|
func (d *Daemon) patchRootOverlay(ctx context.Context, vm model.VMRecord, image model.Image) error {
|
|
resolv := []byte(fmt.Sprintf("nameserver %s\n", d.config.DefaultDNS))
|
|
hostname := []byte(vm.Name + "\n")
|
|
hosts := []byte(fmt.Sprintf("127.0.0.1 localhost\n127.0.1.1 %s\n", vm.Name))
|
|
sshdConfig := []byte(strings.Join([]string{
|
|
"LogLevel DEBUG3",
|
|
"PermitRootLogin yes",
|
|
"PubkeyAuthentication yes",
|
|
"AuthorizedKeysFile /root/.ssh/authorized_keys",
|
|
"StrictModes no",
|
|
"",
|
|
}, "\n"))
|
|
fstab, err := system.ReadDebugFSText(ctx, d.runner, vm.Runtime.DMDev, "/etc/fstab")
|
|
if err != nil {
|
|
fstab = ""
|
|
}
|
|
newFSTab := system.UpdateFSTab(fstab)
|
|
for guestPath, data := range map[string][]byte{
|
|
"/etc/resolv.conf": resolv,
|
|
"/etc/hostname": hostname,
|
|
"/etc/hosts": hosts,
|
|
"/etc/fstab": []byte(newFSTab),
|
|
"/etc/ssh/sshd_config.d/99-banger.conf": sshdConfig,
|
|
} {
|
|
if err := system.WriteExt4File(ctx, d.runner, vm.Runtime.DMDev, guestPath, data); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *Daemon) ensureWorkDisk(ctx context.Context, vm *model.VMRecord) error {
|
|
if exists(vm.Runtime.WorkDiskPath) {
|
|
return nil
|
|
}
|
|
if _, err := d.runner.Run(ctx, "truncate", "-s", strconv.FormatInt(vm.Spec.WorkDiskSizeBytes, 10), vm.Runtime.WorkDiskPath); err != nil {
|
|
return err
|
|
}
|
|
if _, err := d.runner.Run(ctx, "mkfs.ext4", "-F", vm.Runtime.WorkDiskPath); err != nil {
|
|
return err
|
|
}
|
|
rootMount, cleanupRoot, err := system.MountTempDir(ctx, d.runner, vm.Runtime.DMDev, true)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer cleanupRoot()
|
|
workMount, cleanupWork, err := system.MountTempDir(ctx, d.runner, vm.Runtime.WorkDiskPath, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer cleanupWork()
|
|
if err := system.CopyDirContents(ctx, d.runner, filepath.Join(rootMount, "root"), workMount, true); err != nil {
|
|
return err
|
|
}
|
|
if err := d.flattenNestedWorkHome(ctx, workMount); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (d *Daemon) flattenNestedWorkHome(ctx context.Context, workMount string) error {
|
|
nestedHome := filepath.Join(workMount, "root")
|
|
if !exists(nestedHome) {
|
|
return nil
|
|
}
|
|
if _, err := d.runner.RunSudo(ctx, "chmod", "755", nestedHome); err != nil {
|
|
return err
|
|
}
|
|
entries, err := os.ReadDir(nestedHome)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, entry := range entries {
|
|
sourcePath := filepath.Join(nestedHome, entry.Name())
|
|
if _, err := d.runner.RunSudo(ctx, "cp", "-a", sourcePath, workMount+"/"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
_, err = d.runner.RunSudo(ctx, "rm", "-rf", nestedHome)
|
|
return err
|
|
}
|
|
|
|
func (d *Daemon) ensureBridge(ctx context.Context) error {
|
|
if _, err := d.runner.Run(ctx, "ip", "link", "show", d.config.BridgeName); err == nil {
|
|
_, err = d.runner.RunSudo(ctx, "ip", "link", "set", d.config.BridgeName, "up")
|
|
return err
|
|
}
|
|
if _, err := d.runner.RunSudo(ctx, "ip", "link", "add", "name", d.config.BridgeName, "type", "bridge"); err != nil {
|
|
return err
|
|
}
|
|
if _, err := d.runner.RunSudo(ctx, "ip", "addr", "add", fmt.Sprintf("%s/%s", d.config.BridgeIP, d.config.CIDR), "dev", d.config.BridgeName); err != nil {
|
|
return err
|
|
}
|
|
_, err := d.runner.RunSudo(ctx, "ip", "link", "set", d.config.BridgeName, "up")
|
|
return err
|
|
}
|
|
|
|
func (d *Daemon) ensureSocketDir() error {
|
|
return os.MkdirAll(d.layout.RuntimeDir, 0o755)
|
|
}
|
|
|
|
func (d *Daemon) createTap(ctx context.Context, tap string) error {
|
|
if _, err := d.runner.Run(ctx, "ip", "link", "show", tap); err == nil {
|
|
_, _ = d.runner.RunSudo(ctx, "ip", "link", "del", tap)
|
|
}
|
|
if _, err := d.runner.RunSudo(ctx, "ip", "tuntap", "add", "dev", tap, "mode", "tap", "user", strconv.Itoa(os.Getuid()), "group", strconv.Itoa(os.Getgid())); err != nil {
|
|
return err
|
|
}
|
|
if _, err := d.runner.RunSudo(ctx, "ip", "link", "set", tap, "master", d.config.BridgeName); err != nil {
|
|
return err
|
|
}
|
|
if _, err := d.runner.RunSudo(ctx, "ip", "link", "set", tap, "up"); err != nil {
|
|
return err
|
|
}
|
|
_, err := d.runner.RunSudo(ctx, "ip", "link", "set", d.config.BridgeName, "up")
|
|
return err
|
|
}
|
|
|
|
func (d *Daemon) firecrackerBinary() (string, error) {
|
|
if d.config.FirecrackerBin == "" {
|
|
return "", fmt.Errorf("firecracker binary not configured; %s", paths.RuntimeBundleHint())
|
|
}
|
|
path := d.config.FirecrackerBin
|
|
if !exists(path) {
|
|
return "", fmt.Errorf("firecracker binary not found at %s; %s", path, paths.RuntimeBundleHint())
|
|
}
|
|
return path, nil
|
|
}
|
|
|
|
func (d *Daemon) ensureSocketAccess(ctx context.Context, apiSock string) error {
|
|
if _, err := d.runner.RunSudo(ctx, "chown", fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), apiSock); err != nil {
|
|
return err
|
|
}
|
|
_, err := d.runner.RunSudo(ctx, "chmod", "600", apiSock)
|
|
return err
|
|
}
|
|
|
|
func (d *Daemon) findFirecrackerPID(ctx context.Context, apiSock string) (int, error) {
|
|
out, err := d.runner.Run(ctx, "pgrep", "-n", "-f", apiSock)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return strconv.Atoi(strings.TrimSpace(string(out)))
|
|
}
|
|
|
|
func (d *Daemon) resolveFirecrackerPID(ctx context.Context, machine *firecracker.Machine, apiSock string) int {
|
|
if pid, err := d.findFirecrackerPID(ctx, apiSock); err == nil && pid > 0 {
|
|
return pid
|
|
}
|
|
if machine != nil {
|
|
if pid, err := machine.PID(); err == nil && pid > 0 {
|
|
return pid
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (d *Daemon) sendCtrlAltDel(ctx context.Context, vm model.VMRecord) error {
|
|
if err := d.ensureSocketAccess(ctx, vm.Runtime.APISockPath); err != nil {
|
|
return err
|
|
}
|
|
client := firecracker.New(vm.Runtime.APISockPath, d.logger)
|
|
return client.SendCtrlAltDel(ctx)
|
|
}
|
|
|
|
func (d *Daemon) waitForExit(ctx context.Context, pid int, apiSock string, timeout time.Duration) error {
|
|
deadline := time.Now().Add(timeout)
|
|
for {
|
|
if !system.ProcessRunning(pid, apiSock) {
|
|
return nil
|
|
}
|
|
if time.Now().After(deadline) {
|
|
return fmt.Errorf("timed out waiting for VM to exit")
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(100 * time.Millisecond):
|
|
}
|
|
}
|
|
}
|
|
|
|
func (d *Daemon) cleanupRuntime(ctx context.Context, vm model.VMRecord, preserveDisks bool) error {
|
|
if d.logger != nil {
|
|
d.logger.Debug("cleanup runtime", append(vmLogAttrs(vm), "preserve_disks", preserveDisks)...)
|
|
}
|
|
cleanupPID := vm.Runtime.PID
|
|
if vm.Runtime.APISockPath != "" {
|
|
if pid, err := d.findFirecrackerPID(ctx, vm.Runtime.APISockPath); err == nil && pid > 0 {
|
|
cleanupPID = pid
|
|
}
|
|
}
|
|
if cleanupPID > 0 && system.ProcessRunning(cleanupPID, vm.Runtime.APISockPath) {
|
|
_ = d.killVMProcess(ctx, cleanupPID)
|
|
if err := d.waitForExit(ctx, cleanupPID, vm.Runtime.APISockPath, 30*time.Second); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if vm.Runtime.TapDevice != "" {
|
|
_, _ = d.runner.RunSudo(ctx, "ip", "link", "del", vm.Runtime.TapDevice)
|
|
}
|
|
if vm.Runtime.APISockPath != "" {
|
|
_ = os.Remove(vm.Runtime.APISockPath)
|
|
}
|
|
snapshotErr := d.cleanupDMSnapshot(ctx, dmSnapshotHandles{
|
|
BaseLoop: vm.Runtime.BaseLoop,
|
|
COWLoop: vm.Runtime.COWLoop,
|
|
DMName: vm.Runtime.DMName,
|
|
DMDev: vm.Runtime.DMDev,
|
|
})
|
|
if vm.Spec.NATEnabled {
|
|
_ = d.ensureNAT(ctx, vm, false)
|
|
}
|
|
_ = d.removeDNS(ctx, vm.Runtime.DNSName)
|
|
if !preserveDisks && vm.Runtime.VMDir != "" {
|
|
return errors.Join(snapshotErr, os.RemoveAll(vm.Runtime.VMDir))
|
|
}
|
|
return snapshotErr
|
|
}
|
|
|
|
func clearRuntimeHandles(vm *model.VMRecord) {
|
|
vm.Runtime.PID = 0
|
|
vm.Runtime.APISockPath = ""
|
|
vm.Runtime.TapDevice = ""
|
|
vm.Runtime.BaseLoop = ""
|
|
vm.Runtime.COWLoop = ""
|
|
vm.Runtime.DMName = ""
|
|
vm.Runtime.DMDev = ""
|
|
}
|
|
|
|
func (d *Daemon) setDNS(ctx context.Context, vmName, guestIP string) error {
|
|
if d.vmDNS == nil {
|
|
return nil
|
|
}
|
|
return d.vmDNS.Set(vmdns.RecordName(vmName), guestIP)
|
|
}
|
|
|
|
func (d *Daemon) removeDNS(ctx context.Context, dnsName string) error {
|
|
if dnsName == "" {
|
|
return nil
|
|
}
|
|
if d.vmDNS == nil {
|
|
return nil
|
|
}
|
|
return d.vmDNS.Remove(dnsName)
|
|
}
|
|
|
|
func (d *Daemon) rebuildDNS(ctx context.Context) error {
|
|
if d.vmDNS == nil {
|
|
return nil
|
|
}
|
|
vms, err := d.store.ListVMs(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
records := make(map[string]string)
|
|
for _, vm := range vms {
|
|
if vm.State != model.VMStateRunning {
|
|
continue
|
|
}
|
|
if !system.ProcessRunning(vm.Runtime.PID, vm.Runtime.APISockPath) {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(vm.Runtime.GuestIP) == "" {
|
|
continue
|
|
}
|
|
records[vmdns.RecordName(vm.Name)] = vm.Runtime.GuestIP
|
|
}
|
|
return d.vmDNS.Replace(records)
|
|
}
|
|
|
|
func (d *Daemon) killVMProcess(ctx context.Context, pid int) error {
|
|
_, err := d.runner.RunSudo(ctx, "kill", "-KILL", strconv.Itoa(pid))
|
|
return err
|
|
}
|
|
|
|
func (d *Daemon) generateName(ctx context.Context) (string, error) {
|
|
if exists(d.config.NamegenPath) {
|
|
out, err := d.runner.Run(ctx, d.config.NamegenPath)
|
|
if err == nil {
|
|
name := strings.TrimSpace(string(out))
|
|
if name != "" {
|
|
return name, nil
|
|
}
|
|
}
|
|
}
|
|
return "vm-" + strconv.FormatInt(time.Now().Unix(), 10), nil
|
|
}
|
|
|
|
func bridgePrefix(bridgeIP string) string {
|
|
parts := strings.Split(bridgeIP, ".")
|
|
if len(parts) < 3 {
|
|
return bridgeIP
|
|
}
|
|
return strings.Join(parts[:3], ".")
|
|
}
|
|
|
|
func optionalIntOrDefault(value *int, fallback int) int {
|
|
if value != nil {
|
|
return *value
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
func validateOptionalPositiveSetting(label string, value *int) error {
|
|
if value == nil {
|
|
return nil
|
|
}
|
|
if *value <= 0 {
|
|
return fmt.Errorf("%s must be a positive integer", label)
|
|
}
|
|
return nil
|
|
}
|