package daemon import ( "bufio" "bytes" "context" "errors" "fmt" "io" "net" "os" "path/filepath" "sort" "strconv" "strings" "syscall" "time" "banger/internal/guest" "banger/internal/model" "banger/internal/system" "golang.org/x/crypto/ssh" ) const ( guestSessionBackendSSH = "ssh" guestSessionAttachBackendNone = "none" guestSessionAttachBackendSSHBridge = "ssh_rehydratable" guestSessionAttachModeExclusive = "exclusive" guestSessionTransportUnixSocket = "unix_socket" guestSessionStateRoot = "/root/.local/state/banger/sessions" guestSessionLogTailLine = 200 ) var guestSessionHostCommandOutputFunc = func(ctx context.Context, name string, args ...string) ([]byte, error) { runner := system.NewRunner() output, err := runner.Run(ctx, name, args...) if err == nil { return output, nil } command := strings.TrimSpace(strings.Join(append([]string{name}, args...), " ")) detail := strings.TrimSpace(string(output)) if detail == "" { return output, fmt.Errorf("%s: %w", command, err) } return output, fmt.Errorf("%s: %w: %s", command, err, detail) } type guestSSHClient interface { Close() error RunScript(context.Context, string, io.Writer) error RunScriptOutput(context.Context, string) ([]byte, error) UploadFile(context.Context, string, os.FileMode, []byte, io.Writer) error StreamTar(context.Context, string, string, io.Writer) error StreamTarEntries(context.Context, string, []string, string, io.Writer) error } func (d *Daemon) waitForGuestSSH(ctx context.Context, address string, interval time.Duration) error { if d != nil && d.guestWaitForSSH != nil { return d.guestWaitForSSH(ctx, address, d.config.SSHKeyPath, interval) } return guest.WaitForSSH(ctx, address, d.config.SSHKeyPath, interval) } func (d *Daemon) dialGuest(ctx context.Context, address string) (guestSSHClient, error) { if d != nil && d.guestDial != nil { return d.guestDial(ctx, address, d.config.SSHKeyPath) } return guest.Dial(ctx, address, d.config.SSHKeyPath) } func (d *Daemon) waitForGuestSessionReadyHook(ctx context.Context, vm model.VMRecord, session model.GuestSession) (model.GuestSession, error) { if d != nil && d.waitForGuestSessionReady != nil { return d.waitForGuestSessionReady(ctx, vm, session) } return d.waitForGuestSessionReadyDefault(ctx, vm, session) } func (d *Daemon) waitForGuestSessionReadyDefault(ctx context.Context, vm model.VMRecord, session model.GuestSession) (model.GuestSession, error) { for { updated, err := d.refreshGuestSession(ctx, vm, session) if err == nil { session = updated if session.GuestPID != 0 || session.ExitCode != nil || session.Status == model.GuestSessionStatusRunning || session.Status == model.GuestSessionStatusFailed || session.Status == model.GuestSessionStatusExited { return session, nil } } select { case <-ctx.Done(): return session, ctx.Err() case <-time.After(100 * time.Millisecond): } } } func (d *Daemon) refreshGuestSession(ctx context.Context, vm model.VMRecord, session model.GuestSession) (model.GuestSession, error) { if session.Status != model.GuestSessionStatusStarting && session.Status != model.GuestSessionStatusRunning && session.Status != model.GuestSessionStatusStopping { return session, nil } snapshot, err := d.inspectGuestSessionState(ctx, vm, session) if err != nil { return session, err } original := session applyGuestSessionSnapshot(&session, snapshot, vm.State == model.VMStateRunning && system.ProcessRunning(vm.Runtime.PID, vm.Runtime.APISockPath)) if guestSessionStateChanged(original, session) { session.UpdatedAt = model.Now() if err := d.store.UpsertGuestSession(ctx, session); err != nil { return session, err } } return session, nil } func applyGuestSessionSnapshot(session *model.GuestSession, snapshot guestSessionStateSnapshot, vmRunning bool) { if session == nil { return } if snapshot.GuestPID != 0 { session.GuestPID = snapshot.GuestPID } if snapshot.LastError != "" { session.LastError = snapshot.LastError } if snapshot.ExitCode != nil { session.ExitCode = snapshot.ExitCode session.Attachable = false session.Reattachable = false if session.StartedAt.IsZero() { session.StartedAt = model.Now() } if session.EndedAt.IsZero() { session.EndedAt = model.Now() } if *snapshot.ExitCode == 0 { session.Status = model.GuestSessionStatusExited } else { session.Status = model.GuestSessionStatusFailed } return } if snapshot.Alive { if session.StartedAt.IsZero() { session.StartedAt = model.Now() } session.Status = model.GuestSessionStatusRunning return } if !vmRunning && (session.Status == model.GuestSessionStatusStarting || session.Status == model.GuestSessionStatusRunning || session.Status == model.GuestSessionStatusStopping) { session.Status = model.GuestSessionStatusFailed session.Attachable = false session.Reattachable = false if session.LastError == "" { session.LastError = "vm is not running" } if session.EndedAt.IsZero() { session.EndedAt = model.Now() } return } if snapshot.Status == string(model.GuestSessionStatusRunning) { if session.StartedAt.IsZero() { session.StartedAt = model.Now() } session.Status = model.GuestSessionStatusRunning } if session.Status == model.GuestSessionStatusRunning && session.StdinMode == model.GuestSessionStdinPipe { session.Attachable = true session.Reattachable = true if session.AttachBackend == "" { session.AttachBackend = guestSessionAttachBackendSSHBridge } if session.AttachMode == "" { session.AttachMode = guestSessionAttachModeExclusive } } } func (d *Daemon) inspectGuestSessionState(ctx context.Context, vm model.VMRecord, session model.GuestSession) (guestSessionStateSnapshot, error) { if vm.State == model.VMStateRunning && system.ProcessRunning(vm.Runtime.PID, vm.Runtime.APISockPath) { client, err := guest.Dial(ctx, net.JoinHostPort(vm.Runtime.GuestIP, "22"), d.config.SSHKeyPath) if err != nil { return guestSessionStateSnapshot{}, err } defer client.Close() var output bytes.Buffer if err := client.RunScript(ctx, guestSessionInspectScript(session.ID), &output); err != nil { return guestSessionStateSnapshot{}, formatGuestSessionStepError("inspect guest session state", err, output.String()) } return parseGuestSessionState(output.String()) } return d.inspectGuestSessionStateFromWorkDisk(ctx, vm, session.ID) } func (d *Daemon) inspectGuestSessionStateFromWorkDisk(ctx context.Context, vm model.VMRecord, sessionID string) (guestSessionStateSnapshot, error) { runner := d.runner if runner == nil { runner = system.NewRunner() } workMount, cleanup, err := system.MountTempDir(ctx, runner, vm.Runtime.WorkDiskPath, false) if err != nil { return guestSessionStateSnapshot{}, err } defer cleanup() stateDir := filepath.Join(workMount, guestSessionRelativeStateDir(sessionID)) return inspectGuestSessionStateFromDir(stateDir) } func inspectGuestSessionStateFromDir(stateDir string) (guestSessionStateSnapshot, error) { var snapshot guestSessionStateSnapshot statusData, _ := os.ReadFile(filepath.Join(stateDir, "status")) snapshot.Status = strings.TrimSpace(string(statusData)) pidData, _ := os.ReadFile(filepath.Join(stateDir, "pid")) if pidValue, err := strconv.Atoi(strings.TrimSpace(string(pidData))); err == nil { snapshot.GuestPID = pidValue } exitData, _ := os.ReadFile(filepath.Join(stateDir, "exit_code")) if exitValue, err := strconv.Atoi(strings.TrimSpace(string(exitData))); err == nil { snapshot.ExitCode = &exitValue } errorData, _ := os.ReadFile(filepath.Join(stateDir, "error")) snapshot.LastError = strings.TrimSpace(string(errorData)) if snapshot.GuestPID != 0 { snapshot.Alive = processAlive(snapshot.GuestPID) } return snapshot, nil } func (d *Daemon) findGuestSession(ctx context.Context, vmID, idOrName string) (model.GuestSession, error) { if strings.TrimSpace(idOrName) == "" { return model.GuestSession{}, errors.New("session id or name is required") } if session, err := d.store.GetGuestSession(ctx, vmID, idOrName); err == nil { return session, nil } sessions, err := d.store.ListGuestSessionsByVM(ctx, vmID) if err != nil { return model.GuestSession{}, err } matches := make([]model.GuestSession, 0, 1) for _, session := range sessions { if strings.HasPrefix(session.ID, idOrName) || strings.HasPrefix(session.Name, idOrName) { matches = append(matches, session) } } switch len(matches) { case 0: return model.GuestSession{}, fmt.Errorf("session %q not found", idOrName) case 1: return matches[0], nil default: return model.GuestSession{}, fmt.Errorf("multiple sessions match %q", idOrName) } } func guestSessionScript(session model.GuestSession) string { var script strings.Builder script.WriteString("set -euo pipefail\n") fmt.Fprintf(&script, "STATE_DIR=%s\n", guestShellQuote(session.GuestStateDir)) fmt.Fprintf(&script, "STDOUT_LOG=%s\n", guestShellQuote(session.StdoutLogPath)) fmt.Fprintf(&script, "STDERR_LOG=%s\n", guestShellQuote(session.StderrLogPath)) fmt.Fprintf(&script, "PID_FILE=%s\n", guestShellQuote(guestSessionPIDPath(session.ID))) fmt.Fprintf(&script, "MONITOR_PID_FILE=%s\n", guestShellQuote(guestSessionMonitorPIDPath(session.ID))) fmt.Fprintf(&script, "EXIT_FILE=%s\n", guestShellQuote(guestSessionExitCodePath(session.ID))) fmt.Fprintf(&script, "STATUS_FILE=%s\n", guestShellQuote(guestSessionStatusPath(session.ID))) fmt.Fprintf(&script, "ERROR_FILE=%s\n", guestShellQuote(guestSessionErrorPath(session.ID))) fmt.Fprintf(&script, "STDIN_PIPE=%s\n", guestShellQuote(guestSessionStdinPipePath(session.ID))) fmt.Fprintf(&script, "STDIN_KEEPALIVE_PID_FILE=%s\n", guestShellQuote(guestSessionStdinKeepalivePIDPath(session.ID))) fmt.Fprintf(&script, "SESSION_CWD=%s\n", guestShellQuote(defaultGuestSessionCWD(session.CWD))) script.WriteString("mkdir -p \"$STATE_DIR\"\n") script.WriteString(": >\"$STDOUT_LOG\"\n") script.WriteString(": >\"$STDERR_LOG\"\n") script.WriteString("rm -f \"$EXIT_FILE\" \"$ERROR_FILE\" \"$STDIN_KEEPALIVE_PID_FILE\"\n") if session.StdinMode == model.GuestSessionStdinPipe { script.WriteString("rm -f \"$STDIN_PIPE\"\n") script.WriteString("mkfifo -m 600 \"$STDIN_PIPE\"\n") } script.WriteString("printf '%s\\n' \"${BASHPID:-$$}\" >\"$MONITOR_PID_FILE\"\n") script.WriteString("printf 'starting\\n' >\"$STATUS_FILE\"\n") script.WriteString("cd \"$SESSION_CWD\"\n") script.WriteString("exec > >(tee -a \"$STDOUT_LOG\") 2> >(tee -a \"$STDERR_LOG\" >&2)\n") for _, line := range guestSessionEnvLines(session.Env) { script.WriteString(line) script.WriteByte('\n') } script.WriteString("COMMAND=(") for _, value := range append([]string{session.Command}, session.Args...) { script.WriteByte(' ') script.WriteString(guestShellQuote(value)) } script.WriteString(" )\n") if session.StdinMode == model.GuestSessionStdinPipe { script.WriteString("( while :; do sleep 3600; done ) >\"$STDIN_PIPE\" &\n") script.WriteString("keepalive=$!\n") script.WriteString("printf '%s\\n' \"$keepalive\" >\"$STDIN_KEEPALIVE_PID_FILE\"\n") script.WriteString("\"${COMMAND[@]}\" <\"$STDIN_PIPE\" &\n") } else { script.WriteString("\"${COMMAND[@]}\" &\n") } script.WriteString("child=$!\n") script.WriteString("printf '%s\\n' \"$child\" >\"$PID_FILE\"\n") script.WriteString("printf 'running\\n' >\"$STATUS_FILE\"\n") script.WriteString("wait \"$child\"\n") script.WriteString("rc=$?\n") if session.StdinMode == model.GuestSessionStdinPipe { script.WriteString("if [ -f \"$STDIN_KEEPALIVE_PID_FILE\" ]; then kill \"$(cat \"$STDIN_KEEPALIVE_PID_FILE\")\" 2>/dev/null || true; fi\n") } script.WriteString("printf '%s\\n' \"$rc\" >\"$EXIT_FILE\"\n") script.WriteString("if [ \"$rc\" -eq 0 ]; then printf 'exited\\n' >\"$STATUS_FILE\"; else printf 'failed\\n' >\"$STATUS_FILE\"; fi\n") script.WriteString("exit \"$rc\"\n") return script.String() } func guestSessionInspectScript(sessionID string) string { var script strings.Builder script.WriteString("set -euo pipefail\n") fmt.Fprintf(&script, "DIR=%s\n", guestShellQuote(guestSessionStateDir(sessionID))) script.WriteString("status=''\n") script.WriteString("pid=''\n") script.WriteString("exit_code=''\n") script.WriteString("last_error=''\n") script.WriteString("alive=false\n") script.WriteString("[ -f \"$DIR/status\" ] && status=\"$(cat \"$DIR/status\")\"\n") script.WriteString("[ -f \"$DIR/pid\" ] && pid=\"$(cat \"$DIR/pid\")\"\n") script.WriteString("[ -f \"$DIR/exit_code\" ] && exit_code=\"$(cat \"$DIR/exit_code\")\"\n") script.WriteString("[ -f \"$DIR/error\" ] && last_error=\"$(cat \"$DIR/error\")\"\n") script.WriteString("if [ -n \"$pid\" ] && kill -0 \"$pid\" 2>/dev/null; then alive=true; fi\n") script.WriteString("printf 'status=%s\\n' \"$status\"\n") script.WriteString("printf 'pid=%s\\n' \"$pid\"\n") script.WriteString("printf 'exit=%s\\n' \"$exit_code\"\n") script.WriteString("printf 'alive=%s\\n' \"$alive\"\n") script.WriteString("printf 'error=%s\\n' \"$last_error\"\n") return script.String() } func guestSessionSignalScript(sessionID, signal string) string { var script strings.Builder script.WriteString("set -euo pipefail\n") fmt.Fprintf(&script, "DIR=%s\n", guestShellQuote(guestSessionStateDir(sessionID))) fmt.Fprintf(&script, "SIGNAL=%s\n", guestShellQuote(signal)) script.WriteString("pid=''\n") script.WriteString("monitor=''\n") script.WriteString("keepalive=''\n") script.WriteString("[ -f \"$DIR/pid\" ] && pid=\"$(cat \"$DIR/pid\")\"\n") script.WriteString("[ -f \"$DIR/monitor_pid\" ] && monitor=\"$(cat \"$DIR/monitor_pid\")\"\n") script.WriteString("[ -f \"$DIR/stdin_keepalive.pid\" ] && keepalive=\"$(cat \"$DIR/stdin_keepalive.pid\")\"\n") script.WriteString("printf 'stopping\\n' >\"$DIR/status\"\n") script.WriteString("if [ -n \"$pid\" ]; then kill -${SIGNAL} \"$pid\" 2>/dev/null || true; fi\n") script.WriteString("if [ -n \"$monitor\" ]; then kill -${SIGNAL} \"$monitor\" 2>/dev/null || true; fi\n") script.WriteString("if [ -n \"$keepalive\" ]; then kill -${SIGNAL} \"$keepalive\" 2>/dev/null || true; fi\n") return script.String() } func guestSessionStateDir(id string) string { return filepath.ToSlash(filepath.Join(guestSessionStateRoot, id)) } func guestSessionRelativeStateDir(id string) string { return strings.TrimPrefix(guestSessionStateDir(id), "/root/") } func guestSessionScriptPath(id string) string { return filepath.ToSlash(filepath.Join(guestSessionStateDir(id), "run.sh")) } func guestSessionPIDPath(id string) string { return filepath.ToSlash(filepath.Join(guestSessionStateDir(id), "pid")) } func guestSessionMonitorPIDPath(id string) string { return filepath.ToSlash(filepath.Join(guestSessionStateDir(id), "monitor_pid")) } func guestSessionExitCodePath(id string) string { return filepath.ToSlash(filepath.Join(guestSessionStateDir(id), "exit_code")) } func guestSessionStdinPipePath(id string) string { return filepath.ToSlash(filepath.Join(guestSessionStateDir(id), "stdin.pipe")) } func guestSessionStdinKeepalivePIDPath(id string) string { return filepath.ToSlash(filepath.Join(guestSessionStateDir(id), "stdin_keepalive.pid")) } func guestSessionStatusPath(id string) string { return filepath.ToSlash(filepath.Join(guestSessionStateDir(id), "status")) } func guestSessionErrorPath(id string) string { return filepath.ToSlash(filepath.Join(guestSessionStateDir(id), "error")) } func guestSessionStdoutLogPath(id string) string { return filepath.ToSlash(filepath.Join(guestSessionStateDir(id), "stdout.log")) } func guestSessionStderrLogPath(id string) string { return filepath.ToSlash(filepath.Join(guestSessionStateDir(id), "stderr.log")) } func defaultGuestSessionName(id, command, explicit string) string { if trimmed := strings.TrimSpace(explicit); trimmed != "" { return trimmed } base := filepath.Base(strings.TrimSpace(command)) if base == "." || base == string(filepath.Separator) || base == "" { base = "session" } return base + "-" + system.ShortID(id) } func defaultGuestSessionCWD(value string) string { if trimmed := strings.TrimSpace(value); trimmed != "" { return trimmed } return "/root" } func failGuestSessionLaunch(session model.GuestSession, stage, message, rawLog string) model.GuestSession { now := model.Now() session.Status = model.GuestSessionStatusFailed session.LastError = strings.TrimSpace(message) session.Attachable = false session.Reattachable = false session.LaunchStage = strings.TrimSpace(stage) session.LaunchMessage = strings.TrimSpace(message) session.LaunchRawLog = strings.TrimSpace(rawLog) session.UpdatedAt = now session.EndedAt = now return session } func normalizeGuestSessionRequiredCommands(command string, extras []string) []string { ordered := make([]string, 0, len(extras)+1) seen := map[string]struct{}{} appendValue := func(value string) { trimmed := strings.TrimSpace(value) if trimmed == "" { return } if _, ok := seen[trimmed]; ok { return } seen[trimmed] = struct{}{} ordered = append(ordered, trimmed) } appendValue(command) for _, extra := range extras { appendValue(extra) } return ordered } func guestSessionCWDPreflightScript(cwd string) string { var script strings.Builder script.WriteString("set -euo pipefail\n") fmt.Fprintf(&script, "DIR=%s\n", guestShellQuote(defaultGuestSessionCWD(cwd))) script.WriteString("if [ ! -d \"$DIR\" ]; then echo \"missing cwd: $DIR\"; exit 1; fi\n") return script.String() } func guestSessionCommandPreflightScript(commands []string) string { var script strings.Builder script.WriteString("set -euo pipefail\n") script.WriteString("check_command() {\n") script.WriteString(" cmd=\"$1\"\n") script.WriteString(" case \"$cmd\" in\n") script.WriteString(" */*) [ -x \"$cmd\" ] || { echo \"missing command: $cmd\"; exit 1; } ;;\n") script.WriteString(" *) command -v \"$cmd\" >/dev/null 2>&1 || { echo \"missing command: $cmd\"; exit 1; } ;;\n") script.WriteString(" esac\n") script.WriteString("}\n") for _, command := range commands { fmt.Fprintf(&script, "check_command %s\n", guestShellQuote(command)) } return script.String() } func guestSessionAttachInputCommand(sessionID string) string { path := guestSessionStdinPipePath(sessionID) return "bash -lc " + guestShellQuote(fmt.Sprintf("set -euo pipefail\n[ -p %s ] || mkfifo -m 600 %s\nexec cat > %s\n", guestShellQuote(path), guestShellQuote(path), guestShellQuote(path))) } func guestSessionAttachTailCommand(path string) string { return "bash -lc " + guestShellQuote(fmt.Sprintf("set -euo pipefail\ntouch %s\nexec tail -n 0 -F %s 2>/dev/null\n", guestShellQuote(path), guestShellQuote(path))) } func guestSessionEnvLines(values map[string]string) []string { if len(values) == 0 { return nil } keys := make([]string, 0, len(values)) for key := range values { keys = append(keys, key) } sort.Strings(keys) lines := make([]string, 0, len(keys)) for _, key := range keys { lines = append(lines, "export "+key+"="+guestShellQuote(values[key])) } return lines } func guestShellQuote(value string) string { return "'" + strings.ReplaceAll(value, "'", `'"'"'`) + "'" } func parseGuestSessionState(raw string) (guestSessionStateSnapshot, error) { var snapshot guestSessionStateSnapshot scanner := bufio.NewScanner(strings.NewReader(raw)) for scanner.Scan() { line := scanner.Text() key, value, ok := strings.Cut(line, "=") if !ok { continue } switch strings.TrimSpace(key) { case "status": snapshot.Status = strings.TrimSpace(value) case "pid": if pid, err := strconv.Atoi(strings.TrimSpace(value)); err == nil { snapshot.GuestPID = pid } case "exit": if exitCode, err := strconv.Atoi(strings.TrimSpace(value)); err == nil { snapshot.ExitCode = &exitCode } case "alive": snapshot.Alive = strings.TrimSpace(value) == "true" case "error": snapshot.LastError = strings.TrimSpace(value) } } return snapshot, scanner.Err() } func guestSessionExitCode(err error) (int, bool) { if err == nil { return 0, true } var exitErr *ssh.ExitError if errors.As(err, &exitErr) { return exitErr.ExitStatus(), true } return 0, false } func cloneStringMap(values map[string]string) map[string]string { if len(values) == 0 { return nil } cloned := make(map[string]string, len(values)) for key, value := range values { cloned[key] = value } return cloned } func tailFileContent(path string, lines int) (string, error) { data, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { return "", nil } return "", err } if lines <= 0 { return string(data), nil } parts := strings.Split(string(data), "\n") if len(parts) <= lines { return string(data), nil } return strings.Join(parts[len(parts)-lines-1:], "\n"), nil } func processAlive(pid int) bool { if pid <= 0 { return false } return syscallKill(pid, syscall.Signal(0)) == nil } var syscallKill = func(pid int, signal os.Signal) error { proc, err := os.FindProcess(pid) if err != nil { return err } return proc.Signal(signal) } func formatGuestSessionStepError(action string, err error, log string) error { log = strings.TrimSpace(log) if log == "" { return fmt.Errorf("%s: %w", action, err) } return fmt.Errorf("%s: %w: %s", action, err, log) } func guestSessionStateChanged(before, after model.GuestSession) bool { if before.Status != after.Status || before.GuestPID != after.GuestPID || before.LastError != after.LastError || before.Attachable != after.Attachable || before.Reattachable != after.Reattachable || before.AttachBackend != after.AttachBackend || before.AttachMode != after.AttachMode || before.LaunchStage != after.LaunchStage || before.LaunchMessage != after.LaunchMessage || before.LaunchRawLog != after.LaunchRawLog { return true } if before.StartedAt != after.StartedAt || before.EndedAt != after.EndedAt { return true } switch { case before.ExitCode == nil && after.ExitCode == nil: return false case before.ExitCode == nil || after.ExitCode == nil: return true default: return *before.ExitCode != *after.ExitCode } }