#!/usr/bin/env bash set -euo pipefail log() { printf '[spawn] %s\n' "$*" } usage() { cat <<'EOF' Usage: ./run.sh [options] Options: --name VM name (lowercase letters, digits, -) --vcpu vCPU count (default: 2) --ram RAM in MiB (default: 1024) --overlay-size Writable overlay size (e.g. 8G, 16384M) --rootfs Root filesystem image (default: ./rootfs-docker.ext4) --kernel Kernel image (default: ./vmlinux) --initrd Initrd image (optional) -h, --help Show this help EOF } log "starting" DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$DIR/dns.sh" source "$DIR/packages.sh" STATE="$DIR/state" VM_ROOT="$STATE/vms" mkdir -p "$VM_ROOT" FC_BIN="$DIR/firecracker" DEFAULT_KERNEL="$DIR/wtf/root/boot/vmlinux-6.8.0-94-generic" DEFAULT_ROOTFS="$DIR/rootfs-docker.ext4" DEFAULT_INITRD="$DIR/wtf/root/boot/initrd.img-6.8.0-94-generic" SSH_KEY="$DIR/id_ed25519" NAMEGEN="$DIR/namegen" BR_DEV="br-fc" BR_IP="172.16.0.1" CIDR="24" DEFAULT_VCPU=2 DEFAULT_RAM=1024 DEFAULT_OVERLAY_SIZE="8G" MIN_VCPU=1 MAX_VCPU=16 MIN_RAM=256 MAX_RAM=32768 MAX_DISK_BYTES=$((128 * 1024 * 1024 * 1024)) DNS_SERVER="1.1.1.1" VCPU_COUNT="$DEFAULT_VCPU" RAM_MIB="$DEFAULT_RAM" OVERLAY_SIZE="$DEFAULT_OVERLAY_SIZE" KERNEL="$DEFAULT_KERNEL" ROOTFS="$DEFAULT_ROOTFS" INITRD="$DEFAULT_INITRD" VM_NAME="" shopt -s nullglob name_taken() { local candidate="$1" local vm_json existing_name for vm_json in "$VM_ROOT"/*/vm.json; do existing_name="$(jq -r '.meta.name // empty' "$vm_json" 2>/dev/null)" if [[ "$existing_name" == "$candidate" ]]; then return 0 fi done return 1 } parse_disk_size() { local raw="$1" if [[ "$raw" =~ ^([0-9]+)([KMG])?$ ]]; then local num="${BASH_REMATCH[1]}" local unit="${BASH_REMATCH[2]}" case "$unit" in K) echo $((num * 1024)) ;; M|"") echo $((num * 1024 * 1024)) ;; G) echo $((num * 1024 * 1024 * 1024)) ;; esac return 0 fi return 1 } while [[ $# -gt 0 ]]; do case "$1" in --name) VM_NAME="${2:-}" shift 2 ;; --vcpu) VCPU_COUNT="${2:-}" shift 2 ;; --ram) RAM_MIB="${2:-}" shift 2 ;; --overlay-size) OVERLAY_SIZE="${2:-}" shift 2 ;; --rootfs) ROOTFS="${2:-}" shift 2 ;; --kernel) KERNEL="${2:-}" shift 2 ;; --initrd) INITRD="${2:-}" shift 2 ;; -h|--help) usage exit 0 ;; *) log "unknown option: $1" usage exit 1 ;; esac done if ! [[ "$VCPU_COUNT" =~ ^[0-9]+$ ]]; then log "invalid --vcpu value: $VCPU_COUNT" exit 1 fi if (( VCPU_COUNT < MIN_VCPU || VCPU_COUNT > MAX_VCPU )); then log "vcpu must be between $MIN_VCPU and $MAX_VCPU" exit 1 fi if ! [[ "$RAM_MIB" =~ ^[0-9]+$ ]]; then log "invalid --ram value: $RAM_MIB" exit 1 fi if (( RAM_MIB < MIN_RAM || RAM_MIB > MAX_RAM )); then log "ram must be between $MIN_RAM and $MAX_RAM MiB" exit 1 fi if ! OVERLAY_BYTES="$(parse_disk_size "$OVERLAY_SIZE")"; then log "invalid --overlay-size value: $OVERLAY_SIZE" exit 1 fi if (( OVERLAY_BYTES > MAX_DISK_BYTES )); then log "overlay-size exceeds max of $((MAX_DISK_BYTES / 1024 / 1024 / 1024))G" exit 1 fi if [[ -z "$VM_NAME" ]]; then if [[ ! -x "$NAMEGEN" ]]; then log "name generator not found: $NAMEGEN" exit 1 fi for _ in $(seq 1 20); do VM_NAME="$("$NAMEGEN")" if ! name_taken "$VM_NAME"; then break fi done fi if [[ -z "$VM_NAME" ]]; then log "failed to generate a unique name" exit 1 fi if ! [[ "$VM_NAME" =~ ^[a-z0-9][a-z0-9-]{0,63}$ ]]; then log "invalid --name value: $VM_NAME" exit 1 fi if [[ ! -f "$ROOTFS" ]]; then if [[ "$ROOTFS" == "$DEFAULT_ROOTFS" && -x "$DIR/make-rootfs.sh" ]]; then log "rootfs missing; building via make-rootfs.sh" "$DIR/make-rootfs.sh" fi fi if [[ ! -f "$ROOTFS" ]]; then log "rootfs not found: $ROOTFS" exit 1 fi if [[ "$ROOTFS" == "$DEFAULT_ROOTFS" ]]; then ROOTFS_WARNING="$(banger_rootfs_manifest_warning "$ROOTFS" || true)" if [[ -n "$ROOTFS_WARNING" ]]; then log "warning: $ROOTFS_WARNING" fi fi if [[ ! -f "$KERNEL" ]]; then log "kernel not found: $KERNEL" exit 1 fi if [[ -n "$INITRD" && ! -f "$INITRD" ]]; then log "initrd not found: $INITRD" exit 1 fi if name_taken "$VM_NAME"; then log "name already in use: $VM_NAME" exit 1 fi VM_ID="$(head -c 32 /dev/urandom | xxd -p -c 256)" VM_TAG="${VM_ID:0:8}" VM_DIR="$VM_ROOT/$VM_ID" mkdir -p "$VM_DIR" RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" SOCK_DIR="$RUNTIME_DIR/banger" sudo mkdir -p "$SOCK_DIR" sudo chown "$(id -u):$(id -g)" "$SOCK_DIR" API_SOCK="$SOCK_DIR/fc-$VM_TAG.sock" LOG_FILE="$VM_DIR/firecracker.log" TAP_DEV="tap-fc-$VM_TAG" # Allocate guest IP NEXT_IP_FILE="$STATE/next_ip" NEXT_IP="$(cat "$NEXT_IP_FILE" 2>/dev/null || echo 2)" GUEST_IP="172.16.0.$NEXT_IP" echo "$((NEXT_IP + 1))" > "$NEXT_IP_FILE" log "vm id: $VM_ID" log "vm name: $VM_NAME" log "allocated guest ip: $GUEST_IP" sudo -v VM_STARTED=0 CLEANUP_ON_EXIT=0 KEEP_VM_DIR_ON_FAIL=1 COW_FILE="$VM_DIR/cow.ext4" BASE_LOOP="" COW_LOOP="" DM_NAME="fc-rootfs-$VM_TAG" DM_DEV="" DNS_NAME="" cleanup() { local exit_code=$? if [[ "$VM_STARTED" -eq 1 && "$CLEANUP_ON_EXIT" -eq 0 ]]; then return fi log "cleaning up" if [[ -n "${FC_PID:-}" ]]; then sudo kill "$FC_PID" 2>/dev/null || true fi if [[ -n "${TAP_DEV:-}" ]]; then sudo ip link del "$TAP_DEV" 2>/dev/null || true fi rm -f "${API_SOCK:-}" banger_dns_remove_record_name "${DNS_NAME:-}" if [[ -n "${DM_NAME:-}" ]]; then sudo dmsetup remove "$DM_NAME" 2>/dev/null || true fi if [[ -n "${COW_LOOP:-}" ]]; then sudo losetup -d "$COW_LOOP" 2>/dev/null || true fi if [[ -n "${BASE_LOOP:-}" ]]; then sudo losetup -d "$BASE_LOOP" 2>/dev/null || true fi if [[ -n "${VM_DIR:-}" && -d "$VM_DIR" ]]; then if [[ "$KEEP_VM_DIR_ON_FAIL" -eq 0 || "$VM_STARTED" -eq 1 || "$CLEANUP_ON_EXIT" -eq 1 ]]; then rm -rf "$VM_DIR" fi fi return "$exit_code" } on_signal() { CLEANUP_ON_EXIT=1 exit 1 } trap cleanup EXIT trap on_signal INT TERM FC_USE_SUDO="${FC_USE_SUDO:-1}" FC_RUN=("$FC_BIN") CURL_CMD=(curl) if [[ "$FC_USE_SUDO" == "1" ]]; then log "running firecracker with sudo (FC_USE_SUDO=1)" FC_RUN=(sudo -E "$FC_BIN") CURL_CMD=(sudo -E curl) fi if command -v setcap >/dev/null 2>&1; then if ! getcap "$FC_BIN" 2>/dev/null | rg -q "cap_net_admin"; then log "granting cap_net_admin to firecracker binary" sudo setcap cap_net_admin+ep "$FC_BIN" fi else log "setcap not available; firecracker may need root to open TAP" fi if ! command -v dmsetup >/dev/null 2>&1 || ! command -v losetup >/dev/null 2>&1 || ! command -v blockdev >/dev/null 2>&1; then log "dmsetup, losetup, and blockdev are required for rootfs snapshots" exit 1 fi if ! command -v e2cp >/dev/null 2>&1 || ! command -v e2rm >/dev/null 2>&1; then log "e2cp and e2rm are required to set hostname and resolv.conf" exit 1 fi if ! command -v jq >/dev/null 2>&1; then log "jq is required to persist VM metadata" exit 1 fi BASE_LOOP="$(sudo losetup -f --show --read-only "$ROOTFS")" truncate -s "$OVERLAY_BYTES" "$COW_FILE" COW_LOOP="$(sudo losetup -f --show "$COW_FILE")" SECTORS="$(sudo blockdev --getsz "$BASE_LOOP")" sudo dmsetup create "$DM_NAME" --table "0 $SECTORS snapshot $BASE_LOOP $COW_LOOP P 8" DM_DEV="/dev/mapper/$DM_NAME" RESOLV_TMP="$VM_DIR/resolv.conf" HOSTNAME_TMP="$VM_DIR/hostname" HOSTS_TMP="$VM_DIR/hosts" printf 'nameserver %s\n' "$DNS_SERVER" >"$RESOLV_TMP" printf '%s\n' "$VM_NAME" >"$HOSTNAME_TMP" printf '127.0.0.1 localhost\n127.0.1.1 %s\n' "$VM_NAME" >"$HOSTS_TMP" sudo e2rm "$DM_DEV:/etc/resolv.conf" >/dev/null 2>&1 || true sudo e2rm "$DM_DEV:/etc/hostname" >/dev/null 2>&1 || true sudo e2rm "$DM_DEV:/etc/hosts" >/dev/null 2>&1 || true sudo e2cp "$RESOLV_TMP" "$DM_DEV:/etc/resolv.conf" >/dev/null 2>&1 || { log "failed to write /etc/resolv.conf into rootfs snapshot" exit 1 } sudo e2cp "$HOSTNAME_TMP" "$DM_DEV:/etc/hostname" >/dev/null 2>&1 || { log "failed to write /etc/hostname into rootfs snapshot" exit 1 } sudo e2cp "$HOSTS_TMP" "$DM_DEV:/etc/hosts" >/dev/null 2>&1 || { log "failed to write /etc/hosts into rootfs snapshot" exit 1 } # Host bridge if ! ip link show "$BR_DEV" >/dev/null 2>&1; then log "creating host bridge $BR_DEV ($BR_IP/$CIDR)" sudo ip link add name "$BR_DEV" type bridge sudo ip addr add "${BR_IP}/${CIDR}" dev "$BR_DEV" sudo ip link set "$BR_DEV" up else log "host bridge $BR_DEV already exists" # Ensure existing bridge is up in case it was left down. sudo ip link set "$BR_DEV" up fi # Per-VM TAP log "creating tap device $TAP_DEV" TAP_USER="${SUDO_UID:-$(id -u)}" TAP_GROUP="${SUDO_GID:-$(id -g)}" sudo ip tuntap add dev "$TAP_DEV" mode tap user "$TAP_USER" group "$TAP_GROUP" sudo ip link set "$TAP_DEV" master "$BR_DEV" sudo ip link set "$TAP_DEV" up sudo ip link set "$BR_DEV" up # Start Firecracker log "starting firecracker process" rm -f "$API_SOCK" nohup "${FC_RUN[@]}" --api-sock "$API_SOCK" >"$LOG_FILE" 2>&1 & FC_PID="$!" log "firecracker pid: $FC_PID" # Wait for API socket log "waiting for firecracker api socket" for _ in $(seq 1 200); do [[ -S "$API_SOCK" ]] && break sleep 0.02 done if [[ ! -S "$API_SOCK" ]]; then log "firecracker api socket not ready" if [[ -f "$LOG_FILE" ]]; then log "firecracker log (tail)" tail -n 20 "$LOG_FILE" || true fi exit 1 fi log "api socket ready" if [[ "$FC_USE_SUDO" == "1" ]]; then SUDO_CHILD_PID="$(pgrep -n -f "$API_SOCK" || true)" if [[ -n "$SUDO_CHILD_PID" ]]; then FC_PID="$SUDO_CHILD_PID" log "firecracker child pid: $FC_PID" fi fi echo "$FC_PID" > "$VM_DIR/pid" # Machine config log "configuring machine" "${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/machine-config \ -H "Content-Type: application/json" \ -d "{ \"vcpu_count\": $VCPU_COUNT, \"mem_size_mib\": $RAM_MIB, \"smt\": false }" >/dev/null # Boot source log "configuring boot source" KCMD="console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rw ip=${GUEST_IP}::${BR_IP}:255.255.255.0::eth0:off:${DNS_SERVER} hostname=${VM_NAME} systemd.mask=home.mount systemd.mask=var.mount" INITRD_JSON="" if [[ -n "$INITRD" ]]; then INITRD_JSON=", \"initrd_path\": \"$INITRD\"" fi "${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/boot-source \ -H "Content-Type: application/json" \ -d "{ \"kernel_image_path\": \"$KERNEL\", \"boot_args\": \"$KCMD\"${INITRD_JSON} }" >/dev/null # Root filesystem log "attaching root filesystem" "${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/drives/rootfs \ -H "Content-Type: application/json" \ -d "{ \"drive_id\": \"rootfs\", \"path_on_host\": \"$DM_DEV\", \"is_root_device\": true, \"is_read_only\": false }" >/dev/null # Network interface log "configuring network interface" "${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/network-interfaces/eth0 \ -H "Content-Type: application/json" \ -d "{ \"iface_id\": \"eth0\", \"host_dev_name\": \"$TAP_DEV\" }" >/dev/null # Start VM log "starting virtual machine" "${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/actions \ -H "Content-Type: application/json" \ -d '{ "action_type": "InstanceStart" }' >/dev/null VM_CONFIG_JSON="$("${CURL_CMD[@]}" --unix-socket "$API_SOCK" -sS http://localhost/vm/config)" CREATED_AT="$(date -Iseconds)" DNS_NAME="$(banger_dns_name "$VM_NAME")" banger_dns_write_record "$VM_NAME" "$GUEST_IP" jq -n \ --arg id "$VM_ID" \ --arg name "$VM_NAME" \ --arg pid "$FC_PID" \ --arg created_at "$CREATED_AT" \ --arg guest_ip "$GUEST_IP" \ --arg tap "$TAP_DEV" \ --arg api_sock "$API_SOCK" \ --arg log "$LOG_FILE" \ --arg rootfs "$ROOTFS" \ --arg kernel "$KERNEL" \ --arg base_loop "$BASE_LOOP" \ --arg cow_file "$COW_FILE" \ --arg cow_loop "$COW_LOOP" \ --arg dm_name "$DM_NAME" \ --arg dm_dev "$DM_DEV" \ --arg dns_name "$DNS_NAME" \ --argjson config "$VM_CONFIG_JSON" \ '{meta:{id:$id,name:$name,pid:$pid,created_at:$created_at,guest_ip:$guest_ip,tap:$tap,api_sock:$api_sock,log:$log,rootfs:$rootfs,kernel:$kernel,base_loop:$base_loop,cow_file:$cow_file,cow_loop:$cow_loop,dm_name:$dm_name,dm_dev:$dm_dev,dns_name:$dns_name},config:$config}' \ > "$VM_DIR/vm.json" VM_STARTED=1 log "vm started successfully" log "guest ip: $GUEST_IP" log "ssh: ssh -i \"$SSH_KEY\" root@$GUEST_IP" log "logs: $LOG_FILE"