banger/run.sh

525 lines
13 KiB
Bash
Executable file

#!/usr/bin/env bash
set -euo pipefail
log() {
printf '[spawn] %s\n' "$*"
}
usage() {
cat <<'EOF'
Usage: ./run.sh [options]
Options:
--name <name> VM name (lowercase letters, digits, -)
--vcpu <count> vCPU count (default: 2)
--ram <mib> RAM in MiB (default: 1024)
--rootfs <path> Root filesystem image (default: ./rootfs.ext4)
--kernel <path> Kernel image (default: ./vmlinux)
--initrd <path> Initrd image (optional)
--home-size <size> Home disk size (e.g. 4G, 10240M)
--var-size <size> Var disk size (e.g. 4G, 10240M)
-h, --help Show this help
EOF
}
log "starting"
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
STATE="$DIR/state"
VM_ROOT="$STATE/vms"
mkdir -p "$VM_ROOT"
FC_BIN="$DIR/firecracker"
DEFAULT_KERNEL="$DIR/wtf/root/boot/vmlinux-6.8.0-94-generic"
DEFAULT_ROOTFS="$DIR/rootfs-docker.ext4"
DEFAULT_INITRD="$DIR/wtf/root/boot/initrd.img-6.8.0-94-generic"
SSH_KEY="$DIR/id_ed25519"
NAMEGEN="$DIR/namegen"
BR_DEV="br-fc"
BR_IP="172.16.0.1"
CIDR="24"
DEFAULT_VCPU=2
DEFAULT_RAM=1024
DEFAULT_HOME_SIZE="2G"
DEFAULT_VAR_SIZE="2G"
MIN_VCPU=1
MAX_VCPU=16
MIN_RAM=256
MAX_RAM=32768
MAX_DISK_BYTES=$((128 * 1024 * 1024 * 1024))
DNS_SERVER="1.1.1.1"
COW_SIZE="2G"
VCPU_COUNT="$DEFAULT_VCPU"
RAM_MIB="$DEFAULT_RAM"
HOME_SIZE="$DEFAULT_HOME_SIZE"
VAR_SIZE="$DEFAULT_VAR_SIZE"
KERNEL="$DEFAULT_KERNEL"
ROOTFS="$DEFAULT_ROOTFS"
INITRD="$DEFAULT_INITRD"
VM_NAME=""
shopt -s nullglob
name_taken() {
local candidate="$1"
local vm_json existing_name
for vm_json in "$VM_ROOT"/*/vm.json; do
existing_name="$(jq -r '.meta.name // empty' "$vm_json" 2>/dev/null)"
if [[ "$existing_name" == "$candidate" ]]; then
return 0
fi
done
return 1
}
parse_disk_size() {
local raw="$1"
if [[ "$raw" =~ ^([0-9]+)([KMG])?$ ]]; then
local num="${BASH_REMATCH[1]}"
local unit="${BASH_REMATCH[2]}"
case "$unit" in
K) echo $((num * 1024)) ;;
M|"") echo $((num * 1024 * 1024)) ;;
G) echo $((num * 1024 * 1024 * 1024)) ;;
esac
return 0
fi
return 1
}
while [[ $# -gt 0 ]]; do
case "$1" in
--name)
VM_NAME="${2:-}"
shift 2
;;
--vcpu)
VCPU_COUNT="${2:-}"
shift 2
;;
--ram)
RAM_MIB="${2:-}"
shift 2
;;
--rootfs)
ROOTFS="${2:-}"
shift 2
;;
--kernel)
KERNEL="${2:-}"
shift 2
;;
--initrd)
INITRD="${2:-}"
shift 2
;;
--home-size)
HOME_SIZE="${2:-}"
shift 2
;;
--var-size)
VAR_SIZE="${2:-}"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
log "unknown option: $1"
usage
exit 1
;;
esac
done
if ! [[ "$VCPU_COUNT" =~ ^[0-9]+$ ]]; then
log "invalid --vcpu value: $VCPU_COUNT"
exit 1
fi
if (( VCPU_COUNT < MIN_VCPU || VCPU_COUNT > MAX_VCPU )); then
log "vcpu must be between $MIN_VCPU and $MAX_VCPU"
exit 1
fi
if ! [[ "$RAM_MIB" =~ ^[0-9]+$ ]]; then
log "invalid --ram value: $RAM_MIB"
exit 1
fi
if (( RAM_MIB < MIN_RAM || RAM_MIB > MAX_RAM )); then
log "ram must be between $MIN_RAM and $MAX_RAM MiB"
exit 1
fi
HOME_BYTES=""
if ! HOME_BYTES="$(parse_disk_size "$HOME_SIZE")"; then
log "invalid --home-size value: $HOME_SIZE"
exit 1
fi
if (( HOME_BYTES > MAX_DISK_BYTES )); then
log "home-size exceeds max of $((MAX_DISK_BYTES / 1024 / 1024 / 1024))G"
exit 1
fi
VAR_BYTES=""
if ! VAR_BYTES="$(parse_disk_size "$VAR_SIZE")"; then
log "invalid --var-size value: $VAR_SIZE"
exit 1
fi
if (( VAR_BYTES > MAX_DISK_BYTES )); then
log "var-size exceeds max of $((MAX_DISK_BYTES / 1024 / 1024 / 1024))G"
exit 1
fi
if [[ -z "$VM_NAME" ]]; then
if [[ ! -x "$NAMEGEN" ]]; then
log "name generator not found: $NAMEGEN"
exit 1
fi
for _ in $(seq 1 20); do
VM_NAME="$("$NAMEGEN")"
if ! name_taken "$VM_NAME"; then
break
fi
done
fi
if [[ -z "$VM_NAME" ]]; then
log "failed to generate a unique name"
exit 1
fi
if ! [[ "$VM_NAME" =~ ^[a-z0-9][a-z0-9-]{0,63}$ ]]; then
log "invalid --name value: $VM_NAME"
exit 1
fi
if [[ ! -f "$ROOTFS" ]]; then
if [[ "$ROOTFS" == "$DEFAULT_ROOTFS" && -x "$DIR/make-rootfs.sh" ]]; then
log "rootfs missing; building via make-rootfs.sh"
"$DIR/make-rootfs.sh"
fi
fi
if [[ ! -f "$ROOTFS" ]]; then
log "rootfs not found: $ROOTFS"
exit 1
fi
if [[ ! -f "$KERNEL" ]]; then
log "kernel not found: $KERNEL"
exit 1
fi
if [[ -n "$INITRD" && ! -f "$INITRD" ]]; then
log "initrd not found: $INITRD"
exit 1
fi
if name_taken "$VM_NAME"; then
log "name already in use: $VM_NAME"
exit 1
fi
VM_ID="$(head -c 32 /dev/urandom | xxd -p -c 256)"
VM_TAG="${VM_ID:0:8}"
VM_DIR="$VM_ROOT/$VM_ID"
mkdir -p "$VM_DIR"
RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}"
SOCK_DIR="$RUNTIME_DIR/banger"
sudo mkdir -p "$SOCK_DIR"
sudo chown "$(id -u):$(id -g)" "$SOCK_DIR"
API_SOCK="$SOCK_DIR/fc-$VM_TAG.sock"
LOG_FILE="$VM_DIR/firecracker.log"
TAP_DEV="tap-fc-$VM_TAG"
# Allocate guest IP
NEXT_IP_FILE="$STATE/next_ip"
NEXT_IP="$(cat "$NEXT_IP_FILE" 2>/dev/null || echo 2)"
GUEST_IP="172.16.0.$NEXT_IP"
echo "$((NEXT_IP + 1))" > "$NEXT_IP_FILE"
log "vm id: $VM_ID"
log "vm name: $VM_NAME"
log "allocated guest ip: $GUEST_IP"
sudo -v
VM_STARTED=0
CLEANUP_ON_EXIT=0
KEEP_VM_DIR_ON_FAIL=1
HOME_PATH="$VM_DIR/home.ext4"
VAR_PATH="$VM_DIR/var.ext4"
COW_FILE="$VM_DIR/cow.ext4"
BASE_LOOP=""
COW_LOOP=""
DM_NAME="fc-rootfs-$VM_TAG"
DM_DEV=""
cleanup() {
local exit_code=$?
if [[ "$VM_STARTED" -eq 1 && "$CLEANUP_ON_EXIT" -eq 0 ]]; then
return
fi
log "cleaning up"
if [[ -n "${FC_PID:-}" ]]; then
sudo kill "$FC_PID" 2>/dev/null || true
fi
if [[ -n "${TAP_DEV:-}" ]]; then
sudo ip link del "$TAP_DEV" 2>/dev/null || true
fi
rm -f "${API_SOCK:-}"
if [[ -n "${DM_NAME:-}" ]]; then
sudo dmsetup remove "$DM_NAME" 2>/dev/null || true
fi
if [[ -n "${COW_LOOP:-}" ]]; then
sudo losetup -d "$COW_LOOP" 2>/dev/null || true
fi
if [[ -n "${BASE_LOOP:-}" ]]; then
sudo losetup -d "$BASE_LOOP" 2>/dev/null || true
fi
if [[ -n "${VM_DIR:-}" && -d "$VM_DIR" ]]; then
if [[ "$KEEP_VM_DIR_ON_FAIL" -eq 0 || "$VM_STARTED" -eq 1 || "$CLEANUP_ON_EXIT" -eq 1 ]]; then
rm -rf "$VM_DIR"
fi
fi
return "$exit_code"
}
on_signal() {
CLEANUP_ON_EXIT=1
exit 1
}
trap cleanup EXIT
trap on_signal INT TERM
FC_USE_SUDO="${FC_USE_SUDO:-1}"
FC_RUN=("$FC_BIN")
CURL_CMD=(curl)
if [[ "$FC_USE_SUDO" == "1" ]]; then
log "running firecracker with sudo (FC_USE_SUDO=1)"
FC_RUN=(sudo -E "$FC_BIN")
CURL_CMD=(sudo -E curl)
fi
if command -v setcap >/dev/null 2>&1; then
if ! getcap "$FC_BIN" 2>/dev/null | rg -q "cap_net_admin"; then
log "granting cap_net_admin to firecracker binary"
sudo setcap cap_net_admin+ep "$FC_BIN"
fi
else
log "setcap not available; firecracker may need root to open TAP"
fi
if ! command -v mkfs.ext4 >/dev/null 2>&1; then
log "mkfs.ext4 required to create home/var disks"
exit 1
fi
truncate -s "$HOME_BYTES" "$HOME_PATH"
mkfs.ext4 -F "$HOME_PATH" >/dev/null
truncate -s "$VAR_BYTES" "$VAR_PATH"
mkfs.ext4 -F "$VAR_PATH" >/dev/null
if ! command -v dmsetup >/dev/null 2>&1 || ! command -v losetup >/dev/null 2>&1 || ! command -v blockdev >/dev/null 2>&1; then
log "dmsetup, losetup, and blockdev are required for rootfs snapshots"
exit 1
fi
if ! command -v e2cp >/dev/null 2>&1 || ! command -v e2rm >/dev/null 2>&1; then
log "e2cp and e2rm are required to set hostname and resolv.conf"
exit 1
fi
if ! command -v jq >/dev/null 2>&1; then
log "jq is required to persist VM metadata"
exit 1
fi
COW_BYTES="$(parse_disk_size "$COW_SIZE")"
if [[ -z "$COW_BYTES" ]]; then
log "invalid COW size: $COW_SIZE"
exit 1
fi
BASE_LOOP="$(sudo losetup -f --show --read-only "$ROOTFS")"
truncate -s "$COW_BYTES" "$COW_FILE"
COW_LOOP="$(sudo losetup -f --show "$COW_FILE")"
SECTORS="$(sudo blockdev --getsz "$BASE_LOOP")"
sudo dmsetup create "$DM_NAME" --table "0 $SECTORS snapshot $BASE_LOOP $COW_LOOP P 8"
DM_DEV="/dev/mapper/$DM_NAME"
RESOLV_TMP="$VM_DIR/resolv.conf"
HOSTNAME_TMP="$VM_DIR/hostname"
HOSTS_TMP="$VM_DIR/hosts"
printf 'nameserver %s\n' "$DNS_SERVER" >"$RESOLV_TMP"
printf '%s\n' "$VM_NAME" >"$HOSTNAME_TMP"
printf '127.0.0.1 localhost\n127.0.1.1 %s\n' "$VM_NAME" >"$HOSTS_TMP"
sudo e2rm "$DM_DEV:/etc/resolv.conf" >/dev/null 2>&1 || true
sudo e2rm "$DM_DEV:/etc/hostname" >/dev/null 2>&1 || true
sudo e2rm "$DM_DEV:/etc/hosts" >/dev/null 2>&1 || true
sudo e2cp "$RESOLV_TMP" "$DM_DEV:/etc/resolv.conf" >/dev/null 2>&1 || {
log "failed to write /etc/resolv.conf into rootfs snapshot"
exit 1
}
sudo e2cp "$HOSTNAME_TMP" "$DM_DEV:/etc/hostname" >/dev/null 2>&1 || {
log "failed to write /etc/hostname into rootfs snapshot"
exit 1
}
sudo e2cp "$HOSTS_TMP" "$DM_DEV:/etc/hosts" >/dev/null 2>&1 || {
log "failed to write /etc/hosts into rootfs snapshot"
exit 1
}
# Host bridge
if ! ip link show "$BR_DEV" >/dev/null 2>&1; then
log "creating host bridge $BR_DEV ($BR_IP/$CIDR)"
sudo ip link add name "$BR_DEV" type bridge
sudo ip addr add "${BR_IP}/${CIDR}" dev "$BR_DEV"
sudo ip link set "$BR_DEV" up
else
log "host bridge $BR_DEV already exists"
# Ensure existing bridge is up in case it was left down.
sudo ip link set "$BR_DEV" up
fi
# Per-VM TAP
log "creating tap device $TAP_DEV"
TAP_USER="${SUDO_UID:-$(id -u)}"
TAP_GROUP="${SUDO_GID:-$(id -g)}"
sudo ip tuntap add dev "$TAP_DEV" mode tap user "$TAP_USER" group "$TAP_GROUP"
sudo ip link set "$TAP_DEV" master "$BR_DEV"
sudo ip link set "$TAP_DEV" up
sudo ip link set "$BR_DEV" up
# Start Firecracker
log "starting firecracker process"
rm -f "$API_SOCK"
nohup "${FC_RUN[@]}" --api-sock "$API_SOCK" >"$LOG_FILE" 2>&1 &
FC_PID="$!"
log "firecracker pid: $FC_PID"
# Wait for API socket
log "waiting for firecracker api socket"
for _ in $(seq 1 200); do
[[ -S "$API_SOCK" ]] && break
sleep 0.02
done
if [[ ! -S "$API_SOCK" ]]; then
log "firecracker api socket not ready"
if [[ -f "$LOG_FILE" ]]; then
log "firecracker log (tail)"
tail -n 20 "$LOG_FILE" || true
fi
exit 1
fi
log "api socket ready"
if [[ "$FC_USE_SUDO" == "1" ]]; then
SUDO_CHILD_PID="$(pgrep -n -f "$API_SOCK" || true)"
if [[ -n "$SUDO_CHILD_PID" ]]; then
FC_PID="$SUDO_CHILD_PID"
log "firecracker child pid: $FC_PID"
fi
fi
echo "$FC_PID" > "$VM_DIR/pid"
# Machine config
log "configuring machine"
"${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/machine-config \
-H "Content-Type: application/json" \
-d "{
\"vcpu_count\": $VCPU_COUNT,
\"mem_size_mib\": $RAM_MIB,
\"smt\": false
}" >/dev/null
# Boot source
log "configuring boot source"
KCMD="console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rw ip=${GUEST_IP}::${BR_IP}:255.255.255.0::eth0:off:${DNS_SERVER} hostname=${VM_NAME}"
INITRD_JSON=""
if [[ -n "$INITRD" ]]; then
INITRD_JSON=", \"initrd_path\": \"$INITRD\""
fi
"${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/boot-source \
-H "Content-Type: application/json" \
-d "{
\"kernel_image_path\": \"$KERNEL\",
\"boot_args\": \"$KCMD\"${INITRD_JSON}
}" >/dev/null
# Root filesystem
log "attaching root filesystem"
"${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/drives/rootfs \
-H "Content-Type: application/json" \
-d "{
\"drive_id\": \"rootfs\",
\"path_on_host\": \"$DM_DEV\",
\"is_root_device\": true,
\"is_read_only\": false
}" >/dev/null
# Home filesystem
log "attaching home filesystem"
"${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/drives/home \
-H "Content-Type: application/json" \
-d "{
\"drive_id\": \"home\",
\"path_on_host\": \"$HOME_PATH\",
\"is_root_device\": false,
\"is_read_only\": false
}" >/dev/null
# Var filesystem
log "attaching var filesystem"
"${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/drives/var \
-H "Content-Type: application/json" \
-d "{
\"drive_id\": \"var\",
\"path_on_host\": \"$VAR_PATH\",
\"is_root_device\": false,
\"is_read_only\": false
}" >/dev/null
# Network interface
log "configuring network interface"
"${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/network-interfaces/eth0 \
-H "Content-Type: application/json" \
-d "{
\"iface_id\": \"eth0\",
\"host_dev_name\": \"$TAP_DEV\"
}" >/dev/null
# Start VM
log "starting virtual machine"
"${CURL_CMD[@]}" --unix-socket "$API_SOCK" -X PUT http://localhost/actions \
-H "Content-Type: application/json" \
-d '{ "action_type": "InstanceStart" }' >/dev/null
VM_STARTED=1
VM_CONFIG_JSON="$("${CURL_CMD[@]}" --unix-socket "$API_SOCK" -sS http://localhost/vm/config)"
CREATED_AT="$(date -Iseconds)"
jq -n \
--arg id "$VM_ID" \
--arg name "$VM_NAME" \
--arg pid "$FC_PID" \
--arg created_at "$CREATED_AT" \
--arg guest_ip "$GUEST_IP" \
--arg tap "$TAP_DEV" \
--arg api_sock "$API_SOCK" \
--arg log "$LOG_FILE" \
--arg rootfs "$ROOTFS" \
--arg kernel "$KERNEL" \
--arg home_path "$HOME_PATH" \
--arg var_path "$VAR_PATH" \
--arg base_loop "$BASE_LOOP" \
--arg cow_file "$COW_FILE" \
--arg cow_loop "$COW_LOOP" \
--arg dm_name "$DM_NAME" \
--arg dm_dev "$DM_DEV" \
--argjson config "$VM_CONFIG_JSON" \
'{meta:{id:$id,name:$name,pid:$pid,created_at:$created_at,guest_ip:$guest_ip,tap:$tap,api_sock:$api_sock,log:$log,rootfs:$rootfs,kernel:$kernel,home_path:$home_path,var_path:$var_path,base_loop:$base_loop,cow_file:$cow_file,cow_loop:$cow_loop,dm_name:$dm_name,dm_dev:$dm_dev},config:$config}' \
> "$VM_DIR/vm.json"
log "vm started successfully"
log "guest ip: $GUEST_IP"
log "ssh: ssh -i \"$SSH_KEY\" root@$GUEST_IP"
log "logs: $LOG_FILE"