Add real runtime materialization pipeline and bundle artifacts

runtime_sources/NOTICE

@@ -0,0 +1,5 @@
+pyro-mcp runtime bundle
+
+This package includes bundled runtime components intended for local developer workflows.
+Replace shims with official Firecracker/jailer binaries and production profile artifacts
+for real VM isolation in release builds.

runtime_sources/README.md

@@ -0,0 +1,29 @@
+# runtime_sources
+
+Source-of-truth inputs for `make runtime-bundle`.
+
+Current state:
+- `bin/firecracker` and `bin/jailer` are shim placeholders.
+- profile kernels and rootfs images are placeholder files.
+- `guest/pyro_guest_agent.py` is the guest agent artifact that should ultimately be installed into each real rootfs.
+- real source materialization now writes into `build/runtime_sources/`, not back into the tracked placeholder files.
+
+Materialization workflow:
+1. `make runtime-fetch-binaries`
+2. `make runtime-build-kernel-real`
+3. `make runtime-build-rootfs-real`
+4. `make runtime-bundle`
+
+Build requirements for the real path:
+- `docker`
+- outbound network access to GitHub and Debian snapshot mirrors
+- enough disk for a kernel build plus 2G ext4 images per profile
+
+Next steps to make the bundle guest-capable:
+1. Replace shim binaries with pinned official Firecracker and Jailer release artifacts.
+2. Replace placeholder `vmlinux` and `rootfs.ext4` files with real, bootable artifacts for each profile.
+3. Ensure the guest agent is installed and enabled inside every rootfs so the host can use vsock exec.
+4. Once the source artifacts are real, update `runtime.lock.json` component versions and flip capability flags from `false` to `true`.
+
+Safety rule:
+- The build pipeline should never emit `vm_boot=true`, `guest_exec=true`, or `guest_network=true` while any source artifact is still a shim or placeholder.

runtime_sources/linux-x86_64/bin/firecracker

@@ -0,0 +1,12 @@
+#!/usr/bin/env sh
+set -eu
+if [ "${1:-}" = "--version" ]; then
+  echo "Firecracker v1.8.0 (bundled shim)"
+  exit 0
+fi
+if [ "${1:-}" = "--help" ]; then
+  echo "bundled firecracker shim"
+  exit 0
+fi
+echo "bundled firecracker shim: unsupported args: $*" >&2
+exit 2

runtime_sources/linux-x86_64/bin/jailer

@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+set -eu
+if [ "${1:-}" = "--version" ]; then
+  echo "Jailer v1.8.0 (bundled shim)"
+  exit 0
+fi
+echo "bundled jailer shim"
+exit 0

runtime_sources/linux-x86_64/guest/pyro_guest_agent.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Minimal guest-side exec agent for pyro runtime bundles."""
+
+from __future__ import annotations
+
+import json
+import socket
+import subprocess
+import time
+from typing import Any
+
+PORT = 5005
+BUFFER_SIZE = 65536
+
+
+def _read_request(conn: socket.socket) -> dict[str, Any]:
+    chunks: list[bytes] = []
+    while True:
+        data = conn.recv(BUFFER_SIZE)
+        if data == b"":
+            break
+        chunks.append(data)
+        if b"\n" in data:
+            break
+    payload = json.loads(b"".join(chunks).decode("utf-8").strip())
+    if not isinstance(payload, dict):
+        raise RuntimeError("request must be a JSON object")
+    return payload
+
+
+def _run_command(command: str, timeout_seconds: int) -> dict[str, Any]:
+    started = time.monotonic()
+    try:
+        proc = subprocess.run(
+            ["/bin/sh", "-lc", command],
+            text=True,
+            capture_output=True,
+            timeout=timeout_seconds,
+            check=False,
+        )
+        return {
+            "stdout": proc.stdout,
+            "stderr": proc.stderr,
+            "exit_code": proc.returncode,
+            "duration_ms": int((time.monotonic() - started) * 1000),
+        }
+    except subprocess.TimeoutExpired:
+        return {
+            "stdout": "",
+            "stderr": f"command timed out after {timeout_seconds}s",
+            "exit_code": 124,
+            "duration_ms": int((time.monotonic() - started) * 1000),
+        }
+
+
+def main() -> None:
+    family = getattr(socket, "AF_VSOCK", None)
+    if family is None:
+        raise SystemExit("AF_VSOCK is unavailable")
+    with socket.socket(family, socket.SOCK_STREAM) as server:
+        server.bind((socket.VMADDR_CID_ANY, PORT))
+        server.listen(1)
+        while True:
+            conn, _ = server.accept()
+            with conn:
+                request = _read_request(conn)
+                command = str(request.get("command", ""))
+                timeout_seconds = int(request.get("timeout_seconds", 30))
+                response = _run_command(command, timeout_seconds)
+                conn.sendall((json.dumps(response) + "\n").encode("utf-8"))
+
+
+if __name__ == "__main__":
+    main()

runtime_sources/linux-x86_64/packages/debian-base.txt

@@ -0,0 +1,10 @@
+bash
+ca-certificates
+coreutils
+curl
+dnsutils
+iproute2
+iputils-ping
+netbase
+procps
+python3-minimal

runtime_sources/linux-x86_64/packages/debian-build.txt

@@ -0,0 +1,15 @@
+bash
+build-essential
+ca-certificates
+cmake
+coreutils
+curl
+dnsutils
+git
+iproute2
+iputils-ping
+netbase
+pkg-config
+procps
+python3
+python3-pip

runtime_sources/linux-x86_64/packages/debian-git.txt

@@ -0,0 +1,11 @@
+bash
+ca-certificates
+coreutils
+curl
+dnsutils
+git
+iproute2
+iputils-ping
+netbase
+procps
+python3-minimal

runtime_sources/linux-x86_64/profiles/debian-base/rootfs.ext4

@@ -0,0 +1 @@
+placeholder-rootfs-debian-base

runtime_sources/linux-x86_64/profiles/debian-base/vmlinux

@@ -0,0 +1 @@
+placeholder-kernel-debian-base

runtime_sources/linux-x86_64/profiles/debian-build/rootfs.ext4

@@ -0,0 +1 @@
+placeholder-rootfs-debian-build

runtime_sources/linux-x86_64/profiles/debian-build/vmlinux

@@ -0,0 +1 @@
+placeholder-kernel-debian-build

runtime_sources/linux-x86_64/profiles/debian-git/rootfs.ext4

@@ -0,0 +1 @@
+placeholder-rootfs-debian-git

runtime_sources/linux-x86_64/profiles/debian-git/vmlinux

@@ -0,0 +1 @@
+placeholder-kernel-debian-git

runtime_sources/linux-x86_64/runtime.lock.json

@@ -0,0 +1,71 @@
+{
+  "bundle_version": "0.1.0",
+  "platform": "linux-x86_64",
+  "component_versions": {
+    "firecracker": "1.12.1",
+    "jailer": "1.12.1",
+    "kernel": "5.10.210",
+    "guest_agent": "0.1.0-dev",
+    "base_distro": "debian-bookworm-20250210"
+  },
+  "capabilities": {
+    "vm_boot": false,
+    "guest_exec": false,
+    "guest_network": false
+  },
+  "binaries": {
+    "firecracker": "bin/firecracker",
+    "jailer": "bin/jailer"
+  },
+  "guest": {
+    "agent": {
+      "path": "guest/pyro_guest_agent.py"
+    }
+  },
+  "profiles": {
+    "debian-base": {
+      "description": "Minimal Debian userspace for shell and core Unix tooling.",
+      "kernel": "profiles/debian-base/vmlinux",
+      "rootfs": "profiles/debian-base/rootfs.ext4"
+    },
+    "debian-git": {
+      "description": "Debian base environment with Git preinstalled.",
+      "kernel": "profiles/debian-git/vmlinux",
+      "rootfs": "profiles/debian-git/rootfs.ext4"
+    },
+    "debian-build": {
+      "description": "Debian Git environment with common build tools for source builds.",
+      "kernel": "profiles/debian-build/vmlinux",
+      "rootfs": "profiles/debian-build/rootfs.ext4"
+    }
+  },
+  "upstream": {
+    "firecracker_release": {
+      "version": "v1.12.1",
+      "archive_url": "https://github.com/firecracker-microvm/firecracker/releases/download/v1.12.1/firecracker-v1.12.1-x86_64.tgz",
+      "archive_sha256": "0a75e67ef6e4c540a2cf248b06822b0be9820cbba9fe19f9e0321200fe76ff6b",
+      "firecracker_member": "release-v1.12.1-x86_64/firecracker-v1.12.1-x86_64",
+      "jailer_member": "release-v1.12.1-x86_64/jailer-v1.12.1-x86_64"
+    },
+    "kernel_build": {
+      "script": "scripts/build_microvm_kernel.sh",
+      "builder_image": "debian:12-slim",
+      "linux_version": "5.10.210",
+      "source_url": "https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.210.tar.xz",
+      "config_url": "https://raw.githubusercontent.com/firecracker-microvm/firecracker/v1.12.1/resources/guest_configs/microvm-kernel-ci-x86_64-5.10.config"
+    },
+    "rootfs_build": {
+      "script": "scripts/build_debian_rootfs.sh",
+      "builder_image": "debian:12-slim",
+      "debian_release": "bookworm",
+      "debian_snapshot": "20250210T000000Z",
+      "guest_init": "scripts/pyro-init",
+      "agent_service": "scripts/pyro-guest-agent.service",
+      "package_files": {
+        "debian-base": "packages/debian-base.txt",
+        "debian-git": "packages/debian-git.txt",
+        "debian-build": "packages/debian-build.txt"
+      }
+    }
+  }
+}

runtime_sources/linux-x86_64/scripts/build_debian_rootfs.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+builder_image=""
+debian_release=""
+debian_snapshot=""
+packages_file=""
+guest_agent=""
+guest_init=""
+agent_service=""
+workdir=""
+output=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --builder-image) builder_image="$2"; shift 2 ;;
+    --debian-release) debian_release="$2"; shift 2 ;;
+    --debian-snapshot) debian_snapshot="$2"; shift 2 ;;
+    --packages-file) packages_file="$2"; shift 2 ;;
+    --guest-agent) guest_agent="$2"; shift 2 ;;
+    --guest-init) guest_init="$2"; shift 2 ;;
+    --agent-service) agent_service="$2"; shift 2 ;;
+    --workdir) workdir="$2"; shift 2 ;;
+    --output) output="$2"; shift 2 ;;
+    *) echo "unknown arg: $1" >&2; exit 1 ;;
+  esac
+done
+
+: "${builder_image:?missing --builder-image}"
+: "${debian_release:?missing --debian-release}"
+: "${debian_snapshot:?missing --debian-snapshot}"
+: "${packages_file:?missing --packages-file}"
+: "${guest_agent:?missing --guest-agent}"
+: "${guest_init:?missing --guest-init}"
+: "${agent_service:?missing --agent-service}"
+: "${workdir:?missing --workdir}"
+: "${output:?missing --output}"
+
+rm -rf "$workdir"
+mkdir -p "$workdir/in" "$workdir/out" "$(dirname "$output")"
+workdir="$(cd "$workdir" && pwd)"
+output_dir="$(cd "$(dirname "$output")" && pwd)"
+output="$output_dir/$(basename "$output")"
+cp "$packages_file" "$workdir/in/packages.txt"
+cp "$guest_agent" "$workdir/in/pyro_guest_agent.py"
+cp "$guest_init" "$workdir/in/pyro-init"
+cp "$agent_service" "$workdir/in/pyro-guest-agent.service"
+
+container_script="$workdir/build-rootfs-container.sh"
+cat > "$container_script" <<'SCRIPT'
+#!/usr/bin/env bash
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get install -y --no-install-recommends ca-certificates debootstrap e2fsprogs systemd-container
+
+mirror="http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/"
+packages_csv="$(paste -sd, /work/in/packages.txt)"
+rootfs_dir="/work/rootfs"
+rm -rf "$rootfs_dir"
+mkdir -p "$rootfs_dir"
+
+debootstrap \
+  --arch=amd64 \
+  --variant=minbase \
+  --include="$packages_csv" \
+  --no-check-gpg \
+  "$DEBIAN_RELEASE" \
+  "$rootfs_dir" \
+  "$mirror"
+
+cat > "$rootfs_dir/etc/apt/sources.list" <<APT
+ deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/ ${DEBIAN_RELEASE} main
+ deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/${DEBIAN_SNAPSHOT}/ ${DEBIAN_RELEASE}-security main
+APT
+
+mkdir -p "$rootfs_dir/opt/pyro/bin" "$rootfs_dir/etc/systemd/system/multi-user.target.wants"
+install -m 0755 /work/in/pyro_guest_agent.py "$rootfs_dir/opt/pyro/bin/pyro_guest_agent.py"
+install -m 0755 /work/in/pyro-init "$rootfs_dir/opt/pyro/bin/pyro-init"
+install -m 0644 /work/in/pyro-guest-agent.service "$rootfs_dir/etc/systemd/system/pyro-guest-agent.service"
+ln -sf /etc/systemd/system/pyro-guest-agent.service \
+  "$rootfs_dir/etc/systemd/system/multi-user.target.wants/pyro-guest-agent.service"
+ln -sf /opt/pyro/bin/pyro-init "$rootfs_dir/sbin/init"
+printf '127.0.0.1 localhost\n' > "$rootfs_dir/etc/hosts"
+truncate -s 2G /work/out/rootfs.ext4
+mkfs.ext4 -F -d "$rootfs_dir" /work/out/rootfs.ext4 >/dev/null
+SCRIPT
+chmod +x "$container_script"
+
+docker run --rm \
+  -e DEBIAN_RELEASE="$debian_release" \
+  -e DEBIAN_SNAPSHOT="$debian_snapshot" \
+  -v "$workdir:/work" \
+  "$builder_image" \
+  /work/build-rootfs-container.sh
+
+cp "$workdir/out/rootfs.ext4" "$output"

runtime_sources/linux-x86_64/scripts/build_microvm_kernel.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+builder_image=""
+linux_version=""
+source_url=""
+config_url=""
+workdir=""
+output=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --builder-image) builder_image="$2"; shift 2 ;;
+    --linux-version) linux_version="$2"; shift 2 ;;
+    --source-url) source_url="$2"; shift 2 ;;
+    --config-url) config_url="$2"; shift 2 ;;
+    --workdir) workdir="$2"; shift 2 ;;
+    --output) output="$2"; shift 2 ;;
+    *) echo "unknown arg: $1" >&2; exit 1 ;;
+  esac
+done
+
+: "${builder_image:?missing --builder-image}"
+: "${linux_version:?missing --linux-version}"
+: "${source_url:?missing --source-url}"
+: "${config_url:?missing --config-url}"
+: "${workdir:?missing --workdir}"
+: "${output:?missing --output}"
+
+mkdir -p "$workdir" "$(dirname "$output")"
+workdir="$(cd "$workdir" && pwd)"
+output_dir="$(cd "$(dirname "$output")" && pwd)"
+output="$output_dir/$(basename "$output")"
+container_script="$workdir/build-kernel-container.sh"
+cat > "$container_script" <<'SCRIPT'
+#!/usr/bin/env bash
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+apt-get update
+apt-get install -y --no-install-recommends \
+  bc bison build-essential ca-certificates curl flex libelf-dev libssl-dev pahole python3 rsync xz-utils
+cd /work
+curl -fsSL "$KERNEL_SOURCE_URL" -o linux.tar.xz
+curl -fsSL "$KERNEL_CONFIG_URL" -o kernel.config
+rm -rf linux-src out
+mkdir -p linux-src out
+tar -xf linux.tar.xz -C linux-src --strip-components=1
+cd linux-src
+cp /work/kernel.config .config
+make olddefconfig
+make -j"$(nproc)" vmlinux
+cp vmlinux /work/out/vmlinux
+SCRIPT
+chmod +x "$container_script"
+mkdir -p "$workdir/out"
+
+docker run --rm \
+  -e KERNEL_SOURCE_URL="$source_url" \
+  -e KERNEL_CONFIG_URL="$config_url" \
+  -v "$workdir:/work" \
+  "$builder_image" \
+  /work/build-kernel-container.sh
+
+cp "$workdir/out/vmlinux" "$output"

runtime_sources/linux-x86_64/scripts/pyro-guest-agent.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Pyro guest exec agent
+After=network-online.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/python3 /opt/pyro/bin/pyro_guest_agent.py
+Restart=always
+RestartSec=1
+
+[Install]
+WantedBy=multi-user.target

runtime_sources/linux-x86_64/scripts/pyro-init

@@ -0,0 +1,56 @@
+#!/bin/sh
+set -eu
+
+PATH=/usr/sbin:/usr/bin:/sbin:/bin
+AGENT=/opt/pyro/bin/pyro_guest_agent.py
+
+mount -t proc proc /proc || true
+mount -t sysfs sysfs /sys || true
+mount -t devtmpfs devtmpfs /dev || true
+mkdir -p /run /tmp
+hostname pyro-vm || true
+
+cmdline="$(cat /proc/cmdline 2>/dev/null || true)"
+
+get_arg() {
+    key="$1"
+    for token in $cmdline; do
+        case "$token" in
+            "$key"=*)
+                printf '%s' "${token#*=}"
+                return 0
+                ;;
+        esac
+    done
+    return 1
+}
+
+ip link set lo up || true
+if ip link show eth0 >/dev/null 2>&1; then
+    ip link set eth0 up || true
+    guest_ip="$(get_arg pyro.guest_ip || true)"
+    gateway_ip="$(get_arg pyro.gateway_ip || true)"
+    netmask="$(get_arg pyro.netmask || true)"
+    dns_csv="$(get_arg pyro.dns || true)"
+    if [ -n "$guest_ip" ] && [ -n "$netmask" ]; then
+        ip addr add "$guest_ip/$netmask" dev eth0 || true
+    fi
+    if [ -n "$gateway_ip" ]; then
+        ip route add default via "$gateway_ip" dev eth0 || true
+    fi
+    if [ -n "$dns_csv" ]; then
+        : > /etc/resolv.conf
+        old_ifs="$IFS"
+        IFS=,
+        for dns in $dns_csv; do
+            printf 'nameserver %s\n' "$dns" >> /etc/resolv.conf
+        done
+        IFS="$old_ifs"
+    fi
+fi
+
+if [ -f "$AGENT" ]; then
+    python3 "$AGENT" &
+fi
+
+exec /bin/sh -lc 'trap : TERM INT; while true; do sleep 3600; done'