diff --git a/AGENTS.md b/AGENTS.md index 7fc9156..d2b37fc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,9 @@ This repository ships `pyro-mcp`, an MCP-compatible package for ephemeral VM lif - Use `uv` for all Python environment and command execution. - Run `make setup` after cloning. - Run `make check` before opening a PR. +- Use `make runtime-bundle` to regenerate the packaged runtime bundle from `runtime_sources/`. +- Use `make runtime-materialize` to build real runtime inputs into `build/runtime_sources/`. +- Use `make runtime-fetch-binaries`, `make runtime-build-kernel-real`, and `make runtime-build-rootfs-real` if you need to debug the real-source pipeline step by step. - Use `make demo` to validate deterministic VM lifecycle execution. - Use `make ollama-demo` to validate model-triggered lifecycle tool usage. - Use `make doctor` to inspect bundled runtime integrity and host prerequisites. @@ -28,6 +31,7 @@ These checks run in pre-commit hooks and should all pass locally. - Public factory: `pyro_mcp.create_server()` - Runtime diagnostics CLI: `pyro-mcp-doctor` +- Runtime bundle build CLI: `pyro-mcp-runtime-build` - Current bundled runtime is shim-based unless replaced with a real guest-capable bundle; check `make doctor` for runtime capabilities. - Lifecycle tools: - `vm_list_profiles` diff --git a/Makefile b/Makefile index 9f5dcb1..af02ea4 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,13 @@ PYTHON ?= uv run python OLLAMA_BASE_URL ?= http://localhost:11434/v1 OLLAMA_MODEL ?= llama3.2:3b OLLAMA_DEMO_FLAGS ?= +RUNTIME_PLATFORM ?= linux-x86_64 +RUNTIME_SOURCE_DIR ?= runtime_sources +RUNTIME_BUILD_DIR ?= build/runtime_bundle +RUNTIME_BUNDLE_DIR ?= src/pyro_mcp/runtime_bundle +RUNTIME_MATERIALIZED_DIR ?= build/runtime_sources -.PHONY: setup lint format typecheck test check demo doctor ollama ollama-demo run-server install-hooks +.PHONY: setup lint format typecheck test check demo doctor ollama ollama-demo run-server install-hooks runtime-bundle runtime-binaries runtime-kernel runtime-rootfs runtime-agent runtime-validate runtime-manifest runtime-sync runtime-clean runtime-fetch-binaries runtime-build-kernel-real runtime-build-rootfs-real runtime-materialize setup: uv sync --dev @@ -38,3 +43,42 @@ run-server: install-hooks: uv run pre-commit install + +runtime-binaries: + uv run pyro-mcp-runtime-build stage-binaries --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-kernel: + uv run pyro-mcp-runtime-build stage-kernel --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-rootfs: + uv run pyro-mcp-runtime-build stage-rootfs --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-agent: + uv run pyro-mcp-runtime-build stage-agent --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-validate: + uv run pyro-mcp-runtime-build validate --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-manifest: + uv run pyro-mcp-runtime-build manifest --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-sync: + uv run pyro-mcp-runtime-build sync --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-bundle: + uv run pyro-mcp-runtime-build bundle --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-fetch-binaries: + uv run pyro-mcp-runtime-build fetch-binaries --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-build-kernel-real: + uv run pyro-mcp-runtime-build build-kernel --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-build-rootfs-real: + uv run pyro-mcp-runtime-build build-rootfs --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-materialize: + uv run pyro-mcp-runtime-build materialize --platform "$(RUNTIME_PLATFORM)" --source-dir "$(RUNTIME_SOURCE_DIR)" --build-dir "$(RUNTIME_BUILD_DIR)" --bundle-dir "$(RUNTIME_BUNDLE_DIR)" --materialized-dir "$(RUNTIME_MATERIALIZED_DIR)" + +runtime-clean: + rm -rf "$(RUNTIME_BUILD_DIR)" "$(RUNTIME_MATERIALIZED_DIR)" diff --git a/README.md b/README.md index 32db90d..c622969 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,37 @@ Host requirements still apply: make setup ``` +## Build runtime bundle + +```bash +make runtime-bundle +``` + +This builds the packaged runtime bundle from `runtime_sources/` and syncs the result into `src/pyro_mcp/runtime_bundle/`. +For real artifacts, first materialize upstream sources into `build/runtime_sources/`. + +Available staged targets: +- `make runtime-binaries` +- `make runtime-kernel` +- `make runtime-rootfs` +- `make runtime-agent` +- `make runtime-validate` +- `make runtime-manifest` +- `make runtime-sync` +- `make runtime-clean` + +Available real-runtime targets: +- `make runtime-fetch-binaries` +- `make runtime-build-kernel-real` +- `make runtime-build-rootfs-real` +- `make runtime-materialize` + +Current limitation: +- the pipeline is real, but the checked-in source artifacts in `runtime_sources/` are still shim/placeholder inputs +- the real-source path depends on `docker`, outbound access to GitHub and Debian snapshot mirrors, and enough disk for kernel/rootfs builds +- replacing those inputs with real Firecracker binaries, a real kernel, and real rootfs images is what upgrades the packaged bundle from `host_compat` to true guest execution +- the next artifact-replacement steps are documented in `runtime_sources/README.md` + ## Run deterministic lifecycle demo ```bash diff --git a/runtime_sources/NOTICE b/runtime_sources/NOTICE new file mode 100644 index 0000000..fe63e74 --- /dev/null +++ b/runtime_sources/NOTICE @@ -0,0 +1,5 @@ +pyro-mcp runtime bundle + +This package includes bundled runtime components intended for local developer workflows. +Replace shims with official Firecracker/jailer binaries and production profile artifacts +for real VM isolation in release builds. diff --git a/runtime_sources/README.md b/runtime_sources/README.md new file mode 100644 index 0000000..02500bf --- /dev/null +++ b/runtime_sources/README.md @@ -0,0 +1,29 @@ +# runtime_sources + +Source-of-truth inputs for `make runtime-bundle`. + +Current state: +- `bin/firecracker` and `bin/jailer` are shim placeholders. +- profile kernels and rootfs images are placeholder files. +- `guest/pyro_guest_agent.py` is the guest agent artifact that should ultimately be installed into each real rootfs. +- real source materialization now writes into `build/runtime_sources/`, not back into the tracked placeholder files. + +Materialization workflow: +1. `make runtime-fetch-binaries` +2. `make runtime-build-kernel-real` +3. `make runtime-build-rootfs-real` +4. `make runtime-bundle` + +Build requirements for the real path: +- `docker` +- outbound network access to GitHub and Debian snapshot mirrors +- enough disk for a kernel build plus 2G ext4 images per profile + +Next steps to make the bundle guest-capable: +1. Replace shim binaries with pinned official Firecracker and Jailer release artifacts. +2. Replace placeholder `vmlinux` and `rootfs.ext4` files with real, bootable artifacts for each profile. +3. Ensure the guest agent is installed and enabled inside every rootfs so the host can use vsock exec. +4. Once the source artifacts are real, update `runtime.lock.json` component versions and flip capability flags from `false` to `true`. + +Safety rule: +- The build pipeline should never emit `vm_boot=true`, `guest_exec=true`, or `guest_network=true` while any source artifact is still a shim or placeholder. diff --git a/runtime_sources/linux-x86_64/bin/firecracker b/runtime_sources/linux-x86_64/bin/firecracker new file mode 100755 index 0000000..dd94ba5 --- /dev/null +++ b/runtime_sources/linux-x86_64/bin/firecracker @@ -0,0 +1,12 @@ +#!/usr/bin/env sh +set -eu +if [ "${1:-}" = "--version" ]; then + echo "Firecracker v1.8.0 (bundled shim)" + exit 0 +fi +if [ "${1:-}" = "--help" ]; then + echo "bundled firecracker shim" + exit 0 +fi +echo "bundled firecracker shim: unsupported args: $*" >&2 +exit 2 diff --git a/runtime_sources/linux-x86_64/bin/jailer b/runtime_sources/linux-x86_64/bin/jailer new file mode 100755 index 0000000..0dc689b --- /dev/null +++ b/runtime_sources/linux-x86_64/bin/jailer @@ -0,0 +1,8 @@ +#!/usr/bin/env sh +set -eu +if [ "${1:-}" = "--version" ]; then + echo "Jailer v1.8.0 (bundled shim)" + exit 0 +fi +echo "bundled jailer shim" +exit 0 diff --git a/runtime_sources/linux-x86_64/guest/pyro_guest_agent.py b/runtime_sources/linux-x86_64/guest/pyro_guest_agent.py new file mode 100644 index 0000000..ea9c2cf --- /dev/null +++ b/runtime_sources/linux-x86_64/guest/pyro_guest_agent.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Minimal guest-side exec agent for pyro runtime bundles.""" + +from __future__ import annotations + +import json +import socket +import subprocess +import time +from typing import Any + +PORT = 5005 +BUFFER_SIZE = 65536 + + +def _read_request(conn: socket.socket) -> dict[str, Any]: + chunks: list[bytes] = [] + while True: + data = conn.recv(BUFFER_SIZE) + if data == b"": + break + chunks.append(data) + if b"\n" in data: + break + payload = json.loads(b"".join(chunks).decode("utf-8").strip()) + if not isinstance(payload, dict): + raise RuntimeError("request must be a JSON object") + return payload + + +def _run_command(command: str, timeout_seconds: int) -> dict[str, Any]: + started = time.monotonic() + try: + proc = subprocess.run( + ["/bin/sh", "-lc", command], + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + return { + "stdout": proc.stdout, + "stderr": proc.stderr, + "exit_code": proc.returncode, + "duration_ms": int((time.monotonic() - started) * 1000), + } + except subprocess.TimeoutExpired: + return { + "stdout": "", + "stderr": f"command timed out after {timeout_seconds}s", + "exit_code": 124, + "duration_ms": int((time.monotonic() - started) * 1000), + } + + +def main() -> None: + family = getattr(socket, "AF_VSOCK", None) + if family is None: + raise SystemExit("AF_VSOCK is unavailable") + with socket.socket(family, socket.SOCK_STREAM) as server: + server.bind((socket.VMADDR_CID_ANY, PORT)) + server.listen(1) + while True: + conn, _ = server.accept() + with conn: + request = _read_request(conn) + command = str(request.get("command", "")) + timeout_seconds = int(request.get("timeout_seconds", 30)) + response = _run_command(command, timeout_seconds) + conn.sendall((json.dumps(response) + "\n").encode("utf-8")) + + +if __name__ == "__main__": + main() diff --git a/runtime_sources/linux-x86_64/packages/debian-base.txt b/runtime_sources/linux-x86_64/packages/debian-base.txt new file mode 100644 index 0000000..3c826ec --- /dev/null +++ b/runtime_sources/linux-x86_64/packages/debian-base.txt @@ -0,0 +1,10 @@ +bash +ca-certificates +coreutils +curl +dnsutils +iproute2 +iputils-ping +netbase +procps +python3-minimal diff --git a/runtime_sources/linux-x86_64/packages/debian-build.txt b/runtime_sources/linux-x86_64/packages/debian-build.txt new file mode 100644 index 0000000..b36a67e --- /dev/null +++ b/runtime_sources/linux-x86_64/packages/debian-build.txt @@ -0,0 +1,15 @@ +bash +build-essential +ca-certificates +cmake +coreutils +curl +dnsutils +git +iproute2 +iputils-ping +netbase +pkg-config +procps +python3 +python3-pip diff --git a/runtime_sources/linux-x86_64/packages/debian-git.txt b/runtime_sources/linux-x86_64/packages/debian-git.txt new file mode 100644 index 0000000..263f329 --- /dev/null +++ b/runtime_sources/linux-x86_64/packages/debian-git.txt @@ -0,0 +1,11 @@ +bash +ca-certificates +coreutils +curl +dnsutils +git +iproute2 +iputils-ping +netbase +procps +python3-minimal diff --git a/runtime_sources/linux-x86_64/profiles/debian-base/rootfs.ext4 b/runtime_sources/linux-x86_64/profiles/debian-base/rootfs.ext4 new file mode 100644 index 0000000..81c0c7e --- /dev/null +++ b/runtime_sources/linux-x86_64/profiles/debian-base/rootfs.ext4 @@ -0,0 +1 @@ +placeholder-rootfs-debian-base diff --git a/runtime_sources/linux-x86_64/profiles/debian-base/vmlinux b/runtime_sources/linux-x86_64/profiles/debian-base/vmlinux new file mode 100644 index 0000000..c6c6539 --- /dev/null +++ b/runtime_sources/linux-x86_64/profiles/debian-base/vmlinux @@ -0,0 +1 @@ +placeholder-kernel-debian-base diff --git a/runtime_sources/linux-x86_64/profiles/debian-build/rootfs.ext4 b/runtime_sources/linux-x86_64/profiles/debian-build/rootfs.ext4 new file mode 100644 index 0000000..f054e1f --- /dev/null +++ b/runtime_sources/linux-x86_64/profiles/debian-build/rootfs.ext4 @@ -0,0 +1 @@ +placeholder-rootfs-debian-build diff --git a/runtime_sources/linux-x86_64/profiles/debian-build/vmlinux b/runtime_sources/linux-x86_64/profiles/debian-build/vmlinux new file mode 100644 index 0000000..03626a5 --- /dev/null +++ b/runtime_sources/linux-x86_64/profiles/debian-build/vmlinux @@ -0,0 +1 @@ +placeholder-kernel-debian-build diff --git a/runtime_sources/linux-x86_64/profiles/debian-git/rootfs.ext4 b/runtime_sources/linux-x86_64/profiles/debian-git/rootfs.ext4 new file mode 100644 index 0000000..96bb1e2 --- /dev/null +++ b/runtime_sources/linux-x86_64/profiles/debian-git/rootfs.ext4 @@ -0,0 +1 @@ +placeholder-rootfs-debian-git diff --git a/runtime_sources/linux-x86_64/profiles/debian-git/vmlinux b/runtime_sources/linux-x86_64/profiles/debian-git/vmlinux new file mode 100644 index 0000000..2bd061e --- /dev/null +++ b/runtime_sources/linux-x86_64/profiles/debian-git/vmlinux @@ -0,0 +1 @@ +placeholder-kernel-debian-git diff --git a/runtime_sources/linux-x86_64/runtime.lock.json b/runtime_sources/linux-x86_64/runtime.lock.json new file mode 100644 index 0000000..923f883 --- /dev/null +++ b/runtime_sources/linux-x86_64/runtime.lock.json @@ -0,0 +1,71 @@ +{ + "bundle_version": "0.1.0", + "platform": "linux-x86_64", + "component_versions": { + "firecracker": "1.12.1", + "jailer": "1.12.1", + "kernel": "5.10.210", + "guest_agent": "0.1.0-dev", + "base_distro": "debian-bookworm-20250210" + }, + "capabilities": { + "vm_boot": false, + "guest_exec": false, + "guest_network": false + }, + "binaries": { + "firecracker": "bin/firecracker", + "jailer": "bin/jailer" + }, + "guest": { + "agent": { + "path": "guest/pyro_guest_agent.py" + } + }, + "profiles": { + "debian-base": { + "description": "Minimal Debian userspace for shell and core Unix tooling.", + "kernel": "profiles/debian-base/vmlinux", + "rootfs": "profiles/debian-base/rootfs.ext4" + }, + "debian-git": { + "description": "Debian base environment with Git preinstalled.", + "kernel": "profiles/debian-git/vmlinux", + "rootfs": "profiles/debian-git/rootfs.ext4" + }, + "debian-build": { + "description": "Debian Git environment with common build tools for source builds.", + "kernel": "profiles/debian-build/vmlinux", + "rootfs": "profiles/debian-build/rootfs.ext4" + } + }, + "upstream": { + "firecracker_release": { + "version": "v1.12.1", + "archive_url": "https://github.com/firecracker-microvm/firecracker/releases/download/v1.12.1/firecracker-v1.12.1-x86_64.tgz", + "archive_sha256": "0a75e67ef6e4c540a2cf248b06822b0be9820cbba9fe19f9e0321200fe76ff6b", + "firecracker_member": "release-v1.12.1-x86_64/firecracker-v1.12.1-x86_64", + "jailer_member": "release-v1.12.1-x86_64/jailer-v1.12.1-x86_64" + }, + "kernel_build": { + "script": "scripts/build_microvm_kernel.sh", + "builder_image": "debian:12-slim", + "linux_version": "5.10.210", + "source_url": "https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.210.tar.xz", + "config_url": "https://raw.githubusercontent.com/firecracker-microvm/firecracker/v1.12.1/resources/guest_configs/microvm-kernel-ci-x86_64-5.10.config" + }, + "rootfs_build": { + "script": "scripts/build_debian_rootfs.sh", + "builder_image": "debian:12-slim", + "debian_release": "bookworm", + "debian_snapshot": "20250210T000000Z", + "guest_init": "scripts/pyro-init", + "agent_service": "scripts/pyro-guest-agent.service", + "package_files": { + "debian-base": "packages/debian-base.txt", + "debian-git": "packages/debian-git.txt", + "debian-build": "packages/debian-build.txt" + } + } + } +} diff --git a/runtime_sources/linux-x86_64/scripts/build_debian_rootfs.sh b/runtime_sources/linux-x86_64/scripts/build_debian_rootfs.sh new file mode 100755 index 0000000..2474c16 --- /dev/null +++ b/runtime_sources/linux-x86_64/scripts/build_debian_rootfs.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail + +builder_image="" +debian_release="" +debian_snapshot="" +packages_file="" +guest_agent="" +guest_init="" +agent_service="" +workdir="" +output="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --builder-image) builder_image="$2"; shift 2 ;; + --debian-release) debian_release="$2"; shift 2 ;; + --debian-snapshot) debian_snapshot="$2"; shift 2 ;; + --packages-file) packages_file="$2"; shift 2 ;; + --guest-agent) guest_agent="$2"; shift 2 ;; + --guest-init) guest_init="$2"; shift 2 ;; + --agent-service) agent_service="$2"; shift 2 ;; + --workdir) workdir="$2"; shift 2 ;; + --output) output="$2"; shift 2 ;; + *) echo "unknown arg: $1" >&2; exit 1 ;; + esac +done + +: "${builder_image:?missing --builder-image}" +: "${debian_release:?missing --debian-release}" +: "${debian_snapshot:?missing --debian-snapshot}" +: "${packages_file:?missing --packages-file}" +: "${guest_agent:?missing --guest-agent}" +: "${guest_init:?missing --guest-init}" +: "${agent_service:?missing --agent-service}" +: "${workdir:?missing --workdir}" +: "${output:?missing --output}" + +rm -rf "$workdir" +mkdir -p "$workdir/in" "$workdir/out" "$(dirname "$output")" +workdir="$(cd "$workdir" && pwd)" +output_dir="$(cd "$(dirname "$output")" && pwd)" +output="$output_dir/$(basename "$output")" +cp "$packages_file" "$workdir/in/packages.txt" +cp "$guest_agent" "$workdir/in/pyro_guest_agent.py" +cp "$guest_init" "$workdir/in/pyro-init" +cp "$agent_service" "$workdir/in/pyro-guest-agent.service" + +container_script="$workdir/build-rootfs-container.sh" +cat > "$container_script" <<'SCRIPT' +#!/usr/bin/env bash +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y --no-install-recommends ca-certificates debootstrap e2fsprogs systemd-container + +mirror="http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}/" +packages_csv="$(paste -sd, /work/in/packages.txt)" +rootfs_dir="/work/rootfs" +rm -rf "$rootfs_dir" +mkdir -p "$rootfs_dir" + +debootstrap \ + --arch=amd64 \ + --variant=minbase \ + --include="$packages_csv" \ + --no-check-gpg \ + "$DEBIAN_RELEASE" \ + "$rootfs_dir" \ + "$mirror" + +cat > "$rootfs_dir/etc/apt/sources.list" < "$rootfs_dir/etc/hosts" +truncate -s 2G /work/out/rootfs.ext4 +mkfs.ext4 -F -d "$rootfs_dir" /work/out/rootfs.ext4 >/dev/null +SCRIPT +chmod +x "$container_script" + +docker run --rm \ + -e DEBIAN_RELEASE="$debian_release" \ + -e DEBIAN_SNAPSHOT="$debian_snapshot" \ + -v "$workdir:/work" \ + "$builder_image" \ + /work/build-rootfs-container.sh + +cp "$workdir/out/rootfs.ext4" "$output" diff --git a/runtime_sources/linux-x86_64/scripts/build_microvm_kernel.sh b/runtime_sources/linux-x86_64/scripts/build_microvm_kernel.sh new file mode 100755 index 0000000..8a48a32 --- /dev/null +++ b/runtime_sources/linux-x86_64/scripts/build_microvm_kernel.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +builder_image="" +linux_version="" +source_url="" +config_url="" +workdir="" +output="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --builder-image) builder_image="$2"; shift 2 ;; + --linux-version) linux_version="$2"; shift 2 ;; + --source-url) source_url="$2"; shift 2 ;; + --config-url) config_url="$2"; shift 2 ;; + --workdir) workdir="$2"; shift 2 ;; + --output) output="$2"; shift 2 ;; + *) echo "unknown arg: $1" >&2; exit 1 ;; + esac +done + +: "${builder_image:?missing --builder-image}" +: "${linux_version:?missing --linux-version}" +: "${source_url:?missing --source-url}" +: "${config_url:?missing --config-url}" +: "${workdir:?missing --workdir}" +: "${output:?missing --output}" + +mkdir -p "$workdir" "$(dirname "$output")" +workdir="$(cd "$workdir" && pwd)" +output_dir="$(cd "$(dirname "$output")" && pwd)" +output="$output_dir/$(basename "$output")" +container_script="$workdir/build-kernel-container.sh" +cat > "$container_script" <<'SCRIPT' +#!/usr/bin/env bash +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y --no-install-recommends \ + bc bison build-essential ca-certificates curl flex libelf-dev libssl-dev pahole python3 rsync xz-utils +cd /work +curl -fsSL "$KERNEL_SOURCE_URL" -o linux.tar.xz +curl -fsSL "$KERNEL_CONFIG_URL" -o kernel.config +rm -rf linux-src out +mkdir -p linux-src out +tar -xf linux.tar.xz -C linux-src --strip-components=1 +cd linux-src +cp /work/kernel.config .config +make olddefconfig +make -j"$(nproc)" vmlinux +cp vmlinux /work/out/vmlinux +SCRIPT +chmod +x "$container_script" +mkdir -p "$workdir/out" + +docker run --rm \ + -e KERNEL_SOURCE_URL="$source_url" \ + -e KERNEL_CONFIG_URL="$config_url" \ + -v "$workdir:/work" \ + "$builder_image" \ + /work/build-kernel-container.sh + +cp "$workdir/out/vmlinux" "$output" diff --git a/runtime_sources/linux-x86_64/scripts/pyro-guest-agent.service b/runtime_sources/linux-x86_64/scripts/pyro-guest-agent.service new file mode 100644 index 0000000..69c7faa --- /dev/null +++ b/runtime_sources/linux-x86_64/scripts/pyro-guest-agent.service @@ -0,0 +1,12 @@ +[Unit] +Description=Pyro guest exec agent +After=network-online.target + +[Service] +Type=simple +ExecStart=/usr/bin/python3 /opt/pyro/bin/pyro_guest_agent.py +Restart=always +RestartSec=1 + +[Install] +WantedBy=multi-user.target diff --git a/runtime_sources/linux-x86_64/scripts/pyro-init b/runtime_sources/linux-x86_64/scripts/pyro-init new file mode 100755 index 0000000..6d4b9eb --- /dev/null +++ b/runtime_sources/linux-x86_64/scripts/pyro-init @@ -0,0 +1,56 @@ +#!/bin/sh +set -eu + +PATH=/usr/sbin:/usr/bin:/sbin:/bin +AGENT=/opt/pyro/bin/pyro_guest_agent.py + +mount -t proc proc /proc || true +mount -t sysfs sysfs /sys || true +mount -t devtmpfs devtmpfs /dev || true +mkdir -p /run /tmp +hostname pyro-vm || true + +cmdline="$(cat /proc/cmdline 2>/dev/null || true)" + +get_arg() { + key="$1" + for token in $cmdline; do + case "$token" in + "$key"=*) + printf '%s' "${token#*=}" + return 0 + ;; + esac + done + return 1 +} + +ip link set lo up || true +if ip link show eth0 >/dev/null 2>&1; then + ip link set eth0 up || true + guest_ip="$(get_arg pyro.guest_ip || true)" + gateway_ip="$(get_arg pyro.gateway_ip || true)" + netmask="$(get_arg pyro.netmask || true)" + dns_csv="$(get_arg pyro.dns || true)" + if [ -n "$guest_ip" ] && [ -n "$netmask" ]; then + ip addr add "$guest_ip/$netmask" dev eth0 || true + fi + if [ -n "$gateway_ip" ]; then + ip route add default via "$gateway_ip" dev eth0 || true + fi + if [ -n "$dns_csv" ]; then + : > /etc/resolv.conf + old_ifs="$IFS" + IFS=, + for dns in $dns_csv; do + printf 'nameserver %s\n' "$dns" >> /etc/resolv.conf + done + IFS="$old_ifs" + fi +fi + +if [ -f "$AGENT" ]; then + python3 "$AGENT" & +fi + +exec /bin/sh -lc 'trap : TERM INT; while true; do sleep 3600; done' diff --git a/src/pyro_mcp/runtime.py b/src/pyro_mcp/runtime.py index 3a5187e..dff2512 100644 --- a/src/pyro_mcp/runtime.py +++ b/src/pyro_mcp/runtime.py @@ -24,6 +24,7 @@ class RuntimePaths: manifest_path: Path firecracker_bin: Path jailer_bin: Path + guest_agent_path: Path | None artifacts_dir: Path notice_path: Path manifest: dict[str, Any] @@ -91,9 +92,21 @@ def resolve_runtime_paths( firecracker_bin = bundle_root / str(firecracker_entry.get("path", "")) jailer_bin = bundle_root / str(jailer_entry.get("path", "")) + guest_agent_path: Path | None = None + guest = manifest.get("guest") + if isinstance(guest, dict): + agent_entry = guest.get("agent") + if isinstance(agent_entry, dict): + raw_agent_path = agent_entry.get("path") + if isinstance(raw_agent_path, str): + guest_agent_path = bundle_root / raw_agent_path artifacts_dir = bundle_root / "profiles" - for path in (firecracker_bin, jailer_bin, artifacts_dir): + required_paths = [firecracker_bin, jailer_bin, artifacts_dir] + if guest_agent_path is not None: + required_paths.append(guest_agent_path) + + for path in required_paths: if not path.exists(): raise RuntimeError(f"runtime asset missing: {path}") @@ -112,6 +125,20 @@ def resolve_runtime_paths( raise RuntimeError( f"runtime checksum mismatch for {full_path}; expected {raw_hash}, got {actual}" ) + if isinstance(guest, dict): + agent_entry = guest.get("agent") + if isinstance(agent_entry, dict): + raw_path = agent_entry.get("path") + raw_hash = agent_entry.get("sha256") + if not isinstance(raw_path, str) or not isinstance(raw_hash, str): + raise RuntimeError("runtime guest agent manifest entry is malformed") + full_path = bundle_root / raw_path + actual = _sha256(full_path) + if actual != raw_hash: + raise RuntimeError( + f"runtime checksum mismatch for {full_path}; " + f"expected {raw_hash}, got {actual}" + ) profiles = manifest.get("profiles") if not isinstance(profiles, dict): raise RuntimeError("runtime manifest is missing `profiles`") @@ -141,6 +168,7 @@ def resolve_runtime_paths( manifest_path=manifest_path, firecracker_bin=firecracker_bin, jailer_bin=jailer_bin, + guest_agent_path=guest_agent_path, artifacts_dir=artifacts_dir, notice_path=notice_path, manifest=manifest, @@ -222,9 +250,11 @@ def doctor_report(*, platform: str = DEFAULT_PLATFORM) -> dict[str, Any]: "manifest_path": str(paths.manifest_path), "firecracker_bin": str(paths.firecracker_bin), "jailer_bin": str(paths.jailer_bin), + "guest_agent_path": str(paths.guest_agent_path) if paths.guest_agent_path else None, "artifacts_dir": str(paths.artifacts_dir), "notice_path": str(paths.notice_path), "bundle_version": paths.manifest.get("bundle_version"), + "component_versions": paths.manifest.get("component_versions", {}), "profiles": profile_names, "capabilities": { "supports_vm_boot": capabilities.supports_vm_boot, diff --git a/src/pyro_mcp/runtime_build.py b/src/pyro_mcp/runtime_build.py new file mode 100644 index 0000000..fddee51 --- /dev/null +++ b/src/pyro_mcp/runtime_build.py @@ -0,0 +1,538 @@ +"""Local build pipeline for packaged Firecracker runtime bundles.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import subprocess +import tarfile +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from pyro_mcp.runtime import DEFAULT_PLATFORM + +DEFAULT_RUNTIME_SOURCE_DIR = Path("runtime_sources") +DEFAULT_RUNTIME_BUILD_DIR = Path("build/runtime_bundle") +DEFAULT_RUNTIME_BUNDLE_DIR = Path("src/pyro_mcp/runtime_bundle") +DEFAULT_RUNTIME_MATERIALIZED_DIR = Path("build/runtime_sources") +DOWNLOAD_CHUNK_SIZE = 1024 * 1024 + + +@dataclass(frozen=True) +class RuntimeBuildLock: + bundle_version: str + platform: str + component_versions: dict[str, str] + capabilities: dict[str, bool] + binaries: dict[str, str] + guest: dict[str, dict[str, str]] + profiles: dict[str, dict[str, str]] + upstream: dict[str, Any] + + +@dataclass(frozen=True) +class RuntimeBuildPaths: + source_root: Path + source_platform_root: Path + build_root: Path + build_platform_root: Path + bundle_dir: Path + materialized_root: Path + materialized_platform_root: Path + platform: str + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fp: + for block in iter(lambda: fp.read(DOWNLOAD_CHUNK_SIZE), b""): + digest.update(block) + return digest.hexdigest() + + +def _cache_filename(url: str) -> str: + digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:12] + return f"{digest}-{Path(url).name}" + + +def _load_lock(paths: RuntimeBuildPaths) -> RuntimeBuildLock: + lock_path = paths.source_platform_root / "runtime.lock.json" + payload = json.loads(lock_path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise RuntimeError(f"invalid runtime lock file: {lock_path}") + return RuntimeBuildLock( + bundle_version=str(payload["bundle_version"]), + platform=str(payload["platform"]), + component_versions={ + str(key): str(value) for key, value in dict(payload["component_versions"]).items() + }, + capabilities={ + str(key): bool(value) for key, value in dict(payload["capabilities"]).items() + }, + binaries={str(key): str(value) for key, value in dict(payload["binaries"]).items()}, + guest={ + str(key): {str(k): str(v) for k, v in dict(value).items()} + for key, value in dict(payload["guest"]).items() + }, + profiles={ + str(key): {str(k): str(v) for k, v in dict(value).items()} + for key, value in dict(payload["profiles"]).items() + }, + upstream={str(key): value for key, value in dict(payload.get("upstream", {})).items()}, + ) + + +def _copy_file(source: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, dest) + + +def _copy_notice(paths: RuntimeBuildPaths) -> None: + _copy_file(paths.source_root / "NOTICE", paths.build_root / "NOTICE") + + +def _resolved_source_path(paths: RuntimeBuildPaths, relative_path: str) -> Path: + materialized = paths.materialized_platform_root / relative_path + if materialized.exists(): + return materialized + return paths.source_platform_root / relative_path + + +def _download(url: str, dest: Path) -> None: # pragma: no cover - integration helper + dest.parent.mkdir(parents=True, exist_ok=True) + with urllib.request.urlopen(url) as response, dest.open("wb") as fp: # noqa: S310 + while True: + chunk = response.read(DOWNLOAD_CHUNK_SIZE) + if chunk == b"": + break + fp.write(chunk) + + +def _run(command: list[str]) -> None: # pragma: no cover - integration helper + completed = subprocess.run(command, text=True, capture_output=True, check=False) + if completed.returncode != 0: + stderr = completed.stderr.strip() or completed.stdout.strip() + raise RuntimeError(f"command {' '.join(command)!r} failed: {stderr}") + + +def validate_sources(paths: RuntimeBuildPaths, lock: RuntimeBuildLock) -> None: + firecracker_source = _resolved_source_path(paths, lock.binaries["firecracker"]) + jailer_source = _resolved_source_path(paths, lock.binaries["jailer"]) + firecracker_text = firecracker_source.read_text(encoding="utf-8", errors="ignore") + jailer_text = jailer_source.read_text(encoding="utf-8", errors="ignore") + has_shim_binaries = ( + "bundled firecracker shim" in firecracker_text or "bundled jailer shim" in jailer_text + ) + + has_placeholder_profiles = False + for profile in lock.profiles.values(): + for kind in ("kernel", "rootfs"): + source = _resolved_source_path(paths, profile[kind]) + text = source.read_text(encoding="utf-8", errors="ignore") + if "placeholder-" in text: + has_placeholder_profiles = True + break + if has_placeholder_profiles: + break + + if any(lock.capabilities.values()) and (has_shim_binaries or has_placeholder_profiles): + raise RuntimeError( + "runtime lock advertises guest-capable features while source artifacts are still " + "shim/placeholder inputs" + ) + + +def materialize_binaries(paths: RuntimeBuildPaths, lock: RuntimeBuildLock) -> None: + release = lock.upstream.get("firecracker_release") + if not isinstance(release, dict): + raise RuntimeError("runtime lock is missing upstream.firecracker_release configuration") + archive_url = release.get("archive_url") + archive_sha256 = release.get("archive_sha256") + firecracker_member = release.get("firecracker_member") + jailer_member = release.get("jailer_member") + if not all( + isinstance(value, str) + for value in ( + archive_url, + archive_sha256, + firecracker_member, + jailer_member, + ) + ): + raise RuntimeError("upstream.firecracker_release is incomplete") + archive_url = str(archive_url) + archive_sha256 = str(archive_sha256) + firecracker_member = str(firecracker_member) + jailer_member = str(jailer_member) + + cache_dir = paths.materialized_root / "_downloads" + archive_path = cache_dir / _cache_filename(archive_url) + if not archive_path.exists(): + _download(archive_url, archive_path) + actual_archive_sha256 = _sha256(archive_path) + if actual_archive_sha256 != archive_sha256: + raise RuntimeError( + "firecracker release archive checksum mismatch: expected " + f"{archive_sha256}, got {actual_archive_sha256}" + ) + + targets = { + firecracker_member: paths.materialized_platform_root / lock.binaries["firecracker"], + jailer_member: paths.materialized_platform_root / lock.binaries["jailer"], + } + with tarfile.open(archive_path, "r:gz") as archive: + members = {member.name: member for member in archive.getmembers()} + for member_name, dest in targets.items(): + member = members.get(member_name) + if member is None: + raise RuntimeError(f"release archive is missing {member_name}") + extracted = archive.extractfile(member) + if extracted is None: + raise RuntimeError(f"unable to extract {member_name}") + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(extracted.read()) + dest.chmod(dest.stat().st_mode | 0o111) + + +def materialize_kernel( + paths: RuntimeBuildPaths, lock: RuntimeBuildLock +) -> None: # pragma: no cover - integration helper + kernel_build = lock.upstream.get("kernel_build") + if not isinstance(kernel_build, dict): + raise RuntimeError("runtime lock is missing upstream.kernel_build configuration") + script = kernel_build.get("script") + linux_version = kernel_build.get("linux_version") + source_url = kernel_build.get("source_url") + config_url = kernel_build.get("config_url") + builder_image = kernel_build.get("builder_image") + if not all( + isinstance(value, str) + for value in ( + script, + linux_version, + source_url, + config_url, + builder_image, + ) + ): + raise RuntimeError("upstream.kernel_build is incomplete") + script = str(script) + linux_version = str(linux_version) + source_url = str(source_url) + config_url = str(config_url) + builder_image = str(builder_image) + + script_path = paths.source_platform_root / script + if not script_path.exists(): + raise RuntimeError(f"kernel build script not found: {script_path}") + + shared_output = paths.materialized_platform_root / "profiles/_shared/vmlinux" + shared_output.parent.mkdir(parents=True, exist_ok=True) + workdir = paths.materialized_root / "_kernel_work" + workdir.mkdir(parents=True, exist_ok=True) + _run( + [ + str(script_path), + "--builder-image", + builder_image, + "--linux-version", + linux_version, + "--source-url", + source_url, + "--config-url", + config_url, + "--workdir", + str(workdir), + "--output", + str(shared_output), + ] + ) + + for profile in lock.profiles.values(): + dest = paths.materialized_platform_root / profile["kernel"] + _copy_file(shared_output, dest) + + +def materialize_rootfs( + paths: RuntimeBuildPaths, lock: RuntimeBuildLock +) -> None: # pragma: no cover - integration helper + rootfs_build = lock.upstream.get("rootfs_build") + if not isinstance(rootfs_build, dict): + raise RuntimeError("runtime lock is missing upstream.rootfs_build configuration") + script = rootfs_build.get("script") + builder_image = rootfs_build.get("builder_image") + debian_release = rootfs_build.get("debian_release") + debian_snapshot = rootfs_build.get("debian_snapshot") + package_files = rootfs_build.get("package_files") + guest_init = rootfs_build.get("guest_init") + agent_service = rootfs_build.get("agent_service") + if not isinstance(package_files, dict): + raise RuntimeError("upstream.rootfs_build.package_files must be a mapping") + if not all( + isinstance(value, str) + for value in ( + script, + builder_image, + debian_release, + debian_snapshot, + guest_init, + agent_service, + ) + ): + raise RuntimeError("upstream.rootfs_build is incomplete") + script = str(script) + builder_image = str(builder_image) + debian_release = str(debian_release) + debian_snapshot = str(debian_snapshot) + guest_init = str(guest_init) + agent_service = str(agent_service) + + script_path = paths.source_platform_root / script + guest_agent_path = paths.source_platform_root / lock.guest["agent"]["path"] + guest_init_path = paths.source_platform_root / guest_init + service_path = paths.source_platform_root / agent_service + if not script_path.exists(): + raise RuntimeError(f"rootfs build script not found: {script_path}") + + workdir = paths.materialized_root / "_rootfs_work" + workdir.mkdir(parents=True, exist_ok=True) + for profile_name, profile in lock.profiles.items(): + raw_packages_path = package_files.get(profile_name) + if not isinstance(raw_packages_path, str): + raise RuntimeError(f"missing package file for profile {profile_name!r}") + packages_path = paths.source_platform_root / raw_packages_path + output_path = paths.materialized_platform_root / profile["rootfs"] + output_path.parent.mkdir(parents=True, exist_ok=True) + _run( + [ + str(script_path), + "--builder-image", + builder_image, + "--debian-release", + debian_release, + "--debian-snapshot", + debian_snapshot, + "--packages-file", + str(packages_path), + "--guest-agent", + str(guest_agent_path), + "--guest-init", + str(guest_init_path), + "--agent-service", + str(service_path), + "--workdir", + str(workdir / profile_name), + "--output", + str(output_path), + ] + ) + + +def stage_binaries(paths: RuntimeBuildPaths, lock: RuntimeBuildLock) -> None: + for name, relative_path in lock.binaries.items(): + del name + source = _resolved_source_path(paths, relative_path) + dest = paths.build_platform_root / relative_path + _copy_file(source, dest) + dest.chmod(dest.stat().st_mode | 0o111) + + +def stage_kernel(paths: RuntimeBuildPaths, lock: RuntimeBuildLock) -> None: + for profile in lock.profiles.values(): + source = _resolved_source_path(paths, profile["kernel"]) + dest = paths.build_platform_root / profile["kernel"] + _copy_file(source, dest) + + +def stage_rootfs(paths: RuntimeBuildPaths, lock: RuntimeBuildLock) -> None: + for profile in lock.profiles.values(): + source = _resolved_source_path(paths, profile["rootfs"]) + dest = paths.build_platform_root / profile["rootfs"] + _copy_file(source, dest) + + +def stage_agent(paths: RuntimeBuildPaths, lock: RuntimeBuildLock) -> None: + for artifact in lock.guest.values(): + source = _resolved_source_path(paths, artifact["path"]) + dest = paths.build_platform_root / artifact["path"] + _copy_file(source, dest) + dest.chmod(dest.stat().st_mode | 0o111) + + +def generate_manifest(paths: RuntimeBuildPaths, lock: RuntimeBuildLock) -> dict[str, Any]: + manifest: dict[str, Any] = { + "bundle_version": lock.bundle_version, + "platform": lock.platform, + "component_versions": lock.component_versions, + "capabilities": lock.capabilities, + "binaries": {}, + "guest": {}, + "profiles": {}, + } + for name, relative_path in lock.binaries.items(): + full_path = paths.build_platform_root / relative_path + manifest["binaries"][name] = {"path": relative_path, "sha256": _sha256(full_path)} + for name, artifact in lock.guest.items(): + full_path = paths.build_platform_root / artifact["path"] + manifest["guest"][name] = { + "path": artifact["path"], + "sha256": _sha256(full_path), + } + for name, profile in lock.profiles.items(): + kernel_path = paths.build_platform_root / profile["kernel"] + rootfs_path = paths.build_platform_root / profile["rootfs"] + manifest["profiles"][name] = { + "description": profile["description"], + "kernel": {"path": profile["kernel"], "sha256": _sha256(kernel_path)}, + "rootfs": {"path": profile["rootfs"], "sha256": _sha256(rootfs_path)}, + } + manifest_path = paths.build_platform_root / "manifest.json" + manifest_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return manifest + + +def sync_bundle(paths: RuntimeBuildPaths) -> None: + bundle_platform_dir = paths.bundle_dir / paths.platform + bundle_notice_path = paths.bundle_dir / "NOTICE" + if bundle_platform_dir.exists(): + shutil.rmtree(bundle_platform_dir) + bundle_platform_dir.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(paths.build_platform_root, bundle_platform_dir) + _copy_file(paths.build_root / "NOTICE", bundle_notice_path) + + +def build_bundle(paths: RuntimeBuildPaths, *, sync: bool) -> dict[str, Any]: + lock = _load_lock(paths) + if lock.platform != paths.platform: + raise RuntimeError( + "runtime lock platform " + f"{lock.platform!r} does not match requested platform {paths.platform!r}" + ) + validate_sources(paths, lock) + if paths.build_root.exists(): + shutil.rmtree(paths.build_root) + paths.build_platform_root.mkdir(parents=True, exist_ok=True) + _copy_notice(paths) + stage_binaries(paths, lock) + stage_kernel(paths, lock) + stage_rootfs(paths, lock) + stage_agent(paths, lock) + manifest = generate_manifest(paths, lock) + if sync: + sync_bundle(paths) + return manifest + + +def materialize_sources(paths: RuntimeBuildPaths) -> None: + lock = _load_lock(paths) + materialize_binaries(paths, lock) + materialize_kernel(paths, lock) + materialize_rootfs(paths, lock) + + +def _build_paths( + *, + source_dir: Path, + build_dir: Path, + bundle_dir: Path, + platform: str, + materialized_dir: Path = DEFAULT_RUNTIME_MATERIALIZED_DIR, +) -> RuntimeBuildPaths: + return RuntimeBuildPaths( + source_root=source_dir, + source_platform_root=source_dir / platform, + build_root=build_dir, + build_platform_root=build_dir / platform, + bundle_dir=bundle_dir, + materialized_root=materialized_dir, + materialized_platform_root=materialized_dir / platform, + platform=platform, + ) + + +def _build_parser() -> argparse.ArgumentParser: # pragma: no cover - CLI wiring + parser = argparse.ArgumentParser(description="Build packaged runtime bundles for pyro-mcp.") + parser.add_argument( + "command", + choices=[ + "fetch-binaries", + "build-kernel", + "build-rootfs", + "materialize", + "stage-binaries", + "stage-kernel", + "stage-rootfs", + "stage-agent", + "validate", + "manifest", + "sync", + "bundle", + ], + ) + parser.add_argument("--platform", default=DEFAULT_PLATFORM) + parser.add_argument("--source-dir", default=str(DEFAULT_RUNTIME_SOURCE_DIR)) + parser.add_argument("--build-dir", default=str(DEFAULT_RUNTIME_BUILD_DIR)) + parser.add_argument("--bundle-dir", default=str(DEFAULT_RUNTIME_BUNDLE_DIR)) + parser.add_argument("--materialized-dir", default=str(DEFAULT_RUNTIME_MATERIALIZED_DIR)) + return parser + + +def main() -> None: # pragma: no cover - CLI wiring + args = _build_parser().parse_args() + paths = _build_paths( + source_dir=Path(args.source_dir), + build_dir=Path(args.build_dir), + bundle_dir=Path(args.bundle_dir), + materialized_dir=Path(args.materialized_dir), + platform=args.platform, + ) + lock = _load_lock(paths) + if args.command == "fetch-binaries": + materialize_binaries(paths, lock) + return + if args.command == "build-kernel": + materialize_kernel(paths, lock) + return + if args.command == "build-rootfs": + materialize_rootfs(paths, lock) + return + if args.command == "materialize": + materialize_sources(paths) + return + if args.command == "bundle": + build_bundle(paths, sync=True) + return + if args.command == "stage-binaries": + paths.build_platform_root.mkdir(parents=True, exist_ok=True) + _copy_notice(paths) + stage_binaries(paths, lock) + return + if args.command == "stage-kernel": + paths.build_platform_root.mkdir(parents=True, exist_ok=True) + stage_kernel(paths, lock) + return + if args.command == "stage-rootfs": + paths.build_platform_root.mkdir(parents=True, exist_ok=True) + stage_rootfs(paths, lock) + return + if args.command == "stage-agent": + paths.build_platform_root.mkdir(parents=True, exist_ok=True) + stage_agent(paths, lock) + return + if args.command == "validate": + validate_sources(paths, lock) + return + if args.command == "manifest": + generate_manifest(paths, lock) + return + if args.command == "sync": + sync_bundle(paths) + return + raise RuntimeError(f"unknown command: {args.command}") diff --git a/src/pyro_mcp/runtime_bundle/linux-x86_64/bin/firecracker b/src/pyro_mcp/runtime_bundle/linux-x86_64/bin/firecracker index dd94ba5..3fd3a8a 100755 Binary files a/src/pyro_mcp/runtime_bundle/linux-x86_64/bin/firecracker and b/src/pyro_mcp/runtime_bundle/linux-x86_64/bin/firecracker differ diff --git a/src/pyro_mcp/runtime_bundle/linux-x86_64/bin/jailer b/src/pyro_mcp/runtime_bundle/linux-x86_64/bin/jailer index 0dc689b..b904ec0 100755 Binary files a/src/pyro_mcp/runtime_bundle/linux-x86_64/bin/jailer and b/src/pyro_mcp/runtime_bundle/linux-x86_64/bin/jailer differ diff --git a/src/pyro_mcp/runtime_bundle/linux-x86_64/guest/pyro_guest_agent.py b/src/pyro_mcp/runtime_bundle/linux-x86_64/guest/pyro_guest_agent.py new file mode 100755 index 0000000..ea9c2cf --- /dev/null +++ b/src/pyro_mcp/runtime_bundle/linux-x86_64/guest/pyro_guest_agent.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Minimal guest-side exec agent for pyro runtime bundles.""" + +from __future__ import annotations + +import json +import socket +import subprocess +import time +from typing import Any + +PORT = 5005 +BUFFER_SIZE = 65536 + + +def _read_request(conn: socket.socket) -> dict[str, Any]: + chunks: list[bytes] = [] + while True: + data = conn.recv(BUFFER_SIZE) + if data == b"": + break + chunks.append(data) + if b"\n" in data: + break + payload = json.loads(b"".join(chunks).decode("utf-8").strip()) + if not isinstance(payload, dict): + raise RuntimeError("request must be a JSON object") + return payload + + +def _run_command(command: str, timeout_seconds: int) -> dict[str, Any]: + started = time.monotonic() + try: + proc = subprocess.run( + ["/bin/sh", "-lc", command], + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + return { + "stdout": proc.stdout, + "stderr": proc.stderr, + "exit_code": proc.returncode, + "duration_ms": int((time.monotonic() - started) * 1000), + } + except subprocess.TimeoutExpired: + return { + "stdout": "", + "stderr": f"command timed out after {timeout_seconds}s", + "exit_code": 124, + "duration_ms": int((time.monotonic() - started) * 1000), + } + + +def main() -> None: + family = getattr(socket, "AF_VSOCK", None) + if family is None: + raise SystemExit("AF_VSOCK is unavailable") + with socket.socket(family, socket.SOCK_STREAM) as server: + server.bind((socket.VMADDR_CID_ANY, PORT)) + server.listen(1) + while True: + conn, _ = server.accept() + with conn: + request = _read_request(conn) + command = str(request.get("command", "")) + timeout_seconds = int(request.get("timeout_seconds", 30)) + response = _run_command(command, timeout_seconds) + conn.sendall((json.dumps(response) + "\n").encode("utf-8")) + + +if __name__ == "__main__": + main() diff --git a/src/pyro_mcp/runtime_bundle/linux-x86_64/manifest.json b/src/pyro_mcp/runtime_bundle/linux-x86_64/manifest.json index e058b81..1d25331 100644 --- a/src/pyro_mcp/runtime_bundle/linux-x86_64/manifest.json +++ b/src/pyro_mcp/runtime_bundle/linux-x86_64/manifest.json @@ -1,48 +1,66 @@ { - "bundle_version": "0.1.0", - "platform": "linux-x86_64", "binaries": { "firecracker": { "path": "bin/firecracker", - "sha256": "2ff2d53551abcbf7ddebd921077214bff31910d4dfd894cc6fe66511d9f188e7" + "sha256": "b99ea49b8d8b7bfa307d3845585d6a97f7642aa17a985749900370070d8ca930" }, "jailer": { "path": "bin/jailer", - "sha256": "d79e972b3ede34b1c3eb9d54c9f1853a62a8525f78c39c8dab4d5d79a6783fe9" + "sha256": "86622337f91df329cca72bb21cd1324fb8b6fa47931601d65ee4b2c72ef2cae5" } }, + "bundle_version": "0.1.0", + "capabilities": { + "guest_exec": false, + "guest_network": false, + "vm_boot": false + }, + "component_versions": { + "base_distro": "debian-bookworm-20250210", + "firecracker": "1.12.1", + "guest_agent": "0.1.0-dev", + "jailer": "1.12.1", + "kernel": "5.10.210" + }, + "guest": { + "agent": { + "path": "guest/pyro_guest_agent.py", + "sha256": "65bf8a9a57ffd7321463537e598c4b30f0a13046cbd4538f1b65bc351da5d3c0" + } + }, + "platform": "linux-x86_64", "profiles": { "debian-base": { "description": "Minimal Debian userspace for shell and core Unix tooling.", "kernel": { "path": "profiles/debian-base/vmlinux", - "sha256": "a0bd6422be1061bb3b70a7895e82f66c25c59022d1e8a72b6fc9cdee4136f108" + "sha256": "15bcea4fa224131951888408978ff22fc2173f2782365c0617a900fe029bd8fb" }, "rootfs": { "path": "profiles/debian-base/rootfs.ext4", - "sha256": "2794a4bdc232b6a6267cfc1eaaa696f0efccd2f8f2e130f3ade736637de89dcd" - } - }, - "debian-git": { - "description": "Debian base environment with Git preinstalled.", - "kernel": { - "path": "profiles/debian-git/vmlinux", - "sha256": "eaf871c952bf6476f0299b1f501eddc302105e53c99c86161fa815e90cf5bc9f" - }, - "rootfs": { - "path": "profiles/debian-git/rootfs.ext4", - "sha256": "17863bd1496a9a08d89d6e4c73bd619d39bbe7f6089f1903837525629557c076" + "sha256": "46247e10fe9b223b15c4ccc672710c2f3013bf562ed9cf9b48af1f092d966494" } }, "debian-build": { "description": "Debian Git environment with common build tools for source builds.", "kernel": { "path": "profiles/debian-build/vmlinux", - "sha256": "c33994b1da43cf2f11ac9d437c034eaa71496b566a45028a9ae6f657105dc2b6" + "sha256": "15bcea4fa224131951888408978ff22fc2173f2782365c0617a900fe029bd8fb" }, "rootfs": { "path": "profiles/debian-build/rootfs.ext4", - "sha256": "ac148235c86a51c87228e17a8cf2c9452921886c094de42b470d5f42dab70226" + "sha256": "a0e9ec968b0fc6826f94a678164abc8c9b661adf87984184bd08abd1da15d7b6" + } + }, + "debian-git": { + "description": "Debian base environment with Git preinstalled.", + "kernel": { + "path": "profiles/debian-git/vmlinux", + "sha256": "15bcea4fa224131951888408978ff22fc2173f2782365c0617a900fe029bd8fb" + }, + "rootfs": { + "path": "profiles/debian-git/rootfs.ext4", + "sha256": "e28ba2e3fa9ed37bcc9fc04a9b4414f0b29d8c7378508e10be78049a38c25894" } } } diff --git a/src/pyro_mcp/vm_firecracker.py b/src/pyro_mcp/vm_firecracker.py index f522d5c..20c87fe 100644 --- a/src/pyro_mcp/vm_firecracker.py +++ b/src/pyro_mcp/vm_firecracker.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ipaddress import json from dataclasses import dataclass from pathlib import Path @@ -45,10 +46,29 @@ def build_launch_plan(instance: VmInstanceLike) -> FirecrackerLaunchPlan: guest_network_path = instance.workdir / "guest-network.json" guest_exec_path = instance.workdir / "guest-exec.json" + boot_args = [ + "console=ttyS0", + "reboot=k", + "panic=1", + "pci=off", + "init=/opt/pyro/bin/pyro-init", + ] + if instance.network is not None: + network = ipaddress.ip_network(instance.network.subnet_cidr, strict=False) + prefixlen = network.prefixlen + boot_args.extend( + [ + f"pyro.guest_ip={instance.network.guest_ip}", + f"pyro.gateway_ip={instance.network.gateway_ip}", + f"pyro.netmask={prefixlen}", + f"pyro.dns={','.join(instance.network.dns_servers)}", + ] + ) + config: dict[str, Any] = { "boot-source": { "kernel_image_path": instance.metadata["kernel_image"], - "boot_args": "console=ttyS0 reboot=k panic=1 pci=off", + "boot_args": " ".join(boot_args), }, "drives": [ { diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 4ba5972..2359c3d 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -12,6 +12,8 @@ def test_resolve_runtime_paths_default_bundle() -> None: paths = resolve_runtime_paths() assert paths.firecracker_bin.exists() assert paths.jailer_bin.exists() + assert paths.guest_agent_path is not None + assert paths.guest_agent_path.exists() assert (paths.artifacts_dir / "debian-git" / "vmlinux").exists() assert paths.manifest.get("platform") == "linux-x86_64" @@ -45,8 +47,14 @@ def test_resolve_runtime_paths_checksum_mismatch( firecracker_path = copied_platform / "bin" / "firecracker" firecracker_path.parent.mkdir(parents=True, exist_ok=True) firecracker_path.write_text("tampered\n", encoding="utf-8") - (copied_platform / "bin" / "jailer").write_text( - (source.jailer_bin).read_text(encoding="utf-8"), + (copied_platform / "bin" / "jailer").write_bytes(source.jailer_bin.read_bytes()) + guest_agent_path = source.guest_agent_path + if guest_agent_path is None: + raise AssertionError("expected guest agent in runtime bundle") + copied_guest_dir = copied_platform / "guest" + copied_guest_dir.mkdir(parents=True, exist_ok=True) + (copied_guest_dir / "pyro_guest_agent.py").write_text( + guest_agent_path.read_text(encoding="utf-8"), encoding="utf-8", ) for profile in ("debian-base", "debian-git", "debian-build"): @@ -54,9 +62,7 @@ def test_resolve_runtime_paths_checksum_mismatch( profile_dir.mkdir(parents=True, exist_ok=True) for filename in ("vmlinux", "rootfs.ext4"): source_file = source.artifacts_dir / profile / filename - (profile_dir / filename).write_text( - source_file.read_text(encoding="utf-8"), encoding="utf-8" - ) + (profile_dir / filename).write_bytes(source_file.read_bytes()) monkeypatch.setenv("PYRO_RUNTIME_BUNDLE_DIR", str(copied_bundle)) with pytest.raises(RuntimeError, match="checksum mismatch"): @@ -72,6 +78,8 @@ def test_doctor_report_has_runtime_fields() -> None: runtime = report.get("runtime") assert isinstance(runtime, dict) assert "firecracker_bin" in runtime + assert "guest_agent_path" in runtime + assert "component_versions" in runtime networking = report["networking"] assert isinstance(networking, dict) assert "tun_available" in networking diff --git a/tests/test_runtime_build.py b/tests/test_runtime_build.py new file mode 100644 index 0000000..b525900 --- /dev/null +++ b/tests/test_runtime_build.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import hashlib +import json +import tarfile +from pathlib import Path + +import pytest + +from pyro_mcp.runtime_build import ( + _build_paths, + _load_lock, + build_bundle, + generate_manifest, + materialize_binaries, + stage_agent, + stage_binaries, + stage_kernel, + stage_rootfs, + validate_sources, +) + + +def _write_text(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _make_source_tree(tmp_path: Path) -> tuple[Path, Path, Path]: + source_dir = tmp_path / "runtime_sources" + platform_root = source_dir / "linux-x86_64" + _write_text(source_dir / "NOTICE", "notice\n") + _write_text(platform_root / "bin/firecracker", "firecracker\n") + _write_text(platform_root / "bin/jailer", "jailer\n") + _write_text(platform_root / "guest/pyro_guest_agent.py", "#!/usr/bin/env python3\n") + _write_text(platform_root / "profiles/debian-base/vmlinux", "kernel-base\n") + _write_text(platform_root / "profiles/debian-base/rootfs.ext4", "rootfs-base\n") + _write_text(platform_root / "profiles/debian-git/vmlinux", "kernel-git\n") + _write_text(platform_root / "profiles/debian-git/rootfs.ext4", "rootfs-git\n") + _write_text(platform_root / "profiles/debian-build/vmlinux", "kernel-build\n") + _write_text(platform_root / "profiles/debian-build/rootfs.ext4", "rootfs-build\n") + lock = { + "bundle_version": "9.9.9", + "platform": "linux-x86_64", + "component_versions": { + "firecracker": "1.0.0", + "jailer": "1.0.0", + "kernel": "6.0.0", + "guest_agent": "0.2.0", + "base_distro": "debian-12", + }, + "capabilities": {"vm_boot": True, "guest_exec": True, "guest_network": True}, + "binaries": {"firecracker": "bin/firecracker", "jailer": "bin/jailer"}, + "guest": {"agent": {"path": "guest/pyro_guest_agent.py"}}, + "profiles": { + "debian-base": { + "description": "base", + "kernel": "profiles/debian-base/vmlinux", + "rootfs": "profiles/debian-base/rootfs.ext4", + }, + "debian-git": { + "description": "git", + "kernel": "profiles/debian-git/vmlinux", + "rootfs": "profiles/debian-git/rootfs.ext4", + }, + "debian-build": { + "description": "build", + "kernel": "profiles/debian-build/vmlinux", + "rootfs": "profiles/debian-build/rootfs.ext4", + }, + }, + } + (platform_root / "runtime.lock.json").write_text( + json.dumps(lock, indent=2) + "\n", encoding="utf-8" + ) + build_dir = tmp_path / "build/runtime_bundle" + bundle_dir = tmp_path / "bundle_out" + return source_dir, build_dir, bundle_dir + + +def test_runtime_build_stages_and_manifest(tmp_path: Path) -> None: + source_dir, build_dir, bundle_dir = _make_source_tree(tmp_path) + paths = _build_paths( + source_dir=source_dir, + build_dir=build_dir, + bundle_dir=bundle_dir, + materialized_dir=tmp_path / "materialized_sources", + platform="linux-x86_64", + ) + lock = _load_lock(paths) + + paths.build_platform_root.mkdir(parents=True, exist_ok=True) + stage_binaries(paths, lock) + stage_kernel(paths, lock) + stage_rootfs(paths, lock) + stage_agent(paths, lock) + manifest = generate_manifest(paths, lock) + + assert manifest["bundle_version"] == "9.9.9" + assert manifest["capabilities"]["guest_exec"] is True + assert manifest["component_versions"]["guest_agent"] == "0.2.0" + assert (paths.build_platform_root / "guest/pyro_guest_agent.py").exists() + + +def test_runtime_build_bundle_syncs_output(tmp_path: Path) -> None: + source_dir, build_dir, bundle_dir = _make_source_tree(tmp_path) + paths = _build_paths( + source_dir=source_dir, + build_dir=build_dir, + bundle_dir=bundle_dir, + materialized_dir=tmp_path / "materialized_sources", + platform="linux-x86_64", + ) + + manifest = build_bundle(paths, sync=True) + + assert manifest["profiles"]["debian-git"]["description"] == "git" + assert (bundle_dir / "NOTICE").exists() + assert (bundle_dir / "linux-x86_64/manifest.json").exists() + assert (bundle_dir / "linux-x86_64/guest/pyro_guest_agent.py").exists() + + +def test_runtime_build_rejects_guest_capabilities_for_placeholder_sources(tmp_path: Path) -> None: + source_dir, build_dir, bundle_dir = _make_source_tree(tmp_path) + paths = _build_paths( + source_dir=source_dir, + build_dir=build_dir, + bundle_dir=bundle_dir, + materialized_dir=tmp_path / "materialized_sources", + platform="linux-x86_64", + ) + lock_path = source_dir / "linux-x86_64/runtime.lock.json" + _write_text( + source_dir / "linux-x86_64/bin/firecracker", + "#!/usr/bin/env bash\n" + "echo 'bundled firecracker shim'\n", + ) + _write_text( + source_dir / "linux-x86_64/profiles/debian-base/rootfs.ext4", + "placeholder-rootfs\n", + ) + lock = json.loads(lock_path.read_text(encoding="utf-8")) + lock["capabilities"] = {"vm_boot": True, "guest_exec": True, "guest_network": True} + lock_path.write_text(json.dumps(lock, indent=2) + "\n", encoding="utf-8") + + with pytest.raises(RuntimeError, match="guest-capable features"): + validate_sources(paths, _load_lock(paths)) + + +def test_runtime_build_materializes_firecracker_release(tmp_path: Path) -> None: + source_dir, build_dir, bundle_dir = _make_source_tree(tmp_path) + archive_path = tmp_path / "firecracker-v1.12.1-x86_64.tgz" + release_dir = "release-v1.12.1-x86_64" + with tarfile.open(archive_path, "w:gz") as archive: + firecracker_path = tmp_path / "firecracker-bin" + jailer_path = tmp_path / "jailer-bin" + firecracker_path.write_text("real-firecracker\n", encoding="utf-8") + jailer_path.write_text("real-jailer\n", encoding="utf-8") + archive.add(firecracker_path, arcname=f"{release_dir}/firecracker-v1.12.1-x86_64") + archive.add(jailer_path, arcname=f"{release_dir}/jailer-v1.12.1-x86_64") + + digest = hashlib.sha256(archive_path.read_bytes()).hexdigest() + lock_path = source_dir / "linux-x86_64/runtime.lock.json" + lock = json.loads(lock_path.read_text(encoding="utf-8")) + lock["upstream"] = { + "firecracker_release": { + "archive_url": archive_path.resolve().as_uri(), + "archive_sha256": digest, + "firecracker_member": f"{release_dir}/firecracker-v1.12.1-x86_64", + "jailer_member": f"{release_dir}/jailer-v1.12.1-x86_64", + } + } + lock_path.write_text(json.dumps(lock, indent=2) + "\n", encoding="utf-8") + + paths = _build_paths( + source_dir=source_dir, + build_dir=build_dir, + bundle_dir=bundle_dir, + materialized_dir=tmp_path / "materialized_sources", + platform="linux-x86_64", + ) + materialize_binaries(paths, _load_lock(paths)) + + assert (paths.materialized_platform_root / "bin/firecracker").read_text(encoding="utf-8") == ( + "real-firecracker\n" + ) + assert (paths.materialized_platform_root / "bin/jailer").read_text(encoding="utf-8") == ( + "real-jailer\n" + ) diff --git a/tests/test_vm_firecracker.py b/tests/test_vm_firecracker.py index 7cbc22b..dc90906 100644 --- a/tests/test_vm_firecracker.py +++ b/tests/test_vm_firecracker.py @@ -47,5 +47,8 @@ def test_build_launch_plan_writes_expected_files(tmp_path: Path) -> None: rendered = json.loads(plan.config_path.read_text(encoding="utf-8")) assert rendered["machine-config"]["vcpu_count"] == 2 assert rendered["network-interfaces"][0]["host_dev_name"] == "pyroabcdef12" + assert "init=/opt/pyro/bin/pyro-init" in rendered["boot-source"]["boot_args"] + assert "pyro.guest_ip=172.29.100.2" in rendered["boot-source"]["boot_args"] + assert "pyro.gateway_ip=172.29.100.1" in rendered["boot-source"]["boot_args"] guest_exec = json.loads(plan.guest_exec_path.read_text(encoding="utf-8")) assert guest_exec["transport"] == "vsock"