Make concurrent workspaces easier to rediscover and resume without relying on opaque IDs alone. Add optional workspace names, key/value labels, workspace list, and workspace update across the CLI, Python SDK, and MCP surface, and persist last_activity_at so list ordering reflects real mutating activity. Update the stable contract, install/first-run docs, roadmap, and Python workspace example to teach the new discovery flow, and validate it with focused manager/CLI/API/server coverage plus uv lock, make check, make dist-check, and a real multi-workspace smoke for create, list, update, exec, reorder, and delete.
6317 lines
245 KiB
Python
6317 lines
245 KiB
Python
"""Lifecycle manager for ephemeral VM environments and persistent workspaces."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import difflib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shlex
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any, Literal, cast
|
|
|
|
from pyro_mcp.runtime import (
|
|
RuntimeCapabilities,
|
|
RuntimePaths,
|
|
resolve_runtime_paths,
|
|
runtime_capabilities,
|
|
)
|
|
from pyro_mcp.vm_environments import EnvironmentStore, default_cache_dir, get_environment
|
|
from pyro_mcp.vm_firecracker import build_launch_plan
|
|
from pyro_mcp.vm_guest import VsockExecClient
|
|
from pyro_mcp.vm_network import NetworkConfig, TapNetworkManager
|
|
from pyro_mcp.workspace_disk import (
|
|
export_workspace_disk_image,
|
|
list_workspace_disk,
|
|
read_workspace_disk_file,
|
|
scrub_workspace_runtime_paths,
|
|
)
|
|
from pyro_mcp.workspace_files import (
|
|
DEFAULT_WORKSPACE_FILE_READ_MAX_BYTES as DEFAULT_WORKSPACE_FILE_READ_LIMIT,
|
|
)
|
|
from pyro_mcp.workspace_files import (
|
|
WORKSPACE_FILE_MAX_BYTES as WORKSPACE_FILE_MAX_LIMIT,
|
|
)
|
|
from pyro_mcp.workspace_files import (
|
|
WORKSPACE_PATCH_MAX_BYTES as WORKSPACE_PATCH_MAX_LIMIT,
|
|
)
|
|
from pyro_mcp.workspace_files import (
|
|
WorkspaceTextPatch,
|
|
apply_unified_text_patch,
|
|
delete_workspace_path,
|
|
list_workspace_files,
|
|
normalize_workspace_path,
|
|
parse_unified_text_patch,
|
|
read_workspace_file,
|
|
write_workspace_file,
|
|
)
|
|
from pyro_mcp.workspace_ports import DEFAULT_PUBLISHED_PORT_HOST
|
|
from pyro_mcp.workspace_shells import (
|
|
create_local_shell,
|
|
get_local_shell,
|
|
remove_local_shell,
|
|
shell_signal_names,
|
|
)
|
|
|
|
VmState = Literal["created", "started", "stopped"]
|
|
WorkspaceShellState = Literal["running", "stopped"]
|
|
WorkspaceServiceState = Literal["running", "exited", "stopped", "failed"]
|
|
WorkspaceNetworkPolicy = Literal["off", "egress", "egress+published-ports"]
|
|
|
|
DEFAULT_VCPU_COUNT = 1
|
|
DEFAULT_MEM_MIB = 1024
|
|
DEFAULT_TIMEOUT_SECONDS = 30
|
|
DEFAULT_TTL_SECONDS = 600
|
|
DEFAULT_ALLOW_HOST_COMPAT = False
|
|
|
|
WORKSPACE_LAYOUT_VERSION = 8
|
|
WORKSPACE_BASELINE_DIRNAME = "baseline"
|
|
WORKSPACE_BASELINE_ARCHIVE_NAME = "workspace.tar"
|
|
WORKSPACE_SNAPSHOTS_DIRNAME = "snapshots"
|
|
WORKSPACE_DIRNAME = "workspace"
|
|
WORKSPACE_COMMANDS_DIRNAME = "commands"
|
|
WORKSPACE_SHELLS_DIRNAME = "shells"
|
|
WORKSPACE_SERVICES_DIRNAME = "services"
|
|
WORKSPACE_SECRETS_DIRNAME = "secrets"
|
|
WORKSPACE_RUNTIME_DIRNAME = "runtime"
|
|
WORKSPACE_GUEST_PATH = "/workspace"
|
|
WORKSPACE_GUEST_AGENT_PATH = "/opt/pyro/bin/pyro_guest_agent.py"
|
|
WORKSPACE_GUEST_INIT_PATH = "/opt/pyro/bin/pyro-init"
|
|
WORKSPACE_GUEST_SECRETS_PATH = "/run/pyro-secrets"
|
|
WORKSPACE_ARCHIVE_UPLOAD_TIMEOUT_SECONDS = 60
|
|
WORKSPACE_SECRET_MAX_BYTES = 64 * 1024
|
|
DEFAULT_SHELL_COLS = 120
|
|
DEFAULT_SHELL_ROWS = 30
|
|
DEFAULT_SHELL_MAX_CHARS = 65536
|
|
DEFAULT_WORKSPACE_DISK_READ_MAX_BYTES = 65536
|
|
DEFAULT_WORKSPACE_FILE_READ_MAX_BYTES = DEFAULT_WORKSPACE_FILE_READ_LIMIT
|
|
WORKSPACE_FILE_MAX_BYTES = WORKSPACE_FILE_MAX_LIMIT
|
|
WORKSPACE_PATCH_MAX_BYTES = WORKSPACE_PATCH_MAX_LIMIT
|
|
DEFAULT_SERVICE_READY_TIMEOUT_SECONDS = 30
|
|
DEFAULT_SERVICE_READY_INTERVAL_MS = 500
|
|
DEFAULT_SERVICE_LOG_TAIL_LINES = 200
|
|
DEFAULT_WORKSPACE_NETWORK_POLICY: WorkspaceNetworkPolicy = "off"
|
|
WORKSPACE_SHELL_SIGNAL_NAMES = shell_signal_names()
|
|
WORKSPACE_SERVICE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
|
|
WORKSPACE_SNAPSHOT_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
|
|
WORKSPACE_SECRET_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$")
|
|
WORKSPACE_LABEL_KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
|
|
|
|
WorkspaceSeedMode = Literal["empty", "directory", "tar_archive"]
|
|
WorkspaceArtifactType = Literal["file", "directory", "symlink"]
|
|
WorkspaceServiceReadinessType = Literal["file", "tcp", "http", "command"]
|
|
WorkspaceSnapshotKind = Literal["baseline", "named"]
|
|
WorkspaceSecretSourceKind = Literal["literal", "file"]
|
|
|
|
|
|
@dataclass
|
|
class VmInstance:
|
|
"""In-memory VM lifecycle record."""
|
|
|
|
vm_id: str
|
|
environment: str
|
|
vcpu_count: int
|
|
mem_mib: int
|
|
ttl_seconds: int
|
|
created_at: float
|
|
expires_at: float
|
|
workdir: Path
|
|
state: VmState = "created"
|
|
network_requested: bool = False
|
|
allow_host_compat: bool = DEFAULT_ALLOW_HOST_COMPAT
|
|
firecracker_pid: int | None = None
|
|
last_error: str | None = None
|
|
metadata: dict[str, str] = field(default_factory=dict)
|
|
network: NetworkConfig | None = None
|
|
|
|
|
|
@dataclass
|
|
class WorkspaceRecord:
|
|
"""Persistent workspace metadata stored on disk."""
|
|
|
|
workspace_id: str
|
|
environment: str
|
|
vcpu_count: int
|
|
mem_mib: int
|
|
ttl_seconds: int
|
|
created_at: float
|
|
expires_at: float
|
|
state: VmState
|
|
network_policy: WorkspaceNetworkPolicy
|
|
allow_host_compat: bool
|
|
firecracker_pid: int | None = None
|
|
last_error: str | None = None
|
|
metadata: dict[str, str] = field(default_factory=dict)
|
|
network: NetworkConfig | None = None
|
|
name: str | None = None
|
|
labels: dict[str, str] = field(default_factory=dict)
|
|
last_activity_at: float = 0.0
|
|
command_count: int = 0
|
|
last_command: dict[str, Any] | None = None
|
|
workspace_seed: dict[str, Any] = field(default_factory=dict)
|
|
secrets: list[WorkspaceSecretRecord] = field(default_factory=list)
|
|
reset_count: int = 0
|
|
last_reset_at: float | None = None
|
|
|
|
@classmethod
|
|
def from_instance(
|
|
cls,
|
|
instance: VmInstance,
|
|
*,
|
|
network_policy: WorkspaceNetworkPolicy = DEFAULT_WORKSPACE_NETWORK_POLICY,
|
|
command_count: int = 0,
|
|
last_command: dict[str, Any] | None = None,
|
|
workspace_seed: dict[str, Any] | None = None,
|
|
secrets: list[WorkspaceSecretRecord] | None = None,
|
|
name: str | None = None,
|
|
labels: dict[str, str] | None = None,
|
|
) -> WorkspaceRecord:
|
|
return cls(
|
|
workspace_id=instance.vm_id,
|
|
environment=instance.environment,
|
|
vcpu_count=instance.vcpu_count,
|
|
mem_mib=instance.mem_mib,
|
|
ttl_seconds=instance.ttl_seconds,
|
|
created_at=instance.created_at,
|
|
expires_at=instance.expires_at,
|
|
state=instance.state,
|
|
network_policy=network_policy,
|
|
allow_host_compat=instance.allow_host_compat,
|
|
firecracker_pid=instance.firecracker_pid,
|
|
last_error=instance.last_error,
|
|
metadata=dict(instance.metadata),
|
|
network=instance.network,
|
|
name=name,
|
|
labels=dict(labels or {}),
|
|
last_activity_at=instance.created_at,
|
|
command_count=command_count,
|
|
last_command=last_command,
|
|
workspace_seed=dict(workspace_seed or _empty_workspace_seed_payload()),
|
|
secrets=list(secrets or []),
|
|
reset_count=0,
|
|
last_reset_at=None,
|
|
)
|
|
|
|
def to_instance(self, *, workdir: Path) -> VmInstance:
|
|
return VmInstance(
|
|
vm_id=self.workspace_id,
|
|
environment=self.environment,
|
|
vcpu_count=self.vcpu_count,
|
|
mem_mib=self.mem_mib,
|
|
ttl_seconds=self.ttl_seconds,
|
|
created_at=self.created_at,
|
|
expires_at=self.expires_at,
|
|
workdir=workdir,
|
|
state=self.state,
|
|
network_requested=self.network_policy != "off",
|
|
allow_host_compat=self.allow_host_compat,
|
|
firecracker_pid=self.firecracker_pid,
|
|
last_error=self.last_error,
|
|
metadata=dict(self.metadata),
|
|
network=self.network,
|
|
)
|
|
|
|
def to_payload(self) -> dict[str, Any]:
|
|
return {
|
|
"layout_version": WORKSPACE_LAYOUT_VERSION,
|
|
"workspace_id": self.workspace_id,
|
|
"environment": self.environment,
|
|
"vcpu_count": self.vcpu_count,
|
|
"mem_mib": self.mem_mib,
|
|
"ttl_seconds": self.ttl_seconds,
|
|
"created_at": self.created_at,
|
|
"expires_at": self.expires_at,
|
|
"state": self.state,
|
|
"network_policy": self.network_policy,
|
|
"allow_host_compat": self.allow_host_compat,
|
|
"firecracker_pid": self.firecracker_pid,
|
|
"last_error": self.last_error,
|
|
"metadata": self.metadata,
|
|
"network": _serialize_network(self.network),
|
|
"name": self.name,
|
|
"labels": self.labels,
|
|
"last_activity_at": self.last_activity_at,
|
|
"command_count": self.command_count,
|
|
"last_command": self.last_command,
|
|
"workspace_seed": self.workspace_seed,
|
|
"secrets": [secret.to_payload() for secret in self.secrets],
|
|
"reset_count": self.reset_count,
|
|
"last_reset_at": self.last_reset_at,
|
|
}
|
|
|
|
@classmethod
|
|
def from_payload(cls, payload: dict[str, Any]) -> WorkspaceRecord:
|
|
return cls(
|
|
workspace_id=str(payload["workspace_id"]),
|
|
environment=str(payload["environment"]),
|
|
vcpu_count=int(payload["vcpu_count"]),
|
|
mem_mib=int(payload["mem_mib"]),
|
|
ttl_seconds=int(payload["ttl_seconds"]),
|
|
created_at=float(payload["created_at"]),
|
|
expires_at=float(payload["expires_at"]),
|
|
state=cast(VmState, str(payload.get("state", "stopped"))),
|
|
network_policy=_workspace_network_policy_from_payload(payload),
|
|
allow_host_compat=bool(payload.get("allow_host_compat", DEFAULT_ALLOW_HOST_COMPAT)),
|
|
firecracker_pid=_optional_int(payload.get("firecracker_pid")),
|
|
last_error=_optional_str(payload.get("last_error")),
|
|
metadata=_string_dict(payload.get("metadata")),
|
|
network=_deserialize_network(payload.get("network")),
|
|
name=_normalize_workspace_name(_optional_str(payload.get("name")), allow_none=True),
|
|
labels=_normalize_workspace_labels(payload.get("labels")),
|
|
last_activity_at=float(
|
|
payload.get("last_activity_at", float(payload["created_at"]))
|
|
),
|
|
command_count=int(payload.get("command_count", 0)),
|
|
last_command=_optional_dict(payload.get("last_command")),
|
|
workspace_seed=_workspace_seed_dict(payload.get("workspace_seed")),
|
|
secrets=_workspace_secret_records(payload.get("secrets")),
|
|
reset_count=int(payload.get("reset_count", 0)),
|
|
last_reset_at=(
|
|
None
|
|
if payload.get("last_reset_at") is None
|
|
else float(payload.get("last_reset_at", 0.0))
|
|
),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WorkspaceSecretRecord:
|
|
"""Persistent secret metadata stored on disk per workspace."""
|
|
|
|
name: str
|
|
source_kind: WorkspaceSecretSourceKind
|
|
stored_path: str
|
|
|
|
def to_payload(self) -> dict[str, Any]:
|
|
return {
|
|
"name": self.name,
|
|
"source_kind": self.source_kind,
|
|
"stored_path": self.stored_path,
|
|
}
|
|
|
|
@classmethod
|
|
def from_payload(cls, payload: dict[str, Any]) -> WorkspaceSecretRecord:
|
|
return cls(
|
|
name=str(payload["name"]),
|
|
source_kind=cast(WorkspaceSecretSourceKind, str(payload.get("source_kind", "literal"))),
|
|
stored_path=str(payload["stored_path"]),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class WorkspaceSnapshotRecord:
|
|
"""Persistent snapshot metadata stored on disk per workspace."""
|
|
|
|
workspace_id: str
|
|
snapshot_name: str
|
|
kind: WorkspaceSnapshotKind
|
|
created_at: float
|
|
entry_count: int
|
|
bytes_written: int
|
|
|
|
def to_payload(self) -> dict[str, Any]:
|
|
return {
|
|
"layout_version": WORKSPACE_LAYOUT_VERSION,
|
|
"workspace_id": self.workspace_id,
|
|
"snapshot_name": self.snapshot_name,
|
|
"kind": self.kind,
|
|
"created_at": self.created_at,
|
|
"entry_count": self.entry_count,
|
|
"bytes_written": self.bytes_written,
|
|
}
|
|
|
|
@classmethod
|
|
def from_payload(cls, payload: dict[str, Any]) -> WorkspaceSnapshotRecord:
|
|
return cls(
|
|
workspace_id=str(payload["workspace_id"]),
|
|
snapshot_name=str(payload["snapshot_name"]),
|
|
kind=cast(WorkspaceSnapshotKind, str(payload.get("kind", "named"))),
|
|
created_at=float(payload["created_at"]),
|
|
entry_count=int(payload.get("entry_count", 0)),
|
|
bytes_written=int(payload.get("bytes_written", 0)),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class WorkspaceShellRecord:
|
|
"""Persistent shell metadata stored on disk per workspace."""
|
|
|
|
workspace_id: str
|
|
shell_id: str
|
|
cwd: str
|
|
cols: int
|
|
rows: int
|
|
state: WorkspaceShellState
|
|
started_at: float
|
|
ended_at: float | None = None
|
|
exit_code: int | None = None
|
|
execution_mode: str = "pending"
|
|
metadata: dict[str, str] = field(default_factory=dict)
|
|
|
|
def to_payload(self) -> dict[str, Any]:
|
|
return {
|
|
"layout_version": WORKSPACE_LAYOUT_VERSION,
|
|
"workspace_id": self.workspace_id,
|
|
"shell_id": self.shell_id,
|
|
"cwd": self.cwd,
|
|
"cols": self.cols,
|
|
"rows": self.rows,
|
|
"state": self.state,
|
|
"started_at": self.started_at,
|
|
"ended_at": self.ended_at,
|
|
"exit_code": self.exit_code,
|
|
"execution_mode": self.execution_mode,
|
|
"metadata": dict(self.metadata),
|
|
}
|
|
|
|
@classmethod
|
|
def from_payload(cls, payload: dict[str, Any]) -> WorkspaceShellRecord:
|
|
return cls(
|
|
workspace_id=str(payload["workspace_id"]),
|
|
shell_id=str(payload["shell_id"]),
|
|
cwd=str(payload.get("cwd", WORKSPACE_GUEST_PATH)),
|
|
cols=int(payload.get("cols", DEFAULT_SHELL_COLS)),
|
|
rows=int(payload.get("rows", DEFAULT_SHELL_ROWS)),
|
|
state=cast(WorkspaceShellState, str(payload.get("state", "stopped"))),
|
|
started_at=float(payload.get("started_at", 0.0)),
|
|
ended_at=(
|
|
None if payload.get("ended_at") is None else float(payload.get("ended_at", 0.0))
|
|
),
|
|
exit_code=(
|
|
None if payload.get("exit_code") is None else int(payload.get("exit_code", 0))
|
|
),
|
|
execution_mode=str(payload.get("execution_mode", "pending")),
|
|
metadata=_string_dict(payload.get("metadata")),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class WorkspaceServiceRecord:
|
|
"""Persistent service metadata stored on disk per workspace."""
|
|
|
|
workspace_id: str
|
|
service_name: str
|
|
command: str
|
|
cwd: str
|
|
state: WorkspaceServiceState
|
|
started_at: float
|
|
readiness: dict[str, Any] | None = None
|
|
ready_at: float | None = None
|
|
ended_at: float | None = None
|
|
exit_code: int | None = None
|
|
pid: int | None = None
|
|
execution_mode: str = "pending"
|
|
stop_reason: str | None = None
|
|
published_ports: list[WorkspacePublishedPortRecord] = field(default_factory=list)
|
|
metadata: dict[str, str] = field(default_factory=dict)
|
|
|
|
def to_payload(self) -> dict[str, Any]:
|
|
return {
|
|
"layout_version": WORKSPACE_LAYOUT_VERSION,
|
|
"workspace_id": self.workspace_id,
|
|
"service_name": self.service_name,
|
|
"command": self.command,
|
|
"cwd": self.cwd,
|
|
"state": self.state,
|
|
"started_at": self.started_at,
|
|
"readiness": self.readiness,
|
|
"ready_at": self.ready_at,
|
|
"ended_at": self.ended_at,
|
|
"exit_code": self.exit_code,
|
|
"pid": self.pid,
|
|
"execution_mode": self.execution_mode,
|
|
"stop_reason": self.stop_reason,
|
|
"published_ports": [
|
|
published_port.to_payload() for published_port in self.published_ports
|
|
],
|
|
"metadata": dict(self.metadata),
|
|
}
|
|
|
|
@classmethod
|
|
def from_payload(cls, payload: dict[str, Any]) -> WorkspaceServiceRecord:
|
|
readiness_payload = payload.get("readiness")
|
|
readiness = None
|
|
if isinstance(readiness_payload, dict):
|
|
readiness = dict(readiness_payload)
|
|
return cls(
|
|
workspace_id=str(payload["workspace_id"]),
|
|
service_name=str(payload["service_name"]),
|
|
command=str(payload.get("command", "")),
|
|
cwd=str(payload.get("cwd", WORKSPACE_GUEST_PATH)),
|
|
state=cast(WorkspaceServiceState, str(payload.get("state", "stopped"))),
|
|
started_at=float(payload.get("started_at", 0.0)),
|
|
readiness=readiness,
|
|
ready_at=(
|
|
None if payload.get("ready_at") is None else float(payload.get("ready_at", 0.0))
|
|
),
|
|
ended_at=(
|
|
None if payload.get("ended_at") is None else float(payload.get("ended_at", 0.0))
|
|
),
|
|
exit_code=(
|
|
None if payload.get("exit_code") is None else int(payload.get("exit_code", 0))
|
|
),
|
|
pid=None if payload.get("pid") is None else int(payload.get("pid", 0)),
|
|
execution_mode=str(payload.get("execution_mode", "pending")),
|
|
stop_reason=_optional_str(payload.get("stop_reason")),
|
|
published_ports=_workspace_published_port_records(payload.get("published_ports")),
|
|
metadata=_string_dict(payload.get("metadata")),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WorkspacePublishedPortRecord:
|
|
"""Persisted localhost published-port metadata for one service."""
|
|
|
|
guest_port: int
|
|
host_port: int
|
|
host: str = DEFAULT_PUBLISHED_PORT_HOST
|
|
protocol: str = "tcp"
|
|
proxy_pid: int | None = None
|
|
|
|
def to_payload(self) -> dict[str, Any]:
|
|
return {
|
|
"guest_port": self.guest_port,
|
|
"host_port": self.host_port,
|
|
"host": self.host,
|
|
"protocol": self.protocol,
|
|
"proxy_pid": self.proxy_pid,
|
|
}
|
|
|
|
@classmethod
|
|
def from_payload(cls, payload: dict[str, Any]) -> WorkspacePublishedPortRecord:
|
|
return cls(
|
|
guest_port=int(payload["guest_port"]),
|
|
host_port=int(payload["host_port"]),
|
|
host=str(payload.get("host", DEFAULT_PUBLISHED_PORT_HOST)),
|
|
protocol=str(payload.get("protocol", "tcp")),
|
|
proxy_pid=(
|
|
None
|
|
if payload.get("proxy_pid") is None
|
|
else int(payload.get("proxy_pid", 0))
|
|
),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WorkspacePublishedPortSpec:
|
|
"""Requested published-port configuration for one service."""
|
|
|
|
guest_port: int
|
|
host_port: int | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PreparedWorkspaceSeed:
|
|
"""Prepared host-side seed archive plus metadata."""
|
|
|
|
mode: WorkspaceSeedMode
|
|
source_path: str | None
|
|
archive_path: Path | None = None
|
|
entry_count: int = 0
|
|
bytes_written: int = 0
|
|
cleanup_dir: Path | None = None
|
|
|
|
def to_payload(
|
|
self,
|
|
*,
|
|
destination: str = WORKSPACE_GUEST_PATH,
|
|
path_key: str = "seed_path",
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"mode": self.mode,
|
|
path_key: self.source_path,
|
|
"destination": destination,
|
|
"entry_count": self.entry_count,
|
|
"bytes_written": self.bytes_written,
|
|
}
|
|
|
|
def cleanup(self) -> None:
|
|
if self.cleanup_dir is not None:
|
|
shutil.rmtree(self.cleanup_dir, ignore_errors=True)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VmExecResult:
|
|
"""Command execution output."""
|
|
|
|
stdout: str
|
|
stderr: str
|
|
exit_code: int
|
|
duration_ms: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ExportedWorkspaceArchive:
|
|
workspace_path: str
|
|
artifact_type: WorkspaceArtifactType
|
|
archive_path: Path
|
|
entry_count: int
|
|
bytes_written: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class WorkspaceTreeEntry:
|
|
path: str
|
|
artifact_type: WorkspaceArtifactType
|
|
disk_path: Path
|
|
size_bytes: int = 0
|
|
link_target: str | None = None
|
|
|
|
|
|
def _optional_int(value: object) -> int | None:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, bool):
|
|
return int(value)
|
|
if isinstance(value, int):
|
|
return value
|
|
if isinstance(value, float):
|
|
return int(value)
|
|
if isinstance(value, str):
|
|
return int(value)
|
|
raise TypeError("expected integer-compatible payload")
|
|
|
|
|
|
def _optional_str(value: object) -> str | None:
|
|
if value is None:
|
|
return None
|
|
return str(value)
|
|
|
|
|
|
def _optional_dict(value: object) -> dict[str, Any] | None:
|
|
if value is None:
|
|
return None
|
|
if not isinstance(value, dict):
|
|
raise TypeError("expected dictionary payload")
|
|
return dict(value)
|
|
|
|
|
|
def _string_dict(value: object) -> dict[str, str]:
|
|
if not isinstance(value, dict):
|
|
return {}
|
|
return {str(key): str(item) for key, item in value.items()}
|
|
|
|
|
|
def _empty_workspace_seed_payload() -> dict[str, Any]:
|
|
return {
|
|
"mode": "empty",
|
|
"seed_path": None,
|
|
"destination": WORKSPACE_GUEST_PATH,
|
|
"entry_count": 0,
|
|
"bytes_written": 0,
|
|
}
|
|
|
|
|
|
def _workspace_seed_dict(value: object) -> dict[str, Any]:
|
|
if not isinstance(value, dict):
|
|
return _empty_workspace_seed_payload()
|
|
payload = _empty_workspace_seed_payload()
|
|
payload.update(
|
|
{
|
|
"mode": str(value.get("mode", payload["mode"])),
|
|
"seed_path": _optional_str(value.get("seed_path")),
|
|
"destination": str(value.get("destination", payload["destination"])),
|
|
"entry_count": int(value.get("entry_count", payload["entry_count"])),
|
|
"bytes_written": int(value.get("bytes_written", payload["bytes_written"])),
|
|
}
|
|
)
|
|
return payload
|
|
|
|
|
|
def _normalize_workspace_network_policy(policy: str) -> WorkspaceNetworkPolicy:
|
|
normalized = policy.strip().lower()
|
|
if normalized not in {"off", "egress", "egress+published-ports"}:
|
|
raise ValueError("network_policy must be one of: off, egress, egress+published-ports")
|
|
return cast(WorkspaceNetworkPolicy, normalized)
|
|
|
|
|
|
def _workspace_network_policy_from_payload(payload: dict[str, Any]) -> WorkspaceNetworkPolicy:
|
|
raw_policy = payload.get("network_policy")
|
|
if raw_policy is not None:
|
|
return _normalize_workspace_network_policy(str(raw_policy))
|
|
raw_network_requested = payload.get("network_requested", False)
|
|
if isinstance(raw_network_requested, str):
|
|
network_requested = raw_network_requested.strip().lower() in {"1", "true", "yes", "on"}
|
|
else:
|
|
network_requested = bool(raw_network_requested)
|
|
if network_requested:
|
|
return "egress"
|
|
return DEFAULT_WORKSPACE_NETWORK_POLICY
|
|
|
|
|
|
def _serialize_workspace_published_port_public(
|
|
published_port: WorkspacePublishedPortRecord,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"host": published_port.host,
|
|
"host_port": published_port.host_port,
|
|
"guest_port": published_port.guest_port,
|
|
"protocol": published_port.protocol,
|
|
}
|
|
|
|
|
|
def _workspace_published_port_records(value: object) -> list[WorkspacePublishedPortRecord]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
records: list[WorkspacePublishedPortRecord] = []
|
|
for item in value:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
records.append(WorkspacePublishedPortRecord.from_payload(item))
|
|
return records
|
|
|
|
|
|
def _workspace_secret_records(value: object) -> list[WorkspaceSecretRecord]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
records: list[WorkspaceSecretRecord] = []
|
|
for item in value:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
records.append(WorkspaceSecretRecord.from_payload(item))
|
|
return records
|
|
|
|
|
|
def _serialize_workspace_secret_public(secret: WorkspaceSecretRecord) -> dict[str, Any]:
|
|
return {
|
|
"name": secret.name,
|
|
"source_kind": secret.source_kind,
|
|
}
|
|
|
|
|
|
def _redact_text(text: str, secret_values: list[str]) -> str:
|
|
redacted = text
|
|
for secret_value in sorted(
|
|
{value for value in secret_values if value != ""},
|
|
key=len,
|
|
reverse=True,
|
|
):
|
|
redacted = redacted.replace(secret_value, "[REDACTED]")
|
|
return redacted
|
|
|
|
|
|
def _redact_exception(exc: Exception, secret_values: list[str]) -> Exception:
|
|
redacted_message = _redact_text(str(exc), secret_values)
|
|
if redacted_message == str(exc):
|
|
return exc
|
|
return exc.__class__(redacted_message)
|
|
|
|
|
|
def _serialize_network(network: NetworkConfig | None) -> dict[str, Any] | None:
|
|
if network is None:
|
|
return None
|
|
return {
|
|
"vm_id": network.vm_id,
|
|
"tap_name": network.tap_name,
|
|
"guest_ip": network.guest_ip,
|
|
"gateway_ip": network.gateway_ip,
|
|
"subnet_cidr": network.subnet_cidr,
|
|
"mac_address": network.mac_address,
|
|
"dns_servers": list(network.dns_servers),
|
|
}
|
|
|
|
|
|
def _deserialize_network(payload: object) -> NetworkConfig | None:
|
|
if payload is None:
|
|
return None
|
|
if not isinstance(payload, dict):
|
|
raise TypeError("expected dictionary payload")
|
|
dns_servers = payload.get("dns_servers", [])
|
|
dns_values = tuple(str(item) for item in dns_servers) if isinstance(dns_servers, list) else ()
|
|
return NetworkConfig(
|
|
vm_id=str(payload["vm_id"]),
|
|
tap_name=str(payload["tap_name"]),
|
|
guest_ip=str(payload["guest_ip"]),
|
|
gateway_ip=str(payload["gateway_ip"]),
|
|
subnet_cidr=str(payload["subnet_cidr"]),
|
|
mac_address=str(payload["mac_address"]),
|
|
dns_servers=dns_values,
|
|
)
|
|
|
|
|
|
def _run_host_command(
|
|
workdir: Path,
|
|
command: str,
|
|
timeout_seconds: int,
|
|
*,
|
|
env_overrides: dict[str, str] | None = None,
|
|
) -> VmExecResult:
|
|
started = time.monotonic()
|
|
env = {"PATH": os.environ.get("PATH", ""), "HOME": str(workdir)}
|
|
if env_overrides is not None:
|
|
env.update(env_overrides)
|
|
try:
|
|
proc = subprocess.run( # noqa: S603
|
|
["bash", "-lc", command], # noqa: S607
|
|
cwd=workdir,
|
|
env=env,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=timeout_seconds,
|
|
check=False,
|
|
)
|
|
return VmExecResult(
|
|
stdout=proc.stdout,
|
|
stderr=proc.stderr,
|
|
exit_code=proc.returncode,
|
|
duration_ms=int((time.monotonic() - started) * 1000),
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
return VmExecResult(
|
|
stdout="",
|
|
stderr=f"command timed out after {timeout_seconds}s",
|
|
exit_code=124,
|
|
duration_ms=int((time.monotonic() - started) * 1000),
|
|
)
|
|
|
|
|
|
def _copy_rootfs(source: Path, dest: Path) -> str:
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
proc = subprocess.run( # noqa: S603
|
|
["cp", "--reflink=auto", str(source), str(dest)],
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
if proc.returncode == 0:
|
|
return "reflink_or_copy"
|
|
except OSError:
|
|
pass
|
|
shutil.copy2(source, dest)
|
|
return "copy2"
|
|
|
|
|
|
def _wrap_guest_command(command: str, *, cwd: str | None = None) -> str:
|
|
if cwd is None:
|
|
return command
|
|
quoted_cwd = shlex.quote(cwd)
|
|
return f"mkdir -p {quoted_cwd} && cd {quoted_cwd} && {command}"
|
|
|
|
|
|
def _is_supported_seed_archive(path: Path) -> bool:
|
|
name = path.name.lower()
|
|
return name.endswith(".tar") or name.endswith(".tar.gz") or name.endswith(".tgz")
|
|
|
|
|
|
def _normalize_workspace_destination(destination: str) -> tuple[str, PurePosixPath]:
|
|
candidate = destination.strip()
|
|
if candidate == "":
|
|
raise ValueError("workspace destination must not be empty")
|
|
destination_path = PurePosixPath(candidate)
|
|
if any(part == ".." for part in destination_path.parts):
|
|
raise ValueError("workspace destination must stay inside /workspace")
|
|
workspace_root = PurePosixPath(WORKSPACE_GUEST_PATH)
|
|
if not destination_path.is_absolute():
|
|
destination_path = workspace_root / destination_path
|
|
parts = [part for part in destination_path.parts if part not in {"", "."}]
|
|
normalized = PurePosixPath("/") / PurePosixPath(*parts)
|
|
if normalized == PurePosixPath("/"):
|
|
raise ValueError("workspace destination must stay inside /workspace")
|
|
if normalized.parts[: len(workspace_root.parts)] != workspace_root.parts:
|
|
raise ValueError("workspace destination must stay inside /workspace")
|
|
suffix = normalized.relative_to(workspace_root)
|
|
return str(normalized), suffix
|
|
|
|
|
|
def _workspace_host_destination(workspace_dir: Path, destination: str) -> Path:
|
|
_, suffix = _normalize_workspace_destination(destination)
|
|
if str(suffix) in {"", "."}:
|
|
return workspace_dir
|
|
return workspace_dir.joinpath(*suffix.parts)
|
|
|
|
|
|
def _normalize_workspace_disk_path(path: str) -> str:
|
|
candidate = path.strip()
|
|
if candidate == "":
|
|
raise ValueError("workspace disk path must not be empty")
|
|
if candidate.startswith("/"):
|
|
raw_path = PurePosixPath(candidate)
|
|
normalized_parts: list[str] = []
|
|
for part in raw_path.parts:
|
|
if part in {"", "/", "."}:
|
|
continue
|
|
if part == "..":
|
|
if normalized_parts:
|
|
normalized_parts.pop()
|
|
continue
|
|
normalized_parts.append(part)
|
|
if not normalized_parts:
|
|
return "/"
|
|
return str(PurePosixPath("/") / PurePosixPath(*normalized_parts))
|
|
normalized, _ = _normalize_workspace_destination(candidate)
|
|
return normalized
|
|
|
|
|
|
def _normalize_workspace_file_path(path: str) -> str:
|
|
return normalize_workspace_path(path)
|
|
|
|
|
|
def _validate_workspace_file_read_max_bytes(max_bytes: int) -> int:
|
|
if max_bytes <= 0:
|
|
raise ValueError("max_bytes must be positive")
|
|
if max_bytes > WORKSPACE_FILE_MAX_BYTES:
|
|
raise ValueError(
|
|
f"max_bytes must be at most {WORKSPACE_FILE_MAX_BYTES} bytes"
|
|
)
|
|
return max_bytes
|
|
|
|
|
|
def _validate_workspace_text_payload(text: str, *, field_name: str) -> str:
|
|
encoded = text.encode("utf-8")
|
|
if len(encoded) > WORKSPACE_FILE_MAX_BYTES:
|
|
raise ValueError(
|
|
f"{field_name} must be at most {WORKSPACE_FILE_MAX_BYTES} bytes when encoded as UTF-8"
|
|
)
|
|
return text
|
|
|
|
|
|
def _validate_workspace_patch_text(patch: str) -> str:
|
|
if patch.strip() == "":
|
|
raise ValueError("patch must not be empty")
|
|
encoded = patch.encode("utf-8")
|
|
if len(encoded) > WORKSPACE_PATCH_MAX_BYTES:
|
|
raise ValueError(
|
|
f"patch must be at most {WORKSPACE_PATCH_MAX_BYTES} bytes when encoded as UTF-8"
|
|
)
|
|
return patch
|
|
|
|
|
|
def _decode_workspace_patch_text(path: str, content_bytes: bytes) -> str:
|
|
try:
|
|
return content_bytes.decode("utf-8")
|
|
except UnicodeDecodeError as exc:
|
|
raise RuntimeError(
|
|
f"workspace patch only supports UTF-8 text files: {path}"
|
|
) from exc
|
|
|
|
|
|
def _normalize_archive_member_name(name: str) -> PurePosixPath:
|
|
candidate = name.strip()
|
|
if candidate == "":
|
|
raise RuntimeError("archive member path is empty")
|
|
member_path = PurePosixPath(candidate)
|
|
if member_path.is_absolute():
|
|
raise RuntimeError(f"absolute archive member paths are not allowed: {name}")
|
|
parts = [part for part in member_path.parts if part not in {"", "."}]
|
|
if any(part == ".." for part in parts):
|
|
raise RuntimeError(f"unsafe archive member path: {name}")
|
|
normalized = PurePosixPath(*parts)
|
|
if str(normalized) in {"", "."}:
|
|
raise RuntimeError(f"unsafe archive member path: {name}")
|
|
return normalized
|
|
|
|
|
|
def _validate_archive_symlink_target(member_name: PurePosixPath, link_target: str) -> None:
|
|
target = link_target.strip()
|
|
if target == "":
|
|
raise RuntimeError(f"symlink {member_name} has an empty target")
|
|
link_path = PurePosixPath(target)
|
|
if link_path.is_absolute():
|
|
raise RuntimeError(f"symlink {member_name} escapes the workspace")
|
|
combined = member_name.parent.joinpath(link_path)
|
|
parts = [part for part in combined.parts if part not in {"", "."}]
|
|
if any(part == ".." for part in parts):
|
|
raise RuntimeError(f"symlink {member_name} escapes the workspace")
|
|
|
|
|
|
def _inspect_seed_archive(archive_path: Path) -> tuple[int, int]:
|
|
entry_count = 0
|
|
bytes_written = 0
|
|
with tarfile.open(archive_path, "r:*") as archive:
|
|
for member in archive.getmembers():
|
|
member_name = _normalize_archive_member_name(member.name)
|
|
entry_count += 1
|
|
if member.isdir():
|
|
continue
|
|
if member.isfile():
|
|
bytes_written += member.size
|
|
continue
|
|
if member.issym():
|
|
_validate_archive_symlink_target(member_name, member.linkname)
|
|
continue
|
|
if member.islnk():
|
|
raise RuntimeError(
|
|
f"hard links are not allowed in workspace archives: {member.name}"
|
|
)
|
|
raise RuntimeError(f"unsupported archive member type: {member.name}")
|
|
return entry_count, bytes_written
|
|
|
|
|
|
def _write_directory_seed_archive(source_dir: Path, archive_path: Path) -> None:
|
|
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with tarfile.open(archive_path, "w") as archive:
|
|
for child in sorted(source_dir.iterdir(), key=lambda item: item.name):
|
|
archive.add(child, arcname=child.name, recursive=True)
|
|
|
|
|
|
def _write_empty_seed_archive(archive_path: Path) -> None:
|
|
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with tarfile.open(archive_path, "w"):
|
|
pass
|
|
|
|
|
|
def _prepare_workspace_secrets(
|
|
secrets: list[dict[str, str]] | None,
|
|
*,
|
|
secrets_dir: Path,
|
|
) -> tuple[list[WorkspaceSecretRecord], dict[str, str]]:
|
|
if not secrets:
|
|
return [], {}
|
|
secrets_dir.mkdir(parents=True, exist_ok=True)
|
|
records: list[WorkspaceSecretRecord] = []
|
|
values_by_name: dict[str, str] = {}
|
|
for index, item in enumerate(secrets, start=1):
|
|
if not isinstance(item, dict):
|
|
raise ValueError(f"secret #{index} must be a dictionary")
|
|
raw_name = item.get("name")
|
|
if raw_name is None:
|
|
raise ValueError(f"secret #{index} is missing 'name'")
|
|
name = _normalize_workspace_secret_name(str(raw_name))
|
|
if name in values_by_name:
|
|
raise ValueError(f"duplicate secret name: {name}")
|
|
has_value = "value" in item
|
|
has_file_path = "file_path" in item
|
|
if has_value == has_file_path:
|
|
raise ValueError(
|
|
f"secret {name!r} must provide exactly one of 'value' or 'file_path'"
|
|
)
|
|
source_kind: WorkspaceSecretSourceKind
|
|
if has_value:
|
|
value = _validate_workspace_secret_value(name, str(item["value"]))
|
|
source_kind = "literal"
|
|
else:
|
|
raw_file_path = str(item["file_path"]).strip()
|
|
if raw_file_path == "":
|
|
raise ValueError(f"secret {name!r} file_path must not be empty")
|
|
resolved_file_path = Path(raw_file_path).expanduser().resolve()
|
|
if not resolved_file_path.exists() or not resolved_file_path.is_file():
|
|
raise ValueError(f"secret file for {name!r} does not exist: {resolved_file_path}")
|
|
try:
|
|
raw_bytes = resolved_file_path.read_bytes()
|
|
except OSError as exc:
|
|
raise ValueError(
|
|
f"failed to read secret file for {name!r}: {resolved_file_path}"
|
|
) from exc
|
|
if len(raw_bytes) > WORKSPACE_SECRET_MAX_BYTES:
|
|
raise ValueError(
|
|
f"secret {name!r} must be at most {WORKSPACE_SECRET_MAX_BYTES} bytes"
|
|
)
|
|
try:
|
|
value = raw_bytes.decode("utf-8")
|
|
except UnicodeDecodeError as exc:
|
|
raise ValueError(f"secret {name!r} must be valid UTF-8 text") from exc
|
|
value = _validate_workspace_secret_value(name, value)
|
|
source_kind = "file"
|
|
stored_path = f"{name}.secret"
|
|
secret_path = secrets_dir / stored_path
|
|
secret_path.write_text(value, encoding="utf-8")
|
|
secret_path.chmod(0o600)
|
|
values_by_name[name] = value
|
|
records.append(
|
|
WorkspaceSecretRecord(
|
|
name=name,
|
|
source_kind=source_kind,
|
|
stored_path=stored_path,
|
|
)
|
|
)
|
|
secrets_dir.chmod(0o700)
|
|
records.sort(key=lambda item: item.name)
|
|
return records, {record.name: values_by_name[record.name] for record in records}
|
|
|
|
|
|
def _load_workspace_secret_values(
|
|
*,
|
|
workspace_dir: Path,
|
|
secrets: list[WorkspaceSecretRecord],
|
|
) -> dict[str, str]:
|
|
values: dict[str, str] = {}
|
|
for secret in secrets:
|
|
secret_path = workspace_dir / WORKSPACE_SECRETS_DIRNAME / secret.stored_path
|
|
if not secret_path.exists() or not secret_path.is_file():
|
|
raise RuntimeError(f"secret material is unavailable for {secret.name!r}")
|
|
values[secret.name] = secret_path.read_text(encoding="utf-8")
|
|
return values
|
|
|
|
|
|
def _build_workspace_secret_archive(
|
|
*,
|
|
workspace_dir: Path,
|
|
secrets: list[WorkspaceSecretRecord],
|
|
archive_path: Path,
|
|
) -> tuple[int, int]:
|
|
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
|
entry_count = 0
|
|
bytes_written = 0
|
|
with tarfile.open(archive_path, "w") as archive:
|
|
for secret in secrets:
|
|
secret_path = workspace_dir / WORKSPACE_SECRETS_DIRNAME / secret.stored_path
|
|
value = secret_path.read_bytes()
|
|
info = tarfile.TarInfo(name=secret.name)
|
|
info.size = len(value)
|
|
info.mode = 0o600
|
|
archive.addfile(info, io.BytesIO(value))
|
|
entry_count += 1
|
|
bytes_written += len(value)
|
|
return entry_count, bytes_written
|
|
|
|
|
|
def _persist_workspace_baseline(
|
|
prepared_seed: PreparedWorkspaceSeed,
|
|
*,
|
|
baseline_archive_path: Path,
|
|
) -> None:
|
|
baseline_archive_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if prepared_seed.archive_path is None:
|
|
_write_empty_seed_archive(baseline_archive_path)
|
|
return
|
|
shutil.copy2(prepared_seed.archive_path, baseline_archive_path)
|
|
|
|
|
|
def _write_workspace_export_archive(
|
|
source_path: Path,
|
|
*,
|
|
archive_path: Path,
|
|
) -> WorkspaceArtifactType:
|
|
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if source_path.is_symlink():
|
|
artifact_type: WorkspaceArtifactType = "symlink"
|
|
elif source_path.is_file():
|
|
artifact_type = "file"
|
|
elif source_path.is_dir():
|
|
artifact_type = "directory"
|
|
else:
|
|
raise RuntimeError(f"unsupported workspace path type: {source_path}")
|
|
|
|
def validate_source(current_path: Path, relative_path: PurePosixPath) -> None:
|
|
if current_path.is_symlink():
|
|
_validate_archive_symlink_target(relative_path, os.readlink(current_path))
|
|
return
|
|
if current_path.is_file():
|
|
return
|
|
if current_path.is_dir():
|
|
for child in sorted(current_path.iterdir(), key=lambda item: item.name):
|
|
validate_source(child, relative_path / child.name)
|
|
return
|
|
raise RuntimeError(f"unsupported workspace path type: {current_path}")
|
|
|
|
if artifact_type == "directory":
|
|
for child in sorted(source_path.iterdir(), key=lambda item: item.name):
|
|
validate_source(child, PurePosixPath(child.name))
|
|
else:
|
|
validate_source(source_path, PurePosixPath(source_path.name))
|
|
with tarfile.open(archive_path, "w") as archive:
|
|
archive.dereference = False
|
|
if artifact_type == "directory":
|
|
for child in sorted(source_path.iterdir(), key=lambda item: item.name):
|
|
archive.add(child, arcname=child.name, recursive=True)
|
|
else:
|
|
archive.add(source_path, arcname=source_path.name, recursive=False)
|
|
return artifact_type
|
|
|
|
|
|
def _extract_seed_archive_to_host_workspace(
|
|
archive_path: Path,
|
|
*,
|
|
workspace_dir: Path,
|
|
destination: str,
|
|
) -> dict[str, Any]:
|
|
normalized_destination, _ = _normalize_workspace_destination(destination)
|
|
destination_root = _workspace_host_destination(workspace_dir, normalized_destination)
|
|
destination_root.mkdir(parents=True, exist_ok=True)
|
|
entry_count = 0
|
|
bytes_written = 0
|
|
with tarfile.open(archive_path, "r:*") as archive:
|
|
for member in archive.getmembers():
|
|
member_name = _normalize_archive_member_name(member.name)
|
|
target_path = destination_root.joinpath(*member_name.parts)
|
|
entry_count += 1
|
|
_ensure_no_symlink_parents(workspace_dir, target_path, member.name)
|
|
if member.isdir():
|
|
if target_path.is_symlink() or (target_path.exists() and not target_path.is_dir()):
|
|
raise RuntimeError(f"directory conflicts with existing path: {member.name}")
|
|
target_path.mkdir(parents=True, exist_ok=True)
|
|
continue
|
|
if member.isfile():
|
|
target_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if target_path.is_symlink() or target_path.is_dir():
|
|
raise RuntimeError(f"file conflicts with existing path: {member.name}")
|
|
source = archive.extractfile(member)
|
|
if source is None:
|
|
raise RuntimeError(f"failed to read archive member: {member.name}")
|
|
with target_path.open("wb") as handle:
|
|
shutil.copyfileobj(source, handle)
|
|
bytes_written += member.size
|
|
continue
|
|
if member.issym():
|
|
_validate_archive_symlink_target(member_name, member.linkname)
|
|
target_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if target_path.exists() and not target_path.is_symlink():
|
|
raise RuntimeError(f"symlink conflicts with existing path: {member.name}")
|
|
if target_path.is_symlink():
|
|
target_path.unlink()
|
|
os.symlink(member.linkname, target_path)
|
|
continue
|
|
if member.islnk():
|
|
raise RuntimeError(
|
|
f"hard links are not allowed in workspace archives: {member.name}"
|
|
)
|
|
raise RuntimeError(f"unsupported archive member type: {member.name}")
|
|
return {
|
|
"destination": normalized_destination,
|
|
"entry_count": entry_count,
|
|
"bytes_written": bytes_written,
|
|
}
|
|
|
|
|
|
def _prepare_workspace_export_archive(
|
|
*,
|
|
workspace_dir: Path,
|
|
workspace_path: str,
|
|
archive_path: Path,
|
|
) -> ExportedWorkspaceArchive:
|
|
normalized_workspace_path, _ = _normalize_workspace_destination(workspace_path)
|
|
source_path = _workspace_host_destination(workspace_dir, normalized_workspace_path)
|
|
if not source_path.exists() and not source_path.is_symlink():
|
|
raise RuntimeError(f"workspace path does not exist: {normalized_workspace_path}")
|
|
artifact_type = _write_workspace_export_archive(source_path, archive_path=archive_path)
|
|
entry_count, bytes_written = _inspect_seed_archive(archive_path)
|
|
return ExportedWorkspaceArchive(
|
|
workspace_path=normalized_workspace_path,
|
|
artifact_type=artifact_type,
|
|
archive_path=archive_path,
|
|
entry_count=entry_count,
|
|
bytes_written=bytes_written,
|
|
)
|
|
|
|
|
|
def _extract_workspace_export_archive(
|
|
archive_path: Path,
|
|
*,
|
|
output_path: Path,
|
|
artifact_type: WorkspaceArtifactType,
|
|
) -> dict[str, Any]:
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if output_path.exists() or output_path.is_symlink():
|
|
raise RuntimeError(f"output_path already exists: {output_path}")
|
|
|
|
entry_count = 0
|
|
bytes_written = 0
|
|
if artifact_type == "directory":
|
|
output_path.mkdir(parents=True, exist_ok=False)
|
|
with tarfile.open(archive_path, "r:*") as archive:
|
|
for member in archive.getmembers():
|
|
member_name = _normalize_archive_member_name(member.name)
|
|
target_path = output_path.joinpath(*member_name.parts)
|
|
entry_count += 1
|
|
_ensure_no_symlink_parents(output_path, target_path, member.name)
|
|
if member.isdir():
|
|
if target_path.is_symlink() or (
|
|
target_path.exists() and not target_path.is_dir()
|
|
):
|
|
raise RuntimeError(f"directory conflicts with existing path: {member.name}")
|
|
target_path.mkdir(parents=True, exist_ok=True)
|
|
continue
|
|
if member.isfile():
|
|
target_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if target_path.is_symlink() or target_path.is_dir():
|
|
raise RuntimeError(f"file conflicts with existing path: {member.name}")
|
|
source = archive.extractfile(member)
|
|
if source is None:
|
|
raise RuntimeError(f"failed to read archive member: {member.name}")
|
|
with target_path.open("wb") as handle:
|
|
shutil.copyfileobj(source, handle)
|
|
bytes_written += member.size
|
|
continue
|
|
if member.issym():
|
|
_validate_archive_symlink_target(member_name, member.linkname)
|
|
target_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if target_path.exists() and not target_path.is_symlink():
|
|
raise RuntimeError(f"symlink conflicts with existing path: {member.name}")
|
|
if target_path.is_symlink():
|
|
target_path.unlink()
|
|
os.symlink(member.linkname, target_path)
|
|
continue
|
|
if member.islnk():
|
|
raise RuntimeError(
|
|
f"hard links are not allowed in workspace archives: {member.name}"
|
|
)
|
|
raise RuntimeError(f"unsupported archive member type: {member.name}")
|
|
return {
|
|
"output_path": str(output_path),
|
|
"artifact_type": artifact_type,
|
|
"entry_count": entry_count,
|
|
"bytes_written": bytes_written,
|
|
}
|
|
|
|
with tarfile.open(archive_path, "r:*") as archive:
|
|
members = archive.getmembers()
|
|
if len(members) != 1:
|
|
raise RuntimeError(
|
|
"expected exactly one archive member for "
|
|
f"{artifact_type} export, got {len(members)}"
|
|
)
|
|
member = members[0]
|
|
_normalize_archive_member_name(member.name)
|
|
entry_count = 1
|
|
if artifact_type == "file":
|
|
if not member.isfile():
|
|
raise RuntimeError("exported archive did not contain a regular file")
|
|
source = archive.extractfile(member)
|
|
if source is None:
|
|
raise RuntimeError(f"failed to read archive member: {member.name}")
|
|
with output_path.open("wb") as handle:
|
|
shutil.copyfileobj(source, handle)
|
|
bytes_written = member.size
|
|
elif artifact_type == "symlink":
|
|
if not member.issym():
|
|
raise RuntimeError("exported archive did not contain a symlink")
|
|
_validate_archive_symlink_target(PurePosixPath(member.name), member.linkname)
|
|
os.symlink(member.linkname, output_path)
|
|
else:
|
|
raise RuntimeError(f"unsupported artifact type: {artifact_type}")
|
|
return {
|
|
"output_path": str(output_path),
|
|
"artifact_type": artifact_type,
|
|
"entry_count": entry_count,
|
|
"bytes_written": bytes_written,
|
|
}
|
|
|
|
|
|
def _normalize_workspace_service_name(service_name: str) -> str:
|
|
normalized = service_name.strip()
|
|
if normalized == "":
|
|
raise ValueError("service_name must not be empty")
|
|
if WORKSPACE_SERVICE_NAME_RE.fullmatch(normalized) is None:
|
|
raise ValueError(
|
|
"service_name must match "
|
|
r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$"
|
|
)
|
|
return normalized
|
|
|
|
|
|
def _normalize_workspace_name(
|
|
name: str | None,
|
|
*,
|
|
allow_none: bool = False,
|
|
) -> str | None:
|
|
if name is None:
|
|
if allow_none:
|
|
return None
|
|
raise ValueError("name must not be empty")
|
|
normalized = name.strip()
|
|
if normalized == "":
|
|
if allow_none:
|
|
return None
|
|
raise ValueError("name must not be empty")
|
|
if len(normalized) > 120:
|
|
raise ValueError("name must be at most 120 characters")
|
|
return normalized
|
|
|
|
|
|
def _normalize_workspace_label_key(label_key: str) -> str:
|
|
normalized = label_key.strip()
|
|
if normalized == "":
|
|
raise ValueError("label key must not be empty")
|
|
if WORKSPACE_LABEL_KEY_RE.fullmatch(normalized) is None:
|
|
raise ValueError(
|
|
"label key must match "
|
|
r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$"
|
|
)
|
|
return normalized
|
|
|
|
|
|
def _normalize_workspace_label_value(label_key: str, label_value: str) -> str:
|
|
normalized = label_value.strip()
|
|
if normalized == "":
|
|
raise ValueError(f"label {label_key!r} must not be empty")
|
|
if len(normalized) > 120:
|
|
raise ValueError(f"label {label_key!r} must be at most 120 characters")
|
|
if "\n" in normalized or "\r" in normalized:
|
|
raise ValueError(f"label {label_key!r} must not contain newlines")
|
|
try:
|
|
normalized.encode("utf-8")
|
|
except UnicodeEncodeError as exc:
|
|
raise ValueError(f"label {label_key!r} must be valid UTF-8 text") from exc
|
|
return normalized
|
|
|
|
|
|
def _normalize_workspace_labels(value: object) -> dict[str, str]:
|
|
if value is None:
|
|
return {}
|
|
if not isinstance(value, dict):
|
|
raise ValueError("labels must be an object mapping keys to values")
|
|
normalized: dict[str, str] = {}
|
|
for raw_key, raw_value in value.items():
|
|
key = _normalize_workspace_label_key(str(raw_key))
|
|
label_value = _normalize_workspace_label_value(key, str(raw_value))
|
|
normalized[key] = label_value
|
|
return dict(sorted(normalized.items()))
|
|
|
|
|
|
def _normalize_workspace_snapshot_name(
|
|
snapshot_name: str,
|
|
*,
|
|
allow_baseline: bool = False,
|
|
) -> str:
|
|
normalized = snapshot_name.strip()
|
|
if normalized == "":
|
|
raise ValueError("snapshot_name must not be empty")
|
|
if normalized == "baseline" and not allow_baseline:
|
|
raise ValueError("snapshot_name 'baseline' is reserved")
|
|
if WORKSPACE_SNAPSHOT_NAME_RE.fullmatch(normalized) is None:
|
|
raise ValueError(
|
|
"snapshot_name must match "
|
|
r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$"
|
|
)
|
|
return normalized
|
|
|
|
|
|
def _normalize_workspace_secret_name(secret_name: str) -> str:
|
|
normalized = secret_name.strip()
|
|
if normalized == "":
|
|
raise ValueError("secret name must not be empty")
|
|
if WORKSPACE_SECRET_NAME_RE.fullmatch(normalized) is None:
|
|
raise ValueError(
|
|
"secret name must match "
|
|
r"^[A-Za-z_][A-Za-z0-9_]{0,63}$"
|
|
)
|
|
return normalized
|
|
|
|
|
|
def _validate_workspace_secret_value(secret_name: str, value: str) -> str:
|
|
try:
|
|
encoded = value.encode("utf-8")
|
|
except UnicodeEncodeError as exc:
|
|
raise ValueError(f"secret {secret_name!r} must be valid UTF-8 text") from exc
|
|
if value == "":
|
|
raise ValueError(f"secret {secret_name!r} must not be empty")
|
|
if len(encoded) > WORKSPACE_SECRET_MAX_BYTES:
|
|
raise ValueError(
|
|
f"secret {secret_name!r} must be at most {WORKSPACE_SECRET_MAX_BYTES} bytes"
|
|
)
|
|
return value
|
|
|
|
|
|
def _normalize_workspace_secret_env_mapping(
|
|
secret_env: dict[str, str] | None,
|
|
) -> dict[str, str]:
|
|
if secret_env is None:
|
|
return {}
|
|
normalized: dict[str, str] = {}
|
|
for secret_name, env_name in secret_env.items():
|
|
normalized_secret_name = _normalize_workspace_secret_name(str(secret_name))
|
|
normalized_env_name = _normalize_workspace_secret_name(str(env_name))
|
|
if normalized_secret_name in normalized:
|
|
raise ValueError(
|
|
f"secret_env references secret {normalized_secret_name!r} more than once"
|
|
)
|
|
normalized[normalized_secret_name] = normalized_env_name
|
|
return normalized
|
|
|
|
|
|
def _normalize_workspace_published_port(
|
|
*,
|
|
guest_port: object,
|
|
host_port: object | None = None,
|
|
) -> WorkspacePublishedPortSpec:
|
|
if isinstance(guest_port, bool) or not isinstance(guest_port, int | str):
|
|
raise ValueError("published guest_port must be an integer")
|
|
try:
|
|
normalized_guest_port = int(guest_port)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError("published guest_port must be an integer") from exc
|
|
if normalized_guest_port <= 0 or normalized_guest_port > 65535:
|
|
raise ValueError("published guest_port must be between 1 and 65535")
|
|
normalized_host_port: int | None = None
|
|
if host_port is not None:
|
|
if isinstance(host_port, bool) or not isinstance(host_port, int | str):
|
|
raise ValueError("published host_port must be an integer")
|
|
try:
|
|
normalized_host_port = int(host_port)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError("published host_port must be an integer") from exc
|
|
if normalized_host_port <= 1024 or normalized_host_port > 65535:
|
|
raise ValueError("published host_port must be between 1025 and 65535")
|
|
return WorkspacePublishedPortSpec(
|
|
guest_port=normalized_guest_port,
|
|
host_port=normalized_host_port,
|
|
)
|
|
|
|
|
|
def _normalize_workspace_published_port_specs(
|
|
published_ports: list[dict[str, Any]] | None,
|
|
) -> list[WorkspacePublishedPortSpec]:
|
|
if not published_ports:
|
|
return []
|
|
normalized: list[WorkspacePublishedPortSpec] = []
|
|
seen_guest_ports: set[tuple[int | None, int]] = set()
|
|
for index, item in enumerate(published_ports, start=1):
|
|
if not isinstance(item, dict):
|
|
raise ValueError(f"published port #{index} must be a dictionary")
|
|
spec = _normalize_workspace_published_port(
|
|
guest_port=item.get("guest_port"),
|
|
host_port=item.get("host_port"),
|
|
)
|
|
dedupe_key = (spec.host_port, spec.guest_port)
|
|
if dedupe_key in seen_guest_ports:
|
|
raise ValueError(
|
|
"published ports must not repeat the same host/guest port mapping"
|
|
)
|
|
seen_guest_ports.add(dedupe_key)
|
|
normalized.append(spec)
|
|
return normalized
|
|
|
|
|
|
def _normalize_workspace_service_readiness(
|
|
readiness: dict[str, Any] | None,
|
|
) -> dict[str, Any] | None:
|
|
if readiness is None:
|
|
return None
|
|
readiness_type = str(readiness.get("type", "")).strip().lower()
|
|
if readiness_type not in {"file", "tcp", "http", "command"}:
|
|
raise ValueError("readiness.type must be one of: file, tcp, http, command")
|
|
if readiness_type == "file":
|
|
path = str(readiness.get("path", "")).strip()
|
|
if path == "":
|
|
raise ValueError("readiness.path is required for file readiness")
|
|
normalized_path, _ = _normalize_workspace_destination(path)
|
|
return {"type": "file", "path": normalized_path}
|
|
if readiness_type == "tcp":
|
|
address = str(readiness.get("address", "")).strip()
|
|
if ":" not in address:
|
|
raise ValueError("readiness.address must be in HOST:PORT format")
|
|
host, raw_port = address.rsplit(":", 1)
|
|
host = host.strip()
|
|
if host == "":
|
|
raise ValueError("readiness.address host must not be empty")
|
|
try:
|
|
port = int(raw_port)
|
|
except ValueError as exc:
|
|
raise ValueError("readiness.address port must be an integer") from exc
|
|
if port <= 0 or port > 65535:
|
|
raise ValueError("readiness.address port must be between 1 and 65535")
|
|
return {"type": "tcp", "address": f"{host}:{port}"}
|
|
if readiness_type == "http":
|
|
url = str(readiness.get("url", "")).strip()
|
|
if url == "":
|
|
raise ValueError("readiness.url is required for http readiness")
|
|
return {"type": "http", "url": url}
|
|
command = str(readiness.get("command", "")).strip()
|
|
if command == "":
|
|
raise ValueError("readiness.command is required for command readiness")
|
|
return {"type": "command", "command": command}
|
|
|
|
|
|
def _workspace_service_status_path(services_dir: Path, service_name: str) -> Path:
|
|
return services_dir / f"{service_name}.status"
|
|
|
|
|
|
def _workspace_service_stdout_path(services_dir: Path, service_name: str) -> Path:
|
|
return services_dir / f"{service_name}.stdout"
|
|
|
|
|
|
def _workspace_service_stderr_path(services_dir: Path, service_name: str) -> Path:
|
|
return services_dir / f"{service_name}.stderr"
|
|
|
|
|
|
def _workspace_service_runner_path(services_dir: Path, service_name: str) -> Path:
|
|
return services_dir / f"{service_name}.runner.sh"
|
|
|
|
|
|
def _workspace_service_port_ready_path(
|
|
services_dir: Path,
|
|
service_name: str,
|
|
host_port: int,
|
|
guest_port: int,
|
|
) -> Path:
|
|
return services_dir / f"{service_name}.port-{host_port}-to-{guest_port}.ready.json"
|
|
|
|
|
|
def _read_service_exit_code(status_path: Path) -> int | None:
|
|
if not status_path.exists():
|
|
return None
|
|
raw_value = status_path.read_text(encoding="utf-8", errors="ignore").strip()
|
|
if raw_value == "":
|
|
return None
|
|
return int(raw_value)
|
|
|
|
|
|
def _tail_text(path: Path, *, tail_lines: int | None) -> tuple[str, bool]:
|
|
if not path.exists():
|
|
return "", False
|
|
text = path.read_text(encoding="utf-8", errors="replace")
|
|
if tail_lines is None:
|
|
return text, False
|
|
lines = text.splitlines(keepends=True)
|
|
if len(lines) <= tail_lines:
|
|
return text, False
|
|
return "".join(lines[-tail_lines:]), True
|
|
|
|
|
|
def _stop_process_group(pid: int, *, wait_seconds: int = 5) -> tuple[bool, bool]:
|
|
try:
|
|
os.killpg(pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
return False, False
|
|
deadline = time.monotonic() + wait_seconds
|
|
while time.monotonic() < deadline:
|
|
if not _pid_is_running(pid):
|
|
return True, False
|
|
time.sleep(0.1)
|
|
try:
|
|
os.killpg(pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
return True, False
|
|
deadline = time.monotonic() + wait_seconds
|
|
while time.monotonic() < deadline:
|
|
if not _pid_is_running(pid):
|
|
return True, True
|
|
time.sleep(0.1)
|
|
return True, True
|
|
|
|
|
|
def _run_service_probe_command(
|
|
cwd: Path,
|
|
command: str,
|
|
*,
|
|
env_overrides: dict[str, str] | None = None,
|
|
) -> int:
|
|
env = {"PATH": os.environ.get("PATH", ""), "HOME": str(cwd)}
|
|
if env_overrides is not None:
|
|
env.update(env_overrides)
|
|
proc = subprocess.run( # noqa: S603
|
|
["bash", "-lc", command], # noqa: S607
|
|
cwd=cwd,
|
|
env=env,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
return proc.returncode
|
|
|
|
|
|
def _service_ready_on_host(
|
|
*,
|
|
readiness: dict[str, Any] | None,
|
|
workspace_dir: Path,
|
|
cwd: Path,
|
|
env_overrides: dict[str, str] | None = None,
|
|
) -> bool:
|
|
if readiness is None:
|
|
return True
|
|
readiness_type = str(readiness["type"])
|
|
if readiness_type == "file":
|
|
ready_path = _workspace_host_destination(workspace_dir, str(readiness["path"]))
|
|
return ready_path.exists()
|
|
if readiness_type == "tcp":
|
|
host, raw_port = str(readiness["address"]).rsplit(":", 1)
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
|
sock.settimeout(1)
|
|
try:
|
|
sock.connect((host, int(raw_port)))
|
|
except OSError:
|
|
return False
|
|
return True
|
|
if readiness_type == "http":
|
|
request = urllib.request.Request(str(readiness["url"]), method="GET")
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=2) as response: # noqa: S310
|
|
return 200 <= int(response.status) < 400
|
|
except (urllib.error.URLError, TimeoutError, ValueError):
|
|
return False
|
|
if readiness_type == "command":
|
|
try:
|
|
return (
|
|
_run_service_probe_command(
|
|
cwd,
|
|
str(readiness["command"]),
|
|
env_overrides=env_overrides,
|
|
)
|
|
== 0
|
|
)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
return False
|
|
raise RuntimeError(f"unsupported readiness type: {readiness_type}")
|
|
|
|
|
|
def _refresh_local_service_record(
|
|
service: WorkspaceServiceRecord,
|
|
*,
|
|
services_dir: Path,
|
|
) -> WorkspaceServiceRecord:
|
|
if service.state != "running" or service.pid is None:
|
|
return service
|
|
if _pid_is_running(service.pid):
|
|
return service
|
|
refreshed = WorkspaceServiceRecord(
|
|
workspace_id=service.workspace_id,
|
|
service_name=service.service_name,
|
|
command=service.command,
|
|
cwd=service.cwd,
|
|
state="exited",
|
|
started_at=service.started_at,
|
|
readiness=dict(service.readiness) if service.readiness is not None else None,
|
|
ready_at=service.ready_at,
|
|
ended_at=service.ended_at or time.time(),
|
|
exit_code=_read_service_exit_code(
|
|
_workspace_service_status_path(services_dir, service.service_name)
|
|
),
|
|
pid=service.pid,
|
|
execution_mode=service.execution_mode,
|
|
stop_reason=service.stop_reason,
|
|
published_ports=list(service.published_ports),
|
|
metadata=dict(service.metadata),
|
|
)
|
|
return refreshed
|
|
|
|
|
|
def _start_local_service(
|
|
*,
|
|
services_dir: Path,
|
|
workspace_dir: Path,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
command: str,
|
|
cwd_text: str,
|
|
readiness: dict[str, Any] | None,
|
|
ready_timeout_seconds: int,
|
|
ready_interval_ms: int,
|
|
env_overrides: dict[str, str] | None = None,
|
|
) -> WorkspaceServiceRecord:
|
|
services_dir.mkdir(parents=True, exist_ok=True)
|
|
cwd = _workspace_host_destination(workspace_dir, cwd_text)
|
|
cwd.mkdir(parents=True, exist_ok=True)
|
|
stdout_path = _workspace_service_stdout_path(services_dir, service_name)
|
|
stderr_path = _workspace_service_stderr_path(services_dir, service_name)
|
|
status_path = _workspace_service_status_path(services_dir, service_name)
|
|
runner_path = _workspace_service_runner_path(services_dir, service_name)
|
|
stdout_path.write_text("", encoding="utf-8")
|
|
stderr_path.write_text("", encoding="utf-8")
|
|
status_path.unlink(missing_ok=True)
|
|
runner_path.write_text(
|
|
"\n".join(
|
|
[
|
|
"#!/bin/sh",
|
|
"set +e",
|
|
f"cd {shlex.quote(str(cwd))}",
|
|
(
|
|
f"/bin/sh -lc {shlex.quote(command)}"
|
|
f" >> {shlex.quote(str(stdout_path))}"
|
|
f" 2>> {shlex.quote(str(stderr_path))}"
|
|
),
|
|
"status=$?",
|
|
f"printf '%s' \"$status\" > {shlex.quote(str(status_path))}",
|
|
"exit \"$status\"",
|
|
]
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
runner_path.chmod(0o700)
|
|
env = {"PATH": os.environ.get("PATH", ""), "HOME": str(cwd)}
|
|
if env_overrides is not None:
|
|
env.update(env_overrides)
|
|
process = subprocess.Popen( # noqa: S603
|
|
[str(runner_path)],
|
|
cwd=str(cwd),
|
|
env=env,
|
|
text=True,
|
|
start_new_session=True,
|
|
)
|
|
started_at = time.time()
|
|
service = WorkspaceServiceRecord(
|
|
workspace_id=workspace_id,
|
|
service_name=service_name,
|
|
command=command,
|
|
cwd=cwd_text,
|
|
state="running",
|
|
started_at=started_at,
|
|
readiness=dict(readiness) if readiness is not None else None,
|
|
ready_at=None,
|
|
ended_at=None,
|
|
exit_code=None,
|
|
pid=process.pid,
|
|
execution_mode="host_compat",
|
|
stop_reason=None,
|
|
)
|
|
deadline = time.monotonic() + ready_timeout_seconds
|
|
while True:
|
|
service = _refresh_local_service_record(service, services_dir=services_dir)
|
|
if service.state != "running":
|
|
service.state = "failed"
|
|
service.stop_reason = "process_exited_before_ready"
|
|
if service.ended_at is None:
|
|
service.ended_at = time.time()
|
|
return service
|
|
if _service_ready_on_host(
|
|
readiness=readiness,
|
|
workspace_dir=workspace_dir,
|
|
cwd=cwd,
|
|
env_overrides=env_overrides,
|
|
):
|
|
service.ready_at = time.time()
|
|
return service
|
|
if time.monotonic() >= deadline:
|
|
_stop_process_group(process.pid)
|
|
service = _refresh_local_service_record(service, services_dir=services_dir)
|
|
service.state = "failed"
|
|
service.stop_reason = "readiness_timeout"
|
|
if service.ended_at is None:
|
|
service.ended_at = time.time()
|
|
return service
|
|
time.sleep(max(ready_interval_ms, 1) / 1000)
|
|
|
|
|
|
def _stop_local_service(
|
|
service: WorkspaceServiceRecord,
|
|
*,
|
|
services_dir: Path,
|
|
) -> WorkspaceServiceRecord:
|
|
if service.pid is None:
|
|
return service
|
|
stopped, killed = _stop_process_group(service.pid)
|
|
refreshed = _refresh_local_service_record(service, services_dir=services_dir)
|
|
if stopped:
|
|
refreshed.state = "stopped"
|
|
refreshed.stop_reason = "sigkill" if killed else "sigterm"
|
|
refreshed.ended_at = refreshed.ended_at or time.time()
|
|
return refreshed
|
|
|
|
|
|
def _start_workspace_published_port_proxy(
|
|
*,
|
|
services_dir: Path,
|
|
service_name: str,
|
|
workspace_id: str,
|
|
guest_ip: str,
|
|
spec: WorkspacePublishedPortSpec,
|
|
) -> WorkspacePublishedPortRecord:
|
|
ready_path = _workspace_service_port_ready_path(
|
|
services_dir,
|
|
service_name,
|
|
spec.host_port or 0,
|
|
spec.guest_port,
|
|
)
|
|
ready_path.unlink(missing_ok=True)
|
|
command = [
|
|
sys.executable,
|
|
"-m",
|
|
"pyro_mcp.workspace_ports",
|
|
"--listen-host",
|
|
DEFAULT_PUBLISHED_PORT_HOST,
|
|
"--listen-port",
|
|
str(spec.host_port or 0),
|
|
"--target-host",
|
|
guest_ip,
|
|
"--target-port",
|
|
str(spec.guest_port),
|
|
"--ready-file",
|
|
str(ready_path),
|
|
]
|
|
process = subprocess.Popen( # noqa: S603
|
|
command,
|
|
text=True,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
start_new_session=True,
|
|
)
|
|
deadline = time.monotonic() + 5
|
|
while time.monotonic() < deadline:
|
|
if ready_path.exists():
|
|
payload = json.loads(ready_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError("published port proxy ready payload is invalid")
|
|
ready_path.unlink(missing_ok=True)
|
|
return WorkspacePublishedPortRecord(
|
|
guest_port=int(payload.get("target_port", spec.guest_port)),
|
|
host_port=int(payload["host_port"]),
|
|
host=str(payload.get("host", DEFAULT_PUBLISHED_PORT_HOST)),
|
|
protocol=str(payload.get("protocol", "tcp")),
|
|
proxy_pid=process.pid,
|
|
)
|
|
if process.poll() is not None:
|
|
raise RuntimeError(
|
|
"failed to start published port proxy for "
|
|
f"service {service_name!r} in workspace {workspace_id!r}"
|
|
)
|
|
time.sleep(0.05)
|
|
_stop_workspace_published_port_proxy(
|
|
WorkspacePublishedPortRecord(
|
|
guest_port=spec.guest_port,
|
|
host_port=spec.host_port or 0,
|
|
proxy_pid=process.pid,
|
|
)
|
|
)
|
|
ready_path.unlink(missing_ok=True)
|
|
raise RuntimeError(
|
|
"timed out waiting for published port proxy readiness for "
|
|
f"service {service_name!r} in workspace {workspace_id!r}"
|
|
)
|
|
|
|
|
|
def _stop_workspace_published_port_proxy(published_port: WorkspacePublishedPortRecord) -> None:
|
|
if published_port.proxy_pid is None:
|
|
return
|
|
try:
|
|
os.killpg(published_port.proxy_pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
return
|
|
deadline = time.monotonic() + 5
|
|
while time.monotonic() < deadline:
|
|
if not _pid_is_running(published_port.proxy_pid):
|
|
return
|
|
time.sleep(0.05)
|
|
try:
|
|
os.killpg(published_port.proxy_pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
return
|
|
|
|
|
|
def _instance_workspace_host_dir(instance: VmInstance) -> Path:
|
|
raw_value = instance.metadata.get("workspace_host_dir")
|
|
if raw_value is None or raw_value == "":
|
|
raise RuntimeError("workspace host directory is unavailable")
|
|
return Path(raw_value)
|
|
|
|
|
|
def _patch_rootfs_runtime_file(
|
|
rootfs_image: Path,
|
|
*,
|
|
source_path: Path,
|
|
destination_path: str,
|
|
asset_label: str,
|
|
file_mode: str | None = None,
|
|
) -> None:
|
|
debugfs_path = shutil.which("debugfs")
|
|
if debugfs_path is None:
|
|
raise RuntimeError(
|
|
"debugfs is required to seed workspaces on guest-backed runtimes"
|
|
)
|
|
with tempfile.TemporaryDirectory(prefix=f"pyro-{asset_label}-") as temp_dir:
|
|
staged_path = Path(temp_dir) / Path(destination_path).name
|
|
shutil.copy2(source_path, staged_path)
|
|
subprocess.run( # noqa: S603
|
|
[debugfs_path, "-w", "-R", f"rm {destination_path}", str(rootfs_image)],
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
proc = subprocess.run( # noqa: S603
|
|
[
|
|
debugfs_path,
|
|
"-w",
|
|
"-R",
|
|
f"write {staged_path} {destination_path}",
|
|
str(rootfs_image),
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
if proc.returncode == 0 and file_mode is not None:
|
|
proc = subprocess.run( # noqa: S603
|
|
[
|
|
debugfs_path,
|
|
"-w",
|
|
"-R",
|
|
f"set_inode_field {destination_path} mode {file_mode}",
|
|
str(rootfs_image),
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(
|
|
f"failed to patch {asset_label} into workspace rootfs: "
|
|
f"{proc.stderr.strip() or proc.stdout.strip()}"
|
|
)
|
|
|
|
|
|
def _ensure_no_symlink_parents(root: Path, target_path: Path, member_name: str) -> None:
|
|
relative_path = target_path.relative_to(root)
|
|
current = root
|
|
for part in relative_path.parts[:-1]:
|
|
current = current / part
|
|
if current.is_symlink():
|
|
raise RuntimeError(
|
|
f"archive member would traverse through a symlinked path: {member_name}"
|
|
)
|
|
|
|
|
|
def _pid_is_running(pid: int | None) -> bool:
|
|
if pid is None:
|
|
return False
|
|
try:
|
|
os.kill(pid, 0)
|
|
except ProcessLookupError:
|
|
return False
|
|
except PermissionError:
|
|
return True
|
|
return True
|
|
|
|
|
|
def _collect_workspace_tree(root: Path) -> dict[str, WorkspaceTreeEntry]:
|
|
entries: dict[str, WorkspaceTreeEntry] = {}
|
|
|
|
def walk(current: Path, relative_parts: tuple[str, ...] = ()) -> bool:
|
|
has_entries = False
|
|
for child in sorted(current.iterdir(), key=lambda item: item.name):
|
|
child_relative_parts = relative_parts + (child.name,)
|
|
relative_path = "/".join(child_relative_parts)
|
|
if child.is_symlink():
|
|
entries[relative_path] = WorkspaceTreeEntry(
|
|
path=relative_path,
|
|
artifact_type="symlink",
|
|
disk_path=child,
|
|
link_target=os.readlink(child),
|
|
)
|
|
has_entries = True
|
|
continue
|
|
if child.is_file():
|
|
entries[relative_path] = WorkspaceTreeEntry(
|
|
path=relative_path,
|
|
artifact_type="file",
|
|
disk_path=child,
|
|
size_bytes=child.stat().st_size,
|
|
)
|
|
has_entries = True
|
|
continue
|
|
if child.is_dir():
|
|
child_has_entries = walk(child, child_relative_parts)
|
|
if not child_has_entries:
|
|
entries[relative_path] = WorkspaceTreeEntry(
|
|
path=relative_path,
|
|
artifact_type="directory",
|
|
disk_path=child,
|
|
)
|
|
has_entries = True
|
|
else:
|
|
has_entries = True
|
|
continue
|
|
raise RuntimeError(f"unsupported workspace artifact type: {child}")
|
|
return has_entries
|
|
|
|
walk(root)
|
|
return entries
|
|
|
|
|
|
def _is_probably_text(data: bytes) -> bool:
|
|
if b"\x00" in data:
|
|
return False
|
|
try:
|
|
data.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
return False
|
|
return True
|
|
|
|
|
|
def _build_text_patch(
|
|
*,
|
|
path: str,
|
|
before_text: str,
|
|
after_text: str,
|
|
status: str,
|
|
) -> str:
|
|
if status == "added":
|
|
fromfile = "/dev/null"
|
|
tofile = f"b/{path}"
|
|
elif status == "deleted":
|
|
fromfile = f"a/{path}"
|
|
tofile = "/dev/null"
|
|
else:
|
|
fromfile = f"a/{path}"
|
|
tofile = f"b/{path}"
|
|
lines = list(
|
|
difflib.unified_diff(
|
|
before_text.splitlines(keepends=True),
|
|
after_text.splitlines(keepends=True),
|
|
fromfile=fromfile,
|
|
tofile=tofile,
|
|
n=3,
|
|
)
|
|
)
|
|
if not lines:
|
|
return ""
|
|
return "".join(lines)
|
|
|
|
|
|
def _diff_workspace_trees(
|
|
baseline_root: Path,
|
|
current_root: Path,
|
|
) -> dict[str, Any]:
|
|
baseline_entries = _collect_workspace_tree(baseline_root)
|
|
current_entries = _collect_workspace_tree(current_root)
|
|
changed_entries: list[dict[str, Any]] = []
|
|
patch_parts: list[str] = []
|
|
summary = {
|
|
"total": 0,
|
|
"added": 0,
|
|
"modified": 0,
|
|
"deleted": 0,
|
|
"type_changed": 0,
|
|
"text_patched": 0,
|
|
"non_text": 0,
|
|
}
|
|
|
|
for path in sorted(set(baseline_entries) | set(current_entries)):
|
|
baseline_entry = baseline_entries.get(path)
|
|
current_entry = current_entries.get(path)
|
|
entry_payload: dict[str, Any] | None = None
|
|
text_patch = ""
|
|
|
|
if baseline_entry is None and current_entry is not None:
|
|
entry_payload = {
|
|
"path": path,
|
|
"status": "added",
|
|
"artifact_type": current_entry.artifact_type,
|
|
"text_patch": None,
|
|
}
|
|
if current_entry.artifact_type == "file":
|
|
current_bytes = current_entry.disk_path.read_bytes()
|
|
if _is_probably_text(current_bytes):
|
|
text_patch = _build_text_patch(
|
|
path=path,
|
|
before_text="",
|
|
after_text=current_bytes.decode("utf-8"),
|
|
status="added",
|
|
)
|
|
elif current_entry is None and baseline_entry is not None:
|
|
entry_payload = {
|
|
"path": path,
|
|
"status": "deleted",
|
|
"artifact_type": baseline_entry.artifact_type,
|
|
"text_patch": None,
|
|
}
|
|
if baseline_entry.artifact_type == "file":
|
|
baseline_bytes = baseline_entry.disk_path.read_bytes()
|
|
if _is_probably_text(baseline_bytes):
|
|
text_patch = _build_text_patch(
|
|
path=path,
|
|
before_text=baseline_bytes.decode("utf-8"),
|
|
after_text="",
|
|
status="deleted",
|
|
)
|
|
elif baseline_entry is not None and current_entry is not None:
|
|
if baseline_entry.artifact_type != current_entry.artifact_type:
|
|
entry_payload = {
|
|
"path": path,
|
|
"status": "type_changed",
|
|
"artifact_type": current_entry.artifact_type,
|
|
"text_patch": None,
|
|
}
|
|
elif current_entry.artifact_type == "directory":
|
|
continue
|
|
elif current_entry.artifact_type == "symlink":
|
|
if baseline_entry.link_target != current_entry.link_target:
|
|
entry_payload = {
|
|
"path": path,
|
|
"status": "modified",
|
|
"artifact_type": current_entry.artifact_type,
|
|
"text_patch": None,
|
|
}
|
|
else:
|
|
baseline_bytes = baseline_entry.disk_path.read_bytes()
|
|
current_bytes = current_entry.disk_path.read_bytes()
|
|
if baseline_bytes == current_bytes:
|
|
continue
|
|
entry_payload = {
|
|
"path": path,
|
|
"status": "modified",
|
|
"artifact_type": current_entry.artifact_type,
|
|
"text_patch": None,
|
|
}
|
|
if _is_probably_text(baseline_bytes) and _is_probably_text(current_bytes):
|
|
text_patch = _build_text_patch(
|
|
path=path,
|
|
before_text=baseline_bytes.decode("utf-8"),
|
|
after_text=current_bytes.decode("utf-8"),
|
|
status="modified",
|
|
)
|
|
|
|
if entry_payload is None:
|
|
continue
|
|
|
|
summary["total"] += 1
|
|
summary[str(entry_payload["status"])] += 1
|
|
if text_patch != "":
|
|
entry_payload["text_patch"] = text_patch
|
|
patch_parts.append(text_patch)
|
|
summary["text_patched"] += 1
|
|
else:
|
|
summary["non_text"] += 1
|
|
changed_entries.append(entry_payload)
|
|
|
|
return {
|
|
"changed": bool(changed_entries),
|
|
"summary": summary,
|
|
"entries": changed_entries,
|
|
"patch": "".join(patch_parts),
|
|
}
|
|
|
|
|
|
class VmBackend:
|
|
"""Backend interface for lifecycle operations."""
|
|
|
|
def create(self, instance: VmInstance) -> None: # pragma: no cover
|
|
raise NotImplementedError
|
|
|
|
def start(self, instance: VmInstance) -> None: # pragma: no cover
|
|
raise NotImplementedError
|
|
|
|
def exec( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
command: str,
|
|
timeout_seconds: int,
|
|
*,
|
|
workdir: Path | None = None,
|
|
env: dict[str, str] | None = None,
|
|
) -> VmExecResult:
|
|
raise NotImplementedError
|
|
|
|
def stop(self, instance: VmInstance) -> None: # pragma: no cover
|
|
raise NotImplementedError
|
|
|
|
def delete(self, instance: VmInstance) -> None: # pragma: no cover
|
|
raise NotImplementedError
|
|
|
|
def import_archive( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
archive_path: Path,
|
|
destination: str,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def install_secrets( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
archive_path: Path,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def export_archive( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
archive_path: Path,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def list_workspace_entries( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
recursive: bool,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def read_workspace_file( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
max_bytes: int,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def write_workspace_file( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
text: str,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def delete_workspace_path( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def open_shell( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
cwd: str,
|
|
cols: int,
|
|
rows: int,
|
|
env: dict[str, str] | None = None,
|
|
redact_values: list[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def read_shell( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
cursor: int,
|
|
max_chars: int,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def write_shell( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
input_text: str,
|
|
append_newline: bool,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def signal_shell( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
signal_name: str,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def close_shell( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def start_service( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
command: str,
|
|
cwd: str,
|
|
readiness: dict[str, Any] | None,
|
|
ready_timeout_seconds: int,
|
|
ready_interval_ms: int,
|
|
env: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def status_service( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def logs_service( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
tail_lines: int | None,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
def stop_service( # pragma: no cover
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
) -> dict[str, Any]:
|
|
raise NotImplementedError
|
|
|
|
|
|
class MockBackend(VmBackend):
|
|
"""Host-process backend used for development and testability."""
|
|
|
|
def create(self, instance: VmInstance) -> None:
|
|
instance.workdir.mkdir(parents=True, exist_ok=False)
|
|
|
|
def start(self, instance: VmInstance) -> None:
|
|
marker_path = instance.workdir / ".started"
|
|
marker_path.write_text("started\n", encoding="utf-8")
|
|
|
|
def exec(
|
|
self,
|
|
instance: VmInstance,
|
|
command: str,
|
|
timeout_seconds: int,
|
|
*,
|
|
workdir: Path | None = None,
|
|
env: dict[str, str] | None = None,
|
|
) -> VmExecResult:
|
|
return _run_host_command(
|
|
workdir or instance.workdir,
|
|
command,
|
|
timeout_seconds,
|
|
env_overrides=env,
|
|
)
|
|
|
|
def stop(self, instance: VmInstance) -> None:
|
|
marker_path = instance.workdir / ".stopped"
|
|
marker_path.write_text("stopped\n", encoding="utf-8")
|
|
|
|
def delete(self, instance: VmInstance) -> None:
|
|
shutil.rmtree(instance.workdir, ignore_errors=True)
|
|
|
|
def import_archive(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
archive_path: Path,
|
|
destination: str,
|
|
) -> dict[str, Any]:
|
|
return _extract_seed_archive_to_host_workspace(
|
|
archive_path,
|
|
workspace_dir=_instance_workspace_host_dir(instance),
|
|
destination=destination,
|
|
)
|
|
|
|
def install_secrets(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
archive_path: Path,
|
|
) -> dict[str, Any]:
|
|
del instance
|
|
entry_count, bytes_written = _inspect_seed_archive(archive_path)
|
|
return {
|
|
"destination": WORKSPACE_GUEST_SECRETS_PATH,
|
|
"entry_count": entry_count,
|
|
"bytes_written": bytes_written,
|
|
}
|
|
|
|
def export_archive(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
archive_path: Path,
|
|
) -> dict[str, Any]:
|
|
exported = _prepare_workspace_export_archive(
|
|
workspace_dir=_instance_workspace_host_dir(instance),
|
|
workspace_path=workspace_path,
|
|
archive_path=archive_path,
|
|
)
|
|
return {
|
|
"workspace_path": exported.workspace_path,
|
|
"artifact_type": exported.artifact_type,
|
|
"entry_count": exported.entry_count,
|
|
"bytes_written": exported.bytes_written,
|
|
"execution_mode": "host_compat",
|
|
}
|
|
|
|
def list_workspace_entries(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
recursive: bool,
|
|
) -> dict[str, Any]:
|
|
listing = list_workspace_files(
|
|
_instance_workspace_host_dir(instance),
|
|
workspace_path=workspace_path,
|
|
recursive=recursive,
|
|
)
|
|
return {
|
|
"path": listing.path,
|
|
"artifact_type": listing.artifact_type,
|
|
"entries": [entry.to_payload() for entry in listing.entries],
|
|
"execution_mode": "host_compat",
|
|
}
|
|
|
|
def read_workspace_file(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
max_bytes: int,
|
|
) -> dict[str, Any]:
|
|
file_result = read_workspace_file(
|
|
_instance_workspace_host_dir(instance),
|
|
workspace_path=workspace_path,
|
|
max_bytes=max_bytes,
|
|
)
|
|
return {
|
|
"path": file_result.path,
|
|
"size_bytes": file_result.size_bytes,
|
|
"content_bytes": file_result.content_bytes,
|
|
"execution_mode": "host_compat",
|
|
}
|
|
|
|
def write_workspace_file(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
text: str,
|
|
) -> dict[str, Any]:
|
|
result = write_workspace_file(
|
|
_instance_workspace_host_dir(instance),
|
|
workspace_path=workspace_path,
|
|
text=text,
|
|
)
|
|
return {
|
|
"path": result.path,
|
|
"size_bytes": result.size_bytes,
|
|
"bytes_written": result.bytes_written,
|
|
"execution_mode": "host_compat",
|
|
}
|
|
|
|
def delete_workspace_path(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
) -> dict[str, Any]:
|
|
result = delete_workspace_path(
|
|
_instance_workspace_host_dir(instance),
|
|
workspace_path=workspace_path,
|
|
)
|
|
return {
|
|
"path": result.path,
|
|
"deleted": result.deleted,
|
|
"execution_mode": "host_compat",
|
|
}
|
|
|
|
def open_shell(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
cwd: str,
|
|
cols: int,
|
|
rows: int,
|
|
env: dict[str, str] | None = None,
|
|
redact_values: list[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
session = create_local_shell(
|
|
workspace_id=workspace_id,
|
|
shell_id=shell_id,
|
|
cwd=_workspace_host_destination(_instance_workspace_host_dir(instance), cwd),
|
|
display_cwd=cwd,
|
|
cols=cols,
|
|
rows=rows,
|
|
env_overrides=env,
|
|
redact_values=redact_values,
|
|
)
|
|
summary = session.summary()
|
|
summary["execution_mode"] = "host_compat"
|
|
return summary
|
|
|
|
def read_shell(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
cursor: int,
|
|
max_chars: int,
|
|
) -> dict[str, Any]:
|
|
del instance
|
|
session = get_local_shell(workspace_id=workspace_id, shell_id=shell_id)
|
|
payload = session.read(cursor=cursor, max_chars=max_chars)
|
|
payload["execution_mode"] = "host_compat"
|
|
return payload
|
|
|
|
def write_shell(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
input_text: str,
|
|
append_newline: bool,
|
|
) -> dict[str, Any]:
|
|
del instance
|
|
session = get_local_shell(workspace_id=workspace_id, shell_id=shell_id)
|
|
payload = session.write(input_text, append_newline=append_newline)
|
|
payload["execution_mode"] = "host_compat"
|
|
return payload
|
|
|
|
def signal_shell(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
signal_name: str,
|
|
) -> dict[str, Any]:
|
|
del instance
|
|
session = get_local_shell(workspace_id=workspace_id, shell_id=shell_id)
|
|
payload = session.send_signal(signal_name)
|
|
payload["execution_mode"] = "host_compat"
|
|
return payload
|
|
|
|
def close_shell(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
) -> dict[str, Any]:
|
|
del instance
|
|
session = remove_local_shell(workspace_id=workspace_id, shell_id=shell_id)
|
|
if session is None:
|
|
raise ValueError(f"shell {shell_id!r} does not exist in workspace {workspace_id!r}")
|
|
payload = session.close()
|
|
payload["execution_mode"] = "host_compat"
|
|
return payload
|
|
|
|
def start_service(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
command: str,
|
|
cwd: str,
|
|
readiness: dict[str, Any] | None,
|
|
ready_timeout_seconds: int,
|
|
ready_interval_ms: int,
|
|
env: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
services_dir = instance.workdir.parent / WORKSPACE_SERVICES_DIRNAME
|
|
service = _start_local_service(
|
|
services_dir=services_dir,
|
|
workspace_dir=_instance_workspace_host_dir(instance),
|
|
workspace_id=workspace_id,
|
|
service_name=service_name,
|
|
command=command,
|
|
cwd_text=cwd,
|
|
readiness=readiness,
|
|
ready_timeout_seconds=ready_timeout_seconds,
|
|
ready_interval_ms=ready_interval_ms,
|
|
env_overrides=env,
|
|
)
|
|
return service.to_payload()
|
|
|
|
def status_service(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
) -> dict[str, Any]:
|
|
services_dir = instance.workdir.parent / WORKSPACE_SERVICES_DIRNAME
|
|
service = self._load_workspace_service(services_dir, workspace_id, service_name)
|
|
refreshed = _refresh_local_service_record(
|
|
service,
|
|
services_dir=services_dir,
|
|
)
|
|
return refreshed.to_payload()
|
|
|
|
def logs_service(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
tail_lines: int | None,
|
|
) -> dict[str, Any]:
|
|
services_dir = instance.workdir.parent / WORKSPACE_SERVICES_DIRNAME
|
|
service = self._load_workspace_service(services_dir, workspace_id, service_name)
|
|
refreshed = _refresh_local_service_record(service, services_dir=services_dir)
|
|
payload = refreshed.to_payload()
|
|
stdout, stdout_truncated = _tail_text(
|
|
_workspace_service_stdout_path(services_dir, service_name),
|
|
tail_lines=tail_lines,
|
|
)
|
|
stderr, stderr_truncated = _tail_text(
|
|
_workspace_service_stderr_path(services_dir, service_name),
|
|
tail_lines=tail_lines,
|
|
)
|
|
payload.update(
|
|
{
|
|
"stdout": stdout,
|
|
"stderr": stderr,
|
|
"tail_lines": tail_lines,
|
|
"truncated": stdout_truncated or stderr_truncated,
|
|
}
|
|
)
|
|
return payload
|
|
|
|
def stop_service(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
) -> dict[str, Any]:
|
|
services_dir = instance.workdir.parent / WORKSPACE_SERVICES_DIRNAME
|
|
service = self._load_workspace_service(services_dir, workspace_id, service_name)
|
|
stopped = _stop_local_service(
|
|
service,
|
|
services_dir=services_dir,
|
|
)
|
|
return stopped.to_payload()
|
|
|
|
def _load_workspace_service(
|
|
self,
|
|
services_dir: Path,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
) -> WorkspaceServiceRecord:
|
|
record_path = services_dir / f"{service_name}.json"
|
|
if not record_path.exists():
|
|
raise ValueError(
|
|
f"service {service_name!r} does not exist in workspace {workspace_id!r}"
|
|
)
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError(f"service record at {record_path} is invalid")
|
|
return WorkspaceServiceRecord.from_payload(payload)
|
|
|
|
|
|
class FirecrackerBackend(VmBackend): # pragma: no cover
|
|
"""Host-gated backend that validates Firecracker prerequisites."""
|
|
|
|
def __init__(
|
|
self,
|
|
environment_store: EnvironmentStore,
|
|
firecracker_bin: Path,
|
|
jailer_bin: Path,
|
|
runtime_capabilities: RuntimeCapabilities,
|
|
network_manager: TapNetworkManager | None = None,
|
|
guest_exec_client: VsockExecClient | None = None,
|
|
) -> None:
|
|
self._environment_store = environment_store
|
|
self._firecracker_bin = firecracker_bin
|
|
self._jailer_bin = jailer_bin
|
|
self._runtime_capabilities = runtime_capabilities
|
|
self._network_manager = network_manager or TapNetworkManager()
|
|
self._guest_exec_client = guest_exec_client or VsockExecClient()
|
|
self._processes: dict[str, subprocess.Popen[str]] = {}
|
|
if not self._firecracker_bin.exists():
|
|
raise RuntimeError(f"bundled firecracker binary not found at {self._firecracker_bin}")
|
|
if not self._jailer_bin.exists():
|
|
raise RuntimeError(f"bundled jailer binary not found at {self._jailer_bin}")
|
|
if not Path("/dev/kvm").exists():
|
|
raise RuntimeError("/dev/kvm is not available on this host")
|
|
|
|
def create(self, instance: VmInstance) -> None:
|
|
instance.workdir.mkdir(parents=True, exist_ok=False)
|
|
try:
|
|
installed_environment = self._environment_store.ensure_installed(instance.environment)
|
|
if (
|
|
not installed_environment.kernel_image.exists()
|
|
or not installed_environment.rootfs_image.exists()
|
|
):
|
|
raise RuntimeError(
|
|
f"missing environment artifacts for {instance.environment}; expected "
|
|
f"{installed_environment.kernel_image} and {installed_environment.rootfs_image}"
|
|
)
|
|
instance.metadata["environment_version"] = installed_environment.version
|
|
instance.metadata["environment_source"] = installed_environment.source
|
|
if installed_environment.source_digest is not None:
|
|
instance.metadata["environment_digest"] = installed_environment.source_digest
|
|
instance.metadata["environment_install_dir"] = str(installed_environment.install_dir)
|
|
instance.metadata["kernel_image"] = str(installed_environment.kernel_image)
|
|
rootfs_copy = instance.workdir / "rootfs.ext4"
|
|
instance.metadata["rootfs_clone_mode"] = _copy_rootfs(
|
|
installed_environment.rootfs_image,
|
|
rootfs_copy,
|
|
)
|
|
instance.metadata["rootfs_image"] = str(rootfs_copy)
|
|
if instance.network_requested:
|
|
network = self._network_manager.allocate(instance.vm_id)
|
|
instance.network = network
|
|
instance.metadata.update(self._network_manager.to_metadata(network))
|
|
else:
|
|
instance.network = None
|
|
instance.metadata["network_enabled"] = "false"
|
|
except Exception:
|
|
shutil.rmtree(instance.workdir, ignore_errors=True)
|
|
raise
|
|
|
|
def start(self, instance: VmInstance) -> None:
|
|
launch_plan = build_launch_plan(instance)
|
|
for stale_socket_path in (
|
|
launch_plan.api_socket_path,
|
|
instance.workdir / "vsock.sock",
|
|
):
|
|
stale_socket_path.unlink(missing_ok=True)
|
|
instance.metadata["firecracker_config_path"] = str(launch_plan.config_path)
|
|
instance.metadata["guest_network_path"] = str(launch_plan.guest_network_path)
|
|
instance.metadata["guest_exec_path"] = str(launch_plan.guest_exec_path)
|
|
instance.metadata["guest_cid"] = str(launch_plan.guest_cid)
|
|
instance.metadata["guest_exec_port"] = str(launch_plan.vsock_port)
|
|
instance.metadata["guest_exec_uds_path"] = str(instance.workdir / "vsock.sock")
|
|
serial_log_path = instance.workdir / "serial.log"
|
|
firecracker_log_path = instance.workdir / "firecracker.log"
|
|
firecracker_log_path.touch()
|
|
instance.metadata["serial_log_path"] = str(serial_log_path)
|
|
instance.metadata["firecracker_log_path"] = str(firecracker_log_path)
|
|
proc = subprocess.run( # noqa: S603
|
|
[str(self._firecracker_bin), "--version"],
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f"firecracker startup preflight failed: {proc.stderr.strip()}")
|
|
instance.metadata["firecracker_version"] = proc.stdout.strip()
|
|
instance.metadata["jailer_path"] = str(self._jailer_bin)
|
|
if not self._runtime_capabilities.supports_vm_boot:
|
|
instance.metadata["execution_mode"] = "host_compat"
|
|
instance.metadata["boot_mode"] = "shim"
|
|
if self._runtime_capabilities.reason is not None:
|
|
instance.metadata["runtime_reason"] = self._runtime_capabilities.reason
|
|
return
|
|
with serial_log_path.open("w", encoding="utf-8") as serial_fp:
|
|
process = subprocess.Popen( # noqa: S603
|
|
[
|
|
str(self._firecracker_bin),
|
|
"--no-api",
|
|
"--config-file",
|
|
str(launch_plan.config_path),
|
|
"--log-path",
|
|
str(firecracker_log_path),
|
|
"--level",
|
|
"Info",
|
|
],
|
|
stdout=serial_fp,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
start_new_session=True,
|
|
)
|
|
self._processes[instance.vm_id] = process
|
|
time.sleep(2)
|
|
if process.poll() is not None:
|
|
serial_log = serial_log_path.read_text(encoding="utf-8", errors="ignore")
|
|
firecracker_log = firecracker_log_path.read_text(encoding="utf-8", errors="ignore")
|
|
self._processes.pop(instance.vm_id, None)
|
|
raise RuntimeError(
|
|
"firecracker microVM exited during startup: "
|
|
f"{(serial_log or firecracker_log).strip()}"
|
|
)
|
|
instance.firecracker_pid = process.pid
|
|
instance.metadata["execution_mode"] = (
|
|
"guest_vsock" if self._runtime_capabilities.supports_guest_exec else "guest_boot_only"
|
|
)
|
|
instance.metadata["boot_mode"] = "native"
|
|
|
|
def exec(
|
|
self,
|
|
instance: VmInstance,
|
|
command: str,
|
|
timeout_seconds: int,
|
|
*,
|
|
workdir: Path | None = None,
|
|
env: dict[str, str] | None = None,
|
|
) -> VmExecResult:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
deadline = time.monotonic() + min(timeout_seconds, 10)
|
|
while True:
|
|
try:
|
|
response = self._guest_exec_client.exec(
|
|
guest_cid,
|
|
port,
|
|
command,
|
|
timeout_seconds,
|
|
env=env,
|
|
uds_path=uds_path,
|
|
)
|
|
break
|
|
except (OSError, RuntimeError) as exc:
|
|
if time.monotonic() >= deadline:
|
|
raise RuntimeError(
|
|
f"guest exec transport did not become ready: {exc}"
|
|
) from exc
|
|
time.sleep(0.2)
|
|
return VmExecResult(
|
|
stdout=response.stdout,
|
|
stderr=response.stderr,
|
|
exit_code=response.exit_code,
|
|
duration_ms=response.duration_ms,
|
|
)
|
|
instance.metadata["execution_mode"] = "host_compat"
|
|
return _run_host_command(
|
|
workdir or instance.workdir,
|
|
command,
|
|
timeout_seconds,
|
|
env_overrides=env,
|
|
)
|
|
|
|
def stop(self, instance: VmInstance) -> None:
|
|
process = self._processes.pop(instance.vm_id, None)
|
|
if process is not None:
|
|
process.terminate()
|
|
try:
|
|
process.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.wait(timeout=5)
|
|
instance.firecracker_pid = None
|
|
return
|
|
if instance.firecracker_pid is None:
|
|
return
|
|
try:
|
|
os.kill(instance.firecracker_pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
instance.firecracker_pid = None
|
|
return
|
|
deadline = time.monotonic() + 5
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
os.kill(instance.firecracker_pid, 0)
|
|
except ProcessLookupError:
|
|
instance.firecracker_pid = None
|
|
return
|
|
time.sleep(0.1)
|
|
os.kill(instance.firecracker_pid, signal.SIGKILL)
|
|
instance.firecracker_pid = None
|
|
|
|
def delete(self, instance: VmInstance) -> None:
|
|
self._processes.pop(instance.vm_id, None)
|
|
if instance.network is not None:
|
|
self._network_manager.cleanup(instance.network)
|
|
shutil.rmtree(instance.workdir, ignore_errors=True)
|
|
|
|
def import_archive(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
archive_path: Path,
|
|
destination: str,
|
|
) -> dict[str, Any]:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
deadline = time.monotonic() + 10
|
|
while True:
|
|
try:
|
|
response = self._guest_exec_client.upload_archive(
|
|
guest_cid,
|
|
port,
|
|
archive_path,
|
|
destination=destination,
|
|
timeout_seconds=WORKSPACE_ARCHIVE_UPLOAD_TIMEOUT_SECONDS,
|
|
uds_path=uds_path,
|
|
)
|
|
return {
|
|
"destination": response.destination,
|
|
"entry_count": response.entry_count,
|
|
"bytes_written": response.bytes_written,
|
|
}
|
|
except (OSError, RuntimeError) as exc:
|
|
if time.monotonic() >= deadline:
|
|
raise RuntimeError(
|
|
f"guest archive transport did not become ready: {exc}"
|
|
) from exc
|
|
time.sleep(0.2)
|
|
instance.metadata["execution_mode"] = "host_compat"
|
|
return _extract_seed_archive_to_host_workspace(
|
|
archive_path,
|
|
workspace_dir=_instance_workspace_host_dir(instance),
|
|
destination=destination,
|
|
)
|
|
|
|
def install_secrets(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
archive_path: Path,
|
|
) -> dict[str, Any]:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
deadline = time.monotonic() + 10
|
|
while True:
|
|
try:
|
|
response = self._guest_exec_client.install_secrets(
|
|
guest_cid,
|
|
port,
|
|
archive_path,
|
|
timeout_seconds=WORKSPACE_ARCHIVE_UPLOAD_TIMEOUT_SECONDS,
|
|
uds_path=uds_path,
|
|
)
|
|
return {
|
|
"destination": response.destination,
|
|
"entry_count": response.entry_count,
|
|
"bytes_written": response.bytes_written,
|
|
}
|
|
except (OSError, RuntimeError) as exc:
|
|
if time.monotonic() >= deadline:
|
|
raise RuntimeError(
|
|
f"guest secret transport did not become ready: {exc}"
|
|
) from exc
|
|
time.sleep(0.2)
|
|
raise RuntimeError("workspace secrets require guest execution")
|
|
|
|
def export_archive(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
archive_path: Path,
|
|
) -> dict[str, Any]:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
deadline = time.monotonic() + 10
|
|
while True:
|
|
try:
|
|
response = self._guest_exec_client.export_archive(
|
|
guest_cid,
|
|
port,
|
|
workspace_path=workspace_path,
|
|
archive_path=archive_path,
|
|
timeout_seconds=WORKSPACE_ARCHIVE_UPLOAD_TIMEOUT_SECONDS,
|
|
uds_path=uds_path,
|
|
)
|
|
return {
|
|
"workspace_path": response.workspace_path,
|
|
"artifact_type": response.artifact_type,
|
|
"entry_count": response.entry_count,
|
|
"bytes_written": response.bytes_written,
|
|
"execution_mode": instance.metadata.get("execution_mode", "pending"),
|
|
}
|
|
except (OSError, RuntimeError) as exc:
|
|
if time.monotonic() >= deadline:
|
|
raise RuntimeError(
|
|
f"guest export transport did not become ready: {exc}"
|
|
) from exc
|
|
time.sleep(0.2)
|
|
instance.metadata["execution_mode"] = "host_compat"
|
|
exported = _prepare_workspace_export_archive(
|
|
workspace_dir=_instance_workspace_host_dir(instance),
|
|
workspace_path=workspace_path,
|
|
archive_path=archive_path,
|
|
)
|
|
return {
|
|
"workspace_path": exported.workspace_path,
|
|
"artifact_type": exported.artifact_type,
|
|
"entry_count": exported.entry_count,
|
|
"bytes_written": exported.bytes_written,
|
|
"execution_mode": "host_compat",
|
|
}
|
|
|
|
def list_workspace_entries(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
recursive: bool,
|
|
) -> dict[str, Any]:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
payload = self._guest_exec_client.list_workspace_entries(
|
|
guest_cid,
|
|
port,
|
|
workspace_path=workspace_path,
|
|
recursive=recursive,
|
|
uds_path=uds_path,
|
|
)
|
|
payload["execution_mode"] = instance.metadata.get("execution_mode", "pending")
|
|
return payload
|
|
instance.metadata["execution_mode"] = "host_compat"
|
|
listing = list_workspace_files(
|
|
_instance_workspace_host_dir(instance),
|
|
workspace_path=workspace_path,
|
|
recursive=recursive,
|
|
)
|
|
return {
|
|
"path": listing.path,
|
|
"artifact_type": listing.artifact_type,
|
|
"entries": [entry.to_payload() for entry in listing.entries],
|
|
"execution_mode": "host_compat",
|
|
}
|
|
|
|
def read_workspace_file(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
max_bytes: int,
|
|
) -> dict[str, Any]:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
payload = self._guest_exec_client.read_workspace_file(
|
|
guest_cid,
|
|
port,
|
|
workspace_path=workspace_path,
|
|
max_bytes=max_bytes,
|
|
uds_path=uds_path,
|
|
)
|
|
payload["execution_mode"] = instance.metadata.get("execution_mode", "pending")
|
|
return payload
|
|
instance.metadata["execution_mode"] = "host_compat"
|
|
file_result = read_workspace_file(
|
|
_instance_workspace_host_dir(instance),
|
|
workspace_path=workspace_path,
|
|
max_bytes=max_bytes,
|
|
)
|
|
return {
|
|
"path": file_result.path,
|
|
"size_bytes": file_result.size_bytes,
|
|
"content_bytes": file_result.content_bytes,
|
|
"execution_mode": "host_compat",
|
|
}
|
|
|
|
def write_workspace_file(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
text: str,
|
|
) -> dict[str, Any]:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
payload = self._guest_exec_client.write_workspace_file(
|
|
guest_cid,
|
|
port,
|
|
workspace_path=workspace_path,
|
|
text=text,
|
|
uds_path=uds_path,
|
|
)
|
|
payload["execution_mode"] = instance.metadata.get("execution_mode", "pending")
|
|
return payload
|
|
instance.metadata["execution_mode"] = "host_compat"
|
|
result = write_workspace_file(
|
|
_instance_workspace_host_dir(instance),
|
|
workspace_path=workspace_path,
|
|
text=text,
|
|
)
|
|
return {
|
|
"path": result.path,
|
|
"size_bytes": result.size_bytes,
|
|
"bytes_written": result.bytes_written,
|
|
"execution_mode": "host_compat",
|
|
}
|
|
|
|
def delete_workspace_path(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_path: str,
|
|
) -> dict[str, Any]:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
payload = self._guest_exec_client.delete_workspace_path(
|
|
guest_cid,
|
|
port,
|
|
workspace_path=workspace_path,
|
|
uds_path=uds_path,
|
|
)
|
|
payload["execution_mode"] = instance.metadata.get("execution_mode", "pending")
|
|
return payload
|
|
instance.metadata["execution_mode"] = "host_compat"
|
|
result = delete_workspace_path(
|
|
_instance_workspace_host_dir(instance),
|
|
workspace_path=workspace_path,
|
|
)
|
|
return {
|
|
"path": result.path,
|
|
"deleted": result.deleted,
|
|
"execution_mode": "host_compat",
|
|
}
|
|
|
|
def open_shell(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
cwd: str,
|
|
cols: int,
|
|
rows: int,
|
|
env: dict[str, str] | None = None,
|
|
redact_values: list[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
del workspace_id
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
response = self._guest_exec_client.open_shell(
|
|
guest_cid,
|
|
port,
|
|
shell_id=shell_id,
|
|
cwd=cwd,
|
|
cols=cols,
|
|
rows=rows,
|
|
env=env,
|
|
redact_values=redact_values,
|
|
uds_path=uds_path,
|
|
)
|
|
return {
|
|
"shell_id": response.shell_id or shell_id,
|
|
"cwd": response.cwd,
|
|
"cols": response.cols,
|
|
"rows": response.rows,
|
|
"state": response.state,
|
|
"started_at": response.started_at,
|
|
"ended_at": response.ended_at,
|
|
"exit_code": response.exit_code,
|
|
"execution_mode": instance.metadata.get("execution_mode", "pending"),
|
|
}
|
|
|
|
def read_shell(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
cursor: int,
|
|
max_chars: int,
|
|
) -> dict[str, Any]:
|
|
del workspace_id
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
response = self._guest_exec_client.read_shell(
|
|
guest_cid,
|
|
port,
|
|
shell_id=shell_id,
|
|
cursor=cursor,
|
|
max_chars=max_chars,
|
|
uds_path=uds_path,
|
|
)
|
|
return {
|
|
"shell_id": response.shell_id,
|
|
"cwd": response.cwd,
|
|
"cols": response.cols,
|
|
"rows": response.rows,
|
|
"state": response.state,
|
|
"started_at": response.started_at,
|
|
"ended_at": response.ended_at,
|
|
"exit_code": response.exit_code,
|
|
"cursor": response.cursor,
|
|
"next_cursor": response.next_cursor,
|
|
"output": response.output,
|
|
"truncated": response.truncated,
|
|
"execution_mode": instance.metadata.get("execution_mode", "pending"),
|
|
}
|
|
|
|
def write_shell(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
input_text: str,
|
|
append_newline: bool,
|
|
) -> dict[str, Any]:
|
|
del workspace_id
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
payload = self._guest_exec_client.write_shell(
|
|
guest_cid,
|
|
port,
|
|
shell_id=shell_id,
|
|
input_text=input_text,
|
|
append_newline=append_newline,
|
|
uds_path=uds_path,
|
|
)
|
|
payload["execution_mode"] = instance.metadata.get("execution_mode", "pending")
|
|
return payload
|
|
|
|
def signal_shell(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
signal_name: str,
|
|
) -> dict[str, Any]:
|
|
del workspace_id
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
payload = self._guest_exec_client.signal_shell(
|
|
guest_cid,
|
|
port,
|
|
shell_id=shell_id,
|
|
signal_name=signal_name,
|
|
uds_path=uds_path,
|
|
)
|
|
payload["execution_mode"] = instance.metadata.get("execution_mode", "pending")
|
|
return payload
|
|
|
|
def close_shell(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
) -> dict[str, Any]:
|
|
del workspace_id
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
payload = self._guest_exec_client.close_shell(
|
|
guest_cid,
|
|
port,
|
|
shell_id=shell_id,
|
|
uds_path=uds_path,
|
|
)
|
|
payload["execution_mode"] = instance.metadata.get("execution_mode", "pending")
|
|
return payload
|
|
|
|
def start_service(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
command: str,
|
|
cwd: str,
|
|
readiness: dict[str, Any] | None,
|
|
ready_timeout_seconds: int,
|
|
ready_interval_ms: int,
|
|
env: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
payload = self._guest_exec_client.start_service(
|
|
guest_cid,
|
|
port,
|
|
service_name=service_name,
|
|
command=command,
|
|
cwd=cwd,
|
|
readiness=readiness,
|
|
ready_timeout_seconds=ready_timeout_seconds,
|
|
ready_interval_ms=ready_interval_ms,
|
|
env=env,
|
|
uds_path=uds_path,
|
|
)
|
|
payload["execution_mode"] = instance.metadata.get("execution_mode", "pending")
|
|
return payload
|
|
if not instance.allow_host_compat:
|
|
raise RuntimeError("services require guest execution or explicit host compatibility")
|
|
service = _start_local_service(
|
|
services_dir=instance.workdir.parent / WORKSPACE_SERVICES_DIRNAME,
|
|
workspace_dir=_instance_workspace_host_dir(instance),
|
|
workspace_id=workspace_id,
|
|
service_name=service_name,
|
|
command=command,
|
|
cwd_text=cwd,
|
|
readiness=readiness,
|
|
ready_timeout_seconds=ready_timeout_seconds,
|
|
ready_interval_ms=ready_interval_ms,
|
|
env_overrides=env,
|
|
)
|
|
return service.to_payload()
|
|
|
|
def status_service(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
) -> dict[str, Any]:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
payload = self._guest_exec_client.status_service(
|
|
guest_cid,
|
|
port,
|
|
service_name=service_name,
|
|
uds_path=uds_path,
|
|
)
|
|
payload["execution_mode"] = instance.metadata.get("execution_mode", "pending")
|
|
return payload
|
|
if not instance.allow_host_compat:
|
|
raise RuntimeError("services require guest execution or explicit host compatibility")
|
|
services_dir = instance.workdir.parent / WORKSPACE_SERVICES_DIRNAME
|
|
record_path = services_dir / f"{service_name}.json"
|
|
if not record_path.exists():
|
|
raise ValueError(
|
|
f"service {service_name!r} does not exist in workspace {workspace_id!r}"
|
|
)
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError(f"service record at {record_path} is invalid")
|
|
service = WorkspaceServiceRecord.from_payload(payload)
|
|
refreshed = _refresh_local_service_record(service, services_dir=services_dir)
|
|
return refreshed.to_payload()
|
|
|
|
def logs_service(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
tail_lines: int | None,
|
|
) -> dict[str, Any]:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
payload = self._guest_exec_client.logs_service(
|
|
guest_cid,
|
|
port,
|
|
service_name=service_name,
|
|
tail_lines=tail_lines,
|
|
uds_path=uds_path,
|
|
)
|
|
payload["execution_mode"] = instance.metadata.get("execution_mode", "pending")
|
|
return payload
|
|
if not instance.allow_host_compat:
|
|
raise RuntimeError("services require guest execution or explicit host compatibility")
|
|
services_dir = instance.workdir.parent / WORKSPACE_SERVICES_DIRNAME
|
|
record_path = services_dir / f"{service_name}.json"
|
|
if not record_path.exists():
|
|
raise ValueError(
|
|
f"service {service_name!r} does not exist in workspace {workspace_id!r}"
|
|
)
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError(f"service record at {record_path} is invalid")
|
|
service = _refresh_local_service_record(
|
|
WorkspaceServiceRecord.from_payload(payload),
|
|
services_dir=services_dir,
|
|
)
|
|
response = service.to_payload()
|
|
stdout, stdout_truncated = _tail_text(
|
|
_workspace_service_stdout_path(services_dir, service_name),
|
|
tail_lines=tail_lines,
|
|
)
|
|
stderr, stderr_truncated = _tail_text(
|
|
_workspace_service_stderr_path(services_dir, service_name),
|
|
tail_lines=tail_lines,
|
|
)
|
|
response.update(
|
|
{
|
|
"stdout": stdout,
|
|
"stderr": stderr,
|
|
"tail_lines": tail_lines,
|
|
"truncated": stdout_truncated or stderr_truncated,
|
|
}
|
|
)
|
|
return response
|
|
|
|
def stop_service(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
) -> dict[str, Any]:
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
guest_cid = int(instance.metadata["guest_cid"])
|
|
port = int(instance.metadata["guest_exec_port"])
|
|
uds_path = instance.metadata.get("guest_exec_uds_path")
|
|
payload = self._guest_exec_client.stop_service(
|
|
guest_cid,
|
|
port,
|
|
service_name=service_name,
|
|
uds_path=uds_path,
|
|
)
|
|
payload["execution_mode"] = instance.metadata.get("execution_mode", "pending")
|
|
return payload
|
|
if not instance.allow_host_compat:
|
|
raise RuntimeError("services require guest execution or explicit host compatibility")
|
|
services_dir = instance.workdir.parent / WORKSPACE_SERVICES_DIRNAME
|
|
record_path = services_dir / f"{service_name}.json"
|
|
if not record_path.exists():
|
|
raise ValueError(
|
|
f"service {service_name!r} does not exist in workspace {workspace_id!r}"
|
|
)
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError(f"service record at {record_path} is invalid")
|
|
service = WorkspaceServiceRecord.from_payload(payload)
|
|
stopped = _stop_local_service(service, services_dir=services_dir)
|
|
return stopped.to_payload()
|
|
|
|
|
|
class VmManager:
|
|
"""In-process lifecycle manager for ephemeral VM environments and workspaces."""
|
|
|
|
MIN_VCPUS = 1
|
|
MAX_VCPUS = 8
|
|
MIN_MEM_MIB = 256
|
|
MAX_MEM_MIB = 32768
|
|
MIN_TTL_SECONDS = 60
|
|
MAX_TTL_SECONDS = 3600
|
|
DEFAULT_VCPU_COUNT = DEFAULT_VCPU_COUNT
|
|
DEFAULT_MEM_MIB = DEFAULT_MEM_MIB
|
|
DEFAULT_TIMEOUT_SECONDS = DEFAULT_TIMEOUT_SECONDS
|
|
DEFAULT_TTL_SECONDS = DEFAULT_TTL_SECONDS
|
|
DEFAULT_ALLOW_HOST_COMPAT = DEFAULT_ALLOW_HOST_COMPAT
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
backend_name: str | None = None,
|
|
base_dir: Path | None = None,
|
|
cache_dir: Path | None = None,
|
|
max_active_vms: int = 4,
|
|
runtime_paths: RuntimePaths | None = None,
|
|
network_manager: TapNetworkManager | None = None,
|
|
) -> None:
|
|
self._backend_name = backend_name or "firecracker"
|
|
self._base_dir = base_dir or Path("/tmp/pyro-mcp")
|
|
self._workspaces_dir = self._base_dir / "workspaces"
|
|
resolved_cache_dir = cache_dir or default_cache_dir()
|
|
self._runtime_paths = runtime_paths
|
|
if self._backend_name == "firecracker":
|
|
self._runtime_paths = self._runtime_paths or resolve_runtime_paths()
|
|
self._runtime_capabilities = runtime_capabilities(self._runtime_paths)
|
|
self._environment_store = EnvironmentStore(
|
|
runtime_paths=self._runtime_paths,
|
|
cache_dir=resolved_cache_dir,
|
|
)
|
|
else:
|
|
self._runtime_capabilities = RuntimeCapabilities(
|
|
supports_vm_boot=False,
|
|
supports_guest_exec=False,
|
|
supports_guest_network=False,
|
|
reason="mock backend does not boot a guest",
|
|
)
|
|
if self._runtime_paths is None:
|
|
self._runtime_paths = resolve_runtime_paths(verify_checksums=False)
|
|
self._environment_store = EnvironmentStore(
|
|
runtime_paths=self._runtime_paths,
|
|
cache_dir=resolved_cache_dir,
|
|
)
|
|
self._max_active_vms = max_active_vms
|
|
if network_manager is not None:
|
|
self._network_manager = network_manager
|
|
elif self._backend_name == "firecracker":
|
|
self._network_manager = TapNetworkManager(enabled=True)
|
|
else:
|
|
self._network_manager = TapNetworkManager(enabled=False)
|
|
self._lock = threading.Lock()
|
|
self._instances: dict[str, VmInstance] = {}
|
|
self._base_dir.mkdir(parents=True, exist_ok=True)
|
|
self._workspaces_dir.mkdir(parents=True, exist_ok=True)
|
|
self._backend = self._build_backend()
|
|
|
|
def _build_backend(self) -> VmBackend:
|
|
if self._backend_name == "mock":
|
|
return MockBackend()
|
|
if self._backend_name == "firecracker":
|
|
if self._runtime_paths is None:
|
|
raise RuntimeError("runtime paths were not initialized for firecracker backend")
|
|
return FirecrackerBackend(
|
|
self._environment_store,
|
|
firecracker_bin=self._runtime_paths.firecracker_bin,
|
|
jailer_bin=self._runtime_paths.jailer_bin,
|
|
runtime_capabilities=self._runtime_capabilities,
|
|
network_manager=self._network_manager,
|
|
)
|
|
raise ValueError("invalid backend; expected one of: mock, firecracker")
|
|
|
|
def list_environments(self) -> list[dict[str, object]]:
|
|
return self._environment_store.list_environments()
|
|
|
|
def pull_environment(self, environment: str) -> dict[str, object]:
|
|
return self._environment_store.pull_environment(environment)
|
|
|
|
def inspect_environment(self, environment: str) -> dict[str, object]:
|
|
return self._environment_store.inspect_environment(environment)
|
|
|
|
def prune_environments(self) -> dict[str, object]:
|
|
return self._environment_store.prune_environments()
|
|
|
|
def create_vm(
|
|
self,
|
|
*,
|
|
environment: str,
|
|
vcpu_count: int = DEFAULT_VCPU_COUNT,
|
|
mem_mib: int = DEFAULT_MEM_MIB,
|
|
ttl_seconds: int = DEFAULT_TTL_SECONDS,
|
|
network: bool = False,
|
|
allow_host_compat: bool = DEFAULT_ALLOW_HOST_COMPAT,
|
|
) -> dict[str, Any]:
|
|
self._validate_limits(vcpu_count=vcpu_count, mem_mib=mem_mib, ttl_seconds=ttl_seconds)
|
|
get_environment(environment, runtime_paths=self._runtime_paths)
|
|
now = time.time()
|
|
with self._lock:
|
|
self._reap_expired_locked(now)
|
|
self._reap_expired_workspaces_locked(now)
|
|
active_count = len(self._instances) + self._count_workspaces_locked()
|
|
if active_count >= self._max_active_vms:
|
|
raise RuntimeError(
|
|
f"max active VMs reached ({self._max_active_vms}); delete old VMs first"
|
|
)
|
|
vm_id = uuid.uuid4().hex[:12]
|
|
instance = VmInstance(
|
|
vm_id=vm_id,
|
|
environment=environment,
|
|
vcpu_count=vcpu_count,
|
|
mem_mib=mem_mib,
|
|
ttl_seconds=ttl_seconds,
|
|
created_at=now,
|
|
expires_at=now + ttl_seconds,
|
|
workdir=self._base_dir / vm_id,
|
|
network_requested=network,
|
|
allow_host_compat=allow_host_compat,
|
|
)
|
|
instance.metadata["allow_host_compat"] = str(allow_host_compat).lower()
|
|
self._backend.create(instance)
|
|
self._instances[vm_id] = instance
|
|
return self._serialize(instance)
|
|
|
|
def run_vm(
|
|
self,
|
|
*,
|
|
environment: str,
|
|
command: str,
|
|
vcpu_count: int = DEFAULT_VCPU_COUNT,
|
|
mem_mib: int = DEFAULT_MEM_MIB,
|
|
timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
|
|
ttl_seconds: int = DEFAULT_TTL_SECONDS,
|
|
network: bool = False,
|
|
allow_host_compat: bool = DEFAULT_ALLOW_HOST_COMPAT,
|
|
) -> dict[str, Any]:
|
|
created = self.create_vm(
|
|
environment=environment,
|
|
vcpu_count=vcpu_count,
|
|
mem_mib=mem_mib,
|
|
ttl_seconds=ttl_seconds,
|
|
network=network,
|
|
allow_host_compat=allow_host_compat,
|
|
)
|
|
vm_id = str(created["vm_id"])
|
|
try:
|
|
self.start_vm(vm_id)
|
|
return self.exec_vm(vm_id, command=command, timeout_seconds=timeout_seconds)
|
|
except Exception:
|
|
try:
|
|
self.delete_vm(vm_id, reason="run_vm_error_cleanup")
|
|
except ValueError:
|
|
pass
|
|
raise
|
|
|
|
def start_vm(self, vm_id: str) -> dict[str, Any]:
|
|
with self._lock:
|
|
instance = self._get_instance_locked(vm_id)
|
|
self._ensure_not_expired_locked(instance, time.time())
|
|
self._start_instance_locked(instance)
|
|
return self._serialize(instance)
|
|
|
|
def exec_vm(self, vm_id: str, *, command: str, timeout_seconds: int) -> dict[str, Any]:
|
|
with self._lock:
|
|
instance = self._get_instance_locked(vm_id)
|
|
self._ensure_not_expired_locked(instance, time.time())
|
|
exec_instance = instance
|
|
exec_result, execution_mode = self._exec_instance(
|
|
exec_instance,
|
|
command=command,
|
|
timeout_seconds=timeout_seconds,
|
|
)
|
|
cleanup = self.delete_vm(vm_id, reason="post_exec_cleanup")
|
|
return {
|
|
"vm_id": vm_id,
|
|
"environment": exec_instance.environment,
|
|
"environment_version": exec_instance.metadata.get("environment_version"),
|
|
"command": command,
|
|
"stdout": exec_result.stdout,
|
|
"stderr": exec_result.stderr,
|
|
"exit_code": exec_result.exit_code,
|
|
"duration_ms": exec_result.duration_ms,
|
|
"execution_mode": execution_mode,
|
|
"cleanup": cleanup,
|
|
}
|
|
|
|
def stop_vm(self, vm_id: str) -> dict[str, Any]:
|
|
with self._lock:
|
|
instance = self._get_instance_locked(vm_id)
|
|
self._backend.stop(instance)
|
|
instance.state = "stopped"
|
|
return self._serialize(instance)
|
|
|
|
def delete_vm(self, vm_id: str, *, reason: str = "explicit_delete") -> dict[str, Any]:
|
|
with self._lock:
|
|
instance = self._get_instance_locked(vm_id)
|
|
if instance.state == "started":
|
|
self._backend.stop(instance)
|
|
instance.state = "stopped"
|
|
self._backend.delete(instance)
|
|
del self._instances[vm_id]
|
|
return {"vm_id": vm_id, "deleted": True, "reason": reason}
|
|
|
|
def status_vm(self, vm_id: str) -> dict[str, Any]:
|
|
with self._lock:
|
|
instance = self._get_instance_locked(vm_id)
|
|
self._ensure_not_expired_locked(instance, time.time())
|
|
return self._serialize(instance)
|
|
|
|
def network_info_vm(self, vm_id: str) -> dict[str, Any]:
|
|
with self._lock:
|
|
instance = self._get_instance_locked(vm_id)
|
|
self._ensure_not_expired_locked(instance, time.time())
|
|
if instance.network is None:
|
|
return {
|
|
"vm_id": vm_id,
|
|
"network_enabled": False,
|
|
"outbound_connectivity_expected": False,
|
|
"reason": "network configuration is unavailable for this VM",
|
|
}
|
|
return {"vm_id": vm_id, **self._network_manager.network_info(instance.network)}
|
|
|
|
def reap_expired(self) -> dict[str, Any]:
|
|
now = time.time()
|
|
with self._lock:
|
|
expired_vm_ids = [
|
|
vm_id for vm_id, inst in self._instances.items() if inst.expires_at <= now
|
|
]
|
|
for vm_id in expired_vm_ids:
|
|
instance = self._instances[vm_id]
|
|
if instance.state == "started":
|
|
self._backend.stop(instance)
|
|
instance.state = "stopped"
|
|
self._backend.delete(instance)
|
|
del self._instances[vm_id]
|
|
return {"deleted_vm_ids": expired_vm_ids, "count": len(expired_vm_ids)}
|
|
|
|
def create_workspace(
|
|
self,
|
|
*,
|
|
environment: str,
|
|
vcpu_count: int = DEFAULT_VCPU_COUNT,
|
|
mem_mib: int = DEFAULT_MEM_MIB,
|
|
ttl_seconds: int = DEFAULT_TTL_SECONDS,
|
|
network_policy: WorkspaceNetworkPolicy | str = DEFAULT_WORKSPACE_NETWORK_POLICY,
|
|
allow_host_compat: bool = DEFAULT_ALLOW_HOST_COMPAT,
|
|
seed_path: str | Path | None = None,
|
|
secrets: list[dict[str, str]] | None = None,
|
|
name: str | None = None,
|
|
labels: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
self._validate_limits(vcpu_count=vcpu_count, mem_mib=mem_mib, ttl_seconds=ttl_seconds)
|
|
get_environment(environment, runtime_paths=self._runtime_paths)
|
|
normalized_network_policy = _normalize_workspace_network_policy(str(network_policy))
|
|
normalized_name = None if name is None else _normalize_workspace_name(name)
|
|
normalized_labels = _normalize_workspace_labels(labels)
|
|
prepared_seed = self._prepare_workspace_seed(seed_path)
|
|
now = time.time()
|
|
workspace_id = uuid.uuid4().hex[:12]
|
|
workspace_dir = self._workspace_dir(workspace_id)
|
|
runtime_dir = self._workspace_runtime_dir(workspace_id)
|
|
host_workspace_dir = self._workspace_host_dir(workspace_id)
|
|
commands_dir = self._workspace_commands_dir(workspace_id)
|
|
shells_dir = self._workspace_shells_dir(workspace_id)
|
|
services_dir = self._workspace_services_dir(workspace_id)
|
|
secrets_dir = self._workspace_secrets_dir(workspace_id)
|
|
snapshots_dir = self._workspace_snapshots_dir(workspace_id)
|
|
baseline_archive_path = self._workspace_baseline_archive_path(workspace_id)
|
|
workspace_dir.mkdir(parents=True, exist_ok=False)
|
|
host_workspace_dir.mkdir(parents=True, exist_ok=True)
|
|
commands_dir.mkdir(parents=True, exist_ok=True)
|
|
shells_dir.mkdir(parents=True, exist_ok=True)
|
|
services_dir.mkdir(parents=True, exist_ok=True)
|
|
secrets_dir.mkdir(parents=True, exist_ok=True)
|
|
snapshots_dir.mkdir(parents=True, exist_ok=True)
|
|
secret_records, _ = _prepare_workspace_secrets(secrets, secrets_dir=secrets_dir)
|
|
_persist_workspace_baseline(
|
|
prepared_seed,
|
|
baseline_archive_path=baseline_archive_path,
|
|
)
|
|
instance = VmInstance(
|
|
vm_id=workspace_id,
|
|
environment=environment,
|
|
vcpu_count=vcpu_count,
|
|
mem_mib=mem_mib,
|
|
ttl_seconds=ttl_seconds,
|
|
created_at=now,
|
|
expires_at=now + ttl_seconds,
|
|
workdir=runtime_dir,
|
|
network_requested=normalized_network_policy != "off",
|
|
allow_host_compat=allow_host_compat,
|
|
)
|
|
instance.metadata["allow_host_compat"] = str(allow_host_compat).lower()
|
|
instance.metadata["workspace_path"] = WORKSPACE_GUEST_PATH
|
|
instance.metadata["workspace_host_dir"] = str(host_workspace_dir)
|
|
instance.metadata["network_policy"] = normalized_network_policy
|
|
try:
|
|
with self._lock:
|
|
self._reap_expired_locked(now)
|
|
self._reap_expired_workspaces_locked(now)
|
|
active_count = len(self._instances) + self._count_workspaces_locked()
|
|
if active_count >= self._max_active_vms:
|
|
raise RuntimeError(
|
|
f"max active VMs reached ({self._max_active_vms}); delete old VMs first"
|
|
)
|
|
self._require_workspace_network_policy_support(
|
|
network_policy=normalized_network_policy
|
|
)
|
|
self._backend.create(instance)
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
self._ensure_workspace_guest_bootstrap_support(instance)
|
|
with self._lock:
|
|
self._start_instance_locked(instance)
|
|
workspace = WorkspaceRecord.from_instance(
|
|
instance,
|
|
network_policy=normalized_network_policy,
|
|
workspace_seed=prepared_seed.to_payload(),
|
|
secrets=secret_records,
|
|
name=normalized_name,
|
|
labels=normalized_labels,
|
|
)
|
|
if workspace.secrets:
|
|
self._install_workspace_secrets_locked(workspace, instance)
|
|
self._require_guest_exec_or_opt_in(instance)
|
|
import_summary = self._backend.import_archive(
|
|
instance,
|
|
archive_path=baseline_archive_path,
|
|
destination=WORKSPACE_GUEST_PATH,
|
|
)
|
|
workspace_seed = dict(workspace.workspace_seed)
|
|
workspace_seed["entry_count"] = int(import_summary["entry_count"])
|
|
workspace_seed["bytes_written"] = int(import_summary["bytes_written"])
|
|
workspace_seed["destination"] = str(import_summary["destination"])
|
|
workspace.workspace_seed = workspace_seed
|
|
self._save_workspace_locked(workspace)
|
|
return self._serialize_workspace(workspace)
|
|
except Exception:
|
|
if runtime_dir.exists():
|
|
try:
|
|
if instance.state == "started":
|
|
self._backend.stop(instance)
|
|
instance.state = "stopped"
|
|
except Exception:
|
|
pass
|
|
try:
|
|
self._backend.delete(instance)
|
|
except Exception:
|
|
pass
|
|
shutil.rmtree(workspace_dir, ignore_errors=True)
|
|
raise
|
|
finally:
|
|
prepared_seed.cleanup()
|
|
|
|
def push_workspace_sync(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
source_path: str | Path,
|
|
dest: str = WORKSPACE_GUEST_PATH,
|
|
) -> dict[str, Any]:
|
|
prepared_seed = self._prepare_workspace_seed(source_path)
|
|
if prepared_seed.archive_path is None:
|
|
prepared_seed.cleanup()
|
|
raise ValueError("source_path is required")
|
|
normalized_destination, _ = _normalize_workspace_destination(dest)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
self._refresh_workspace_liveness_locked(workspace)
|
|
if workspace.state != "started":
|
|
raise RuntimeError(
|
|
f"workspace {workspace_id} must be in 'started' state "
|
|
"before workspace_sync_push"
|
|
)
|
|
instance = workspace.to_instance(
|
|
workdir=self._workspace_runtime_dir(workspace.workspace_id)
|
|
)
|
|
try:
|
|
import_summary = self._backend.import_archive(
|
|
instance,
|
|
archive_path=prepared_seed.archive_path,
|
|
destination=normalized_destination,
|
|
)
|
|
finally:
|
|
prepared_seed.cleanup()
|
|
workspace_sync = prepared_seed.to_payload(
|
|
destination=normalized_destination,
|
|
path_key="source_path",
|
|
)
|
|
workspace_sync["entry_count"] = int(import_summary["entry_count"])
|
|
workspace_sync["bytes_written"] = int(import_summary["bytes_written"])
|
|
workspace_sync["destination"] = str(import_summary["destination"])
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"execution_mode": instance.metadata.get("execution_mode", "pending"),
|
|
"workspace_sync": workspace_sync,
|
|
}
|
|
|
|
def list_workspaces(self) -> dict[str, Any]:
|
|
with self._lock:
|
|
now = time.time()
|
|
self._reap_expired_workspaces_locked(now)
|
|
workspaces: list[WorkspaceRecord] = []
|
|
for metadata_path in self._workspaces_dir.glob("*/workspace.json"):
|
|
payload = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
continue
|
|
workspace = WorkspaceRecord.from_payload(payload)
|
|
self._refresh_workspace_liveness_locked(workspace)
|
|
self._refresh_workspace_service_counts_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
workspaces.append(workspace)
|
|
workspaces.sort(
|
|
key=lambda item: (
|
|
-item.last_activity_at,
|
|
-item.created_at,
|
|
item.workspace_id,
|
|
)
|
|
)
|
|
return {
|
|
"count": len(workspaces),
|
|
"workspaces": [
|
|
self._serialize_workspace_list_item(workspace) for workspace in workspaces
|
|
],
|
|
}
|
|
|
|
def update_workspace(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
name: str | None = None,
|
|
clear_name: bool = False,
|
|
labels: dict[str, str] | None = None,
|
|
clear_labels: list[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
if name is not None and clear_name:
|
|
raise ValueError("name and clear_name cannot be used together")
|
|
normalized_name = None if name is None else _normalize_workspace_name(name)
|
|
normalized_labels = None if labels is None else _normalize_workspace_labels(labels)
|
|
normalized_clear_labels = [
|
|
_normalize_workspace_label_key(label_key) for label_key in (clear_labels or [])
|
|
]
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
updated = False
|
|
if clear_name:
|
|
if workspace.name is not None:
|
|
workspace.name = None
|
|
updated = True
|
|
elif normalized_name is not None and workspace.name != normalized_name:
|
|
workspace.name = normalized_name
|
|
updated = True
|
|
if normalized_labels is not None:
|
|
for label_key, label_value in normalized_labels.items():
|
|
if workspace.labels.get(label_key) != label_value:
|
|
workspace.labels[label_key] = label_value
|
|
updated = True
|
|
for label_key in normalized_clear_labels:
|
|
if label_key in workspace.labels:
|
|
del workspace.labels[label_key]
|
|
updated = True
|
|
workspace.labels = dict(sorted(workspace.labels.items()))
|
|
if not updated:
|
|
raise ValueError("workspace update requested no effective metadata change")
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
return self._serialize_workspace(workspace)
|
|
|
|
def export_workspace(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
path: str,
|
|
output_path: str | Path,
|
|
) -> dict[str, Any]:
|
|
normalized_path, _ = _normalize_workspace_destination(path)
|
|
raw_output_path = str(output_path).strip()
|
|
if raw_output_path == "":
|
|
raise ValueError("output_path must not be empty")
|
|
resolved_output_path = Path(output_path).expanduser().resolve()
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_operation_locked(
|
|
workspace,
|
|
operation_name="workspace_export",
|
|
)
|
|
with tempfile.TemporaryDirectory(prefix="pyro-workspace-export-") as temp_dir:
|
|
archive_path = Path(temp_dir) / "workspace-export.tar"
|
|
exported = self._backend.export_archive(
|
|
instance,
|
|
workspace_path=normalized_path,
|
|
archive_path=archive_path,
|
|
)
|
|
extracted = _extract_workspace_export_archive(
|
|
archive_path,
|
|
output_path=resolved_output_path,
|
|
artifact_type=cast(WorkspaceArtifactType, str(exported["artifact_type"])),
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._save_workspace_locked(workspace)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"workspace_path": normalized_path,
|
|
"output_path": str(Path(str(extracted["output_path"]))),
|
|
"artifact_type": extracted["artifact_type"],
|
|
"entry_count": int(extracted["entry_count"]),
|
|
"bytes_written": int(extracted["bytes_written"]),
|
|
"execution_mode": str(
|
|
exported.get("execution_mode", instance.metadata.get("execution_mode", "pending"))
|
|
),
|
|
}
|
|
|
|
def diff_workspace(self, workspace_id: str) -> dict[str, Any]:
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_operation_locked(
|
|
workspace,
|
|
operation_name="workspace_diff",
|
|
)
|
|
baseline_archive_path = self._workspace_baseline_archive_path(workspace_id)
|
|
if not baseline_archive_path.exists():
|
|
raise RuntimeError(
|
|
"workspace diff requires a baseline snapshot. Recreate the workspace to use diff."
|
|
)
|
|
with tempfile.TemporaryDirectory(prefix="pyro-workspace-diff-") as temp_dir:
|
|
temp_root = Path(temp_dir)
|
|
current_archive_path = temp_root / "current.tar"
|
|
baseline_root = temp_root / "baseline"
|
|
current_root = temp_root / "current"
|
|
self._backend.export_archive(
|
|
instance,
|
|
workspace_path=WORKSPACE_GUEST_PATH,
|
|
archive_path=current_archive_path,
|
|
)
|
|
_extract_seed_archive_to_host_workspace(
|
|
baseline_archive_path,
|
|
workspace_dir=baseline_root,
|
|
destination=WORKSPACE_GUEST_PATH,
|
|
)
|
|
_extract_seed_archive_to_host_workspace(
|
|
current_archive_path,
|
|
workspace_dir=current_root,
|
|
destination=WORKSPACE_GUEST_PATH,
|
|
)
|
|
diff_payload = _diff_workspace_trees(baseline_root, current_root)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._save_workspace_locked(workspace)
|
|
diff_payload["workspace_id"] = workspace_id
|
|
return diff_payload
|
|
|
|
def list_workspace_files(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
path: str = WORKSPACE_GUEST_PATH,
|
|
recursive: bool = False,
|
|
) -> dict[str, Any]:
|
|
normalized_path = _normalize_workspace_file_path(path)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_operation_locked(
|
|
workspace,
|
|
operation_name="workspace_file_list",
|
|
)
|
|
listing = self._backend.list_workspace_entries(
|
|
instance,
|
|
workspace_path=normalized_path,
|
|
recursive=recursive,
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._save_workspace_locked(workspace)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"path": str(listing["path"]),
|
|
"recursive": recursive,
|
|
"entries": cast(list[dict[str, Any]], list(listing.get("entries", []))),
|
|
"execution_mode": str(
|
|
listing.get("execution_mode", instance.metadata.get("execution_mode", "pending"))
|
|
),
|
|
}
|
|
|
|
def read_workspace_file(
|
|
self,
|
|
workspace_id: str,
|
|
path: str,
|
|
*,
|
|
max_bytes: int = DEFAULT_WORKSPACE_FILE_READ_MAX_BYTES,
|
|
) -> dict[str, Any]:
|
|
normalized_path = _normalize_workspace_file_path(path)
|
|
normalized_max_bytes = _validate_workspace_file_read_max_bytes(max_bytes)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_operation_locked(
|
|
workspace,
|
|
operation_name="workspace_file_read",
|
|
)
|
|
payload = self._backend.read_workspace_file(
|
|
instance,
|
|
workspace_path=normalized_path,
|
|
max_bytes=WORKSPACE_FILE_MAX_BYTES,
|
|
)
|
|
raw_bytes = cast(bytes, payload["content_bytes"])
|
|
content = raw_bytes[:normalized_max_bytes].decode("utf-8", errors="replace")
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"path": str(payload["path"]),
|
|
"size_bytes": int(payload["size_bytes"]),
|
|
"max_bytes": normalized_max_bytes,
|
|
"content": content,
|
|
"truncated": len(raw_bytes) > normalized_max_bytes,
|
|
"execution_mode": str(
|
|
payload.get("execution_mode", instance.metadata.get("execution_mode", "pending"))
|
|
),
|
|
}
|
|
|
|
def write_workspace_file(
|
|
self,
|
|
workspace_id: str,
|
|
path: str,
|
|
*,
|
|
text: str,
|
|
) -> dict[str, Any]:
|
|
normalized_path = _normalize_workspace_file_path(path)
|
|
normalized_text = _validate_workspace_text_payload(text, field_name="text")
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_operation_locked(
|
|
workspace,
|
|
operation_name="workspace_file_write",
|
|
)
|
|
payload = self._backend.write_workspace_file(
|
|
instance,
|
|
workspace_path=normalized_path,
|
|
text=normalized_text,
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._save_workspace_locked(workspace)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"path": str(payload["path"]),
|
|
"size_bytes": int(payload["size_bytes"]),
|
|
"bytes_written": int(payload["bytes_written"]),
|
|
"execution_mode": str(
|
|
payload.get("execution_mode", instance.metadata.get("execution_mode", "pending"))
|
|
),
|
|
}
|
|
|
|
def apply_workspace_patch(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
patch: str,
|
|
) -> dict[str, Any]:
|
|
patch_text = _validate_workspace_patch_text(patch)
|
|
parsed_patches = parse_unified_text_patch(patch_text)
|
|
patch_by_path: dict[str, WorkspaceTextPatch] = {}
|
|
for text_patch in parsed_patches:
|
|
if text_patch.path in patch_by_path:
|
|
raise ValueError(f"patch contains duplicate file entries for {text_patch.path}")
|
|
patch_by_path[text_patch.path] = text_patch
|
|
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_operation_locked(
|
|
workspace,
|
|
operation_name="workspace_patch_apply",
|
|
)
|
|
|
|
planned_writes: dict[str, str] = {}
|
|
planned_deletes: list[str] = []
|
|
summary = {
|
|
"total": 0,
|
|
"added": 0,
|
|
"modified": 0,
|
|
"deleted": 0,
|
|
}
|
|
entries: list[dict[str, str]] = []
|
|
|
|
for path_text in sorted(patch_by_path):
|
|
file_patch = patch_by_path[path_text]
|
|
listing: dict[str, Any] | None = None
|
|
current_text: str | None = None
|
|
exists = True
|
|
try:
|
|
listing = self._backend.list_workspace_entries(
|
|
instance,
|
|
workspace_path=file_patch.path,
|
|
recursive=False,
|
|
)
|
|
except RuntimeError as exc:
|
|
if "does not exist" in str(exc):
|
|
exists = False
|
|
else:
|
|
raise
|
|
if exists:
|
|
if listing is None:
|
|
raise RuntimeError(
|
|
f"workspace patch could not inspect current path: {file_patch.path}"
|
|
)
|
|
artifact_type = str(listing["artifact_type"])
|
|
if artifact_type != "file":
|
|
raise RuntimeError(
|
|
f"workspace patch only supports regular files: {file_patch.path}"
|
|
)
|
|
current_payload = self._backend.read_workspace_file(
|
|
instance,
|
|
workspace_path=file_patch.path,
|
|
max_bytes=WORKSPACE_FILE_MAX_BYTES,
|
|
)
|
|
current_text = _decode_workspace_patch_text(
|
|
file_patch.path,
|
|
cast(bytes, current_payload["content_bytes"]),
|
|
)
|
|
if file_patch.status == "added" and exists:
|
|
raise RuntimeError(
|
|
f"workspace patch cannot add an existing path: {file_patch.path}"
|
|
)
|
|
if file_patch.status in {"modified", "deleted"} and not exists:
|
|
raise RuntimeError(
|
|
f"workspace patch cannot modify a missing path: {file_patch.path}"
|
|
)
|
|
after_text = apply_unified_text_patch(
|
|
path=file_patch.path,
|
|
patch=file_patch,
|
|
before_text=current_text,
|
|
)
|
|
if after_text is None:
|
|
planned_deletes.append(file_patch.path)
|
|
else:
|
|
planned_writes[file_patch.path] = after_text
|
|
summary["total"] += 1
|
|
summary[file_patch.status] += 1
|
|
entries.append({"path": file_patch.path, "status": file_patch.status})
|
|
|
|
for path_text in sorted(planned_writes):
|
|
self._backend.write_workspace_file(
|
|
instance,
|
|
workspace_path=path_text,
|
|
text=planned_writes[path_text],
|
|
)
|
|
for path_text in sorted(planned_deletes):
|
|
self._backend.delete_workspace_path(
|
|
instance,
|
|
workspace_path=path_text,
|
|
)
|
|
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"changed": bool(entries),
|
|
"summary": summary,
|
|
"entries": entries,
|
|
"patch": patch_text,
|
|
"execution_mode": instance.metadata.get("execution_mode", "pending"),
|
|
}
|
|
|
|
def create_snapshot(
|
|
self,
|
|
workspace_id: str,
|
|
snapshot_name: str,
|
|
) -> dict[str, Any]:
|
|
normalized_snapshot_name = _normalize_workspace_snapshot_name(snapshot_name)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
self._workspace_baseline_snapshot_locked(workspace)
|
|
if (
|
|
self._load_workspace_snapshot_locked_optional(
|
|
workspace_id,
|
|
normalized_snapshot_name,
|
|
)
|
|
is not None
|
|
):
|
|
raise ValueError(
|
|
f"snapshot {normalized_snapshot_name!r} already exists in workspace "
|
|
f"{workspace_id!r}"
|
|
)
|
|
instance = self._workspace_instance_for_live_operation_locked(
|
|
workspace,
|
|
operation_name="workspace_snapshot_create",
|
|
)
|
|
with tempfile.TemporaryDirectory(prefix="pyro-workspace-snapshot-") as temp_dir:
|
|
temp_archive_path = Path(temp_dir) / f"{normalized_snapshot_name}.tar"
|
|
exported = self._backend.export_archive(
|
|
instance,
|
|
workspace_path=WORKSPACE_GUEST_PATH,
|
|
archive_path=temp_archive_path,
|
|
)
|
|
snapshot = WorkspaceSnapshotRecord(
|
|
workspace_id=workspace_id,
|
|
snapshot_name=normalized_snapshot_name,
|
|
kind="named",
|
|
created_at=time.time(),
|
|
entry_count=int(exported["entry_count"]),
|
|
bytes_written=int(exported["bytes_written"]),
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
if (
|
|
self._load_workspace_snapshot_locked_optional(
|
|
workspace_id,
|
|
normalized_snapshot_name,
|
|
)
|
|
is not None
|
|
):
|
|
raise ValueError(
|
|
f"snapshot {normalized_snapshot_name!r} already exists in workspace "
|
|
f"{workspace_id!r}"
|
|
)
|
|
archive_path = self._workspace_snapshot_archive_path(
|
|
workspace_id,
|
|
normalized_snapshot_name,
|
|
)
|
|
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(temp_archive_path, archive_path)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
self._save_workspace_snapshot_locked(snapshot)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"snapshot": self._serialize_workspace_snapshot(snapshot),
|
|
"execution_mode": instance.metadata.get("execution_mode", "pending"),
|
|
}
|
|
|
|
def list_snapshots(self, workspace_id: str) -> dict[str, Any]:
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
snapshots = self._list_workspace_snapshots_locked(workspace)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"count": len(snapshots),
|
|
"snapshots": [
|
|
self._serialize_workspace_snapshot(snapshot) for snapshot in snapshots
|
|
],
|
|
}
|
|
|
|
def delete_snapshot(self, workspace_id: str, snapshot_name: str) -> dict[str, Any]:
|
|
normalized_snapshot_name = _normalize_workspace_snapshot_name(
|
|
snapshot_name,
|
|
allow_baseline=True,
|
|
)
|
|
if normalized_snapshot_name == "baseline":
|
|
raise ValueError("cannot delete the baseline snapshot")
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
self._workspace_baseline_snapshot_locked(workspace)
|
|
self._load_workspace_snapshot_locked(workspace_id, normalized_snapshot_name)
|
|
self._delete_workspace_snapshot_locked(workspace_id, normalized_snapshot_name)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"snapshot_name": normalized_snapshot_name,
|
|
"deleted": True,
|
|
}
|
|
|
|
def reset_workspace(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
snapshot: str = "baseline",
|
|
) -> dict[str, Any]:
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
self._refresh_workspace_liveness_locked(workspace)
|
|
selected_snapshot, archive_path = self._resolve_workspace_snapshot_locked(
|
|
workspace,
|
|
snapshot,
|
|
)
|
|
instance = workspace.to_instance(
|
|
workdir=self._workspace_runtime_dir(workspace.workspace_id)
|
|
)
|
|
self._stop_workspace_services_locked(workspace, instance)
|
|
self._close_workspace_shells_locked(workspace, instance)
|
|
if workspace.state == "started":
|
|
self._backend.stop(instance)
|
|
workspace.state = "stopped"
|
|
self._backend.delete(instance)
|
|
workspace.state = "stopped"
|
|
workspace.firecracker_pid = None
|
|
workspace.last_error = None
|
|
self._reset_workspace_runtime_dirs(workspace_id)
|
|
self._save_workspace_locked(workspace)
|
|
recreated: VmInstance | None = None
|
|
try:
|
|
recreated = workspace.to_instance(
|
|
workdir=self._workspace_runtime_dir(workspace.workspace_id)
|
|
)
|
|
self._require_workspace_network_policy_support(
|
|
network_policy=workspace.network_policy
|
|
)
|
|
self._backend.create(recreated)
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
self._ensure_workspace_guest_bootstrap_support(recreated)
|
|
with self._lock:
|
|
self._start_instance_locked(recreated)
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
if workspace.secrets:
|
|
self._install_workspace_secrets_locked(workspace, recreated)
|
|
self._require_guest_exec_or_opt_in(recreated)
|
|
reset_summary = self._backend.import_archive(
|
|
recreated,
|
|
archive_path=archive_path,
|
|
destination=WORKSPACE_GUEST_PATH,
|
|
)
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = recreated.state
|
|
workspace.firecracker_pid = recreated.firecracker_pid
|
|
workspace.last_error = recreated.last_error
|
|
workspace.metadata = dict(recreated.metadata)
|
|
workspace.command_count = 0
|
|
workspace.last_command = None
|
|
workspace.reset_count += 1
|
|
workspace.last_reset_at = time.time()
|
|
self._touch_workspace_activity_locked(workspace, when=workspace.last_reset_at)
|
|
self._save_workspace_locked(workspace)
|
|
payload = self._serialize_workspace(workspace)
|
|
except Exception:
|
|
try:
|
|
if recreated is not None and recreated.state == "started":
|
|
self._backend.stop(recreated)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
if recreated is not None:
|
|
self._backend.delete(recreated)
|
|
except Exception:
|
|
pass
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = "stopped"
|
|
workspace.firecracker_pid = None
|
|
workspace.last_error = None
|
|
self._save_workspace_locked(workspace)
|
|
raise
|
|
payload["workspace_reset"] = {
|
|
"snapshot_name": selected_snapshot.snapshot_name,
|
|
"kind": selected_snapshot.kind,
|
|
"destination": str(reset_summary["destination"]),
|
|
"entry_count": int(reset_summary["entry_count"]),
|
|
"bytes_written": int(reset_summary["bytes_written"]),
|
|
}
|
|
return payload
|
|
|
|
def exec_workspace(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
command: str,
|
|
timeout_seconds: int = 30,
|
|
secret_env: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
if timeout_seconds <= 0:
|
|
raise ValueError("timeout_seconds must be positive")
|
|
normalized_secret_env = _normalize_workspace_secret_env_mapping(secret_env)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
self._refresh_workspace_liveness_locked(workspace)
|
|
if workspace.state != "started":
|
|
raise RuntimeError(
|
|
f"workspace {workspace_id} must be in 'started' state before workspace_exec"
|
|
)
|
|
instance = workspace.to_instance(
|
|
workdir=self._workspace_runtime_dir(workspace.workspace_id)
|
|
)
|
|
redact_values = self._workspace_secret_redact_values_locked(workspace)
|
|
env_values = self._workspace_secret_env_values_locked(workspace, normalized_secret_env)
|
|
if workspace.secrets and normalized_secret_env:
|
|
self._install_workspace_secrets_locked(workspace, instance)
|
|
try:
|
|
exec_result, execution_mode = self._exec_instance(
|
|
instance,
|
|
command=command,
|
|
timeout_seconds=timeout_seconds,
|
|
host_workdir=self._workspace_host_dir(workspace.workspace_id),
|
|
guest_cwd=WORKSPACE_GUEST_PATH,
|
|
env=env_values or None,
|
|
)
|
|
except Exception as exc:
|
|
raise _redact_exception(exc, redact_values) from exc
|
|
redacted_exec_result = VmExecResult(
|
|
stdout=_redact_text(exec_result.stdout, redact_values),
|
|
stderr=_redact_text(exec_result.stderr, redact_values),
|
|
exit_code=exec_result.exit_code,
|
|
duration_ms=exec_result.duration_ms,
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
entry = self._record_workspace_command_locked(
|
|
workspace,
|
|
command=command,
|
|
exec_result=redacted_exec_result,
|
|
execution_mode=execution_mode,
|
|
cwd=WORKSPACE_GUEST_PATH,
|
|
)
|
|
self._save_workspace_locked(workspace)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"environment": workspace.environment,
|
|
"environment_version": workspace.metadata.get("environment_version"),
|
|
"command": command,
|
|
"stdout": redacted_exec_result.stdout,
|
|
"stderr": redacted_exec_result.stderr,
|
|
"exit_code": redacted_exec_result.exit_code,
|
|
"duration_ms": redacted_exec_result.duration_ms,
|
|
"execution_mode": execution_mode,
|
|
"sequence": entry["sequence"],
|
|
"cwd": WORKSPACE_GUEST_PATH,
|
|
}
|
|
|
|
def open_shell(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
cwd: str = WORKSPACE_GUEST_PATH,
|
|
cols: int = DEFAULT_SHELL_COLS,
|
|
rows: int = DEFAULT_SHELL_ROWS,
|
|
secret_env: dict[str, str] | None = None,
|
|
) -> dict[str, Any]:
|
|
if cols <= 0:
|
|
raise ValueError("cols must be positive")
|
|
if rows <= 0:
|
|
raise ValueError("rows must be positive")
|
|
normalized_cwd, _ = _normalize_workspace_destination(cwd)
|
|
normalized_secret_env = _normalize_workspace_secret_env_mapping(secret_env)
|
|
shell_id = uuid.uuid4().hex[:12]
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_shell_locked(workspace)
|
|
redact_values = self._workspace_secret_redact_values_locked(workspace)
|
|
env_values = self._workspace_secret_env_values_locked(workspace, normalized_secret_env)
|
|
if workspace.secrets and normalized_secret_env:
|
|
self._install_workspace_secrets_locked(workspace, instance)
|
|
try:
|
|
payload = self._backend.open_shell(
|
|
instance,
|
|
workspace_id=workspace_id,
|
|
shell_id=shell_id,
|
|
cwd=normalized_cwd,
|
|
cols=cols,
|
|
rows=rows,
|
|
env=env_values or None,
|
|
redact_values=redact_values,
|
|
)
|
|
except Exception as exc:
|
|
raise _redact_exception(exc, redact_values) from exc
|
|
shell = self._workspace_shell_record_from_payload(
|
|
workspace_id=workspace_id,
|
|
shell_id=shell_id,
|
|
payload=payload,
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
self._save_workspace_shell_locked(shell)
|
|
return self._serialize_workspace_shell(shell)
|
|
|
|
def read_shell(
|
|
self,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
*,
|
|
cursor: int = 0,
|
|
max_chars: int = DEFAULT_SHELL_MAX_CHARS,
|
|
) -> dict[str, Any]:
|
|
if cursor < 0:
|
|
raise ValueError("cursor must not be negative")
|
|
if max_chars <= 0:
|
|
raise ValueError("max_chars must be positive")
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_shell_locked(workspace)
|
|
shell = self._load_workspace_shell_locked(workspace_id, shell_id)
|
|
redact_values = self._workspace_secret_redact_values_locked(workspace)
|
|
try:
|
|
payload = self._backend.read_shell(
|
|
instance,
|
|
workspace_id=workspace_id,
|
|
shell_id=shell_id,
|
|
cursor=cursor,
|
|
max_chars=max_chars,
|
|
)
|
|
except Exception as exc:
|
|
raise _redact_exception(exc, redact_values) from exc
|
|
updated_shell = self._workspace_shell_record_from_payload(
|
|
workspace_id=workspace_id,
|
|
shell_id=shell_id,
|
|
payload=payload,
|
|
metadata=shell.metadata,
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._save_workspace_locked(workspace)
|
|
self._save_workspace_shell_locked(updated_shell)
|
|
response = self._serialize_workspace_shell(updated_shell)
|
|
response.update(
|
|
{
|
|
"cursor": int(payload.get("cursor", cursor)),
|
|
"next_cursor": int(payload.get("next_cursor", cursor)),
|
|
"output": _redact_text(str(payload.get("output", "")), redact_values),
|
|
"truncated": bool(payload.get("truncated", False)),
|
|
}
|
|
)
|
|
return response
|
|
|
|
def write_shell(
|
|
self,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
*,
|
|
input_text: str,
|
|
append_newline: bool = True,
|
|
) -> dict[str, Any]:
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_shell_locked(workspace)
|
|
shell = self._load_workspace_shell_locked(workspace_id, shell_id)
|
|
redact_values = self._workspace_secret_redact_values_locked(workspace)
|
|
try:
|
|
payload = self._backend.write_shell(
|
|
instance,
|
|
workspace_id=workspace_id,
|
|
shell_id=shell_id,
|
|
input_text=input_text,
|
|
append_newline=append_newline,
|
|
)
|
|
except Exception as exc:
|
|
raise _redact_exception(exc, redact_values) from exc
|
|
updated_shell = self._workspace_shell_record_from_payload(
|
|
workspace_id=workspace_id,
|
|
shell_id=shell_id,
|
|
payload=payload,
|
|
metadata=shell.metadata,
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
self._save_workspace_shell_locked(updated_shell)
|
|
response = self._serialize_workspace_shell(updated_shell)
|
|
response.update(
|
|
{
|
|
"input_length": int(payload.get("input_length", len(input_text))),
|
|
"append_newline": bool(payload.get("append_newline", append_newline)),
|
|
}
|
|
)
|
|
return response
|
|
|
|
def signal_shell(
|
|
self,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
*,
|
|
signal_name: str = "INT",
|
|
) -> dict[str, Any]:
|
|
normalized_signal = signal_name.upper()
|
|
if normalized_signal not in WORKSPACE_SHELL_SIGNAL_NAMES:
|
|
raise ValueError(
|
|
f"signal_name must be one of: {', '.join(WORKSPACE_SHELL_SIGNAL_NAMES)}"
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_shell_locked(workspace)
|
|
shell = self._load_workspace_shell_locked(workspace_id, shell_id)
|
|
redact_values = self._workspace_secret_redact_values_locked(workspace)
|
|
try:
|
|
payload = self._backend.signal_shell(
|
|
instance,
|
|
workspace_id=workspace_id,
|
|
shell_id=shell_id,
|
|
signal_name=normalized_signal,
|
|
)
|
|
except Exception as exc:
|
|
raise _redact_exception(exc, redact_values) from exc
|
|
updated_shell = self._workspace_shell_record_from_payload(
|
|
workspace_id=workspace_id,
|
|
shell_id=shell_id,
|
|
payload=payload,
|
|
metadata=shell.metadata,
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
self._save_workspace_shell_locked(updated_shell)
|
|
response = self._serialize_workspace_shell(updated_shell)
|
|
response["signal"] = str(payload.get("signal", normalized_signal))
|
|
return response
|
|
|
|
def close_shell(
|
|
self,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
) -> dict[str, Any]:
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_shell_locked(workspace)
|
|
shell = self._load_workspace_shell_locked(workspace_id, shell_id)
|
|
redact_values = self._workspace_secret_redact_values_locked(workspace)
|
|
try:
|
|
payload = self._backend.close_shell(
|
|
instance,
|
|
workspace_id=workspace_id,
|
|
shell_id=shell_id,
|
|
)
|
|
except Exception as exc:
|
|
raise _redact_exception(exc, redact_values) from exc
|
|
closed_shell = self._workspace_shell_record_from_payload(
|
|
workspace_id=workspace_id,
|
|
shell_id=shell_id,
|
|
payload=payload,
|
|
metadata=shell.metadata,
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
self._delete_workspace_shell_locked(workspace_id, shell_id)
|
|
response = self._serialize_workspace_shell(closed_shell)
|
|
response["closed"] = bool(payload.get("closed", True))
|
|
return response
|
|
|
|
def start_service(
|
|
self,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
*,
|
|
command: str,
|
|
cwd: str = WORKSPACE_GUEST_PATH,
|
|
readiness: dict[str, Any] | None = None,
|
|
ready_timeout_seconds: int = DEFAULT_SERVICE_READY_TIMEOUT_SECONDS,
|
|
ready_interval_ms: int = DEFAULT_SERVICE_READY_INTERVAL_MS,
|
|
secret_env: dict[str, str] | None = None,
|
|
published_ports: list[dict[str, Any]] | None = None,
|
|
) -> dict[str, Any]:
|
|
normalized_service_name = _normalize_workspace_service_name(service_name)
|
|
normalized_cwd, _ = _normalize_workspace_destination(cwd)
|
|
normalized_readiness = _normalize_workspace_service_readiness(readiness)
|
|
normalized_secret_env = _normalize_workspace_secret_env_mapping(secret_env)
|
|
normalized_published_ports = _normalize_workspace_published_port_specs(published_ports)
|
|
if ready_timeout_seconds <= 0:
|
|
raise ValueError("ready_timeout_seconds must be positive")
|
|
if ready_interval_ms <= 0:
|
|
raise ValueError("ready_interval_ms must be positive")
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_service_locked(workspace)
|
|
if normalized_published_ports:
|
|
if workspace.network_policy != "egress+published-ports":
|
|
raise RuntimeError(
|
|
"published ports require workspace network_policy "
|
|
"'egress+published-ports'"
|
|
)
|
|
if instance.network is None:
|
|
raise RuntimeError(
|
|
"published ports require an active guest network configuration"
|
|
)
|
|
redact_values = self._workspace_secret_redact_values_locked(workspace)
|
|
env_values = self._workspace_secret_env_values_locked(workspace, normalized_secret_env)
|
|
if workspace.secrets and normalized_secret_env:
|
|
self._install_workspace_secrets_locked(workspace, instance)
|
|
existing = self._load_workspace_service_locked_optional(
|
|
workspace_id,
|
|
normalized_service_name,
|
|
)
|
|
if existing is not None:
|
|
existing = self._refresh_workspace_service_locked(
|
|
workspace,
|
|
instance,
|
|
existing,
|
|
)
|
|
if existing.state == "running":
|
|
raise RuntimeError(
|
|
f"service {normalized_service_name!r} is already running in "
|
|
f"workspace {workspace_id!r}"
|
|
)
|
|
self._delete_workspace_service_artifacts_locked(
|
|
workspace_id,
|
|
normalized_service_name,
|
|
)
|
|
try:
|
|
payload = self._backend.start_service(
|
|
instance,
|
|
workspace_id=workspace_id,
|
|
service_name=normalized_service_name,
|
|
command=command,
|
|
cwd=normalized_cwd,
|
|
readiness=normalized_readiness,
|
|
ready_timeout_seconds=ready_timeout_seconds,
|
|
ready_interval_ms=ready_interval_ms,
|
|
env=env_values or None,
|
|
)
|
|
except Exception as exc:
|
|
raise _redact_exception(exc, redact_values) from exc
|
|
service = self._workspace_service_record_from_payload(
|
|
workspace_id=workspace_id,
|
|
service_name=normalized_service_name,
|
|
payload=payload,
|
|
)
|
|
if normalized_published_ports:
|
|
assert instance.network is not None # guarded above
|
|
try:
|
|
service.published_ports = self._start_workspace_service_published_ports(
|
|
workspace=workspace,
|
|
service=service,
|
|
guest_ip=instance.network.guest_ip,
|
|
published_ports=normalized_published_ports,
|
|
)
|
|
except Exception:
|
|
try:
|
|
failed_payload = self._backend.stop_service(
|
|
instance,
|
|
workspace_id=workspace_id,
|
|
service_name=normalized_service_name,
|
|
)
|
|
service = self._workspace_service_record_from_payload(
|
|
workspace_id=workspace_id,
|
|
service_name=normalized_service_name,
|
|
payload=failed_payload,
|
|
published_ports=[],
|
|
)
|
|
except Exception:
|
|
service.state = "failed"
|
|
service.stop_reason = "published_port_failed"
|
|
service.ended_at = service.ended_at or time.time()
|
|
else:
|
|
service.state = "failed"
|
|
service.stop_reason = "published_port_failed"
|
|
service.ended_at = service.ended_at or time.time()
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
self._save_workspace_service_locked(service)
|
|
return self._serialize_workspace_service(service)
|
|
|
|
def list_services(self, workspace_id: str) -> dict[str, Any]:
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_service_locked(workspace)
|
|
services = self._refresh_workspace_services_locked(workspace, instance)
|
|
self._save_workspace_locked(workspace)
|
|
serialized = [self._serialize_workspace_service(service) for service in services]
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"count": len(serialized),
|
|
"running_count": sum(1 for item in serialized if item["state"] == "running"),
|
|
"services": serialized,
|
|
}
|
|
|
|
def status_service(self, workspace_id: str, service_name: str) -> dict[str, Any]:
|
|
normalized_service_name = _normalize_workspace_service_name(service_name)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_service_locked(workspace)
|
|
service = self._load_workspace_service_locked(workspace_id, normalized_service_name)
|
|
service = self._refresh_workspace_service_locked(workspace, instance, service)
|
|
self._save_workspace_locked(workspace)
|
|
self._save_workspace_service_locked(service)
|
|
return self._serialize_workspace_service(service)
|
|
|
|
def logs_service(
|
|
self,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
*,
|
|
tail_lines: int | None = DEFAULT_SERVICE_LOG_TAIL_LINES,
|
|
) -> dict[str, Any]:
|
|
normalized_service_name = _normalize_workspace_service_name(service_name)
|
|
if tail_lines is not None and tail_lines <= 0:
|
|
raise ValueError("tail_lines must be positive")
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_service_locked(workspace)
|
|
service = self._load_workspace_service_locked(workspace_id, normalized_service_name)
|
|
redact_values = self._workspace_secret_redact_values_locked(workspace)
|
|
try:
|
|
payload = self._backend.logs_service(
|
|
instance,
|
|
workspace_id=workspace_id,
|
|
service_name=normalized_service_name,
|
|
tail_lines=tail_lines,
|
|
)
|
|
except Exception as exc:
|
|
raise _redact_exception(exc, redact_values) from exc
|
|
service = self._workspace_service_record_from_payload(
|
|
workspace_id=workspace_id,
|
|
service_name=normalized_service_name,
|
|
payload=payload,
|
|
metadata=service.metadata,
|
|
published_ports=service.published_ports,
|
|
)
|
|
if service.published_ports:
|
|
for published_port in service.published_ports:
|
|
_stop_workspace_published_port_proxy(published_port)
|
|
service.published_ports = [
|
|
WorkspacePublishedPortRecord(
|
|
guest_port=published_port.guest_port,
|
|
host_port=published_port.host_port,
|
|
host=published_port.host,
|
|
protocol=published_port.protocol,
|
|
proxy_pid=None,
|
|
)
|
|
for published_port in service.published_ports
|
|
]
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
self._save_workspace_service_locked(service)
|
|
response = self._serialize_workspace_service(service)
|
|
response.update(
|
|
{
|
|
"stdout": _redact_text(str(payload.get("stdout", "")), redact_values),
|
|
"stderr": _redact_text(str(payload.get("stderr", "")), redact_values),
|
|
"tail_lines": tail_lines,
|
|
"truncated": bool(payload.get("truncated", False)),
|
|
}
|
|
)
|
|
return response
|
|
|
|
def stop_service(self, workspace_id: str, service_name: str) -> dict[str, Any]:
|
|
normalized_service_name = _normalize_workspace_service_name(service_name)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = self._workspace_instance_for_live_service_locked(workspace)
|
|
service = self._load_workspace_service_locked(workspace_id, normalized_service_name)
|
|
redact_values = self._workspace_secret_redact_values_locked(workspace)
|
|
try:
|
|
payload = self._backend.stop_service(
|
|
instance,
|
|
workspace_id=workspace_id,
|
|
service_name=normalized_service_name,
|
|
)
|
|
except Exception as exc:
|
|
raise _redact_exception(exc, redact_values) from exc
|
|
service = self._workspace_service_record_from_payload(
|
|
workspace_id=workspace_id,
|
|
service_name=normalized_service_name,
|
|
payload=payload,
|
|
metadata=service.metadata,
|
|
published_ports=service.published_ports,
|
|
)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = instance.last_error
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._save_workspace_locked(workspace)
|
|
self._save_workspace_service_locked(service)
|
|
return self._serialize_workspace_service(service)
|
|
|
|
def status_workspace(self, workspace_id: str) -> dict[str, Any]:
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
self._refresh_workspace_liveness_locked(workspace)
|
|
self._refresh_workspace_service_counts_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
return self._serialize_workspace(workspace)
|
|
|
|
def logs_workspace(self, workspace_id: str) -> dict[str, Any]:
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
self._refresh_workspace_liveness_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
entries = self._read_workspace_logs_locked(workspace.workspace_id)
|
|
redact_values = self._workspace_secret_redact_values_locked(workspace)
|
|
redacted_entries = []
|
|
for entry in entries:
|
|
redacted_entry = dict(entry)
|
|
redacted_entry["stdout"] = _redact_text(str(entry.get("stdout", "")), redact_values)
|
|
redacted_entry["stderr"] = _redact_text(str(entry.get("stderr", "")), redact_values)
|
|
redacted_entries.append(redacted_entry)
|
|
return {
|
|
"workspace_id": workspace.workspace_id,
|
|
"count": len(redacted_entries),
|
|
"entries": redacted_entries,
|
|
}
|
|
|
|
def stop_workspace(self, workspace_id: str) -> dict[str, Any]:
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
self._refresh_workspace_liveness_locked(workspace)
|
|
instance = workspace.to_instance(
|
|
workdir=self._workspace_runtime_dir(workspace.workspace_id)
|
|
)
|
|
try:
|
|
self._stop_workspace_services_locked(workspace, instance)
|
|
self._close_workspace_shells_locked(workspace, instance)
|
|
self._flush_workspace_filesystem_locked(workspace, instance)
|
|
if workspace.state == "started":
|
|
self._backend.stop(instance)
|
|
workspace.state = "stopped"
|
|
workspace.firecracker_pid = None
|
|
workspace.last_error = None
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._scrub_workspace_runtime_state_locked(workspace)
|
|
except Exception as exc:
|
|
workspace.state = "stopped"
|
|
workspace.firecracker_pid = None
|
|
workspace.last_error = str(exc)
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
raise
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
return self._serialize_workspace(workspace)
|
|
|
|
def start_workspace(self, workspace_id: str) -> dict[str, Any]:
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
self._refresh_workspace_liveness_locked(workspace)
|
|
if workspace.state == "started":
|
|
self._refresh_workspace_service_counts_locked(workspace)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
return self._serialize_workspace(workspace)
|
|
instance = workspace.to_instance(
|
|
workdir=self._workspace_runtime_dir(workspace.workspace_id)
|
|
)
|
|
self._stop_workspace_services_locked(workspace, instance)
|
|
self._close_workspace_shells_locked(workspace, instance)
|
|
try:
|
|
self._require_workspace_network_policy_support(
|
|
network_policy=workspace.network_policy
|
|
)
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
self._ensure_workspace_guest_bootstrap_support(instance)
|
|
with self._lock:
|
|
self._start_instance_locked(instance)
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
if workspace.secrets:
|
|
self._install_workspace_secrets_locked(workspace, instance)
|
|
workspace.state = instance.state
|
|
workspace.firecracker_pid = instance.firecracker_pid
|
|
workspace.last_error = None
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
return self._serialize_workspace(workspace)
|
|
except Exception as exc:
|
|
try:
|
|
if instance.state == "started":
|
|
self._backend.stop(instance)
|
|
except Exception:
|
|
pass
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
workspace.state = "stopped"
|
|
workspace.firecracker_pid = None
|
|
workspace.last_error = str(exc)
|
|
workspace.metadata = dict(instance.metadata)
|
|
self._touch_workspace_activity_locked(workspace)
|
|
self._save_workspace_locked(workspace)
|
|
raise
|
|
|
|
def export_workspace_disk(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
output_path: str | Path,
|
|
) -> dict[str, Any]:
|
|
raw_output_path = str(output_path).strip()
|
|
if raw_output_path == "":
|
|
raise ValueError("output_path must not be empty")
|
|
resolved_output_path = Path(output_path).expanduser().resolve()
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
rootfs_path = self._workspace_stopped_disk_rootfs_locked(
|
|
workspace,
|
|
operation_name="workspace_disk_export",
|
|
)
|
|
self._scrub_workspace_runtime_state_locked(workspace, rootfs_path=rootfs_path)
|
|
self._save_workspace_locked(workspace)
|
|
exported = export_workspace_disk_image(rootfs_path, output_path=resolved_output_path)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"output_path": str(Path(str(exported["output_path"]))),
|
|
"disk_format": str(exported["disk_format"]),
|
|
"bytes_written": int(exported["bytes_written"]),
|
|
}
|
|
|
|
def list_workspace_disk(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
path: str = WORKSPACE_GUEST_PATH,
|
|
recursive: bool = False,
|
|
) -> dict[str, Any]:
|
|
normalized_path = _normalize_workspace_disk_path(path)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
rootfs_path = self._workspace_stopped_disk_rootfs_locked(
|
|
workspace,
|
|
operation_name="workspace_disk_list",
|
|
)
|
|
self._scrub_workspace_runtime_state_locked(workspace, rootfs_path=rootfs_path)
|
|
self._save_workspace_locked(workspace)
|
|
entries = list_workspace_disk(
|
|
rootfs_path,
|
|
guest_path=normalized_path,
|
|
recursive=recursive,
|
|
)
|
|
return {
|
|
"workspace_id": workspace_id,
|
|
"path": normalized_path,
|
|
"recursive": recursive,
|
|
"entries": entries,
|
|
}
|
|
|
|
def read_workspace_disk(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
path: str,
|
|
max_bytes: int = DEFAULT_WORKSPACE_DISK_READ_MAX_BYTES,
|
|
) -> dict[str, Any]:
|
|
normalized_path = _normalize_workspace_disk_path(path)
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
rootfs_path = self._workspace_stopped_disk_rootfs_locked(
|
|
workspace,
|
|
operation_name="workspace_disk_read",
|
|
)
|
|
self._scrub_workspace_runtime_state_locked(workspace, rootfs_path=rootfs_path)
|
|
self._save_workspace_locked(workspace)
|
|
payload = read_workspace_disk_file(
|
|
rootfs_path,
|
|
guest_path=normalized_path,
|
|
max_bytes=max_bytes,
|
|
)
|
|
payload["workspace_id"] = workspace_id
|
|
return payload
|
|
|
|
def delete_workspace(
|
|
self,
|
|
workspace_id: str,
|
|
*,
|
|
reason: str = "explicit_delete",
|
|
) -> dict[str, Any]:
|
|
with self._lock:
|
|
workspace = self._load_workspace_locked(workspace_id)
|
|
instance = workspace.to_instance(
|
|
workdir=self._workspace_runtime_dir(workspace.workspace_id)
|
|
)
|
|
self._stop_workspace_services_locked(workspace, instance)
|
|
self._close_workspace_shells_locked(workspace, instance)
|
|
if workspace.state == "started":
|
|
self._backend.stop(instance)
|
|
workspace.state = "stopped"
|
|
self._backend.delete(instance)
|
|
shutil.rmtree(self._workspace_dir(workspace_id), ignore_errors=True)
|
|
return {"workspace_id": workspace_id, "deleted": True, "reason": reason}
|
|
|
|
def _validate_limits(self, *, vcpu_count: int, mem_mib: int, ttl_seconds: int) -> None:
|
|
if not self.MIN_VCPUS <= vcpu_count <= self.MAX_VCPUS:
|
|
raise ValueError(f"vcpu_count must be between {self.MIN_VCPUS} and {self.MAX_VCPUS}")
|
|
if not self.MIN_MEM_MIB <= mem_mib <= self.MAX_MEM_MIB:
|
|
raise ValueError(f"mem_mib must be between {self.MIN_MEM_MIB} and {self.MAX_MEM_MIB}")
|
|
if not self.MIN_TTL_SECONDS <= ttl_seconds <= self.MAX_TTL_SECONDS:
|
|
raise ValueError(
|
|
f"ttl_seconds must be between {self.MIN_TTL_SECONDS} and {self.MAX_TTL_SECONDS}"
|
|
)
|
|
|
|
def _serialize(self, instance: VmInstance) -> dict[str, Any]:
|
|
return {
|
|
"vm_id": instance.vm_id,
|
|
"environment": instance.environment,
|
|
"environment_version": instance.metadata.get("environment_version"),
|
|
"vcpu_count": instance.vcpu_count,
|
|
"mem_mib": instance.mem_mib,
|
|
"ttl_seconds": instance.ttl_seconds,
|
|
"created_at": instance.created_at,
|
|
"expires_at": instance.expires_at,
|
|
"state": instance.state,
|
|
"network_enabled": instance.network is not None,
|
|
"allow_host_compat": instance.allow_host_compat,
|
|
"guest_ip": instance.network.guest_ip if instance.network is not None else None,
|
|
"tap_name": instance.network.tap_name if instance.network is not None else None,
|
|
"execution_mode": instance.metadata.get("execution_mode", "pending"),
|
|
"metadata": instance.metadata,
|
|
}
|
|
|
|
def _serialize_workspace(self, workspace: WorkspaceRecord) -> dict[str, Any]:
|
|
service_count, running_service_count = self._workspace_service_counts_locked(
|
|
workspace.workspace_id
|
|
)
|
|
return {
|
|
"workspace_id": workspace.workspace_id,
|
|
"name": workspace.name,
|
|
"labels": dict(workspace.labels),
|
|
"environment": workspace.environment,
|
|
"environment_version": workspace.metadata.get("environment_version"),
|
|
"vcpu_count": workspace.vcpu_count,
|
|
"mem_mib": workspace.mem_mib,
|
|
"ttl_seconds": workspace.ttl_seconds,
|
|
"created_at": workspace.created_at,
|
|
"last_activity_at": workspace.last_activity_at,
|
|
"expires_at": workspace.expires_at,
|
|
"state": workspace.state,
|
|
"network_policy": workspace.network_policy,
|
|
"network_enabled": workspace.network is not None,
|
|
"allow_host_compat": workspace.allow_host_compat,
|
|
"guest_ip": workspace.network.guest_ip if workspace.network is not None else None,
|
|
"tap_name": workspace.network.tap_name if workspace.network is not None else None,
|
|
"execution_mode": workspace.metadata.get("execution_mode", "pending"),
|
|
"workspace_path": WORKSPACE_GUEST_PATH,
|
|
"workspace_seed": _workspace_seed_dict(workspace.workspace_seed),
|
|
"secrets": [
|
|
_serialize_workspace_secret_public(secret) for secret in workspace.secrets
|
|
],
|
|
"command_count": workspace.command_count,
|
|
"last_command": workspace.last_command,
|
|
"reset_count": workspace.reset_count,
|
|
"last_reset_at": workspace.last_reset_at,
|
|
"service_count": service_count,
|
|
"running_service_count": running_service_count,
|
|
"metadata": workspace.metadata,
|
|
}
|
|
|
|
def _serialize_workspace_list_item(self, workspace: WorkspaceRecord) -> dict[str, Any]:
|
|
service_count, running_service_count = self._workspace_service_counts_locked(
|
|
workspace.workspace_id
|
|
)
|
|
return {
|
|
"workspace_id": workspace.workspace_id,
|
|
"name": workspace.name,
|
|
"labels": dict(workspace.labels),
|
|
"environment": workspace.environment,
|
|
"state": workspace.state,
|
|
"created_at": workspace.created_at,
|
|
"last_activity_at": workspace.last_activity_at,
|
|
"expires_at": workspace.expires_at,
|
|
"command_count": workspace.command_count,
|
|
"service_count": service_count,
|
|
"running_service_count": running_service_count,
|
|
}
|
|
|
|
def _serialize_workspace_shell(self, shell: WorkspaceShellRecord) -> dict[str, Any]:
|
|
return {
|
|
"workspace_id": shell.workspace_id,
|
|
"shell_id": shell.shell_id,
|
|
"cwd": shell.cwd,
|
|
"cols": shell.cols,
|
|
"rows": shell.rows,
|
|
"state": shell.state,
|
|
"started_at": shell.started_at,
|
|
"ended_at": shell.ended_at,
|
|
"exit_code": shell.exit_code,
|
|
"execution_mode": shell.execution_mode,
|
|
}
|
|
|
|
def _serialize_workspace_service(self, service: WorkspaceServiceRecord) -> dict[str, Any]:
|
|
return {
|
|
"workspace_id": service.workspace_id,
|
|
"service_name": service.service_name,
|
|
"state": service.state,
|
|
"command": service.command,
|
|
"cwd": service.cwd,
|
|
"started_at": service.started_at,
|
|
"ended_at": service.ended_at,
|
|
"exit_code": service.exit_code,
|
|
"pid": service.pid,
|
|
"execution_mode": service.execution_mode,
|
|
"readiness": dict(service.readiness) if service.readiness is not None else None,
|
|
"ready_at": service.ready_at,
|
|
"stop_reason": service.stop_reason,
|
|
"published_ports": [
|
|
_serialize_workspace_published_port_public(published_port)
|
|
for published_port in service.published_ports
|
|
],
|
|
}
|
|
|
|
def _serialize_workspace_snapshot(self, snapshot: WorkspaceSnapshotRecord) -> dict[str, Any]:
|
|
return {
|
|
"workspace_id": snapshot.workspace_id,
|
|
"snapshot_name": snapshot.snapshot_name,
|
|
"kind": snapshot.kind,
|
|
"created_at": snapshot.created_at,
|
|
"entry_count": snapshot.entry_count,
|
|
"bytes_written": snapshot.bytes_written,
|
|
"deletable": snapshot.kind != "baseline",
|
|
}
|
|
|
|
def _require_guest_boot_or_opt_in(self, instance: VmInstance) -> None:
|
|
if self._runtime_capabilities.supports_vm_boot or instance.allow_host_compat:
|
|
return
|
|
reason = self._runtime_capabilities.reason or "runtime does not support real VM boot"
|
|
raise RuntimeError(
|
|
"guest boot is unavailable and host compatibility mode is disabled: "
|
|
f"{reason}. Set allow_host_compat=True (CLI: --allow-host-compat) to opt into "
|
|
"host execution."
|
|
)
|
|
|
|
def _require_guest_exec_or_opt_in(self, instance: VmInstance) -> None:
|
|
if self._runtime_capabilities.supports_guest_exec or instance.allow_host_compat:
|
|
return
|
|
reason = self._runtime_capabilities.reason or (
|
|
"runtime does not support guest command execution"
|
|
)
|
|
raise RuntimeError(
|
|
"guest command execution is unavailable and host compatibility mode is disabled: "
|
|
f"{reason}. Set allow_host_compat=True (CLI: --allow-host-compat) to opt into "
|
|
"host execution."
|
|
)
|
|
|
|
def _require_workspace_shell_support(self, instance: VmInstance) -> None:
|
|
if self._backend_name == "mock":
|
|
return
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
return
|
|
reason = self._runtime_capabilities.reason or (
|
|
"runtime does not support guest interactive shell sessions"
|
|
)
|
|
raise RuntimeError(
|
|
"interactive shells require guest execution and are unavailable for this "
|
|
f"workspace: {reason}"
|
|
)
|
|
|
|
def _require_workspace_service_support(self, instance: VmInstance) -> None:
|
|
if self._backend_name == "mock":
|
|
return
|
|
if self._runtime_capabilities.supports_guest_exec or instance.allow_host_compat:
|
|
return
|
|
reason = self._runtime_capabilities.reason or (
|
|
"runtime does not support guest-backed or host-compatible service execution"
|
|
)
|
|
raise RuntimeError(
|
|
"workspace services are unavailable for this workspace: "
|
|
f"{reason}. Recreate the workspace with --allow-host-compat to opt into "
|
|
"host compatibility when guest execution is unavailable."
|
|
)
|
|
|
|
def _require_workspace_secret_support(self, instance: VmInstance) -> None:
|
|
if self._backend_name == "mock":
|
|
return
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
return
|
|
reason = self._runtime_capabilities.reason or (
|
|
"runtime does not support guest-backed secret installation"
|
|
)
|
|
raise RuntimeError(
|
|
"workspace secrets require guest execution and are unavailable for this "
|
|
f"workspace: {reason}"
|
|
)
|
|
|
|
def _require_workspace_network_policy_support(
|
|
self,
|
|
*,
|
|
network_policy: WorkspaceNetworkPolicy,
|
|
) -> None:
|
|
if network_policy == "off":
|
|
return
|
|
if self._runtime_capabilities.supports_guest_network:
|
|
return
|
|
reason = self._runtime_capabilities.reason or (
|
|
"runtime does not support guest-backed workspace networking"
|
|
)
|
|
raise RuntimeError(
|
|
"workspace network_policy requires guest networking and is unavailable for this "
|
|
f"workspace: {reason}"
|
|
)
|
|
|
|
def _workspace_secret_values_locked(self, workspace: WorkspaceRecord) -> dict[str, str]:
|
|
return _load_workspace_secret_values(
|
|
workspace_dir=self._workspace_dir(workspace.workspace_id),
|
|
secrets=workspace.secrets,
|
|
)
|
|
|
|
def _workspace_secret_redact_values_locked(self, workspace: WorkspaceRecord) -> list[str]:
|
|
return list(self._workspace_secret_values_locked(workspace).values())
|
|
|
|
def _workspace_secret_env_values_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
secret_env: dict[str, str],
|
|
) -> dict[str, str]:
|
|
secret_values = self._workspace_secret_values_locked(workspace)
|
|
env_values: dict[str, str] = {}
|
|
for secret_name, env_name in secret_env.items():
|
|
if secret_name not in secret_values:
|
|
raise ValueError(
|
|
f"secret_env references unknown workspace secret {secret_name!r}"
|
|
)
|
|
env_values[env_name] = secret_values[secret_name]
|
|
return env_values
|
|
|
|
def _touch_workspace_activity_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
*,
|
|
when: float | None = None,
|
|
) -> None:
|
|
workspace.last_activity_at = time.time() if when is None else when
|
|
|
|
def _install_workspace_secrets_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
instance: VmInstance,
|
|
) -> None:
|
|
if not workspace.secrets:
|
|
return
|
|
self._require_workspace_secret_support(instance)
|
|
with tempfile.TemporaryDirectory(prefix="pyro-workspace-secrets-") as temp_dir:
|
|
archive_path = Path(temp_dir) / "workspace-secrets.tar"
|
|
_build_workspace_secret_archive(
|
|
workspace_dir=self._workspace_dir(workspace.workspace_id),
|
|
secrets=workspace.secrets,
|
|
archive_path=archive_path,
|
|
)
|
|
self._backend.install_secrets(instance, archive_path=archive_path)
|
|
|
|
def _get_instance_locked(self, vm_id: str) -> VmInstance:
|
|
try:
|
|
return self._instances[vm_id]
|
|
except KeyError as exc:
|
|
raise ValueError(f"vm {vm_id!r} does not exist") from exc
|
|
|
|
def _reap_expired_locked(self, now: float) -> None:
|
|
expired_vm_ids = [
|
|
vm_id for vm_id, inst in self._instances.items() if inst.expires_at <= now
|
|
]
|
|
for vm_id in expired_vm_ids:
|
|
instance = self._instances[vm_id]
|
|
if instance.state == "started":
|
|
self._backend.stop(instance)
|
|
instance.state = "stopped"
|
|
self._backend.delete(instance)
|
|
del self._instances[vm_id]
|
|
|
|
def _ensure_not_expired_locked(self, instance: VmInstance, now: float) -> None:
|
|
if instance.expires_at <= now:
|
|
vm_id = instance.vm_id
|
|
self._reap_expired_locked(now)
|
|
raise RuntimeError(f"vm {vm_id!r} expired and was automatically deleted")
|
|
|
|
def _start_instance_locked(self, instance: VmInstance) -> None:
|
|
if instance.state not in {"created", "stopped"}:
|
|
raise RuntimeError(
|
|
f"vm {instance.vm_id} cannot be started from state {instance.state!r}"
|
|
)
|
|
self._require_guest_boot_or_opt_in(instance)
|
|
if not self._runtime_capabilities.supports_vm_boot:
|
|
instance.metadata["execution_mode"] = "host_compat"
|
|
instance.metadata["boot_mode"] = "compat"
|
|
if self._runtime_capabilities.reason is not None:
|
|
instance.metadata["runtime_reason"] = self._runtime_capabilities.reason
|
|
self._backend.start(instance)
|
|
instance.state = "started"
|
|
|
|
def _exec_instance(
|
|
self,
|
|
instance: VmInstance,
|
|
*,
|
|
command: str,
|
|
timeout_seconds: int,
|
|
host_workdir: Path | None = None,
|
|
guest_cwd: str | None = None,
|
|
env: dict[str, str] | None = None,
|
|
) -> tuple[VmExecResult, str]:
|
|
if timeout_seconds <= 0:
|
|
raise ValueError("timeout_seconds must be positive")
|
|
if instance.state != "started":
|
|
raise RuntimeError(f"vm {instance.vm_id} must be in 'started' state before execution")
|
|
self._require_guest_exec_or_opt_in(instance)
|
|
prepared_command = command
|
|
if self._runtime_capabilities.supports_guest_exec:
|
|
prepared_command = _wrap_guest_command(command, cwd=guest_cwd)
|
|
workdir = None
|
|
else:
|
|
instance.metadata["execution_mode"] = "host_compat"
|
|
workdir = host_workdir
|
|
if env is None:
|
|
exec_result = self._backend.exec(
|
|
instance,
|
|
prepared_command,
|
|
timeout_seconds,
|
|
workdir=workdir,
|
|
)
|
|
else:
|
|
exec_result = self._backend.exec(
|
|
instance,
|
|
prepared_command,
|
|
timeout_seconds,
|
|
workdir=workdir,
|
|
env=env,
|
|
)
|
|
execution_mode = instance.metadata.get("execution_mode", "unknown")
|
|
return exec_result, execution_mode
|
|
|
|
def _prepare_workspace_seed(self, seed_path: str | Path | None) -> PreparedWorkspaceSeed:
|
|
if seed_path is None:
|
|
return PreparedWorkspaceSeed(mode="empty", source_path=None)
|
|
resolved_source_path = Path(seed_path).expanduser().resolve()
|
|
if not resolved_source_path.exists():
|
|
raise ValueError(f"seed_path {resolved_source_path} does not exist")
|
|
if resolved_source_path.is_dir():
|
|
cleanup_dir = Path(tempfile.mkdtemp(prefix="pyro-workspace-seed-"))
|
|
archive_path = cleanup_dir / "workspace-seed.tar"
|
|
try:
|
|
_write_directory_seed_archive(resolved_source_path, archive_path)
|
|
entry_count, bytes_written = _inspect_seed_archive(archive_path)
|
|
except Exception:
|
|
shutil.rmtree(cleanup_dir, ignore_errors=True)
|
|
raise
|
|
return PreparedWorkspaceSeed(
|
|
mode="directory",
|
|
source_path=str(resolved_source_path),
|
|
archive_path=archive_path,
|
|
entry_count=entry_count,
|
|
bytes_written=bytes_written,
|
|
cleanup_dir=cleanup_dir,
|
|
)
|
|
if (
|
|
not resolved_source_path.is_file()
|
|
or not _is_supported_seed_archive(resolved_source_path)
|
|
):
|
|
raise ValueError(
|
|
"seed_path must be a directory or a .tar/.tar.gz/.tgz archive"
|
|
)
|
|
entry_count, bytes_written = _inspect_seed_archive(resolved_source_path)
|
|
return PreparedWorkspaceSeed(
|
|
mode="tar_archive",
|
|
source_path=str(resolved_source_path),
|
|
archive_path=resolved_source_path,
|
|
entry_count=entry_count,
|
|
bytes_written=bytes_written,
|
|
)
|
|
|
|
def _ensure_workspace_guest_bootstrap_support(self, instance: VmInstance) -> None:
|
|
if (
|
|
self._runtime_paths is None
|
|
or self._runtime_paths.guest_agent_path is None
|
|
or self._runtime_paths.guest_init_path is None
|
|
):
|
|
raise RuntimeError(
|
|
"runtime bundle does not provide guest bootstrap assets for workspace operations"
|
|
)
|
|
rootfs_image = instance.metadata.get("rootfs_image")
|
|
if rootfs_image is None or rootfs_image == "":
|
|
raise RuntimeError("workspace rootfs image is unavailable for guest operations")
|
|
rootfs_path = Path(rootfs_image)
|
|
_patch_rootfs_runtime_file(
|
|
rootfs_path,
|
|
source_path=self._runtime_paths.guest_init_path,
|
|
destination_path=WORKSPACE_GUEST_INIT_PATH,
|
|
asset_label="guest-init",
|
|
file_mode="0100755",
|
|
)
|
|
_patch_rootfs_runtime_file(
|
|
rootfs_path,
|
|
source_path=self._runtime_paths.guest_agent_path,
|
|
destination_path=WORKSPACE_GUEST_AGENT_PATH,
|
|
asset_label="guest-agent",
|
|
file_mode="0100755",
|
|
)
|
|
|
|
def _workspace_dir(self, workspace_id: str) -> Path:
|
|
return self._workspaces_dir / workspace_id
|
|
|
|
def _workspace_runtime_dir(self, workspace_id: str) -> Path:
|
|
return self._workspace_dir(workspace_id) / WORKSPACE_RUNTIME_DIRNAME
|
|
|
|
def _workspace_host_dir(self, workspace_id: str) -> Path:
|
|
return self._workspace_dir(workspace_id) / WORKSPACE_DIRNAME
|
|
|
|
def _workspace_baseline_dir(self, workspace_id: str) -> Path:
|
|
return self._workspace_dir(workspace_id) / WORKSPACE_BASELINE_DIRNAME
|
|
|
|
def _workspace_baseline_archive_path(self, workspace_id: str) -> Path:
|
|
return self._workspace_baseline_dir(workspace_id) / WORKSPACE_BASELINE_ARCHIVE_NAME
|
|
|
|
def _workspace_snapshots_dir(self, workspace_id: str) -> Path:
|
|
return self._workspace_dir(workspace_id) / WORKSPACE_SNAPSHOTS_DIRNAME
|
|
|
|
def _workspace_snapshot_archive_path(self, workspace_id: str, snapshot_name: str) -> Path:
|
|
return self._workspace_snapshots_dir(workspace_id) / f"{snapshot_name}.tar"
|
|
|
|
def _workspace_snapshot_metadata_path(self, workspace_id: str, snapshot_name: str) -> Path:
|
|
return self._workspace_snapshots_dir(workspace_id) / f"{snapshot_name}.json"
|
|
|
|
def _workspace_commands_dir(self, workspace_id: str) -> Path:
|
|
return self._workspace_dir(workspace_id) / WORKSPACE_COMMANDS_DIRNAME
|
|
|
|
def _workspace_shells_dir(self, workspace_id: str) -> Path:
|
|
return self._workspace_dir(workspace_id) / WORKSPACE_SHELLS_DIRNAME
|
|
|
|
def _workspace_services_dir(self, workspace_id: str) -> Path:
|
|
return self._workspace_dir(workspace_id) / WORKSPACE_SERVICES_DIRNAME
|
|
|
|
def _workspace_secrets_dir(self, workspace_id: str) -> Path:
|
|
return self._workspace_dir(workspace_id) / WORKSPACE_SECRETS_DIRNAME
|
|
|
|
def _workspace_metadata_path(self, workspace_id: str) -> Path:
|
|
return self._workspace_dir(workspace_id) / "workspace.json"
|
|
|
|
def _workspace_shell_record_path(self, workspace_id: str, shell_id: str) -> Path:
|
|
return self._workspace_shells_dir(workspace_id) / f"{shell_id}.json"
|
|
|
|
def _workspace_service_record_path(self, workspace_id: str, service_name: str) -> Path:
|
|
return self._workspace_services_dir(workspace_id) / f"{service_name}.json"
|
|
|
|
def _workspace_rootfs_image_path_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
) -> Path:
|
|
raw_rootfs_image = workspace.metadata.get("rootfs_image")
|
|
if raw_rootfs_image is None or raw_rootfs_image == "":
|
|
raise RuntimeError(
|
|
f"workspace {workspace.workspace_id!r} does not have a persisted rootfs image"
|
|
)
|
|
rootfs_path = Path(raw_rootfs_image)
|
|
if not rootfs_path.exists():
|
|
raise RuntimeError(
|
|
f"workspace {workspace.workspace_id!r} rootfs image is unavailable at "
|
|
f"{rootfs_path}"
|
|
)
|
|
return rootfs_path
|
|
|
|
def _workspace_stopped_disk_rootfs_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
*,
|
|
operation_name: str,
|
|
) -> Path:
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
self._refresh_workspace_liveness_locked(workspace)
|
|
if workspace.state != "stopped":
|
|
raise RuntimeError(
|
|
f"workspace {workspace.workspace_id!r} must be stopped before {operation_name}"
|
|
)
|
|
if workspace.metadata.get("execution_mode") == "host_compat":
|
|
raise RuntimeError(
|
|
f"{operation_name} is unavailable for host_compat workspaces"
|
|
)
|
|
return self._workspace_rootfs_image_path_locked(workspace)
|
|
|
|
def _scrub_workspace_runtime_state_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
*,
|
|
rootfs_path: Path | None = None,
|
|
) -> None:
|
|
execution_mode = workspace.metadata.get("execution_mode")
|
|
if execution_mode == "host_compat":
|
|
return
|
|
scrub_workspace_runtime_paths(
|
|
rootfs_path or self._workspace_rootfs_image_path_locked(workspace)
|
|
)
|
|
|
|
def _flush_workspace_filesystem_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
instance: VmInstance,
|
|
) -> None:
|
|
if workspace.state != "started":
|
|
return
|
|
if self._backend_name == "mock":
|
|
return
|
|
if not self._runtime_capabilities.supports_guest_exec:
|
|
return
|
|
self._backend.exec(instance, "sync", 10)
|
|
|
|
def _count_workspaces_locked(self) -> int:
|
|
return sum(1 for _ in self._workspaces_dir.glob("*/workspace.json"))
|
|
|
|
def _load_workspace_locked(self, workspace_id: str) -> WorkspaceRecord:
|
|
metadata_path = self._workspace_metadata_path(workspace_id)
|
|
if not metadata_path.exists():
|
|
raise ValueError(f"workspace {workspace_id!r} does not exist")
|
|
payload = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError(f"workspace record at {metadata_path} is invalid")
|
|
return WorkspaceRecord.from_payload(payload)
|
|
|
|
def _save_workspace_locked(self, workspace: WorkspaceRecord) -> None:
|
|
metadata_path = self._workspace_metadata_path(workspace.workspace_id)
|
|
metadata_path.parent.mkdir(parents=True, exist_ok=True)
|
|
metadata_path.write_text(
|
|
json.dumps(workspace.to_payload(), indent=2, sort_keys=True),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
def _reap_expired_workspaces_locked(self, now: float) -> None:
|
|
for metadata_path in list(self._workspaces_dir.glob("*/workspace.json")):
|
|
payload = json.loads(metadata_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
shutil.rmtree(metadata_path.parent, ignore_errors=True)
|
|
continue
|
|
workspace = WorkspaceRecord.from_payload(payload)
|
|
if workspace.expires_at > now:
|
|
continue
|
|
instance = workspace.to_instance(
|
|
workdir=self._workspace_runtime_dir(workspace.workspace_id)
|
|
)
|
|
self._close_workspace_shells_locked(workspace, instance)
|
|
if workspace.state == "started":
|
|
self._backend.stop(instance)
|
|
workspace.state = "stopped"
|
|
self._backend.delete(instance)
|
|
shutil.rmtree(self._workspace_dir(workspace.workspace_id), ignore_errors=True)
|
|
|
|
def _ensure_workspace_not_expired_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
now: float,
|
|
) -> None:
|
|
if workspace.expires_at <= now:
|
|
workspace_id = workspace.workspace_id
|
|
self._reap_expired_workspaces_locked(now)
|
|
raise RuntimeError(f"workspace {workspace_id!r} expired and was automatically deleted")
|
|
|
|
def _refresh_workspace_liveness_locked(self, workspace: WorkspaceRecord) -> None:
|
|
if workspace.state != "started":
|
|
return
|
|
execution_mode = workspace.metadata.get("execution_mode")
|
|
if execution_mode == "host_compat":
|
|
return
|
|
if _pid_is_running(workspace.firecracker_pid):
|
|
return
|
|
workspace.state = "stopped"
|
|
workspace.firecracker_pid = None
|
|
workspace.last_error = "backing guest process is no longer running"
|
|
|
|
def _record_workspace_command_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
*,
|
|
command: str,
|
|
exec_result: VmExecResult,
|
|
execution_mode: str,
|
|
cwd: str,
|
|
) -> dict[str, Any]:
|
|
sequence = workspace.command_count + 1
|
|
commands_dir = self._workspace_commands_dir(workspace.workspace_id)
|
|
commands_dir.mkdir(parents=True, exist_ok=True)
|
|
base_name = f"{sequence:06d}"
|
|
stdout_path = commands_dir / f"{base_name}.stdout"
|
|
stderr_path = commands_dir / f"{base_name}.stderr"
|
|
record_path = commands_dir / f"{base_name}.json"
|
|
stdout_path.write_text(exec_result.stdout, encoding="utf-8")
|
|
stderr_path.write_text(exec_result.stderr, encoding="utf-8")
|
|
entry: dict[str, Any] = {
|
|
"sequence": sequence,
|
|
"command": command,
|
|
"cwd": cwd,
|
|
"exit_code": exec_result.exit_code,
|
|
"duration_ms": exec_result.duration_ms,
|
|
"execution_mode": execution_mode,
|
|
"stdout_file": stdout_path.name,
|
|
"stderr_file": stderr_path.name,
|
|
"recorded_at": time.time(),
|
|
}
|
|
record_path.write_text(json.dumps(entry, indent=2, sort_keys=True), encoding="utf-8")
|
|
workspace.command_count = sequence
|
|
workspace.last_command = {
|
|
"sequence": sequence,
|
|
"command": command,
|
|
"cwd": cwd,
|
|
"exit_code": exec_result.exit_code,
|
|
"duration_ms": exec_result.duration_ms,
|
|
"execution_mode": execution_mode,
|
|
}
|
|
self._touch_workspace_activity_locked(workspace)
|
|
return entry
|
|
|
|
def _read_workspace_logs_locked(self, workspace_id: str) -> list[dict[str, Any]]:
|
|
entries: list[dict[str, Any]] = []
|
|
commands_dir = self._workspace_commands_dir(workspace_id)
|
|
if not commands_dir.exists():
|
|
return entries
|
|
for record_path in sorted(commands_dir.glob("*.json")):
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
continue
|
|
stdout_name = str(payload.get("stdout_file", ""))
|
|
stderr_name = str(payload.get("stderr_file", ""))
|
|
stdout = ""
|
|
stderr = ""
|
|
if stdout_name != "":
|
|
stdout_path = commands_dir / stdout_name
|
|
if stdout_path.exists():
|
|
stdout = stdout_path.read_text(encoding="utf-8")
|
|
if stderr_name != "":
|
|
stderr_path = commands_dir / stderr_name
|
|
if stderr_path.exists():
|
|
stderr = stderr_path.read_text(encoding="utf-8")
|
|
entry = dict(payload)
|
|
entry["stdout"] = stdout
|
|
entry["stderr"] = stderr
|
|
entries.append(entry)
|
|
return entries
|
|
|
|
def _workspace_instance_for_live_shell_locked(self, workspace: WorkspaceRecord) -> VmInstance:
|
|
instance = self._workspace_instance_for_live_operation_locked(
|
|
workspace,
|
|
operation_name="shell operations",
|
|
)
|
|
self._require_workspace_shell_support(instance)
|
|
return instance
|
|
|
|
def _workspace_instance_for_live_service_locked(self, workspace: WorkspaceRecord) -> VmInstance:
|
|
instance = self._workspace_instance_for_live_operation_locked(
|
|
workspace,
|
|
operation_name="service operations",
|
|
)
|
|
self._require_workspace_service_support(instance)
|
|
return instance
|
|
|
|
def _workspace_instance_for_live_operation_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
*,
|
|
operation_name: str,
|
|
) -> VmInstance:
|
|
self._ensure_workspace_not_expired_locked(workspace, time.time())
|
|
self._refresh_workspace_liveness_locked(workspace)
|
|
if workspace.state != "started":
|
|
raise RuntimeError(
|
|
"workspace "
|
|
f"{workspace.workspace_id} must be in 'started' state before {operation_name}"
|
|
)
|
|
instance = workspace.to_instance(
|
|
workdir=self._workspace_runtime_dir(workspace.workspace_id)
|
|
)
|
|
return instance
|
|
|
|
def _workspace_shell_record_from_payload(
|
|
self,
|
|
*,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
payload: dict[str, Any],
|
|
metadata: dict[str, str] | None = None,
|
|
) -> WorkspaceShellRecord:
|
|
return WorkspaceShellRecord(
|
|
workspace_id=workspace_id,
|
|
shell_id=str(payload.get("shell_id", shell_id)),
|
|
cwd=str(payload.get("cwd", WORKSPACE_GUEST_PATH)),
|
|
cols=int(payload.get("cols", DEFAULT_SHELL_COLS)),
|
|
rows=int(payload.get("rows", DEFAULT_SHELL_ROWS)),
|
|
state=cast(WorkspaceShellState, str(payload.get("state", "stopped"))),
|
|
started_at=float(payload.get("started_at", time.time())),
|
|
ended_at=(
|
|
None if payload.get("ended_at") is None else float(payload.get("ended_at", 0.0))
|
|
),
|
|
exit_code=(
|
|
None if payload.get("exit_code") is None else int(payload.get("exit_code", 0))
|
|
),
|
|
execution_mode=str(payload.get("execution_mode", "pending")),
|
|
metadata=dict(metadata or {}),
|
|
)
|
|
|
|
def _workspace_service_record_from_payload(
|
|
self,
|
|
*,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
payload: dict[str, Any],
|
|
metadata: dict[str, str] | None = None,
|
|
published_ports: list[WorkspacePublishedPortRecord] | None = None,
|
|
) -> WorkspaceServiceRecord:
|
|
readiness_payload = payload.get("readiness")
|
|
readiness = dict(readiness_payload) if isinstance(readiness_payload, dict) else None
|
|
normalized_published_ports = _workspace_published_port_records(
|
|
payload.get("published_ports")
|
|
)
|
|
if not normalized_published_ports and published_ports is not None:
|
|
normalized_published_ports = list(published_ports)
|
|
return WorkspaceServiceRecord(
|
|
workspace_id=workspace_id,
|
|
service_name=str(payload.get("service_name", service_name)),
|
|
command=str(payload.get("command", "")),
|
|
cwd=str(payload.get("cwd", WORKSPACE_GUEST_PATH)),
|
|
state=cast(WorkspaceServiceState, str(payload.get("state", "stopped"))),
|
|
started_at=float(payload.get("started_at", time.time())),
|
|
readiness=readiness,
|
|
ready_at=(
|
|
None if payload.get("ready_at") is None else float(payload.get("ready_at", 0.0))
|
|
),
|
|
ended_at=(
|
|
None if payload.get("ended_at") is None else float(payload.get("ended_at", 0.0))
|
|
),
|
|
exit_code=(
|
|
None if payload.get("exit_code") is None else int(payload.get("exit_code", 0))
|
|
),
|
|
pid=None if payload.get("pid") is None else int(payload.get("pid", 0)),
|
|
execution_mode=str(payload.get("execution_mode", "pending")),
|
|
stop_reason=_optional_str(payload.get("stop_reason")),
|
|
published_ports=normalized_published_ports,
|
|
metadata=dict(metadata or {}),
|
|
)
|
|
|
|
def _load_workspace_shell_locked(
|
|
self,
|
|
workspace_id: str,
|
|
shell_id: str,
|
|
) -> WorkspaceShellRecord:
|
|
record_path = self._workspace_shell_record_path(workspace_id, shell_id)
|
|
if not record_path.exists():
|
|
raise ValueError(f"shell {shell_id!r} does not exist in workspace {workspace_id!r}")
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError(f"shell record at {record_path} is invalid")
|
|
return WorkspaceShellRecord.from_payload(payload)
|
|
|
|
def _workspace_service_counts_locked(self, workspace_id: str) -> tuple[int, int]:
|
|
services = self._list_workspace_services_locked(workspace_id)
|
|
return len(services), sum(1 for service in services if service.state == "running")
|
|
|
|
def _start_workspace_service_published_ports(
|
|
self,
|
|
*,
|
|
workspace: WorkspaceRecord,
|
|
service: WorkspaceServiceRecord,
|
|
guest_ip: str,
|
|
published_ports: list[WorkspacePublishedPortSpec],
|
|
) -> list[WorkspacePublishedPortRecord]:
|
|
services_dir = self._workspace_services_dir(workspace.workspace_id)
|
|
started: list[WorkspacePublishedPortRecord] = []
|
|
try:
|
|
for spec in published_ports:
|
|
started.append(
|
|
_start_workspace_published_port_proxy(
|
|
services_dir=services_dir,
|
|
service_name=service.service_name,
|
|
workspace_id=workspace.workspace_id,
|
|
guest_ip=guest_ip,
|
|
spec=spec,
|
|
)
|
|
)
|
|
except Exception:
|
|
for published_port in started:
|
|
_stop_workspace_published_port_proxy(published_port)
|
|
raise
|
|
return started
|
|
|
|
def _workspace_baseline_snapshot_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
) -> WorkspaceSnapshotRecord:
|
|
baseline_archive_path = self._workspace_baseline_archive_path(workspace.workspace_id)
|
|
if not baseline_archive_path.exists():
|
|
raise RuntimeError(
|
|
"workspace snapshots and reset require a baseline snapshot. "
|
|
"Recreate the workspace to use snapshot/reset features."
|
|
)
|
|
entry_count, bytes_written = _inspect_seed_archive(baseline_archive_path)
|
|
return WorkspaceSnapshotRecord(
|
|
workspace_id=workspace.workspace_id,
|
|
snapshot_name="baseline",
|
|
kind="baseline",
|
|
created_at=workspace.created_at,
|
|
entry_count=entry_count,
|
|
bytes_written=bytes_written,
|
|
)
|
|
|
|
def _resolve_workspace_snapshot_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
snapshot_name: str,
|
|
) -> tuple[WorkspaceSnapshotRecord, Path]:
|
|
normalized_name = _normalize_workspace_snapshot_name(snapshot_name, allow_baseline=True)
|
|
if normalized_name == "baseline":
|
|
baseline = self._workspace_baseline_snapshot_locked(workspace)
|
|
return baseline, self._workspace_baseline_archive_path(workspace.workspace_id)
|
|
snapshot = self._load_workspace_snapshot_locked(workspace.workspace_id, normalized_name)
|
|
return (
|
|
snapshot,
|
|
self._workspace_snapshot_archive_path(workspace.workspace_id, normalized_name),
|
|
)
|
|
|
|
def _load_workspace_service_locked(
|
|
self,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
) -> WorkspaceServiceRecord:
|
|
record_path = self._workspace_service_record_path(workspace_id, service_name)
|
|
if not record_path.exists():
|
|
raise ValueError(
|
|
f"service {service_name!r} does not exist in workspace {workspace_id!r}"
|
|
)
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError(f"service record at {record_path} is invalid")
|
|
return WorkspaceServiceRecord.from_payload(payload)
|
|
|
|
def _load_workspace_snapshot_locked(
|
|
self,
|
|
workspace_id: str,
|
|
snapshot_name: str,
|
|
) -> WorkspaceSnapshotRecord:
|
|
record_path = self._workspace_snapshot_metadata_path(workspace_id, snapshot_name)
|
|
if not record_path.exists():
|
|
raise ValueError(
|
|
f"snapshot {snapshot_name!r} does not exist in workspace {workspace_id!r}"
|
|
)
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError(f"snapshot record at {record_path} is invalid")
|
|
return WorkspaceSnapshotRecord.from_payload(payload)
|
|
|
|
def _load_workspace_snapshot_locked_optional(
|
|
self,
|
|
workspace_id: str,
|
|
snapshot_name: str,
|
|
) -> WorkspaceSnapshotRecord | None:
|
|
record_path = self._workspace_snapshot_metadata_path(workspace_id, snapshot_name)
|
|
if not record_path.exists():
|
|
return None
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError(f"snapshot record at {record_path} is invalid")
|
|
return WorkspaceSnapshotRecord.from_payload(payload)
|
|
|
|
def _load_workspace_service_locked_optional(
|
|
self,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
) -> WorkspaceServiceRecord | None:
|
|
record_path = self._workspace_service_record_path(workspace_id, service_name)
|
|
if not record_path.exists():
|
|
return None
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError(f"service record at {record_path} is invalid")
|
|
return WorkspaceServiceRecord.from_payload(payload)
|
|
|
|
def _save_workspace_service_locked(self, service: WorkspaceServiceRecord) -> None:
|
|
record_path = self._workspace_service_record_path(
|
|
service.workspace_id,
|
|
service.service_name,
|
|
)
|
|
record_path.parent.mkdir(parents=True, exist_ok=True)
|
|
record_path.write_text(
|
|
json.dumps(service.to_payload(), indent=2, sort_keys=True),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
def _save_workspace_snapshot_locked(self, snapshot: WorkspaceSnapshotRecord) -> None:
|
|
record_path = self._workspace_snapshot_metadata_path(
|
|
snapshot.workspace_id,
|
|
snapshot.snapshot_name,
|
|
)
|
|
record_path.parent.mkdir(parents=True, exist_ok=True)
|
|
record_path.write_text(
|
|
json.dumps(snapshot.to_payload(), indent=2, sort_keys=True),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
def _delete_workspace_service_artifacts_locked(
|
|
self,
|
|
workspace_id: str,
|
|
service_name: str,
|
|
) -> None:
|
|
existing = self._load_workspace_service_locked_optional(workspace_id, service_name)
|
|
if existing is not None:
|
|
for published_port in existing.published_ports:
|
|
_stop_workspace_published_port_proxy(published_port)
|
|
self._workspace_service_record_path(workspace_id, service_name).unlink(missing_ok=True)
|
|
services_dir = self._workspace_services_dir(workspace_id)
|
|
_workspace_service_stdout_path(services_dir, service_name).unlink(missing_ok=True)
|
|
_workspace_service_stderr_path(services_dir, service_name).unlink(missing_ok=True)
|
|
_workspace_service_status_path(services_dir, service_name).unlink(missing_ok=True)
|
|
_workspace_service_runner_path(services_dir, service_name).unlink(missing_ok=True)
|
|
for ready_path in services_dir.glob(f"{service_name}.port-*.ready.json"):
|
|
ready_path.unlink(missing_ok=True)
|
|
|
|
def _delete_workspace_snapshot_locked(self, workspace_id: str, snapshot_name: str) -> None:
|
|
self._workspace_snapshot_metadata_path(workspace_id, snapshot_name).unlink(missing_ok=True)
|
|
self._workspace_snapshot_archive_path(workspace_id, snapshot_name).unlink(missing_ok=True)
|
|
|
|
def _list_workspace_services_locked(self, workspace_id: str) -> list[WorkspaceServiceRecord]:
|
|
services_dir = self._workspace_services_dir(workspace_id)
|
|
if not services_dir.exists():
|
|
return []
|
|
services: list[WorkspaceServiceRecord] = []
|
|
for record_path in sorted(services_dir.glob("*.json")):
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
continue
|
|
services.append(WorkspaceServiceRecord.from_payload(payload))
|
|
return services
|
|
|
|
def _list_workspace_snapshots_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
) -> list[WorkspaceSnapshotRecord]:
|
|
snapshots_dir = self._workspace_snapshots_dir(workspace.workspace_id)
|
|
snapshots: list[WorkspaceSnapshotRecord] = [
|
|
self._workspace_baseline_snapshot_locked(workspace)
|
|
]
|
|
if not snapshots_dir.exists():
|
|
return snapshots
|
|
named_snapshots: list[WorkspaceSnapshotRecord] = []
|
|
for record_path in snapshots_dir.glob("*.json"):
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
continue
|
|
named_snapshots.append(WorkspaceSnapshotRecord.from_payload(payload))
|
|
named_snapshots.sort(key=lambda item: (-item.created_at, item.snapshot_name))
|
|
snapshots.extend(named_snapshots)
|
|
return snapshots
|
|
|
|
def _save_workspace_shell_locked(self, shell: WorkspaceShellRecord) -> None:
|
|
record_path = self._workspace_shell_record_path(shell.workspace_id, shell.shell_id)
|
|
record_path.parent.mkdir(parents=True, exist_ok=True)
|
|
record_path.write_text(
|
|
json.dumps(shell.to_payload(), indent=2, sort_keys=True),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
def _delete_workspace_shell_locked(self, workspace_id: str, shell_id: str) -> None:
|
|
record_path = self._workspace_shell_record_path(workspace_id, shell_id)
|
|
if record_path.exists():
|
|
record_path.unlink()
|
|
|
|
def _list_workspace_shells_locked(self, workspace_id: str) -> list[WorkspaceShellRecord]:
|
|
shells_dir = self._workspace_shells_dir(workspace_id)
|
|
if not shells_dir.exists():
|
|
return []
|
|
shells: list[WorkspaceShellRecord] = []
|
|
for record_path in sorted(shells_dir.glob("*.json")):
|
|
payload = json.loads(record_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
continue
|
|
shells.append(WorkspaceShellRecord.from_payload(payload))
|
|
return shells
|
|
|
|
def _close_workspace_shells_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
instance: VmInstance,
|
|
) -> None:
|
|
for shell in self._list_workspace_shells_locked(workspace.workspace_id):
|
|
try:
|
|
self._backend.close_shell(
|
|
instance,
|
|
workspace_id=workspace.workspace_id,
|
|
shell_id=shell.shell_id,
|
|
)
|
|
except Exception:
|
|
pass
|
|
self._delete_workspace_shell_locked(workspace.workspace_id, shell.shell_id)
|
|
|
|
def _reset_workspace_runtime_dirs(self, workspace_id: str) -> None:
|
|
shutil.rmtree(self._workspace_runtime_dir(workspace_id), ignore_errors=True)
|
|
shutil.rmtree(self._workspace_host_dir(workspace_id), ignore_errors=True)
|
|
shutil.rmtree(self._workspace_commands_dir(workspace_id), ignore_errors=True)
|
|
shutil.rmtree(self._workspace_shells_dir(workspace_id), ignore_errors=True)
|
|
shutil.rmtree(self._workspace_services_dir(workspace_id), ignore_errors=True)
|
|
self._workspace_host_dir(workspace_id).mkdir(parents=True, exist_ok=True)
|
|
self._workspace_commands_dir(workspace_id).mkdir(parents=True, exist_ok=True)
|
|
self._workspace_shells_dir(workspace_id).mkdir(parents=True, exist_ok=True)
|
|
self._workspace_services_dir(workspace_id).mkdir(parents=True, exist_ok=True)
|
|
|
|
def _refresh_workspace_service_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
instance: VmInstance,
|
|
service: WorkspaceServiceRecord,
|
|
) -> WorkspaceServiceRecord:
|
|
payload = self._backend.status_service(
|
|
instance,
|
|
workspace_id=workspace.workspace_id,
|
|
service_name=service.service_name,
|
|
)
|
|
refreshed = self._workspace_service_record_from_payload(
|
|
workspace_id=workspace.workspace_id,
|
|
service_name=service.service_name,
|
|
payload=payload,
|
|
metadata=service.metadata,
|
|
published_ports=service.published_ports,
|
|
)
|
|
if refreshed.state != "running" and refreshed.published_ports:
|
|
refreshed.published_ports = [
|
|
WorkspacePublishedPortRecord(
|
|
guest_port=published_port.guest_port,
|
|
host_port=published_port.host_port,
|
|
host=published_port.host,
|
|
protocol=published_port.protocol,
|
|
proxy_pid=None,
|
|
)
|
|
for published_port in refreshed.published_ports
|
|
]
|
|
for published_port in service.published_ports:
|
|
_stop_workspace_published_port_proxy(published_port)
|
|
self._save_workspace_service_locked(refreshed)
|
|
return refreshed
|
|
|
|
def _refresh_workspace_services_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
instance: VmInstance,
|
|
) -> list[WorkspaceServiceRecord]:
|
|
services = self._list_workspace_services_locked(workspace.workspace_id)
|
|
refreshed: list[WorkspaceServiceRecord] = []
|
|
for service in services:
|
|
refreshed.append(self._refresh_workspace_service_locked(workspace, instance, service))
|
|
return refreshed
|
|
|
|
def _refresh_workspace_service_counts_locked(self, workspace: WorkspaceRecord) -> None:
|
|
services = self._list_workspace_services_locked(workspace.workspace_id)
|
|
if not services:
|
|
return
|
|
if workspace.state != "started":
|
|
changed = False
|
|
for service in services:
|
|
if service.state == "running":
|
|
for published_port in service.published_ports:
|
|
_stop_workspace_published_port_proxy(published_port)
|
|
service.state = "stopped"
|
|
service.stop_reason = "workspace_stopped"
|
|
service.ended_at = service.ended_at or time.time()
|
|
self._save_workspace_service_locked(service)
|
|
changed = True
|
|
if changed:
|
|
return
|
|
return
|
|
instance = workspace.to_instance(
|
|
workdir=self._workspace_runtime_dir(workspace.workspace_id)
|
|
)
|
|
self._require_workspace_service_support(instance)
|
|
self._refresh_workspace_services_locked(workspace, instance)
|
|
|
|
def _stop_workspace_services_locked(
|
|
self,
|
|
workspace: WorkspaceRecord,
|
|
instance: VmInstance,
|
|
) -> None:
|
|
for service in self._list_workspace_services_locked(workspace.workspace_id):
|
|
if workspace.state == "started":
|
|
try:
|
|
payload = self._backend.stop_service(
|
|
instance,
|
|
workspace_id=workspace.workspace_id,
|
|
service_name=service.service_name,
|
|
)
|
|
stopped = self._workspace_service_record_from_payload(
|
|
workspace_id=workspace.workspace_id,
|
|
service_name=service.service_name,
|
|
payload=payload,
|
|
metadata=service.metadata,
|
|
published_ports=service.published_ports,
|
|
)
|
|
self._save_workspace_service_locked(stopped)
|
|
except Exception:
|
|
pass
|
|
self._delete_workspace_service_artifacts_locked(
|
|
workspace.workspace_id,
|
|
service.service_name,
|
|
)
|