From 6988d85f7db52744cb39c0fe125471daa8d9b1dc Mon Sep 17 00:00:00 2001 From: Thales Maciel Date: Mon, 9 Mar 2026 17:39:18 -0300 Subject: [PATCH] Switch official environment publishing to Docker Hub --- .github/workflows/publish-environments.yml | 7 +- AGENTS.md | 2 +- Makefile | 1 - README.md | 12 +- docs/public-contract.md | 2 +- docs/troubleshooting.md | 2 +- runtime_sources/README.md | 3 +- src/pyro_mcp/runtime_build.py | 177 ++++++++- src/pyro_mcp/vm_environments.py | 12 +- tests/test_runtime_build.py | 410 +++++++++++++++++++-- tests/test_vm_environments.py | 35 +- 11 files changed, 590 insertions(+), 73 deletions(-) diff --git a/.github/workflows/publish-environments.yml b/.github/workflows/publish-environments.yml index 6089bbd..e46446b 100644 --- a/.github/workflows/publish-environments.yml +++ b/.github/workflows/publish-environments.yml @@ -8,7 +8,6 @@ on: permissions: contents: read - packages: write concurrency: group: publish-environments-${{ github.ref }} @@ -19,8 +18,8 @@ jobs: runs-on: ubuntu-24.04 env: UV_CACHE_DIR: .uv-cache - OCI_REGISTRY_USERNAME: ${{ github.actor }} - OCI_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + OCI_REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + OCI_REGISTRY_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} steps: - name: Check out source uses: actions/checkout@v4 @@ -42,5 +41,5 @@ jobs: - name: Build real runtime inputs run: make runtime-materialize - - name: Publish official environments to GHCR + - name: Publish official environments to Docker Hub run: make runtime-publish-official-environments-oci diff --git a/AGENTS.md b/AGENTS.md index 2910223..9836499 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,7 +32,7 @@ This repository ships `pyro-mcp`, an MCP-compatible package for ephemeral VM lif - After heavy runtime work, reclaim local space with `rm -rf build` and `git lfs prune`. - The pre-migration `pre-lfs-*` tag is local backup material only; do not push it or it will keep the old giant blobs reachable. - Public contract documentation lives in `docs/public-contract.md`. -- Official GHCR publication workflow lives in `.github/workflows/publish-environments.yml`. +- Official Docker Hub publication workflow lives in `.github/workflows/publish-environments.yml`. ## Quality Gates diff --git a/Makefile b/Makefile index 975796c..3a65546 100644 --- a/Makefile +++ b/Makefile @@ -146,7 +146,6 @@ runtime-publish-environment-oci: runtime-publish-official-environments-oci: @for environment in $(RUNTIME_ENVIRONMENTS); do \ - $(MAKE) runtime-export-environment-oci RUNTIME_ENVIRONMENT="$$environment"; \ $(MAKE) runtime-publish-environment-oci RUNTIME_ENVIRONMENT="$$environment"; \ done diff --git a/README.md b/README.md index 143588d..5c34f26 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Current official environments in the shipped catalog: - `debian:12-build` The package ships the embedded Firecracker runtime and a package-controlled environment catalog. -Official environments are pulled as OCI artifacts from GHCR into a local cache on first use or +Official environments are pulled as OCI artifacts from Docker Hub into a local cache on first use or through `pyro env pull`. ## CLI @@ -195,10 +195,16 @@ Contributor runtime source artifacts are still maintained under `src/pyro_mcp/ru Official environment publication is automated through `.github/workflows/publish-environments.yml`. -For a local publish dry run against GHCR-compatible credentials: +For a local publish against Docker Hub: ```bash make runtime-materialize -OCI_REGISTRY_USERNAME="$GITHUB_USER" OCI_REGISTRY_PASSWORD="$GITHUB_TOKEN" \ +OCI_REGISTRY_USERNAME="$DOCKERHUB_USERNAME" OCI_REGISTRY_PASSWORD="$DOCKERHUB_TOKEN" \ make runtime-publish-official-environments-oci ``` + +`make runtime-publish-environment-oci` auto-exports the OCI layout for the selected +environment if it is missing. +Docker Hub uploads are chunked by default for large rootfs layers; if you need to tune a slow +link, use `PYRO_OCI_UPLOAD_TIMEOUT_SECONDS`, `PYRO_OCI_UPLOAD_CHUNK_SIZE_BYTES`, and +`PYRO_OCI_REQUEST_TIMEOUT_SECONDS`. diff --git a/docs/public-contract.md b/docs/public-contract.md index e8b09e0..1497958 100644 --- a/docs/public-contract.md +++ b/docs/public-contract.md @@ -34,7 +34,7 @@ Stable `pyro run` interface: Behavioral guarantees: -- `pyro run -- ` returns structured JSON. +- `pyro run --vcpu-count --mem-mib -- ` returns structured JSON. - `pyro env list`, `pyro env pull`, `pyro env inspect`, and `pyro env prune` return structured JSON. - `pyro doctor` returns structured JSON diagnostics. - `pyro demo ollama` prints log lines plus a final summary line. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 059f276..5bd9cef 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -5,7 +5,7 @@ Cause: - the environment cache directory is not writable -- the configured GHCR environment artifact is unavailable +- the configured registry artifact is unavailable - the environment download was interrupted Fix: diff --git a/runtime_sources/README.md b/runtime_sources/README.md index bd128a1..356beed 100644 --- a/runtime_sources/README.md +++ b/runtime_sources/README.md @@ -18,7 +18,8 @@ Materialization workflow: Official environment publication workflow: 1. `make runtime-materialize` 2. `OCI_REGISTRY_USERNAME=... OCI_REGISTRY_PASSWORD=... make runtime-publish-official-environments-oci` -3. or run the repo workflow at `.github/workflows/publish-environments.yml` +3. or run the repo workflow at `.github/workflows/publish-environments.yml` with Docker Hub credentials +4. if your uplink is slow, tune publishing with `PYRO_OCI_UPLOAD_TIMEOUT_SECONDS`, `PYRO_OCI_UPLOAD_CHUNK_SIZE_BYTES`, and `PYRO_OCI_REQUEST_TIMEOUT_SECONDS` Build requirements for the real path: - `docker` diff --git a/src/pyro_mcp/runtime_build.py b/src/pyro_mcp/runtime_build.py index e7f9203..7863210 100644 --- a/src/pyro_mcp/runtime_build.py +++ b/src/pyro_mcp/runtime_build.py @@ -37,6 +37,15 @@ OCI_LAYOUT_VERSION = "1.0.0" VALIDATION_READ_LIMIT = 1024 * 1024 DEFAULT_OCI_USERNAME_ENV = "OCI_REGISTRY_USERNAME" DEFAULT_OCI_PASSWORD_ENV = "OCI_REGISTRY_PASSWORD" +DEFAULT_DOCKERHUB_USERNAME_ENV = "DOCKERHUB_USERNAME" +DEFAULT_DOCKERHUB_TOKEN_ENV = "DOCKERHUB_TOKEN" +DEFAULT_OCI_REQUEST_TIMEOUT_ENV = "PYRO_OCI_REQUEST_TIMEOUT_SECONDS" +DEFAULT_OCI_UPLOAD_TIMEOUT_ENV = "PYRO_OCI_UPLOAD_TIMEOUT_SECONDS" +DEFAULT_OCI_UPLOAD_CHUNK_SIZE_ENV = "PYRO_OCI_UPLOAD_CHUNK_SIZE_BYTES" +DEFAULT_OCI_REQUEST_TIMEOUT_SECONDS = 90 +DEFAULT_OCI_UPLOAD_TIMEOUT_SECONDS = 900 +DEFAULT_OCI_UPLOAD_CHUNK_SIZE_BYTES = 8 * 1024 * 1024 +_REGISTRY_BEARER_TOKENS: dict[tuple[str, str, str, str], str] = {} @dataclass(frozen=True) @@ -269,7 +278,24 @@ def _basic_auth_header(username: str | None, password: str | None) -> str | None def _parse_authenticate_parameters(raw: str) -> dict[str, str]: params: dict[str, str] = {} - for segment in raw.split(","): + segments: list[str] = [] + current: list[str] = [] + in_quotes = False + for char in raw: + if char == '"': + in_quotes = not in_quotes + if char == "," and not in_quotes: + segment = "".join(current).strip() + if segment: + segments.append(segment) + current = [] + continue + current.append(char) + tail = "".join(current).strip() + if tail: + segments.append(tail) + + for segment in segments: if "=" not in segment: continue key, value = segment.split("=", 1) @@ -286,8 +312,8 @@ def _registry_credentials( username_env: str = DEFAULT_OCI_USERNAME_ENV, password_env: str = DEFAULT_OCI_PASSWORD_ENV, ) -> tuple[str | None, str | None]: - username = os.environ.get(username_env) - password = os.environ.get(password_env) + username = os.environ.get(username_env) or os.environ.get(DEFAULT_DOCKERHUB_USERNAME_ENV) + password = os.environ.get(password_env) or os.environ.get(DEFAULT_DOCKERHUB_TOKEN_ENV) return ( username if isinstance(username, str) and username != "" else None, password if isinstance(password, str) and password != "" else None, @@ -305,6 +331,55 @@ def _read_response_body(response: Any) -> bytes: return b"" +def _positive_int_from_env(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None or raw == "": + return default + try: + value = int(raw) + except ValueError as exc: + raise RuntimeError(f"{name} must be an integer; got {raw!r}") from exc + if value <= 0: + raise RuntimeError(f"{name} must be greater than zero; got {value}") + return value + + +def _oci_request_timeout_seconds() -> int: + return _positive_int_from_env( + DEFAULT_OCI_REQUEST_TIMEOUT_ENV, + DEFAULT_OCI_REQUEST_TIMEOUT_SECONDS, + ) + + +def _oci_upload_timeout_seconds() -> int: + return _positive_int_from_env( + DEFAULT_OCI_UPLOAD_TIMEOUT_ENV, + DEFAULT_OCI_UPLOAD_TIMEOUT_SECONDS, + ) + + +def _oci_upload_chunk_size_bytes() -> int: + return _positive_int_from_env( + DEFAULT_OCI_UPLOAD_CHUNK_SIZE_ENV, + DEFAULT_OCI_UPLOAD_CHUNK_SIZE_BYTES, + ) + + +def _registry_token_cache_key( + *, + url: str, + repository: str, + scope_actions: str, + username: str | None, +) -> tuple[str, str, str, str]: + return ( + urllib.parse.urlsplit(url).netloc, + repository, + scope_actions, + username or "", + ) + + def _fetch_registry_token( authenticate: str, *, @@ -332,7 +407,7 @@ def _fetch_registry_token( headers=headers, method="GET", ) - with urllib.request.urlopen(request, timeout=90) as response: # noqa: S310 + with urllib.request.urlopen(request, timeout=_oci_request_timeout_seconds()) as response: # noqa: S310 payload = json.loads(_read_response_body(response).decode("utf-8")) if not isinstance(payload, dict): raise RuntimeError("OCI auth token response was not a JSON object") @@ -353,8 +428,19 @@ def _request_registry( allow_statuses: tuple[int, ...] = (), username: str | None = None, password: str | None = None, + timeout_seconds: int | None = None, ) -> tuple[int, bytes, dict[str, str]]: request_headers = dict(headers or {}) + request_timeout = timeout_seconds or _oci_request_timeout_seconds() + cache_key = _registry_token_cache_key( + url=url, + repository=repository, + scope_actions=scope_actions, + username=username, + ) + cached_token = _REGISTRY_BEARER_TOKENS.get(cache_key) + if cached_token is not None and "Authorization" not in request_headers: + request_headers["Authorization"] = f"Bearer {cached_token}" request = urllib.request.Request( url, data=cast(Any, data), @@ -362,7 +448,7 @@ def _request_registry( method=method, ) try: - with urllib.request.urlopen(request, timeout=90) as response: # noqa: S310 + with urllib.request.urlopen(request, timeout=request_timeout) as response: # noqa: S310 return ( response.status, _read_response_body(response), @@ -380,6 +466,7 @@ def _request_registry( username=username, password=password, ) + _REGISTRY_BEARER_TOKENS[cache_key] = token authenticated_headers = {**request_headers, "Authorization": f"Bearer {token}"} _rewind_body(data) retry = urllib.request.Request( @@ -389,7 +476,7 @@ def _request_registry( method=method, ) try: - with urllib.request.urlopen(retry, timeout=90) as response: # noqa: S310 + with urllib.request.urlopen(retry, timeout=request_timeout) as response: # noqa: S310 return ( response.status, _read_response_body(response), @@ -403,9 +490,13 @@ def _request_registry( _normalize_headers(retry_exc.headers), ) raise RuntimeError(f"registry request failed for {url}: {retry_exc}") from retry_exc + except urllib.error.URLError as retry_exc: + raise RuntimeError(f"registry request failed for {url}: {retry_exc}") from retry_exc if exc.code in allow_statuses: return exc.code, _read_response_body(exc), _normalize_headers(exc.headers) raise RuntimeError(f"registry request failed for {url}: {exc}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"registry request failed for {url}: {exc}") from exc def _load_oci_layout_manifest( @@ -489,24 +580,71 @@ def _upload_blob( if location is None: raise RuntimeError("registry did not return a blob upload location") upload_url = urllib.parse.urljoin(f"https://{registry}", location) - separator = "&" if "?" in upload_url else "?" - upload_url = f"{upload_url}{separator}{urllib.parse.urlencode({'digest': digest})}" size = blob_path.stat().st_size + upload_timeout_seconds = _oci_upload_timeout_seconds() + upload_chunk_size = _oci_upload_chunk_size_bytes() with blob_path.open("rb") as blob_fp: + uploaded_bytes = 0 + chunk_index = 0 + while True: + chunk = blob_fp.read(upload_chunk_size) + if chunk == b"": + break + try: + status, _, headers = _request_registry( + upload_url, + method="PATCH", + repository=repository, + scope_actions="pull,push", + headers={ + "Content-Length": str(len(chunk)), + "Content-Type": "application/octet-stream", + }, + data=chunk, + allow_statuses=(202,), + username=username, + password=password, + timeout_seconds=upload_timeout_seconds, + ) + except RuntimeError as exc: + raise RuntimeError( + "registry blob upload patch failed for " + f"{digest} at byte offset {uploaded_bytes} (chunk {chunk_index}): {exc}" + ) from exc + if status != 202: + raise RuntimeError( + f"unexpected registry status when uploading blob chunk {chunk_index}: {status}" + ) + location = headers.get("location") + if location is None: + raise RuntimeError( + "registry did not return a blob upload location after chunk " + f"{chunk_index}" + ) + upload_url = urllib.parse.urljoin(f"https://{registry}", location) + uploaded_bytes += len(chunk) + chunk_index += 1 + + separator = "&" if "?" in upload_url else "?" + finalize_url = f"{upload_url}{separator}{urllib.parse.urlencode({'digest': digest})}" + try: status, _, _ = _request_registry( - upload_url, + finalize_url, method="PUT", repository=repository, scope_actions="pull,push", headers={ - "Content-Length": str(size), + "Content-Length": "0", "Content-Type": "application/octet-stream", }, - data=blob_fp, + data=b"", allow_statuses=(201,), username=username, password=password, + timeout_seconds=upload_timeout_seconds, ) + except RuntimeError as exc: + raise RuntimeError(f"registry blob upload finalize failed for {digest}: {exc}") from exc if status != 201: raise RuntimeError(f"unexpected registry status when uploading blob: {status}") return {"digest": digest, "size": size, "uploaded": True} @@ -535,6 +673,12 @@ def publish_environment_oci_layout( username_env=username_env, password_env=password_env, ) + if username is None or password is None: + raise RuntimeError( + "OCI registry credentials are not configured; set " + f"{username_env}/{password_env} or " + f"{DEFAULT_DOCKERHUB_USERNAME_ENV}/{DEFAULT_DOCKERHUB_TOKEN_ENV}" + ) config = manifest.get("config") layers = manifest.get("layers") @@ -1166,9 +1310,18 @@ def main() -> None: # pragma: no cover - CLI wiring if args.command == "publish-environment-oci": if not isinstance(args.environment, str) or args.environment == "": raise RuntimeError("--environment is required for publish-environment-oci") + layout_root = Path(args.layout_root) + layout_index = layout_root / _environment_slug(args.environment) / "index.json" + if not layout_index.exists(): + export_environment_oci_layout( + paths, + environment=args.environment, + output_dir=layout_root, + reference=args.reference, + ) result = publish_environment_oci_layout( environment=args.environment, - layout_root=Path(args.layout_root), + layout_root=layout_root, registry=args.registry, repository=args.repository, reference=args.reference, diff --git a/src/pyro_mcp/vm_environments.py b/src/pyro_mcp/vm_environments.py index 79392cc..2017a1a 100644 --- a/src/pyro_mcp/vm_environments.py +++ b/src/pyro_mcp/vm_environments.py @@ -74,8 +74,8 @@ CATALOG: dict[str, VmEnvironment] = { distribution="debian", distribution_version="12", source_profile="debian-git", - oci_registry="ghcr.io", - oci_repository="thaloco/pyro-environments/debian-12", + oci_registry="registry-1.docker.io", + oci_repository="thalesmaciel/pyro-environment-debian-12", oci_reference=DEFAULT_ENVIRONMENT_VERSION, ), "debian:12-base": VmEnvironment( @@ -86,8 +86,8 @@ CATALOG: dict[str, VmEnvironment] = { distribution="debian", distribution_version="12", source_profile="debian-base", - oci_registry="ghcr.io", - oci_repository="thaloco/pyro-environments/debian-12-base", + oci_registry="registry-1.docker.io", + oci_repository="thalesmaciel/pyro-environment-debian-12-base", oci_reference=DEFAULT_ENVIRONMENT_VERSION, ), "debian:12-build": VmEnvironment( @@ -98,8 +98,8 @@ CATALOG: dict[str, VmEnvironment] = { distribution="debian", distribution_version="12", source_profile="debian-build", - oci_registry="ghcr.io", - oci_repository="thaloco/pyro-environments/debian-12-build", + oci_registry="registry-1.docker.io", + oci_repository="thalesmaciel/pyro-environment-debian-12-build", oci_reference=DEFAULT_ENVIRONMENT_VERSION, ), } diff --git a/tests/test_runtime_build.py b/tests/test_runtime_build.py index 6c66464..eea2e3c 100644 --- a/tests/test_runtime_build.py +++ b/tests/test_runtime_build.py @@ -13,9 +13,14 @@ from pathlib import Path import pytest from pyro_mcp.runtime_build import ( + _REGISTRY_BEARER_TOKENS, + DEFAULT_DOCKERHUB_TOKEN_ENV, + DEFAULT_DOCKERHUB_USERNAME_ENV, _build_paths, _load_lock, _load_oci_layout_manifest, + _parse_authenticate_parameters, + _registry_credentials, _request_registry, _upload_blob, build_bundle, @@ -362,13 +367,77 @@ def test_runtime_build_main_dispatches_export( assert payload["manifest_digest"] == "sha256:test" +def test_runtime_build_main_dispatches_publish_with_implicit_export( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + tmp_path: Path, +) -> None: + source_dir, build_dir, bundle_dir = _make_source_tree(tmp_path) + layout_root = tmp_path / "oci_layouts" + calls: list[str] = [] + + class StubParser: + def parse_args(self) -> argparse.Namespace: + return argparse.Namespace( + command="publish-environment-oci", + platform="linux-x86_64", + source_dir=str(source_dir), + build_dir=str(build_dir), + bundle_dir=str(bundle_dir), + materialized_dir=str(tmp_path / "materialized"), + environment="debian:12-base", + output_dir=str(layout_root), + layout_root=str(layout_root), + registry=None, + repository=None, + reference="1.0.0", + username_env="OCI_REGISTRY_USERNAME", + password_env="OCI_REGISTRY_PASSWORD", + ) + + monkeypatch.setattr("pyro_mcp.runtime_build._build_parser", lambda: StubParser()) + def fake_export(paths: object, **kwargs: object) -> dict[str, str]: + del paths + environment = str(kwargs["environment"]) + output_dir = Path(str(kwargs["output_dir"])) + calls.append(f"export:{environment}") + return { + "environment": environment, + "layout_dir": str(output_dir / "debian_12-base"), + "manifest_digest": "sha256:exported", + } + + def fake_publish(**kwargs: object) -> dict[str, str]: + environment = str(kwargs["environment"]) + layout_root = Path(str(kwargs["layout_root"])) + calls.append(f"publish:{environment}") + return { + "environment": environment, + "layout_dir": str(layout_root / "debian_12-base"), + "manifest_digest": "sha256:published", + } + + monkeypatch.setattr("pyro_mcp.runtime_build.export_environment_oci_layout", fake_export) + monkeypatch.setattr("pyro_mcp.runtime_build.publish_environment_oci_layout", fake_publish) + + main() + + payload = json.loads(capsys.readouterr().out) + assert calls == ["export:debian:12-base", "publish:debian:12-base"] + assert payload["environment"] == "debian:12-base" + assert payload["manifest_digest"] == "sha256:published" + + def test_request_registry_retries_with_bearer_token(monkeypatch: pytest.MonkeyPatch) -> None: requested_auth: list[str | None] = [] + requested_token_urls: list[str] = [] + _REGISTRY_BEARER_TOKENS.clear() def fake_urlopen(request: object, timeout: int = 90) -> FakeResponse: del timeout url = request.full_url if isinstance(request, urllib.request.Request) else str(request) - if url.startswith("https://ghcr.io/token?"): + if url.startswith("https://auth.docker.io/token?"): + requested_token_urls.append(url) authorization = ( _request_header(request, "Authorization") if isinstance(request, urllib.request.Request) @@ -389,9 +458,9 @@ def test_request_registry_retries_with_bearer_token(monkeypatch: pytest.MonkeyPa _http_headers( { "WWW-Authenticate": ( - 'Bearer realm="https://ghcr.io/token",' - 'service="ghcr.io",' - 'scope="repository:thaloco/pyro-environments/debian-12-base:pull,push"' + 'Bearer realm="https://auth.docker.io/token",' + 'service="registry.docker.io",' + 'scope="repository:thalesmaciel/pyro-environment-debian-12-base:pull,push"' ) } ), @@ -402,9 +471,9 @@ def test_request_registry_retries_with_bearer_token(monkeypatch: pytest.MonkeyPa monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) status, payload, headers = _request_registry( - "https://ghcr.io/v2/thaloco/pyro-environments/debian-12-base/test", + "https://registry-1.docker.io/v2/thalesmaciel/pyro-environment-debian-12-base/test", method="GET", - repository="thaloco/pyro-environments/debian-12-base", + repository="thalesmaciel/pyro-environment-debian-12-base", scope_actions="pull,push", username="user", password="token", @@ -416,9 +485,45 @@ def test_request_registry_retries_with_bearer_token(monkeypatch: pytest.MonkeyPa assert requested_auth[0] is None assert requested_auth[1] is not None and requested_auth[1].startswith("Basic ") assert requested_auth[2] == "Bearer secret-token" + assert requested_token_urls == [ + ( + "https://auth.docker.io/token?service=registry.docker.io&" + "scope=repository%3Athalesmaciel%2Fpyro-environment-debian-12-base%3Apull%2Cpush" + ) + ] + + +def test_parse_authenticate_parameters_preserves_quoted_commas() -> None: + params = _parse_authenticate_parameters( + 'realm="https://auth.docker.io/token",' + 'service="registry.docker.io",' + 'scope="repository:thalesmaciel/pyro-environment-debian-12:pull,push"' + ) + + assert params == { + "realm": "https://auth.docker.io/token", + "service": "registry.docker.io", + "scope": "repository:thalesmaciel/pyro-environment-debian-12:pull,push", + } + + +def test_registry_credentials_fall_back_to_dockerhub_envs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("OCI_REGISTRY_USERNAME", raising=False) + monkeypatch.delenv("OCI_REGISTRY_PASSWORD", raising=False) + monkeypatch.setenv(DEFAULT_DOCKERHUB_USERNAME_ENV, "docker-user") + monkeypatch.setenv(DEFAULT_DOCKERHUB_TOKEN_ENV, "docker-token") + + username, password = _registry_credentials() + + assert username == "docker-user" + assert password == "docker-token" def test_request_registry_requires_auth_challenge(monkeypatch: pytest.MonkeyPatch) -> None: + _REGISTRY_BEARER_TOKENS.clear() + def fake_urlopen(request: object, timeout: int = 90) -> FakeResponse: del timeout url = request.full_url if isinstance(request, urllib.request.Request) else str(request) @@ -434,18 +539,20 @@ def test_request_registry_requires_auth_challenge(monkeypatch: pytest.MonkeyPatc with pytest.raises(RuntimeError, match="denied access without an auth challenge"): _request_registry( - "https://ghcr.io/v2/thaloco/pyro-environments/debian-12-base/test", + "https://registry-1.docker.io/v2/thalesmaciel/pyro-environment-debian-12-base/test", method="GET", - repository="thaloco/pyro-environments/debian-12-base", + repository="thalesmaciel/pyro-environment-debian-12-base", scope_actions="pull,push", ) def test_request_registry_returns_allowed_retry_status(monkeypatch: pytest.MonkeyPatch) -> None: + _REGISTRY_BEARER_TOKENS.clear() + def fake_urlopen(request: object, timeout: int = 90) -> FakeResponse: del timeout url = request.full_url if isinstance(request, urllib.request.Request) else str(request) - if url.startswith("https://ghcr.io/token?"): + if url.startswith("https://auth.docker.io/token?"): return FakeResponse(b'{"token":"secret-token"}') authorization = ( _request_header(request, "Authorization") @@ -460,9 +567,9 @@ def test_request_registry_returns_allowed_retry_status(monkeypatch: pytest.Monke _http_headers( { "WWW-Authenticate": ( - 'Bearer realm="https://ghcr.io/token",' - 'service="ghcr.io",' - 'scope="repository:thaloco/pyro-environments/debian-12-base:pull,push"' + 'Bearer realm="https://auth.docker.io/token",' + 'service="registry.docker.io",' + 'scope="repository:thalesmaciel/pyro-environment-debian-12-base:pull,push"' ) } ), @@ -479,9 +586,9 @@ def test_request_registry_returns_allowed_retry_status(monkeypatch: pytest.Monke monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) status, payload, headers = _request_registry( - "https://ghcr.io/v2/thaloco/pyro-environments/debian-12-base/test", + "https://registry-1.docker.io/v2/thalesmaciel/pyro-environment-debian-12-base/test", method="GET", - repository="thaloco/pyro-environments/debian-12-base", + repository="thalesmaciel/pyro-environment-debian-12-base", scope_actions="pull,push", allow_statuses=(404,), ) @@ -491,6 +598,78 @@ def test_request_registry_returns_allowed_retry_status(monkeypatch: pytest.Monke assert headers["docker-content-digest"] == "sha256:missing" +def test_request_registry_reuses_cached_bearer_token(monkeypatch: pytest.MonkeyPatch) -> None: + requested_auth: list[str | None] = [] + requested_token_urls: list[str] = [] + _REGISTRY_BEARER_TOKENS.clear() + + def fake_urlopen(request: object, timeout: int = 90) -> FakeResponse: + del timeout + url = request.full_url if isinstance(request, urllib.request.Request) else str(request) + authorization = ( + _request_header(request, "Authorization") + if isinstance(request, urllib.request.Request) + else None + ) + requested_auth.append(authorization if isinstance(authorization, str) else None) + if url.startswith("https://auth.docker.io/token?"): + requested_token_urls.append(url) + return FakeResponse(b'{"token":"secret-token"}') + if url.endswith("/first"): + if authorization is None: + raise urllib.error.HTTPError( + url, + 401, + "Unauthorized", + _http_headers( + { + "WWW-Authenticate": ( + 'Bearer realm="https://auth.docker.io/token",' + 'service="registry.docker.io",' + 'scope="repository:thalesmaciel/pyro-environment-debian-12:pull,push"' + ) + } + ), + io.BytesIO(b""), + ) + return FakeResponse(b"ok-first") + if url.endswith("/second"): + if authorization != "Bearer secret-token": + raise AssertionError("expected cached bearer token on second request") + return FakeResponse(b"ok-second") + raise AssertionError(f"unexpected registry request: {url}") + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + + first_status, first_payload, _ = _request_registry( + "https://registry-1.docker.io/v2/thalesmaciel/pyro-environment-debian-12/first", + method="GET", + repository="thalesmaciel/pyro-environment-debian-12", + scope_actions="pull,push", + username="user", + password="token", + ) + second_status, second_payload, _ = _request_registry( + "https://registry-1.docker.io/v2/thalesmaciel/pyro-environment-debian-12/second", + method="PUT", + repository="thalesmaciel/pyro-environment-debian-12", + scope_actions="pull,push", + username="user", + password="token", + ) + + assert first_status == 200 + assert first_payload == b"ok-first" + assert second_status == 200 + assert second_payload == b"ok-second" + assert requested_token_urls == [ + ( + "https://auth.docker.io/token?service=registry.docker.io&" + "scope=repository%3Athalesmaciel%2Fpyro-environment-debian-12%3Apull%2Cpush" + ) + ] + + @pytest.mark.parametrize( ("index_payload", "message"), [ @@ -523,8 +702,8 @@ def test_upload_blob_skips_existing_blob(monkeypatch: pytest.MonkeyPatch, tmp_pa monkeypatch.setattr("pyro_mcp.runtime_build._blob_exists", lambda **_: True) result = _upload_blob( - registry="ghcr.io", - repository="thaloco/pyro-environments/debian-12-base", + registry="registry-1.docker.io", + repository="thalesmaciel/pyro-environment-debian-12-base", digest="sha256:abc123", blob_path=blob_path, username=None, @@ -549,8 +728,132 @@ def test_upload_blob_requires_location(monkeypatch: pytest.MonkeyPatch, tmp_path with pytest.raises(RuntimeError, match="did not return a blob upload location"): _upload_blob( - registry="ghcr.io", - repository="thaloco/pyro-environments/debian-12-base", + registry="registry-1.docker.io", + repository="thalesmaciel/pyro-environment-debian-12-base", + digest="sha256:abc123", + blob_path=blob_path, + username=None, + password=None, + ) + + +def test_upload_blob_uses_patch_chunks_and_finalize_put( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + blob_path = tmp_path / "blob.tar" + blob_path.write_bytes(b"abcdefghij") + calls: list[tuple[str, str, int, int | None]] = [] + monkeypatch.setenv("PYRO_OCI_UPLOAD_CHUNK_SIZE_BYTES", "4") + monkeypatch.setenv("PYRO_OCI_UPLOAD_TIMEOUT_SECONDS", "123") + monkeypatch.setattr("pyro_mcp.runtime_build._blob_exists", lambda **_: False) + + def fake_request_registry( + url: str, + *, + method: str, + repository: str, + scope_actions: str, + headers: dict[str, str] | None = None, + data: object = None, + allow_statuses: tuple[int, ...] = (), + username: str | None = None, + password: str | None = None, + timeout_seconds: int | None = None, + ) -> tuple[int, bytes, dict[str, str]]: + del repository, scope_actions, allow_statuses, username, password + if isinstance(data, bytes): + payload_length = len(data) + else: + payload_length = 0 + calls.append((method, url, payload_length, timeout_seconds)) + if method == "POST": + return 202, b"", {"location": "/upload/session-1"} + if method == "PATCH": + return 202, b"", {"location": "/upload/session-1"} + if method == "PUT": + assert isinstance(headers, dict) + assert headers["Content-Length"] == "0" + return 201, b"", {} + raise AssertionError(f"unexpected method: {method}") + + monkeypatch.setattr("pyro_mcp.runtime_build._request_registry", fake_request_registry) + + result = _upload_blob( + registry="registry-1.docker.io", + repository="thalesmaciel/pyro-environment-debian-12-base", + digest="sha256:abc123", + blob_path=blob_path, + username=None, + password=None, + ) + + assert result == {"digest": "sha256:abc123", "size": 10, "uploaded": True} + assert calls == [ + ( + "POST", + "https://registry-1.docker.io/v2/thalesmaciel/pyro-environment-debian-12-base/blobs/uploads/", + 0, + None, + ), + ("PATCH", "https://registry-1.docker.io/upload/session-1", 4, 123), + ("PATCH", "https://registry-1.docker.io/upload/session-1", 4, 123), + ("PATCH", "https://registry-1.docker.io/upload/session-1", 2, 123), + ("PUT", "https://registry-1.docker.io/upload/session-1?digest=sha256%3Aabc123", 0, 123), + ] + + +def test_upload_blob_reports_patch_offset_on_failure( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + blob_path = tmp_path / "blob.tar" + blob_path.write_bytes(b"abcdefghij") + monkeypatch.setenv("PYRO_OCI_UPLOAD_CHUNK_SIZE_BYTES", "4") + monkeypatch.setattr("pyro_mcp.runtime_build._blob_exists", lambda **_: False) + + def fake_request_registry( + url: str, + *, + method: str, + repository: str, + scope_actions: str, + headers: dict[str, str] | None = None, + data: object = None, + allow_statuses: tuple[int, ...] = (), + username: str | None = None, + password: str | None = None, + timeout_seconds: int | None = None, + ) -> tuple[int, bytes, dict[str, str]]: + del ( + url, + repository, + scope_actions, + headers, + data, + allow_statuses, + username, + password, + timeout_seconds, + ) + if method == "POST": + return 202, b"", {"location": "/upload/session-1"} + if method == "PATCH": + raise RuntimeError( + "registry request failed for " + "https://registry-1.docker.io/upload/session-1: timed out" + ) + raise AssertionError(f"unexpected method: {method}") + + monkeypatch.setattr("pyro_mcp.runtime_build._request_registry", fake_request_registry) + + with pytest.raises( + RuntimeError, + match=r"patch failed.*byte offset 0 \(chunk 0\)", + ): + _upload_blob( + registry="registry-1.docker.io", + repository="thalesmaciel/pyro-environment-debian-12-base", digest="sha256:abc123", blob_path=blob_path, username=None, @@ -602,9 +905,13 @@ def test_publish_environment_oci_layout_uploads_blobs_and_manifest( output_dir=tmp_path / "oci_layouts", ) - uploads: list[tuple[str, int]] = [] + patch_uploads: list[tuple[str, int]] = [] + finalize_uploads: list[str] = [] manifest_publish: list[tuple[str, str, int]] = [] upload_counter = 0 + monkeypatch.setenv("OCI_REGISTRY_USERNAME", "registry-user") + monkeypatch.setenv("OCI_REGISTRY_PASSWORD", "registry-token") + monkeypatch.setenv("PYRO_OCI_UPLOAD_CHUNK_SIZE_BYTES", "1048576") def fake_urlopen(request: object, timeout: int = 90) -> FakeResponse: del timeout @@ -627,16 +934,20 @@ def test_publish_environment_oci_layout_uploads_blobs_and_manifest( status=202, headers={"Location": f"/upload/session-{upload_counter}"}, ) - if method == "PUT" and "/upload/session-" in url: - digest = url.split("digest=", 1)[1] + if method == "PATCH" and "/upload/session-" in url: body = request.data if isinstance(body, bytes): payload = body - elif body is not None and hasattr(body, "read"): - payload = body.read() else: raise AssertionError("expected upload body") - uploads.append((digest, len(payload))) + patch_uploads.append((url, len(payload))) + session_id = url.rsplit("-", 1)[1] + return FakeResponse(status=202, headers={"Location": f"/upload/session-{session_id}"}) + if method == "PUT" and "/upload/session-" in url: + body = request.data + if body != b"": + raise AssertionError("expected finalize PUT to have an empty body") + finalize_uploads.append(url) return FakeResponse(status=201) if method == "PUT" and "/manifests/" in url: body = request.data @@ -660,20 +971,65 @@ def test_publish_environment_oci_layout_uploads_blobs_and_manifest( result = publish_environment_oci_layout( environment="debian:12-base", layout_root=tmp_path / "oci_layouts", - registry="ghcr.io", - repository="thaloco/pyro-environments/debian-12-base", + registry="registry-1.docker.io", + repository="thalesmaciel/pyro-environment-debian-12-base", reference="1.0.0", ) assert result["manifest_digest"] == "sha256:published-manifest" assert len(result["uploaded_blobs"]) == 4 assert all(bool(entry["uploaded"]) for entry in result["uploaded_blobs"]) - assert len(uploads) == 4 + assert len(patch_uploads) == 4 + assert len(finalize_uploads) == 4 + assert all( + url.startswith("https://registry-1.docker.io/upload/session-") + for url in finalize_uploads + ) + assert all("digest=sha256%3A" in url for url in finalize_uploads) assert len(manifest_publish) == 1 assert manifest_publish[0][0].endswith("/manifests/1.0.0") assert manifest_publish[0][1] == "application/vnd.oci.image.manifest.v1+json" +def test_publish_environment_oci_layout_requires_credentials( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + source_dir, build_dir, bundle_dir = _make_source_tree(tmp_path) + paths = _build_paths( + source_dir=source_dir, + build_dir=build_dir, + bundle_dir=bundle_dir, + materialized_dir=tmp_path / "materialized_sources", + platform="linux-x86_64", + ) + export_environment_oci_layout( + paths, + environment="debian:12-base", + output_dir=tmp_path / "oci_layouts", + ) + monkeypatch.delenv("OCI_REGISTRY_USERNAME", raising=False) + monkeypatch.delenv("OCI_REGISTRY_PASSWORD", raising=False) + monkeypatch.delenv(DEFAULT_DOCKERHUB_USERNAME_ENV, raising=False) + monkeypatch.delenv(DEFAULT_DOCKERHUB_TOKEN_ENV, raising=False) + + with pytest.raises( + RuntimeError, + match=( + "OCI registry credentials are not configured; set " + "OCI_REGISTRY_USERNAME/OCI_REGISTRY_PASSWORD or " + "DOCKERHUB_USERNAME/DOCKERHUB_TOKEN" + ), + ): + publish_environment_oci_layout( + environment="debian:12-base", + layout_root=tmp_path / "oci_layouts", + registry="registry-1.docker.io", + repository="thalesmaciel/pyro-environment-debian-12-base", + reference="1.0.0", + ) + + def test_sync_bundle_replaces_existing_bundle_dir(tmp_path: Path) -> None: source_dir, build_dir, bundle_dir = _make_source_tree(tmp_path) paths = _build_paths( diff --git a/tests/test_vm_environments.py b/tests/test_vm_environments.py index 8d6cede..a5cd6d9 100644 --- a/tests/test_vm_environments.py +++ b/tests/test_vm_environments.py @@ -246,15 +246,15 @@ def test_fetch_oci_manifest_resolves_linux_amd64_index_with_bearer_auth( runtime_paths = _fake_runtime_paths(tmp_path) store = EnvironmentStore(runtime_paths=runtime_paths, cache_dir=tmp_path / "cache") spec = VmEnvironment( - name="debian:12-ghcr", + name="debian:12-registry", version="1.0.0", description="OCI-backed environment", default_packages=("bash", "git"), distribution="debian", distribution_version="12", source_profile="missing-profile", - oci_registry="ghcr.io", - oci_repository="thaloco/pyro-environments/debian-12", + oci_registry="registry-1.docker.io", + oci_repository="thalesmaciel/pyro-environment-debian-12", oci_reference="1.0.0", ) @@ -288,7 +288,7 @@ def test_fetch_oci_manifest_resolves_linux_amd64_index_with_bearer_auth( def fake_urlopen(request: object, timeout: int = 90) -> FakeResponse: del timeout url = request.full_url if isinstance(request, urllib.request.Request) else str(request) - if url.startswith("https://ghcr.io/token?"): + if url.startswith("https://auth.docker.io/token?"): return FakeResponse(b'{"token":"secret-token"}') authorization = _authorization_header(request) if url.endswith("/manifests/1.0.0"): @@ -300,9 +300,9 @@ def test_fetch_oci_manifest_resolves_linux_amd64_index_with_bearer_auth( _http_headers( { "WWW-Authenticate": ( - 'Bearer realm="https://ghcr.io/token",' - 'service="ghcr.io",' - 'scope="repository:thaloco/pyro-environments/debian-12:pull"' + 'Bearer realm="https://auth.docker.io/token",' + 'service="registry.docker.io",' + 'scope="repository:thalesmaciel/pyro-environment-debian-12:pull"' ) } ), @@ -322,9 +322,9 @@ def test_fetch_oci_manifest_resolves_linux_amd64_index_with_bearer_auth( _http_headers( { "WWW-Authenticate": ( - 'Bearer realm="https://ghcr.io/token",' - 'service="ghcr.io",' - 'scope="repository:thaloco/pyro-environments/debian-12:pull"' + 'Bearer realm="https://auth.docker.io/token",' + 'service="registry.docker.io",' + 'scope="repository:thalesmaciel/pyro-environment-debian-12:pull"' ) } ), @@ -345,8 +345,11 @@ def test_fetch_oci_manifest_resolves_linux_amd64_index_with_bearer_auth( assert manifest == child_manifest assert resolved_digest == child_digest assert authorized_urls == [ - "https://ghcr.io/v2/thaloco/pyro-environments/debian-12/manifests/1.0.0", - f"https://ghcr.io/v2/thaloco/pyro-environments/debian-12/manifests/{child_digest}", + "https://registry-1.docker.io/v2/thalesmaciel/pyro-environment-debian-12/manifests/1.0.0", + ( + "https://registry-1.docker.io/v2/thalesmaciel/pyro-environment-debian-12/" + f"manifests/{child_digest}" + ), ] @@ -377,15 +380,15 @@ def test_environment_store_installs_from_oci_when_runtime_source_missing( manifest_payload = json.dumps(manifest).encode("utf-8") manifest_digest = _sha256_digest(manifest_payload) environment = VmEnvironment( - name="debian:12-ghcr", + name="debian:12-registry", version="1.0.0", description="OCI-backed environment", default_packages=("bash", "git"), distribution="debian", distribution_version="12", source_profile="missing-profile", - oci_registry="ghcr.io", - oci_repository="thaloco/pyro-environments/debian-12", + oci_registry="registry-1.docker.io", + oci_repository="thalesmaciel/pyro-environment-debian-12", oci_reference="1.0.0", ) @@ -415,7 +418,7 @@ def test_environment_store_installs_from_oci_when_runtime_source_missing( assert installed.kernel_image.read_text(encoding="utf-8") == "kernel\n" assert installed.rootfs_image.read_text(encoding="utf-8") == "rootfs\n" assert installed.source == ( - "oci://ghcr.io/thaloco/pyro-environments/debian-12" + "oci://registry-1.docker.io/thalesmaciel/pyro-environment-debian-12" f"@{manifest_digest}" ) metadata = json.loads((installed.install_dir / "environment.json").read_text(encoding="utf-8"))