Switch to fetched runtime bundles
Stop treating Firecracker, kernels, modules, and guest images as tracked source files. Source checkouts now resolve runtime assets from ./runtime, while installed binaries keep using ../lib/banger. Add a small runtimebundle helper plus runtime-bundle.toml so make can bootstrap, package, and install a runtime bundle with checksum validation. Update the shell helpers and daemon path hints to fail clearly when the bundle is missing instead of assuming repo-root artifacts. This removes the tracked runtime blobs from HEAD in favor of an ignored local runtime/ tree. Verified with go test ./..., make build, bash -n on the shell helpers, make -n install, and a temporary package/fetch smoke test. The manifest URL/SHA still need a published bundle before fresh clones can bootstrap, and history rewrite remains a separate rollout step.
This commit is contained in:
parent
ce1be52047
commit
238bb8a020
6512 changed files with 1019 additions and 65372 deletions
326
internal/runtimebundle/bundle.go
Normal file
326
internal/runtimebundle/bundle.go
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
package runtimebundle
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
toml "github.com/pelletier/go-toml"
|
||||
)
|
||||
|
||||
type Manifest struct {
|
||||
Version string `toml:"version"`
|
||||
URL string `toml:"url"`
|
||||
SHA256 string `toml:"sha256"`
|
||||
BundleRoot string `toml:"bundle_root"`
|
||||
RequiredPaths []string `toml:"required_paths"`
|
||||
}
|
||||
|
||||
func LoadManifest(path string) (Manifest, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
var manifest Manifest
|
||||
if err := toml.Unmarshal(data, &manifest); err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
manifest.BundleRoot = strings.TrimSpace(manifest.BundleRoot)
|
||||
manifest.URL = strings.TrimSpace(manifest.URL)
|
||||
manifest.SHA256 = strings.ToLower(strings.TrimSpace(manifest.SHA256))
|
||||
for i, required := range manifest.RequiredPaths {
|
||||
manifest.RequiredPaths[i] = filepath.Clean(strings.TrimSpace(required))
|
||||
}
|
||||
sort.Strings(manifest.RequiredPaths)
|
||||
if len(manifest.RequiredPaths) == 0 {
|
||||
return Manifest{}, fmt.Errorf("runtime bundle manifest %s has no required_paths", path)
|
||||
}
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func Bootstrap(ctx context.Context, manifest Manifest, manifestPath, outDir string) error {
|
||||
if manifest.URL == "" {
|
||||
return fmt.Errorf("runtime bundle manifest %s has no url; publish a runtime bundle and update the manifest", manifestPath)
|
||||
}
|
||||
if manifest.SHA256 == "" {
|
||||
return fmt.Errorf("runtime bundle manifest %s has no sha256", manifestPath)
|
||||
}
|
||||
manifestDir := filepath.Dir(manifestPath)
|
||||
parentDir := filepath.Dir(outDir)
|
||||
if err := os.MkdirAll(parentDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
workDir, err := os.MkdirTemp(parentDir, ".runtime-bundle-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(workDir)
|
||||
|
||||
archivePath := filepath.Join(workDir, "bundle.tar.gz")
|
||||
if err := downloadArchive(ctx, resolveSource(manifestDir, manifest.URL), archivePath); err != nil {
|
||||
return err
|
||||
}
|
||||
sum, err := fileSHA256(archivePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sum != manifest.SHA256 {
|
||||
return fmt.Errorf("runtime bundle checksum mismatch: got %s want %s", sum, manifest.SHA256)
|
||||
}
|
||||
|
||||
extractDir := filepath.Join(workDir, "extract")
|
||||
if err := extractTarGz(archivePath, extractDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bundleDir := extractDir
|
||||
if manifest.BundleRoot != "" {
|
||||
bundleDir = filepath.Join(extractDir, manifest.BundleRoot)
|
||||
}
|
||||
if err := ValidateBundle(bundleDir, manifest.RequiredPaths); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stageDir := filepath.Join(workDir, "stage")
|
||||
if err := os.Rename(bundleDir, stageDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(outDir); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(stageDir, outDir); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateBundle(bundleDir string, requiredPaths []string) error {
|
||||
for _, rel := range requiredPaths {
|
||||
if rel == "." || strings.HasPrefix(rel, "..") {
|
||||
return fmt.Errorf("invalid required bundle path: %s", rel)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(bundleDir, rel)); err != nil {
|
||||
return fmt.Errorf("runtime bundle missing %s", rel)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Package(runtimeDir, outArchive string, manifest Manifest) (string, error) {
|
||||
if err := ValidateBundle(runtimeDir, manifest.RequiredPaths); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(outArchive), 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
file, err := os.Create(outArchive)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hash := sha256.New()
|
||||
multi := io.MultiWriter(file, hash)
|
||||
gz := gzip.NewWriter(multi)
|
||||
defer gz.Close()
|
||||
tw := tar.NewWriter(gz)
|
||||
defer tw.Close()
|
||||
|
||||
for _, rel := range manifest.RequiredPaths {
|
||||
if err := addPathToArchive(tw, runtimeDir, manifest.BundleRoot, rel); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func addPathToArchive(tw *tar.Writer, runtimeDir, bundleRoot, rel string) error {
|
||||
srcPath := filepath.Join(runtimeDir, rel)
|
||||
info, err := os.Lstat(srcPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
archiveName := rel
|
||||
if bundleRoot != "" {
|
||||
archiveName = filepath.Join(bundleRoot, rel)
|
||||
}
|
||||
if info.IsDir() {
|
||||
header, err := tar.FileInfoHeader(info, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header.Name = filepath.ToSlash(archiveName) + "/"
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
entries, err := os.ReadDir(srcPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
childRel := filepath.Join(rel, entry.Name())
|
||||
if err := addPathToArchive(tw, runtimeDir, bundleRoot, childRel); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
header, err := tar.FileInfoHeader(info, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header.Name = filepath.ToSlash(archiveName)
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.Open(srcPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
_, err = io.Copy(tw, file)
|
||||
return err
|
||||
}
|
||||
|
||||
func resolveSource(manifestDir, source string) string {
|
||||
parsed, err := url.Parse(source)
|
||||
if err == nil && parsed.Scheme != "" {
|
||||
return source
|
||||
}
|
||||
if filepath.IsAbs(source) {
|
||||
return source
|
||||
}
|
||||
return filepath.Join(manifestDir, source)
|
||||
}
|
||||
|
||||
func downloadArchive(ctx context.Context, source, dst string) error {
|
||||
switch {
|
||||
case strings.HasPrefix(source, "http://"), strings.HasPrefix(source, "https://"):
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("download runtime bundle: %s", resp.Status)
|
||||
}
|
||||
return writeFileFromReader(dst, resp.Body)
|
||||
case strings.HasPrefix(source, "file://"):
|
||||
parsed, err := url.Parse(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return copyFile(parsed.Path, dst)
|
||||
default:
|
||||
return copyFile(source, dst)
|
||||
}
|
||||
}
|
||||
|
||||
func writeFileFromReader(dst string, reader io.Reader) error {
|
||||
file, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
_, err = io.Copy(file, reader)
|
||||
return err
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
return writeFileFromReader(dst, in)
|
||||
}
|
||||
|
||||
func extractTarGz(archivePath, outDir string) error {
|
||||
if err := os.MkdirAll(outDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
gz, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gz.Close()
|
||||
tr := tar.NewReader(gz)
|
||||
for {
|
||||
header, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := filepath.Clean(header.Name)
|
||||
if name == "." || strings.HasPrefix(name, "..") || filepath.IsAbs(name) {
|
||||
return fmt.Errorf("invalid archive entry: %s", header.Name)
|
||||
}
|
||||
target := filepath.Join(outDir, name)
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(target, os.FileMode(header.Mode)); err != nil {
|
||||
return err
|
||||
}
|
||||
case tar.TypeReg, tar.TypeRegA:
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.FileMode(header.Mode))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(file, tr); err != nil {
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unsupported archive entry type: %s", header.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func fileSHA256(path string) (string, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer file.Close()
|
||||
hash := sha256.New()
|
||||
if _, err := io.Copy(hash, file); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
150
internal/runtimebundle/bundle_test.go
Normal file
150
internal/runtimebundle/bundle_test.go
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
package runtimebundle
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBootstrapExtractsBundleAndValidatesChecksum(t *testing.T) {
|
||||
manifestDir := t.TempDir()
|
||||
bundleData := buildArchive(t, map[string]string{
|
||||
"runtime/firecracker": "fc",
|
||||
"runtime/customize.sh": "#!/bin/bash\n",
|
||||
"runtime/packages.apt": "vim\n",
|
||||
"runtime/rootfs-docker.ext4": "rootfs",
|
||||
"runtime/wtf/root/boot/vmlinux-6.8.0-94-generic": "kernel",
|
||||
"runtime/wtf/root/boot/initrd.img-6.8.0-94-generic": "initrd",
|
||||
"runtime/wtf/root/lib/modules/6.8.0-94-generic/modules.dep": "dep",
|
||||
})
|
||||
archivePath := filepath.Join(manifestDir, "bundle.tar.gz")
|
||||
if err := os.WriteFile(archivePath, bundleData, 0o644); err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
|
||||
manifest := Manifest{
|
||||
URL: "./bundle.tar.gz",
|
||||
SHA256: sha256Hex(bundleData),
|
||||
BundleRoot: "runtime",
|
||||
RequiredPaths: []string{"firecracker", "customize.sh", "packages.apt", "rootfs-docker.ext4", "wtf/root/boot/vmlinux-6.8.0-94-generic", "wtf/root/lib/modules/6.8.0-94-generic"},
|
||||
}
|
||||
outDir := filepath.Join(t.TempDir(), "runtime")
|
||||
if err := Bootstrap(context.Background(), manifest, filepath.Join(manifestDir, "runtime-bundle.toml"), outDir); err != nil {
|
||||
t.Fatalf("Bootstrap: %v", err)
|
||||
}
|
||||
for _, rel := range manifest.RequiredPaths {
|
||||
if _, err := os.Stat(filepath.Join(outDir, rel)); err != nil {
|
||||
t.Fatalf("runtime missing %s: %v", rel, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapRejectsChecksumMismatch(t *testing.T) {
|
||||
manifestDir := t.TempDir()
|
||||
archivePath := filepath.Join(manifestDir, "bundle.tar.gz")
|
||||
if err := os.WriteFile(archivePath, []byte("not-a-tarball"), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
manifest := Manifest{
|
||||
URL: "./bundle.tar.gz",
|
||||
SHA256: strings.Repeat("0", 64),
|
||||
BundleRoot: "runtime",
|
||||
RequiredPaths: []string{"firecracker"},
|
||||
}
|
||||
err := Bootstrap(context.Background(), manifest, filepath.Join(manifestDir, "runtime-bundle.toml"), filepath.Join(t.TempDir(), "runtime"))
|
||||
if err == nil || !strings.Contains(err.Error(), "checksum mismatch") {
|
||||
t.Fatalf("Bootstrap() error = %v, want checksum mismatch", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackageWritesArchive(t *testing.T) {
|
||||
runtimeDir := t.TempDir()
|
||||
for _, rel := range []string{
|
||||
"firecracker",
|
||||
"customize.sh",
|
||||
"packages.apt",
|
||||
"rootfs-docker.ext4",
|
||||
"wtf/root/boot/vmlinux-6.8.0-94-generic",
|
||||
"wtf/root/boot/initrd.img-6.8.0-94-generic",
|
||||
"wtf/root/lib/modules/6.8.0-94-generic",
|
||||
} {
|
||||
path := filepath.Join(runtimeDir, rel)
|
||||
if rel == "wtf/root/lib/modules/6.8.0-94-generic" {
|
||||
if err := os.MkdirAll(path, 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(path, "modules.dep"), []byte(rel), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("MkdirAll: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(rel), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
}
|
||||
manifest := Manifest{
|
||||
BundleRoot: "runtime",
|
||||
RequiredPaths: []string{
|
||||
"firecracker",
|
||||
"customize.sh",
|
||||
"packages.apt",
|
||||
"rootfs-docker.ext4",
|
||||
"wtf/root/boot/vmlinux-6.8.0-94-generic",
|
||||
"wtf/root/boot/initrd.img-6.8.0-94-generic",
|
||||
"wtf/root/lib/modules/6.8.0-94-generic",
|
||||
},
|
||||
}
|
||||
outArchive := filepath.Join(t.TempDir(), "bundle.tar.gz")
|
||||
sum, err := Package(runtimeDir, outArchive, manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("Package: %v", err)
|
||||
}
|
||||
if sum == "" {
|
||||
t.Fatalf("Package() returned empty checksum")
|
||||
}
|
||||
if _, err := os.Stat(outArchive); err != nil {
|
||||
t.Fatalf("archive missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func buildArchive(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gz)
|
||||
for name, contents := range files {
|
||||
header := &tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(contents)),
|
||||
}
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
t.Fatalf("WriteHeader(%s): %v", name, err)
|
||||
}
|
||||
if _, err := tw.Write([]byte(contents)); err != nil {
|
||||
t.Fatalf("Write(%s): %v", name, err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatalf("Close tar: %v", err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatalf("Close gzip: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func sha256Hex(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue