From 78376ba6ec8b0f666b70fbc93dd28a3c626c5f4a Mon Sep 17 00:00:00 2001 From: Thales Maciel Date: Thu, 16 Apr 2026 17:22:13 -0300 Subject: [PATCH] =?UTF-8?q?Phase=201:=20imagepull=20package=20=E2=80=94=20?= =?UTF-8?q?pull,=20flatten,=20ext4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New internal/imagepull/ subpackage. Three concerns, each independently testable: Pull (imagepull.go): - github.com/google/go-containerregistry's remote.Image with the linux/amd64 platform pinned. Anonymous pulls only for v1. - Layer blobs cached on disk via cache.NewFilesystemCache under /blobs/sha256/ — OCI-standard layout so skopeo/crane could co-exist later. - Eagerly touches every layer once so network errors surface at Pull time, not deep in Flatten. Flatten (flatten.go): - Replays layers oldest-first into destDir. - Whiteout-aware: .wh. deletes the named entry, .wh..wh..opq wipes the parent directory's contents from prior layers. - Path-traversal hardening mirrored from kernelcat extractTar: reject .., absolute paths, and symlinks/hardlinks whose resolved target escapes destDir. - Handles tar.TypeReg, TypeDir, TypeSymlink, TypeLink. Skips device/fifo nodes silently (need privilege; udev/devtmpfs handles them in the guest). BuildExt4 (ext4.go): - Truncates outFile to sizeBytes, then runs `mkfs.ext4 -F -d -E root_owner=0:0`. No mount, no sudo, no loopback. - 64 MiB floor; callers handle real sizing with content-aware headroom. - File ownership in the resulting ext4 reflects srcDir's on-disk ownership — runner's uid/gid since extraction was unprivileged. Documented in package doc as a Phase A v1 limitation; Phase B will add a debugfs- or tar2ext4-based ownership fixup. paths.Layout gains OCICacheDir at $XDG_CACHE_HOME/banger/oci/, ensured at startup alongside the other dirs. Tests use go-containerregistry's in-process registry to push and pull synthetic multi-layer images. Cover: layer caching round-trip, whiteout + opaque-marker handling, path-traversal rejection, unsafe symlink rejection, real mkfs.ext4 round-trip (skipped if mkfs.ext4 absent), and tiny-size rejection. go-containerregistry v0.21.5 added as a direct dep, plus its transitive closure (containerd/stargz, opencontainers/go-digest, docker/cli config helpers, etc). Co-Authored-By: Claude Sonnet 4.6 --- go.mod | 28 ++- go.sum | 63 ++++-- internal/imagepull/ext4.go | 70 +++++++ internal/imagepull/flatten.go | 201 ++++++++++++++++++ internal/imagepull/imagepull.go | 102 +++++++++ internal/imagepull/imagepull_test.go | 298 +++++++++++++++++++++++++++ internal/paths/paths.go | 4 +- 7 files changed, 733 insertions(+), 33 deletions(-) create mode 100644 internal/imagepull/ext4.go create mode 100644 internal/imagepull/flatten.go create mode 100644 internal/imagepull/imagepull.go create mode 100644 internal/imagepull/imagepull_test.go diff --git a/go.mod b/go.mod index 2ddb7c4..6067e9e 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,14 @@ go 1.25.0 require ( github.com/firecracker-microvm/firecracker-go-sdk v1.0.0 + github.com/google/go-containerregistry v0.21.5 + github.com/klauspost/compress v1.18.5 github.com/miekg/dns v1.1.72 github.com/pelletier/go-toml v1.9.5 github.com/sirupsen/logrus v1.9.4 - github.com/spf13/cobra v1.8.1 - golang.org/x/crypto v0.46.0 - golang.org/x/sys v0.39.0 + github.com/spf13/cobra v1.10.2 + golang.org/x/crypto v0.50.0 + golang.org/x/sys v0.43.0 modernc.org/sqlite v1.38.2 ) @@ -18,8 +20,11 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/containerd/fifo v1.0.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect github.com/containernetworking/cni v1.0.1 // indirect github.com/containernetworking/plugins v1.0.1 // indirect + github.com/docker/cli v29.4.0+incompatible // indirect + github.com/docker/docker-credential-helpers v0.9.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-openapi/analysis v0.21.2 // indirect github.com/go-openapi/errors v0.20.2 // indirect @@ -37,27 +42,30 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.18.5 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mdlayher/socket v0.2.0 // indirect github.com/mdlayher/vsock v1.1.1 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/oklog/ulid v1.3.1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/vbatts/tar-split v0.12.2 // indirect github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 // indirect github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect go.mongodb.org/mongo-driver v1.8.3 // indirect golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect - golang.org/x/mod v0.31.0 // indirect - golang.org/x/net v0.48.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/text v0.32.0 // indirect - golang.org/x/tools v0.40.0 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/text v0.36.0 // indirect + golang.org/x/tools v0.44.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect modernc.org/libc v1.66.3 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/go.sum b/go.sum index 9547056..ca330a4 100644 --- a/go.sum +++ b/go.sum @@ -162,6 +162,8 @@ github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJ github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/stargz-snapshotter/estargz v0.18.2 h1:yXkZFYIzz3eoLwlTUZKz2iQ4MrckBxJjkmD16ynUTrw= +github.com/containerd/stargz-snapshotter/estargz v0.18.2/go.mod h1:XyVU5tcJ3PRpkA9XS2T5us6Eg35yM0214Y+wvrZTBrY= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= @@ -206,7 +208,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= @@ -222,9 +224,13 @@ github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/cli v29.4.0+incompatible h1:+IjXULMetlvWJiuSI0Nbor36lcJ5BTcVpUmB21KBoVM= +github.com/docker/cli v29.4.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= @@ -384,8 +390,10 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.21.5 h1:KTJG9Pn/jC0VdZR6ctV3/jcN+q6/Iqlx0sTVz3ywZlM= +github.com/google/go-containerregistry v0.21.5/go.mod h1:ySvMuiWg+dOsRW0Hw8GYwfMwBlNRTmpYBFJPlkco5zU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -503,6 +511,7 @@ github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -558,9 +567,12 @@ github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -654,15 +666,17 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -694,6 +708,8 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vbatts/tar-split v0.12.2 h1:w/Y6tjxpeiFMR47yzZPlPj/FcPLpXbTUi/9H7d3CPa4= +github.com/vbatts/tar-split v0.12.2/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= @@ -737,6 +753,7 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -754,8 +771,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -788,8 +805,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= -golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -830,8 +847,8 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -848,8 +865,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -923,13 +940,13 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -939,8 +956,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -988,8 +1005,8 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1094,8 +1111,10 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/imagepull/ext4.go b/internal/imagepull/ext4.go new file mode 100644 index 0000000..9c31ed5 --- /dev/null +++ b/internal/imagepull/ext4.go @@ -0,0 +1,70 @@ +package imagepull + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" + + "banger/internal/system" +) + +// MinExt4Size is the smallest ext4 image we'll create. mkfs.ext4 needs a +// few megabytes for its bookkeeping; for a real rootfs the staging tree +// will dominate anyway. +const MinExt4Size int64 = 1 << 20 * 64 // 64 MiB + +// BuildExt4 creates outFile as a sparse ext4 image of sizeBytes and +// populates it from srcDir using `mkfs.ext4 -F -d`. No mount, no sudo. +// +// sizeBytes must be at least MinExt4Size. Callers are expected to size +// the file with headroom over the staged tree (the daemon orchestrator +// does this; this function only enforces a sanity floor). +// +// The resulting image's file ownership reflects srcDir's on-disk +// ownership — see the package doc for the implications. +func BuildExt4(ctx context.Context, runner system.CommandRunner, srcDir, outFile string, sizeBytes int64) error { + if sizeBytes < MinExt4Size { + return fmt.Errorf("ext4 size %d below minimum %d", sizeBytes, MinExt4Size) + } + info, err := os.Stat(srcDir) + if err != nil { + return fmt.Errorf("stat source: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("%s is not a directory", srcDir) + } + + if err := os.Remove(outFile); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + f, err := os.OpenFile(outFile, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o644) + if err != nil { + return err + } + if err := f.Truncate(sizeBytes); err != nil { + _ = f.Close() + _ = os.Remove(outFile) + return err + } + if err := f.Close(); err != nil { + _ = os.Remove(outFile) + return err + } + + out, runErr := runner.Run(ctx, "mkfs.ext4", + "-F", + "-q", + "-d", srcDir, + "-L", "banger-rootfs", + "-E", "root_owner=0:0", + outFile, + strconv.FormatInt(sizeBytes/4096, 10), // size in 4 KiB blocks + ) + if runErr != nil { + _ = os.Remove(outFile) + return fmt.Errorf("mkfs.ext4 -d: %w: %s", runErr, string(out)) + } + return nil +} diff --git a/internal/imagepull/flatten.go b/internal/imagepull/flatten.go new file mode 100644 index 0000000..7404ca9 --- /dev/null +++ b/internal/imagepull/flatten.go @@ -0,0 +1,201 @@ +package imagepull + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +const ( + whiteoutPrefix = ".wh." + // whiteoutOpaque marks the parent directory as opaque: every entry + // from previous layers should be removed, but entries from the + // current layer (siblings of this marker) are preserved. + whiteoutOpaque = ".wh..wh..opq" +) + +// Flatten replays the image's layers in oldest-first order into destDir. +// destDir must exist and ideally be empty. Path-traversal members and +// symlink targets that escape destDir are rejected. +// +// File ownership in destDir reflects the running user, not the tar +// header's uid/gid (Phase A v1 limitation; see package docs). +func Flatten(ctx context.Context, img PulledImage, destDir string) error { + absDest, err := filepath.Abs(destDir) + if err != nil { + return err + } + layers, err := img.Image.Layers() + if err != nil { + return fmt.Errorf("read layers: %w", err) + } + for i, layer := range layers { + if err := ctx.Err(); err != nil { + return err + } + if err := applyLayer(layer, absDest); err != nil { + return fmt.Errorf("apply layer %d/%d: %w", i+1, len(layers), err) + } + } + return nil +} + +func applyLayer(layer interface { + Uncompressed() (io.ReadCloser, error) +}, dest string) error { + rc, err := layer.Uncompressed() + if err != nil { + return err + } + defer rc.Close() + + tr := tar.NewReader(rc) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("read tar entry: %w", err) + } + if err := applyEntry(tr, hdr, dest); err != nil { + return err + } + } +} + +func applyEntry(tr *tar.Reader, hdr *tar.Header, dest string) error { + rel := filepath.Clean(hdr.Name) + if rel == "." || rel == string(filepath.Separator) { + return nil + } + if filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("unsafe path in layer: %q", hdr.Name) + } + + base := filepath.Base(rel) + parent := filepath.Dir(rel) + + // Whiteouts come in two flavors: opaque-dir markers and per-file + // deletes. Both are resolved relative to the parent directory. + if base == whiteoutOpaque { + parentAbs, err := safeJoin(dest, parent) + if err != nil { + return err + } + return clearDirContents(parentAbs) + } + if strings.HasPrefix(base, whiteoutPrefix) { + target := strings.TrimPrefix(base, whiteoutPrefix) + victim, err := safeJoin(dest, filepath.Join(parent, target)) + if err != nil { + return err + } + if err := os.RemoveAll(victim); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("apply whiteout %s: %w", hdr.Name, err) + } + return nil + } + + abs, err := safeJoin(dest, rel) + if err != nil { + return err + } + + switch hdr.Typeflag { + case tar.TypeDir: + return os.MkdirAll(abs, 0o755) + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + return err + } + // Replace any prior file/dir in this slot — later layers + // shadow earlier ones. + if err := os.RemoveAll(abs); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + f, err := os.OpenFile(abs, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)|0o600) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { + _ = f.Close() + return err + } + return f.Close() + case tar.TypeSymlink: + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + return err + } + // Resolve the link target relative to the link's parent and + // require that it stays inside dest. Absolute targets that + // resolve outside dest are also rejected. + resolved := hdr.Linkname + if !filepath.IsAbs(resolved) { + resolved = filepath.Join(filepath.Dir(abs), resolved) + } + resolved = filepath.Clean(resolved) + if resolved != dest && !strings.HasPrefix(resolved, dest+string(filepath.Separator)) { + return fmt.Errorf("unsafe symlink in layer: %q -> %q", hdr.Name, hdr.Linkname) + } + if err := os.RemoveAll(abs); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return os.Symlink(hdr.Linkname, abs) + case tar.TypeLink: + // Hardlink: target must already exist inside dest from this or + // a previous layer, and must not escape. + linkTarget, err := safeJoin(dest, filepath.Clean(hdr.Linkname)) + if err != nil { + return err + } + if _, err := os.Lstat(linkTarget); err != nil { + return fmt.Errorf("hardlink target %q missing: %w", hdr.Linkname, err) + } + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + return err + } + if err := os.RemoveAll(abs); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return os.Link(linkTarget, abs) + default: + // TypeChar / TypeBlock / TypeFifo / TypeXGlobalHeader / etc. + // Container layers occasionally include /dev nodes — they need + // privilege we don't have. Skip silently; udev/devtmpfs in the + // guest will create them at boot. + return nil + } +} + +// safeJoin returns dest+rel after verifying the result lies under dest. +func safeJoin(dest, rel string) (string, error) { + joined := filepath.Join(dest, rel) + if joined != dest && !strings.HasPrefix(joined, dest+string(filepath.Separator)) { + return "", fmt.Errorf("unsafe path: %q escapes %q", rel, dest) + } + return joined, nil +} + +// clearDirContents removes every entry under dir but leaves dir itself. +// Used for opaque-whiteout markers. +func clearDirContents(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return os.MkdirAll(dir, 0o755) + } + return err + } + for _, entry := range entries { + if err := os.RemoveAll(filepath.Join(dir, entry.Name())); err != nil { + return err + } + } + return nil +} diff --git a/internal/imagepull/imagepull.go b/internal/imagepull/imagepull.go new file mode 100644 index 0000000..c9da7c3 --- /dev/null +++ b/internal/imagepull/imagepull.go @@ -0,0 +1,102 @@ +// Package imagepull pulls OCI container images from registries and lays +// them down as banger-ready ext4 rootfs files. The package is a primitive: +// it produces an ext4 file plus per-file ownership metadata. Higher layers +// (the daemon's PullImage orchestrator) decide where the file lands and +// how it gets registered. +// +// Three concerns: +// - Pull resolves an OCI reference, selects the linux/amd64 platform, +// and returns a v1.Image whose layer blobs are cached on disk so +// re-pulls are cheap. +// - Flatten replays the layers in order into a staging directory, +// applies whiteouts, and rejects unsafe paths/symlinks. +// - BuildExt4 turns that staging directory into an ext4 file via +// `mkfs.ext4 -d` (no mount, no sudo). +// +// Limitations (Phase A v1): +// - Anonymous registry pulls only. Auth is deferred. +// - Hardcoded linux/amd64. Other platforms reject at Pull time. +// - File ownership in the resulting ext4 is the runner's uid/gid; +// setuid binaries and root-owned config files lose their original +// ownership. Phase B will add a debugfs- or tar2ext4-based fixup +// pass; until then the produced image is suitable as input to +// `image build` but not directly bootable. +package imagepull + +import ( + "context" + "fmt" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/cache" + "github.com/google/go-containerregistry/pkg/v1/remote" + + "github.com/google/go-containerregistry/pkg/name" +) + +// Platform is the only platform Phase A produces. Adding arm64 later is a +// matter of letting callers override this. +var Platform = v1.Platform{OS: "linux", Architecture: "amd64"} + +// PulledImage is what Pull returns: the resolved OCI image plus enough +// reference metadata to identify it later (digest for cache keys, +// canonical name for logs). +type PulledImage struct { + Reference string // user-supplied reference, parsed and re-stringified + Digest string // image manifest digest (sha256:...) + Platform string // "linux/amd64" + Image v1.Image // go-containerregistry handle; layers, manifest, etc. +} + +// Pull resolves ref against the public registry, selects the linux/amd64 +// platform from any manifest list, and ensures the layer blobs are cached +// on disk under cacheDir/blobs/sha256/. Subsequent Pulls of the same +// digest are local-only. +func Pull(ctx context.Context, ref, cacheDir string) (PulledImage, error) { + parsed, err := name.ParseReference(ref) + if err != nil { + return PulledImage{}, fmt.Errorf("parse oci ref %q: %w", ref, err) + } + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return PulledImage{}, err + } + + img, err := remote.Image(parsed, + remote.WithContext(ctx), + remote.WithPlatform(Platform), + ) + if err != nil { + return PulledImage{}, fmt.Errorf("fetch %q: %w", ref, err) + } + + cached := cache.Image(img, cache.NewFilesystemCache(filepath.Join(cacheDir, "blobs"))) + + digest, err := cached.Digest() + if err != nil { + return PulledImage{}, fmt.Errorf("resolve digest for %q: %w", ref, err) + } + + // Touch the layers once so they are guaranteed present in the cache + // before Flatten runs; surfaces network errors here, not deep inside + // Flatten's hot loop. + layers, err := cached.Layers() + if err != nil { + return PulledImage{}, fmt.Errorf("read layers for %q: %w", ref, err) + } + for i, layer := range layers { + rc, err := layer.Compressed() + if err != nil { + return PulledImage{}, fmt.Errorf("fetch layer %d for %q: %w", i, ref, err) + } + _ = rc.Close() + } + + return PulledImage{ + Reference: parsed.String(), + Digest: digest.String(), + Platform: Platform.OS + "/" + Platform.Architecture, + Image: cached, + }, nil +} diff --git a/internal/imagepull/imagepull_test.go b/internal/imagepull/imagepull_test.go new file mode 100644 index 0000000..2532fdc --- /dev/null +++ b/internal/imagepull/imagepull_test.go @@ -0,0 +1,298 @@ +package imagepull + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "io" + "log" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "banger/internal/system" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// tarMember is a single entry to put into a fake layer tarball. +type tarMember struct { + name string + mode int64 + body []byte + link string // for symlinks / hardlinks + dir bool + symlink bool + hardlink bool +} + +func buildTar(t *testing.T, members []tarMember) []byte { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, m := range members { + hdr := &tar.Header{Name: m.name, Mode: m.mode} + switch { + case m.dir: + hdr.Typeflag = tar.TypeDir + if hdr.Mode == 0 { + hdr.Mode = 0o755 + } + case m.symlink: + hdr.Typeflag = tar.TypeSymlink + hdr.Linkname = m.link + case m.hardlink: + hdr.Typeflag = tar.TypeLink + hdr.Linkname = m.link + default: + hdr.Typeflag = tar.TypeReg + hdr.Size = int64(len(m.body)) + if hdr.Mode == 0 { + hdr.Mode = 0o644 + } + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("tar header: %v", err) + } + if hdr.Typeflag == tar.TypeReg && len(m.body) > 0 { + if _, err := tw.Write(m.body); err != nil { + t.Fatalf("tar write: %v", err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatalf("tar close: %v", err) + } + return buf.Bytes() +} + +func startRegistry(t *testing.T) string { + t.Helper() + srv := httptest.NewServer(registry.New(registry.Logger(log.New(io.Discard, "", 0)))) + t.Cleanup(srv.Close) + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + return u.Host +} + +func makeLayer(t *testing.T, members []tarMember) v1.Layer { + t.Helper() + body := buildTar(t, members) + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(body)), nil + }) + if err != nil { + t.Fatalf("LayerFromOpener: %v", err) + } + return layer +} + +// pushImage assembles a multi-layer image with linux/amd64 platform and +// pushes it under repo:tag. Returns the canonical reference. +func pushImage(t *testing.T, host, repo, tag string, layers ...v1.Layer) string { + t.Helper() + img, err := mutate.AppendLayers(empty.Image, layers...) + if err != nil { + t.Fatalf("AppendLayers: %v", err) + } + cfg, err := img.ConfigFile() + if err != nil { + t.Fatalf("ConfigFile: %v", err) + } + cfg.Architecture = "amd64" + cfg.OS = "linux" + img, err = mutate.ConfigFile(img, cfg) + if err != nil { + t.Fatalf("ConfigFile mutate: %v", err) + } + ref, err := name.NewTag(host + "/" + repo + ":" + tag) + if err != nil { + t.Fatalf("NewTag: %v", err) + } + if err := remote.Write(ref, img); err != nil { + t.Fatalf("remote.Write: %v", err) + } + return ref.String() +} + +func TestPullCachesLayersAndReturnsImage(t *testing.T) { + host := startRegistry(t) + ref := pushImage(t, host, "banger/test", "v1", + makeLayer(t, []tarMember{ + {name: "etc/", dir: true}, + {name: "etc/hello", body: []byte("world")}, + }), + ) + + cacheDir := t.TempDir() + pulled, err := Pull(context.Background(), ref, cacheDir) + if err != nil { + t.Fatalf("Pull: %v", err) + } + if pulled.Digest == "" { + t.Fatalf("Digest empty") + } + if pulled.Platform != "linux/amd64" { + t.Fatalf("Platform = %q", pulled.Platform) + } + // Cache should now hold at least one blob. + blobsRoot := filepath.Join(cacheDir, "blobs") + count := 0 + _ = filepath.WalkDir(blobsRoot, func(_ string, d os.DirEntry, _ error) error { + if d != nil && !d.IsDir() { + count++ + } + return nil + }) + if count == 0 { + t.Fatalf("no blobs cached under %s", blobsRoot) + } +} + +func TestFlattenAppliesLayersAndWhiteouts(t *testing.T) { + host := startRegistry(t) + ref := pushImage(t, host, "banger/test", "wh", + makeLayer(t, []tarMember{ + {name: "etc/", dir: true}, + {name: "etc/keep", body: []byte("keep")}, + {name: "etc/old", body: []byte("old")}, + }), + makeLayer(t, []tarMember{ + {name: "etc/.wh.old"}, // delete etc/old + {name: "etc/new", body: []byte("new")}, // add etc/new + {name: "var/", dir: true}, + {name: "var/log/", dir: true}, + {name: "var/log/file", body: []byte("log")}, + }), + makeLayer(t, []tarMember{ + {name: "var/log/.wh..wh..opq"}, // wipe var/log contents from prior layers + {name: "var/log/fresh", body: []byte("fresh")}, + }), + ) + + pulled, err := Pull(context.Background(), ref, t.TempDir()) + if err != nil { + t.Fatalf("Pull: %v", err) + } + dest := t.TempDir() + if err := Flatten(context.Background(), pulled, dest); err != nil { + t.Fatalf("Flatten: %v", err) + } + + checkFile := func(rel, want string) { + t.Helper() + data, err := os.ReadFile(filepath.Join(dest, rel)) + if err != nil { + t.Errorf("read %s: %v", rel, err) + return + } + if string(data) != want { + t.Errorf("%s = %q, want %q", rel, string(data), want) + } + } + checkFile("etc/keep", "keep") + checkFile("etc/new", "new") + checkFile("var/log/fresh", "fresh") + + if _, err := os.Stat(filepath.Join(dest, "etc/old")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("etc/old should have been whited out: stat err=%v", err) + } + if _, err := os.Stat(filepath.Join(dest, "var/log/file")); !errors.Is(err, os.ErrNotExist) { + t.Errorf("var/log/file should have been wiped by opaque marker: stat err=%v", err) + } +} + +func TestFlattenRejectsPathTraversal(t *testing.T) { + host := startRegistry(t) + ref := pushImage(t, host, "banger/test", "evil", + makeLayer(t, []tarMember{ + {name: "../escape", body: []byte("bad")}, + }), + ) + pulled, err := Pull(context.Background(), ref, t.TempDir()) + if err != nil { + t.Fatalf("Pull: %v", err) + } + dest := t.TempDir() + err = Flatten(context.Background(), pulled, dest) + if err == nil || !strings.Contains(err.Error(), "unsafe path") { + t.Fatalf("Flatten escape: err=%v, want unsafe path", err) + } + escape := filepath.Join(filepath.Dir(dest), "escape") + if _, statErr := os.Stat(escape); !errors.Is(statErr, os.ErrNotExist) { + t.Errorf("escape file should not exist: %v", statErr) + } +} + +func TestFlattenRejectsUnsafeSymlink(t *testing.T) { + host := startRegistry(t) + ref := pushImage(t, host, "banger/test", "evil-sym", + makeLayer(t, []tarMember{ + {name: "evil", symlink: true, link: "/etc/passwd"}, // absolute target outside dest + }), + ) + pulled, err := Pull(context.Background(), ref, t.TempDir()) + if err != nil { + t.Fatalf("Pull: %v", err) + } + err = Flatten(context.Background(), pulled, t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "unsafe symlink") { + t.Fatalf("Flatten unsafe symlink: err=%v", err) + } +} + +func TestBuildExt4ProducesValidImage(t *testing.T) { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + t.Skip("mkfs.ext4 not available; skipping") + } + src := t.TempDir() + if err := os.MkdirAll(filepath.Join(src, "etc"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "etc", "hello"), []byte("hi"), 0o644); err != nil { + t.Fatal(err) + } + out := filepath.Join(t.TempDir(), "rootfs.ext4") + if err := BuildExt4(context.Background(), system.NewRunner(), src, out, MinExt4Size); err != nil { + t.Fatalf("BuildExt4: %v", err) + } + info, err := os.Stat(out) + if err != nil { + t.Fatalf("stat output: %v", err) + } + if info.Size() != MinExt4Size { + t.Errorf("ext4 size = %d, want %d", info.Size(), MinExt4Size) + } + // Quick sanity via file(1) — the ext4 superblock should be detectable. + if _, err := exec.LookPath("file"); err == nil { + out, _ := exec.Command("file", "-b", out).Output() + if !bytes.Contains(out, []byte("ext")) { + t.Errorf("file(1) does not see an ext filesystem: %s", out) + } + } +} + +func TestBuildExt4RejectsTinySize(t *testing.T) { + src := t.TempDir() + out := filepath.Join(t.TempDir(), "rootfs.ext4") + err := BuildExt4(context.Background(), system.NewRunner(), src, out, 1024) + if err == nil || !strings.Contains(err.Error(), "below minimum") { + t.Fatalf("BuildExt4 tiny: err=%v", err) + } + if _, statErr := os.Stat(out); !errors.Is(statErr, os.ErrNotExist) { + t.Errorf("output file should not exist on rejection: %v", statErr) + } +} diff --git a/internal/paths/paths.go b/internal/paths/paths.go index 4ae0b46..ce9ef96 100644 --- a/internal/paths/paths.go +++ b/internal/paths/paths.go @@ -23,6 +23,7 @@ type Layout struct { VMsDir string ImagesDir string KernelsDir string + OCICacheDir string } func Resolve() (Layout, error) { @@ -54,11 +55,12 @@ func Resolve() (Layout, error) { layout.VMsDir = filepath.Join(layout.StateDir, "vms") layout.ImagesDir = filepath.Join(layout.StateDir, "images") layout.KernelsDir = filepath.Join(layout.StateDir, "kernels") + layout.OCICacheDir = filepath.Join(layout.CacheDir, "oci") return layout, nil } func Ensure(layout Layout) error { - for _, dir := range []string{layout.ConfigDir, layout.StateDir, layout.CacheDir, layout.RuntimeDir, layout.VMsDir, layout.ImagesDir, layout.KernelsDir} { + for _, dir := range []string{layout.ConfigDir, layout.StateDir, layout.CacheDir, layout.RuntimeDir, layout.VMsDir, layout.ImagesDir, layout.KernelsDir, layout.OCICacheDir} { if err := os.MkdirAll(dir, 0o755); err != nil { return err }