daemon: tighten concurrency around pulls, cleanup, and handle persistence

Four targeted fixes from a race-condition audit of the daemon package.
None change behaviour on the happy path; each closes a window where a
concurrent or interrupted RPC could strand state on the host.

  - KernelDelete now holds the same per-name lock as KernelPull /
    readOrAutoPullKernel. Without it, a delete racing a concurrent
    pull could remove files mid-write or land between the pull's
    manifest write and its first use.

  - cleanupRuntime no longer early-returns on an inner waitForExit
    failure; DM snapshot, capability, and tap teardown always run and
    every error is folded into the returned errors.Join. EBUSY against
    a still-alive firecracker is benign and surfaces in the joined
    error rather than stranding kernel state across daemon restarts.

  - Per-name image / kernel pull locks switch from *sync.Mutex to a
    1-buffered chan struct{}. Acquire is a select on ctx.Done(), so a
    peer waiting behind a pull whose RPC was cancelled can bail out
    instead of blocking forever on a pull nobody is consuming.

  - setVMHandles writes the per-VM scratch file before updating the
    in-memory cache. A daemon crash between the two now leaves disk
    ahead of memory (recoverable: reconcile re-seeds the cache from
    the file on next start) rather than memory ahead of disk (lost
    handles → stranded DM/loops/tap).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Thales Maciel 2026-04-27 19:32:43 -03:00
parent 777b597a1e
commit c4e1cb5953
No known key found for this signature in database
GPG key ID: 33112E6833C34679
6 changed files with 99 additions and 39 deletions

View file

@ -39,15 +39,20 @@ type ImageService struct {
imageOpsMu sync.Mutex
// kernelPullLocksMu guards the kernelPullLocks map itself. Per-name
// mutexes inside the map serialise concurrent pulls of the same
// kernel ref. Without this, two parallel `vm run` callers that
// auto-pull the same kernel race on
// channel locks inside the map serialise concurrent pulls of the
// same kernel ref. Without this, two parallel `vm run` callers
// that auto-pull the same kernel race on
// /var/lib/banger/kernels/<name>/manifest.json: one is mid-write
// from kernelcat.Fetch's WriteLocal while the other is reading it
// back, yielding "unexpected end of JSON input". The map keeps
// pulls of *different* kernels parallel.
//
// chan struct{} (cap 1) instead of sync.Mutex: acquire is a
// `select` that respects ctx.Done(), so a peer waiting behind a
// pull whose RPC was cancelled can bail out instead of blocking
// forever on a pull that nobody is consuming.
kernelPullLocksMu sync.Mutex
kernelPullLocks map[string]*sync.Mutex
kernelPullLocks map[string]chan struct{}
// imagePullLocksMu / imagePullLocks: same per-name pattern for
// image auto-pulls. Without this, parallel `vm.create` callers
@ -59,7 +64,7 @@ type ImageService struct {
// per image name; peers see the freshly-published image on the
// post-lock recheck.
imagePullLocksMu sync.Mutex
imagePullLocks map[string]*sync.Mutex
imagePullLocks map[string]chan struct{}
// Test seams; nil → real implementation.
pullAndFlatten func(ctx context.Context, ref, cacheDir, destDir string) (imagepull.Metadata, error)
@ -96,39 +101,61 @@ func newImageService(deps imageServiceDeps) *ImageService {
}
}
// kernelPullLock returns the per-name mutex used to serialise kernel
// pulls of `name`. The map entry is created on first access and lives
// for the daemon's lifetime — kernels rarely churn and keeping the
// entry around saves the allocation and the second-acquire path stays
// branchless. Callers Lock() / Unlock() the returned mutex directly.
func (s *ImageService) kernelPullLock(name string) *sync.Mutex {
// acquireKernelPullLock blocks until the per-name lock for `name` is
// free or ctx is cancelled. On success returns a release func that
// the caller must invoke (typically via defer). On ctx cancellation
// returns ctx.Err() and a nil release. The map entry is created on
// first access and lives for the daemon's lifetime — kernels rarely
// churn and keeping the entry around keeps the second-acquire path
// branchless.
func (s *ImageService) acquireKernelPullLock(ctx context.Context, name string) (func(), error) {
ch := s.kernelPullLockChan(name)
select {
case ch <- struct{}{}:
return func() { <-ch }, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (s *ImageService) kernelPullLockChan(name string) chan struct{} {
s.kernelPullLocksMu.Lock()
defer s.kernelPullLocksMu.Unlock()
if s.kernelPullLocks == nil {
s.kernelPullLocks = make(map[string]*sync.Mutex)
s.kernelPullLocks = make(map[string]chan struct{})
}
m, ok := s.kernelPullLocks[name]
ch, ok := s.kernelPullLocks[name]
if !ok {
m = &sync.Mutex{}
s.kernelPullLocks[name] = m
ch = make(chan struct{}, 1)
s.kernelPullLocks[name] = ch
}
return m
return ch
}
// imagePullLock is the image-name peer of kernelPullLock; same lifetime
// and zero-allocation properties on the second-acquire path.
func (s *ImageService) imagePullLock(name string) *sync.Mutex {
// acquireImagePullLock is the image-name peer of acquireKernelPullLock;
// same semantics and lifetime.
func (s *ImageService) acquireImagePullLock(ctx context.Context, name string) (func(), error) {
ch := s.imagePullLockChan(name)
select {
case ch <- struct{}{}:
return func() { <-ch }, nil
case <-ctx.Done():
return nil, ctx.Err()
}
}
func (s *ImageService) imagePullLockChan(name string) chan struct{} {
s.imagePullLocksMu.Lock()
defer s.imagePullLocksMu.Unlock()
if s.imagePullLocks == nil {
s.imagePullLocks = make(map[string]*sync.Mutex)
s.imagePullLocks = make(map[string]chan struct{})
}
m, ok := s.imagePullLocks[name]
ch, ok := s.imagePullLocks[name]
if !ok {
m = &sync.Mutex{}
s.imagePullLocks[name] = m
ch = make(chan struct{}, 1)
s.imagePullLocks[name] = ch
}
return m
return ch
}
// FindImage is the service-owned lookup helper. It falls back from