daemon: tighten concurrency around pulls, cleanup, and handle persistence
Four targeted fixes from a race-condition audit of the daemon package.
None change behaviour on the happy path; each closes a window where a
concurrent or interrupted RPC could strand state on the host.
- KernelDelete now holds the same per-name lock as KernelPull /
readOrAutoPullKernel. Without it, a delete racing a concurrent
pull could remove files mid-write or land between the pull's
manifest write and its first use.
- cleanupRuntime no longer early-returns on an inner waitForExit
failure; DM snapshot, capability, and tap teardown always run and
every error is folded into the returned errors.Join. EBUSY against
a still-alive firecracker is benign and surfaces in the joined
error rather than stranding kernel state across daemon restarts.
- Per-name image / kernel pull locks switch from *sync.Mutex to a
1-buffered chan struct{}. Acquire is a select on ctx.Done(), so a
peer waiting behind a pull whose RPC was cancelled can bail out
instead of blocking forever on a pull nobody is consuming.
- setVMHandles writes the per-VM scratch file before updating the
in-memory cache. A daemon crash between the two now leaves disk
ahead of memory (recoverable: reconcile re-seeds the cache from
the file on next start) rather than memory ahead of disk (lost
handles → stranded DM/loops/tap).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
777b597a1e
commit
c4e1cb5953
6 changed files with 99 additions and 39 deletions
|
|
@ -39,15 +39,20 @@ type ImageService struct {
|
|||
imageOpsMu sync.Mutex
|
||||
|
||||
// kernelPullLocksMu guards the kernelPullLocks map itself. Per-name
|
||||
// mutexes inside the map serialise concurrent pulls of the same
|
||||
// kernel ref. Without this, two parallel `vm run` callers that
|
||||
// auto-pull the same kernel race on
|
||||
// channel locks inside the map serialise concurrent pulls of the
|
||||
// same kernel ref. Without this, two parallel `vm run` callers
|
||||
// that auto-pull the same kernel race on
|
||||
// /var/lib/banger/kernels/<name>/manifest.json: one is mid-write
|
||||
// from kernelcat.Fetch's WriteLocal while the other is reading it
|
||||
// back, yielding "unexpected end of JSON input". The map keeps
|
||||
// pulls of *different* kernels parallel.
|
||||
//
|
||||
// chan struct{} (cap 1) instead of sync.Mutex: acquire is a
|
||||
// `select` that respects ctx.Done(), so a peer waiting behind a
|
||||
// pull whose RPC was cancelled can bail out instead of blocking
|
||||
// forever on a pull that nobody is consuming.
|
||||
kernelPullLocksMu sync.Mutex
|
||||
kernelPullLocks map[string]*sync.Mutex
|
||||
kernelPullLocks map[string]chan struct{}
|
||||
|
||||
// imagePullLocksMu / imagePullLocks: same per-name pattern for
|
||||
// image auto-pulls. Without this, parallel `vm.create` callers
|
||||
|
|
@ -59,7 +64,7 @@ type ImageService struct {
|
|||
// per image name; peers see the freshly-published image on the
|
||||
// post-lock recheck.
|
||||
imagePullLocksMu sync.Mutex
|
||||
imagePullLocks map[string]*sync.Mutex
|
||||
imagePullLocks map[string]chan struct{}
|
||||
|
||||
// Test seams; nil → real implementation.
|
||||
pullAndFlatten func(ctx context.Context, ref, cacheDir, destDir string) (imagepull.Metadata, error)
|
||||
|
|
@ -96,39 +101,61 @@ func newImageService(deps imageServiceDeps) *ImageService {
|
|||
}
|
||||
}
|
||||
|
||||
// kernelPullLock returns the per-name mutex used to serialise kernel
|
||||
// pulls of `name`. The map entry is created on first access and lives
|
||||
// for the daemon's lifetime — kernels rarely churn and keeping the
|
||||
// entry around saves the allocation and the second-acquire path stays
|
||||
// branchless. Callers Lock() / Unlock() the returned mutex directly.
|
||||
func (s *ImageService) kernelPullLock(name string) *sync.Mutex {
|
||||
// acquireKernelPullLock blocks until the per-name lock for `name` is
|
||||
// free or ctx is cancelled. On success returns a release func that
|
||||
// the caller must invoke (typically via defer). On ctx cancellation
|
||||
// returns ctx.Err() and a nil release. The map entry is created on
|
||||
// first access and lives for the daemon's lifetime — kernels rarely
|
||||
// churn and keeping the entry around keeps the second-acquire path
|
||||
// branchless.
|
||||
func (s *ImageService) acquireKernelPullLock(ctx context.Context, name string) (func(), error) {
|
||||
ch := s.kernelPullLockChan(name)
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
return func() { <-ch }, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ImageService) kernelPullLockChan(name string) chan struct{} {
|
||||
s.kernelPullLocksMu.Lock()
|
||||
defer s.kernelPullLocksMu.Unlock()
|
||||
if s.kernelPullLocks == nil {
|
||||
s.kernelPullLocks = make(map[string]*sync.Mutex)
|
||||
s.kernelPullLocks = make(map[string]chan struct{})
|
||||
}
|
||||
m, ok := s.kernelPullLocks[name]
|
||||
ch, ok := s.kernelPullLocks[name]
|
||||
if !ok {
|
||||
m = &sync.Mutex{}
|
||||
s.kernelPullLocks[name] = m
|
||||
ch = make(chan struct{}, 1)
|
||||
s.kernelPullLocks[name] = ch
|
||||
}
|
||||
return m
|
||||
return ch
|
||||
}
|
||||
|
||||
// imagePullLock is the image-name peer of kernelPullLock; same lifetime
|
||||
// and zero-allocation properties on the second-acquire path.
|
||||
func (s *ImageService) imagePullLock(name string) *sync.Mutex {
|
||||
// acquireImagePullLock is the image-name peer of acquireKernelPullLock;
|
||||
// same semantics and lifetime.
|
||||
func (s *ImageService) acquireImagePullLock(ctx context.Context, name string) (func(), error) {
|
||||
ch := s.imagePullLockChan(name)
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
return func() { <-ch }, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ImageService) imagePullLockChan(name string) chan struct{} {
|
||||
s.imagePullLocksMu.Lock()
|
||||
defer s.imagePullLocksMu.Unlock()
|
||||
if s.imagePullLocks == nil {
|
||||
s.imagePullLocks = make(map[string]*sync.Mutex)
|
||||
s.imagePullLocks = make(map[string]chan struct{})
|
||||
}
|
||||
m, ok := s.imagePullLocks[name]
|
||||
ch, ok := s.imagePullLocks[name]
|
||||
if !ok {
|
||||
m = &sync.Mutex{}
|
||||
s.imagePullLocks[name] = m
|
||||
ch = make(chan struct{}, 1)
|
||||
s.imagePullLocks[name] = ch
|
||||
}
|
||||
return m
|
||||
return ch
|
||||
}
|
||||
|
||||
// FindImage is the service-owned lookup helper. It falls back from
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue