Rollback partial dm snapshot startup
Prevent partial VM startup failures from leaking loop devices and dm state on the host. Move root snapshot setup into a rollback-safe helper that records loop and mapper handles incrementally, tears them down in reverse order on failure, and reuses the same dm/loop cleanup path during normal runtime teardown. Also switch the daemon runner field to a small command-runner interface so the snapshot path can be tested with injected failures. Add failure-injection coverage for losetup, blockdev, dmsetup, partial teardown, and joined rollback errors. Validated with go test ./... and make build.
This commit is contained in:
parent
171009b30b
commit
375900cf65
5 changed files with 401 additions and 47 deletions
294
internal/daemon/snapshot_test.go
Normal file
294
internal/daemon/snapshot_test.go
Normal file
|
|
@ -0,0 +1,294 @@
|
|||
package daemon
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type runnerCall struct {
|
||||
sudo bool
|
||||
name string
|
||||
args []string
|
||||
}
|
||||
|
||||
type runnerStep struct {
|
||||
call runnerCall
|
||||
out []byte
|
||||
err error
|
||||
}
|
||||
|
||||
type scriptedRunner struct {
|
||||
t *testing.T
|
||||
steps []runnerStep
|
||||
calls []runnerCall
|
||||
}
|
||||
|
||||
func (r *scriptedRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
return r.next(runnerCall{name: name, args: append([]string(nil), args...)})
|
||||
}
|
||||
|
||||
func (r *scriptedRunner) RunSudo(ctx context.Context, args ...string) ([]byte, error) {
|
||||
return r.next(runnerCall{sudo: true, args: append([]string(nil), args...)})
|
||||
}
|
||||
|
||||
func (r *scriptedRunner) next(call runnerCall) ([]byte, error) {
|
||||
r.t.Helper()
|
||||
r.calls = append(r.calls, call)
|
||||
if len(r.steps) == 0 {
|
||||
r.t.Fatalf("unexpected call: %+v", call)
|
||||
}
|
||||
step := r.steps[0]
|
||||
r.steps = r.steps[1:]
|
||||
if step.call.sudo != call.sudo || step.call.name != call.name || !slices.Equal(step.call.args, call.args) {
|
||||
r.t.Fatalf("call mismatch:\n got: %+v\n want: %+v", call, step.call)
|
||||
}
|
||||
return step.out, step.err
|
||||
}
|
||||
|
||||
func (r *scriptedRunner) assertExhausted() {
|
||||
r.t.Helper()
|
||||
if len(r.steps) != 0 {
|
||||
r.t.Fatalf("unconsumed steps: %+v", r.steps)
|
||||
}
|
||||
}
|
||||
|
||||
func sudoStep(out string, err error, args ...string) runnerStep {
|
||||
return runnerStep{
|
||||
call: runnerCall{sudo: true, args: append([]string(nil), args...)},
|
||||
out: []byte(out),
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateDMSnapshotFailsWithoutRollbackWhenBaseLoopSetupFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
attachErr := errors.New("attach base loop")
|
||||
runner := &scriptedRunner{
|
||||
t: t,
|
||||
steps: []runnerStep{
|
||||
sudoStep("", attachErr, "losetup", "-f", "--show", "--read-only", "/rootfs.ext4"),
|
||||
},
|
||||
}
|
||||
d := &Daemon{runner: runner}
|
||||
|
||||
_, err := d.createDMSnapshot(context.Background(), "/rootfs.ext4", "/cow.ext4", "fc-rootfs-test")
|
||||
if !errors.Is(err, attachErr) {
|
||||
t.Fatalf("error = %v, want %v", err, attachErr)
|
||||
}
|
||||
runner.assertExhausted()
|
||||
if len(runner.calls) != 1 {
|
||||
t.Fatalf("call count = %d, want 1", len(runner.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateDMSnapshotRollsBackBaseLoopWhenCowLoopSetupFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
attachErr := errors.New("attach cow loop")
|
||||
runner := &scriptedRunner{
|
||||
t: t,
|
||||
steps: []runnerStep{
|
||||
sudoStep("/dev/loop10\n", nil, "losetup", "-f", "--show", "--read-only", "/rootfs.ext4"),
|
||||
sudoStep("", attachErr, "losetup", "-f", "--show", "/cow.ext4"),
|
||||
sudoStep("", nil, "losetup", "-d", "/dev/loop10"),
|
||||
},
|
||||
}
|
||||
d := &Daemon{runner: runner}
|
||||
|
||||
_, err := d.createDMSnapshot(context.Background(), "/rootfs.ext4", "/cow.ext4", "fc-rootfs-test")
|
||||
if !errors.Is(err, attachErr) {
|
||||
t.Fatalf("error = %v, want %v", err, attachErr)
|
||||
}
|
||||
runner.assertExhausted()
|
||||
}
|
||||
|
||||
func TestCreateDMSnapshotRollsBackBothLoopsWhenBlockdevFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
blockdevErr := errors.New("read sectors")
|
||||
runner := &scriptedRunner{
|
||||
t: t,
|
||||
steps: []runnerStep{
|
||||
sudoStep("/dev/loop10\n", nil, "losetup", "-f", "--show", "--read-only", "/rootfs.ext4"),
|
||||
sudoStep("/dev/loop11\n", nil, "losetup", "-f", "--show", "/cow.ext4"),
|
||||
sudoStep("", blockdevErr, "blockdev", "--getsz", "/dev/loop10"),
|
||||
sudoStep("", nil, "losetup", "-d", "/dev/loop11"),
|
||||
sudoStep("", nil, "losetup", "-d", "/dev/loop10"),
|
||||
},
|
||||
}
|
||||
d := &Daemon{runner: runner}
|
||||
|
||||
_, err := d.createDMSnapshot(context.Background(), "/rootfs.ext4", "/cow.ext4", "fc-rootfs-test")
|
||||
if !errors.Is(err, blockdevErr) {
|
||||
t.Fatalf("error = %v, want %v", err, blockdevErr)
|
||||
}
|
||||
runner.assertExhausted()
|
||||
}
|
||||
|
||||
func TestCreateDMSnapshotRollsBackLoopsWhenDMSetupFails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dmErr := errors.New("create dm snapshot")
|
||||
runner := &scriptedRunner{
|
||||
t: t,
|
||||
steps: []runnerStep{
|
||||
sudoStep("/dev/loop10\n", nil, "losetup", "-f", "--show", "--read-only", "/rootfs.ext4"),
|
||||
sudoStep("/dev/loop11\n", nil, "losetup", "-f", "--show", "/cow.ext4"),
|
||||
sudoStep("12345\n", nil, "blockdev", "--getsz", "/dev/loop10"),
|
||||
sudoStep("", dmErr, "dmsetup", "create", "fc-rootfs-test", "--table", "0 12345 snapshot /dev/loop10 /dev/loop11 P 8"),
|
||||
sudoStep("", nil, "losetup", "-d", "/dev/loop11"),
|
||||
sudoStep("", nil, "losetup", "-d", "/dev/loop10"),
|
||||
},
|
||||
}
|
||||
d := &Daemon{runner: runner}
|
||||
|
||||
_, err := d.createDMSnapshot(context.Background(), "/rootfs.ext4", "/cow.ext4", "fc-rootfs-test")
|
||||
if !errors.Is(err, dmErr) {
|
||||
t.Fatalf("error = %v, want %v", err, dmErr)
|
||||
}
|
||||
runner.assertExhausted()
|
||||
for _, call := range runner.calls {
|
||||
if call.sudo && len(call.args) >= 2 && call.args[0] == "dmsetup" && call.args[1] == "remove" {
|
||||
t.Fatalf("unexpected dmsetup remove call: %+v", call)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateDMSnapshotJoinsRollbackErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
blockdevErr := errors.New("read sectors")
|
||||
detachErr := errors.New("detach cow loop")
|
||||
runner := &scriptedRunner{
|
||||
t: t,
|
||||
steps: []runnerStep{
|
||||
sudoStep("/dev/loop10\n", nil, "losetup", "-f", "--show", "--read-only", "/rootfs.ext4"),
|
||||
sudoStep("/dev/loop11\n", nil, "losetup", "-f", "--show", "/cow.ext4"),
|
||||
sudoStep("", blockdevErr, "blockdev", "--getsz", "/dev/loop10"),
|
||||
sudoStep("", detachErr, "losetup", "-d", "/dev/loop11"),
|
||||
sudoStep("", nil, "losetup", "-d", "/dev/loop10"),
|
||||
},
|
||||
}
|
||||
d := &Daemon{runner: runner}
|
||||
|
||||
_, err := d.createDMSnapshot(context.Background(), "/rootfs.ext4", "/cow.ext4", "fc-rootfs-test")
|
||||
if err == nil {
|
||||
t.Fatal("expected createDMSnapshot to return an error")
|
||||
}
|
||||
if !errors.Is(err, blockdevErr) || !errors.Is(err, detachErr) {
|
||||
t.Fatalf("error = %v, want joined blockdev and rollback errors", err)
|
||||
}
|
||||
runner.assertExhausted()
|
||||
}
|
||||
|
||||
func TestCreateDMSnapshotReturnsHandlesOnSuccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
runner := &scriptedRunner{
|
||||
t: t,
|
||||
steps: []runnerStep{
|
||||
sudoStep("/dev/loop10\n", nil, "losetup", "-f", "--show", "--read-only", "/rootfs.ext4"),
|
||||
sudoStep("/dev/loop11\n", nil, "losetup", "-f", "--show", "/cow.ext4"),
|
||||
sudoStep("12345\n", nil, "blockdev", "--getsz", "/dev/loop10"),
|
||||
sudoStep("", nil, "dmsetup", "create", "fc-rootfs-test", "--table", "0 12345 snapshot /dev/loop10 /dev/loop11 P 8"),
|
||||
},
|
||||
}
|
||||
d := &Daemon{runner: runner}
|
||||
|
||||
handles, err := d.createDMSnapshot(context.Background(), "/rootfs.ext4", "/cow.ext4", "fc-rootfs-test")
|
||||
if err != nil {
|
||||
t.Fatalf("createDMSnapshot returned error: %v", err)
|
||||
}
|
||||
want := dmSnapshotHandles{
|
||||
BaseLoop: "/dev/loop10",
|
||||
COWLoop: "/dev/loop11",
|
||||
DMName: "fc-rootfs-test",
|
||||
DMDev: "/dev/mapper/fc-rootfs-test",
|
||||
}
|
||||
if handles != want {
|
||||
t.Fatalf("handles = %+v, want %+v", handles, want)
|
||||
}
|
||||
runner.assertExhausted()
|
||||
}
|
||||
|
||||
func TestCleanupDMSnapshotRemovesResourcesInReverseOrder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
runner := &scriptedRunner{
|
||||
t: t,
|
||||
steps: []runnerStep{
|
||||
sudoStep("", nil, "dmsetup", "remove", "fc-rootfs-test"),
|
||||
sudoStep("", nil, "losetup", "-d", "/dev/loop11"),
|
||||
sudoStep("", nil, "losetup", "-d", "/dev/loop10"),
|
||||
},
|
||||
}
|
||||
d := &Daemon{runner: runner}
|
||||
|
||||
err := d.cleanupDMSnapshot(context.Background(), dmSnapshotHandles{
|
||||
BaseLoop: "/dev/loop10",
|
||||
COWLoop: "/dev/loop11",
|
||||
DMName: "fc-rootfs-test",
|
||||
DMDev: "/dev/mapper/fc-rootfs-test",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("cleanupDMSnapshot returned error: %v", err)
|
||||
}
|
||||
runner.assertExhausted()
|
||||
}
|
||||
|
||||
func TestCleanupDMSnapshotUsesPartialHandles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
runner := &scriptedRunner{
|
||||
t: t,
|
||||
steps: []runnerStep{
|
||||
sudoStep("", nil, "dmsetup", "remove", "/dev/mapper/fc-rootfs-test"),
|
||||
sudoStep("", nil, "losetup", "-d", "/dev/loop10"),
|
||||
},
|
||||
}
|
||||
d := &Daemon{runner: runner}
|
||||
|
||||
err := d.cleanupDMSnapshot(context.Background(), dmSnapshotHandles{
|
||||
BaseLoop: "/dev/loop10",
|
||||
DMDev: "/dev/mapper/fc-rootfs-test",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("cleanupDMSnapshot returned error: %v", err)
|
||||
}
|
||||
runner.assertExhausted()
|
||||
}
|
||||
|
||||
func TestCleanupDMSnapshotJoinsTeardownErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dmErr := errors.New("remove dm")
|
||||
cowErr := errors.New("detach cow")
|
||||
baseErr := errors.New("detach base")
|
||||
runner := &scriptedRunner{
|
||||
t: t,
|
||||
steps: []runnerStep{
|
||||
sudoStep("", dmErr, "dmsetup", "remove", "fc-rootfs-test"),
|
||||
sudoStep("", cowErr, "losetup", "-d", "/dev/loop11"),
|
||||
sudoStep("", baseErr, "losetup", "-d", "/dev/loop10"),
|
||||
},
|
||||
}
|
||||
d := &Daemon{runner: runner}
|
||||
|
||||
err := d.cleanupDMSnapshot(context.Background(), dmSnapshotHandles{
|
||||
BaseLoop: "/dev/loop10",
|
||||
COWLoop: "/dev/loop11",
|
||||
DMName: "fc-rootfs-test",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected cleanupDMSnapshot to return an error")
|
||||
}
|
||||
for _, expected := range []error{dmErr, cowErr, baseErr} {
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatalf("cleanup error %q not joined into %v", expected, err)
|
||||
}
|
||||
}
|
||||
runner.assertExhausted()
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue