Speed up vm run repo import
Replace the post-boot full-history git bundle path with a shallow repo copy so vm run no longer spends its quiet time shipping and cloning every object in the source repository. Stage a depth-10 no-checkout clone from the host repo, fetch the requested checkout commit only when it is outside the shallow window, rewrite origin back to the host repo's origin URL, and keep the existing guest checkout plus working-tree overlay behavior. Add explicit [vm run] progress lines after [vm create] ready so the user can see the SSH wait, shallow repo prep, guest copy, overlay, and opencode attach phases instead of a silent pause. Validated with GOCACHE=/tmp/banger-gocache go test ./..., make build, and a local payload comparison showing the banger repo dropping from a ~400 MB full bundle to a ~294 KB shallow metadata copy.
This commit is contained in:
parent
42b4a18c63
commit
1e967140c3
2 changed files with 299 additions and 83 deletions
|
|
@ -477,6 +477,26 @@ func TestVMCreateProgressRendererSuppressesDuplicateLines(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestVMRunProgressRendererSuppressesDuplicateLines(t *testing.T) {
|
||||
var stderr bytes.Buffer
|
||||
renderer := newVMRunProgressRenderer(&stderr)
|
||||
|
||||
renderer.render("waiting for guest ssh")
|
||||
renderer.render("waiting for guest ssh")
|
||||
renderer.render("overlaying host working tree")
|
||||
|
||||
lines := strings.Split(strings.TrimSpace(stderr.String()), "\n")
|
||||
if len(lines) != 2 {
|
||||
t.Fatalf("rendered lines = %q, want 2 lines", stderr.String())
|
||||
}
|
||||
if lines[0] != "[vm run] waiting for guest ssh" {
|
||||
t.Fatalf("first line = %q", lines[0])
|
||||
}
|
||||
if lines[1] != "[vm run] overlaying host working tree" {
|
||||
t.Fatalf("second line = %q", lines[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestVMSetParamsFromFlagsConflict(t *testing.T) {
|
||||
if _, err := vmSetParamsFromFlags("devbox", -1, -1, "", true, true); err == nil {
|
||||
t.Fatal("expected nat conflict error")
|
||||
|
|
@ -923,6 +943,7 @@ func TestInspectVMRunRepoUsesRepoRootAndOverlayPaths(t *testing.T) {
|
|||
testRunGit(t, repoRoot, "config", "--global", "user.email", "global@example.com")
|
||||
testRunGit(t, repoRoot, "config", "--global", "user.name", "Global User")
|
||||
testRunGit(t, repoRoot, "init")
|
||||
testRunGit(t, repoRoot, "remote", "add", "origin", "https://example.com/repo.git")
|
||||
testRunGit(t, repoRoot, "config", "user.email", "test@example.com")
|
||||
testRunGit(t, repoRoot, "config", "user.name", "Banger Test")
|
||||
|
||||
|
|
@ -972,6 +993,9 @@ func TestInspectVMRunRepoUsesRepoRootAndOverlayPaths(t *testing.T) {
|
|||
if spec.BaseCommit != spec.HeadCommit {
|
||||
t.Fatalf("BaseCommit = %q, want head %q", spec.BaseCommit, spec.HeadCommit)
|
||||
}
|
||||
if spec.OriginURL != "https://example.com/repo.git" {
|
||||
t.Fatalf("OriginURL = %q, want https://example.com/repo.git", spec.OriginURL)
|
||||
}
|
||||
if spec.GitUserName != "Banger Test" {
|
||||
t.Fatalf("GitUserName = %q, want Banger Test", spec.GitUserName)
|
||||
}
|
||||
|
|
@ -1018,13 +1042,14 @@ func TestInspectVMRunRepoRejectsSubmodules(t *testing.T) {
|
|||
|
||||
func TestRunVMRunCreatesImportsAndAttaches(t *testing.T) {
|
||||
repoRoot := t.TempDir()
|
||||
repoCopyDir := filepath.Join(t.TempDir(), "repo-copy")
|
||||
|
||||
origBegin := vmCreateBeginFunc
|
||||
origStatus := vmCreateStatusFunc
|
||||
origCancel := vmCreateCancelFunc
|
||||
origWaitForSSH := guestWaitForSSHFunc
|
||||
origGuestDial := guestDialFunc
|
||||
origHostCommandOutput := hostCommandOutputFunc
|
||||
origPrepareVMRunRepoCopy := prepareVMRunRepoCopyFunc
|
||||
origOpencodeExec := opencodeExecFunc
|
||||
t.Cleanup(func() {
|
||||
vmCreateBeginFunc = origBegin
|
||||
|
|
@ -1032,7 +1057,7 @@ func TestRunVMRunCreatesImportsAndAttaches(t *testing.T) {
|
|||
vmCreateCancelFunc = origCancel
|
||||
guestWaitForSSHFunc = origWaitForSSH
|
||||
guestDialFunc = origGuestDial
|
||||
hostCommandOutputFunc = origHostCommandOutput
|
||||
prepareVMRunRepoCopyFunc = origPrepareVMRunRepoCopy
|
||||
opencodeExecFunc = origOpencodeExec
|
||||
})
|
||||
|
||||
|
|
@ -1082,20 +1107,11 @@ func TestRunVMRunCreatesImportsAndAttaches(t *testing.T) {
|
|||
dialKeyPath = privateKeyPath
|
||||
return fakeClient, nil
|
||||
}
|
||||
hostCommandOutputFunc = func(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
if name != "git" {
|
||||
t.Fatalf("command = %q, want git", name)
|
||||
prepareVMRunRepoCopyFunc = func(ctx context.Context, spec vmRunRepoSpec) (string, func(), error) {
|
||||
if spec.RepoRoot != repoRoot {
|
||||
t.Fatalf("spec.RepoRoot = %q, want %q", spec.RepoRoot, repoRoot)
|
||||
}
|
||||
if len(args) < 7 || args[0] != "-C" || args[1] != repoRoot || args[2] != "bundle" || args[3] != "create" || args[5] != "--all" {
|
||||
t.Fatalf("unexpected bundle args: %v", args)
|
||||
}
|
||||
if !reflect.DeepEqual(args[6:], []string{"deadbeef", "cafebabe"}) {
|
||||
t.Fatalf("bundle revs = %v, want deadbeef/cafebabe", args[6:])
|
||||
}
|
||||
if err := os.WriteFile(args[4], []byte("bundle-data"), 0o600); err != nil {
|
||||
t.Fatalf("WriteFile(bundle): %v", err)
|
||||
}
|
||||
return nil, nil
|
||||
return repoCopyDir, func() {}, nil
|
||||
}
|
||||
var attachArgs []string
|
||||
opencodeExecFunc = func(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, args []string) error {
|
||||
|
|
@ -1143,21 +1159,18 @@ func TestRunVMRunCreatesImportsAndAttaches(t *testing.T) {
|
|||
if dialKeyPath != waitKeyPath {
|
||||
t.Fatalf("dialKeyPath = %q, want %q", dialKeyPath, waitKeyPath)
|
||||
}
|
||||
if fakeClient.uploadPath != vmRunGuestBundlePath {
|
||||
t.Fatalf("uploadPath = %q, want %q", fakeClient.uploadPath, vmRunGuestBundlePath)
|
||||
if fakeClient.tarSourceDir != repoCopyDir {
|
||||
t.Fatalf("tarSourceDir = %q, want %q", fakeClient.tarSourceDir, repoCopyDir)
|
||||
}
|
||||
if fakeClient.uploadMode != 0o600 {
|
||||
t.Fatalf("uploadMode = %v, want 0600", fakeClient.uploadMode)
|
||||
}
|
||||
if string(fakeClient.uploadData) != "bundle-data" {
|
||||
t.Fatalf("uploadData = %q, want bundle-data", string(fakeClient.uploadData))
|
||||
}
|
||||
if !strings.Contains(fakeClient.script, `git clone "$BUNDLE" "$DIR"`) {
|
||||
t.Fatalf("script = %q, want clone command", fakeClient.script)
|
||||
if fakeClient.tarCommand != "rm -rf '/root/repo' && mkdir -p '/root/repo' && tar -o -C '/root/repo' --strip-components=1 -xf -" {
|
||||
t.Fatalf("tarCommand = %q", fakeClient.tarCommand)
|
||||
}
|
||||
if !strings.Contains(fakeClient.script, `git -C "$DIR" checkout -B 'feature' 'cafebabe'`) {
|
||||
t.Fatalf("script = %q, want guest branch checkout", fakeClient.script)
|
||||
}
|
||||
if !strings.Contains(fakeClient.script, `find "$DIR" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} +`) {
|
||||
t.Fatalf("script = %q, want guest worktree reset", fakeClient.script)
|
||||
}
|
||||
if !strings.Contains(fakeClient.script, `git config --global --add safe.directory "$DIR"`) {
|
||||
t.Fatalf("script = %q, want guest safe.directory config", fakeClient.script)
|
||||
}
|
||||
|
|
@ -1185,8 +1198,147 @@ func TestRunVMRunCreatesImportsAndAttaches(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestVMRunCloneScriptSkipsRepoGitIdentityWhenIncomplete(t *testing.T) {
|
||||
script := vmRunCloneScript(vmRunRepoSpec{
|
||||
func TestVMRunPrintsPostCreateProgress(t *testing.T) {
|
||||
origBegin := vmCreateBeginFunc
|
||||
origStatus := vmCreateStatusFunc
|
||||
origCancel := vmCreateCancelFunc
|
||||
origWaitForSSH := guestWaitForSSHFunc
|
||||
origGuestDial := guestDialFunc
|
||||
origPrepareVMRunRepoCopy := prepareVMRunRepoCopyFunc
|
||||
origOpencodeExec := opencodeExecFunc
|
||||
t.Cleanup(func() {
|
||||
vmCreateBeginFunc = origBegin
|
||||
vmCreateStatusFunc = origStatus
|
||||
vmCreateCancelFunc = origCancel
|
||||
guestWaitForSSHFunc = origWaitForSSH
|
||||
guestDialFunc = origGuestDial
|
||||
prepareVMRunRepoCopyFunc = origPrepareVMRunRepoCopy
|
||||
opencodeExecFunc = origOpencodeExec
|
||||
})
|
||||
|
||||
vm := model.VMRecord{
|
||||
ID: "vm-id",
|
||||
Name: "devbox",
|
||||
Runtime: model.VMRuntime{
|
||||
State: model.VMStateRunning,
|
||||
GuestIP: "172.16.0.2",
|
||||
},
|
||||
}
|
||||
vmCreateBeginFunc = func(context.Context, string, api.VMCreateParams) (api.VMCreateBeginResult, error) {
|
||||
return api.VMCreateBeginResult{
|
||||
Operation: api.VMCreateOperation{
|
||||
ID: "op-1",
|
||||
Stage: "ready",
|
||||
Detail: "vm is ready",
|
||||
Done: true,
|
||||
Success: true,
|
||||
VM: &vm,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
vmCreateStatusFunc = func(context.Context, string, string) (api.VMCreateStatusResult, error) {
|
||||
t.Fatal("vmCreateStatusFunc should not be called")
|
||||
return api.VMCreateStatusResult{}, nil
|
||||
}
|
||||
vmCreateCancelFunc = func(context.Context, string, string) error {
|
||||
t.Fatal("vmCreateCancelFunc should not be called")
|
||||
return nil
|
||||
}
|
||||
guestWaitForSSHFunc = func(ctx context.Context, address, privateKeyPath string, interval time.Duration) error {
|
||||
return nil
|
||||
}
|
||||
guestDialFunc = func(ctx context.Context, address, privateKeyPath string) (vmRunGuestClient, error) {
|
||||
return &testVMRunGuestClient{}, nil
|
||||
}
|
||||
prepareVMRunRepoCopyFunc = func(ctx context.Context, spec vmRunRepoSpec) (string, func(), error) {
|
||||
return t.TempDir(), func() {}, nil
|
||||
}
|
||||
opencodeExecFunc = func(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var stderr bytes.Buffer
|
||||
err := runVMRun(
|
||||
context.Background(),
|
||||
"/tmp/bangerd.sock",
|
||||
model.DaemonConfig{SSHKeyPath: "/tmp/id_ed25519"},
|
||||
strings.NewReader(""),
|
||||
&bytes.Buffer{},
|
||||
&stderr,
|
||||
api.VMCreateParams{Name: "devbox"},
|
||||
vmRunRepoSpec{RepoRoot: t.TempDir(), RepoName: "repo", HeadCommit: "deadbeef"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("runVMRun: %v", err)
|
||||
}
|
||||
|
||||
output := stderr.String()
|
||||
for _, want := range []string{
|
||||
"[vm run] waiting for guest ssh",
|
||||
"[vm run] preparing shallow repo",
|
||||
"[vm run] copying repo metadata to guest",
|
||||
"[vm run] preparing guest checkout",
|
||||
"[vm run] overlaying host working tree",
|
||||
"[vm run] attaching opencode",
|
||||
} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("stderr = %q, want %q", output, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareVMRunRepoCopyCreatesShallowMetadataCopy(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("git not installed")
|
||||
}
|
||||
|
||||
repoRoot := t.TempDir()
|
||||
testRunGit(t, repoRoot, "init")
|
||||
testRunGit(t, repoRoot, "remote", "add", "origin", "https://example.com/repo.git")
|
||||
for i := 0; i < 12; i++ {
|
||||
name := fmt.Sprintf("file-%02d.txt", i)
|
||||
if err := os.WriteFile(filepath.Join(repoRoot, name), []byte(fmt.Sprintf("commit-%02d\n", i)), 0o644); err != nil {
|
||||
t.Fatalf("WriteFile(%s): %v", name, err)
|
||||
}
|
||||
testRunGit(t, repoRoot, "add", name)
|
||||
testRunGit(t, repoRoot, "commit", "-m", fmt.Sprintf("commit-%02d", i))
|
||||
}
|
||||
baseCommit := strings.TrimSpace(testRunGit(t, repoRoot, "rev-parse", "HEAD~5"))
|
||||
|
||||
repoCopyDir, cleanup, err := prepareVMRunRepoCopy(context.Background(), vmRunRepoSpec{
|
||||
RepoRoot: repoRoot,
|
||||
RepoName: "repo",
|
||||
BranchName: "feature",
|
||||
BaseCommit: baseCommit,
|
||||
HeadCommit: strings.TrimSpace(testRunGit(t, repoRoot, "rev-parse", "HEAD")),
|
||||
OriginURL: "https://example.com/repo.git",
|
||||
OverlayPaths: []string{"file-11.txt"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepareVMRunRepoCopy: %v", err)
|
||||
}
|
||||
defer cleanup()
|
||||
|
||||
entries, err := os.ReadDir(repoCopyDir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir(repoCopyDir): %v", err)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].Name() != ".git" {
|
||||
t.Fatalf("repo copy entries = %v, want only .git", entries)
|
||||
}
|
||||
if got := strings.TrimSpace(testRunGit(t, repoCopyDir, "rev-parse", "--is-shallow-repository")); got != "true" {
|
||||
t.Fatalf("is-shallow-repository = %q, want true", got)
|
||||
}
|
||||
if got := strings.TrimSpace(testRunGit(t, repoCopyDir, "config", "--get", "remote.origin.url")); got != "https://example.com/repo.git" {
|
||||
t.Fatalf("remote.origin.url = %q, want https://example.com/repo.git", got)
|
||||
}
|
||||
if _, err := exec.Command("git", "-C", repoCopyDir, "cat-file", "-e", baseCommit+"^{commit}").CombinedOutput(); err != nil {
|
||||
t.Fatalf("cat-file -e %s^{commit}: %v", baseCommit, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVMRunCheckoutScriptSkipsRepoGitIdentityWhenIncomplete(t *testing.T) {
|
||||
script := vmRunCheckoutScript(vmRunRepoSpec{
|
||||
RepoName: "repo",
|
||||
HeadCommit: "deadbeef",
|
||||
CurrentBranch: "main",
|
||||
|
|
@ -1388,10 +1540,9 @@ func testRunGit(t *testing.T, dir string, args ...string) string {
|
|||
|
||||
type testVMRunGuestClient struct {
|
||||
closed bool
|
||||
uploadPath string
|
||||
uploadMode os.FileMode
|
||||
uploadData []byte
|
||||
script string
|
||||
tarSourceDir string
|
||||
tarCommand string
|
||||
streamSourceDir string
|
||||
streamEntries []string
|
||||
streamCommand string
|
||||
|
|
@ -1402,10 +1553,9 @@ func (c *testVMRunGuestClient) Close() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c *testVMRunGuestClient) UploadFile(ctx context.Context, remotePath string, mode os.FileMode, data []byte, logWriter io.Writer) error {
|
||||
c.uploadPath = remotePath
|
||||
c.uploadMode = mode
|
||||
c.uploadData = append([]byte(nil), data...)
|
||||
func (c *testVMRunGuestClient) StreamTar(ctx context.Context, sourceDir, remoteCommand string, logWriter io.Writer) error {
|
||||
c.tarSourceDir = sourceDir
|
||||
c.tarCommand = remoteCommand
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue