e2e/reap_test.go
246 lines · 10069 bytes
1package e2e
2
3import (
4 "crypto/sha256"
5 "encoding/hex"
6 "encoding/json"
7 "fmt"
8 "os"
9 "path/filepath"
10 "strings"
11 "testing"
12 "time"
13)
14
15// A build claimed by a runner that never reports is failed by the
16// scheduler's tick, with no runner alive to trigger it.
17func TestStaleBuildReapedWithoutRunner(t *testing.T) {
18 t.Setenv("GITBAY_SCHED_TICK", "500ms")
19 t.Setenv("GITBAY_STALE_BUILD_DEADLINE", "2s")
20 inst := startInstance(t)
21 aliceKey := inst.newKey(t, "alice")
22 runnerKey := inst.newKey(t, "ci")
23 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
24 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
25 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
26 t.Fatal("repo create failed")
27 }
28 work := t.TempDir()
29 env := inst.gitEnv(aliceKey)
30 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
31 dir := filepath.Join(work, "w")
32 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
33 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs:\n ok:\n steps:\n - echo fine\n"), 0o644)
34 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
35 mustGit(t, dir, env, "add", ".")
36 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
37 mustGit(t, dir, env, "push", "-q", "origin", "main")
38
39 // Claim the build the way a runner does, then vanish.
40 out, errOut, code := inst.ssh(t, runnerKey, "", "runner", "next", "--json")
41 if code != 0 || !strings.Contains(out, `"job":"ok"`) {
42 t.Fatalf("runner next: exit %d\n%s%s", code, out, errOut)
43 }
44 var claim struct {
45 Data struct {
46 SHA string `json:"sha"`
47 } `json:"data"`
48 }
49 json.Unmarshal([]byte(out), &claim)
50
51 status := func() (string, string) {
52 out, _, _ := inst.ssh(t, aliceKey, "", "build", "list", "alice/app", "--json")
53 var env struct {
54 Data []struct {
55 Status string `json:"status"`
56 } `json:"data"`
57 }
58 json.Unmarshal([]byte(out), &env)
59 st := ""
60 if len(env.Data) > 0 {
61 st = env.Data[0].Status
62 }
63 cs, _, _ := inst.ssh(t, aliceKey, "", "status", "list", "alice/app", claim.Data.SHA)
64 return st, cs
65 }
66 if st, _ := status(); st != "running" {
67 t.Fatalf("claimed build is %q, want running", st)
68 }
69 deadline := time.Now().Add(20 * time.Second)
70 for {
71 st, cs := status()
72 if st == "failure" && strings.Contains(cs, "build abandoned") {
73 break
74 }
75 if time.Now().After(deadline) {
76 t.Fatalf("never reaped: build %q, statuses:\n%s", st, cs)
77 }
78 time.Sleep(250 * time.Millisecond)
79 }
80 if out, _, _ := inst.ssh(t, aliceKey, "", "build", "log", "alice/app", "1"); !strings.Contains(out, "build abandoned") {
81 t.Fatalf("log lacks the abandonment note:\n%s", out)
82 }
83}
84
85func TestAdminRunners(t *testing.T) {
86 inst := startInstance(t)
87 rootKey := inst.newKey(t, "root")
88 aliceKey := inst.newKey(t, "alice")
89 runnerKey := inst.newKey(t, "ci")
90 inst.admin(t, "admin", "user", "create", "root", "--key", rootKey+".pub", "--admin")
91 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
92 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
93 if _, _, code := inst.ssh(t, aliceKey, "", "admin", "runners"); code != 4 {
94 t.Fatal("non-admin listed runners")
95 }
96 if out, _, code := inst.ssh(t, rootKey, "", "admin", "runners", "--json"); code != 0 || strings.TrimSpace(out) != `{"protocol_version":1,"data":[]}` {
97 t.Fatalf("no runners yet: exit %d %s", code, out)
98 }
99 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/app"); code != 0 {
100 t.Fatal("repo create failed")
101 }
102 // An idle poll registers the runner with its scope.
103 if _, _, code := inst.ssh(t, runnerKey, "", "runner", "next", "alice/app"); code != 0 {
104 t.Fatal("runner next failed")
105 }
106 out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners")
107 if !strings.HasPrefix(out, "ci\t") || !strings.Contains(out, "\talice/app\tidle") {
108 t.Fatalf("idle runner row:\n%s", out)
109 }
110 work := t.TempDir()
111 env := inst.gitEnv(aliceKey)
112 mustGit(t, work, env, "clone", inst.sshURL("alice/app"), "w")
113 dir := filepath.Join(work, "w")
114 os.MkdirAll(filepath.Join(dir, ".gitbay"), 0o755)
115 os.WriteFile(filepath.Join(dir, ".gitbay", "ci.yml"), []byte("jobs:\n ok:\n steps:\n - echo fine\n"), 0o644)
116 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
117 mustGit(t, dir, env, "add", ".")
118 mustGit(t, dir, env, "commit", "-q", "-m", "ci")
119 mustGit(t, dir, env, "push", "-q", "origin", "main")
120 out, _, code := inst.ssh(t, runnerKey, "", "runner", "next", "--json")
121 if code != 0 || !strings.Contains(out, `"job":"ok"`) {
122 t.Fatalf("claim: %s", out)
123 }
124 var claim struct {
125 Data struct {
126 ID int64 `json:"id"`
127 } `json:"data"`
128 }
129 json.Unmarshal([]byte(out), &claim)
130 if out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners"); !strings.Contains(out, "\tany\talice/app #1 ok since ") {
131 t.Fatalf("holding runner row:\n%s", out)
132 }
133 if _, _, code := inst.ssh(t, runnerKey, "", "runner", "done", fmt.Sprint(claim.Data.ID), "success"); code != 0 {
134 t.Fatal("runner done failed")
135 }
136 if out, _, _ := inst.ssh(t, rootKey, "", "admin", "runners"); !strings.Contains(out, "\tany\tidle") {
137 t.Fatalf("runner still holds a build after done:\n%s", out)
138 }
139 // Host-local, the same read.
140 if out := inst.admin(t, "admin", "runners", "--json"); !strings.Contains(out, `"username":"ci"`) {
141 t.Fatalf("host runners:\n%s", out)
142 }
143}
144
145// /healthz is unauthenticated, cache-free, and says which build serves.
146func TestHealthz(t *testing.T) {
147 inst := startInstance(t)
148 status, body := inst.get(t, "/healthz")
149 if status != 200 || !strings.Contains(body, `"ok":true`) || !strings.Contains(body, `"commit":"`) {
150 t.Fatalf("healthz: %d %s", status, body)
151 }
152 // The name is reserved: no account can shadow the route.
153 if _, errOut, code := inst.ssh(t, inst.newKey(t, "x"), "", "register", "--username", "healthz"); code == 0 {
154 t.Fatalf("healthz registered as a username: %s", errOut)
155 }
156}
157
158// gc --lfs removes objects no pointer names, keeps referenced ones, and
159// leaves anything young enough to be an upload ahead of its push.
160func TestGCLFSOrphans(t *testing.T) {
161 inst := startInstance(t)
162 aliceKey := inst.newKey(t, "alice")
163 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
164 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/big"); code != 0 {
165 t.Fatal("repo create failed")
166 }
167 put := func(content string, age time.Duration) string {
168 t.Helper()
169 sum := sha256.Sum256([]byte(content))
170 oid := hex.EncodeToString(sum[:])
171 path := filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid)
172 os.MkdirAll(filepath.Dir(path), 0o755)
173 os.WriteFile(path, []byte(content), 0o644)
174 os.Chtimes(path, time.Now().Add(-age), time.Now().Add(-age))
175 return oid
176 }
177 kept := put("referenced payload", 48*time.Hour)
178 orphan := put("nobody points here", 48*time.Hour)
179 young := put("just uploaded", time.Minute)
180
181 work := t.TempDir()
182 env := inst.gitEnv(aliceKey)
183 mustGit(t, work, env, "clone", inst.sshURL("alice/big"), "w")
184 dir := filepath.Join(work, "w")
185 pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", kept, len("referenced payload"))
186 os.WriteFile(filepath.Join(dir, "data.bin"), []byte(pointer), 0o644)
187 mustGit(t, dir, env, "checkout", "-q", "-b", "main")
188 mustGit(t, dir, env, "add", ".")
189 mustGit(t, dir, env, "commit", "-q", "-m", "pointer")
190 mustGit(t, dir, env, "push", "-q", "origin", "main")
191
192 if out := inst.admin(t, "admin", "stats", "--json"); !strings.Contains(out, `"lfs_bytes":`) || strings.Contains(out, `"lfs_bytes":0`) {
193 t.Fatalf("stats lfs bytes:\n%s", out)
194 }
195 out := inst.admin(t, "admin", "gc", "--lfs")
196 if !strings.Contains(out, "lfs\t1 referenced, removed 1 orphans") {
197 t.Fatalf("gc --lfs:\n%s", out)
198 }
199 exists := func(oid string) bool {
200 _, err := os.Stat(filepath.Join(inst.root, "lfs", oid[:2], oid[2:4], oid))
201 return err == nil
202 }
203 if !exists(kept) || exists(orphan) || !exists(young) {
204 t.Fatalf("after gc: kept=%v orphan=%v young=%v", exists(kept), exists(orphan), exists(young))
205 }
206}
207
208// backup --verify reads an archive back and says whether a restore would
209// have what the database expects.
210func TestBackupVerify(t *testing.T) {
211 inst := startInstance(t)
212 aliceKey := inst.newKey(t, "alice")
213 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
214 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/keep"); code != 0 {
215 t.Fatal("repo create failed")
216 }
217 full := filepath.Join(t.TempDir(), "full.tar.gz")
218 inst.admin(t, "admin", "backup", "--out", full)
219 if out := inst.admin(t, "admin", "backup", "--verify", full); !strings.Contains(out, "integrity ok, 1 repositories in the database, 1 in the archive") {
220 t.Fatalf("verify full:\n%s", out)
221 }
222 dbOnly := filepath.Join(t.TempDir(), "db.tar.gz")
223 inst.admin(t, "admin", "backup", "--db-only", "--out", dbOnly)
224 if out := inst.admin(t, "admin", "backup", "--verify", dbOnly); !strings.Contains(out, "database only; integrity ok, 1 repositories in the database") {
225 t.Fatalf("verify db-only:\n%s", out)
226 }
227 // A truncated archive is named as damaged, not reported healthy.
228 raw, _ := os.ReadFile(full)
229 cut := filepath.Join(t.TempDir(), "cut.tar.gz")
230 os.WriteFile(cut, raw[:len(raw)/2], 0o644)
231 if out := inst.forgedAdminErr(t, "admin", "backup", "--verify", cut); !strings.Contains(out, "damaged") && !strings.Contains(out, "unexpected EOF") {
232 t.Fatalf("verify truncated:\n%s", out)
233 }
234 // A repository the database names but the archive lacks fails it.
235 if _, _, code := inst.ssh(t, aliceKey, "", "repo", "create", "alice/late"); code != 0 {
236 t.Fatal("repo create failed")
237 }
238 late := filepath.Join(t.TempDir(), "late.tar.gz")
239 inst.admin(t, "admin", "backup", "--out", late)
240 os.RemoveAll(filepath.Join(inst.root, "repos", "alice", "late.git"))
241 stale := filepath.Join(t.TempDir(), "stale.tar.gz")
242 inst.admin(t, "admin", "backup", "--out", stale)
243 if out := inst.forgedAdminErr(t, "admin", "backup", "--verify", stale); !strings.Contains(out, "not in the archive: alice/late") {
244 t.Fatalf("verify missing repo:\n%s", out)
245 }
246}