e2e/shutdown_test.go
159 lines · 5464 bytes
1package e2e
2
3import (
4 "errors"
5 "fmt"
6 "net/http"
7 "os/exec"
8 "path/filepath"
9 "strings"
10 "syscall"
11 "testing"
12 "time"
13)
14
15// SIGTERM is how a deploy restarts the daemon. It used to be a plain
16// kill: no listener closed, no request or push allowed to finish. The
17// daemon now stops its listeners, drains, and exits 0 (#105).
18func TestServeStopsOnSIGTERM(t *testing.T) {
19 t.Parallel()
20 inst := startInstance(t)
21 if resp, err := http.Get(inst.base() + "/healthz"); err != nil || resp.StatusCode != 200 {
22 t.Fatalf("healthz before shutdown: %v", err)
23 }
24 if err := inst.proc.Process.Signal(syscall.SIGTERM); err != nil {
25 t.Fatal(err)
26 }
27 done := make(chan error, 1)
28 go func() { done <- inst.proc.Wait() }()
29 select {
30 case err := <-done:
31 if err != nil {
32 t.Fatalf("daemon did not exit cleanly on SIGTERM: %v", err)
33 }
34 case <-time.After(15 * time.Second):
35 t.Fatal("daemon still running 15s after SIGTERM")
36 }
37 if _, err := http.Get(inst.base() + "/healthz"); err == nil {
38 t.Fatal("http listener still answering after shutdown")
39 }
40}
41
42// A CLI keeps a shared connection open between commands. On shutdown that
43// idle connection is closed at once rather than holding the drain for its
44// full 30 s; only a session mid-command is waited for (#141).
45func TestShutdownClosesIdleConnections(t *testing.T) {
46 t.Parallel()
47 inst := startInstance(t)
48 aliceKey := inst.newKey(t, "alice")
49 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
50
51 // A control master with no session: -N, backgrounded, persistent.
52 // A unix socket path has a short limit; t.TempDir is far past it here.
53 sock := fmt.Sprintf("/tmp/gitbay-e2e-cm-%d", inst.port)
54 base := []string{"-p", fmt.Sprint(inst.port), "-i", aliceKey, "-o", "IdentitiesOnly=yes",
55 "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=" + filepath.Join(inst.sshDir, "known_hosts"),
56 "-o", "BatchMode=yes", "-o", "ControlPath=" + sock}
57 master := exec.Command("ssh", append(append([]string{}, base...), "-o", "ControlMaster=yes", "-o", "ControlPersist=yes", "-N", "-f", "git@127.0.0.1")...)
58 if out, err := master.CombinedOutput(); err != nil {
59 t.Fatalf("control master: %v\n%s", err, out)
60 }
61 t.Cleanup(func() {
62 exec.Command("ssh", append(append([]string{}, base...), "-O", "exit", "git@127.0.0.1")...).Run()
63 })
64 // The shared connection works.
65 if out, err := exec.Command("ssh", append(append([]string{}, base...), "git@127.0.0.1", "whoami")...).Output(); err != nil || strings.TrimSpace(string(out)) != "alice" {
66 t.Fatalf("multiplexed whoami: %v %q", err, out)
67 }
68
69 start := time.Now()
70 if err := inst.proc.Process.Signal(syscall.SIGTERM); err != nil {
71 t.Fatal(err)
72 }
73 done := make(chan error, 1)
74 go func() { done <- inst.proc.Wait() }()
75 select {
76 case err := <-done:
77 if err != nil {
78 t.Fatalf("daemon did not exit cleanly: %v", err)
79 }
80 case <-time.After(20 * time.Second):
81 t.Fatal("daemon still running 20s after SIGTERM with only an idle connection open")
82 }
83 if took := time.Since(start); took > 5*time.Second {
84 t.Fatalf("shutdown took %s with only an idle connection open", took)
85 }
86}
87
88// A deploy restarts the daemon while someone follows a build. The follows
89// end at once with a message saying so, rather than holding the drain for
90// its full 30 s and then being cut off mid-page.
91func TestShutdownEndsFollows(t *testing.T) {
92 t.Parallel()
93 inst := startInstance(t)
94 aliceKey := inst.newKey(t, "alice")
95 runnerKey := inst.newKey(t, "ci")
96 inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
97 inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
98 queueBuild(t, inst, aliceKey)
99 // A line in the log is how the test knows both follows are streaming
100 // before the signal; a follow still connecting is an idle connection,
101 // which shutdown closes without a word.
102 id := claimBuild(t, inst, runnerKey)
103 if _, errOut, code := inst.ssh(t, runnerKey, "started\n", "runner", "log", id); code != 0 {
104 t.Fatalf("runner log: %s", errOut)
105 }
106
107 cmd := inst.sshCmd(aliceKey, "build", "log", "alice/app", "1", "--follow")
108 stdout, err := cmd.StdoutPipe()
109 if err != nil {
110 t.Fatal(err)
111 }
112 var stderr strings.Builder
113 cmd.Stderr = &stderr
114 if err := cmd.Start(); err != nil {
115 t.Fatal(err)
116 }
117 defer func() {
118 if cmd.ProcessState == nil {
119 cmd.Process.Kill()
120 cmd.Wait()
121 }
122 }()
123
124 page, err := http.Get(inst.base() + "/alice/app/builds/1")
125 if err != nil {
126 t.Fatal(err)
127 }
128 defer page.Body.Close()
129 web := newStreamReader(page.Body)
130 web.waitFor(t, "started")
131 newStreamReader(stdout).waitFor(t, "started")
132
133 start := time.Now()
134 if err := inst.proc.Process.Signal(syscall.SIGTERM); err != nil {
135 t.Fatal(err)
136 }
137 web.waitFor(t, `<p class="notice" role="status">gitbay is restarting`)
138 web.waitFor(t, "</html>")
139 var exit *exec.ExitError
140 if err := cmd.Wait(); !errors.As(err, &exit) || exit.ExitCode() != 1 {
141 t.Fatalf("follow ended with %v, want exit 1\n%s", err, stderr.String())
142 }
143 if !strings.Contains(stderr.String(), "gitbay is restarting") {
144 t.Errorf("follow stderr %q", stderr.String())
145 }
146 done := make(chan error, 1)
147 go func() { done <- inst.proc.Wait() }()
148 select {
149 case err := <-done:
150 if err != nil {
151 t.Fatalf("daemon did not exit cleanly: %v", err)
152 }
153 case <-time.After(20 * time.Second):
154 t.Fatal("daemon still running 20s after SIGTERM")
155 }
156 if took := time.Since(start); took > 5*time.Second {
157 t.Fatalf("shutdown took %s with two follows open", took)
158 }
159}