e2e/shutdown_test.go

  1package e2e
  2
  3import (
  4	"errors"
  5	"fmt"
  6	"net/http"
  7	"os/exec"
  8	"path/filepath"
  9	"strings"
 10	"syscall"
 11	"testing"
 12	"time"
 13)
 14
 15// SIGTERM is how a deploy restarts the daemon. It used to be a plain
 16// kill: no listener closed, no request or push allowed to finish. The
 17// daemon now stops its listeners, drains, and exits 0 (#105).
 18func TestServeStopsOnSIGTERM(t *testing.T) {
 19	t.Parallel()
 20	inst := startInstance(t)
 21	if resp, err := http.Get(inst.base() + "/healthz"); err != nil || resp.StatusCode != 200 {
 22		t.Fatalf("healthz before shutdown: %v", err)
 23	}
 24	if err := inst.proc.Process.Signal(syscall.SIGTERM); err != nil {
 25		t.Fatal(err)
 26	}
 27	done := make(chan error, 1)
 28	go func() { done <- inst.proc.Wait() }()
 29	select {
 30	case err := <-done:
 31		if err != nil {
 32			t.Fatalf("daemon did not exit cleanly on SIGTERM: %v", err)
 33		}
 34	case <-time.After(15 * time.Second):
 35		t.Fatal("daemon still running 15s after SIGTERM")
 36	}
 37	if _, err := http.Get(inst.base() + "/healthz"); err == nil {
 38		t.Fatal("http listener still answering after shutdown")
 39	}
 40}
 41
 42// A CLI keeps a shared connection open between commands. On shutdown that
 43// idle connection is closed at once rather than holding the drain for its
 44// full 30 s; only a session mid-command is waited for (#141).
 45func TestShutdownClosesIdleConnections(t *testing.T) {
 46	t.Parallel()
 47	inst := startInstance(t)
 48	aliceKey := inst.newKey(t, "alice")
 49	inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
 50
 51	// A control master with no session: -N, backgrounded, persistent.
 52	// A unix socket path has a short limit; t.TempDir is far past it here.
 53	sock := fmt.Sprintf("/tmp/gitbay-e2e-cm-%d", inst.port)
 54	base := []string{"-p", fmt.Sprint(inst.port), "-i", aliceKey, "-o", "IdentitiesOnly=yes",
 55		"-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=" + filepath.Join(inst.sshDir, "known_hosts"),
 56		"-o", "BatchMode=yes", "-o", "ControlPath=" + sock}
 57	master := exec.Command("ssh", append(append([]string{}, base...), "-o", "ControlMaster=yes", "-o", "ControlPersist=yes", "-N", "-f", "git@127.0.0.1")...)
 58	if out, err := master.CombinedOutput(); err != nil {
 59		t.Fatalf("control master: %v\n%s", err, out)
 60	}
 61	t.Cleanup(func() {
 62		exec.Command("ssh", append(append([]string{}, base...), "-O", "exit", "git@127.0.0.1")...).Run()
 63	})
 64	// The shared connection works.
 65	if out, err := exec.Command("ssh", append(append([]string{}, base...), "git@127.0.0.1", "whoami")...).Output(); err != nil || strings.TrimSpace(string(out)) != "alice" {
 66		t.Fatalf("multiplexed whoami: %v %q", err, out)
 67	}
 68
 69	start := time.Now()
 70	if err := inst.proc.Process.Signal(syscall.SIGTERM); err != nil {
 71		t.Fatal(err)
 72	}
 73	done := make(chan error, 1)
 74	go func() { done <- inst.proc.Wait() }()
 75	select {
 76	case err := <-done:
 77		if err != nil {
 78			t.Fatalf("daemon did not exit cleanly: %v", err)
 79		}
 80	case <-time.After(20 * time.Second):
 81		t.Fatal("daemon still running 20s after SIGTERM with only an idle connection open")
 82	}
 83	if took := time.Since(start); took > 5*time.Second {
 84		t.Fatalf("shutdown took %s with only an idle connection open", took)
 85	}
 86}
 87
 88// A deploy restarts the daemon while someone follows a build. The follows
 89// end at once with a message saying so, rather than holding the drain for
 90// its full 30 s and then being cut off mid-page.
 91func TestShutdownEndsFollows(t *testing.T) {
 92	t.Parallel()
 93	inst := startInstance(t)
 94	aliceKey := inst.newKey(t, "alice")
 95	runnerKey := inst.newKey(t, "ci")
 96	inst.admin(t, "admin", "user", "create", "alice", "--key", aliceKey+".pub")
 97	inst.admin(t, "admin", "user", "create", "ci", "--key", runnerKey+".pub", "--admin")
 98	queueBuild(t, inst, aliceKey)
 99	// A line in the log is how the test knows both follows are streaming
100	// before the signal; a follow still connecting is an idle connection,
101	// which shutdown closes without a word.
102	id := claimBuild(t, inst, runnerKey)
103	if _, errOut, code := inst.ssh(t, runnerKey, "started\n", "runner", "log", id); code != 0 {
104		t.Fatalf("runner log: %s", errOut)
105	}
106
107	cmd := inst.sshCmd(aliceKey, "build", "log", "alice/app", "1", "--follow")
108	stdout, err := cmd.StdoutPipe()
109	if err != nil {
110		t.Fatal(err)
111	}
112	var stderr strings.Builder
113	cmd.Stderr = &stderr
114	if err := cmd.Start(); err != nil {
115		t.Fatal(err)
116	}
117	defer func() {
118		if cmd.ProcessState == nil {
119			cmd.Process.Kill()
120			cmd.Wait()
121		}
122	}()
123
124	page, err := http.Get(inst.base() + "/alice/app/builds/1")
125	if err != nil {
126		t.Fatal(err)
127	}
128	defer page.Body.Close()
129	web := newStreamReader(page.Body)
130	web.waitFor(t, "started")
131	newStreamReader(stdout).waitFor(t, "started")
132
133	start := time.Now()
134	if err := inst.proc.Process.Signal(syscall.SIGTERM); err != nil {
135		t.Fatal(err)
136	}
137	web.waitFor(t, `<p class="notice" role="status">gitbay is restarting`)
138	web.waitFor(t, "</html>")
139	var exit *exec.ExitError
140	if err := cmd.Wait(); !errors.As(err, &exit) || exit.ExitCode() != 1 {
141		t.Fatalf("follow ended with %v, want exit 1\n%s", err, stderr.String())
142	}
143	if !strings.Contains(stderr.String(), "gitbay is restarting") {
144		t.Errorf("follow stderr %q", stderr.String())
145	}
146	done := make(chan error, 1)
147	go func() { done <- inst.proc.Wait() }()
148	select {
149	case err := <-done:
150		if err != nil {
151			t.Fatalf("daemon did not exit cleanly: %v", err)
152		}
153	case <-time.After(20 * time.Second):
154		t.Fatal("daemon still running 20s after SIGTERM")
155	}
156	if took := time.Since(start); took > 5*time.Second {
157		t.Fatalf("shutdown took %s with two follows open", took)
158	}
159}