Commit eed104c810
Verified · cmc
Layout: unified · split
cmd/gitbay/main.go +1 −1
| @@ -48,7 +48,7 @@ func newRoot() *cobra.Command { | |||
| 48 | group("build", "CI builds", | 48 | group("build", "CI builds", |
| 49 | pass("list", "recent builds: <owner/name>", passOpts{server: []string{"build", "list"}, needsRepo: true}), | 49 | pass("list", "recent builds: <owner/name>", passOpts{server: []string{"build", "list"}, needsRepo: true}), |
| 50 | pass("show", "one build: <owner/name> <n>", passOpts{server: []string{"build", "show"}, needsRepo: true}), | 50 | pass("show", "one build: <owner/name> <n>", passOpts{server: []string{"build", "show"}, needsRepo: true}), |
| 51 | pass("log", "a build's log: <owner/name> <n>", passOpts{server: []string{"build", "log"}, needsRepo: true}), | 51 | pass("log", "a build's log: <owner/name> <n> [--follow]", passOpts{server: []string{"build", "log"}, needsRepo: true}), |
| 52 | pass("jobs", "list the jobs a trigger can name", passOpts{server: []string{"build", "jobs"}, needsRepo: true}), | 52 | pass("jobs", "list the jobs a trigger can name", passOpts{server: []string{"build", "jobs"}, needsRepo: true}), |
| 53 | pass("trigger", "queue a job now: <job>", passOpts{server: []string{"build", "trigger"}, needsRepo: true}), | 53 | pass("trigger", "queue a job now: <job>", passOpts{server: []string{"build", "trigger"}, needsRepo: true}), |
| 54 | pass("cancel", "withdraw a queued build: <n>", passOpts{server: []string{"build", "cancel"}, needsRepo: true}), | 54 | pass("cancel", "withdraw a queued build: <n>", passOpts{server: []string{"build", "cancel"}, needsRepo: true}), |
internal/control/build.go +10 −3
| @@ -27,8 +27,8 @@ func init() { | |||
| 27 | Summary: "show one build", | 27 | Summary: "show one build", |
| 28 | Usage: "build show <owner/name> <n>", ReadOnly: true, Run: runBuildShow}) | 28 | Usage: "build show <owner/name> <n>", ReadOnly: true, Run: runBuildShow}) |
| 29 | register(Command{Path: []string{"build", "log"}, | 29 | register(Command{Path: []string{"build", "log"}, |
| 30 | Summary: "print a build's log", | 30 | Summary: "print a build's log, or follow it until the build ends", |
| 31 | Usage: "build log <owner/name> <n>", ReadOnly: true, Run: runBuildLog}) | 31 | Usage: "build log <owner/name> <n> [--follow]", ReadOnly: true, Run: runBuildLog}) |
| 32 | 32 | ||
| 33 | register(Command{Path: []string{"build", "jobs"}, | 33 | register(Command{Path: []string{"build", "jobs"}, |
| 34 | Summary: "list the jobs a trigger can name", | 34 | Summary: "list the jobs a trigger can name", |
| @@ -196,10 +196,17 @@ func runBuildShow(c *Ctx, args []string) int { | |||
| 196 | } | 196 | } |
| 197 | 197 | ||
| 198 | func runBuildLog(c *Ctx, args []string) int { | 198 | func runBuildLog(c *Ctx, args []string) int { |
| 199 | _, b, code := buildRef(c, args) | 199 | f, err := parseFlags(args, flagSpec{Bools: []string{"--follow"}, MaxPos: 2, Usage: c.Cmd.Usage}) |
| 200 | if err != nil { | ||
| 201 | return c.fail(protocol.ExitUsage, "%v", err) | ||
| 202 | } | ||
| 203 | _, b, code := buildRef(c, f.Pos) | ||
| 200 | if code >= 0 { | 204 | if code >= 0 { |
| 201 | return code | 205 | return code |
| 202 | } | 206 | } |
| 207 | if f.Has("--follow") { | ||
| 208 | return followBuildLog(c, b) | ||
| 209 | } | ||
| 203 | log, err := c.Store.BuildLog(b.ID) | 210 | log, err := c.Store.BuildLog(b.ID) |
| 204 | if err != nil { | 211 | if err != nil { |
| 205 | return c.fail(protocol.ExitFailure, "%v", err) | 212 | return c.fail(protocol.ExitFailure, "%v", err) |
internal/control/build_test.go +5 −1
| @@ -42,9 +42,13 @@ func gitRunner(t *testing.T) func(dir string, args ...string) string { | |||
| 42 | 42 | ||
| 43 | // newQueueTestRepo returns a store with one public repo (default branch | 43 | // newQueueTestRepo returns a store with one public repo (default branch |
| 44 | // "main", matching the schema default) and the uid to queue builds as. | 44 | // "main", matching the schema default) and the uid to queue builds as. |
| 45 | // | ||
| 46 | // A real file, not ":memory:": ":memory:" gives each connection its own | ||
| 47 | // database, so a goroutine querying while another writes (build log | ||
| 48 | // --follow) sees an empty schema (see internal/store/contention_test.go). | ||
| 45 | func newQueueTestRepo(t *testing.T) (*store.Store, store.Repo, int64) { | 49 | func newQueueTestRepo(t *testing.T) (*store.Store, store.Repo, int64) { |
| 46 | t.Helper() | 50 | t.Helper() |
| 47 | st, err := store.Open(":memory:") | 51 | st, err := store.Open(filepath.Join(t.TempDir(), "gitbay.db")) |
| 48 | if err != nil { | 52 | if err != nil { |
| 49 | t.Fatal(err) | 53 | t.Fatal(err) |
| 50 | } | 54 | } |
internal/control/buildfollow.go added +95
| @@ -0,0 +1,95 @@ | |||
| 1 | package control | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "fmt" | ||
| 5 | "sync" | ||
| 6 | "time" | ||
| 7 | |||
| 8 | "gitbay.org/gitbay/internal/protocol" | ||
| 9 | "gitbay.org/gitbay/internal/store" | ||
| 10 | ) | ||
| 11 | |||
| 12 | // maxFollows is how many build log follows one account holds open at | ||
| 13 | // once. Signed-out web viewers are account 0 and share it. | ||
| 14 | const maxFollows = 8 | ||
| 15 | |||
| 16 | var ( | ||
| 17 | // followPoll bounds a wait with no wake. A write from another process | ||
| 18 | // (gitbayd admin, or any session under gitbayd shell) wakes nobody; | ||
| 19 | // this is how its bytes still arrive. | ||
| 20 | followPoll = 2 * time.Second | ||
| 21 | // followSettle is how long a follow keeps reading after the build has | ||
| 22 | // an outcome: a cancel appends its line after the status changes, and | ||
| 23 | // a cancelled runner's stream runs on until its next check. | ||
| 24 | followSettle = time.Second | ||
| 25 | ) | ||
| 26 | |||
| 27 | var ( | ||
| 28 | followMu sync.Mutex | ||
| 29 | follows = map[int64]int{} | ||
| 30 | ) | ||
| 31 | |||
| 32 | func takeFollow(uid int64) bool { | ||
| 33 | followMu.Lock() | ||
| 34 | defer followMu.Unlock() | ||
| 35 | if follows[uid] >= maxFollows { | ||
| 36 | return false | ||
| 37 | } | ||
| 38 | follows[uid]++ | ||
| 39 | return true | ||
| 40 | } | ||
| 41 | |||
| 42 | func dropFollow(uid int64) { | ||
| 43 | followMu.Lock() | ||
| 44 | defer followMu.Unlock() | ||
| 45 | if follows[uid]--; follows[uid] <= 0 { | ||
| 46 | delete(follows, uid) | ||
| 47 | } | ||
| 48 | } | ||
| 49 | |||
| 50 | // followBuildLog writes the build's log as it grows and returns once the | ||
| 51 | // build has an outcome and its last bytes are written. The outcome goes | ||
| 52 | // to stderr, so stdout is the log byte for byte. | ||
| 53 | func followBuildLog(c *Ctx, b store.Build) int { | ||
| 54 | if !takeFollow(c.User.ID) { | ||
| 55 | return c.fail(protocol.ExitDenied, "%d follows are already open for this account; close one and retry", maxFollows) | ||
| 56 | } | ||
| 57 | defer dropFollow(c.User.ID) | ||
| 58 | |||
| 59 | var off int64 | ||
| 60 | var settleBy time.Time | ||
| 61 | for { | ||
| 62 | wake := c.Store.BuildLogWait(b.ID) | ||
| 63 | status, chunk, err := c.Store.BuildLogFrom(b.ID, off) | ||
| 64 | if err != nil { | ||
| 65 | return c.fail(protocol.ExitFailure, "%v", err) | ||
| 66 | } | ||
| 67 | if len(chunk) > 0 { | ||
| 68 | if _, err := c.Stdout.Write(chunk); err != nil { | ||
| 69 | return protocol.ExitFailure | ||
| 70 | } | ||
| 71 | off += int64(len(chunk)) | ||
| 72 | } | ||
| 73 | wait := followPoll | ||
| 74 | if status != "pending" && status != "running" { | ||
| 75 | if settleBy.IsZero() { | ||
| 76 | settleBy = time.Now().Add(followSettle) | ||
| 77 | } | ||
| 78 | left := time.Until(settleBy) | ||
| 79 | if left <= 0 && len(chunk) == 0 { | ||
| 80 | fmt.Fprintf(c.Stderr, "build %d %s\n", b.Number, status) | ||
| 81 | return protocol.ExitOK | ||
| 82 | } | ||
| 83 | wait = min(wait, max(left, 0)) | ||
| 84 | } | ||
| 85 | t := time.NewTimer(wait) | ||
| 86 | select { | ||
| 87 | case <-wake: | ||
| 88 | case <-t.C: | ||
| 89 | case <-c.Done: | ||
| 90 | t.Stop() | ||
| 91 | return protocol.ExitFailure | ||
| 92 | } | ||
| 93 | t.Stop() | ||
| 94 | } | ||
| 95 | } | ||
internal/control/buildfollow_test.go added +138
| @@ -0,0 +1,138 @@ | |||
| 1 | package control | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "bytes" | ||
| 5 | "strings" | ||
| 6 | "testing" | ||
| 7 | "time" | ||
| 8 | |||
| 9 | "gitbay.org/gitbay/internal/protocol" | ||
| 10 | "gitbay.org/gitbay/internal/store" | ||
| 11 | ) | ||
| 12 | |||
| 13 | // follow starts build log --follow on build 1 of repo and returns the | ||
| 14 | // buffers and a channel carrying the exit code. | ||
| 15 | func follow(t *testing.T, st *store.Store, uid int64, repo store.Repo, done <-chan struct{}) (*bytes.Buffer, *bytes.Buffer, chan int) { | ||
| 16 | t.Helper() | ||
| 17 | u, err := st.UserByID(uid) | ||
| 18 | if err != nil { | ||
| 19 | t.Fatal(err) | ||
| 20 | } | ||
| 21 | var out, errOut bytes.Buffer | ||
| 22 | c := &Ctx{User: u, Scope: "full", Store: st, Stdin: strings.NewReader(""), | ||
| 23 | Stdout: &out, Stderr: &errOut, Done: done} | ||
| 24 | res := make(chan int, 1) | ||
| 25 | go func() { res <- Dispatch(c, []string{"build", "log", repo.Path(), "1", "--follow"}) }() | ||
| 26 | return &out, &errOut, res | ||
| 27 | } | ||
| 28 | |||
| 29 | func waitExit(t *testing.T, res chan int) int { | ||
| 30 | t.Helper() | ||
| 31 | select { | ||
| 32 | case code := <-res: | ||
| 33 | return code | ||
| 34 | case <-time.After(10 * time.Second): | ||
| 35 | t.Fatal("follow did not end") | ||
| 36 | return -1 | ||
| 37 | } | ||
| 38 | } | ||
| 39 | |||
| 40 | func shortFollowTimers(t *testing.T) { | ||
| 41 | settle, poll := followSettle, followPoll | ||
| 42 | followSettle, followPoll = 200*time.Millisecond, 50*time.Millisecond | ||
| 43 | t.Cleanup(func() { followSettle, followPoll = settle, poll }) | ||
| 44 | } | ||
| 45 | |||
| 46 | // The follow prints the stored log, then what arrives, and ends with the | ||
| 47 | // outcome on stderr once the build finishes. | ||
| 48 | func TestBuildLogFollow(t *testing.T) { | ||
| 49 | shortFollowTimers(t) | ||
| 50 | st, repo, uid := newQueueTestRepo(t) | ||
| 51 | id, err := st.CreateBuild(repo.ID, "unit", "abc", "main", `["true"]`, "", "", true) | ||
| 52 | if err != nil { | ||
| 53 | t.Fatal(err) | ||
| 54 | } | ||
| 55 | st.AppendBuildLog(id, []byte("queued\n")) | ||
| 56 | out, errOut, res := follow(t, st, uid, repo, nil) | ||
| 57 | |||
| 58 | if _, ok, err := st.ClaimBuild([]int64{repo.ID}, false); err != nil || !ok { | ||
| 59 | t.Fatalf("claim: %v %v", ok, err) | ||
| 60 | } | ||
| 61 | st.AppendBuildLog(id, []byte("step one\n")) | ||
| 62 | st.AppendBuildLog(id, []byte("step two\n")) | ||
| 63 | if err := st.FinishBuild(id, "success"); err != nil { | ||
| 64 | t.Fatal(err) | ||
| 65 | } | ||
| 66 | if code := waitExit(t, res); code != protocol.ExitOK { | ||
| 67 | t.Fatalf("exit %d: %s", code, errOut) | ||
| 68 | } | ||
| 69 | if got := out.String(); got != "queued\nstep one\nstep two\n" { | ||
| 70 | t.Errorf("stdout %q", got) | ||
| 71 | } | ||
| 72 | if got := strings.TrimSpace(errOut.String()); got != "build 1 success" { | ||
| 73 | t.Errorf("stderr %q", got) | ||
| 74 | } | ||
| 75 | } | ||
| 76 | |||
| 77 | // A cancel ends the follow, and the line the cancel appends after the | ||
| 78 | // status change still arrives. | ||
| 79 | func TestBuildLogFollowCancel(t *testing.T) { | ||
| 80 | shortFollowTimers(t) | ||
| 81 | st, repo, uid := newQueueTestRepo(t) | ||
| 82 | id, err := st.CreateBuild(repo.ID, "unit", "abc", "main", `["true"]`, "", "", true) | ||
| 83 | if err != nil { | ||
| 84 | t.Fatal(err) | ||
| 85 | } | ||
| 86 | out, errOut, res := follow(t, st, uid, repo, nil) | ||
| 87 | if err := st.CancelBuild(id); err != nil { | ||
| 88 | t.Fatal(err) | ||
| 89 | } | ||
| 90 | st.AppendBuildLog(id, []byte("cancelled by alice before a runner claimed it\n")) | ||
| 91 | if code := waitExit(t, res); code != protocol.ExitOK { | ||
| 92 | t.Fatalf("exit %d: %s", code, errOut) | ||
| 93 | } | ||
| 94 | if !strings.Contains(out.String(), "cancelled by alice") { | ||
| 95 | t.Errorf("the cancel line did not arrive: %q", out) | ||
| 96 | } | ||
| 97 | if got := strings.TrimSpace(errOut.String()); got != "build 1 cancelled" { | ||
| 98 | t.Errorf("stderr %q", got) | ||
| 99 | } | ||
| 100 | } | ||
| 101 | |||
| 102 | // Closing Done ends a follow of a build that is still running. | ||
| 103 | func TestBuildLogFollowDone(t *testing.T) { | ||
| 104 | shortFollowTimers(t) | ||
| 105 | st, repo, uid := newQueueTestRepo(t) | ||
| 106 | if _, err := st.CreateBuild(repo.ID, "unit", "abc", "main", `["true"]`, "", "", true); err != nil { | ||
| 107 | t.Fatal(err) | ||
| 108 | } | ||
| 109 | done := make(chan struct{}) | ||
| 110 | _, _, res := follow(t, st, uid, repo, done) | ||
| 111 | close(done) | ||
| 112 | if code := waitExit(t, res); code != protocol.ExitFailure { | ||
| 113 | t.Fatalf("exit %d, want %d", code, protocol.ExitFailure) | ||
| 114 | } | ||
| 115 | } | ||
| 116 | |||
| 117 | // An account holding maxFollows is refused another. | ||
| 118 | func TestBuildLogFollowCap(t *testing.T) { | ||
| 119 | st, repo, uid := newQueueTestRepo(t) | ||
| 120 | if _, err := st.CreateBuild(repo.ID, "unit", "abc", "main", `["true"]`, "", "", true); err != nil { | ||
| 121 | t.Fatal(err) | ||
| 122 | } | ||
| 123 | followMu.Lock() | ||
| 124 | follows[uid] = maxFollows | ||
| 125 | followMu.Unlock() | ||
| 126 | t.Cleanup(func() { | ||
| 127 | followMu.Lock() | ||
| 128 | delete(follows, uid) | ||
| 129 | followMu.Unlock() | ||
| 130 | }) | ||
| 131 | _, errOut, res := follow(t, st, uid, repo, nil) | ||
| 132 | if code := waitExit(t, res); code != protocol.ExitDenied { | ||
| 133 | t.Fatalf("exit %d, want %d", code, protocol.ExitDenied) | ||
| 134 | } | ||
| 135 | if !strings.Contains(errOut.String(), "8 follows are already open") { | ||
| 136 | t.Errorf("stderr %q", errOut) | ||
| 137 | } | ||
| 138 | } | ||
internal/control/control.go +4
| @@ -40,6 +40,10 @@ type Ctx struct { | |||
| 40 | // Cmd is the command being run, set by Dispatch, so a usage error can | 40 | // Cmd is the command being run, set by Dispatch, so a usage error can |
| 41 | // print the registered usage rather than a copy of it. | 41 | // print the registered usage rather than a copy of it. |
| 42 | Cmd Command | 42 | Cmd Command |
| 43 | // Done, when the surface has one, closes when nobody is reading any | ||
| 44 | // more: the SSH channel closed or the HTTP request ended. A command | ||
| 45 | // that runs until something happens (build log --follow) stops on it. | ||
| 46 | Done <-chan struct{} | ||
| 43 | } | 47 | } |
| 44 | 48 | ||
| 45 | // usage reports a bad invocation with the command's registered usage, | 49 | // usage reports a bad invocation with the command's registered usage, |