Commit ec2d05573b
ec2d05573b7c9da0a02dc143f0dcaa0f75592d0c
parent: 38bdad0c9f
Verified · cmc ci/build: success ci/test: success
cmc <hello@cleberg.net> · 2026-09-07 04:20 UTC
runner: retry reporting a build's outcome when the server is unreachable
A finished build's result was dropped on the first failed report, so a
gitbayd restart at that moment left the build running on the server for
good. ssh exit 255 is retried four times over about thirty seconds; any
other exit is the server's answer and is not.
Ref #179
Layout: unified · split
cmd/gitbay-runner/main.go
+2 −2
| @@ -184,8 +184,8 @@ func (r *runner) step() (bool, error) { |
| 184 | 184 | if r.run(j) { |
| 185 | 185 | status = "success" |
| 186 | 186 | } |
| 187 | | if out, err := r.ssh(nil, "runner", "done", fmt.Sprint(j.ID), status); err != nil { |
| 188 | | return true, fmt.Errorf("reporting build %d: %w (%s)", j.ID, err, out) |
| 187 | if err := r.reportDone(j.ID, status); err != nil { |
| 188 | return true, err |
| 189 | 189 | } |
| 190 | 190 | log.Printf("build %d: %s", j.ID, status) |
| 191 | 191 | return true, nil |
cmd/gitbay-runner/report.go
added
+47
| @@ -0,0 +1,47 @@ |
| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "errors" |
| 5 | "fmt" |
| 6 | "os/exec" |
| 7 | "time" |
| 8 | ) |
| 9 | |
| 10 | // reportDone tells the server a build's outcome, retrying when the |
| 11 | // server cannot be reached. A finished build's result used to be dropped |
| 12 | // on the first failure: gitbayd restarting for a deploy at the moment the |
| 13 | // runner reported left the build "running" on the server for good, with |
| 14 | // the runner already on to the next one (#179). |
| 15 | // |
| 16 | // Only a connection-level failure is retried — ssh exits 255 for those. |
| 17 | // Any other exit is the server's answer, and asking again would not |
| 18 | // change it. Four retries over about thirty seconds outlasts a restart. |
| 19 | func (r *runner) reportDone(id int64, status string) error { |
| 20 | return reportWithRetry(func() (string, error) { |
| 21 | return r.ssh(nil, "runner", "done", fmt.Sprint(id), status) |
| 22 | }, id, retryDelays) |
| 23 | } |
| 24 | |
| 25 | var retryDelays = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second} |
| 26 | |
| 27 | func reportWithRetry(report func() (string, error), id int64, delays []time.Duration) error { |
| 28 | var out string |
| 29 | var err error |
| 30 | for attempt := 0; ; attempt++ { |
| 31 | out, err = report() |
| 32 | if err == nil { |
| 33 | return nil |
| 34 | } |
| 35 | if !connectionFailed(err) || attempt >= len(delays) { |
| 36 | return fmt.Errorf("reporting build %d: %w (%s)", id, err, out) |
| 37 | } |
| 38 | time.Sleep(delays[attempt]) |
| 39 | } |
| 40 | } |
| 41 | |
| 42 | // connectionFailed reports whether ssh itself failed to reach the server, |
| 43 | // which is exit status 255, as opposed to the remote command exiting. |
| 44 | func connectionFailed(err error) bool { |
| 45 | var ee *exec.ExitError |
| 46 | return errors.As(err, &ee) && ee.ExitCode() == 255 |
| 47 | } |
cmd/gitbay-runner/report_test.go
added
+63
| @@ -0,0 +1,63 @@ |
| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "errors" |
| 5 | "os/exec" |
| 6 | "testing" |
| 7 | "time" |
| 8 | ) |
| 9 | |
| 10 | // exitErr fabricates the error ssh returns for a given exit status. |
| 11 | func exitErr(t *testing.T, code int) error { |
| 12 | t.Helper() |
| 13 | err := exec.Command("sh", "-c", "exit "+itoa(code)).Run() |
| 14 | var ee *exec.ExitError |
| 15 | if !errors.As(err, &ee) || ee.ExitCode() != code { |
| 16 | t.Fatalf("could not fabricate exit %d: %v", code, err) |
| 17 | } |
| 18 | return err |
| 19 | } |
| 20 | |
| 21 | func itoa(n int) string { |
| 22 | return string(rune('0'+n/100)) + string(rune('0'+n/10%10)) + string(rune('0'+n%10)) |
| 23 | } |
| 24 | |
| 25 | // A connection failure (ssh exit 255) is retried until it succeeds; the |
| 26 | // result is not lost to a server that was restarting (#179). |
| 27 | func TestReportRetriesConnectionFailure(t *testing.T) { |
| 28 | calls := 0 |
| 29 | err := reportWithRetry(func() (string, error) { |
| 30 | calls++ |
| 31 | if calls < 3 { |
| 32 | return "ssh: connect to host 127.0.0.1 port 22: Connection refused", exitErr(t, 255) |
| 33 | } |
| 34 | return "", nil |
| 35 | }, 7, []time.Duration{0, 0, 0, 0}) |
| 36 | if err != nil || calls != 3 { |
| 37 | t.Fatalf("err=%v calls=%d, want success on the third attempt", err, calls) |
| 38 | } |
| 39 | } |
| 40 | |
| 41 | // The server's own refusal is an answer, not a transient: no retry. |
| 42 | func TestReportDoesNotRetryServerAnswer(t *testing.T) { |
| 43 | calls := 0 |
| 44 | err := reportWithRetry(func() (string, error) { |
| 45 | calls++ |
| 46 | return `{"error":"build 7 is not running"}`, exitErr(t, 3) |
| 47 | }, 7, []time.Duration{0, 0}) |
| 48 | if err == nil || calls != 1 { |
| 49 | t.Fatalf("err=%v calls=%d, want one attempt and an error", err, calls) |
| 50 | } |
| 51 | } |
| 52 | |
| 53 | // Retries are bounded: after the delays are spent the error surfaces. |
| 54 | func TestReportGivesUp(t *testing.T) { |
| 55 | calls := 0 |
| 56 | err := reportWithRetry(func() (string, error) { |
| 57 | calls++ |
| 58 | return "", exitErr(t, 255) |
| 59 | }, 7, []time.Duration{0, 0}) |
| 60 | if err == nil || calls != 3 { |
| 61 | t.Fatalf("err=%v calls=%d, want three attempts then an error", err, calls) |
| 62 | } |
| 63 | } |