cmd/gitbay-runner/report.go

 1package main
 2
 3import (
 4	"errors"
 5	"fmt"
 6	"os/exec"
 7	"time"
 8)
 9
10// reportDone tells the server a build's outcome, retrying when the
11// server cannot be reached. A finished build's result used to be dropped
12// on the first failure: gitbayd restarting for a deploy at the moment the
13// runner reported left the build "running" on the server for good, with
14// the runner already on to the next one (#179).
15//
16// Only a connection-level failure is retried — ssh exits 255 for those.
17// Any other exit is the server's answer, and asking again would not
18// change it. Four retries over about thirty seconds outlasts a restart.
19func (r *runner) reportDone(id int64, status string) error {
20	return reportWithRetry(func() (string, error) {
21		return r.ssh(nil, "runner", "done", fmt.Sprint(id), status)
22	}, id, retryDelays)
23}
24
25var retryDelays = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second}
26
27func reportWithRetry(report func() (string, error), id int64, delays []time.Duration) error {
28	var out string
29	var err error
30	for attempt := 0; ; attempt++ {
31		out, err = report()
32		if err == nil {
33			return nil
34		}
35		if !connectionFailed(err) || attempt >= len(delays) {
36			return fmt.Errorf("reporting build %d: %w (%s)", id, err, out)
37		}
38		time.Sleep(delays[attempt])
39	}
40}
41
42// connectionFailed reports whether ssh itself failed to reach the server,
43// which is exit status 255, as opposed to the remote command exiting.
44func connectionFailed(err error) bool {
45	var ee *exec.ExitError
46	return errors.As(err, &ee) && ee.ExitCode() == 255
47}