cmd/gitbay-runner/report.go

 1package main
 2
 3import (
 4	"errors"
 5	"fmt"
 6	"os/exec"
 7	"strconv"
 8	"strings"
 9	"time"
10)
11
12// reportDone tells the server a build's outcome, retrying when the
13// server cannot be reached. A finished build's result used to be dropped
14// on the first failure: gitbayd restarting for a deploy at the moment the
15// runner reported left the build "running" on the server for good, with
16// the runner already on to the next one (#179).
17//
18// Only a connection-level failure is retried — ssh exits 255 for those.
19// Any other exit is the server's answer, and asking again would not
20// change it. Four retries over about thirty seconds outlasts a restart.
21func (r *runner) reportDone(id int64, status string, f *failure) error {
22	args := doneArgs(id, status, f)
23	return reportWithRetry(func() (string, error) {
24		return r.ssh(nil, args...)
25	}, id, retryDelays)
26}
27
28// doneArgs is the runner done command for a build's outcome. The reason
29// is single-quoted: ssh joins arguments with spaces, and the server
30// splits the line again with POSIX rules.
31func doneArgs(id int64, status string, f *failure) []string {
32	args := []string{"runner", "done", fmt.Sprint(id), status}
33	if f == nil {
34		return args
35	}
36	if f.Step > 0 {
37		args = append(args, "--step", strconv.Itoa(f.Step))
38	}
39	if f.Reason != "" {
40		args = append(args, "--reason", "'"+strings.ReplaceAll(f.Reason, "'", `'\''`)+"'")
41	}
42	return args
43}
44
45var retryDelays = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second}
46
47func reportWithRetry(report func() (string, error), id int64, delays []time.Duration) error {
48	var out string
49	var err error
50	for attempt := 0; ; attempt++ {
51		out, err = report()
52		if err == nil {
53			return nil
54		}
55		if !connectionFailed(err) || attempt >= len(delays) {
56			return fmt.Errorf("reporting build %d: %w (%s)", id, err, out)
57		}
58		time.Sleep(delays[attempt])
59	}
60}
61
62// connectionFailed reports whether ssh itself failed to reach the server,
63// which is exit status 255, as opposed to the remote command exiting.
64func connectionFailed(err error) bool {
65	var ee *exec.ExitError
66	return errors.As(err, &ee) && ee.ExitCode() == 255
67}