Commit 65f453dc9b
65f453dc9bd4da2eb9fee17025aa6b6e2d0160ae
parent: 631ccdc137
Verified · cmc ci/build: success ci/test: success
cmc <hello@cleberg.net> · 2026-09-07 05:06 UTC
runner, deploy, wiki: drain on SIGTERM
The runner stops claiming on the first signal, finishes and reports the
build in flight, then exits; the unit gives it 50 minutes. A second
signal exits at once. A restart no longer leaves a build running on the
server with nothing executing.
Closes #179
Layout: unified · split
.gitbay/wiki/Admin.org
+8
| @@ -476,6 +476,14 @@ cannot call =newuidmap= and the runner refuses to start. That is a |
| 476 | 476 | considered trade, explained in the file and in the Threat-Model; if you |
| 477 | 477 | run with =-isolation none=, set it back to =yes=. |
| 478 | 478 | |
| 479 | *Restarting the runner is safe.* On SIGTERM it stops claiming, finishes |
| 480 | the build in flight, reports it, and exits; the drop-in's |
| 481 | =TimeoutStopSec=50min= covers the longest build. So =make deploy-runner= |
| 482 | waits for a running build rather than orphaning it, and a build's result |
| 483 | is retried for half a minute if gitbayd is restarting at that moment. A |
| 484 | second SIGTERM ends the runner at once, abandoning the build to the |
| 485 | reaper. |
| 486 | |
| 479 | 487 | *Validate podman mode on a scratch repository before pointing the runner |
| 480 | 488 | at real ones.* Every deploy that switched the whole instance to |
| 481 | 489 | containers and failed took CI down with it. Instead: create a throwaway |
cmd/gitbay-runner/env_test.go
+46
| @@ -4,6 +4,7 @@ import ( |
| 4 | 4 | "os" |
| 5 | 5 | "strings" |
| 6 | 6 | "testing" |
| 7 | "time" |
| 7 | 8 | ) |
| 8 | 9 | |
| 9 | 10 | // A step's environment is constructed, not inherited: repository content |
| @@ -129,3 +130,48 @@ func TestLimitArgs(t *testing.T) { |
| 129 | 130 | t.Errorf("limitArgs = %v, want %v", got, want) |
| 130 | 131 | } |
| 131 | 132 | } |
| 133 | |
| 134 | // Closing stop drains: the build in flight finishes and is reported, and |
| 135 | // no further build is claimed (#179). |
| 136 | func TestServeDrainsOnStop(t *testing.T) { |
| 137 | stop := make(chan struct{}) |
| 138 | started := make(chan struct{}) |
| 139 | release := make(chan struct{}) |
| 140 | calls := 0 |
| 141 | r := &runner{stepFn: func() (bool, error) { |
| 142 | calls++ |
| 143 | if calls == 1 { |
| 144 | close(started) |
| 145 | <-release // the build is in flight while stop closes |
| 146 | } |
| 147 | return true, nil |
| 148 | }} |
| 149 | done := make(chan struct{}) |
| 150 | go func() { r.serve(1, false, time.Millisecond, stop); close(done) }() |
| 151 | <-started |
| 152 | close(stop) |
| 153 | close(release) |
| 154 | select { |
| 155 | case <-done: |
| 156 | case <-time.After(2 * time.Second): |
| 157 | t.Fatal("serve did not return after the in-flight build finished") |
| 158 | } |
| 159 | if calls != 1 { |
| 160 | t.Errorf("claimed %d builds after stop, want the one already in flight", calls-1) |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | // An idle worker leaves promptly on stop rather than sleeping out a poll. |
| 165 | func TestServeStopsWhileIdle(t *testing.T) { |
| 166 | stop := make(chan struct{}) |
| 167 | r := &runner{stepFn: func() (bool, error) { return false, nil }} |
| 168 | done := make(chan struct{}) |
| 169 | go func() { r.serve(1, false, time.Hour, stop); close(done) }() |
| 170 | time.Sleep(20 * time.Millisecond) |
| 171 | close(stop) |
| 172 | select { |
| 173 | case <-done: |
| 174 | case <-time.After(2 * time.Second): |
| 175 | t.Fatal("idle worker did not stop") |
| 176 | } |
| 177 | } |
cmd/gitbay-runner/main.go
+41 −8
| @@ -17,6 +17,7 @@ import ( |
| 17 | 17 | "log" |
| 18 | 18 | "os" |
| 19 | 19 | "os/exec" |
| 20 | "os/signal" |
| 20 | 21 | "path/filepath" |
| 21 | 22 | "strings" |
| 22 | 23 | "sync" |
| @@ -52,6 +53,8 @@ type runner struct { |
| 52 | 53 | // memory and cpus cap one build's container; empty means no cap. |
| 53 | 54 | memory string |
| 54 | 55 | cpus string |
| 56 | // stepFn is step, replaceable by tests. |
| 57 | stepFn func() (bool, error) |
| 55 | 58 | // repos limits which repositories this runner claims builds for. Empty |
| 56 | 59 | // means any, which is what a runner on the server itself wants; a runner |
| 57 | 60 | // somewhere that should not execute every repository's steps names them. |
| @@ -133,28 +136,58 @@ func main() { |
| 133 | 136 | // claiming at once is already safe; the runner just never used that. |
| 134 | 137 | // Each build works in its own build-<id> directory, so they do not |
| 135 | 138 | // meet on disk either. |
| 139 | // A stop signal drains: no build is claimed after it, and each build |
| 140 | // already in flight runs to completion and is reported. The old |
| 141 | // behaviour was to die mid-build, which left the build "running" on |
| 142 | // the server with nothing executing (#179). The unit's |
| 143 | // TimeoutStopSec bounds the drain; a second signal ends it now. |
| 144 | stop := make(chan struct{}) |
| 145 | go func() { |
| 146 | sigs := make(chan os.Signal, 2) |
| 147 | signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT) |
| 148 | <-sigs |
| 149 | log.Printf("draining: finishing builds in flight, claiming no more") |
| 150 | close(stop) |
| 151 | <-sigs |
| 152 | log.Printf("second signal: exiting without draining") |
| 153 | os.Exit(1) |
| 154 | }() |
| 155 | r.serve(n, *once, *poll, stop) |
| 156 | } |
| 157 | |
| 158 | // serve runs n workers until stop closes. A worker checks stop only |
| 159 | // between builds, so closing it never interrupts one. |
| 160 | func (r *runner) serve(n int, once bool, poll time.Duration, stop <-chan struct{}) { |
| 161 | if r.stepFn == nil { |
| 162 | r.stepFn = r.step |
| 163 | } |
| 136 | 164 | var wg sync.WaitGroup |
| 137 | 165 | for i := 0; i < n; i++ { |
| 138 | 166 | wg.Add(1) |
| 139 | 167 | go func(i int) { |
| 140 | 168 | defer wg.Done() |
| 141 | | // Spread the idle polls across the interval rather than |
| 142 | | // having every worker wake together: n workers asking the |
| 143 | | // same question in the same instant is n times the load for |
| 144 | | // one answer. |
| 145 | 169 | if n > 1 { |
| 146 | | time.Sleep(time.Duration(i) * *poll / time.Duration(n)) |
| 170 | time.Sleep(time.Duration(i) * poll / time.Duration(n)) |
| 147 | 171 | } |
| 148 | 172 | for { |
| 149 | | ran, err := r.step() |
| 173 | select { |
| 174 | case <-stop: |
| 175 | return |
| 176 | default: |
| 177 | } |
| 178 | ran, err := r.stepFn() |
| 150 | 179 | if err != nil { |
| 151 | 180 | log.Printf("runner: %v", err) |
| 152 | 181 | } |
| 153 | | if *once { |
| 182 | if once { |
| 154 | 183 | return |
| 155 | 184 | } |
| 156 | 185 | if !ran { |
| 157 | | time.Sleep(*poll) |
| 186 | select { |
| 187 | case <-stop: |
| 188 | return |
| 189 | case <-time.After(poll): |
| 190 | } |
| 158 | 191 | } |
| 159 | 192 | } |
| 160 | 193 | }(i) |
deploy/gitbay-runner.override.conf
+5
| @@ -44,6 +44,11 @@ |
| 44 | 44 | # start joins its namespaces — including a /tmp that no longer exists. |
| 45 | 45 | # End it with the service. |
| 46 | 46 | ExecStopPost=-/usr/bin/pkill -u ci-runner -x catatonit |
| 47 | # On stop the runner drains: it claims nothing more and finishes the |
| 48 | # build in flight, then exits. Give it long enough — the per-build limit |
| 49 | # is 45m plus half a minute of report retries — before systemd kills it. |
| 50 | # `make deploy-runner` therefore waits for a running build (#179). |
| 51 | TimeoutStopSec=50min |
| 47 | 52 | ExecStart= |
| 48 | 53 | ExecStart=/usr/local/bin/gitbay-runner -remote git@127.0.0.1 -workdir /var/lib/gitbay-runner/work -poll 5s -timeout 45m -repos krz/gitbay,cmc/ci-smoke -isolation podman -image localhost/gitbay-ci:1 -cpus 3 |
| 49 | 54 | Nice=10 |