Commit `579da6b329`

579da6b32974f51e1ce5b475529a573e18fddf27

parent: 5b0b085f73

Verified · cmc ci/build: error ci/test: error ci/vuln: error

cmc <hello@cleberg.net> · 2026-09-02 04:21 UTC

runner: every phase of a build is cancellable

Cancel reached the runner only in the step loop. The clone and checkout
ran to completion first, so on a loaded host a build cancelled during
its clone kept going until git finished, and the e2e test that gives
the runner twenty seconds failed on the CI host both times while
passing locally in two.

One runStep drives the clone, the checkout and each step: start, then
wait for the process, the cancel signal, or the deadline, and check the
signal before starting at all. The runner also logs how its log session
ended when that was not a cancel, so a lost stream is named. The test's
budget grows to a minute, still well inside the step's two-minute
sleep, so the assertion stays that the runner stopped because it was
told to.

Ref #90

Layout: unified · split

cmd/gitbay-runner/main.go +42 −26

 	logExited := make(chan struct{})
 	go func() {
 		defer close(logExited)
-		if err := logCmd.Wait(); err != nil {
+		err := logCmd.Wait()
-			if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() == 3 {
+		if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() == 3 {
-				close(cancelled)
+			close(cancelled)
-			}
+			return
+		}
+		if err != nil {
+			log.Printf("build %d: log session ended: %v", j.ID, err)
 		}
 	}()
+	// runStep starts cmd and waits for it, the cancel signal, or the
+	// deadline. Every phase goes through it, so a cancel during the clone
+	// lands as fast as one during a step.
+	runStep := func(cmd *exec.Cmd, deadline time.Time) (bool, string) {
+		select {
+		case <-cancelled:
+			return false, "cancelled"
+		default:
+		}
+		if err := cmd.Start(); err != nil {
+			return false, fmt.Sprintf("start: %v", err)
+		}
+		done := make(chan error, 1)
+		go func() { done <- cmd.Wait() }()
+		select {
+		case err := <-done:
+			if err != nil {
+				return false, fmt.Sprintf("step failed: %v", err)
+			}
+			return true, ""
+		case <-cancelled:
+			cmd.Process.Kill()
+			<-done
+			return false, "cancelled"
+		case <-time.After(time.Until(deadline)):
+			cmd.Process.Kill()
+			<-done
+			return false, fmt.Sprintf("build timed out after %s", r.timeout)
+		}
+	}
 	defer func() {
 		select {
 		case <-cancelled:
 	gitSSH := strings.TrimSpace("ssh " + strings.Join(r.sshOpts, " "))
 	cloneURL := r.cloneBase + "/" + j.Repo + ".git"
+	deadline := time.Now().Add(r.timeout)
 	fmt.Fprintf(sink, "$ git clone %s (%.10s)\n", cloneURL, j.SHA)
 	for _, args := range [][]string{
 		{"clone", "-q", cloneURL, dir},
 		cmd := exec.Command("git", args...)
 		cmd.Env = append(os.Environ(), "GIT_SSH_COMMAND="+gitSSH, "GIT_TERMINAL_PROMPT=0")
 		cmd.Stdout, cmd.Stderr = sink, sink
-		if err := cmd.Run(); err != nil {
+		if ok, why := runStep(cmd, deadline); !ok {
-			fmt.Fprintf(sink, "git %s: %v\n", args[0], err)
+			fmt.Fprintf(sink, "git %s: %s\n", args[0], why)
 			return false
 		}
 	}
-	deadline := time.Now().Add(r.timeout)
 	for _, step := range j.Steps {
 		fmt.Fprintf(sink, "$ %s\n", step)
 		cmd := exec.Command("sh", "-c", step)
 			cmd.Env = append(cmd.Env, name+"="+value)
 		}
 		cmd.Stdout, cmd.Stderr = sink, sink
-		if err := cmd.Start(); err != nil {
+		if ok, why := runStep(cmd, deadline); !ok {
-			fmt.Fprintf(sink, "start: %v\n", err)
+			fmt.Fprintf(sink, "%s\n", why)
-			return false
-		}
-		done := make(chan error, 1)
-		go func() { done <- cmd.Wait() }()
-		select {
-		case err := <-done:
-			if err != nil {
-				fmt.Fprintf(sink, "step failed: %v\n", err)
-				return false
-			}
-		case <-cancelled:
-			cmd.Process.Kill()
-			<-done
-			return false
-		case <-time.After(time.Until(deadline)):
-			cmd.Process.Kill()
-			fmt.Fprintf(sink, "build timed out after %s\n", r.timeout)
 			return false
 		}
 	}

e2e/build_cancel_test.go +5 −3

 	}
 	select {
 	case <-exited:
-	case <-time.After(20 * time.Second):
+	case <-time.After(60 * time.Second):
-		t.Fatalf("runner still running 20s after cancel:\n%s", runnerOut.String())
+		t.Fatalf("runner still running 60s after cancel:\n%s", runnerOut.String())
 	}
-	if took := time.Since(started); took > 15*time.Second {
+	// Well inside the step's 120s sleep: the runner stopped because it
+	// was told to, not because the step ended.
+	if took := time.Since(started); took > 45*time.Second {
 		t.Fatalf("runner took %s to stop", took)
 	}
 	if !strings.Contains(runnerOut.String(), "cancelled") {

Commit 579da6b329

Commit `579da6b329`