cmd/gitbay-runner/cgroup_linux.go

  1//go:build linux
  2
  3package main
  4
  5import (
  6	"fmt"
  7	"log"
  8	"os"
  9	"os/exec"
 10	"path/filepath"
 11	"strconv"
 12	"syscall"
 13	"time"
 14)
 15
 16// buildCgroups is the runner's own cgroup subtree for builds. The unit's
 17// Delegate=yes hands the runner its service cgroup; the runner parks
 18// itself in a leaf so the service cgroup can enable controllers for
 19// children (a cgroup may hold processes or controller-enabled children,
 20// not both), and creates one child per build under builds/.
 21type buildCgroups struct {
 22	builds string // <service cgroup>/builds
 23}
 24
 25const cgroupControllers = "+cpu +memory +pids"
 26
 27// prepareBuildCgroups moves the runner into <own>/runner, enables the
 28// controllers on its original cgroup, and creates builds/. It fails
 29// where the cgroup is not writable, which is a unit without
 30// Delegate=yes; the caller decides whether that is fatal.
 31func prepareBuildCgroups() (*buildCgroups, error) {
 32	raw, err := os.ReadFile("/proc/self/cgroup")
 33	if err != nil {
 34		return nil, err
 35	}
 36	own, err := ownCgroupPath(string(raw))
 37	if err != nil {
 38		return nil, err
 39	}
 40	root := filepath.Join("/sys/fs/cgroup", own)
 41	leaf := filepath.Join(root, "runner")
 42	if err := os.MkdirAll(leaf, 0o755); err != nil {
 43		return nil, fmt.Errorf("%s is not writable; the unit needs Delegate=yes: %w", root, err)
 44	}
 45	if err := os.WriteFile(filepath.Join(leaf, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil {
 46		return nil, fmt.Errorf("moving into %s: %w", leaf, err)
 47	}
 48	if err := os.WriteFile(filepath.Join(root, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil {
 49		return nil, fmt.Errorf("enabling controllers on %s: %w", root, err)
 50	}
 51	builds := filepath.Join(root, "builds")
 52	if err := os.MkdirAll(builds, 0o755); err != nil {
 53		return nil, err
 54	}
 55	if err := os.WriteFile(filepath.Join(builds, "cgroup.subtree_control"), []byte(cgroupControllers), 0o644); err != nil {
 56		return nil, fmt.Errorf("enabling controllers on %s: %w", builds, err)
 57	}
 58	return &buildCgroups{builds: builds}, nil
 59}
 60
 61// create makes the cgroup for one build with its limits written, and
 62// returns its path and an open directory fd for placing processes.
 63func (c *buildCgroups) create(id int64, memory, cpus string) (string, *os.File, error) {
 64	dir := filepath.Join(c.builds, fmt.Sprintf("build-%d", id))
 65	if err := os.Mkdir(dir, 0o755); err != nil {
 66		return "", nil, err
 67	}
 68	if err := writeLimits(dir, memory, cpus); err != nil {
 69		os.Remove(dir)
 70		return "", nil, err
 71	}
 72	f, err := os.Open(dir)
 73	if err != nil {
 74		os.Remove(dir)
 75		return "", nil, err
 76	}
 77	return dir, f, nil
 78}
 79
 80// remove kills whatever is still in the build's cgroup — conmon, the
 81// pause process, a step's stray child — and removes it. rmdir fails
 82// until the kernel has reaped every process, so it retries briefly.
 83func (c *buildCgroups) remove(dir string) {
 84	if err := os.WriteFile(filepath.Join(dir, "cgroup.kill"), []byte("1"), 0o644); err != nil {
 85		log.Printf("cgroup %s: kill: %v", dir, err)
 86	}
 87	for i := 0; i < 50; i++ {
 88		if err := os.Remove(dir); err == nil {
 89			return
 90		}
 91		time.Sleep(100 * time.Millisecond)
 92	}
 93	log.Printf("cgroup %s: still populated after kill; left in place", dir)
 94}
 95
 96// intoCgroup starts cmd inside the cgroup fd refers to, so podman,
 97// conmon and everything they start inherit the build's limits. A podman
 98// exec started from the runner's own cgroup lands there, outside the
 99// limit, which is why every invocation for a build goes through this.
100func intoCgroup(cmd *exec.Cmd, f *os.File) {
101	if f == nil {
102		return
103	}
104	if cmd.SysProcAttr == nil {
105		cmd.SysProcAttr = &syscall.SysProcAttr{}
106	}
107	cmd.SysProcAttr.UseCgroupFD = true
108	cmd.SysProcAttr.CgroupFD = int(f.Fd())
109}