Commit `0b531fd6ad`

0b531fd6ad0c601a672051022509c7816964f245

parent: badc110670

Verified · cmc ci/build: success ci/test: success ci/vuln: success

cmc <hello@cleberg.net> · 2026-09-02 02:53 UTC

monitor: backup freshness and the daemon's own healthz

The hourly heartbeat checked systemd, disk and the certificate. It did
not check that either backup timer had produced anything lately, and a
timer failing quietly is the most damaging silent failure this host has.

The reading now carries the age of the newest full archive and the
newest database snapshot, alerting at 25 and 2 hours, and the daemon's
own /healthz answer read through site_url. All three ride the journald
line and the webhook body.

Closes #78
Ref #28

Layout: unified · split

deploy/cloud-init.yaml +29 −2

		@@ -91,10 +91,37 @@ write_files:
91	91	days=$(( (soonest - $(date -u +%s)) / 86400 ))
92	92	fi
93	93	fi
	94	# Backups are timers, and a timer failing quietly is the most
	95	# damaging silent failure this host has. Age of the newest full
	96	# archive and the newest database snapshot, in hours.
	97	now=$(date -u +%s)
	98	age_h() {
	99	f=$(ls -t "$1"/*.tar.gz 2>/dev/null \| head -1)
	100	[ -n "$f" ] \|\| { echo ""; return; }
	101	echo $(( (now - $(stat -c %Y "$f")) / 3600 ))
	102	}
	103	full_age=$(age_h /var/backups/gitbay)
	104	db_age=$(age_h /var/backups/gitbay/db)
	105	# The daemon's own word, from inside the process.
	106	site=$(sed -n 's/^site_url = "$.*$"/\1/p' /etc/gitbay/config.toml \| head -1)
	107	health="n/a"
	108	if [ -n "$site" ]; then
	109	health=$(curl -fsS -m 10 "$site/healthz" 2>/dev/null \| grep -o '"ok":[a-z]*' \| head -1 \| cut -d: -f2)
	110	[ -n "$health" ] \|\| health="unreachable"
	111	fi
94	112	alert=""
95	113	if [ "$svc" != "active" ]; then
96	114	alert="gitbayd is $svc; "
97	115	fi
	116	if [ "$health" != "true" ]; then
	117	alert="${alert}healthz $health; "
	118	fi
	119	if [ -z "$full_age" ] \|\| [ "$full_age" -ge 25 ]; then
	120	alert="${alert}full backup ${full_age:-missing}h old; "
	121	fi
	122	if [ -z "$db_age" ] \|\| [ "$db_age" -ge 2 ]; then
	123	alert="${alert}db snapshot ${db_age:-missing}h old; "
	124	fi
98	125	pct=$(echo "$disk" \| tr -d '%')
99	126	if [ "$pct" -ge 85 ]; then
100	127	alert="${alert}disk ${disk}; "
		@@ -104,10 +131,10 @@ write_files:
104	131	fi
105	132	# journald always gets the reading, so an unset webhook cannot make a
106	133	# sick host look like a quiet one.
107		echo "disk=$disk service=$svc cert_expires=$exp${days:+ cert_days=$days}"
	134	echo "disk=$disk service=$svc healthz=$health cert_expires=$exp${days:+ cert_days=$days} full_backup_h=${full_age:-missing} db_snapshot_h=${db_age:-missing}"
108	135	url_file=/etc/gitbay/monitor.url
109	136	if [ -f "$url_file" ]; then
110		body=$(printf '{"disk":"%s","service":"%s","cert_expires":"%s","alert":"%s"}' "$disk" "$svc" "$exp" "$alert")
	137	body=$(printf '{"disk":"%s","service":"%s","healthz":"%s","cert_expires":"%s","full_backup_h":"%s","db_snapshot_h":"%s","alert":"%s"}' "$disk" "$svc" "$health" "$exp" "${full_age:-missing}" "${db_age:-missing}" "$alert")
111	138	if ! curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$(cat "$url_file")" >/dev/null; then
112	139	echo "monitor webhook post failed" >&2
113	140	fi

Commit 0b531fd6ad

Commit `0b531fd6ad`