Commit `0b531fd6ad`

0b531fd6ad0c601a672051022509c7816964f245

parent: badc110670

Verified · cmc ci/build: success ci/test: success ci/vuln: success

cmc <hello@cleberg.net> · 2026-09-02 02:53 UTC

monitor: backup freshness and the daemon's own healthz

The hourly heartbeat checked systemd, disk and the certificate. It did
not check that either backup timer had produced anything lately, and a
timer failing quietly is the most damaging silent failure this host has.

The reading now carries the age of the newest full archive and the
newest database snapshot, alerting at 25 and 2 hours, and the daemon's
own /healthz answer read through site_url. All three ride the journald
line and the webhook body.

Closes #78
Ref #28

Layout: unified · split

deploy/cloud-init.yaml +29 −2

	@@ -91,10 +91,37 @@ write_files:
91	days=$(( (soonest - $(date -u +%s)) / 86400 ))	91	days=$(( (soonest - $(date -u +%s)) / 86400 ))
92	fi	92	fi
93	fi	93	fi
		94	# Backups are timers, and a timer failing quietly is the most
		95	# damaging silent failure this host has. Age of the newest full
		96	# archive and the newest database snapshot, in hours.
		97	now=$(date -u +%s)
		98	age_h() {
		99	f=$(ls -t "$1"/*.tar.gz 2>/dev/null \| head -1)
		100	[ -n "$f" ] \|\| { echo ""; return; }
		101	echo $(( (now - $(stat -c %Y "$f")) / 3600 ))
		102	}
		103	full_age=$(age_h /var/backups/gitbay)
		104	db_age=$(age_h /var/backups/gitbay/db)
		105	# The daemon's own word, from inside the process.
		106	site=$(sed -n 's/^site_url = "$.*$"/\1/p' /etc/gitbay/config.toml \| head -1)
		107	health="n/a"
		108	if [ -n "$site" ]; then
		109	health=$(curl -fsS -m 10 "$site/healthz" 2>/dev/null \| grep -o '"ok":[a-z]*' \| head -1 \| cut -d: -f2)
		110	[ -n "$health" ] \|\| health="unreachable"
		111	fi
94	alert=""	112	alert=""
95	if [ "$svc" != "active" ]; then	113	if [ "$svc" != "active" ]; then
96	alert="gitbayd is $svc; "	114	alert="gitbayd is $svc; "
97	fi	115	fi
		116	if [ "$health" != "true" ]; then
		117	alert="${alert}healthz $health; "
		118	fi
		119	if [ -z "$full_age" ] \|\| [ "$full_age" -ge 25 ]; then
		120	alert="${alert}full backup ${full_age:-missing}h old; "
		121	fi
		122	if [ -z "$db_age" ] \|\| [ "$db_age" -ge 2 ]; then
		123	alert="${alert}db snapshot ${db_age:-missing}h old; "
		124	fi
98	pct=$(echo "$disk" \| tr -d '%')	125	pct=$(echo "$disk" \| tr -d '%')
99	if [ "$pct" -ge 85 ]; then	126	if [ "$pct" -ge 85 ]; then
100	alert="${alert}disk ${disk}; "	127	alert="${alert}disk ${disk}; "
	@@ -104,10 +131,10 @@ write_files:
104	fi	131	fi
105	# journald always gets the reading, so an unset webhook cannot make a	132	# journald always gets the reading, so an unset webhook cannot make a
106	# sick host look like a quiet one.	133	# sick host look like a quiet one.
107	echo "disk=$disk service=$svc cert_expires=$exp${days:+ cert_days=$days}"	134	echo "disk=$disk service=$svc healthz=$health cert_expires=$exp${days:+ cert_days=$days} full_backup_h=${full_age:-missing} db_snapshot_h=${db_age:-missing}"
108	url_file=/etc/gitbay/monitor.url	135	url_file=/etc/gitbay/monitor.url
109	if [ -f "$url_file" ]; then	136	if [ -f "$url_file" ]; then
110	body=$(printf '{"disk":"%s","service":"%s","cert_expires":"%s","alert":"%s"}' "$disk" "$svc" "$exp" "$alert")	137	body=$(printf '{"disk":"%s","service":"%s","healthz":"%s","cert_expires":"%s","full_backup_h":"%s","db_snapshot_h":"%s","alert":"%s"}' "$disk" "$svc" "$health" "$exp" "${full_age:-missing}" "${db_age:-missing}" "$alert")
111	if ! curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$(cat "$url_file")" >/dev/null; then	138	if ! curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$(cat "$url_file")" >/dev/null; then
112	echo "monitor webhook post failed" >&2	139	echo "monitor webhook post failed" >&2
113	fi	140	fi

Commit 0b531fd6ad

Commit `0b531fd6ad`