monitor: backup freshness and the daemon's own healthz !162
1 file changed, +29 −2
Layout: unified · split
deploy/cloud-init.yaml +29 −2
| @@ -91,10 +91,37 @@ write_files: | ||
| 91 | 91 | days=$(( (soonest - $(date -u +%s)) / 86400 )) |
| 92 | 92 | fi |
| 93 | 93 | fi |
| 94 | # Backups are timers, and a timer failing quietly is the most | |
| 95 | # damaging silent failure this host has. Age of the newest full | |
| 96 | # archive and the newest database snapshot, in hours. | |
| 97 | now=$(date -u +%s) | |
| 98 | age_h() { | |
| 99 | f=$(ls -t "$1"/*.tar.gz 2>/dev/null | head -1) | |
| 100 | [ -n "$f" ] || { echo ""; return; } | |
| 101 | echo $(( (now - $(stat -c %Y "$f")) / 3600 )) | |
| 102 | } | |
| 103 | full_age=$(age_h /var/backups/gitbay) | |
| 104 | db_age=$(age_h /var/backups/gitbay/db) | |
| 105 | # The daemon's own word, from inside the process. | |
| 106 | site=$(sed -n 's/^site_url *= *"\(.*\)"/\1/p' /etc/gitbay/config.toml | head -1) | |
| 107 | health="n/a" | |
| 108 | if [ -n "$site" ]; then | |
| 109 | health=$(curl -fsS -m 10 "$site/healthz" 2>/dev/null | grep -o '"ok":[a-z]*' | head -1 | cut -d: -f2) | |
| 110 | [ -n "$health" ] || health="unreachable" | |
| 111 | fi | |
| 94 | 112 | alert="" |
| 95 | 113 | if [ "$svc" != "active" ]; then |
| 96 | 114 | alert="gitbayd is $svc; " |
| 97 | 115 | fi |
| 116 | if [ "$health" != "true" ]; then | |
| 117 | alert="${alert}healthz $health; " | |
| 118 | fi | |
| 119 | if [ -z "$full_age" ] || [ "$full_age" -ge 25 ]; then | |
| 120 | alert="${alert}full backup ${full_age:-missing}h old; " | |
| 121 | fi | |
| 122 | if [ -z "$db_age" ] || [ "$db_age" -ge 2 ]; then | |
| 123 | alert="${alert}db snapshot ${db_age:-missing}h old; " | |
| 124 | fi | |
| 98 | 125 | pct=$(echo "$disk" | tr -d '%') |
| 99 | 126 | if [ "$pct" -ge 85 ]; then |
| 100 | 127 | alert="${alert}disk ${disk}; " |
| @@ -104,10 +131,10 @@ write_files: | ||
| 104 | 131 | fi |
| 105 | 132 | # journald always gets the reading, so an unset webhook cannot make a |
| 106 | 133 | # sick host look like a quiet one. |
| 107 | echo "disk=$disk service=$svc cert_expires=$exp${days:+ cert_days=$days}" | |
| 134 | echo "disk=$disk service=$svc healthz=$health cert_expires=$exp${days:+ cert_days=$days} full_backup_h=${full_age:-missing} db_snapshot_h=${db_age:-missing}" | |
| 108 | 135 | url_file=/etc/gitbay/monitor.url |
| 109 | 136 | if [ -f "$url_file" ]; then |
| 110 | body=$(printf '{"disk":"%s","service":"%s","cert_expires":"%s","alert":"%s"}' "$disk" "$svc" "$exp" "$alert") | |
| 137 | body=$(printf '{"disk":"%s","service":"%s","healthz":"%s","cert_expires":"%s","full_backup_h":"%s","db_snapshot_h":"%s","alert":"%s"}' "$disk" "$svc" "$health" "$exp" "${full_age:-missing}" "${db_age:-missing}" "$alert") | |
| 111 | 138 | if ! curl -fsS -m 10 -H 'Content-Type: application/json' -d "$body" "$(cat "$url_file")" >/dev/null; then |
| 112 | 139 | echo "monitor webhook post failed" >&2 |
| 113 | 140 | fi |