diff options
| -rw-r--r-- | statuspage/README.md | 45 | ||||
| -rwxr-xr-x | statuspage/checks/jail-cgit.sh | 13 | ||||
| -rwxr-xr-x | statuspage/checks/jail-www.sh | 13 | ||||
| -rwxr-xr-x | statuspage/checks/jails.sh | 41 | ||||
| -rwxr-xr-x | statuspage/checks/wireguard.sh | 36 |
5 files changed, 119 insertions, 29 deletions
diff --git a/statuspage/README.md b/statuspage/README.md index 3c2ae85..2b1026d 100644 --- a/statuspage/README.md +++ b/statuspage/README.md @@ -63,6 +63,22 @@ That's it - no changes to `render.sh`, no template edits. It picks up any executable `checks/*.sh` file automatically and renders whatever sections it finds. +### Jails are auto-discovered, not one script per jail + +`checks/jails.sh` is the exception to "one script per metric" - instead of +a `jail-<name>.sh` per jail, it calls `jls -n name` once and reports every +*currently running* jail it finds, resolving each one's IPv4 address (via +the `ip4.addr` jail parameter for classic jails, falling back to `jexec +<name> ifconfig` for VNET jails, which manage their own network stack and +usually don't have `ip4.addr` set) and probing port 80 on it. + +Roll a new jail and it shows up on the next render automatically - no file +to add. The trade-off: a jail that's *supposed* to exist but isn't running +won't show up as a `down` row, it just won't appear at all, since `jls` +only lists running jails. If you need "this jail should exist and +doesn't" alerting, that needs an explicit expected-jails list, which isn't +implemented here. + ## Failure isolation `render.sh` runs each check script and handles three failure modes without @@ -86,23 +102,46 @@ ever need to emit richer JSON (nested objects, arrays), the awk parser in ## Deployment +`/usr/local/etc/statuspage` is a symlink to this checkout +(`~/projects/console/statuspage` on the server). Deploying an update is +just `git pull` there - cron picks up the change on its next run. + Cron (as root): ``` * * * * * /usr/local/etc/statuspage/render.sh ``` -Caddyfile (bind to the WireGuard interface IP only): +Caddyfile - served over a real, browser-trusted cert via DNS-01 (Porkbun), +but bound only to the WireGuard interface so it's unreachable from the +public internet even though the hostname resolves via public DNS: ``` -172.16.0.1:8080 { +status.dandokmang.com { + bind 172.16.0.1 + tls { + dns porkbun { + api_key {env.PORKBUN_API_KEY} + api_secret_key {env.PORKBUN_SECRET_KEY} + } + } root * /usr/local/www/status file_server } ``` +This requires: +- A Caddy build with the `github.com/caddy-dns/porkbun` module (the + pkg-installed FreeBSD binary doesn't include it - get one from + [caddyserver.com/download](https://caddyserver.com/download)). +- `status.dandokmang.com` set as an explicit A record -> `172.16.0.1` in + Porkbun DNS, overriding the `*.dandokmang.com` wildcard that otherwise + points to the public IP. +- `PORKBUN_API_KEY` / `PORKBUN_SECRET_KEY` set in the environment Caddy's + service reads (Porkbun account -> API Access, enabled per-domain). + Reload Caddy after editing: `service caddy reload`. -Access from a WireGuard peer: `http://172.16.0.1:8080`. +Access from a WireGuard peer: `https://status.dandokmang.com`. `render.sh` writes to `/usr/local/www/status/index.html` by default; override with the `STATUSPAGE_OUT` environment variable (useful for local diff --git a/statuspage/checks/jail-cgit.sh b/statuspage/checks/jail-cgit.sh deleted file mode 100755 index aabeacf..0000000 --- a/statuspage/checks/jail-cgit.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -# checks/jail-cgit.sh -# cgit VNET jail: jail-up check, then nginx-inside-jail http check. -set -eu - -DIR=$(dirname "$0") -. "$DIR/../lib/common.sh" - -JAIL_STATUS=$(jail_status cgit) -json_line "jails" "cgit jail" "" "$JAIL_STATUS" - -HTTP_STATUS=$(port_check 192.168.100.10 80) -json_line "jails" " -> http :80" "" "$HTTP_STATUS" diff --git a/statuspage/checks/jail-www.sh b/statuspage/checks/jail-www.sh deleted file mode 100755 index 7b6274e..0000000 --- a/statuspage/checks/jail-www.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -# checks/jail-www.sh -# www VNET jail: jail-up check, then nginx-inside-jail http check. -set -eu - -DIR=$(dirname "$0") -. "$DIR/../lib/common.sh" - -JAIL_STATUS=$(jail_status www) -json_line "jails" "www jail" "" "$JAIL_STATUS" - -HTTP_STATUS=$(port_check 192.168.100.20 80) -json_line "jails" " -> http :80" "" "$HTTP_STATUS" diff --git a/statuspage/checks/jails.sh b/statuspage/checks/jails.sh new file mode 100755 index 0000000..71024cc --- /dev/null +++ b/statuspage/checks/jails.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# checks/jails.sh +# Discovers every currently running jail via jls - no separate script or +# hardcoded name/IP needed per jail. Roll a new jail and it shows up on +# the next render with no changes here. +# +# Trade-off: since jls only lists running jails, a jail that's supposed +# to exist but isn't running just doesn't appear on the page - it won't +# show as "down". If you need to detect an unexpectedly-stopped jail, +# that needs an explicit expected-jails list (not implemented here). +set -eu + +DIR=$(dirname "$0") +. "$DIR/../lib/common.sh" + +# jail_ip4 NAME +# Prints the jail's IPv4 address, or nothing if none could be found. +jail_ip4() { + ip=$(jls -j "$1" -n ip4.addr 2>/dev/null | sed -n 's/.*ip4\.addr=\([^ ]*\).*/\1/p') + case "$ip" in + ""|-|0.0.0.0) ;; + *) printf '%s' "$ip"; return ;; + esac + # VNET jails run their own network stack, so ip4.addr (a classic-jail + # restriction parameter) is usually unset. Ask the jail directly for + # the first non-loopback address it sees instead. + jexec "$1" ifconfig -f inet:cidr 2>/dev/null \ + | awk '/inet /{split($2,a,"/"); if (a[1] !~ /^127\./) {print a[1]; exit}}' +} + +jls -n name 2>/dev/null | sed -n 's/.*name=\([^ ]*\).*/\1/p' | while IFS= read -r name; do + [ -n "$name" ] || continue + + ip=$(jail_ip4 "$name") + json_line "jails" "$name jail" "$ip" "ok" + + if [ -n "$ip" ]; then + http_status=$(port_check "$ip" 80) + json_line "jails" " -> http :80" "" "$http_status" + fi +done diff --git a/statuspage/checks/wireguard.sh b/statuspage/checks/wireguard.sh index 3a899e5..2928e3e 100755 --- a/statuspage/checks/wireguard.sh +++ b/statuspage/checks/wireguard.sh @@ -1,6 +1,7 @@ #!/bin/sh # checks/wireguard.sh # wg0 interface: at least one recent handshake means the tunnel is alive. +# Also lists each configured peer with time since its last handshake. set -eu DIR=$(dirname "$0") @@ -8,3 +9,38 @@ DIR=$(dirname "$0") STATUS=$(wg_status wg0) json_line "host services" "wireguard (wg0)" "" "$STATUS" + +NOW=$(date +%s) +TAB=$(printf '\t') + +wg show wg0 dump 2>/dev/null | tail -n +2 | while IFS="$TAB" read -r pubkey psk endpoint allowedips handshake rx tx keepalive; do + [ -n "$pubkey" ] || continue + peer_id=$(printf '%s' "$pubkey" | cut -c1-8) + label="peer $peer_id ($allowedips)" + + if [ "$handshake" = "0" ]; then + json_line "wireguard peers" "$label" "never" "down" + continue + fi + + age=$((NOW - handshake)) + if [ "$age" -lt 180 ]; then + pstatus="ok" + elif [ "$age" -lt 600 ]; then + pstatus="warn" + else + pstatus="down" + fi + + if [ "$age" -lt 60 ]; then + rel="${age}s ago" + elif [ "$age" -lt 3600 ]; then + rel="$((age / 60))m ago" + elif [ "$age" -lt 86400 ]; then + rel="$((age / 3600))h ago" + else + rel="$((age / 86400))d ago" + fi + + json_line "wireguard peers" "$label" "$rel" "$pstatus" +done |