Add mediapi-watchdog: reboot the Pi if the mpv player wedges

On 2026-07-09 a kernel keyring-GC oops left the player stuck in
uninterruptible D-state while systemd, SSH, and the Flask app stayed
alive -- so the box sat dead all night instead of recovering. A plain
systemd/hardware watchdog only fires on a TOTAL hang and would not have
caught that partial wedge.

mediapi-watchdog.service (Type=simple, Restart=always, runs as root)
pings mpv over its JSON IPC socket every 30s; after ~3 min of continuous
failure it forces a reboot (systemctl reboot -ff, then SysRq as a
kernel-level fallback that works even when userspace is wedged). A
D-state mpv accepts the socket connect but never replies, so the ping
times out and is caught. install.sh installs + enables it.

Verified: no false positives while mpv is healthy; correctly detects a
stopped mpv and reaches the reboot decision (tested in dry-run).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

install.sh

@@ -116,7 +116,7 @@ bootstrap_system() {
 
 # --- render + install systemd units (used again on rollback) --------
 install_units() {
-  for unit in mediapi-mpv mediapi-app; do
+  for unit in mediapi-mpv mediapi-app mediapi-watchdog; do
     sed -e "s|\${MEDIAPI_USER}|${MEDIAPI_USER}|g" \
         -e "s|\${PROJECT_DIR}|${PROJECT_DIR}|g" \
         -e "s|\${UV}|${UV}|g" \
@@ -184,7 +184,7 @@ deploy_current() {
 
   echo "==> Installing systemd units ..."
   install_units
-  sudo systemctl enable mediapi-mpv mediapi-app >/dev/null 2>&1 || true
+  sudo systemctl enable mediapi-mpv mediapi-app mediapi-watchdog >/dev/null 2>&1 || true
 
   # Migration cleanup: earlier versions ran Kodi as the player. A still-running
   # Kodi keeps the GPU's DRM master and the tty1 seat, which stops mpv from ever
@@ -203,6 +203,7 @@ deploy_current() {
   echo "==> Starting services ..."
   sudo systemctl restart mediapi-mpv
   sudo systemctl restart mediapi-app
+  sudo systemctl restart mediapi-watchdog
 
   # Wait for mpv's IPC socket so a first-run deploy leaves a driveable player.
   echo "==> Waiting for mpv IPC socket (${MEDIAPI_MPV_SOCKET:-/run/mediapi/mpv.sock}) ..."
@@ -235,7 +236,7 @@ echo "==> Health check on http://127.0.0.1:${MEDIAPI_PORT}/login ..."
 if app_healthy; then
   echo "$CURRENT_REF" > "$DEPLOYED_REF_FILE"
   echo "==> Deploy OK. $CURRENT_DESC healthy on port ${MEDIAPI_PORT}."
-  systemctl --no-pager --lines=0 status mediapi-mpv mediapi-app || true
+  systemctl --no-pager --lines=0 status mediapi-mpv mediapi-app mediapi-watchdog || true
   exit 0
 fi
 

scripts/mediapi-watchdog.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+#
+# mediapi-watchdog -- reboot the Pi if the mpv player wedges.
+#
+# Guards against the failure seen 2026-07-09: a kernel oops left the player
+# stuck in uninterruptible (D) state while the rest of the box -- systemd, SSH,
+# the Flask app -- stayed alive and responsive. A plain systemd/hardware
+# watchdog only fires on a TOTAL system hang, so it would NOT have caught that
+# partial wedge. Instead we ping mpv over its JSON IPC socket; if mpv stays
+# unresponsive for ~3 minutes we force a reboot, turning a dead-all-night wedge
+# into a ~30s auto-recovery.
+#
+# Runs as root (needs to force a reboot). Installed + enabled by install.sh.
+set -u
+
+SOCK="${MEDIAPI_MPV_SOCKET:-/run/mediapi/mpv.sock}"
+INTERVAL="${MEDIAPI_WATCHDOG_INTERVAL:-30}"        # seconds between checks
+FAILS_TO_REBOOT="${MEDIAPI_WATCHDOG_FAILS:-6}"     # consecutive fails -> reboot (~3 min)
+PING_TIMEOUT="${MEDIAPI_WATCHDOG_PING_TIMEOUT:-6}" # per-check hard timeout
+DRYRUN="${MEDIAPI_WATCHDOG_DRYRUN:-}"              # non-empty: log instead of rebooting
+
+# Return 0 iff mpv replies to a JSON IPC command within PING_TIMEOUT. A player
+# wedged in D-state accepts the socket connect but never replies, so the recv
+# blocks and `timeout` trips it -- exactly the case we want to catch.
+ping_mpv() {
+  timeout "$PING_TIMEOUT" python3 - "$SOCK" <<'PY'
+import socket, json, sys
+try:
+    s = socket.socket(socket.AF_UNIX); s.settimeout(4); s.connect(sys.argv[1])
+    s.sendall(b'{"command":["get_property","mpv-version"],"request_id":1}\n')
+    buf = b""
+    while b"\n" not in buf:
+        chunk = s.recv(4096)
+        if not chunk:
+            sys.exit(1)
+        buf += chunk
+    for line in buf.decode(errors="replace").splitlines():
+        m = json.loads(line)
+        if m.get("request_id") == 1:
+            sys.exit(0 if m.get("error") == "success" else 1)
+    sys.exit(1)
+except Exception:
+    sys.exit(1)
+PY
+}
+
+do_reboot() {
+  if [ -n "$DRYRUN" ]; then
+    echo "mediapi-watchdog: DRYRUN -- would reboot now" >&2
+    return
+  fi
+  # Best-effort clean reboot first; fall back to SysRq, which reboots at the
+  # kernel level even when userspace is wedged in D-state (the case we guard).
+  sync &
+  systemctl reboot -ff &
+  sleep 12
+  echo 1 > /proc/sys/kernel/sysrq 2>/dev/null || true
+  echo b > /proc/sysrq-trigger 2>/dev/null || true
+}
+
+echo "mediapi-watchdog: watching $SOCK (reboot after ${FAILS_TO_REBOOT}x${INTERVAL}s unresponsive)" >&2
+fails=0
+while true; do
+  if ping_mpv; then
+    fails=0
+  else
+    fails=$((fails + 1))
+    echo "mediapi-watchdog: mpv ping FAILED ($fails/$FAILS_TO_REBOOT)" >&2
+    if [ "$fails" -ge "$FAILS_TO_REBOOT" ]; then
+      echo "mediapi-watchdog: mpv unresponsive ~$((INTERVAL * FAILS_TO_REBOOT))s -- rebooting" >&2
+      do_reboot
+      fails=0
+    fi
+  fi
+  sleep "$INTERVAL"
+done

systemd/mediapi-watchdog.service.template

@@ -0,0 +1,15 @@
+[Unit]
+Description=MediaPi watchdog -- reboots the Pi if the mpv player wedges
+After=mediapi-mpv.service
+Wants=mediapi-mpv.service
+
+[Service]
+Type=simple
+# Runs as root (no User=): must be able to force a reboot, including via SysRq,
+# to recover a box whose player is wedged in uninterruptible D-state.
+ExecStart=${PROJECT_DIR}/scripts/mediapi-watchdog.sh
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target