fix(update_validation): poll system health instead of broken state wait

Cargo.toml

@@ -8,7 +8,7 @@ license = "MIT OR Apache-2.0"
 name = "omnect-device-service"
 readme = "README.md"
 repository = "https://github.com/omnect/omnect-device-service.git"
-version = "0.44.1"
+version = "0.45.0"
 
 [dependencies]
 actix-server = { version = "2.6", default-features = false }

README.md

@@ -923,6 +923,12 @@ curl -X POST --unix-socket /run/omnect-device-service/api.sock http://localhost/
 curl -X POST --unix-socket /run/omnect-device-service/api.sock http://localhost/healthcheck/v1
 ```
 
+Independent of this endpoint, the [healthcheck](healthcheck/) directory contains
+scripts that check specific system functions and rate each one green, yellow, or
+red. `omnect_health_check.sh` runs the checks configured in
+`omnect_health_checks.json` and derives the script name from each entry's type,
+so the configuration file lists what a device actually checks.
+
 ### Status updates
 
 omnect-device-service is capable to publish certain properties to a list of defined endpoints. Currently the following properties are published:

healthcheck/omnect_health__crash_loop.sh

@@ -0,0 +1,73 @@
+#!/bin/sh
+#
+
+. healthchecklib.sh
+
+# prints crash-looping services to stdout; returns 1 if they cannot be determined
+function find_crash_loops() {
+    local unit props active sub found="" units
+
+    units=$(systemctl list-units --all --type=service --no-legend --plain | awk '{print $1}')
+    if [ -z "${units}" ]; then
+        return 1
+    fi
+
+    for unit in ${units}; do
+        props=$(systemctl show "${unit}" -p ActiveState,SubState 2>/dev/null) || continue
+        active=$(echo "${props}" | sed -n 's/^ActiveState=//p')
+        sub=$(echo "${props}" | sed -n 's/^SubState=//p')
+
+        # only a live loop is rated red: NRestarts has no time window and keeps
+        # counting occasional restarts, and a unit that gave up restarting shows
+        # up as a failed unit in the system-running check
+        if [ "${active}" = "activating" ]; then
+            case "${sub}" in
+                # 'auto-restart-queued' is the same pending restart with the
+                # restart job already queued
+                auto-restart|auto-restart-queued)
+                    found="${found} ${unit}(${sub})"
+                    ;;
+            esac
+        fi
+    done
+
+    echo "${found}"
+}
+
+function do_check() {
+    local found rating=0
+
+    found=$(find_crash_loops) || rating=2
+    [ -z "${found}" ] || rating=2
+    print_rating ${rating} crash_loop "$ME"
+    do_rate ${rating}
+    return ${rating}
+}
+
+function do_get_infos() {
+    local found rating=0 error=""
+
+    found=$(find_crash_loops) || { rating=2; error="failed to list services"; }
+    [ -z "${found}" ] || rating=2
+    print_info_header "${ME}" "${rating}"
+    [ -z "${error}" ] || echo "${error}"
+    [ -z "${found}" ] || echo "crash-looping services:${found}"
+    return ${rating}
+}
+
+command="${1:-check}"
+[ "$1" ] && shift
+check_command_arg "$command"
+
+case "$command" in
+    check)
+	do_check "$@"
+	retval=$?
+	;;
+    get-infos)
+	do_get_infos "$@"
+	retval=$?
+	;;
+esac
+
+exit $retval

healthcheck/omnect_health__system_running.sh

@@ -19,9 +19,10 @@ function do_get_infos() {
     local ret
     checkit
     ret=$?
-    # TODO/FIXME: how to find and show reasons for this state?
     print_info_header "${ME}" "$ret"
-    [ $ret = 0 ] || { systemctl is-system-running; systemctl --failed; }
+    # a state other than "running" comes from failed units or from jobs that
+    # are still pending
+    [ $ret = 0 ] || { systemctl is-system-running; systemctl --failed; systemctl list-jobs; }
     return $ret
 }
 

healthcheck/omnect_health_checks.json

@@ -42,6 +42,10 @@
 	"name": "system_running",
 	"type": "system_running"
     },
+    {
+	"name": "crash_loop",
+	"type": "crash_loop"
+    },
     {
 	"name": "timesync",
 	"type": "timesync"

healthcheck/omnect_health_checks.json.template

@@ -75,6 +75,10 @@
 	"name": "system_running",
 	"type": "system_running"
     },
+    {
+	"name": "crash_loop",
+	"type": "crash_loop"
+    },
     {
 	"name": "timesync",
 	"type": "timesync"