#!/bin/bash
################################################################################
# nvr — Unified CLI for NVR Docker management
#
# Runs on the HOST. Shipped inside the backend image and synced to
# /opt/dividia/ on container startup, so it stays current with the
# running software version.
#
# Usage: nvr <command> [args...]
################################################################################

set -e

INSTALL_DIR="${NVR_INSTALL_DIR:-/opt/dividia}"
HOST_DVIEW_ROOT="$INSTALL_DIR/dview-host"
HOST_DVIEW_ENABLED="$HOST_DVIEW_ROOT/enabled"
HOST_DVIEW_SERVICE="dividia-host-dview"
CLOUDAPI_PROFILE_ROOT="$INSTALL_DIR/config/cloudapi-profiles"
CLOUDAPI_ACTIVE_LINK="$INSTALL_DIR/config/cloudapi-active"
CLOUDAPI_STAGING_MARKER="$INSTALL_DIR/config/cloudapi-staging-approved"
CLOUDAPI_PROFILE_HELPER="$INSTALL_DIR/cloudapi-profile-util.py"

# --- Optional aiengine add-on -------------------------------------------------
# Durable host intent, credential directory, and overlay for the local aiengine
# add-on (LPR / object detection on the NVR). See
# docs/plans/aiengine-docker-addon.md. All paths are overridable so the
# behavioral tests can point them at a temp dir; production uses the real tree.
#
# Intent is the long-term service switch (enabled | legacy-provisioned |
# disabled). An absent file means disabled. It is deliberately NOT database
# state: a local engine is a licensed, intentional host capability that must
# stay available while an operator changes camera configuration.
AIENGINE_ENV_FILE="${NVR_ENV_FILE:-$INSTALL_DIR/.env}"
AIENGINE_INTENT_DIR="${NVR_AIENGINE_INTENT_DIR:-$INSTALL_DIR/data/config/addons/aiengine}"
AIENGINE_INTENT_FILE="$AIENGINE_INTENT_DIR/intent"
AIENGINE_CONFIG_DIR="${NVR_AIENGINE_CONFIG_DIR:-$INSTALL_DIR/data/config/aiengine}"
AIENGINE_OVERLAY="docker-compose.aiengine.yml"
AIENGINE_SERVICE="aiengine"

# HME drive-thru addon (see `nvr addon hme` + docker-compose.hme.yml). The
# marker holds durable operator intent (enabled/disabled/absent=auto); the
# overlay token in COMPOSE_FILE is what actually runs the container. Cron/lock
# are overridable so the behavioral test can point them at a temp dir.
HME_STATE_DIR="${NVR_HME_STATE_DIR:-$INSTALL_DIR/hme}"
HME_MARKER_FILE="$HME_STATE_DIR/marker"
HME_ABSENT_STREAK_FILE="$HME_STATE_DIR/absent-streak"
HME_LAST_ERROR_FILE="$HME_STATE_DIR/last-error"
HME_OVERLAY_FILE="docker-compose.hme.yml"
HME_CONF_PATH="${NVR_HME_CONF_PATH:-$INSTALL_DIR/data/config/hme-stream.conf}"
HME_RECONCILE_CRON="${NVR_HME_CRON:-/etc/cron.d/dividia-nvr-hme}"
HME_LOCK="${NVR_HME_LOCK:-/var/lock/nvr-hme.lock}"

# Host-config self-heal targets (FIX 4 / ensure_host_config). Overridable so
# the behavioral test can point them at a temp dir; production uses the real
# /etc paths that install-nvr.sh writes.
SUDOERS_FILE="${NVR_SUDOERS_FILE:-/etc/sudoers.d/dividia}"
DOCKER_DAEMON_JSON="${NVR_DOCKER_DAEMON_JSON:-/etc/docker/daemon.json}"
SYSCTL_RESERVED_PORTS_FILE="${NVR_SYSCTL_RESERVED_PORTS_FILE:-/etc/sysctl.d/90-dividia-nvr-ports.conf}"
NVR_SERVICE_PORT_RANGE="43202-43210"

# CO6/CO7 hosts install docker at /usr/local/bin/docker, which sudo's
# secure_path omits by default. Without this export, the `docker compose
# version` detection below fails when nvr is invoked via non-interactive
# sudo (e.g. cron, remote ssh exec) and the script exits with "Docker
# Compose not found". Prepending these dirs is safe on CO9/UB24 hosts
# too, where docker is at /usr/bin/docker.
#
# Default fallback (the :- branch) avoids a trailing colon when PATH is
# unset / empty, which bash would interpret as "current directory in
# PATH". Since this script then `cd "$INSTALL_DIR"` and INSTALL_DIR is
# /opt/dividia (writable by the dividia user per install-nvr.sh), a
# trailing-colon PATH would let any local dividia-user plant a malicious
# tar/grep/mv at /opt/dividia/<bin> and have it executed by root on the
# next `sudo nvr update` or scheduled cron run.
export PATH="/usr/local/bin:/usr/local/sbin:${PATH:-/usr/sbin:/usr/bin:/sbin:/bin}"

# Detect compose command: prefer v2 plugin, fall back to standalone v1
if docker compose version &>/dev/null 2>&1; then
    COMPOSE="docker compose"
elif command -v docker-compose &>/dev/null; then
    COMPOSE="docker-compose"
else
    echo "ERROR: Docker Compose not found"
    exit 1
fi

cd "$INSTALL_DIR"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

################################################################################
# Commands
#
# Windows-side access to `db / shell / channel <set> / vm-shell /
# migrate-scalewatcher` is gated at the SSH layer: the ro-key in
# ~/.ssh/authorized_keys uses a forced-command wrapper that only lets a
# subset of subcommands through, and the admin key is passphrase-protected
# on the Windows host. On Linux-host Docker installs there is no
# script-level gate — the tech is already authenticated as `dividia` via
# SSH public-key or console login before they ever type `nvr`.
################################################################################

cmd_status() {
    echo -e "${BLUE}=== NVR Docker Stack ===${NC}"
    $COMPOSE ps
    echo ""
    echo -e "${BLUE}=== Identity ===${NC}"
    # Serial + hostname are read offline (dvs.conf / hostname) so they always
    # render; only the name comes from the db and degrades if it's down. Each
    # assignment is guarded (declare `local` separately, then `|| ...`) so a
    # failed read never trips `set -e` and aborts the rest of status. Serial
    # uses the same dvs.conf path/grep as update_cron_jitter().
    local _name _serial _host
    _name=$(nvr_name) || _name=""
    [[ -z "$_name" ]] && _name="(unavailable - db not running?)"
    _serial=$(grep '^ID=' data/config/dvs.conf 2>/dev/null | cut -d= -f2-) || true
    _host=$(hostname 2>/dev/null) || true
    echo "Name:     ${_name}"
    echo "Serial:   ${_serial:-<unknown>}"
    echo "Hostname: ${_host:-<unknown>}"
    echo ""
    echo -e "${BLUE}=== Configuration ===${NC}"
    grep -E '^(CHANNEL|REGISTRY)=' .env 2>/dev/null || true
    echo ""
    echo -e "${BLUE}=== Update cron ===${NC}"
    if [[ -f /etc/cron.d/dividia-nvr-update ]]; then
        # `|| true` keeps cmd_status surviving a future `set -o pipefail`
        # roll-out: head closes early → grep gets SIGPIPE → exit 141.
        grep -E '^[0-9]+ ' /etc/cron.d/dividia-nvr-update 2>/dev/null | head -1 || true
    else
        echo "(update cron not installed; will land on next nvr update)"
    fi
    echo ""
    cmd_display status
}

cmd_logs() {
    $COMPOSE logs "$@"
}

cloudapi_profile_python() {
    if command -v python3 >/dev/null 2>&1; then
        echo python3
    elif command -v python >/dev/null 2>&1; then
        echo python
    else
        echo "ERROR: Python is required for cloudapi profile validation" >&2
        return 1
    fi
}

cloudapi_profile_describe() {
    local profile_dir="$1"
    local expected="${2:-}"
    local python_cmd

    [[ -f "$CLOUDAPI_PROFILE_HELPER" ]] || {
        echo "ERROR: $CLOUDAPI_PROFILE_HELPER is missing; run nvr update first" >&2
        return 1
    }
    python_cmd=$(cloudapi_profile_python) || return 1
    if [[ -n "$expected" ]]; then
        "$python_cmd" "$CLOUDAPI_PROFILE_HELPER" "$profile_dir" "$expected"
    else
        "$python_cmd" "$CLOUDAPI_PROFILE_HELPER" "$profile_dir"
    fi
}

cloudapi_profile_require_root() {
    [[ $EUID -eq 0 ]] || {
        echo "ERROR: run 'sudo nvr cloudapi-profile $1'" >&2
        return 1
    }
}

cloudapi_profile_channel() {
    sed -n 's/^CHANNEL=//p' "$INSTALL_DIR/.env" 2>/dev/null | head -1
}

cloudapi_profile_staging_allowed() {
    local channel
    channel=$(cloudapi_profile_channel)
    case "$channel" in
        dev|dev-*) ;;
        *)
            echo "ERROR: staging requires a dev or dev-* update channel" >&2
            return 1
            ;;
    esac
    [[ -f "$CLOUDAPI_STAGING_MARKER" ]] || {
        echo "ERROR: staging requires $CLOUDAPI_STAGING_MARKER" >&2
        return 1
    }
}

cloudapi_profile_set_link() {
    local target="$1"
    local python_cmd

    mkdir -p "$(dirname "$CLOUDAPI_ACTIVE_LINK")"
    if [[ -d "$CLOUDAPI_ACTIVE_LINK" && ! -L "$CLOUDAPI_ACTIVE_LINK" ]]; then
        rmdir "$CLOUDAPI_ACTIVE_LINK" 2>/dev/null || {
            echo "ERROR: $CLOUDAPI_ACTIVE_LINK is a non-empty directory" >&2
            return 1
        }
    elif [[ -e "$CLOUDAPI_ACTIVE_LINK" && ! -L "$CLOUDAPI_ACTIVE_LINK" ]]; then
        echo "ERROR: $CLOUDAPI_ACTIVE_LINK is not a managed symlink" >&2
        return 1
    fi

    python_cmd=$(cloudapi_profile_python) || return 1
    "$python_cmd" "$CLOUDAPI_PROFILE_HELPER" \
        activate "$target" "$CLOUDAPI_ACTIVE_LINK"
}

cloudapi_profile_wait_backend() {
    local container=""
    local status=""
    local attempt=0

    while [[ $attempt -lt 60 ]]; do
        container=$($COMPOSE ps -q backend 2>/dev/null || true)
        if [[ -n "$container" ]]; then
            status=$(docker inspect -f '{{.State.Health.Status}}' "$container" 2>/dev/null || true)
            [[ "$status" == "healthy" ]] && return 0
            [[ "$status" == "unhealthy" ]] && break
        fi
        attempt=$((attempt + 1))
        sleep 2
    done
    echo "ERROR: backend did not become healthy" >&2
    return 1
}

cloudapi_profile_apply_backend() {
    $COMPOSE up -d --force-recreate backend
    cloudapi_profile_wait_backend
    local probe_rc=0
    $COMPOSE exec -T backend /usr/local/bin/cloudapi-profile-probe || probe_rc=$?
    if [[ $probe_rc -eq 0 ]]; then
        # Repoint the appliance web viewer's Apache /cloudapi/ reverse proxy at
        # the newly-selected endpoint. viewer-start renders that target from the
        # mounted profile (config/cloudapi-active), so a recreate is required —
        # a switch that only recreates the backend would leave the web viewer's
        # OAuth calls proxying to the previous environment. Non-fatal: a viewer
        # hiccup must not fail an otherwise-good profile switch or trip rollback,
        # but surface it so an operator knows the web viewer may be stale/down.
        $COMPOSE up -d --force-recreate viewer \
            || echo "WARNING: viewer recreate failed; web viewer may still proxy OAuth to the previous cloudapi endpoint" >&2
    fi
    return $probe_rc
}

cmd_cloudapi_profile_show() {
    if [[ ! -L "$CLOUDAPI_ACTIVE_LINK" ]]; then
        echo "Cloudapi profile: unconfigured"
        return 0
    fi

    local target
    local metadata
    target=$(readlink "$CLOUDAPI_ACTIVE_LINK")
    metadata=$(cloudapi_profile_describe "$target") || return 1
    echo "Cloudapi profile: ${metadata%%|*}"
    metadata="${metadata#*|}"
    echo "Endpoint: ${metadata%%|*}"
    echo "Trusted kids: ${metadata#*|}"
    echo "Selected at: $(stat -c '%y' "$CLOUDAPI_ACTIVE_LINK" 2>/dev/null || echo unknown)"
}

cmd_cloudapi_profile_select() {
    local environment="$1"
    local candidate="$CLOUDAPI_PROFILE_ROOT/$environment"
    local previous=""

    cloudapi_profile_require_root "$environment" || return 1
    [[ "$environment" == "prod" || "$environment" == "staging" ]] || {
        echo "ERROR: profile must be prod or staging" >&2
        return 1
    }
    if [[ "$environment" == "staging" ]]; then
        cloudapi_profile_staging_allowed || return 1
    fi
    cloudapi_profile_describe "$candidate" "$environment" >/dev/null || return 1

    if [[ -L "$CLOUDAPI_ACTIVE_LINK" ]]; then
        previous=$(readlink "$CLOUDAPI_ACTIVE_LINK")
        if [[ "$previous" == "$candidate" ]]; then
            echo "Cloudapi profile is already $environment"
            return 0
        fi
    fi

    cloudapi_profile_set_link "$candidate" || return 1
    if cloudapi_profile_apply_backend; then
        echo "Cloudapi profile switched to $environment"
        cmd_cloudapi_profile_show
        return 0
    fi

    echo "ERROR: cloudapi profile probe failed; restoring prior profile" >&2
    if [[ -n "$previous" ]]; then
        cloudapi_profile_set_link "$previous"
        cloudapi_profile_apply_backend || {
            echo "ERROR: prior cloudapi profile did not recover" >&2
            return 1
        }
    else
        rm -f "$CLOUDAPI_ACTIVE_LINK"
        $COMPOSE up -d --force-recreate backend || true
    fi
    return 1
}

cmd_cloudapi_profile() {
    case "${1:-show}" in
        show) cmd_cloudapi_profile_show ;;
        prod|staging) cmd_cloudapi_profile_select "$1" ;;
        *)
            echo "Usage: nvr cloudapi-profile show|prod|staging" >&2
            return 1
            ;;
    esac
}

host_display_os_major() {
    local major=""
    if command -v rpm >/dev/null 2>&1; then
        major=$(rpm -E '%{rhel}' 2>/dev/null || true)
    fi
    if [[ ! "$major" =~ ^[0-9]+$ ]] && [[ -r /etc/redhat-release ]]; then
        major=$(sed -nE 's/.*release[[:space:]]+([0-9]+).*/\1/p' /etc/redhat-release | head -1)
    fi
    echo "$major"
}

host_display_supported() {
    local major
    major=$(host_display_os_major)
    [[ "$major" == "6" || "$major" == "7" ]]
}

host_display_is_enabled() {
    [[ -f "$HOST_DVIEW_ENABLED" ]]
}

host_display_require_root() {
    [[ $EUID -eq 0 ]] || { echo "ERROR: run 'sudo nvr display $1'" >&2; return 1; }
}

host_display_service_active() {
    if command -v systemctl >/dev/null 2>&1; then
        systemctl is-active --quiet "$HOST_DVIEW_SERVICE.service"
    else
        service "$HOST_DVIEW_SERVICE" status >/dev/null 2>&1
    fi
}

host_display_stop_disable_service() {
    if command -v systemctl >/dev/null 2>&1; then
        systemctl stop "$HOST_DVIEW_SERVICE.service" 2>/dev/null || true
        systemctl disable "$HOST_DVIEW_SERVICE.service" 2>/dev/null || true
    else
        service "$HOST_DVIEW_SERVICE" stop 2>/dev/null || true
        chkconfig "$HOST_DVIEW_SERVICE" off 2>/dev/null || true
    fi
}

host_display_install_service() {
    [[ -x "$INSTALL_DIR/host-dview" ]] || {
        echo "ERROR: $INSTALL_DIR/host-dview is missing; run nvr update first" >&2
        return 1
    }

    if command -v systemctl >/dev/null 2>&1; then
        cat > "/etc/systemd/system/$HOST_DVIEW_SERVICE.service" <<EOF
[Unit]
Description=Dividia host desktop viewer (CO6/CO7 Docker deployment)
Requires=docker.service
Wants=network-online.target
After=docker.service network-online.target

[Service]
Type=simple
# CO7's systemd (219) predates StandardOutput=append:. Keep the redirection
# in a shell so the stable host log works on both CO7 and newer systemd.
ExecStart=/bin/bash -c 'exec $INSTALL_DIR/host-dview >>/var/log/dividia-host-dview.log 2>&1'
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF
        systemctl daemon-reload
    else
        cat > "/etc/init.d/$HOST_DVIEW_SERVICE" <<EOF
#!/bin/bash
# chkconfig: 2345 97 03
# description: Dividia host desktop viewer
### BEGIN INIT INFO
# Provides:          $HOST_DVIEW_SERVICE
# Required-Start:    \$network \$docker
# Required-Stop:     \$network \$docker
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Dividia host desktop viewer
### END INIT INFO

PIDFILE=/var/run/$HOST_DVIEW_SERVICE.pid
LOGFILE=/var/log/dividia-host-dview.log
LAUNCHER=$INSTALL_DIR/host-dview

start() {
    if [[ -f "\$PIDFILE" ]] && kill -0 "\$(cat "\$PIDFILE")" 2>/dev/null; then
        return 0
    fi
    mkdir -p "\$(dirname "\$LOGFILE")"
    nohup "\$LAUNCHER" >>"\$LOGFILE" 2>&1 < /dev/null &
    echo \$! > "\$PIDFILE"
}

stop() {
    if [[ -f "\$PIDFILE" ]]; then
        kill "\$(cat "\$PIDFILE")" 2>/dev/null || true
        rm -f "\$PIDFILE"
    fi
}

status() {
    [[ -f "\$PIDFILE" ]] && kill -0 "\$(cat "\$PIDFILE")" 2>/dev/null
}

case "\${1:-}" in
    start) start ;;
    stop) stop ;;
    restart) stop; start ;;
    status) status ;;
    *) echo "Usage: \$0 {start|stop|restart|status}"; exit 2 ;;
esac
EOF
        chmod 755 "/etc/init.d/$HOST_DVIEW_SERVICE"
    fi
}

host_display_enable_start_service() {
    if command -v systemctl >/dev/null 2>&1; then
        systemctl enable "$HOST_DVIEW_SERVICE.service"
        systemctl restart "$HOST_DVIEW_SERVICE.service"
    else
        chkconfig "$HOST_DVIEW_SERVICE" on
        service "$HOST_DVIEW_SERVICE" restart
    fi

    # A Java process can fail after the service manager accepts the start.
    # Give its native library/Xorg initialization a short grace period before
    # declaring the staged payload healthy.
    sleep 3
    host_display_service_active
}

host_display_preflight() {
    local failed=0 pkg
    echo "Checking host dview prerequisites..."

    if ! host_display_supported; then
        echo "  FAIL: host must be CentOS 6 or 7 (detected: $(host_display_os_major || true))" >&2
        failed=1
    fi
    if [[ "$(uname -m)" != "x86_64" ]]; then
        echo "  FAIL: host architecture must be x86_64" >&2
        failed=1
    fi
    for pkg in Xorg xinit metacity unzip strings sha256sum; do
        if ! command -v "$pkg" >/dev/null 2>&1; then
            echo "  FAIL: required host command missing: $pkg" >&2
            failed=1
        fi
    done
    if ! rpm -q xorg-x11-server-Xorg xorg-x11-drv-fbdev xorg-x11-drv-evdev metacity >/dev/null 2>&1; then
        echo "  FAIL: legacy Xorg/video/input packages are not all installed" >&2
        failed=1
    fi
    if [[ ! -c /dev/tty1 ]]; then
        echo "  FAIL: local virtual console /dev/tty1 is unavailable" >&2
        failed=1
    fi
    if [[ ! -e /dev/fb0 && ! -d /dev/dri ]]; then
        echo "  FAIL: no framebuffer or DRM device is available" >&2
        failed=1
    fi
    [[ $failed -eq 0 ]] || return 1
    echo "  OK: host is eligible for the optional local viewer"
}

host_display_payload_image() {
    $COMPOSE config --images 2>/dev/null | awk '/(^|\/)nvr-viewer:/{ print; exit }'
}

host_display_payload_ref() {
    local image="$1" ref
    ref=$(docker image inspect --format '{{index .Config.Labels "io.dividia.host-dview.payload"}}' "$image" 2>/dev/null || true)
    [[ "$ref" != "<no value>" ]] || ref=""
    [[ "$ref" =~ ^[^[:space:]@]+@sha256:[a-f0-9]{64}$ ]] || {
        echo "ERROR: viewer image does not name an immutable nvr-dview-host payload" >&2
        return 1
    }
    echo "$ref"
}

host_display_native_sha256_label() {
    local image="$1" major="$2" label sha
    label="io.dividia.host-dview.native.co${major}.sha256"
    sha=$(docker image inspect --format "{{index .Config.Labels \"$label\"}}" "$image" 2>/dev/null || true)
    [[ "$sha" != "<no value>" && "$sha" =~ ^[a-f0-9]{64}$ ]] || {
        echo "ERROR: image lacks a valid $label label" >&2
        return 1
    }
    echo "$sha"
}

host_display_sha256() {
    sha256sum "$1" | awk '{print $1}'
}

host_display_validate_payload() {
    local payload="$1" expected_native_sha="${2:-}" major lib host native java_bin java_version actual_native_sha
    major=$(host_display_os_major)
    [[ "$major" == "6" || "$major" == "7" ]] || return 1
    lib="$payload/lib"
    host="$payload/host"
    native="$host/native/ffmpeg-arch-x86_64-pc-linux-gnu-co${major}.jar"
    java_bin="$host/jre/bin/java"
    [[ -s "$lib/dview.jar" ]] || return 1
    [[ -s "$lib/logging-server.properties" ]] || return 1
    [[ -s "$lib/ffmpeg-noarch.jar" ]] || return 1
    [[ ! -e "$lib/ffmpeg-arch-x86_64-pc-linux-gnu-co6.jar" ]] || return 1
    [[ ! -e "$lib/ffmpeg-arch-x86_64-pc-linux-gnu-co7.jar" ]] || return 1
    [[ -s "$native" ]] || return 1
    [[ -f "$host/co6-java-ok" ]] || return 1
    if [[ -n "$expected_native_sha" ]]; then
        actual_native_sha=$(host_display_sha256 "$native") || return 1
        [[ "$actual_native_sha" == "$expected_native_sha" ]] || return 1
    fi
    grep -qx 'format=1' "$host/manifest.properties" || return 1
    grep -qx 'java=21' "$host/manifest.properties" || return 1
    [[ -x "$java_bin" ]] || return 1
    java_version=$("$java_bin" -version 2>&1 | head -1 || true)
    [[ "$java_version" == *'version "21.'* ]] || return 1
    unzip -tqq "$native" >/dev/null
    # The currently published legacy JAR predates cooperative cancellation
    # and the cloud-precache guard. Reject it rather than running a current
    # Java viewer against a stale JNI library (which can throw later during
    # playback shutdown). This also makes release publication verifiable on
    # the actual host without trusting a mutable CDN path.
    unzip -p "$native" libFFmpeg.so \
        | strings \
        | grep -qx 'Java_net_dividia_ffmpeg_FFmpeg_requestCancel'
    unzip -p "$native" libFFmpeg.so \
        | strings \
        | grep -qx 'Java_net_dividia_ffmpeg_FFmpeg_hasPrecache'
}

host_display_activate_payload() {
    local target="$1" previous="" link_tmp
    [[ -L "$HOST_DVIEW_ROOT/current" ]] && previous=$(readlink "$HOST_DVIEW_ROOT/current")
    link_tmp="$HOST_DVIEW_ROOT/.current.$$"
    ln -s "versions/$target" "$link_tmp"
    mv -Tf "$link_tmp" "$HOST_DVIEW_ROOT/current"
    HOST_DVIEW_PREVIOUS="$previous"
}

host_display_restore_previous() {
    local link_tmp
    if [[ -n "${HOST_DVIEW_PREVIOUS:-}" ]]; then
        link_tmp="$HOST_DVIEW_ROOT/.current.rollback.$$"
        ln -s "$HOST_DVIEW_PREVIOUS" "$link_tmp"
        mv -Tf "$link_tmp" "$HOST_DVIEW_ROOT/current"
    else
        rm -f "$HOST_DVIEW_ROOT/current"
    fi
}

# Pull the private runtime/JNI payload named by the exact locally pulled viewer
# image. The CO6/CO7 viewer container stays web-only; the separate OCI payload
# is downloaded only for a host that explicitly enabled local dview. The pair
# of immutable image IDs names the staged directory.
# Sets HOST_DVIEW_CHANGED=1 only after the active symlink changed.
host_display_stage_payload() {
    local image viewer_id host_ref host_id payload_id target_name target stage viewer_cid="" host_cid="" expected_native_sha host_native_sha
    HOST_DVIEW_CHANGED=0
    HOST_DVIEW_PREVIOUS=""
    image=$(host_display_payload_image)
    [[ -n "$image" ]] || { echo "ERROR: viewer image is not configured" >&2; return 1; }
    viewer_id=$(docker image inspect --format '{{.Id}}' "$image" 2>/dev/null || true)
    viewer_id=${viewer_id#sha256:}
    [[ "$viewer_id" =~ ^[a-f0-9]{32,}$ ]] || { echo "ERROR: cannot determine immutable viewer image ID" >&2; return 1; }
    expected_native_sha=$(host_display_native_sha256_label "$image" "$(host_display_os_major)") || return 1
    # A viewer config/image ID includes the payload label. If it has not
    # changed, neither has the immutable payload reference; avoid even pulling
    # the host artifact (and, critically, avoid restarting dview) on a no-op
    # `nvr update`.
    local current_target
    current_target=$(readlink "$HOST_DVIEW_ROOT/current" 2>/dev/null || true)
    if [[ "$current_target" == "versions/${viewer_id}-"* ]] && \
       host_display_validate_payload "$HOST_DVIEW_ROOT/$current_target" "$expected_native_sha"; then
        return 0
    fi
    host_ref=$(host_display_payload_ref "$image") || return 1
    echo "Pulling host dview payload: $host_ref"
    docker pull "$host_ref" >/dev/null || { echo "ERROR: unable to pull host dview payload" >&2; return 1; }
    host_native_sha=$(host_display_native_sha256_label "$host_ref" "$(host_display_os_major)") || return 1
    [[ "$host_native_sha" == "$expected_native_sha" ]] || {
        echo "ERROR: viewer and host payload disagree on CO$(host_display_os_major) JNI artifact identity" >&2
        return 1
    }
    host_id=$(docker image inspect --format '{{.Id}}' "$host_ref" 2>/dev/null || true)
    host_id=${host_id#sha256:}
    [[ "$host_id" =~ ^[a-f0-9]{32,}$ ]] || { echo "ERROR: cannot determine immutable host payload ID" >&2; return 1; }
    payload_id="${viewer_id}-${host_id}"

    mkdir -p "$HOST_DVIEW_ROOT/versions"
    target_name="$payload_id"
    target="$HOST_DVIEW_ROOT/versions/$target_name"
    if [[ -d "$target" ]] && host_display_validate_payload "$target" "$expected_native_sha"; then
        if [[ "$(readlink "$HOST_DVIEW_ROOT/current" 2>/dev/null || true)" == "versions/$payload_id" ]]; then
            return 0
        fi
        host_display_activate_payload "$payload_id"
        HOST_DVIEW_CHANGED=1
        return 0
    fi

    # Never replace an invalid directory in place: it may still be the active
    # payload of the running service. Stage a repaired sibling and switch the
    # symlink only after validation, preserving the previous bytes for the
    # required update rollback path.
    if [[ -e "$target" ]]; then
        target_name="${payload_id}.repair.$(date +%s).$$"
        target="$HOST_DVIEW_ROOT/versions/$target_name"
    fi

    stage=$(mktemp -d "$HOST_DVIEW_ROOT/.stage.XXXXXX") || return 1
    viewer_cid=$(docker create "$image" 2>/dev/null || true)
    host_cid=$(docker create "$host_ref" 2>/dev/null || true)
    if [[ -z "$viewer_cid" || -z "$host_cid" ]] || \
       ! docker cp "$viewer_cid:/usr/share/dview/lib" "$stage/lib" 2>/dev/null || \
       ! docker cp "$host_cid:/usr/share/nvr-dview-host" "$stage/host" 2>/dev/null; then
        [[ -n "$viewer_cid" ]] && docker rm "$viewer_cid" >/dev/null 2>&1 || true
        [[ -n "$host_cid" ]] && docker rm "$host_cid" >/dev/null 2>&1 || true
        rm -rf "$stage"
        echo "ERROR: unable to extract the matching host dview payload" >&2
        return 1
    fi
    docker rm "$viewer_cid" >/dev/null 2>&1 || true
    docker rm "$host_cid" >/dev/null 2>&1 || true
    if ! host_display_validate_payload "$stage" "$expected_native_sha"; then
        rm -rf "$stage"
        echo "ERROR: viewer/host image pair lacks a valid CO$(host_display_os_major) dview payload" >&2
        return 1
    fi
    mv "$stage" "$target"
    host_display_activate_payload "$target_name"
    HOST_DVIEW_CHANGED=1
}

host_display_refresh_after_update() {
    host_display_install_service || return 1
    host_display_stage_payload || return 1
    [[ "$HOST_DVIEW_CHANGED" -eq 1 ]] || return 0
    if host_display_enable_start_service; then
        return 0
    fi

    echo "WARN: new host dview payload did not stay running; restoring prior payload" >&2
    host_display_restore_previous
    host_display_enable_start_service >/dev/null 2>&1 || true
    return 1
}

cmd_display() {
    local action="${1:-status}"
    case "$action" in
        status)
            echo -e "${BLUE}=== Optional Host dview ===${NC}"
            if host_display_is_enabled; then
                echo "enabled: yes"
            else
                echo "enabled: no (default; CO6/CO7 viewer container remains web-only)"
            fi
            echo "eligible host: $(host_display_supported && echo yes || echo no)"
            echo "active payload: $(readlink "$HOST_DVIEW_ROOT/current" 2>/dev/null || echo none)"
            if host_display_service_active; then
                echo "service: running"
            else
                echo "service: stopped"
            fi
            ;;
        enable)
            host_display_require_root enable || return 1
            host_display_preflight || return 1
            host_display_stage_payload || return 1
            host_display_install_service || return 1
            touch "$HOST_DVIEW_ENABLED"
            if ! host_display_enable_start_service; then
                echo "ERROR: host dview failed startup; leaving display disabled" >&2
                rm -f "$HOST_DVIEW_ENABLED"
                host_display_stop_disable_service
                if [[ "${HOST_DVIEW_CHANGED:-0}" -eq 1 ]]; then
                    host_display_restore_previous
                fi
                return 1
            fi
            echo "Host dview enabled. Future successful 'nvr update' runs will stage and restart the matching viewer payload."
            ;;
        disable)
            host_display_require_root disable || return 1
            rm -f "$HOST_DVIEW_ENABLED"
            host_display_stop_disable_service
            echo "Host dview disabled; cached payloads were preserved."
            ;;
        # Internal handoff used by a non-root operator's `nvr update`.
        # Container updates are intentionally usable by the `dividia` user,
        # while payload activation writes /opt/dividia and systemd state.
        # Re-exec just this small privileged portion instead of making the
        # entire update (or its Docker commands) require root.
        refresh)
            host_display_require_root refresh || return 1
            host_display_is_enabled || return 0
            host_display_refresh_after_update
            ;;
        *)
            echo "Usage: nvr display [enable|disable|status]" >&2
            return 1
            ;;
    esac
}

validate_extracted_host_tool() {
    local dest="$1"
    local staged="$2"

    case "$dest" in
        "$INSTALL_DIR/nvr"|"$INSTALL_DIR/install-nvr.sh"|"$INSTALL_DIR/host-dview")
            bash -n "$staged" >/dev/null 2>&1
            ;;
        *)
            return 0
            ;;
    esac
}

# --- Core-stack convergence (the update self-heal loop) --------------------
#
# WHY THIS EXISTS. BCC fleet 2026-08-05: cs1686 + cs1129 finished `nvr update`
# with the recording pipeline DOWN and cron still reported success (exit 0).
# Root cause: a single `compose up -d`. Backend, recreated on a freshly pulled
# image, warms up (DB connect + XML-RPC on :43204 + VideoStore mounts) and its
# healthcheck transiently reads `unhealthy`. The moment it does, compose aborts
# the whole bring-up with "dependency failed to start: container backend is
# unhealthy" (every recording service has `depends_on: backend
# service_healthy`) and leaves engine/connector/viewer/playback in `Created`.
# Backend self-recovers ~30s later, but nothing re-runs the `up`, so the box
# sits not-recording for up to 24h until the next daily cron.
#
# This was tuned open-loop four times before (flock the detached docker-start
# `up`, --force-recreate, --remove-orphans, repair_untracked_compose_containers)
# — each tried to make one `up` win a timing window. None added a POST-CONDITION.
# The durable fix is a closed loop: re-apply the desired state (idempotent
# `up -d`), then VERIFY every core service is actually running, retrying with
# backoff until a deadline. A transient backend-unhealthy no longer strands the
# stack, because the next pass re-runs `up` once backend has settled.
#
# Portability: sh/bash on CO6..Ubuntu24. Uses only docker inspect + compose ps.
core_stack_converged() {
    local svc cid running health
    for svc in $1; do
        cid=$($COMPOSE ps -q "$svc" 2>/dev/null | head -1)
        # `ps -q` lists only running containers under Compose v2, so a service
        # stranded in `created` (the exact failure this loop fixes) yields no
        # id here -> not converged. That is correct for this gate; the report
        # path below uses `-aq` so it can still name a stranded service.
        [[ -n "$cid" ]] || return 1
        running=$(docker inspect -f '{{.State.Running}}' "$cid" 2>/dev/null)
        [[ "$running" == "true" ]] || return 1
        # Every core service must be RUNNING. The RECORDING-PATH services must
        # additionally be `healthy` (not `starting`, not `unhealthy`): backend
        # (the dependency gate everything waits on), db, engine (mpengine
        # capture) and connector (recorder). We deliberately do NOT require
        # viewer/playback to be healthy -- only running -- so a viewing-side
        # healthcheck quirk (e.g. a headless dview) cannot wedge `nvr update`
        # into a nightly `exit 1`. Their health is the fleet monitor's job.
        case " backend db engine connector " in
            *" $svc "*)
                health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$cid" 2>/dev/null)
                [[ "$health" == "none" || "$health" == "healthy" ]] || return 1
                ;;
        esac
    done
    return 0
}

core_stack_report() {
    local svc cid
    for svc in $1; do
        # `-aq` (not `-q`): the whole point of the report is to name services
        # stranded in `created`/`exited`, which Compose v2's `ps -q` omits.
        cid=$($COMPOSE ps -aq "$svc" 2>/dev/null | head -1)
        if [[ -z "$cid" ]]; then
            echo "  $svc: NO CONTAINER"
            continue
        fi
        docker inspect -f "  $svc: running={{.State.Running}} status={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}" "$cid" 2>/dev/null \
            || echo "  $svc: inspect failed"
    done
}

# Re-apply desired state, then verify, retrying until every core service is up
# or the deadline passes. Returns 0 only when the whole core stack is running
# (and healthy where a healthcheck is defined). Never aborts the caller: a
# failed `up` attempt is swallowed and retried, because the abort itself is the
# condition we are converging past.
converge_core_stack() {
    local core_svcs="$1"
    local timeout="${NVR_UPDATE_CONVERGE_TIMEOUT:-360}"
    local interval="${NVR_UPDATE_CONVERGE_INTERVAL:-10}"
    local start now attempt=0
    start=$(date +%s)
    while :; do
        attempt=$((attempt + 1))
        # Idempotent: a no-op for services already up, and (once backend is
        # healthy) the step that finally starts any stranded dependent. When
        # backend is still unhealthy this returns non-zero fast; we retry.
        # shellcheck disable=SC2086 — intentional word-split of the service list.
        $COMPOSE up -d --remove-orphans $core_svcs >/dev/null 2>&1 || true
        if core_stack_converged "$core_svcs"; then
            echo "Core stack converged after ${attempt} attempt(s)."
            return 0
        fi
        now=$(date +%s)
        if [[ $((now - start)) -ge $timeout ]]; then
            echo "ERROR: core stack did not converge within ${timeout}s (${attempt} attempts)." >&2
            core_stack_report "$core_svcs" >&2
            return 1
        fi
        sleep "$interval"
    done
}

cmd_update() {
    # Serialize concurrent invocations: the cron-fired `nvr update` and
    # an operator who types `sudo nvr update` at the same minute must
    # not race. Same-shell FD form: the kernel attaches the lock to
    # FD 9 and auto-releases when this process exits (normal exit,
    # SIGKILL, OOM, hardware reset, anything) — no stale-lockfile risk.
    # Non-blocking `-n`: if the lock is already held, the second
    # invocation exits 0 (no cron email) instead of piling up.
    # Skipped for non-root callers (dev mode, container) so /var/lock
    # permission errors don't break local iteration.
    if [[ $EUID -eq 0 ]]; then
        exec 9>/var/lock/nvr-update.lock
        flock -n 9 || {
            echo "another nvr update in progress, exiting"
            exit 0
        }
    fi

    # Idempotent maintenance crons. Land here (not just install-nvr.sh)
    # so NVRs that were installed before each cron existed pick them up
    # on their next update without needing a re-install.
    ensure_prune_cron
    ensure_update_cron

    # Idempotent host-config self-heal (FIX 4). Re-asserts passwordless sudo
    # for dividia and the Docker log cap on every update, repairing boxes left
    # half-configured by an interrupted migration (cs2565). `|| true`: a
    # self-heal must never abort the update; ensure_host_config also no-ops for
    # non-root callers internally.
    ensure_host_config || true

    # Self-heal containers that compose has lost track of. Without this,
    # a single untracked container aborts `compose up -d` mid-recreate
    # and leaves the stack half-broken in the field. See function for
    # full root-cause notes.
    repair_untracked_compose_containers

    echo "Pulling latest images..."
    # Pull required CORE services by name, never the whole project. Once an
    # optional add-on overlay (aiengine or hme) is in COMPOSE_FILE, a
    # whole-project pull would try to fetch it too; those images publish out of
    # band and can lag or be absent (canary dev-<workspace>), so an add-on
    # registry outage or bad publish would block the core update fleet-wide.
    # core_service_pull excludes both add-ons; each add-on image is pulled
    # separately in its own reconcile (aiengine_reconcile / hme_reconcile).
    core_service_pull

    # Extract updated compose files + CLI from new backend image
    echo "Extracting updated files from backend image..."
    local image
    image=$($COMPOSE config --images 2>/dev/null | grep backend | head -1)
    if [[ -n "$image" ]]; then
        local cid
        cid=$(docker create "$image" 2>/dev/null) || true
        if [[ -n "$cid" ]]; then
            # Extract files via tar stream so they're owned by calling user
            # (not root). Use atomic write-temp-then-rename: a failed
            # `docker cp | tar -xO` would otherwise truncate the destination
            # to zero bytes via the `>` redirect, leaving (e.g.)
            # /usr/local/bin/nvr-ro-wrap as an empty file — which sshd's
            # forced-command exec can't run, locking out ALL Windows-side
            # ro-key SSH until manual recovery. Verify the temp file is
            # non-empty before swapping it in.
            local extract_files=(
                "/usr/share/nvr/compose/docker-compose.yml:$INSTALL_DIR/docker-compose.yml"
                "/usr/share/nvr/compose/docker-compose.prod.yml:$INSTALL_DIR/docker-compose.prod.yml"
                # Optional HME overlay: not OS-specific, so it refreshes on every
                # box (like prod.yml) rather than being presence-gated like the
                # co6/windows overlays below. Inert unless COMPOSE_FILE names it.
                "/usr/share/nvr/compose/docker-compose.hme.yml:$INSTALL_DIR/docker-compose.hme.yml"
                "/usr/share/nvr/bin/nvr:$INSTALL_DIR/nvr"
                "/usr/share/nvr/bin/install-nvr.sh:$INSTALL_DIR/install-nvr.sh"
                "/usr/share/nvr/bin/host-dview:$INSTALL_DIR/host-dview"
                "/usr/local/bin/nvr-ro-wrap:/usr/local/bin/nvr-ro-wrap"
                # Distribute the aiengine overlay UNCONDITIONALLY (unlike the
                # co6/windows host overlays below). A feature overlay must land
                # on a box before it is ever enabled; it is inert on disk while
                # intent is disabled because it is not in COMPOSE_FILE.
                "/usr/share/nvr/compose/docker-compose.aiengine.yml:$INSTALL_DIR/docker-compose.aiengine.yml"
            )
            # Platform-specific compose overlays: refresh from the image only
            # when the host already has the file on disk. Presence at install
            # time = host needs this overlay (install-nvr.sh writes it
            # conditionally per OS). Without this, co6.yml / windows.yml fixes
            # never reach customers via `nvr update` — they stay on the
            # version that install-nvr.sh originally downloaded. CO9/UB24
            # hosts have no co6.yml on disk and the entry is skipped, so we
            # don't sprout an unused overlay on the wrong platform.
            local optional_extract_files=(
                "/usr/share/nvr/compose/docker-compose.co6.yml:$INSTALL_DIR/docker-compose.co6.yml"
                "/usr/share/nvr/compose/docker-compose.windows.yml:$INSTALL_DIR/docker-compose.windows.yml"
            )
            for entry in "${optional_extract_files[@]}"; do
                if [[ -f "${entry#*:}" ]]; then
                    extract_files+=("$entry")
                fi
            done
            for entry in "${extract_files[@]}"; do
                local src="${entry%%:*}" dest="${entry#*:}"
                local tmp="${dest}.new.$$"
                if docker cp "$cid:$src" - 2>/dev/null | tar -xO > "$tmp" && [[ -s "$tmp" ]]; then
                    # The 6.2 backend image once shipped an nvr script that
                    # parsed on the build host but failed under Bash 4.1 on
                    # 60 migrated BCC appliances. A fleet stopgap repaired
                    # /opt/dividia/nvr, then the next nightly update blindly
                    # extracted the invalid image copy and broke it again.
                    # Validate host shell tools with the HOST'S Bash before
                    # replacing the known-working copy. Compose/config files
                    # are not shell scripts and intentionally skip this gate.
                    if validate_extracted_host_tool "$dest" "$tmp"; then
                        mv -f "$tmp" "$dest"
                    else
                        rm -f "$tmp" 2>/dev/null || true
                        echo "WARN: extracted $src fails host Bash syntax; keeping existing $dest" >&2
                    fi
                else
                    rm -f "$tmp" 2>/dev/null || true
                    echo "WARN: extract of $src failed or produced empty file; keeping existing $dest" >&2
                fi
            done
            chmod 555 "$INSTALL_DIR/nvr" "$INSTALL_DIR/install-nvr.sh" "$INSTALL_DIR/host-dview" 2>/dev/null || true
            chmod 555 /usr/local/bin/nvr-ro-wrap 2>/dev/null || true
            docker rm "$cid" > /dev/null
            echo "Files updated from image."
        fi
    fi

    # BOOT SERVICE, and it must come AFTER the extract above -- that is the whole
    # correctness condition, not a style preference.
    #
    # `install-nvr.sh` runs exactly once in an NVR's life, so anything only it writes
    # can never be fixed on a deployed box. ADR-045 proved the cost: it put
    # `ensure_videostore_mounts` INLINE in the generated /etc/init.d/nvr and made the
    # managed fstab entry `noauto` so the OS deliberately skips it, which makes that
    # generated script the only thing that mounts a VideoStore at boot on CentOS 6.
    # cs2 (real CO6 6.10) has ZERO occurrences of it, so `nvr update` handed it a
    # `noauto` entry nothing at boot honoured -- safe (backend Phase 2 still mounts and
    # compose depends_on keeps engine behind it; measured on the CO6 VM, no root fill)
    # but INERT, while the entry made the box look fixed.
    #
    # ORDERING. ensure_boot_service SOURCES the host's install-nvr.sh to invoke its one
    # generator. The extract above is what replaces that file with the version from the
    # image just pulled. Running this BEFORE the extract regenerates the boot service
    # from the OLD on-disk generator, so a newly shipped boot-path fix would not land
    # until the NEXT update -- a silent one-cycle lag, and for cs2 it would mean the
    # first update still left it without the mounter. Caught in review; the ordering is
    # pinned by test_nvr_ensure_boot_service.sh.
    #
    # `|| true`: a self-heal must never abort the update, same rule as
    # ensure_host_config.
    ensure_boot_service || true

    # Converge the optional aiengine overlay to host intent BEFORE the compose
    # up below, and pull its image in an isolated step. Must run AFTER the
    # extract above (so the overlay file is the freshly shipped version) and
    # BEFORE `up` (so COMPOSE_FILE already reflects intent). `|| true`: an
    # add-on reconcile failure must never abort a core update.
    aiengine_reconcile || true

    # Converge the optional HME addon's HOST state (COMPOSE_FILE overlay + conf +
    # DeviceType seed) BEFORE the main compose up, so the hme container is
    # started or orphan-removed in the SAME pass as everything else. State-only
    # (no compose call here); fail-open on any DB error and never aborts the
    # update. The db container from the prior version is still up at this point,
    # so the Device-signal read succeeds.
    hme_reconcile_config || true

    echo "Restarting services..."
    # --remove-orphans: when an NVR migrates off watchtower for the first
    # time, the existing dividia-nvr-watchtower-1 container becomes an
    # orphan of the new compose project (the watchtower service block
    # was stripped from prod.yml in feature/replace-watchtower-with-nvr-cron).
    # Without --remove-orphans, compose emits a "Found orphan containers"
    # warning but leaves it alive — cmd_update_disable_watchtower then has
    # to catch it at the tail of cmd_update. That tail path turned out to
    # be set-e-fragile (cs2427 2026-05-28: chain broke between
    # ensure_update_cron at L106 and the disable call, leaving cron
    # installed but watchtower running). With --remove-orphans, compose
    # itself kills the orphan as part of the upgrade.
    #
    # Safe across all dividia-nvr-* compose project owners: --remove-orphans
    # only touches containers labeled com.docker.compose.project=dividia-nvr
    # that aren't declared in the current compose files. Other compose
    # projects on the host (homebridge, dragon-pilot, etc.) carry their
    # own project label and stay untouched.
    local compose_up_ok=0
    # Bring up CORE services BY NAME. A whole-project `up` aborts entirely if
    # one service's image is missing, so an enabled add-on whose image failed
    # to pull (and has no local copy) would take the core lifecycle down with
    # it. Naming core services isolates that. No --quiet-pull: core images were
    # already pulled by name above; the add-on image was pulled separately in
    # aiengine_reconcile. --remove-orphans still prunes project orphans (e.g. a
    # just-disabled aiengine whose overlay reconcile dropped from COMPOSE_FILE).
    local _core_svcs
    _core_svcs=$(core_services)
    if [[ -n "$_core_svcs" ]]; then
        # Converge + verify instead of a single fire-and-forget `up`. A
        # transient backend-unhealthy during warmup used to abort the bring-up
        # and strand engine/connector/viewer in `Created` with cron still
        # reporting success (BCC cs1686/cs1129 2026-08-05). converge_core_stack
        # re-applies the desired state until every core service is actually
        # running, or fails loudly so the tail below leaves the rollback path
        # intact and cmd_update exits non-zero.
        converge_core_stack "$_core_svcs" && compose_up_ok=1 || true
    else
        $COMPOSE up -d --remove-orphans && compose_up_ok=1 || true
    fi
    # Start the optional add-on in its OWN step, never gating compose_up_ok. An
    # add-on registry/image outage must not block the watchtower handoff or the
    # host-dview refresh below. Reconcile already set the overlay + pulled.
    if aiengine_intent_is_on; then
        $COMPOSE up -d "$AIENGINE_SERVICE" || echo "WARN: aiengine did not start; core services are up" >&2
    fi

    # Reclaim disk from images obsoleted by the pull. 168h = 7 days:
    # anything not used for a week is gone. Closes the gap that used
    # to be covered partly by watchtower's WATCHTOWER_CLEANUP=true.
    # `|| true` so a prune failure (transient docker error, race with
    # another process pruning concurrently, etc.) doesn't break the
    # watchtower-drop contract below — that contract is gated on
    # compose-up success, not on cmd_prune success.
    cmd_prune --quiet || true

    # Atomic handoff from watchtower to cron-driven updates. Gated on the
    # explicit compose_up_ok flag, not on `set -e` chain reaching this
    # line. cs2427 2026-05-28: prior version sat at the tail of cmd_update
    # under set -e and silently never ran, even though compose up had
    # succeeded — leaving the migrated NVR half-handed-off (cron in place,
    # watchtower still active). See
    # operational_nvr_update_first_migration_skip_watchtower_drop.md.
    #
    # If compose-up succeeded we know the new stack is at least running
    # (subsequent health checks may still fail, but watchtower can't help
    # with that either — watchtower 1.7.1 is the broken version we're
    # trying to escape). If compose-up failed, leave watchtower in place
    # as the rollback fallback (original atomic-handoff intent).
    if [[ $compose_up_ok -eq 1 ]]; then
        cmd_update_disable_watchtower
    fi

    # HME addon: the main compose up above already started/removed the hme
    # container per the COMPOSE_FILE hme_reconcile_config just set. Two tails
    # remain: a changed conf must be forced into the running container (a
    # bind-mount content change does not recreate on its own), and the 2-minute
    # reconcile cron must track provisioning. Both `|| true` — never break the
    # update.
    if [[ $compose_up_ok -eq 1 && "${HME_DESIRED:-}" == "on" && "${HME_CONF_CHANGED:-0}" -eq 1 ]] && hme_container_running; then
        $COMPOSE up -d --force-recreate hme || true
    fi
    ensure_hme_reconcile_cron || true

    # The host viewer is opt-in. A bad payload must never turn a successful
    # container update into a failed NVR update, nor displace a working prior
    # desktop viewer; refresh performs its own atomic rollback.
    if [[ $compose_up_ok -eq 1 ]] && host_display_is_enabled; then
        local refresh_cmd=(host_display_refresh_after_update)
        if [[ $EUID -ne 0 ]]; then
            refresh_cmd=(sudo -n "$INSTALL_DIR/nvr" display refresh)
        fi
        if ! "${refresh_cmd[@]}"; then
            echo "WARN: host dview was not updated; container services remain updated" >&2
        fi
    fi

    if [[ $compose_up_ok -ne 1 ]]; then
        # The core stack did not converge. Say so loudly and return non-zero so
        # the daily cron emails instead of silently reporting success while the
        # recording pipeline is down. `return 1` (not `exit 1`): under `set -e`
        # the `update) cmd_update "$@"` dispatch propagates it to a non-zero
        # script exit for cron, and it stays catchable by callers/tests. The
        # watchtower rollback fallback is preserved -- its drop is gated on
        # compose_up_ok=1, which stayed 0.
        echo "Update FINISHED WITH ERRORS: core recording stack is not fully up." >&2
        $COMPOSE ps >&2 || true
        return 1
    fi

    echo ""
    echo "Update complete!"
    $COMPOSE ps
}

ensure_prune_cron() {
    local cron_path="/etc/cron.d/dividia-docker-prune"
    local cron_body
    # Do not nest heredocs inside command substitutions: Bash 4.1 on CO6 can
    # lose the closing `)` when the heredoc body later gains apostrophes.
    IFS= read -r -d '' cron_body <<'EOF' || true
# Daily Docker image prune for Dividia NVR.
#
# Reclaims disk from images that have been replaced (by watchtower auto-pull,
# by `nvr update`, by `nvr channel <new>` + manual `docker compose pull`, or
# by ad-hoc operator pulls). Two passes:
#   - dangling (no -a): immediately removes <none>:<none> images that were
#     replaced. Safe by definition; nothing references them.
#   - 168h time-filtered (-a): catches still-tagged images that haven't
#     been used in a week.
# Without the dangling pass, active dev iteration on dev/dev-* channels
# accumulates GB of dangling images that the time filter holds for a week.
#
# Owned by /usr/share/nvr/bin/nvr ensure_prune_cron; do not edit by hand.
SHELL=/bin/sh
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
30 3 * * * root (/usr/bin/docker image prune -f; /usr/bin/docker image prune -a -f --filter "until=168h") >/var/log/dividia-docker-prune.log 2>&1
EOF
    # Skip if not root (containers, dev mode); only the bare-metal install
    # needs the cron managed.
    if [[ $EUID -ne 0 ]]; then
        if ! sudo -n true 2>/dev/null; then return 0; fi
        if [[ -f "$cron_path" ]] && sudo cmp -s <(echo "$cron_body") "$cron_path"; then return 0; fi
        echo "$cron_body" | sudo tee "$cron_path" >/dev/null && \
            sudo chmod 0644 "$cron_path"
    else
        if [[ -f "$cron_path" ]] && cmp -s <(echo "$cron_body") "$cron_path"; then return 0; fi
        echo "$cron_body" > "$cron_path" && chmod 0644 "$cron_path"
    fi
}

# Compute a stable per-host minute jitter in [0,59] so the fleet
# doesn't all hit DockerHub at the same instant. Primary source:
# the bSerial (customer ID) recorded in dvs.conf at install time —
# stable across reboots, replayable for support. Fallback: a hash
# of the hostname so even a zero-ID dev install gets spread.
#
# Echoes the integer to stdout. Always produces a value in [0,59];
# never errors. Safe to invoke from cron-install paths.
update_cron_jitter() {
    local dvs_conf="/opt/dividia/data/config/dvs.conf"
    local id=""
    if [[ -f "$dvs_conf" ]]; then
        id=$(grep -E '^ID=' "$dvs_conf" 2>/dev/null \
             | head -1 \
             | sed -E 's/^ID="?([^"]*)"?.*$/\1/' \
             | tr -d '[:space:]')
    fi
    if [[ "$id" =~ ^[0-9]+$ ]] && [[ "$id" -gt 0 ]]; then
        echo $(( id % 60 ))
        return 0
    fi
    # Fallback: hostname hash. cksum is on every distro back to
    # CO6; awk does the modulo so the value lands in [0,59].
    local h
    h=$(hostname 2>/dev/null | cksum 2>/dev/null | awk '{print $1 % 60}')
    [[ -n "$h" ]] && echo "$h" || echo 0
}

ensure_update_cron() {
    local cron_path="/etc/cron.d/dividia-nvr-update"
    local jitter
    jitter=$(update_cron_jitter)
    # Note: $jitter interpolates because this heredoc is NOT quoted.
    # Everything else is a literal comment or PATH/SHELL line.
    local cron_body
    # Keep the heredoc outside command substitution for Bash 4.1 (CO6).
    IFS= read -r -d '' cron_body <<EOF || true
# Daily nvr update for Dividia NVR.
#
# Replaces watchtower auto-pull (containrrr/watchtower 1.7.1, unmaintained
# since 2024-01 and known to strip com.docker.compose.project labels
# during recreate — see [[pitfall-watchtower-strips-compose-labels]]).
#
# nvr update is the right loop: self-heals orphaned compose project
# labels, pulls images, extracts updated compose + CLI from the new
# backend image, calls compose up -d, prunes obsolete images, then
# removes the watchtower container at the tail of a successful run.
#
# Schedule: 02:NN host-local, where NN = bSerial % 60 from dvs.conf.
# Spreads the fleet across the 02:00-02:59 hour so DockerHub doesn't
# get a thundering herd. Concurrent operator-typed and cron-fired
# invocations serialize via flock inside cmd_update.
#
# Owned by /usr/share/nvr/bin/nvr ensure_update_cron; do not edit by hand.
SHELL=/bin/sh
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
$jitter 2 * * * root /opt/dividia/nvr update >/var/log/dividia-nvr-update.log 2>&1
EOF
    # Same root + sudo idempotent-write pattern as ensure_prune_cron.
    if [[ $EUID -ne 0 ]]; then
        if ! sudo -n true 2>/dev/null; then return 0; fi
        if [[ -f "$cron_path" ]] && sudo cmp -s <(echo "$cron_body") "$cron_path"; then return 0; fi
        echo "$cron_body" | sudo tee "$cron_path" >/dev/null && \
            sudo chmod 0644 "$cron_path"
    else
        if [[ -f "$cron_path" ]] && cmp -s <(echo "$cron_body") "$cron_path"; then return 0; fi
        echo "$cron_body" > "$cron_path" && chmod 0644 "$cron_path"
    fi
}

# ensure_host_config: idempotently re-assert host-level config that
# install-nvr.sh writes ONCE at install time but a steady-state `nvr update`
# never re-checks. An interrupted migration could leave a box with no
# passwordless sudo for the dividia user, or a daemon.json missing the
# container log cap. `nvr update` runs as root from cron on the whole fleet
# nightly, and `nvr start` runs before the containers bind their ports, so those
# are the right places to self-heal these settings. Everything is guarded so a
# non-root caller or a box missing these paths degrades gracefully — a self-heal
# must NEVER hard-fail an update or start (call sites use `|| true` too).
# Re-generate the host boot service (systemd nvr.service, or the SysV
# /etc/init.d/nvr on CentOS 6) from the CURRENT generator, on every update.
#
# WHY THIS IS A CLI FUNCTION AND NOT INSTALLER-ONLY. `install-nvr.sh` runs once per
# NVR lifetime. Every other host artifact already converges from here -- the crons,
# host config, sudoers, reserved ports, the rc.local block, the Docker log cap -- and
# the boot service was the last one that did not, so a boot-path fix could never
# reach a deployed box. ADR-045 is the proof: its CentOS 6 mounter lives inline in the
# generated init script, and cs2 does not have it.
#
# ONE IMPLEMENTATION, deliberately. The generator stays in install-nvr.sh and is
# INVOKED here, rather than copied: ADR-045 forbids a second implementation of
# anything mount-related, and the generated init script carries the CentOS 6 copy of
# ensure_videostore_mounts. Copying the generator would fork the mount logic three
# ways. install-nvr.sh is re-extracted from the backend image by cmd_update (see the
# extract_files list), so the generator invoked here is always as new as the image.
#
# Sourcing it is safe and is an established contract: the file ends with
# `if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then main "$@"; fi` precisely so it can be
# sourced without running the installer, which docker/tests/*.sh already rely on. Its
# top level is only `set -eE`, a PATH export and variable defaults -- no traps, no
# commands with side effects.
#
# Run in a SUBSHELL. install-nvr.sh sets `set -eE` and defines ~70 globals (CHANNEL,
# INSTALL_DIR, colors, MIGRATE_MODE...); leaking those into the CLI would be a
# silent behaviour change for every later subcommand, and its `set -eE` would make
# any subsequent non-zero fatal here.
ensure_boot_service() {
    local installer="$INSTALL_DIR/install-nvr.sh"

    # Works whether invoked as root or as the `dividia` user.
    #
    # WHY THIS IS NOT JUST A ROOT GUARD. The boot files live in /etc, so the work needs
    # root -- but on the CentOS 6 fleet the CLI is routinely run as `dividia` (uid 491 on
    # cs1383), and `install-nvr.sh` gives that user passwordless sudo via
    # /etc/sudoers.d/dividia. The first version returned 0 SILENTLY for non-root, so an
    # operator-run `nvr update` skipped the boot-service convergence and said nothing,
    # while the root-owned nightly cron did it correctly. That split is the worst shape:
    # the gap only appears when a human does it, and appears as no output at all. Found
    # on cs1383 2026-07-27 -- the box took the update, got its `noauto` fstab entry, and
    # kept an /etc/init.d/nvr with no mounter in it.
    #
    # So: elevate rather than skip. NVR_BOOT_SERVICE_ELEVATED stops a second pass from
    # re-sudoing if sudo somehow did not actually raise privileges, which would otherwise
    # recurse.
    if [ "$(id -u)" != "0" ]; then
        if [ -n "${NVR_BOOT_SERVICE_ELEVATED:-}" ]; then
            echo "  NOTE: still not root after elevating; boot service not refreshed"
            return 0
        fi
        local self="$INSTALL_DIR/nvr"
        [ -x "$self" ] || self="$0"
        if command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then
            NVR_BOOT_SERVICE_ELEVATED=1 sudo -n \
                NVR_BOOT_SERVICE_ELEVATED=1 "$self" ensure-boot-service || \
                echo "  WARNING: elevated boot-service refresh failed; boot service left as-is"
            return 0
        fi
        # LOUD, never silent. A boot service that has quietly stopped converging is the
        # exact failure this function exists to prevent.
        echo "  NOTE: not root and passwordless sudo unavailable; boot service NOT refreshed"
        echo "  NOTE: run 'sudo $self ensure-boot-service' to converge it now"
        return 0
    fi

    # No installer on disk => nothing to invoke. Do NOT fall back to a local copy of
    # the generator; a second implementation is the thing this function exists to
    # avoid. The next update re-extracts the installer and this converges then.
    if [ ! -r "$installer" ]; then
        echo "  NOTE: $installer not present; boot service left as-is (next update will refresh it)"
        return 0
    fi

    # Guard against invoking a generator that is not there (an older installer, or a
    # future rename). Silent no-op would mean the boot service quietly stops
    # converging, which is exactly the failure mode this function was written for.
    if ! grep -q '^create_boot_service()' "$installer"; then
        echo "  NOTE: $installer has no create_boot_service; boot service not refreshed"
        return 0
    fi

    (
        # shellcheck disable=SC1090
        . "$installer" >/dev/null 2>&1 || exit 1
        INSTALL_DIR="${NVR_INSTALL_DIR:-/opt/dividia}"
        create_boot_service
    ) || {
        echo "  WARNING: could not refresh the host boot service from $installer"
        return 0
    }
    return 0
}

ensure_host_config() {
    # Only the bare-metal root install owns these host files. Skip quietly for
    # non-root callers (dev mode, in-container invocations).
    if [[ $EUID -ne 0 ]]; then
        return 0
    fi
    ensure_dividia_sudoers || true
    ensure_docker_log_cap || true
    ensure_reserved_service_ports || true
    ensure_rc_local_legacy_block || true
    return 0
}

# Canonicalize a comma-separated Linux ip_local_reserved_ports value plus one
# required range. POSIX awk keeps this host-side code working on CO6/CO7 as
# well as CO9/UB24. Expanding at most 65,535 array entries is cheap and lets us
# coalesce overlaps rather than asking the kernel to accept duplicate ranges.
merge_reserved_port_ranges() {
    local current="${1:-}"
    local required="${2:-$NVR_SERVICE_PORT_RANGE}"
    printf '%s\n' "${current}${current:+,}${required}" | awk -F, '
        function trim(s) {
            gsub(/^[[:space:]]+|[[:space:]]+$/, "", s)
            return s
        }
        {
            for (i = 1; i <= NF; i++) {
                token = trim($i)
                if (token == "") continue
                count = split(token, edge, "-")
                if (count == 1 && edge[1] ~ /^[0-9]+$/) {
                    first = edge[1] + 0
                    last = first
                } else if (count == 2 && edge[1] ~ /^[0-9]+$/ && edge[2] ~ /^[0-9]+$/) {
                    first = edge[1] + 0
                    last = edge[2] + 0
                } else {
                    invalid = 1
                    continue
                }
                if (first < 1 || last > 65535 || first > last) {
                    invalid = 1
                    continue
                }
                for (port = first; port <= last; port++) reserved[port] = 1
            }
        }
        END {
            if (invalid) exit 2
            separator = ""
            port = 1
            while (port <= 65535) {
                if (!(port in reserved)) {
                    port++
                    continue
                }
                first = port
                while (port < 65535 && ((port + 1) in reserved)) port++
                last = port
                if (first == last) printf "%s%d", separator, first
                else printf "%s%d-%d", separator, first, last
                separator = ","
                port++
            }
            print ""
        }
    '
}

# Keep fixed NVR listeners out of Linux's ephemeral client-port allocator.
#
# All NVR containers use network_mode: host. An outbound camera connection can
# otherwise receive a local source port before the corresponding NVR service
# binds its listener, causing that service to crash-loop. Reserving the complete
# 43202-43210 product range prevents the allocator from creating that collision.
#
# Merge rather than replace: other software may own reservations on the same
# appliance. Apply live with sysctl -w and persist the exact same canonical
# union for reboot. This does not evict a socket that already owns a port; the
# monitor's notification-port collision alert still identifies that recovery
# case.
ensure_reserved_service_ports() {
    local current merged wanted file tmp mode
    file="$SYSCTL_RESERVED_PORTS_FILE"

    if ! current=$(sysctl -n net.ipv4.ip_local_reserved_ports 2>/dev/null); then
        echo "WARN: kernel does not expose net.ipv4.ip_local_reserved_ports; NVR service ports were not reserved" >&2
        return 0
    fi
    if ! merged=$(merge_reserved_port_ranges "$current" "$NVR_SERVICE_PORT_RANGE"); then
        echo "WARN: current ip_local_reserved_ports value is invalid; left live and persistent settings untouched" >&2
        return 0
    fi
    wanted="net.ipv4.ip_local_reserved_ports = $merged"

    if [[ "$current" != "$merged" ]]; then
        if ! sysctl -w "net.ipv4.ip_local_reserved_ports=$merged" >/dev/null 2>&1; then
            echo "WARN: could not reserve NVR service ports live; left $file untouched" >&2
            return 0
        fi
        echo "nvr: reserved host service ports $NVR_SERVICE_PORT_RANGE from the ephemeral allocator"
    fi

    if ! mkdir -p "$(dirname "$file")" 2>/dev/null; then
        echo "WARN: could not create service-port reservation directory for $file" >&2
        return 0
    fi
    if [[ -f "$file" ]] && grep -qxF "$wanted" "$file" 2>/dev/null; then
        mode=$(stat -c '%a' "$file" 2>/dev/null || stat -f '%Lp' "$file" 2>/dev/null || echo "")
        [[ -z "$mode" || "$mode" == "644" ]] || chmod 0644 "$file" 2>/dev/null || true
        return 0
    fi

    tmp="${file}.new.$$"
    if printf '%s\n' "$wanted" > "$tmp" 2>/dev/null; then
        chmod 0644 "$tmp" 2>/dev/null || true
        if ! mv -f "$tmp" "$file" 2>/dev/null; then
            rm -f "$tmp" 2>/dev/null || true
            echo "WARN: could not persist NVR service-port reservation to $file" >&2
        fi
    else
        rm -f "$tmp" 2>/dev/null || true
        echo "WARN: could not write NVR service-port reservation to $file" >&2
    fi
    return 0
}

# (c) Neutralize the RPM-era `### dvs start ###` block in rc.local on a migrated
#     Docker host, keeping the one part of it that still matters.
#
# install-nvr.sh has removed this block since the migration flow was written, but
# it edited the WRONG FILE. On CentOS 7/9 `/etc/rc.local` is a symlink to
# `/etc/rc.d/rc.local`, and `sed -i` on a symlink REPLACES the symlink with a
# regular file: the edit lands on a brand-new /etc/rc.local that systemd never
# reads, while `rc-local.service` keeps running the untouched
# `/etc/rc.d/rc.local`. It logged "Removed DVS block from rc.local" and changed
# nothing. Found on cs256 2026-07-26 (473-byte /etc/rc.local dated the migration,
# 1586-byte /etc/rc.d/rc.local dated before it and still carrying the block).
#
# What the surviving block does on every boot of a migrated NVR:
#   systemctl start mariadb          <- a SECOND MariaDB. The containers run
#                                       network_mode: host, so both want :3306.
#                                       Harmless only because the migration
#                                       masks/removes the unit; on any box where
#                                       it is still installed and unmasked this
#                                       is a stack-down race for the port.
#   systemctl start rda-backend      <- unit gone; fails
#   /usr/local/bin/videostore-conf-sync.py   <- shipped by rda-autofs, removed
#   /usr/bin/rdalog --msg ...        <- shipped by an rda RPM, removed
# The failures are why a migrated box reports systemd `degraded`.
#
# It does NOT delete the whole block. The block is also the only thing on a
# migrated NVR that sets the CPU governor (`tuned-adm profile
# throughput-performance`, or the per-CPU scaling_governor loop on CO6) -- I
# checked, nothing in the Docker install replaces it -- and dropping that on a
# fleet of video encoders to silence a log line is the wrong trade. So the block
# is REPLACED by a Docker-appropriate one that keeps the tuning and drops the
# dead service starts.
ensure_rc_local_legacy_block() {
    local rc real backup
    # The file systemd actually executes. rc-local.service on CO7/CO9 declares
    # ExecStart=/etc/rc.d/rc.local, so prefer it, and resolve any symlink so the
    # edit cannot land on the link instead of the target (the original bug).
    for rc in /etc/rc.d/rc.local /etc/rc.local; do
        [[ -f "$rc" ]] || continue
        real=$(readlink -f "$rc" 2>/dev/null || echo "$rc")
        [[ -n "$real" && -f "$real" ]] || continue
        # The trigger is simply "a legacy block is present". No separate
        # already-converted check is needed, because the replacement does not
        # emit a `### dvs start ###` line -- so a converted file no longer
        # matches and a second run is a no-op by construction. A marker-based
        # skip would ALSO have made the write-once backup guard below
        # unreachable, and therefore untestable.
        grep -q '^### dvs start ###' "$real" 2>/dev/null || continue

        # REFUSE without a matching column-0 end marker.
        #
        # The awk below clears `inblock` only on /^### dvs end ###/, so a start
        # marker with no such line deletes everything from it to EOF -- and the
        # "refuse a truncated result" guard cannot see that, because the shebang
        # and non-empty tests both live ABOVE the block. Measured by review against
        # this function: with the end marker INDENTED, three operator lines below
        # the block went to zero, rc=0, and it printed the success line. A
        # tech-added VLAN route or ethtool line would be gone from the file root
        # runs at boot, silently, from the nightly root cron.
        #
        # Reachable without anyone hand-editing anything: the generator that wrote
        # these blocks appends the start and end markers in separate steps
        # (rda-db/src/setup/scripts/type/rc-local), so a run killed between them
        # leaves a start with no end.
        if ! grep -q '^### dvs end ###' "$real" 2>/dev/null; then
            echo "WARNING: $real has '### dvs start ###' with no matching '### dvs end ###' at column 0;"
            echo "WARNING: refusing to touch it (converting would delete every line below the marker)."
            continue
        fi

        # If our block is already there, the tuning is in place, so a legacy
        # block that reappeared (an operator paste, an old installer re-run) is
        # dropped rather than replaced -- otherwise the tuning is emitted twice.
        local mode=replace
        grep -q '^### dvs docker-migrated ###' "$real" 2>/dev/null && mode=drop

        backup="${real}.pre-docker"
        [[ -f "$backup" ]] || cp -p "$real" "$backup" 2>/dev/null || true

        local tmp
        tmp=$(mktemp "${real}.dividia.XXXXXX" 2>/dev/null) || return 0
        # Replace the marked region; everything outside it is the operator's and
        # is copied through untouched.
        awk -v mode="$mode" '
            /^### dvs start ###/ { inblock=1
                if( mode == "drop" ) next
                print "### dvs docker-migrated ###"
                print "# RPM-era NVR startup block, replaced on Docker migration."
                print "#"
                print "# Removed: the host-service starts for the database and the backend"
                print "# (the container stack owns both, and a host database daemon would"
                print "# contend for port 3306 under network_mode: host), and two helper"
                print "# binaries that shipped in RPMs no longer installed here. Also the"
                print "# rda_postboot run-once handler: nothing in the Docker tree writes"
                print "# that directory any more, and the handler referenced a log variable"
                print "# this file never defined, so every line of it errored anyway."
                print "#"
                print "# Kept: the CPU governor tuning below. Nothing else on a Docker NVR"
                print "# sets it, and these are video encoders."
                print "#"
                print "# Deliberately no literal command names above: an operator grepping"
                print "# this file for a service start must not get a hit from a comment"
                print "# saying it was taken out."
                print "if [ -e /usr/sbin/tuned-adm ] ; then"
                print "\ttuned-adm profile throughput-performance"
                print "else"
                print "\tfor CPU in 0 1 2 3 4 5 6 7 8 9 ; do"
                print "\t\tif [ -e /sys/devices/system/cpu/cpu${CPU}/cpufreq/scaling_governor ] ; then"
                print "\t\t\techo performance >/sys/devices/system/cpu/cpu${CPU}/cpufreq/scaling_governor"
                print "\t\tfi"
                print "\tdone"
                print "fi"
                print "### dvs end ###"
                next }
            /^### dvs end ###/ {
                # ONLY the marker that closes a block we are inside. Dropping
                # every end marker also removed the converted block own closer
                # (it is preceded by the docker-migrated marker, not by a start),
                # leaving the file with no end marker at all -- after which the
                # guard above correctly refuses to touch it, forever.
                if( inblock ) { inblock=0; next }
            }
            !inblock { print }
        ' "$real" > "$tmp" 2>/dev/null || { rm -f "$tmp"; return 0; }

        # Refuse to install a truncated or non-script result. rc.local runs as
        # root at boot; a mangled one is worse than a noisy one.
        if [[ ! -s "$tmp" ]] || ! head -1 "$tmp" | grep -q '^#!'; then
            rm -f "$tmp"
            echo "WARNING: refusing to rewrite $real (unexpected result); left as-is"
            return 0
        fi
        # Count the lines OUTSIDE our markers before and after. The shebang and
        # non-empty checks above only look at the top of the file, so they cannot
        # detect a truncated tail; this can.
        local before_outside after_outside
        # SAME anchor set on both sides, or this compares apples to oranges: the
        # first version anchored the "before" count on `dvs start` and the "after"
        # count on `dvs docker-migrated`, so a file containing both counted
        # different regions and the check refused a correct conversion.
        # PLAIN awk. `awk -E` is not "use ERE" -- it is gawk's --exec, so it took
        # the program text as a FILENAME, printed nothing, and both counts came
        # back empty; the fallback below then set them equal and silently disabled
        # this whole check. POSIX awk already supports alternation inside /.../,
        # so no flag is needed, and the fallback is gone so a future failure is
        # visible instead of self-muting.
        local vs_outside='/^### dvs (start|docker-migrated) ###/{i=1} /^### dvs end ###/{i=0;next} !i'
        before_outside=$( awk "$vs_outside" "$real" 2>/dev/null | wc -l | tr -d ' ' )
        after_outside=$( awk "$vs_outside" "$tmp" 2>/dev/null | wc -l | tr -d ' ' )
        if [ "${after_outside:-0}" -lt "${before_outside:-0}" ]; then
            rm -f "$tmp"
            echo "WARNING: refusing to rewrite $real: $before_outside lines outside the block became $after_outside"
            continue
        fi
        chmod --reference="$real" "$tmp" 2>/dev/null || chmod 755 "$tmp"
        mv -f "$tmp" "$real" 2>/dev/null || { rm -f "$tmp"; return 0; }
        echo "Replaced the RPM-era dvs block in $real (backup: $backup)"
    done
    return 0
}

# (a) /etc/sudoers.d/dividia = `dividia ALL=(ALL) NOPASSWD: ALL`, mode 440.
#     Only rewrites when missing or wrong, and NEVER installs a sudoers file
#     that fails `visudo -cf` validation (a bad drop-in can lock every user
#     out of sudo) — validate on a temp file, then atomically move into place.
ensure_dividia_sudoers() {
    local want="dividia ALL=(ALL) NOPASSWD: ALL"
    mkdir -p "$(dirname "$SUDOERS_FILE")" 2>/dev/null || true

    # Fast idempotent path: correct content already present. Fix only the mode
    # if it drifted off 440.
    if [[ -f "$SUDOERS_FILE" ]] && grep -qxF "$want" "$SUDOERS_FILE" 2>/dev/null; then
        local mode
        mode=$(stat -c '%a' "$SUDOERS_FILE" 2>/dev/null || stat -f '%Lp' "$SUDOERS_FILE" 2>/dev/null || echo "")
        if [[ -n "$mode" && "$mode" != "440" ]]; then
            chmod 440 "$SUDOERS_FILE" 2>/dev/null || true
            echo "nvr: fixed mode on $SUDOERS_FILE ($mode -> 440)"
        fi
        return 0
    fi

    # Missing or wrong: write a temp file, set 440, validate, then atomic move.
    local tmp
    tmp=$(mktemp "${SUDOERS_FILE}.new.XXXXXX" 2>/dev/null) || tmp="${SUDOERS_FILE}.new.$$"
    printf '%s\n' "$want" > "$tmp" 2>/dev/null || { rm -f "$tmp" 2>/dev/null || true; return 0; }
    chmod 440 "$tmp" 2>/dev/null || true
    if command -v visudo >/dev/null 2>&1; then
        if ! visudo -cf "$tmp" >/dev/null 2>&1; then
            rm -f "$tmp" 2>/dev/null || true
            echo "WARN: generated dividia sudoers failed visudo validation; left $SUDOERS_FILE untouched" >&2
            return 0
        fi
    fi
    if mv -f "$tmp" "$SUDOERS_FILE" 2>/dev/null; then
        echo "nvr: (re)installed $SUDOERS_FILE (dividia passwordless sudo)"
    else
        rm -f "$tmp" 2>/dev/null || true
        echo "WARN: could not install $SUDOERS_FILE" >&2
    fi
    return 0
}

# (b) /etc/docker/daemon.json must carry the json-file log cap
#     (max-size 10m, max-file 3), MERGED into any existing JSON so data-root /
#     storage-driver etc. are preserved. Only touches the file when the cap is
#     missing. Does NOT restart docker (too disruptive from an update): a
#     daemon.json change only affects containers created after a daemon
#     reload, and the compose per-service logging cap already bounds the
#     running stack — so we just fix the file and log that a docker restart
#     applies it to running containers.
ensure_docker_log_cap() {
    local daemon="$DOCKER_DAEMON_JSON"
    mkdir -p "$(dirname "$daemon")" 2>/dev/null || true

    local py=""
    if command -v python3 >/dev/null 2>&1; then py=python3
    elif command -v python >/dev/null 2>&1; then py=python; fi

    if [[ -n "$py" ]]; then
        # Authoritative JSON merge. Prints CHANGED / UNCHANGED so we know
        # whether to advise a restart, and only rewrites the file when a value
        # actually differs (byte-identical no-op on the second run).
        local py_body result
        # Bash 4.1 cannot reliably parse a heredoc nested inside $(). Capture
        # the program first, then keep the command substitution itself plain.
        IFS= read -r -d '' py_body <<'PYEOF' || true
import json, os, sys
path = sys.argv[1]
try:
    with open(path) as f:
        cfg = json.load(f)
    if not isinstance(cfg, dict):
        cfg = {}
except FileNotFoundError:
    cfg = {}
except Exception:
    # Unparseable existing daemon.json: do NOT rebuild from empty (that would
    # silently drop data-root and any other keys). Leave the file untouched and
    # report so an operator can fix the JSON by hand; the log cap is simply not
    # applied this run.
    print("PARSE_ERROR")
    sys.exit(0)

want_opts = {'max-size': '10m', 'max-file': '3'}
changed = False
if cfg.get('log-driver') != 'json-file':
    cfg['log-driver'] = 'json-file'
    changed = True
opts = cfg.get('log-opts')
if not isinstance(opts, dict):
    opts = {}
for k, v in want_opts.items():
    if opts.get(k) != v:
        opts[k] = v
        changed = True
cfg['log-opts'] = opts
if changed:
    tmp = path + '.new'
    with open(tmp, 'w') as f:
        json.dump(cfg, f, indent=2)
    os.replace(tmp, path)
    print('CHANGED')
else:
    print('UNCHANGED')
PYEOF
        result=$("$py" -c "$py_body" "$daemon" 2>/dev/null) || result="ERROR"
        case "$result" in
            *CHANGED*)
                echo "nvr: added Docker log cap to $daemon (10m x 3). Run 'systemctl restart docker' to apply it to running containers (the compose per-service logging cap already bounds the current stack)." ;;
            *UNCHANGED*)
                : ;;
            *PARSE_ERROR*)
                echo "WARN: $daemon is not valid JSON; left untouched (fix by hand, then re-run). Docker log cap NOT applied." >&2 ;;
            *)
                echo "WARN: could not update Docker log cap in $daemon" >&2 ;;
        esac
        return 0
    fi

    # No python: safe fallback. Write a fresh capped file only if none exists;
    # never text-merge into an existing daemon.json (would risk clobbering
    # data-root etc. without a real JSON parser).
    if [[ ! -f "$daemon" ]]; then
        cat > "$daemon" <<'DAEMONJSON'
{
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "10m",
        "max-file": "3"
    }
}
DAEMONJSON
        echo "nvr: wrote $daemon with Docker log cap (10m x 3). Restart docker to apply to running containers."
    elif ! grep -q 'max-size' "$daemon" 2>/dev/null; then
        echo "WARN: $daemon exists without a log cap and python is unavailable to merge safely; left it untouched." >&2
    fi
    return 0
}

# Atomic handoff from watchtower to cron. Invoked from cmd_update after
# `$COMPOSE up -d --quiet-pull --remove-orphans` returns success — gated on
# an explicit `compose_up_ok` flag, not on the surrounding `set -e` chain.
# The flag-based gate exists because cs2427 2026-05-28 surfaced a silent
# chain-break: cron was installed (L106 of cmd_update fires) but this
# function never ran (set -e aborted somewhere between L106 and the
# original tail-of-function call site). See
# operational_nvr_update_first_migration_skip_watchtower_drop.md.
#
# Always-attempt + silent-on-missing: handles fresh-install (no watchtower
# yet), already-removed (subsequent updates), and partial states (e.g.
# previous disable killed the container but `docker rm` raced with a
# manual cleanup). The `|| true` is load-bearing — a missing-container
# error here under set -e would abort cmd_update mid-flight on every
# steady-state run.
cmd_update_disable_watchtower() {
    docker stop dividia-nvr-watchtower-1 2>/dev/null || true
    docker rm dividia-nvr-watchtower-1 2>/dev/null || true
}

cmd_prune() {
    local quiet=0
    if [[ "${1:-}" == "--quiet" ]]; then quiet=1; fi

    # Two-pass prune. Dangling-only pass runs first with no time filter:
    # those <none>:<none> images are by definition replaced/orphaned and
    # immediately safe to remove. The 168h pass catches still-tagged
    # images that haven't been used in a week. Without the dangling pass,
    # active dev iteration (e.g. a feature channel rebuilt several times
    # in one day) accumulates GB of dangling images that the time filter
    # won't release until a week later — surfaced when cs256 hit /opt full
    # mid-pilot.
    [[ $quiet -eq 0 ]] && echo "Pruning unused images..."
    local out
    out=$(docker image prune -f 2>&1; docker image prune -a -f --filter "until=168h" 2>&1) || {
        [[ $quiet -eq 0 ]] && echo -e "${RED}prune failed:${NC} $out"
        return 0
    }

    # Sum the two "Total reclaimed space:" lines into a single number.
    # docker prints sizes like "4.154GB" or "523.1MB"; convert to bytes,
    # add, format back so the user sees one tidy number.
    local total_bytes=0 line size unit bytes
    while IFS= read -r line; do
        size=$(echo "$line" | sed -E 's/^Total reclaimed space: //; s/([0-9.]+)([A-Za-z]+)$/\1 \2/')
        [[ -z "$size" ]] && continue
        bytes=$(awk -v s="$size" 'BEGIN {
            split(s, parts, " ")
            n = parts[1] + 0
            unit = parts[2]
            mult = 1
            if (unit == "kB" || unit == "KB") mult = 1024
            else if (unit == "MB") mult = 1024 * 1024
            else if (unit == "GB") mult = 1024 * 1024 * 1024
            else if (unit == "TB") mult = 1024 * 1024 * 1024 * 1024
            printf "%d", n * mult
        }')
        total_bytes=$((total_bytes + bytes))
    done < <(echo "$out" | grep -E "^Total reclaimed space")

    local reclaimed
    reclaimed=$(awk -v b="$total_bytes" 'BEGIN {
        if (b >= 1024^3)      printf "Total reclaimed space: %.2fGB", b / 1024^3
        else if (b >= 1024^2) printf "Total reclaimed space: %.1fMB", b / 1024^2
        else if (b >= 1024)   printf "Total reclaimed space: %.1fkB", b / 1024
        else                  printf "Total reclaimed space: %dB", b
    }')

    if [[ $quiet -eq 1 ]]; then
        # Only print when we actually freed something, to keep nvr update output tight.
        [[ $total_bytes -gt 0 ]] && echo "$reclaimed"
    else
        echo "$reclaimed"
    fi
}

cmd_backup() {
    if ! $COMPOSE ps --format '{{.Service}}' 2>/dev/null | grep -q backend; then
        echo -e "${RED}ERROR: NVR backend container is not running${NC}"
        exit 1
    fi

    echo "Starting NVR backup..."
    $COMPOSE exec backend rda-db --backup
    local result=$?

    if [[ $result -ne 0 ]]; then
        echo -e "${RED}ERROR: Backup failed (exit code: $result)${NC}"
        exit 1
    fi

    local backup_dir
    backup_dir=$($COMPOSE exec -T backend sh -c 'ls -dt /videostore/vs1/backups/[0-9]* 2>/dev/null | head -1')
    [[ -n "$backup_dir" ]] && echo "Backup saved to: $backup_dir"

    # Save .env to VideoStore via backend container's /videostore bind mount
    if [[ -f .env ]]; then
        $COMPOSE cp .env backend:/videostore/vs1/backups/.env.save 2>/dev/null \
            && echo "Saved .env to VideoStore" \
            || echo -e "${YELLOW}WARN: Could not save .env to VideoStore${NC}"
    fi

    echo "Backup complete!"
}

cmd_channel() {
    if [[ -z "$1" ]]; then
        echo "Current channel: $(grep '^CHANNEL=' .env 2>/dev/null | cut -d= -f2)"
        echo ""
        echo "Usage: nvr channel <version|dev|dev-<suffix>>  (e.g., nvr channel 6.2)"
        return
    fi

    # Channel: "dev", "dev-<suffix>" (per-branch test channel), or major.minor (e.g., 6.2, 7.0).
    if [[ ! "$1" =~ ^(dev(-[a-z0-9-]+)?|[0-9]+\.[0-9]+)$ ]]; then
        echo -e "${RED}Invalid channel: $1 (must be 'dev', 'dev-<suffix>', or a version like 6.2)${NC}"; exit 1
    fi

    sed -i "s/^CHANNEL=.*/CHANNEL=$1/" .env
    echo "Channel switched to: $1"
    echo "Run 'nvr update' to pull images from the new channel."
}

cmd_db() {
    local db_pass
    db_pass=$(grep '^MYSQL_ROOT_PASSWORD=' .env 2>/dev/null | cut -d= -f2-)
    $COMPOSE exec db mariadb -u root -p"${db_pass:-lynn1094}" "$@" dtech
}

# Read the NVR's human name from dtech.Misc('system','name') — the same value
# the legacy /usr/local/bin/server_name script printed on RPM installs.
# `-N -B` = no column header, batch/tab mode, so callers get a bare value with
# no `grep -v sValue` post-processing (the legacy script only needed that
# because it didn't pass -N). `-T` (no pseudo-TTY) so the exec works when
# stdout is captured / run non-interactively (cron, ssh exec). Returns
# non-zero (and empty stdout, stderr swallowed) if the db container is down.
nvr_name() {
    local db_pass
    db_pass=$(grep '^MYSQL_ROOT_PASSWORD=' .env 2>/dev/null | cut -d= -f2-)
    $COMPOSE exec -T db mariadb -u root -p"${db_pass:-lynn1094}" -N -B dtech \
        -e "SELECT sValue FROM Misc WHERE sModule='system' AND sName='name'" 2>/dev/null
}

cmd_name() {
    local name
    if ! name=$(nvr_name); then
        echo "ERROR: could not read NVR name (is the db container running?)" >&2
        exit 1
    fi
    if [[ -z "$name" ]]; then
        echo "(name unset)"
    else
        echo "$name"
    fi
}

cmd_start() {
    # Reassert immediately before container startup as well as during update.
    # CentOS 6 boot does not reliably consume /etc/sysctl.d, and a reboot must
    # not reopen a window where an outbound socket can claim a service port.
    ensure_host_config || true
    ensure_videostore_mounts
    repair_untracked_compose_containers
    # Converge the optional aiengine overlay to host intent before boot-time
    # startup. `|| true`: never let an add-on reconcile block core startup.
    aiengine_reconcile || true
    # Core by name, add-on separately: an add-on image missing at boot (e.g.
    # registry unreachable on a cold start) must not fail core startup.
    local _core_svcs
    _core_svcs=$(core_services)
    if [[ -n "$_core_svcs" ]]; then
        # shellcheck disable=SC2086 — intentional word-split of the service list.
        $COMPOSE up -d $_core_svcs
    else
        $COMPOSE up -d
    fi
    if aiengine_intent_is_on; then
        $COMPOSE up -d "$AIENGINE_SERVICE" || echo "WARN: aiengine did not start; core services are up" >&2
    fi
}

# Mount every managed VideoStore. DO NOT DELETE THIS AS REDUNDANT WITH fstab.
#
# Since 2026-07-27 this is the PRIMARY VideoStore mounter, not a retry. The managed
# entry is `noauto`, so `mount -a` deliberately skips it (rc.sysinit on CentOS 6,
# local-fs.target on systemd) and nothing else on the box mounts a store. The reason
# is the shadow refusal below: `mount -a` knows nothing about it and would mount
# straight over root-backed video, burying footage that still fills root while
# thread_rollover deletes the Event rows pointing at it. Reproduced on a CentOS 6 box
# 2026-07-26 (80 MB buried); the refusal now runs at boot instead, because this
# function does the mounting. See ADR-045.
#
# ADR-045 moved VideoStore mounting host-side: docker/backend/docker-start Phase 2
# writes an /etc/fstab entry for every store whose mount it verified by identity.
# Those entries all carry `nofail`, which is mandatory (a missing disk must never
# strand a box at an emergency prompt) and which had a consequence people miss back
# when the entries were `auto`: systemd.mount(5) says nofail also removes the
# ordering, so the mount was only WANTED by local-fs.target and NOT ordered before
# it. Boot proceeded without waiting, a late-enumerating SATA/USB/HBA disk missed its
# window, and nothing tried again -- /videostore/vsN stayed a plain directory on root
# and mpengine recorded onto the root filesystem, which is the 2026-07-24 BCC
# failure. Under `noauto` that race is gone by construction (boot never attempts the
# mount at all), and this function is what closes it either way.
#
# `RequiresMountsFor=/videostore` in nvr.service does NOT cover this. It adds
# dependencies on the mounts required to REACH the path, and /videostore is a plain
# directory on root, so it resolves to `-.mount`. The per-store mounts live BELOW
# it. Covering them that way would mean enumerating vs1..vsN into the unit at
# install time, and stores get added later. So the mounting lives here instead: one
# place, no per-store knowledge, correct for a box that grows a third disk in 2028,
# and it also runs for an operator typing `nvr start` by hand. The CentOS 6 SysV
# script (61 of the 101 migrated BCC boxes have no systemd) carries the same logic
# inline because it calls compose directly rather than going through this CLI.
#
# It runs BEFORE compose on both init systems (nvr.service ExecStart here,
# /etc/init.d/nvr start on CentOS 6), which is what keeps ADR-045's mount-before-
# containers ordering under `noauto`.
#
# Only touches mount points that ALREADY have an fstab entry: it executes a decision
# Phase 2 already made and verified by identity, never a new mount of its own.
NVR_FSTAB="${NVR_FSTAB:-/etc/fstab}"
NVR_VIDEOSTORE_ROOT="${NVR_VIDEOSTORE_ROOT:-/videostore}"
# The marker docker-start Phase 2 writes above each entry it owns, as
# "<sentinel> <mountpoint>". Authorship is decided by this and nothing else.
NVR_FSTAB_SENTINEL="${NVR_FSTAB_SENTINEL:-# dividia-nvr videostore, managed}"
# Bytes of root-backed content above which mounting would hide real video. Must
# equal VIDEOSTORE_SHADOW_LIMIT_BYTES in docker/backend/docker-start and
# SHADOW_LIMIT_BYTES in rda-backend/src/lib/mount_check.py.
NVR_VIDEOSTORE_SHADOW_LIMIT_BYTES="${NVR_VIDEOSTORE_SHADOW_LIMIT_BYTES:-67108864}"
# Test seam for the host-only guard. Production leaves this at /.dockerenv.
NVR_DOCKERENV="${NVR_DOCKERENV:-/.dockerenv}"

ensure_videostore_mounts() {
    local fstab="$NVR_FSTAB"
    local root="$NVR_VIDEOSTORE_ROOT"
    local mp shadow

    # HOST only. This script also ships inside the backend image (it is synced to
    # /opt/dividia from there), and the backend runs `privileged`, so a `nvr start`
    # typed inside the container really could mount. It must not: the /videostore
    # bind carries `propagation: slave`, so a container-side mount does NOT reach
    # the host -- it would create a mount only the backend can see, while the host
    # and every other writer keep looking at the empty root-backed directory. That
    # is a worse version of the bug this exists to prevent.
    if [ -f "$NVR_DOCKERENV" ]; then
        return 0
    fi

    [ -d "$root" ] || return 0
    [ -r "$fstab" ] || return 0

    for mp in "$root"/vs*; do
        [ -d "$mp" ] || continue

        # OUR entry only, decided by the sentinel -- not by "some line names this
        # path". An operator's own line for the same mount point is theirs:
        # videostore_persist_mount explicitly declines to manage it, and mounting it
        # here would act on a decision Phase 2 refused to make.
        grep -qF "$NVR_FSTAB_SENTINEL $mp" "$fstab" 2>/dev/null || continue

        mountpoint -q "$mp" 2>/dev/null && continue

        # Refuse to mount over root-backed video, the same refusal Phase 2 and the
        # 5-minute self-heal both make. Mounting would make those bytes unreachable
        # while they still fill root -- and rollover would then delete the Event
        # rows pointing at them, because their globs resolve under the new mount and
        # find nothing. cs170 / cs662 / cs1181 all had to be relocated by hand for
        # exactly this. -x so the count never descends into an already-mounted disk.
        shadow=$( du -sx --block-size=1 "$mp" 2>/dev/null | awk '{print $1+0; exit}' )
        if [ "${shadow:-0}" -gt "$NVR_VIDEOSTORE_SHADOW_LIMIT_BYTES" ]; then
            echo "WARNING: NOT mounting $mp: it holds ${shadow} bytes on the ROOT filesystem."
            echo "WARNING: Mounting would hide that video while it still fills root. Relocate it"
            echo "WARNING: onto the physical store first, then start the NVR again."
            continue
        fi

        # NOT "fstab entry did not take at boot". Since the managed entry became
        # `noauto` (2026-07-27) the OS is SUPPOSED to leave it alone, so this is the
        # normal path on every boot rather than a fault being recovered. Measured on
        # cs256: this line prints once per boot. The old wording read as an anomaly,
        # which is alarm fatigue on every box and would send a tech looking for a
        # broken fstab entry that is working exactly as designed.
        echo "Mounting VideoStore $mp (managed noauto entry; guarded mount before the stack)"
        # Never fatal. `nvr start` must still bring the stack up: mpengine now
        # refuses to record to an unmounted store rather than filling root
        # (ADR-045), and rda-backend retries the mount every 5 minutes and alarms
        # [51201]. A stopped NVR is strictly worse than a loud degraded one.
        mount "$mp" || echo "WARNING: could not mount $mp; rda-backend will retry and alarm [51201]"
    done

    return 0
}

# Remove containers whose name belongs to this compose project but whose
# com.docker.compose.project label is missing or wrong. They look like
# orphans to compose, so the next `compose up -d` collides with their
# names ("Conflict. The container name '/dividia-nvr-<svc>-1' is already
# in use") and aborts mid-recreate, leaving the stack half-broken until
# the operator manually `docker rm -f`s the offender.
#
# Why a container ends up in this state:
#   - Older watchtower releases recreate containers via the Docker API
#     without preserving compose's labels. The new container has the
#     right name and image but no `com.docker.compose.project` — invisible
#     to `compose ps` / `compose up`. cs2585 hit this 2026-05 after
#     watchtower's session updated engine + connector four days earlier.
#   - Manual `docker run --name dividia-nvr-...` (rare).
#   - A killed-mid-create container left behind by a prior failed up.
#
# Removal is safe: anything matching the project name prefix is by
# convention owned by this compose project. The next `compose up -d`
# recreates it cleanly, restoring the label set.
#
# The function sweeps THREE distinct blockers, all of which stall `compose up`:
#
#   (a) Mislabeled orphans -- correct name, missing/wrong compose-project
#       label (the watchtower / manual-run cases above). Compose sees a name
#       conflict and aborts.
#
#   (b) Dead-state containers -- correctly labeled, so invisible to (a)'s
#       label filter, but Docker CANNOT start a container in `Dead` state, so
#       the next `compose up -d` errors and aborts the whole bring-up. Docker
#       leaves a container Dead when it fails to tear down its filesystem
#       layer, which an unclean power loss reliably produces. cs8 (Rosa's Cafe
#       #18) hit this 2026-08-03: a site power cycle left dividia-nvr-playback-1
#       Dead, every `nvr start` (the systemd boot service included) exited 1,
#       and the box sat unplayable for 80 minutes until removed by hand. A Dead
#       container holds no running state and cannot be restarted, so
#       force-removing it is safe; compose recreates it cleanly.
#
#   (c) Lost-RW-layer containers -- correctly labeled AND not Dead (they sit
#       `exited`/`created`), so invisible to BOTH filters above, but their overlay
#       read-write layer is gone from the layer store, so Docker fails every start
#       with "RWLayer of container <id> is unexpectedly nil" (recorded in
#       .State.Error) and the next `compose up -d` aborts. Same unclean-power-loss
#       root cause as (b), a different torn-write outcome: the container record
#       survived but its layer-metadata entry did not. cs2616 (Trade Show / Test)
#       hit this 2026-08-05 after five power-cycles -- the viewer stranded and the
#       local video wall stayed black until manual removal. Force-removing it is
#       safe (a viewer/playback container holds no persistent state; config lives
#       in the DB and bind mounts) and compose recreates it with a fresh layer.
#       See the (c) block below for why detection uses .State.Error, not
#       .GraphDriver, on the containerd-snapshotter fleet.
#
#       The CentOS 6 SysV boot script already clears this with `docker compose
#       rm -f` + runc-state cleanup before its inline `compose up`; this is what
#       brings the systemd path (ExecStart=/opt/dividia/nvr start -> cmd_start)
#       to parity.
#
#       Scope: this removes the Dead container with `docker rm -f`, which
#       cleared it on cs8 (a systemd box) with no further action. It does NOT
#       replicate the SysV path's `rm -rf /run/containerd/.../moby/*` runtime
#       -state wipe -- that is a CentOS-6-kernel-4.4 hard-reboot workaround, and
#       blindly wiping task state under systemd's own containerd is unproven and
#       risky. If `docker rm -f` ever cannot clear a Dead container, the loop
#       below warns and `compose up` proceeds exactly as it did before this
#       change: never worse than baseline, strictly better for the observed case.
repair_untracked_compose_containers() {
    local project="dividia-nvr"
    local untracked dead broken_layer offenders name err
    untracked=$(docker ps -a \
        --filter "name=^${project}-" \
        --format '{{.Names}}|{{.Label "com.docker.compose.project"}}' \
        2>/dev/null \
        | awk -F'|' -v p="$project" 'NF>=2 && $2 != p { print $1 }')
    # `|| true` is load-bearing, not decoration. The script runs under `set -e`
    # (no pipefail) and this function is called bare before `$COMPOSE up -d` in
    # both cmd_start (the systemd boot ExecStart) and cmd_update. A bare
    # `dead=$(docker ps ...)` propagates docker's exit status, so a transient
    # `docker ps` failure -- daemon not yet accepting connections at boot, the
    # exact race this function exists to survive -- would abort before compose
    # ever runs, making `nvr start` LESS reliable, not more. (The `untracked=`
    # query above dodges this only incidentally: it ends in a pipe, whose status
    # is awk's 0.) Tolerate the failure and let compose surface any real problem.
    dead=$(docker ps -a \
        --filter "name=^${project}-" \
        --filter "status=dead" \
        --format '{{.Names}}' \
        2>/dev/null) || true
    # (c) Lost-RW-layer containers -- a THIRD unclean-power-loss failure mode.
    #     Docker records the container config but its overlay RW-layer entry never
    #     fsync'd to the layer store, so on the next start the daemon returns
    #     "RWLayer of container <id> is unexpectedly nil" and records it in
    #     .State.Error. The container sits `exited`/`created` (observed exited on
    #     cs2616; `restarting` is included too for a box whose `restart: always`
    #     policy is mid-cycle when the daemon reloads), NOT `dead`, so the
    #     status=dead query above misses it, and its compose-project label is intact
    #     (so the untracked/label query misses it too), and every `compose up -d`
    #     -- hence every `nvr start` and the systemd boot service -- aborts on that
    #     start error. cs2616 (Trade Show / Test) hit this 2026-08-05 after five
    #     power-cycles: dividia-nvr-viewer-1 stranded and the local video wall stayed
    #     black until the container was removed by hand. Force-removing it lets the
    #     following `compose up -d` recreate it with a FRESH RW layer, which clears
    #     it. autoheal cannot: it only `docker restart`s running+unhealthy
    #     containers, and a restart re-issues the same start that hits the nil layer.
    #
    #     Detection keys off .State.Error, NOT .GraphDriver. On this fleet Docker 23
    #     uses the containerd snapshotter (driver=overlayfs, root /opt/docker), which
    #     leaves .GraphDriver null on HEALTHY containers too -- a null-GraphDriver
    #     test would force-remove the entire running stack. The match is deliberately
    #     narrow (the specific moby layer-corruption strings) so an unrelated start
    #     error -- a port already allocated, an OOM kill -- is never mistaken for
    #     corruption and removed. `|| true` for the same set -e reason as the dead
    #     query: a transient docker failure at boot must not abort the bring-up.
    broken_layer=$(
        for name in $(docker ps -a \
            --filter "name=^${project}-" \
            --filter "status=exited" \
            --filter "status=created" \
            --filter "status=restarting" \
            --format '{{.Names}}' 2>/dev/null); do
            err=$(docker inspect -f '{{.State.Error}}' "$name" 2>/dev/null) || continue
            # Leading `(` on the pattern is REQUIRED, not style: a case pattern's
            # bare `)` inside this `$(...)` command substitution is misparsed as the
            # end of the substitution on CentOS 6's Bash 4.1 ("syntax error near
            # unexpected token `;;'"). `(pattern)` balances the parens.
            case "$err" in
                (*"unexpectedly nil"*|*"RW layer for container"*)
                    printf '%s\n' "$name" ;;
            esac
        done
    ) || true
    # A single container can satisfy more than one query (a Dead container may also
    # be mislabeled); dedupe so it is only removed (and reported) once.
    offenders=$(printf '%s\n%s\n%s\n' "$untracked" "$dead" "$broken_layer" | awk 'NF' | sort -u)
    if [[ -z "$offenders" ]]; then
        return 0
    fi
    local count
    count=$(printf '%s\n' "$offenders" | wc -l | tr -d ' ')
    echo -e "${YELLOW}Detected $count container(s) with missing/stale compose labels, in Dead state, or with a lost RW layer; removing so compose can recreate cleanly:${NC}" >&2
    while IFS= read -r name; do
        [[ -z "$name" ]] && continue
        echo "  - $name" >&2
        docker rm -f "$name" >/dev/null 2>&1 || \
            echo -e "    ${RED}WARN: failed to remove $name${NC}" >&2
    done <<< "$offenders"
}

cmd_stop() {
    $COMPOSE down
}

cmd_restart() {
    # `compose up -d --force-recreate`, NOT `compose restart`.
    #
    # `compose restart` bounces containers in place and does NOT evaluate
    # depends_on conditions, so `nvr restart engine` can bring mpengine back
    # against a backend that is still starting. That is the ordering bypass
    # behind the 2026-07-24 root-recording incident (and cs1666, where a plain
    # restart did not clear a wedged recorder but --force-recreate did).
    # `up -d` honors `condition: service_healthy`, so the engine waits for
    # backend, which waits for the DB.
    #
    # --force-recreate is required, not decoration: on an unchanged config a
    # bare `up -d` is a NO-OP, so `nvr restart` would silently do nothing.
    # Recreating also re-applies the compose spec, which is how the per-service
    # json-file log cap (70aff728c) reaches a container at all; `restart` keeps
    # whatever LogConfig the container was created with.
    #
    # Recreation is safe here: every piece of state lives in a bind mount or a
    # named volume, never in the container's writable layer.
    $COMPOSE up -d --force-recreate "$@"
}

cmd_version() {
    echo -e "${BLUE}=== NVR Image Versions ===${NC}"
    for svc in $($COMPOSE config --services 2>/dev/null); do
        local cid
        cid=$($COMPOSE ps -q "$svc" 2>/dev/null)
        if [[ -n "$cid" ]]; then
            local ver commit channel
            ver=$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.version"}}' "$cid" 2>/dev/null || echo "?")
            commit=$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$cid" 2>/dev/null || echo "?")
            channel=$(docker inspect --format '{{index .Config.Labels "channel"}}' "$cid" 2>/dev/null || echo "?")
            printf "  %-12s version=%-8s commit=%-10s channel=%s\n" "$svc" "$ver" "$commit" "$channel"
        else
            printf "  %-12s (not running)\n" "$svc"
        fi
    done
}

cmd_shell() {
    local svc="${1:-backend}"
    $COMPOSE exec "$svc" bash
}

cmd_vm_shell() {
    # Called by the Windows nvr.cmd `nvr shell` branch. The Windows-side
    # admin-key passphrase prompt is what authenticated the tech; by the
    # time we reach this function, SSH has already accepted the admin
    # key and `sudo` has run without prompt. We drop back to the ssh
    # login user with `sudo -u dividia -i` — root would give an unexpected
    # shell with the wrong home/prompt.
    #
    # Hardcoded to `dividia` (not `${SUDO_USER}`) because cloud-init only
    # provisions the `dividia` account. Trusting SUDO_USER would silently
    # drop into whichever account sudo happened to be invoked from — if
    # ops later adds another sudoer (e.g. a `deployer` service account),
    # `nvr shell` would become identity-laundering. Fail loud instead.
    exec sudo -u dividia -i
}

cmd_migrate_scalewatcher() {
    # Import a legacy 2014 Windows Scale Watcher backup zip into this NVR.
    # $1 must be an absolute path to the zip file, either:
    #   - inside the backend container's /videostore mount (customer placed
    #     the zip on the Windows-host SMB share at C:\Dividia\VideoStore\
    #     migrate-staging\ — shows up as /videostore/vs1/migrate-staging/
    #     inside the container), OR
    #   - any Windows-host path — we docker cp it into the container
    # $2+ optional --dry-run flag.
    local zip_path="$1"
    local dry_flag=""
    shift || true
    for arg in "$@"; do
        case "$arg" in
            --dry-run) dry_flag="--dry-run" ;;
            *)
                echo -e "${RED}ERROR: unknown flag: $arg${NC}" >&2
                exit 1
                ;;
        esac
    done

    if [[ -z "$zip_path" ]]; then
        echo -e "${RED}ERROR: usage: nvr migrate-scalewatcher <path-to-zip> [--dry-run]${NC}" >&2
        exit 1
    fi

    # Defense against shell-injection via zip_path: reject anything that
    # isn't plain filesystem-path-looking (letters, digits, /, \, :, ., _,
    # -, space).  We do NOT shell-interpolate this value into bash -c
    # below, but keeping a strict charset avoids pushing the problem to
    # downstream tools that may be less careful.
    if [[ "$zip_path" =~ [\`\$\;\"\'\&\|\<\>] ]]; then
        echo -e "${RED}ERROR: zip path contains shell metacharacters${NC}" >&2
        exit 1
    fi
    # Defense against path-traversal: reject any ../ component.  Charset
    # guard above allows dots, so '/videostore/../etc/passwd' passes the
    # prefix check below.  realpath-based canonicalization would be
    # stronger but realpath(1) is inconsistent across BSD/GNU; the explicit
    # ..-rejection is portable and sufficient.
    if [[ "$zip_path" == *'/..'* || "$zip_path" == *'..'/* || "$zip_path" == '..' || "$zip_path" == *'/../'* ]]; then
        echo -e "${RED}ERROR: zip path contains .. traversal components${NC}" >&2
        exit 1
    fi

    if ! $COMPOSE ps --format '{{.Service}}' 2>/dev/null | grep -q backend; then
        echo -e "${RED}ERROR: backend container is not running${NC}" >&2
        exit 1
    fi

    # Stage inside the backend container so cleanup is bounded to one mount.
    local stage_dir="/tmp/migrate-scalewatcher-$$"
    $COMPOSE exec -T backend mkdir -p "$stage_dir"

    # Resolve the in-container path for the zip.  Inside the backend
    # container the SMB-mapped VideoStore share is mounted at /videostore/
    # vs1/ (the host-side path /mnt/videostore/ documented in the plan
    # does NOT exist inside the container).  If the caller handed us a
    # /videostore path, it's already container-local.  Everything else
    # goes through docker cp.
    local container_zip
    if [[ "$zip_path" == /videostore/* ]]; then
        container_zip="$zip_path"
    elif [[ "$zip_path" == /mnt/videostore/* ]]; then
        # Rewrite the documented host alias to the real container path
        container_zip="/videostore/vs1/${zip_path#/mnt/videostore/}"
    else
        # Arbitrary host path: docker cp into the stage dir
        container_zip="$stage_dir/scalewatcher-backup.zip"
        $COMPOSE cp "$zip_path" "backend:$container_zip" || {
            echo -e "${RED}ERROR: cannot copy $zip_path into backend container${NC}" >&2
            exit 1
        }
    fi

    # Pass zip path via env, not string interpolation — prevents shell
    # injection even if earlier guards are bypassed.  `bash -c '<script>' _
    # arg1 arg2` style with "$1"/"$2" inside the script is the one safe way
    # to forward user input through bash -c.
    # Use `|| result=$?` so `set -e` on the outer script doesn't abort
    # before we can capture the exit status and clean up stage_dir.  The
    # naive `cmd; local result=$?` idiom is dead code under set -e: if cmd
    # fails, the script exits immediately and the cleanup + pretty-error
    # block never run, leaving /tmp/migrate-scalewatcher-<pid> behind on
    # the backend container.
    local result=0
    $COMPOSE exec -T \
        -e MIGRATE_STAGE="$stage_dir" \
        -e MIGRATE_ZIP="$container_zip" \
        -e MIGRATE_DRY_FLAG="$dry_flag" \
        backend bash -c '
            set -e
            cd "$MIGRATE_STAGE"
            unzip -o "$MIGRATE_ZIP" -d unpacked/
            # Zip unpacks to a single top-level directory containing
            # manifest.json + dtech.sql.  Pin the exact expected shape
            # (exactly one top-level dir, with a manifest).
            mapfile -t export_dirs < <(find unpacked -maxdepth 1 -mindepth 1 -type d)
            if [[ ${#export_dirs[@]} -eq 0 ]]; then
                export_dir="unpacked"
            elif [[ ${#export_dirs[@]} -eq 1 ]]; then
                export_dir="${export_dirs[0]}"
            else
                echo "ERROR: zip contains multiple top-level directories" >&2
                exit 1
            fi
            if [[ ! -f "$export_dir/manifest.json" ]]; then
                echo "ERROR: manifest.json not found under $export_dir" >&2
                exit 1
            fi
            # Quote MIGRATE_DRY_FLAG to prevent word-splitting surprises if
            # parent-side validation ever loosens.  Empty string is a valid
            # argv that Python getopt/argparse rejects cleanly.
            if [[ -n "$MIGRATE_DRY_FLAG" ]]; then
                rda-db --migrate-scalewatcher "$export_dir" "$MIGRATE_DRY_FLAG"
            else
                rda-db --migrate-scalewatcher "$export_dir"
            fi
        ' || result=$?

    # Cleanup — always, even on failure, to avoid /tmp buildup.
    $COMPOSE exec -T backend rm -rf "$stage_dir" 2>/dev/null || true

    if [[ $result -ne 0 ]]; then
        echo -e "${RED}ERROR: migrate-scalewatcher failed (exit $result)${NC}" >&2
        exit 1
    fi

    if [[ -z "$dry_flag" ]]; then
        echo -e "${GREEN}Migration complete. Restarting services so new Camera/Device/POS config takes effect...${NC}"
        # Full restart: engine/playback/viewer all cache dvs.conf + DB
        # rows at startup and won't see imported data otherwise.  Skip db
        # (kept up).
        #
        # up -d --force-recreate rather than `compose restart`, for the same
        # reason as cmd_restart: `restart` ignores depends_on conditions, so the
        # engine could come back ahead of the backend it depends on.
        $COMPOSE up -d --force-recreate backend engine connector playback viewer
    fi
}

cmd_find() {
    # Discover hosts on the local network using arp-scan in the backend
    # container.  Default mode auto-detects physical IPv4 NICs (eth*, en*,
    # bond*, br0) and runs arp-scan --localnet against each.  --interface
    # and --cidr override.  Vendor names come from the IEEE OUI database
    # bundled with the arp-scan package; duplicates are NOT deduplicated
    # so IP conflicts are visible.
    local iface=""
    local cidr=""

    while [[ $# -gt 0 ]]; do
        case "$1" in
            -i|--interface)
                if [[ -z "${2:-}" || "${2:0:1}" == "-" ]]; then
                    echo -e "${RED}ERROR: --interface requires a value${NC}" >&2
                    return 1
                fi
                iface="$2"; shift 2 ;;
            -c|--cidr)
                if [[ -z "${2:-}" || "${2:0:1}" == "-" ]]; then
                    echo -e "${RED}ERROR: --cidr requires a value${NC}" >&2
                    return 1
                fi
                cidr="$2"; shift 2 ;;
            -h|--help)
                cat <<'FINDHELP'
Usage: nvr find [options]

Discover hosts on the local network using arp-scan. Shows IP, MAC,
and vendor (looked up from the IEEE OUI database). Duplicates are
NOT deduplicated -- that's how you spot IP conflicts.

Options:
  -i, --interface IFACE   Scan only the named interface
  -c, --cidr CIDR         Scan a specific subnet (e.g. 192.168.0.0/24)
  -h, --help              Show this help

With no options, scans every IPv4-bearing physical NIC (eth*, en*,
bond*, br0). Docker bridges, veth, VPN tunnels are skipped.

Examples:
  nvr find                       # all physical NICs
  nvr find -i eth0               # eth0 only
  nvr find -c 10.0.0.0/24        # specific subnet on default iface
  nvr find -i eth1 -c 10.0.0.0/24
FINDHELP
                return 0 ;;
            *)
                echo -e "${RED}ERROR: unknown option: $1${NC}" >&2
                cmd_find --help >&2
                return 1 ;;
        esac
    done

    # Validate option values.  docker compose exec passes argv directly,
    # so shell-injection isn't possible -- this is defense in depth and
    # gives the user a clearer error than arp-scan's own complaint.
    # Iface regex anchors the first char to alnum so a value like `-rf`
    # can't survive validation and end up parsed as a flag by arp-scan.
    if [[ -n "$iface" && ! "$iface" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
        echo -e "${RED}ERROR: interface name contains illegal characters${NC}" >&2
        return 1
    fi
    if [[ -n "$cidr" ]]; then
        if [[ ! "$cidr" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}(/[0-9]{1,2})?$ ]]; then
            echo -e "${RED}ERROR: --cidr must be a dotted-quad with optional /mask${NC}" >&2
            return 1
        fi
        # Bounds-check octets and prefix.  Loose regex would accept
        # 999.999.999.999/99 and let arp-scan emit a less actionable
        # error 100ms later.
        local cidr_addr="${cidr%%/*}"
        local cidr_pfx="${cidr#*/}"
        [[ "$cidr_pfx" == "$cidr" ]] && cidr_pfx=""
        local IFS_save="$IFS"; IFS=.
        local octets=($cidr_addr)
        IFS="$IFS_save"
        for o in "${octets[@]}"; do
            if (( o > 255 )); then
                echo -e "${RED}ERROR: CIDR octet $o > 255${NC}" >&2
                return 1
            fi
        done
        if [[ -n "$cidr_pfx" ]] && (( cidr_pfx > 32 )); then
            echo -e "${RED}ERROR: CIDR prefix /$cidr_pfx > 32${NC}" >&2
            return 1
        fi
    fi

    # `grep -qx backend` (exact-line match) so a future service named
    # `backend-foo` doesn't accidentally satisfy the gate.
    if ! $COMPOSE ps --format '{{.Service}}' 2>/dev/null | grep -qx backend; then
        echo -e "${RED}ERROR: backend container is not running${NC}" >&2
        exit 1
    fi

    if [[ -n "$cidr" && -n "$iface" ]]; then
        $COMPOSE exec -T backend arp-scan -I "$iface" --plain "$cidr"
    elif [[ -n "$cidr" ]]; then
        $COMPOSE exec -T backend arp-scan --plain "$cidr"
    elif [[ -n "$iface" ]]; then
        $COMPOSE exec -T backend arp-scan -I "$iface" --localnet --plain
    else
        # Default: physical NICs only.  Filter is name-prefix + IPv4-
        # bearing.  Track per-iface success so an all-failed run exits
        # 1 (e.g. NET_RAW missing) -- partial success still exits 0.
        # No user input reaches the bash -c body; shell-quoting is fine.
        $COMPOSE exec -T backend bash -c '
            set -u
            mapfile -t candidates < <(
                ip -4 -o addr show \
                    | awk "{print \$2}" \
                    | sort -u \
                    | grep -E "^(eth|en|bond|br0)" \
                    | grep -vE "^(docker|veth|br-|tun|wg|virbr)"
            )
            if [[ ${#candidates[@]} -eq 0 ]]; then
                echo "ERROR: no physical IPv4 interfaces found (looking for eth*, en*, bond*, br0)" >&2
                exit 1
            fi
            ok=0
            first=1
            for i in "${candidates[@]}"; do
                [[ $first -eq 1 ]] || echo
                echo "=== $i ==="
                if arp-scan -I "$i" --localnet --plain; then
                    ok=$((ok+1))
                fi
                first=0
            done
            if [[ $ok -eq 0 ]]; then
                echo "ERROR: every interface scan failed (NET_RAW missing? arp-scan absent?)" >&2
                exit 1
            fi
            exit 0
        '
    fi
}

################################################################################
# Optional aiengine add-on lifecycle
#
# The whole add-on is a compose overlay gated on durable host intent. `enable`
# appends the overlay LAST in COMPOSE_FILE and starts the service; `disable`
# drops the overlay and removes the container; reconcile (from update, start,
# boot, and the CLI) converges live state to intent. Only root, holding the
# existing `nvr update` lock, mutates .env / intent / the overlay / containers.
# A non-root caller is read-only: it detects drift and warns, never writes.
#
# Image build, model files, the device-ID generator, key validation, and the
# healthcheck all ship from the EXTERNAL aiengine repository. This code never
# reimplements the licensing algorithm.
################################################################################

# Copy the existing file's owner + mode onto a temp replacement. A fresh temp
# file lands as root:root 0644 under root's umask; renaming that over .env would
# DOWNGRADE it from the dividia:docker 0640 contract and expose
# MYSQL_ROOT_PASSWORD to every local user. `--reference` on GNU coreutils (all
# four supported hosts); stat/octal fallback keeps it working on a dev Mac test.
env_preserve_perms() {
    local ref="$1" tmp="$2"
    chmod --reference="$ref" "$tmp" 2>/dev/null \
        || chmod "$(stat -c '%a' "$ref" 2>/dev/null || echo 640)" "$tmp" 2>/dev/null || true
    chown --reference="$ref" "$tmp" 2>/dev/null || true
}

# Atomic single-variable rewrite of the host .env. Preserves every other line
# (unknown vars, comments, blanks) byte-for-byte, then swaps in via rename so a
# crash mid-write can never truncate .env (which would strand COMPOSE_FILE and
# break every subsequent compose call). Replaces the first matching KEY= line;
# appends if absent. Keys here are fixed identifiers, so the anchor is safe.
# The value is passed through ENVIRON (not `awk -v`) so a backslash in a value
# is never interpreted as an escape sequence.
env_write_var() {
    local key="$1" value="$2"
    [[ -f "$AIENGINE_ENV_FILE" ]] || { echo "ERROR: $AIENGINE_ENV_FILE missing" >&2; return 1; }
    local tmp="${AIENGINE_ENV_FILE}.new.$$"
    AENV_K="$key" AENV_V="$value" awk '
        BEGIN { k=ENVIRON["AENV_K"]; v=ENVIRON["AENV_V"]; done=0 }
        !done && index($0, k "=") == 1 { print k "=" v; done=1; next }
        { print }
        END { if (!done) print k "=" v }
    ' "$AIENGINE_ENV_FILE" > "$tmp" || { rm -f "$tmp"; return 1; }
    env_preserve_perms "$AIENGINE_ENV_FILE" "$tmp"
    mv -f "$tmp" "$AIENGINE_ENV_FILE"
}

# Atomic removal of a KEY= line from .env (same temp+rename discipline).
env_remove_var() {
    local key="$1"
    [[ -f "$AIENGINE_ENV_FILE" ]] || return 0
    local tmp="${AIENGINE_ENV_FILE}.new.$$"
    AENV_K="$key" awk 'BEGIN { k=ENVIRON["AENV_K"] } index($0, k "=") == 1 { next } { print }' \
        "$AIENGINE_ENV_FILE" > "$tmp" || { rm -f "$tmp"; return 1; }
    env_preserve_perms "$AIENGINE_ENV_FILE" "$tmp"
    mv -f "$tmp" "$AIENGINE_ENV_FILE"
}

compose_file_get() {
    sed -n 's/^COMPOSE_FILE=//p' "$AIENGINE_ENV_FILE" 2>/dev/null | head -1
}

# Recompute COMPOSE_FILE so the aiengine overlay appears EXACTLY once and LAST,
# preserving the base + host overlay (prod/co6/windows) order. `want` is
# present | absent. Deduplicates a doubly-appended overlay as a side effect.
compose_overlay_reconcile() {
    local want="$1"
    local current
    current=$(compose_file_get)
    [[ -n "$current" ]] || { echo "ERROR: COMPOSE_FILE not set in $AIENGINE_ENV_FILE" >&2; return 1; }
    local rebuilt=() part
    local IFS=':'
    for part in $current; do
        [[ "$part" == "$AIENGINE_OVERLAY" ]] && continue
        [[ -n "$part" ]] && rebuilt+=("$part")
    done
    [[ "$want" == "present" ]] && rebuilt+=("$AIENGINE_OVERLAY")
    local joined
    joined=$(IFS=':'; echo "${rebuilt[*]}")
    # Skip the rewrite when COMPOSE_FILE already matches. On a steady-state boot
    # the overlay state is already correct, so reconcile does not touch .env at
    # all — which also shrinks the (lockless) boot-reconcile vs update-cron race
    # to the rare moment the overlay actually has to change.
    [[ "$joined" == "$current" ]] && return 0
    env_write_var COMPOSE_FILE "$joined"
}

# True when the aiengine overlay is currently listed in COMPOSE_FILE.
compose_overlay_present() {
    compose_file_get | tr ':' '\n' | grep -qxF "$AIENGINE_OVERLAY"
}

# Print the intent: enabled | legacy-provisioned | disabled. An absent or
# unrecognized file reads as disabled (fail-safe off on a fresh NVR).
aiengine_intent_read() {
    local v=""
    if [[ -f "$AIENGINE_INTENT_FILE" ]]; then
        v=$(tr -d '[:space:]' < "$AIENGINE_INTENT_FILE" 2>/dev/null) || true
    fi
    case "$v" in
        enabled|legacy-provisioned|disabled) echo "$v" ;;
        *) echo "disabled" ;;
    esac
}

# enabled and legacy-provisioned both mean "run the local container".
aiengine_intent_is_on() {
    local v; v=$(aiengine_intent_read)
    [[ "$v" == "enabled" || "$v" == "legacy-provisioned" ]]
}

# Atomic intent write (root-only mutation). Directory root:docker 0750; file
# 0640 so a non-root `nvr update` reconcile can still READ existing intent.
aiengine_intent_write() {
    local value="$1"
    mkdir -p "$AIENGINE_INTENT_DIR"
    chown root:docker "$AIENGINE_INTENT_DIR" 2>/dev/null || true
    chmod 0750 "$AIENGINE_INTENT_DIR" 2>/dev/null || true
    local tmp="${AIENGINE_INTENT_FILE}.new.$$"
    printf '%s\n' "$value" > "$tmp" || { rm -f "$tmp"; return 1; }
    chmod 0640 "$tmp" 2>/dev/null || true
    mv -f "$tmp" "$AIENGINE_INTENT_FILE"
}

aiengine_require_root() {
    [[ $EUID -eq 0 ]] || { echo -e "${RED}ERROR: run 'sudo nvr addon aiengine $1'${NC}" >&2; return 1; }
}

# Take the SAME lock as `nvr update` so an operator `nvr addon` and the nightly
# update cron cannot race on .env / intent / containers. Non-blocking: a second
# holder exits cleanly rather than piling up.
aiengine_lock() {
    exec 9>/var/lock/nvr-update.lock
    flock -n 9 || { echo "another nvr update/addon operation is in progress" >&2; return 1; }
}

# Core-service pull, isolated from optional add-on images. The whole-project
# `$COMPOSE pull` / `up --quiet-pull` would try to fetch aiengine too once its
# overlay is in COMPOSE_FILE, so an add-on registry outage would block core
# updates. Pull required core services BY NAME instead; the add-on image is
# pulled separately in aiengine_reconcile. Required-core pull failure stays
# fatal, exactly as the prior whole-project pull was.
core_services() {
    # Exclude the aiengine add-on: it has its OWN guarded `up` step (its image
    # can lag/be-absent, and must never take the core lifecycle down). hme stays
    # in the core set on purpose — hme_reconcile_config runs BEFORE the core up
    # and only leaves the hme overlay in COMPOSE_FILE when its image is proven
    # pullable, so the by-name core up is the step that starts a validated hme.
    $COMPOSE config --services 2>/dev/null | grep -vxF "$AIENGINE_SERVICE" || true
}

core_service_pull() {
    local svcs
    # This pull runs BEFORE hme_reconcile_config has validated hme's tag this
    # cycle, so also drop hme here (core_services keeps it for the post-reconcile
    # up). A lagging/absent hme ${CHANNEL} tag would otherwise abort the whole
    # pre-reconcile core pull under `set -e` and block core updates fleet-wide.
    # hme is pulled+started later by the validated by-name core up; aiengine is
    # already excluded by core_services and pulled in aiengine_reconcile.
    svcs=$(core_services | grep -vxF "hme")
    # Fall back to the whole-project pull only if the service list is
    # unreadable (compose config failed) — never silently pull nothing.
    if [[ -z "$svcs" ]]; then
        $COMPOSE pull --quiet
        return
    fi
    # shellcheck disable=SC2086 — intentional word-split of the service list.
    $COMPOSE pull --quiet $svcs
}

# Pull ONLY the aiengine image, in its own step. Never folded into the core
# pull. A failure here is a warning to the caller, not a fatal core error.
aiengine_pull() {
    $COMPOSE pull --quiet "$AIENGINE_SERVICE"
}

# The single reconcile entry point, called from update, start, boot, and the
# CLI. Converges COMPOSE_FILE + the add-on image to host intent.
#   enabled | legacy-provisioned -> overlay present, add-on image pulled
#   disabled | absent            -> overlay absent
# Root mutates; non-root is READ-ONLY (drift warning only). The `up` that
# actually creates/removes the container is owned by the caller (cmd_update /
# cmd_start / cmd_addon_aiengine), so reconcile stays side-effect-light and can
# run from every boot path including CentOS 6.
aiengine_reconcile() {
    local on=0
    aiengine_intent_is_on && on=1

    if [[ $EUID -ne 0 ]]; then
        local have=0
        compose_overlay_present && have=1
        if [[ $on -ne $have ]]; then
            echo "WARN: aiengine intent ($(aiengine_intent_read)) does not match COMPOSE_FILE overlay state; run 'sudo nvr addon aiengine status'" >&2
        fi
        return 0
    fi

    if [[ $on -eq 1 ]]; then
        # Cred-gate, matching aiengine_env_normalize. Intent alone is NOT
        # enough: without the license key the container crash-loops keyless
        # every update and boot. Gate on the KEY only -- the key is the license;
        # the device-id is derived from hardware and its file is optional (a
        # legacy image regenerates it at start and never ships a file writer).
        # Requiring a device-id file here would wrongly strip the add-on on
        # every boot/update of a legacy-image box.
        if [[ ! -f "$AIENGINE_CONFIG_DIR/aiengine-key" ]]; then
            echo "WARN: aiengine intent is '$(aiengine_intent_read)' but aiengine-key is missing; leaving the add-on off" >&2
            compose_overlay_reconcile absent || return 1
            return 0
        fi
        compose_overlay_reconcile present || return 1
        # Optional-image pull in its OWN step. On failure keep any prior
        # healthy image/container: an add-on registry outage must never block
        # the core update. First `enable` is transactional (see cmd below).
        aiengine_pull || echo "WARN: aiengine image pull failed; keeping any existing image/container" >&2
    else
        compose_overlay_reconcile absent || return 1
    fi
}

# Restore normalizer. A restored .env can carry an aiengine overlay reference or
# AIENGINE_* vars onto a box whose intent or credential files are missing, which
# would re-enable a broken container. Recompute COMPOSE_FILE from intent and
# strip AIENGINE_* when intent is disabled or the credentials are absent. Restore
# never re-enables aiengine implicitly. Invoked after any .env restore
# (rda-db/src/backup/dvs30.py and install-nvr.sh .env.save copy-back).
aiengine_env_normalize() {
    # Gate on the KEY only (the license); the device-id file is optional and
    # derived from hardware, so a legacy-image box has a valid key but no
    # device-id file. Matches aiengine_reconcile.
    local creds_ok=0
    [[ -f "$AIENGINE_CONFIG_DIR/aiengine-key" ]] && creds_ok=1
    if aiengine_intent_is_on && [[ $creds_ok -eq 1 ]]; then
        compose_overlay_reconcile present || return 1
    else
        compose_overlay_reconcile absent || return 1
        env_remove_var AIENGINE_TAG || true
        env_remove_var AIENGINE_REGISTRY || true
        env_remove_var AIENGINE_IMAGE || true
    fi
}

# Boot-time compose bring-up used by the CentOS 6 SysV init script and the
# migration finalize. Same reconcile + pull isolation as cmd_update/cmd_start,
# but WITHOUT the host-config/videostore self-heal the SysV script already runs
# inline before it. Core up by name so a missing add-on image cannot fail the
# boot; add-on started separately and non-fatally.
cmd_boot_up() {
    aiengine_reconcile || true
    local _core_svcs
    _core_svcs=$(core_services)
    if [[ -n "$_core_svcs" ]]; then
        # shellcheck disable=SC2086 — intentional word-split of the service list.
        $COMPOSE up -d $_core_svcs || return 1
    else
        $COMPOSE up -d || return 1
    fi
    if aiengine_intent_is_on; then
        $COMPOSE up -d "$AIENGINE_SERVICE" || echo "WARN: aiengine did not start; core services are up" >&2
    fi
}

# --- Local database demand ----------------------------------------------------
# Detect whether configured LPR/object work can call a LOCAL endpoint. Reads
# BOTH config tables (never Camera.fLPR or a bare DeviceType 61 row). Prints one
# "kind|bCamera|bID|sName|sIP|bPort" line per enabled row referencing a type-61
# Device. bPort is read as-is (no COALESCE 88): a NULL bPort is a broken config
# at runtime today, not implicit-88 local demand. Best-effort: an unreachable DB
# yields no rows and the caller decides (fail-open lives in migration, not here).
aiengine_demand_rows() {
    cmd_db --skip-column-names -B -e "
        SELECT 'lpr', L.bCamera, D.bID, D.sName, D.sIP, D.bPort
        FROM LprConfig L JOIN Device D ON D.bID = L.bDeviceID
        WHERE L.fEnable = 1 AND D.bType = 61 AND D.bPort IS NOT NULL
        UNION ALL
        SELECT 'object', O.bCamera, D.bID, D.sName, D.sIP, D.bPort
        FROM ObjectDetectConfig O JOIN Device D ON D.bID = O.bDeviceID
        WHERE O.fEnable = 1 AND D.bType = 61 AND D.bPort IS NOT NULL;
    " 2>/dev/null || true
}

# Classify an address as local: loopback, ::1, this host's name(s), or an
# address on a current interface. Empty/unresolvable is NOT local.
aiengine_addr_is_local() {
    local addr="$1"
    [[ -z "$addr" ]] && return 1
    case "$addr" in
        localhost|127.*|::1) return 0 ;;
    esac
    local host fqdn
    host=$(hostname 2>/dev/null) || host=""
    fqdn=$(hostname -f 2>/dev/null) || fqdn=""
    [[ -n "$host" && "$addr" == "$host" ]] && return 0
    [[ -n "$fqdn" && "$addr" == "$fqdn" ]] && return 0
    # An address on a current interface.
    if command -v ip >/dev/null 2>&1; then
        ip -o addr show 2>/dev/null | grep -qw "$addr" && return 0
    fi
    return 1
}

# Print local direct-demand camera IDs (unique). Used by disable/status.
aiengine_local_camera_ids() {
    local rows kind cam bid name ip port
    rows=$(aiengine_demand_rows)
    [[ -n "$rows" ]] || return 0
    while IFS=$'\t' read -r kind cam bid name ip port; do
        [[ -z "$kind" ]] && continue
        if aiengine_addr_is_local "$ip"; then
            echo "$cam"
        fi
    done <<< "$rows" | sort -un
}

aiengine_has_local_demand() {
    [[ -n "$(aiengine_local_camera_ids)" ]]
}

# --- CLI: nvr addon aiengine <prepare|enable|disable|status> ------------------

cmd_addon() {
    local sub="${1:-}"; shift || true
    case "$sub" in
        aiengine) cmd_addon_aiengine "$@" ;;
        ""|help|-h|--help)
            cat <<'EOF'
Usage: nvr addon aiengine <command> [options]

Manage the optional local aiengine add-on (LPR / object detection on the NVR,
at most two cameras). Mutating commands require root and hold the update lock.

  prepare [--tag TAG]                 Pull image, create the credential dir,
                                      create+print device-id (never the key)
  enable [--key-file PATH] [--tag TAG]  Install key, enable intent, start service
  disable [--force]                   Stop and drop the overlay (keeps identity)
  status                              Report intent, image, health, demand
EOF
            ;;
        *) echo "Unknown addon: $sub" >&2; return 1 ;;
    esac
}

cmd_addon_aiengine() {
    local action="${1:-status}"; shift || true
    case "$action" in
        prepare) cmd_addon_aiengine_prepare "$@" ;;
        enable)  cmd_addon_aiengine_enable "$@" ;;
        disable) cmd_addon_aiengine_disable "$@" ;;
        status)  cmd_addon_aiengine_status "$@" ;;
        # Hidden: print local direct-demand camera IDs (one per line, empty if
        # none). RPM migration uses it to choose enabled vs legacy-provisioned.
        local-demand) aiengine_local_camera_ids ;;
        *) echo "Unknown: nvr addon aiengine $action" >&2; return 1 ;;
    esac
}

# Validate host architecture: the local NVR image is amd64 only.
aiengine_require_amd64() {
    local arch
    arch=$(uname -m 2>/dev/null) || arch=""
    case "$arch" in
        x86_64|amd64) return 0 ;;
        *) echo -e "${RED}ERROR: aiengine local add-on requires x86_64/amd64 (host is '$arch'). Use a Jetson for arm.${NC}" >&2; return 1 ;;
    esac
}

# Create the credential directory (root:docker 0710). The group-execute bit lets
# a non-root `nvr update` stat known paths without letting the docker group list
# the directory or read the key.
aiengine_ensure_config_dir() {
    mkdir -p "$AIENGINE_CONFIG_DIR"
    chown root:docker "$AIENGINE_CONFIG_DIR" 2>/dev/null || true
    chmod 0710 "$AIENGINE_CONFIG_DIR" 2>/dev/null || true
}

# Create device-id via the EXTERNAL image's canonical generator, ONLY when
# absent. Never regenerate half of an existing (device-id, aiengine-key) pair —
# that would disable the licensed engine. The NVR never computes the ID itself.
aiengine_create_device_id() {
    local tag="$1"
    # Full image ref (honor a digest pin). One-shot read-write mount, used ONLY
    # here; the compose service mounts the same directory read-write (the Track 1
    # image rewrites device-id every boot), so this one-shot mount just seeds the
    # id before the first licensed start so `prepare` can print it for key issuance.
    local image="${AIENGINE_IMAGE:-${AIENGINE_REGISTRY:-docker.io}/dividia/aiengine:${tag:-stable}}"
    # The tool WRITES the inode-encrypted /srv/data/device-id (the engine reads
    # it) AND prints {"dev_id":"..."} to stdout. Idempotent: the id is derived
    # from the host MAC, so re-running yields the same id. We capture stdout and
    # return the human-readable id, because the stored file is encrypted binary
    # and can never be cat'd for display.
    local out
    if out=$(docker run --rm -v "$AIENGINE_CONFIG_DIR:/srv/data" "$image" \
            /usr/local/bin/aiengine-make-device-id 2>/dev/null); then
        chmod 0600 "$AIENGINE_CONFIG_DIR/device-id" 2>/dev/null || true
        chown root:root "$AIENGINE_CONFIG_DIR/device-id" 2>/dev/null || true
    else
        # Legacy image without the make-device-id helper: read the id via the
        # engine's own -d flag. No file is written; the engine regenerates the
        # id from hardware at start (so the licensed pair still matches).
        # --network host so it sees the real host NICs, not a container veth.
        # NOTE: `aiengine -d` prints the id and then exit(1) BY DESIGN, so do
        # NOT gate on the exit code -- capture stdout regardless.
        out=$(docker run --rm --network host "$image" -d x 2>/dev/null) || true
    fi
    # Extract the 12-hex device id from the JSON (or a bare token) and return it.
    local id
    id=$(printf '%s' "$out" | grep -oE '[0-9A-Fa-f]{12}' | head -1)
    [[ -n "$id" ]] && printf '%s\n' "$id"
    return 0
}

cmd_addon_aiengine_prepare() {
    aiengine_require_root prepare || return 1
    local tag="stable"
    while [[ $# -gt 0 ]]; do
        case "$1" in
            --tag) tag="$2"; shift 2 ;;
            *) echo "Unknown prepare option: $1" >&2; return 1 ;;
        esac
    done
    aiengine_lock || return 1
    aiengine_require_amd64 || return 1

    # Record the requested tag so reconcile/compose render the same image.
    env_write_var AIENGINE_TAG "$tag" || return 1

    echo "Pulling aiengine:$tag ..."
    aiengine_pull || { echo -e "${YELLOW}WARN: image pull failed; a later pull may still succeed${NC}" >&2; }

    aiengine_ensure_config_dir
    local dev_id
    dev_id=$(aiengine_create_device_id "$tag") || dev_id=""

    if [[ -n "$dev_id" ]]; then
        echo ""
        echo "Device ID (needed for out-of-band key issuance):"
        echo "  $dev_id"
        case "$dev_id" in
            000000000000|DEADBEEFDEAD)
                echo -e "${YELLOW}  WARNING: this is not a valid device id. The image could not read a hardware MAC.${NC}" >&2 ;;
        esac
    else
        echo -e "${YELLOW}No device-id could be generated. Is the pulled image the amd64 build that ships aiengine-make-device-id?${NC}"
    fi
    # NEVER print an existing product key.
}

# Install a supplied key file atomically at 0600, rejecting symlinks,
# directories, and group/other-readable sources (a key must be secret).
aiengine_install_key() {
    local src="$1"
    [[ -e "$src" ]] || { echo -e "${RED}ERROR: key file not found: $src${NC}" >&2; return 1; }
    if [[ -L "$src" ]]; then
        echo -e "${RED}ERROR: refusing a symlink key file: $src${NC}" >&2; return 1
    fi
    if [[ -d "$src" ]]; then
        echo -e "${RED}ERROR: key path is a directory: $src${NC}" >&2; return 1
    fi
    # Reject a group/other-readable source to avoid adopting a leaked key.
    local mode
    mode=$(stat -c '%a' "$src" 2>/dev/null || stat -f '%A' "$src" 2>/dev/null) || mode=""
    if [[ -n "$mode" && "${mode: -2}" != "00" ]]; then
        echo -e "${RED}ERROR: key file $src is group/other-readable (mode $mode); tighten to 0600 first${NC}" >&2
        return 1
    fi
    aiengine_ensure_config_dir
    local dest="$AIENGINE_CONFIG_DIR/aiengine-key"
    local tmp="${dest}.new.$$"
    cp -f "$src" "$tmp" || { rm -f "$tmp"; return 1; }
    chmod 0600 "$tmp" 2>/dev/null || true
    chown root:root "$tmp" 2>/dev/null || true
    mv -f "$tmp" "$dest"
}

cmd_addon_aiengine_enable() {
    aiengine_require_root enable || return 1
    local tag="" key_file=""
    while [[ $# -gt 0 ]]; do
        case "$1" in
            --key-file) key_file="$2"; shift 2 ;;
            --tag) tag="$2"; shift 2 ;;
            *) echo "Unknown enable option: $1" >&2; return 1 ;;
        esac
    done
    aiengine_lock || return 1
    aiengine_require_amd64 || return 1

    [[ -f "$INSTALL_DIR/$AIENGINE_OVERLAY" ]] || {
        echo -e "${RED}ERROR: $AIENGINE_OVERLAY not on disk; run 'nvr update' first to receive the overlay${NC}" >&2
        return 1
    }

    # prepare idempotently (pull, config dir, device-id).
    local eff_tag="${tag:-$(sed -n 's/^AIENGINE_TAG=//p' "$AIENGINE_ENV_FILE" 2>/dev/null | head -1)}"
    eff_tag="${eff_tag:-stable}"
    env_write_var AIENGINE_TAG "$eff_tag" || return 1
    aiengine_pull || echo -e "${YELLOW}WARN: image pull failed${NC}" >&2
    aiengine_ensure_config_dir
    aiengine_create_device_id "$eff_tag" || true

    if [[ -n "$key_file" ]]; then
        aiengine_install_key "$key_file" || return 1
    fi

    # The key is required. The device-id FILE is optional: a legacy image
    # without the make-device-id helper regenerates the id from hardware at
    # start, so the licensed pair still matches without a stored file.
    [[ -f "$AIENGINE_CONFIG_DIR/aiengine-key" ]] || { echo -e "${RED}ERROR: aiengine-key missing; pass --key-file PATH${NC}" >&2; return 1; }
    [[ -f "$AIENGINE_CONFIG_DIR/device-id" ]] || echo -e "${YELLOW}NOTE: no device-id file; the image will regenerate it from hardware at start.${NC}" >&2

    # Transactional first enable: capture prior compose/intent to roll back if
    # the container never comes up healthy.
    local prev_compose prev_intent
    prev_compose=$(compose_file_get)
    prev_intent=$(aiengine_intent_read)

    aiengine_intent_write enabled || return 1
    compose_overlay_reconcile present || return 1

    echo "Starting aiengine ..."
    # `up -d` returns 0 as soon as the container is CREATED, even if it then
    # crash-loops OR stays up but never loads its key/model (unhealthy). Wait for
    # the healthcheck to RESOLVE before committing intent, and roll back on
    # unhealthy/exited/timeout so a broken first enable never leaves
    # intent=enabled to retry every boot. Accept a container with NO healthcheck
    # once it is Running (the external image may not ship the probe yet). The
    # bound (~180s) exceeds the overlay's 120s start_period. v1-compatible poll
    # (no `--wait`).
    local _i=0 _running="" _health="" _ok=0
    if $COMPOSE up -d "$AIENGINE_SERVICE"; then
        while [[ $_i -lt 90 ]]; do
            _running=$(docker inspect -f '{{.State.Running}}' "$AIENGINE_SERVICE" 2>/dev/null || true)
            [[ "$_running" == "true" ]] || { _ok=0; break; }   # exited -> fail
            _health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$AIENGINE_SERVICE" 2>/dev/null || true)
            case "$_health" in
                healthy|none) _ok=1; break ;;
                unhealthy)    _ok=0; break ;;
                *)            : ;;              # starting -> keep waiting
            esac
            sleep 2; _i=$((_i+1))
        done
    fi
    if [[ $_ok -ne 1 ]]; then
        echo -e "${RED}ERROR: aiengine did not become healthy (state=${_running:-gone} health=${_health:-n/a}); rolling back${NC}" >&2
        docker logs --tail 20 "$AIENGINE_SERVICE" 2>&1 | head -20 || true
        env_write_var COMPOSE_FILE "$prev_compose" || true
        aiengine_intent_write "$prev_intent" || true
        $COMPOSE rm -fs "$AIENGINE_SERVICE" >/dev/null 2>&1 || true
        return 1
    fi
    echo "aiengine enabled (tag $eff_tag). Configure an 'AI Engine' Device at 127.0.0.1:88 in dview LPR/object setup."
}

cmd_addon_aiengine_disable() {
    aiengine_require_root disable || return 1
    local force=0
    while [[ $# -gt 0 ]]; do
        case "$1" in
            --force) force=1; shift ;;
            *) echo "Unknown disable option: $1" >&2; return 1 ;;
        esac
    done
    aiengine_lock || return 1

    local intent; intent=$(aiengine_intent_read)
    if [[ $force -ne 1 ]]; then
        local ids; ids=$(aiengine_local_camera_ids)
        if [[ -n "$ids" ]]; then
            echo -e "${YELLOW}Refusing: local LPR/object cameras still use this engine: $(echo "$ids" | tr '\n' ' ')${NC}" >&2
            echo "Inbound clients on OTHER NVRs cannot be detected locally. Re-run with --force to disable anyway." >&2
            return 1
        fi
        if [[ "$intent" == "legacy-provisioned" ]]; then
            echo -e "${YELLOW}Refusing: intent is legacy-provisioned (a licensed install another NVR may use). Re-run with --force.${NC}" >&2
            return 1
        fi
    fi

    aiengine_intent_write disabled || return 1
    compose_overlay_reconcile absent || return 1
    # Drop the container; --remove-orphans clears it now that the overlay is
    # out of COMPOSE_FILE. Customer DB rows are never touched.
    $COMPOSE up -d --remove-orphans >/dev/null 2>&1 || true
    $COMPOSE rm -fs "$AIENGINE_SERVICE" >/dev/null 2>&1 || true
    # Identity (device-id, aiengine-key, tag) is preserved for a safe re-enable.
    echo "aiengine disabled. Credential identity preserved."
    local ids; ids=$(aiengine_local_camera_ids)
    [[ -n "$ids" ]] && echo -e "${YELLOW}NOTE: local type-61 Device rows remain; recorder LPR fallback can still select a stopped endpoint.${NC}"
}

cmd_addon_aiengine_status() {
    local intent overlay
    intent=$(aiengine_intent_read)
    overlay="absent"; compose_overlay_present && overlay="present"

    echo -e "${BLUE}=== aiengine add-on ===${NC}"
    echo "Intent:        $intent"
    echo "Overlay:       $overlay in COMPOSE_FILE"
    local tag pinned
    tag=$(sed -n 's/^AIENGINE_TAG=//p' "$AIENGINE_ENV_FILE" 2>/dev/null | head -1)
    pinned=$(sed -n 's/^AIENGINE_IMAGE=//p' "$AIENGINE_ENV_FILE" 2>/dev/null | head -1)
    echo "Requested tag: ${tag:-<unset, defaults to stable>}"
    [[ -n "$pinned" ]] && echo "Pinned image:  $pinned (digest pin from migration)"

    # Image + container state, best-effort.
    local img_id digest
    img_id=$(docker image inspect --format '{{.Id}}' "${AIENGINE_REGISTRY:-docker.io}/dividia/aiengine:${tag:-stable}" 2>/dev/null) || img_id=""
    digest=$(docker image inspect --format '{{join .RepoDigests ","}}' "${AIENGINE_REGISTRY:-docker.io}/dividia/aiengine:${tag:-stable}" 2>/dev/null) || digest=""
    echo "Image ID:      ${img_id:-<not present locally>}"
    [[ -n "$digest" ]] && echo "Digest:        $digest"

    local cstate health restarts
    cstate=$(docker inspect --format '{{.State.Status}}' "$AIENGINE_SERVICE" 2>/dev/null) || cstate="(no container)"
    health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}n/a{{end}}' "$AIENGINE_SERVICE" 2>/dev/null) || health="n/a"
    restarts=$(docker inspect --format '{{.RestartCount}}' "$AIENGINE_SERVICE" 2>/dev/null) || restarts="?"
    echo "Container:     $cstate (health $health, restarts $restarts)"

    # Credential presence + mode ONLY — never the key contents or hash.
    local did_mode key_mode
    did_mode=$(stat -c '%a' "$AIENGINE_CONFIG_DIR/device-id" 2>/dev/null || echo "absent")
    key_mode=$(stat -c '%a' "$AIENGINE_CONFIG_DIR/aiengine-key" 2>/dev/null || echo "absent")
    echo "device-id:     $did_mode"
    echo "aiengine-key:  $key_mode"

    # Direct local demand.
    local ids; ids=$(aiengine_local_camera_ids)
    if [[ -n "$ids" ]]; then
        local count; count=$(echo "$ids" | grep -c .)
        echo "Local cameras: $(echo "$ids" | tr '\n' ' ')(count $count)"
        [[ "$count" -gt 2 ]] && echo -e "${YELLOW}NOTE: more than two local cameras; the CPU image targets at most two.${NC}"
    else
        echo "Local cameras: none detected"
    fi
    echo "(Remote AI Engine / Jetson endpoints are unchanged and not managed here.)"
}

cmd_help() {
    # Quoted heredoc terminator (<<'EOF') disables variable expansion AND
    # command substitution inside the body. The line below referencing
    # `nvr update` in backticks would otherwise be evaluated as a command
    # substitution by bash on every `nvr help` invocation, recursively
    # invoking `nvr update` (= docker compose pull/up) for every help-text
    # render. Verified via `bash -x /opt/dividia/nvr help` showing
    # `++ nvr update` mid-cat. Don't drop the quotes again.
    cat <<'EOF'
NVR Docker Management CLI

Usage: nvr <command> [args...]

Commands:
  status              Show service status and configuration
  logs [svc] [-f]     View logs (pass-through to docker compose logs)
  update              Pull latest images, extract files, restart
  backup              Backup NVR to VideoStore
  channel [name]      Show or set update channel (setter gated on Windows
                      via admin-key passphrase; open on Linux)
  db [args...]        Connect to MariaDB (dtech database)
  name                Show this NVR's name (like the legacy server_name)
  start               Start all services
  stop                Stop all services
  restart [svc]       Restart all or specific service
  version             Show image version labels
  display [action]    Manage optional CO6/CO7 host dview: enable, disable, status
  cloudapi-profile    Show or select the prod/staging cloud trust profile
  shell [svc]         Open a shell in a container
  find [options]      Scan local network for hosts (vendor + dup detection)
  prune               Reclaim disk from unused images >7 days old
                      (also runs automatically at the end of `nvr update`)
  addon aiengine ...  Manage the optional local aiengine add-on
                      (prepare | enable | disable | status)
  migrate-scalewatcher <zip> [--dry-run]
                      Import legacy 2014 Scale Watcher export (see runbook)
  addon hme <action>  Manage the optional HME drive-thru timer container:
                      enable | disable [--force] | auto | status
  help                Show this help message

Examples:
  nvr status
  nvr logs backend -f --tail=50
  nvr update
  nvr backup
  nvr channel dev
  nvr channel dev-smartrec       # per-branch test channel
  sudo nvr display enable        # preflight + enable local host dview on CO6/CO7
  sudo nvr display disable
  nvr cloudapi-profile show
  sudo nvr cloudapi-profile staging
  sudo nvr cloudapi-profile prod
  nvr db -e "SELECT COUNT(*) FROM Camera"
  nvr name
  nvr restart engine
  nvr shell backend
  nvr find
  nvr find -i eth0
  nvr find -c 192.168.0.0/24
  nvr prune                                          # reclaim disk now
  nvr migrate-scalewatcher /videostore/vs1/migrate-staging/scalewatcher-backup.zip --dry-run
  sudo nvr addon hme enable       # provision + start the HME drive-thru timer
  sudo nvr addon hme disable      # stop it and remove DeviceType 64 (orphan-guarded)
  nvr addon hme status

EOF
}

################################################################################
# HME drive-thru addon
#
# Converts the retired hme-stream RPM (+ init/systemd unit + watchprog cron)
# into an optional per-system compose service. Three separate signals, kept
# distinct on purpose:
#
#   1. DeviceType 64 'HME Stream' row  — provisioning marker "this box supports
#      HME". Seeded by `enable` (idempotent, name-gated), removed by `disable`.
#      Drives dview's camera-type dropdown, so it must exist before a tech can
#      add an HME camera.
#   2. Device (bType 64) row           — the configured camera + store ID. The
#      run trigger and the conf source (ip/port/storeid).
#   3. Host marker (HME_MARKER_FILE)   — short-term operator override.
#
# Precedence for whether the container runs: marker=disabled > marker=enabled >
# DB derive (a Device 64 exists) > off. Reconcile is FAIL-OPEN: any DB error
# keeps the prior state, and an auto-mode turn-off is debounced across two
# consecutive successful "absent" reads. Reconcile NEVER deletes DB rows.
################################################################################

# --- host marker + debounce state ------------------------------------------

hme_get_marker() {
    # Echoes enabled|disabled|"" (empty == auto).
    [[ -f "$HME_MARKER_FILE" ]] || return 0
    head -n1 "$HME_MARKER_FILE" 2>/dev/null | tr -d '[:space:]'
}

hme_set_marker() {
    # $1 = enabled|disabled|auto
    mkdir -p "$HME_STATE_DIR" 2>/dev/null || true
    case "$1" in
        enabled|disabled) printf '%s\n' "$1" > "$HME_MARKER_FILE" ;;
        auto)             rm -f "$HME_MARKER_FILE" 2>/dev/null || true ;;
    esac
}

hme_get_streak() {
    local s=0
    [[ -f "$HME_ABSENT_STREAK_FILE" ]] && s=$(tr -cd '0-9' < "$HME_ABSENT_STREAK_FILE" 2>/dev/null)
    [[ "$s" =~ ^[0-9]+$ ]] && echo "$s" || echo 0
}

hme_set_streak() {
    mkdir -p "$HME_STATE_DIR" 2>/dev/null || true
    printf '%s\n' "${1:-0}" > "$HME_ABSENT_STREAK_FILE" 2>/dev/null || true
}

hme_record_error() {
    mkdir -p "$HME_STATE_DIR" 2>/dev/null || true
    printf '%s\n' "$1" > "$HME_LAST_ERROR_FILE" 2>/dev/null || true
    echo -e "${YELLOW}HME reconcile: $1${NC}" >&2
}

hme_clear_error() { rm -f "$HME_LAST_ERROR_FILE" 2>/dev/null || true; }

# --- DB signals -------------------------------------------------------------

# Echo the winning HME Device row as "sIP<TAB>bPort<TAB>sPath"; return mariadb's
# exit code (non-zero == DB unreadable, which the caller treats as fail-open).
# ORDER BY D.bID picks the lowest-bID Device deterministically when a box has
# more than one (the retired launcher took "whatever row order returned").
hme_query_device() {
    local db_pass
    db_pass=$(grep '^MYSQL_ROOT_PASSWORD=' .env 2>/dev/null | cut -d= -f2-)
    $COMPOSE exec -T db mariadb -u root -p"${db_pass:-lynn1094}" -N -B dtech \
        -e "SELECT D.sIP, D.bPort, D.sPath FROM Device D, DeviceType T WHERE D.bType = T.bID AND T.sName = 'HME Stream' ORDER BY D.bID LIMIT 1" 2>/dev/null
}

# Seed DeviceType 64 'HME Stream' — idempotent + name-gated, mirroring the
# retired RPM %post. INSERT only when bID 64 is ABSENT, so it never overwrites a
# pre-2020 'Object Engine' 64 on a field box and never rewrites an existing HME
# row's sPath. The 5 explicit columns match the RPM; the rest take their
# dtech.sql defaults, so the row is identical to existing HME field boxes.
hme_seed_devicetype() {
    local db_pass
    db_pass=$(grep '^MYSQL_ROOT_PASSWORD=' .env 2>/dev/null | cut -d= -f2-)
    $COMPOSE exec -T db mariadb -u root -p"${db_pass:-lynn1094}" dtech -e "
        INSERT INTO DeviceType (bID, sName, sPath, bNumDevice, bNumInput)
        SELECT 64, 'HME Stream', 'rtsp://localhost:8554/timer', 1, 1 FROM DUAL
        WHERE NOT EXISTS (SELECT 1 FROM DeviceType WHERE bID = 64);" 2>/dev/null
}

# Remove DeviceType 64, ORPHAN-GUARDED. If a live Device of bType 64 still
# exists, refuse (deleting the type would strand a live camera — the hazard
# test_update_integration.py guards) unless $1 == --force, which deletes the
# Device rows first. Returns non-zero when it refused OR when the count read
# failed (fail-SAFE: an unreadable count must never be treated as zero).
hme_remove_devicetype() {
    local force="${1:-}" db_pass count
    db_pass=$(grep '^MYSQL_ROOT_PASSWORD=' .env 2>/dev/null | cut -d= -f2-)
    count=$($COMPOSE exec -T db mariadb -u root -p"${db_pass:-lynn1094}" -N -B dtech \
        -e "SELECT COUNT(*) FROM Device WHERE bType = 64" 2>/dev/null | tr -cd '0-9')
    if [[ -z "$count" ]]; then
        echo -e "${YELLOW}WARN: could not read Device count (db down?); leaving DeviceType 64 in place${NC}" >&2
        return 1
    fi
    if [[ "$count" -gt 0 && "$force" != "--force" ]]; then
        echo -e "${RED}Refusing to remove DeviceType 64: $count live 'HME Stream' Device(s) still exist.${NC}" >&2
        echo    "Remove the HME camera(s) in dview Setup first, or re-run with --force to delete them." >&2
        return 1
    fi
    if [[ "$count" -gt 0 ]]; then
        # Delete the Device rows FIRST, and only proceed to drop the type if that
        # succeeded — otherwise a failed Device delete + a successful type delete
        # would strand a live Device 64 on a missing type (the exact orphan the
        # guard above exists to prevent) while reporting success.
        if ! $COMPOSE exec -T db mariadb -u root -p"${db_pass:-lynn1094}" dtech \
            -e "DELETE FROM Device WHERE bType = 64" 2>/dev/null; then
            echo -e "${YELLOW}WARN: failed to delete Device 64 rows; leaving DeviceType 64 in place${NC}" >&2
            return 1
        fi
    fi
    if ! $COMPOSE exec -T db mariadb -u root -p"${db_pass:-lynn1094}" dtech \
        -e "DELETE FROM DeviceType WHERE bID = 64" 2>/dev/null; then
        echo -e "${YELLOW}WARN: failed to delete DeviceType 64${NC}" >&2
        return 1
    fi
    return 0
}

# --- pure decision ----------------------------------------------------------

# Given the host marker, the DB query exit code, whether a Device is present,
# and the current debounce streak, decide the desired container state WITHOUT
# touching anything. Echoes "<state> <new_streak>" where state is on|off|keep.
# Debounce applies ONLY to auto-mode turn-off (two consecutive successful
# "absent" reads); marker=disabled turns off immediately.
hme_decide() {
    local marker="$1" query_rc="$2" device_present="$3" streak="$4"
    # Pinning resets the streak: a stale count left from before a pin must not
    # shorten the auto-off debounce below two reads after a later return to auto.
    case "$marker" in
        disabled) echo "off 0"; return 0 ;;
        enabled)  echo "on 0";  return 0 ;;
    esac
    # auto (marker empty)
    if [[ "$query_rc" -ne 0 ]]; then
        # DB error: FAIL-OPEN. Reset the streak — an unreadable DB is not a
        # "successful absent read", so it breaks the consecutive chain and a
        # flaky DB can never auto-off HME.
        echo "keep 0"; return 0
    fi
    if [[ "$device_present" -eq 1 ]]; then
        echo "on 0"; return 0                  # device present: on, reset streak
    fi
    streak=$(( streak + 1 ))                    # absent + query OK: debounce
    if [[ "$streak" -ge 2 ]]; then
        echo "off $streak"; return 0
    fi
    echo "keep $streak"; return 0               # first absent read: hold
}

# --- conf + overlay ---------------------------------------------------------

# Generate data/config/hme-stream.conf from a Device row ("sIP<TAB>bPort<TAB>sPath")
# and write it atomically IF the content changed (sets HME_CONF_CHANGED=1).
# Mirrors the legacy start-hme-stream conf format. Returns non-zero WITHOUT
# creating the file if the row lacks a usable ip/port, so the caller never
# enables the overlay against a broken conf (Docker would auto-create a
# directory at the read-only bind path).
hme_write_conf() {
    local row="$1" ip port storeid new
    ip=$(printf '%s' "$row" | cut -f1)
    port=$(printf '%s' "$row" | cut -f2)
    storeid=$(printf '%s' "$row" | cut -f3)
    # mariadb -N prints a literal "NULL" for SQL NULL.
    [[ "$storeid" == "NULL" ]] && storeid=""
    if [[ -z "$ip" || "$ip" == "NULL" || -z "$port" || "$port" == "NULL" ]]; then
        return 1
    fi
    new=$(printf '[server]\nip = %s\nport = %s\nstoreid = %s\ndebug = off' "$ip" "$port" "$storeid")
    if [[ "$new" == "$(cat "$HME_CONF_PATH" 2>/dev/null || true)" ]]; then
        return 0    # unchanged
    fi
    mkdir -p "$(dirname "$HME_CONF_PATH")" 2>/dev/null || true
    local tmp="${HME_CONF_PATH}.new.$$"
    printf '%s\n' "$new" > "$tmp" || { rm -f "$tmp" 2>/dev/null || true; return 1; }
    chmod 640 "$tmp" 2>/dev/null || true
    chown dividia:docker "$tmp" 2>/dev/null || true
    mv -f "$tmp" "$HME_CONF_PATH" || { rm -f "$tmp" 2>/dev/null || true; return 1; }
    HME_CONF_CHANGED=1
    return 0
}

# Add ($1=on) or drop ($1=off) docker-compose.hme.yml in COMPOSE_FILE. Keeps the
# base file + any OS overlay first and appends the HME overlay LAST so its
# service block merges on top. Idempotent; rewrites .env atomically.
hme_set_overlay() {
    local want="$1" env_file="$INSTALL_DIR/.env"
    [[ -f "$env_file" ]] || return 0
    local current rebuilt="" part
    current=$(grep -E '^COMPOSE_FILE=' "$env_file" 2>/dev/null | head -1)
    current="${current#COMPOSE_FILE=}"
    [[ -z "$current" ]] && current="docker-compose.yml"
    local IFS_save="$IFS" ; IFS=':'
    local -a parts=($current)
    IFS="$IFS_save"
    for part in "${parts[@]}"; do
        [[ -z "$part" || "$part" == "$HME_OVERLAY_FILE" ]] && continue
        if [[ -z "$rebuilt" ]]; then rebuilt="$part"; else rebuilt="$rebuilt:$part"; fi
    done
    [[ "$want" == "on" ]] && rebuilt="${rebuilt:+$rebuilt:}$HME_OVERLAY_FILE"
    [[ "$rebuilt" == "$current" ]] && return 0
    local tmp="${env_file}.new.$$"
    if grep -qE '^COMPOSE_FILE=' "$env_file"; then
        sed "s|^COMPOSE_FILE=.*|COMPOSE_FILE=$rebuilt|" "$env_file" > "$tmp" || { rm -f "$tmp" 2>/dev/null || true; return 1; }
    else
        { cat "$env_file"; echo "COMPOSE_FILE=$rebuilt"; } > "$tmp" || { rm -f "$tmp" 2>/dev/null || true; return 1; }
    fi
    chmod 640 "$tmp" 2>/dev/null || true
    chown dividia:docker "$tmp" 2>/dev/null || true
    mv -f "$tmp" "$env_file" || { rm -f "$tmp" 2>/dev/null || true; return 1; }
    return 0
}

hme_overlay_is_on() {
    grep -qE "^COMPOSE_FILE=.*${HME_OVERLAY_FILE//./\\.}" "$INSTALL_DIR/.env" 2>/dev/null
}

hme_container_running() {
    docker inspect -f '{{.State.Running}}' hme-stream 2>/dev/null | grep -q true
}

hme_stack_is_up() {
    # True if any CORE dividia-nvr container is running, i.e. the stack was not
    # intentionally stopped (`nvr stop`) and is past boot (`nvr start`). Used to
    # stop the 2-minute cron from resurrecting hme against an operator stop.
    # `grep -qvx hme-stream`: succeeds only if some running project container is
    # NOT hme itself (so a lone hme container does not count as "stack up").
    docker ps --filter label=com.docker.compose.project=dividia-nvr \
        --filter status=running --format '{{.Names}}' 2>/dev/null \
        | grep -qvx 'hme-stream'
}

hme_container_is_managed() {
    # True only when the hme-stream container is OUR compose service (carries the
    # project label), NOT a leftover foreign `docker run --name hme-stream` from
    # the RPM init or the pre-addon restore path.
    [[ "$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}' hme-stream 2>/dev/null || true)" == "dividia-nvr" ]]
}

# Resolve the hme image ref the overlay would pull (must match the ${REGISTRY}/
# ${CHANNEL} pins in docker-compose.hme.yml). Used to prove the image is present
# BEFORE evicting a working foreign container.
hme_image_ref() {
    local reg chan
    reg=$(grep '^REGISTRY=' .env 2>/dev/null | cut -d= -f2- | tr -d '[:space:]'); reg=${reg:-docker.io}
    chan=$(grep '^CHANNEL=' .env 2>/dev/null | cut -d= -f2- | tr -d '[:space:]'); chan=${chan:-dev}
    echo "${reg}/dividia/hme-stream:${chan}"
}

# Fully REMOVE the legacy RPM hme-stream launcher once the addon takes over, so
# no leftover init/systemd unit or watchprog entry tries to (re)start a foreign
# container that the addon has replaced — including "start an image that has been
# pruned". Masking is not enough; the package files must go (the unit, the
# watchprog conf, and /usr/local/bin/start-hme-stream all belong to the RPM).
#
# CRITICAL: use `rpm -e --noscripts`. The RPM's %preun does `docker rm -f
# hme-stream` + `docker rmi dividia/hme-stream:*`, which would kill the MANAGED
# compose container and delete the image the addon just pulled. It does NOT touch
# DeviceType 64, so removing the package with --noscripts preserves the row the
# addon relies on. Best-effort + root-only; a non-RPM host (Ubuntu) has no
# `rpm`/hme-stream and this no-ops.
hme_remove_legacy_rpm() {
    # Stop + de-register the running unit first, while its script still exists.
    if command -v systemctl >/dev/null 2>&1 && systemctl cat hme-stream.service >/dev/null 2>&1; then
        systemctl stop hme-stream 2>/dev/null || true
        systemctl disable hme-stream 2>/dev/null || true
    fi
    if [[ -x /etc/init.d/hme-stream ]]; then
        service hme-stream stop 2>/dev/null || true
        chkconfig hme-stream off 2>/dev/null || true
    fi
    # Remove the package WITHOUT its destructive %preun.
    if command -v rpm >/dev/null 2>&1 && rpm -q hme-stream >/dev/null 2>&1; then
        rpm -e --noscripts hme-stream 2>/dev/null || true
    fi
    # Belt-and-suspenders for a box whose RPM db no longer owns these (botched
    # prior removal): drop the launcher + watchprog entry directly.
    rm -f /etc/watchprog.conf.d/hme-stream.conf /etc/watchprog.d/hme-stream.conf \
          /usr/local/bin/start-hme-stream 2>/dev/null || true
}

# --- reconcile --------------------------------------------------------------

# Converge HME HOST state (COMPOSE_FILE + conf + DeviceType seed) WITHOUT
# pulling or starting containers. Shared by cmd_update (which runs its own
# compose up afterward) and hme_reconcile (which adds the compose up). Records
# HME_DESIRED and HME_CONF_CHANGED for the caller, and any error for `status`.
# Always returns 0 — a reconcile must never abort an update.
hme_reconcile_config() {
    HME_CONF_CHANGED=0
    HME_DESIRED=""
    local marker streak rc=0 row present=0 decision new_streak
    marker=$(hme_get_marker)
    streak=$(hme_get_streak)
    row=$(hme_query_device) || rc=$?
    [[ $rc -eq 0 && -n "$row" ]] && present=1
    read -r decision new_streak <<< "$(hme_decide "$marker" "$rc" "$present" "$streak")"
    hme_set_streak "$new_streak"
    HME_DESIRED="$decision"

    case "$decision" in
        keep)
            [[ $rc -ne 0 ]] && hme_record_error "db query failed (rc=$rc); kept prior HME state"
            return 0 ;;
        off)
            hme_set_overlay off
            hme_clear_error
            return 0 ;;
    esac

    # decision == on
    if [[ "$marker" == "enabled" ]]; then
        hme_seed_devicetype >/dev/null 2>&1 || true
    fi
    if [[ $present -eq 1 ]]; then
        if ! hme_write_conf "$row"; then
            # Fail-SAFE: never enable the overlay against a missing/broken conf.
            # HME_DESIRED=keep so hme_reconcile leaves any running container on
            # its last-good conf rather than tearing it down over a bad read.
            hme_record_error "conf generation failed (bad Device row); keeping prior state"
            HME_DESIRED="keep"
            return 0
        fi
    elif [[ ! -f "$HME_CONF_PATH" ]]; then
        # Pinned on (marker=enabled) but no Device 64 and no prior conf: nothing
        # to stream. Hold OFF rather than crash-loop the container on empty conf,
        # and reflect that in HME_DESIRED so hme_reconcile does not try to start it.
        hme_record_error "pinned on but no Device 64 configured yet; waiting"
        hme_set_overlay off
        HME_DESIRED="off"
        return 0
    fi
    # Prove the managed image is AVAILABLE before enabling the overlay. This
    # single gate protects two things:
    #   1. It never evicts a working foreign container (below) for an image that
    #      is not here — otherwise a missing/bad ${CHANNEL} tag or a registry blip
    #      takes HME dark with no fallback (a Docker-only box has no RPM to
    #      restore), and because this fires unattended on every HME box's nightly
    #      update it would be a synchronized fleet-wide outage.
    #   2. It never lets the hme overlay onto COMPOSE_FILE for an unpullable tag,
    #      which would make the core `nvr update` compose-up fail on the hme pull
    #      and skip its post-success steps (watchtower handoff, dview refresh).
    #      This is the real path for a per-branch `nvr channel dev-<workspace>`
    #      canary, where dividia/hme-stream:dev-<workspace> may not exist, or when
    #      the out-of-band hme publish lags a core release.
    # `docker image inspect` first so steady state is a cheap local check, not a
    # network pull. Keep prior state and retry next reconcile if unavailable.
    local img; img=$(hme_image_ref)
    if ! { docker image inspect "$img" >/dev/null 2>&1 || docker pull "$img" >/dev/null 2>&1; }; then
        hme_record_error "managed hme image $img unavailable; not enabling (kept prior state)"
        HME_DESIRED="keep"
        return 0
    fi

    # Takeover: now that the image is local, evict a leftover FOREIGN or STOPPED
    # hme-stream container (RPM init / pre-addon `docker run`) that holds the name
    # + :8554. Gate on EXISTENCE, not running state: a stopped container still
    # owns the name.
    if docker container inspect hme-stream >/dev/null 2>&1 && ! hme_container_is_managed; then
        docker rm -f hme-stream >/dev/null 2>&1 || true
    fi
    # Remove the legacy RPM launcher entirely so nothing tries to restart it.
    # Run it whenever HME is being enabled, NOT only on the foreign-eviction path,
    # because the migration cutover already `docker rm -f`'s the foreign container
    # before it calls `nvr addon hme enable` — so gating on the container would
    # skip RPM removal during a fresh migration. Self-gates on `rpm -q`; a no-op
    # once the package is gone or on a non-RPM host.
    hme_remove_legacy_rpm
    hme_set_overlay on
    hme_clear_error
    return 0
}

# Full reconcile: converge host state, then converge the container — but only
# act on the container when reality differs from the desired state, so the
# 2-minute cron is a cheap DB-read poll in steady state (no compose churn).
# Serialized on its own lock (FD 8) so the cron and an operator command cannot
# overlap; also DEFERS to an in-progress `nvr update` (which converges HME
# itself) so the cron's compose call can't race the update's and disturb a core
# service. Its on-path `up` is scoped to the hme service alone, never the whole
# project, so even a residual race touches only hme.
hme_reconcile() {
    # $1 == --from-cron marks the unattended 2-minute run: it must not resurrect
    # hme against an operator `nvr stop` / at boot before `nvr start`. An explicit
    # enable/auto is a deliberate "on now" and skips that guard.
    local from_cron="${1:-}"
    if [[ $EUID -eq 0 ]]; then
        # Probe the nvr-update lock WITHOUT holding it (subshell releases it on
        # exit), so we never make a real update skip. If an update holds it,
        # skip — it reconciles HME via hme_reconcile_config.
        if ! ( exec 9>/var/lock/nvr-update.lock 2>/dev/null && flock -n 9 ) 2>/dev/null; then
            echo "nvr update in progress; skipping HME reconcile"
            return 0
        fi
        exec 8>"$HME_LOCK" 2>/dev/null || true
        flock -n 8 || { echo "another HME reconcile in progress, skipping"; return 0; }
    fi
    hme_reconcile_config
    local running="no"; hme_container_running && running="yes"
    case "$HME_DESIRED" in
        on)
            if [[ "$running" == "no" ]]; then
                # Overlay now in COMPOSE_FILE. Scope to the hme service so this
                # never recreates core containers (the image is already local from
                # the availability gate, so this create is fast). Skip on the cron
                # path when the stack is intentionally down, so the poll does not
                # undo `nvr stop`.
                if [[ "$from_cron" != "--from-cron" ]] || hme_stack_is_up; then
                    $COMPOSE up -d hme || true
                fi
            elif [[ "${HME_CONF_CHANGED:-0}" -eq 1 ]]; then
                # A bind-mount content change does not recreate on its own.
                $COMPOSE up -d --force-recreate hme || true
            fi ;;
        off)
            # reconcile_config dropped the overlay from COMPOSE_FILE. Remove the
            # hme container directly BY NAME rather than a project-wide
            # `compose up --remove-orphans`: the latter reconciles the WHOLE
            # project, so it could recreate/disturb core containers and race a
            # concurrent `nvr update` (the exact race this cron's lock probe is
            # only best-effort against). `docker rm -f hme-stream` removes a
            # managed OR a pre-addon foreign `docker run --name hme-stream` alike.
            if [[ "$running" == "yes" ]]; then
                docker container inspect hme-stream >/dev/null 2>&1 && docker rm -f hme-stream >/dev/null 2>&1 || true
            fi ;;
        keep)
            : ;;  # fail-open / debounce hold: touch nothing
    esac
    ensure_hme_reconcile_cron
}

# --- 2-minute reconcile cron ------------------------------------------------

hme_devicetype_exists() {
    # 0 = DeviceType 64 present, OR the DB is unreadable (fail-SAFE: a DB blip
    # must never deprovision the reconcile cron). 1 = DB readable and 64 absent.
    local db_pass out rc=0
    db_pass=$(grep '^MYSQL_ROOT_PASSWORD=' .env 2>/dev/null | cut -d= -f2-)
    out=$($COMPOSE exec -T db mariadb -u root -p"${db_pass:-lynn1094}" -N -B dtech \
        -e "SELECT 1 FROM DeviceType WHERE bID=64 LIMIT 1" 2>/dev/null) || rc=$?
    [[ $rc -ne 0 ]] && return 0
    [[ -n "$out" ]]
}

hme_is_provisioned() {
    # Reactive (cron-worthy) when pinned on, or in auto while the box is
    # HME-CAPABLE. "Capable" = the overlay is on OR DeviceType 64 exists (the
    # durable "installed with HME support" marker), so an auto box that auto-offed
    # KEEPS the 2-minute cron and re-enables within 2 min when a Device is
    # re-added — instead of going dark until the next nightly `nvr update`. A
    # pinned-OFF box is never reactive.
    local marker; marker=$(hme_get_marker)
    [[ "$marker" == "disabled" ]] && return 1
    [[ "$marker" == "enabled" ]] && return 0
    hme_overlay_is_on && return 0
    hme_devicetype_exists
}

_hme_remove_cron_file() {
    [[ -f "$HME_RECONCILE_CRON" ]] || return 0
    if [[ $EUID -ne 0 ]]; then
        sudo -n rm -f "$HME_RECONCILE_CRON" 2>/dev/null || true
    else
        rm -f "$HME_RECONCILE_CRON" 2>/dev/null || true
    fi
}

ensure_hme_reconcile_cron() {
    # Install the 2-minute reconcile cron only on HME-provisioned boxes so a
    # non-HME NVR never carries it; remove it once the box is pinned off /
    # deprovisioned. Matches the retired watchprog cadence (a cheap DB read +
    # idempotent converge). Owned here; do not edit by hand.
    if ! hme_is_provisioned; then
        _hme_remove_cron_file
        return 0
    fi
    local cron_body
    IFS= read -r -d '' cron_body <<'EOF' || true
# HME drive-thru reconcile for Dividia NVR.
#
# Every 2 minutes, converge the optional hme-stream container to the host
# marker + the Device (bType 64) signal, matching the retired watchprog
# cadence. Installed by `nvr addon hme enable`, removed by `disable`.
# Owned by /opt/dividia/nvr ensure_hme_reconcile_cron; do not edit by hand.
SHELL=/bin/sh
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
*/2 * * * * root /opt/dividia/nvr addon hme reconcile >/var/log/dividia-nvr-hme.log 2>&1
EOF
    if [[ $EUID -ne 0 ]]; then
        if ! sudo -n true 2>/dev/null; then return 0; fi
        if [[ -f "$HME_RECONCILE_CRON" ]] && sudo cmp -s <(echo "$cron_body") "$HME_RECONCILE_CRON"; then return 0; fi
        echo "$cron_body" | sudo tee "$HME_RECONCILE_CRON" >/dev/null && sudo chmod 0644 "$HME_RECONCILE_CRON"
    else
        if [[ -f "$HME_RECONCILE_CRON" ]] && cmp -s <(echo "$cron_body") "$HME_RECONCILE_CRON"; then return 0; fi
        echo "$cron_body" > "$HME_RECONCILE_CRON" && chmod 0644 "$HME_RECONCILE_CRON"
    fi
}

# --- status -----------------------------------------------------------------

hme_status() {
    local marker db_pass
    marker=$(hme_get_marker); [[ -z "$marker" ]] && marker="auto"
    db_pass=$(grep '^MYSQL_ROOT_PASSWORD=' .env 2>/dev/null | cut -d= -f2-)

    echo -e "${BLUE}=== HME addon ===${NC}"
    echo "Host intent (marker):   $marker"

    local dt
    dt=$($COMPOSE exec -T db mariadb -u root -p"${db_pass:-lynn1094}" -N -B dtech \
        -e "SELECT sName FROM DeviceType WHERE bID=64" 2>/dev/null | head -1)
    [[ -n "$dt" ]] && echo "DeviceType 64:          present ('$dt')" || echo "DeviceType 64:          absent"

    local rc=0 row
    row=$(hme_query_device) || rc=$?
    if [[ $rc -ne 0 ]]; then
        echo "Device (bType 64):      DB unreadable (fail-open: prior state kept)"
    elif [[ -n "$row" ]]; then
        echo "Device (bType 64):      configured (ip=$(printf '%s' "$row" | cut -f1) port=$(printf '%s' "$row" | cut -f2) storeid=$(printf '%s' "$row" | cut -f3))"
    else
        echo "Device (bType 64):      none configured"
    fi

    hme_overlay_is_on && echo "COMPOSE_FILE overlay:   on" || echo "COMPOSE_FILE overlay:   off"
    hme_container_running && echo "Container hme-stream:   running" || echo "Container hme-stream:   not running"

    if [[ -f "$HME_CONF_PATH" ]]; then
        local mt hash
        mt=$(stat -c '%y' "$HME_CONF_PATH" 2>/dev/null || stat -f '%Sm' "$HME_CONF_PATH" 2>/dev/null || echo '?')
        hash=$( { md5sum "$HME_CONF_PATH" 2>/dev/null || md5 -q "$HME_CONF_PATH" 2>/dev/null; } | awk '{print $1}')
        echo "Conf:                   $HME_CONF_PATH (mtime=$mt md5=$hash)"
    else
        echo "Conf:                   absent"
    fi

    local img
    img=$(docker inspect -f '{{.Config.Image}} {{.Image}}' hme-stream 2>/dev/null || true)
    [[ -n "$img" ]] && echo "Image:                  $img"

    local owner
    owner=$( { ss -ltnp 2>/dev/null || netstat -ltnp 2>/dev/null; } | grep -E '[:.]8554[[:space:]]' | head -1 | sed 's/^[[:space:]]*//')
    [[ -n "$owner" ]] && echo "Port 8554:              $owner" || echo "Port 8554:              (unbound)"

    echo "Auto-disable streak:    $(hme_get_streak)/2"
    [[ -f "$HME_LAST_ERROR_FILE" ]] && echo "Last reconcile error:   $(head -1 "$HME_LAST_ERROR_FILE")" || echo "Last reconcile error:   (none)"
}

# --- command surface --------------------------------------------------------

cmd_addon() {
    local addon="${1:-}"; shift 2>/dev/null || true
    case "$addon" in
        hme) cmd_addon_hme "$@" ;;
        ""|help|-h|--help) echo "Usage: nvr addon hme {enable|disable [--force]|auto|status}" ;;
        *) echo "Unknown addon: $addon (known: hme)"; exit 1 ;;
    esac
}

cmd_addon_hme() {
    local action="${1:-status}"; shift 2>/dev/null || true
    # Mutating actions rewrite .env / markers, write the DB, manage the host cron,
    # and take the reconcile flock — all of which only work reliably as root, and
    # the flock is skipped for non-root (matching cmd_update). Enforce root so a
    # non-root `nvr addon hme enable` can't run lock-free and race the root cron.
    # `status` stays open (read-only). Matches the `sudo nvr addon hme ...` help.
    case "$action" in
        enable|disable|auto|reconcile)
            if [[ $EUID -ne 0 ]]; then
                echo "nvr addon hme $action must run as root (use: sudo nvr addon hme $action)" >&2
                exit 1
            fi ;;
    esac
    case "$action" in
        enable)
            # `enable` OWNS the DeviceType 64 seed (the only creator now the RPM
            # is gone) and reconciles immediately so a tech doesn't wait for the
            # 02:xx update cron before dview offers the camera type.
            hme_set_marker enabled
            hme_reconcile
            echo "HME enabled. dview now offers the 'HME Stream' camera type; add the Device in Setup."
            hme_status ;;
        disable)
            local force=""
            [[ "${1:-}" == "--force" ]] && force="--force"
            hme_set_marker disabled
            hme_reconcile   # drops overlay, removes container, frees :8554
            if hme_remove_devicetype "$force"; then
                echo "HME disabled; DeviceType 64 removed."
            else
                echo "HME disabled (container stopped); DeviceType 64 left in place — see the warning above."
            fi
            hme_status ;;
        auto)
            hme_set_marker auto
            hme_reconcile
            echo "HME set to auto (follows the Device bType 64 signal)."
            hme_status ;;
        reconcile)
            # Internal: driven by the 2-minute cron. --from-cron applies the
            # stack-up guard so the poll never resurrects hme against `nvr stop`.
            hme_reconcile --from-cron ;;
        status)
            hme_status ;;
        *)
            echo "Usage: nvr addon hme {enable|disable [--force]|auto|status}"; exit 1 ;;
    esac
}

################################################################################
# Main
################################################################################

if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
case "${1:-help}" in
    status)   shift; cmd_status "$@" ;;
    logs)     shift; cmd_logs "$@" ;;
    update)   shift; cmd_update "$@" ;;
    backup)   shift; cmd_backup "$@" ;;
    channel)  shift; cmd_channel "$@" ;;
    db)       shift; cmd_db "$@" ;;
    name)     shift; cmd_name "$@" ;;
    start)    shift; cmd_start "$@" ;;
    stop)     shift; cmd_stop "$@" ;;
    restart)  shift; cmd_restart "$@" ;;
    version)  shift; cmd_version "$@" ;;
    display)  shift; cmd_display "$@" ;;
    cloudapi-profile) shift; cmd_cloudapi_profile "$@" ;;
    shell)    shift; cmd_shell "$@" ;;
    vm-shell) shift; cmd_vm_shell "$@" ;;
    find)     shift; cmd_find "$@" ;;
    prune)    shift; cmd_prune "$@" ;;
    migrate-scalewatcher) shift; cmd_migrate_scalewatcher "$@" ;;
    addon)    shift; cmd_addon "$@" ;;
    # Re-assert host-level config (sudoers, docker log cap, rc.local dvs block).
    # cmd_update calls this on every run; exposed as a subcommand so
    # install-nvr.sh's migration can trigger the SAME implementation instead of
    # carrying its own copy.
    ensure-host-config) shift; ensure_host_config "$@" ;;
    # Same shape as ensure-host-config: lets a tech converge one box by hand
    # (and lets the runbook name a command) without waiting for the update cron.
    ensure-boot-service) shift; ensure_boot_service "$@" ;;
    # Hidden: recompute COMPOSE_FILE from aiengine intent and strip AIENGINE_*
    # when the add-on is not fully present. The backup restore (dvs30.py) and
    # install-nvr.sh .env.save copy-back call this so a restored .env can never
    # implicitly re-enable a broken aiengine container. Not in help on purpose.
    normalize-addon-env) shift; aiengine_env_normalize "$@" ;;
    # Hidden: reconciled boot bring-up for the CentOS 6 SysV init script and
    # migration finalize (core up by name + add-on isolated). Not in help.
    boot-up) shift; cmd_boot_up "$@" ;;
    help|--help|-h) cmd_help ;;
    *)        echo "Unknown command: $1"; echo ""; cmd_help; exit 1 ;;
esac
fi