#!/bin/bash
################################################################################
# NVR Docker Installation Script
#
# Installs Docker-based NVR on a fresh CentOS 6+, CentOS Stream 9, or Ubuntu system.
# Downloads compose files from files.dividia.net, pulls images, and
# configures a host cron for daily automatic updates via `nvr update`.
#
# Usage:
#   sudo ./install-nvr.sh [options]
#
# Options:
#   --channel <name>        Version channel (e.g., 6.2), "dev", or "dev-<suffix>"
#                           for per-branch test images [default: 6.2]
#   --registry <url>        Docker registry URL [default: docker.io]
#   --data-dir <path>       Data directory [default: /opt/dividia/data]
#   --video-device <path>   Block device for VideoStore (e.g., /dev/sdb)
#   --no-video-device       Acknowledge no dedicated VideoStore drive (dev/test)
#   --help                  Show this help message
#
# URL: http://files.dividia.net/software/nvr-docker/install-nvr.sh
################################################################################

# set -eE: errexit + errtrace. The -E (errtrace) is load-bearing for the
# rollback contract in migrate_flow. Without it, the ERR trap installed at
# line 2712 fires only on commands at migrate_flow's own scope; failures
# inside the 14+ functions it calls (restore_from_backup, verify_migration,
# wait_for_backend_healthy, etc.) silently abort the shell without firing
# rollback_rpm_services. Empirically verified: bare `false` inside a nested
# function aborts the script under `set -e` alone but does NOT fire the
# parent's ERR trap; under `set -eE` it does. Pass-3 review converted
# `exit 1` → `return 1` in trap-window functions to fix one half of this;
# `set -E` is the other half.
set -eE

# Ensure /usr/local/bin is in PATH (Docker static binaries on CentOS 6)
export PATH="/usr/local/bin:$PATH"

# Default configuration
CHANNEL="6.2"
CHANNEL_EXPLICIT=false
REGISTRY="docker.io"
DATA_DIR="/opt/dividia/data"
INSTALL_DIR="/opt/dividia"
# Completion marker: written as the LAST action of each install flow (fresh /
# upgrade / migrate), only after every functional step ran. Its presence means
# "this box finished an install"; its ABSENCE next to a live Docker stack means
# a prior run was interrupted after the point of no return but before the
# finalize tail (configure_user_access, crons) executed — the cs2565 failure
# mode that left a migrated box with no passwordless sudo. main() resumes that
# tail instead of re-running a destructive full flow. See resume_finalize().
MARKER_FILE="$INSTALL_DIR/.install-complete"
VIDEO_DEVICE=""
NO_VIDEO_DEVICE=false
DETECTED_VS_LABELS=""  # Set by check_video_device_or_fail if pre-labeled vs[N] partitions exist
NVR_ID=""
WATCHTOWER_SCHEDULE="0 0 7 * * *"  # 07:00 UTC daily = 02:00 CDT / 01:00 CST (quiet window for US Central NVRs)
BASE_URL="${NVR_BASE_URL:-http://files.dividia.net/software/nvr-docker}"
UPGRADE_MODE=false
UPGRADE_BACKUP_DIR=""
MIGRATE_MODE=false
ASSUME_YES=false
RPM_SEED=""
RPM_DVS_CONF=""
KEEP_LEGACY=false
RPM_VIDEOSTORE_MOUNTS_FILE="/tmp/nvr-rpm-videostore-mounts"
SYSCTL_RESERVED_PORTS_FILE="${NVR_SYSCTL_RESERVED_PORTS_FILE:-/etc/sysctl.d/90-dividia-nvr-ports.conf}"
NVR_SERVICE_PORT_RANGE="43202-43210"
# Root-resident boot storage artifacts. These cannot live below /opt because
# /opt can itself be the bind target that they must establish before Docker.
# Tests override each path so generation never touches the developer host.
NVR_BOOT_STORAGE_CONFIG="${NVR_BOOT_STORAGE_CONFIG:-/etc/dividia-nvr/boot-storage.conf}"
NVR_BOOT_STORAGE_HELPER="${NVR_BOOT_STORAGE_HELPER:-/usr/local/sbin/dividia-nvr-boot-storage}"
NVR_DOCKER_BOOT_STORAGE_DROPIN="${NVR_DOCKER_BOOT_STORAGE_DROPIN:-/etc/systemd/system/docker.service.d/10-dividia-boot-storage.conf}"
NVR_CONTAINERD_BOOT_STORAGE_DROPIN="${NVR_CONTAINERD_BOOT_STORAGE_DROPIN:-/etc/systemd/system/containerd.service.d/10-dividia-boot-storage.conf}"
NVR_DOCKER_INIT="${NVR_DOCKER_INIT:-/etc/init.d/docker}"
NVR_BOOT_STORAGE_COMPOSE_OVERLAY="${NVR_BOOT_STORAGE_COMPOSE_OVERLAY:-/etc/dividia-nvr/docker-compose.boot-storage.yml}"
NVR_BOOT_STORAGE_CONTAINER_MASK="${NVR_BOOT_STORAGE_CONTAINER_MASK:-/etc/dividia-nvr/container-mask}"
# Set only when repair_docker_firewall_after_restore restarts dockerd.  Docker
# preserves a container's previous health state across that daemon restart;
# Compose can therefore start host-network dependents before MariaDB has bound
# 3306 again (cs460).  The post-repair recovery below recreates DB and then
# restarts those dependents only after a real TCP/authenticated DB probe.
DOCKER_DAEMON_RESTARTED=false
# Set when migration takes ownership of an RPM-managed HME service. Rollback
# uses the captured state so a failed migration does not enable/start a service
# that was intentionally disabled before the attempt.
LEGACY_HME_HANDOFF=false
LEGACY_HME_WAS_ENABLED=""
LEGACY_HME_WAS_ACTIVE=""
LEGACY_HME_WAS_MASKED=false

# Same capture-for-rollback shape for the optional legacy aiengine service
# (LPR / object detection wrapper container). Only a handoff this migration
# actually performed is rolled back.
LEGACY_AIENGINE_HANDOFF=false
LEGACY_AIENGINE_WAS_ENABLED=""
LEGACY_AIENGINE_WAS_ACTIVE=""
LEGACY_AIENGINE_WAS_MASKED=false

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info()  { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn()  { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
# WINDOWS-INSTALLER-CONTRACT: log_step emits a machine-readable "NVR-STEP:" line
# that the Windows Hyper-V installer (windows/scripts/Install-NVR.ps1, step 14)
# parses to show sub-step progress in the InnoSetup UI. Keep this line format
# stable. If you change it, update Install-NVR.ps1's NVR-STEP regex and the
# contract test at docker/tests/test_install_nvr_contract.sh.
log_step()  { echo "NVR-STEP: $1"; echo -e "\n${BLUE}==>${NC} $1"; }

################################################################################
# Logging — tee all output to a persistent log file for post-mortem
################################################################################

LOG_FILE=""

setup_logging() {
    local timestamp
    timestamp=$(date +%Y%m%d-%H%M%S)
    LOG_FILE="/var/log/nvr-install-${timestamp}.log"

    # Fall back to /tmp if /var/log is not writable (shouldn't happen as root,
    # but better to keep some log than none).
    if ! { mkdir -p /var/log 2>/dev/null && touch "$LOG_FILE" 2>/dev/null; }; then
        LOG_FILE="/tmp/nvr-install-${timestamp}.log"
        if ! touch "$LOG_FILE" 2>/dev/null; then
            log_warn "Cannot create install log file — proceeding without"
            LOG_FILE=""
            return
        fi
    fi
    chmod 0644 "$LOG_FILE" 2>/dev/null || true

    # Keep a stable "latest" symlink so post-mortem is `cat /var/log/nvr-install.log`.
    local log_dir="${LOG_FILE%/*}"
    ln -sf "$(basename "$LOG_FILE")" "$log_dir/nvr-install.log" 2>/dev/null || true

    # Tee both stdout and stderr; strip ANSI color escapes from the file copy
    # so it greps cleanly later. Terminal output stays colored.
    exec > >(tee >(sed -u 's/\x1b\[[0-9;]*m//g' >> "$LOG_FILE"))
    exec 2> >(tee >(sed -u 's/\x1b\[[0-9;]*m//g' >> "$LOG_FILE") >&2)

    log_info "Install log: $LOG_FILE"
}

################################################################################
# Signal handling — keep a dropped SSH session from silently stranding a
# migration mid-run.
#
# cs2565 2026: the SSH session was killed while the installer waited at the
# live-MPE gate; the box was left mid-migration BEFORE configure_user_access
# ran, so it had no passwordless sudo. The fix has two halves: (1) ignore the
# signals a disconnect delivers so the run finishes to LOG_FILE regardless,
# and (2) log loudly (not die silently) on a deliberate INT/TERM abort.
################################################################################

# Logged handler for INT (Ctrl-C) and TERM (kill). The default disposition is
# silent process death; this prints an un-missable line to stdout AND the
# tee'd LOG_FILE so a post-mortem shows the run was cut short, then exits with
# the conventional 128+signal code.
#
# This does NOT touch migrate_flow's ERR-trap rollback: ERR is a bash
# pseudo-signal distinct from the OS signals trapped here, and (matching the
# pre-existing behavior) neither the old default INT disposition nor this
# handler fires rollback_rpm_services — an operator abort mid-migration is
# picked up by main()'s resume_finalize on the next run, not auto-rolled-back.
on_interrupt() {
    local sig="${1:-?}"
    log_error "MIGRATION/INSTALL INTERRUPTED (SIG ${sig}) — state may be incomplete."
    log_error "Re-run install-nvr.sh to resume: a half-finished migration past the"
    log_error "point of no return is detected and finalized idempotently."
    exit 130
}

install_signal_traps() {
    # Ignore SIGHUP and SIGPIPE. A dropped SSH session delivers SIGHUP; a
    # broken terminal pipe on the setup_logging `exec > >(tee ...)` fan-out
    # delivers SIGPIPE. Ignoring both (nohup semantics: trap '' , not a
    # handler) lets the installer keep running to completion, still writing
    # to LOG_FILE, instead of dying partway through and stranding the box.
    #
    # Deliberately SEPARATE from the ERR trap migrate_flow installs/clears for
    # rollback (trap ... ERR). ERR is a bash pseudo-signal unaffected by these
    # OS-signal traps, so there is nothing to fold. Verified: the only other
    # traps in this script are that migrate_flow ERR rollback; there is no
    # EXIT trap to preserve.
    trap '' HUP PIPE

    # A deliberate Ctrl-C / kill still stops the run (obey the operator) but
    # through on_interrupt so the truncated state is loud in the log.
    trap 'on_interrupt INT' INT
    trap 'on_interrupt TERM' TERM
}

# Best-effort nudge: interactive over SSH without a terminal multiplexer means
# a disconnect races the run. The HUP trap above mitigates, but screen/tmux is
# the real safety net (survives the disconnect, lets the operator reattach).
warn_if_not_multiplexed() {
    if [[ -t 1 ]] && [[ -z "${STY:-}" ]] && [[ -z "${TMUX:-}" ]]; then
        log_warn "Not running under screen/tmux. If this is a remote SSH session and it"
        log_warn "  drops, re-run under 'screen' or 'tmux' so the install survives a disconnect."
    fi
}

################################################################################
# Completion marker + interrupted-run resume
################################################################################

# write_completion_marker <flow>: drop $MARKER_FILE recording the flow name and
# a UTC timestamp. Called from the tail of each flow, gated on an explicit
# success flag (per .claude/rules/bash-scripting.md: a critical after-success
# step must not rely solely on the set -e chain reaching its line). Best-effort
# on the write itself — a marker we couldn't write is a warning, never fatal.
write_completion_marker() {
    local flow="${1:-unknown}"
    mkdir -p "$INSTALL_DIR" 2>/dev/null || true
    if {
        echo "flow=$flow"
        echo "completed=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date)"
        echo "channel=$CHANNEL"
    } > "$MARKER_FILE" 2>/dev/null; then
        chmod 0644 "$MARKER_FILE" 2>/dev/null || true
        log_info "Wrote completion marker: $MARKER_FILE ($flow)"
    else
        log_warn "Could not write completion marker $MARKER_FILE (install still finished)"
    fi
}

# docker_stack_present: true when a compose stack has containers on this host.
# Guarded so a fresh box (no docker, no compose file) returns false cleanly.
docker_stack_present() {
    [[ -f "$INSTALL_DIR/docker-compose.yml" ]] || return 1
    command -v docker &>/dev/null || return 1
    local ids
    ids=$( (cd "$INSTALL_DIR" 2>/dev/null && docker compose ps -q 2>/dev/null) || true )
    [[ -n "$ids" ]]
}

# detect_incomplete_install: true iff a prior run was interrupted PAST the point
# of no return but before it finalized. Conservative on purpose — all three must
# hold: (1) no completion marker, (2) a live Docker stack exists, (3) no RPM NVR
# services remain (i.e. RPMs already removed, or a fresh install that never had
# them). Requiring RPMs-gone excludes the rolled-back state (rollback_rpm_services
# runs `docker compose down -v`, so the stack is gone there anyway) and the
# pre-point-of-no-return migrate state (RPMs still present → re-run migrates).
detect_incomplete_install() {
    [[ -f "$MARKER_FILE" ]] && return 1
    docker_stack_present || return 1
    detect_rpm_install 2>/dev/null && return 1
    # A missing marker alone is NOT proof of an interrupted run: boxes installed
    # before the marker feature existed also lack it. Require POSITIVE evidence
    # the finalize tail did not complete, namely its passwordless-sudo grant
    # (written by configure_user_access, one of the last finalize steps) is
    # absent. A fully-configured pre-marker box HAS it and stays on the normal
    # path, so an explicit --upgrade still upgrades and dividia's password is
    # not needlessly reset. A box stranded mid-finalize (cs2565/cs93) lacks it,
    # which is exactly the case resume_finalize exists for.
    configure_user_access_ran && return 1
    return 0
}

# Split out so behavioral tests can stub it (the real path is root-owned and
# not present on a CI/dev host). True when configure_user_access has run.
configure_user_access_ran() {
    # /etc/profile.d/nvr.sh, NOT /etc/sudoers.d/dividia.
    #
    # This is positive evidence that configure_user_access completed, and it has
    # to be a file ONLY that function writes. The sudoers drop-in stopped
    # qualifying the moment `nvr ensure-host-config` began self-healing it: the
    # backend entrypoint runs that on every container start, so pull_and_start
    # plants /etc/sudoers.d/dividia ~60-100 lines BEFORE configure_user_access
    # would have. detect_incomplete_install then reads "finalize already ran" on a
    # box that was interrupted in the documented cs2565/cs93 SSH-drop window,
    # main() skips resume_finalize, the upgrade auto-detect sees the migration
    # backup still on the VideoStore, and upgrade_flow runs `rda-db --restore`
    # OVER the live restored database -- discarding every configuration change and
    # every Event row since the migration, leaving recorded footage on disk with no
    # index. Exactly what the comment above resume exists to prevent.
    #
    # /etc/profile.d/nvr.sh has one writer in the whole tree
    # (configure_user_access, below) and is written unconditionally, while the
    # sudoers write there is additionally gated on `id dividia`. Strictly better
    # sentinel.
    [[ -f /etc/profile.d/nvr.sh ]]
}

# resume_finalize: run ONLY the idempotent finalize tail against an already-
# running stack, then mark complete. Every step here overwrites in place
# (boot service unit, extracted CLI, sudoers/PATH, crons), so it is safe to
# re-run. It deliberately SKIPS finalize_migrated_stack (a Docker+service
# restart, too disruptive when the stack is already healthy) and the full
# restore/verify chain (would re-restore over a live DB). The point is to
# repair the cs2565 gap — passwordless sudo + update cron + marker — not to
# redo the migration.
resume_finalize() {
    log_step "Detected incomplete prior install/migration — resuming finalize steps"
    log_warn "A live Docker stack is running but $MARKER_FILE is absent: a prior run"
    log_warn "  was interrupted after the point of no return. Re-running the idempotent"
    log_warn "  finalize tail (boot service, CLI, host access, crons) instead of a full flow."
    check_root
    cd "$INSTALL_DIR"
    create_boot_service
    create_management_scripts
    configure_reserved_service_ports
    configure_user_access
    install_prune_cron
    install_update_cron

    # Gate the marker on an explicit flag rather than the set -e chain.
    local resume_ok=1
    if [[ $resume_ok -eq 1 ]]; then
        write_completion_marker "resume"
    fi

    log_step "Finalize complete"
    log_info "The interrupted install/migration has been finalized."
}

################################################################################
# Parse Arguments
################################################################################

parse_args() {
    while [[ $# -gt 0 ]]; do
        case $1 in
            --channel)       CHANNEL="$2"; CHANNEL_EXPLICIT=true; shift 2 ;;
            --registry)      REGISTRY="$2";     shift 2 ;;
            --data-dir)      DATA_DIR="$2";     shift 2 ;;
            --video-device)  VIDEO_DEVICE="$2"; shift 2 ;;
            --no-video-device) NO_VIDEO_DEVICE=true; shift ;;
            --id)            NVR_ID="$2";      shift 2 ;;
            --upgrade)       UPGRADE_MODE=true; shift ;;
            --migrate)       MIGRATE_MODE=true; shift ;;
            -y|--yes)        ASSUME_YES=true; shift ;;
            --keep-legacy)   KEEP_LEGACY=true; shift ;;
            --help)          show_help; exit 0 ;;
            *)               log_error "Unknown option: $1"; show_help; exit 1 ;;
        esac
    done
}

show_help() {
    cat <<EOF
NVR Docker Installation Script

Usage: sudo $0 [options]

Options:
  --channel <name>        Version channel (e.g., 6.2), "dev", or "dev-<suffix>"
                          for per-branch test images [default: 6.2]
  --registry <url>        Docker registry URL [default: docker.io]
  --data-dir <path>       Data directory [default: /opt/dividia/data]
  --video-device <path>   Block device for VideoStore (e.g., /dev/sdb).
                          Script partitions, formats ext4, labels vs1.
  --no-video-device       Acknowledge no dedicated VideoStore drive.
                          Recordings go to \$DATA_DIR/videostore/vs1 (will
                          fill the root partition under sustained motion).
                          Dev/test only, NOT for production.
  --upgrade               Upgrade mode: restore from VideoStore backup after install
  --migrate               Migrate from RPM install to Docker
  -y, --yes               Skip confirmation prompts
  --keep-legacy           Skip post-migration cleanup of RPM-era artifacts
                          (firefox, httpd, yum cache, old logs, /var/lib/mysql).
                          Useful for debugging or if you need to roll back.
  --help                  Show this help message

Channels:
  X.Y       Version channel (e.g., 6.2). Auto-receives patches within this
            major.minor version line.
  dev       Shared development builds (latest features)
  dev-<sfx> Per-branch test channel (e.g., dev-smartrec). The update cron
            polls this exact tag, so a system on dev-smartrec stays on the
            feature-branch image and never auto-updates to plain :dev.

Example:
  sudo $0 --channel 6.2
  sudo $0 --channel dev --data-dir /data/nvr
  sudo $0 --channel dev-smartrec     # install per-branch test build
  sudo $0 --video-device /dev/sdb
  sudo $0 --upgrade --channel 6.2
  sudo $0 --migrate --channel dev

Migrating from a 2014 Windows Scale Watcher install? Import via
\`nvr migrate-scalewatcher <zip>\` on the new NVR after this script completes.
See docs/runbooks/migrate-scalewatcher.md in the repo for the full flow.

EOF
}

################################################################################
# System Checks
################################################################################

check_root() {
    if [[ $EUID -ne 0 ]]; then
        log_error "This script must be run as root"
        exit 1
    fi
}

validate_channel() {
    # Channel: "dev", "dev-<suffix>" (per-branch test channel), or major.minor (e.g., 6.2, 7.0).
    if [[ ! "$CHANNEL" =~ ^(dev(-[a-z0-9-]+)?|[0-9]+\.[0-9]+)$ ]]; then
        log_error "Invalid channel: $CHANNEL (must be 'dev', 'dev-<suffix>', or a version like 6.2)"
        exit 1
    fi
}

detect_os() {
    if [[ -f /etc/os-release ]]; then
        . /etc/os-release
        OS=$ID
        OS_VERSION=$VERSION_ID
        log_info "Detected OS: $PRETTY_NAME"
    elif [[ -f /etc/redhat-release ]]; then
        # CentOS 6 (no /etc/os-release): "CentOS release 6.10 (Final)"
        OS="centos"
        OS_VERSION=$(sed 's/.*release \([0-9]*\).*/\1/' /etc/redhat-release)
        PRETTY_NAME=$(cat /etc/redhat-release)
        log_info "Detected OS: $PRETTY_NAME"
    else
        log_error "Cannot detect OS. /etc/os-release not found."
        exit 1
    fi

    case "$OS" in
        centos|rhel|rocky|almalinux)
            OS_FAMILY="rhel"
            ;;
        ubuntu|debian)
            OS_FAMILY="debian"
            ;;
        *)
            log_error "Unsupported OS: $OS"
            exit 1
            ;;
    esac

    # Detect Windows deployment (WSL2 or Hyper-V VM with --windows flag)
    IS_WINDOWS=false
    if grep -qi microsoft /proc/version 2>/dev/null; then
        IS_WINDOWS=true
        log_info "Running inside WSL2 (Windows host detected)"
    elif [[ "${NVR_PLATFORM:-}" == "windows" ]]; then
        IS_WINDOWS=true
        log_info "Windows platform flag set (Hyper-V VM deployment)"
    fi
}

################################################################################
# CentOS 6 kernel gate
#
# Docker 27 has NO usable storage driver on CentOS 6's stock 2.6.32 kernel:
# devicemapper was removed in Docker 25, and overlay2 needs kernel 4.x. So dockerd
# silently falls back to `vfs`, which makes a FULL COPY of the filesystem for every
# image layer instead of stacking them.
#
# Measured on a bento/centos-6.9 guest 2026-07-26: six of the seven NVR images
# consumed 42 GB of /var/lib/docker/vfs and the migration hit ENOSPC on a 50 GB
# root that started with 44 GB free. The same box on kernel-lt 4.4.180 used 5.6 GB
# for the whole set. configure_docker_storage() sizes for "~2.7G resting, 8 GB root
# floor", which is derived from overlay2 on cs256 and is off by roughly 15x here --
# so the preflight passes and the migration dies partway, after stop_rpm_services.
#
# The fix is nearly free, because the rda repo already ships kernel-lt and rda-release
# pulls it in: on most CentOS 6 boxes 4.4.180 is ALREADY INSTALLED and already grub
# default=0, and the only thing missing is a reboot. So this refuses early and says
# exactly that, rather than letting the box discover it 40 minutes in.
#
# Deliberately does NOT reboot. Rebooting a customer's NVR is an operator decision,
# and this runs before the backup, so refusing costs nothing but a re-run.
NVR_PROC_VERSION="${NVR_PROC_VERSION:-/proc/version}"   # test seam
NVR_KERNEL_GATE_SKIP="${NVR_KERNEL_GATE_SKIP:-0}"

check_centos6_kernel_or_fail() {
    # Only CentOS/RHEL 6 is affected. Everything newer has overlay2.
    [[ "$OS_FAMILY" == "rhel" && "${OS_VERSION%%.*}" == "6" ]] || return 0

    # NUMERIC compare on major.minor, not a `2.6.*` glob. overlay2 requires kernel
    # >= 4.0 (the 3.10 exception is RHEL 7's heavily patched kernel, NOT an elrepo
    # 3.x on EL6), so an EL6 box on a 3.x kernel passed a `2.6.*` test and then hit
    # the same 42 GB vfs blowup -- after stop_rpm_services, which is the expensive
    # place this gate exists to avoid.
    #
    # An unreadable or unparseable /proc/version REFUSES. Fail-open is the one thing
    # a fail-closed gate must never do, and the glob version returned 0 on an empty
    # release string.
    local release major minor
    release=$(awk '{print $3; exit}' "$NVR_PROC_VERSION" 2>/dev/null || true)
    major=${release%%.*}
    minor=${release#*.}; minor=${minor%%.*}
    if [[ "$major" =~ ^[0-9]+$ ]] && [[ "$minor" =~ ^[0-9]+$ ]]; then
        # 4.0 or newer: overlay2 is available, nothing to do.
        if [ "$major" -ge 4 ]; then
            return 0
        fi
    else
        log_warn "Could not parse a kernel release from $NVR_PROC_VERSION (got '${release:-empty}')"
        log_warn "Treating that as unsafe: this gate fails CLOSED."
    fi

    log_error "================================================================"
    log_error "REFUSING TO CONTINUE: this CentOS 6 box is running kernel ${release:-<unreadable>}"
    log_error "================================================================"
    log_error ""
    log_error "Docker has no usable storage driver below kernel 4.0. It falls back to"
    log_error "'vfs', which copies the entire filesystem for every image layer: the NVR"
    log_error "image set needs ~42GB there instead of ~3GB, and the install runs the root"
    log_error "filesystem out of space partway through."
    log_error ""

    # kernel-lt is normally already installed (rda-release requires it). Say which
    # of the two situations this box is in, because the remedy differs by one step.
    local have_lt=""
    if command -v rpm >/dev/null 2>&1; then
        # Gate on the EXIT STATUS, never on the message text: rpm translates
        # "package kernel-lt is not installed", so under a non-English locale the
        # substring guard missed and the installer told the tech "ALREADY INSTALLED,
        # just reboot" when it was not -- burning a maintenance window on a reboot
        # that changes nothing.
        if rpm -q kernel-lt >/dev/null 2>&1; then
            have_lt=$(LC_ALL=C rpm -q --qf '%{VERSION}-%{RELEASE}\n' kernel-lt 2>/dev/null | head -1 || true)
        fi
    fi

    if [[ -n "$have_lt" ]]; then
        log_error "kernel-lt $have_lt is ALREADY INSTALLED on this box."
        log_error ""
        log_error "  DO THIS:  reboot, confirm 'uname -r' shows 4.4 or newer, re-run this"
        log_error "            installer. Nothing else is needed."
    else
        log_error "kernel-lt is NOT installed. Install it, reboot into it, then re-run:"
        log_error ""
        log_error "  DO THIS:  yum -y install kernel-lt"
        log_error "            grub.conf 'default' must point at the kernel-lt entry"
        log_error "            reboot"
        log_error "            uname -r   # expect 4.4.x or newer"
        log_error "            re-run this installer"
    fi
    log_error ""
    log_error "Nothing has been changed on this box yet: this check runs before the"
    log_error "backup and before any RPM service is stopped, so a re-run is safe."
    log_error ""
    log_error "Override with NVR_KERNEL_GATE_SKIP=1 only if you have confirmed"
    log_error "'docker info' reports a storage driver other than vfs."

    if [[ "$NVR_KERNEL_GATE_SKIP" == "1" ]]; then
        log_warn "NVR_KERNEL_GATE_SKIP=1 set; continuing onto a 2.6 kernel anyway"
        return 0
    fi
    # return 1, not exit 1. NOTE, corrected 2026-07-27: no caller is currently inside
    # the `trap rollback_rpm_services ERR` window -- there is exactly one such trap,
    # installed partway through migrate_flow, and all three call sites are before it.
    # `return` is still the right form because it KEEPS this safe if a caller is ever
    # moved inside that window, whereas `exit` would bypass the trap. Do not "simplify"
    # it to exit on the grounds that there is no trap today.
    return 1
}

################################################################################
# Init System Helpers
#
# CentOS 6 uses SysV init (no systemd). These helpers abstract service
# management so the rest of the script works on both init systems.
################################################################################

has_systemd() { command -v systemctl &>/dev/null; }

svc_start()   { if has_systemd; then systemctl start "$1";  else service "$1" start;  fi; }
svc_stop()    { if has_systemd; then systemctl stop "$1";   else service "$1" stop;   fi; }
svc_enable()  { if has_systemd; then systemctl enable "$1"; else chkconfig "$1" on 2>/dev/null; fi; }
svc_disable() { if has_systemd; then systemctl disable "$1"; else chkconfig "$1" off 2>/dev/null; fi; }
svc_active()  { if has_systemd; then systemctl is-active --quiet "$1"; else service "$1" status &>/dev/null; fi; }

# RPM restore can resurrect a SysV service whose stop action waits forever for
# an old daemon. Keep this migration-only path bounded; the caller follows it
# by killing known non-container processes and verifying no legacy port owner
# remains before Docker is allowed to continue.
migration_svc_stop() {
    local svc="$1"
    if has_systemd; then
        systemctl stop "$svc" 2>/dev/null || true
    elif command -v timeout &>/dev/null; then
        local stop_rc=0
        if timeout --help 2>&1 | grep -q -- '--kill-after'; then
            timeout --signal=TERM --kill-after=5s 30s service "$svc" stop 2>/dev/null || stop_rc=$?
        else
            # CentOS 6 coreutils timeout has --signal but no --kill-after.
            # Passing the newer option exits 125 before service(8) runs.
            timeout --signal=TERM 30s service "$svc" stop 2>/dev/null || stop_rc=$?
        fi
        if [[ "$stop_rc" -eq 124 || "$stop_rc" -eq 137 ]]; then
            log_warn "Timed out stopping legacy service $svc; forcing remaining non-container processes down"
        fi
    else
        svc_stop "$svc" 2>/dev/null || true
    fi
}

# Refuse to proceed if the NVR's current version is below 6.0. The current
# rda-db image ships only the 37 post-6.0 update scripts (the 206 pre-6.0
# scripts were purged — see PR1 of feature/rda-db-py3-cleanup). A <6.0
# customer running update.py against this tree would skip directly from
# whatever pre-6.0 schema they're on to the 6.0+ scripts without running
# the intermediate migrations, producing a subtly broken DB.
#
# Bypasses on fresh-install signals (missing dvs.conf, VERSION=0, -1, empty)
# so a fresh install path is never gated. Only fires on an actual upgrade
# or migration from a pre-6.0 system.
#
# Called from migrate_flow (reads /etc/dvs.conf on the RPM host) and
# upgrade_flow (reads the restored /data/config/dvs.conf post-restore).
check_version_6_0_or_fail() {
    local conf="$1"
    if [[ ! -f "$conf" ]]; then
        return 0  # Fresh install or missing config — nothing to gate
    fi
    local version
    version=$(grep -E '^VERSION=' "$conf" | head -1 | cut -d= -f2 | tr -d '\r\n ')
    case "$version" in
        ''|'0'|'-1')
            return 0  # Fresh-install sentinel values — bypass
            ;;
    esac
    local major
    major=$(echo "$version" | cut -d. -f1)
    if ! [[ "$major" =~ ^[0-9]+$ ]]; then
        log_error "Cannot parse VERSION=$version from $conf."
        log_error "Expected format: X.Y or X.Y.Z  (e.g. 6.0, 6.2.1)"
        # return 1 (not exit 1): two of the three callers run inside
        # migrate_flow's trap window (line 2388 in restore_from_backup,
        # line 2692 at migrate_flow entry). exit 1 bypasses the ERR
        # trap regardless of set -E; return 1 propagates via set -e
        # so the rollback fires. Non-trap callers (line 2623 in
        # upgrade_flow) also still abort because set -e propagates the
        # non-zero return to the call site.
        return 1
    fi
    if [[ $major -lt 6 ]]; then
        log_error "This NVR is running version $version, which is below 6.0."
        log_error ""
        log_error "The Docker release only supports migration from 6.0 or newer."
        log_error "The 206 pre-6.0 database migration scripts were removed from"
        log_error "the current image. Running it against a pre-6.0 database would"
        log_error "skip intermediate migrations and leave the DB in a broken state."
        log_error ""
        log_error "Stepping-stone path:"
        log_error "  1. Install the 6.0 RPM release from the legacy repo"
        log_error "     (files.dividia.net) and run yum update to bring this NVR"
        log_error "     up to 6.0 or newer."
        log_error "  2. Verify /etc/dvs.conf shows VERSION=6.0 (or higher)."
        log_error "  3. Re-run this installer."
        return 1
    fi
}

check_virtualbox_graphics() {
    if [[ -f /sys/class/dmi/id/product_name ]] && grep -qi "virtualbox" /sys/class/dmi/id/product_name; then
        if command -v lspci &> /dev/null && lspci | grep -qi "vmware svga"; then
            log_warn "VirtualBox detected with VMSVGA graphics controller"
            log_warn "The NVR viewer requires VBoxVGA for Java OpenGL compatibility"
            log_warn "Fix: VBoxManage modifyvm <VM> --graphicscontroller vboxvga --vram 128"
        fi
    fi
}

################################################################################
# Docker Installation
################################################################################

check_docker() {
    if command -v docker &> /dev/null; then
        # Verify docker compose v2 is available (Docker 18.x only has v1)
        if docker compose version &>/dev/null; then
            log_info "Docker is already installed: $(docker --version)"
            return 0
        fi
        log_warn "Docker found but too old (no compose v2 support) — upgrading"
        return 1
    fi
    return 1
}

install_docker_rhel() {
    log_step "Installing Docker on RHEL-based system"

    if command -v dnf &>/dev/null; then
        dnf remove -y docker docker-client docker-client-latest docker-common \
            docker-latest docker-latest-logrotate docker-logrotate docker-engine \
            podman runc 2>/dev/null || true

        dnf install -y dnf-plugins-core
        dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
        dnf install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
    else
        # CentOS 7: yum-based Docker install
        yum remove -y docker docker-client docker-client-latest docker-common \
            docker-latest docker-latest-logrotate docker-logrotate docker-engine 2>/dev/null || true

        yum install -y yum-utils
        yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo

        # Docker deps (container-selinux, slirp4netns, fuse-overlayfs) are shipped in
        # the NVR's rda repo under os-updates/.  Install deps first, then Docker CE.
        yum install -y container-selinux slirp4netns fuse-overlayfs fuse3-libs 2>/dev/null || true
        yum install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
    fi

    svc_enable docker
    svc_start docker
    log_info "Docker installed successfully"
}

install_docker_static() {
    # Install Docker CE from static binaries — works on any Linux x86_64.
    # Avoids yum/dnf dependency hell. On CentOS 6 (no systemd), creates a
    # SysV init script. On CentOS 7+/systemd, creates a systemd unit.
    log_step "Installing Docker via static binaries"

    local docker_ver="27.5.1"
    local compose_ver="v2.32.4"
    local url="https://download.docker.com/linux/static/stable/x86_64/docker-${docker_ver}.tgz"

    # Stop old Docker and containerd if running (e.g., Docker 1.7 from EPEL,
    # docker-ce RPM, or old containerd RPM). The old containerd must be stopped
    # because dockerd will connect to a running containerd socket — if the old
    # v1.2 containerd is running, the new Docker 27 fails with "unknown method
    # AddResource: not implemented". Don't remove RPMs — they may have dependents.
    if has_systemd; then
        systemctl stop docker 2>/dev/null || true
        systemctl disable docker 2>/dev/null || true
        systemctl stop containerd 2>/dev/null || true
        systemctl disable containerd 2>/dev/null || true
        # Kill any lingering containerd — if an old version (e.g., 1.2.x from
        # docker-ce RPM) is running, the new dockerd connects to it and fails.
        pkill -9 containerd 2>/dev/null || true
        sleep 1
    else
        service docker stop 2>/dev/null || true
    fi

    # Install Docker CE static binaries
    log_info "Downloading Docker CE ${docker_ver}..."
    curl -fsSL "$url" | tar xz -C /usr/local/bin --strip-components=1
    log_info "Docker binaries installed to /usr/local/bin"

    # Install Docker Compose v2 plugin
    log_info "Downloading Docker Compose ${compose_ver}..."
    # The CLI plugin is used by the non-root `dividia` operator account too.
    # A restrictive inherited umask on older RPM hosts otherwise makes the
    # directory and plugin root-only: Docker works for root but `nvr status`
    # under SSH fails with "Docker Compose not found".
    install -d -m 755 /usr/local/lib/docker/cli-plugins
    curl -fsSL "https://github.com/docker/compose/releases/download/${compose_ver}/docker-compose-linux-x86_64" \
        -o /usr/local/lib/docker/cli-plugins/docker-compose
    chmod 755 /usr/local/lib/docker /usr/local/lib/docker/cli-plugins \
        /usr/local/lib/docker/cli-plugins/docker-compose

    # Create docker group and data directories
    groupadd -f docker
    mkdir -p /var/run/docker /var/lib/docker /var/lib/containerd /etc/docker

    # Docker 27+ requires explicit userland-proxy-path when using static binaries
    cat > /etc/docker/daemon.json <<'DAEMONJSON'
{
    "userland-proxy-path": "/usr/local/bin/docker-proxy"
}
DAEMONJSON

    if has_systemd; then
        # Systemd unit for containerd (matched to the new static /usr/local/bin
        # binary). MUST be installed alongside docker.service — without it,
        # systemd keeps the EPEL-installed /usr/lib/systemd/system/containerd.service
        # which has ExecStart=/usr/bin/containerd (the OLD v1.2 binary). On the
        # next boot, the old containerd starts under that unit and the new
        # dockerd 27 talks to its socket — fails with "Unimplemented: unknown
        # method AddResource: not implemented" on every image pull. Discovered
        # on cs50 ACS-Odessa migration 2026-05-15. The override in
        # /etc/systemd/system/ takes precedence over /usr/lib/systemd/system/.
        cat > /etc/systemd/system/containerd.service <<'CTRDUNIT'
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
KillMode=process
Delegate=yes
LimitNOFILE=1048576
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity

[Install]
WantedBy=multi-user.target
CTRDUNIT

        # Systemd unit for Docker daemon (CentOS 7+, CentOS 9, Ubuntu).
        # Requires=/After= containerd so systemd starts containerd first on
        # boot — without these directives, docker.service starts before
        # containerd.service and dockerd briefly fails before retry.
        cat > /etc/systemd/system/docker.service <<'UNITEOF'
[Unit]
Description=Docker Application Container Engine
After=network-online.target containerd.service
Wants=network-online.target
Requires=containerd.service

[Service]
Type=notify
ExecStart=/usr/local/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutStartSec=0
RestartSec=2
Restart=always
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
Delegate=yes
KillMode=process
OOMScoreAdjust=-500

[Install]
WantedBy=multi-user.target
UNITEOF
        systemctl daemon-reload
        systemctl enable containerd docker
        systemctl start containerd
        systemctl start docker
    else
        # SysV init script for Docker daemon (CentOS 6)
        cat > /etc/init.d/docker <<'INITEOF'
#!/bin/bash
# chkconfig: 2345 95 05
# description: Docker daemon

### BEGIN INIT INFO
# Provides:       docker
# Required-Start: $network $syslog
# Required-Stop:  $network $syslog
# Default-Start:  2 3 4 5
# Default-Stop:   0 1 6
# Description:    Docker daemon
### END INIT INFO

DOCKERD=/usr/local/bin/dockerd
PIDFILE=/var/run/docker.pid
LOGFILE=/var/log/docker.log
export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin

mount_cgroups() {
    # CentOS 6 does not auto-mount cgroups; Docker requires them.
    if mountpoint -q /sys/fs/cgroup 2>/dev/null; then return 0; fi
    # On CentOS 6, /sys/fs is sysfs and cannot create the cgroup mount-point
    # directly.  Overlay it with tmpfs before creating /sys/fs/cgroup.
    if ! mountpoint -q /sys/fs 2>/dev/null; then
        mount -t tmpfs cgroup_fs /sys/fs 2>/dev/null || true
    fi
    mkdir -p /sys/fs/cgroup
    mount -t tmpfs cgroup_root /sys/fs/cgroup 2>/dev/null || true
    for s in memory cpuset cpuacct devices freezer blkio perf_event pids net_cls net_prio; do
        mkdir -p /sys/fs/cgroup/$s
        mount -t cgroup -o $s cgroup_$s /sys/fs/cgroup/$s 2>/dev/null || true
    done
    mkdir -p /sys/fs/cgroup/systemd
    mount -t cgroup -o none,name=systemd cgroup_systemd /sys/fs/cgroup/systemd 2>/dev/null || true
    # Enable shared mount propagation so bind mounts with :slave work
    mount --make-rshared / 2>/dev/null || true
}

start() {
    echo -n "Starting Docker daemon: "
    if [ -f "$PIDFILE" ] && kill -0 $(cat "$PIDFILE") 2>/dev/null; then
        echo "already running"
        return 0
    fi
    mount_cgroups
    $DOCKERD --pidfile "$PIDFILE" >> "$LOGFILE" 2>&1 &
    local i=0
    while [ $i -lt 30 ]; do
        if docker info &>/dev/null; then
            echo "OK"
            return 0
        fi
        sleep 1
        i=$((i + 1))
    done
    echo "FAILED (timeout)"
    return 1
}

stop() {
    echo -n "Stopping Docker daemon: "
    if [ ! -f "$PIDFILE" ] || ! kill -0 $(cat "$PIDFILE") 2>/dev/null; then
        echo "not running"
        return 0
    fi
    kill $(cat "$PIDFILE")
    local i=0
    while [ $i -lt 30 ] && kill -0 $(cat "$PIDFILE") 2>/dev/null; do
        sleep 1
        i=$((i + 1))
    done
    echo "OK"
}

status() {
    if [ -f "$PIDFILE" ] && kill -0 $(cat "$PIDFILE") 2>/dev/null; then
        echo "Docker daemon is running (PID $(cat $PIDFILE))"
        return 0
    else
        echo "Docker daemon is not running"
        return 3
    fi
}

case "$1" in
    start)   start ;;
    stop)    stop ;;
    restart) stop; start ;;
    status)  status ;;
    *)       echo "Usage: $0 {start|stop|restart|status}"; exit 1 ;;
esac
INITEOF
        chmod 755 /etc/init.d/docker
        chkconfig docker on
        # A predeclared VideoStore dependency must guard the first daemon
        # start. The canonical init template above replaces any older hook.
        prepare_configured_boot_storage_guard
        service docker start
    fi

    hash -r  # Clear bash's cached binary paths after install
    log_info "Docker installed successfully (static binaries)"
}

install_docker_debian() {
    log_step "Installing Docker on Debian-based system"

    apt-get remove -y docker docker-engine docker.io containerd runc 2>/dev/null || true
    apt-get update
    apt-get install -y ca-certificates curl gnupg lsb-release

    mkdir -p /etc/apt/keyrings
    curl -fsSL "https://download.docker.com/linux/$OS/gpg" | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg

    echo \
        "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/$OS \
        $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null

    apt-get update
    apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

    svc_enable docker
    svc_start docker
    log_info "Docker installed successfully"
}

install_docker() {
    # A migration can establish and declare the /opt bind before Docker is
    # installed. Load its guard before a package script or existing daemon can
    # start Docker. The static CO6 path repeats this after rewriting its init
    # script.
    prepare_configured_boot_storage_guard
    if check_docker; then
        return 0
    fi

    case "$OS_FAMILY" in
        rhel)
            local os_major="${OS_VERSION%%.*}"
            if [[ "$os_major" -le 7 ]]; then
                # CentOS 6/7: static binaries (yum dependency hell, old containerd)
                install_docker_static
            else
                # CentOS 8+/Stream 9/RHEL 9+: docker-ce repo works fine
                install_docker_rhel
            fi
            ;;
        debian) install_docker_debian ;;
    esac

    docker --version
    docker compose version
}

################################################################################
# Log Rotation (Docker + journald)
#
# Docker's default json-file driver has NO log rotation — a chatty container
# can fill the disk. Cap each container at 10MB × 3 files = 30MB max.
# journald defaults to ~4GB on a 40GB disk; cap at 200MB for scale-watcher VMs.
# Called after install_docker so it applies to all install paths.
################################################################################
configure_docker_logging() {
    log_step "Configuring log rotation (Docker + journald)"

    # --- Docker log rotation ---
    mkdir -p /etc/docker
    local json_python=""
    if command -v python3 &>/dev/null; then
        json_python=python3
    elif command -v python &>/dev/null; then
        json_python=python
    fi

    if [[ -f /etc/docker/daemon.json ]] && [[ -n "$json_python" ]]; then
        "$json_python" <<'PYEOF'
import json
try:
    with open('/etc/docker/daemon.json') as f:
        cfg = json.load(f)
except Exception:
    cfg = {}
cfg['log-driver'] = 'json-file'
cfg['log-opts'] = {'max-size': '10m', 'max-file': '3'}
with open('/etc/docker/daemon.json', 'w') as f:
    json.dump(cfg, f, indent=2)
PYEOF
    else
        cat > /etc/docker/daemon.json <<'DAEMONJSON'
{
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "10m",
        "max-file": "3"
    }
}
DAEMONJSON
    fi

    svc_stop docker 2>/dev/null || true
    svc_start docker
    log_info "Docker log rotation: 10MB × 3 files per container (30MB max each)"

    # --- journald cap (systemd systems only) ---
    if has_systemd && [[ -d /etc/systemd/journald.conf.d ]] || mkdir -p /etc/systemd/journald.conf.d 2>/dev/null; then
        cat > /etc/systemd/journald.conf.d/nvr-cap.conf <<'JOURNALD'
[Journal]
SystemMaxUse=200M
SystemKeepFree=500M
JOURNALD
        systemctl restart systemd-journald 2>/dev/null || true
        log_info "journald cap: 200MB max, 500MB min free"
    fi
}

# Canonicalize existing Linux port reservations plus the NVR service range.
# Kept POSIX-awk compatible because this installer runs on CO6/CO7/CO9/UB24.
merge_reserved_port_ranges() {
    local current="${1:-}"
    local required="${2:-$NVR_SERVICE_PORT_RANGE}"
    printf '%s\n' "${current}${current:+,}${required}" | awk -F, '
        function trim(s) {
            gsub(/^[[:space:]]+|[[:space:]]+$/, "", s)
            return s
        }
        {
            for (i = 1; i <= NF; i++) {
                token = trim($i)
                if (token == "") continue
                count = split(token, edge, "-")
                if (count == 1 && edge[1] ~ /^[0-9]+$/) {
                    first = edge[1] + 0
                    last = first
                } else if (count == 2 && edge[1] ~ /^[0-9]+$/ && edge[2] ~ /^[0-9]+$/) {
                    first = edge[1] + 0
                    last = edge[2] + 0
                } else {
                    invalid = 1
                    continue
                }
                if (first < 1 || last > 65535 || first > last) {
                    invalid = 1
                    continue
                }
                for (port = first; port <= last; port++) reserved[port] = 1
            }
        }
        END {
            if (invalid) exit 2
            separator = ""
            port = 1
            while (port <= 65535) {
                if (!(port in reserved)) {
                    port++
                    continue
                }
                first = port
                while (port < 65535 && ((port + 1) in reserved)) port++
                last = port
                if (first == last) printf "%s%d", separator, first
                else printf "%s%d-%d", separator, first, last
                separator = ","
                port++
            }
            print ""
        }
    '
}

# All containers share the host network namespace. Keep their fixed listeners
# out of Linux's ephemeral outbound-port pool before the first container starts.
# Merge with existing reservations so installing NVR cannot erase another
# product's host configuration.
configure_reserved_service_ports() {
    log_step "Reserving NVR host service ports"

    local current merged wanted file tmp mode
    file="$SYSCTL_RESERVED_PORTS_FILE"

    if ! current=$(sysctl -n net.ipv4.ip_local_reserved_ports 2>/dev/null); then
        log_warn "Kernel does not expose net.ipv4.ip_local_reserved_ports; NVR service ports were not reserved"
        return 0
    fi
    if ! merged=$(merge_reserved_port_ranges "$current" "$NVR_SERVICE_PORT_RANGE"); then
        log_warn "Current ip_local_reserved_ports value is invalid; left live and persistent settings untouched"
        return 0
    fi
    wanted="net.ipv4.ip_local_reserved_ports = $merged"

    if [[ "$current" != "$merged" ]]; then
        if ! sysctl -w "net.ipv4.ip_local_reserved_ports=$merged" >/dev/null 2>&1; then
            log_warn "Could not reserve NVR service ports live; left $file untouched"
            return 0
        fi
        log_info "Reserved host service ports $NVR_SERVICE_PORT_RANGE from the ephemeral allocator"
    fi

    if ! mkdir -p "$(dirname "$file")" 2>/dev/null; then
        log_warn "Could not create service-port reservation directory for $file"
        return 0
    fi
    if [[ -f "$file" ]] && grep -qxF "$wanted" "$file" 2>/dev/null; then
        mode=$(stat -c '%a' "$file" 2>/dev/null || stat -f '%Lp' "$file" 2>/dev/null || echo "")
        [[ -z "$mode" || "$mode" == "644" ]] || chmod 0644 "$file" 2>/dev/null || true
        return 0
    fi

    tmp="${file}.new.$$"
    if printf '%s\n' "$wanted" > "$tmp" 2>/dev/null; then
        chmod 0644 "$tmp" 2>/dev/null || true
        if ! mv -f "$tmp" "$file" 2>/dev/null; then
            rm -f "$tmp" 2>/dev/null || true
            log_warn "Could not persist NVR service-port reservation to $file"
        fi
    else
        rm -f "$tmp" 2>/dev/null || true
        log_warn "Could not write NVR service-port reservation to $file"
    fi
    return 0
}

################################################################################
# Docker Storage Configuration
#
# NVR images are large (~8GB total). Many NVR appliances have small root
# partitions (10-20GB) with a large /opt partition for data. Docker and
# containerd both default to /var/lib which lives on root. This function
# detects low root space and relocates both to /opt before any images are
# pulled.
################################################################################

configure_docker_storage() {
    log_step "Checking Docker storage location"

    local root_avail
    # CentOS 6 df lacks --output; fall back to awk parsing
    root_avail=$(df -P / | awk 'NR==2{print $4}')  # in 1K blocks

    local docker_root
    docker_root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null)

    # If Docker data is already on /opt, nothing to do.
    if [[ "$docker_root" == /opt/* ]]; then
        log_info "Docker data already on /opt ($docker_root)"
        return 0
    fi

    # Thresholds re-tuned 2026-05-15 after further image-shrink work that
    # the original 15G/10G numbers predated. Actual sizing today:
    #
    #   Fresh-install image footprint (sum of pulled tags on cs256):
    #     viewer 957M + backend 601M + connector 313M + engine 195M +
    #     playback 185M + mariadb 395M + autoheal/watchtower ~50M = ~2.7G
    #   Pull-time scratch:          ~500M transient
    #   Watchtower update overlap:  up to one image-set extra until the
    #                               daily 03:30 prune drops the old layers
    #                               (worst case +1G for ~24h)
    #   Sustainable floor:          ~5G on /opt; ~8G on / if Docker stays
    #                               on the root partition.
    #
    # 5G/8G is the floor the Windows installer also recommends. Headroom
    # above that comes from /opt sizing on real-disk NVRs (cs256/cs2427
    # have ~50G /opt today) and from the EBS-grow path for cloud NVRs
    # (cs1018 root grew 10→20G during 2026-05-15 migration).
    local ROOT_MIN_GB_KB=$(( 8 * 1024 * 1024 ))   # 8 GB
    local OPT_MIN_GB_KB=$(( 5 * 1024 * 1024 ))    # 5 GB

    if [[ $root_avail -gt $ROOT_MIN_GB_KB ]]; then
        log_info "Root filesystem has sufficient space ($(( root_avail / 1048576 ))GB), using default locations"
        return 0
    fi

    # Check /opt is a separate mount with enough space
    local opt_avail
    opt_avail=$(df -P /opt 2>/dev/null | awk 'NR==2{print $4}')

    if [[ -z "$opt_avail" ]] || [[ $opt_avail -lt $OPT_MIN_GB_KB ]]; then
        log_warn "Root has only $(( root_avail / 1048576 ))GB free but /opt doesn't have enough space either"
        log_warn "NVR images need ~3GB resting + headroom for daily nvr update pulls. Installation may fail."
        return 0
    fi

    log_info "Root filesystem has only $(( root_avail / 1048576 ))GB free — relocating Docker storage to /opt"

    # Stop Docker and containerd
    svc_stop docker 2>/dev/null || true
    svc_stop containerd 2>/dev/null || true
    pkill -TERM -x dockerd 2>/dev/null || true
    pkill -TERM -x containerd 2>/dev/null || true
    sleep 2
    pkill -KILL -x dockerd 2>/dev/null || true
    pkill -KILL -x containerd 2>/dev/null || true

    # Configure Docker data-root
    mkdir -p /opt/docker
    mkdir -p /etc/docker
    if [[ -f /etc/docker/daemon.json ]]; then
        # Merge data-root into existing config
        if command -v python3 &> /dev/null; then
            python3 -c "
import json
with open('/etc/docker/daemon.json') as f:
    cfg = json.load(f)
cfg['data-root'] = '/opt/docker'
with open('/etc/docker/daemon.json', 'w') as f:
    json.dump(cfg, f, indent=2)
"
        else
            echo '{"data-root": "/opt/docker"}' > /etc/docker/daemon.json
        fi
    else
        echo '{"data-root": "/opt/docker"}' > /etc/docker/daemon.json
    fi
    log_info "Docker data-root set to /opt/docker"

    # Configure containerd root (Docker 29+ uses containerd for snapshots)
    mkdir -p /opt/containerd
    mkdir -p /etc/containerd
    cat > /etc/containerd/config.toml <<TOML
version = 2
root = "/opt/containerd"
TOML
    log_info "Containerd root set to /opt/containerd"

    # Clean up old data from root partition
    if ! rm -rf /var/lib/docker /var/lib/containerd; then
        log_warn "Could not remove old /var/lib Docker storage; continuing with /opt data-root"
    fi

    # Restart services
    svc_start containerd 2>/dev/null || true
    svc_start docker
    log_info "Docker storage relocated to /opt ($(( opt_avail / 1048576 ))GB available)"
}

################################################################################
# Docker Hub Authentication
################################################################################

docker_login() {
    log_step "Docker Hub Authentication"

    local DEFAULT_DOCKER_USER="nvrservice"
    local DEFAULT_DOCKER_PASS="dckr_pat_LHFmDIppHGAOH9llE3hKl1ZTu8o"  # nvr-pull read-only PAT

    # Priority: env vars > existing config > built-in defaults
    if [[ -n "$DOCKER_USER" ]] && [[ -n "$DOCKER_PASS" ]]; then
        log_info "Using credentials from environment variables"
    elif grep -q '"auths"' ~/.docker/config.json 2>/dev/null && \
         grep -q 'docker.io\|index.docker' ~/.docker/config.json 2>/dev/null; then
        log_info "Docker Hub credentials already configured"
        return
    else
        log_info "Using built-in service credentials"
        DOCKER_USER="$DEFAULT_DOCKER_USER"
        DOCKER_PASS="$DEFAULT_DOCKER_PASS"
    fi

    echo "$DOCKER_PASS" | docker login --username "$DOCKER_USER" --password-stdin "$REGISTRY"
    log_info "Docker Hub authentication configured"
}

################################################################################
# NVR Installation
################################################################################

create_directory_structure() {
    log_step "Creating directory structure"

    # Ensure dividia user and docker group exist before any chown
    if ! getent group docker &>/dev/null; then
        groupadd docker
        log_info "Created docker group"
    fi
    if ! id "dividia" &>/dev/null; then
        useradd -r -m -G docker -s /bin/bash dividia
        log_info "Created dividia user"
    fi

    mkdir -p "$INSTALL_DIR"
    chown dividia:docker "$INSTALL_DIR"
    mkdir -p "$DATA_DIR"
    mkdir -p "$DATA_DIR/db_data"
    mkdir -p "$DATA_DIR/config"
    # Shared Apache Listen overrides dir. Backend writes dvs-extra.conf here
    # when Server.bPort != 80; viewer reads via Include in httpd.conf. Empty by
    # default — rda-backend at startup syncs from DB and populates if needed.
    mkdir -p "$DATA_DIR/apache-extra"

    # VideoStore data-dir layout decisions:
    #   1. --video-device  → prepare_video_device handles the partition; we
    #      just need $DATA_DIR/videostore to exist as the bind-mount source.
    #   2. Auto-detected pre-labeled vs[N] (kickstart path) → same; Phase 1
    #      in backend's docker-start mounts the real partition over
    #      /videostore/vs<N> via nsenter on first container start.
    #   3. --no-video-device → operator explicitly accepted the directory
    #      fallback. Create $DATA_DIR/videostore/vs1/dividia/tickets so
    #      Phase 1's directory-mode discovery has somewhere to record to.
    #      Warn loudly that recordings will fill root.
    #   4. Neither flag and no auto-detect: only reachable from upgrade_flow
    #      or migrate_flow (fresh_install_flow has been gated by
    #      check_video_device_or_fail above). The prior VideoStore config
    #      lives in the DB; Phase 1+2 will mount it. Create the fallback dir
    #      as backward-compat (Phase 2 mount shadows it on success).
    if [[ -n "$VIDEO_DEVICE" ]]; then
        mkdir -p "$DATA_DIR/videostore"
    elif [[ -n "$DETECTED_VS_LABELS" ]]; then
        mkdir -p "$DATA_DIR/videostore"
    elif [[ "$NO_VIDEO_DEVICE" == "true" ]]; then
        log_warn "Creating directory-based VideoStore at $DATA_DIR/videostore/vs1"
        log_warn "Recordings will go to root partition — dev/test only"
        mkdir -p "$DATA_DIR/videostore/vs1/dividia/tickets"
    else
        # upgrade_flow / migrate_flow path: VideoStore comes from preserved DB
        mkdir -p "$DATA_DIR/videostore/vs1/dividia/tickets"
    fi

    # Seed timezone files — must exist before docker compose up
    # (Docker creates empty files for missing bind-mount sources, causing UTC fallback)
    mkdir -p "$DATA_DIR/timezone"
    if [[ ! -s "$DATA_DIR/timezone/localtime" ]]; then
        cp /usr/share/zoneinfo/America/Chicago "$DATA_DIR/timezone/localtime"
        # Also set host timezone to match
        if command -v timedatectl &>/dev/null; then
            timedatectl set-timezone America/Chicago
        else
            cp /usr/share/zoneinfo/America/Chicago /etc/localtime
        fi
        log_info "Seeded timezone: America/Chicago"
    fi
    if [[ ! -s "$DATA_DIR/timezone/clock" ]]; then
        cat > "$DATA_DIR/timezone/clock" <<TZEOF
ZONE="America/Chicago"
UTC=false
ARC=false
TZEOF
    fi

    log_info "Created directories:"
    log_info "  Install: $INSTALL_DIR"
    log_info "  Data:    $DATA_DIR"
}

download_compose_files() {
    log_step "Downloading compose files"

    curl -fsSL "$BASE_URL/docker-compose.yml" -o "$INSTALL_DIR/docker-compose.yml"
    log_info "Downloaded docker-compose.yml"

    curl -fsSL "$BASE_URL/docker-compose.prod.yml" -o "$INSTALL_DIR/docker-compose.prod.yml"
    log_info "Downloaded docker-compose.prod.yml"

    # Optional HME drive-thru overlay. Not OS-specific: download it on every
    # host (like prod.yml) so `nvr addon hme enable` can reference it without a
    # first update round-trip. Inert until COMPOSE_FILE names it.
    curl -fsSL "$BASE_URL/docker-compose.hme.yml" -o "$INSTALL_DIR/docker-compose.hme.yml"
    log_info "Downloaded docker-compose.hme.yml"

    # CentOS 6 overlay (headless prod without start_interval)
    if [[ "$OS" == "centos" && "${OS_VERSION%%.*}" == "6" ]]; then
        curl -fsSL "$BASE_URL/docker-compose.co6.yml" -o "$INSTALL_DIR/docker-compose.co6.yml"
        log_info "Downloaded docker-compose.co6.yml"
    fi

    # Windows overlay (no privileged mode, directory VideoStore, no host mounts)
    # Used for both WSL2 and Hyper-V VM Windows deployments
    if [[ "$IS_WINDOWS" == "true" ]]; then
        curl -fsSL "$BASE_URL/docker-compose.windows.yml" -o "$INSTALL_DIR/docker-compose.windows.yml"
        log_info "Downloaded docker-compose.windows.yml"
    fi

    # Optional aiengine add-on overlay, downloaded UNCONDITIONALLY. It is inert
    # on disk (not in COMPOSE_FILE) until `nvr addon aiengine enable` or an RPM
    # migration adopts it, but it must be present so enable/adoption never has to
    # wait for a first `nvr update` to receive it.
    curl -fsSL "$BASE_URL/docker-compose.aiengine.yml" -o "$INSTALL_DIR/docker-compose.aiengine.yml"
    log_info "Downloaded docker-compose.aiengine.yml"
}

# Replace or append a single KEY=VALUE in the host .env. Atomic-ish (write
# temp, rename). Used by aiengine adoption to pin the image ref. VALUE may
# contain '/', '@', ':' (image digests do), so use a delimiter that cannot
# appear in a shell-safe env value and escape nothing.
set_host_env_var() {
    local key="$1" val="$2" f="$INSTALL_DIR/.env"
    [[ -f "$f" ]] || return 1
    local tmp="${f}.new.$$"
    # Value via ENVIRON, not `awk -v`, so a backslash is never escape-processed.
    AENV_K="$key" AENV_V="$val" awk '
        BEGIN { k=ENVIRON["AENV_K"]; v=ENVIRON["AENV_V"]; done=0 }
        !done && index($0, k "=") == 1 { print k "=" v; done=1; next }
        { print }
        END { if (!done) print k "=" v }
    ' "$f" > "$tmp" || { rm -f "$tmp"; return 1; }
    # Preserve owner+mode: a fresh temp lands root:root 0644 under root's umask,
    # which would leak MYSQL_ROOT_PASSWORD from .env. Match the dividia:docker
    # 0640 contract the installer set.
    chmod --reference="$f" "$tmp" 2>/dev/null \
        || chmod "$(stat -c '%a' "$f" 2>/dev/null || echo 640)" "$tmp" 2>/dev/null || true
    chown --reference="$f" "$tmp" 2>/dev/null || true
    mv -f "$tmp" "$f"
}

create_env_file() {
    log_step "Creating environment configuration"

    # Preserve an existing DB root password from a prior .env. The MariaDB
    # bind-mount at data/db_data initializes once and keeps whatever password
    # was generated on that first run; if we wrote a fresh random password
    # here, the backend would get "Access denied" against the existing data.
    # Regenerate only when .env is absent or has no MYSQL_ROOT_PASSWORD line.
    local db_pass
    if [[ -f "$INSTALL_DIR/.env" ]]; then
        db_pass=$(grep '^MYSQL_ROOT_PASSWORD=' "$INSTALL_DIR/.env" 2>/dev/null | cut -d= -f2- | tr -d '\r\n')
    fi
    if [[ -z "$db_pass" ]]; then
        db_pass=$(openssl rand -base64 24 | tr -d '/+=' | head -c 24)
        log_info "Generated new MYSQL_ROOT_PASSWORD"
    else
        log_info "Preserving existing MYSQL_ROOT_PASSWORD from .env"
    fi
    # Detect Docker API version for the .env DOCKER_API_VERSION field.
    # Historical: required by watchtower; kept after watchtower removal
    # to avoid breaking customer .env files that already reference it.
    local api_ver
    api_ver=$(docker version --format '{{.Server.APIVersion}}' 2>/dev/null || echo "1.44")

    cat > "$INSTALL_DIR/.env" <<EOF
# NVR Docker Environment Configuration
# Compose overlay: base + production (videostore + headless)
COMPOSE_FILE=docker-compose.yml:docker-compose.prod.yml
CHANNEL=$CHANNEL
REGISTRY=$REGISTRY
WATCHTOWER_SCHEDULE="$WATCHTOWER_SCHEDULE"
DOCKER_API_VERSION=$api_ver
MYSQL_ROOT_PASSWORD=$db_pass
# NOTE: To change the DB password on a running system, you must first run:
#   nvr db -e "ALTER USER 'root'@'localhost' IDENTIFIED BY 'new-password';"
#   nvr db -e "ALTER USER 'root'@'%' IDENTIFIED BY 'new-password';"
# Then update this value and restart: docker compose down && docker compose up -d
EOF

    # CentOS 6/7: force headless mode (kernel too old for container Xorg/DRM)
    local os_major="${OS_VERSION%%.*}"
    if [[ "$OS" == "centos" && ("$os_major" == "6" || "$os_major" == "7") ]]; then
        echo "NVR_HEADLESS=true" >> "$INSTALL_DIR/.env"
        log_warn "CentOS ${os_major} detected - viewer will run in web-only mode (no local display)"
    fi

    # CentOS 6: use CO6 compose overlay (no start_interval, headless prod settings)
    if [[ "$OS" == "centos" && "$os_major" == "6" ]]; then
        sed -i 's|^COMPOSE_FILE=.*|COMPOSE_FILE=docker-compose.yml:docker-compose.co6.yml|' "$INSTALL_DIR/.env"
        log_info "Using CentOS 6 compose overlay"
    fi

    # Windows: use Windows compose overlay (no privileged, directory VideoStore)
    if [[ "$IS_WINDOWS" == "true" ]]; then
        sed -i 's|^COMPOSE_FILE=.*|COMPOSE_FILE=docker-compose.yml:docker-compose.windows.yml|' "$INSTALL_DIR/.env"
        cat >> "$INSTALL_DIR/.env" <<'WINENV'
NVR_PLATFORM=windows
NVR_LOCAL_NET_SERVICES=0
NVR_HOST_MOUNTS=0
NVR_SVIEW_MODE=1
WINENV
        log_info "Using Windows compose overlay"
    fi

    chown dividia:docker "$INSTALL_DIR/.env"
    chmod 640 "$INSTALL_DIR/.env"
    boot_storage_compose_overlay_reconcile
    log_info "Created .env file"
}

################################################################################
# Start Services
################################################################################

# Backfill VideoStore table rows with sDevice / sLabel / sUUID / sFSType
# from blkid — so docker-start's mount logic (Phase 2) can re-mount every
# drive cleanly after a reboot.
#
# Why: RPM NVRs running rda-autofs stored the mount key (often just sUUID,
# sometimes only sMountPoint) in VideoStore. migrate_flow removes
# rda-autofs, but docker/backend/docker-start keeps doing the mount at
# backend-container start via nsenter. Its Phase 2 query needs at least
# one of sDevice / sLabel / sUUID populated to mount the right device.
#
# If the migrated row has only sMountPoint (cs256 2026-04-24), Phase 2
# skips it. First reboot: backend mounts nothing, mpengine writes to
# /videostore/vs1 on the root partition, root fills in ~1.5 hours,
# pbserver rejects every /videostore/vs1/* path because the files are
# on the unmounted drive.
#
# The mount reaches the writers because the docker-compose bind is
# `propagation: slave`: an nsenter mount in the backend container shows up
# in engine/connector/playback too.
#
# What used to be claimed here and is NOT true: that `depends_on: backend:
# service_healthy` means the writers "never start before the mount lands".
# depends_on gates START ORDER, not correctness, and it says nothing about
# whether Phase 2 actually succeeded -- the backend reports healthy either
# way, because an entrypoint that exits here takes all five containers down
# under restart:always (cs50, 14c8528c5). On the BCC fleet Phase 2 failed,
# the backend went healthy, the writers started, and mpengine recorded onto
# the ROOT filesystem for four days (2026-07-24).
#
# So the mount is now persisted HOST-side, before Docker starts at all:
# docker-start Phase 2 writes a managed fstab entry for every store whose
# mount it has verified by identity. See ADR-045 and
# OneDrive/Dividia/Software Plans/VideoStore-Persistent-Host-Mounts-Plan.
# remount_videostores_after_backfill below re-runs that phase once this
# backfill has given it something to mount, so a migrated box is persistent
# from its first boot rather than its second.
#
# Idempotent: updates rows to match the device currently mounted at the
# row's /videostore/vsN mountpoint. RPM-era systems can carry stale duplicate
# labels in the DB even after the filesystem label is corrected.
backfill_videostore_fields() {
    log_step "Backfilling VideoStore device info from blkid"

    if [[ "$IS_WINDOWS" == "true" ]]; then
        log_info "Windows deployment: VideoStore is a container bind; no device to backfill"
        return 0
    fi

    cd "$INSTALL_DIR"

    # Pull mountpoint + whatever's already filled in so we can refuse to
    # overwrite operator-set values.
    local vs_rows
    # Use a non-whitespace delimiter. `read` treats tab as IFS whitespace and
    # collapses empty fields, so a blank sLabel shifts sUUID/sFSType left and
    # can make a multi-VideoStore row escape the repair path.
    vs_rows=$(docker compose exec -T db sh -c 'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e "SELECT CONCAT(sName,CHAR(31),sMountPoint,CHAR(31),COALESCE(sDevice,\"\"),CHAR(31),COALESCE(sLabel,\"\"),CHAR(31),COALESCE(sUUID,\"\"),CHAR(31),COALESCE(sFSType,\"\")) FROM VideoStore WHERE fEnable=1"' 2>/dev/null) || {
        # Return 0 even on DB-query failure: we run inside migrate_flow under
        # `set -e` + `trap rollback_rpm_services ERR`, and a transient docker
        # compose exec failure must NOT trigger RPM resurrection on a host
        # whose Docker stack is already up with restored data. Backfill
        # failures degrade gracefully — docker-start still has the best-effort
        # mount logic for rows whose fields are already populated.
        log_warn "Could not query VideoStore from DB — skipping backfill"
        log_warn "docker-start will only mount rows whose sDevice/sLabel/sUUID are set."
        return 0
    }

    if [[ -z "$vs_rows" ]]; then
        log_info "No enabled VideoStore entries in DB — nothing to backfill"
        return 0
    fi

    local updated=0 unresolved=0
    # Read all rows into an array BEFORE the processing loop, then iterate the
    # array. The loop body runs `docker compose exec -T` (the UPDATE), and `-T`
    # forwards stdin to the container — run inside a `while read <<< "$vs_rows"`
    # it drains the here-string holding the not-yet-read rows, so only the FIRST
    # VideoStore is processed and every later row silently vanishes. That
    # dropped vs2 on cs1934 / TB06-1 (2026-07-21): only vs1 was backfilled, the
    # blank vs2 label tripped the provenance gate (and on pre-gate builds vs2
    # unmounted on the next reboot and filled root). Iterating a pre-read array
    # keeps stdin out of the loop entirely, immune to any stdin-reading command
    # in the body. See .claude/rules/bash-scripting.md and the mariadb-CLI
    # stdin-consumption pitfall.
    local -a vs_row_list=()
    local vs_row
    while IFS= read -r vs_row; do
        [[ -n "$vs_row" ]] && vs_row_list+=("$vs_row")
    done <<< "$vs_rows"

    for vs_row in "${vs_row_list[@]}"; do
        IFS=$'\037' read -r name mount cur_device cur_label cur_uuid cur_fstype <<< "$vs_row"
        if [[ -z "$mount" ]]; then
            log_warn "VideoStore '$name' has no mount point — cannot backfill"
            unresolved=$((unresolved + 1))
            continue
        fi
        # Expect /videostore/<label>. Skip if mountpoint doesn't fit the pattern.
        local expected_label="${mount##*/}"
        if [[ -z "$expected_label" ]]; then
            log_warn "VideoStore '$name' has no parseable mount point — skipping"
            unresolved=$((unresolved + 1))
            continue
        fi

        # Reject operator-set names that contain anything other than
        # alphanumerics, underscore, dash. VideoStore names in practice are
        # always 'vs1', 'vs2', ... — anything weirder is either operator
        # error or an injection attempt. We splice $name into a SQL WHERE
        # clause below; a lone apostrophe breaks the statement.
        if ! [[ "$name" =~ ^[A-Za-z0-9_-]+$ ]]; then
            log_warn "VideoStore sName '$name' contains unsafe characters — skipping backfill for this row"
            unresolved=$((unresolved + 1))
            continue
        fi

        # blkid lookup by the label we derived from the mountpoint.
        local blkid_out
        blkid_out=$(blkid -L "$expected_label" 2>/dev/null) || blkid_out=""
        if [[ -z "$blkid_out" ]]; then
            # FALLBACK PATH for RPM-era NVRs.
            #
            # rda-autofs mounted /videostore/vs1 by sDevice or sUUID and never
            # set a LABEL on the ext4 filesystem. So `blkid -L vs1` returns
            # nothing on a freshly-migrated RPM NVR even though the videostore
            # disk is right there in the DB at sDevice. Without this branch,
            # backfill silently no-ops, and on the next host reboot
            # docker-start Phase 2 races with LVM activation / udev — Phase 2
            # tries to mount via sDevice before /dev/mapper/<vg>-<lv> exists,
            # fails silently, mpengine writes mp4s to /videostore/vs1 on the
            # ROOT partition, root fills in ~3 hours.
            #
            # Surfaced by cs999 (Lawn Mower Sales) 2026-05-15 — ISP outage
            # triggered a host reboot, came back with /videostore/vs1 NOT
            # mounted, root climbed from 76% to 97% in ~2 hours before manual
            # remount. cs50 + cs1129 had identical empty rows but hadn't yet
            # rebooted.
            #
            # Fallback: if the DB row already has sDevice populated, blkid
            # that device directly to read UUID/FSType. Also try e2label to
            # add the missing label for future Phase 1 auto-discovery — only
            # when the FS is ext2/3/4 (other FS types need different label
            # tools that may not be installed).
            if [[ -z "$cur_device" || ! -b "$cur_device" ]]; then
                log_warn "No block device labeled '$expected_label' and no usable DB sDevice — '$name' must be hand-fixed"
                log_warn "  Operator: 'e2label <device> $expected_label' + re-run --migrate"
                unresolved=$((unresolved + 1))
                continue
            fi

            log_info "No '$expected_label' label found; falling back to DB sDevice=$cur_device"
            blkid_out="$cur_device"

            # Best-effort label add (ext only). Failure is non-fatal — UUID
            # in the DB is what Phase 2 actually mounts by.
            # `|| true`: blkid returns non-zero on "no such device" or "no such
            # field"; under set -eE inside migrate_flow's trap window this
            # would fire rollback_rpm_services on a benign probe failure.
            # Empty fs_for_label falls through to the "not ext" branch which
            # is the correct skip-e2label path.
            local fs_for_label
            fs_for_label=$(blkid -s TYPE -o value "$cur_device" 2>/dev/null || true)
            if [[ "$fs_for_label" =~ ^ext[234]$ ]]; then
                if e2label "$cur_device" "$expected_label" 2>/dev/null; then
                    log_info "  e2label set '$expected_label' on $cur_device (Phase 1 auto-discovery enabled)"
                else
                    log_warn "  e2label failed (root needed? read-only FS?) — Phase 1 will skip, Phase 2 still works via UUID"
                fi
            else
                log_info "  $cur_device is $fs_for_label, not ext — skipping e2label; Phase 2 still works via UUID"
            fi
        fi

        # Refuse ambiguous labels — if multiple partitions claim the same
        # label (USB hot-swap, drive clone, leftover partition), blkid -L
        # returns whichever the kernel registered first. Mounting the wrong
        # disk silently loses customer recordings. Bail loudly and name
        # both devices so the operator can unlabel one with e2label.
        local blkid_all
        blkid_all=$(blkid | awk -v lbl="$expected_label" -F: '
            $0 ~ "LABEL=\""lbl"\"" {print $1}
        ')
        local blkid_count
        # `grep -c .` exits 1 on empty stdin (no matching lines). When
        # blkid_all is empty (the cs999 fallthrough case where awk found
        # no LABEL-tagged devices), bare assignment under set -e would
        # abort the script — and now under set -eE that abort would
        # fire the migrate_flow ERR trap on a non-error condition.
        # The intent here is "count lines, treat empty as zero".
        # An LVM LV can be listed through both /dev/mapper and /dev/<vg>.
        # Count canonical devices so those aliases are not treated as two
        # distinct VideoStores.
        blkid_count=$(while IFS= read -r dev; do
            [[ -n "$dev" ]] || continue
            readlink -f "$dev" 2>/dev/null || echo "$dev"
        done <<< "$blkid_all" | sort -u | grep -c . || true)
        if [[ "$blkid_count" -gt 1 ]]; then
            log_warn "Multiple devices labeled '$expected_label' — refusing to backfill '$name':"
            while IFS= read -r dev; do log_warn "    $dev"; done <<< "$blkid_all"
            log_warn "  Remove or re-label one with 'e2label <dev> <new-label>' and re-run migration"
            unresolved=$((unresolved + 1))
            continue
        fi

        local new_device new_uuid new_fstype
        new_device="$blkid_out"
        # `|| true`: blkid can return non-zero on transient kernel state (FS
        # not yet probable, device just attached). Under set -eE inside the
        # trap window this would fire rollback on a benign probe. The
        # validation block immediately below already handles empty / malformed
        # outputs by skipping the row.
        new_uuid=$(blkid -s UUID -o value "$new_device" 2>/dev/null || true)
        new_fstype=$(blkid -s TYPE -o value "$new_device" 2>/dev/null || true)

        # Defense in depth: validate blkid outputs match the expected shapes
        # before we splice them into SQL. Anything that doesn't match is
        # either pathological or hostile — skip the row rather than risk
        # running arbitrary SQL as DB root.
        if ! [[ "$new_device" =~ ^/dev/[A-Za-z0-9/_-]+$ ]]; then
            log_warn "blkid returned unexpected device path '$new_device' — skipping '$name'"
            unresolved=$((unresolved + 1))
            continue
        fi
        if [[ -n "$new_uuid" ]] && ! [[ "$new_uuid" =~ ^[0-9A-Fa-f-]+$ ]]; then
            log_warn "blkid returned unexpected UUID '$new_uuid' — skipping '$name'"
            unresolved=$((unresolved + 1))
            continue
        fi
        if [[ -n "$new_fstype" ]] && ! [[ "$new_fstype" =~ ^[a-z0-9_]+$ ]]; then
            log_warn "blkid returned unexpected fstype '$new_fstype' — skipping '$name'"
            unresolved=$((unresolved + 1))
            continue
        fi

        # Keep DB mount metadata aligned with the filesystem mounted at this
        # row's /videostore/vsN path. Stale duplicate labels are migration
        # blockers, so a mismatch must be corrected rather than preserved.
        local set_clauses=""
        [[ "$cur_device" != "$new_device" ]] && set_clauses+="sDevice='$new_device',"
        [[ "$cur_label"  != "$expected_label" ]] && set_clauses+="sLabel='$expected_label',"
        [[ "$cur_uuid"   != "$new_uuid" ]] && set_clauses+="sUUID='$new_uuid',"
        [[ "$cur_fstype" != "${new_fstype:-ext4}" ]] && set_clauses+="sFSType='${new_fstype:-ext4}',"
        set_clauses="${set_clauses%,}"

        if [[ -z "$set_clauses" ]]; then
            log_info "VideoStore '$name' already populated — leaving"
            continue
        fi

        # `</dev/null`: never let `docker compose exec -T` read the caller's
        # stdin. Defense-in-depth against the here-string drain described at
        # the top of the loop — belt to the array-iteration suspenders.
        if docker compose exec -T db sh -c "mysql -uroot -p\"\$MYSQL_ROOT_PASSWORD\" dtech -e \"UPDATE VideoStore SET $set_clauses WHERE sName='$name'\"" </dev/null >/dev/null 2>&1; then
            log_info "VideoStore '$name' -> device=$new_device uuid=$new_uuid label=$expected_label"
            updated=$((updated + 1))
        else
            log_warn "Failed to update VideoStore '$name'"
            unresolved=$((unresolved + 1))
        fi
    done

    if [[ "$unresolved" -gt 0 ]]; then
        log_error "VideoStore backfill left $unresolved unresolved row(s); aborting migration before RPM cleanup"
        return 1
    fi

    log_info "Backfill complete ($updated rows updated)"
}

# The migration cannot be considered complete merely because the VideoStore
# row count survived restore. Each enabled physical row must carry mount
# metadata that docker-start can parse after a future backend restart, and its
# mountpoint must be backed by a real filesystem rather than root's fallback
# directory. TG's #1 (cs1934, 2026-07-21) had two enabled rows but only vs1
# was backfilled; vs2 later unmounted and 6.19 GiB of recordings filled root.
# Re-run the backend entrypoint so Phase 2 sees the metadata the backfill just
# wrote, mounts each store, and persists a host-side fstab entry for it.
#
# Why a restart rather than mounting and writing fstab here: docker-start Phase 2
# is the ONE implementation of "mount this VideoStore, verify it is the right disk,
# then write a managed fstab line". That code carries a pile of hard-won
# constraints -- the sentinel that decides authorship, `nofail` plus pass 0 so a
# missing disk cannot strand a box at an emergency prompt, the FSROOT guard against
# persisting a bind mount as a whole device, the whitespace guard on sMountPoint,
# the write-once pristine .bak, tune2fs -c 0 -i 0 for RPM-era ext disks. A second
# copy of that in the installer would be a second thing to keep correct, and it
# would drift the way the two brace parsers in mpengine/tests drifted. So the
# installer triggers the shipped writer instead of reimplementing it.
#
# The ordering is why this is needed at all: the migration starts the backend in
# Step 11 (before the DB restore completes) so Phase 2 has already run against
# rows that carried only sMountPoint. Without this, a migrated box gets its fstab
# entry on its SECOND boot, and the first reboot is exactly when the 2026-07-24
# incident happened.
#
# Never fatal. This runs inside migrate_flow's `set -eE` + `trap
# rollback_rpm_services ERR` window, and a transient compose hiccup here must not
# resurrect RPM services on a host whose Docker stack is already serving restored
# data. verify_videostore_mount_metadata below is the gate that actually decides.
remount_videostores_after_backfill() {
    log_step "Re-running backend VideoStore mount phase with backfilled metadata"

    if [[ "$IS_WINDOWS" == "true" ]]; then
        log_info "Windows deployment: VideoStore is a container bind; nothing to mount"
        return 0
    fi

    # `|| return 0`, never a bare cd. This runs inside migrate_flow's `set -eE` +
    # `trap rollback_rpm_services ERR` window, so a bare failing command here
    # resurrects RPM services on a host already serving the restored Docker
    # database. Caught by review: the grep-for-`return 1` assertion that was
    # supposed to prove this could not happen did not cover it.
    cd "$INSTALL_DIR" || { log_warn "Could not cd to $INSTALL_DIR; skipping the VideoStore remount"; return 0; }

    # --no-deps: db is already up and healthy, and recreating it here would take
    # the restored database down mid-migration. Writers keep running; they hold no
    # handle on the mount point that a fresh mount would break, and their
    # depends_on only gates START.
    if ! docker compose up -d --force-recreate --no-deps backend; then
        log_warn "Could not recreate backend to re-run the VideoStore mount phase"
        log_warn "  The stores may only mount on the next boot; verification below will say so."
        return 0
    fi

    # Phase 1 + Phase 2 run before `exec rda-backend`, so the container being
    # healthy means they finished. 120s: Phase 2 retries once after a udev settle
    # for late-enumerating disks.
    local deadline=$((SECONDS + 120)) cid health
    while [[ "$SECONDS" -lt "$deadline" ]]; do
        cid=$(docker compose ps -q backend 2>/dev/null || true)
        if [[ -n "$cid" ]]; then
            health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$cid" 2>/dev/null || echo unknown)
            [[ "$health" == "healthy" || "$health" == "none" ]] && break
        fi
        sleep 5
    done

    # Surface what Phase 2 decided. The operator reading a migration log should
    # see the mount outcome here, not have to go find it in `nvr logs backend`.
    docker compose logs --no-log-prefix --tail 200 backend 2>/dev/null \
        | grep -E 'Phase (1|2)|VideoStore|Persisted|not mounted|identity mismatch' \
        | tail -20 || true

    return 0
}

# Report video stranded on the ROOT filesystem underneath a VideoStore mount, at
# the one moment a human is definitely watching.
#
# ONE implementation, not a second copy: this shells into the backend container and
# runs the same `nvr-maintenance.sh shadow-report` the hourly cron and the runbook
# use. ADR-045 already insists there be a single implementation of mount-verify-
# persist for exactly this reason, and a bind-of-/ measurement reimplemented here
# would be a second thing to keep correct.
#
# Runs AFTER the stack is up, which is also when it is most accurate: Phase 2 never
# unmounts anything, so a store that was already shadowing footage before the
# migration is still shadowing it now, and the numbers are the same either way.
#
# Never fatal, and never blocks. NOTE, corrected 2026-07-27: this call site is AFTER
# migrate_flow's `trap - ERR`, so it is not actually inside the rollback window -- the
# earlier version of this comment claimed it was. It stays written this way anyway: a
# report that could abort a completed migration would be indefensible, and the guards
# cost nothing.
report_shadowed_video() {
    log_step "Checking for video stranded underneath a VideoStore mount"

    if [[ "$IS_WINDOWS" == "true" ]]; then
        log_info "Windows deployment: VideoStore is a container bind; nothing to check"
        return 0
    fi

    cd "$INSTALL_DIR" || return 0

    local out=""
    out=$(docker compose exec -T backend /usr/local/bin/nvr-maintenance.sh shadow-report 2>&1 </dev/null) || {
        log_warn "Could not run the stranded-video report (backend not ready?); the hourly check will cover it"
        return 0
    }
    printf '%s\n' "$out" | sed 's/^/  /'

    # `grep -c` and a comparison, never `grep -q` in a pipeline under pipefail.
    if printf '%s\n' "$out" | grep -q 'Stranded footage found'; then
        log_error "================================================================"
        log_error "STRANDED VIDEO FOUND UNDERNEATH A VIDEOSTORE MOUNT"
        log_error "================================================================"
        log_error "Video was recorded to the ROOT filesystem while a disk was down, and a"
        log_error "mount now hides it: nothing can reach those bytes, they still consume"
        log_error "root, and rollover will delete the Event rows that point at them."
        log_error ""
        log_error "  DO THIS:  docs/runbooks/migrate-rpm-to-docker.md, section"
        log_error "            'Stranded video underneath a VideoStore mount'"
        log_error ""
        log_error "Do NOT unmount the store to 'fix' this without reading that first."
        log_error "Alarm [51203] has also been raised, so this is on the Monarch board."
    fi
    return 0
}

verify_videostore_mount_metadata() {
    log_step "Verifying VideoStore mount metadata and provenance"

    if [[ "$IS_WINDOWS" == "true" ]]; then
        log_info "Windows deployment: directory VideoStore is expected"
        return 0
    fi

    cd "$INSTALL_DIR"
    local vs_rows
    vs_rows=$(docker compose exec -T db sh -c 'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e "SELECT CONCAT(sName,CHAR(31),sMountPoint,CHAR(31),COALESCE(sDevice,\"\"),CHAR(31),COALESCE(sLabel,\"\"),CHAR(31),COALESCE(sUUID,\"\"),CHAR(31),COALESCE(sFSType,\"\")) FROM VideoStore WHERE fEnable=1"' 2>/dev/null) || {
        log_error "Could not query enabled VideoStore metadata after backfill"
        return 1
    }
    if [[ -z "$vs_rows" ]]; then
        log_error "No enabled VideoStore rows found after backfill"
        return 1
    fi

    local root_source failures=0 vs_row name mount device label uuid fstype expected_label source fs_label fs_uuid fs_fstype canonical_device canonical_source
    root_source=$(findmnt -rn -T / -o SOURCE 2>/dev/null || true)
    while IFS= read -r vs_row; do
        IFS=$'\037' read -r name mount device label uuid fstype <<< "$vs_row"
        expected_label="${mount##*/}"
        if [[ -z "$mount" || -z "$expected_label" || -z "$device" || -z "$label" || -z "$uuid" || -z "$fstype" ]]; then
            log_error "VideoStore '$name' has incomplete Docker mount metadata: mount=$mount device=$device label=$label uuid=$uuid fstype=$fstype"
            failures=$((failures + 1))
            continue
        fi
        if [[ "$label" != "$expected_label" ]]; then
            log_error "VideoStore '$name' label mismatch: DB=$label expected=$expected_label"
            failures=$((failures + 1))
            continue
        fi
        source=$(findmnt -rn -T "$mount" -o SOURCE 2>/dev/null || true)
        if [[ -z "$source" || "$source" == "$root_source" ]]; then
            log_error "VideoStore '$name' is not mounted on physical storage: mount=$mount source=${source:-missing} root=$root_source"
            failures=$((failures + 1))
            continue
        fi
        fs_label=$(blkid -s LABEL -o value "$source" 2>/dev/null || true)
        if [[ "$fs_label" != "$expected_label" ]]; then
            log_error "VideoStore '$name' filesystem label mismatch: source=$source label=${fs_label:-missing} expected=$expected_label"
            failures=$((failures + 1))
            continue
        fi
        fs_uuid=$(blkid -s UUID -o value "$source" 2>/dev/null || true)
        fs_fstype=$(blkid -s TYPE -o value "$source" 2>/dev/null || true)
        canonical_device=$(readlink -f "$device" 2>/dev/null || printf '%s' "$device")
        canonical_source=$(readlink -f "$source" 2>/dev/null || printf '%s' "$source")
        if [[ "$canonical_device" != "$canonical_source" || "$uuid" != "$fs_uuid" || "$fstype" != "$fs_fstype" ]]; then
            log_error "VideoStore '$name' DB metadata does not match mounted filesystem: db_device=$device db_uuid=$uuid db_fstype=$fstype source=$source source_uuid=${fs_uuid:-missing} source_fstype=${fs_fstype:-missing}"
            failures=$((failures + 1))
            continue
        fi
        log_info "VideoStore '$name' verified: $source -> $mount (label=$label uuid=$uuid)"
    done <<< "$vs_rows"

    if [[ "$failures" -gt 0 ]]; then
        log_error "VideoStore metadata verification found $failures problem(s); refusing RPM cleanup"
        return 1
    fi
}

newest_backup_sql_path() {
    local newest_mtime=0
    local backup_sql=""
    local search_root search_depth sql_path sql_mtime

    for search_root in /videostore/vs1/backups /rda/backups; do
        [[ -d "$search_root" ]] || continue
        search_depth=4
        [[ "$search_root" == "/rda/backups" ]] && search_depth=3

        while IFS= read -r sql_path; do
            sql_mtime=$(stat -c '%Y' "$sql_path" 2>/dev/null || stat -f '%m' "$sql_path" 2>/dev/null || true)
            if [[ -n "$sql_mtime" ]] && [[ "$sql_mtime" -gt "$newest_mtime" ]]; then
                newest_mtime="$sql_mtime"
                backup_sql="$sql_path"
            fi
        done < <(find "$search_root" -maxdepth "$search_depth" -path '*/db/dtech.sql' -type f 2>/dev/null || true)
    done

    [[ -n "$backup_sql" ]] && printf '%s\n' "$backup_sql"
}

backup_sql_insert_count() {
    local backup_sql="$1"
    local table="$2"

    awk -v table="$table" '
        $0 ~ "^INSERT INTO `" table "` VALUES" {
            line = $0
            rows = gsub(/\),\(/, "),(", line) + 1
            total += rows
        }
        END { print total + 0 }
    ' "$backup_sql" 2>/dev/null || echo 0
}

verify_restored_customer_config() {
    log_step "Verifying restored customer camera/device configuration"

    cd "$INSTALL_DIR"

    local stats
    stats=$(docker compose exec -T db sh -c 'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e "
SELECT
  COUNT(*) AS cameras,
  COALESCE(SUM(fEnable),0) AS enabled_cameras,
  COALESCE(SUM(bDeviceID IS NOT NULL),0) AS assigned_cameras
FROM Camera;
SELECT COUNT(*) FROM Device;
SELECT COUNT(*) FROM VideoStore;
SELECT COUNT(*) FROM VideoStore WHERE fEnable=1 AND COALESCE(sMountPoint,\"\") <> \"\";
SELECT COUNT(*) FROM Misc WHERE sModule=\"system\" AND sName IN (\"smart-fill-enable\",\"smart-fill-chunk-seconds\",\"smart-fill-days\");
SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=\"Event\" AND COLUMN_NAME IN (\"bVideoStoreID\",\"sTags\");
SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=\"Camera\" AND COLUMN_NAME=\"fSmartmask\";
"' 2>/dev/null) || {
        log_error "Could not query restored Camera/Device/VideoStore/smart-fill schema"
        return 1
    }

    local cameras enabled assigned devices videostores enabled_videostores smart_misc event_smart_cols camera_smart_cols
    read -r cameras enabled assigned <<< "$(echo "$stats" | sed -n '1p')"
    devices=$(echo "$stats" | sed -n '2p')
    videostores=$(echo "$stats" | sed -n '3p')
    enabled_videostores=$(echo "$stats" | sed -n '4p')
    smart_misc=$(echo "$stats" | sed -n '5p')
    event_smart_cols=$(echo "$stats" | sed -n '6p')
    camera_smart_cols=$(echo "$stats" | sed -n '7p')

    local backup_sql expected_cameras expected_devices expected_videostores
    backup_sql=$(newest_backup_sql_path || true)
    if [[ -n "$backup_sql" && -f "$backup_sql" ]]; then
        expected_cameras=$(backup_sql_insert_count "$backup_sql" "Camera")
        expected_devices=$(backup_sql_insert_count "$backup_sql" "Device")
        expected_videostores=$(backup_sql_insert_count "$backup_sql" "VideoStore")

        if [[ "$expected_cameras" -gt 0 && "${cameras:-0}" -ne "$expected_cameras" ]] \
            || [[ "$expected_devices" -gt 0 && "${devices:-0}" -ne "$expected_devices" ]] \
            || [[ "$expected_videostores" -gt 0 && "${videostores:-0}" -ne "$expected_videostores" ]]; then
            log_error "Restored customer config does not match backup dump:"
            log_error "  live:   cameras=$cameras enabled=$enabled assigned=$assigned devices=$devices videostores=$videostores enabled_mounted_videostores=$enabled_videostores"
            log_error "  backup: cameras=$expected_cameras devices=$expected_devices videostores=$expected_videostores ($backup_sql)"
            log_error "This usually means the restore hit the wrong DB endpoint or Docker re-seeded dtech after restore."
            return 1
        fi
    else
        log_warn "No backup dtech.sql found for count comparison; using live DB/schema sanity only"
    fi

    # Older 6.2 systems can have all cameras represented without the newer
    # fEnable/bDeviceID/Device conventions (RC54 has 47 Camera rows and no
    # Device rows). Require the durable restore invariants instead: real
    # camera and VideoStore content plus the post-update schema/misc markers.
    if [[ "${cameras:-0}" -lt 1 || "${videostores:-0}" -lt 1 || "${enabled_videostores:-0}" -lt 1 || "${smart_misc:-0}" -lt 3 || "${event_smart_cols:-0}" -lt 2 || "${camera_smart_cols:-0}" -lt 1 ]]; then
        log_error "Restored customer config failed sanity check:"
        log_error "  cameras=$cameras enabled=$enabled assigned=$assigned devices=$devices videostores=$videostores enabled_mounted_videostores=$enabled_videostores"
        log_error "  smart_fill_misc=$smart_misc event_smart_cols=$event_smart_cols camera_smart_cols=$camera_smart_cols"
        log_error "This looks like a placeholder/fresh DB, not the restored customer configuration."
        return 1
    fi

    log_info "Restored customer config OK: cameras=$cameras enabled=$enabled assigned=$assigned devices=$devices videostores=$videostores enabled_mounted_videostores=$enabled_videostores smart_fill_misc=$smart_misc event_smart_cols=$event_smart_cols camera_smart_cols=$camera_smart_cols"
}

disable_host_services() {
    # Windows deployments (WSL2/Hyper-V) have no host services to conflict with
    if [[ "$IS_WINDOWS" == "true" ]]; then
        log_info "Windows deployment: skipping host service checks"
        return
    fi

    log_step "Disabling conflicting host services"

    # Stop, disable, and mask host services that conflict with Docker containers
    # using host networking: httpd (port 80), smbd/nmbd (port 445).
    #
    # Mask unconditionally, not just when currently active. A boot-time race —
    # systemd transitioning, service not yet active when install-nvr.sh hits
    # this step — can slip past an active-only check and let systemd start
    # the service moments later, stealing the port before `docker compose up`
    # brings the container online. Surfaced on the macbuilder CentOS 9
    # claude-centos9-stream VM 2026-04-24: viewer container restart-looped
    # with "Address already in use: 0.0.0.0:80" because host httpd.service
    # came up after this step logged empty.
    #
    # autofs is handled separately in ensure_videostore_writable — masking it
    # unconditionally would unmount /videostore on real-disk NVRs (cs256 /
    # cs2427 / cs50) during the migration window, before docker-start Phase 2
    # remounts. The host-service mask here stays scoped to port-conflict
    # services only.
    #
    # Mask is idempotent and reversible via `systemctl unmask`; no harm on
    # hosts that never had the services installed.
    for svc in httpd smbd nmbd; do
        svc_stop "$svc" 2>/dev/null || true
        svc_disable "$svc" 2>/dev/null || true
        if has_systemd; then
            systemctl mask "$svc" 2>/dev/null || true
        fi
    done
    # Belt and suspenders: pkill after mask. On CentOS 7, systemctl stop
    # can return before the process fully exits; on any host, systemd may
    # have respawned the service in the window between stop and mask.
    pkill -9 httpd 2>/dev/null || true
    pkill -9 smbd 2>/dev/null || true
    pkill -9 nmbd 2>/dev/null || true
}

pid_is_containerized() {
    local pid="$1"

    [[ -r "/proc/$pid/cgroup" ]] || return 1
    grep -Eq '(/docker/|docker-|/containerd/|libpod|kubepods)' "/proc/$pid/cgroup"
}

list_listening_pids_for_ports() {
    local port_regex="$1"

    (ss -tlnp 2>/dev/null || netstat -tlnp 2>/dev/null || true) | awk -v port_regex="$port_regex" '
        $0 ~ ":(" port_regex ")([[:space:]]|$)" {
            line = $0
            while (match(line, /pid=[0-9]+/)) {
                print substr(line, RSTART + 4, RLENGTH - 4)
                line = substr(line, RSTART + RLENGTH)
            }

            line = $0
            while (match(line, /[0-9]+\/[A-Za-z0-9_.-]+/)) {
                pidproc = substr(line, RSTART, RLENGTH)
                split(pidproc, parts, "/")
                print parts[1]
                line = substr(line, RSTART + RLENGTH)
            }
        }
    ' | sort -u
}

kill_non_container_pids_by_name() {
    local name pid

    for name in "$@"; do
        while read -r pid; do
            [[ -n "$pid" ]] || continue
            if pid_is_containerized "$pid"; then
                continue
            fi
            kill -9 "$pid" 2>/dev/null || true
        done < <(pgrep -x "$name" 2>/dev/null || true)
    done
}

kill_non_container_port_holders() {
    local port_regex="$1"
    local pid

    while read -r pid; do
        [[ -n "$pid" ]] || continue
        if pid_is_containerized "$pid"; then
            continue
        fi
        log_warn "Killing non-Docker process $pid on legacy port(s) $port_regex"
        kill -9 "$pid" 2>/dev/null || true
    done < <(list_listening_pids_for_ports "$port_regex")
}

wait_for_no_non_container_port_holders() {
    local port_regex="$1"
    local timeout="${2:-10}"
    local waited=0

    while [[ $waited -lt $timeout ]]; do
        local found=0 pid
        while read -r pid; do
            [[ -n "$pid" ]] || continue
            if ! pid_is_containerized "$pid"; then
                found=1
                break
            fi
        done < <(list_listening_pids_for_ports "$port_regex")

        [[ "$found" -eq 0 ]] && return 0

        sleep 1
        waited=$((waited + 1))
    done

    return 1
}

host_mysql_process_present() {
    local name pid

    for name in mysqld_safe mysqld mariadbd; do
        while read -r pid; do
            [[ -n "$pid" ]] || continue
            # The Docker DB process is visible in the host process table, so
            # only a non-containerized process counts as a legacy host DB.
            if ! pid_is_containerized "$pid"; then
                return 0
            fi
        done < <(pgrep -x "$name" 2>/dev/null || true)
    done

    return 1
}

legacy_mysql_log_files() {
    # Only enumerate regular files. In particular, never turn this into an
    # rm -rf of /var/log/mysql: an operator may have mounted that directory
    # separately or placed unrelated diagnostics there.
    find /var/log -xdev -maxdepth 1 -type f \
        \( -name 'mysqld.log*' -o -name 'mysql.log*' \) -print 2>/dev/null || true

    if [[ -d /var/log/mysql ]] && command -v mountpoint &>/dev/null \
       && ! mountpoint -q /var/log/mysql 2>/dev/null; then
        find /var/log/mysql -xdev -maxdepth 1 -type f -name '*.log*' \
            -print 2>/dev/null || true
    fi
}

legacy_mysql_cleanup_gate() {
    # Host MySQL artifacts may be removed only after proving the running DB is
    # the Docker bind-mounted database. This is defense in depth for reruns or
    # partially completed migrations where an RPM daemon could still exist.
    local db_container db_state db_health db_source expected_source

    cd "$INSTALL_DIR" 2>/dev/null || return 1
    [[ -f "$INSTALL_DIR/docker-compose.yml" ]] || return 1
    command -v pgrep >/dev/null 2>&1 || return 1
    if ! command -v ss >/dev/null 2>&1 && ! command -v netstat >/dev/null 2>&1; then
        return 1
    fi

    db_container=$(docker compose ps -q db 2>/dev/null || true)
    [[ -n "$db_container" ]] || return 1

    db_state=$(docker inspect -f '{{.State.Status}}' "$db_container" 2>/dev/null || true)
    db_health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
        "$db_container" 2>/dev/null || true)
    [[ "$db_state" == "running" && "$db_health" == "healthy" ]] || return 1

    # Health status alone can be stale across a daemon restart; require a live
    # authenticated probe before touching RPM-era database artifacts.
    docker compose exec -T db sh -c \
        'mysqladmin -uroot -p"$MYSQL_ROOT_PASSWORD" ping -h 127.0.0.1 --silent' \
        >/dev/null 2>&1 || return 1

    # The live DB must be the compose bind mount. A named volume or a host
    # mounted /var/lib/mysql is intentionally not accepted by this cleanup.
    db_source=$(docker inspect -f \
        '{{range .Mounts}}{{if eq .Destination "/var/lib/mysql"}}{{.Source}}{{end}}{{end}}' \
        "$db_container" 2>/dev/null || true)
    [[ -n "$db_source" ]] || return 1
    expected_source=$(readlink -f "$INSTALL_DIR/data/db_data" 2>/dev/null || true)
    db_source=$(readlink -f "$db_source" 2>/dev/null || true)
    [[ -n "$expected_source" && "$db_source" == "$expected_source" ]] || return 1

    # Never remove a path that is itself mounted, and never remove logs while
    # a non-containerized mysqld/mariadbd still exists or owns 3306.
    if [[ -d /var/lib/mysql ]] && { ! command -v mountpoint &>/dev/null \
        || mountpoint -q /var/lib/mysql 2>/dev/null; }; then
        return 1
    fi
    if [[ -d /var/log/mysql ]] && { ! command -v mountpoint &>/dev/null \
        || mountpoint -q /var/log/mysql 2>/dev/null; }; then
        return 1
    fi
    host_mysql_process_present && return 1
    wait_for_no_non_container_port_holders "3306" 1 || return 1

    return 0
}

stop_host_database_service_for_docker() {
    # Docker DB runs with host networking on 3306. RPM-era restore scripts can
    # re-enable or restart host MariaDB after the initial service stop, so call
    # this both before first Docker startup and after rda-db --restore.
    for db_svc in mariadb mysqld; do
        migration_svc_stop "$db_svc"
        svc_disable "$db_svc" 2>/dev/null || true
        if has_systemd; then
            # A restored unit can live under /etc/systemd/system, where a
            # persistent mask cannot replace it. Keep the migration-only
            # guard in /run so rollback can remove it reliably.
            systemctl mask --runtime "$db_svc" 2>/dev/null || true
        fi
    done
    kill_non_container_pids_by_name mysqld_safe mysqld mariadbd
    kill_non_container_port_holders "3306"

    if ! wait_for_no_non_container_port_holders "3306" 10; then
        log_error "Non-Docker process still owns port 3306 after 10s — refusing to continue migration"
        return 1
    fi
}

stop_host_backend_service_for_docker() {
    # rda-db --restore invokes the RPM-era service restoration path on some
    # legacy hosts. Those old services bind Docker host-network ports:
    # httpd=:80, MariaDB=:3306, pbserver=:43203, backend=:43204/43208,
    # mpengine=:43209. Stop and mask them again before restarting compose.
    local services=(watchprog rda-backend recorder mpengine pbserver ptzd rdafw logmuxd optician dview ipsetup httpd)
    local svc
    for svc in "${services[@]}"; do
        migration_svc_stop "$svc"
        svc_disable "$svc" 2>/dev/null || true
        if has_systemd; then
            systemctl mask --runtime "$svc" 2>/dev/null || true
        fi
    done

    kill_legacy_rpm_processes

    if has_systemd; then
        systemctl kill rda-backend recorder mpengine pbserver httpd dview ipsetup ptzd --kill-who=all 2>/dev/null || true
        # rda-db restore can schedule a unit shortly after its first stop.
        # Re-run the stop after the runtime masks are in place.
        sleep 2
        for svc in "${services[@]}"; do
            systemctl stop "$svc" 2>/dev/null || true
        done
    fi

    kill_non_container_port_holders "80|43203|43204|43208|43209"

    if ! wait_for_no_non_container_port_holders "80|43203|43204|43208|43209" 10; then
        log_error "Non-Docker legacy NVR ports still in use after 10s — refusing to continue migration"
        return 1
    fi
}

legacy_migration_services() {
    printf '%s\n' watchprog rda-backend recorder mpengine pbserver ptzd rdafw logmuxd optician dview ipsetup httpd mariadb mysqld
}

suspend_legacy_cron_starters() {
    # Some old appliances used cron as a poor-man's service watchdog. During
    # migration, any cron line that starts watchprog/RPM services can race the
    # Docker restore window. Disable only files that explicitly start/restart
    # known RPM-era NVR services; rollback restores them if migration fails.
    local hold_dir="/var/lib/dividia-nvr-migration/cron-disabled"
    local file rel dest
    local legacy_names='watchprog|dvs-up|rda-backend|recorder|mpengine|pbserver|ptzd|httpd|mariadb|mysqld'
    local pattern="((^|[[:space:]])service[[:space:]]+($legacy_names)[[:space:]]+(start|restart)|(^|[[:space:]])systemctl[[:space:]]+(start|restart)[[:space:]]+($legacy_names)(\\.service)?([[:space:]]|$)|(^|[[:space:]])initctl[[:space:]]+(start|restart)[[:space:]]+($legacy_names)([[:space:]]|$)|/etc/init\\.d/($legacy_names)[[:space:]]+(start|restart))"

    mkdir -p "$hold_dir"

    while IFS= read -r file; do
        [[ -f "$file" ]] || continue
        if grep -Eq "$pattern" "$file" 2>/dev/null; then
            rel="${file#/}"
            dest="$hold_dir/$rel"
            mkdir -p "$(dirname "$dest")"
            if [[ ! -e "$dest" ]]; then
                mv "$file" "$dest"
                log_info "Disabled legacy service cron during migration: $file"
            fi
        fi
    done < <(
        find /etc/cron.d -maxdepth 1 -type f 2>/dev/null
        find /etc/cron.hourly /etc/cron.daily /etc/cron.weekly /etc/cron.monthly -maxdepth 1 -type f 2>/dev/null
    )
}

restore_legacy_cron_starters() {
    local hold_dir="/var/lib/dividia-nvr-migration/cron-disabled"
    local held rel dest

    [[ -d "$hold_dir" ]] || return 0

    while IFS= read -r held; do
        [[ -f "$held" ]] || continue
        rel="${held#"$hold_dir"/}"
        dest="/$rel"
        mkdir -p "$(dirname "$dest")"
        if [[ ! -e "$dest" ]]; then
            mv "$held" "$dest"
            log_info "Restored legacy service cron after rollback: $dest"
        fi
    done < <(find "$hold_dir" -type f 2>/dev/null)
}

assert_migration_lock() {
    local context="${1:-migration}"
    local failed=0 svc pid

    log_step "Asserting RPM service migration lock ($context)"

    stop_host_database_service_for_docker || failed=1
    stop_host_backend_service_for_docker || failed=1

    if has_systemd; then
        while read -r svc; do
            [[ -n "$svc" ]] || continue
            if systemctl is-active --quiet "$svc" 2>/dev/null; then
                log_error "Legacy service $svc is active during $context"
                failed=1
            fi
        done < <(legacy_migration_services)
    fi

    while read -r pid; do
        [[ -n "$pid" ]] || continue
        if ! pid_is_containerized "$pid"; then
            log_error "watchprog process $pid is still running during $context"
            failed=1
        fi
    done < <(pgrep -x watchprog 2>/dev/null || true)

    if ! wait_for_no_non_container_port_holders "80|3306|43203|43204|43208|43209" 1; then
        log_error "A non-Docker process owns an NVR port during $context"
        failed=1
    fi

    [[ "$failed" -eq 0 ]] || return 1
}

repair_docker_firewall_after_restore() {
    # Restoring RPM firewall configuration can flush Docker's nat/DOCKER
    # chain. Compose then fails with "No chain/target/match by that name".
    # Only restart dockerd when the chain is actually absent: a restart can
    # briefly interrupt unrelated containers managed on the same appliance.
    local docker_chain=""
    if command -v iptables &>/dev/null; then
        docker_chain=$(iptables -w -t nat -S DOCKER 2>/dev/null || true)
    fi
    if ! command -v iptables &>/dev/null || grep -qx -- '-N DOCKER' <<< "$docker_chain"; then
        return 0
    fi

    log_warn "Docker DOCKER iptables chain is missing after restore; restarting Docker to rebuild networking"
    if has_systemd; then
        systemctl restart docker
    else
        service docker restart
    fi
    DOCKER_DAEMON_RESTARTED=true
    sleep 3

    docker_chain=$(iptables -w -t nat -S DOCKER 2>/dev/null || true)
    if command -v iptables &>/dev/null && ! grep -qx -- '-N DOCKER' <<< "$docker_chain"; then
        log_warn "Docker DOCKER iptables chain is still missing after Docker restart; continuing to compose readiness gate"
    fi
}

recover_compose_after_docker_daemon_restart() {
    [[ "$DOCKER_DAEMON_RESTARTED" == "true" ]] || return 0

    # `restart: always` brings the old compose containers back as dockerd
    # returns.  On Docker 27/CO6 their old Health.Status can be reported as
    # healthy while MariaDB is still initializing, so Compose's depends_on
    # condition is not sufficient here.  Recreate only DB, prove the exact
    # host-network endpoint the engine uses is available, then restart all
    # DB-dependent services so Record threads cannot have exited during that
    # window.
    log_step "Recovering Docker services after daemon restart"
    cd "$INSTALL_DIR"
    docker compose up -d --force-recreate --no-deps db
    ensure_docker_db_ready
    local restart_services=(backend engine connector playback viewer autoheal)
    if docker compose config --services 2>/dev/null | grep -qx ptz; then
        restart_services+=(ptz)
    fi
    docker compose restart "${restart_services[@]}"
}

ensure_videostore_writable() {
    # Make sure /videostore is a directory the docker viewer container can write
    # to when it bind-mounts /videostore on first start. Without this, the
    # viewer crash-loops on `mkdir /videostore/vs1: EACCES` and the migration
    # leaves a broken NVR.
    #
    # Two host shapes seen in the field:
    #
    #   1. Real-disk NVRs (cs256 / cs2427 / cs50): rda-autofs binds /videostore
    #      to the videostore disk. vs1 already exists on the disk with content.
    #      The mkdir below is a no-op — the disk stays mounted, recording
    #      continues, and docker-start Phase 2 later assumes ownership of the
    #      mount via nsenter.
    #
    #   2. Cloud-shape NVRs (cs1018): the autofs daemon is still running but
    #      its map is empty (rda-autofs config gone, or never had a real disk).
    #      mkdir fails with EACCES because autofs refuses to materialize an
    #      entry the map doesn't know about. We detect that, stop+mask autofs
    #      to release the binding, then create vs1 on the root partition so
    #      the viewer can write.
    #
    # Probe-then-act is more reliable than file-presence detection — we react
    # to the actual symptom (mkdir EACCES), not a proxy that could mis-classify
    # an unusual host (e.g., a CloudNVR that historically had a disk and so
    # has both a /etc/auto.videostore file AND no real volume to point at).
    if [[ "$IS_WINDOWS" == "true" ]]; then
        log_info "Windows deployment: skipping /videostore writability check"
        return 0
    fi

    log_step "Ensuring /videostore is writable for viewer bind-mount"

    # Probe by attempting the actual mkdir we need. Use `if` so the
    # failure path doesn't trip the script-wide `set -e` (a bare
    # assignment from a failing $(...) substitution would abort here).
    if mkdir -p /videostore/vs1 2>/dev/null; then
        # mkdir -p returns 0 even when /videostore/vs1 pre-exists with
        # restrictive ownership (e.g., root:root 700 left by an earlier
        # bootstrap that failed before the dividia user was created).
        # Only fix the mountpoint itself. A recursive chown over a real
        # multi-TB VideoStore can run for hours and blocks migration startup;
        # the recorder/engine keep ownership of existing clip trees.
        chown dividia:docker /videostore/vs1 2>/dev/null \
            || chown root:root /videostore/vs1 2>/dev/null || true
        log_info "/videostore/vs1 ready (writable real disk or empty host directory)"
        return 0
    fi

    # mkdir failed. Distinguish "autofs has stale/empty map at /videostore"
    # (the cs1018 case we built this function for) from "something else is
    # broken — read-only FS, EIO, ENOSPC, SELinux denial, missing parent".
    # Masking autofs on a non-autofs failure would redirect writes to the
    # root partition and hide the underlying fault.
    #
    # Discriminate by querying the filesystem type at /videostore, not by
    # mkdir's stderr text: stderr is locale-dependent ("Keine Berechtigung"
    # on a German host) and SELinux denials surface as "Permission denied"
    # but aren't autofs-related. findmnt -o FSTYPE returns exactly the
    # mount type for the target, so a real-disk ext4/xfs at /videostore
    # with EROFS won't be misclassified as autofs.
    local autofs_at_videostore=0
    if has_systemd && systemctl is-active --quiet autofs 2>/dev/null; then
        local vs_fstype
        vs_fstype=$(findmnt -n -o FSTYPE /videostore 2>/dev/null || true)
        if [[ "$vs_fstype" == "autofs" ]]; then
            autofs_at_videostore=1
        fi
    fi

    if [[ "$autofs_at_videostore" != "1" ]]; then
        log_error "/videostore not writable AND autofs is not bound there — underlying fault not the stale-map case"
        log_error "  mount: $(mount 2>/dev/null | grep -E 'videostore' || echo '(no /videostore mount)')"
        log_error "Refusing to mask autofs — would redirect writes to root partition and hide the real fault."
        log_error "Operator: investigate the underlying mount/disk/SELinux state at /videostore before re-running."
        # Use return 1 (not exit 1): in migrate_flow, this function runs
        # inside the rollback-trap window (line 2686 `trap ERR ... ` →
        # line 2742 `trap - ERR`). exit 1 would bypass the trap, leaving
        # the host with RPM services stopped and Docker half-installed.
        # return 1 propagates via set -e to migrate_flow's call site and
        # fires rollback_rpm_services.
        return 1
    fi

    log_warn "/videostore not writable AND autofs is bound there — stale/empty map case"
    log_info "Masking autofs and re-seeding /videostore"
    svc_stop autofs    2>/dev/null || true
    svc_disable autofs 2>/dev/null || true
    if has_systemd; then
        systemctl mask autofs 2>/dev/null || true
    fi
    pkill -9 automount 2>/dev/null || true

    mkdir -p /videostore/vs1
    chown dividia:docker /videostore /videostore/vs1 2>/dev/null \
        || chown root:root /videostore /videostore/vs1 2>/dev/null || true
    log_info "/videostore/vs1 created on root partition"
}

install_host_firefox() {
    # Install firefox + fonts on host so the backend can launch Utilities → Support
    # → Open Browser via nsenter into the host, avoiding firefox in the viewer image.
    # Create bind-mount host dirs for sharing the X socket and Xauthority cookie
    # between the viewer container (writer) and host firefox (reader).
    #
    # On install failure, the flag file at $DATA_DIR/config/nvr-host-firefox-available
    # is left absent. Open Browser will silently no-op for that install until firefox
    # is manually installed and the flag is re-created (viewer image no longer ships
    # firefox as a fallback — saved 343 MB from the viewer image).
    log_step "Configuring host firefox for Open Browser feature"

    # Host dirs are required by docker-compose.yml bind mounts regardless of whether
    # host firefox is available — viewer's Xorg always writes the X socket to
    # /var/run/dividia/x11 (bind-mounted from /tmp/.X11-unix inside the container).
    mkdir -p /var/run/dividia/x11
    chmod 1777 /var/run/dividia/x11
    mkdir -p /var/run/dividia/xauth
    chmod 755 /var/run/dividia/xauth
    log_info "Created host X11 + xauth share dirs at /var/run/dividia/"

    local flag_file="$DATA_DIR/config/nvr-host-firefox-available"
    rm -f "$flag_file"

    local os_major="${OS_VERSION%%.*}"
    local rc=0

    # Run install in a non-fatal block (script runs under `set -e`).
    # Ubuntu server-minimal installs are missing fonts + a few X libs that
    # firefox (especially snap firefox on 24.04) needs for non-Latin text
    # rendering and shared-memory compositing. Add them defensively even
    # though the firefox snap bundles most of its own deps — host-side
    # fonts still matter for fontconfig resolution.
    case "$OS_FAMILY" in
        debian)
            # DEBIAN_FRONTEND=noninteractive is essential here, NOT cosmetic.
            # On Ubuntu 24.04, firefox apt is a transitional shim to firefox
            # snap. If snapd cannot reach api.snapcraft.io (slow network,
            # pre-NTP-sync VM, restricted egress) the snap preinst hits a
            # debconf "Retry / Abort / Skip" prompt that `-y` does NOT
            # auto-answer — bare `apt-get install -y` only handles apt's own
            # prompts, not preinst-script debconf dialogs. Without this env
            # the script hangs forever waiting on stdin that's not connected.
            #
            # The `timeout 300` wrapper is a second line of defense. Once
            # debconf auto-skips, the snap preinst falls into its OWN retry
            # loop ("Unable to contact the store, trying every minute for
            # the next 30 minutes"). That's wasted wall-clock time — if the
            # store wasn't reachable in 30s, it's not coming back in 30min.
            # 300s is generous enough for slow-but-working installs (snap
            # downloads firefox at ~250 MB) and bails fast on no-connectivity.
            # On timeout, exit code 124 falls through `|| rc=$?` into the
            # graceful-degradation warn — Open Browser feature no-ops.
            DEBIAN_FRONTEND=noninteractive timeout 300 apt-get install -y \
                firefox \
                fonts-liberation \
                fonts-dejavu \
                fonts-noto-core \
                libxshmfence1 \
                xauth \
                || rc=$?
            ;;
        rhel)
            # NVR server hosts (CO6/CO7/CO9+) run headless: viewer container only
            # starts apache2 + php-fpm, never Xorg + dview. Backend's nsenter→
            # host-firefox path needs a DISPLAY target that doesn't exist on
            # headless deployments, so installing firefox here just wastes
            # ~300 MB on / (verified empirically across cs999/cs1018/cs50/cs1129
            # 2026-05-15 — none of them had Xorg running on host or container).
            #
            # If a future deployment shape adds Xorg+dview to the viewer
            # container on RHEL (e.g., kiosk mode with local monitor), un-skip
            # this block. Open Browser will no-op gracefully on headless: the
            # flag file stays absent and the menu item silently degrades.
            log_info "RHEL ${os_major} server host runs headless — skipping host firefox install"
            return 0
            ;;
        *)
            log_warn "Unknown OS family '$OS_FAMILY' — skipping host firefox install"
            return 0
            ;;
    esac

    if [[ $rc -ne 0 ]] || ! command -v firefox &>/dev/null; then
        log_warn "host firefox not available; Utilities → Support → Open Browser will no-op until firefox is installed and /data/nvr-host-firefox-available is created"
        return 0
    fi

    # Ubuntu 24.04's firefox apt package is a transitional shim to the firefox snap.
    # `command -v firefox` resolves to the shim regardless of whether the snap itself
    # actually installed (snap download can fail silently on restricted networks).
    # Verify the snap binary exists before marking firefox "available" — otherwise the
    # shim prints "Command requires the firefox snap to be installed" at runtime and
    # Support → Open Browser silently no-ops forever.
    if [[ "$OS_FAMILY" == "debian" ]] && \
       command -v firefox &>/dev/null && \
       file "$(command -v firefox)" 2>/dev/null | grep -q 'shell script' && \
       [[ ! -x /snap/bin/firefox ]]; then
        log_warn "firefox apt shim is installed but /snap/bin/firefox is missing — snap install did not complete; skipping flag file"
        return 0
    fi

    touch "$flag_file"
    log_info "Host firefox installed; backend will launch via nsenter (flag: $flag_file)"
}

pull_and_start() {
    log_step "Pulling and starting NVR services"

    cd "$INSTALL_DIR"
    docker compose pull --quiet

    # up -d may return non-zero if dependent services time out waiting for
    # backend's health check (which can be slow on first run: DB seeding,
    # schema setup, etc). Dependent services end up in "Created" state.
    # Retry after backend is healthy to bring them up.
    if ! docker compose up -d --quiet-pull 2>&1; then
        log_warn "Some services failed to start (backend may still be initializing)"
        wait_for_backend_healthy
        log_info "Retrying service start..."
        docker compose up -d --quiet-pull 2>/dev/null || true
    fi

    log_info "NVR services started"
}

# Preserve the RPM-era HME workload when the NVR Docker root is relocated. The
# old init script expects the legacy host MariaDB and bridge networking; both
# are intentionally unavailable during migration (and CO6 can have a missing
# DOCKER iptables chain after rda-db --restore). Cut it over to the managed
# compose addon on host networking (see restore_legacy_hme_workload below) so
# the restored HME camera stays a valid live-MPE/recording gate AND `nvr update`
# owns it going forward. CO9 systems may still have an active systemd
# hme-stream.service; stop and mask it before replacing its container or the old
# launcher races this function and restores bridge-mode/--rm behavior.
legacy_hme_service_exists() {
    if has_systemd; then
        systemctl cat hme-stream.service >/dev/null 2>&1
    else
        [[ -x /etc/init.d/hme-stream ]]
    fi
}

stop_legacy_hme_service_for_docker() {
    legacy_hme_service_exists || return 0

    if [[ "$LEGACY_HME_HANDOFF" != "true" ]]; then
        if has_systemd; then
            LEGACY_HME_WAS_ENABLED=$(systemctl is-enabled hme-stream 2>/dev/null || true)
            LEGACY_HME_WAS_ACTIVE=$(systemctl is-active hme-stream 2>/dev/null || true)
            [[ "$LEGACY_HME_WAS_ENABLED" == "masked" ]] && LEGACY_HME_WAS_MASKED=true
        else
            if chkconfig --list hme-stream 2>/dev/null | grep -Eq '[2-5]:on'; then
                LEGACY_HME_WAS_ENABLED=enabled
            else
                LEGACY_HME_WAS_ENABLED=disabled
            fi
            if service hme-stream status >/dev/null 2>&1; then
                LEGACY_HME_WAS_ACTIVE=active
            else
                LEGACY_HME_WAS_ACTIVE=inactive
            fi
        fi
        LEGACY_HME_HANDOFF=true
    fi

    if has_systemd && systemctl cat hme-stream.service >/dev/null 2>&1; then
        # Mask first so a pending Restart= job cannot recreate the old
        # container between stop and docker rm. The runtime mask disappears
        # on reboot; the unit is also disabled below, while Docker's restart
        # policy owns the migrated workload.
        systemctl mask --runtime hme-stream 2>/dev/null || true
    fi
    migration_svc_stop hme-stream
    svc_disable hme-stream 2>/dev/null || true
    if svc_active hme-stream 2>/dev/null; then
        log_error "Legacy HME service is still active after stop — refusing Docker HME handoff"
        return 1
    fi
}

# Port-8554 preflight for the HME cutover. Free port => OK. Held by an HME
# container (hme-stream / hme-stream-foreign, which is what the RPM init unit
# and the earlier foreign-container path both run) => OK, we will replace it.
# Held by anything else => NOT OK: refuse so we never fight a stranger's port
# or roll one back. Uses ss or netstat (both present on all four supported host
# OSes) plus a container-name check rather than fragile PID->container mapping.
hme_port_8554_is_hme_or_free() {
    if ! { ss -ltn 2>/dev/null || netstat -ltn 2>/dev/null; } | grep -qE '[:.]8554[[:space:]]'; then
        return 0  # nobody is listening on :8554
    fi
    if docker ps --format '{{.Names}}' 2>/dev/null | grep -qxE 'hme-stream|hme-stream-foreign'; then
        return 0  # an HME container owns it; safe to replace
    fi
    return 1      # a non-HME process owns :8554
}

# Wait (up to ~30s) for the compose hme service to come up: container running
# AND :8554 bound. Robust regardless of whether the image ships the freshness
# healthcheck binary yet, so it does not gate the cutover on that dependency.
hme_compose_service_healthy() {
    local i
    for i in $(seq 1 15); do
        if [[ "$(docker inspect -f '{{.State.Running}}' hme-stream 2>/dev/null || true)" == "true" ]] \
           && { ss -ltn 2>/dev/null || netstat -ltn 2>/dev/null; } | grep -qE '[:.]8554[[:space:]]'; then
            return 0
        fi
        sleep 2
    done
    return 1
}

# Cut a migrated RPM/foreign HME workload over to the compose addon. The legacy
# workload was an init/systemd unit or a bare `docker run --name hme-stream`
# foreign container; replace both with the managed compose service so `nvr
# update` owns HME like every other container. Atomic and port-8554-aware:
#
#   1. Find the legacy conf (its own path, or the newest copy in the backup tree).
#   2. Port-8554 preflight: bail cleanly if a non-HME process owns it.
#   3. Stop/mask the legacy service (existing handoff code).
#   4. Seed data/config/hme-stream.conf from the legacy conf as a DB-unavailable
#      fallback; `nvr addon hme enable` regenerates it from the Device row when
#      the DB is reachable (the source of truth). PRESERVE the legacy conf until
#      the compose service is confirmed healthy, so rollback still has it.
#   5. Remove the foreign hme-stream / hme-stream-foreign container BEFORE the
#      compose service of the same name starts (name + :8554 collision).
#   6. `nvr addon hme enable` (seeds DeviceType 64 idempotently — it already
#      exists from the RPM — sets marker=enable, reconciles the conf + overlay,
#      compose up) and probe :8554.
#   7. On success drop the legacy conf; on failure revert the host intent and
#      return non-zero so the migration's own rollback restores the RPM service.
# Retire a legacy migrated unit on a CONFIRMED cutover. The addon owns the workload
# and the RPM is gone, so the orphaned unit file is dead weight: remove it outright
# rather than leaving a masked tombstone (a persistent mask still shows as an installed
# unit and reports masked-runtime, which the fleet monitor pages as an incomplete
# migration; that class paged 2026-08-08). Only remove a unit file we can prove is NOT
# owned by an installed RPM; if rpm still owns it (RPM not fully removed) or we cannot
# locate it, fall back to a persistent mask so we never fight rpm or leave the old unit
# runnable. The reversible `--runtime` mask from stop_legacy_*_service_for_docker still
# covers the in-window rollback path; this runs only after the cutover is confirmed
# healthy. FragmentPath is parsed without `--value` (systemd 219 on CO7 lacks it).
remove_migrated_legacy_unit() {
    local svc="$1" frag=""
    if has_systemd; then
        systemctl unmask --runtime "$svc" 2>/dev/null || true
        frag=$(systemctl show -p FragmentPath "$svc" 2>/dev/null | sed -n 's/^FragmentPath=//p')
        if [[ -n "$frag" && -f "$frag" ]] && ! rpm -qf "$frag" >/dev/null 2>&1; then
            rm -f "$frag"
            systemctl daemon-reload 2>/dev/null || true
            systemctl reset-failed "$svc" 2>/dev/null || true
            log_info "Retired orphaned legacy unit $svc ($frag)"
        else
            systemctl mask "$svc" 2>/dev/null || true
            log_info "Legacy unit $svc is RPM-owned or unlocatable; masked persistently instead of removing"
        fi
    else
        # SysV (CentOS 6): drop the init script unless rpm still owns it.
        if [[ -f "/etc/init.d/$svc" ]] && ! rpm -qf "/etc/init.d/$svc" >/dev/null 2>&1; then
            chkconfig --del "$svc" 2>/dev/null || true
            rm -f "/etc/init.d/$svc"
            log_info "Retired orphaned legacy init script /etc/init.d/$svc"
        fi
    fi
}

restore_legacy_hme_workload() {
    local conf=/usr/local/etc/hme-stream.conf
    local new_conf=/opt/dividia/data/config/hme-stream.conf
    local saved_conf newest_mtime=0 candidate candidate_mtime

    if [[ ! -f "$conf" ]]; then
        while IFS= read -r candidate; do
            candidate_mtime=$(stat -c '%Y' "$candidate" 2>/dev/null || stat -f '%m' "$candidate" 2>/dev/null || true)
            if [[ -n "$candidate_mtime" && "$candidate_mtime" -gt "$newest_mtime" ]]; then
                newest_mtime="$candidate_mtime"
                saved_conf="$candidate"
            fi
        # HME config is captured in the normal NVR backup tree.  Never scan
        # the recording trees themselves: on a multi-TB VideoStore that can
        # turn a no-HME migration into an hours-long outage.
        done < <(find /videostore/vs*/backups /rda/backups -maxdepth 8 -type f \
            -name hme-stream.conf 2>/dev/null || true)
        if [[ -n "${saved_conf:-}" ]]; then
            mkdir -p /usr/local/etc
            cp "$saved_conf" "$conf"
            log_info "Restored HME config from $saved_conf"
        fi
    fi

    [[ -f "$conf" ]] || return 0

    log_step "Cutting over HME to the compose addon"

    # 2. Port-8554 preflight.
    if ! hme_port_8554_is_hme_or_free; then
        log_error "Port 8554 is held by a non-HME process; refusing HME cutover"
        return 1
    fi

    # 2b. Prove the managed image is AVAILABLE before we stop/mask the legacy
    #     service or remove the working foreign container. The reconcile path
    #     already gates this way; the migration path must too. If the ${CHANNEL}
    #     tag is missing (out-of-band hme build not yet published, a canary
    #     dev-<workspace> channel, or a registry blip), stopping the legacy owner
    #     first would take HME dark with the RPM unit masked and NO auto-restore
    #     (migrate_flow calls this `|| log_warn`, so the rm -rf ERR-trap rollback
    #     is deliberately suppressed). Leave the legacy workload running and let
    #     the 2-minute `nvr addon hme` reconcile take over once the tag publishes.
    local hme_reg hme_chan hme_img
    hme_reg=$(grep '^REGISTRY=' /opt/dividia/.env 2>/dev/null | cut -d= -f2- | tr -d '[:space:]'); hme_reg=${hme_reg:-docker.io}
    hme_chan=$(grep '^CHANNEL=' /opt/dividia/.env 2>/dev/null | cut -d= -f2- | tr -d '[:space:]'); hme_chan=${hme_chan:-dev}
    hme_img="${hme_reg}/dividia/hme-stream:${hme_chan}"
    if ! { docker image inspect "$hme_img" >/dev/null 2>&1 || docker pull "$hme_img" >/dev/null 2>&1; }; then
        log_warn "Managed HME image $hme_img unavailable; leaving the legacy HME workload in place (reconcile will cut over when the tag publishes)"
        return 0
    fi

    # 3. Stop/mask the legacy service (records prior state for rollback).
    #    Reached only after the image is confirmed local, so we never strand a
    #    site with the legacy owner masked and no runnable replacement.
    stop_legacy_hme_service_for_docker

    # 4. Seed the new conf location from the preserved legacy conf as a
    #    DB-unavailable fallback. `nvr addon hme enable` regenerates it from the
    #    Device row when the DB is up. The legacy conf is NOT deleted yet.
    mkdir -p "$(dirname "$new_conf")" 2>/dev/null || true
    cp -f "$conf" "$new_conf" 2>/dev/null || true
    chown dividia:docker "$new_conf" 2>/dev/null || true
    chmod 640 "$new_conf" 2>/dev/null || true

    # 5. Remove the foreign container so the compose service name + :8554 are free.
    docker rm -f hme-stream hme-stream-foreign >/dev/null 2>&1 || true
    chkconfig hme-stream off 2>/dev/null || true

    # 6. Enable the compose addon (single source of truth for seed + overlay +
    #    conf-gen + compose up). It pulls dividia/hme-stream:${CHANNEL}, so the
    #    :latest pull is gone — the hme-stream image must publish the channel tag.
    if [[ -x /opt/dividia/nvr ]]; then
        /opt/dividia/nvr addon hme enable >/dev/null 2>&1 \
            || log_warn "nvr addon hme enable returned non-zero; verifying health directly"
    else
        log_error "/opt/dividia/nvr not present; cannot cut HME over to the compose addon"
        return 1
    fi

    # 6b/7. Probe, then commit or roll back.
    if hme_compose_service_healthy; then
        log_info "HME compose service healthy on host network (port 8554; container hme-stream)"
        # Cutover confirmed: now safe to drop the legacy conf.
        rm -f "$conf" 2>/dev/null || true
        # On a CONFIRMED cutover, retire the legacy unit for good: the compose
        # addon owns the workload and the RPM is gone. remove_migrated_legacy_unit
        # deletes the orphaned unit file when it is not RPM-owned (else falls back to
        # a persistent mask) instead of leaving a masked tombstone that
        # `systemctl is-enabled` reports as masked-runtime and the fleet monitor pages
        # as an incomplete migration (cs989/cs1078/cs2204/cs2598/cs1027/cs2418,
        # 2026-08-08). The reversible `--runtime` mask from
        # stop_legacy_hme_service_for_docker still covers the in-window rollback path;
        # this runs only here, after the cutover is confirmed healthy. Gated on the
        # handoff flag so a box whose unit was already fully removed is untouched.
        if [[ "$LEGACY_HME_HANDOFF" == "true" ]]; then
            remove_migrated_legacy_unit hme-stream
        fi
        return 0
    fi

    log_error "HME compose service did not come healthy on :8554 — reverting host intent"
    # The image was confirmed local at step 2b, so this is a runtime miss (device
    # unreachable, slow start), not a missing tag. Revert the compose intent and
    # let the unattended 2-minute `nvr addon hme` reconcile retry the managed
    # container once conditions clear. Do NOT rely on rollback_rpm_services: the
    # migrate_flow call is `|| log_warn`, which suppresses the ERR trap on
    # purpose (an HME miss must never fire the rm -rf db_data rollback), and the
    # legacy RPM is being retired by the addon anyway, so we do not restore it.
    docker rm -f hme-stream hme-stream-foreign >/dev/null 2>&1 || true
    rm -f /opt/dividia/hme/marker 2>/dev/null || true
    sed -i 's/:docker-compose\.hme\.yml//g' /opt/dividia/.env 2>/dev/null || true
    return 1
}

################################################################################
# Legacy aiengine adoption (optional local LPR / object detection)
#
# Unlike HME (a foreign `docker run` workload), aiengine is adopted INTO the
# compose project as the optional add-on: config directory + host intent +
# digest-pinned overlay, so `nvr update`, backup/restore, and boot all manage
# it thereafter. See docs/plans/aiengine-docker-addon.md and the `nvr addon
# aiengine` lifecycle in the nvr CLI. Mirrors the HME service capture/rollback.
################################################################################

legacy_aiengine_service_exists() {
    if has_systemd; then
        systemctl cat aiengine.service >/dev/null 2>&1
    else
        [[ -x /etc/init.d/aiengine ]]
    fi
}

stop_legacy_aiengine_service_for_docker() {
    legacy_aiengine_service_exists || return 0

    if [[ "$LEGACY_AIENGINE_HANDOFF" != "true" ]]; then
        if has_systemd; then
            LEGACY_AIENGINE_WAS_ENABLED=$(systemctl is-enabled aiengine 2>/dev/null || true)
            LEGACY_AIENGINE_WAS_ACTIVE=$(systemctl is-active aiengine 2>/dev/null || true)
            [[ "$LEGACY_AIENGINE_WAS_ENABLED" == "masked" ]] && LEGACY_AIENGINE_WAS_MASKED=true
        else
            if chkconfig --list aiengine 2>/dev/null | grep -Eq '[2-5]:on'; then
                LEGACY_AIENGINE_WAS_ENABLED=enabled
            else
                LEGACY_AIENGINE_WAS_ENABLED=disabled
            fi
            if service aiengine status >/dev/null 2>&1; then
                LEGACY_AIENGINE_WAS_ACTIVE=active
            else
                LEGACY_AIENGINE_WAS_ACTIVE=inactive
            fi
        fi
        LEGACY_AIENGINE_HANDOFF=true
    fi

    if has_systemd && systemctl cat aiengine.service >/dev/null 2>&1; then
        # Mask first so a pending Restart= job cannot recreate the old
        # container between stop and docker rm. The runtime mask disappears on
        # reboot; the unit is also disabled below, while compose owns the
        # migrated workload.
        systemctl mask --runtime aiengine 2>/dev/null || true
    fi
    migration_svc_stop aiengine
    svc_disable aiengine 2>/dev/null || true
    if svc_active aiengine 2>/dev/null; then
        log_error "Legacy aiengine service is still active after stop — refusing Docker aiengine handoff"
        return 1
    fi
}

# Adopt a complete, licensed legacy aiengine install into the compose add-on.
# Fail-safe retention: a complete licensed install is retained even without
# provable local demand (another NVR may consume it), marked legacy-provisioned.
# An incomplete install (no key / no device-id) is left dormant, not adopted.
restore_legacy_aiengine_workload() {
    local legacy_dir=/usr/local/etc/aiengine
    local dst_dir="$INSTALL_DIR/data/config/aiengine"
    local intent_dir="$INSTALL_DIR/data/config/addons/aiengine"

    legacy_aiengine_service_exists || [[ -f "$legacy_dir/aiengine-key" ]] || return 0

    # Require a COMPLETE license identity. device-id + aiengine-key are one
    # pair; without both this is not an adoptable licensed install.
    if [[ ! -f "$legacy_dir/device-id" || ! -f "$legacy_dir/aiengine-key" ]]; then
        log_info "aiengine: no complete legacy license identity at $legacy_dir; leaving dormant, not adopting"
        return 0
    fi

    log_step "Adopting legacy aiengine workload into the compose add-on"

    # Capture the RUNNING legacy image's digest BEFORE stopping it. The RPM runs
    # dividia/aiengine:latest, so the digest is the pin, not the tag.
    local image_ref="" legacy_img legacy_digest legacy_cfg_image
    legacy_img=$(docker inspect -f '{{.Image}}' aiengine 2>/dev/null || true)
    if [[ -n "$legacy_img" ]]; then
        legacy_digest=$(docker image inspect -f '{{if .RepoDigests}}{{index .RepoDigests 0}}{{end}}' "$legacy_img" 2>/dev/null || true)
        legacy_cfg_image=$(docker inspect -f '{{.Config.Image}}' aiengine 2>/dev/null || true)
        # Only adopt a container that really is dividia/aiengine.
        if [[ -n "$legacy_cfg_image" && "$legacy_cfg_image" != dividia/aiengine* ]]; then
            log_warn "aiengine: running container image '$legacy_cfg_image' is not dividia/aiengine; not adopting"
            return 0
        fi
        if [[ -n "$legacy_digest" ]]; then
            image_ref="$legacy_digest"
        elif [[ -n "$legacy_cfg_image" ]]; then
            image_ref="$legacy_cfg_image"
        fi
    fi

    # Stop + mask the legacy service, then remove its container so it releases
    # port 88 before the compose service claims it. Non-fatal: if the legacy
    # service will not stop, skip adoption rather than reverting the whole
    # migration for an optional add-on.
    if ! stop_legacy_aiengine_service_for_docker; then
        log_warn "aiengine: could not stop the legacy service; skipping adoption (run 'nvr addon aiengine enable' post-migration). Core migration is unaffected."
        return 0
    fi
    docker rm -f aiengine >/dev/null 2>&1 || true
    # Also drop any other RUNNING dividia/aiengine wrapper by ancestor image, so
    # a non-standard legacy container name cannot keep port 88 bound. On a box
    # being migrated, any dividia/aiengine container is the legacy wrapper.
    local _other_ai
    _other_ai=$(docker ps -q --filter ancestor=dividia/aiengine 2>/dev/null || true)
    [[ -n "$_other_ai" ]] && docker rm -f $_other_ai >/dev/null 2>&1 || true

    # Stage the credential + model directory (root:docker 0710; secrets 0600).
    mkdir -p "$dst_dir"
    cp -a "$legacy_dir/." "$dst_dir/" 2>/dev/null || cp -R "$legacy_dir/." "$dst_dir/" 2>/dev/null || true
    chown -R root:docker "$dst_dir" 2>/dev/null || true
    chmod 0710 "$dst_dir" 2>/dev/null || true
    chmod 0600 "$dst_dir/device-id" "$dst_dir/aiengine-key" 2>/dev/null || true

    # The device-id file is inode-encrypted (anti-copy): the copied one is now
    # bound to the WRONG inode, so the engine would read it as DEADBEEFDEAD and
    # reject the key. Drop it and regenerate a valid file bound to the new inode.
    # The id is MAC-derived, so on this same host it is identical and the
    # existing key still validates. If regeneration fails, no file remains and
    # the engine regenerates from hardware at start (same result).
    rm -f "$dst_dir/device-id"
    if [[ -n "$image_ref" ]]; then
        docker run --rm -v "$dst_dir:/srv/data" "$image_ref" \
            /usr/local/bin/aiengine-make-device-id >/dev/null 2>&1 \
            || log_warn "aiengine: could not regenerate device-id; engine will regenerate from hardware at start"
        chmod 0600 "$dst_dir/device-id" 2>/dev/null || true
    fi

    # Choose intent from local demand against the RESTORED DB. Fail-safe: any
    # error or no demand retains as legacy-provisioned (cross-NVR inbound use is
    # undetectable locally). Strong local demand marks enabled.
    local intent=legacy-provisioned demand=""
    if [[ -x "$INSTALL_DIR/nvr" ]]; then
        demand=$("$INSTALL_DIR/nvr" addon aiengine local-demand 2>/dev/null || true)
    fi
    [[ -n "$demand" ]] && intent=enabled
    mkdir -p "$intent_dir"
    chown root:docker "$intent_dir" 2>/dev/null || true
    chmod 0750 "$intent_dir" 2>/dev/null || true
    printf '%s\n' "$intent" > "$intent_dir/intent"
    chmod 0640 "$intent_dir/intent" 2>/dev/null || true

    # Pin the adopted image (digest when available) in the host .env.
    if [[ -n "$image_ref" ]]; then
        set_host_env_var AIENGINE_IMAGE "$image_ref"
    fi

    # Append the overlay to COMPOSE_FILE via the same reconcile the CLI uses
    # (intent is on + creds present, so the normalizer adds it), then start the
    # add-on from the local image. No pull: the exact bytes are already local.
    if [[ -x "$INSTALL_DIR/nvr" ]]; then
        "$INSTALL_DIR/nvr" normalize-addon-env || true
    fi
    # Add-on start failures are NON-FATAL to the migration. aiengine is an
    # optional <=2-camera add-on; it must never trigger rollback_rpm_services
    # and revert a completed, healthy NVR migration back to RPM (this function
    # runs inside migrate_flow's ERR-trap window). On failure, cleanly back the
    # add-on out (intent disabled, overlay + pin stripped, container removed) so
    # it does not crash-loop, warn the operator to re-enable, and return 0. The
    # legacy service is already stopped/masked; local AI stays off until the
    # operator runs `nvr addon aiengine enable`.
    if ! ( cd "$INSTALL_DIR" && docker compose up -d aiengine ); then
        log_warn "aiengine: compose service failed to start after adoption; leaving add-on OFF (run 'nvr addon aiengine enable' post-migration). Core migration is unaffected."
        back_out_aiengine_adoption
        return 0
    fi
    sleep 3
    if [[ "$(docker inspect -f '{{.State.Running}}' aiengine 2>/dev/null || true)" != "true" ]]; then
        log_warn "aiengine: adopted container did not stay running; leaving add-on OFF (run 'nvr addon aiengine enable' post-migration). Core migration is unaffected."
        docker logs --tail 20 aiengine 2>&1 | head -20 || true
        back_out_aiengine_adoption
        return 0
    fi
    # On a CONFIRMED cutover, retire the legacy unit for good (mirrors HME):
    # remove_migrated_legacy_unit deletes the orphaned aiengine unit file when it is
    # not RPM-owned (else falls back to a persistent mask), rather than leaving a
    # masked tombstone the fleet monitor pages as an incomplete migration. The
    # reversible `--runtime` mask from stop_legacy_aiengine_service_for_docker still
    # covers the in-window rollback; this is reached only after the adopted container
    # is confirmed Running (the failure paths back out and return 0 above).
    if [[ "$LEGACY_AIENGINE_HANDOFF" == "true" ]]; then
        remove_migrated_legacy_unit aiengine
    fi
    log_info "aiengine adopted into compose (intent=$intent, image=${image_ref:-stable})"
}

# Cleanly reverse a partial aiengine adoption: disable intent, remove the
# container, and strip the overlay + AIENGINE_IMAGE from .env via the shared
# normalizer. Used on a non-fatal adoption failure and by the rollback path.
back_out_aiengine_adoption() {
    printf 'disabled\n' > "$INSTALL_DIR/data/config/addons/aiengine/intent" 2>/dev/null || true
    ( cd "$INSTALL_DIR" && docker compose rm -fs aiengine >/dev/null 2>&1 ) || true
    docker rm -f aiengine >/dev/null 2>&1 || true
    if [[ -x "$INSTALL_DIR/nvr" ]]; then
        "$INSTALL_DIR/nvr" normalize-addon-env >/dev/null 2>&1 || true
    fi
}

################################################################################
# Boot Service
################################################################################

# Commit a generated BOOT FILE atomically, or not at all.
#
# WHY THIS EXISTS. `cat > /etc/init.d/nvr` truncates in place and is not atomic, and
# nothing used to check the result. That was tolerable while create_boot_service ran
# ONCE per NVR lifetime with a human watching the install. It is not tolerable now that
# `nvr update` calls it through ensure_boot_service on every update -- nightly,
# unattended, across the CentOS 6 fleet. An interrupted write, or a generator that dies
# partway through its heredoc, would leave a TRUNCATED boot file that is then chmod 755
# and chkconfig'd on: the OS still boots, the NVR stack does not start, and it needs a
# hand fix on a customer box.
#
# Mirrors what videostore_persist_mount already does for /etc/fstab: temp file in the
# SAME directory (so the rename is atomic on one filesystem), validate, then commit. On
# any failure the existing boot file is left exactly as it was -- a stale-but-working
# boot file beats a fresh broken one.
#
# Validation is deliberately about SHAPE, not content equality: the file must parse
# under the HOST's own bash (CentOS 6 is bash 4.1, and validating with the host shell is
# the same rule validate_extracted_host_tool follows in the nvr CLI), be non-trivial,
# and carry the markers that prove the heredoc completed rather than stopping early.
#
# Args: <dest> <tmp> <kind: shell|unit> <octal-mode> [required-marker ...]
#
# KIND is EXPLICIT, not sniffed. An earlier version gated the `bash -n` parse check on
# `head -1 | grep '^#!.*sh'`, so a generated script that lost its shebang skipped the
# parse check entirely and could still be published. Passing the kind removes that
# whole class: `shell` always requires BOTH a shebang and a successful parse.
#
# This function also owns the chmod, and a chmod FAILURE aborts the commit. Doing the
# chmod at the call site with `|| true` meant a failed chmod still renamed a 0600 file
# onto /etc/init.d/nvr -- destroying the working script and replacing it with one the
# boot cannot execute, which is the exact failure this hardening exists to prevent.
commit_boot_file() {
    local dest="$1"; shift
    local tmp="$1"; shift
    local kind="$1"; shift
    local mode="$1"; shift
    local marker published

    if [[ ! -s "$tmp" ]]; then
        log_warn "generated $dest is empty; keeping the existing file"
        rm -f "$tmp"
        return 1
    fi
    if [[ "$kind" == "shell" ]]; then
        if ! head -1 "$tmp" | grep -q '^#!'; then
            log_warn "generated $dest has no shebang; keeping the existing file"
            rm -f "$tmp"
            return 1
        fi
        if ! bash -n "$tmp" 2>/dev/null; then
            log_warn "generated $dest does not parse under this host's bash; keeping the existing file"
            rm -f "$tmp"
            return 1
        fi
    fi
    # Permissions BEFORE the rename, so the destination is never briefly unusable -- and
    # a failure here must abort rather than publish an unusable file.
    if ! chmod "$mode" "$tmp" 2>/dev/null; then
        log_warn "could not set mode $mode on the generated $dest; keeping the existing file"
        rm -f "$tmp"
        return 1
    fi
    # Markers are literal by default. A marker prefixed `re:` is an extended regex, which
    # some checks NEED: a literal "ensure_videostore_mounts" also matches the function
    # DEFINITION inside the generated script, so dropping the CALL would still pass. That
    # exact hole was caught by test_nvr_ensure_boot_service.sh before this shipped.
    for marker in "$@"; do
        case "$marker" in
            re:*)
                if ! grep -qE -- "${marker#re:}" "$tmp"; then
                    log_warn "generated $dest does not match /${marker#re:}/ (truncated or altered write?); keeping the existing file"
                    rm -f "$tmp"
                    return 1
                fi
                ;;
            *)
                if ! grep -qF -- "$marker" "$tmp"; then
                    log_warn "generated $dest is missing '$marker' (truncated write?); keeping the existing file"
                    rm -f "$tmp"
                    return 1
                fi
                ;;
        esac
    done
    if ! mv -f "$tmp" "$dest"; then
        log_warn "could not move generated boot file into $dest; keeping the existing file"
        rm -f "$tmp"
        return 1
    fi
    # Verify what actually landed. Cheap, and it turns a silent wrong-mode publish into a
    # loud one -- the caller has already lost the old file by this point, so the operator
    # needs to know rather than discover it at the next reboot.
    published=$(stat -c %a "$dest" 2>/dev/null || stat -f %Lp "$dest" 2>/dev/null || echo "")
    if [[ -n "$published" && "$published" != "$mode" ]]; then
        log_warn "$dest published with mode $published, expected $mode; fix by hand before the next reboot"
    fi
    return 0
}

# Generate a root-owned Compose overlay that hides the reserved host-storage
# directory from every container that receives the read-write /videostore bind.
# Without this nested mask, an engine compromise could overwrite the host NVR
# script, Docker state, database files, or .env below the adopted source.
boot_storage_read_mapping_file() {
    local config="$1"
    awk '
        $1 ~ /^#/ || NF == 0 { next }
        NF != 2 { exit 2 }
        { count++; source=$1; target=$2 }
        END {
            if (count != 1) exit 2
            print source, target
        }
    ' "$config" 2>/dev/null
}

create_boot_storage_container_mask() {
    local config="${1:-$NVR_BOOT_STORAGE_CONFIG}" mapping source target relative store_name store_digits overlay_dir overlay_tmp
    [[ -r "$config" ]] || return 1
    mapping=$(boot_storage_read_mapping_file "$config") || {
        log_warn "cannot generate the container mask from invalid config $config"
        return 1
    }
    read -r source target <<< "$mapping"
    case "$source$target" in
        *[!A-Za-z0-9_./-]*|*'/../'*|*'/./'*|*'//'*|*'/..'|*'/.' )
            log_warn "cannot generate the container mask from unsafe paths in $config"
            return 1 ;;
    esac
    case "$source" in /videostore/vs*/?*) ;; *)
        log_warn "container-mask source is outside /videostore/vsN: $source"
        return 1 ;;
    esac
    relative="${source#/videostore/}"
    store_name="${relative%%/*}"
    store_digits="${store_name#vs}"
    case "$store_digits" in ''|*[!0-9]*)
        log_warn "container-mask source is outside /videostore/vsN: $source"
        return 1 ;;
    esac
    [[ "$target" == "/opt" ]] || {
        log_warn "container-mask target is unsupported: $target"
        return 1
    }

    if [[ -L "$NVR_BOOT_STORAGE_CONTAINER_MASK" ]]; then
        log_warn "container-mask directory cannot be a symlink: $NVR_BOOT_STORAGE_CONTAINER_MASK"
        return 1
    fi
    if [[ -d "$NVR_BOOT_STORAGE_CONTAINER_MASK" ]] \
            && find "$NVR_BOOT_STORAGE_CONTAINER_MASK" -mindepth 1 -maxdepth 1 2>/dev/null | grep -q .; then
        log_warn "container-mask directory is not empty: $NVR_BOOT_STORAGE_CONTAINER_MASK"
        return 1
    fi
    mkdir -p "$NVR_BOOT_STORAGE_CONTAINER_MASK" || return 1
    chown root:root "$NVR_BOOT_STORAGE_CONTAINER_MASK" 2>/dev/null || true
    chmod 0555 "$NVR_BOOT_STORAGE_CONTAINER_MASK" || return 1

    overlay_dir="$(dirname "$NVR_BOOT_STORAGE_COMPOSE_OVERLAY")"
    mkdir -p "$overlay_dir" || return 1
    overlay_tmp=$(mktemp "$overlay_dir/.docker-compose.boot-storage.XXXXXX") || return 1
    cat > "$overlay_tmp" <<EOF
# Managed by Dividia NVR. The final nested bind hides host control data from
# containers that otherwise receive all of /videostore read-write.
services:
  backend:
    volumes:
      - type: bind
        source: $NVR_BOOT_STORAGE_CONTAINER_MASK
        target: $source
        read_only: true
  engine:
    volumes:
      - type: bind
        source: $NVR_BOOT_STORAGE_CONTAINER_MASK
        target: $source
        read_only: true
  connector:
    volumes:
      - type: bind
        source: $NVR_BOOT_STORAGE_CONTAINER_MASK
        target: $source
        read_only: true
  playback:
    volumes:
      - type: bind
        source: $NVR_BOOT_STORAGE_CONTAINER_MASK
        target: $source
        read_only: true
  viewer:
    volumes:
      - type: bind
        source: $NVR_BOOT_STORAGE_CONTAINER_MASK
        target: $source
        read_only: true
EOF
    commit_boot_file "$NVR_BOOT_STORAGE_COMPOSE_OVERLAY" "$overlay_tmp" config 644 \
        "Managed by Dividia NVR" \
        "source: $NVR_BOOT_STORAGE_CONTAINER_MASK" \
        "target: $source" \
        "read_only: true"
}

boot_storage_compose_overlay_reconcile() {
    [[ -e "$NVR_BOOT_STORAGE_CONFIG" || -n "${NVR_BOOT_STORAGE_STAGED_CONFIG:-}" ]] || return 0
    [[ -f "$INSTALL_DIR/.env" ]] || return 0
    local current rebuilt="" part
    current=$(sed -n 's/^COMPOSE_FILE=//p' "$INSTALL_DIR/.env" 2>/dev/null | head -1)
    [[ -n "$current" ]] || return 1
    local old_ifs="$IFS"
    IFS=':'
    for part in $current; do
        [[ -z "$part" || "$part" == "$NVR_BOOT_STORAGE_COMPOSE_OVERLAY" ]] && continue
        rebuilt="${rebuilt:+$rebuilt:}$part"
    done
    IFS="$old_ifs"
    rebuilt="${rebuilt:+$rebuilt:}$NVR_BOOT_STORAGE_COMPOSE_OVERLAY"
    [[ "$rebuilt" == "$current" ]] || set_host_env_var COMPOSE_FILE "$rebuilt"
}

# Install the only code allowed to establish a VideoStore-backed Docker path
# during boot. It is root-resident by design. The normal launcher lives below
# /opt and systemd starts it after Docker, so it cannot break that dependency
# cycle when /opt is the configured target.
create_boot_storage_helper() {
    local helper_dir helper_tmp
    helper_dir="$(dirname "$NVR_BOOT_STORAGE_HELPER")"
    if ! mkdir -p "$helper_dir" 2>/dev/null; then
        log_warn "could not create $helper_dir; boot-storage helper not refreshed"
        return 1
    fi
    rm -f "$helper_dir"/.dividia-nvr-boot-storage.?????? 2>/dev/null || true
    if ! helper_tmp="$(mktemp "$helper_dir/.dividia-nvr-boot-storage.XXXXXX" 2>/dev/null)" \
            || [[ -z "$helper_tmp" ]]; then
        log_warn "could not create a temp file for $NVR_BOOT_STORAGE_HELPER"
        return 1
    fi

    cat > "$helper_tmp" <<'BOOTEOF'
#!/bin/bash
# Mount one explicitly adopted VideoStore-backed host path before containerd
# and Docker. Normal hosts do not install this helper. Once installed, a
# missing or invalid configuration fails closed to protect the root filesystem.

set -u

CONFIG="${NVR_BOOT_STORAGE_CONFIG:-/etc/dividia-nvr/boot-storage.conf}"
FSTAB="${NVR_FSTAB:-/etc/fstab}"
VIDEOSTORE_ROOT="${NVR_VIDEOSTORE_ROOT:-/videostore}"
TARGET_ROOT="${NVR_BOOT_STORAGE_TARGET_ROOT:-}"
SHADOW_LIMIT_BYTES="${NVR_BOOT_STORAGE_SHADOW_LIMIT_BYTES:-67108864}"
RETRIES="${NVR_BOOT_STORAGE_RETRIES:-12}"
RETRY_SECONDS="${NVR_BOOT_STORAGE_RETRY_SECONDS:-5}"

fail() {
    echo "dividia-nvr boot storage: ERROR: $1" >&2
    return 1
}

shadow_bytes() {
    local output bytes
    output=$(du -sx --block-size=1 "$1" 2>/dev/null) || return 1
    bytes=$(printf '%s\n' "$output" | awk 'NR == 1 { print $1; exit }')
    case "$bytes" in ''|*[!0-9]*) return 1 ;; esac
    printf '%s\n' "$bytes"
}

fstab_entry_for_store() {
    local mount_point="$1"
    awk -v mp="$mount_point" '
        $1 !~ /^#/ && $2 == mp { count++; source=$1; options=$4 }
        END {
            if (count != 1) exit 2
            print source, options
        }
    ' "$FSTAB" 2>/dev/null
}

fstab_target_is_absent() {
    local target="$1"
    ! awk -v mp="$target" '
        $1 !~ /^#/ && $2 == mp { found=1 }
        END { exit !found }
    ' "$FSTAB" 2>/dev/null
}

resolve_block_spec() {
    local spec="$1" resolved=""
    resolved=$(findfs "$spec" 2>/dev/null) || resolved=""
    if [ -z "$resolved" ]; then
        case "$spec" in
            LABEL=*) resolved=$(blkid -L "${spec#LABEL=}" 2>/dev/null) || resolved="" ;;
            UUID=*) resolved=$(blkid -U "${spec#UUID=}" 2>/dev/null) || resolved="" ;;
            /dev/*) resolved="$spec" ;;
        esac
    fi
    [ -n "$resolved" ] || return 1
    readlink -f "$resolved" 2>/dev/null
}

verify_store_identity() {
    local store="$1" entry spec options expected live
    entry=$(fstab_entry_for_store "$store") || {
        fail "$store needs exactly one fstab entry"
        return 1
    }
    read -r spec options <<EOF
$entry
EOF
    case ",${options}," in
        *,noauto,*) ;;
        *) fail "$store fstab entry has no noauto option"; return 1 ;;
    esac
    expected=$(resolve_block_spec "$spec") || {
        fail "cannot resolve fstab source $spec for $store"
        return 1
    }
    live=$(findmnt -rn -T "$store" -o SOURCE 2>/dev/null) || live=""
    live=$(readlink -f "$live" 2>/dev/null) || live=""
    if [ -z "$live" ] || [ "$live" != "$expected" ]; then
        fail "$store is mounted from ${live:-unknown}, not fstab source $expected"
        return 1
    fi
    return 0
}

source_directory_safe() {
    local source="$1" store="$2" canonical_source canonical_store source_dev store_dev uid mode
    canonical_source=$(readlink -f "$source" 2>/dev/null) || canonical_source=""
    canonical_store=$(readlink -f "$store" 2>/dev/null) || canonical_store=""
    [ -n "$canonical_source" ] && [ "$canonical_source" = "$source" ] || {
        fail "source must be canonical and cannot contain symlinks: $source"
        return 1
    }
    case "$canonical_source" in "$canonical_store"/*) ;; *)
        fail "source resolves outside $store: $source"
        return 1 ;;
    esac
    if mountpoint -q "$source" 2>/dev/null; then
        fail "source cannot be a separate mount point: $source"
        return 1
    fi
    source_dev=$(stat -c %d "$source" 2>/dev/null) || source_dev=""
    store_dev=$(stat -c %d "$store" 2>/dev/null) || store_dev=""
    [ -n "$source_dev" ] && [ "$source_dev" = "$store_dev" ] || {
        fail "source is not on the $store filesystem: $source"
        return 1
    }
    uid=$(stat -c %u "$source" 2>/dev/null) || uid=""
    mode=$(stat -c %a "$source" 2>/dev/null) || mode=""
    [ "$uid" = "0" ] || { fail "source must be owned by root: $source"; return 1; }
    case "$mode" in ''|*[!0-7]*) fail "cannot verify source permissions: $source"; return 1 ;; esac
    [ $((8#$mode & 8#22)) -eq 0 ] || {
        fail "source cannot be group or world writable: $source"
        return 1
    }
    return 0
}

mount_videostore() {
    local store="$1" shadow i
    if mountpoint -q "$store" 2>/dev/null; then
        verify_store_identity "$store"
        return $?
    fi

    if ! shadow=$(shadow_bytes "$store"); then
        fail "could not measure root-backed data at $store"
        return 1
    fi
    if [ "$shadow" -gt "$SHADOW_LIMIT_BYTES" ]; then
        fail "$store holds ${shadow} root-backed bytes; refusing to hide them"
        return 1
    fi

    command -v udevadm >/dev/null 2>&1 && udevadm settle --timeout=10 >/dev/null 2>&1 || true
    i=0
    while [ "$i" -lt "$RETRIES" ]; do
        mount "$store" >/dev/null 2>&1 || true
        if mountpoint -q "$store" 2>/dev/null; then
            verify_store_identity "$store"
            return $?
        fi
        i=$((i + 1))
        [ "$i" -lt "$RETRIES" ] && sleep "$RETRY_SECONDS"
    done

    # Do not trust mount's exit status. CentOS 6 can return zero for an absent
    # device when the fstab row includes nofail.
    fail "$store is still not mounted after $RETRIES attempts"
    return 1
}

apply_mapping() {
    local source="$1" target="$2" relative store_name store_digits store shadow

    case "$source$target" in
        *[!A-Za-z0-9_./-]*|*'/../'*|*'/./'*|*'//'*|*'/..'|*'/.' )
            fail "paths must be absolute, normalized, and use safe characters"
            return 1
            ;;
    esac
    case "$source" in
        "$VIDEOSTORE_ROOT"/vs*/?*) ;;
        *) fail "source must be below $VIDEOSTORE_ROOT/vsN: $source"; return 1 ;;
    esac
    case "$target" in
        /opt) ;;
        *)
            if [ -z "$TARGET_ROOT" ] || [ "$target" != "$TARGET_ROOT/opt" ]; then
                fail "unsupported bind target: $target"
                return 1
            fi
            ;;
    esac

    relative="${source#"$VIDEOSTORE_ROOT"/}"
    store_name="${relative%%/*}"
    store_digits="${store_name#vs}"
    case "$store_digits" in
        ''|*[!0-9]*) fail "source must be below $VIDEOSTORE_ROOT/vsN: $source"; return 1 ;;
    esac
    store="$VIDEOSTORE_ROOT/$store_name"

    [ -r "$FSTAB" ] || { fail "cannot read $FSTAB"; return 1; }
    fstab_target_is_absent "$target" || {
        fail "$target has a competing fstab entry"
        return 1
    }
    mount_videostore "$store" || return 1

    [ -d "$source" ] || { fail "source directory does not exist: $source"; return 1; }
    [ -d "$target" ] || { fail "target directory does not exist: $target"; return 1; }
    source_directory_safe "$source" "$store" || return 1

    if mountpoint -q "$target" 2>/dev/null; then
        if [ "$source" -ef "$target" ]; then
            return 0
        fi
        fail "$target is mounted from a different source"
        return 1
    fi

    if ! shadow=$(shadow_bytes "$target"); then
        fail "could not measure root-backed data at $target"
        return 1
    fi
    if [ "$shadow" -gt "$SHADOW_LIMIT_BYTES" ]; then
        fail "$target holds ${shadow} root-backed bytes; refusing to hide them"
        return 1
    fi

    if ! mount --bind "$source" "$target" >/dev/null 2>&1; then
        fail "could not bind $source to $target"
        return 1
    fi
    if ! mountpoint -q "$target" 2>/dev/null || ! [ "$source" -ef "$target" ]; then
        fail "bind verification failed for $target"
        return 1
    fi
    return 0
}

case "$RETRIES" in ''|*[!0-9]*) RETRIES=12 ;; esac
case "$RETRY_SECONDS" in ''|*[!0-9]*) RETRY_SECONDS=5 ;; esac
case "$SHADOW_LIMIT_BYTES" in ''|*[!0-9]*) SHADOW_LIMIT_BYTES=67108864 ;; esac
[ "$RETRIES" -gt 0 ] 2>/dev/null || RETRIES=1
[ -e "$CONFIG" ] || { fail "boot-storage config is missing: $CONFIG"; exit 1; }
[ -r "$CONFIG" ] || { fail "cannot read $CONFIG"; exit 1; }

configured=0
while read -r source target extra || [ -n "${source:-}${target:-}${extra:-}" ]; do
    case "${source:-}" in ''|'#'*) continue ;; esac
    if [ -z "${target:-}" ] || [ -n "${extra:-}" ]; then
        fail "invalid mapping in $CONFIG; expected: SOURCE TARGET"
        exit 1
    fi
    configured=$((configured + 1))
    if [ "$configured" -gt 1 ]; then
        fail "$CONFIG declares more than one mapping"
        exit 1
    fi
    apply_mapping "$source" "$target" || exit 1
done < "$CONFIG"

[ "$configured" -eq 1 ] || { fail "$CONFIG has no mapping"; exit 1; }
exit 0
BOOTEOF

    if ! commit_boot_file "$NVR_BOOT_STORAGE_HELPER" "$helper_tmp" shell 755 \
            "re:^#!/bin/bash$" \
            "re:^apply_mapping\(\) \{" \
            "mount --bind" \
            "still not mounted after"; then
        log_warn "boot-storage helper NOT replaced; the previous helper is still in place"
        return 1
    fi
    return 0
}

# ExecStartPre runs for every daemon start, not only during boot. This catches a
# lost or wrong bind even when an operator restarts Docker after the one-shot
# boot sequence already completed.
create_systemd_boot_storage_hooks() {
    local dropin dropin_dir dropin_tmp
    for dropin in "$NVR_CONTAINERD_BOOT_STORAGE_DROPIN" "$NVR_DOCKER_BOOT_STORAGE_DROPIN"; do
        dropin_dir="$(dirname "$dropin")"
        if ! mkdir -p "$dropin_dir" 2>/dev/null; then
            log_warn "could not create $dropin_dir; boot-storage hook not refreshed"
            return 1
        fi
        rm -f "$dropin_dir"/.10-dividia-boot-storage.conf.?????? 2>/dev/null || true
        if ! dropin_tmp="$(mktemp "$dropin_dir/.10-dividia-boot-storage.conf.XXXXXX" 2>/dev/null)" \
                || [[ -z "$dropin_tmp" ]]; then
            log_warn "could not create a temp file for $dropin"
            return 1
        fi
        cat > "$dropin_tmp" <<EOF
[Service]
ExecStartPre=$NVR_BOOT_STORAGE_HELPER
EOF
        if ! commit_boot_file "$dropin" "$dropin_tmp" unit 644 \
                "re:^\[Service\]$" \
                "re:^ExecStartPre=$NVR_BOOT_STORAGE_HELPER$"; then
            log_warn "systemd boot-storage hook NOT replaced: $dropin"
            return 1
        fi
    done
    systemctl daemon-reload || return 1
    return 0
}

# SysV does not stop a later service when an earlier S-number fails. Patch the
# Dividia-owned static Docker launcher so the helper can fail dockerd directly.
ensure_sysv_docker_boot_storage_hook() {
    local marker="# dividia-nvr boot-storage hook" init_dir init_tmp expected_call hook_line dockerd_line
    [ -f "$NVR_DOCKER_INIT" ] || {
        log_warn "$NVR_DOCKER_INIT is missing; CentOS 6 boot-storage hook not installed"
        return 1
    }
    grep -q '^DOCKERD=/usr/local/bin/dockerd$' "$NVR_DOCKER_INIT" 2>/dev/null || {
        log_warn "$NVR_DOCKER_INIT is not the Dividia static Docker launcher; boot-storage hook not installed"
        return 1
    }
    expected_call="    \"$NVR_BOOT_STORAGE_HELPER\" || return 1"
    hook_line=$(grep -nF "$expected_call" "$NVR_DOCKER_INIT" 2>/dev/null | head -1 | cut -d: -f1)
    dockerd_line=$(grep -n '^[[:space:]]*\$DOCKERD[[:space:]]' "$NVR_DOCKER_INIT" 2>/dev/null | head -1 | cut -d: -f1)
    if grep -qF "$marker" "$NVR_DOCKER_INIT" 2>/dev/null \
            && [[ -n "$hook_line" && -n "$dockerd_line" && "$hook_line" -lt "$dockerd_line" ]]; then
        return 0
    fi

    init_dir="$(dirname "$NVR_DOCKER_INIT")"
    rm -f "$init_dir"/.docker-dividia-boot-storage.?????? 2>/dev/null || true
    if ! init_tmp="$(mktemp "$init_dir/.docker-dividia-boot-storage.XXXXXX" 2>/dev/null)" \
            || [[ -z "$init_tmp" ]]; then
        log_warn "could not create a temp file next to $NVR_DOCKER_INIT"
        return 1
    fi
    if ! awk -v helper="$NVR_BOOT_STORAGE_HELPER" -v marker="$marker" -v call="$expected_call" '
        $0 == "    " marker || $0 == call { next }
        { print }
        !done && /^start\(\)[[:space:]]*\{/ {
            print "    # dividia-nvr boot-storage hook"
            print "    \"" helper "\" || return 1"
            done=1
        }
        END { if (!done) exit 42 }
    ' "$NVR_DOCKER_INIT" > "$init_tmp"; then
        log_warn "could not insert the boot-storage hook into $NVR_DOCKER_INIT"
        rm -f "$init_tmp"
        return 1
    fi
    if ! commit_boot_file "$NVR_DOCKER_INIT" "$init_tmp" shell 755 \
            "$marker" \
            "re:^[[:space:]]+\"?$NVR_BOOT_STORAGE_HELPER\"?[[:space:]]+\|\|[[:space:]]+return 1$" \
            "DOCKERD=/usr/local/bin/dockerd"; then
        log_warn "CentOS 6 Docker launcher NOT replaced; the previous script is still in place"
        return 1
    fi
    return 0
}

# Install the guard before Docker can start when a migration predeclares the
# VideoStore-backed /opt bind. Normal hosts have no config and take no action.
prepare_configured_boot_storage_guard() {
    [[ -e "$NVR_BOOT_STORAGE_CONFIG" ]] || return 0
    create_boot_storage_container_mask "$NVR_BOOT_STORAGE_CONFIG" || return 1
    create_boot_storage_helper || return 1
    if has_systemd; then
        create_systemd_boot_storage_hooks || return 1
    elif [[ -f "$NVR_DOCKER_INIT" ]] \
            && grep -q '^DOCKERD=/usr/local/bin/dockerd$' "$NVR_DOCKER_INIT" 2>/dev/null; then
        ensure_sysv_docker_boot_storage_hook || return 1
    fi
    # Do not continue an install or update on a configured host until the live
    # bind itself is correct. Hooks protect the next daemon start; this protects
    # the current process when Docker was already running or the bind was lost.
    "$NVR_BOOT_STORAGE_HELPER" || return 1
    return 0
}

create_boot_service() {
    local boot_storage_helper_ready=false
    # Normal fleet hosts do not need another daemon hook. Install these files
    # only for an existing mapping or while the adopt command prepares one.
    if [[ -e "$NVR_BOOT_STORAGE_CONFIG" || "${NVR_BOOT_STORAGE_PREPARE:-}" == "1" ]]; then
        local boot_storage_mapping="${NVR_BOOT_STORAGE_STAGED_CONFIG:-$NVR_BOOT_STORAGE_CONFIG}"
        if [[ -r "$boot_storage_mapping" ]]; then
            create_boot_storage_container_mask "$boot_storage_mapping" || return 1
        fi
        if create_boot_storage_helper; then
            boot_storage_helper_ready=true
        else
            log_warn "boot-storage hooks were not refreshed because the helper is unavailable"
        fi
    fi
    if has_systemd; then
        if [[ "$boot_storage_helper_ready" == "true" ]]; then
            create_systemd_boot_storage_hooks || return 1
        fi
        create_systemd_unit
    else
        if [[ "$boot_storage_helper_ready" == "true" ]]; then
            ensure_sysv_docker_boot_storage_hook || return 1
        fi
        create_sysv_init_script
    fi
    boot_storage_compose_overlay_reconcile || return 1
}

create_systemd_unit() {
    log_step "Creating systemd service"

    # Deliberately NO RequiresMountsFor=/videostore. It looks like the right
    # directive and is the wrong one twice over:
    #   - Scope. It adds Requires= + After= on the mounts needed to REACH the
    #     path, and on a normal box /videostore is a plain directory on root, so
    #     it resolves to `-.mount`. The mounts that matter live BELOW it. Covering
    #     those would mean baking vs1..vsN into the unit at install time, which
    #     goes stale the first time a store is added.
    #   - Coupling. On the one shape where it is NOT inert -- a site that mounts a
    #     disk AT /videostore, which this file handles elsewhere as "a real-disk
    #     ext4/xfs at /videostore" -- Requires= means one dead video disk refuses
    #     to start the whole NVR: no live view, no playback, and no XML-RPC, which
    #     is the surface support needs to repair that very disk. ADR-045 rejects
    #     that trade explicitly. `nofail` on the operator's own line does not
    #     help; it drops the local-fs.target requirement, not another unit's
    #     Requires=.
    # The layer that actually covers /videostore/vsN is ensure_videostore_mounts
    # in the `nvr` CLI, which ExecStart below runs (and which the SysV script
    # carries inline). It has to exist regardless: every managed fstab entry
    # carries `nofail`, and systemd.mount(5) is explicit that nofail removes the
    # ORDERING too -- the mount is only wanted by local-fs.target, not ordered
    # before it -- so boot never waits for a late-enumerating disk.
    local unit_tmp
    # Pre-clean debris from a previous run killed between mktemp and the commit. Anchored
    # to our own template in a root-owned 0755 directory, so this is not the unbounded
    # sweeper pattern that went wrong in the [51203] work.
    rm -f "$(dirname /etc/systemd/system/nvr.service)"/.nvr.service.?????? 2>/dev/null || true
    # A failed mktemp must not abort the installer: install-nvr.sh runs `set -eE`, so an
    # empty var here makes `cat > ""` an ambiguous-redirect error that would kill the
    # whole run. The old `cat > <live path>` had no such failure point, so this guard is
    # what keeps the hardening from being a net regression.
    if ! unit_tmp="$(mktemp "$(dirname /etc/systemd/system/nvr.service)/.nvr.service.XXXXXX" 2>/dev/null)" \
            || [[ -z "$unit_tmp" ]]; then
        log_warn "could not create a temp file next to /etc/systemd/system/nvr.service; keeping the existing unit"
        return 0
    fi
    cat > "$unit_tmp" <<EOF
[Unit]
Description=NVR Docker Stack
After=docker.service
Requires=docker.service

[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory=$INSTALL_DIR
ExecStartPre=/bin/mkdir -p /videostore
ExecStart=$INSTALL_DIR/nvr start
ExecStop=$INSTALL_DIR/nvr stop
TimeoutStartSec=0

[Install]
WantedBy=multi-user.target
EOF

    # Commit or keep the old unit. ExecStart is the marker that matters: without it the
    # unit is syntactically fine and starts nothing.
    # Markers ANCHORED. Unanchored substrings are satisfied by a COMMENTED-OUT unit
    # ("# [Unit]", "# ExecStart=..."), which would replace the live unit with one that
    # starts nothing and then fail systemctl daemon-reload after the damage is done.
    if ! commit_boot_file /etc/systemd/system/nvr.service "$unit_tmp" unit 644 \
            "re:^\\[Unit\\]$" "re:^\\[Service\\]$" "re:^\\[Install\\]$" \
            "re:^ExecStart=$INSTALL_DIR/nvr start$" "re:^WantedBy=multi-user\\.target$"; then
        log_warn "systemd unit NOT replaced; the previous nvr.service is still in place"
        return 0
    fi

    systemctl daemon-reload
    systemctl enable nvr.service

    log_info "Created systemd service: nvr.service"
}

create_sysv_init_script() {
    log_step "Creating SysV init script"

    local init_tmp
    rm -f "$(dirname /etc/init.d/nvr)"/.nvr.?????? 2>/dev/null || true
    if ! init_tmp="$(mktemp "$(dirname /etc/init.d/nvr)/.nvr.XXXXXX" 2>/dev/null)" \
            || [[ -z "$init_tmp" ]]; then
        log_warn "could not create a temp file next to /etc/init.d/nvr; keeping the existing init script"
        return 0
    fi
    cat > "$init_tmp" <<INITEOF
#!/bin/bash
# chkconfig: 2345 96 04
# description: NVR Docker Stack

### BEGIN INIT INFO
# Provides:       nvr
# Required-Start: docker
# Required-Stop:  docker
# Default-Start:  2 3 4 5
# Default-Stop:   0 1 6
# Description:    NVR Docker Stack
### END INIT INFO

PATH=/usr/local/bin:\$PATH
INSTALL_DIR="$INSTALL_DIR"

ensure_docker() {
    # chkconfig orders docker before nvr, but retain a local guard for hosts
    # where service startup raced or Docker was restarted during boot.  Compose
    # otherwise only reports a daemon-connect error and exits successfully
    # from this SysV script, leaving every recorder container offline.
    docker info >/dev/null 2>&1 && return 0

    echo -n "Docker daemon unavailable; starting it... "
    if ! service docker start >/dev/null 2>&1; then
        echo "FAILED"
        return 1
    fi

    local i=0
    while [ \$i -lt 30 ]; do
        if docker info >/dev/null 2>&1; then
            echo "OK"
            return 0
        fi
        sleep 1
        i=\$((i + 1))
    done
    echo "FAILED (timeout)"
    return 1
}

# DO NOT DELETE THIS AS REDUNDANT WITH fstab. Since 2026-07-27 this is the
# PRIMARY VideoStore mounter on CentOS 6, not a retry: the managed entry is
# noauto, so rc.sysinit's mount -a deliberately skips it and nothing else on the
# box mounts a store. The reason is the shadow refusal below -- mount -a knows
# nothing about it and would mount straight over root-backed video, burying
# footage that still fills root while rollover deletes the Event rows pointing
# at it (reproduced on a CentOS 6 box 2026-07-26, 80 MB). Doing the mount here
# is what makes that refusal run at boot. See ADR-045.
#
# CentOS 6 has no systemd, so there is no RequiresMountsFor and no
# fstab-generated .mount unit to order against -- and 61 of the 101 migrated BCC
# Docker NVRs are CentOS 6. Back when the entries were auto, rc.sysinit ran
# mount -a at boot but a late-enumerating SATA/USB/HBA disk could miss that
# window entirely and nothing tried again: /videostore/vsN stayed a plain
# directory on root and mpengine recorded onto the root filesystem
# (2026-07-24). Under noauto that race is gone by construction.
#
# This runs right before the stack comes up, which is what keeps ADR-045's
# mount-before-containers ordering. Each mount is attempted only when its path
# is not already a mount point, so it is a no-op on a healthy boot and cannot
# disturb a store that is already up. The managed entries carry nofail and pass
# 0, so a genuinely absent disk still cannot block a boot.
#
# NOTE: no backticks and no unescaped dollar signs anywhere in this heredoc. The
# INITEOF delimiter is unquoted, so the installing shell expands both -- a
# backtick pair here would EXECUTE on the customer host at install time and
# interpolate its output into /etc/init.d/nvr. Caught while writing this block.
NVR_FSTAB="\${NVR_FSTAB:-/etc/fstab}"
NVR_VIDEOSTORE_ROOT="\${NVR_VIDEOSTORE_ROOT:-/videostore}"
NVR_FSTAB_SENTINEL="\${NVR_FSTAB_SENTINEL:-# dividia-nvr videostore, managed}"
NVR_VIDEOSTORE_SHADOW_LIMIT_BYTES="\${NVR_VIDEOSTORE_SHADOW_LIMIT_BYTES:-67108864}"

ensure_videostore_mounts() {
    [ -d "\$NVR_VIDEOSTORE_ROOT" ] || return 0
    [ -r "\$NVR_FSTAB" ] || return 0

    for mp in "\$NVR_VIDEOSTORE_ROOT"/vs*; do
        [ -d "\$mp" ] || continue
        # OUR entry only, by sentinel. An operator line for the same mount point is
        # theirs; Phase 2 declines to manage it and so must this.
        grep -qF "\$NVR_FSTAB_SENTINEL \$mp" "\$NVR_FSTAB" 2>/dev/null || continue
        if mountpoint -q "\$mp" 2>/dev/null; then continue; fi
        # Refuse to mount over root-backed video: mounting hides bytes that still
        # fill root, and rollover then deletes the Event rows pointing at them.
        # awk in SINGLE quotes, with the field reference escaped. Double quotes let
        # the INSTALLING shell expand it to empty, so the generated line printed
        # plus-zero unconditionally -- always 0, guard silently disabled on every
        # CentOS 6 box. Caught by the behavioral test, the only thing that could
        # catch it: the template reads fine. (And no backticks in this comment,
        # for the reason stated at the top of this heredoc.)
        shadow=\$( du -sx --block-size=1 "\$mp" 2>/dev/null | awk '{print \$1+0; exit}' )
        if [ "\${shadow:-0}" -gt "\$NVR_VIDEOSTORE_SHADOW_LIMIT_BYTES" ]; then
            echo -n "NOT mounting \$mp (holds \${shadow} bytes on ROOT; relocate first) "
            continue
        fi
        echo -n "mounting \$mp... "
        mount "\$mp" >/dev/null 2>&1 || echo -n "FAILED "
    done
}

start() {
    echo -n "Starting NVR Docker Stack: "
    ensure_docker || return 1
    # The host CLI reasserts the fixed-port reservation before any container
    # can open outbound camera sockets. This is required on CentOS 6, whose
    # boot path does not reliably load /etc/sysctl.d drop-ins.
    if [ -x "\$INSTALL_DIR/nvr" ]; then
        "\$INSTALL_DIR/nvr" ensure-host-config >/dev/null 2>&1 || true
    fi
    # Before the writers start, not after: a container that comes up pointed at an
    # unmounted store records to root, and the DB rows it writes name paths that
    # will not resolve once the disk is back.
    ensure_videostore_mounts
    # rda-db's legacy firewall restore on CentOS 6 can remove Docker's custom
    # chains.  Docker 27 then cannot create the compose bridge because old
    # iptables mistakes a missing user chain for an unavailable target.
    # Recreate the empty chains idempotently; dockerd populates their rules.
    iptables -t nat -N DOCKER 2>/dev/null || true
    iptables -N DOCKER 2>/dev/null || true
    iptables -N DOCKER-ISOLATION-STAGE-1 2>/dev/null || true
    iptables -N DOCKER-ISOLATION-STAGE-2 2>/dev/null || true
    # Clean stale containerd/runc state from hard reboot (kernel 4.4 doesn't
    # always clean these up on shutdown)
    rm -rf /var/run/docker/containerd/daemon/io.containerd.runtime.v2.task/moby/* 2>/dev/null
    rm -rf /run/containerd/runc/moby/* 2>/dev/null
    # Remove stale containers so compose up doesn't fail with "already exists"
    cd "\$INSTALL_DIR" && docker compose rm -f 2>/dev/null
    # Reconciled bring-up: the host CLI converges the optional aiengine add-on
    # to host intent and brings CORE services up BY NAME, so a missing add-on
    # image can never fail boot (the same reconcile helper nvr start uses). Fall
    # back to a raw whole-project compose up when the CLI is unavailable.
    if [ -x "\$INSTALL_DIR/nvr" ] && "\$INSTALL_DIR/nvr" boot-up; then
        echo "OK"
    elif docker compose up -d --quiet-pull; then
        echo "OK"
    else
        echo "FAILED"
        return 1
    fi
}

stop() {
    echo -n "Stopping NVR Docker Stack: "
    cd "\$INSTALL_DIR" && docker compose down && echo "OK" || echo "FAILED"
}

status() {
    cd "\$INSTALL_DIR" && docker compose ps
}

case "\$1" in
    start)   start ;;
    stop)    stop ;;
    restart) stop; start ;;
    status)  status ;;
    *)       echo "Usage: \$0 {start|stop|restart|status}"; exit 1 ;;
esac
INITEOF
    # Commit or keep the old script. The markers are the ones whose absence would be
    # SILENT: the mounter call is what makes the noauto fstab entry mean anything on
    # CentOS 6 (ADR-045), and the case dispatch is the last thing in the heredoc, so its
    # presence is what proves the write completed instead of stopping midway.
    if ! commit_boot_file /etc/init.d/nvr "$init_tmp" shell 755 \
            "re:^#!/bin/bash$" \
            "re:^# chkconfig: 2345" \
            "re:^[[:space:]]+ensure_videostore_mounts[[:space:]]*$" \
            "re:^ensure_videostore_mounts\\(\\) \\{" \
            "docker compose up -d" "Usage:"; then
        log_warn "SysV init script NOT replaced; the previous /etc/init.d/nvr is still in place"
        return 0
    fi
    chkconfig nvr on

    log_info "Created SysV init script: /etc/init.d/nvr"
}

wait_for_compose_services_ready() {
    log_step "Waiting for all NVR containers to become ready"

    cd "$INSTALL_DIR"

    # PTZ did not exist in the published 6.2 compose file during the Lubbock
    # migration. Require it when the selected compose bundle defines it, but
    # do not roll a healthy core stack back because an optional service is not
    # part of that channel's bundle.
    local services=(db backend engine playback viewer connector autoheal)
    if docker compose config --services 2>/dev/null | grep -qx ptz; then
        services+=(ptz)
    fi
    local deadline=$((SECONDS + 180))
    local last_status=""

    while [[ "$SECONDS" -lt "$deadline" ]]; do
        local bad=0 starting=0 status_lines=()

        for svc in "${services[@]}"; do
            local cid state health
            cid=$(docker compose ps -q "$svc" 2>/dev/null || true)
            if [[ -z "$cid" ]]; then
                status_lines+=("$svc:missing")
                bad=1
                continue
            fi

            state=$(docker inspect -f '{{.State.Status}}' "$cid" 2>/dev/null || echo unknown)
            health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$cid" 2>/dev/null || echo unknown)
            status_lines+=("$svc:$state/$health")

            if [[ "$state" != "running" ]]; then
                bad=1
            elif [[ "$health" == "starting" ]]; then
                starting=1
            elif [[ "$health" == "unhealthy" || "$health" == "unknown" ]]; then
                bad=1
            fi
        done

        local status
        status="${status_lines[*]}"
        if [[ "$bad" -eq 0 && "$starting" -eq 0 ]]; then
            log_info "All NVR containers ready: $status"
            return 0
        fi
        if [[ "$status" != "$last_status" ]]; then
            log_info "Container readiness: $status"
            last_status="$status"
        fi
        sleep 5
    done

    log_error "NVR containers did not become ready in time"
    docker compose ps || true
    return 1
}

ensure_docker_db_ready() {
    cd "$INSTALL_DIR"

    docker compose up -d db >/dev/null 2>&1 || true

    local waited=0
    while [[ $waited -lt 60 ]]; do
        if docker compose exec -T db sh -c 'mysqladmin -uroot -p"$MYSQL_ROOT_PASSWORD" ping -h 127.0.0.1 --silent' >/dev/null 2>&1; then
            return 0
        fi
        sleep 2
        waited=$((waited + 2))
    done

    log_error "Docker DB did not become reachable within 60s"
    docker compose ps db || true
    return 1
}

mark_restored_sw_paired_update_if_schema_present() {
    cd "$INSTALL_DIR"

    local present_count
    present_count=$(docker compose exec -T db sh -c 'mysql -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -NBe "
SELECT COUNT(*)
FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
  AND (
    (TABLE_NAME = '\''ApexInboundLog'\'' AND COLUMN_NAME IN ('\''sProvider'\'','\''sPairedState'\''))
    OR (TABLE_NAME = '\''PosType19'\'' AND COLUMN_NAME IN ('\''fPairTareLoadout'\'','\''sTareField'\'','\''sTareValue'\'','\''bPairTimeout'\''))
  );
"' 2>/dev/null | tail -1 | tr -d '\r[:space:]') || present_count=0

    if [[ "${present_count:-0}" -eq 6 ]]; then
        # restore_from_backup intentionally keeps the regular backend stopped
        # until the database is restored and migrated.  Use a one-shot
        # container for this marker so it cannot race the restore lifecycle.
        docker compose run --rm --no-deps -T --entrypoint "" backend \
            sh -c 'mkdir -p /var/lib/rda-db && touch /var/lib/rda-db/20260522-sw-paired-tare-loadout'
        log_info "Marked 20260522 SW paired update complete; restored DB already has all paired-mode columns"
    fi
}

# The post-restore update runs in `docker compose run`, which creates a fresh
# one-shot container from the image. A script copied into the stopped regular
# backend container is therefore only a defense for its later normal startup.
# Resolve the restored Event index through the DB container itself, repair an
# older same-name definition, and write the shared rda-db marker before the
# one-shot update can execute the pre-fix image module.
ensure_restored_event_bcamera_index() {
    cd "$INSTALL_DIR"

    local index_columns
    index_columns=$(docker compose exec -T db sh -c \
        'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e "SHOW INDEX FROM Event" 2>/dev/null' \
        2>/dev/null | awk -F '\t' '
            tolower($3) == "bcamera" {
                columns[$4] = tolower($5)
                if ($4 > maximum) maximum = $4
            }
            END {
                for (i = 1; i <= maximum; i++)
                    printf "%s%s", (i > 1 ? "," : ""), columns[i]
            }
        ') || index_columns=""

    if [[ -n "$index_columns" && "$index_columns" != "bcamera,stimestamp" ]]; then
        log_warn "Replacing restored Event bCamera index definition: $index_columns"
        docker compose exec -T db sh -c \
            'mysql -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e \
            "ALTER TABLE Event DROP KEY bCamera, ADD KEY bCamera (bCamera, sTimestamp)"' \
            >/dev/null
        index_columns="bcamera,stimestamp"
    fi

    if [[ "$index_columns" == "bcamera,stimestamp" ]]; then
        docker compose run --rm --no-deps -T --entrypoint "" backend \
            sh -c 'mkdir -p /var/lib/rda-db && touch /var/lib/rda-db/20260804-event-bcamera-index'
        log_info "Marked 20260804 Event bCamera index update complete; restored DB already has the composite key"
    fi
}

patch_backend_sw_update_scripts() {
    cd "$INSTALL_DIR"

    local container tmp_script
    # The regular backend is deliberately stopped during restore. docker cp
    # works on stopped containers, so include all compose states here.
    container=$(docker compose ps -aq backend 2>/dev/null)
    if [[ -z "$container" ]]; then
        log_warn "Backend container does not exist — cannot patch SW update scripts"
        return 0
    fi

    tmp_script=$(mktemp /tmp/nvr-sw-paired-update.XXXXXX.py)
    cat > "$tmp_script" <<'PY'
# Python Update Module

import os
from lib.globals import GB

class UpdateSwPairedTareLoadout20260522:

	_sDescription = 'Add SW paired-mode (tare+loadout merge) config + ApexInboundLog provider/paired-state columns.'

	def __init__( self, sName, oDB ):
		self._sName = sName
		self._oDB = oDB

	def getName( self ):
		return self._sName

	def getDescription( self ):
		return self._sDescription

	def _columnExists( self, sTable, sColumn ):
		rgo = self._oDB.query(
			'SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=%s AND COLUMN_NAME=%s',
			sTable,
			sColumn,
		)
		return len( rgo ) > 0 and int( rgo[0][0] ) > 0

	def _addColumnIfMissing( self, sTable, sColumn, sAlter ):
		if self._columnExists( sTable, sColumn ):
			return
		try:
			self._oDB.query( sAlter )
		except Exception as e:
			if 'Duplicate column name' in str(e):
				return
			raise

	def runUpdate( self ):
		try:
			fVerboseFlag = self._oDB.getVerbose()
			self._oDB.setVerbose( False )

			self._addColumnIfMissing( 'ApexInboundLog', 'sProvider', "ALTER TABLE ApexInboundLog ADD COLUMN sProvider VARCHAR(32) NOT NULL default 'apex'" )

			self._addColumnIfMissing( 'ApexInboundLog', 'sPairedState', "ALTER TABLE ApexInboundLog ADD COLUMN sPairedState LONGBLOB" )

			self._addColumnIfMissing( 'PosType19', 'fPairTareLoadout', "ALTER TABLE PosType19 ADD COLUMN fPairTareLoadout BOOLEAN NOT NULL default '0'" )

			self._addColumnIfMissing( 'PosType19', 'sTareField', "ALTER TABLE PosType19 ADD COLUMN sTareField VARCHAR(32) NOT NULL default 'bTicket'" )

			self._addColumnIfMissing( 'PosType19', 'sTareValue', "ALTER TABLE PosType19 ADD COLUMN sTareValue VARCHAR(32) NOT NULL default '0'" )

			self._addColumnIfMissing( 'PosType19', 'bPairTimeout', "ALTER TABLE PosType19 ADD COLUMN bPairTimeout MEDIUMINT UNSIGNED NOT NULL default '480'" )

		finally:
			self._oDB.setVerbose( fVerboseFlag )
PY

    if docker cp "$tmp_script" "$container:/usr/share/rda-db/setup/scripts/update/20260522-sw-paired-tare-loadout.py" 2>/dev/null; then
        log_info "Patched backend 20260522 SW paired update script for idempotent restore migration"
    else
        log_warn "Could not patch backend SW paired update script; existing image script will run"
    fi
    rm -f "$tmp_script"

    tmp_script=$(mktemp /tmp/nvr-sw-feature-stack-update.XXXXXX.py)
    cat > "$tmp_script" <<'PY'
# Python Update Module

import os
from lib.globals import GB

class UpdateSwFeatureStack20260523:

	_sDescription = 'Add sTicketSource column to PosType19 + create TicketLog generic per-receive archive table.'

	def __init__( self, sName, oDB ):
		self._sName = sName
		self._oDB = oDB

	def getName( self ):
		return self._sName

	def getDescription( self ):
		return self._sDescription

	def _columnExists( self, sTable, sColumn ):
		rgo = self._oDB.query(
			'SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=%s AND COLUMN_NAME=%s',
			sTable,
			sColumn,
		)
		return len( rgo ) > 0 and int( rgo[0][0] ) > 0

	def _tableExists( self, sTable ):
		rgo = self._oDB.query(
			'SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=%s',
			sTable,
		)
		return len( rgo ) > 0 and int( rgo[0][0] ) > 0

	def _addColumnIfMissing( self, sTable, sColumn, sAlter ):
		if self._columnExists( sTable, sColumn ):
			return
		try:
			self._oDB.query( sAlter )
		except Exception as e:
			if 'Duplicate column name' in str(e):
				return
			raise

	def _createTableIfMissing( self, sTable, sCreate ):
		if self._tableExists( sTable ):
			return
		try:
			self._oDB.query( sCreate )
		except Exception as e:
			if 'already exists' in str(e):
				return
			raise

	def runUpdate( self ):
		try:
			fVerboseFlag = self._oDB.getVerbose()
			self._oDB.setVerbose( False )

			self._addColumnIfMissing( 'PosType19', 'sTicketSource', "ALTER TABLE PosType19 ADD COLUMN sTicketSource VARCHAR(16) NOT NULL default 'apex'" )

			self._createTableIfMissing(
				'TicketLog',
					'CREATE TABLE TicketLog ('
					' bID BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,'
					' bPos BIGINT UNSIGNED NOT NULL default 0,'
					" sTimestamp VARCHAR(14) NOT NULL default '00000000000000',"
					" sProvider VARCHAR(32) NOT NULL default 'apex',"
					" sTicket VARCHAR(64) NOT NULL default '0',"
					" sUniqueID VARCHAR(64) NOT NULL default '',"
					' sPayload LONGTEXT,'
					' sError TEXT,'
					' PRIMARY KEY (bID),'
					' INDEX idx_pos_timestamp (bPos, sTimestamp),'
					' INDEX idx_unique_id (sUniqueID),'
					' INDEX idx_provider (sProvider)'
					') ENGINE=MyISAM DEFAULT CHARSET=latin1'
			)

		finally:
			self._oDB.setVerbose( fVerboseFlag )
PY

    if docker cp "$tmp_script" "$container:/usr/share/rda-db/setup/scripts/update/20260523-sw-feature-stack.py" 2>/dev/null; then
        log_info "Patched backend 20260523 SW feature-stack update script for idempotent restore migration"
    else
        log_warn "Could not patch backend SW feature-stack update script; existing image script will run"
    fi
    rm -f "$tmp_script"

    tmp_script=$(mktemp /tmp/nvr-sw-globalize-paired-update.XXXXXX.py)
    cat > "$tmp_script" <<'PY'
# Python Update Module

import os
from lib.globals import GB

class UpdateSwGlobalizePairedToggle20260602:

	_sDescription = 'Promote paired-mode toggle from per-POS PosType19.fPairTareLoadout to NVR-wide Misc(system, paired-tare-loadout). OR-merges existing per-POS values into the new system row before dropping the column.'

	def __init__( self, sName, oDB ):
		self._sName = sName
		self._oDB = oDB

	def getName( self ):
		return self._sName

	def getDescription( self ):
		return self._sDescription

	def _dropColumnIfPresent( self, sTable, sColumn ):
		rgoCol = self._oDB.query(
			'SHOW columns FROM %s WHERE field=%%s' % sTable,
			sColumn
		)
		if len( rgoCol ) == 0:
			return
		try:
			self._oDB.query( 'ALTER TABLE %s DROP COLUMN %s' % ( sTable, sColumn ) )
		except Exception as e:
			if "Can't DROP COLUMN" in str(e) or "check that it exists" in str(e):
				return
			raise

	def runUpdate( self ):
		try:
			fVerboseFlag = self._oDB.getVerbose()
			self._oDB.setVerbose( False )

			rgo = self._oDB.query(
				'SELECT sValue FROM Misc WHERE sModule=%s AND sName=%s',
				'system', 'paired-tare-loadout'
			)
			if len( rgo ) == 0:
				rgoCol = self._oDB.query(
					'SHOW columns FROM PosType19 WHERE field=%s',
					'fPairTareLoadout'
				)
				sInitialValue = 'off'
				if len( rgoCol ) != 0:
					rgoAny = self._oDB.query(
						'SELECT COUNT(*) AS bCount FROM PosType19 WHERE fPairTareLoadout=1'
					)
					if len( rgoAny ) > 0:
						oRow = rgoAny[0]
						bCount = oRow.get( 'bCount', 0 ) if hasattr( oRow, 'get' ) else oRow[0]
						if bCount and int( bCount ) > 0:
							sInitialValue = 'on'

				self._oDB.query(
					'INSERT IGNORE INTO Misc (sModule, sName, sValue) VALUES (%s, %s, %s)',
					'system', 'paired-tare-loadout', sInitialValue
				)

			self._dropColumnIfPresent( 'PosType19', 'fPairTareLoadout' )

		finally:
			self._oDB.setVerbose( fVerboseFlag )
PY

    if docker cp "$tmp_script" "$container:/usr/share/rda-db/setup/scripts/update/20260602-sw-globalize-paired-toggle.py" 2>/dev/null; then
        log_info "Patched backend 20260602 SW globalize-paired update script for idempotent restore migration"
    else
        log_warn "Could not patch backend SW globalize-paired update script; existing image script will run"
    fi
    rm -f "$tmp_script"

    tmp_script=$(mktemp /tmp/nvr-event-bcamera-index-update.XXXXXX.py)
    cat > "$tmp_script" <<'PY'
# Python Update Module

class UpdateEventBcameraIndex20260804:

	_sDescription = 'Add KEY (bCamera, sTimestamp) to Event.'

	def __init__( self, sName, oDB ):
		self._sName = sName
		self._oDB = oDB

	def getName( self ):
		return self._sName

	def getDescription( self ):
		return self._sDescription

	def _indexColumns( self ):
		rgo = self._oDB.query( 'SHOW INDEX FROM Event' )
		rgoColumns = []
		for oRow in rgo:
			if hasattr( oRow, 'get' ):
				sKey = oRow.get( 'Key_name', oRow.get( 'key_name', '' ) )
				bSequence = oRow.get( 'Seq_in_index', oRow.get( 'seq_in_index', 0 ) )
				sValue = oRow.get( 'Column_name', oRow.get( 'column_name', '' ) )
			else:
				sKey = oRow[2]
				bSequence = oRow[3]
				sValue = oRow[4]
			try:
				sKey = sKey.decode( 'utf-8' )
				sValue = sValue.decode( 'utf-8' )
			except ( AttributeError, UnicodeDecodeError ):
				pass
			if sKey.lower() == 'bcamera':
				rgoColumns.append( ( int( bSequence ), sValue.lower() ) )
		rgoColumns.sort()
		return [ sValue for bSequence, sValue in rgoColumns ]

	def runUpdate( self ):
		try:
			fVerboseFlag = self._oDB.getVerbose()
			self._oDB.setVerbose( False )
			rgsColumns = self._indexColumns()
			if rgsColumns == [ 'bcamera', 'stimestamp' ]:
				return
			if len( rgsColumns ) != 0:
				sAlter = 'ALTER TABLE Event DROP KEY bCamera, ADD KEY bCamera ( bCamera, sTimestamp )'
			else:
				sAlter = 'ALTER TABLE Event ADD KEY bCamera ( bCamera, sTimestamp )'
			try:
				self._oDB.query( sAlter )
			except Exception as e:
				if 'Duplicate key name' not in str( e ) or self._indexColumns() != [ 'bcamera', 'stimestamp' ]:
					raise
		finally:
			self._oDB.setVerbose( fVerboseFlag )
PY

    if docker cp "$tmp_script" "$container:/usr/share/rda-db/setup/scripts/update/20260804-event-bcamera-index.py" 2>/dev/null; then
        log_info "Patched backend 20260804 Event bCamera index update for case-safe restore migration"
    else
        log_warn "Could not patch backend Event bCamera index update; existing image script will run"
    fi
    rm -f "$tmp_script"
}

# The migration image may still contain the pre-fix 20260602 update module.
# Seed its destination row before the one-shot update so that an old module's
# SELECT/INSERT race cannot abort the entire migration. INSERT IGNORE keeps
# this safe when the restored backup already contains the row or when a newer
# image has already applied the update.
ensure_sw_globalize_paired_misc_row() {
    cd "$INSTALL_DIR"

    local pos_type19_present pair_column_present paired_value=off
    pos_type19_present=$(docker compose exec -T db sh -c \
        'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e \
        "SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=\"PosType19\"" 2>/dev/null' \
        2>/dev/null | tail -1 | tr -d '\r[:space:]') || pos_type19_present=0

    if [[ "${pos_type19_present:-0}" != "1" ]]; then
        log_info "SW paired-row seed skipped: PosType19 is not present"
        return 0
    fi

    pair_column_present=$(docker compose exec -T db sh -c \
        'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e \
        "SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=\"PosType19\" AND COLUMN_NAME=\"fPairTareLoadout\"" 2>/dev/null' \
        2>/dev/null | tail -1 | tr -d '\r[:space:]') || pair_column_present=0
    if [[ "${pair_column_present:-0}" == "1" ]]; then
        local paired_count
        paired_count=$(docker compose exec -T db sh -c \
            'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e \
            "SELECT COUNT(*) FROM PosType19 WHERE fPairTareLoadout=1" 2>/dev/null' \
            2>/dev/null | tail -1 | tr -d '\r[:space:]') || paired_count=0
        [[ "${paired_count:-0}" != "0" ]] && paired_value=on
    fi

    docker compose exec -T db sh -c \
        "mysql -uroot -p\"\$MYSQL_ROOT_PASSWORD\" dtech -e \"INSERT IGNORE INTO Misc (sModule, sName, sValue) VALUES ('system', 'paired-tare-loadout', '$paired_value')\"" \
        >/dev/null
    log_info "Seeded SW paired Misc row before database migrations"
}

finalize_migrated_stack() {
    log_step "Finalizing Docker service startup"

    cd "$INSTALL_DIR"

    # Stop compose-managed containers before restarting Docker. On CentOS 6
    # static Docker, restarting dockerd while containers are running can leave
    # stale containerd task/shim state that makes healthcheck execs fail with a
    # missing log.json until the daemon is hard-restarted.
    docker compose down --remove-orphans 2>/dev/null || true

    if has_systemd; then
        # RPM uninstall scripts and restored firewall rules can leave Docker's
        # iptables chains stale after the migration stack was first verified.
        # Restart Docker once after legacy cleanup, then start the installed
        # boot service and verify final runtime state before reporting success.
        log_info "Restarting Docker after RPM cleanup/firewall restore"
        systemctl restart docker
        sleep 3

        systemctl reset-failed nvr.service 2>/dev/null || true
        if ! systemctl start nvr.service; then
            log_warn "nvr.service did not start cleanly on first attempt; waiting for backend and retrying"
            wait_for_backend_healthy || true
            systemctl reset-failed nvr.service 2>/dev/null || true
            systemctl start nvr.service
        fi
    else
        service docker restart 2>/dev/null || true
        sleep 3
        service nvr start
    fi

    wait_for_backend_healthy
    # Reconciled bring-up so an adopted aiengine add-on comes up isolated from
    # core (core by name; a missing add-on image cannot fail this belt). Raw
    # fallback preserves behavior when the CLI is unavailable.
    if [[ -x "$INSTALL_DIR/nvr" ]]; then
        "$INSTALL_DIR/nvr" boot-up || docker compose up -d --quiet-pull
    else
        docker compose up -d --quiet-pull
    fi
    wait_for_compose_services_ready
}

################################################################################
# Management Scripts
################################################################################

create_management_scripts() {
    log_step "Installing management CLI"

    # Extract host management tools and the RO forced-command wrapper from the
    # backend image. The wrapper lands at /usr/local/bin/nvr-ro-wrap on the VM
    # host filesystem so sshd's authorized_keys `command="..."` entry can
    # exec it when the customer-readable RO key connects.
    local image
    image=$(cd "$INSTALL_DIR" && docker compose config --images 2>/dev/null | grep backend | head -1)
    if [[ -n "$image" ]]; then
        local cid
        cid=$(docker create "$image" 2>/dev/null) || true
        if [[ -n "$cid" ]]; then
            docker cp "$cid:/usr/share/nvr/bin/nvr" "$INSTALL_DIR/nvr" 2>/dev/null || true
            docker cp "$cid:/usr/share/nvr/bin/install-nvr.sh" "$INSTALL_DIR/install-nvr.sh" 2>/dev/null || true
            docker cp "$cid:/usr/share/nvr/bin/host-dview" "$INSTALL_DIR/host-dview" 2>/dev/null || true
            docker cp "$cid:/usr/local/bin/nvr-ro-wrap" "/usr/local/bin/nvr-ro-wrap" 2>/dev/null || true
            docker rm "$cid" > /dev/null
        fi
    fi

    chmod 555 "$INSTALL_DIR/nvr" "$INSTALL_DIR/install-nvr.sh" "$INSTALL_DIR/host-dview" 2>/dev/null || true
    chmod 555 /usr/local/bin/nvr-ro-wrap 2>/dev/null || true
    log_info "Installed nvr CLI to $INSTALL_DIR/nvr"
    [ -x /usr/local/bin/nvr-ro-wrap ] && log_info "Installed RO wrapper to /usr/local/bin/nvr-ro-wrap"
}

################################################################################
# Image-prune cron
################################################################################

install_prune_cron() {
    log_step "Installing daily Docker image prune cron"

    cat > /etc/cron.d/dividia-docker-prune <<'CRON'
# Daily Docker image prune for Dividia NVR.
#
# Reclaims disk from images that have been replaced (by watchtower auto-pull,
# by `nvr update`, by `nvr channel <new>` + manual `docker compose pull`, or
# by ad-hoc operator pulls). Two passes:
#   - dangling (no -a): immediately removes <none>:<none> images that were
#     replaced. Safe by definition; nothing references them.
#   - 168h time-filtered (-a): catches still-tagged images that haven't
#     been used in a week.
# Without the dangling pass, active dev iteration on dev/dev-* channels
# accumulates GB of dangling images that the time filter holds for a week.
#
# Owned by /usr/share/nvr/bin/nvr ensure_prune_cron; do not edit by hand.
SHELL=/bin/sh
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
30 3 * * * root (/usr/bin/docker image prune -f; /usr/bin/docker image prune -a -f --filter "until=168h") >/var/log/dividia-docker-prune.log 2>&1
CRON
    chmod 0644 /etc/cron.d/dividia-docker-prune
    log_info "  Cron: /etc/cron.d/dividia-docker-prune (03:30 daily)"
}

################################################################################
# nvr update cron
################################################################################
#
# Replaces watchtower auto-pull (pitfall_watchtower_strips_compose_labels).
# Mirrors `ensure_update_cron` in docker/nvr — both scripts MUST write the
# same cron body or test_update_cron_contract.sh fails. Inline duplication
# matches the install_prune_cron / ensure_prune_cron precedent.

install_update_cron() {
    log_step "Installing daily nvr update cron"

    # Compute a stable per-host minute jitter in [0,59] so the fleet
    # doesn't all hit DockerHub at the same instant. Same algorithm as
    # nvr CLI's update_cron_jitter() — duplicated rather than shared
    # because install-nvr.sh runs before the nvr CLI is on disk.
    local jitter id=""
    local dvs_conf="$DATA_DIR/config/dvs.conf"
    if [[ -f "$dvs_conf" ]]; then
        id=$(grep -E '^ID=' "$dvs_conf" 2>/dev/null \
             | head -1 \
             | sed -E 's/^ID="?([^"]*)"?.*$/\1/' \
             | tr -d '[:space:]')
    fi
    if [[ "$id" =~ ^[0-9]+$ ]] && [[ "$id" -gt 0 ]]; then
        jitter=$(( id % 60 ))
    else
        jitter=$(hostname 2>/dev/null | cksum 2>/dev/null | awk '{print $1 % 60}')
        [[ -z "$jitter" ]] && jitter=0
    fi

    # Non-quoted heredoc so $jitter interpolates. Body MUST match nvr
    # CLI's ensure_update_cron output character-for-character (modulo the
    # jitter value, which the contract test ignores).
    cat > /etc/cron.d/dividia-nvr-update <<EOF
# Daily nvr update for Dividia NVR.
#
# Replaces watchtower auto-pull (containrrr/watchtower 1.7.1, unmaintained
# since 2024-01 and known to strip com.docker.compose.project labels
# during recreate — see [[pitfall-watchtower-strips-compose-labels]]).
#
# nvr update is the right loop: self-heals orphaned compose project
# labels, pulls images, extracts updated compose + CLI from the new
# backend image, calls compose up -d, prunes obsolete images, then
# removes the watchtower container at the tail of a successful run.
#
# Schedule: 02:NN host-local, where NN = bSerial % 60 from dvs.conf.
# Spreads the fleet across the 02:00-02:59 hour so DockerHub doesn't
# get a thundering herd. Concurrent operator-typed and cron-fired
# invocations serialize via flock inside cmd_update.
#
# Owned by /usr/share/nvr/bin/nvr ensure_update_cron; do not edit by hand.
SHELL=/bin/sh
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
$jitter 2 * * * root /opt/dividia/nvr update >/var/log/dividia-nvr-update.log 2>&1
EOF
    chmod 0644 /etc/cron.d/dividia-nvr-update
    log_info "  Cron: /etc/cron.d/dividia-nvr-update (02:${jitter} daily host-local)"
}

################################################################################
# User Access
################################################################################

configure_user_access() {
    log_step "Configuring user access"

    # Add /opt/dividia to PATH for all users via profile.d
    cat > /etc/profile.d/nvr.sh <<'PROFILE'
# NVR management CLI
if [ -d /opt/dividia ]; then
    PATH="/opt/dividia:$PATH"
fi
PROFILE
    chmod 644 /etc/profile.d/nvr.sh
    log_info "Added $INSTALL_DIR to PATH via /etc/profile.d/nvr.sh"

    # Add dividia user to docker group for passwordless docker access
    if id "dividia" &>/dev/null; then
        usermod -aG docker dividia
        log_info "Added dividia user to docker group"

        # Passwordless sudo for dividia user (NVR management)
        cat > /etc/sudoers.d/dividia <<'SUDOEOF'
Defaults:dividia !requiretty
dividia ALL=(ALL) NOPASSWD: ALL
SUDOEOF
        chmod 440 /etc/sudoers.d/dividia
        log_info "Added dividia to sudoers"

        # Merge root's SSH authorized_keys into dividia's (dedup by exact line).
        # CentOS/Ubuntu path: root has the develop tech support key — needed for
        # passwordless SSH to all NVRs. Windows Hyper-V path: cloud-init puts
        # the Windows host key in dividia's authorized_keys directly. Merging
        # preserves both so tech support can reach either install type.
        if [[ -f /root/.ssh/authorized_keys ]]; then
            mkdir -p /home/dividia/.ssh
            touch /home/dividia/.ssh/authorized_keys
            # Pass files directly to awk instead of piping through `cat`. When
            # one of the input files lacks a trailing newline, `cat` splices
            # its last line onto the next file's first line, producing a single
            # mashed key line that sshd rejects. awk treats each file's EOF as
            # a record boundary, so this is safe regardless of trailing-newline
            # state.
            awk 'NF && !seen[$0]++' \
                /home/dividia/.ssh/authorized_keys \
                /root/.ssh/authorized_keys \
                > /home/dividia/.ssh/authorized_keys.new
            mv /home/dividia/.ssh/authorized_keys.new /home/dividia/.ssh/authorized_keys
            chown -R dividia:dividia /home/dividia/.ssh
            chmod 700 /home/dividia/.ssh
            chmod 600 /home/dividia/.ssh/authorized_keys
            log_info "Merged root's SSH authorized_keys into dividia"
        fi

        # Sync dividia's password to root's. Migrations from old RPM NVRs
        # often inherit a pre-2024 dividia user with a password nobody
        # remembers; techs know root's password (shared) and expect dividia
        # to accept the same one.
        #
        # TRADEOFF: if a tech intentionally set a different dividia password
        # AFTER the initial install, re-running install-nvr.sh (e.g. for an
        # --upgrade or --migrate) will clobber it. install-nvr.sh isn't run
        # on steady-state (nvr update uses docker-compose only), so this is
        # bounded to explicit maintenance invocations. If you need a distinct
        # dividia password that survives reinstalls, set it AFTER the last
        # install-nvr.sh run in the NVR's lifecycle.
        local root_hash
        root_hash=$(getent shadow root | cut -d: -f2)
        if [[ -n "$root_hash" && "$root_hash" != "!"* && "$root_hash" != "*" ]]; then
            # Surface the clobber at runtime so an operator who set a distinct
            # dividia password post-install notices before they lose it.
            log_warn "Resetting dividia password to match root (any distinct password set earlier is lost)"
            usermod -p "$root_hash" dividia
            log_info "Copied root password hash to dividia"
        fi
    fi
}

################################################################################
# Upgrade Detection
################################################################################

detect_existing_install() {
    log_step "Checking for existing NVR backup on VideoStore"

    # Find vs1 partition
    local vs_dev
    vs_dev=$(blkid -l -t LABEL=vs1 -o device 2>/dev/null || true)
    if [[ -z "$vs_dev" ]]; then
        log_info "No VideoStore partition (vs1) found"
        return 1
    fi

    # Mount temporarily if not already mounted
    local vs_mount=""
    local did_mount=false
    vs_mount=$(findmnt -rn -o TARGET -S LABEL=vs1 2>/dev/null || true)
    if [[ -z "$vs_mount" ]]; then
        vs_mount="/tmp/vs1-detect"
        mkdir -p "$vs_mount"
        if ! mount -o ro "$vs_dev" "$vs_mount" 2>/dev/null; then
            log_info "Could not mount $vs_dev"
            rmdir "$vs_mount" 2>/dev/null || true
            return 1
        fi
        did_mount=true
    fi

    # Look for backup directories (not .restored)
    local backup_found=false
    if [[ -d "$vs_mount/backups" ]]; then
        for dir in "$vs_mount"/backups/[0-9]*; do
            [[ -d "$dir" ]] || continue
            # Skip .restored directories
            [[ "$dir" == *.restored ]] && continue
            # Verify it has a conf directory (valid backup)
            if [[ -d "$dir/conf" ]]; then
                UPGRADE_BACKUP_DIR="$dir"
                backup_found=true
                break
            fi
        done
    fi

    # Check for saved .env
    local env_saved=false
    if [[ -f "$vs_mount/backups/.env.save" ]]; then
        env_saved=true
    fi

    if $did_mount; then
        umount "$vs_mount" 2>/dev/null || true
        rmdir "$vs_mount" 2>/dev/null || true
    fi

    if $backup_found; then
        log_info "Found backup: $UPGRADE_BACKUP_DIR"
        [[ "$env_saved" == "true" ]] && log_info "Found saved .env file"
        return 0
    fi

    log_info "No backup found on VideoStore"
    return 1
}

restore_saved_env() {
    log_step "Restoring saved .env from VideoStore"

    local vs_mount
    vs_mount=$(findmnt -rn -o TARGET -S LABEL=vs1 2>/dev/null || true)
    if [[ -z "$vs_mount" ]]; then
        # VideoStore might not be mounted yet — try /videostore/vs1
        vs_mount="/videostore/vs1"
    fi

    if [[ -f "$vs_mount/backups/.env.save" ]]; then
        # Restore .env but preserve any channel override from command line
        local saved_env="$vs_mount/backups/.env.save"
        local current_channel="$CHANNEL"

        cp "$saved_env" "$INSTALL_DIR/.env"
        chown dividia:docker "$INSTALL_DIR/.env" 2>/dev/null || true
        chmod 640 "$INSTALL_DIR/.env"

        # If channel was explicitly passed, override what was saved
        if [[ "$CHANNEL_EXPLICIT" == "true" ]]; then
            sed -i "s/^CHANNEL=.*/CHANNEL=$current_channel/" "$INSTALL_DIR/.env"
        fi

        # Normalize the restored .env for the aiengine add-on: a saved .env can
        # carry an aiengine overlay reference or AIENGINE_* onto a box whose
        # intent or credential files are missing, which would re-enable a broken
        # container. `nvr normalize-addon-env` recomputes COMPOSE_FILE from
        # intent and strips AIENGINE_* when the add-on is not fully present.
        # Guarded: never let a normalize hiccup abort the restore.
        if [[ -x "$INSTALL_DIR/nvr" ]]; then
            "$INSTALL_DIR/nvr" normalize-addon-env || log_warn "aiengine .env normalize skipped (nvr normalize-addon-env failed)"
        fi

        log_info "Restored .env from backup"
    else
        log_info "No saved .env found, using freshly created one"
    fi
}

wait_for_backend_healthy() {
    log_step "Waiting for backend to become healthy"

    local max_wait=120
    local waited=0
    while [[ $waited -lt $max_wait ]]; do
        local health
        health=$(docker compose -f "$INSTALL_DIR/docker-compose.yml" ps backend --format '{{.Health}}' 2>/dev/null || true)
        if [[ "$health" == "healthy" ]]; then
            log_info "Backend is healthy"
            return 0
        fi

        # Also check if the container is running at all
        local state
        state=$(docker compose -f "$INSTALL_DIR/docker-compose.yml" ps backend --format '{{.State}}' 2>/dev/null || true)
        if [[ "$state" == "exited" ]] || [[ "$state" == "dead" ]]; then
            log_error "Backend container exited unexpectedly"
            return 1
        fi

        sleep 5
        waited=$((waited + 5))
        [[ $((waited % 15)) -eq 0 ]] && log_info "Still waiting... ($waited/${max_wait}s)"
    done

    log_warn "Backend did not become healthy within ${max_wait}s (proceeding anyway)"
    return 0
}

################################################################################
# Video Device Preparation
################################################################################

# Scan blkid for any vs[N]-labeled partitions on the host. Returns 0 and sets
# DETECTED_VS_LABELS=" vs1 vs2 ..." if any are found; returns 1 (no labels)
# otherwise.
#
# Used by check_video_device_or_fail to allow kickstart-driven fresh installs
# (which pre-label vs1 at OS-install time via anaconda/curtin) to proceed
# without --video-device. Without this auto-detect bypass, the fail-closed
# gate below would break every kickstart install.
detect_existing_vs_labels() {
    local n found=""
    # Refresh blkid cache so labels written between OS install and this
    # script's invocation are visible.
    blkid -g 2>/dev/null || true
    for n in 1 2 3 4 5 6 7 8 9; do
        if blkid -L "vs$n" >/dev/null 2>&1; then
            found="$found vs$n"
        fi
    done
    DETECTED_VS_LABELS="${found# }"
    [[ -n "$DETECTED_VS_LABELS" ]]
}

# Fail-closed gate for fresh installs. ONLY called from fresh_install_flow —
# upgrade_flow and migrate_flow preserve the prior VideoStore configuration
# via the database and have no need for an install-time gate.
#
# Three paths are valid:
#   1. --video-device <path> — operator explicitly designates a drive
#   2. --no-video-device      — operator opts into directory fallback
#   3. Pre-labeled vs[N] partition exists (kickstart path) — auto-detect
#
# Anything else exits non-zero with an explicit error naming both flags.
# This closes the silent-root-fallback footgun where a manual operator
# forgets --video-device on a fresh disk with no pre-labeled vs[N] and
# recordings overflow the root partition within hours.
check_video_device_or_fail() {
    if [[ -n "$VIDEO_DEVICE" ]]; then
        return 0
    fi
    if [[ "$NO_VIDEO_DEVICE" == "true" ]]; then
        return 0
    fi
    if detect_existing_vs_labels; then
        log_info "Detected pre-labeled VideoStore partition(s): $DETECTED_VS_LABELS"
        log_info "Backend Phase 1 will discover and mount on first startup"
        return 0
    fi

    cat >&2 <<'EOF'

ERROR: No VideoStore configured.

  No vs[N]-labeled partitions found on this host (checked vs1..vs9
  via blkid). install-nvr.sh requires one of the following:

    --video-device /dev/sdX       Dedicated drive (production).
                                  Script will partition, format, label vs1.

    --no-video-device             Directory-only fallback (dev/test only).
                                  Recordings go to root; will fill within
                                  hours of sustained motion.

  If this hardware was provisioned via kickstart, the vs[N] label should
  already exist. If `blkid -L vs1` returns nothing, the kickstart
  partition step likely failed — check OS install logs before retrying.

EOF
    exit 2
}

prepare_video_device() {
    log_step "Preparing video device: $VIDEO_DEVICE"

    # Validate it's a block device
    if [[ ! -b "$VIDEO_DEVICE" ]]; then
        log_error "$VIDEO_DEVICE is not a block device"
        exit 1
    fi

    local PART_DEV="$VIDEO_DEVICE"

    # If raw disk (no trailing digit), check for existing partition first
    if [[ ! "$VIDEO_DEVICE" =~ [0-9]$ ]]; then
        PART_DEV="${VIDEO_DEVICE}1"
        if [[ -b "$PART_DEV" ]]; then
            log_info "Existing partition $PART_DEV found — skipping partitioning"
        else
            log_info "Partitioning $VIDEO_DEVICE..."
            if command -v parted &>/dev/null; then
                parted "$VIDEO_DEVICE" --script -- mktable gpt
                parted "$VIDEO_DEVICE" --script -- mkpart primary 0% 100%
            else
                # CentOS 6 may lack parted — fall back to fdisk (MBR)
                echo -e "n\np\n1\n\n\nw" | fdisk "$VIDEO_DEVICE" || true
            fi

            # Wait for partition to appear
            local WAIT=0
            while [[ ! -b "$PART_DEV" ]] && [[ $WAIT -lt 10 ]]; do
                sleep 1
                WAIT=$((WAIT + 1))
            done
            if [[ ! -b "$PART_DEV" ]]; then
                log_error "Partition $PART_DEV did not appear after partitioning"
                exit 1
            fi
        fi
    fi

    # Check existing label. Three cases:
    #   1. Already labeled vs1 — skip format (existing behavior; partition is
    #      already a VideoStore in the expected state).
    #   2. Labeled vs[N] for N != 1 — refuse. This disk holds another
    #      VideoStore's recordings. Operator typo against the wrong /dev/sdX
    #      would destroy that data. Operator must clear the label first
    #      (`e2label DEV ""`) to acknowledge they are intentionally
    #      repurposing the disk.
    #   3. Any other label or no label — proceed with format + vs1 label.
    local EXISTING_LABEL
    EXISTING_LABEL=$(blkid -s LABEL -o value "$PART_DEV" 2>/dev/null || true)
    if [[ "$EXISTING_LABEL" == "vs1" ]]; then
        log_info "$PART_DEV already labeled vs1 — skipping format"
    elif [[ "$EXISTING_LABEL" =~ ^vs[0-9]+$ ]]; then
        log_error "$PART_DEV is already a VideoStore (label: $EXISTING_LABEL)"
        log_error "Re-formatting would destroy any recordings on that disk."
        log_error "If this is intentional, clear the label first:"
        log_error "    e2label $PART_DEV \"\""
        log_error "then re-run this installer."
        exit 1
    else
        log_info "Formatting $PART_DEV with ext4 (label: vs1)..."
        mkfs.ext4 -L vs1 "$PART_DEV" -E lazy_itable_init 1>/dev/null 2>/dev/null
        tune2fs -c 0 -i 0 "$PART_DEV" 1>/dev/null 2>/dev/null
        log_info "Formatted and tuned $PART_DEV"
    fi

    # Ensure /videostore exists on the host — Docker bind mounts require the
    # source path to exist.  The backend container handles the actual drive
    # mount via nsenter on startup.
    mkdir -p /videostore

    log_info "Video device ready: $PART_DEV (label: vs1)"
    log_info "The backend container will auto-detect and mount it on startup"
}

seed_nvr_id() {
    local dvs_conf="$DATA_DIR/config/dvs.conf"
    local nvr_id="${NVR_ID:-0}"

    if [[ -f "$dvs_conf" ]]; then
        # Update existing dvs.conf with the requested ID
        if [[ "$nvr_id" != "0" ]]; then
            sed -i "s/^ID=.*/ID=$nvr_id/" "$dvs_conf"
            log_info "Updated NVR ID to $nvr_id in existing dvs.conf"
        fi
    else
        # Create dvs.conf with all required defaults
        cat > "$dvs_conf" <<EOF
ID=$nvr_id
INSTALLTYPE=default
CONF=/usr/local/etc
CAMSERV=/rda/work/camserv
DEVICE=VideoStore
NUMCAMS=4
VERSION=-1
KEY=0
POSLOCK=0
LPRLOCK=0
EOF
        log_info "Created dvs.conf with ID=$nvr_id"
    fi

    # Set hostname on the host to match the NVR ID
    if [[ "$nvr_id" != "0" ]]; then
        local new_hostname="cs${nvr_id}.dividia.net"
        if command -v hostnamectl &>/dev/null; then
            hostnamectl set-hostname "$new_hostname" 2>/dev/null || true
        else
            hostname "$new_hostname" 2>/dev/null || true
            echo "HOSTNAME=$new_hostname" >> /etc/sysconfig/network 2>/dev/null || true
        fi
        log_info "Set hostname to $new_hostname"
    fi
}

################################################################################
# RPM Migration
################################################################################

detect_rpm_install() {
    local count=0
    if has_systemd; then
        for svc in rda-backend.service mpengine.service recorder.service; do
            [[ -f "/etc/systemd/system/$svc" ]] || [[ -f "/usr/lib/systemd/system/$svc" ]] && count=$((count + 1))
        done
    else
        for svc in rda-backend mpengine recorder; do
            [[ -f "/etc/init.d/$svc" ]] && count=$((count + 1))
        done
    fi
    [[ $count -ge 2 ]]
}

# RPM NVR installs binaries to /usr/local/bin which may not be in sudo's
# secure_path. Ensure it's available for mpengine, rda-db, etc.
ensure_rpm_path() {
    case ":$PATH:" in
        *:/usr/local/bin:*) ;;
        *) export PATH="/usr/local/bin:$PATH" ;;
    esac
}

capture_rpm_seed() {
    log_step "Capturing RPM hardware seed"

    if command -v mpengine &>/dev/null; then
        RPM_SEED=$(mpengine -S 2>/dev/null | tr -d '[:space:]')
        if [[ -n "$RPM_SEED" ]]; then
            log_info "Captured hardware seed: $RPM_SEED"
            return 0
        fi
    fi

    log_warn "Could not capture hardware seed (mpengine not available)"
    log_warn "Product key will need to be re-entered after migration"
    return 0
}

# Docker's backend initializes an empty database from data/config/dvs.conf on
# its first start.  During an RPM migration that first start happens before
# rda-db restores the customer backup.  Preserve the RPM config up front so
# its NUMCAMS (and other hardware defaults) are used instead of Docker's
# generic four-camera default; otherwise the initial schema can retain a
# phantom default Camera row after restore.
capture_rpm_dvs_conf() {
    log_step "Capturing RPM runtime configuration"

    if [[ ! -s /etc/dvs.conf ]]; then
        log_warn "RPM /etc/dvs.conf is unavailable; Docker will use its default runtime configuration"
        return 0
    fi

    RPM_DVS_CONF=$(mktemp /tmp/nvr-rpm-dvs-conf.XXXXXX)
    cp -p /etc/dvs.conf "$RPM_DVS_CONF"
    log_info "Captured RPM dvs.conf for Docker first boot"
}

seed_rpm_dvs_conf_for_docker() {
    [[ -n "$RPM_DVS_CONF" && -s "$RPM_DVS_CONF" ]] || return 0

    local docker_dvs_conf="$DATA_DIR/config/dvs.conf"
    cp -p "$RPM_DVS_CONF" "$docker_dvs_conf"
    chown dividia:docker "$docker_dvs_conf" 2>/dev/null || true
    chmod 640 "$docker_dvs_conf" 2>/dev/null || true
    log_info "Seeded Docker first-boot dvs.conf from RPM configuration"
}

rpm_backup() {
    log_step "Backing up RPM NVR"

    # Verify MariaDB is running (required for mysqldump)
    if ! svc_active mariadb 2>/dev/null && ! svc_active mysqld 2>/dev/null; then
        log_error "MariaDB is not running — cannot create backup"
        exit 1
    fi

    # rda-db --backup is the RPM-installed backup tool
    if ! rda-db --backup; then
        log_error "Backup failed — aborting migration"
        exit 1
    fi

    log_info "RPM backup completed successfully"
}

ensure_rpm_videostore_labels() {
    log_step "Checking RPM VideoStore filesystem labels"

    local vs_rows
    vs_rows=$(findmnt -rn -o SOURCE,TARGET,FSTYPE 2>/dev/null \
        | awk '$2 ~ "^/videostore/vs[0-9]+$" && $3 != "autofs" { print $1 "\t" $2 "\t" $3 }') || vs_rows=""

    if [[ -z "$vs_rows" ]]; then
        log_info "No mounted RPM VideoStores found to label"
        return 0
    fi

    local failed=0
    while IFS=$'\t' read -r source target fstype; do
        [[ -n "$source" && -n "$target" ]] || continue

        local expected_label="${target##*/}"
        if ! [[ "$expected_label" =~ ^vs[0-9]+$ ]]; then
            log_warn "$target does not map to a standard vsN label — skipping"
            failed=1
            continue
        fi

        if [[ ! -b "$source" ]]; then
            log_warn "$target is mounted from '$source', which is not a block device"
            failed=1
            continue
        fi

        local source_real
        source_real=$(readlink -f "$source" 2>/dev/null || echo "$source")

        local labeled_devices label_count duplicate=0
        labeled_devices=$(blkid | awk -v lbl="$expected_label" -F: '
            $0 ~ "LABEL=\""lbl"\"" {print $1}
        ')
        # LVM exposes the same device through both /dev/mapper and /dev/<vg>.
        # Count canonical devices, not their aliases, so a correctly labeled
        # mounted VideoStore is not mistaken for a duplicate label.
        label_count=$(while IFS= read -r labeled_device; do
            [[ -n "$labeled_device" ]] || continue
            readlink -f "$labeled_device" 2>/dev/null || echo "$labeled_device"
        done <<< "$labeled_devices" | sort -u | grep -c . || true)
        if [[ "$label_count" -gt 0 ]]; then
            while IFS= read -r labeled_device; do
                [[ -n "$labeled_device" ]] || continue
                local labeled_real
                labeled_real=$(readlink -f "$labeled_device" 2>/dev/null || echo "$labeled_device")
                if [[ "$labeled_real" != "$source_real" ]]; then
                    duplicate=1
                fi
            done <<< "$labeled_devices"
        fi
        if [[ "$label_count" -gt 1 || "$duplicate" -eq 1 ]]; then
            log_warn "Label '$expected_label' is already present on another device:"
            while IFS= read -r dev; do [[ -n "$dev" ]] && log_warn "    $dev"; done <<< "$labeled_devices"
            log_warn "  Mounted $target source is $source"
            failed=1
            continue
        fi

        local current_label
        current_label=$(blkid -s LABEL -o value "$source" 2>/dev/null || true)
        if [[ "$current_label" == "$expected_label" ]]; then
            log_info "$target is labeled '$expected_label'"
            continue
        fi

        if [[ -n "$current_label" ]]; then
            log_warn "$target source $source has label '$current_label', expected '$expected_label'"
            failed=1
            continue
        fi

        if [[ "$fstype" =~ ^ext[234]$ ]]; then
            if e2label "$source" "$expected_label" 2>/dev/null; then
                log_info "Set label '$expected_label' on $source for $target"
            else
                log_warn "Failed to set label '$expected_label' on $source"
                failed=1
                continue
            fi
        else
            log_warn "$target source $source is $fstype, not ext2/3/4; cannot auto-label"
            failed=1
            continue
        fi

        local resolved
        resolved=$(blkid -L "$expected_label" 2>/dev/null || true)
        if [[ -z "$resolved" || "$(readlink -f "$resolved" 2>/dev/null || echo "$resolved")" != "$source_real" ]]; then
            log_warn "Label '$expected_label' did not resolve back to $source after labeling"
            failed=1
        fi
    done <<< "$vs_rows"

    if [[ "$failed" -ne 0 ]]; then
        log_error "RPM VideoStore labels are not safe for migration; fix labels and re-run --migrate"
        return 1
    fi
}

capture_rpm_videostore_mounts() {
    # RPM installs can use rda-autofs to keep /videostore/vsN mounted by
    # device path without LABEL=vsN. Once the recording stack is stopped,
    # autofs may expire those mounts before Docker restore looks for the
    # freshly-created backup. Capture the concrete mounts while RPM is still
    # fully online so migrate/rollback can remount the real disks explicitly.
    : > "$RPM_VIDEOSTORE_MOUNTS_FILE"
    findmnt -rn -o SOURCE,TARGET,FSTYPE 2>/dev/null \
        | awk '$2 ~ "^/videostore/vs[0-9]+$" && $3 != "autofs" { print $1 "\t" $2 "\t" $3 }' \
        > "$RPM_VIDEOSTORE_MOUNTS_FILE" || true

    if [[ -s "$RPM_VIDEOSTORE_MOUNTS_FILE" ]]; then
        log_info "Captured RPM VideoStore mounts:"
        sed 's/^/  /' "$RPM_VIDEOSTORE_MOUNTS_FILE" | while read -r line; do log_info "$line"; done
    else
        log_info "No mounted RPM VideoStores captured"
    fi
}

remount_rpm_videostores() {
    local reason="${1:-migration}"
    [[ -s "$RPM_VIDEOSTORE_MOUNTS_FILE" ]] || return 0

    log_step "Ensuring RPM VideoStores are mounted ($reason)"

    local failed=0
    while IFS=$'\t' read -r source target fstype; do
        [[ -n "$source" && -n "$target" ]] || continue
        if mountpoint -q "$target" 2>/dev/null; then
            log_info "$target already mounted"
            continue
        fi

        mkdir -p "$target"
        if mount -t "$fstype" "$source" "$target" 2>/dev/null || mount "$source" "$target" 2>/dev/null; then
            log_info "Mounted $source -> $target"
        else
            log_warn "Failed to mount $source -> $target"
            failed=1
        fi
    done < "$RPM_VIDEOSTORE_MOUNTS_FILE"

    return "$failed"
}

kill_legacy_rpm_processes() {
    # Some RPM services launch child processes that can outlive both
    # `systemctl stop` and RPM file removal. systemd then reports units as
    # active/not-found, while the new Docker stack competes for CPU/ports.
    # Keep patterns specific to host RPM shapes so we do not kill Docker
    # containers with similar process names.
    log_step "Killing leftover RPM-era processes"

    systemctl kill dview ipsetup ptzd pbserver --kill-who=all 2>/dev/null || true

    pkill -f 'net.dividia.dview.Main --server=127.0.0.1 --local' 2>/dev/null || true
    pkill -f '/usr/bin/Xorg :1' 2>/dev/null || true
    pkill -f '/usr/bin/python ./pbserver.pyc --debug --nodaemon' 2>/dev/null || true
    pkill -x ptzd 2>/dev/null || true

    systemctl reset-failed dview ipsetup ptzd pbserver mpengine recorder 2>/dev/null || true
}

stop_rpm_services() {
    log_step "Stopping all RPM NVR services"

    suspend_legacy_cron_starters

    # RPM NVRs use two autofs service shapes.  rda-autofs is the newer
    # appliance wrapper, while CentOS 6 commonly runs the stock `autofs`
    # daemon directly.  Leaving the latter active lets docker-start Phase
    # 1.5 lazy-unmount /videostore after its child VideoStore mounts have
    # been captured.  That detaches the mount tree mid-migration, so a
    # later explicit remount can fail even though the physical disk is good.
    # Stop either daemon only after rpm_backup/capture have completed; the
    # caller remounts the captured real disks immediately after this function.
    svc_stop rda-autofs 2>/dev/null || true
    svc_disable rda-autofs 2>/dev/null || true
    svc_stop autofs 2>/dev/null || true
    svc_disable autofs 2>/dev/null || true
    pkill -9 automount 2>/dev/null || true

    # Stopping automount does not necessarily detach its indirect
    # /videostore mount while its captured vsN children are still mounted.
    # That leaves an autofs-owned parent which rejects `mkdir -p vsN` in
    # remount_rpm_videostores (RC52, 2026-07-16).  The RPM services are down
    # and the concrete device paths were captured before this function, so
    # detach the old tree and let the next step remount those disks directly.
    if [[ -s "$RPM_VIDEOSTORE_MOUNTS_FILE" ]]; then
        while IFS=$'\t' read -r _source target _fstype; do
            [[ -n "$target" ]] || continue
            mountpoint -q "$target" 2>/dev/null && umount -l "$target" 2>/dev/null || true
        done < "$RPM_VIDEOSTORE_MOUNTS_FILE"
    fi
    if [[ "$(findmnt -n -o FSTYPE /videostore 2>/dev/null || true)" == "autofs" ]]; then
        umount -l /videostore 2>/dev/null || true
    fi

    # 1. Kill watchprog first — it monitors and auto-restarts all NVR services.
    #    On CentOS 6, watchprog is started by the Upstart dvs-up job.
    svc_stop watchprog 2>/dev/null || true
    svc_disable watchprog 2>/dev/null || true
    pkill -9 watchprog 2>/dev/null || true

    # Disable Upstart dvs-up job (CentOS 6) — starts watchprog on runlevel 4
    if [ -f /etc/init/dvs-up.conf ]; then
        initctl stop dvs-up 2>/dev/null || true
        mv /etc/init/dvs-up.conf /etc/init/dvs-up.conf.disabled 2>/dev/null || true
        log_info "Disabled Upstart dvs-up job"
    fi

    # Neutralize the RPM NVR startup block in rc.local (it hardcodes
    # "systemctl start mariadb" and "systemctl start rda-backend", which on a
    # host-network Docker NVR means a second MariaDB racing the container for
    # :3306, plus two RPM binaries that no longer exist).
    #
    # Delegated to `nvr ensure-host-config`, which is the ONE implementation and
    # also self-heals the ~100 boxes already migrated by the version of this
    # function that edited the wrong file. What it did wrong, kept here because it
    # is easy to reintroduce: it ran `sed -i /etc/rc.local`, but on CentOS 7/9
    # that path is a SYMLINK to /etc/rc.d/rc.local and `sed -i` replaces a symlink
    # with a regular file -- so the edit created a new /etc/rc.local that systemd
    # never reads (rc-local.service declares ExecStart=/etc/rc.d/rc.local) and
    # left the real boot file untouched, while logging success. Confirmed on cs256
    # 2026-07-26.
    if [[ -x /opt/dividia/nvr ]]; then
        /opt/dividia/nvr ensure-host-config || \
            log_warn "Could not run ensure-host-config for the rc.local cleanup; the nightly nvr update will retry"
    else
        log_warn "/opt/dividia/nvr not present yet; the rc.local dvs block will be cleaned by the first nvr update"
    fi

    # 2. Stop all NVR services. HME is handled separately because a CO9
    # systemd unit can restart its bridge-mode container while migration is
    # replacing it with the host-network Docker workload.
    stop_legacy_hme_service_for_docker
    local services=(rda-backend recorder mpengine pbserver ptzd rdafw logmuxd optician dview ipsetup httpd)
    for svc in "${services[@]}"; do
        svc_stop "$svc" 2>/dev/null || true
        svc_disable "$svc" 2>/dev/null || true
    done
    kill_legacy_rpm_processes

    # 3. Stop MariaDB (backup already completed) — CO6 uses "mysqld", CO7+ uses "mariadb"
    stop_host_database_service_for_docker

    # Do not remove any RPMs here. The migration commit point is after Docker
    # DB restore, update logs, key, VideoStore, MPE, and recording checks pass.
    # remove_rpm_packages uses --noscripts at that point so old RPM uninstall
    # hooks cannot stop Docker or remove live containers.
    assert_migration_lock "after stopping RPM services"

    log_info "All RPM services stopped and disabled"
}

capture_migration_failure_evidence() {
    # The rollback path deliberately removes the failed Docker stack and its
    # partial dtech bind mount so the next migration begins cleanly.  Preserve
    # the information needed to diagnose that failure *before* doing so.  In
    # particular, the engine healthcheck is intentionally permissive while
    # mpengine is waiting for MariaDB, so `healthy` alone does not explain a
    # failed live-MPE gate.
    local stamp evidence_dir service container_id
    stamp=$(date +%Y%m%d-%H%M%S)
    evidence_dir="/videostore/vs1/admin/nvr-migration-failures/$stamp"
    if ! mkdir -p "$evidence_dir" 2>/dev/null; then
        evidence_dir="/root/nvr-migration-failures/$stamp"
        mkdir -p "$evidence_dir" 2>/dev/null || {
            log_warn "Could not create migration-failure evidence directory"
            return 0
        }
    fi

    log_error "Preserving Docker failure evidence at $evidence_dir before rollback"

    {
        echo "captured_at=$(date -Is)"
        echo "install_dir=$INSTALL_DIR"
        echo
        echo '=== docker compose ps -a ==='
        cd "$INSTALL_DIR" 2>/dev/null && docker compose ps -a 2>&1 || true
        echo
        echo '=== 43209 listener ==='
        (ss -lntp 2>/dev/null || netstat -lntp 2>/dev/null || true) | grep -E '(:43209|Local Address)' || true
        echo
        echo '=== Docker daemon tail ==='
        tail -n 400 /var/log/docker.log 2>/dev/null || true
        echo
        echo '=== kernel tail ==='
        dmesg 2>/dev/null | tail -n 200 || true
    } > "$evidence_dir/summary.txt" 2>&1 || true

    # Keep per-service logs separate so the MPE failure is not buried in
    # noisy viewer/backend output. `docker compose down` below removes these
    # json-file logs, so this must stay before it.
    for service in db backend engine connector playback viewer ptz autoheal; do
        (
            cd "$INSTALL_DIR" 2>/dev/null \
                && docker compose logs --no-color --timestamps --tail 1000 "$service" 2>&1
        ) > "$evidence_dir/${service}.log" 2>&1 || true
    done

    # Docker's inspect state preserves exit code, restart count and health
    # history without copying container environment variables (which could
    # contain database credentials) into the diagnostic bundle.
    (
        cd "$INSTALL_DIR" 2>/dev/null || exit 0
        for container_id in $(docker compose ps -aq 2>/dev/null); do
            docker inspect --format '{{.Name}} id={{.Id}} image={{.Config.Image}} restart={{.RestartCount}} state={{json .State}}' "$container_id" 2>&1 || true
        done
    ) > "$evidence_dir/container-state.txt" 2>&1 || true

    [[ -n "${LOG_FILE:-}" && -f "$LOG_FILE" ]] \
        && cp "$LOG_FILE" "$evidence_dir/installer.log" 2>/dev/null || true
    log_error "Docker failure evidence preserved at $evidence_dir"
}

rollback_rpm_services() {
    log_error "Migration failed — rolling back to RPM services"

    capture_migration_failure_evidence

    # Stop Docker containers if any started AND drop the persistent dtech
    # state. `docker compose down -v` removes NAMED volumes (seed,
    # dstream-cache) and anonymous volumes — but NOT bind mounts.
    # The db container's dtech state lives at "$INSTALL_DIR/data/db_data"
    # as a bind mount (compose YAML: "./data/db_data:/var/lib/mysql").
    # We must explicitly rm -rf that directory to drop the
    # docker-start Phase 1 VideoStore INSERTs + any partial
    # rda-db --restore data. Without this, those orphan rows would
    # poison the next migration (Phase 1 skip-if-exists logic would
    # treat them as authoritative).
    #
    # Pre-fix the rollback trap was effectively dead code because
    # rda-db --restore silently returned 0; the loud-failure fix
    # 2026-05-28 made the trap fire for real, surfacing this gap.
    cd "$INSTALL_DIR" 2>/dev/null && docker compose down -v 2>/dev/null || true
    if [[ -n "${INSTALL_DIR:-}" && -d "$INSTALL_DIR/data/db_data" ]]; then
        rm -rf "$INSTALL_DIR/data/db_data" 2>/dev/null || true
        log_info "Dropped $INSTALL_DIR/data/db_data — next install will reseed dtech from scratch"
    fi

    # Only undo an HME handoff that this migration actually performed. Remove
    # the replacement, release the runtime mask, and restore the service's
    # pre-migration enabled/active state; do not disturb an unrelated Docker
    # container or a service that was intentionally disabled before migration.
    if [[ "$LEGACY_HME_HANDOFF" == "true" ]]; then
        docker rm -f hme-stream hme-stream-foreign >/dev/null 2>&1 || true
        if has_systemd; then
            systemctl unmask --runtime hme-stream 2>/dev/null || true
            if [[ "$LEGACY_HME_WAS_MASKED" == "true" ]]; then
                systemctl mask hme-stream 2>/dev/null || true
            else
                systemctl unmask hme-stream 2>/dev/null || true
            fi
        fi
        if [[ "$LEGACY_HME_WAS_ENABLED" == "enabled" ]]; then
            svc_enable hme-stream 2>/dev/null || true
        else
            svc_disable hme-stream 2>/dev/null || true
        fi
        if [[ "$LEGACY_HME_WAS_ACTIVE" == "active" ]]; then
            svc_start hme-stream 2>/dev/null || true
        else
            svc_stop hme-stream 2>/dev/null || true
        fi
    fi

    # Only undo an aiengine handoff this migration performed. First back the
    # adopted state out of the host tree: `docker compose down -v` above removed
    # containers/volumes but NOT the host intent file or the AIENGINE_IMAGE /
    # overlay entries in .env, which a later `nvr start` would read to re-enable
    # a half-adopted add-on. Then restore the legacy service's pre-migration
    # enabled/active state so the RPM engine comes back.
    if [[ "$LEGACY_AIENGINE_HANDOFF" == "true" ]]; then
        back_out_aiengine_adoption
        docker rm -f aiengine >/dev/null 2>&1 || true
        if has_systemd; then
            systemctl unmask --runtime aiengine 2>/dev/null || true
            if [[ "$LEGACY_AIENGINE_WAS_MASKED" == "true" ]]; then
                systemctl mask aiengine 2>/dev/null || true
            else
                systemctl unmask aiengine 2>/dev/null || true
            fi
        fi
        if [[ "$LEGACY_AIENGINE_WAS_ENABLED" == "enabled" ]]; then
            svc_enable aiengine 2>/dev/null || true
        else
            svc_disable aiengine 2>/dev/null || true
        fi
        if [[ "$LEGACY_AIENGINE_WAS_ACTIVE" == "active" ]]; then
            svc_start aiengine 2>/dev/null || true
        else
            svc_stop aiengine 2>/dev/null || true
        fi
    fi

    # Unmount any /videostore/vs[0-9]+ mounts that docker-start's Phase 2
    # nsenter-mounted into PID 1's namespace. The Docker container that
    # ran nsenter is now stopped, but the mounts persist in the host
    # mount table and would clash with rda-autofs when it comes back up.
    # Lazy unmount handles cases where a process inside the (now-stopped)
    # container is still holding a reference.
    if mount | grep -qE 'on /videostore/vs[0-9]+ '; then
        while read -r vs_mount; do
            log_info "Unmounting docker-start nsenter mount: $vs_mount"
            umount -l "$vs_mount" 2>/dev/null || true
        done < <(mount | grep -oE 'on /videostore/vs[0-9]+ ' | awk '{print $2}')
    fi

    # Re-enable Upstart dvs-up job (CentOS 6)
    if [ -f /etc/init/dvs-up.conf.disabled ]; then
        mv /etc/init/dvs-up.conf.disabled /etc/init/dvs-up.conf 2>/dev/null || true
    fi
    restore_legacy_cron_starters

    # Restore rc.local if we backed it up
    if [ -f /etc/rc.local.pre-migration ]; then
        mv /etc/rc.local.pre-migration /etc/rc.local
    fi

    # Unmask and re-enable MariaDB
    for db_svc in mariadb mysqld; do
        if has_systemd; then
            systemctl unmask "$db_svc" 2>/dev/null || true
        fi
        svc_enable "$db_svc" 2>/dev/null || true
        svc_start "$db_svc" 2>/dev/null || true
    done

    # Restart the RPM autofs service BEFORE the NVR services. docker-start Phase 1.5
    # may have lazy-unmounted any leftover autofs at /videostore (the
    # "autofs hangover" cs50 2026-05-27 workaround); the rollback must
    # rebuild that automount before mpengine starts. mpengine writes
    # MP4 files to /videostore/vs1 on its first recording event — if
    # autofs isn't claiming /videostore yet, those writes go to a bare
    # /videostore/vs1 directory on the root partition (cs256 2026-04-24
    # root-fill failure shape). Idempotent: no-op on systems that
    # never had rda-autofs.
    svc_enable rda-autofs 2>/dev/null || true
    svc_start rda-autofs 2>/dev/null || true
    svc_enable autofs 2>/dev/null || true
    svc_start autofs 2>/dev/null || true
    remount_rpm_videostores "rollback before RPM service restart" || true

    # disable_host_services masks these to keep host-network Docker services
    # from losing ports. A migration rollback returns the RPM stack to service,
    # so release the masks before the legacy httpd restart below.
    if has_systemd; then
        for svc in watchprog rda-backend recorder mpengine pbserver ptzd rdafw \
                   logmuxd optician dview ipsetup httpd smbd nmbd mariadb mysqld; do
            systemctl unmask --runtime "$svc" 2>/dev/null || true
            systemctl unmask "$svc" 2>/dev/null || true
        done
    fi

    # Re-enable and start NVR services
    local services=(rda-backend recorder mpengine pbserver ptzd rdafw logmuxd optician dview ipsetup httpd)
    for svc in "${services[@]}"; do
        svc_enable "$svc" 2>/dev/null || true
        svc_start "$svc" 2>/dev/null || true
    done

    # Re-enable watchprog
    svc_enable watchprog 2>/dev/null || true
    svc_start watchprog 2>/dev/null || true

    log_info "RPM services restored — system should be back to pre-migration state"
    [[ -n "$LOG_FILE" ]] && log_error "Migration log saved to: $LOG_FILE"
}

write_rpm_seed_to_docker() {
    if [[ -z "$RPM_SEED" ]]; then
        log_info "No RPM seed captured — skipping seed write"
        return 0
    fi

    log_step "Writing RPM hardware seed to Docker"

    cd "$INSTALL_DIR"
    if ! docker compose exec -T backend nvr-check-key --write-seed /seed/device-id "$RPM_SEED"; then
        log_warn "Failed to write seed to Docker — product key will need re-entry"
        return 0
    fi

    # Verify the seed was written correctly
    local docker_seed
    docker_seed=$(docker compose exec -T backend nvr-check-key -S 2>/dev/null | tr -d '[:space:]')
    if [[ "$docker_seed" == "$RPM_SEED" ]]; then
        log_info "Seed verified: $docker_seed"
    else
        log_warn "Seed mismatch (wrote=$RPM_SEED, read=$docker_seed) — product key may need re-entry"
    fi
    return 0
}

ensure_backup_accessible() {
    log_step "Making RPM backup accessible to Docker"

    cd "$INSTALL_DIR"

    # Find the NEWEST backup on host (the freshly-created one from
    # rpm_backup, which writes a timestamp dir to either /rda/backups/
    # or /videostore/vs1/backups/ depending on legacy dvs.conf
    # CAMSERV setting). DO NOT short-circuit on "any backup is already
    # visible to container" — a stale /videostore/vs1/backups/<old-ts>
    # (years-old snapshot, previous failed migration, hand-copied dir)
    # would otherwise satisfy that gate and rda-db --restore would
    # silently use the stale snapshot instead of the fresh one.
    #
    # Walk both canonical layouts, sort by mtime descending, take the
    # most-recently-modified backup that has the expected /conf/
    # marker dir (the rda-db --backup output contract).
    local backup_dir="" newest_mtime=0
    for search_path in /rda/backups /videostore/*/backups; do
        [[ -d "$search_path" ]] || continue
        for dir in "$search_path"/[0-9]*; do
            [[ -d "$dir" ]] || continue
            [[ "$dir" == *.restored ]] && continue
            [[ -d "$dir/conf" ]] || continue

            local dir_mtime
            dir_mtime=$(stat -c '%Y' "$dir" 2>/dev/null || stat -f '%m' "$dir" 2>/dev/null || true)
            [[ -n "$dir_mtime" ]] || continue
            if [[ "$dir_mtime" -gt "$newest_mtime" ]]; then
                newest_mtime="$dir_mtime"
                backup_dir="$dir"
            fi
        done
    done

    if [[ -z "$backup_dir" ]]; then
        log_warn "No backup found on host — restore will create a fresh configuration"
        return 0
    fi

    local container
    container=$(docker compose ps -q backend 2>/dev/null)
    if [[ -z "$container" ]]; then
        log_warn "Backend container not running — cannot copy backup"
        return 0
    fi

    # Stage backup where rda-db --restore looks for it.
    #
    # rda-db --restore inside the backend container defaults to
    # /videostore/vs1/backups. Pre-2022 NVRs (cs1681, cs2 era) have
    # dvs.conf with CAMSERV=/rda/work/camserv and rda-db --backup wrote
    # to /rda/backups; newer NVRs (cs1129, cs50, cs999, cs1018, cs2427)
    # use /videostore as the convention and write to /videostore/vs1/
    # backups. The container has /videostore bind-mounted from host but
    # NOT /rda, so a docker cp into /rda/backups inside the container
    # is invisible to /videostore-scoped tools.
    #
    # Bind-mount-first: if the backup is already under /videostore on
    # host, the container sees it via the existing bind mount — no copy
    # needed. Otherwise copy to host /videostore/vs1/backups/ so the
    # bind mount surfaces it inside the container. This also unifies
    # the canonical layout for both /rda and /videostore legacy paths.
    #
    # Silent restore failure was the worst-case outcome here: rda-db's
    # `Backup::doRestore` logs "could not find a backup archive to
    # restore from" but install-nvr.sh's wrapper logged "Restore
    # completed successfully" because the underlying tool exit-coded 0
    # on the empty-search-result path. Hit on cs1681 BCC Controller
    # 2026-05-26: 135-server enterprise sync registry restored as
    # empty, full manual recovery required.
    if [[ "$backup_dir" == /videostore/* ]]; then
        log_info "Backup at $backup_dir is already visible via /videostore bind mount"
    else
        log_info "Staging host backup $backup_dir → /videostore/vs1/backups/ (visible to container via bind mount)"
        mkdir -p /videostore/vs1/backups
        # No-clobber: if a previous failed install-nvr.sh run left the same
        # timestamp dir under /videostore/vs1/backups, don't silently merge
        # file-by-file (cp -a default) — that produces a hybrid snapshot
        # that rda-db --restore would happily consume. Bail loud instead so
        # the operator can investigate. -e + -L catches both real files and
        # dangling symlinks (an operator may have manually-symlinked the dir
        # during recovery).
        local dest
        dest="/videostore/vs1/backups/$(basename "$backup_dir")"
        if [[ -e "$dest" || -L "$dest" ]]; then
            log_warn "Destination $dest already exists — not overwriting"
            log_warn "  resolves to: $(readlink -f "$dest" 2>/dev/null || echo "(unknown)")"
            log_warn "  rda-db --restore will use whatever's already there. Investigate before re-running install-nvr.sh."
        else
            cp -a "$backup_dir" /videostore/vs1/backups/
        fi
    fi

    # Verify the backend container can see the bind-mounted backup. On CO6,
    # docker-start can remount /videostore/vs1 in the host namespace after the
    # backend container has already started, and that mount event may not
    # propagate into the running container. Host-side "backup is under
    # /videostore" is not enough; rda-db --restore runs inside the container.
    local backup_name canonical_container_path canonical_visible=0
    backup_name=$(basename "$backup_dir")
    canonical_container_path="/videostore/vs1/backups/$backup_name"
    if docker exec "$container" test -f "$canonical_container_path/conf/dvs.conf" 2>/dev/null; then
        canonical_visible=1
        log_info "Backend container can see backup at $canonical_container_path"
    else
        log_warn "Backend container cannot see $canonical_container_path via /videostore bind mount"
    fi

    # Defensive belt: copy the backup into the backend container overlay at
    # /rda/backups too. rda-db --restore searches this legacy path on Docker
    # after /videostore. Current backend images may not ship an /rda directory,
    # but the rootfs is writable during migration, so create it if possible.
    #
    # Keep the copy itself non-fatal when canonical /videostore is visible,
    # but fail before restore if neither path is visible. Otherwise the next
    # step rolls into a guaranteed rda-db --restore failure and rollback.
    local belt_visible=0
    if ! docker exec "$container" mkdir -p /rda/backups 2>/dev/null; then
        log_warn "Could not create container /rda/backups — skipping defensive belt; restore will use /videostore"
    elif docker exec "$container" test -e "/rda/backups/$backup_name"; then
        belt_visible=1
        log_warn "Container's /rda/backups/$backup_name already exists — skipping defensive belt"
        log_warn "  rda-db --restore will read from /videostore/vs1/backups via the Backup heuristic. Investigate before re-running install-nvr.sh if you wanted a clean belt."
    elif docker cp "$backup_dir" "$container:/rda/backups/" 2>/dev/null; then
        belt_visible=1
        log_info "Copied backup $backup_name into container at /rda/backups/ (defensive belt)"
    else
        log_warn "Could not copy backup into container /rda/backups — skipping defensive belt; restore will use /videostore"
    fi

    if [[ "$canonical_visible" -ne 1 && "$belt_visible" -ne 1 ]]; then
        log_error "Backup $backup_name is not visible inside backend via /videostore or /rda/backups"
        return 1
    fi
}

restore_rda_marker_cache_from_backup() {
    log_step "Restoring rda-db update marker cache"

    local cache_tgz="" newest_mtime=0
    local search_root maxdepth cache_path cache_mtime
    for search_root in /videostore/vs1/backups /rda/backups; do
        [[ -d "$search_root" ]] || continue
        if [[ "$search_root" == /rda/backups ]]; then
            maxdepth=3
        else
            maxdepth=4
        fi
        while IFS= read -r cache_path; do
            [[ -n "$cache_path" ]] || continue
            cache_mtime=$(stat -c '%Y' "$cache_path" 2>/dev/null || stat -f '%m' "$cache_path" 2>/dev/null || true)
            [[ -n "$cache_mtime" ]] || continue
            if [[ "$cache_mtime" -gt "$newest_mtime" ]]; then
                newest_mtime="$cache_mtime"
                cache_tgz="$cache_path"
            fi
        done < <(find "$search_root" -maxdepth "$maxdepth" -path '*/db/cache.tgz' -type f 2>/dev/null)
    done

    if [[ -z "$cache_tgz" ]]; then
        log_warn "No rda-db cache.tgz found in backup; backend may rerun old update scripts"
        return 0
    fi

    local tmp_dir
    tmp_dir=$(mktemp -d /tmp/nvr-rda-db-cache.XXXXXX)
    if ! tar xzf "$cache_tgz" -C "$tmp_dir" 2>/dev/null; then
        rm -rf "$tmp_dir"
        log_warn "Could not extract rda-db marker cache from $cache_tgz"
        return 0
    fi

    if [[ ! -d "$tmp_dir/var/lib/rda-db" ]]; then
        rm -rf "$tmp_dir"
        log_warn "Backup marker cache $cache_tgz did not contain var/lib/rda-db"
        return 0
    fi

    mkdir -p "$DATA_DIR/config/rda-db"
    if [[ -d "$DATA_DIR/config/rda-db" ]] && find "$DATA_DIR/config/rda-db" -mindepth 1 -maxdepth 1 | read -r _; then
        cp -a "$DATA_DIR/config/rda-db" "$DATA_DIR/config/rda-db.pre-cache-restore-$(date +%Y%m%d%H%M%S)"
    fi

    rm -rf "$DATA_DIR/config/rda-db"/*
    cp -a "$tmp_dir/var/lib/rda-db/." "$DATA_DIR/config/rda-db/"
    rm -rf "$tmp_dir"

    local marker_count
    marker_count=$(find "$DATA_DIR/config/rda-db" -type f 2>/dev/null | wc -l)
    log_info "Restored rda-db marker cache from $cache_tgz ($marker_count markers)"
}

remove_rpm_packages() {
    log_step "Removing NVR RPM packages"

    # On CentOS 6 the static Docker install deliberately supplies
    # /etc/init.d/docker. That path is also owned by the legacy docker-engine
    # RPM, so `rpm -e --noscripts docker-engine` still removes the file as an
    # owned payload. Preserve and restore our active static service around
    # the RPM removal; otherwise the NVR works until its first reboot, when
    # the nvr boot service runs before any Docker daemon exists.
    local static_docker_init_backup=""
    if ! has_systemd && [[ -f /etc/init.d/docker ]]; then
        static_docker_init_backup=$(mktemp /tmp/nvr-docker-init.XXXXXX)
        cp -p /etc/init.d/docker "$static_docker_init_backup"
    fi

    # All NVR packages plus their database and Docker engine dependencies.
    # Include both mysql-server (CentOS 6) and mariadb-server (CentOS 7+).
    # Remove one-at-a-time: rpm -e with a list treats it as one transaction
    # and a single %preun failure aborts the entire removal.
    local packages=(rda-release rda-backend recorder mpengine rda-db rda-scripts rdafw
                    logmuxd pbserver ptzd optician dview dview-lib dvsconf ipsetup
                    ffmpeg darknet rda-autofs rda-desktop pelco aiengine rda-snmpd
                    watchprog
                    mariadb-server mysql-server docker-engine)

    for pkg in "${packages[@]}"; do
        # Do not run legacy RPM uninstall scripts during Docker migration.
        # Some of those scripts stop the host docker service and remove Docker
        # containers/images as cleanup for RPM-era auxiliary services. At this
        # point the restored Docker stack, DB volume, and product-key seed are
        # the live system; package scripts must not be allowed to mutate them.
        rpm -e --nodeps --noscripts "$pkg" 2>/dev/null || true
    done

    if [[ -n "$static_docker_init_backup" ]]; then
        install -m 755 "$static_docker_init_backup" /etc/init.d/docker
        rm -f "$static_docker_init_backup"
        chkconfig docker on
    fi

    log_info "RPM packages removed"
}

verify_rda_db_update_log_complete() {
    log_step "Verifying rda-db update log completion"

    cd "$INSTALL_DIR"

    local update_tail
    update_tail=$(docker compose exec -T backend sh -c 'tail -n 120 /var/log/rda-db.log 2>/dev/null' 2>/dev/null || true)
    if [[ -z "$update_tail" ]]; then
        log_error "Could not read /var/log/rda-db.log from backend"
        return 1
    fi

    if ! grep -q 'successfully updated to' <<< "$update_tail" || ! grep -q 'finished' <<< "$update_tail"; then
        log_error "rda-db.log does not show a completed update run"
        echo "$update_tail" >&2
        return 1
    fi

    if grep -Eiq '(traceback|error running update|database migration failed|failed after restore)' <<< "$update_tail"; then
        log_error "rda-db.log contains update failure markers"
        echo "$update_tail" >&2
        return 1
    fi

    log_info "rda-db update log completed normally"
}

verify_live_mpe_and_recording() {
    log_step "Verifying live MPE and recording cadence"

    cd "$INSTALL_DIR"

    local cam_bounds min_cam max_cam
    # A migration validates that the restored engine can serve live video; it
    # must not reject an otherwise healthy NVR because an already-failed
    # camera (often a foreign HME feed) happens to have the highest bID.
    # Prefer two enabled cameras that MPE currently considers healthy.  The
    # legacy fallback below retains coverage for older DBs without fEnable.
    cam_bounds=$(docker compose exec -T db sh -c 'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e "SELECT COALESCE(MIN(bID),0), COALESCE(MAX(bID),0) FROM Camera WHERE fEnable=1 AND fFail=0" 2>/dev/null' 2>/dev/null | tr '\t' ' ' | tr -d '\r' || true)
    read -r min_cam max_cam <<< "$cam_bounds"

    if [[ -z "${min_cam:-}" || -z "${max_cam:-}" || "$min_cam" == "0" || "$max_cam" == "0" ]]; then
        # An empty healthy-camera result has two different meanings. Very old
        # schemas may not have fFail, while a current schema can deliberately
        # mark every enabled camera failed. The latter must not make a
        # migration probe a known-bad camera and roll back a healthy stack.
        local has_failed_column enabled_cameras
        has_failed_column=$(docker compose exec -T db sh -c 'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e "SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=\"dtech\" AND TABLE_NAME=\"Camera\" AND COLUMN_NAME=\"fFail\""' 2>/dev/null || true)
        if [[ "${has_failed_column:-0}" -gt 0 ]]; then
            enabled_cameras=$(docker compose exec -T db sh -c 'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e "SELECT COUNT(*) FROM Camera WHERE fEnable=1"' 2>/dev/null || true)
            if [[ "${enabled_cameras:-0}" -gt 0 ]]; then
                log_warn "All enabled cameras are marked failed; skipping live MPE and recording cadence verification"
                return 0
            fi
            # Modern schema (fFail present) with EXPLICITLY ZERO enabled
            # cameras: there is no video source, so the engine can serve no
            # frame and write no recording, and the hard recording gate below
            # can NEVER pass. This is a pre-provisioned bench box whose cameras
            # are added at the customer site later, not a broken migration. Skip
            # verification rather than falling through to the legacy all-rows
            # fallback, which would probe DISABLED placeholder cameras and
            # guarantee a rollback (cs2615, 2026-08-03). fEnable is authoritative
            # on a modern schema, so zero enabled == nothing to verify.
            #
            # Match "0" EXACTLY, not `-le 0` / `${x:-0}`: an EMPTY count is a
            # FAILED read (transient DB drop/lock), NOT a real zero. Reading it
            # as zero and skipping would fail OPEN — a box that may have enabled
            # cameras ships without proving it records. An empty count falls
            # through to the all-rows fallback and the recording gate (fail-safe).
            if [[ "$enabled_cameras" == "0" ]]; then
                log_warn "No cameras are enabled; skipping live MPE and recording cadence verification (nothing to record yet)"
                return 0
            fi
        fi

        local all_cam_bounds
        all_cam_bounds=$(docker compose exec -T db sh -c 'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e "SELECT COALESCE(MIN(bID),0), COALESCE(MAX(bID),0) FROM Camera" 2>/dev/null' 2>/dev/null | tr '\t' ' ' | tr -d '\r' || true)
        read -r min_cam max_cam <<< "$all_cam_bounds"
        if [[ -z "${min_cam:-}" || -z "${max_cam:-}" || "$min_cam" == "0" || "$max_cam" == "0" ]]; then
            log_error "No cameras found for live MPE verification"
            return 1
        fi
        log_warn "No enabled non-failed cameras found; falling back to all Camera rows for legacy live MPE verification"
    fi

    # FIX 1: the live-JPEG probe is now ADVISORY; the recent-*.mp4 recording
    # probe is the only HARD gate. A restored DB carries STALE camera health
    # (fFail=0 for a camera that is actually offline now → persistent HTTP
    # 400), so a single offline-but-"healthy" camera used to hard-fail this
    # check and roll an otherwise-healthy migration back to RPM (cs93 rolled
    # back; cs2565 hung ~110s here, then the SSH session was killed before
    # configure_user_access ran). We now require ANY ONE enabled camera to
    # serve a live JPEG within a single BOUNDED TOTAL window (not per-camera),
    # and only WARN — never return 1 — if none do.
    #
    # The engine healthcheck intentionally stays healthy while mpengine waits
    # for its initial database connection (restarting it there causes a
    # cascade), so it is not proof of live video; the bounded probe below
    # gives the post-restore engine time to bind before we fall through to the
    # recording gate.
    local jpeg_total_budget_seconds=150   # whole-probe budget across ALL cams
    local jpeg_retry_seconds=5
    local -a probe_cams=("$min_cam")
    [[ "$max_cam" != "$min_cam" ]] && probe_cams+=("$max_cam")

    local jpeg_ok=0 jpeg_elapsed=0 cam tmp
    while (( jpeg_elapsed < jpeg_total_budget_seconds )); do
        for cam in "${probe_cams[@]}"; do
            tmp="/tmp/nvr-migrate-cam${cam}.jpg"
            rm -f "$tmp"
            if curl -fsS --max-time 12 -o "$tmp" "http://127.0.0.1:43209/cam${cam}.jpg?sess=dtech&overlay=0" \
                && file "$tmp" 2>/dev/null | grep -q 'JPEG image data'; then
                jpeg_ok=1
                log_info "Live MPE JPEG OK from camera $cam"
                break 2
            fi
        done
        log_warn "No live MPE JPEG yet from cameras ${probe_cams[*]} (elapsed ${jpeg_elapsed}s/${jpeg_total_budget_seconds}s); retrying in ${jpeg_retry_seconds}s"
        sleep "$jpeg_retry_seconds"
        jpeg_elapsed=$((jpeg_elapsed + jpeg_retry_seconds))
    done

    if [[ $jpeg_ok -ne 1 ]]; then
        # ADVISORY only — do NOT return 1. A camera genuinely offline at
        # migration time (stale fFail=0) is a camera problem, not a migration
        # failure; rolling back to RPM over it stranded healthy boxes. The
        # recording-cadence gate below is the real proof the engine works.
        log_warn "No enabled camera served a live JPEG within ${jpeg_total_budget_seconds}s — continuing (advisory)."
        log_warn "This is expected if a camera is offline; verifying recording cadence instead."
    fi

    # Recording cadence: the HARD gate. Because the JPEG probe above no longer
    # blocks long enough to guarantee the engine finished arming its record
    # threads, wait (bounded) for the first recent *.mp4 to appear before
    # failing. return 1 (NOT exit 1) still fires migrate_flow's ERR rollback —
    # this is the one signal we trust that the migrated engine is recording.
    # NVR_VIDEOSTORE_ROOT overrides the search root for behavioral tests only;
    # production is unchanged (defaults to /videostore).
    local videostore_root="${NVR_VIDEOSTORE_ROOT:-/videostore}"
    # A stale *.mp4 written by the RPM recorder shortly before migration must
    # NOT satisfy this gate (that would mark a non-recording migrated engine as
    # healthy). Require the recording to be NEWER than the Docker engine
    # container's start: an in-progress segment carries a current mtime (so long
    # segment lengths still pass), while any pre-migration RPM file is older than
    # the container and is rejected. Fall back to a 10-minute window only if the
    # container start can't be read, so the gate is never stricter than the
    # engine can satisfy. NVR_REC_REF_EPOCH overrides the reference for tests.
    local rec_ref_epoch="${NVR_REC_REF_EPOCH:-}"
    if [[ -z "$rec_ref_epoch" ]]; then
        local started
        started=$(docker inspect -f '{{.State.StartedAt}}' dividia-nvr-engine-1 2>/dev/null || true)
        [[ -n "$started" ]] && rec_ref_epoch=$(date -d "$started" +%s 2>/dev/null || true)
    fi
    # Only trust a clean positive integer; anything else (unparseable StartedAt,
    # a date(1) without -d) drops to the freshness-window fallback below.
    [[ "$rec_ref_epoch" =~ ^[0-9]+$ ]] || rec_ref_epoch=""
    local -a rec_time_pred
    if [[ -n "$rec_ref_epoch" ]]; then
        rec_time_pred=(-newermt "@$rec_ref_epoch")
        log_info "Recording gate: requiring an *.mp4 newer than engine start (@$rec_ref_epoch), so a stale RPM-era file cannot pass"
    else
        rec_time_pred=(-mmin -10)
        log_warn "Recording gate: could not read engine start; falling back to a 10-minute freshness window"
    fi
    local rec_total_budget_seconds=120
    local rec_retry_seconds=5
    local rec_elapsed=0 today recent_mp4="" dir
    while true; do
        recent_mp4=""
        today=$(date +%Y/%m/%d)
        shopt -s nullglob
        for dir in "$videostore_root"/vs*/dividia/cam*/"$today"; do
            recent_mp4=$(find "$dir" -maxdepth 1 -type f -name '*.mp4' "${rec_time_pred[@]}" -print -quit 2>/dev/null || true)
            [[ -n "$recent_mp4" ]] && break
        done
        shopt -u nullglob
        [[ -n "$recent_mp4" ]] && break

        if (( rec_elapsed >= rec_total_budget_seconds )); then
            log_error "No fresh MP4 recording (newer than engine start) found under $videostore_root (waited ${rec_total_budget_seconds}s) — migrated engine is not recording"
            return 1
        fi
        log_warn "No recent recording yet (elapsed ${rec_elapsed}s/${rec_total_budget_seconds}s); retrying in ${rec_retry_seconds}s"
        sleep "$rec_retry_seconds"
        rec_elapsed=$((rec_elapsed + rec_retry_seconds))
    done

    if [[ $jpeg_ok -eq 1 ]]; then
        log_info "Live MPE OK for cameras ${probe_cams[*]}; recent recording: $recent_mp4"
    else
        log_info "Recording cadence verified (live JPEG was advisory-only); recent recording: $recent_mp4"
    fi
}

cleanup_legacy_artifacts() {
    # Post-migration cleanup of RPM-era artifacts that the migration leaves
    # behind on /. remove_rpm_packages uses `rpm -e --nodeps`, which does NOT
    # cascade to deps — orphan packages like firefox, httpd, php (pulled in
    # by rda-release sub-package Requires:) stay on disk. This function
    # reclaims them, plus yum metadata cache, RPM-era log files, and the
    # /var/lib/mysql data dir (gated on the DB sanity check from verify_migration
    # and a live Docker DB/mount/service check below).
    #
    # Verified empirically on cs999/cs1018/cs50/cs1129 2026-05-15:
    # 400-870 MB recoverable per host depending on what was installed.
    log_step "Cleaning up legacy artifacts"

    if $KEEP_LEGACY; then
        log_info "--keep-legacy set, skipping cleanup (firefox, httpd, yum cache, old logs all preserved)"
        return 0
    fi

    local before_kb after_kb
    before_kb=$(df -k / | tail -1 | awk '{print $3}')

    # 1. Orphan RPM packages. firefox is conditional on RPM presence (CO6 NVRs
    #    had it from RPM era; CO7 NVRs that ran the pre-RHEL-skip install-nvr
    #    also have it; new CO7 installs do not).
    log_info "Removing orphan RPM packages"
    local orphan_pkgs=(firefox
                       httpd-manual httpd-tools httpd
                       liberation-fonts dejavu-sans-fonts google-noto-sans-fonts
                       php php-cli php-common php-mysqlnd
                       traffic-shaper)
    for pkg in "${orphan_pkgs[@]}"; do
        if rpm -q "$pkg" &>/dev/null; then
            rpm -e --nodeps "$pkg" 2>/dev/null && log_info "  removed: $pkg"
        fi
    done

    # Firefox flag file: stale "yes, host firefox is available" claim. The
    # Open Browser menu in backend gates on this file; with firefox gone it
    # must be absent so the menu item correctly no-ops.
    rm -f "$DATA_DIR/config/nvr-host-firefox-available"

    # /root/.mozilla profile: RPM-era usage state, never touched by Docker stack.
    [[ -d /root/.mozilla ]] && rm -rf /root/.mozilla

    # 2. yum / dnf metadata cache.
    log_info "Cleaning package manager cache"
    if command -v yum &>/dev/null; then
        yum clean all >/dev/null 2>&1 || true
        # CO6 yum doesn't fully clean (leaves ~50 MB of repodata). Nuke residual.
        rm -rf /var/cache/yum/*/*/repodata /var/cache/yum/*/*/gen 2>/dev/null
    fi
    if command -v dnf &>/dev/null; then
        dnf clean all >/dev/null 2>&1 || true
    fi

    # 3. RPM-era log files. Docker stack logs to container json-file logs — these
    #    paths are not written to anymore. Wildcard catches both rotated
    #    (.log-YYYYMMDD) and current (.log) files.
    log_info "Removing RPM-era log files"
    rm -f /var/log/rda-backend.log* \
          /var/log/recorder.log* \
          /var/log/pbserver.log* \
          /var/log/mpengine.log* \
          /var/log/logmuxd.log* \
          /var/log/optician.log* \
          /var/log/ptzd.log* \
          /var/log/dview.log* \
          /var/log/aiengine.log* 2>/dev/null

    # Host MySQL logs need a stricter gate than the other RPM-era logs: the
    # Docker DB is still actively serving the restored database, and a host
    # MariaDB process must never be mistaken for that container process.
    local mysql_log_files=()
    local mysql_cleanup_safe=0
    mapfile -t mysql_log_files < <(legacy_mysql_log_files)
    if ((${#mysql_log_files[@]} > 0)) || [[ -d /var/lib/mysql ]]; then
        if legacy_mysql_cleanup_gate; then
            mysql_cleanup_safe=1
        fi
    fi
    if ((${#mysql_log_files[@]} > 0)); then
        if [[ "$mysql_cleanup_safe" == "1" ]]; then
            local mysql_log_file
            for mysql_log_file in "${mysql_log_files[@]}"; do
                rm -f -- "$mysql_log_file"
            done
            log_info "Removed ${#mysql_log_files[@]} confirmed-unused host MySQL log file(s)"
        else
            log_warn "Docker DB/mount/service safety gate failed — keeping host MySQL logs"
        fi
    fi

    # 4. /var/lib/mysql: orphan after rpm_backup → mysqldump → rda-db --restore
    #    moved everything to /opt/dividia/data/db_data inside the db container.
    #    The mariadb-server / mysql-server RPM was already removed by
    #    remove_rpm_packages, so the data dir is unowned and has no daemon.
    #
    #    Gated on DB_SANITY_PASSED from verify_migration, AND we re-verify
    #    the backup file is still readable + still has its mysqldump header
    #    immediately before the rm. The verify ran minutes ago and several
    #    rpm -e calls have run since (some have %preun scripts that could
    #    touch /videostore), so the gate-time check is not enough by itself.
    if [[ -d /var/lib/mysql ]]; then
        if [[ "$mysql_cleanup_safe" == "1" ]] \
           && [[ "${DB_SANITY_PASSED:-0}" == "1" ]] \
           && [[ -n "${BACKUP_SQL_PATH:-}" ]] \
           && [[ -f "$BACKUP_SQL_PATH" ]] \
           && head -c 200 "$BACKUP_SQL_PATH" 2>/dev/null | grep -qE '^-- (MySQL|MariaDB) dump'; then
            local mysql_kb
            mysql_kb=$(du -sk /var/lib/mysql 2>/dev/null | awk '{print $1}')
            rm -rf /var/lib/mysql
            log_info "Removed /var/lib/mysql (~${mysql_kb} KB; rollback dtech.sql verified at $BACKUP_SQL_PATH)"
        elif [[ "$mysql_cleanup_safe" != "1" ]]; then
            log_warn "Docker DB/mount/service safety gate failed — keeping /var/lib/mysql"
        elif [[ "${DB_SANITY_PASSED:-0}" == "1" ]]; then
            # Gate passed at verify time but backup file is now gone/corrupt.
            log_warn "DB sanity gate passed but backup file ${BACKUP_SQL_PATH:-?} is missing/corrupt at cleanup time"
            log_warn "  keeping /var/lib/mysql; investigate before running cleanup again"
        else
            log_warn "DB sanity gate not set — keeping /var/lib/mysql for safety"
        fi
    fi

    # 5. Old kernels (yum-utils' package-cleanup). Keeps the running kernel
    #    plus N-1; clears anything older. No-op when only one kernel is
    #    installed.
    if command -v package-cleanup &>/dev/null; then
        local kernel_count
        kernel_count=$(rpm -q kernel 2>/dev/null | grep -c '^kernel-' || true)
        if [[ "$kernel_count" -gt 1 ]]; then
            package-cleanup --oldkernels --count=1 -y >/dev/null 2>&1 \
                && log_info "Pruned old kernels (kept running + 1 previous)"
        fi
    fi

    # 6. Stale autofs binding at /videostore from the now-removed rda-autofs RPM.
    #
    # rda-autofs (removed by remove_rpm_packages above) owned
    # /etc/auto.videostore and added a `/videostore /etc/auto.videostore
    # --timeout=300` line to /etc/auto.master. After the RPM is gone,
    # /etc/auto.videostore is deleted but the /etc/auto.master entry
    # survives. autofs then returns EACCES whenever anything (container
    # restart, watchtower update, manual docker compose recreate)
    # accesses /videostore — backend's recreate triggers /videostore/vs1
    # access during startup, the bind mount fails, backend exits 143,
    # dependent containers (engine, connector, playback, viewer) crash.
    #
    # Surfaced on cs1681 BCC Controller 2026-05-26 hours after the
    # migration completed cleanly — the dead-binding fault stayed
    # dormant until a backend recreate forced a /videostore access.
    # By the time the symptom hits, install-nvr.sh has long exited;
    # the operator just sees an inexplicable stack-down event.
    #
    # Remove the orphan entry here. autofs is reloaded so it forgets
    # the dead mountpoint (other entries like /var/autofs/removable
    # may still be in use — don't disable autofs entirely).
    if [[ -f /etc/auto.master ]] && grep -qE '^/videostore[[:space:]]' /etc/auto.master; then
        log_info "Removing orphan /videostore line from /etc/auto.master (rda-autofs RPM gone)"
        # Preserve the pre-cleanup snapshot only on first run — re-running
        # install-nvr.sh later must not clobber the original backup with a
        # post-cleanup copy.
        [[ -f /etc/auto.master.pre-migration-cleanup ]] \
            || cp /etc/auto.master /etc/auto.master.pre-migration-cleanup
        sed -i '\|^/videostore[[:space:]]|d' /etc/auto.master
        # Reload autofs so the orphan /videostore binding is purged.
        # `service autofs reload` is the standard cross-distro reload —
        # on CO6 SysV (autofs 5.0.5), the init script SIGHUPs the master
        # PID specifically; on CO7+ systemd, systemctl reload autofs
        # equivalent. Both re-read /etc/auto.master without unmounting
        # sub-mounts that came from external mechanisms (e.g., docker-
        # start Phase 2's nsenter-mounted /videostore/vs1 on recording
        # NVRs). Falling back to a stop+start cycle would briefly
        # collapse those sub-mounts and crash running containers.
        if svc_active autofs 2>/dev/null; then
            service autofs reload 2>/dev/null \
                || systemctl reload autofs 2>/dev/null \
                || pkill -HUP automount 2>/dev/null \
                || log_warn "  autofs reload signal could not be delivered; orphan /videostore binding persists until next reboot"

            # Verify the orphan binding actually went away. Some autofs
            # versions (notably CO6 5.0.5 per-mountpoint daemon model)
            # don't drop runtime mountpoints on SIGHUP — the file change
            # persists but the live binding survives until the next
            # reboot. If we still see autofs at /videostore, warn loudly
            # so the operator schedules a planned reboot before the next
            # docker compose recreate.
            sleep 1
            if findmnt -t autofs /videostore >/dev/null 2>&1; then
                log_warn "  autofs still bound at /videostore after reload — runtime state not cleared"
                log_warn "  the orphan binding will crash the stack on next backend recreate; schedule a reboot"
            else
                log_info "  autofs /videostore binding cleared"
            fi
        fi
    fi

    after_kb=$(df -k / | tail -1 | awk '{print $3}')
    local reclaimed_kb=$(( before_kb - after_kb ))
    log_info "Legacy cleanup complete: reclaimed ~$((reclaimed_kb / 1024)) MB on /"
}

verify_migration() {
    log_step "Verifying migration"

    cd "$INSTALL_DIR"
    assert_migration_lock "pre-final verification"

    # Block on backend health before any downstream steps — we're about to
    # return control to migrate_flow which clears the rollback trap and
    # runs remove_rpm_packages (the point of no return). A backend that
    # isn't actually healthy here means RPMs get removed on top of a broken
    # Docker stack, leaving the NVR with no running recorder.
    wait_for_backend_healthy

    local health
    health=$(docker compose ps backend --format '{{.Health}}' 2>/dev/null || true)
    if [[ "$health" == "healthy" ]]; then
        log_info "Backend container: healthy"
    else
        log_error "Backend container is not healthy after wait ($health) — refusing to remove RPMs"
        log_error "Migration is NOT safe to finalize. Inspect the backend logs and retry."
        # return 1 (not exit 1): verify_migration is called inside the
        # rollback-trap window in migrate_flow (line 2686 → 2742). exit 1
        # bypasses the trap; return 1 propagates via set -e and fires
        # rollback_rpm_services so the host returns to RPM operation.
        return 1
    fi

    # Validate product key if seed was preserved
    if [[ -n "$RPM_SEED" ]]; then
        local key
        key=$(docker compose exec -T backend bash -c 'grep "^KEY=" /etc/dvs.conf | cut -d= -f2' 2>/dev/null || true)
        if [[ -n "$key" ]] && [[ "$key" != "0" ]]; then
            local key_result
            key_result=$(docker compose exec -T backend nvr-check-key -V "$key" 2>/dev/null || true)
            if [[ "$key_result" == "pass" ]]; then
                log_info "Product key: valid"
            else
                log_error "Product key validation failed — refusing to remove RPMs"
                return 1
            fi
        else
            log_error "RPM seed was captured but Docker dvs.conf has no product key"
            return 1
        fi
    fi

    verify_restored_customer_config
    verify_rda_db_update_log_complete
    verify_live_mpe_and_recording

    # Report camera count
    local cam_count
    if ! cam_count=$(docker compose exec -T db sh -c \
        'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" dtech -e "SELECT COUNT(*) FROM Camera" 2>/dev/null' \
        2>/dev/null | tr -d '\r\n '); then
        cam_count=""
    fi
    if [[ -n "$cam_count" ]] && [[ "$cam_count" != "0" ]]; then
        log_info "Cameras in database: $cam_count"
    fi

    # DB sanity gate for cleanup_legacy_artifacts: only remove the orphan
    # /var/lib/mysql if ALL of these hold:
    #   - the newest dtech.sql backup file lives at a canonical
    #     rda-db --backup output path (NOT just anywhere under /videostore)
    #   - the file is large enough to be a real dump (> DB_BACKUP_MIN_BYTES)
    #   - the file begins with a mysqldump/MariaDB header (NOT a customer
    #     export or stale junk that happens to be named dtech.sql)
    #   - the live container has ≥1 Camera row, confirming restore succeeded
    #
    # rda-db --backup writes to one of these canonical layouts:
    #   /videostore/vs1/backups/<ts>.restored/db/dtech.sql  (local-rdavol)
    #   /rda/backups/<ts>/db/dtech.sql                       (CloudNVR EBS)
    #
    # BACKUP_SQL_PATH is exported so cleanup_legacy_artifacts can
    # re-verify the file is still present and headered at delete-time
    # rather than trusting this gate from minutes ago.
    DB_SANITY_PASSED=0
    BACKUP_SQL_PATH=""
    local DB_BACKUP_MIN_BYTES=$((100 * 1024))   # 100 KB; real dump is multi-MB
    local backup_sql backup_size=0

    # Newest first across both canonical layouts. We constrain by path
    # (matching `*/db/dtech.sql` under each canonical root) and sort by
    # mtime descending so multiple snapshots don't yield a stale 2024 file.
    local newest_mtime=0
    local search_root search_depth sql_path sql_mtime
    for search_root in /videostore/vs1/backups /rda/backups; do
        [[ -d "$search_root" ]] || continue
        search_depth=4
        [[ "$search_root" == "/rda/backups" ]] && search_depth=3

        while IFS= read -r sql_path; do
            sql_mtime=$(stat -c '%Y' "$sql_path" 2>/dev/null || stat -f '%m' "$sql_path" 2>/dev/null || true)
            if [[ -n "$sql_mtime" ]] && [[ "$sql_mtime" -gt "$newest_mtime" ]]; then
                newest_mtime="$sql_mtime"
                backup_sql="$sql_path"
            fi
        done < <(find "$search_root" -maxdepth "$search_depth" -path '*/db/dtech.sql' -type f 2>/dev/null || true)
    done

    if [[ -n "$backup_sql" ]]; then
        backup_size=$(stat -c '%s' "$backup_sql" 2>/dev/null || stat -f '%z' "$backup_sql" 2>/dev/null || echo 0)
    fi

    # Header check: mysqldump (and mariadb-dump) write a recognizable banner
    # in the first ~120 bytes ("-- MySQL dump" or "-- MariaDB dump"). A
    # customer recording, junk file, or partial-restore stub will not have
    # this. Cheap to check; catches the "stale dtech.sql somewhere in
    # /videostore satisfies the gate" failure mode.
    local backup_is_mysqldump=0
    if [[ -n "$backup_sql" ]] && head -c 200 "$backup_sql" 2>/dev/null | grep -qE '^-- (MySQL|MariaDB) dump'; then
        backup_is_mysqldump=1
    fi

    if [[ -n "$backup_sql" ]] \
       && [[ "$backup_size" -gt "$DB_BACKUP_MIN_BYTES" ]] \
       && [[ "$backup_is_mysqldump" == "1" ]] \
       && [[ -n "$cam_count" ]] \
       && [[ "$cam_count" != "0" ]]; then
        DB_SANITY_PASSED=1
        BACKUP_SQL_PATH="$backup_sql"
        log_info "DB sanity check: PASS"
        log_info "  backup:   $backup_sql ($backup_size bytes, mysqldump header verified)"
        log_info "  cameras:  $cam_count restored"
    else
        log_warn "DB sanity check: SKIP cleanup"
        log_warn "  backup:        ${backup_sql:-NONE FOUND}"
        log_warn "  size:          ${backup_size}B (min ${DB_BACKUP_MIN_BYTES})"
        log_warn "  mysqldump hdr: $([[ "$backup_is_mysqldump" == "1" ]] && echo yes || echo no)"
        log_warn "  cameras:       ${cam_count:-?}"
    fi
}

################################################################################
# Shared Restore Helper
################################################################################

restore_from_backup() {
    cd "$INSTALL_DIR"

    # The regular backend starts before the restore so ensure_backup_accessible
    # can stage its defensive /rda/backups copy.  It must not remain running
    # while dtech is empty, though: its startup threads query Misc and exit,
    # which kills a concurrent `docker compose exec backend rda-db --restore`
    # with exit 137.  cs146 RC25 exposed that race.  Keep only MariaDB up and
    # run rda-db in an isolated one-shot backend container; start the normal
    # stack only after the database is fully restored below.
    docker compose stop backend engine connector playback viewer ptz autoheal \
        >/dev/null 2>&1 || true
    # Compose stop can return while an old host-PID backend is still being
    # reaped.  Remove those app containers before the restore worker starts:
    # they are recreated after the database has been restored.
    docker compose rm -sf backend engine connector playback viewer ptz autoheal \
        >/dev/null 2>&1 || true
    stop_host_database_service_for_docker
    ensure_docker_db_ready

    log_step "Restoring NVR from backup"
    # Gate the restore with `if !` so set -eE doesn't silently abort on a
    # non-zero exit before the diagnostic block prints. The bare-command
    # pattern this replaced was dead code under set -e (and now under
    # set -eE the rollback trap would fire but the operator never sees
    # the "rda-db --backup file is on /videostore" guidance). Mirror the
    # --update-only branch a few lines below that already uses if-then.
    #
    # `return 1` (NOT `exit 1`): bash's ERR trap fires on commands that
    # return non-zero when `set -e` is active, but does NOT fire on a
    # direct `exit` — `exit` terminates the shell, bypassing trap.
    # Using `return 1` lets `set -e` (line 23) propagate the failure
    # up to migrate_flow's `restore_from_backup` call, which fires the
    # ERR trap set at line 2674 and runs `rollback_rpm_services`.
    # set -E (line 23) inherits that trap into this function so it fires
    # at restore_from_backup's scope if a nested call fails too.
    #
    # upgrade_flow has no rollback trap; the return 1 just exits the
    # script via set -e with the failed restore exit code. Operator
    # can re-run `docker compose exec backend rda-db --restore` once
    # the underlying issue is fixed.
    # Capture exit code via `|| rc=$?` rather than `if ! cmd; then rc=$?`.
    # Bash gotcha: inside `if ! cmd; then ...`, `$?` reflects the inverted
    # test result (0 because `! failing-cmd` succeeds), NOT cmd's actual
    # exit code. PIPESTATUS[0] doesn't help either — the pipeline IS
    # `! cmd`, status 0. Verified empirically with /tmp/test-if-bang-rc.sh.
    # `|| rc=$?` correctly captures the underlying command's code and also
    # prevents set -eE from firing the trap before we print diagnostics.
    local RESTORE_RESULT=0
    # The production backend is privileged and shares the host PID namespace
    # for VideoStore mounting.  rda-db restore needs neither capability; using
    # that namespace here lets its process-control cleanup collide with the
    # stopped production stack on old CentOS 6 hosts.  Make this one-shot
    # worker private and unprivileged.
    local restore_override="$INSTALL_DIR/.compose-restore-isolated.yml"
    # `!reset null` removes the production overlay's pid: host key.  Docker
    # Engine 27 rejects the literal value `private`; omitting/resetting the
    # key is how Compose requests Docker's default private PID namespace.
    printf 'services:\n  backend:\n    pid: !reset null\n    privileged: false\n' > "$restore_override"
    local compose_files
    compose_files=$(sed -n 's/^COMPOSE_FILE=//p' "$INSTALL_DIR/.env" | head -1)
    COMPOSE_FILE="${compose_files}:$(basename "$restore_override")" \
        docker compose run --rm --no-deps -T --entrypoint "" backend rda-db --restore || RESTORE_RESULT=$?
    rm -f "$restore_override"
    if [[ $RESTORE_RESULT -ne 0 ]]; then
        # FATAL: continuing past restore failure was the worst-case outcome
        # for migrate_flow — verify_migration only checks backend health
        # (which can be green against an EMPTY DB), then the script clears
        # the rollback trap and removes the RPM stack. Net result: working
        # RPM NVR → empty Docker NVR + RPMs gone, no way back without a
        # manual rebuild from /rda/backups/<ts>/db/dtech.sql.
        log_error "Restore failed (exit code: $RESTORE_RESULT) — aborting before RPM removal"
        log_error "Operator: rda-db --backup file is on /videostore or /rda/backups."
        log_error "  Migrate path: rollback_rpm_services ERR trap will restore RPM services."
        log_error "  Upgrade path: rerun 'docker compose exec backend rda-db --restore' after investigation."
        return 1
    fi
    log_info "Restore completed successfully"

    # rda-db --restore replays RPM-era network/firewall/service state. On
    # host-network Docker installs that can resurrect host MariaDB/backend
    # before the post-restore update runs, sending the next 127.0.0.1 DB call
    # to the wrong process. Reassert the Docker endpoint immediately.
    stop_host_database_service_for_docker
    stop_host_backend_service_for_docker
    ensure_docker_db_ready
    patch_backend_sw_update_scripts
    mark_restored_sw_paired_update_if_schema_present
    ensure_sw_globalize_paired_misc_row
    ensure_restored_event_bcamera_index

    # Version gate AFTER restore: dvs.conf has been materialized from the
    # backup. Refuse to run --update-only against a pre-6.0 restore (the
    # image no longer ships the pre-6.0 migration scripts, so skipping
    # directly to post-6.0 scripts would corrupt the schema).
    check_version_6_0_or_fail "$DATA_DIR/config/dvs.conf"

    log_step "Running database migrations"
    # Fail-closed: a failed migration after restore leaves the DB in an
    # inconsistent state. Abort instead of restarting services against a
    # half-migrated schema. rda-db --update-only propagates script failures
    # correctly (see rda-db/src/setup/update.py).
    if ! docker compose run --rm --no-deps -T --entrypoint "" backend rda-db --update-only; then
        log_error "Database migration failed after restore."
        log_error "The backend is running but the DB is partially migrated."
        log_error "Investigate the update script failure in the logs, fix the"
        log_error "root cause, then re-run: docker compose exec backend rda-db --update-only"
        # return 1 (not exit 1): restore_from_backup is called inside the
        # migrate_flow rollback-trap window (line 2686 → 2742). exit 1
        # bypasses the trap; return 1 propagates via set -e and fires
        # rollback_rpm_services so a half-migrated DB doesn't strand the
        # host with RPM services stopped and Docker running broken state.
        # Same rationale as the --restore branch above.
        return 1
    fi

    # Re-sync host hostname + yum repo csid to the restored NVR ID.
    # During pull_and_start, backend's docker-start ran rda-db --update-only
    # against an empty DB, which hit the fresh-install path and called
    # setSerial(0) — stamping /etc/hostname as cs0.dividia.net on the host
    # via nsenter. The restore above then populated the DB with the real ID,
    # but docker-start's first-boot set-serial block skips when DB_SERIAL is
    # no longer 0. Invoke set-serial explicitly here so the hostname matches
    # the restored NVR ID instead of staying stuck on cs0.
    #
    # Read the serial directly from the DB, NOT from dvs.conf. rda-db's
    # doRestore (backup.py) wraps the entire sequence in a single try/except
    # that swallows exceptions and returns normally. A partial restore
    # (config restored, DB half-restored) would leave dvs.conf with the real
    # serial but the DB in an inconsistent state. Passing the DB-resident
    # value means setSerial can't propagate a stale dvs.conf into the DB —
    # it's either a no-op (serials match → just hostname/yum side effects)
    # or skipped (DB serial is 0/missing → bail out, hostname stays whatever
    # it was; a broken restore has bigger problems than a stale hostname).
    local db_serial
    db_serial=$(docker compose exec -T db sh -c \
        'mysql -N -B -uroot -p"$MYSQL_ROOT_PASSWORD" -e "SELECT sValue FROM dtech.Misc WHERE sModule='"'"'general'"'"' AND sName='"'"'serial'"'"'" 2>/dev/null' \
        2>/dev/null | tr -d '\r\n ' || true)
    # Reject non-numeric serials before passing to rda-db. A carriage-return
    # or unexpected character from the mysql output makes the downstream
    # rda-db --set-serial choke with a ValueError that surfaces as a generic
    # log_warn, hiding the real cause. Requiring ^[0-9]+$ here gives us a
    # targeted error message and also plugs a theoretical SQL-injection
    # path via seed_apache_extra_from_db's bport SELECT that interpolates
    # $db_serial directly.
    if [[ -n "$db_serial" && "$db_serial" != "0" && "$db_serial" =~ ^[0-9]+$ ]]; then
        log_info "Syncing hostname + yum csid to DB-resident NVR ID ($db_serial)"
        docker compose run --rm --no-deps -T --entrypoint "" backend rda-db --set-serial "$db_serial" \
            || log_warn "set-serial failed; hostname may still read cs0"
    elif [[ -n "$db_serial" && ! "$db_serial" =~ ^[0-9]+$ ]]; then
        log_warn "DB serial contains non-numeric characters [$db_serial] — rejecting to avoid corrupting downstream calls"
        log_warn "Investigate the restore; hostname will stay at its current value"
        db_serial=""
    else
        log_warn "DB serial is empty or 0 — skipping hostname sync (check restore completeness)"
    fi

    # Pre-seed apache Listen override before the final restart so viewer comes up
    # already configured for customer's PublicPort. rda-backend's startup sync
    # would do this too, but writing it here avoids a second viewer restart
    # (content-compare in rda-backend makes the startup pass a no-op).
    seed_apache_extra_from_db "$db_serial"

    log_step "Restarting services with restored configuration"
    # Start the regular stack only after restore, migration, and set-serial
    # have completed. docker-start sees the restored serial and therefore does
    # not enter the fresh-install reallocate-devices path.
    # Tolerate a restart hiccup (image-pull race, daemon flake): the DB is
    # already migrated and we're still inside migrate_flow's trap window,
    # but rollback at this point would un-migrate a working DB. Better to
    # warn and let the operator finish with `nvr restart`.
    docker compose up -d --quiet-pull \
        || log_warn "Service start failed — DB is migrated; finish manually with: nvr restart"
}

# Write $DATA_DIR/apache-extra/dvs-extra.conf from DB Server.bPort so the viewer
# container's apache binds on the customer PublicPort (e.g. 8888) at startup.
# Called after restore when the DB holds the authoritative bPort value.
# Safe to call with empty serial (no-ops); safe when bPort == 80 (clears file).
seed_apache_extra_from_db() {
    local db_serial="$1"
    local conf_file="$DATA_DIR/apache-extra/dvs-extra.conf"
    mkdir -p "$DATA_DIR/apache-extra"

    if [[ -z "$db_serial" || "$db_serial" == "0" ]]; then
        log_warn "Apache extra seed skipped: no DB serial"
        return 0
    fi

    local bport
    bport=$(docker compose exec -T db sh -c \
        "mysql -N -B -uroot -p\"\$MYSQL_ROOT_PASSWORD\" dtech -e \"SELECT bPort FROM Server WHERE bSerial=$db_serial\" 2>/dev/null" \
        2>/dev/null | tr -d '\r\n ' || true)

    if [[ -z "$bport" ]]; then
        log_warn "Apache extra seed: no Server.bPort row for serial $db_serial"
        return 0
    fi

    if [[ "$bport" == "80" ]]; then
        rm -f "$conf_file"
        log_info "Apache extra seed: bPort=80, no override needed"
        return 0
    fi

    cat > "$conf_file" <<EOF
# Customer PublicPort — managed by rda-backend _changePublicPort
### dvs listen start ###
Listen $bport
### dvs listen end ###
EOF
    log_info "Apache extra seed: Listen $bport written to dvs-extra.conf"
}

# The Docker viewer owns the public Apache listener. Preserve a Dragon-pilot
# reverse-proxy installed on an RPM NVR by moving it into the viewer's mounted
# Apache-extra directory before the viewer is first started. Without this, the
# viewer's SPA fallback serves index.html for /dragon-api/ and the cloud
# backend receives HTTP 200 with HTML instead of shard JSON.
preserve_legacy_dragon_apache_config() {
    local legacy_conf="/etc/httpd/conf.d/dragon-pilot.conf"
    local docker_conf="$DATA_DIR/apache-extra/dragon-pilot.conf"

    [[ -f "$legacy_conf" ]] || return 0
    if ! grep -qE 'ProxyPass[[:space:]].*/dragon-api/' "$legacy_conf"; then
        log_warn "Legacy Dragon Apache config has no /dragon-api/ proxy; not copying it"
        return 0
    fi

    mkdir -p "$DATA_DIR/apache-extra"
    if cmp -s "$legacy_conf" "$docker_conf" 2>/dev/null; then
        log_info "Legacy Dragon Apache config already present in viewer mount"
        return 0
    fi

    cp "$legacy_conf" "$docker_conf"
    chmod 0644 "$docker_conf"
    log_info "Preserved legacy Dragon Apache config for Docker viewer"
}

################################################################################
# Main
################################################################################

fresh_install_flow() {
    check_root
    # Fail-closed VideoStore gate: refuses install unless --video-device,
    # --no-video-device, or a pre-labeled vs[N] partition is present
    # (kickstart path). Runs BEFORE any side-effecting work so an error
    # leaves the host unchanged. NOT called from upgrade_flow or
    # migrate_flow — both preserve the prior VideoStore via the DB.
    check_video_device_or_fail
    detect_os
    check_centos6_kernel_or_fail
    check_virtualbox_graphics
    install_docker
    configure_docker_storage
    configure_docker_logging
    configure_reserved_service_ports
    docker_login
    create_directory_structure
    seed_nvr_id
    [[ -n "$VIDEO_DEVICE" ]] && prepare_video_device
    download_compose_files
    create_env_file
    disable_host_services
    ensure_videostore_writable
    install_host_firefox
    pull_and_start
    create_boot_service
    create_management_scripts
    configure_user_access
    install_prune_cron
    install_update_cron

    # FIX 3: mark complete only after every functional step above ran. Gated
    # on an explicit flag, not on trusting the set -e chain reached this line.
    local fresh_ok=1
    [[ $fresh_ok -eq 1 ]] && write_completion_marker "fresh"

    echo ""
    log_step "Installation Complete!"
    echo ""
    log_info "NVR installed to: $INSTALL_DIR"
    log_info "Data directory:   $DATA_DIR"
    log_info "Channel:          $CHANNEL"
    echo ""
    log_info "Management commands:"
    log_info "  nvr status          — Check status"
    log_info "  nvr logs [service]  — View logs"
    log_info "  nvr update          — Update to latest"
    log_info "  nvr channel <name>  — Switch channel"
    log_info "  nvr backup          — Backup to VideoStore"
    log_info "  nvr help            — Show all commands"
    echo ""
    log_info "Access the NVR:"
    local ip
    ip=$(hostname -I 2>/dev/null | awk '{print $1}')
    log_info "  Web interface: http://${ip:-YOUR-SERVER-IP}"
    log_info "  XML-RPC API:   http://${ip:-YOUR-SERVER-IP}:43204"
    echo ""
    log_info "Auto-updates: /etc/cron.d/dividia-nvr-update (host cron, 02:xx daily)"
    echo ""
    log_warn "Next steps:"
    log_warn "  1. Enter your product key in the web interface"
    log_warn "  2. Add cameras through Setup > Cameras"
    log_warn "  3. Configure VideoStores if needed"
    echo ""
    [[ -n "$LOG_FILE" ]] && log_info "Install log saved to: $LOG_FILE"
    echo ""
}

upgrade_flow() {
    log_step "UPGRADE MODE: Restoring from VideoStore backup"
    echo ""

    # Steps 1-8: same as fresh install (Docker, storage, dirs, compose, .env)
    check_root
    detect_os
    check_centos6_kernel_or_fail
    check_virtualbox_graphics
    install_docker
    configure_docker_storage
    configure_docker_logging
    configure_reserved_service_ports
    docker_login
    create_directory_structure
    seed_nvr_id
    [[ -n "$VIDEO_DEVICE" ]] && prepare_video_device
    download_compose_files
    create_env_file

    # Step 9: Restore saved .env from VideoStore (preserves channel/registry)
    restore_saved_env
    boot_storage_compose_overlay_reconcile

    # Step 10: Disable conflicting host services
    disable_host_services
    ensure_videostore_writable
    install_host_firefox

    # Step 11: Pull images, start DB only, restore before backend fully starts
    log_step "Pulling NVR images"
    cd "$INSTALL_DIR"
    docker compose pull --quiet

    log_step "Starting database for restore"
    docker compose up -d db
    # Wait for DB healthy
    local max_wait=60
    local waited=0
    while [[ $waited -lt $max_wait ]]; do
        local health
        health=$(docker compose ps db --format '{{.Health}}' 2>/dev/null || true)
        if [[ "$health" == "healthy" ]]; then
            break
        fi
        sleep 2
        waited=$((waited + 2))
    done

    # Run restore as a one-shot command using the backend image.
    # This runs rda-db --restore BEFORE docker-start, so the database,
    # dvs.conf, SSH keys, and network config are all in place before the
    # backend's full initialization (autoKey, seeding, etc.).
    log_step "Restoring NVR from backup"
    if docker compose run --rm --no-deps -T --entrypoint "" backend rda-db --restore; then
        log_info "Restore completed successfully"
    else
        log_warn "Restore failed — backend will start with fresh configuration"
    fi

    # Version gate AFTER restore: dvs.conf has been materialized from the
    # backup. If the backup was from a pre-6.0 NVR, refuse to run --update-only
    # against it (it would skip the 206 missing pre-6.0 scripts and produce
    # a subtly broken DB).
    check_version_6_0_or_fail "$DATA_DIR/config/dvs.conf"

    log_step "Running database migrations"
    # Fail-closed: a failed migration on the pre-start path leaves the
    # DB half-migrated. Starting services against that state corrupts
    # further. Abort with explicit recovery instructions instead.
    if ! docker compose run --rm --no-deps -T --entrypoint "" backend rda-db --update-only; then
        log_error "Database migration failed before services started."
        log_error "The restored DB is partially migrated. Services were NOT started."
        log_error "Investigate the update script failure above, fix the root cause,"
        log_error "then re-run: $0 --upgrade"
        exit 1
    fi

    # Now start all services — backend's docker-start will find the restored
    # database and dvs.conf already in place
    log_step "Starting all NVR services"
    if ! docker compose up -d --quiet-pull 2>&1; then
        log_warn "Some services failed to start (backend may still be initializing)"
        wait_for_backend_healthy
        log_info "Retrying service start..."
        docker compose up -d --quiet-pull 2>/dev/null || true
    fi

    # Create systemd service, management scripts, user access
    create_boot_service
    create_management_scripts
    configure_user_access
    install_prune_cron
    install_update_cron

    # FIX 3: mark complete only after every functional step ran (explicit flag).
    local upgrade_ok=1
    [[ $upgrade_ok -eq 1 ]] && write_completion_marker "upgrade"

    echo ""
    log_step "Upgrade Complete!"
    echo ""
    log_info "NVR upgraded and restored to: $INSTALL_DIR"
    log_info "Data directory:   $DATA_DIR"
    echo ""
    log_info "Management commands:"
    log_info "  nvr status          — Check status"
    log_info "  nvr logs [service]  — View logs"
    log_info "  nvr update          — Update to latest"
    log_info "  nvr backup          — Backup to VideoStore"
    log_info "  nvr help            — Show all commands"
    echo ""
    local ip
    ip=$(hostname -I 2>/dev/null | awk '{print $1}')
    log_info "Access the NVR:"
    log_info "  Web interface: http://${ip:-YOUR-SERVER-IP}"
    log_info "  XML-RPC API:   http://${ip:-YOUR-SERVER-IP}:43204"
    echo ""
    log_warn "Note: Product key may need re-entry if the device seed changed"
    echo ""
    [[ -n "$LOG_FILE" ]] && log_info "Install log saved to: $LOG_FILE"
    echo ""
}

migrate_flow() {
    log_step "RPM-TO-DOCKER MIGRATION"
    echo ""
    log_warn "This will migrate your RPM-based NVR installation to Docker."
    log_warn "The process will:"
    log_warn "  1. Back up your current NVR configuration and database"
    log_warn "  2. Stop all RPM NVR services"
    log_warn "  3. Install Docker and start NVR containers"
    log_warn "  4. Restore your configuration, database, and product key"
    log_warn "  5. Remove old RPM packages"
    echo ""

    # Version gate BEFORE we take the destructive user-confirmation prompt —
    # user shouldn't have to say yes to a migration we're about to refuse.
    check_version_6_0_or_fail /etc/dvs.conf

    if ! $ASSUME_YES; then
        read -r -p "Proceed with migration? [y/N] " response
        if [[ ! "$response" =~ ^[Yy]$ ]]; then
            log_info "Migration cancelled"
            exit 0
        fi
    fi
    echo ""

    # Phase 1: Pre-migration (RPM still running)
    check_root
    detect_os
    # BEFORE ensure_rpm_path / rpm_backup / stop_rpm_services: on a 2.6 kernel this
    # install cannot succeed, and refusing here leaves the box exactly as it was.
    check_centos6_kernel_or_fail
    ensure_rpm_path
    capture_rpm_seed
    capture_rpm_dvs_conf
    ensure_rpm_videostore_labels
    rpm_backup
    capture_rpm_videostore_mounts

    # Phase 2: Stop all RPM services (with rollback on failure)
    trap 'rollback_rpm_services' ERR
    stop_rpm_services
    remount_rpm_videostores "after stopping RPM services"

    # Phase 3: Docker install (reuses fresh install functions)
    install_docker
    configure_docker_storage
    configure_docker_logging
    configure_reserved_service_ports
    docker_login

    # Pre-seed timezone from host before create_directory_structure
    mkdir -p "$DATA_DIR/timezone"
    if [[ -f /etc/localtime ]]; then
        cp /etc/localtime "$DATA_DIR/timezone/localtime"
    elif command -v timedatectl &>/dev/null; then
        local tz
        tz=$(timedatectl show --property=Timezone --value 2>/dev/null || true)
        if [[ -n "$tz" ]] && [[ -f "/usr/share/zoneinfo/$tz" ]]; then
            cp "/usr/share/zoneinfo/$tz" "$DATA_DIR/timezone/localtime"
        fi
    elif [[ -f /etc/sysconfig/clock ]]; then
        # CentOS 6: read ZONE from /etc/sysconfig/clock
        local tz
        tz=$(. /etc/sysconfig/clock 2>/dev/null && echo "$ZONE")
        if [[ -n "$tz" ]] && [[ -f "/usr/share/zoneinfo/$tz" ]]; then
            cp "/usr/share/zoneinfo/$tz" "$DATA_DIR/timezone/localtime"
        fi
    fi
    [[ -f /etc/sysconfig/clock ]] && cp /etc/sysconfig/clock "$DATA_DIR/timezone/clock"

    create_directory_structure
    seed_rpm_dvs_conf_for_docker
    preserve_legacy_dragon_apache_config
    # Skip seed_nvr_id — restore provides the real dvs.conf
    # Skip prepare_video_device — drive already has backup data and recordings
    mkdir -p /videostore
    download_compose_files
    create_env_file
    disable_host_services
    ensure_videostore_writable
    install_host_firefox
    pull_and_start
    assert_migration_lock "before backup restore"

    # Phase 4: Restore
    write_rpm_seed_to_docker
    remount_rpm_videostores "before backup restore"
    ensure_backup_accessible
    restore_from_backup
    boot_storage_compose_overlay_reconcile
    stop_host_database_service_for_docker
    stop_host_backend_service_for_docker
    restore_rda_marker_cache_from_backup
    assert_migration_lock "after marker-cache restore"
    repair_docker_firewall_after_restore
    assert_migration_lock "after Docker firewall repair"
    recover_compose_after_docker_daemon_restart
    docker compose up -d --quiet-pull || log_warn "Post-restore compose up had a transient failure; readiness wait will continue"
    wait_for_backend_healthy
    write_rpm_seed_to_docker
    # The first post-restart compose up can return while backend health is
    # settling, leaving playback in Created. Retry after backend + seed are
    # ready so dependents are started before the all-services readiness gate.
    docker compose up -d --quiet-pull || log_warn "Post-seed compose retry had a transient failure; readiness wait will continue"
    docker compose restart connector 2>/dev/null || true
    wait_for_compose_services_ready
    # HME and aiengine cutovers are best-effort and MUST NOT fail the
    # migration. The seven core services are already restored and healthy by this
    # point; an optional add-on failing to cut over must never trigger the
    # migration-wide ERR trap (rollback_rpm_services does `compose down -v` +
    # `rm -rf data/db_data`, destroying the just-restored customer DB). Calling
    # each in a `|| ...` context makes bash ignore `set -e`/ERR for the ENTIRE
    # function body, so no internal `return 1` (port preflight, stop race,
    # health probe) can fire rollback. The box migrates with the add-on on its
    # old container or off; the next `nvr update` auto-reconcile (or
    # `nvr addon <name> enable`) picks it up.
    restore_legacy_hme_workload \
        || log_warn "HME cutover incomplete; core migration succeeded — HME will reconcile on the next 'nvr update' (or run 'nvr addon hme enable')"
    restore_legacy_aiengine_workload \
        || log_warn "aiengine cutover incomplete; core migration succeeded — aiengine will reconcile on the next 'nvr update' (or run 'nvr addon aiengine enable')"
    assert_migration_lock "before restored DB gates"

    # Phase 4.5: Make sure every enabled VideoStore row has the device
    # info docker-start needs to re-mount on next backend startup. RPM-era
    # rda-autofs sometimes only stored sUUID (or just sMountPoint);
    # docker-start's Phase 2 re-mount needs at least one of sDevice /
    # sLabel / sUUID. Without backfill, cs256-style disaster on reboot:
    # no mount, writes go to /, root fills, pbserver rejects every path.
    backfill_videostore_fields
    remount_videostores_after_backfill
    if ! verify_videostore_mount_metadata; then
        # Docker has restored the customer configuration by this point. Keep
        # it running for an in-place VideoStore repair, but make RPM cleanup
        # impossible. Re-entering legacy RPM services here would add downtime
        # and risks a second writer stack against the restored database.
        log_error "VideoStore metadata verification failed; Docker remains running for repair and RPM packages were not removed"
        trap - ERR
        return 1
    fi
    verify_restored_customer_config

    # Phase 5: Verify Docker is healthy, then remove RPMs
    verify_migration

    # Clear the rollback trap — Docker is running, safe to remove RPMs
    trap - ERR
    remove_rpm_packages

    # Reclaim ~400-870 MB of RPM-era cruft (firefox, httpd, yum cache, old
    # logs, /var/lib/mysql when DB sanity check passed). Skip with
    # --keep-legacy if debugging or planning a rollback.
    cleanup_legacy_artifacts

    create_boot_service
    create_management_scripts
    configure_user_access
    install_prune_cron
    install_update_cron
    finalize_migrated_stack
    verify_restored_customer_config
    report_shadowed_video

    # FIX 3: mark complete only after the whole migration tail ran (explicit
    # flag). This is the marker whose ABSENCE + a live stack triggers
    # resume_finalize on a later re-run (cs2565 interrupted-migration case).
    local migrate_ok=1
    [[ $migrate_ok -eq 1 ]] && write_completion_marker "migrate"

    echo ""
    log_step "Migration Complete!"
    echo ""
    log_info "NVR migrated from RPM to Docker: $INSTALL_DIR"
    log_info "Data directory:   $DATA_DIR"
    log_info "Channel:          $CHANNEL"
    echo ""
    log_info "Management commands:"
    log_info "  nvr status          — Check status"
    log_info "  nvr logs [service]  — View logs"
    log_info "  nvr update          — Update to latest"
    log_info "  nvr backup          — Backup to VideoStore"
    log_info "  nvr help            — Show all commands"
    echo ""
    local ip
    ip=$(hostname -I 2>/dev/null | awk '{print $1}')
    log_info "Access the NVR:"
    log_info "  Web interface: http://${ip:-YOUR-SERVER-IP}"
    log_info "  XML-RPC API:   http://${ip:-YOUR-SERVER-IP}:43204"
    echo ""
    if [[ -n "$RPM_SEED" ]]; then
        log_info "Product key preserved from RPM install"
    else
        log_warn "Product key needs re-entry — device seed could not be preserved"
    fi
    echo ""
    [[ -n "$LOG_FILE" ]] && log_info "Install log saved to: $LOG_FILE"
    echo ""
}

main() {
    # FIX 2: install the HUP/PIPE-ignore + INT/TERM-logging traps BEFORE
    # setup_logging creates the tee pipe, so a broken tee (SIGPIPE) or a
    # dropped SSH session (SIGHUP) can't kill the run before logging is up.
    install_signal_traps

    echo ""
    echo "============================================"
    echo "  NVR Docker Installation Script"
    echo "============================================"
    echo ""

    parse_args "$@"
    validate_channel
    setup_logging
    warn_if_not_multiplexed

    # Auto-detect RPM install if not explicitly in migrate or upgrade mode
    if [[ "$MIGRATE_MODE" != "true" ]] && [[ "$UPGRADE_MODE" != "true" ]]; then
        if detect_rpm_install 2>/dev/null; then
            log_info "Existing RPM NVR installation detected — switching to migration mode"
            MIGRATE_MODE=true
        fi
    fi

    # FIX 3: resume an interrupted install/migration instead of re-running a
    # destructive full flow. Checked after RPM auto-detect (so a box whose
    # RPMs are already gone isn't misread) but before the upgrade auto-detect
    # below — an interrupted box still has its migration backup on the
    # VideoStore, which would otherwise route it into a full UPGRADE (a
    # re-restore over the already-running stack). Only fires when a live
    # Docker stack has no completion marker and no RPM services remain.
    if detect_incomplete_install; then
        resume_finalize
        return 0
    fi

    # Auto-detect upgrade mode if not migrating
    if [[ "$MIGRATE_MODE" != "true" ]]; then
        if [[ "$UPGRADE_MODE" != "true" ]]; then
            if detect_existing_install 2>/dev/null; then
                log_info "Existing backup detected on VideoStore — switching to upgrade mode"
                UPGRADE_MODE=true
            fi
        else
            # Verify backup exists when --upgrade is explicit
            if ! detect_existing_install 2>/dev/null; then
                log_error "No backup found on VideoStore"
                log_error "Run backup-nvr.sh on the existing system before upgrading"
                exit 1
            fi
        fi
    fi

    # Determine display mode
    local mode="FRESH"
    if $MIGRATE_MODE; then mode="MIGRATE"; elif $UPGRADE_MODE; then mode="UPGRADE"; fi

    log_info "Configuration:"
    log_info "  Mode:         $mode"
    log_info "  Channel:      $CHANNEL"
    log_info "  Registry:     $REGISTRY"
    log_info "  Data Dir:     $DATA_DIR"
    log_info "  Install:      $INSTALL_DIR"
    if [[ -n "$VIDEO_DEVICE" ]]; then
        log_info "  Video Device: $VIDEO_DEVICE"
    elif [[ "$NO_VIDEO_DEVICE" == "true" ]]; then
        log_info "  Video Device: (none — --no-video-device; directory fallback, dev/test only)"
    fi
    # Auto-detected vs[N] partitions are logged from check_video_device_or_fail
    # at fresh_install_flow start; no summary line here because DETECTED_VS_LABELS
    # isn't populated yet at this point.
    echo ""

    if $MIGRATE_MODE; then
        migrate_flow
    elif $UPGRADE_MODE; then
        upgrade_flow
    else
        fresh_install_flow
    fi
}

# Only auto-run when executed directly, not when sourced. docker/tests/*.sh
# behavioral tests source this file to exercise individual functions (e.g.
# verify_live_mpe_and_recording) without running the whole installer. The
# guard is inert for the real `sudo ./install-nvr.sh` path ($0 == this file).
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
    main "$@"
fi