#!/bin/bash

# pve-create-tshoot-image — Build a ReaR troubleshooting / restore ISO from
# a Proxmox VE cloud-init template.
#
# Two modes of operation:
#
#   LOCAL MODE  — run directly on a PVE node (as root).
#     The ISO is built locally and written to --output (default: $PWD).
#     Example:  pve-create-tshoot-image -t 9004 -c hosts.csv
#
#   REMOTE MODE — run from a jump host, specify the PVE node with -S.
#     The script copies itself and the CSV to the remote node, re-executes
#     there, then copies the resulting ISO back to the local --output dir.
#     Example:  pve-create-tshoot-image -t 9004 -c hosts.csv -S root@pve1
#
# The resulting ISO boots into a ReaR rescue environment pre-configured with
# network troubleshooting tools.  On boot it:
#
#   1. Identifies the physical host via DMI serial number and sets hostname,
#      bond members and management IP from a CSV inventory
#      (serial,hostname,bond_members,ip — bond members are :-separated).
#   2. Brings up every physical NIC and starts lldpd for neighbour discovery
#      (unconditional — runs even without IP configuration).
#   3. (Optional) Configures a bonded VLAN management interface using the
#      per-host IP from the CSV and shared network parameters (mask, gateway,
#      DNS) passed as CLI arguments.
#   4. Offers tcpdump and nic-xray for packet-level diagnostics.
#   5. Can run  rear recover  to deploy the base OS to local disks.
#
# The build VM is an intermediate state — host identity files (machine-id,
# SSH host keys, etc.) are wiped so each restored system is unique.
#
# Supported base distributions: Rocky Linux 9, Ubuntu LTS (22.04, 24.04),
# openSUSE Leap 15.x.  openSUSE Leap 16.x builds but ReaR rescue fails.
#
# All VM interaction uses the QEMU guest agent (virtio serial channel) —
# no SSH or network connectivity required between PVE host and build VM.
# The ISO is extracted from the stopped VM's disk via virt-copy-out.
#
# Prerequisites (on the PVE host):
#   - Proxmox VE 8.x or later
#   - libguestfs-tools  (virt-customize, virt-cat, virt-copy-out)
#   - python3 (for parsing guest agent JSON responses)
#   - A cloud-init template with qemu-guest-agent installed
#
# See --help for full usage.

VERSION="1.3.0"

set -euo pipefail

# --- Colours (disabled when stdout is not a terminal) -------------------------

if [[ -t 1 ]]; then
    C_RED='\033[0;31m'
    C_GREEN='\033[0;32m'
    C_YELLOW='\033[0;33m'
    C_CYAN='\033[0;36m'
    C_BOLD='\033[1m'
    C_RESET='\033[0m'
else
    C_RED='' C_GREEN='' C_YELLOW='' C_CYAN='' C_BOLD='' C_RESET=''
fi

# --- Helpers ------------------------------------------------------------------

msg()  { echo -e "${C_BOLD}${C_CYAN}::${C_RESET} $*"; }
ok()   { echo -e "   ${C_GREEN}[+]${C_RESET} $*"; }
warn() { echo -e "   ${C_YELLOW}[!]${C_RESET} $*"; }
err()  { echo -e "   ${C_RED}[-]${C_RESET} $*" >&2; }
die()  { err "$@"; exit 1; }

run() {
    if (( DRY_RUN )); then
        echo -e "   ${C_YELLOW}[dry-run]${C_RESET} $*"
    else
        "$@"
    fi
}

# Convert a dotted-decimal netmask (255.255.255.0) or bare prefix (24) to
# CIDR notation (/24).  Input already in /NN form is returned unchanged.
normalize_mask() {
    local mask="$1"
    [[ "$mask" =~ ^/[0-9]+$ ]] && { echo "$mask"; return; }
    if [[ "$mask" =~ ^[0-9]+$ ]] && (( mask >= 0 && mask <= 32 )); then
        echo "/$mask"; return
    fi
    # Dotted decimal → count set bits
    local IFS='.' bits=0
    local -a octets=($mask)
    for o in "${octets[@]}"; do
        case "$o" in
            255) (( bits += 8 )) ;; 254) (( bits += 7 )) ;; 252) (( bits += 6 )) ;;
            248) (( bits += 5 )) ;; 240) (( bits += 4 )) ;; 224) (( bits += 3 )) ;;
            192) (( bits += 2 )) ;; 128) (( bits += 1 )) ;; 0)   ;;
            *)   die "Invalid netmask: $mask" ;;
        esac
    done
    echo "/$bits"
}

# --- Defaults -----------------------------------------------------------------

DRY_RUN=0
PVE_SERVER=""                # empty = local mode; USER@HOST = remote mode
TEMPLATE=""                  # PVE template VMID  (required)
CSV_FILE=""                  # serial,hostname,bond_members,ip CSV (required)
STORAGE="local-lvm"         # PVE storage for temp VM
OUTPUT_DIR=""                # where to write the ISO (default: $PWD)
RESCUE_ONLY=0                # 1 = mkrescue (no backup), 0 = mkbackup
REAR_TIMEOUT=1800            # max seconds for rear build (30 min)
BOOT_TIMEOUT=180             # max seconds to wait for guest agent

# Target-host network (embedded in the ISO, applied after restore)
BOND_MODE="802.3ad"          # bonding mode
VLAN_ID=""                   # management VLAN ID
NETMASK=""                   # prefix length or dotted mask (e.g. /24)
GATEWAY=""                   # default gateway
DNS=""                       # comma-separated nameservers
PROXY=""                     # HTTP/HTTPS proxy URL

# Target-host identity (embedded in the ISO, applied on first boot)
TARGET_USER=""               # user account to create/configure
TARGET_PASSWORD=""           # plaintext password (hashed before embedding)
TARGET_SSH_KEY=""            # path to SSH public key file

# Build-VM network (used during image preparation)
VM_BRIDGE="vmbr0"            # PVE bridge
VM_VLAN=""                   # optional VLAN tag on the VM NIC
VM_NET_IP="dhcp"             # "dhcp" or static IP/MASK (e.g. 10.0.0.50/24)
VM_NET_GW=""                 # gateway (required when VM_NET_IP is static)
VM_NET_DNS=""                # DNS     (required when VM_NET_IP is static)
VM_PROXY=""                  # HTTP proxy for build VM (package downloads)

# Internal state (set during execution)
TEMP_VMID=""
TEMP_DIR=""
DISK_PATH=""
DISTRO_FAMILY=""
DISTRO_VERSION=""

# --- Cleanup ------------------------------------------------------------------

cleanup() {
    local rc=$?

    if (( DRY_RUN )); then
        [[ -n "$TEMP_DIR" && -d "$TEMP_DIR" ]] && rm -rf "$TEMP_DIR"
        return $rc
    fi

    # Destroy the temporary VM if it exists
    if [[ -n "$TEMP_VMID" ]]; then
        if qm status "$TEMP_VMID" &>/dev/null 2>&1; then
            msg "Cleaning up temporary VM ${TEMP_VMID}..."
            qm stop "$TEMP_VMID" --timeout 30 2>/dev/null || true
            local waited=0
            while (( waited < 30 )); do
                local st
                st=$(qm status "$TEMP_VMID" 2>/dev/null | awk '{print $2}') || break
                [[ "$st" == "stopped" ]] && break
                sleep 2
                (( waited += 2 ))
            done
            qm destroy "$TEMP_VMID" --purge 2>/dev/null || \
                warn "Could not destroy VM ${TEMP_VMID} — clean up manually"
        fi
    fi

    [[ -n "$TEMP_DIR" && -d "$TEMP_DIR" ]] && rm -rf "$TEMP_DIR"

    return $rc
}

trap cleanup EXIT

# --- Distro detection ---------------------------------------------------------

# Read /etc/os-release from a disk image and set DISTRO_FAMILY + DISTRO_VERSION.
detect_distro() {
    local disk="$1"
    local os_release

    msg "Detecting distribution..."

    os_release=$(virt-cat -a "$disk" /etc/os-release 2>/dev/null) || \
        die "Cannot read /etc/os-release — is this a Linux template?"

    local id version_id name
    id=$(echo "$os_release" | sed -n 's/^ID="\?\([^"]*\)"\?$/\1/p' | head -1)
    version_id=$(echo "$os_release" | sed -n 's/^VERSION_ID="\?\([^"]*\)"\?$/\1/p' | head -1)
    name=$(echo "$os_release" | sed -n 's/^PRETTY_NAME="\?\([^"]*\)"\?$/\1/p' | head -1)

    case "$id" in
        rocky)         DISTRO_FAMILY="rocky"   ;;
        ubuntu)        DISTRO_FAMILY="ubuntu"  ;;
        opensuse-leap) DISTRO_FAMILY="opensuse" ;;
        opensuse)      DISTRO_FAMILY="opensuse" ;;
        *)             die "Unsupported distribution: ${name:-$id} ($version_id)" ;;
    esac

    DISTRO_VERSION="$version_id"
    ok "Detected: ${name:-$id $version_id}"
}

# --- File generation (all injection assets) -----------------------------------

# Populate $TEMP_DIR with every file that will be injected into the disk image.
generate_files() {
    TEMP_DIR=$(mktemp -d /tmp/pve-tshoot-XXXXXX)

    msg "Preparing configuration files..."

    # ---- input files ---------------------------------------------------------
    cp "$CSV_FILE" "$TEMP_DIR/hosts.csv"

    # ---- target-network.conf (generated from CLI params) ---------------------
    if [[ -n "$NETMASK" ]]; then
        generate_target_network_conf > "$TEMP_DIR/target-network.conf"
    fi

    # NOTE: ReaR local.conf and dhclient.conf are generated in inject_files()
    # because they depend on DISTRO_FAMILY which is set by detect_distro()
    # after generate_files().

    # ---- DHCP timeout override (runs before 58-start-dhclient.sh) ------------
    cat > "$TEMP_DIR/57-set-dhcp-timeout.sh" <<'DHCP_EOF'
# Fast-fail DHCP for the ReaR rescue system.
# Runs before 58-start-dhclient.sh in system-setup.d.
#
# Two paths:
#   1. dhclient present → patch /etc/dhclient.conf timeout
#   2. dhclient absent  → set up systemd-networkd with short DHCP timeout

if command -v dhclient &>/dev/null && [[ -f /etc/dhclient.conf ]]; then
    # Path 1: patch dhclient.conf
    sed -i 's/^timeout .*/timeout 6;/' /etc/dhclient.conf 2>/dev/null || true
    sed -i 's/^reboot .*/reboot 2;/' /etc/dhclient.conf 2>/dev/null || true
    sed -i 's/^select-timeout .*/select-timeout 0;/' /etc/dhclient.conf 2>/dev/null || true
    sed -i 's/^initial-interval .*/initial-interval 1;/' /etc/dhclient.conf 2>/dev/null || true
elif command -v networkctl &>/dev/null; then
    # Path 2: systemd-networkd DHCP (Ubuntu 24.04+, no dhclient)
    mkdir -p /run/systemd/network
    cat > /run/systemd/network/10-dhcp-rescue.network <<'NETD_EOF'
[Match]
Name=en* eth* eno* ens*

[Network]
DHCP=ipv4

[DHCPv4]
RouteMetric=100
UseDomains=true
MaxAttempts=3
NETD_EOF
    networkctl reload 2>/dev/null || systemctl restart systemd-networkd 2>/dev/null || true
fi
DHCP_EOF
    chmod 755 "$TEMP_DIR/57-set-dhcp-timeout.sh"

    # ---- boot-time setup script (ReaR rescue system-setup.d) -----------------
    generate_tshoot_script > "$TEMP_DIR/90-tshoot-setup.sh"
    chmod 755 "$TEMP_DIR/90-tshoot-setup.sh"

    # ---- firstboot service (runs on restored host) ---------------------------
    generate_firstboot_script > "$TEMP_DIR/tshoot-firstboot.sh"
    chmod 755 "$TEMP_DIR/tshoot-firstboot.sh"
    generate_firstboot_unit > "$TEMP_DIR/tshoot-firstboot.service"
    generate_linkup_unit > "$TEMP_DIR/tshoot-linkup.service"

    # ---- target-identity.conf (user/password/SSH key) ------------------------
    if [[ -n "$TARGET_USER" || -n "$TARGET_PASSWORD" || -n "$TARGET_SSH_KEY" ]]; then
        generate_target_identity_conf > "$TEMP_DIR/target-identity.conf"
    fi

    ok "Configuration files ready"
}

# Write target-network.conf to stdout (shared network params for all hosts).
generate_target_network_conf() {
    echo "# Target-host network — generated by pve-create-tshoot-image"
    echo "# Shared parameters applied to every restored host."
    [[ -n "$BOND_MODE" ]] && echo "BOND_MODE=$BOND_MODE"
    [[ -n "$VLAN_ID" ]]   && echo "VLAN_ID=$VLAN_ID"
    echo "NETMASK=$NETMASK"
    echo "GATEWAY=$GATEWAY"
    echo "DNS_SERVERS=$DNS"
    [[ -n "$PROXY" ]] && echo "PROXY=$PROXY"
    return 0
}

# Write a ReaR local.conf to stdout.
generate_rear_conf() {
    cat <<'EOF'
# /etc/rear/local.conf — generated by pve-create-tshoot-image
#
# Self-contained troubleshooting / restore ISO.

OUTPUT=ISO
OUTPUT_URL=file:///var/lib/rear/output/
EOF

    # Ubuntu 24.04+ dropped isc-dhcp-client; use static networking and let
    # our 57-set-dhcp-timeout.sh handle DHCP via systemd-networkd instead.
    if [[ "$DISTRO_FAMILY" == "ubuntu" ]]; then
        local _ubuntu_major="${DISTRO_VERSION%%.*}"
        if (( _ubuntu_major >= 24 )); then
            cat <<'EOF'

# Ubuntu 24.04+ has no dhclient — our rescue script handles DHCP via networkd.
USE_DHCLIENT=no
USE_STATIC_NETWORKING=y
EOF
        else
            cat <<'EOF'

# Use dhclient in the rescue system (timeout patched by 57-set-dhcp-timeout.sh).
USE_DHCLIENT=yes
EOF
        fi
    else
        cat <<'EOF'

# Use dhclient in the rescue system (timeout patched by 57-set-dhcp-timeout.sh).
USE_DHCLIENT=yes
EOF
    fi

    if (( RESCUE_ONLY )); then
        cat <<'EOF'

# Rescue-only — no backup archive.
BACKUP=REQUESTRESTORE
EOF
    else
        cat <<'EOF'

# Full system backup embedded in the ISO.
BACKUP=NETFS
BACKUP_URL=iso:///backup
BACKUP_PROG_COMPRESS_OPTIONS=( --gzip )
BACKUP_PROG_COMPRESS_SUFFIX=".gz"
EOF
    fi

    cat <<'EOF'

# --- Troubleshooting tools ---------------------------------------------------

# Binaries to carry into the rescue environment.
PROGS+=( lldpd lldpcli tcpdump nic-xray dmidecode ip modprobe )

# Config / data files.
COPY_AS_IS+=( /etc/tshoot /etc/lldpd.conf /etc/lldpd.d )

# Kernel modules for bond + VLAN in the rescue system.
MODULES+=( bonding 8021q )
MODULES_LOAD+=( bonding 8021q )

# Include all firmware for broad hardware compatibility.
FIRMWARE_FILES=( 'yes' )

# Override the build VM's console settings so the rescue ISO works on both
# VGA (IPMI/KVM) and serial (BMC/headless) consoles.  Linux outputs to all
# listed consoles; the last one gets interactive input (tty0 = VGA).
USE_SERIAL_CONSOLE=no
KERNEL_CMDLINE="console=ttyS0,115200 console=tty0"
EOF
}

# Write a dhclient.conf to stdout with a short timeout so the rescue system
# reaches the shell quickly when no DHCP server answers.
generate_dhclient_conf() {
    cat <<'EOF'
# dhclient.conf — generated by pve-create-tshoot-image
# Short timeout so the rescue shell is available quickly when no DHCP
# server is reachable (e.g. bare-metal with static-only network).

option rfc3442-classless-static-routes code 121 = array of unsigned integer 8;

request subnet-mask, broadcast-address, routers,
        rfc3442-classless-static-routes,
        interface-mtu, host-name, domain-name,
        domain-name-servers, ntp-servers;

require subnet-mask;

timeout 6;
retry 60;
reboot 2;
select-timeout 0;
initial-interval 1;

script "/bin/dhclient-script";
EOF
}

# Write the 90-tshoot-setup.sh boot script to stdout.
# This script executes during ReaR rescue system boot and configures the
# troubleshooting environment automatically.
generate_tshoot_script() {
    cat <<'TSHOOT_EOF'
#!/bin/bash
# 90-tshoot-setup.sh — Troubleshooting environment bootstrap
#
# Runs during ReaR rescue system boot (system-setup.d).
#
# Step 1: Identify the host via DMI serial → set hostname + per-host IP
# Step 2: Enable ALL physical NICs (unconditional — required for LLDP)
# Step 3: Start lldpd (unconditional)
# Step 4: Optional management network (bond + VLAN + IP/gw/dns)
#
# INVARIANT: Steps 2 + 3 MUST run before step 4 so that lldpd can
#            communicate on every port regardless of IP configuration.

echo ""
echo "==== Troubleshooting environment setup ===="
echo ""

# --------------------------------------------------------------------------- #
# 1.  Identity from DMI serial number (hostname + IP from CSV)                #
# --------------------------------------------------------------------------- #
SERIAL=$(dmidecode -s system-serial-number 2>/dev/null | tr -d '[:space:]')
CSV="/etc/tshoot/hosts.csv"

# Per-host values (populated by CSV lookup)
HOST_NAME=""
HOST_BOND_MEMBERS=""   # colon-separated in CSV, converted to comma below
HOST_IP=""

if [[ -n "$SERIAL" && "$SERIAL" != "None" && -f "$CSV" ]]; then
    # CSV format: serial,hostname,bond_members(:-separated),ip
    HOST_NAME=$(awk -F, -v s="$SERIAL" \
        '/^[[:space:]]*#/{next} /^[[:space:]]*$/{next}
         {gsub(/[[:space:]]/,"",$1); gsub(/[[:space:]]/,"",$2)}
         $1==s{print $2; exit}' "$CSV")
    HOST_BOND_MEMBERS=$(awk -F, -v s="$SERIAL" \
        '/^[[:space:]]*#/{next} /^[[:space:]]*$/{next}
         {gsub(/[[:space:]]/,"",$1); gsub(/[[:space:]]/,"",$3)}
         $1==s{print $3; exit}' "$CSV")
    HOST_IP=$(awk -F, -v s="$SERIAL" \
        '/^[[:space:]]*#/{next} /^[[:space:]]*$/{next}
         {gsub(/[[:space:]]/,"",$1); gsub(/[[:space:]]/,"",$4)}
         $1==s{print $4; exit}' "$CSV")

    if [[ -n "$HOST_NAME" ]]; then
        hostname "$HOST_NAME"
        echo "$HOST_NAME" > /etc/hostname 2>/dev/null || true
        echo "[tshoot] Hostname  : $HOST_NAME  (serial $SERIAL)"
        [[ -n "$HOST_BOND_MEMBERS" ]] && echo "[tshoot] Bond NICs : $HOST_BOND_MEMBERS"
        [[ -n "$HOST_IP" ]]           && echo "[tshoot] Host IP   : $HOST_IP"
    else
        echo "[tshoot] WARNING — no mapping for serial '$SERIAL'"
        echo "[tshoot] Known mappings:"
        awk -F, '/^[[:space:]]*#/{next} /^[[:space:]]*$/{next} {
            gsub(/[[:space:]]/,"",$1); gsub(/[[:space:]]/,"",$2)
            gsub(/[[:space:]]/,"",$3); gsub(/[[:space:]]/,"",$4)
            printf "            %-16s  %-20s  %-14s  %s\n", $1, $2, $3, $4
        }' "$CSV"
    fi
else
    [[ -z "$SERIAL" || "$SERIAL" == "None" ]] && \
        echo "[tshoot] WARNING — could not read DMI serial number"
    [[ ! -f "$CSV" ]] && \
        echo "[tshoot] WARNING — $CSV not found"
fi

# Fallback hostname when no host was identified
if [[ -z "$HOST_NAME" ]]; then
    hostname "kml"
    echo "kml" > /etc/hostname 2>/dev/null || true
    echo "[tshoot] Hostname  : kml  (fallback — host not identified)"
fi

# --------------------------------------------------------------------------- #
# 2.  Enable every physical NIC  (unconditional — needed for LLDP)            #
# --------------------------------------------------------------------------- #
echo "[tshoot] Enabling network interfaces..."
for dev_link in /sys/class/net/*/device; do
    [[ -e "$dev_link" ]] || continue
    iface=$(basename "$(dirname "$dev_link")")
    ip link set "$iface" up 2>/dev/null \
        && echo "[tshoot]   $iface UP" \
        || echo "[tshoot]   $iface FAILED"
done

# --------------------------------------------------------------------------- #
# 3.  Start lldpd  (unconditional)                                            #
# --------------------------------------------------------------------------- #
if command -v lldpd &>/dev/null; then
    lldpd -c -s -e 2>/dev/null \
        && echo "[tshoot] lldpd started (run 'lldpcli show neighbors' after ~30 s)" \
        || echo "[tshoot] WARNING — lldpd failed to start"
else
    echo "[tshoot] WARNING — lldpd not found in rescue image"
fi

# --------------------------------------------------------------------------- #
# 4.  Optional management network (bond + VLAN + per-host IP)                 #
# --------------------------------------------------------------------------- #
NETCFG="/etc/tshoot/target-network.conf"
if [[ -f "$NETCFG" && -n "$HOST_IP" ]]; then
    echo "[tshoot] Configuring management network..."

    # shellcheck source=/dev/null
    source "$NETCFG"

    # Determine the management interface (built bottom-up)
    _mgmt_iface=""

    # ---- Bond (per-host members from CSV, colon-separated) ----
    if [[ -n "$HOST_BOND_MEMBERS" ]]; then
        modprobe bonding 2>/dev/null || true
        _mode="${BOND_MODE:-802.3ad}"
        case "$_mode" in
            802.3ad|4)       _km=4 ;; active-backup|1) _km=1 ;;
            balance-rr|0)    _km=0 ;; balance-xor|2)   _km=2 ;;
            broadcast|3)     _km=3 ;; balance-tlb|5)   _km=5 ;;
            balance-alb|6)   _km=6 ;; *)               _km=4 ;;
        esac

        ip link add bond0 type bond mode "$_km" 2>/dev/null || true

        # CSV uses ":" as separator for bond members
        IFS=':' read -ra _members <<< "$HOST_BOND_MEMBERS"
        for m in "${_members[@]}"; do
            m=${m// /}
            ip link set "$m" down  2>/dev/null
            ip link set "$m" master bond0 2>/dev/null \
                && echo "[tshoot]   $m -> bond0" \
                || echo "[tshoot]   WARNING — $m could not join bond0"
            ip link set "$m" up 2>/dev/null
        done
        ip link set bond0 up 2>/dev/null
        _mgmt_iface="bond0"
        echo "[tshoot] bond0 created  (mode $_mode)"
    fi

    # ---- VLAN (optional, on top of bond0 or first physical NIC) ----
    if [[ -n "${VLAN_ID:-}" ]]; then
        modprobe 8021q 2>/dev/null || true
        _parent="${_mgmt_iface:-eth0}"
        _vif="${_parent}.${VLAN_ID}"
        ip link add link "$_parent" name "$_vif" type vlan id "$VLAN_ID" 2>/dev/null || true
        ip link set "$_vif" up 2>/dev/null
        _mgmt_iface="$_vif"
        echo "[tshoot] VLAN      $_vif"
    fi

    # Fall back to first physical NIC if no bond/VLAN
    if [[ -z "$_mgmt_iface" ]]; then
        _mgmt_iface=$(ip -o link show | awk -F': ' '/state UP/ && !/lo/{print $2; exit}')
        [[ -z "$_mgmt_iface" ]] && _mgmt_iface="eth0"
    fi

    # ---- Per-host IP (from CSV) + shared netmask ----
    ip addr add "${HOST_IP}${NETMASK}" dev "$_mgmt_iface" 2>/dev/null || true
    echo "[tshoot] IP        ${HOST_IP}${NETMASK} on $_mgmt_iface"

    # ---- Default gateway ----
    [[ -n "${GATEWAY:-}" ]] && {
        ip route add default via "$GATEWAY" 2>/dev/null || true
        echo "[tshoot] gateway   $GATEWAY"
    }

    # ---- DNS ----
    [[ -n "${DNS_SERVERS:-}" ]] && {
        : > /etc/resolv.conf
        IFS=',' read -ra _ns <<< "$DNS_SERVERS"
        for n in "${_ns[@]}"; do echo "nameserver ${n// /}" >> /etc/resolv.conf; done
        echo "[tshoot] DNS       $DNS_SERVERS"
    }

    # ---- HTTP(S) proxy ----
    [[ -n "${PROXY:-}" ]] && {
        export http_proxy="$PROXY" https_proxy="$PROXY"
        export HTTP_PROXY="$PROXY" HTTPS_PROXY="$PROXY"
        echo "[tshoot] proxy     $PROXY"
    }
elif [[ -f "$NETCFG" && -z "$HOST_IP" ]]; then
    echo "[tshoot] WARNING — network config present but no IP for this host (serial: ${SERIAL:-unknown})"
fi

echo ""
echo "==== Troubleshooting environment ready ===="
echo ""
echo "  Tools available:"
command -v lldpcli  &>/dev/null && echo "    lldpcli   — show LLDP neighbours  (lldpcli show neighbors)"
command -v tcpdump  &>/dev/null && echo "    tcpdump   — packet capture"
command -v nic-xray &>/dev/null && echo "    nic-xray  — NIC diagnostics"
echo ""
echo "  To restore the OS to local disks, run:  rear recover"
echo ""
TSHOOT_EOF
}

# Write the firstboot script to stdout.
# Runs once on the restored system to set hostname and configure the target
# user account from DMI serial + CSV + target-identity.conf.
generate_firstboot_script() {
    cat <<'FIRSTBOOT_EOF'
#!/bin/bash
# tshoot-firstboot.sh — First-boot identity setup for restored hosts
#
# Runs once after rear recover deploys the system to disk.
#   1. Reads DMI serial → sets hostname from CSV inventory
#   2. Configures target user account (password + SSH key) if provided
#   3. Disables itself

IDENTITY_CONF="/etc/tshoot/target-identity.conf"
CSV="/etc/tshoot/hosts.csv"
SERIAL=$(dmidecode -s system-serial-number 2>/dev/null | tr -d '[:space:]')

# --- Hostname from DMI serial -------------------------------------------------

if [[ -n "$SERIAL" && "$SERIAL" != "None" && -f "$CSV" ]]; then
    HOST_NAME=$(awk -F, -v s="$SERIAL" \
        '/^[[:space:]]*#/{next} /^[[:space:]]*$/{next}
         {gsub(/[[:space:]]/,"",$1); gsub(/[[:space:]]/,"",$2)}
         $1==s{print $2; exit}' "$CSV")
    if [[ -n "$HOST_NAME" ]]; then
        hostnamectl set-hostname "$HOST_NAME" 2>/dev/null || {
            echo "$HOST_NAME" > /etc/hostname
            hostname "$HOST_NAME"
        }
        logger -t tshoot-firstboot "Hostname set to $HOST_NAME (serial: $SERIAL)"
    else
        logger -t tshoot-firstboot "No CSV mapping for serial '$SERIAL'"
    fi
else
    logger -t tshoot-firstboot "No DMI serial or CSV not found — skipping hostname"
fi

# --- Target user account ------------------------------------------------------

if [[ -f "$IDENTITY_CONF" ]]; then
    # shellcheck source=/dev/null
    source "$IDENTITY_CONF"

    if [[ -n "${TARGET_USER:-}" ]]; then
        # Create user if it does not exist (with sudo access)
        if ! id "$TARGET_USER" &>/dev/null; then
            useradd -m -s /bin/bash "$TARGET_USER" 2>/dev/null || true
            logger -t tshoot-firstboot "Created user $TARGET_USER"
        fi

        # Grant sudo access
        if [[ -d /etc/sudoers.d ]]; then
            echo "$TARGET_USER ALL=(ALL) NOPASSWD:ALL" > "/etc/sudoers.d/90-${TARGET_USER}"
            chmod 440 "/etc/sudoers.d/90-${TARGET_USER}"
        fi

        # Set password (hashed value from config)
        if [[ -n "${TARGET_PASSWORD_HASH:-}" ]]; then
            echo "${TARGET_USER}:${TARGET_PASSWORD_HASH}" | chpasswd -e 2>/dev/null
            logger -t tshoot-firstboot "Password set for $TARGET_USER"
        fi

        # Install SSH public key
        if [[ -n "${TARGET_SSH_KEY:-}" ]]; then
            local_home=$(eval echo "~${TARGET_USER}")
            ssh_dir="${local_home}/.ssh"
            mkdir -p "$ssh_dir"
            echo "$TARGET_SSH_KEY" >> "$ssh_dir/authorized_keys"
            chmod 700 "$ssh_dir"
            chmod 600 "$ssh_dir/authorized_keys"
            chown -R "${TARGET_USER}:${TARGET_USER}" "$ssh_dir" 2>/dev/null || \
                chown -R "${TARGET_USER}:" "$ssh_dir" 2>/dev/null || true
            logger -t tshoot-firstboot "SSH key installed for $TARGET_USER"
        fi
    fi

    # Also apply to root if TARGET_USER is root, or separately configure root
    if [[ "${TARGET_USER:-}" == "root" ]]; then
        if [[ -n "${TARGET_SSH_KEY:-}" ]]; then
            mkdir -p /root/.ssh
            echo "$TARGET_SSH_KEY" >> /root/.ssh/authorized_keys
            chmod 700 /root/.ssh
            chmod 600 /root/.ssh/authorized_keys
        fi
    fi
fi

# --- Network configuration ----------------------------------------------------
# The restored system has the build VM's network config which references
# different interface names.  Replace it with a config that matches the
# actual hardware.  Dispatch to the correct generator for the distro.

NETCFG="/etc/tshoot/target-network.conf"
DISTRO_CONF="/etc/tshoot/distro.conf"

# Detect distro — prefer build-time metadata, fall back to /etc/os-release
if [[ -f "$DISTRO_CONF" ]]; then
    # shellcheck source=/dev/null
    source "$DISTRO_CONF"
else
    . /etc/os-release 2>/dev/null || true
    case "${ID:-}" in
        rocky)                DISTRO_FAMILY="rocky"    ;;
        ubuntu)               DISTRO_FAMILY="ubuntu"   ;;
        opensuse-leap|opensuse) DISTRO_FAMILY="opensuse" ;;
        *)                    DISTRO_FAMILY="unknown"  ;;
    esac
    DISTRO_VERSION="${VERSION_ID:-}"
fi

# Discover physical interfaces (have /sys/class/net/*/device)
declare -a PHYS_IFACES=()
for dev_link in /sys/class/net/*/device; do
    [[ -e "$dev_link" ]] || continue
    iface=$(basename "$(dirname "$dev_link")")
    PHYS_IFACES+=("$iface")
done

if [[ ${#PHYS_IFACES[@]} -gt 0 ]]; then
    # Load target network params if available
    if [[ -f "$NETCFG" ]]; then
        # shellcheck source=/dev/null
        source "$NETCFG"
    fi

    # Look up per-host bond members and IP from CSV
    HOST_BOND_MEMBERS=""
    HOST_IP=""
    if [[ -n "$SERIAL" && "$SERIAL" != "None" && -f "$CSV" ]]; then
        HOST_BOND_MEMBERS=$(awk -F, -v s="$SERIAL" \
            '/^[[:space:]]*#/{next} /^[[:space:]]*$/{next}
             {gsub(/[[:space:]]/,"",$1); gsub(/[[:space:]]/,"",$3)}
             $1==s{print $3; exit}' "$CSV")
        HOST_IP=$(awk -F, -v s="$SERIAL" \
            '/^[[:space:]]*#/{next} /^[[:space:]]*$/{next}
             {gsub(/[[:space:]]/,"",$1); gsub(/[[:space:]]/,"",$4)}
             $1==s{print $4; exit}' "$CSV")
    fi

    # Validate bond members exist on this system
    _bond_valid=true
    if [[ -n "$HOST_BOND_MEMBERS" && -n "${NETMASK:-}" ]]; then
        IFS=':' read -ra _members <<< "$HOST_BOND_MEMBERS"
        for m in "${_members[@]}"; do
            m=${m// /}
            if [[ ! -e "/sys/class/net/$m" ]]; then
                logger -t tshoot-firstboot "Bond member $m not found — skipping bond config"
                _bond_valid=false
                break
            fi
        done
    fi
    _have_bond=false
    [[ -n "$HOST_BOND_MEMBERS" && -n "${NETMASK:-}" && "$_bond_valid" == "true" ]] && _have_bond=true

    # ---- Distro-specific network config generators ---------------------------

    # Determine the network manager in use
    _net_mgr="unknown"
    case "$DISTRO_FAMILY" in
        ubuntu)   _net_mgr="netplan" ;;
        rocky)    _net_mgr="nm" ;;
        opensuse)
            local _suse_major="${DISTRO_VERSION%%.*}"
            if (( _suse_major >= 16 )) || systemctl is-active --quiet NetworkManager 2>/dev/null; then
                _net_mgr="nm"
            else
                _net_mgr="wicked"
            fi
            ;;
    esac

    # ---- Clean stale cloud-init / build-VM network configs -------------------
    case "$_net_mgr" in
        netplan)
            rm -f /etc/netplan/*.yaml 2>/dev/null
            ;;
        nm)
            rm -f /etc/NetworkManager/system-connections/*cloud-init* 2>/dev/null
            rm -f /etc/sysconfig/network-scripts/ifcfg-e* 2>/dev/null
            ;;
        wicked)
            rm -f /etc/sysconfig/network/ifcfg-eth* /etc/sysconfig/network/ifcfg-ens* \
                  /etc/sysconfig/network/ifcfg-en* 2>/dev/null
            ;;
    esac

    # ======================================================================== #
    # NETPLAN (Ubuntu)                                                         #
    # ======================================================================== #
    if [[ "$_net_mgr" == "netplan" ]]; then
        NP="/etc/netplan/50-tshoot.yaml"

        if [[ "$_have_bond" == "true" ]]; then
            IFS=':' read -ra _members <<< "$HOST_BOND_MEMBERS"
            _yaml_members=""
            for m in "${_members[@]}"; do _yaml_members+="        - ${m// /}"$'\n'; done
            _bond_mode="${BOND_MODE:-802.3ad}"

            cat > "$NP" <<NPEOF
# Generated by tshoot-firstboot.sh
network:
  version: 2
  renderer: networkd
  ethernets:
NPEOF
            for iface in "${PHYS_IFACES[@]}"; do
                cat >> "$NP" <<NPEOF
    ${iface}:
      dhcp4: false
      dhcp6: false
      optional: true
NPEOF
            done
            cat >> "$NP" <<NPEOF
  bonds:
    bond0:
      interfaces:
${_yaml_members}      parameters:
        mode: ${_bond_mode}
        mii-monitor-interval: 100
      dhcp4: false
      dhcp6: false
NPEOF
            if [[ -n "${VLAN_ID:-}" ]]; then
                cat >> "$NP" <<NPEOF
  vlans:
    bond0.${VLAN_ID}:
      id: ${VLAN_ID}
      link: bond0
      addresses:
        - ${HOST_IP}${NETMASK}
NPEOF
            else
                cat >> "$NP" <<NPEOF
      addresses:
        - ${HOST_IP}${NETMASK}
NPEOF
            fi
            [[ -n "${GATEWAY:-}" ]] && cat >> "$NP" <<NPEOF
      routes:
        - to: default
          via: ${GATEWAY}
NPEOF
            if [[ -n "${DNS_SERVERS:-}" ]]; then
                IFS=',' read -ra _ns <<< "$DNS_SERVERS"
                _yaml_dns=""
                for n in "${_ns[@]}"; do _yaml_dns+="          - ${n// /}"$'\n'; done
                cat >> "$NP" <<NPEOF
      nameservers:
        addresses:
${_yaml_dns}NPEOF
            fi
        else
            # Fallback: DHCP on all physical interfaces
            cat > "$NP" <<NPEOF
# Generated by tshoot-firstboot.sh (fallback — no matching bond members)
network:
  version: 2
  renderer: networkd
  ethernets:
NPEOF
            for iface in "${PHYS_IFACES[@]}"; do
                cat >> "$NP" <<NPEOF
    ${iface}:
      dhcp4: true
      dhcp6: false
      optional: true
NPEOF
            done
        fi
        chmod 600 "$NP"
        netplan apply 2>/dev/null || true

    # ======================================================================== #
    # NETWORKMANAGER (Rocky, openSUSE 16.x)                                    #
    # ======================================================================== #
    elif [[ "$_net_mgr" == "nm" ]]; then
        _conn_dir="/etc/NetworkManager/system-connections"
        mkdir -p "$_conn_dir"

        if [[ "$_have_bond" == "true" ]]; then
            _bond_mode="${BOND_MODE:-802.3ad}"
            # Map bond mode name to NM numeric mode
            case "$_bond_mode" in
                802.3ad|4)       _nm_mode="802.3ad" ;;
                active-backup|1) _nm_mode="active-backup" ;;
                balance-rr|0)    _nm_mode="balance-rr" ;;
                balance-xor|2)   _nm_mode="balance-xor" ;;
                broadcast|3)     _nm_mode="broadcast" ;;
                balance-tlb|5)   _nm_mode="balance-tlb" ;;
                balance-alb|6)   _nm_mode="balance-alb" ;;
                *)               _nm_mode="802.3ad" ;;
            esac

            # Bond master — no IP on bond0 when VLAN is used
            if [[ -n "${VLAN_ID:-}" ]]; then
                cat > "${_conn_dir}/bond0.nmconnection" <<NMEOF
[connection]
id=bond0
type=bond
interface-name=bond0

[bond]
mode=${_nm_mode}
miimon=100

[ipv4]
method=disabled

[ipv6]
method=disabled
NMEOF
            else
                # IP directly on bond0
                _dns_list=""
                if [[ -n "${DNS_SERVERS:-}" ]]; then
                    _dns_list=$(echo "$DNS_SERVERS" | tr ',' ';')
                    _dns_list="${_dns_list};"
                fi
                cat > "${_conn_dir}/bond0.nmconnection" <<NMEOF
[connection]
id=bond0
type=bond
interface-name=bond0

[bond]
mode=${_nm_mode}
miimon=100

[ipv4]
method=manual
address1=${HOST_IP}${NETMASK},${GATEWAY:-}
dns=${_dns_list}

[ipv6]
method=disabled
NMEOF
            fi

            # Bond slaves
            IFS=':' read -ra _members <<< "$HOST_BOND_MEMBERS"
            for m in "${_members[@]}"; do
                m=${m// /}
                cat > "${_conn_dir}/bond0-slave-${m}.nmconnection" <<NMEOF
[connection]
id=bond0-slave-${m}
type=ethernet
interface-name=${m}
master=bond0
slave-type=bond
NMEOF
            done

            # VLAN on bond0
            if [[ -n "${VLAN_ID:-}" ]]; then
                _dns_list=""
                if [[ -n "${DNS_SERVERS:-}" ]]; then
                    _dns_list=$(echo "$DNS_SERVERS" | tr ',' ';')
                    _dns_list="${_dns_list};"
                fi
                cat > "${_conn_dir}/bond0.${VLAN_ID}.nmconnection" <<NMEOF
[connection]
id=bond0.${VLAN_ID}
type=vlan
interface-name=bond0.${VLAN_ID}

[vlan]
id=${VLAN_ID}
parent=bond0

[ipv4]
method=manual
address1=${HOST_IP}${NETMASK},${GATEWAY:-}
dns=${_dns_list}

[ipv6]
method=disabled
NMEOF
            fi
        else
            # Fallback: DHCP on all physical interfaces
            for iface in "${PHYS_IFACES[@]}"; do
                cat > "${_conn_dir}/tshoot-${iface}.nmconnection" <<NMEOF
[connection]
id=tshoot-${iface}
type=ethernet
interface-name=${iface}

[ipv4]
method=auto

[ipv6]
method=disabled
NMEOF
            done
        fi

        chmod 600 "${_conn_dir}"/*.nmconnection 2>/dev/null
        nmcli connection reload 2>/dev/null || true
        systemctl restart NetworkManager 2>/dev/null || true

    # ======================================================================== #
    # WICKED (openSUSE Leap 15.x)                                             #
    # ======================================================================== #
    elif [[ "$_net_mgr" == "wicked" ]]; then
        _net_dir="/etc/sysconfig/network"

        if [[ "$_have_bond" == "true" ]]; then
            _bond_mode="${BOND_MODE:-802.3ad}"
            IFS=':' read -ra _members <<< "$HOST_BOND_MEMBERS"

            # Build BONDING_SLAVEn entries
            _slave_lines=""
            _idx=0
            for m in "${_members[@]}"; do
                m=${m// /}
                _slave_lines+="BONDING_SLAVE_${_idx}='${m}'"$'\n'
                (( _idx++ ))
                # Create a minimal ifcfg for each slave
                cat > "${_net_dir}/ifcfg-${m}" <<WEOF
STARTMODE='auto'
BOOTPROTO='none'
WEOF
            done

            # Bond master
            cat > "${_net_dir}/ifcfg-bond0" <<WEOF
STARTMODE='auto'
BONDING_MASTER='yes'
BONDING_MODULE_OPTS='mode=${_bond_mode} miimon=100'
${_slave_lines}BOOTPROTO='none'
WEOF

            if [[ -n "${VLAN_ID:-}" ]]; then
                # VLAN on bond0
                cat > "${_net_dir}/ifcfg-bond0.${VLAN_ID}" <<WEOF
STARTMODE='auto'
ETHERDEVICE='bond0'
VLAN_ID='${VLAN_ID}'
BOOTPROTO='static'
IPADDR='${HOST_IP}${NETMASK}'
WEOF
                # Default route
                [[ -n "${GATEWAY:-}" ]] && echo "default ${GATEWAY} - -" > "${_net_dir}/routes"
            else
                # IP directly on bond0
                cat >> "${_net_dir}/ifcfg-bond0" <<WEOF
IPADDR='${HOST_IP}${NETMASK}'
WEOF
                [[ -n "${GATEWAY:-}" ]] && echo "default ${GATEWAY} - -" > "${_net_dir}/routes"
            fi

            # DNS
            if [[ -n "${DNS_SERVERS:-}" ]]; then
                _dns_static=$(echo "$DNS_SERVERS" | tr ',' ' ')
                sed -i "s/^NETCONFIG_DNS_STATIC_SERVERS=.*/NETCONFIG_DNS_STATIC_SERVERS=\"${_dns_static}\"/" \
                    "${_net_dir}/config" 2>/dev/null || \
                    echo "NETCONFIG_DNS_STATIC_SERVERS=\"${_dns_static}\"" >> "${_net_dir}/config"
            fi
        else
            # Fallback: DHCP on all physical interfaces
            for iface in "${PHYS_IFACES[@]}"; do
                cat > "${_net_dir}/ifcfg-${iface}" <<WEOF
STARTMODE='auto'
BOOTPROTO='dhcp'
WEOF
            done
        fi

        systemctl restart wicked 2>/dev/null || wicked ifup all 2>/dev/null || true
    fi

    logger -t tshoot-firstboot "Network configured ($_net_mgr): ${PHYS_IFACES[*]}"
else
    logger -t tshoot-firstboot "No physical interfaces found — skipping network"
fi

# --- Disable after first run --------------------------------------------------
systemctl disable tshoot-firstboot.service 2>/dev/null || true
logger -t tshoot-firstboot "Firstboot complete — service disabled"
FIRSTBOOT_EOF
}

# Write the systemd unit for the firstboot service to stdout.
generate_firstboot_unit() {
    cat <<'UNIT_EOF'
[Unit]
Description=Set hostname, user credentials, and network from DMI serial (tshoot firstboot)
After=network.target local-fs.target
ConditionPathExists=/etc/tshoot/hosts.csv

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/tshoot-firstboot.sh
RemainAfterExit=no

[Install]
WantedBy=multi-user.target
UNIT_EOF
}

# Write the persistent link-up service unit to stdout.
# Runs every boot AFTER networkd to bring up all physical NICs at the link
# layer — required for LLDP and diagnostics regardless of IP configuration.
generate_linkup_unit() {
    cat <<'LINKUP_EOF'
[Unit]
Description=Bring all physical NICs to link-up state (tshoot)
After=systemd-networkd.service NetworkManager-wait-online.service wicked.service network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/bin/bash -c 'for d in /sys/class/net/*/device; do [ -e "$d" ] && ip link set "$(basename "$(dirname "$d")")" up 2>/dev/null; done'
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
LINKUP_EOF
}

# Write target-identity.conf to stdout (user account params for restored hosts).
generate_target_identity_conf() {
    echo "# Target-host identity — generated by pve-create-tshoot-image"
    echo "# Applied on first boot of the restored system."
    [[ -n "$TARGET_USER" ]] && echo "TARGET_USER=$TARGET_USER"
    if [[ -n "$TARGET_PASSWORD" ]]; then
        local hash
        hash=$(openssl passwd -6 "$TARGET_PASSWORD" 2>/dev/null) || \
            hash=$(echo "$TARGET_PASSWORD" | mkpasswd -m sha-512 --stdin 2>/dev/null) || \
            die "Cannot hash password — install openssl or whois (mkpasswd)"
        echo "TARGET_PASSWORD_HASH='$hash'"
    fi
    if [[ -n "$TARGET_SSH_KEY" ]]; then
        local key_content
        if [[ -f "$TARGET_SSH_KEY" ]]; then
            key_content=$(cat "$TARGET_SSH_KEY")
        else
            key_content="$TARGET_SSH_KEY"
        fi
        echo "TARGET_SSH_KEY='$key_content'"
    fi
    return 0
}

# --- Offline file injection (via virt-customize) ------------------------------

# Inject configuration files into the disk image.  Lightweight — no package
# installation, no network access.  The VM must be stopped when this runs.
inject_files() {
    local disk="$1"

    msg "Injecting configuration files into disk image..."

    # Generate files that depend on detected distro (DISTRO_FAMILY/VERSION)
    generate_rear_conf > "$TEMP_DIR/rear-local.conf"

    local _need_dhclient=true
    if [[ "$DISTRO_FAMILY" == "ubuntu" ]]; then
        local _ub_major="${DISTRO_VERSION%%.*}"
        (( _ub_major >= 24 )) && _need_dhclient=false
    fi
    if [[ "$_need_dhclient" == "true" ]]; then
        generate_dhclient_conf > "$TEMP_DIR/dhclient.conf"
    fi

    local -a vc=( -a "$disk" )

    # ---- create target directories -------------------------------------------
    vc+=( --run-command "mkdir -p /etc/tshoot" )
    vc+=( --run-command "mkdir -p /etc/rear" )
    vc+=( --run-command "mkdir -p /usr/share/rear/skel/default/etc/scripts/system-setup.d" )
    vc+=( --run-command "mkdir -p /usr/share/rear/skel/default/etc/tshoot" )

    # ---- inject files --------------------------------------------------------

    # CSV (serial,hostname,bond_members,ip) — in /etc/tshoot (backed up by
    # ReaR) and in the skel tree (copied into the rescue system at ISO build)
    vc+=( --upload "$TEMP_DIR/hosts.csv:/etc/tshoot/hosts.csv" )
    vc+=( --upload "$TEMP_DIR/hosts.csv:/usr/share/rear/skel/default/etc/tshoot/hosts.csv" )

    # dhclient.conf with short timeout (skipped for distros without dhclient)
    if [[ -f "$TEMP_DIR/dhclient.conf" ]]; then
        vc+=( --run-command "mkdir -p /etc/dhcp" )
        vc+=( --upload "$TEMP_DIR/dhclient.conf:/etc/dhclient.conf" )
        vc+=( --upload "$TEMP_DIR/dhclient.conf:/etc/dhcp/dhclient.conf" )
        vc+=( --upload "$TEMP_DIR/dhclient.conf:/usr/share/rear/skel/default/etc/dhclient.conf" )
    fi

    # DHCP timeout patch → runs before 58-start-dhclient.sh in rescue system
    vc+=( --upload "$TEMP_DIR/57-set-dhcp-timeout.sh:/usr/share/rear/skel/default/etc/scripts/system-setup.d/57-set-dhcp-timeout.sh" )
    vc+=( --chmod "0755:/usr/share/rear/skel/default/etc/scripts/system-setup.d/57-set-dhcp-timeout.sh" )

    # Boot-time setup script → rescue system-setup.d
    vc+=( --upload "$TEMP_DIR/90-tshoot-setup.sh:/usr/share/rear/skel/default/etc/scripts/system-setup.d/90-tshoot-setup.sh" )
    vc+=( --chmod "0755:/usr/share/rear/skel/default/etc/scripts/system-setup.d/90-tshoot-setup.sh" )

    # Target-host network configuration (optional)
    if [[ -f "$TEMP_DIR/target-network.conf" ]]; then
        vc+=( --upload "$TEMP_DIR/target-network.conf:/etc/tshoot/target-network.conf" )
        vc+=( --upload "$TEMP_DIR/target-network.conf:/usr/share/rear/skel/default/etc/tshoot/target-network.conf" )
    fi

    # ReaR local.conf
    vc+=( --upload "$TEMP_DIR/rear-local.conf:/etc/rear/local.conf" )

    # Firstboot service — sets hostname + user credentials on restored system
    vc+=( --upload "$TEMP_DIR/tshoot-firstboot.sh:/usr/local/sbin/tshoot-firstboot.sh" )
    vc+=( --chmod "0755:/usr/local/sbin/tshoot-firstboot.sh" )
    vc+=( --upload "$TEMP_DIR/tshoot-firstboot.service:/etc/systemd/system/tshoot-firstboot.service" )

    # Persistent link-up service — brings all physical NICs up every boot
    vc+=( --upload "$TEMP_DIR/tshoot-linkup.service:/etc/systemd/system/tshoot-linkup.service" )

    # Distro metadata for firstboot runtime detection (generated here because
    # detect_distro() runs after generate_files())
    cat > "$TEMP_DIR/distro.conf" <<DISTRO_EOF
# Distro metadata — generated by pve-create-tshoot-image at build time.
DISTRO_FAMILY=$DISTRO_FAMILY
DISTRO_VERSION=$DISTRO_VERSION
DISTRO_EOF
    vc+=( --upload "$TEMP_DIR/distro.conf:/etc/tshoot/distro.conf" )
    vc+=( --upload "$TEMP_DIR/distro.conf:/usr/share/rear/skel/default/etc/tshoot/distro.conf" )

    # Target-host identity configuration (optional)
    if [[ -f "$TEMP_DIR/target-identity.conf" ]]; then
        vc+=( --upload "$TEMP_DIR/target-identity.conf:/etc/tshoot/target-identity.conf" )
        vc+=( --chmod "0600:/etc/tshoot/target-identity.conf" )
    fi

    # ---- enable guest-exec in the QEMU guest agent ---------------------------
    # Cloud images ship with an allow-list that excludes guest-exec.
    # We need it for package installation and running ReaR without SSH.
    # Clear the filter so all RPCs are allowed (this is a temp build VM).
    # Add guest-exec and guest-exec-status to the guest agent allow-list.
    # Cloud images ship with an allow-list that excludes exec RPCs.
    vc+=( --run-command "if grep -q '^FILTER_RPC_ARGS=.*--allow-rpcs=' /etc/sysconfig/qemu-ga 2>/dev/null; then \
        sed -i 's/--allow-rpcs=\\(.*\\)\"/--allow-rpcs=\\1,guest-exec,guest-exec-status\"/' /etc/sysconfig/qemu-ga; \
    elif [ -f /etc/sysconfig/qemu-ga ]; then \
        sed -i 's/^FILTER_RPC_ARGS=.*/FILTER_RPC_ARGS=\"--allow-rpcs=guest-ping,guest-info,guest-sync,guest-sync-delimited,guest-exec,guest-exec-status,guest-network-get-interfaces,guest-get-osinfo,guest-shutdown,guest-get-host-name\"/' /etc/sysconfig/qemu-ga; \
    fi || true" )
    vc+=( --run-command "mkdir -p /etc/qemu" )

    # ---- set SELinux to permissive for the build VM --------------------------
    # On RHEL/Rocky, SELinux confines the guest agent to virt_qemu_ga_t which
    # blocks command execution and file writes.  Permissive mode allows
    # guest-exec to work.  This only affects the build VM, not the final ISO.
    vc+=( --run-command "sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config 2>/dev/null || true" )

    # ---- wipe host identity --------------------------------------------------
    # Each restored system must regenerate unique IDs on first boot.
    vc+=( --run-command 'truncate -s 0 /etc/machine-id 2>/dev/null || true' )
    vc+=( --run-command 'rm -f /var/lib/dbus/machine-id; ln -sf /etc/machine-id /var/lib/dbus/machine-id 2>/dev/null || true' )
    vc+=( --run-command 'rm -f /etc/ssh/ssh_host_*' )
    vc+=( --run-command 'rm -f /var/lib/systemd/random-seed 2>/dev/null || true' )
    vc+=( --run-command 'echo localhost > /etc/hostname' )

    run virt-customize "${vc[@]}"

    ok "File injection complete"
}

# --- Online package installation (via guest agent) ----------------------------

# Install packages inside the running VM via qm guest exec.  Called after
# cloud-init has grown the filesystem and network is available.
install_packages() {
    local major="${DISTRO_VERSION%%.*}"

    msg "Installing packages ($DISTRO_FAMILY $DISTRO_VERSION)..."

    # Wait for cloud-init AND background package managers to finish.
    # Poll instead of running a long-lived command — the guest agent may
    # restart during cloud-init, killing any in-flight guest-exec.
    msg "Waiting for cloud-init to finish..."
    local ci_attempts=0
    while (( ci_attempts < 60 )); do
        local ci_out
        ci_out=$(vm_exec 15 "cloud-init status 2>/dev/null; pgrep -x 'zypper|dnf|apt-get|dpkg' > /dev/null && echo PKG_RUNNING || echo PKG_IDLE" 2>/dev/null) || {
            # Agent might have restarted — wait and retry
            sleep 10
            wait_for_agent "$TEMP_VMID" 60 || true
            (( ci_attempts++ )) || true
            continue
        }
        if echo "$ci_out" | grep -q 'done\|error' && echo "$ci_out" | grep -q 'PKG_IDLE'; then
            break
        fi
        sleep 10
        (( ci_attempts++ )) || true
    done
    ok "Cloud-init complete"

    # Configure HTTP proxy in the build VM if --vm-proxy is specified.
    if [[ -n "$VM_PROXY" ]]; then
        ok "Build-VM proxy: $VM_PROXY"
        vm_exec 30 "
            echo 'proxy=${VM_PROXY}' >> /etc/dnf/dnf.conf 2>/dev/null
            printf 'Acquire::http::Proxy \"${VM_PROXY}\";\nAcquire::https::Proxy \"${VM_PROXY}\";\n' > /etc/apt/apt.conf.d/99proxy 2>/dev/null
            mkdir -p /etc/sysconfig 2>/dev/null
            printf 'PROXY_ENABLED=\"yes\"\nHTTP_PROXY=\"${VM_PROXY}\"\nHTTPS_PROXY=\"${VM_PROXY}\"\n' >> /etc/sysconfig/proxy 2>/dev/null
            true
        " > /dev/null 2>&1 || true
    fi

    # Build install script in TEMP_DIR, write it to the VM, then execute.
    # This avoids quoting/length issues with qm guest exec.
    local install_script="$TEMP_DIR/vm-install.sh"
    {
        echo '#!/bin/bash'
        # Ensure full PATH — guest agent may run with a restricted PATH
        echo 'export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
        # Redirect all output to a log file inside the VM — the guest agent
        # response buffer cannot handle large package manager output.
        echo 'exec > /var/log/tshoot-install.log 2>&1'
        # Do NOT use set -e — package managers may return non-zero for
        # non-fatal warnings (e.g. zypper exit 106 = repos skipped).
        # Each critical command is checked explicitly with || exit 1.
        # Proxy env (if set)
        [[ -n "$VM_PROXY" ]] && cat <<PROXY_EOF
export http_proxy=${VM_PROXY}
export https_proxy=${VM_PROXY}
export HTTP_PROXY=${VM_PROXY}
export HTTPS_PROXY=${VM_PROXY}
PROXY_EOF

        case "$DISTRO_FAMILY" in
            rocky)
                cat <<ROCKY_EOF
cat > /etc/yum.repos.d/nic-xray.repo << 'REPO'
[home_ciriarte_network-tools]
name=Network Tools (home:ciriarte:network-tools)
baseurl=https://download.opensuse.org/repositories/home:/ciriarte:/network-tools/Rocky_${major}/
enabled=1
gpgcheck=0
type=rpm-md
REPO
dnf install -y epel-release || exit 1
dnf install -y rear lldpd tcpdump nic-xray dmidecode efibootmgr genisoimage || exit 1
ROCKY_EOF
                if (( major >= 9 )); then
                    echo "echo 'BOOTLOADER=GRUB2-EFI' >> /etc/rear/local.conf"
                fi
                ;;
            ubuntu)
                local ub_repo="https://download.opensuse.org/repositories/home:/ciriarte:/network-tools/Ubuntu_${DISTRO_VERSION}"
                cat <<UBUNTU_EOF
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq || exit 1
apt-get install -y -qq curl gnupg || exit 1
mkdir -p /etc/apt/keyrings
curl -fsSL "${ub_repo}/Release.key" | gpg --dearmor -o /etc/apt/keyrings/nic-xray.gpg || exit 1
echo "deb [signed-by=/etc/apt/keyrings/nic-xray.gpg] ${ub_repo}/ /" > /etc/apt/sources.list.d/nic-xray.list
apt-get update -qq || exit 1
apt-get install -y -qq -o Dpkg::Options::="--force-confold" rear lldpd tcpdump nic-xray dmidecode efibootmgr genisoimage || exit 1
UBUNTU_EOF
                ;;
            opensuse)
                local suse_repo="https://download.opensuse.org/repositories/home:/ciriarte:/network-tools/openSUSE_Leap_${DISTRO_VERSION}"
                local suse_archiving="https://download.opensuse.org/repositories/Archiving/${DISTRO_VERSION}"
                cat <<SUSE_EOF
# Wait for DNS to be ready — cloud-init may report done before resolv.conf is usable
for _dns_try in \$(seq 1 30); do
    host download.opensuse.org >/dev/null 2>&1 && break
    getent hosts download.opensuse.org >/dev/null 2>&1 && break
    echo "Waiting for DNS (\${_dns_try}/30)..."
    sleep 2
done
zypper addrepo --no-gpgcheck "${suse_repo}/" nic-xray 2>/dev/null || true
zypper --gpg-auto-import-keys --no-gpg-checks refresh 2>/dev/null || true
zypper -n --no-gpg-checks install lldpd tcpdump nic-xray dmidecode efibootmgr xorriso sysvinit-tools 2>/dev/null || true
zypper -n --no-gpg-checks install "${suse_archiving}/x86_64/rear-2.9-archiving.91.1.x86_64.rpm" 2>/dev/null || zypper -n --no-gpg-checks install rear 2>/dev/null || true
# Verify critical packages installed — zypper exit codes are unreliable
command -v rear > /dev/null || { echo "FATAL: rear not installed"; exit 1; }
command -v tcpdump > /dev/null || { echo "FATAL: tcpdump not installed"; exit 1; }
command -v lldpd > /dev/null || { echo "FATAL: lldpd not installed"; exit 1; }
SUSE_EOF
                ;;
        esac

        echo 'systemctl enable lldpd 2>/dev/null || true'
    } > "$install_script"

    # Write the install script into the VM via guest agent.
    # Re-check agent availability — cloud-init may restart the agent service.
    wait_for_agent "$TEMP_VMID" 60 || die "Guest agent lost during package setup"

    local script_b64
    script_b64=$(base64 -w0 "$install_script")
    vm_exec 10 "echo '${script_b64}' | base64 -d > /tmp/install.sh; chmod +x /tmp/install.sh" > /dev/null 2>&1 || \
        die "Failed to write install script to VM"

    # Execute the install script in the background inside the VM.
    # Long-running guest-exec calls are unreliable (agent restarts, buffer
    # overflows, timeouts).  Instead: start in background, poll for completion.
    vm_exec 10 "nohup bash -c 'bash /tmp/install.sh; echo \$? > /tmp/install.exitcode' > /dev/null 2>&1 &" > /dev/null 2>&1 || true

    local elapsed=0 install_status=""
    while (( elapsed < REAR_TIMEOUT )); do
        sleep 15
        (( elapsed += 15 )) || true
        install_status=$(vm_exec 10 "cat /tmp/install.exitcode 2>/dev/null" 2>/dev/null) || {
            # Agent might have restarted — retry after wait
            wait_for_agent "$TEMP_VMID" 30 || true
            continue
        }
        install_status=$(echo "$install_status" | tr -d '[:space:]')
        [[ -n "$install_status" ]] && break
    done

    if [[ "$install_status" != "0" ]]; then
        err "Package installation failed (exit code: ${install_status:-timeout})"
        err "Last 20 lines from VM install log:"
        vm_exec 10 "tail -20 /var/log/tshoot-install.log 2>/dev/null" >&2 || true
        die "Package installation failed on $DISTRO_FAMILY"
    fi

    # Remove build-VM proxy config so it doesn't leak into the ISO
    if [[ -n "$VM_PROXY" ]]; then
        vm_exec 10 "sed -i '/^proxy=/d' /etc/dnf/dnf.conf 2>/dev/null; rm -f /etc/apt/apt.conf.d/99proxy 2>/dev/null; sed -i '/PROXY/d' /etc/sysconfig/proxy 2>/dev/null; true" > /dev/null 2>&1 || true
    fi

    # Enable the firstboot service via systemd (must be done while VM is
    # running so systemd creates the proper symlinks in the wants directory).
    vm_exec 10 "systemctl enable tshoot-firstboot.service" > /dev/null 2>&1 || \
        warn "Could not enable tshoot-firstboot.service"
    vm_exec 10 "systemctl enable tshoot-linkup.service" > /dev/null 2>&1 || \
        warn "Could not enable tshoot-linkup.service"

    ok "Package installation complete"
}

# --- VM helpers ---------------------------------------------------------------

# Return the next free cluster-wide VMID.
find_free_vmid() {
    local vmid
    vmid=$(pvesh get /cluster/nextid 2>/dev/null | tr -d '"' | tr -d '\n')
    [[ -n "$vmid" && "$vmid" =~ ^[0-9]+$ ]] || die "Cannot determine next free VMID"
    echo "$vmid"
}

# Resolve the first disk of a VM to a system device / file path.
get_disk_path() {
    local vmid="$1"
    local disk_line ref path

    disk_line=$(qm config "$vmid" 2>/dev/null \
        | grep -E '^(scsi|virtio|ide|sata)[0-9]+:' \
        | grep -v 'cloudinit' \
        | head -1) || true

    [[ -z "$disk_line" ]] && die "VM ${vmid} has no data disks"

    ref=$(echo "$disk_line" | sed 's/^[^:]*: //' | cut -d, -f1 | tr -d '[:space:]')
    path=$(pvesm path "$ref" 2>/dev/null) || die "Cannot resolve disk path for '$ref'"

    echo "$path"
}

# Poll the QEMU guest agent until it responds or timeout.
wait_for_agent() {
    local vmid="$1" timeout="${2:-$BOOT_TIMEOUT}"
    local elapsed=0

    while (( elapsed < timeout )); do
        qm guest cmd "$vmid" ping &>/dev/null && return 0
        sleep 5
        (( elapsed += 5 ))
    done
    return 1
}

# Run a command inside the VM via the QEMU guest agent (virtio serial).
# No network connectivity required between host and VM.
# Usage: vm_exec TIMEOUT_SECS "command string"
vm_exec() {
    local timeout="$1"; shift
    local cmd="$*"
    local result qm_rc=0

    result=$(qm guest exec "$TEMP_VMID" --timeout "$timeout" -- bash -c "export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; $cmd" 2>&1) || qm_rc=$?

    # If qm itself failed (agent unreachable, timeout), there's no JSON to parse
    if ! echo "$result" | grep -q '"exitcode"'; then
        err "Guest agent error (qm exit $qm_rc): $result"
        return 1
    fi

    # Parse the JSON response — PVE 9.x returns plain-text out-data/err-data
    python3 -c "
import json, sys
d = json.load(sys.stdin)
for key, fd in [('out-data', sys.stdout), ('err-data', sys.stderr)]:
    v = d.get(key, '')
    if v:
        print(v, end='', file=fd)
sys.exit(d.get('exitcode', 1))
" <<< "$result"
}

# --- Remote mode (delegation) -------------------------------------------------

# In remote mode (-S user@host), the script copies itself and the CSV to the
# target PVE node, re-executes in local mode there, then downloads the
# resulting ISO back to the jump host.
delegate_to_server() {
    msg "Remote mode: delegating to ${PVE_SERVER}..."

    [[ -f "$0" ]] || die "--server requires running from a script file, not a pipe"

    local -a ssh_opts=( -o BatchMode=yes -o StrictHostKeyChecking=accept-new )

    # Create a scratch directory on the remote host
    local rdir
    rdir=$(ssh "${ssh_opts[@]}" "$PVE_SERVER" "mktemp -d /tmp/pve-tshoot-XXXXXX") || \
        die "Cannot create working directory on ${PVE_SERVER}"

    # Upload script and input files
    scp "${ssh_opts[@]}" "$0"        "${PVE_SERVER}:${rdir}/pve-create-tshoot-image"
    scp "${ssh_opts[@]}" "$CSV_FILE" "${PVE_SERVER}:${rdir}/hosts.csv"

    # Build remote invocation (without --server to avoid infinite delegation)
    local -a rcmd=( bash "${rdir}/pve-create-tshoot-image" )
    rcmd+=( --template "$TEMPLATE" )
    rcmd+=( --csv "${rdir}/hosts.csv" )
    rcmd+=( --storage "$STORAGE" )
    rcmd+=( --output "${rdir}/output" )
    rcmd+=( --rear-timeout "$REAR_TIMEOUT" )
    # Target-host network
    [[ -n "$BOND_MODE" ]]    && rcmd+=( --bond-mode "$BOND_MODE" )
    [[ -n "$VLAN_ID" ]]      && rcmd+=( --vlan-id "$VLAN_ID" )
    [[ -n "$NETMASK" ]]      && rcmd+=( --netmask "$NETMASK" )
    [[ -n "$GATEWAY" ]]      && rcmd+=( --gateway "$GATEWAY" )
    [[ -n "$DNS" ]]          && rcmd+=( --dns "$DNS" )
    [[ -n "$PROXY" ]]        && rcmd+=( --proxy "$PROXY" )
    # Target-host identity
    [[ -n "$TARGET_USER" ]]     && rcmd+=( --target-user "$TARGET_USER" )
    [[ -n "$TARGET_PASSWORD" ]] && rcmd+=( --target-password "$TARGET_PASSWORD" )
    # SSH key: resolve file to content before passing to remote (file is local)
    if [[ -n "$TARGET_SSH_KEY" ]]; then
        local key_val="$TARGET_SSH_KEY"
        [[ -f "$key_val" ]] && key_val=$(cat "$key_val")
        rcmd+=( --target-ssh-key "$key_val" )
    fi
    # Build-VM network
    rcmd+=( --vm-bridge "$VM_BRIDGE" )
    [[ -n "$VM_VLAN" ]]      && rcmd+=( --vm-vlan "$VM_VLAN" )
    [[ "$VM_NET_IP" != "dhcp" ]] && rcmd+=( --vm-ip "$VM_NET_IP" )
    [[ -n "$VM_NET_GW" ]]    && rcmd+=( --vm-gateway "$VM_NET_GW" )
    [[ -n "$VM_NET_DNS" ]]   && rcmd+=( --vm-dns "$VM_NET_DNS" )
    [[ -n "$VM_PROXY" ]]     && rcmd+=( --vm-proxy "$VM_PROXY" )
    # Flags
    (( RESCUE_ONLY )) && rcmd+=( --rescue-only )
    (( DRY_RUN ))     && rcmd+=( --dry-run )

    # Allocate a pseudo-terminal for coloured output
    local -a tty_flag=()
    [[ -t 1 ]] && tty_flag=( -t )

    ssh "${ssh_opts[@]}" "${tty_flag[@]}" "$PVE_SERVER" "${rcmd[@]}"

    # Download resulting ISO(s)
    mkdir -p "$OUTPUT_DIR"
    scp "${ssh_opts[@]}" "${PVE_SERVER}:${rdir}/output/*.iso" "$OUTPUT_DIR/" 2>/dev/null || \
        die "No ISO file produced on remote server"

    # Clean up remote scratch directory
    ssh "${ssh_opts[@]}" "$PVE_SERVER" "rm -rf ${rdir}" 2>/dev/null || true

    local iso
    iso=$(find "$OUTPUT_DIR" -maxdepth 1 -name '*.iso' -printf '%T@ %p\n' 2>/dev/null \
        | sort -rn | head -1 | cut -d' ' -f2-)
    [[ -n "$iso" ]] && msg "ISO downloaded: ${iso}"
    return 0
}

# --- Input validation ---------------------------------------------------------

validate_csv() {
    local file="$1"
    [[ -f "$file" ]] || die "CSV file not found: $file"

    local ipv4_re='^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
    local iface_re='^[a-zA-Z][a-zA-Z0-9._-]*(:[a-zA-Z][a-zA-Z0-9._-]*)*$'
    local line_no=0 data_lines=0
    while IFS= read -r line || [[ -n "$line" ]]; do
        (( line_no++ )) || true
        [[ "$line" =~ ^[[:space:]]*# ]] && continue
        [[ -z "${line// /}" ]]           && continue
        (( data_lines++ )) || true

        local nf
        nf=$(awk -F, '{print NF}' <<< "$line")
        (( nf >= 4 )) || die "CSV line ${line_no}: expected 'serial,hostname,bond_members,ip', got: $line"

        # Validate bond members (3rd field — colon-separated interface names)
        local members
        members=$(awk -F, '{gsub(/[[:space:]]/,"",$3); print $3}' <<< "$line")
        [[ "$members" =~ $iface_re ]] || die "CSV line ${line_no}: invalid bond members '$members' (use iface1:iface2 format)"

        # Validate IP address (4th field)
        local ip
        ip=$(awk -F, '{gsub(/[[:space:]]/,"",$4); print $4}' <<< "$line")
        [[ "$ip" =~ $ipv4_re ]] || die "CSV line ${line_no}: invalid IP address '$ip'"
    done < "$file"

    (( data_lines > 0 )) || die "CSV file contains no data lines: $file"
    ok "CSV validated: ${data_lines} host mapping(s)"
}

# Validate target-host network params — either all provided or none.
validate_target_network() {
    # If none of the key params are set, target networking is optional
    if [[ -z "$NETMASK" && -z "$GATEWAY" && -z "$DNS" ]]; then
        return 0
    fi

    local missing=""
    [[ -z "$NETMASK" ]]  && missing+="--netmask "
    [[ -z "$GATEWAY" ]]  && missing+="--gateway "
    [[ -z "$DNS" ]]      && missing+="--dns "
    [[ -n "$missing" ]]  && die "Target network: if any of --netmask/--gateway/--dns are set, all three are required. Missing: ${missing}"

    # Normalize netmask to CIDR prefix (/NN) — ip(8) requires this form.
    NETMASK=$(normalize_mask "$NETMASK")

    ok "Target network parameters validated"
}

# Validate build-VM network params.
validate_vm_network() {
    if [[ "$VM_NET_IP" == "dhcp" ]]; then
        return 0
    fi

    # Static mode — require gateway
    [[ -n "$VM_NET_GW" ]] || die "Build-VM network: --vm-gateway is required when --vm-ip is static"
    ok "Build-VM network parameters validated"
}

# --- Usage / help -------------------------------------------------------------

usage() {
    cat <<EOF
Usage: $(basename "$0") [OPTIONS]

Build a ReaR troubleshooting / restore ISO from a Proxmox VE cloud-init
template.  The ISO boots into a rescue environment with lldpd, tcpdump,
and nic-xray.  On boot it identifies the physical host by DMI serial
number, assigns hostname + IP from a CSV, and can restore the base OS
via  rear recover .

${C_BOLD}Modes of operation:${C_RESET}

  LOCAL   Run directly on a PVE node (as root).  The ISO is written to
          the output directory on the local filesystem.

  REMOTE  Run from a jump host and specify the target PVE node with -S.
          The script copies itself and the CSV to the remote node,
          builds the ISO there, then copies it back to the local output
          directory.  Only ssh and scp are required on the jump host.

${C_BOLD}Required:${C_RESET}
  -t, --template VMID      PVE template VMID to use as the base image
  -c, --csv FILE           Host inventory CSV (serial,hostname,bond_members,ip)

${C_BOLD}Target-host network${C_RESET} (applied on physical hosts after restore):
      --bond-mode MODE     Bond mode                       [802.3ad]
      --vlan-id ID         Management VLAN ID
      --netmask MASK       Network mask (e.g. /24 or 255.255.255.0)
      --gateway GW         Default gateway
      --dns SERVERS        Comma-separated DNS servers
      --proxy URL          HTTP/HTTPS proxy for restored hosts

  If --netmask, --gateway, and --dns are all provided, a management
  interface is configured automatically on each restored host using the
  per-host IP from the CSV.

${C_BOLD}Target-host identity${C_RESET} (applied on first boot after restore):
      --target-user USER   User account to create/configure
      --target-password PW Password for the target user (hashed before embedding)
      --target-ssh-key KEY SSH public key file (or literal key string)

${C_BOLD}Build-VM network${C_RESET} (used during image preparation):
      --vm-bridge BRIDGE   PVE bridge for the build VM     [vmbr0]
      --vm-vlan ID         VLAN tag on the build VM NIC
      --vm-ip IP/MASK      Build VM IP (default: dhcp)
      --vm-gateway GW      Build VM gateway (required if --vm-ip is static)
      --vm-dns DNS         Build VM DNS    (optional for static)
      --vm-proxy URL       HTTP/HTTPS proxy for build VM (package downloads)

${C_BOLD}General:${C_RESET}
  -s, --storage STORE      PVE storage for temp VM          [local-lvm]
  -o, --output DIR         Output directory for ISO file    [\$PWD]
  -S, --server USER@HOST   Remote mode — target PVE node
      --rescue-only        Rescue ISO only — no backup (smaller image)
      --rear-timeout SEC   Timeout for rear build           [1800]
      --dry-run            Show actions without executing
  -h, --help               Show this help
  -v, --version            Print version

${C_BOLD}CSV format${C_RESET} (one host per line, # for comments):
  # serial,hostname,bond_members,ip
  SVR001,web-server-01,eth0:eth1,10.0.0.11
  SVR002,db-server-01,eno1:eno2,10.0.0.12

${C_BOLD}Supported distributions:${C_RESET}
  Rocky Linux (9)    Ubuntu LTS (22.04, 24.04)    openSUSE Leap (15.x)
  Note: openSUSE Leap 16.x builds succeed but ReaR rescue boot fails
  due to udev incompatibility — not yet supported for restore.

${C_BOLD}Examples:${C_RESET}
  # Local mode — run on a PVE node, ISO stays in current directory
  $(basename "$0") -t 9000 -c hosts.csv

  # Local mode — ISO to a specific directory
  $(basename "$0") -t 9000 -c hosts.csv -o /var/tmp/iso/

  # Local mode — with VLAN management network
  $(basename "$0") -t 9000 -c hosts.csv \\
      --vlan-id 100 --netmask /24 --gateway 10.0.0.1 --dns 8.8.8.8

  # Remote mode — build on a PVE node, ISO downloaded to jump host
  $(basename "$0") -t 9000 -c hosts.csv -S root@pve1.example.com

  # Remote mode — custom VM networking on the PVE node
  $(basename "$0") -t 9000 -c hosts.csv -S root@192.168.20.5 \\
      --vm-bridge vmbr0 --vm-vlan 200 \\
      --vm-ip 10.0.200.50/24 --vm-gateway 10.0.200.1

  # Rescue-only (smaller ISO, no full backup)
  $(basename "$0") -t 9000 -c hosts.csv --rescue-only
EOF
}

# --- Main ---------------------------------------------------------------------

main() {
    # ---- argument parsing ----------------------------------------------------
    while [[ $# -gt 0 ]]; do
        case "$1" in
            # --- core ---
            -t|--template)    TEMPLATE="$2";       shift 2 ;;
            -c|--csv)         CSV_FILE="$2";       shift 2 ;;
            -s|--storage)     STORAGE="$2";        shift 2 ;;
            -o|--output)      OUTPUT_DIR="$2";     shift 2 ;;
            -S|--server)      PVE_SERVER="$2";     shift 2 ;;
            --rescue-only)    RESCUE_ONLY=1;       shift   ;;
            --rear-timeout)   REAR_TIMEOUT="$2";   shift 2 ;;
            --dry-run)        DRY_RUN=1;           shift   ;;
            # --- target-host network ---
            --bond-mode)      BOND_MODE="$2";      shift 2 ;;
            --vlan-id)        VLAN_ID="$2";        shift 2 ;;
            --netmask)        NETMASK="$2";        shift 2 ;;
            --gateway)        GATEWAY="$2";        shift 2 ;;
            --dns)            DNS="$2";            shift 2 ;;
            --proxy)          PROXY="$2";          shift 2 ;;
            # --- target-host identity ---
            --target-user)    TARGET_USER="$2";    shift 2 ;;
            --target-password) TARGET_PASSWORD="$2"; shift 2 ;;
            --target-ssh-key) TARGET_SSH_KEY="$2"; shift 2 ;;
            # --- build-VM network ---
            --vm-bridge)      VM_BRIDGE="$2";      shift 2 ;;
            --vm-vlan)        VM_VLAN="$2";        shift 2 ;;
            --vm-ip)          VM_NET_IP="$2";      shift 2 ;;
            --vm-gateway)     VM_NET_GW="$2";      shift 2 ;;
            --vm-dns)         VM_NET_DNS="$2";     shift 2 ;;
            --vm-proxy)       VM_PROXY="$2";       shift 2 ;;
            # --- info ---
            -h|--help)        usage;               exit 0  ;;
            -v|--version)     echo "pve-create-tshoot-image $VERSION"; exit 0 ;;
            -*)               die "Unknown option: $1 (see --help)" ;;
            *)                die "Unexpected argument: $1 (see --help)" ;;
        esac
    done

    # ---- required arguments / defaults ----------------------------------------
    [[ -n "$TEMPLATE" ]] || die "Missing required option: --template VMID"
    [[ -n "$CSV_FILE" ]] || die "Missing required option: --csv FILE"
    [[ -z "$OUTPUT_DIR" ]] && OUTPUT_DIR="$PWD"

    (( DRY_RUN )) && echo -e "\n${C_YELLOW}=== DRY RUN ===${C_RESET}\n"

    # ---- validate inputs --------------------------------------------------------
    validate_csv "$CSV_FILE"
    validate_target_network
    validate_vm_network

    # ---- select mode of operation --------------------------------------------
    if [[ -n "$PVE_SERVER" ]]; then
        # REMOTE MODE: delegate to the target PVE node
        delegate_to_server
        exit 0
    fi

    # LOCAL MODE: run directly on this PVE node
    if [[ $EUID -ne 0 ]]; then
        die "Local mode requires root on a PVE host (or use -S user@host for remote mode)"
    fi

    for cmd in qm pvesm pvesh virt-customize virt-cat virt-copy-out python3; do
        command -v "$cmd" &>/dev/null || die "Required command not found: $cmd"
    done

    # Verify the template exists
    qm status "$TEMPLATE" &>/dev/null 2>&1 || die "VMID $TEMPLATE does not exist"

    local tmpl_flag
    tmpl_flag=$(qm config "$TEMPLATE" 2>/dev/null | grep -c '^template:' || true)
    (( tmpl_flag == 0 )) && warn "VMID $TEMPLATE is not marked as a template"

    mkdir -p "$OUTPUT_DIR"

    # ==== STEP 1: prepare injection files =====================================
    generate_files

    # ==== STEP 2: clone template ==============================================
    TEMP_VMID=$(find_free_vmid)
    msg "Cloning template ${TEMPLATE} -> temporary VM ${TEMP_VMID}..."
    run qm clone "$TEMPLATE" "$TEMP_VMID" --full --name "tshoot-build-$$"

    if (( DRY_RUN )); then
        msg "Dry run complete — no further actions taken."
        TEMP_VMID=""  # prevent cleanup from destroying
        return 0
    fi

    # ==== STEP 3: detect distro ===============================================
    DISK_PATH=$(get_disk_path "$TEMP_VMID")
    ok "Disk path: $DISK_PATH"
    detect_distro "$DISK_PATH"

    # ==== STEP 4: resize disk ================================================
    # Cloud images are very small (1-3 GB).  Package installation + ReaR
    # workspace need more room.  Resize the block device — cloud-init will
    # grow the partition and filesystem automatically at first boot.
    msg "Resizing disk (+10G for packages and ReaR workspace)..."
    local first_disk
    first_disk=$(qm config "$TEMP_VMID" \
        | grep -E '^(scsi|virtio|ide|sata)[0-9]+:' \
        | grep -v 'cloudinit' \
        | head -1 | cut -d: -f1)
    qm resize "$TEMP_VMID" "$first_disk" +10G
    ok "Disk resized"

    # ==== STEP 5: inject configuration files (offline, lightweight) ==========
    inject_files "$DISK_PATH"

    # ==== STEP 6: configure build-VM networking + start VM =====================
    msg "Configuring build-VM networking..."

    # NIC: bridge + optional VLAN tag
    local net0_spec="virtio,bridge=${VM_BRIDGE}"
    [[ -n "$VM_VLAN" ]] && net0_spec+=",tag=${VM_VLAN}"
    qm set "$TEMP_VMID" --net0 "$net0_spec"

    # Cloud-init: IP config (no SSH key needed — we use guest agent)
    local ipconfig0
    if [[ "$VM_NET_IP" == "dhcp" ]]; then
        ipconfig0="ip=dhcp"
    else
        ipconfig0="ip=${VM_NET_IP},gw=${VM_NET_GW}"
    fi
    qm set "$TEMP_VMID" --ciuser root --ipconfig0 "$ipconfig0"

    # Set DNS via cloud-init when using static IP
    if [[ "$VM_NET_IP" != "dhcp" && -n "$VM_NET_DNS" ]]; then
        qm set "$TEMP_VMID" --nameserver "$VM_NET_DNS"
    fi

    # Ensure a cloud-init drive is attached
    if ! qm config "$TEMP_VMID" | grep -q 'cloudinit'; then
        qm set "$TEMP_VMID" --ide2 "${STORAGE}:cloudinit"
    fi

    msg "Starting temporary VM ${TEMP_VMID}..."
    qm start "$TEMP_VMID"

    # ==== STEP 7: wait for guest agent ========================================
    msg "Waiting for guest agent (timeout: ${BOOT_TIMEOUT}s)..."
    if ! wait_for_agent "$TEMP_VMID" "$BOOT_TIMEOUT"; then
        die "Guest agent did not respond within ${BOOT_TIMEOUT}s"
    fi
    ok "Guest agent ready"

    # ==== STEP 8: install packages via guest agent ============================
    install_packages

    # ==== STEP 9: run ReaR ====================================================
    local rear_cmd="mkbackup"
    (( RESCUE_ONLY )) && rear_cmd="mkrescue"

    msg "Running rear ${rear_cmd} inside VM (timeout: ${REAR_TIMEOUT}s)..."
    msg "This may take several minutes..."

    # Locate rear binary (may be at /usr/sbin or /sbin depending on distro)
    local rear_bin
    rear_bin=$(vm_exec 10 "command -v rear 2>/dev/null || find /usr/sbin /usr/bin /sbin /bin -name rear -type f 2>/dev/null | head -1" 2>/dev/null) || true
    rear_bin=$(echo "$rear_bin" | tr -d '[:space:]')
    [[ -z "$rear_bin" ]] && die "rear binary not found in VM — package installation may have failed"
    ok "rear found at: $rear_bin"
    if ! vm_exec "$REAR_TIMEOUT" "${rear_bin} -v ${rear_cmd}" > /dev/null 2>&1; then
        die "rear ${rear_cmd} failed — check VM ${TEMP_VMID} logs"
    fi
    ok "ReaR build complete"

    # Flush filesystem buffers and record expected ISO size before stopping.
    # Without sync the disk image may contain unflushed data, causing
    # virt-copy-out to extract a truncated file.
    vm_exec 30 "sync" > /dev/null 2>&1 || true
    local expected_iso_size
    expected_iso_size=$(vm_exec 10 \
        "stat -c '%s' /var/lib/rear/output/*.iso 2>/dev/null | head -1" 2>/dev/null) || true
    expected_iso_size=$(echo "$expected_iso_size" | tr -d '[:space:]')
    if [[ -n "$expected_iso_size" ]]; then
        ok "ISO inside VM: $(( expected_iso_size / 1048576 ))M ($expected_iso_size bytes)"
    fi

    # ==== STEP 10: extract ISO via virt-copy-out ==============================
    # Stop the VM so we can access the disk, then extract the ISO directly
    # from the filesystem.  No network/SSH needed.
    msg "Stopping VM to extract ISO..."
    qm stop "$TEMP_VMID" --timeout 60 2>/dev/null || true
    local waited=0
    while (( waited < 60 )); do
        local st
        st=$(qm status "$TEMP_VMID" 2>/dev/null | awk '{print $2}') || break
        [[ "$st" == "stopped" ]] && break
        sleep 2
        (( waited += 2 ))
    done

    DISK_PATH=$(get_disk_path "$TEMP_VMID")
    msg "Extracting ISO from disk image..."
    local iso_tmp="${TEMP_DIR}/iso-extract"
    mkdir -p "$iso_tmp"
    virt-copy-out -a "$DISK_PATH" /var/lib/rear/output/ "$iso_tmp/" 2>/dev/null || \
        die "Failed to extract ISO from VM disk"

    local source_iso
    source_iso=$(find "$iso_tmp" -name '*.iso' -type f | head -1)
    [[ -n "$source_iso" ]] || die "No ISO found in /var/lib/rear/output/ on VM disk"

    # Verify extracted ISO is not truncated
    if [[ -n "$expected_iso_size" && "$expected_iso_size" =~ ^[0-9]+$ ]]; then
        local actual_iso_size
        actual_iso_size=$(stat -c '%s' "$source_iso" 2>/dev/null || echo 0)
        if (( actual_iso_size < expected_iso_size )); then
            die "Extracted ISO is truncated: expected ${expected_iso_size} bytes but got ${actual_iso_size} bytes"
        fi
        ok "ISO size verified: ${actual_iso_size} bytes"
    fi

    # Rename to a descriptive filename
    local timestamp final_iso
    timestamp=$(date +%Y%m%d%H%M)
    final_iso="tshoot-${DISTRO_FAMILY}-${DISTRO_VERSION}-${timestamp}.iso"
    mv "$source_iso" "$OUTPUT_DIR/$final_iso"

    ok "ISO saved: ${OUTPUT_DIR}/${final_iso}"

    # ==== STEP 11: destroy temp VM (cleanup trap is the safety net) ===========
    msg "Destroying temporary VM ${TEMP_VMID}..."
    qm destroy "$TEMP_VMID" --purge 2>/dev/null || \
        warn "Could not destroy VM ${TEMP_VMID} — clean up manually"
    TEMP_VMID=""  # cleared so the EXIT trap skips destroy

    # ==== Summary =============================================================
    local iso_size mode_text
    iso_size=$(du -h "$OUTPUT_DIR/$final_iso" | cut -f1)
    mode_text="full backup"
    (( RESCUE_ONLY )) && mode_text="rescue only"

    local csv_count
    csv_count=$(grep -cE '^[^#[:space:]]' "$CSV_FILE" 2>/dev/null || echo 0)

    echo ""
    msg "Build complete!"
    echo ""
    echo "   ISO file : ${OUTPUT_DIR}/${final_iso}"
    echo "   Size     : ${iso_size}"
    echo "   Base     : ${DISTRO_FAMILY} ${DISTRO_VERSION} (template ${TEMPLATE})"
    echo "   Hosts    : ${csv_count} mapping(s)"
    echo "   Mode     : ${mode_text}"
    echo ""
    echo "   Boot the ISO on a target server and run 'rear recover' to deploy."
    echo ""
}

main "$@"
