#!/usr/bin/env bash

set -Eeuo pipefail
umask 077

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
SCRIPT_NAME=$(basename "$0")

TARGET_HOST=sqjcnginx02
TARGET_IP=10.2.203.103
TARGET_ROLE=nginx
TARGET_NODE=nginx02
EXPECTED_SECRET_SOURCE=Q101_NGINX01_CLONE_PRESERVED_CONFIRMED_BY_GLOBAL
REMOTE_BUNDLE=/opt/zhct-deploy
BACKUP_ROOT=/var/backups/zhct-q102/files
REMOTE_EVIDENCE_ROOT=/var/backups/zhct-q102/evidence

usage() {
    cat <<'USAGE'
Usage:
  q102-nginx02.sh --static-validate
  q102-nginx02.sh preflight|execute|verify|rollback|archive

Non-static modes require:
  Q102_RUN_ID=q102-<fixed-run-id>
  Q102_REMOTE_LOCK_GRANTED=1
  Q102_REMOTE_LOCK_OWNER=NGINX_Q102

preflight/execute/verify also require:
  Q102_SECRET_SOURCE_CONFIRMATION=Q101_NGINX01_CLONE_PRESERVED_CONFIRMED_BY_GLOBAL
USAGE
}

die() {
    printf 'ERROR: %s\n' "$*" >&2
    return 1
}

require_command() {
    command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
}

mode=${1:-}
[[ $# -eq 1 ]] || {
    usage
    exit 2
}
case "$mode" in
    --static-validate|preflight|execute|verify|rollback|archive) ;;
    *)
        usage
        exit 2
        ;;
esac

run_id=${Q102_RUN_ID:-}
if [[ "$mode" == --static-validate ]]; then
    run_id=${run_id:-q102-static-validate}
    evidence_root=${Q102_EVIDENCE_ROOT:-$(mktemp -d "${TMPDIR:-/tmp}/q102-static-evidence.XXXXXX")}
    evidence_dir="$evidence_root/$run_id"
else
    [[ "$run_id" =~ ^q102-[A-Za-z0-9._+-]+$ ]] || die 'Q102_RUN_ID is missing or invalid'
    evidence_root=${Q102_EVIDENCE_ROOT:-$REMOTE_EVIDENCE_ROOT}
    evidence_dir="$evidence_root/$run_id"
fi

mkdir -p "$evidence_dir"
timestamp=$(date -u +%Y%m%dT%H%M%SZ)
mode_name=${mode#--}
log_file="$evidence_dir/${timestamp}-${mode_name}.log"
rc_file="$evidence_dir/${timestamp}-${mode_name}.rc"

if [[ "$mode" == --static-validate ]]; then
    exec >>"$log_file" 2>&1
else
    exec > >(tee -a "$log_file") 2>&1
fi

start_time=$(date '+%Y-%m-%dT%H:%M:%S%z')
finalized=0
finalize() {
    local rc=$?
    if [[ "$finalized" -eq 0 ]]; then
        finalized=1
        {
            printf 'Q102_MODE_END=%s\n' "$mode"
            printf 'Q102_END_TIME=%s\n' "$(date '+%Y-%m-%dT%H:%M:%S%z')"
            printf 'Q102_TARGET_HOST=%s\n' "$TARGET_HOST"
            printf 'Q102_TARGET_IP=%s\n' "$TARGET_IP"
            printf 'Q102_TARGET_ROLE=%s\n' "$TARGET_ROLE"
            printf 'Q102_RUN_ID=%s\n' "$run_id"
            printf 'Q102_RC=%s\n' "$rc"
        } | tee -a "$rc_file"
    fi
    return "$rc"
}
trap finalize EXIT

printf 'Q102_MODE_BEGIN=%s\n' "$mode"
printf 'Q102_START_TIME=%s\n' "$start_time"
printf 'Q102_TARGET_HOST=%s\n' "$TARGET_HOST"
printf 'Q102_TARGET_IP=%s\n' "$TARGET_IP"
printf 'Q102_TARGET_ROLE=%s\n' "$TARGET_ROLE"
printf 'Q102_RUN_ID=%s\n' "$run_id"

static_validate() {
    local required_mode
    for required_mode in --static-validate preflight execute verify rollback archive; do
        grep -Fq -- "$required_mode" "$SCRIPT_DIR/$SCRIPT_NAME" ||
            die "mode is not implemented: $required_mode"
    done
    bash -n "$SCRIPT_DIR/$SCRIPT_NAME"
    (
        cd "$SCRIPT_DIR"
        sha256sum -c SHA256SUMS
    )
    grep -Fq 'RaftEnabled' "$SCRIPT_DIR/$SCRIPT_NAME"
    grep -Fq 'Q102_SECRET_SOURCE_CONFIRMATION' "$SCRIPT_DIR/$SCRIPT_NAME"
    grep -Fq 'Q102_REMOTE_LOCK_GRANTED' "$SCRIPT_DIR/$SCRIPT_NAME"
    printf 'Q102_STATIC_VALIDATE_PASS\n'
}

common_remote_gate() {
    [[ $(id -u) -eq 0 ]] || die 'root is required'
    [[ "${Q102_REMOTE_LOCK_GRANTED:-}" == 1 ]] ||
        die 'shared remote lock is not asserted'
    [[ "${Q102_REMOTE_LOCK_OWNER:-}" == NGINX_Q102 ]] ||
        die 'shared remote lock owner is not NGINX_Q102'
    printf 'Q102_REMOTE_LOCK_ASSERTION=PASS\n'

    local command
    for command in awk bash cp curl find grep install ip iptables mv python3 \
        rsync sed sha256sum sort ss stat systemctl tar tee; do
        require_command "$command"
    done

    [[ "$(hostname -s)" == "$TARGET_HOST" ]] ||
        die "hostname is not $TARGET_HOST"
    ip -o -4 addr show scope global | awk '{print $4}' | cut -d/ -f1 |
        grep -Fxq "$TARGET_IP" || die "target IP $TARGET_IP is absent"

    [[ -r /etc/zhct/node.env ]] || die '/etc/zhct/node.env is missing'
    grep -Fxq "ROLE=$TARGET_ROLE" /etc/zhct/node.env ||
        die 'node role mismatch'
    grep -Fxq "NODE_KEY=$TARGET_NODE" /etc/zhct/node.env ||
        die 'node key mismatch'
    grep -Fxq "NODE_IP=$TARGET_IP" /etc/zhct/node.env ||
        die 'node IP mismatch'
    grep -Fxq "NODE_NAME=$TARGET_HOST" /etc/zhct/node.env ||
        die 'node hostname mismatch'
    printf 'Q102_IDENTITY_GATE=PASS\n'
}

secret_metadata_gate() {
    [[ "${Q102_SECRET_SOURCE_CONFIRMATION:-}" == "$EXPECTED_SECRET_SOURCE" ]] ||
        die 'trusted same-source secret confirmation is absent'
    printf 'Q102_SECRET_SOURCE_CONFIRMATION=%s\n' "$EXPECTED_SECRET_SOURCE"

    local secret_dir=/etc/zhct/secrets
    [[ -d "$secret_dir" ]] || die 'secret directory is absent'
    [[ $(stat -c '%U:%G' "$secret_dir") == root:root ]] ||
        die 'secret directory owner/group is unsafe'
    [[ $(stat -c '%a' "$secret_dir") == 700 ]] ||
        die 'secret directory mode is not 0700'

    local -a expected=(
        orchestrator-topology.password
        redis5.password
        redis7.password
        vrrp.auth
    )
    local actual_count name path mode
    actual_count=$(find "$secret_dir" -maxdepth 1 -type f | wc -l)
    [[ "$actual_count" -eq "${#expected[@]}" ]] ||
        die 'secret directory does not contain exactly four files'

    for name in "${expected[@]}"; do
        path="$secret_dir/$name"
        [[ -f "$path" && -s "$path" ]] ||
            die "required secret file is missing or empty: $name"
        [[ $(stat -c '%U:%G' "$path") == root:root ]] ||
            die "secret owner/group is unsafe: $name"
        mode=$(stat -c '%a' "$path")
        [[ "$mode" == 600 || "$mode" == 400 ]] ||
            die "secret mode is unsafe: $name"
        printf 'SECRET_FILE=%s STATUS=PRESENT_NONEMPTY OWNER=root:root MODE=%s\n' \
            "$name" "$mode"
    done
    printf 'Q102_SECRET_METADATA_GATE=PASS\n'
}

print_unit_state() {
    local unit
    for unit in nginx haproxy orchestrator sentinel5 sentinel7 keepalived; do
        printf 'UNIT=%s ENABLED=' "$unit"
        systemctl is-enabled "$unit" 2>/dev/null || true
        printf 'UNIT=%s ACTIVE=' "$unit"
        systemctl is-active "$unit" 2>/dev/null || true
    done
}

assert_all_units_quiet() {
    local unit enabled active
    for unit in nginx haproxy orchestrator sentinel5 sentinel7 keepalived; do
        enabled=$(systemctl is-enabled "$unit" 2>/dev/null || true)
        active=$(systemctl is-active "$unit" 2>/dev/null || true)
        [[ "$enabled" == disabled ]] ||
            die "unit is not disabled: $unit ($enabled)"
        [[ "$active" == inactive ]] ||
            die "unit is not inactive: $unit ($active)"
    done
}

verify_remote_asset_hashes() {
    (
        cd /
        sha256sum -c - <<'HASHES'
e637ca657a897c6d0b689bbd6347f04516baf5c9f32350cd1e8a2071c211a0a5  opt/zhct-deploy/common/scripts/activate-role.sh
e3eae16aa5c87f43b56495fc71de378a947578cabe18b507e8fed01ac1f32025  opt/zhct-deploy/common/bin/role-action.sh
4c06e4ad325de7241607b06d93c99c2f35f3422bfcbdbeeb43b530f2053f4bb6  opt/zhct-deploy/common/lib/runtime.sh
92c3b974b74c9d128c52a989624c97a1bb6ba77057dd02cf6de6421d3087f105  opt/zhct-deploy/roles/nginx/hooks.sh
ce18e7febc7e3105294ddc74d32f10f7fb4da2c4d8e59c3e9642816ec770463f  opt/zhct-deploy/roles/nginx/manifests/nodes.tsv
24ce7c17d7a7cf8070d1cdd9013c54e49a445cd264a0ecafcae4ef1912811d61  opt/zhct-deploy/roles/nginx/manifests/systemd-units.tsv
HASHES
    )
    printf 'Q102_REMOTE_ASSET_HASH_GATE=PASS\n'
}

print_nonsecret_orchestrator_config() {
    python3 - <<'PY'
import json
with open("/etc/orchestrator.conf.json", encoding="utf-8") as handle:
    config = json.load(handle)
for key in ("ListenAddress", "BackendDB", "SQLite3DataFile", "RaftEnabled",
            "RaftDataDir", "RaftBind", "DefaultRaftPort", "RaftNodes"):
    print(f"ORCHESTRATOR_{key}={config.get(key)!r}")
PY
}

assert_no_vip() {
    if ip -o -4 addr show | grep -Eq '10\.2\.203\.(115|116|117)/'; then
        die 'unexpected VIP is configured'
    fi
    printf 'Q102_VIP_ABSENT=PASS\n'
}

preflight() {
    common_remote_gate
    secret_metadata_gate
    verify_remote_asset_hashes
    assert_all_units_quiet
    print_unit_state

    local transient active
    for transient in zhct-transfer zhct-transfer80 zhct-transfer-q140 \
        zhct-transfer-q140b; do
        active=$(systemctl is-active "$transient" 2>/dev/null || true)
        [[ "$active" != active ]] ||
            die "transient transfer unit is active: $transient"
        printf 'TRANSIENT_UNIT=%s ACTIVE=%s\n' "$transient" "$active"
    done
    if iptables -C INPUT -p tcp -s 10.2.203.109 -d 10.2.203.103 \
        --dport 80 -j ACCEPT 2>/dev/null; then
        die 'Q-140 temporary rule from 10.2.203.109 is present'
    fi
    if iptables -C INPUT -p tcp -s 10.2.203.110 -d 10.2.203.103 \
        --dport 80 -j ACCEPT 2>/dev/null; then
        die 'Q-140 temporary rule from 10.2.203.110 is present'
    fi
    [[ ! -e /tmp/zhct-postclone-framework.tar.gz ]] ||
        die 'Q-140 temporary archive is present'
    if ss -lntH | grep -Eq \
        ':(80|18080|8081|1883|8083|3306|3308|6379|6387|26379|26387|3000|10008)[[:space:]]'; then
        die 'target or transient listener exists before Q-102'
    fi
    printf 'Q102_PREEXISTING_TARGET_LISTENERS=ABSENT\n'

    env RUN_ID="$run_id-preflight" BACKUP_ROOT="$BACKUP_ROOT" \
        "$REMOTE_BUNDLE/common/scripts/activate-role.sh" \
        --role nginx --bundle-root "$REMOTE_BUNDLE" --dry-run

    grep -Fq 'nginx02=disabled,inactive' \
        "$REMOTE_BUNDLE/roles/nginx/manifests/systemd-units.tsv"
    grep -Eq '^[[:space:]]*router_id[[:space:]]+SANQUAN_EDGE_02$' \
        /etc/keepalived/keepalived.conf
    print_nonsecret_orchestrator_config

    if [[ -d /var/lib/orchestrator ]]; then
        find /var/lib/orchestrator -maxdepth 1 -mindepth 1 \
            -printf 'ORCHESTRATOR_DATA=%f TYPE=%y MODE=%m OWNER=%u:%g SIZE=%s\n' |
            sort
    else
        printf 'ORCHESTRATOR_DATA_DIR=ABSENT\n'
    fi
    if [[ -d /var/log/orchestrator ]]; then
        stat -c 'ORCHESTRATOR_LOG_DIR MODE=%a OWNER=%U:%G SIZE=%s' \
            /var/log/orchestrator
    else
        printf 'ORCHESTRATOR_LOG_DIR=ABSENT\n'
    fi

    assert_no_vip
    printf 'Q102_PREFLIGHT_PASS\n'
    : >"$evidence_dir/preflight.pass"
}

isolate_orchestrator_state() {
    local run_backup="$BACKUP_ROOT/$run_id"
    local original="$run_backup/q102-original"
    local state_file="$run_backup/q102-state.env"
    local inherited_lib="/var/lib/orchestrator.q102-inherited-$run_id"
    local inherited_log="/var/log/orchestrator.q102-inherited-$run_id"
    local lib_present=0 log_present=0

    [[ ! -e "$run_backup" ]] ||
        die "run backup already exists; refuse replay: $run_backup"
    install -d -m 0750 -o root -g root "$original/etc"
    install -d -m 0750 -o root -g root "$original/var/lib"
    install -d -m 0750 -o root -g root "$original/var/log"
    cp -a /etc/orchestrator.conf.json "$original/etc/orchestrator.conf.json"

    if [[ -d /var/lib/orchestrator ]]; then
        [[ ! -e "$inherited_lib" ]] || die 'inherited lib quarantine already exists'
        rsync -aHAX --numeric-ids /var/lib/orchestrator/ \
            "$original/var/lib/orchestrator/"
        mv /var/lib/orchestrator "$inherited_lib"
        lib_present=1
    fi
    if [[ -d /var/log/orchestrator ]]; then
        [[ ! -e "$inherited_log" ]] || die 'inherited log quarantine already exists'
        rsync -aHAX --numeric-ids /var/log/orchestrator/ \
            "$original/var/log/orchestrator/"
        mv /var/log/orchestrator "$inherited_log"
        log_present=1
    fi
    printf 'LIB_ORIGINAL_PRESENT=%s\nLOG_ORIGINAL_PRESENT=%s\n' \
        "$lib_present" "$log_present" >"$state_file"
    chmod 0600 "$state_file"

    install -d -m 0750 -o root -g root /var/lib/orchestrator
    install -d -m 0750 -o root -g root /var/log/orchestrator

    local temp
    temp=$(mktemp /etc/.orchestrator.conf.q102.XXXXXX)
    python3 - /etc/orchestrator.conf.json "$temp" <<'PY'
import json
import pathlib
import sys

source = pathlib.Path(sys.argv[1])
target = pathlib.Path(sys.argv[2])
config = json.loads(source.read_text(encoding="utf-8"))
assert config.get("ListenAddress") == "10.2.203.103:3000"
assert config.get("RaftBind") == "10.2.203.103"
assert config.get("RaftNodes") == ["10.2.203.102", "10.2.203.103", "10.2.203.112"]
config["RaftEnabled"] = False
target.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
PY
    install -m 0600 -o root -g root "$temp" /etc/orchestrator.conf.json
    rm -f "$temp"
    python3 -m json.tool /etc/orchestrator.conf.json >/dev/null
    print_nonsecret_orchestrator_config
    printf 'Q102_ORCHESTRATOR_STATE_ISOLATED=PASS\n'
}

rollback_internal() {
    local run_backup="$BACKUP_ROOT/$run_id"
    local original="$run_backup/q102-original"
    local state_file="$run_backup/q102-state.env"
    local failed="$run_backup/q102-failed-new-state"
    local inherited_lib="/var/lib/orchestrator.q102-inherited-$run_id"
    local inherited_log="/var/log/orchestrator.q102-inherited-$run_id"
    local lib_present log_present unit

    [[ -r "$state_file" ]] || die "state marker is absent: $state_file"
    lib_present=$(awk -F= '$1=="LIB_ORIGINAL_PRESENT"{print $2}' "$state_file")
    log_present=$(awk -F= '$1=="LOG_ORIGINAL_PRESENT"{print $2}' "$state_file")
    [[ "$lib_present" =~ ^[01]$ && "$log_present" =~ ^[01]$ ]] ||
        die 'state marker is invalid'

    for unit in nginx haproxy orchestrator sentinel5 sentinel7 keepalived; do
        systemctl disable --now "$unit" || true
    done

    [[ ! -e "$failed" ]] || die 'failed-state evidence directory already exists'
    install -d -m 0750 -o root -g root "$failed/var/lib" "$failed/var/log"
    [[ ! -e "$failed/var/lib/orchestrator" ]]
    [[ ! -e "$failed/var/log/orchestrator" ]]
    [[ -d /var/lib/orchestrator ]] &&
        mv /var/lib/orchestrator "$failed/var/lib/orchestrator"
    [[ -d /var/log/orchestrator ]] &&
        mv /var/log/orchestrator "$failed/var/log/orchestrator"

    install -m 0600 -o root -g root \
        "$original/etc/orchestrator.conf.json" /etc/orchestrator.conf.json

    if [[ "$lib_present" == 1 ]]; then
        [[ -d "$inherited_lib" ]] || die 'inherited lib quarantine is missing'
        mv "$inherited_lib" /var/lib/orchestrator
    fi
    if [[ "$log_present" == 1 ]]; then
        [[ -d "$inherited_log" ]] || die 'inherited log quarantine is missing'
        mv "$inherited_log" /var/log/orchestrator
    fi
    systemctl daemon-reload
    print_unit_state
    assert_all_units_quiet
    assert_no_vip
    printf 'Q102_ROLLBACK_PASS\n'
}

execute_mode() {
    common_remote_gate
    secret_metadata_gate
    verify_remote_asset_hashes
    [[ -f "$evidence_dir/preflight.pass" ]] ||
        die 'preflight.pass is absent for this run ID'
    assert_all_units_quiet

    set +e
    (
        set -Eeuo pipefail
        isolate_orchestrator_state
        env RUN_ID="$run_id" BACKUP_ROOT="$BACKUP_ROOT" \
            "$REMOTE_BUNDLE/common/scripts/activate-role.sh" \
            --role nginx --bundle-root "$REMOTE_BUNDLE"
    )
    local rc=$?
    set -e
    if [[ "$rc" -ne 0 ]]; then
        printf 'Q102_EXECUTE_FAILED_RC=%s\n' "$rc" >&2
        rollback_internal || true
        return "$rc"
    fi
    printf 'Q102_EXECUTE_PASS\n'
    : >"$evidence_dir/execute.pass"
}

verify_mode() {
    common_remote_gate
    secret_metadata_gate
    [[ -f "$evidence_dir/execute.pass" ]] ||
        die 'execute.pass is absent for this run ID'

    local unit
    for unit in nginx haproxy orchestrator; do
        systemctl is-enabled --quiet "$unit"
        systemctl is-active --quiet "$unit"
    done
    for unit in sentinel5 sentinel7 keepalived; do
        [[ $(systemctl is-enabled "$unit" 2>/dev/null || true) == disabled ]]
        [[ $(systemctl is-active "$unit" 2>/dev/null || true) == inactive ]]
    done
    print_unit_state

    /usr/local/nginx/sbin/nginx -t
    /usr/local/haproxy/sbin/haproxy -c -f /etc/haproxy/haproxy.cfg
    curl -fsS --max-time 5 http://127.0.0.1:8081/__edge_health
    curl -fsS --max-time 5 "http://$TARGET_IP:3000/api/health"
    print_nonsecret_orchestrator_config
    python3 - <<'PY'
import json
with open("/etc/orchestrator.conf.json", encoding="utf-8") as handle:
    config = json.load(handle)
assert config.get("RaftEnabled") is False
assert config.get("RaftBind") == "10.2.203.103"
PY
    ss -lntup
    ss -lntH | grep -Eq '10\.2\.203\.103:3000[[:space:]]'
    if ss -lntH | grep -Eq '10\.2\.203\.103:10008[[:space:]]'; then
        die 'Q-210 Raft listener appeared during Q-102'
    fi
    printf 'Q102_RAFT_10008_ABSENT=PASS\n'
    assert_no_vip
    printf 'Q102_VERIFY_PASS\n'
    : >"$evidence_dir/verify.pass"
}

rollback_mode() {
    common_remote_gate
    rollback_internal
    : >"$evidence_dir/rollback.pass"
}

archive_mode() {
    common_remote_gate
    local archive_dir="$evidence_root"
    local archive="$archive_dir/q102-evidence-$run_id.tar.gz"
    local checksum="$archive.sha256"
    [[ -d "$evidence_dir" ]] || die 'evidence directory is absent'
    [[ ! -e "$archive" && ! -e "$checksum" ]] ||
        die 'evidence archive already exists; refuse overwrite'
    tar --no-xattrs \
        --exclude="$run_id/$(basename "$log_file")" \
        --exclude="$run_id/$(basename "$rc_file")" \
        -czf "$archive" -C "$evidence_root" "$run_id"
    sha256sum "$archive" >"$checksum"
    printf 'Q102_EVIDENCE_ARCHIVE=%s\n' "$archive"
    printf 'Q102_EVIDENCE_ARCHIVE_SHA256_FILE=%s\n' "$checksum"
    printf 'Q102_ARCHIVE_PASS\n'
}

case "$mode" in
    --static-validate) static_validate ;;
    preflight) preflight ;;
    execute) execute_mode ;;
    verify) verify_mode ;;
    rollback) rollback_mode ;;
    archive) archive_mode ;;
esac