#!/usr/bin/env bash

set -Eeuo pipefail
set +x
umask 077

readonly LABEL=REDIS01_Q131_FAILURE_DIAGNOSIS
readonly EXPECTED_HOST=sqjcredis01
readonly EXPECTED_IP=10.2.203.112
readonly WRAPPER_RUN_ID=q131-postseal-execute-20260730T025009Z
readonly EXECUTE_RUN_ID=q131-postseal-execute-20260730T025009Z-execute
readonly WRAPPER_AUDIT=/var/backups/zhct-q131-postseal-execute/q131-postseal-execute-20260730T025009Z/operator.log
readonly KEY_BACKUP=/var/backups/zhct-q131-postseal-execute/q131-postseal-execute-20260730T025009Z/ssh-host-keys-pre-postclone
readonly EXECUTE_EVIDENCE=/var/backups/zhct-q131-132/q131/q131-postseal-execute-20260730T025009Z-execute
readonly ACTIVATION_BACKUP=/var/backups/zhct-deploy/q131-postseal-execute-20260730T025009Z-execute-activate
readonly AUDIT_ROOT=/var/backups/zhct-q131-failure-diagnosis

MODE=diagnose
AUDIT_DIR=
AUDIT_LOG=

timestamp() {
    date '+%Y-%m-%dT%H:%M:%S%z'
}

die() {
    printf 'ERROR|%s\n' "$*" >&2
    exit 1
}

sanitize() {
    sed -E \
        -e 's/([Pp]assword|[Tt]oken|[Cc]ookie|[Ss]ecret)([[:space:]]*[:=][[:space:]]*)[^[:space:]",}]+/\1\2[REDACTED]/g' \
        -e 's/(requirepass|masterauth|auth-pass)([[:space:]]+).*/\1\2[REDACTED]/g'
}

finalize() {
    local rc=$?
    trap - EXIT
    if [[ "$MODE" == static ]]; then
        printf 'END|%s_STATIC_VALIDATE|RC=%s\n' "$LABEL" "$rc"
    else
        if [[ "$rc" -eq 0 ]]; then
            printf 'DIAGNOSTIC_ONLY=PASS_NO_SERVICE_OR_IDENTITY_MUTATION\n'
        else
            printf 'DIAGNOSTIC_ONLY=FAIL_NO_SERVICE_OR_IDENTITY_MUTATION\n'
        fi
        [[ -n "$AUDIT_LOG" ]] && printf 'AUDIT_LOG=%s\n' "$AUDIT_LOG"
        printf 'END|%s|RC=%s\n' "$LABEL" "$rc"
    fi
    exit "$rc"
}
trap finalize EXIT

static_validate() {
    local script=${BASH_SOURCE[0]}
    bash -n "$script" || die "self bash syntax failed"
    grep -Fqx "readonly EXPECTED_HOST=$EXPECTED_HOST" "$script" ||
        die "hostname constant missing"
    grep -Fqx "readonly EXPECTED_IP=$EXPECTED_IP" "$script" ||
        die "IP constant missing"
    grep -Fqx "readonly EXECUTE_RUN_ID=$EXECUTE_RUN_ID" "$script" ||
        die "execute run constant missing"
    grep -Fq 'Q131_DIAGNOSIS_CAPTURE=PASS_READ_ONLY' "$script" ||
        die "diagnosis completion marker missing"
    if grep -Eq 'systemctl[[:space:]]+(start|stop|restart|enable|disable)|(^|[[:space:]])(mv|rm|cp|install)[[:space:]].*/etc/(ssh|redis|zhct)' "$script"; then
        die "forbidden remote mutation command detected"
    fi
    if grep -Eq '(cat|head|tail|sha256sum)[[:space:]]+.*(redis[57]\.password|ssh_host_.*_key)' "$script"; then
        die "secret or private-key content/hash command detected"
    fi
    printf 'Q131_FAILURE_DIAGNOSIS_STATIC_VALIDATE=PASS\n'
}

if [[ ${1:-} == --static-validate ]]; then
    MODE=static
    static_validate
    exit 0
fi
[[ $# -eq 0 ]] || die "unsupported argument"

printf 'BEGIN|%s|%s\n' "$LABEL" "$(timestamp)"
[[ $(id -u) -eq 0 ]] || die "root is required"
[[ $(hostname -s) == "$EXPECTED_HOST" ]] || die "wrong hostname"
ip -o -4 addr show scope global |
    awk '{split($4,a,"/"); print a[1]}' |
    grep -Fxq "$EXPECTED_IP" ||
    die "expected IP is absent"
if ip -o -4 addr show scope global |
    awk '{split($4,a,"/"); print a[1]}' |
    grep -Eq '^10\.2\.203\.(115|116|117)$'
then
    die "VIP is present before Q-251"
fi
printf 'IDENTITY|hostname=%s|expected=%s|ip=%s\n' \
    "$(hostname -s)" "$EXPECTED_HOST" "$EXPECTED_IP"
printf 'Q131_DIAGNOSIS_TARGET_IDENTITY=PASS\n'

for path in \
    "$WRAPPER_AUDIT" \
    "$EXECUTE_EVIDENCE/stdout-stderr.log" \
    "$EXECUTE_EVIDENCE/result.env" \
    "$EXECUTE_EVIDENCE/execution-receipt.env"
do
    [[ -f "$path" && ! -L "$path" ]] || die "required evidence missing or linked: $path"
    [[ $(stat -c '%U:%G' "$path") == root:root ]] ||
        die "evidence owner is not root:root: $path"
done
[[ -d "$KEY_BACKUP" && ! -L "$KEY_BACKUP" ]] ||
    die "pre-postclone key backup directory missing or linked"
[[ -d "$ACTIVATION_BACKUP" && ! -L "$ACTIVATION_BACKUP" ]] ||
    die "activation backup directory missing or linked"
printf 'Q131_DIAGNOSIS_SOURCE_EVIDENCE_GATE=PASS\n'

diag_run_id="q131-failure-diagnosis-$(date -u +%Y%m%dT%H%M%SZ)"
AUDIT_DIR="$AUDIT_ROOT/$diag_run_id"
AUDIT_LOG="$AUDIT_DIR/operator.log"
[[ ! -e "$AUDIT_DIR" && ! -L "$AUDIT_DIR" ]] ||
    die "diagnosis audit directory already exists"
install -d -m 0700 -o root -g root -- "$AUDIT_ROOT" "$AUDIT_DIR"
exec > >(sanitize | tee -a "$AUDIT_LOG") 2>&1
printf 'AUDIT_BEGIN|%s|%s\n' "$LABEL" "$(timestamp)"
printf 'WRAPPER_RUN_ID=%s\nEXECUTE_RUN_ID=%s\n' "$WRAPPER_RUN_ID" "$EXECUTE_RUN_ID"

printf '%s\n' '=== EXECUTION RECEIPT ==='
sanitize <"$EXECUTE_EVIDENCE/execution-receipt.env"
printf '%s\n' '=== EXECUTION RESULT ==='
sanitize <"$EXECUTE_EVIDENCE/result.env"
printf '%s\n' '=== EXECUTE ERROR / PHASE MARKERS ==='
grep -nE 'ERROR:|FINAL_RC=|ROLE_ACTIVATION|post.?clone|rollback|RUN:|failed|FAIL' \
    "$EXECUTE_EVIDENCE/stdout-stderr.log" | sanitize || true
printf '%s\n' '=== EXECUTE LOG LAST 180 LINES ==='
tail -n 180 "$EXECUTE_EVIDENCE/stdout-stderr.log" | sanitize
printf '%s\n' '=== WRAPPER LOG LAST 120 LINES ==='
tail -n 120 "$WRAPPER_AUDIT" | sanitize

printf '%s\n' '=== IDENTITY METADATA ==='
if [[ -e /etc/machine-id || -L /etc/machine-id ]]; then
    stat -c 'MACHINE_ID|%F|%A|%U:%G|%s|%n' /etc/machine-id
else
    printf 'MACHINE_ID|ABSENT\n'
fi
if [[ -e /etc/zhct/node.env || -L /etc/zhct/node.env ]]; then
    stat -c 'NODE_ENV|%F|%A|%U:%G|%s|%n' /etc/zhct/node.env
    grep -E '^(ROLE|NODE_KEY|NODE_IP|NODE_NAME|APPLIED_AT)=' \
        /etc/zhct/node.env | sanitize
else
    printf 'NODE_ENV|ABSENT\n'
fi
printf 'CURRENT_PRIVATE_HOST_KEY_COUNT=%s\n' \
    "$(find /etc/ssh -maxdepth 1 -type f -name 'ssh_host_*_key' | wc -l)"
printf 'PRE_POSTCLONE_KEY_BACKUP_FILE_COUNT=%s\n' \
    "$(find "$KEY_BACKUP" -maxdepth 1 -type f | wc -l)"
printf 'SSH_HOST_KEY_CONTENT=NOT_READ_NOT_HASHED\n'

printf '%s\n' '=== CONFIG METADATA ==='
for path in \
    /etc/redis/redis5.conf \
    /etc/redis/redis7.conf \
    /etc/redis/sentinel5.conf \
    /etc/redis/sentinel7.conf \
    /etc/orchestrator.conf.json
do
    if [[ -e "$path" || -L "$path" ]]; then
        stat -c 'CONFIG|%F|%A|%U:%G|%s|%n' "$path"
    else
        printf 'CONFIG|ABSENT|%s\n' "$path"
    fi
done
printf 'CONFIG_SECRET_VALUES=NOT_OUTPUT_NOT_HASHED\n'

printf '%s\n' '=== ACTIVATION BACKUP METADATA ==='
find "$ACTIVATION_BACKUP" -maxdepth 4 -printf '%y|%m|%u:%g|%s|%p\n' | sort

printf '%s\n' '=== UNIT STATE ==='
for unit in redis5 redis7 sentinel5 sentinel7 orchestrator; do
    printf '%s.service|load=%s|enabled=%s|active=%s\n' \
        "$unit" \
        "$(systemctl show "$unit.service" -p LoadState --value 2>/dev/null || true)" \
        "$(systemctl is-enabled "$unit.service" 2>/dev/null || true)" \
        "$(systemctl is-active "$unit.service" 2>/dev/null || true)"
done
printf '%s\n' '=== TARGET PROCESSES ==='
pgrep -a -f '[/](opt/redis/.*/bin/redis-(server|sentinel)|usr/local/orchestrator/orchestrator)' ||
    printf 'NONE\n'
printf '%s\n' '=== TARGET LISTENERS ==='
ss -lntupH |
    grep -E ':(6379|6387|26379|26387|3000|10008)([[:space:]]|$)' ||
    printf 'NONE\n'
printf '%s\n' '=== FAILED UNITS ==='
systemctl --failed --no-legend --plain || true
printf '%s\n' '=== RELEVANT JOURNAL ==='
for unit in redis5 redis7 sentinel5 sentinel7 orchestrator; do
    printf '%s\n' "--- $unit.service ---"
    journalctl -u "$unit.service" --since '2026-07-30 10:48:00' \
        --no-pager -n 100 | sanitize
done
printf '%s\n' '=== DNF LATEST ==='
dnf history list 2>/dev/null | sed -n '1,8p'

printf 'Q131_DIAGNOSIS_CAPTURE=PASS_READ_ONLY\n'
printf 'Q131_RETRY=PROHIBITED_PENDING_DIAGNOSIS_REVIEW\n'
printf 'Q132=LOCKED\n'