#!/usr/bin/env bash

set -Eeuo pipefail
umask 077
export LANG=C

EXPECTED_HOSTNAME="sqjcnginx01"
EXPECTED_IPV4="10.2.203.102"
BUNDLE_ROOT="/opt/zhct-deploy"
POST_CLONE_SCRIPT="${BUNDLE_ROOT}/common/scripts/post-clone.sh"
EVIDENCE_ROOT="/var/backups/zhct-q043/evidence"
RUN_ID=${1:-}
MODE=${2:-}
EVIDENCE_DIR="${EVIDENCE_ROOT}/${RUN_ID}"
FAILURE_ENV="${EVIDENCE_DIR}/failure.env"
FINAL_RESULT="${EVIDENCE_DIR}/final-result.env"
FINAL_INDEX="${EVIDENCE_DIR}/q043-final-evidence-index.sha256"
FINAL_ARCHIVE="/root/q043-evidence-${RUN_ID}.tar.gz"
SENSITIVE_SCAN_TMP="${EVIDENCE_ROOT}/.${RUN_ID}.recovery-sensitive-scan.tmp"
TARGET_PORT_RE=':(8081|8443|3306|3308|6379|6387|1883|8083|3000|26379|26387)([[:space:]]|$)'
VIP_RE='10\.2\.203\.(115|116|117)'

TARGET_UNITS=(
  nginx.service
  keepalived.service
  haproxy.service
  orchestrator.service
  sentinel5.service
  sentinel7.service
)

FORMAL_RUNTIME_PATHS=(
  /etc/keepalived/keepalived.conf
  /etc/haproxy/haproxy.cfg
  /etc/orchestrator.conf.json
  /var/lib/redis-sentinel5/sentinel.conf
  /var/lib/redis-sentinel7/sentinel.conf
  /var/lib/orchestrator/.raft
  /var/lib/orchestrator/raft
  /var/lib/orchestrator/orchestrator.db
  /etc/zhct/node.env
  /etc/zhct/secrets
)

die() {
  printf 'ERROR: %s\n' "$*" >&2
  exit 1
}

capture() {
  local name=$1
  shift
  {
    printf '# captured_at=%s\n' "$(date --iso-8601=seconds)"
    printf '# command='
    printf '%q ' "$@"
    printf '\n'
    "$@"
  } >"${EVIDENCE_DIR}/${name}.txt" 2>&1
}

assert_body_unchanged() {
  local before=$1
  local after=$2
  local label=$3
  [[ -f "${before}" && -f "${after}" ]] || die "${label} comparison evidence is missing"
  cmp -s <(sed -n '3,$p' "${before}") <(sed -n '3,$p' "${after}") || \
    die "${label} changed during seal"
}

assert_units_quiescent() {
  local output=$1
  : >"${output}"
  local unit enabled active substate started
  for unit in "${TARGET_UNITS[@]}"; do
    enabled=$(systemctl is-enabled "${unit}" 2>/dev/null || true)
    active=$(systemctl show "${unit}" -p ActiveState --value)
    substate=$(systemctl show "${unit}" -p SubState --value)
    started=$(systemctl show "${unit}" -p ExecMainStartTimestampMonotonic --value)
    printf '%s\t%s\t%s\t%s\t%s\n' \
      "${unit}" "${enabled}" "${active}" "${substate}" "${started}" >>"${output}"
    [[ "${enabled}" == "disabled" ]] || die "unit is not disabled: ${unit}"
    [[ "${active}" == "inactive" ]] || die "unit is not inactive: ${unit}"
    [[ "${substate}" == "dead" ]] || die "unit is not dead: ${unit}"
    [[ "${started}" == "0" ]] || die "unit has a start timestamp: ${unit}"
  done
}

assert_runtime_paths_absent() {
  local output=$1
  : >"${output}"
  local path
  for path in "${FORMAL_RUNTIME_PATHS[@]}"; do
    printf '%s\tABSENT\n' "${path}" >>"${output}"
    [[ ! -e "${path}" && ! -L "${path}" ]] || die "formal runtime path remains: ${path}"
  done
}

assert_clone_identity_removed() {
  local output=$1
  local machine_size key_count
  [[ -f /etc/machine-id && ! -L /etc/machine-id ]] || die "/etc/machine-id is missing or unsafe"
  machine_size=$(stat -c '%s' /etc/machine-id)
  [[ "${machine_size}" -eq 0 ]] || die "/etc/machine-id is not empty"
  [[ ! -e /var/lib/dbus/machine-id && ! -L /var/lib/dbus/machine-id ]] || \
    die "DBus machine-id remains"
  key_count=$(find /etc/ssh -maxdepth 1 -type f -name 'ssh_host_*_key*' -print | wc -l)
  [[ "${key_count}" -eq 0 ]] || die "SSH host key remains"
  {
    printf 'etc_machine_id_bytes=%s\n' "${machine_size}"
    printf 'dbus_machine_id=ABSENT\n'
    printf 'ssh_host_key_file_count=%s\n' "${key_count}"
  } >"${output}"
}

[[ "${EUID}" -eq 0 ]] || die "must run as root"
[[ "${MODE}" == "--finalize-sealed" ]] || die "explicit --finalize-sealed is required"
[[ "${RUN_ID}" =~ ^q043-[0-9]{8}T[0-9]{6}\+0800$ ]] || die "invalid Q043 run id"
[[ "$(hostname -s)" == "${EXPECTED_HOSTNAME}" ]] || die "hostname mismatch"
[[ -d "${EVIDENCE_DIR}" && ! -L "${EVIDENCE_DIR}" ]] || die "Q043 evidence directory is missing"
[[ -f "${FAILURE_ENV}" && ! -L "${FAILURE_ENV}" ]] || die "Q043 failure boundary is missing"
grep -Fxq 'seal_attempted=1' "${FAILURE_ENV}" || die "seal was not attempted"
grep -Fxq 'seal_completed=1' "${FAILURE_ENV}" || die "seal did not complete"
[[ ! -e "${FINAL_RESULT}" && ! -L "${FINAL_RESULT}" ]] || die "final result already exists"
[[ ! -e "${FINAL_ARCHIVE}" && ! -L "${FINAL_ARCHIVE}" ]] || die "final archive already exists"
[[ ! -e "${FINAL_ARCHIVE}.sha256" && ! -L "${FINAL_ARCHIVE}.sha256" ]] || \
  die "final archive checksum already exists"
ip -o -4 addr show scope global | awk '{print $4}' | cut -d/ -f1 | \
  grep -Fxq "${EXPECTED_IPV4}" || die "expected IPv4 is not configured"

cp -a "$0" "${EVIDENCE_DIR}/q043-postseal-recovery-finalize.sh"
chmod 0600 "${EVIDENCE_DIR}/q043-postseal-recovery-finalize.sh"

assert_clone_identity_removed "${EVIDENCE_DIR}/clone-identity-recovery.env"
assert_units_quiescent "${EVIDENCE_DIR}/units-recovery.tsv"
assert_runtime_paths_absent "${EVIDENCE_DIR}/runtime-paths-recovery.tsv"
for dir in /var/lib/redis-sentinel5 /var/lib/redis-sentinel7; do
  [[ -d "${dir}" && ! -L "${dir}" ]] || die "Sentinel state directory is missing or unsafe: ${dir}"
  [[ "$(find "${dir}" -mindepth 1 -print | wc -l)" -eq 0 ]] || \
    die "Sentinel state directory is not empty: ${dir}"
done
if [[ -e /var/log/zhct || -L /var/log/zhct ]]; then
  [[ -d /var/log/zhct && ! -L /var/log/zhct ]] || die "/var/log/zhct is unsafe"
  [[ "$(find /var/log/zhct -mindepth 1 ! -type d -print | wc -l)" -eq 0 ]] || \
    die "/var/log/zhct contains non-directory state"
fi
capture listeners-recovery ss -lntupH
! grep -Eq "${TARGET_PORT_RE}" "${EVIDENCE_DIR}/listeners-recovery.txt" || \
  die "target listener remains"
capture ipv4-full-recovery ip -o -4 addr show
! grep -Eq "${VIP_RE}" "${EVIDENCE_DIR}/ipv4-full-recovery.txt" || die "VIP remains announced"
capture ipv4-recovery ip -4 -br addr
capture routes-recovery ip route show table all
capture dnf-history-ids-recovery bash -Eeuo pipefail -c \
  "dnf history list | awk '\$1 ~ /^[0-9]+$/ {print \$1}'"
capture framework-sha256-recovery sha256sum \
  "${BUNDLE_ROOT}/common/scripts/seal-template.sh" \
  "${POST_CLONE_SCRIPT}" \
  "${BUNDLE_ROOT}/common/bin/role-action.sh" \
  "${BUNDLE_ROOT}/common/lib/runtime.sh" \
  "${BUNDLE_ROOT}/roles/nginx/hooks.sh" \
  "${BUNDLE_ROOT}/roles/nginx/role.env" \
  "${BUNDLE_ROOT}/roles/nginx/manifests/systemd-units.tsv" \
  "${BUNDLE_ROOT}/roles/nginx/manifests/nodes.tsv"
capture binaries-recovery sha256sum \
  /usr/local/nginx/sbin/nginx \
  /usr/local/sbin/keepalived \
  /usr/local/haproxy/sbin/haproxy \
  /usr/local/orchestrator/orchestrator \
  /opt/redis/5.0.14/bin/redis-sentinel \
  /opt/redis/5.0.14/bin/redis-cli \
  /opt/redis/7.2.14/bin/redis-sentinel \
  /opt/redis/7.2.14/bin/redis-cli \
  /usr/bin/orchestrator \
  /usr/bin/orchestrator-client
capture unit-files-recovery sha256sum \
  /etc/systemd/system/nginx.service \
  /etc/systemd/system/keepalived.service \
  /etc/systemd/system/haproxy.service \
  /usr/lib/systemd/system/orchestrator.service \
  /etc/systemd/system/sentinel5.service \
  /etc/systemd/system/sentinel7.service

assert_body_unchanged "${EVIDENCE_DIR}/ipv4-before.txt" \
  "${EVIDENCE_DIR}/ipv4-recovery.txt" "IPv4"
assert_body_unchanged "${EVIDENCE_DIR}/routes-before.txt" \
  "${EVIDENCE_DIR}/routes-recovery.txt" "routes"
assert_body_unchanged "${EVIDENCE_DIR}/dnf-history-ids-before.txt" \
  "${EVIDENCE_DIR}/dnf-history-ids-recovery.txt" "DNF history"
assert_body_unchanged "${EVIDENCE_DIR}/framework-sha256.txt" \
  "${EVIDENCE_DIR}/framework-sha256-recovery.txt" "framework files"
assert_body_unchanged "${EVIDENCE_DIR}/binaries-before.txt" \
  "${EVIDENCE_DIR}/binaries-recovery.txt" "formal binaries"
assert_body_unchanged "${EVIDENCE_DIR}/unit-files-before.txt" \
  "${EVIDENCE_DIR}/unit-files-recovery.txt" "formal unit files"

capture post-clone-dry-run-recovery env RUN_ID="${RUN_ID}-dry-recovery" \
  bash "${POST_CLONE_SCRIPT}" --role nginx --node nginx01 \
  --expected-ip "${EXPECTED_IPV4}" --bundle-root "${BUNDLE_ROOT}" --dry-run

{
  printf 'recovered_run_id=%s\n' "${RUN_ID}"
  printf 'recovered_at=%s\n' "$(date --iso-8601=seconds)"
  printf 'original_failure=timestamp_headers_in_whole_file_comparison\n'
  printf 'seal_attempted=1\n'
  printf 'seal_completed=1\n'
  printf 'seal_reexecuted=NO\n'
  printf 'identity_restored=NO\n'
  printf 'guest_rebooted_or_reconnected=NO\n'
  printf 'postseal_recovery_validation=PASS\n'
} >"${EVIDENCE_DIR}/recovery-result.env"

find "${EVIDENCE_DIR}" -type f \
  \( -name '*.pem' -o -name '*.key' -o -name 'ssh_host_*' \) -print \
  >"${EVIDENCE_DIR}/forbidden-private-files.txt"
[[ ! -s "${EVIDENCE_DIR}/forbidden-private-files.txt" ]] || \
  die "private key-like evidence file found"
if grep -RInE \
  '(BEGIN[[:space:]]+(RSA|EC|OPENSSH)?[[:space:]]*PRIVATE KEY|AKIA[0-9A-Z]{16}|(password|passwd|token|cookie|secret)[[:space:]]*[:=][[:space:]]*[^_[:space:]]{6,})' \
  "${EVIDENCE_DIR}" \
  --exclude='q043-seal-check.sh' \
  --exclude='q043-postseal-recovery-finalize.sh' \
  >"${SENSITIVE_SCAN_TMP}"; then
  mv "${SENSITIVE_SCAN_TMP}" "${EVIDENCE_DIR}/sensitive-scan.txt"
  die "sensitive-looking content found in Q043 evidence"
fi
rm -f "${SENSITIVE_SCAN_TMP}"
printf 'PASS\n' >"${EVIDENCE_DIR}/sensitive-scan.txt"

{
  printf 'run_id=%s\n' "${RUN_ID}"
  printf 'completed_at=%s\n' "$(date --iso-8601=seconds)"
  printf 'v1_q042_confirmation_and_evidence=PASS\n'
  printf 'v2_target_identity=PASS\n'
  printf 'v3_framework_integrity=PASS\n'
  printf 'v4_preseal_quiescence=PASS\n'
  printf 'v5_dry_run=PASS\n'
  printf 'v6_live_seal=PASS\n'
  printf 'v7_clone_identity_removed=PASS\n'
  printf 'v8_runtime_state_removed=PASS\n'
  printf 'v9_immutable_state_and_space=PASS\n'
  printf 'v10_post_clone_dry_run_and_evidence=PASS\n'
  printf 'postseal_recovery_finalizer=PASS\n'
  printf 'target_units_quiescent=PASS\n'
  printf 'target_listener_count=0\n'
  printf 'vip_announcement_count=0\n'
  printf 'machine_id_bytes=0\n'
  printf 'ssh_host_key_file_count=0\n'
  printf 'sensitive_scan=PASS\n'
  printf 'template_state=SEALED_AWAITING_CUSTOMER_APPLICATION\n'
  printf 'guest_reboot_or_reconnect=FORBIDDEN\n'
  printf 'next_task_execution=AUTHORIZED_BY_TARGET_MODE\n'
  printf 'final_check=PASS\n'
} >"${FINAL_RESULT}"
chmod 0600 "${FINAL_RESULT}"

find "${EVIDENCE_DIR}" -type l -print >"${EVIDENCE_DIR}/symlinks.txt"
[[ ! -s "${EVIDENCE_DIR}/symlinks.txt" ]] || die "evidence contains a symbolic link"
(
  cd "${EVIDENCE_DIR}"
  find . -type f \
    ! -name 'q043-final-evidence-index.sha256' \
    -print0 | sort -z | xargs -0 sha256sum >"$(basename "${FINAL_INDEX}")"
)
chmod 0600 "${FINAL_INDEX}"
sha256sum "${FINAL_INDEX}" >"${FINAL_INDEX}.sha256"
chmod 0600 "${FINAL_INDEX}.sha256"
(
  cd "${EVIDENCE_ROOT}"
  tar -czf "${FINAL_ARCHIVE}" "${RUN_ID}"
)
sha256sum "${FINAL_ARCHIVE}" >"${FINAL_ARCHIVE}.sha256"
chmod 0600 "${FINAL_ARCHIVE}" "${FINAL_ARCHIVE}.sha256"

printf 'Q043_POSTSEAL_RECOVERY_OK\n'
printf 'RUN_ID=%s\n' "${RUN_ID}"
printf 'FINAL_ARCHIVE=%s\n' "${FINAL_ARCHIVE}"
printf 'FINAL_ARCHIVE_SHA256=%s\n' "$(awk '{print $1}' "${FINAL_ARCHIVE}.sha256")"
printf 'TEMPLATE_STATE=SEALED_AWAITING_CUSTOMER_APPLICATION\n'
printf 'GUEST_REBOOT_OR_RECONNECT=FORBIDDEN\n'