#!/usr/bin/env bash

set -Eeuo pipefail
export LANG=C
umask 077

readonly EXPECTED_HOSTNAME='sqjcmqtt02'
readonly EXPECTED_IP='10.2.203.109'
readonly ETC_DIR='/etc/emqx'
readonly CONFIG='/etc/emqx/emqx.conf'
readonly ACL='/etc/emqx/acl.conf'
readonly COOKIE='/etc/zhct/secrets/emqx.cluster-cookie'
readonly COOKIE_TRANSFER='/root/emqx.cluster-cookie.transfer'
readonly TARGET_PORT_RE=':(1883|8083|18083|4370|5370|8883|8084)([[:space:]]|$)'
readonly AUDIT_LOG='/root/q142-clear-failed-sealed-recovery.log'

q142_recovery_mutation_started=0
q142_recovery_success=0

q142_recovery_assert_sealed() {
  local q142_recovery_unit q142_recovery_enabled q142_recovery_active
  for q142_recovery_unit in emqx.service mosquitto.service; do
    q142_recovery_enabled=$(
      systemctl is-enabled "${q142_recovery_unit}" 2>&1 || true
    )
    q142_recovery_active=$(
      systemctl is-active "${q142_recovery_unit}" 2>&1 || true
    )
    printf 'SEALED_UNIT=%s enabled=%s active=%s\n' \
      "${q142_recovery_unit}" \
      "${q142_recovery_enabled}" \
      "${q142_recovery_active}"
    test "${q142_recovery_enabled}" = disabled
    test "${q142_recovery_active}" = inactive
  done
  ! pgrep -x beam.smp >/dev/null
  ! pgrep -x mosquitto >/dev/null
  ! ss -lntupH | grep -E "${TARGET_PORT_RE}" >/dev/null
  for q142_recovery_runtime_dir in /var/lib/emqx /var/log/emqx; do
    test -d "${q142_recovery_runtime_dir}"
    ! find "${q142_recovery_runtime_dir}" -mindepth 1 -print -quit |
      grep -q .
    printf 'SEALED_RUNTIME_DIRECTORY_EMPTY=%s\n' \
      "${q142_recovery_runtime_dir}"
  done
  test -f "${CONFIG}"
  grep -Fq '__EMQX_CLUSTER_COOKIE__' "${CONFIG}"
  test ! -e "${COOKIE}"
  test ! -e "${COOKIE_TRANSFER}"
  test ! -e "${ACL}"
  test "$(stat -c '%U:%G:%a' "${ETC_DIR}")" = root:root:750
}

q142_recovery_finish() {
  q142_recovery_rc=$?
  trap - EXIT
  set +e
  if [[ "${q142_recovery_rc}" -ne 0 &&
    "${q142_recovery_mutation_started}" -eq 1 &&
    "${q142_recovery_success}" -ne 1 ]]; then
    printf '%s\n' 'Q142_CLEAR_FAILED_RECOVERY_FALLBACK_BEGIN'
    systemctl reset-failed emqx.service >/dev/null 2>&1 || true
    if [[ -d "${ETC_DIR}" ]]; then
      chown root:root "${ETC_DIR}" || true
      chmod 0750 "${ETC_DIR}" || true
    fi
    if q142_recovery_assert_sealed; then
      printf '%s\n' \
        'Q142_CLEAR_FAILED_RECOVERY_FALLBACK_STATUS=PASS_SEALED_STATE_RESTORED'
    else
      printf '%s\n' \
        'Q142_CLEAR_FAILED_RECOVERY_FALLBACK_STATUS=FAIL_MANUAL_REVIEW_REQUIRED'
    fi
  fi
  printf '%s\n' 'Q142_CLEAR_FAILED_SEALED_RECOVERY_END'
  printf 'Q142_CLEAR_FAILED_SEALED_RECOVERY_RC=%s\n' "${q142_recovery_rc}"
  exit "${q142_recovery_rc}"
}
trap q142_recovery_finish EXIT

test "${EUID}" -eq 0
test "$(hostname -s)" = "${EXPECTED_HOSTNAME}"
ip -o -4 addr show scope global | awk '{print $4}' | cut -d/ -f1 |
  grep -Fxq "${EXPECTED_IP}"

exec > >(tee "${AUDIT_LOG}") 2>&1

printf '%s\n' 'Q142_CLEAR_FAILED_SEALED_RECOVERY_BEGIN'
hostname -s
ip -br -4 addr show ens192
ip -br -4 addr show ens192 | grep -F "${EXPECTED_IP}/24"

printf '%s\n' '=== REQUIRED COMMANDS ==='
for q142_recovery_cmd in find grep pgrep ss stat systemctl; do
  command -v "${q142_recovery_cmd}" >/dev/null
  printf 'COMMAND_PRESENT=%s\n' "${q142_recovery_cmd}"
done

printf '%s\n' '=== PRE-RECOVERY FAILED SEALED-LIKE GATE ==='
test "$(systemctl is-enabled emqx.service 2>&1 || true)" = disabled
test "$(systemctl is-active emqx.service 2>&1 || true)" = failed
test "$(systemctl show emqx.service --property=Result --value)" = timeout
printf '%s\n' 'PRE_RECOVERY_EMQX_UNIT=disabled/failed'
printf '%s\n' 'PRE_RECOVERY_EMQX_RESULT=timeout'
test "$(systemctl is-enabled mosquitto.service 2>&1 || true)" = disabled
test "$(systemctl is-active mosquitto.service 2>&1 || true)" = inactive
printf '%s\n' 'PRE_RECOVERY_MOSQUITTO_UNIT=disabled/inactive'
! pgrep -x beam.smp >/dev/null
! pgrep -x mosquitto >/dev/null
printf '%s\n' 'PRE_RECOVERY_TARGET_PROCESSES_ZERO'
! ss -lntupH | grep -E "${TARGET_PORT_RE}" >/dev/null
printf '%s\n' 'PRE_RECOVERY_TARGET_LISTENERS_ZERO'
for q142_recovery_runtime_dir in /var/lib/emqx /var/log/emqx; do
  test -d "${q142_recovery_runtime_dir}"
  ! find "${q142_recovery_runtime_dir}" -mindepth 1 -print -quit |
    grep -q .
  printf 'PRE_RECOVERY_RUNTIME_DIRECTORY_EMPTY=%s\n' \
    "${q142_recovery_runtime_dir}"
done
test -f "${CONFIG}"
grep -Fq '__EMQX_CLUSTER_COOKIE__' "${CONFIG}"
printf '%s\n' 'PRE_RECOVERY_PLACEHOLDER_CONFIG_PRESENT'
test ! -e "${COOKIE}"
test ! -e "${COOKIE_TRANSFER}"
test ! -e "${ACL}"
printf '%s\n' 'PRE_RECOVERY_COOKIE_ABSENT'
printf '%s\n' 'PRE_RECOVERY_COOKIE_TRANSFER_ABSENT'
printf '%s\n' 'PRE_RECOVERY_RPM_ACL_ABSENT'
q142_recovery_etc_state=$(stat -c '%U:%G:%a' "${ETC_DIR}")
case "${q142_recovery_etc_state}" in
  root:emqx:750|root:root:750) ;;
  *)
    printf 'ERROR: unexpected /etc/emqx metadata: %s\n' \
      "${q142_recovery_etc_state}" >&2
    exit 1
    ;;
esac
printf 'PRE_RECOVERY_ETC_DIR_OWNER_MODE=%s\n' "${q142_recovery_etc_state}"
printf '%s\n' 'PRE_RECOVERY_GATE_PASS'

printf '%s\n' '=== MINIMAL SYSTEMD AND DIRECTORY STATE RECOVERY ==='
q142_recovery_mutation_started=1
systemctl reset-failed emqx.service
test "$(systemctl is-active emqx.service 2>&1 || true)" = inactive
printf '%s\n' 'EMQX_FAILED_STATE_CLEARED'
chown root:root "${ETC_DIR}"
chmod 0750 "${ETC_DIR}"
test "$(stat -c '%U:%G:%a' "${ETC_DIR}")" = root:root:750
printf '%s\n' 'ETC_DIR_ORIGINAL_OWNER_MODE_RESTORED=root:root:750'

printf '%s\n' '=== FINAL SEALED STATE ==='
q142_recovery_assert_sealed
printf '%s\n' 'SEALED_TARGET_PROCESSES_ZERO'
printf '%s\n' 'SEALED_TARGET_LISTENERS_ZERO'
printf '%s\n' 'SEALED_PLACEHOLDER_CONFIG_PRESENT'
printf '%s\n' 'SEALED_COOKIE_ABSENT'
printf '%s\n' 'SEALED_COOKIE_TRANSFER_ABSENT'
printf '%s\n' 'SEALED_RPM_ACL_ABSENT'
printf '%s\n' 'CONFIG_CONTENT_NOT_OUTPUT'
printf '%s\n' 'COOKIE_VALUE_NOT_OUTPUT'
printf '%s\n' 'SERVICE_START_STOP_NOT_PERFORMED'
printf '%s\n' 'Q142_EXECUTE_NOT_RUN'
printf '%s\n' 'Q143_Q200_Q240_BLOCKED'
printf '%s\n' 'Q142_CLEAR_FAILED_SEALED_RECOVERY_PASS'
q142_recovery_success=1