#!/usr/bin/env bash

set -Eeuo pipefail
export LANG=C
umask 077

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
MANIFEST="${SCRIPT_DIR}/SHA256SUMS"
FRAMEWORK_ARCHIVE="${SCRIPT_DIR}/payload/zhct-emqx-framework.tar.gz"
FRAMEWORK_SHA256=489294eb3a5c362a5f403518d9486dc61b26330e98923b006cfd792dfda842cd
ACL_SOURCE="${SCRIPT_DIR}/payload/acl.conf"
ACL_SHA256=970449f3515ffe63683de4eebac29f6c4cc93f11660cbb3f5bcb0b39df5a4aa0
ACL_FILE=/etc/emqx/acl.conf
ETC_DIR=/etc/emqx
BUNDLE_ROOT=/opt/zhct-deploy
COOKIE_FILE=/etc/zhct/secrets/emqx.cluster-cookie
COOKIE_TRANSFER=/root/emqx.cluster-cookie.transfer
EVIDENCE_ROOT=/root/q142-q143-evidence
ROLLBACK_ROOT=/root/q142-q143-rollback
TARGET_PORT_RE=':(1883|8083|18083|4370|5370|8883|8084)([[:space:]]|$)'

usage() {
  cat <<'USAGE'
Usage:
  emqx-q142-q143.sh --static-validate
  emqx-q142-q143.sh q142|q143 preflight [--expected-cookie-sha256 SHA256]
  emqx-q142-q143.sh q142|q143 execute --expected-cookie-sha256 SHA256 --confirm q142|q143-execute
  emqx-q142-q143.sh q142|q143 verify --expected-cookie-sha256 SHA256
  emqx-q142-q143.sh q142|q143 rollback --confirm q142|q143-rollback
  emqx-q142-q143.sh q142|q143 archive
USAGE
}

die() {
  printf 'ERROR: %s\n' "$*" >&2
  exit 1
}

sha256_file() {
  if command -v sha256sum >/dev/null 2>&1; then
    sha256sum "$1" | awk '{print $1}'
  elif command -v shasum >/dev/null 2>&1; then
    shasum -a 256 "$1" | awk '{print $1}'
  else
    die "sha256sum or shasum is required"
  fi
}

verify_manifest() {
  [[ -f "${MANIFEST}" ]] || die "SHA256SUMS is missing"
  local expected path actual
  while read -r expected path; do
    [[ "${expected}" =~ ^[0-9a-f]{64}$ ]] || die "invalid manifest digest"
    path=${path#\*}
    [[ "${path}" != /* && "${path}" != *'..'* ]] || die "unsafe manifest path"
    [[ -f "${SCRIPT_DIR}/${path}" ]] || die "manifest file missing: ${path}"
    actual=$(sha256_file "${SCRIPT_DIR}/${path}")
    [[ "${actual}" == "${expected}" ]] || die "manifest mismatch: ${path}"
    printf '%s: OK\n' "${path}"
  done <"${MANIFEST}"
}

static_validate() {
  verify_manifest
  [[ "$(sha256_file "${FRAMEWORK_ARCHIVE}")" == "${FRAMEWORK_SHA256}" ]] ||
    die "framework archive digest mismatch"
  [[ "$(sha256_file "${ACL_SOURCE}")" == "${ACL_SHA256}" ]] ||
    die "RPM ACL payload digest mismatch"
  bash -n "${SCRIPT_DIR}/emqx-q142-q143.sh"
  local members
  members=$(mktemp)
  tar -tzf "${FRAMEWORK_ARCHIVE}" | LC_ALL=C sort >"${members}"
  grep -Fxq common/scripts/activate-role.sh "${members}" ||
    die "activate-role.sh missing from framework archive"
  grep -Fxq roles/emqx/templates/nodes/emqx02/emqx.conf "${members}" ||
    die "emqx02 template missing from framework archive"
  grep -Fxq roles/emqx/templates/nodes/emqx03/emqx.conf "${members}" ||
    die "emqx03 template missing from framework archive"
  grep -Fxq roles/emqx/templates/runtime/provision-emqx-cookie.sh "${members}" ||
    die "provisioner missing from framework archive"
  rm -f "${members}"
  printf 'PACKAGE_CONTAINS_SECRET=NO\n'
  printf 'RPM_ACL_PAYLOAD_SHA256=%s\n' "${ACL_SHA256}"
  printf 'Q200_NETWORK_GATE=NOT_PERFORMED\n'
  printf 'Q240_FORMAL_CLUSTER_ACCEPTANCE=NOT_PERFORMED\n'
  printf 'EMQX_Q142_Q143_PACKAGE_STATIC_VALIDATE_PASS\n'
}

if [[ "${1:-}" == --static-validate ]]; then
  [[ $# -eq 1 ]] || die "--static-validate takes no other arguments"
  static_validate
  exit 0
fi

QID=${1:-}
ACTION=${2:-}
shift $(( $# >= 2 ? 2 : $# ))
EXPECTED_COOKIE_SHA=
CONFIRM=
while (($#)); do
  case "$1" in
    --expected-cookie-sha256)
      [[ $# -ge 2 ]] || die "missing SHA256 value"
      EXPECTED_COOKIE_SHA=$2
      shift 2
      ;;
    --confirm)
      [[ $# -ge 2 ]] || die "missing confirmation value"
      CONFIRM=$2
      shift 2
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      die "unknown argument: $1"
      ;;
  esac
done

case "${QID}" in
  q142)
    EXPECTED_HOST=sqjcmqtt02
    EXPECTED_IP=10.2.203.109
    NODE_KEY=emqx02
    EXPECTED_NODE='emqx@10.2.203.109'
    TEMPLATE_SHA=5b8a58d992022caa7ee01329a1514b590697fa1adc970db33db25433280e240e
    PREVIOUS_QID=q141
    ;;
  q143)
    EXPECTED_HOST=sqjcmqtt03
    EXPECTED_IP=10.2.203.110
    NODE_KEY=emqx03
    EXPECTED_NODE='emqx@10.2.203.110'
    TEMPLATE_SHA=b3f89486324a77a51902164e5cfaddabe97010c1c41ee2f9ccfd9b947d4dbc2c
    PREVIOUS_QID=q142
    ;;
  *)
    usage >&2
    die "task must be q142 or q143"
    ;;
esac

case "${ACTION}" in
  preflight|execute|verify|rollback|archive) ;;
  *)
    usage >&2
    die "unsupported action"
    ;;
esac

[[ "${EUID}" -eq 0 ]] || die "remote actions require root"
RUN_ID="${QID}-${ACTION}-$(date +%Y%m%dT%H%M%S%z)"
EVIDENCE_DIR="${EVIDENCE_ROOT}/${QID}/${RUN_ID}"
install -d -m 0700 "${EVIDENCE_DIR}"
exec > >(tee -a "${EVIDENCE_DIR}/terminal.log") 2>&1
exec 9>/run/lock/zhct-emqx-q142-q143.lock
flock -n 9 || die "another EMQX Q-142/Q-143 action is running"

COMMAND_RC=0
MUTATION_STARTED=0
SUCCESS=0
finish() {
  local rc=$?
  set +e
  if [[ "${ACTION}" == execute && "${MUTATION_STARTED}" -eq 1 && "${SUCCESS}" -ne 1 ]]; then
    printf 'AUTO_ROLLBACK_BEGIN\n'
    systemctl disable --now emqx.service mosquitto.service
    systemctl reset-failed emqx.service mosquitto.service 2>/dev/null || true
    if [[ -f "${ROLLBACK_ROOT}/${QID}/emqx.conf.placeholder" ]]; then
      install -m 0600 -o emqx -g emqx \
        "${ROLLBACK_ROOT}/${QID}/emqx.conf.placeholder" /etc/emqx/emqx.conf
    fi
    rm -f "${COOKIE_FILE}" "${COOKIE_TRANSFER}" "${ACL_FILE}"
    find /var/lib/emqx /var/log/emqx -mindepth 1 -delete 2>/dev/null
    chown root:root "${ETC_DIR}"
    chmod 0750 "${ETC_DIR}"
    local rollback_ok=1 rollback_enabled rollback_active runtime_dir
    [[ ! -e "${COOKIE_FILE}" ]] || rollback_ok=0
    [[ ! -e "${COOKIE_TRANSFER}" ]] || rollback_ok=0
    [[ ! -e "${ACL_FILE}" ]] || rollback_ok=0
    [[ -f /etc/emqx/emqx.conf ]] || rollback_ok=0
    grep -Fq '__EMQX_CLUSTER_COOKIE__' /etc/emqx/emqx.conf || rollback_ok=0
    [[ "$(stat -c '%U:%G:%a' "${ETC_DIR}")" == root:root:750 ]] ||
      rollback_ok=0
    for rollback_unit in emqx.service mosquitto.service; do
      rollback_enabled=$(systemctl is-enabled "${rollback_unit}" 2>&1 || true)
      rollback_active=$(systemctl is-active "${rollback_unit}" 2>&1 || true)
      printf 'AUTO_ROLLBACK_UNIT=%s enabled=%s active=%s\n' \
        "${rollback_unit}" "${rollback_enabled}" "${rollback_active}"
      [[ "${rollback_enabled}" == disabled ]] || rollback_ok=0
      [[ "${rollback_active}" == inactive ]] || rollback_ok=0
    done
    if pgrep -x beam.smp >/dev/null || pgrep -x mosquitto >/dev/null; then
      printf 'AUTO_ROLLBACK_TARGET_PROCESSES_UNEXPECTED\n'
      rollback_ok=0
    else
      printf 'AUTO_ROLLBACK_TARGET_PROCESSES_ZERO\n'
    fi
    if ss -lntupH | grep -E "${TARGET_PORT_RE}" >/dev/null; then
      printf 'AUTO_ROLLBACK_TARGET_LISTENERS_UNEXPECTED\n'
      rollback_ok=0
    else
      printf 'AUTO_ROLLBACK_TARGET_LISTENERS_ZERO\n'
    fi
    for runtime_dir in /var/lib/emqx /var/log/emqx; do
      if [[ ! -d "${runtime_dir}" ]]; then
        printf 'AUTO_ROLLBACK_RUNTIME_DIRECTORY_MISSING=%s\n' "${runtime_dir}"
        rollback_ok=0
      elif find "${runtime_dir}" -mindepth 1 -print -quit 2>/dev/null |
          grep -q .; then
        printf 'AUTO_ROLLBACK_RUNTIME_DIRECTORY_NOT_EMPTY=%s\n' "${runtime_dir}"
        rollback_ok=0
      else
        printf 'AUTO_ROLLBACK_RUNTIME_DIRECTORY_EMPTY=%s\n' "${runtime_dir}"
      fi
    done
    if [[ "${rollback_ok}" -eq 1 ]]; then
      printf 'AUTO_ROLLBACK_STATUS=PASS_SEALED_STATE_RESTORED\n'
    else
      printf 'AUTO_ROLLBACK_STATUS=FAIL_MANUAL_REVIEW_REQUIRED\n'
    fi
  fi
  printf 'END_TIME=%s\n' "$(date '+%Y-%m-%dT%H:%M:%S%z')"
  printf 'COMMAND_RC=%s\n' "${rc}"
  exit "${rc}"
}
trap finish EXIT

printf 'START_TIME=%s\n' "$(date '+%Y-%m-%dT%H:%M:%S%z')"
printf 'TARGET_HOST=%s\n' "${EXPECTED_HOST}"
printf 'TARGET_IP=%s\n' "${EXPECTED_IP}"
printf 'TASK=%s\n' "${QID}"
printf 'ACTION=%s\n' "${ACTION}"
printf 'ARGV='
printf '%q ' "$0" "${QID}" "${ACTION}"
[[ -n "${EXPECTED_COOKIE_SHA}" ]] &&
  printf '%q ' --expected-cookie-sha256 "${EXPECTED_COOKIE_SHA}"
[[ -n "${CONFIRM}" ]] && printf '%q ' --confirm "${CONFIRM}"
printf '\n'
printf 'EVIDENCE_DIR=%s\n' "${EVIDENCE_DIR}"

require_sha() {
  [[ "${EXPECTED_COOKIE_SHA}" =~ ^[0-9a-f]{64}$ ]] ||
    die "a 64-character lowercase expected cookie SHA256 is required"
}

assert_identity() {
  [[ "$(hostname -s)" == "${EXPECTED_HOST}" ]] || die "hostname mismatch"
  ip -o -4 addr show scope global | awk '{print $4}' | cut -d/ -f1 |
    grep -Fxq "${EXPECTED_IP}" || die "expected IPv4 is not configured"
  [[ "$(uname -m)" == x86_64 ]] || die "architecture mismatch"
  [[ -f /etc/zhct/node.env ]] || die "node.env is missing"
  [[ "$(stat -c '%U:%G' /etc/zhct/node.env)" == root:root ]] ||
    die "node.env owner mismatch"
  [[ "$(stat -c '%a' /etc/zhct/node.env)" == 640 ]] ||
    die "node.env mode mismatch"
  grep -Fxq 'ROLE=emqx' /etc/zhct/node.env || die "node.env role mismatch"
  grep -Fxq "NODE_KEY=${NODE_KEY}" /etc/zhct/node.env || die "node.env node mismatch"
  grep -Fxq "NODE_IP=${EXPECTED_IP}" /etc/zhct/node.env || die "node.env IP mismatch"
  grep -Fxq "NODE_NAME=${EXPECTED_HOST}" /etc/zhct/node.env ||
    die "node.env hostname mismatch"
  printf 'HOSTNAME_IP_IDENTITY=PASS\n'
  printf 'NODE_ENV_SHA256=%s\n' "$(sha256_file /etc/zhct/node.env)"
}

assert_packages() {
  rpm -q emqx-5.8.9-1.el8.x86_64
  rpm -q mosquitto libatomic cjson libwebsockets
  printf 'EMQX_VERSION=5.8.9\n'
}

assert_package_payload() {
  verify_manifest
  [[ "$(sha256_file "${FRAMEWORK_ARCHIVE}")" == "${FRAMEWORK_SHA256}" ]] ||
    die "framework archive digest mismatch"
  printf 'FRAMEWORK_ARCHIVE_SHA256=%s\n' "${FRAMEWORK_SHA256}"
  [[ -f "${ACL_SOURCE}" ]] || die "RPM ACL payload is missing"
  [[ "$(sha256_file "${ACL_SOURCE}")" == "${ACL_SHA256}" ]] ||
    die "RPM ACL payload digest mismatch"
  [[ "$(stat -c '%s' "${ACL_SOURCE}")" == 6763 ]] ||
    die "RPM ACL payload size mismatch"
  printf 'RPM_ACL_PAYLOAD_SHA256=%s\n' "${ACL_SHA256}"
  printf 'RPM_ACL_PAYLOAD_SIZE=6763\n'
}

assert_placeholder_config() {
  [[ -f /etc/emqx/emqx.conf ]] || die "EMQX runtime config is missing"
  grep -Fq '__EMQX_CLUSTER_COOKIE__' /etc/emqx/emqx.conf ||
    die "runtime config is not in placeholder state"
  grep -Fq "name = \"${EXPECTED_NODE}\"" /etc/emqx/emqx.conf ||
    die "EMQX node.name mismatch"
  [[ "$(sha256_file /etc/emqx/emqx.conf)" == "${TEMPLATE_SHA}" ]] ||
    die "runtime placeholder config digest mismatch"
  printf 'NODE_NAME_UNIQUE=%s\n' "${EXPECTED_NODE}"
  printf 'RUNTIME_PLACEHOLDER_CONFIG_SHA256=%s\n' "${TEMPLATE_SHA}"
  printf 'PRE_ACTIVATION_CONFIG_PLACEHOLDER=PASS\n'
}

assert_quiet_state() {
  local enabled active
  enabled=$(systemctl is-enabled emqx.service 2>/dev/null || true)
  active=$(systemctl is-active emqx.service 2>/dev/null || true)
  [[ "${enabled}" == disabled ]] || die "emqx.service is not disabled"
  [[ "${active}" == inactive ]] || die "emqx.service is not inactive"
  enabled=$(systemctl is-enabled mosquitto.service 2>/dev/null || true)
  active=$(systemctl is-active mosquitto.service 2>/dev/null || true)
  [[ "${enabled}" == disabled ]] || die "mosquitto.service is not disabled"
  [[ "${active}" == inactive ]] || die "mosquitto.service is not inactive"
  [[ -z "$(pgrep -fa 'beam.smp|/usr/lib/emqx|mosquitto' || true)" ]] ||
    die "target process exists before activation"
  [[ -z "$(ss -lntupH | grep -E "${TARGET_PORT_RE}" || true)" ]] ||
    die "target listener exists before activation"
  if find /var/lib/emqx /var/log/emqx -mindepth 1 -print -quit 2>/dev/null |
      grep -q .; then
    die "inherited EMQX runtime state is not empty"
  fi
  [[ ! -e "${ACL_FILE}" ]] || die "inherited RPM ACL file exists before activation"
  [[ "$(stat -c '%U:%G:%a' "${ETC_DIR}")" == root:root:750 ]] ||
    die "pre-activation /etc/emqx metadata mismatch"
  printf 'TARGET_UNITS_DISABLED_INACTIVE\n'
  printf 'TARGET_PROCESSES_ZERO\n'
  printf 'TARGET_LISTENERS_ZERO\n'
  printf 'INHERITED_RUNTIME_STATE_EMPTY\n'
  printf 'PRE_ACTIVATION_RPM_ACL_ABSENT\n'
  printf 'PRE_ACTIVATION_ETC_DIR_OWNER_MODE=root:root:750\n'
}

report_cookie_transfer() {
  if [[ -e "${COOKIE_FILE}" ]]; then
    die "destination cookie already exists before this task"
  fi
  if [[ -f "${COOKIE_TRANSFER}" ]]; then
    [[ "$(stat -c '%U:%G' "${COOKIE_TRANSFER}")" == root:root ]] ||
      die "cookie transfer owner mismatch"
    [[ "$(stat -c '%a' "${COOKIE_TRANSFER}")" == 600 ]] ||
      die "cookie transfer mode mismatch"
    [[ -s "${COOKIE_TRANSFER}" ]] || die "cookie transfer is empty"
    local digest
    digest=$(sha256_file "${COOKIE_TRANSFER}")
    printf 'COOKIE_TRANSFER_PRESENT=PASS\n'
    printf 'COOKIE_TRANSFER_OWNER=root:root\n'
    printf 'COOKIE_TRANSFER_MODE=0600\n'
    printf 'COOKIE_TRANSFER_SIZE=%s\n' "$(stat -c '%s' "${COOKIE_TRANSFER}")"
    printf 'COOKIE_TRANSFER_SHA256=%s\n' "${digest}"
    if [[ -n "${EXPECTED_COOKIE_SHA}" ]]; then
      require_sha
      [[ "${digest}" == "${EXPECTED_COOKIE_SHA}" ]] ||
        die "cookie transfer SHA256 mismatch"
      printf 'COOKIE_TRANSFER_SHA256_MATCH=PASS\n'
    fi
  else
    printf 'COOKIE_TRANSFER_PRESENT=NO\n'
    [[ "${ACTION}" != execute ]] || die "cookie transfer file is required"
  fi
  printf 'COOKIE_VALUE_OUTPUT=PROHIBITED\n'
}

preflight_checks() {
  assert_identity
  assert_packages
  assert_package_payload
  assert_placeholder_config
  assert_quiet_state
  report_cookie_transfer
  printf 'PREVIOUS_TASK_REQUIRED=%s_PASS\n' "${PREVIOUS_QID^^}"
  printf 'Q200_NETWORK_GATE=NOT_PERFORMED\n'
  printf 'Q240_FORMAL_CLUSTER_ACCEPTANCE=NOT_PERFORMED\n'
  printf 'KEEPALIVED_VIP=OUT_OF_SCOPE\n'
}

install_framework_payload() {
  install -d -m 0750 -o root -g root "${BUNDLE_ROOT}"
  tar -xzf "${FRAMEWORK_ARCHIVE}" -C "${BUNDLE_ROOT}"
  printf '%s  %s\n' \
    e637ca657a897c6d0b689bbd6347f04516baf5c9f32350cd1e8a2071c211a0a5 \
    "${BUNDLE_ROOT}/common/scripts/activate-role.sh" | sha256sum -c -
  printf '%s  %s\n' "${TEMPLATE_SHA}" \
    "${BUNDLE_ROOT}/roles/emqx/templates/nodes/${NODE_KEY}/emqx.conf" |
    sha256sum -c -
  printf 'LOCKED_FRAMEWORK_RESTORED=PASS\n'
}

local_pubsub() {
  local topic payload sub_output sub_pid received
  topic="sanquan/${QID}/${NODE_KEY}/$(date +%s)-$$"
  payload="${QID}_${NODE_KEY}_$(openssl rand -hex 12)"
  sub_output=$(mktemp)
  timeout 15 mosquitto_sub -h 127.0.0.1 -p 1883 \
    -t "${topic}" -C 1 -W 12 >"${sub_output}" &
  sub_pid=$!
  sleep 1
  mosquitto_pub -h 127.0.0.1 -p 1883 -t "${topic}" -m "${payload}"
  wait "${sub_pid}"
  received=$(<"${sub_output}")
  rm -f "${sub_output}"
  [[ "${received}" == "${payload}" ]] || die "local pub/sub mismatch"
  printf 'PUBSUB_TOPIC=%s\n' "${topic}"
  printf 'PUBSUB_SENT=%s\n' "${payload}"
  printf 'PUBSUB_RECEIVED=%s\n' "${received}"
  printf 'PUBSUB_MATCH=PASS\n'
}

active_verify() {
  local readiness_ready=0 readiness_wait_seconds=0
  require_sha
  assert_identity
  assert_packages
  [[ -s "${COOKIE_FILE}" ]] || die "cookie file is missing or empty"
  [[ "$(stat -c '%U:%G' "${COOKIE_FILE}")" == root:root ]] ||
    die "cookie owner mismatch"
  [[ "$(stat -c '%a' "${COOKIE_FILE}")" == 600 ]] || die "cookie mode mismatch"
  local cookie_sha
  cookie_sha=$(sha256_file "${COOKIE_FILE}")
  [[ "${cookie_sha}" == "${EXPECTED_COOKIE_SHA}" ]] || die "cookie SHA256 mismatch"
  printf 'COOKIE_PRESENT=PASS\n'
  printf 'COOKIE_OWNER=root:root\n'
  printf 'COOKIE_MODE=0600\n'
  printf 'COOKIE_SIZE=%s\n' "$(stat -c '%s' "${COOKIE_FILE}")"
  printf 'COOKIE_SHA256=%s\n' "${cookie_sha}"
  printf 'COOKIE_VALUE_OUTPUT=PROHIBITED\n'
  [[ -f "${ACL_FILE}" ]] || die "RPM ACL file is missing"
  [[ "$(sha256_file "${ACL_FILE}")" == "${ACL_SHA256}" ]] ||
    die "RPM ACL file digest mismatch"
  [[ "$(stat -c '%U:%G:%a:%s' "${ACL_FILE}")" == emqx:emqx:644:6763 ]] ||
    die "RPM ACL file metadata mismatch"
  printf 'RPM_ACL_FILE_SHA256=%s\n' "${ACL_SHA256}"
  printf 'RPM_ACL_FILE_OWNER_MODE_SIZE=emqx:emqx:644:6763\n'
  [[ -f /etc/emqx/emqx.conf ]] || die "runtime config missing"
  ! grep -Fq '__EMQX_CLUSTER_COOKIE__' /etc/emqx/emqx.conf ||
    die "cookie placeholder remains"
  grep -Fq "name = \"${EXPECTED_NODE}\"" /etc/emqx/emqx.conf ||
    die "EMQX node.name mismatch"
  emqx check_config
  printf 'CHECK_CONFIG_RC=0\n'
  [[ "$(systemctl is-enabled emqx.service)" == enabled ]] ||
    die "emqx.service is not enabled"
  [[ "$(systemctl is-enabled mosquitto.service 2>/dev/null || true)" == disabled ]] ||
    die "mosquitto.service is not disabled"
  [[ "$(systemctl is-active mosquitto.service 2>/dev/null || true)" == inactive ]] ||
    die "mosquitto.service is not inactive"
  printf 'EMQX_READINESS_WAIT_BEGIN\n'
  for ((readiness_wait_seconds=0; readiness_wait_seconds<=60;
    readiness_wait_seconds++)); do
    if systemctl is-active --quiet emqx.service &&
      ss -lntH | grep -Eq ':(1883)([[:space:]]|$)' &&
      ss -lntH | grep -Eq ':(8083)([[:space:]]|$)' &&
      ss -lntH | grep -Eq ':(18083)([[:space:]]|$)'; then
      readiness_ready=1
      break
    fi
    systemctl is-failed --quiet emqx.service && break
    sleep 1
  done
  printf 'EMQX_READINESS_WAIT_SECONDS=%s\n' "${readiness_wait_seconds}"
  [[ "${readiness_ready}" -eq 1 ]] ||
    die "EMQX readiness did not reach active listeners within 60 seconds"
  printf 'EMQX_READINESS_WAIT_PASS\n'
  systemctl is-active --quiet emqx.service || die "emqx.service is not active"
  ss -lntH | grep -Eq ':(1883)([[:space:]]|$)' ||
    die "MQTT listener 1883 is absent"
  ss -lntH | grep -Eq ':(8083)([[:space:]]|$)' ||
    die "WebSocket listener 8083 is absent"
  ss -lntH | grep -Eq ':(18083)([[:space:]]|$)' ||
    die "dashboard listener 18083 is absent"
  ! pgrep -x mosquitto >/dev/null || die "mosquitto process must remain absent"
  printf 'LISTENERS_BEGIN\n'
  ss -lntupH | grep -E "${TARGET_PORT_RE}"
  printf 'LISTENERS_END\n'
  local_pubsub
  printf 'NODE_STATUS_BEGIN\n'
  emqx ctl status
  printf 'NODE_STATUS_END\n'
  printf 'CLUSTER_STATUS_OBSERVATION_BEGIN\n'
  emqx ctl cluster status 2>&1 |
    sed -E '/cookie|secret|password|token/Id' || true
  printf 'CLUSTER_STATUS_OBSERVATION_END\n'
  printf 'CLUSTER_STATUS_IS_OBSERVATION_ONLY=YES\n'
  printf 'UNIT_STATE=EMQX_ENABLED_ACTIVE_MOSQUITTO_DISABLED_INACTIVE\n'
  printf 'LOG_FILES_BEGIN\n'
  find /var/log/emqx -maxdepth 2 -type f -printf '%p\t%U:%G\t%m\t%s\n' | sort
  printf 'LOG_FILES_END\n'
  printf 'JOURNAL_ERRORS_BEGIN\n'
  journalctl -u emqx.service -p err -n 50 --no-pager |
    sed -E '/cookie|secret|password|token/Id' || true
  printf 'JOURNAL_ERRORS_END\n'
  printf 'Q200_NETWORK_GATE=NOT_PERFORMED\n'
  printf 'Q240_FORMAL_CLUSTER_ACCEPTANCE=NOT_PERFORMED\n'
  printf 'KEEPALIVED_VIP=OUT_OF_SCOPE\n'
  printf '%s_NODE_ACTIVATION_VERIFY_PASS\n' "${QID^^}"
}

case "${ACTION}" in
  preflight)
    preflight_checks
    printf '%s_PREFLIGHT_PASS\n' "${QID^^}"
    SUCCESS=1
    ;;
  execute)
    require_sha
    [[ "${CONFIRM}" == "${QID}-execute" ]] ||
      die "execute requires --confirm ${QID}-execute"
    for cmd in sha256sum rpm systemctl ss flock openssl timeout \
      mosquitto_pub mosquitto_sub emqx runuser; do
      command -v "${cmd}" >/dev/null || die "required command missing: ${cmd}"
    done
    preflight_checks
    install -d -m 0700 "${ROLLBACK_ROOT}/${QID}"
    install -m 0600 -o root -g root /etc/emqx/emqx.conf \
      "${ROLLBACK_ROOT}/${QID}/emqx.conf.placeholder"
    MUTATION_STARTED=1
    install_framework_payload
    chown root:emqx "${ETC_DIR}"
    chmod 0750 "${ETC_DIR}"
    [[ "$(stat -c '%U:%G:%a' "${ETC_DIR}")" == root:emqx:750 ]] ||
      die "runtime /etc/emqx metadata mismatch"
    runuser -u emqx -- test -x "${ETC_DIR}" ||
      die "emqx user cannot traverse /etc/emqx"
    runuser -u emqx -- test -r /etc/emqx/emqx.conf ||
      die "emqx user cannot read runtime config"
    runuser -u emqx -- test -r /etc/emqx/base.hocon ||
      die "emqx user cannot read base.hocon"
    printf 'RUNTIME_ETC_DIR_OWNER_MODE=root:emqx:750\n'
    printf 'EMQX_USER_RUNTIME_CONFIG_ACCESS=PASS\n'
    install -d -m 0700 -o root -g root /etc/zhct/secrets
    install -m 0600 -o root -g root "${COOKIE_TRANSFER}" "${COOKIE_FILE}"
    rm -f "${COOKIE_TRANSFER}"
    install -m 0644 -o emqx -g emqx "${ACL_SOURCE}" "${ACL_FILE}"
    [[ "$(sha256_file "${ACL_FILE}")" == "${ACL_SHA256}" ]] ||
      die "installed RPM ACL digest mismatch"
    printf 'RPM_ACL_RESTORE_PASS\n'
    [[ "$(sha256_file "${COOKIE_FILE}")" == "${EXPECTED_COOKIE_SHA}" ]] ||
      die "installed cookie SHA256 mismatch"
    "${BUNDLE_ROOT}/common/scripts/activate-role.sh" \
      --role emqx --bundle-root "${BUNDLE_ROOT}"
    active_verify
    SUCCESS=1
    printf '%s_EXECUTE_PASS\n' "${QID^^}"
    ;;
  verify)
    active_verify
    SUCCESS=1
    printf '%s_VERIFY_PASS\n' "${QID^^}"
    ;;
  rollback)
    [[ "${CONFIRM}" == "${QID}-rollback" ]] ||
      die "rollback requires --confirm ${QID}-rollback"
    systemctl disable --now emqx.service mosquitto.service
    systemctl reset-failed emqx.service mosquitto.service 2>/dev/null || true
    if [[ -f "${ROLLBACK_ROOT}/${QID}/emqx.conf.placeholder" ]]; then
      install -m 0600 -o emqx -g emqx \
        "${ROLLBACK_ROOT}/${QID}/emqx.conf.placeholder" /etc/emqx/emqx.conf
    else
      die "rollback placeholder backup is missing"
    fi
    rm -f "${COOKIE_FILE}" "${COOKIE_TRANSFER}" "${ACL_FILE}"
    find /var/lib/emqx /var/log/emqx -mindepth 1 -delete 2>/dev/null
    chown root:root "${ETC_DIR}"
    chmod 0750 "${ETC_DIR}"
    assert_quiet_state
    assert_placeholder_config
    rm -rf "${ROLLBACK_ROOT:?}/${QID}"
    printf 'ROLLBACK_COOKIE_REMOVED=PASS\n'
    printf 'ROLLBACK_RUNTIME_EMPTY=PASS\n'
    printf '%s_ROLLBACK_PASS\n' "${QID^^}"
    SUCCESS=1
    ;;
  archive)
    archive_path="/root/${QID}-evidence-$(date +%Y%m%dT%H%M%S%z).tar.gz"
    tar -czf "${archive_path}" -C "${EVIDENCE_ROOT}" "${QID}"
    printf 'ARCHIVE_PATH=%s\n' "${archive_path}"
    printf 'ARCHIVE_SHA256=%s\n' "$(sha256_file "${archive_path}")"
    printf 'ARCHIVE_MODE=%s\n' "$(stat -c '%a' "${archive_path}")"
    printf '%s_ARCHIVE_PASS\n' "${QID^^}"
    SUCCESS=1
    ;;
esac

trap - EXIT
printf 'END_TIME=%s\n' "$(date '+%Y-%m-%dT%H:%M:%S%z')"
printf 'COMMAND_RC=0\n'
exit 0