#!/usr/bin/env bash

set -Eeuo pipefail
umask 077

EXPECTED_HOST="sqjcapp02"
EXPECTED_IP="10.2.203.105"
PACKAGE_ARCHIVE="/root/sanquan-app-q110-112.tar.gz"
PACKAGE_SHA256="f0133d79e006d3be738ceb38883e740fc895b83578ab28657ce4fe5eaa3f953b"
PACKAGE_ROOT="/root/sanquan-app-q110-112"
ENTRY="${PACKAGE_ROOT}/scripts/app-q110-112.sh"
ENTRY_SHA256="33d10c2407b7382bd43953b6b7b60dd28021eecf599e07a2f963053e87336750"
ACTIVATE_SHA256="1cc3ae5fc7ff7485968018d4ec5ce7a5506539afce90685753cc95c7c40de516"
VERIFY_SHA256="bf81fa1e6ca8db1550e7d1ec665c188cb9808e707094006e3ae109f504f16364"

Q110_STAGE3B_RUN="q110-app02-postclone-execute-20260729T185051+0800"
Q110_STAGE3B_RESULT="PASS"

Q111_RECEIPT="/root/sanquan-app-q110-112-gates/Q111_APP01_ACTIVE.receipt"
Q111_RECEIPT_SHA256="PENDING_Q111_PASS_DO_NOT_EXECUTE"
EXECUTE_GATE="/root/sanquan-app-q110-112-gates/Q112_EXECUTE.pass"
EXECUTE_GATE_SHA256="PENDING_CONTROLLER_Q112_AUTHORIZATION_DO_NOT_EXECUTE"

RUN_ID=""
BACKUP_DIR=""
AUDIT_LOG="STDOUT_ONLY_BEFORE_PERSISTENT_WRITE"
EXECUTION_STARTED=0
ACTIVATION_PASS=0
VERIFY_PASS=0
ARCHIVE_PASS=0
ROLLBACK_STATE="NOT_REQUIRED_PREWRITE_GATE"
FINAL_RESULT="FAIL"

timestamp() {
    date '+%Y-%m-%dT%H:%M:%S%z'
}

assert_unit_state() {
    local unit=$1 expected_enabled=$2 expected_active=$3
    local enabled active
    enabled=$(systemctl is-enabled "$unit" 2>/dev/null || true)
    active=$(systemctl is-active "$unit" 2>/dev/null || true)
    printf '%s enabled=%s active=%s\n' "$unit" "$enabled" "$active"
    [[ "$enabled" == "$expected_enabled" && "$active" == "$expected_active" ]]
}

assert_forbidden_runtime_absent() {
    ! pgrep -x supervisord
    ! pgrep -x crond
    ! pgrep -x php
}

rollback_after_execution_failure() {
    local rollback_bad=0

    echo Q112_APP02_ROLLBACK_BEGIN
    timestamp
    systemctl disable --now \
        nginx.service php-fpm73.service php-fpm74.service \
        supervisor.service crond.service || rollback_bad=1
    systemctl reset-failed \
        nginx.service php-fpm73.service php-fpm74.service \
        supervisor.service crond.service || true

    if [[ -n "$BACKUP_DIR" && -f "$BACKUP_DIR/prechange-files.tar.gz" ]]; then
        tar -xzf "$BACKUP_DIR/prechange-files.tar.gz" -C / || rollback_bad=1
        systemctl daemon-reload || rollback_bad=1
    else
        echo Q112_APP02_ROLLBACK_BACKUP=ABSENT
        rollback_bad=1
    fi

    for unit in nginx.service php-fpm73.service php-fpm74.service supervisor.service crond.service; do
        assert_unit_state "$unit" disabled inactive || rollback_bad=1
    done
    if ss -H -lntup | grep -Eq ':(8081|18081)([[:space:]]|$)'; then
        echo Q112_APP02_ROLLBACK_TARGET_LISTENER=STILL_PRESENT
        rollback_bad=1
    else
        echo Q112_APP02_ROLLBACK_TARGET_LISTENER=ZERO
    fi
    for process_name in nginx php-fpm supervisord crond php; do
        if pgrep -a -x "$process_name"; then
            rollback_bad=1
        fi
    done

    if [[ "$rollback_bad" -eq 0 ]]; then
        ROLLBACK_STATE="PASS_DISABLED_INACTIVE_PRECHANGE_FILES_RESTORED"
    else
        ROLLBACK_STATE="FAIL_MANUAL_REVIEW_REQUIRED"
    fi
    echo "Q112_APP02_ROLLBACK_STATE=$ROLLBACK_STATE"
    echo Q112_APP02_ROLLBACK_END
}

finish() {
    local rc=$?
    trap - EXIT
    set +e

    if [[ "$rc" -ne 0 && "$EXECUTION_STARTED" -eq 1 ]]; then
        rollback_after_execution_failure
    fi
    if [[ "$rc" -eq 0 ]]; then
        FINAL_RESULT="PASS"
        ROLLBACK_STATE="NOT_REQUIRED_ACTIVATE_VERIFY_ARCHIVE_PASS"
    fi

    echo "Q112_APP02_RUN_ID=${RUN_ID:-NOT_CREATED}"
    echo "Q112_APP02_TARGET=${EXPECTED_HOST}/${EXPECTED_IP}"
    echo "Q112_APP02_Q110_STAGE3B_RUN=$Q110_STAGE3B_RUN"
    echo "Q112_APP02_Q110_STAGE3B_RESULT=$Q110_STAGE3B_RESULT"
    echo "Q112_APP02_PACKAGE_SHA256=$PACKAGE_SHA256"
    echo "Q112_APP02_ACTIVATION_PASS=$ACTIVATION_PASS"
    echo "Q112_APP02_VERIFY_PASS=$VERIFY_PASS"
    echo "Q112_APP02_ARCHIVE_PASS=$ARCHIVE_PASS"
    echo "Q112_APP02_BACKUP_DIR=${BACKUP_DIR:-NOT_CREATED}"
    echo "Q112_APP02_AUDIT_LOG=$AUDIT_LOG"
    echo "Q112_APP02_RC=$rc"
    echo "Q112_APP02_RESULT=$FINAL_RESULT"
    echo "Q112_APP02_ROLLBACK_STATE=$ROLLBACK_STATE"
    echo Q112_APP02_ALLOWED_UNITS=nginx.service,php-fpm73.service,php-fpm74.service
    echo Q112_APP02_FORBIDDEN_UNITS=supervisor.service,crond.service
    echo Q112_APP02_WORKERS_CONSUMERS_SINGLETON_TASKS=NOT_STARTED
    echo Q112_APP02_END
    exit "$rc"
}
trap finish EXIT

echo Q112_APP02_BEGIN
timestamp
echo "EXPECTED_HOST=$EXPECTED_HOST EXPECTED_IP=$EXPECTED_IP"
echo CONTROLLER_LOCAL_PREP_AUTHORIZATION=Q112_LOCAL_CANDIDATE_ONLY
echo REMOTE_EXECUTE_AUTHORIZATION=BLOCKED_PENDING_Q111_RECEIPT_AND_Q112_GATE
echo "Q110_STAGE3B_RUN=$Q110_STAGE3B_RUN"
echo "Q110_STAGE3B_RESULT=$Q110_STAGE3B_RESULT"

[[ $# -eq 1 && "$1" == "--execute" ]] || {
    echo "ERROR: exact --execute argument is required" >&2
    exit 2
}
[[ $(id -u) -eq 0 ]] || {
    echo "ERROR: root required" >&2
    exit 10
}

# Everything through Q112_APP02_PREWRITE_GATE_RESULT is read-only. The fixed
# target check happens before any directory, log, backup, or service mutation.
actual_host=$(hostname -s)
echo "ACTUAL_HOST=$actual_host"
[[ "$actual_host" == "$EXPECTED_HOST" ]] || {
    echo "ERROR: hostname mismatch; persistent write not started" >&2
    exit 11
}
ip -o -4 addr show scope global
ip -o -4 addr show scope global |
    grep -Eq "(^|[[:space:]])${EXPECTED_IP//./\\.}/" || {
    echo "ERROR: expected IPv4 missing; persistent write not started" >&2
    exit 12
}

if [[ "$Q111_RECEIPT_SHA256" == PENDING_* ]]; then
    echo "ERROR: Q111 PASS receipt hash is not locked; Q112 remains blocked" >&2
    exit 13
fi
[[ -f "$Q111_RECEIPT" && ! -L "$Q111_RECEIPT" ]] || {
    echo "ERROR: Q111 PASS receipt is absent or unsafe" >&2
    exit 14
}
[[ $(stat -c '%U:%G' "$Q111_RECEIPT") == root:root ]] || {
    echo "ERROR: Q111 PASS receipt owner mismatch" >&2
    exit 15
}
receipt_mode=$(stat -c '%a' "$Q111_RECEIPT")
[[ "$receipt_mode" == 400 || "$receipt_mode" == 600 ]] || {
    echo "ERROR: Q111 PASS receipt mode must be 0400 or 0600" >&2
    exit 16
}
echo "${Q111_RECEIPT_SHA256}  ${Q111_RECEIPT}" | sha256sum -c -
grep -Fxq 'Q111_APP01_RESULT=PASS' "$Q111_RECEIPT"
grep -Fxq 'Q111_APP01_RC=0' "$Q111_RECEIPT"
grep -Fxq 'Q111_APP01_ROLLBACK_STATE=NOT_REQUIRED_ACTIVATE_VERIFY_ARCHIVE_PASS' "$Q111_RECEIPT"
grep -Fxq 'Q111_APP01_TARGET=sqjcapp01/10.2.203.104' "$Q111_RECEIPT"

if [[ "$EXECUTE_GATE_SHA256" == PENDING_* ]]; then
    echo "ERROR: controller Q112 execute gate hash is not locked" >&2
    exit 17
fi
[[ -f "$EXECUTE_GATE" && ! -L "$EXECUTE_GATE" ]] || {
    echo "ERROR: controller Q112 execute gate is absent or unsafe" >&2
    exit 18
}
[[ $(stat -c '%U:%G' "$EXECUTE_GATE") == root:root ]] || {
    echo "ERROR: controller Q112 execute gate owner mismatch" >&2
    exit 19
}
gate_mode=$(stat -c '%a' "$EXECUTE_GATE")
[[ "$gate_mode" == 400 || "$gate_mode" == 600 ]] || {
    echo "ERROR: controller Q112 execute gate mode must be 0400 or 0600" >&2
    exit 20
}
echo "${EXECUTE_GATE_SHA256}  ${EXECUTE_GATE}" | sha256sum -c -
grep -Fxq 'FORMAT=Q112_EXECUTE_GATE_V1' "$EXECUTE_GATE"
grep -Fxq 'CONTROLLER_Q112_EXECUTE_AUTHORIZED' "$EXECUTE_GATE"
grep -Fxq "TARGET=${EXPECTED_HOST}/${EXPECTED_IP}" "$EXECUTE_GATE"
grep -Fxq "PACKAGE_SHA256=${PACKAGE_SHA256}" "$EXECUTE_GATE"
grep -Fxq "Q110_STAGE3B_RUN=${Q110_STAGE3B_RUN}" "$EXECUTE_GATE"

[[ -f "$PACKAGE_ARCHIVE" && ! -L "$PACKAGE_ARCHIVE" ]]
[[ $(stat -c '%U:%G' "$PACKAGE_ARCHIVE") == root:root ]]
echo "${PACKAGE_SHA256}  ${PACKAGE_ARCHIVE}" | sha256sum -c -
[[ -d "$PACKAGE_ROOT" && ! -L "$PACKAGE_ROOT" ]]
echo "${ENTRY_SHA256}  ${ENTRY}" | sha256sum -c -
echo "${ACTIVATE_SHA256}  ${PACKAGE_ROOT}/scripts/app-q111-q112-activate.sh" | sha256sum -c -
echo "${VERIFY_SHA256}  ${PACKAGE_ROOT}/scripts/app-q110-q112-verify.sh" | sha256sum -c -
"$ENTRY" --static-validate

echo "e637ca657a897c6d0b689bbd6347f04516baf5c9f32350cd1e8a2071c211a0a5  /opt/zhct-deploy/common/scripts/activate-role.sh" | sha256sum -c -
echo "e3eae16aa5c87f43b56495fc71de378a947578cabe18b507e8fed01ac1f32025  /opt/zhct-deploy/common/bin/role-action.sh" | sha256sum -c -
echo "03c6b13a0def5bf3954e069581b6f9fbe0f81f24285ba2319b6279c0406d70aa  /opt/zhct-deploy/roles/app/hooks.sh" | sha256sum -c -
echo "d43e179c108f6db551b452256f867d6d4f6fc9e031c1d4def2f343966af8d684  /opt/zhct-deploy/roles/app/manifests/systemd-units.tsv" | sha256sum -c -

grep -Fxq 'ROLE=app' /etc/zhct/node.env
grep -Fxq 'NODE_KEY=app02' /etc/zhct/node.env
grep -Fxq "NODE_IP=$EXPECTED_IP" /etc/zhct/node.env
grep -Fxq "NODE_NAME=$EXPECTED_HOST" /etc/zhct/node.env
[[ -s /etc/machine-id ]]
[[ -s /etc/ssh/ssh_host_ed25519_key ]]

for required_path in \
    /usr/local/nginx/sbin/nginx \
    /usr/local/php7.3/sbin/php-fpm \
    /usr/local/php7.4/sbin/php-fpm \
    /usr/local/sbin/singleton-state-check.sh \
    /etc/supervisor/conf.d/sanquan.conf \
    /etc/cron.d/zhct-sanquan \
    /etc/systemd/system/nginx.service \
    /etc/systemd/system/php-fpm73.service \
    /etc/systemd/system/php-fpm74.service \
    /etc/systemd/system/supervisor.service; do
    [[ -e "$required_path" && ! -L "$required_path" ]]
done

/usr/local/nginx/sbin/nginx -t
/usr/local/php7.3/sbin/php-fpm -tt --fpm-config /usr/local/php7.3/etc/php-fpm.conf
/usr/local/php7.4/sbin/php-fpm -tt --fpm-config /usr/local/php7.4/etc/php-fpm.conf
/usr/local/sbin/singleton-state-check.sh \
    --node app02 \
    --config /etc/supervisor/conf.d/sanquan.conf \
    --cron /etc/cron.d/zhct-sanquan

for unit in nginx.service php-fpm73.service php-fpm74.service supervisor.service crond.service; do
    assert_unit_state "$unit" disabled inactive
done
! ss -H -lntup | grep -Eq ':(8081|18081)([[:space:]]|$)'
for process_name in nginx php-fpm supervisord crond php; do
    ! pgrep -a -x "$process_name"
done

echo Q112_APP02_PREWRITE_GATE_RESULT=PASS
echo Q112_APP02_PREWRITE_PERSISTENT_MUTATION=ZERO

RUN_ID="q112-app02-activate-$(date '+%Y%m%dT%H%M%S%z')"
BACKUP_DIR="/var/backups/zhct-q112/manual-activate/${RUN_ID}"
[[ ! -e "$BACKUP_DIR" && ! -L "$BACKUP_DIR" ]]
install -d -m 0700 -o root -g root "$BACKUP_DIR"
AUDIT_LOG="$BACKUP_DIR/q112-app02-activate-verify.log"
exec > >(tee "$AUDIT_LOG") 2>&1

echo Q112_APP02_PERSISTENT_PHASE_BEGIN
timestamp
echo "Q112_APP02_PREWRITE_GATE_RESULT=PASS"
echo "Q112_APP02_Q111_RECEIPT_SHA256=$Q111_RECEIPT_SHA256"
echo "Q112_APP02_EXECUTE_GATE_SHA256=$EXECUTE_GATE_SHA256"
echo "Q112_APP02_PACKAGE_SHA256=$PACKAGE_SHA256"
echo "Q112_APP02_ENTRY_SHA256=$ENTRY_SHA256"
echo "Q112_APP02_RUN_ID=$RUN_ID"

for unit in nginx.service php-fpm73.service php-fpm74.service supervisor.service crond.service; do
    assert_unit_state "$unit" disabled inactive
done >"$BACKUP_DIR/prechange-unit-state.tsv"
chmod 0600 "$BACKUP_DIR/prechange-unit-state.tsv"

tar -czf "$BACKUP_DIR/prechange-files.tar.gz" -C / \
    etc/zhct/node.env \
    etc/supervisor/conf.d/sanquan.conf \
    etc/cron.d/zhct-sanquan \
    etc/systemd/system/nginx.service \
    etc/systemd/system/php-fpm73.service \
    etc/systemd/system/php-fpm74.service \
    etc/systemd/system/supervisor.service \
    usr/local/nginx/conf/nginx.conf \
    usr/local/nginx/conf/conf.d/app.conf \
    usr/local/php7.3/etc/php.ini \
    usr/local/php7.3/etc/php-fpm.conf \
    usr/local/php7.4/etc/php.ini \
    usr/local/php7.4/etc/php-fpm.conf
chmod 0600 "$BACKUP_DIR/prechange-files.tar.gz"
sha256sum "$BACKUP_DIR/prechange-files.tar.gz" >"$BACKUP_DIR/prechange-files.tar.gz.sha256"
chmod 0600 "$BACKUP_DIR/prechange-files.tar.gz.sha256"
(cd "$BACKUP_DIR" && sha256sum -c prechange-files.tar.gz.sha256)
echo Q112_APP02_BACKUP_RESULT=PASS

EXECUTION_STARTED=1
"$ENTRY" activate app02 "$RUN_ID" --execute
ACTIVATION_PASS=1

"$ENTRY" verify app02 "$RUN_ID" active
VERIFY_PASS=1

/usr/local/nginx/sbin/nginx -t
/usr/local/php7.3/sbin/php-fpm -tt --fpm-config /usr/local/php7.3/etc/php-fpm.conf
/usr/local/php7.4/sbin/php-fpm -tt --fpm-config /usr/local/php7.4/etc/php-fpm.conf
/usr/local/sbin/singleton-state-check.sh \
    --node app02 \
    --config /etc/supervisor/conf.d/sanquan.conf \
    --cron /etc/cron.d/zhct-sanquan

for unit in nginx.service php-fpm73.service php-fpm74.service; do
    assert_unit_state "$unit" enabled active
done
for unit in supervisor.service crond.service; do
    assert_unit_state "$unit" disabled inactive
done
ss -H -ltn | awk '$4 ~ /:8081$/ {found=1} END {exit !found}'
ss -H -ltn | awk '$4 ~ /^127\\.0\\.0\\.1:18081$/ {found=1} END {exit !found}'
pgrep -a -x nginx
pgrep -a -x php-fpm
assert_forbidden_runtime_absent

"$ENTRY" archive q112 "$RUN_ID"
archive_path="/var/backups/zhct-q112/archives/${RUN_ID}.tar.gz"
archive_sha_path="${archive_path}.sha256"
[[ -f "$archive_path" && ! -L "$archive_path" ]]
[[ -f "$archive_sha_path" && ! -L "$archive_sha_path" ]]
(cd "$(dirname "$archive_path")" && sha256sum -c "$(basename "$archive_sha_path")")
ARCHIVE_PASS=1

echo Q112_APP02_HEALTH_RESULT=PASS_CONFIG_LISTENERS_PROCESSES_UNITS
echo Q112_APP02_PERSISTENT_PHASE_END