set -Eeuo pipefail
set +x
umask 077

readonly LABEL='Q102_NGINX01_THIRD_SENTINEL_V5'
readonly HOST='sqjcnginx01'
readonly IP='10.2.203.102'
readonly CLI5='/opt/redis/5.0.14/bin/redis-cli'
readonly CLI7='/opt/redis/7.2.14/bin/redis-cli'
readonly SERVER5='/opt/redis/5.0.14/bin/redis-sentinel'
readonly SERVER7='/opt/redis/7.2.14/bin/redis-sentinel'
readonly CONF5='/var/lib/redis-sentinel5/sentinel.conf'
readonly CONF7='/var/lib/redis-sentinel7/sentinel.conf'
readonly LOG5='/var/log/redis-sentinel5/sentinel.log'
readonly LOG7='/var/log/redis-sentinel7/sentinel.log'
readonly DROPIN5='/etc/systemd/system/sentinel5.service.d/99-zhct-q102-v5.conf'
readonly DROPIN7='/etc/systemd/system/sentinel7.service.d/99-zhct-q102-v5.conf'

backup=''
zone=''
mutation_started=0
success=0
secret_tmp=''
orchestrator_enabled=''
orchestrator_active=''
orchestrator_pid=''

rollback_stage() {
    set +e
    [[ "$mutation_started" -eq 1 && -n "$backup" ]] || return 0

    systemctl disable --now sentinel5.service sentinel7.service >/dev/null 2>&1 || true

    [[ ! -f "$backup/sentinel5.conf" ]] || cp -a --remove-destination "$backup/sentinel5.conf" "$CONF5"
    [[ ! -f "$backup/sentinel7.conf" ]] || cp -a --remove-destination "$backup/sentinel7.conf" "$CONF7"
    for spec in "sentinel5|$LOG5" "sentinel7|$LOG7"; do
        IFS='|' read -r name logfile <<<"$spec"
        if [[ -f "$backup/${name}-log-existed" ]]; then
            cp -a --remove-destination "$backup/${name}.log" "$logfile"
        else
            rm -f -- "$logfile"
        fi
    done

    if [[ -f "$backup/added-rich-rules.txt" && -n "$zone" ]]; then
        while IFS= read -r rule; do
            [[ -n "$rule" ]] || continue
            timeout 15 firewall-cmd --permanent --zone="$zone" --remove-rich-rule="$rule" >/dev/null 2>&1 || true
        done <"$backup/added-rich-rules.txt"
        timeout 15 firewall-cmd --reload >/dev/null 2>&1 || true
    fi

    [[ ! -f "$backup/port-26387-added" ]] || semanage port -d -p tcp 26387 >/dev/null 2>&1 || true

    while IFS='|' read -r marker pattern; do
        [[ -n "$marker" && -f "$backup/$marker" ]] || continue
        semanage fcontext -d "$pattern" >/dev/null 2>&1 || true
    done <<'FCONTEXT_ROLLBACK'
fcontext-lib5-added|/var/lib/redis-sentinel5(/.*)?
fcontext-lib7-added|/var/lib/redis-sentinel7(/.*)?
fcontext-log5-added|/var/log/redis-sentinel5(/.*)?
fcontext-log7-added|/var/log/redis-sentinel7(/.*)?
FCONTEXT_ROLLBACK

    if [[ -f "$backup/path-contexts.tsv" ]]; then
        while IFS='|' read -r path context; do
            [[ -e "$path" ]] || continue
            chcon "$context" "$path" >/dev/null 2>&1 || true
        done <"$backup/path-contexts.tsv"
    fi

    if [[ -f "$backup/directory-metadata.tsv" ]]; then
        while IFS='|' read -r path uid gid mode; do
            [[ -d "$path" ]] || continue
            chown "$uid:$gid" "$path" >/dev/null 2>&1 || true
            chmod "$mode" "$path" >/dev/null 2>&1 || true
        done <"$backup/directory-metadata.tsv"
    fi

    for spec in "sentinel5|$DROPIN5" "sentinel7|$DROPIN7"; do
        IFS='|' read -r name dropin <<<"$spec"
        if [[ -f "$backup/${name}-dropin-existed" ]]; then
            cp -a --remove-destination "$backup/${name}-dropin.conf" "$dropin"
        else
            rm -f -- "$dropin"
        fi
        if [[ -f "$backup/${name}-dropin-dir-created" ]]; then
            rmdir -- "$(dirname "$dropin")" >/dev/null 2>&1 || true
        fi
    done
    systemctl daemon-reload >/dev/null 2>&1 || true

    printf 'Q102_V5_ROLLBACK=COMPLETED BACKUP=%s\n' "$backup"
}

finish() {
    local rc=$?
    local current_enabled current_active current_pid orchestrator_gate
    set +e
    [[ -z "$secret_tmp" || ! -e "$secret_tmp" ]] || rm -f -- "$secret_tmp"

    if [[ "$rc" -ne 0 || "$success" -ne 1 ]]; then
        rollback_stage
        printf 'Q102_NGINX01_THIRD_SENTINEL_V5_FAIL RC=%s\n' "$rc"
    fi

    current_enabled="$(systemctl is-enabled orchestrator.service 2>/dev/null || true)"
    current_active="$(systemctl is-active orchestrator.service 2>/dev/null || true)"
    current_pid="$(systemctl show orchestrator.service -p MainPID --value 2>/dev/null || true)"
    orchestrator_gate=FAIL
    if [[ -n "$orchestrator_enabled" && "$current_enabled" == "$orchestrator_enabled" && "$current_active" == "$orchestrator_active" && "$current_pid" == "$orchestrator_pid" ]]; then
        orchestrator_gate=PASS
    fi
    printf '%s\n' \
        "ORCHESTRATOR_UNCHANGED=${orchestrator_gate} ENABLED=${current_enabled:-UNKNOWN} ACTIVE=${current_active:-UNKNOWN} PID=${current_pid:-UNKNOWN}" \
        "END|${LABEL}|RC=${rc}|BACKUP=${backup:-NOT_CREATED}"
    exit "$rc"
}
trap finish EXIT

printf 'BEGIN|%s|%s\n' "$LABEL" "$(date '+%Y-%m-%dT%H:%M:%S%z')"
printf '%s\n' 'OLD_V2_STATUS=REVOKED_DO_NOT_EXECUTE'
printf '%s\n' 'OLD_V3_STATUS=REVOKED_DO_NOT_EXECUTE'
printf '%s\n' 'OLD_V4_STATUS=REVOKED_DO_NOT_EXECUTE'

[[ "$(id -u)" -eq 0 ]] || { printf '%s\n' 'ERROR: root required' >&2; exit 1; }
[[ "$(hostname -s)" == "$HOST" ]] || { printf '%s\n' 'ERROR: hostname mismatch' >&2; exit 1; }
ip -o -4 addr show scope global | awk '{split($4,a,"/"); print a[1]}' | grep -Fxq "$IP" || {
    printf '%s\n' 'ERROR: target IP absent' >&2
    exit 1
}

for cmd in semanage matchpathcon restorecon chcon firewall-cmd timeout python3 systemctl ss stat awk grep; do
    command -v "$cmd" >/dev/null 2>&1 || { printf 'ERROR: command missing: %s\n' "$cmd" >&2; exit 1; }
done
semanage fcontext -l >/dev/null
semanage port -l >/dev/null

for binary in "$CLI5" "$CLI7" "$SERVER5" "$SERVER7"; do
    [[ -x "$binary" && ! -L "$binary" ]] || { printf 'ERROR: binary invalid: %s\n' "$binary" >&2; exit 1; }
done

for unit in sentinel5.service sentinel7.service; do
    [[ "$(systemctl show "$unit" -p LoadState --value)" == loaded ]] || exit 1
    [[ "$(systemctl is-enabled "$unit" 2>/dev/null || true)" == disabled ]] || exit 1
    [[ "$(systemctl is-active "$unit" 2>/dev/null || true)" == inactive ]] || exit 1
done

for unit in redis5.service redis7.service keepalived.service; do
    [[ "$(systemctl is-active "$unit" 2>/dev/null || true)" != active ]] || exit 1
done

orchestrator_enabled="$(systemctl is-enabled orchestrator.service 2>/dev/null || true)"
orchestrator_active="$(systemctl is-active orchestrator.service 2>/dev/null || true)"
orchestrator_pid="$(systemctl show orchestrator.service -p MainPID --value)"
[[ "$orchestrator_enabled" == enabled && "$orchestrator_active" == active && "$orchestrator_pid" =~ ^[1-9][0-9]*$ ]] || exit 1

for config in "$CONF5" "$CONF7"; do
    [[ -f "$config" && ! -L "$config" ]] || exit 1
    [[ "$(stat -c '%U:%G:%a' "$config")" == redis:redis:600 ]] || exit 1
done
grep -Eq '^sentinel monitor sanquan-redis5 10\.2\.203\.112 6379 2$' "$CONF5"
grep -Eq '^sentinel monitor sanquan-redis7 10\.2\.203\.112 6387 2$' "$CONF7"

for secret in /etc/zhct/secrets/redis5.password /etc/zhct/secrets/redis7.password; do
    [[ -f "$secret" && ! -L "$secret" && -s "$secret" ]] || exit 1
    [[ "$(stat -c '%U:%G' "$secret")" == root:root ]] || exit 1
    case "$(stat -c '%a' "$secret")" in 400|600) ;; *) exit 1 ;; esac
    [[ "$(awk 'END {print NR}' "$secret")" == 1 ]] || exit 1
done

for directory in /var/lib/redis-sentinel5 /var/lib/redis-sentinel7 /var/log/redis-sentinel5 /var/log/redis-sentinel7; do
    [[ -d "$directory" && ! -L "$directory" ]] || exit 1
done

timeout 15 firewall-cmd --state | grep -Fxq running
iface="$(ip -o -4 addr show scope global | awk '$4 ~ /^10\.2\.203\.102\// {print $2; exit}')"
[[ -n "$iface" ]] || exit 1
zone="$(timeout 15 firewall-cmd --get-zone-of-interface="$iface")"
[[ -n "$zone" && "$zone" != 'no zone' ]] || exit 1
[[ "$(systemctl is-active sshd 2>/dev/null || true)" == active ]]
ss -lntH | awk '$4 ~ /:22$/ {found=1} END {exit !found}'
timeout 15 firewall-cmd --permanent --zone="$zone" --query-service=ssh | grep -Fxq yes
for port in 26379 26387; do
    ! timeout 15 firewall-cmd --zone="$zone" --query-port="${port}/tcp" >/dev/null
    ! timeout 15 firewall-cmd --permanent --zone="$zone" --query-port="${port}/tcp" >/dev/null
done

exact_port_types() {
    local port="$1" listing
    listing="$(semanage port -l)"
    PORT_LISTING="$listing" python3 - "$port" <<'PY_PORT'
import os, re, sys
target = int(sys.argv[1])
types = set()
for line in os.environ['PORT_LISTING'].splitlines():
    match = re.match(r'^(\S+)\s+(tcp|udp)\s+(.+)$', line.strip())
    if not match or match.group(2) != 'tcp':
        continue
    for item in match.group(3).split(','):
        item = item.strip()
        if not item:
            continue
        # A broad policy range such as unreserved_port_t 1024-32767 is not an
        # exact assignment and must not override a specific redis_port_t entry.
        if '-' not in item and int(item) == target:
            types.add(match.group(1))
print(','.join(sorted(types)))
PY_PORT
}

[[ "$(exact_port_types 26379)" == redis_port_t ]] || { printf '%s\n' 'ERROR: 26379 has no exclusive exact redis_port_t assignment' >&2; exit 1; }
port26387_before="$(exact_port_types 26387)"
[[ -z "$port26387_before" || "$port26387_before" == redis_port_t ]] || {
    printf 'ERROR: 26387 belongs to unexpected type: %s\n' "$port26387_before" >&2
    exit 1
}

fcontext_plan=''
plan_fcontext() {
    local path="$1" pattern="$2" type="$3" marker="$4" current
    current="$(matchpathcon -n "$path")"
    if grep -Fq ":${type}:" <<<"$current"; then
        printf 'FCONTEXT_PLAN=%s ALREADY_EFFECTIVE=%s\n' "$path" "$type"
        return 0
    fi
    if semanage fcontext -l -C | awk -v p="$pattern" '$1 == p {found=1} END {exit !found}'; then
        printf 'ERROR: conflicting custom fcontext exists: %s\n' "$pattern" >&2
        exit 1
    fi
    fcontext_plan+="${marker}|${pattern}|${type}"$'\n'
}
plan_fcontext /var/lib/redis-sentinel5 '/var/lib/redis-sentinel5(/.*)?' redis_var_lib_t fcontext-lib5-added
plan_fcontext /var/lib/redis-sentinel7 '/var/lib/redis-sentinel7(/.*)?' redis_var_lib_t fcontext-lib7-added
plan_fcontext /var/log/redis-sentinel5 '/var/log/redis-sentinel5(/.*)?' redis_log_t fcontext-log5-added
plan_fcontext /var/log/redis-sentinel7 '/var/log/redis-sentinel7(/.*)?' redis_log_t fcontext-log7-added

for port in 26379 26387; do
    allowed112="rule family=\"ipv4\" source address=\"10.2.203.112/32\" port port=\"${port}\" protocol=\"tcp\" accept"
    allowed113="rule family=\"ipv4\" source address=\"10.2.203.113/32\" port port=\"${port}\" protocol=\"tcp\" accept"
    while IFS= read -r existing_rule; do
        if [[ "$existing_rule" == *"port=\"${port}\""* && "$existing_rule" == *' accept'* && "$existing_rule" != "$allowed112" && "$existing_rule" != "$allowed113" ]]; then
            printf 'ERROR: unexpected existing accept rule for %s: %s\n' "$port" "$existing_rule" >&2
            exit 1
        fi
    done < <(timeout 15 firewall-cmd --permanent --zone="$zone" --list-rich-rules)
done

printf '%s\n' 'Q102_V5_PREWRITE_GATE=PASS'

backup="/var/backups/zhct-sentinel/nginx01-v5-$(date -u +%Y%m%dT%H%M%SZ)"
install -d -o root -g root -m 0700 "$backup"
cp -a "$CONF5" "$backup/sentinel5.conf"
cp -a "$CONF7" "$backup/sentinel7.conf"
for spec in "sentinel5|$LOG5" "sentinel7|$LOG7"; do
    IFS='|' read -r name logfile <<<"$spec"
    if [[ -f "$logfile" ]]; then
        cp -a "$logfile" "$backup/${name}.log"
        : >"$backup/${name}-log-existed"
    fi
done
semanage fcontext -l -C >"$backup/selinux-custom-fcontext.before.txt"
semanage port -l >"$backup/selinux-ports.before.txt"
systemctl cat sentinel5.service >"$backup/sentinel5-unit.before.txt"
systemctl cat sentinel7.service >"$backup/sentinel7-unit.before.txt"
: >"$backup/added-rich-rules.txt"
: >"$backup/directory-metadata.tsv"
: >"$backup/path-contexts.tsv"
chmod 0600 "$backup"/*.txt "$backup"/*.tsv

for directory in /var/lib/redis-sentinel5 /var/lib/redis-sentinel7 /var/log/redis-sentinel5 /var/log/redis-sentinel7; do
    printf '%s|%s|%s|%s\n' "$directory" "$(stat -c '%u' "$directory")" "$(stat -c '%g' "$directory")" "$(stat -c '%a' "$directory")" >>"$backup/directory-metadata.tsv"
done
for path in /var/lib/redis-sentinel5 /var/lib/redis-sentinel7 /var/log/redis-sentinel5 /var/log/redis-sentinel7 "$CONF5" "$CONF7"; do
    printf '%s|%s\n' "$path" "$(stat -c '%C' "$path")" >>"$backup/path-contexts.tsv"
done

for spec in "sentinel5|$DROPIN5" "sentinel7|$DROPIN7"; do
    IFS='|' read -r name dropin <<<"$spec"
    if [[ -f "$dropin" ]]; then
        cp -a "$dropin" "$backup/${name}-dropin.conf"
        : >"$backup/${name}-dropin-existed"
    elif [[ ! -d "$(dirname "$dropin")" ]]; then
        : >"$backup/${name}-dropin-dir-created"
    fi
done

mutation_started=1

while IFS='|' read -r marker pattern type; do
    [[ -n "$marker" ]] || continue
    semanage fcontext -a -t "$type" "$pattern"
    : >"$backup/$marker"
done <<<"$fcontext_plan"

if [[ -z "$port26387_before" ]]; then
    semanage port -a -t redis_port_t -p tcp 26387
    : >"$backup/port-26387-added"
fi

for directory in /var/lib/redis-sentinel5 /var/lib/redis-sentinel7 /var/log/redis-sentinel5 /var/log/redis-sentinel7; do
    chown redis:redis "$directory"
    chmod 0750 "$directory"
done
restorecon -F /var/lib/redis-sentinel5 /var/lib/redis-sentinel7 /var/log/redis-sentinel5 /var/log/redis-sentinel7 "$CONF5" "$CONF7"

materialize_secret() {
    local config="$1" secret="$2" placeholder="$3"
    secret_tmp="$(mktemp /var/tmp/q102-v5-secret.XXXXXX)"
    chmod 0600 "$secret_tmp"
    python3 - "$config" "$secret" "$placeholder" "$secret_tmp" <<'PY_SECRET'
import pathlib, sys
config, secret_file, placeholder, target = sys.argv[1:]
value = pathlib.Path(secret_file).read_text().rstrip('\r\n')
assert value and '\n' not in value and '\r' not in value
text = pathlib.Path(config).read_text()
text = text.replace(placeholder, value) if placeholder in text else text
assert value in text
pathlib.Path(target).write_text(text)
PY_SECRET
    install -o redis -g redis -m 0600 "$secret_tmp" "$config"
    rm -f -- "$secret_tmp"
    secret_tmp=''
}
materialize_secret "$CONF5" /etc/zhct/secrets/redis5.password __REDIS5_PASSWORD__
materialize_secret "$CONF7" /etc/zhct/secrets/redis7.password __REDIS7_PASSWORD__
restorecon -F "$CONF5" "$CONF7"

ensure_execstart() {
    local unit="$1" server="$2" config="$3" dropin="$4" effective
    effective="$(systemctl show "$unit" -p ExecStart --value)"
    if [[ "$effective" != *"path=${server}"* || "$effective" != *"argv[]=${server} ${config}"* ]]; then
        install -d -o root -g root -m 0755 "$(dirname "$dropin")"
        cat >"$dropin" <<EOF_DROPIN
[Service]
ExecStart=
ExecStart=${server} ${config}
EOF_DROPIN
        chmod 0644 "$dropin"
    fi
}
ensure_execstart sentinel5.service "$SERVER5" "$CONF5" "$DROPIN5"
ensure_execstart sentinel7.service "$SERVER7" "$CONF7" "$DROPIN7"
systemctl daemon-reload

for source in 10.2.203.112 10.2.203.113; do
    for port in 26379 26387; do
        rule="rule family=\"ipv4\" source address=\"${source}/32\" port port=\"${port}\" protocol=\"tcp\" accept"
        if ! timeout 15 firewall-cmd --permanent --zone="$zone" --query-rich-rule="$rule" >/dev/null; then
            timeout 15 firewall-cmd --permanent --zone="$zone" --add-rich-rule="$rule"
            printf '%s\n' "$rule" >>"$backup/added-rich-rules.txt"
        fi
    done
done
timeout 15 firewall-cmd --reload
[[ "$(systemctl is-active sshd 2>/dev/null || true)" == active ]]
ss -lntH | awk '$4 ~ /:22$/ {found=1} END {exit !found}'
timeout 15 firewall-cmd --zone="$zone" --query-service=ssh | grep -Fxq yes

systemctl enable --now sentinel5.service sentinel7.service

field() {
    local data="$1" key="$2"
    awk -v key="$key" 'previous == key {print; exit} {previous=$0}' <<<"$data"
}

replica_present() {
    local data="$1" expected_ip="$2" expected_port="$3"
    REPLICA_DATA="$data" python3 - "$expected_ip" "$expected_port" <<'PY_REPLICA'
import os, sys
expected_ip, expected_port = sys.argv[1:]
lines = os.environ['REPLICA_DATA'].splitlines()
assert len(lines) % 2 == 0
records, record = [], {}
for index in range(0, len(lines), 2):
    key, value = lines[index], lines[index + 1]
    if key == 'name' and record:
        records.append(record)
        record = {}
    record[key] = value
if record:
    records.append(record)
raise SystemExit(0 if any(item.get('ip') == expected_ip and item.get('port') == expected_port for item in records) else 1)
PY_REPLICA
}

verify_one() {
    local cli="$1" host="$2" sentinel_port="$3" master_name="$4" redis_port="$5"
    local master replicas quorum_result
    [[ "$(timeout 3 "$cli" --raw -h "$host" -p "$sentinel_port" PING 2>/dev/null)" == PONG ]] || return 1
    master="$(timeout 3 "$cli" --raw -h "$host" -p "$sentinel_port" SENTINEL master "$master_name" 2>/dev/null)" || return 1
    [[ "$(field "$master" ip)" == 10.2.203.112 ]] || return 1
    [[ "$(field "$master" port)" == "$redis_port" ]] || return 1
    [[ "$(field "$master" quorum)" == 2 ]] || return 1
    [[ "$(field "$master" num-slaves)" -ge 1 ]] || return 1
    [[ "$(field "$master" num-other-sentinels)" -ge 2 ]] || return 1
    replicas="$(timeout 3 "$cli" --raw -h "$host" -p "$sentinel_port" SENTINEL replicas "$master_name" 2>/dev/null)" || return 1
    replica_present "$replicas" 10.2.203.113 "$redis_port" || return 1
    quorum_result="$(timeout 3 "$cli" --raw -h "$host" -p "$sentinel_port" SENTINEL ckquorum "$master_name" 2>/dev/null)" || return 1
    grep -q '^OK' <<<"$quorum_result" || return 1
}

verify_all() {
    local host
    [[ "$(systemctl is-active sentinel5.service)" == active ]] || return 1
    [[ "$(systemctl is-active sentinel7.service)" == active ]] || return 1
    ss -lntH | awk '$4 == "10.2.203.102:26379" {found=1} END {exit !found}' || return 1
    ss -lntH | awk '$4 == "10.2.203.102:26387" {found=1} END {exit !found}' || return 1
    for host in 10.2.203.102 10.2.203.112 10.2.203.113; do
        verify_one "$CLI5" "$host" 26379 sanquan-redis5 6379 || return 1
        verify_one "$CLI7" "$host" 26387 sanquan-redis7 6387 || return 1
    done
}

deadline=$((SECONDS + 60))
verified=0
while (( SECONDS < deadline )); do
    if verify_all; then
        verified=1
        break
    fi
    sleep 2
done
[[ "$verified" -eq 1 ]] || { printf '%s\n' 'ERROR: 60-second Sentinel verification timeout' >&2; exit 1; }

for unit in redis5.service redis7.service keepalived.service; do
    [[ "$(systemctl is-active "$unit" 2>/dev/null || true)" != active ]] || exit 1
done
[[ "$(systemctl is-enabled orchestrator.service 2>/dev/null || true)" == "$orchestrator_enabled" ]]
[[ "$(systemctl is-active orchestrator.service 2>/dev/null || true)" == "$orchestrator_active" ]]
[[ "$(systemctl show orchestrator.service -p MainPID --value)" == "$orchestrator_pid" ]]

for host in 10.2.203.102 10.2.203.112 10.2.203.113; do
    printf 'SENTINEL5_V5_PASS HOST=%s PORT=26379 MASTER=10.2.203.112:6379 REPLICA=10.2.203.113 QUORUM=2 PEERS_GE_2 CKQUORUM=OK CLI=%s\n' "$host" "$CLI5"
    printf 'SENTINEL7_V5_PASS HOST=%s PORT=26387 MASTER=10.2.203.112:6387 REPLICA=10.2.203.113 QUORUM=2 PEERS_GE_2 CKQUORUM=OK CLI=%s\n' "$host" "$CLI7"
done

success=1
printf 'Q102_NGINX01_THREE_SENTINELS_V5_PASS BACKUP=%s\n' "$backup"