#!/usr/bin/env bash

set -Eeuo pipefail

run_id='q101-20260727T120000-role-framework-resume'
staging_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
bundle_root='/opt/zhct-deploy'
failed_backup='/var/backups/zhct-q101/files/q101-20260727T112000-bundle-refresh'
backup_root="/var/backups/zhct-q101/files/$run_id"
candidate_root="$bundle_root/.q101-candidate-$run_id"
framework_tar="$staging_dir/nginx-role-framework-8b750344.tar.gz"
framework_files="$staging_dir/nginx-role-framework-8b750344.files.sha256"
source_commit_file="$staging_dir/SOURCE_COMMIT"
framework_sha_file="$staging_dir/FRAMEWORK_TAR_SHA256"

mutation_started=0
live_unit_changed=0
rollback_in_progress=0

print_state() {
    local unit
    for unit in nginx haproxy orchestrator sentinel5 sentinel7 keepalived; do
        printf 'UNIT=%s ENABLED=' "$unit"
        systemctl is-enabled "$unit" 2>/dev/null || true
        printf 'UNIT=%s ACTIVE=' "$unit"
        systemctl is-active "$unit" 2>/dev/null || true
    done
    ip -o -4 addr show || true
}

restore_framework() {
    local saved_root="$backup_root/root-files"
    local old_root="$backup_root/swap-old"
    local failed_new="$backup_root/failed-new"
    local metadata_file target

    install -d -m 0700 -o root -g root "$failed_new" "$failed_new/roles"

    if [[ -d "$old_root/common" ]]; then
        [[ -d "$bundle_root/common" ]] &&
            mv "$bundle_root/common" "$failed_new/common"
        mv "$old_root/common" "$bundle_root/common"
    fi
    if [[ -d "$old_root/roles/nginx" ]]; then
        [[ -d "$bundle_root/roles/nginx" ]] &&
            mv "$bundle_root/roles/nginx" "$failed_new/roles/nginx"
        mv "$old_root/roles/nginx" "$bundle_root/roles/nginx"
    fi

    for file in README.md bundle-sha256.sh .source-commit .framework-bundle-sha256; do
        [[ -e "$saved_root/$file" ]] && cp -a "$saved_root/$file" "$bundle_root/$file"
    done

    if [[ ! -e "$bundle_root/manifests" && -d "$backup_root/non-role-root/manifests" ]]; then
        mv "$backup_root/non-role-root/manifests" "$bundle_root/manifests"
    fi
    if [[ ! -e "$bundle_root/validate-assets.sh" &&
          -f "$backup_root/non-role-root/validate-assets.sh" ]]; then
        mv "$backup_root/non-role-root/validate-assets.sh" "$bundle_root/validate-assets.sh"
    fi

    if [[ -d "$backup_root/appledouble" ]]; then
        while IFS= read -r -d '' metadata_file; do
            target="$bundle_root/${metadata_file#"$backup_root/appledouble/"}"
            install -d -m 0750 -o root -g root "$(dirname "$target")"
            mv "$metadata_file" "$target"
        done < <(find "$backup_root/appledouble" -type f -print0)
    fi
}

on_error() {
    local rc=$?
    local rollback_ok=1
    trap - ERR
    set +e

    echo "Q101_ROLE_FRAMEWORK_RESUME_FAILED_RC=$rc"
    if [[ "$rollback_in_progress" == '0' ]]; then
        rollback_in_progress=1
        if [[ "$live_unit_changed" == '1' ]]; then
            install -m 0644 -o root -g root \
                "$failed_backup/etc/systemd/system/nginx.service" \
                /etc/systemd/system/nginx.service || rollback_ok=0
            systemctl daemon-reload || rollback_ok=0
            /usr/local/nginx/sbin/nginx -t || rollback_ok=0
            systemctl restart nginx.service || rollback_ok=0
        fi
        if [[ "$mutation_started" == '1' ]]; then
            restore_framework || rollback_ok=0
        fi
    fi

    if [[ "$rollback_ok" == '1' ]]; then
        echo 'Q101_ROLE_FRAMEWORK_RESUME_ROLLBACK=PASS'
    else
        echo 'Q101_ROLE_FRAMEWORK_RESUME_ROLLBACK=FAILED'
    fi
    print_state
    systemctl status nginx.service --no-pager -l || true
    echo 'Q101_ROLE_FRAMEWORK_RESUME_FAILED_STOP_NO_RETRY'
    exit "$rc"
}
trap on_error ERR

echo 'Q101_ROLE_FRAMEWORK_RESUME_BEGIN'
date -Is

[[ "$EUID" -eq 0 ]]
[[ "$(hostname -s)" == 'sqjcnginx01' ]]
ip -o -4 addr show scope global | grep -F '10.2.203.102/'

for unit in nginx haproxy orchestrator; do
    systemctl is-enabled --quiet "$unit"
    systemctl is-active --quiet "$unit"
done
for unit in sentinel5 sentinel7 keepalived; do
    [[ "$(systemctl is-enabled "$unit" 2>/dev/null || true)" == 'disabled' ]]
    [[ "$(systemctl is-active "$unit" 2>/dev/null || true)" == 'inactive' ]]
done
if ip -o -4 addr show | grep -Eq '10\.2\.203\.(115|116|117)/'; then
    echo 'Q101_UNEXPECTED_VIP_PRESENT'
    exit 1
fi

printf '%s  %s\n' \
    'b4905680419a0ad3d9a8ea9298dec5f4f2f5e5ef2f3ffe42d2119a1aafe9df62' "$bundle_root/roles/nginx/templates/nginx/nginx.conf" \
    '92c3b974b74c9d128c52a989624c97a1bb6ba77057dd02cf6de6421d3087f105' "$bundle_root/roles/nginx/hooks.sh" \
    'd214722459a5029b601522957dbb6b3b710e87bd2c090768dec965d680bf55e0' "$bundle_root/roles/nginx/templates/systemd/nginx.service" \
    '811409c6721b1c8f5c5b8fc1c8db67a8386cb712a93e617178015b97e3eb5177' "$bundle_root/validate-assets.sh" \
    '128453c793f540126181db45ac3b12e7d0c755156b6b4b1c84cc188f8eaea942' /etc/systemd/system/nginx.service \
    'b4905680419a0ad3d9a8ea9298dec5f4f2f5e5ef2f3ffe42d2119a1aafe9df62' /usr/local/nginx/conf/nginx.conf |
    sha256sum -c -

grep -Fxq '92c3b974b74c9d128c52a989624c97a1bb6ba77057dd02cf6de6421d3087f105  roles/nginx/hooks.sh' \
    "$bundle_root/manifests/bundle.sha256"
grep -Fxq 'b4905680419a0ad3d9a8ea9298dec5f4f2f5e5ef2f3ffe42d2119a1aafe9df62  roles/nginx/templates/nginx/nginx.conf' \
    "$bundle_root/manifests/bundle.sha256"
grep -Fxq 'd214722459a5029b601522957dbb6b3b710e87bd2c090768dec965d680bf55e0  roles/nginx/templates/systemd/nginx.service' \
    "$bundle_root/manifests/bundle.sha256"
grep -Fxq '811409c6721b1c8f5c5b8fc1c8db67a8386cb712a93e617178015b97e3eb5177  validate-assets.sh' \
    "$bundle_root/manifests/bundle.sha256"

printf '%s  %s\n' \
    'be54d0b36f9ae5319f942f36c0f4e9681571c3210e8bd4e3382fe270ec101659' "$failed_backup/opt/zhct-deploy/roles/nginx/templates/nginx/nginx.conf" \
    '52c3261326cd09781b7cef59fe89e1ffa67a6bb40a316ddfc09bd43c5ce246bf' "$failed_backup/opt/zhct-deploy/roles/nginx/hooks.sh" \
    '128453c793f540126181db45ac3b12e7d0c755156b6b4b1c84cc188f8eaea942' "$failed_backup/opt/zhct-deploy/roles/nginx/templates/systemd/nginx.service" \
    '46f2a9bfea97067873ffbdd774d415da762cae8fc59f6ee11a235a86829c05c0' "$failed_backup/opt/zhct-deploy/manifests/bundle.sha256" \
    '128453c793f540126181db45ac3b12e7d0c755156b6b4b1c84cc188f8eaea942' "$failed_backup/etc/systemd/system/nginx.service" |
    sha256sum -c -

[[ "$(tr -d '\r\n' <"$bundle_root/.source-commit")" == '47584918681b8aad36fee3c9eade5cd919c702ab' ]]
[[ "$(tr -d '\r\n' <"$bundle_root/.framework-bundle-sha256")" == 'f74941f6328c752eeefb5fe082c4366dfed5c91ecad36d31ba7e11b7d2cac2d5' ]]

[[ "$(tr -d '\r\n' <"$source_commit_file")" == '8b750344db5d39e8920ba3eedc4309296a49228b' ]]
[[ "$(tr -d '\r\n' <"$framework_sha_file")" == 'cf70ae457bde311e86dc16a6454eab701ff1c78c1eee144913496d6258f1e935' ]]
printf '%s  %s\n' \
    'cf70ae457bde311e86dc16a6454eab701ff1c78c1eee144913496d6258f1e935' "$framework_tar" \
    'd7fd1396522cecd209cfed087a13efc5d65b8bb0db5d1e3517638e65c9ef6000' "$framework_files" |
    sha256sum -c -

[[ ! -e "$candidate_root" ]]
[[ ! -e "$backup_root" ]]
install -d -m 0700 -o root -g root "$candidate_root"
tar -xzf "$framework_tar" -C "$candidate_root"

[[ "$(find "$candidate_root/common" "$candidate_root/roles/nginx" -type f | wc -l)" -eq 42 ]]
[[ -f "$candidate_root/README.md" ]]
[[ -f "$candidate_root/bundle-sha256.sh" ]]
[[ -z "$(find "$candidate_root" -type f \( -name '._*' -o -name '.DS_Store' \) -print -quit)" ]]
(cd "$candidate_root" && sha256sum -c "$framework_files")

while IFS= read -r -d '' script; do
    bash -n "$script"
done < <(find "$candidate_root/common" "$candidate_root/roles/nginx" \
    -type f -name '*.sh' -print0 | sort -z)

grep -Fq 'include conf.d/front.conf;' \
    "$candidate_root/roles/nginx/templates/nginx/nginx.conf"
! grep -Fq 'include conf.d/*.conf;' \
    "$candidate_root/roles/nginx/templates/nginx/nginx.conf"
grep -Fq 'include conf.d/file-primary-routes.conf;' \
    "$candidate_root/roles/nginx/templates/nginx/front.conf"
grep -Fq 'LimitNOFILE=65535' \
    "$candidate_root/roles/nginx/templates/systemd/nginx.service"
grep -Fq 'ensure_dir /var/lib/orchestrator 0750 root root' \
    "$candidate_root/roles/nginx/hooks.sh"
grep -Fq 'ensure_dir /var/log/orchestrator 0750 root root' \
    "$candidate_root/roles/nginx/hooks.sh"

(cd "$bundle_root" &&
    sha256sum -c "$candidate_root/roles/nginx/manifests/packages.sha256")
cmp -s "$candidate_root/roles/nginx/templates/nginx/nginx.conf" \
    /usr/local/nginx/conf/nginx.conf
/usr/local/nginx/sbin/nginx -t

[[ -d "$bundle_root/common" ]]
[[ -d "$bundle_root/roles/nginx" ]]
[[ -d "$bundle_root/manifests" ]]
[[ "$(find "$bundle_root/manifests" -mindepth 1 -maxdepth 1 | wc -l)" -eq 2 ]]
[[ -f "$bundle_root/manifests/bundle.sha256" ]]
[[ -f "$bundle_root/manifests/._bundle.sha256" ]]

install -d -m 0700 -o root -g root \
    "$backup_root/root-files" \
    "$backup_root/swap-old/roles" \
    "$backup_root/non-role-root" \
    "$backup_root/appledouble"
for file in README.md bundle-sha256.sh .source-commit .framework-bundle-sha256; do
    cp -a "$bundle_root/$file" "$backup_root/root-files/$file"
done

mutation_started=1
mv "$bundle_root/common" "$backup_root/swap-old/common"
mv "$bundle_root/roles/nginx" "$backup_root/swap-old/roles/nginx"
mv "$candidate_root/common" "$bundle_root/common"
mv "$candidate_root/roles/nginx" "$bundle_root/roles/nginx"

install -m 0644 -o root -g root "$candidate_root/README.md" "$bundle_root/README.md"
install -m 0750 -o root -g root "$candidate_root/bundle-sha256.sh" "$bundle_root/bundle-sha256.sh"
install -m 0640 -o root -g root "$source_commit_file" "$bundle_root/.source-commit"
install -m 0640 -o root -g root "$framework_sha_file" "$bundle_root/.framework-bundle-sha256"

mv "$bundle_root/manifests" "$backup_root/non-role-root/manifests"
mv "$bundle_root/validate-assets.sh" "$backup_root/non-role-root/validate-assets.sh"

while IFS= read -r -d '' metadata_file; do
    metadata_target="$backup_root/appledouble/${metadata_file#"$bundle_root/"}"
    install -d -m 0700 -o root -g root "$(dirname "$metadata_target")"
    mv "$metadata_file" "$metadata_target"
    printf 'QUARANTINED_METADATA=%s\n' "${metadata_file#"$bundle_root/"}"
done < <(find "$bundle_root" -type f \( -name '._*' -o -name '.DS_Store' \) -print0)

[[ -z "$(find "$bundle_root" -type f \( -name '._*' -o -name '.DS_Store' \) -print -quit)" ]]
[[ ! -e "$bundle_root/manifests" ]]
[[ ! -e "$bundle_root/validate-assets.sh" ]]
(cd "$bundle_root" && sha256sum -c "$framework_files")
(cd "$bundle_root" &&
    sha256sum -c "$bundle_root/roles/nginx/manifests/packages.sha256")

live_unit_changed=1
install -m 0644 -o root -g root \
    "$bundle_root/roles/nginx/templates/systemd/nginx.service" \
    /etc/systemd/system/nginx.service
systemctl daemon-reload
/usr/local/nginx/sbin/nginx -t
systemctl restart nginx.service
systemctl is-enabled --quiet nginx.service
systemctl is-active --quiet nginx.service

nginx_pid=$(systemctl show nginx.service -p MainPID --value)
[[ "$nginx_pid" =~ ^[1-9][0-9]*$ ]]
nginx_soft_limit=$(awk '/Max open files/ {print $4}' "/proc/$nginx_pid/limits")
[[ "$nginx_soft_limit" -eq 65535 ]]
printf 'NGINX_MAIN_PID=%s\n' "$nginx_pid"
printf 'NGINX_OPEN_FILES_SOFT=%s\n' "$nginx_soft_limit"

curl -fsS --max-time 5 http://127.0.0.1:8081/__edge_health
echo

for unit in haproxy orchestrator; do
    systemctl is-enabled --quiet "$unit"
    systemctl is-active --quiet "$unit"
done
for unit in sentinel5 sentinel7 keepalived; do
    [[ "$(systemctl is-enabled "$unit" 2>/dev/null || true)" == 'disabled' ]]
    [[ "$(systemctl is-active "$unit" 2>/dev/null || true)" == 'inactive' ]]
done
if ip -o -4 addr show | grep -Eq '10\.2\.203\.(115|116|117)/'; then
    echo 'Q101_UNEXPECTED_VIP_PRESENT'
    exit 1
fi

[[ "$(tr -d '\r\n' <"$bundle_root/.source-commit")" == '8b750344db5d39e8920ba3eedc4309296a49228b' ]]
[[ "$(tr -d '\r\n' <"$bundle_root/.framework-bundle-sha256")" == 'cf70ae457bde311e86dc16a6454eab701ff1c78c1eee144913496d6258f1e935' ]]

if find "$candidate_root" -mindepth 1 -print -quit | grep -q .; then
    mv "$candidate_root" "$backup_root/candidate-leftovers"
else
    rmdir "$candidate_root"
fi
chmod -R go-rwx "$backup_root"

trap - ERR
echo 'Q101_ROLE_FRAMEWORK_MODEL=PASS'
echo 'Q101_GLOBAL_MANIFEST_QUARANTINED=PASS'
echo 'Q101_APPLEDOUBLE_QUARANTINED=PASS'
echo 'Q101_VIP_ABSENT=PASS'
print_state
echo 'Q101_ROLE_FRAMEWORK_RESUME_PASS'