#!/usr/bin/env bash

set -Eeuo pipefail
umask 077
export LANG=C

EXPECTED_HOSTNAME="sqjcnginx01"
EXPECTED_IPV4="10.2.203.102"
EXPECTED_OS="openEuler release 24.03 (LTS-SP1)"
EXPECTED_RUN_ID="q042-20260723T134134+0800-R1"
EXPECTED_PREFLIGHT_RESULT_SHA256="7728a3d5457740be422698f0f48048eca227240866f166be7f3b9a41ef45d32d"
EXPECTED_B4_SCRIPT_SHA256="8fd82b67efa3d739f41bef9b8c3407d7741fc46f78dd73b9e7e84abcae07f5d6"
EXPECTED_B4_ARTIFACT_SHA256="7c6de7b7570f504f045b9addca8529f9151d145a791972911b57d55dedbd0553"
BUNDLE_ROOT="/opt/zhct-deploy"
BUILD_PARENT="/var/tmp/zhct-q042"
EVIDENCE_ROOT="/var/backups/zhct-q042/evidence"
EVIDENCE_DIR="${EVIDENCE_ROOT}/${EXPECTED_RUN_ID}"
ATTEMPT_ROOT="${EVIDENCE_DIR}/offline-rebuild-attempts"
B4_DIR="${ATTEMPT_ROOT}/B4"
CANONICAL_RESULT="${EVIDENCE_DIR}/offline-rebuild-result.env"
FINAL_RESULT="${EVIDENCE_DIR}/final-audit-result.env"
FINAL_INDEX="${EVIDENCE_DIR}/q042-final-evidence-index.sha256"
FINAL_ARCHIVE="/root/q042-evidence-${EXPECTED_RUN_ID}.tar.gz"
VIP_RE='10\.2\.203\.(115|116|117)'
MIN_FREE_BYTES=$((9 * 1024 * 1024 * 1024))
TARGET_UNITS=(
  nginx.service
  keepalived.service
  haproxy.service
  orchestrator.service
  sentinel5.service
  sentinel7.service
)
FORMAL_BINARIES=(
  /usr/local/nginx/sbin/nginx
  /usr/local/sbin/keepalived
  /usr/local/haproxy/sbin/haproxy
  /usr/local/orchestrator/orchestrator
  /opt/redis/5.0.14/bin/redis-sentinel
  /opt/redis/5.0.14/bin/redis-cli
  /opt/redis/7.2.14/bin/redis-sentinel
  /opt/redis/7.2.14/bin/redis-cli
  /usr/bin/orchestrator
  /usr/bin/orchestrator-client
)
FORMAL_UNIT_FILES=(
  /etc/systemd/system/nginx.service
  /etc/systemd/system/keepalived.service
  /etc/systemd/system/haproxy.service
  /usr/lib/systemd/system/orchestrator.service
  /etc/systemd/system/sentinel5.service
  /etc/systemd/system/sentinel7.service
)
FORMAL_RUNTIME_PATHS=(
  /etc/keepalived/keepalived.conf
  /etc/haproxy/haproxy.cfg
  /etc/orchestrator.conf.json
  /var/lib/redis-sentinel5/sentinel.conf
  /var/lib/redis-sentinel7/sentinel.conf
  /var/lib/orchestrator
  /var/log/orchestrator
  /etc/zhct/node.env
  /etc/zhct/secrets
)

die() {
  printf 'ERROR: %s\n' "$*" >&2
  exit 1
}

assert_result_line() {
  local file=$1
  local line=$2
  grep -Fxq "${line}" "${file}" || die "required result line is missing: ${line}"
}

assert_units_quiescent() {
  local output=$1
  : >"${output}"
  local unit enabled active substate started
  for unit in "${TARGET_UNITS[@]}"; do
    enabled=$(systemctl is-enabled "${unit}" 2>/dev/null || true)
    active=$(systemctl show "${unit}" -p ActiveState --value)
    substate=$(systemctl show "${unit}" -p SubState --value)
    started=$(systemctl show "${unit}" -p ExecMainStartTimestampMonotonic --value)
    printf '%s\t%s\t%s\t%s\t%s\n' \
      "${unit}" "${enabled}" "${active}" "${substate}" "${started}" >>"${output}"
    [[ "${enabled}" == "disabled" ]] || die "unit is not disabled: ${unit}"
    [[ "${active}" == "inactive" ]] || die "unit is not inactive: ${unit}"
    [[ "${substate}" == "dead" ]] || die "unit is not dead: ${unit}"
    [[ "${started}" == "0" ]] || die "unit has a start timestamp: ${unit}"
  done
}

[[ "${EUID}" -eq 0 ]] || die "must run as root"
[[ $# -eq 1 && "$1" == "${EXPECTED_RUN_ID}" ]] || die "Q042 run id mismatch"
[[ "$(hostname -s)" == "${EXPECTED_HOSTNAME}" ]] || die "hostname mismatch"
[[ "$(tr -d '\r' </etc/openEuler-release)" == "${EXPECTED_OS}" ]] || die "OS mismatch"
[[ "$(uname -m)" == "x86_64" ]] || die "CPU architecture mismatch"
ip -o -4 addr show scope global | awk '{print $4}' | cut -d/ -f1 | \
  grep -Fxq "${EXPECTED_IPV4}" || die "expected IPv4 is not configured"

for command in awk bash cat chmod cmp cp cut date df dnf find grep hostname ip \
  ln mkdir mktemp mv nproc pgrep python3 readlink realpath rm sed sha256sum sort \
  ss stat systemctl tail tar tee tr uname wc xargs; do
  command -v "${command}" >/dev/null 2>&1 || die "required command is missing: ${command}"
done

[[ -d "${EVIDENCE_DIR}" && ! -L "${EVIDENCE_DIR}" ]] || \
  die "Q042 evidence directory is missing or unsafe"
[[ ! -e "${FINAL_RESULT}" && ! -L "${FINAL_RESULT}" ]] || \
  die "Q042 final audit result already exists"
[[ ! -e "${FINAL_INDEX}" && ! -L "${FINAL_INDEX}" ]] || \
  die "Q042 final evidence index already exists"
[[ ! -e "${FINAL_ARCHIVE}" && ! -L "${FINAL_ARCHIVE}" ]] || \
  die "Q042 final archive already exists"
[[ ! -e "${FINAL_ARCHIVE}.sha256" && ! -L "${FINAL_ARCHIVE}.sha256" ]] || \
  die "Q042 final archive checksum already exists"
evidence_symlink_count=$(find "${EVIDENCE_DIR}" -type l -print | wc -l)
[[ "${evidence_symlink_count}" -eq 0 ]] || die "Q042 evidence contains a symbolic link"

exec > >(tee "${EVIDENCE_DIR}/final-audit-check.log") 2>&1
SELF_SHA256=$(sha256sum "$0" | awk '{print $1}')
printf 'final_audit_script_sha256=%s\n' "${SELF_SHA256}"
cp -- "$0" "${EVIDENCE_DIR}/q042-final-audit-check.sh"
chmod 0600 "${EVIDENCE_DIR}/q042-final-audit-check.sh"

[[ "$(sha256sum "${EVIDENCE_DIR}/preflight-result.env" | awk '{print $1}')" == \
  "${EXPECTED_PREFLIGHT_RESULT_SHA256}" ]] || die "R1 preflight result SHA256 mismatch"
assert_result_line "${EVIDENCE_DIR}/preflight-result.env" 'preflight_check=PASS'
assert_result_line "${EVIDENCE_DIR}/preflight-result.env" \
  'dnf_cacheonly_reinstall_package_identity=PASS'
assert_result_line "${EVIDENCE_DIR}/preflight-result.env" \
  'isolated_public_network=BLOCKED'
assert_result_line "${EVIDENCE_DIR}/preflight-result.env" 'dnf_history_id_before=2'
assert_result_line "${EVIDENCE_DIR}/preflight-result.env" 'dnf_history_id_after=2'

for attempt in B1 B2 B3 B4; do
  [[ -d "${ATTEMPT_ROOT}/${attempt}" && ! -L "${ATTEMPT_ROOT}/${attempt}" ]] || \
    die "attempt evidence directory is missing: ${attempt}"
done
for attempt in B1 B2 B3; do
  attempt_dir="${ATTEMPT_ROOT}/${attempt}"
  [[ -s "${attempt_dir}/failure.env" ]] || die "failure marker is missing: ${attempt}"
  assert_result_line "${attempt_dir}/failure.env" 'exit_code=1'
  [[ -f "${attempt_dir}/failure-artifacts.tar.gz" &&
    ! -L "${attempt_dir}/failure-artifacts.tar.gz" &&
    -s "${attempt_dir}/failure-artifacts.tar.gz" ]] || \
    die "failure artifact is missing: ${attempt}"
  [[ -f "${attempt_dir}/failure-artifacts.sha256" &&
    ! -L "${attempt_dir}/failure-artifacts.sha256" &&
    -s "${attempt_dir}/failure-artifacts.sha256" ]] || \
    die "failure artifact checksum is missing: ${attempt}"
  sha256sum -c "${attempt_dir}/failure-artifacts.sha256"
  tar -tzf "${attempt_dir}/failure-artifacts.tar.gz" >/dev/null
  [[ ! -e "${attempt_dir}/offline-rebuild-result.env" ]] || \
    die "failed attempt contains a PASS result: ${attempt}"
done

[[ -s "${B4_DIR}/offline-rebuild-result.env" ]] || die "B4 result is missing"
[[ -f "${CANONICAL_RESULT}" && ! -L "${CANONICAL_RESULT}" ]] || \
  die "canonical Q042 result is not a regular non-symlink file"
[[ -f "${B4_DIR}/offline-rebuild-result.env" &&
  ! -L "${B4_DIR}/offline-rebuild-result.env" ]] || \
  die "B4 result is not a regular non-symlink file"
canonical_result_identity=$(stat -c '%d:%i' "${CANONICAL_RESULT}")
b4_result_identity=$(stat -c '%d:%i' "${B4_DIR}/offline-rebuild-result.env")
[[ "${canonical_result_identity}" == "${b4_result_identity}" ]] || \
  die "canonical Q042 result and B4 result do not share an inode"
canonical_result_link_count=$(stat -c '%h' "${CANONICAL_RESULT}")
((canonical_result_link_count >= 2)) || \
  die "canonical Q042 result does not have the expected hard-link count"
[[ "$(sha256sum /root/q042-offline-rebuild-check-b4.sh | awk '{print $1}')" == \
  "${EXPECTED_B4_SCRIPT_SHA256}" ]] || die "B4 remote script SHA256 mismatch"
[[ -f "${B4_DIR}/build-artifacts.tar.gz" &&
  ! -L "${B4_DIR}/build-artifacts.tar.gz" &&
  -s "${B4_DIR}/build-artifacts.tar.gz" ]] || die "B4 build artifact is missing"
[[ "$(sha256sum "${B4_DIR}/build-artifacts.tar.gz" | awk '{print $1}')" == \
  "${EXPECTED_B4_ARTIFACT_SHA256}" ]] || die "B4 build artifact SHA256 mismatch"
tar -tzf "${B4_DIR}/build-artifacts.tar.gz" >/dev/null
[[ ! -e "${B4_DIR}/failure.env" && ! -e "${B4_DIR}/failure-artifacts.tar.gz" ]] || \
  die "B4 contains a failure marker"

required_b4_lines=(
  "run_id=${EXPECTED_RUN_ID}"
  'build_attempt_id=B4'
  "offline_rebuild_script_sha256=${EXPECTED_B4_SCRIPT_SHA256}"
  'source_archive_count=5'
  'source_archive_sha256_check=PASS'
  'archive_path_and_link_safety=PASS'
  'network_namespace_isolation=PASS'
  'pid_namespace_cleanup=PASS'
  'mount_namespace_protected_paths_readonly=PASS'
  'temporary_build_uid=65534'
  'temporary_build_gid=65534'
  'temporary_build_no_new_privs=PASS'
  'temporary_build_all_capability_sets_empty=PASS'
  'nginx_1_26_3_offline_rebuild=PASS'
  'keepalived_2_3_4_offline_rebuild=PASS'
  'haproxy_2_8_26_offline_rebuild=PASS'
  'redis_5_0_14_offline_rebuild=PASS'
  'redis_7_2_14_offline_rebuild=PASS'
  'temporary_binary_dynamic_links=PASS'
  'formal_binary_immutability=PASS'
  'formal_unit_file_immutability=PASS'
  'formal_runtime_paths_immutability=PASS'
  'host_listener_immutability=PASS'
  'host_ipv4_immutability=PASS'
  'host_route_immutability=PASS'
  'dnf_history_id_before=2'
  'dnf_history_id_after=2'
  'target_units_quiescent=PASS'
  'target_listener_count=0'
  'vip_announcement_count=0'
  'temporary_build_root_cleanup=PASS'
  "offline_build_artifacts_sha256=${EXPECTED_B4_ARTIFACT_SHA256}"
  'offline_rebuild_check=PASS'
)
for line in "${required_b4_lines[@]}"; do
  assert_result_line "${CANONICAL_RESULT}" "${line}"
done

for attempt in B1 B2 B3 B4; do
  [[ ! -e "${BUILD_PARENT}/${EXPECTED_RUN_ID}-${attempt}" &&
    ! -L "${BUILD_PARENT}/${EXPECTED_RUN_ID}-${attempt}" ]] || \
    die "temporary build root remains: ${attempt}"
done
remaining_build_processes=$(pgrep -af "${BUILD_PARENT}/${EXPECTED_RUN_ID}" || true)
[[ -z "${remaining_build_processes}" ]] || die "Q042 build process remains"

assert_units_quiescent "${EVIDENCE_DIR}/final-unit-state.tsv"
ss -lntupH >"${EVIDENCE_DIR}/final-listeners.txt"
ip -4 -br addr >"${EVIDENCE_DIR}/final-ipv4.txt"
ip route show table all >"${EVIDENCE_DIR}/final-routes.txt"
sha256sum "${FORMAL_BINARIES[@]}" >"${EVIDENCE_DIR}/final-formal-binaries.sha256"
sha256sum "${FORMAL_UNIT_FILES[@]}" >"${EVIDENCE_DIR}/final-formal-unit-files.sha256"
cmp "${EVIDENCE_DIR}/formal-binaries-after-preflight.sha256" \
  "${EVIDENCE_DIR}/final-formal-binaries.sha256" || die "formal binary changed after R1"
cmp "${EVIDENCE_DIR}/formal-unit-files-after-preflight.sha256" \
  "${EVIDENCE_DIR}/final-formal-unit-files.sha256" || die "formal unit changed after R1"
cmp "${EVIDENCE_DIR}/listeners-after-preflight.txt" \
  "${EVIDENCE_DIR}/final-listeners.txt" || die "host listeners changed after R1"
cmp "${EVIDENCE_DIR}/ipv4-after-preflight.txt" \
  "${EVIDENCE_DIR}/final-ipv4.txt" || die "host IPv4 changed after R1"
cmp "${EVIDENCE_DIR}/routes-after-preflight.txt" \
  "${EVIDENCE_DIR}/final-routes.txt" || die "host routes changed after R1"

target_ports=$(awk -F'\t' 'NR > 1 && $1 ~ /^[0-9]+$/ {
  ports[++count]=$1
} END {
  for (i=1; i<=count; i++) printf "%s%s", (i > 1 ? "|" : ""), ports[i]
}' "${BUNDLE_ROOT}/roles/nginx/manifests/ports.tsv")
[[ -z "$(ss -lntupH | grep -E ":(${target_ports})([[:space:]]|$)" || true)" ]] || \
  die "target port is listening after Q042"
[[ -z "$(ip -o -4 addr show | grep -E "${VIP_RE}" || true)" ]] || \
  die "project VIP is announced after Q042"

: >"${EVIDENCE_DIR}/final-formal-runtime-paths.tsv"
for path in "${FORMAL_RUNTIME_PATHS[@]}"; do
  if [[ -e "${path}" || -L "${path}" ]]; then
    printf '%s\tPRESENT\n' "${path}" >>"${EVIDENCE_DIR}/final-formal-runtime-paths.tsv"
    die "formal runtime path exists after Q042: ${path}"
  fi
  printf '%s\tABSENT\n' "${path}" >>"${EVIDENCE_DIR}/final-formal-runtime-paths.tsv"
done
cmp "${EVIDENCE_DIR}/formal-runtime-paths-after-preflight.tsv" \
  "${EVIDENCE_DIR}/final-formal-runtime-paths.tsv" || \
  die "formal runtime path state changed after R1"

dnf_history_id=$(dnf history list | awk '$1 ~ /^[0-9]+$/ {print $1; exit}')
dnf_history_id=${dnf_history_id:-0}
[[ "${dnf_history_id}" == "2" ]] || die "DNF history changed after Q042"

root_available_bytes=$(df -B1 --output=avail / | tail -n 1 | tr -d ' ')
root_total_bytes=$(df -B1 --output=size / | tail -n 1 | tr -d ' ')
minimum_required_bytes=${MIN_FREE_BYTES}
if ((root_total_bytes / 5 > minimum_required_bytes)); then
  minimum_required_bytes=$((root_total_bytes / 5))
fi
((root_available_bytes >= minimum_required_bytes)) || die "root free space is below Q042 gate"

python3 - "${EVIDENCE_DIR}" "${EVIDENCE_DIR}/q042-sensitive-scan.env" <<'PY'
import pathlib
import re
import sys
import tarfile

root = pathlib.Path(sys.argv[1])
output = pathlib.Path(sys.argv[2])
patterns = {
    "private_key": re.compile(rb"BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY"),
    "credential_url": re.compile(rb"://[^/\s:]+:[^/@\s]+@"),
    "credential_assignment": re.compile(
        rb"""(?ix)
        (?<![A-Za-z0-9_])
        ["']?(?:password|passwd|pwd|secret|token)["']?
        [ \t]*[:=][ \t]*
        (?:
            "[^"\r\n]{4,}"
          | '[^'\r\n]{4,}'
          | [^\s,"'}]{4,}
        )
        """
    ),
}
findings: list[tuple[str, str]] = []
scanned_files = 0
scanned_members = 0

def scan(label: str, data: bytes) -> None:
    for kind, pattern in patterns.items():
        if pattern.search(data):
            findings.append((kind, label))

def scan_stream(label: str, stream) -> None:
    overlap = b""
    while True:
        chunk = stream.read(1024 * 1024)
        if not chunk:
            break
        data = overlap + chunk
        scan(label, data)
        overlap = data[-4096:]

for path in sorted(root.rglob("*")):
    if (
        not path.is_file()
        or path == output
        or path == root / "q042-final-audit-check.sh"
    ):
        continue
    relative = str(path.relative_to(root))
    if path.name.endswith(".tar.gz"):
        with tarfile.open(path, "r:gz") as archive:
            for member in archive:
                if not member.isfile():
                    continue
                extracted = archive.extractfile(member)
                if extracted is None:
                    continue
                scan_stream(f"{relative}:{member.name}", extracted)
                scanned_members += 1
    else:
        with path.open("rb") as stream:
            scan_stream(relative, stream)
        scanned_files += 1

if findings:
    for kind, label in findings:
        print(f"{kind}\t{label}", file=sys.stderr)
    raise SystemExit("sensitive evidence marker found")

output.write_text(
    "\n".join(
        [
            f"scanned_regular_file_count={scanned_files}",
            f"scanned_archive_member_count={scanned_members}",
            "private_key_marker_count=0",
            "credential_url_count=0",
            "credential_assignment_count=0",
            "sensitive_scan=PASS",
            "",
        ]
    ),
    encoding="utf-8",
)
PY

index_temp=$(mktemp "${EVIDENCE_ROOT}/.q042-final-evidence-index.XXXXXX")
result_temp=$(mktemp "${EVIDENCE_ROOT}/.q042-final-audit-result.XXXXXX")
archive_temp=$(mktemp "/root/.q042-evidence-${EXPECTED_RUN_ID}.XXXXXX.tar.gz")
cleanup_final_audit() {
  local rc=$?
  if [[ "${rc}" -ne 0 ]]; then
    rm -f -- \
      "${index_temp}" "${result_temp}" "${archive_temp}" \
      "${FINAL_INDEX}" "${FINAL_INDEX}.sha256" \
      "${FINAL_RESULT}" "${FINAL_ARCHIVE}" "${FINAL_ARCHIVE}.sha256"
  fi
  exit "${rc}"
}
trap cleanup_final_audit EXIT
(
  cd "${EVIDENCE_DIR}"
  find . -type f \
    ! -path './q042-final-evidence-index.sha256' \
    ! -path './q042-final-evidence-index.sha256.sha256' \
    ! -path './final-audit-result.env' \
    ! -path './final-audit-check.log' \
    -print0 | sort -z | xargs -0 sha256sum
) >"${index_temp}"
[[ -s "${index_temp}" ]] || die "Q042 final evidence index is empty"
mv "${index_temp}" "${FINAL_INDEX}"
(
  cd "${EVIDENCE_DIR}"
  sha256sum "$(basename "${FINAL_INDEX}")"
) >"${FINAL_INDEX}.sha256"

evidence_file_count=$(wc -l <"${FINAL_INDEX}")
evidence_index_sha256=$(awk '{print $1}' "${FINAL_INDEX}.sha256")
preflight_result_sha256=$(sha256sum "${EVIDENCE_DIR}/preflight-result.env" | awk '{print $1}')
b4_result_sha256=$(sha256sum "${CANONICAL_RESULT}" | awk '{print $1}')
{
  printf 'captured_at=%s\n' "$(date --iso-8601=seconds)"
  printf 'run_id=%s\n' "${EXPECTED_RUN_ID}"
  printf 'final_audit_script_sha256=%s\n' "${SELF_SHA256}"
  printf 'preflight_result_sha256=%s\n' "${preflight_result_sha256}"
  printf 'b4_script_sha256=%s\n' "${EXPECTED_B4_SCRIPT_SHA256}"
  printf 'b4_result_sha256=%s\n' "${b4_result_sha256}"
  printf 'b4_artifact_sha256=%s\n' "${EXPECTED_B4_ARTIFACT_SHA256}"
  printf 'failed_attempt_count=3\n'
  printf 'successful_attempt=B4\n'
  printf 'evidence_symlink_count=%s\n' "${evidence_symlink_count}"
  printf 'evidence_file_count=%s\n' "${evidence_file_count}"
  printf 'evidence_index_sha256=%s\n' "${evidence_index_sha256}"
  printf 'execution_log_indexing=ARCHIVE_ONLY\n'
  printf 'dnf_history_id=%s\n' "${dnf_history_id}"
  printf 'root_available_bytes=%s\n' "${root_available_bytes}"
  printf 'minimum_required_bytes=%s\n' "${minimum_required_bytes}"
  printf 'v1_identity_and_confirmation=PASS\n'
  printf 'v2_locked_asset_integrity=PASS\n'
  printf 'v3_network_namespace_isolation=PASS\n'
  printf 'v4_dnf_cacheonly_resolution=PASS\n'
  printf 'v5_nginx_offline_rebuild=PASS\n'
  printf 'v6_keepalived_haproxy_offline_rebuild=PASS\n'
  printf 'v7_redis5_redis7_offline_rebuild=PASS\n'
  printf 'v8_formal_state_immutability=PASS\n'
  printf 'v9_cleanup_and_space_gate=PASS\n'
  printf 'v10_evidence_and_sensitive_scan=PASS\n'
  printf 'target_units_quiescent=PASS\n'
  printf 'target_listener_count=0\n'
  printf 'vip_announcement_count=0\n'
  printf 'formal_runtime_paths=ABSENT\n'
  printf 'temporary_build_roots=ABSENT\n'
  printf 'sensitive_scan=PASS\n'
  printf 'q043_execution=LOCKED_PENDING_USER_CONFIRMATION\n'
  printf 'final_audit_check=PASS\n'
} >"${result_temp}"
mv "${result_temp}" "${FINAL_RESULT}"

find "${EVIDENCE_DIR}" -type d -exec chmod 0700 {} +
find "${EVIDENCE_DIR}" -type f -exec chmod 0600 {} +
tar -C "${EVIDENCE_DIR}" -czf "${archive_temp}" .
tar -tzf "${archive_temp}" >/dev/null
mv "${archive_temp}" "${FINAL_ARCHIVE}"
sha256sum "${FINAL_ARCHIVE}" >"${FINAL_ARCHIVE}.sha256"
chmod 0600 "${FINAL_ARCHIVE}" "${FINAL_ARCHIVE}.sha256"
trap - EXIT

printf 'Q042_FINAL_AUDIT_OK\n'
cat "${FINAL_RESULT}"
printf 'final_archive=%s\n' "${FINAL_ARCHIVE}"
cat "${FINAL_ARCHIVE}.sha256"