#!/usr/bin/env python3
"""Crawl tender detail evidence with Scrapling and build coverage workbooks.

The source Excel already contains project-level Yifangbao/Qianlima links.  This
script turns those links into an auditable enrichment queue: every fetch attempt
is recorded, protected endpoints are marked as login-required, and confirmed
detail fields are kept separate from missing or inferred fields.
"""

from __future__ import annotations

import argparse
import html
import json
import re
import sys
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib.parse import quote_plus

from openpyxl import Workbook, load_workbook
from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
from openpyxl.utils import get_column_letter


ROOT = Path(__file__).resolve().parents[1]
SOURCE_XLSX = ROOT / "docs" / "deliverables" / "20260617-raw-excel-entity-tables" / "原始Excel招投标实体抽取表.xlsx"
DEFAULT_OUT_DIR = ROOT / "docs" / "deliverables" / "20260618-scrapling-tender-detail-crawl"
SCRAPLING_SOURCE = Path("/Users/jack/code/099-github/jiangyang118/Scrapling")

FONT = "Microsoft YaHei"
BLUE = "1F4E79"
LIGHT_BLUE = "D9EAF7"
LIGHT_GREEN = "E2F0D9"
LIGHT_YELLOW = "FFF2CC"
LIGHT_RED = "FCE4D6"
WHITE = "FFFFFF"
GRAY = "F6F8FB"
BORDER = Border(
    left=Side(style="thin", color="B7C9DA"),
    right=Side(style="thin", color="B7C9DA"),
    top=Side(style="thin", color="B7C9DA"),
    bottom=Side(style="thin", color="B7C9DA"),
)

DETAIL_FIELDS = [
    ("项目基础信息", ["项目名称", "招标公司", "投标公司", "金额", "发布时间"]),
    ("产品的软件功能", ["软件", "系统", "平台", "功能", "模块", "智慧食堂", "监管"]),
    ("硬件设备参数", ["硬件", "设备", "终端", "型号", "参数", "刷脸", "摄像头", "消费机"]),
    ("公司的资质", ["资质", "证书", "认证", "许可", "专利", "软著", "软件著作权"]),
    ("评分情况", ["评分", "商务", "技术", "报价分", "评审", "评标", "综合得分"]),
    ("中标原因", ["中标原因", "排名第一", "综合排名", "推荐", "成交原因", "中标候选人"]),
    ("3家投标公司", ["投标人", "供应商", "候选人", "第一名", "第二名", "第三名"]),
    ("招标/中标原文件附件", ["附件", "采购文件", "招标文件", "投标文件", "下载"]),
]


@dataclass
class FetchResult:
    channel: str
    url: str
    status: str
    http_status: str = ""
    evidence_level: str = ""
    title: str = ""
    text: str = ""
    raw_excerpt: str = ""
    next_action: str = ""


def import_scrapling_fetcher() -> Any:
    if SCRAPLING_SOURCE.exists() and str(SCRAPLING_SOURCE) not in sys.path:
        sys.path.insert(0, str(SCRAPLING_SOURCE))
    try:
        from scrapling.fetchers import Fetcher
    except Exception as exc:  # pragma: no cover - environment guidance
        raise RuntimeError(
            "Scrapling Fetcher cannot be imported. Use Python 3.10+ and install "
            f"the local project: python -m pip install -e '{SCRAPLING_SOURCE}[fetchers]'"
        ) from exc
    return Fetcher


def clean(value: Any) -> str:
    if value is None:
        return ""
    text = str(value).strip()
    return "" if text in {"None", "nan", "--", "暂未公布"} else text


def short(text: str, limit: int = 360) -> str:
    normalized = re.sub(r"\s+", " ", clean(text))
    return normalized[:limit] + ("..." if len(normalized) > limit else "")


def strip_html(text: str) -> str:
    text = re.sub(r"<script\b[^<]*(?:(?!</script>)<[^<]*)*</script>", " ", text, flags=re.I)
    text = re.sub(r"<style\b[^<]*(?:(?!</style>)<[^<]*)*</style>", " ", text, flags=re.I)
    text = re.sub(r"<[^>]+>", " ", text)
    return html.unescape(re.sub(r"\s+", " ", text)).strip()


def read_projects(source: Path, limit: int | None = None, project_ids: set[str] | None = None) -> list[dict[str, Any]]:
    wb = load_workbook(source, read_only=True, data_only=True)
    ws = wb["招投标项目表"]
    headers = [clean(cell.value) for cell in next(ws.iter_rows(min_row=2, max_row=2))]
    rows: list[dict[str, Any]] = []
    for excel_row in ws.iter_rows(min_row=3, values_only=True):
        item = {header: clean(value) for header, value in zip(headers, excel_row) if header}
        if not item.get("项目ID"):
            continue
        if project_ids and item["项目ID"] not in project_ids:
            continue
        rows.append(item)
        if limit and len(rows) >= limit:
            break
    wb.close()
    return rows


def parse_qianlima_url(url: str) -> dict[str, str]:
    match = re.search(r"/infoDetail/(\d+)/(\d+)/([^/?#]+)", url)
    if not match:
        return {}
    return {"contentId": match.group(1), "areaId": match.group(2), "pageFrom": match.group(3)}


def fetch_static(Fetcher: Any, url: str, timeout: int, headers: dict[str, str]) -> FetchResult:
    try:
        page = Fetcher.get(url, timeout=timeout, retries=1, stealthy_headers=True, headers=headers)
        body = page.body.decode(page.encoding or "utf-8", errors="replace")
        texts = page.css("body ::text").getall()
        text = " ".join(clean(item) for item in texts if clean(item))
        title = clean(page.css("title::text").get() or "")
        status = "confirmed"
        next_action = "从静态页面抽取详情字段。"
        if "页面加载中" in text and len(text) < 2000:
            status = "dynamic-shell"
            next_action = "该链接为前端 SPA 外壳，需要接口认证或浏览器登录后抓取正文。"
        return FetchResult(
            channel="聚合站详情页",
            url=url,
            status=status,
            http_status=str(getattr(page, "status", "")),
            evidence_level="聚合站页面",
            title=title,
            text=text,
            raw_excerpt=short(strip_html(body)),
            next_action=next_action,
        )
    except Exception as exc:
        return FetchResult(
            channel="聚合站详情页",
            url=url,
            status="failed",
            evidence_level="聚合站页面",
            raw_excerpt=short(str(exc)),
            next_action="检查网络、证书或站点访问限制后重试。",
        )


def fetch_qianlima_api(
    Fetcher: Any,
    params: dict[str, str],
    timeout: int,
    headers: dict[str, str],
    auth_token: str = "",
) -> FetchResult:
    if not params:
        return FetchResult(
            channel="乙方宝详情接口",
            url="",
            status="not-applicable",
            evidence_level="接口",
            next_action="来源链接不是乙方宝 infoDetail 格式，需走官方平台或搜索补链。",
        )
    api_url = "https://qiye.qianlima.com/new_qd_yfbsite/api/subZhaobiao/zbDetail"
    api_headers = dict(headers)
    if auth_token:
        api_headers["Authorization"] = f"Bearer {auth_token}"
    try:
        page = Fetcher.get(api_url, params=params, timeout=timeout, retries=1, stealthy_headers=True, headers=api_headers)
        body = page.body.decode(page.encoding or "utf-8", errors="replace")
        status = "partial"
        next_action = "解析接口 JSON 并补齐正文、附件和联系人。"
        text = body
        title = ""
        try:
            payload = json.loads(body)
            code = payload.get("code")
            if code == 401:
                status = "login-required"
                next_action = "乙方宝接口返回 401；需要扫码/登录后带 Authorization 或 Cookie 复跑。"
            elif code not in {0, 200, "0", "200", None}:
                status = "partial"
                next_action = f"接口返回 code={code}，需人工复核接口参数或登录状态。"
            else:
                data = payload.get("data", payload)
                text = strip_html(json.dumps(data, ensure_ascii=False))
                title = clean(data.get("title") or data.get("projectName") or "") if isinstance(data, dict) else ""
                status = "confirmed" if text else "partial"
        except json.JSONDecodeError:
            status = "partial"
            next_action = "接口未返回标准 JSON，需检查登录状态或接口变更。"
        return FetchResult(
            channel="乙方宝详情接口",
            url=api_url + "?" + "&".join(f"{k}={v}" for k, v in params.items()),
            status=status,
            http_status=str(getattr(page, "status", "")),
            evidence_level="聚合站接口",
            title=title,
            text=text,
            raw_excerpt=short(body, 800),
            next_action=next_action,
        )
    except Exception as exc:
        return FetchResult(
            channel="乙方宝详情接口",
            url=api_url,
            status="failed",
            evidence_level="聚合站接口",
            raw_excerpt=short(str(exc)),
            next_action="检查接口访问、认证头和网络后重试。",
        )


def fetch_baidu_search(Fetcher: Any, project: dict[str, Any], timeout: int, headers: dict[str, str]) -> FetchResult:
    query = f"{project.get('项目名称', '')} {project.get('招标公司', '')}".strip()
    url = "https://www.baidu.com/s?wd=" + quote_plus(query)
    try:
        page = Fetcher.get(url, timeout=timeout, retries=1, stealthy_headers=True, headers=headers)
        text = " ".join(clean(item) for item in page.css("body ::text").getall() if clean(item))
        links = [clean(link) for link in page.css("a::attr(href)").getall() if clean(link)]
        interesting = [link for link in links if "baidu.com/link" in link or "http" in link][:8]
        status = "partial" if text else "empty"
        return FetchResult(
            channel="公开搜索",
            url=url,
            status=status,
            http_status=str(getattr(page, "status", "")),
            evidence_level="搜索线索",
            title=clean(page.css("title::text").get() or ""),
            text=text,
            raw_excerpt=short(" | ".join(interesting) + " " + text, 900),
            next_action="逐条打开搜索结果，优先确认官方公告和附件。",
        )
    except Exception as exc:
        return FetchResult(
            channel="公开搜索",
            url=url,
            status="failed",
            evidence_level="搜索线索",
            raw_excerpt=short(str(exc)),
            next_action="搜索引擎限制或网络失败，改用官方站内搜索/浏览器登录。",
        )


def field_status(project: dict[str, Any], evidence_text: str, field_name: str, keywords: list[str]) -> tuple[str, str]:
    base_text = " ".join(
        clean(project.get(key))
        for key in ["项目名称", "招标公司", "投标公司/中标单位", "中标金额_万元", "信息发布时间", "官网查看地址"]
    )
    if field_name == "项目基础信息":
        return "✓", "原始实体 Excel 已确认项目名、招标/中标单位、金额和来源链接。"
    haystack = f"{base_text} {evidence_text}"
    hits = [kw for kw in keywords if kw and kw in haystack]
    if hits and evidence_text and any(kw in evidence_text for kw in hits):
        return "△", "抓取线索命中：" + "、".join(hits[:6]) + "；仍需官方附件确认。"
    if hits:
        return "△", "原始标题/关键词命中：" + "、".join(hits[:6]) + "；未见详细参数。"
    return "✗", "当前公开/匿名抓取未取得该字段，需登录乙方宝或官方平台下载附件。"


def status_label(results: list[FetchResult]) -> str:
    statuses = {item.status for item in results}
    if "confirmed" in statuses:
        return "部分抓到"
    if "login-required" in statuses:
        return "需登录补齐"
    if "dynamic-shell" in statuses:
        return "动态页待补"
    if "failed" in statuses:
        return "抓取失败待重试"
    return "待补"


def build_rows(projects: list[dict[str, Any]], args: argparse.Namespace) -> dict[str, list[dict[str, Any]]]:
    Fetcher = import_scrapling_fetcher()
    headers = {
        "Referer": "https://qiye.qianlima.com/new_qd_yfbsite/",
        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.7",
    }
    if args.cookie:
        headers["Cookie"] = args.cookie

    project_rows: list[dict[str, Any]] = []
    source_rows: list[dict[str, Any]] = []
    coverage_rows: list[dict[str, Any]] = []
    gap_rows: list[dict[str, Any]] = []
    search_rows: list[dict[str, Any]] = []

    for index, project in enumerate(projects, 1):
        url = clean(project.get("官网查看地址"))
        params = parse_qianlima_url(url)
        attempts: list[FetchResult] = []
        if url:
            attempts.append(fetch_static(Fetcher, url, args.timeout, headers))
        attempts.append(fetch_qianlima_api(Fetcher, params, args.timeout, headers, args.auth_token))
        if args.search_limit and index <= args.search_limit:
            attempts.append(fetch_baidu_search(Fetcher, project, args.timeout, headers))

        evidence_text = " ".join(item.text for item in attempts if item.text)
        for attempt in attempts:
            source_rows.append(
                {
                    "项目ID": project.get("项目ID", ""),
                    "项目名称": project.get("项目名称", ""),
                    "渠道": attempt.channel,
                    "状态": attempt.status,
                    "HTTP状态": attempt.http_status,
                    "证据等级": attempt.evidence_level,
                    "URL": attempt.url,
                    "页面标题": attempt.title,
                    "摘录": attempt.raw_excerpt,
                    "下一步": attempt.next_action,
                }
            )
            if attempt.channel == "公开搜索":
                search_rows.append(
                    {
                        "项目ID": project.get("项目ID", ""),
                        "项目名称": project.get("项目名称", ""),
                        "搜索URL": attempt.url,
                        "状态": attempt.status,
                        "线索摘录": attempt.raw_excerpt,
                        "下一步": attempt.next_action,
                    }
                )

        confirmed = partial = missing = 0
        missing_fields: list[str] = []
        for field_name, keywords in DETAIL_FIELDS:
            mark, note = field_status(project, evidence_text, field_name, keywords)
            if mark == "✓":
                confirmed += 1
            elif mark == "△":
                partial += 1
            else:
                missing += 1
                missing_fields.append(field_name)
            row = {
                "项目ID": project.get("项目ID", ""),
                "项目名称": project.get("项目名称", ""),
                "字段": field_name,
                "状态": mark,
                "说明": note,
                "当前来源": "；".join(f"{item.channel}:{item.status}" for item in attempts),
                "下一步": "扫码/登录乙方宝详情页下载附件；并用官方平台复核。" if mark == "✗" else "继续用官方附件确认明细。",
            }
            coverage_rows.append(row)
            if mark == "✗":
                gap_rows.append(row)

        project_rows.append(
            {
                "项目序号": index,
                "项目ID": project.get("项目ID", ""),
                "项目名称": project.get("项目名称", ""),
                "招标公司": project.get("招标公司", ""),
                "中标单位": project.get("投标公司/中标单位", ""),
                "金额_万元": project.get("中标金额_万元", ""),
                "地区": " ".join(clean(project.get(k)) for k in ["发布省份", "发布市级", "发布区级"]).strip(),
                "来源链接": url,
                "乙方宝contentId": params.get("contentId", ""),
                "乙方宝areaId": params.get("areaId", ""),
                "详情采集状态": status_label(attempts),
                "已确认字段数": confirmed,
                "部分字段数": partial,
                "缺失字段数": missing,
                "缺失字段": "、".join(missing_fields),
                "结论": f"这个项目一共要抓 {len(DETAIL_FIELDS)} 个，已确认 {confirmed} 个，部分确认 {partial} 个，缺 {missing} 个。",
                "下一步": "需要登录乙方宝/千里马或官方平台下载详情附件。" if missing else "进入人工复核。",
            }
        )

    summary = build_summary_rows(project_rows, source_rows, args)
    env_rows = [
        {"项目": "Scrapling源码路径", "值": str(SCRAPLING_SOURCE), "说明": "本地开源项目作为抓取引擎来源。"},
        {"项目": "Python解释器", "值": sys.executable, "说明": "Scrapling 0.4.9 需要 Python 3.10+。"},
        {"项目": "输入Excel", "值": str(SOURCE_XLSX.relative_to(ROOT)), "说明": "从实体项目表读取全量项目和来源链接。"},
        {"项目": "认证方式", "值": "已传入" if args.auth_token or args.cookie else "未传入", "说明": "未认证时乙方宝详情接口会返回 401。"},
    ]
    return {
        "运行摘要": summary,
        "项目详情队列": project_rows,
        "来源尝试记录": source_rows,
        "字段覆盖矩阵": coverage_rows,
        "登录补齐清单": gap_rows,
        "搜索线索": search_rows or [{"说明": "本次未启用公开搜索；使用 --search-limit N 可对前 N 个项目补充搜索线索。"}],
        "Scrapling运行环境": env_rows,
    }


def build_summary_rows(project_rows: list[dict[str, Any]], source_rows: list[dict[str, Any]], args: argparse.Namespace) -> list[dict[str, Any]]:
    total = len(project_rows)
    login_required = sum(1 for row in source_rows if row.get("状态") == "login-required")
    dynamic_shell = sum(1 for row in source_rows if row.get("状态") == "dynamic-shell")
    confirmed = sum(int(row.get("已确认字段数") or 0) for row in project_rows)
    partial = sum(int(row.get("部分字段数") or 0) for row in project_rows)
    missing = sum(int(row.get("缺失字段数") or 0) for row in project_rows)
    return [
        {"指标": "采集项目数", "值": total, "说明": "来自实体 Excel 的招投标项目。"},
        {"指标": "来源尝试次数", "值": len(source_rows), "说明": "每个项目至少尝试详情页和乙方宝详情接口。"},
        {"指标": "乙方宝接口401次数", "值": login_required, "说明": "认证失败，需要扫码/登录后复跑。"},
        {"指标": "SPA动态页次数", "值": dynamic_shell, "说明": "静态 HTML 只有页面加载壳，不能当作详情正文。"},
        {"指标": "字段确认数", "值": confirmed, "说明": "原始 Excel 已确认的基础字段。"},
        {"指标": "字段部分确认数", "值": partial, "说明": "标题/搜索/接口文本命中，但缺官方附件细节。"},
        {"指标": "字段缺失数", "值": missing, "说明": "当前公开匿名抓取拿不到的详细字段。"},
        {"指标": "公开搜索项目数", "值": args.search_limit or 0, "说明": "用 Scrapling 抓取百度搜索结果的项目数。"},
    ]


def style_cell(cell: Any, fill: str | None = None, bold: bool = False, color: str = "1F1F1F") -> None:
    cell.font = Font(name=FONT, size=10, bold=bold, color=color)
    cell.alignment = Alignment(wrap_text=True, vertical="top")
    cell.border = BORDER
    if fill:
        cell.fill = PatternFill("solid", fgColor=fill)


def fill_for(value: Any) -> str | None:
    text = clean(value)
    if text in {"✓", "confirmed", "部分抓到"}:
        return LIGHT_GREEN
    if text in {"△", "partial", "dynamic-shell", "需登录补齐", "动态页待补"}:
        return LIGHT_YELLOW
    if text in {"✗", "failed", "login-required", "抓取失败待重试"}:
        return LIGHT_RED
    return None


def write_sheet(wb: Workbook, name: str, rows: list[dict[str, Any]]) -> None:
    ws = wb.create_sheet(name[:31])
    if not rows:
        rows = [{"说明": "无记录"}]
    headers = list(rows[0].keys())
    ws.sheet_view.showGridLines = False
    ws.merge_cells(start_row=1, start_column=1, end_row=1, end_column=len(headers))
    title = ws.cell(1, 1, name)
    title.font = Font(name=FONT, size=15, bold=True, color=WHITE)
    title.fill = PatternFill("solid", fgColor=BLUE)
    title.alignment = Alignment(horizontal="center", vertical="center")
    title.border = BORDER
    for col, header in enumerate(headers, 1):
        cell = ws.cell(2, col, header)
        style_cell(cell, LIGHT_BLUE, True, "14344A")
        cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True)
    for row_index, row in enumerate(rows, 3):
        for col_index, header in enumerate(headers, 1):
            cell = ws.cell(row_index, col_index, row.get(header, ""))
            style_cell(cell, fill_for(row.get(header, "")))
            if header.endswith("URL") or header.endswith("链接"):
                value = clean(row.get(header, ""))
                if value.startswith("http"):
                    cell.hyperlink = value
                    cell.style = "Hyperlink"
                    style_cell(cell, fill_for(row.get(header, "")))
    for idx, header in enumerate(headers, 1):
        max_len = len(str(header))
        for row in rows[:120]:
            max_len = max(max_len, min(len(str(row.get(header, ""))) // 2, 60))
        ws.column_dimensions[get_column_letter(idx)].width = min(max(max_len + 4, 14), 68)
    for row_number in range(1, ws.max_row + 1):
        ws.row_dimensions[row_number].height = 34
    ws.freeze_panes = "A3"
    ws.auto_filter.ref = f"A2:{get_column_letter(len(headers))}{ws.max_row}"


def write_workbook(tables: dict[str, list[dict[str, Any]]], out_dir: Path) -> Path:
    out_dir.mkdir(parents=True, exist_ok=True)
    wb = Workbook()
    wb.remove(wb.active)
    for name, rows in tables.items():
        write_sheet(wb, name, rows)
    output = out_dir / "Scrapling智慧食堂招投标详情采集结果.xlsx"
    wb.save(output)
    return output


def write_html(tables: dict[str, list[dict[str, Any]]], out_dir: Path, xlsx_path: Path) -> Path:
    summary = tables["运行摘要"]
    projects = tables["项目详情队列"]
    gaps = tables["登录补齐清单"]
    total = len(projects)
    login_projects = sum(1 for row in projects if "登录" in clean(row.get("详情采集状态")))
    generated = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    def table_html(rows: list[dict[str, Any]], limit: int = 30) -> str:
        rows = rows[:limit]
        if not rows:
            return "<p class=\"empty\">无记录</p>"
        headers = list(rows[0].keys())
        head = "".join(f"<th>{html.escape(str(h))}</th>" for h in headers)
        body = []
        for row in rows:
            cells = []
            for header in headers:
                value = clean(row.get(header, ""))
                if value.startswith("http"):
                    value_html = f'<a href="{html.escape(value)}" target="_blank" rel="noreferrer">打开链接</a>'
                else:
                    value_html = html.escape(str(value))
                cells.append(f"<td>{value_html}</td>")
            body.append("<tr>" + "".join(cells) + "</tr>")
        return f"<table><thead><tr>{head}</tr></thead><tbody>{''.join(body)}</tbody></table>"

    html_text = f"""<!doctype html>
<html lang="zh-CN">
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>Scrapling 智慧食堂招投标详情采集结果</title>
  <style>
    :root {{
      --page:#f5f7fa; --panel:#ffffff; --ink:#17212b; --muted:#617084;
      --line:#d8e1ea; --accent:#0f766e; --warn:#9a5b00; --bad:#9f2f1f;
      --soft:#e8f4f2; --warn-soft:#fff3d8; --bad-soft:#fde8df;
    }}
    * {{ box-sizing:border-box; }}
    body {{ margin:0; background:var(--page); color:var(--ink); font-family:"Microsoft YaHei","PingFang SC",Arial,sans-serif; line-height:1.56; }}
    main {{ width:min(1440px, calc(100% - 36px)); margin:0 auto; padding:28px 0 56px; }}
    header, section {{ background:var(--panel); border:1px solid var(--line); border-radius:8px; }}
    header {{ padding:24px; margin-bottom:16px; }}
    h1 {{ margin:0; font-size:34px; line-height:1.2; letter-spacing:0; }}
    h2 {{ margin:0 0 14px; font-size:18px; }}
    p {{ margin:10px 0 0; color:var(--muted); }}
    .grid {{ display:grid; grid-template-columns:repeat(4,minmax(0,1fr)); gap:12px; margin-top:18px; }}
    .metric {{ border:1px solid var(--line); border-radius:8px; padding:14px; background:#fbfcfd; min-height:86px; }}
    .metric strong {{ display:block; font-size:24px; }}
    .metric span {{ display:block; margin-top:6px; color:var(--muted); font-size:13px; }}
    section {{ padding:18px; margin:16px 0; overflow:hidden; }}
    .banner {{ border:1px solid #ecd28c; background:var(--warn-soft); color:#573900; border-radius:8px; padding:14px; margin-top:16px; }}
    .actions {{ display:flex; gap:10px; flex-wrap:wrap; margin-top:16px; }}
    .button {{ display:inline-flex; align-items:center; min-height:38px; padding:8px 12px; border-radius:8px; background:var(--accent); color:white; text-decoration:none; font-weight:700; }}
    table {{ width:100%; border-collapse:collapse; table-layout:fixed; font-size:13px; }}
    th,td {{ border:1px solid var(--line); padding:8px 10px; text-align:left; vertical-align:top; word-break:break-word; }}
    th {{ background:#e8f4f2; color:#12433f; }}
    a {{ color:var(--accent); text-decoration:none; }}
    a:hover {{ text-decoration:underline; }}
    .empty {{ color:var(--muted); }}
    @media (max-width:900px) {{ main {{ width:calc(100% - 20px); }} .grid {{ grid-template-columns:1fr 1fr; }} table {{ display:block; overflow-x:auto; }} }}
  </style>
</head>
<body>
<main>
  <header>
    <h1>Scrapling 智慧食堂招投标详情采集结果</h1>
    <p>生成时间：{html.escape(generated)}。本页用于查看全量项目详情抓取状态、字段覆盖和登录补齐清单。</p>
    <div class="actions">
      <a class="button" href="{html.escape(xlsx_path.name)}">下载 Excel 结果</a>
    </div>
    <div class="grid">
      <div class="metric"><strong>{total}</strong><span>采集项目数</span></div>
      <div class="metric"><strong>{login_projects}</strong><span>需登录补齐项目</span></div>
      <div class="metric"><strong>{len(gaps)}</strong><span>字段缺口行</span></div>
      <div class="metric"><strong>Scrapling</strong><span>本地开源抓取引擎</span></div>
    </div>
    <div class="banner">乙方宝详情接口匿名访问返回 401。当前结果已保存抓取证据和缺口；传入登录 Cookie 或 Authorization 后可复跑补齐正文、附件和资质/评分信息。</div>
  </header>
  <section><h2>运行摘要</h2>{table_html(summary, 20)}</section>
  <section><h2>项目详情队列</h2>{table_html(projects, 40)}</section>
  <section><h2>登录补齐清单</h2>{table_html(gaps, 80)}</section>
</main>
</body>
</html>
"""
    output = out_dir / "Scrapling智慧食堂招投标详情采集结果.html"
    output.write_text(html_text, encoding="utf-8")
    return output


def write_run_note(out_dir: Path, xlsx_path: Path, html_path: Path, tables: dict[str, list[dict[str, Any]]]) -> Path:
    note = ROOT / "docs" / "runs" / "20260618-scrapling-tender-detail-crawl.html"
    summary = {row["指标"]: row["值"] for row in tables["运行摘要"] if "指标" in row}
    note.write_text(
        f"""<!doctype html>
<html lang="zh-CN">
<head><meta charset="utf-8"><title>Scrapling 招投标详情采集运行记录</title></head>
<body>
<h1>Scrapling 招投标详情采集运行记录</h1>
<p>用户目标：使用本机 <code>/Users/jack/code/099-github/jiangyang118/Scrapling</code> 抓取智慧食堂招投标详细信息。</p>
<h2>结果</h2>
<ul>
  <li>采集项目数：{summary.get('采集项目数', '')}</li>
  <li>来源尝试次数：{summary.get('来源尝试次数', '')}</li>
  <li>乙方宝接口 401 次数：{summary.get('乙方宝接口401次数', '')}</li>
  <li>输出 Excel：<code>{html.escape(str(xlsx_path.relative_to(ROOT)))}</code></li>
  <li>输出 HTML：<code>{html.escape(str(html_path.relative_to(ROOT)))}</code></li>
</ul>
<h2>结论</h2>
<p>匿名方式可以确认项目基础信息和来源链接，但乙方宝详情接口返回认证失败；需要扫码/登录后带 Cookie 或 Authorization 复跑，才能继续下载附件、评分标准、设备清单和资质证书。</p>
<h2>复用方式</h2>
<pre>python3 scripts/crawl_tender_details_with_scrapling.py --search-limit 10
python3 scripts/crawl_tender_details_with_scrapling.py --auth-token TOKEN --search-limit 20</pre>
</body>
</html>
""",
        encoding="utf-8",
    )
    return note


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Crawl tender details with Scrapling.")
    parser.add_argument("--source", type=Path, default=SOURCE_XLSX, help="Entity workbook path.")
    parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUT_DIR, help="Output deliverable directory.")
    parser.add_argument("--limit", type=int, default=0, help="Limit number of projects; 0 means all.")
    parser.add_argument("--project-id", action="append", default=[], help="Only crawl selected project ID. Can repeat.")
    parser.add_argument("--search-limit", type=int, default=0, help="Use Scrapling to fetch Baidu search for the first N projects.")
    parser.add_argument("--timeout", type=int, default=20, help="Fetch timeout in seconds.")
    parser.add_argument("--auth-token", default="", help="Optional Bearer token for Yifangbao API.")
    parser.add_argument("--cookie", default="", help="Optional Cookie header for logged-in pages.")
    return parser.parse_args()


def main() -> None:
    args = parse_args()
    projects = read_projects(args.source, args.limit or None, set(args.project_id) if args.project_id else None)
    tables = build_rows(projects, args)
    xlsx_path = write_workbook(tables, args.output_dir)
    html_path = write_html(tables, args.output_dir, xlsx_path)
    note_path = write_run_note(args.output_dir, xlsx_path, html_path, tables)
    print(json.dumps({
        "projects": len(projects),
        "xlsx": str(xlsx_path.relative_to(ROOT)),
        "html": str(html_path.relative_to(ROOT)),
        "note": str(note_path.relative_to(ROOT)),
    }, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()