#!/usr/bin/env python3
"""Build derived indexes for company WeDrive knowledge sources.

The script intentionally writes metadata and navigation indexes, not copies of
the original binary documents. The source WeDrive folders remain the raw record.
"""

from __future__ import annotations

import csv
import os
import re
from collections import Counter, defaultdict
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path


ROOTS = {
    "delivery": Path(
        "/Users/jack/Library/Containers/com.tencent.WeWorkMac/Data/WeDrive/"
        "康比特/项目交付管理共享空间"
    ),
    "tech_kb": Path(
        "/Users/jack/Library/Containers/com.tencent.WeWorkMac/Data/WeDrive/"
        "康比特/数字技术中心知识库"
    ),
    "delivery_dept": Path(
        "/Users/jack/Library/Containers/com.tencent.WeWorkMac/Data/WeDrive/"
        "康比特/交付实施部"
    ),
}

OUTPUT_ROOT = Path("work_company_knowledge")
INDEX_DIR = OUTPUT_ROOT / "indexes"
REPORT_DIR = OUTPUT_ROOT / "reports"

DOC_EXTS = {
    ".doc",
    ".docx",
    ".pdf",
    ".ppt",
    ".pptx",
    ".xls",
    ".xlsx",
    ".csv",
    ".md",
    ".txt",
}
IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".heic", ".webp"}
VIDEO_EXTS = {".mp4", ".mov", ".avi", ".wmv", ".m4v"}
ARCHIVE_EXTS = {".zip", ".rar", ".7z", ".tar", ".gz"}
CODE_EXTS = {".php", ".js", ".ts", ".vue", ".py", ".java", ".cs", ".html", ".css", ".sql"}
INSTALL_EXTS = {".exe", ".dll", ".msi", ".cab", ".pak", ".pdb", ".sys"}


@dataclass
class FileRow:
    file_id: str
    source: str
    top_dir: str
    second_dir: str
    year_bucket: str
    project_or_topic: str
    stage: str
    role_lane: str
    knowledge_area: str
    file_kind: str
    ext: str
    size_bytes: int
    mtime: str
    name: str
    rel_path: str
    abs_path: str


def load_existing_file_ids() -> dict[str, str]:
    """Keep file IDs stable across refreshes so index diffs stay reviewable."""
    index_path = INDEX_DIR / "file-inventory.tsv"
    if not index_path.exists():
        return {}

    with index_path.open("r", newline="", encoding="utf-8-sig") as handle:
        reader = csv.DictReader(handle, delimiter="\t")
        return {
            row["abs_path"]: row["file_id"]
            for row in reader
            if row.get("abs_path") and re.fullmatch(r"WK-\d{6}", row.get("file_id", ""))
        }


def normalize_ext(path: Path) -> str:
    return path.suffix.lower() or "[no_ext]"


def classify_kind(ext: str) -> str:
    if ext in DOC_EXTS:
        return "document"
    if ext in IMAGE_EXTS:
        return "image"
    if ext in VIDEO_EXTS:
        return "video"
    if ext in ARCHIVE_EXTS:
        return "archive"
    if ext in CODE_EXTS:
        return "code_or_script"
    if ext in INSTALL_EXTS:
        return "installer_or_runtime"
    return "other"


def pick_stage(text: str) -> str:
    rules = [
        ("售前", "presales"),
        ("需求", "requirements"),
        ("原型", "design"),
        ("方案", "solution"),
        ("合同", "contract"),
        ("报价", "quotation"),
        ("开发", "development"),
        ("部署", "deployment"),
        ("实施", "implementation"),
        ("交付", "delivery"),
        ("培训", "training"),
        ("验收", "acceptance"),
        ("运维", "operations"),
        ("会议", "meeting"),
        ("测试", "testing"),
        ("安全", "security"),
        ("等保", "security"),
        ("证书", "certificate"),
    ]
    for keyword, stage in rules:
        if keyword in text:
            return stage
    return "uncategorized"


def pick_role_lane(text: str) -> str:
    product_keywords = ["产品", "需求", "原型", "PRD", "方案", "用户手册", "操作手册", "行业报告"]
    delivery_keywords = ["交付", "实施", "培训", "验收", "现场", "施工", "运行计划", "会议纪要"]
    dev_keywords = ["研发", "开发", "部署", "运维", "接口", "数据库", "代码", "安全", "等保", "测试"]
    sales_keywords = ["售前", "报价", "合同", "招投标", "销售", "宣传"]
    management_keywords = ["OKR", "岗位", "试用期", "周例会", "总结", "预算", "资产"]

    hits = []
    for lane, keywords in [
        ("product", product_keywords),
        ("delivery", delivery_keywords),
        ("development", dev_keywords),
        ("presales", sales_keywords),
        ("management", management_keywords),
    ]:
        if any(keyword in text for keyword in keywords):
            hits.append(lane)
    return "+".join(hits) if hits else "shared"


def pick_area(text: str, source: str) -> str:
    rules = [
        ("智慧餐厅", "smart-canteen"),
        ("智慧食堂", "smart-canteen"),
        ("营养", "nutrition"),
        ("科训", "sports-training"),
        ("体能", "sports-training"),
        ("康复", "rehab"),
        ("设备", "device-integration"),
        ("闸机", "device-integration"),
        ("食安", "food-safety"),
        ("进销存", "inventory"),
        ("等保", "security-compliance"),
        ("安全", "security-compliance"),
        ("证书", "certificate"),
        ("运维", "operations"),
        ("部署", "operations"),
        ("会议", "management"),
        ("OKR", "management"),
        ("素材", "assets"),
        ("logo", "assets"),
        ("学习", "learning"),
        ("书籍", "learning"),
        ("新员工", "onboarding"),
    ]
    for keyword, area in rules:
        if keyword in text:
            return area
    return "project-delivery" if source == "delivery" else "technical-knowledge"


def year_bucket(parts: tuple[str, ...], name: str) -> str:
    text = "/".join(parts) + "/" + name
    match = re.search(r"(20\d{2})", text)
    return match.group(1) if match else "unknown"


def project_or_topic(source: str, parts: tuple[str, ...]) -> str:
    if not parts:
        return "."
    if source == "delivery":
        if len(parts) >= 2 and ("立项项目" in parts[0] or "未排期项目" in parts[0]):
            return parts[1]
        if parts[0].startswith("⭐"):
            return parts[1] if len(parts) >= 2 else parts[0]
    if source == "tech_kb":
        return parts[1] if len(parts) >= 2 else parts[0]
    if source == "delivery_dept":
        if len(parts) >= 2 and parts[0] in {"003 实施记录", "004项目资料"}:
            return parts[1]
        if len(parts) >= 2 and parts[0] == "002 安装实施手册":
            return parts[1]
        return parts[0]
    return parts[0]


def second_dir(parts: tuple[str, ...]) -> str:
    if len(parts) >= 2:
        return "/".join(parts[:2])
    if len(parts) == 1:
        return parts[0]
    return "."


def walk_rows(existing_ids: dict[str, str]) -> tuple[list[FileRow], list[str]]:
    rows: list[FileRow] = []
    errors: list[str] = []
    seq = max((int(file_id.removeprefix("WK-")) for file_id in existing_ids.values()), default=0)
    for source, root in ROOTS.items():
        if not root.exists():
            errors.append(f"missing root: {root}")
            continue
        for dirpath, dirnames, filenames in os.walk(root, onerror=lambda e: errors.append(str(e))):
            dirnames.sort()
            filenames.sort()
            current = Path(dirpath)
            try:
                rel_dir = current.relative_to(root)
            except ValueError:
                continue
            parts = () if str(rel_dir) == "." else rel_dir.parts
            for filename in filenames:
                path = current / filename
                try:
                    stat = path.stat()
                except OSError as exc:
                    errors.append(f"{path}: {exc}")
                    continue
                abs_path = str(path)
                file_id = existing_ids.get(abs_path)
                if file_id is None:
                    seq += 1
                    file_id = f"WK-{seq:06d}"
                ext = normalize_ext(path)
                text = "/".join(parts + (filename,))
                rows.append(
                    FileRow(
                        file_id=file_id,
                        source=source,
                        top_dir=parts[0] if parts else ".",
                        second_dir=second_dir(parts),
                        year_bucket=year_bucket(parts, filename),
                        project_or_topic=project_or_topic(source, parts),
                        stage=pick_stage(text),
                        role_lane=pick_role_lane(text),
                        knowledge_area=pick_area(text, source),
                        file_kind=classify_kind(ext),
                        ext=ext,
                        size_bytes=stat.st_size,
                        mtime=datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M:%S"),
                        name=filename,
                        rel_path=str(rel_dir / filename) if str(rel_dir) != "." else filename,
                        abs_path=abs_path,
                    )
                )
    return rows, errors


def write_tsv(path: Path, fieldnames: list[str], rows: list[dict[str, object]]) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open("w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter="\t", extrasaction="ignore")
        writer.writeheader()
        writer.writerows(rows)


def row_dict(row: FileRow) -> dict[str, object]:
    return row.__dict__.copy()


def counter_rows(counter: Counter[tuple[str, ...] | str], columns: list[str]) -> list[dict[str, object]]:
    out = []
    for key, count in counter.most_common():
        if not isinstance(key, tuple):
            key = (key,)
        record = {col: key[idx] if idx < len(key) else "" for idx, col in enumerate(columns)}
        record["file_count"] = count
        out.append(record)
    return out


def write_markdown(rows: list[FileRow], errors: list[str]) -> None:
    OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
    INDEX_DIR.mkdir(parents=True, exist_ok=True)
    REPORT_DIR.mkdir(parents=True, exist_ok=True)

    total_size = sum(row.size_bytes for row in rows)
    source_counts = Counter(row.source for row in rows)
    doc_counts = Counter(row.ext for row in rows if row.file_kind == "document")
    area_counts = Counter(row.knowledge_area for row in rows)
    lane_counts = Counter(row.role_lane for row in rows)

    generated_at = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %z")
    readme = f"""# 公司微盘知识库

本目录是 `zhctprompt` 对企业微信微盘资料的派生知识库。原始文件仍在企业微信微盘，本目录只保存索引、分类、入口和重建脚本输出，不复制原始大文件、安装包、合同附件或证书原件。

## 当前资料源

| 来源 | 原始路径 | 文件数 |
| --- | --- | ---: |
| 项目交付管理共享空间 | `{ROOTS['delivery']}` | {source_counts.get('delivery', 0)} |
| 数字技术中心知识库 | `{ROOTS['tech_kb']}` | {source_counts.get('tech_kb', 0)} |
| 交付实施部 | `{ROOTS['delivery_dept']}` | {source_counts.get('delivery_dept', 0)} |

## 角色入口

- 产品人：先看 `indexes/document-index.tsv`，筛选 `role_lane` 包含 `product` 或 `presales`，再按 `knowledge_area` 找方案、需求、原型、行业资料。
- 交付人：先看 `indexes/project-index.tsv` 和 `indexes/document-index.tsv`，筛选 `stage` 为 `implementation`、`delivery`、`training`、`acceptance`、`operations`。
- 开发人：先看 `indexes/document-index.tsv`，筛选 `role_lane` 包含 `development`，再按 `stage` 找接口、部署、运维、测试、安全和设备对接资料。
- 管理者/新人：先看 `knowledge-map.md`、`indexes/category-summary.tsv` 和 `indexes/role-entrypoints.tsv`。

## 索引文件

- `indexes/file-inventory.tsv`：全量文件索引，包含原始绝对路径、来源、项目/主题、文件类型、角色分类、知识域和修改时间。
- `indexes/document-index.tsv`：可作为知识输入的文档类文件索引，过滤出 Word、PDF、PPT、Excel、Markdown、文本、CSV。
- `indexes/project-index.tsv`：项目或主题级汇总，适合查某个项目资料包是否存在、规模多大、主要阶段是什么。
- `indexes/category-summary.tsv`：按来源、知识域、角色和阶段聚合。
- `indexes/role-entrypoints.tsv`：给产品、交付、开发、售前、管理几类人看的入口清单。
- `online-documents/`：企业微信在线文档、在线表格和智能表格导出快照，补足本地微盘同步文件扫描无法覆盖的协作内容。
- `reports/summary.md`：本次抽取摘要和风险边界。

## 重建方式

```bash
cd /Users/jack/code/010-cpt/008-zhct/zhctprompt
python3 control/scripts/build_company_wedrive_knowledge.py
```

## 使用边界

- 不把微盘原始二进制文件批量复制进 Git。
- 不把生产密钥、完整 SQL dump、证书私钥、个人隐私原文抽取进索引。
- 需要正文级抽取时，按具体任务从 `document-index.tsv` 选定少量文件，再用 MarkItDown 或对应解析器定向转换。
- 企业微信在线文档/在线表格不一定以普通文件同步到微盘目录；这类内容从 `online-documents/online-document-index.csv` 进入，按导出快照保存 CSV/Markdown。
- 微盘目录是资料真源；本目录是团队检索和任务启动入口。
"""
    (OUTPUT_ROOT / "README.md").write_text(readme, encoding="utf-8")

    source_roots = f"""# Source Roots

| key | 用途 | 原始路径 |
| --- | --- | --- |
| `delivery` | 项目交付、售前、实施、验收、运维、标准产品资料 | `{ROOTS['delivery']}` |
| `tech_kb` | 产品、研发、部署、运维、安全、素材、会议、学习和资质资料 | `{ROOTS['tech_kb']}` |
| `delivery_dept` | 交付实施部手册、周报、模板、实施记录、项目资料和交付素材 | `{ROOTS['delivery_dept']}` |
| `wedoc_online` | 企业微信在线文档、在线表格和智能表格的长期入口 | `https://doc.weixin.qq.com` |
| `wedoc_export_jack` | 姜阳本机企业微信在线文档导出快照默认落点 | `/Users/jack` |

这些路径是姜阳本机企业微信微盘同步目录。其他同事电脑上的微盘路径可能不同，应登记到 `control/local-paths.md` 后再重建索引。
在线文档的临时 COS 导出链接会过期，长期入口应记录 `doc.weixin.qq.com` URL 或 doc id。
"""
    (OUTPUT_ROOT / "source-roots.md").write_text(source_roots, encoding="utf-8")

    area_lines = "\n".join(
        f"| {area} | {count} |" for area, count in area_counts.most_common()
    )
    lane_lines = "\n".join(
        f"| {lane} | {count} |" for lane, count in lane_counts.most_common()
    )
    doc_lines = "\n".join(f"| {ext} | {count} |" for ext, count in doc_counts.most_common())
    knowledge_map = f"""# Knowledge Map

生成时间：{generated_at}

## 总览

- 总文件数：{len(rows)}
- 总体积：{total_size / 1024 / 1024:.2f} MB
- 文档类文件数：{sum(doc_counts.values())}
- 扫描错误数：{len(errors)}

## 知识域分布

| 知识域 | 文件数 |
| --- | ---: |
{area_lines}

## 角色分布

| 角色分类 | 文件数 |
| --- | ---: |
{lane_lines}

## 文档类型分布

| 扩展名 | 文档数 |
| --- | ---: |
{doc_lines}

## 推荐检索路径

1. 先用 `indexes/project-index.tsv` 判断目标项目或主题是否存在。
2. 再用 `indexes/document-index.tsv` 按 `role_lane`、`stage`、`knowledge_area` 过滤文档。
3. 最后回到 `abs_path` 打开企业微信微盘原始文件。
4. 如果目标是企业微信在线文档/在线表格，进入 `online-documents/online-document-index.csv` 和对应归档目录。
5. 需要沉淀项目专属事实时，再按项目落到 `work_<project>/`；跨项目方法进 `work/`；未来 agent 规则进 `standards-stack/`。
"""
    (OUTPUT_ROOT / "knowledge-map.md").write_text(knowledge_map, encoding="utf-8")

    summary = f"""# 企业微信微盘知识库抽取摘要

时间：{generated_at}

## 目标

把企业微信微盘目录抽取成 `zhctprompt` 内可检索、可复用的团队知识入口，让产品人、交付人、开发人都能从同一个控制项目进入资料，而不是只围绕开发仓库工作。

## 结果

- 扫描 `项目交付管理共享空间`：{source_counts.get('delivery', 0)} 个文件。
- 扫描 `数字技术中心知识库`：{source_counts.get('tech_kb', 0)} 个文件。
- 扫描 `交付实施部`：{source_counts.get('delivery_dept', 0)} 个文件。
- 总文件数：{len(rows)}。
- 总体积：{total_size / 1024 / 1024:.2f} MB。
- 文档类文件数：{sum(doc_counts.values())}。

## 已生成

- `work_company_knowledge/README.md`
- `work_company_knowledge/source-roots.md`
- `work_company_knowledge/knowledge-map.md`
- `work_company_knowledge/indexes/file-inventory.tsv`
- `work_company_knowledge/indexes/document-index.tsv`
- `work_company_knowledge/indexes/project-index.tsv`
- `work_company_knowledge/indexes/category-summary.tsv`
- `work_company_knowledge/indexes/role-entrypoints.tsv`
- `work_company_knowledge/online-documents/online-document-index.csv`

## 抽取边界

本次只抽取路径、文件名、扩展名、大小、mtime 和基于路径/文件名的知识分类；没有批量复制或正文抽取原始合同、证书、安装包、图片、视频和数据库文件。

## 后续规则

- 具体项目沉淀：进入 `work_<project>/`。
- 公司级资料检索：先进入 `work_company_knowledge/`。
- 面向未来 agent 的行为规则：进入 `standards-stack/` 和对应 skill。
- 正文级转换：按任务挑选少量文件，用 MarkItDown 定向转换，避免把敏感资料全文进入仓库。
"""
    (REPORT_DIR / "summary.md").write_text(summary, encoding="utf-8")
    error_text = "\n".join(errors) + ("\n" if errors else "")
    (REPORT_DIR / "scan-errors.txt").write_text(error_text, encoding="utf-8")


def build_indexes(rows: list[FileRow]) -> None:
    write_tsv(
        INDEX_DIR / "file-inventory.tsv",
        list(FileRow.__dataclass_fields__.keys()),
        [row_dict(row) for row in rows],
    )
    document_rows = [row for row in rows if row.file_kind == "document"]
    write_tsv(
        INDEX_DIR / "document-index.tsv",
        list(FileRow.__dataclass_fields__.keys()),
        [row_dict(row) for row in document_rows],
    )

    grouped: dict[tuple[str, str, str, str], dict[str, object]] = {}
    for row in rows:
        key = (row.source, row.top_dir, row.project_or_topic, row.year_bucket)
        record = grouped.setdefault(
            key,
            {
                "source": row.source,
                "top_dir": row.top_dir,
                "project_or_topic": row.project_or_topic,
                "year_bucket": row.year_bucket,
                "file_count": 0,
                "document_count": 0,
                "size_mb": 0.0,
                "top_stages": Counter(),
                "top_areas": Counter(),
                "sample_path": row.abs_path,
            },
        )
        record["file_count"] = int(record["file_count"]) + 1
        record["document_count"] = int(record["document_count"]) + (1 if row.file_kind == "document" else 0)
        record["size_mb"] = float(record["size_mb"]) + row.size_bytes / 1024 / 1024
        record["top_stages"][row.stage] += 1
        record["top_areas"][row.knowledge_area] += 1

    project_rows = []
    for record in grouped.values():
        stages: Counter = record.pop("top_stages")  # type: ignore[assignment]
        areas: Counter = record.pop("top_areas")  # type: ignore[assignment]
        record["size_mb"] = f"{float(record['size_mb']):.2f}"
        record["top_stages"] = ",".join(f"{k}:{v}" for k, v in stages.most_common(5))
        record["top_areas"] = ",".join(f"{k}:{v}" for k, v in areas.most_common(5))
        project_rows.append(record)
    project_rows.sort(key=lambda item: (str(item["source"]), str(item["top_dir"]), str(item["project_or_topic"])))
    write_tsv(
        INDEX_DIR / "project-index.tsv",
        [
            "source",
            "top_dir",
            "project_or_topic",
            "year_bucket",
            "file_count",
            "document_count",
            "size_mb",
            "top_stages",
            "top_areas",
            "sample_path",
        ],
        project_rows,
    )

    category_counter = Counter(
        (row.source, row.knowledge_area, row.role_lane, row.stage, row.file_kind) for row in rows
    )
    write_tsv(
        INDEX_DIR / "category-summary.tsv",
        ["source", "knowledge_area", "role_lane", "stage", "file_kind", "file_count"],
        counter_rows(category_counter, ["source", "knowledge_area", "role_lane", "stage", "file_kind"]),
    )

    role_records = []
    for role in ["product", "delivery", "development", "presales", "management", "shared"]:
        matching = [
            row
            for row in rows
            if row.file_kind == "document" and (role in row.role_lane.split("+") or row.role_lane == role)
        ]
        area = Counter(row.knowledge_area for row in matching)
        stage = Counter(row.stage for row in matching)
        samples = [row.abs_path for row in matching[:12]]
        role_records.append(
            {
                "role": role,
                "document_count": len(matching),
                "top_areas": ",".join(f"{k}:{v}" for k, v in area.most_common(8)),
                "top_stages": ",".join(f"{k}:{v}" for k, v in stage.most_common(8)),
                "sample_paths": " | ".join(samples),
            }
        )
    write_tsv(
        INDEX_DIR / "role-entrypoints.tsv",
        ["role", "document_count", "top_areas", "top_stages", "sample_paths"],
        role_records,
    )


def main() -> int:
    rows, errors = walk_rows(load_existing_file_ids())
    rows.sort(key=lambda row: (row.source, row.rel_path))
    build_indexes(rows)
    write_markdown(rows, errors)
    print(
        f"generated {len(rows)} files, "
        f"{sum(1 for row in rows if row.file_kind == 'document')} documents, "
        f"errors={len(errors)}"
    )
    return 1 if errors else 0


if __name__ == "__main__":
    raise SystemExit(main())