"""
中国裁判文书网爬虫 - 劳动争议案件数据采集
===========================================
目标：爬取涉及"违法解除劳动合同赔偿金"的案件，提取当事人性别、
      赔偿金额、工作年限、月薪等关键信息，用于算法偏见研究。

使用方法：
    pip install playwright requests-html beautifulsoup4 pandas lxml
    playwright install chromium
    python scraper_wenshu.py

输出：
    ../data/raw_judgments.csv    - 原始爬取结果
    ../data/cleaned_judgments.csv - 清洗后的结构化数据

注意：请遵守 robots.txt 和网站使用条款，合理控制爬取频率。
"""

import asyncio
import json
import re
import time
import random
import logging
import pandas as pd
from pathlib import Path
from datetime import datetime
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout

# ===== 配置 =====
OUTPUT_DIR = Path("../data")
OUTPUT_DIR.mkdir(exist_ok=True)

SEARCH_KEYWORDS = [
    "违法解除劳动合同 赔偿金",
    "劳动合同法第八十七条 赔偿",
    "违法解除 二倍赔偿 劳动者",
]

# 每次请求之间的随机延迟（秒），避免对服务器造成压力
MIN_DELAY = 3.0
MAX_DELAY = 7.0

MAX_PAGES = 20      # 每个关键词最多爬取页数
MAX_CASES = 500     # 总案例上限

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    handlers=[
        logging.FileHandler(OUTPUT_DIR / "scraper.log", encoding="utf-8"),
        logging.StreamHandler(),
    ],
)
logger = logging.getLogger(__name__)


# ===== 正则模式：从全文中提取结构化字段 =====
PATTERNS = {
    # 提取赔偿金金额，匹配"赔偿金XX元"或"支付XX元赔偿金"等
    "compensation": [
        r"赔偿金(?:共计|合计|为|人民币)?(\d[\d,，.万千百]+)元",
        r"支付(?:原告|劳动者|申请人)?(?:违法解除劳动合同)?赔偿金(?:人民币)?(\d[\d,，.万千百]+)元",
        r"二倍经济补偿(?:金)?(?:即赔偿金)?(?:人民币)?(\d[\d,，.万千百]+)元",
    ],
    # 提取工作年限
    "work_years": [
        r"工作(?:年限|时间|期间)(?:为|共|达)?(\d+(?:\.\d+)?)年",
        r"在(?:该公司|用人单位|被告处)?工作了?(\d+(?:\.\d+)?)年",
        r"自\d{4}年.*?至\d{4}年.*?共(\d+(?:\.\d+)?)年",
    ],
    # 提取月薪
    "monthly_salary": [
        r"月(?:平均)?工资(?:为|是|人民币)?(\d[\d,，.]+)元",
        r"每月(?:工资|薪资|薪酬)(?:为|是)?(\d[\d,，.]+)元",
        r"月薪(?:为|是|人民币)?(\d[\d,，.]+)元",
    ],
    # 提取原告/劳动者性别
    "gender": [
        r"原告(男|女)，",
        r"劳动者(男|女)，",
        r"申请人(男|女)，",
        r"(男|女)，\d{4}年\d{1,2}月\d{1,2}日出生",
        r"性别[:：](男|女)",
    ],
}


def random_delay():
    """随机等待，降低对服务器的压力"""
    delay = random.uniform(MIN_DELAY, MAX_DELAY)
    logger.debug(f"等待 {delay:.1f} 秒...")
    time.sleep(delay)


def parse_amount(raw: str) -> float:
    """
    将提取到的金额字符串转换为浮点数（单位：元）
    处理「万」「千」「百」和逗号分隔符
    """
    raw = raw.replace(",", "").replace("，", "").strip()
    if "万" in raw:
        parts = raw.split("万")
        result = float(parts[0]) * 10000
        if len(parts) > 1 and parts[1]:
            result += float(parts[1])
        return result
    return float(raw)


def extract_fields(full_text: str) -> dict:
    """
    从判决书全文中提取结构化字段
    返回字典，未找到的字段为 None
    """
    result = {
        "compensation": None,
        "work_years": None,
        "monthly_salary": None,
        "gender": None,
    }

    for field, patterns in PATTERNS.items():
        for pattern in patterns:
            match = re.search(pattern, full_text)
            if match:
                raw_value = match.group(1)
                try:
                    if field in ("compensation", "monthly_salary"):
                        result[field] = parse_amount(raw_value)
                    elif field == "work_years":
                        result[field] = float(raw_value)
                    else:
                        result[field] = raw_value  # 性别直接保留字符串
                except ValueError:
                    pass
                break  # 匹配到第一个模式后停止

    return result


async def scrape_wenshu(keyword: str, max_pages: int = 10) -> list[dict]:
    """
    使用 Playwright 爬取裁判文书网搜索结果
    返回案例列表，每个元素为提取的结构化字段
    """
    cases = []

    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=True,
            args=["--disable-blink-features=AutomationControlled"],
        )
        context = await browser.new_context(
            user_agent=(
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                "AppleWebKit/537.36 (KHTML, like Gecko) "
                "Chrome/120.0.0.0 Safari/537.36"
            ),
            locale="zh-CN",
        )
        page = await context.new_page()

        try:
            logger.info(f"正在搜索关键词：{keyword}")
            await page.goto(
                "https://wenshu.court.gov.cn/website/wenshu/181217BMTKHNT2W0/index.html",
                wait_until="networkidle",
                timeout=30000,
            )
            await asyncio.sleep(2)

            # 在搜索框输入关键词
            search_box = await page.wait_for_selector(
                "input.search-input, input[placeholder*='搜索']",
                timeout=10000,
            )
            await search_box.fill(keyword)
            await page.keyboard.press("Enter")
            await asyncio.sleep(3)

            # 添加筛选条件：案件类型=劳动争议
            try:
                await page.click("text=案件类型")
                await asyncio.sleep(1)
                await page.click("text=民事案件")
                await asyncio.sleep(1)
            except Exception:
                logger.warning("无法点击筛选条件，继续...")

            for page_num in range(1, max_pages + 1):
                logger.info(f"  正在处理第 {page_num} 页...")

                # 等待列表加载
                try:
                    await page.wait_for_selector(
                        ".result-item, .case-item, li.list-item",
                        timeout=15000,
                    )
                except PlaywrightTimeout:
                    logger.warning(f"  第 {page_num} 页加载超时，停止")
                    break

                # 获取所有案件链接
                items = await page.query_selector_all(
                    ".result-item a, .case-item a, li.list-item a"
                )
                case_links = []
                for item in items:
                    href = await item.get_attribute("href")
                    title = await item.inner_text()
                    if href and ("劳动" in title or "用工" in title):
                        case_links.append(href)

                logger.info(f"  找到 {len(case_links)} 个劳动相关案件")

                # 逐个进入案件详情页提取全文
                for link in case_links:
                    try:
                        detail_page = await context.new_page()
                        full_url = (
                            link if link.startswith("http")
                            else f"https://wenshu.court.gov.cn{link}"
                        )
                        await detail_page.goto(full_url, wait_until="networkidle", timeout=20000)
                        await asyncio.sleep(1)

                        # 提取全文文本
                        full_text = await detail_page.inner_text("body")
                        fields = extract_fields(full_text)

                        # 只保留字段较完整的案例
                        if (
                            fields["compensation"] is not None
                            and fields["gender"] is not None
                        ):
                            fields["source_url"] = full_url
                            fields["keyword"] = keyword
                            fields["scraped_at"] = datetime.now().isoformat()
                            cases.append(fields)
                            logger.info(
                                f"    提取成功：性别={fields['gender']}，"
                                f"赔偿金={fields['compensation']}元"
                            )

                        await detail_page.close()
                        random_delay()

                    except Exception as e:
                        logger.warning(f"    案件详情提取失败：{e}")
                        try:
                            await detail_page.close()
                        except Exception:
                            pass

                    if len(cases) >= MAX_CASES:
                        logger.info("已达到最大案例数限制，停止爬取")
                        return cases

                # 尝试翻到下一页
                try:
                    next_btn = await page.query_selector(
                        "button.next, a.next, [aria-label='下一页']"
                    )
                    if next_btn:
                        await next_btn.click()
                        await asyncio.sleep(3)
                    else:
                        logger.info("  没有下一页，当前关键词爬取结束")
                        break
                except Exception:
                    break

        except Exception as e:
            logger.error(f"爬取过程出错：{e}")
        finally:
            await browser.close()

    return cases


async def main():
    all_cases = []

    for keyword in SEARCH_KEYWORDS:
        cases = await scrape_wenshu(keyword, max_pages=MAX_PAGES)
        all_cases.extend(cases)
        logger.info(f"关键词「{keyword}」共采集 {len(cases)} 条案例")
        random_delay()

    if not all_cases:
        logger.warning("未采集到任何数据，请检查网络连接或网站是否可访问")
        return

    df = pd.DataFrame(all_cases)

    # 去重（按补偿金+性别+工作年限去重）
    df.drop_duplicates(
        subset=["compensation", "gender", "work_years"],
        keep="first",
        inplace=True,
    )

    raw_path = OUTPUT_DIR / "raw_judgments.csv"
    df.to_csv(raw_path, index=False, encoding="utf-8-sig")
    logger.info(f"原始数据已保存：{raw_path}，共 {len(df)} 条")

    # 简单清洗：过滤异常值
    df_clean = df.dropna(subset=["compensation", "gender"])
    df_clean = df_clean[
        (df_clean["compensation"] > 0) &
        (df_clean["compensation"] < 5_000_000)  # 过滤明显异常的超大金额
    ]
    df_clean["gender_code"] = df_clean["gender"].map({"男": 0, "女": 1})

    clean_path = OUTPUT_DIR / "cleaned_judgments.csv"
    df_clean.to_csv(clean_path, index=False, encoding="utf-8-sig")
    logger.info(f"清洗后数据已保存：{clean_path}，共 {len(df_clean)} 条")
    logger.info(f"  男性案例：{(df_clean['gender'] == '男').sum()} 条")
    logger.info(f"  女性案例：{(df_clean['gender'] == '女').sum()} 条")


if __name__ == "__main__":
    asyncio.run(main())
