import requests
from bs4 import BeautifulSoup
import json
import os
import time
import re

def crawl_criminal_law():
    # 基础URL
    base_url = "http://xingfa.org/"
    
    # 创建一个会话以保持连接
    session = requests.Session()
    
    # 添加请求头，模拟浏览器访问
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        'Connection': 'keep-alive',
    }
    
    # 获取主页内容
    response = session.get(base_url, headers=headers)
    response.encoding = 'utf-8'  # 确保正确编码中文
    
    if response.status_code != 200:
        print(f"请求失败，状态码: {response.status_code}")
        return []
    
    # 解析HTML内容
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # 找到所有法条内容
    law_articles = []
    
    # 根据网站结构找到包含法条的元素
    # 注意：这里的选择器需要根据实际网站结构调整
    article_elements = soup.select('.article-content p') or soup.select('.content p')
    
    current_article_id = None
    current_article_text = ""
    
    # 正则表达式匹配法条格式
    article_pattern = re.compile(r'^第([一二三四五六七八九十百]+|\d+)条(\s*之\s*[一二三四五六七八九十]+)?')
    
    for element in article_elements:
        text = element.get_text(strip=True)
        if not text:
            continue
            
        # 检查是否是新的条款
        match = article_pattern.match(text)
        if match:
            # 如果已经有正在处理的条款，保存它
            if current_article_id is not None:
                law_articles.append({
                    "id": current_article_id,
                    "fact": current_article_text
                })
            
            # 提取条款编号
            article_num = match.group(1)
            special_part = match.group(2) if match.group(2) else ""
            
            # 将中文数字转换为阿拉伯数字
            if not article_num.isdigit():
                article_num = convert_chinese_to_number(article_num)
                
            current_article_id = f"{article_num}{special_part}"
            current_article_text = text
        else:
            # 如果不是新条款，追加到当前条款内容
            if current_article_id is not None:
                current_article_text += " " + text
    
    # 添加最后一个条款
    if current_article_id is not None:
        law_articles.append({
            "id": current_article_id,
            "fact": current_article_text
        })
    
    return law_articles

def convert_chinese_to_number(chinese):
    """将中文数字转换为阿拉伯数字"""
    chinese_number_map = {
        '一': 1, '二': 2, '三': 3, '四': 4, '五': 5,
        '六': 6, '七': 7, '八': 8, '九': 9, '十': 10,
        '百': 100
    }
    
    result = 0
    temp = 0
    
    for char in chinese:
        if char in chinese_number_map:
            value = chinese_number_map[char]
            if value >= 10:  # 十或百
                if temp == 0:  # 如"十"而非"一十"
                    temp = 1
                result += temp * value
                temp = 0
            else:  # 个位数
                temp = value
        
    # 加上最后的个位数
    if temp > 0:
        result += temp
        
    return str(result)

def save_law_articles(articles, output_folder):
    """将法条保存为单独的JSON文件"""
    os.makedirs(output_folder, exist_ok=True)
    
    for article in articles:
        # 处理特殊编号格式，确保文件名有效
        file_id = article["id"].replace(" ", "").replace("之", "_")
        
        output_file = os.path.join(output_folder, f"{file_id}.json")
        with open(output_file, 'w', encoding='utf-8') as f:
            json.dump(article, f, ensure_ascii=False, indent=2)
    
    print(f"共保存了 {len(articles)} 条法条到 {output_folder}")

if __name__ == "__main__":
    # 爬取法条
    print("开始爬取刑法条款...")
    law_articles = crawl_criminal_law()
    
    if law_articles:
        print(f"成功爬取了 {len(law_articles)} 条法条")
        
        # 保存法条
        output_folder = "/data3/xubuqiang/candidate_law_articles"
        save_law_articles(law_articles, output_folder)
        
        # 输出几个示例
        for i in range(min(5, len(law_articles))):
            print(f"条款 {law_articles[i]['id']}: {law_articles[i]['fact'][:50]}...")
    else:
        print("未能爬取到法条内容，请检查网站结构或网络连接")