law / get_law_article_data.py

Upload folder using huggingface_hub

3c35d3b verified over 1 year ago

4.87 kB

import requests
from bs4 import BeautifulSoup
import json
import os
import time
import re

def crawl_criminal_law():
    # 基础URL
    base_url = "http://xingfa.org/"
    
    # 创建一个会话以保持连接
    session = requests.Session()
    
    # 添加请求头，模拟浏览器访问
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        'Connection': 'keep-alive',
    }
    
    # 获取主页内容
    response = session.get(base_url, headers=headers)
    response.encoding = 'utf-8'  # 确保正确编码中文
    
    if response.status_code != 200:
        print(f"请求失败，状态码: {response.status_code}")
        return []
    
    # 解析HTML内容
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # 找到所有法条内容
    law_articles = []
    
    # 根据网站结构找到包含法条的元素
    # 注意：这里的选择器需要根据实际网站结构调整
    article_elements = soup.select('.article-content p') or soup.select('.content p')
    
    current_article_id = None
    current_article_text = ""
    
    # 正则表达式匹配法条格式
    article_pattern = re.compile(r'^第([一二三四五六七八九十百]+|\d+)条(\s*之\s*[一二三四五六七八九十]+)?')
    
    for element in article_elements:
        text = element.get_text(strip=True)
        if not text:
            continue
            
        # 检查是否是新的条款
        match = article_pattern.match(text)
        if match:
            # 如果已经有正在处理的条款，保存它
            if current_article_id is not None:
                law_articles.append({
                    "id": current_article_id,
                    "fact": current_article_text
                })
            
            # 提取条款编号
            article_num = match.group(1)
            special_part = match.group(2) if match.group(2) else ""
            
            # 将中文数字转换为阿拉伯数字
            if not article_num.isdigit():
                article_num = convert_chinese_to_number(article_num)
                
            current_article_id = f"{article_num}{special_part}"
            current_article_text = text
        else:
            # 如果不是新条款，追加到当前条款内容
            if current_article_id is not None:
                current_article_text += " " + text
    
    # 添加最后一个条款
    if current_article_id is not None:
        law_articles.append({
            "id": current_article_id,
            "fact": current_article_text
        })
    
    return law_articles

def convert_chinese_to_number(chinese):
    """将中文数字转换为阿拉伯数字"""
    chinese_number_map = {
        '一': 1, '二': 2, '三': 3, '四': 4, '五': 5,
        '六': 6, '七': 7, '八': 8, '九': 9, '十': 10,
        '百': 100
    }
    
    result = 0
    temp = 0
    
    for char in chinese:
        if char in chinese_number_map:
            value = chinese_number_map[char]
            if value >= 10:  # 十或百
                if temp == 0:  # 如"十"而非"一十"
                    temp = 1
                result += temp * value
                temp = 0
            else:  # 个位数
                temp = value
        
    # 加上最后的个位数
    if temp > 0:
        result += temp
        
    return str(result)

def save_law_articles(articles, output_folder):
    """将法条保存为单独的JSON文件"""
    os.makedirs(output_folder, exist_ok=True)
    
    for article in articles:
        # 处理特殊编号格式，确保文件名有效
        file_id = article["id"].replace(" ", "").replace("之", "_")
        
        output_file = os.path.join(output_folder, f"{file_id}.json")
        with open(output_file, 'w', encoding='utf-8') as f:
            json.dump(article, f, ensure_ascii=False, indent=2)
    
    print(f"共保存了 {len(articles)} 条法条到 {output_folder}")

if __name__ == "__main__":
    # 爬取法条
    print("开始爬取刑法条款...")
    law_articles = crawl_criminal_law()
    
    if law_articles:
        print(f"成功爬取了 {len(law_articles)} 条法条")
        
        # 保存法条
        output_folder = "/data3/xubuqiang/candidate_law_articles"
        save_law_articles(law_articles, output_folder)
        
        # 输出几个示例
        for i in range(min(5, len(law_articles))):
            print(f"条款 {law_articles[i]['id']}: {law_articles[i]['fact'][:50]}...")
    else:
        print("未能爬取到法条内容，请检查网站结构或网络连接")

	import requests
	from bs4 import BeautifulSoup
	import json
	import os
	import time
	import re

	def crawl_criminal_law():
	# 基础URL
	base_url = "http://xingfa.org/"

	# 创建一个会话以保持连接
	session = requests.Session()

	# 添加请求头，模拟浏览器访问
	headers = {
	'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
	'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,/;q=0.8',
	'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
	'Connection': 'keep-alive',
	}

	# 获取主页内容
	response = session.get(base_url, headers=headers)
	response.encoding = 'utf-8' # 确保正确编码中文

	if response.status_code != 200:
	print(f"请求失败，状态码: {response.status_code}")
	return []

	# 解析HTML内容
	soup = BeautifulSoup(response.text, 'html.parser')

	# 找到所有法条内容
	law_articles = []

	# 根据网站结构找到包含法条的元素
	# 注意：这里的选择器需要根据实际网站结构调整
	article_elements = soup.select('.article-content p') or soup.select('.content p')

	current_article_id = None
	current_article_text = ""

	# 正则表达式匹配法条格式
	article_pattern = re.compile(r'^第([一二三四五六七八九十百]+\|\d+)条(\s之\s[一二三四五六七八九十]+)?')

	for element in article_elements:
	text = element.get_text(strip=True)
	if not text:
	continue

	# 检查是否是新的条款
	match = article_pattern.match(text)
	if match:
	# 如果已经有正在处理的条款，保存它
	if current_article_id is not None:
	law_articles.append({
	"id": current_article_id,
	"fact": current_article_text
	})

	# 提取条款编号
	article_num = match.group(1)
	special_part = match.group(2) if match.group(2) else ""

	# 将中文数字转换为阿拉伯数字
	if not article_num.isdigit():
	article_num = convert_chinese_to_number(article_num)

	current_article_id = f"{article_num}{special_part}"
	current_article_text = text
	else:
	# 如果不是新条款，追加到当前条款内容
	if current_article_id is not None:
	current_article_text += " " + text

	# 添加最后一个条款
	if current_article_id is not None:
	law_articles.append({
	"id": current_article_id,
	"fact": current_article_text
	})

	return law_articles

	def convert_chinese_to_number(chinese):
	"""将中文数字转换为阿拉伯数字"""
	chinese_number_map = {
	'一': 1, '二': 2, '三': 3, '四': 4, '五': 5,
	'六': 6, '七': 7, '八': 8, '九': 9, '十': 10,
	'百': 100
	}

	result = 0
	temp = 0

	for char in chinese:
	if char in chinese_number_map:
	value = chinese_number_map[char]
	if value >= 10: # 十或百
	if temp == 0: # 如"十"而非"一十"
	temp = 1
	result += temp * value
	temp = 0
	else: # 个位数
	temp = value

	# 加上最后的个位数
	if temp > 0:
	result += temp

	return str(result)

	def save_law_articles(articles, output_folder):
	"""将法条保存为单独的JSON文件"""
	os.makedirs(output_folder, exist_ok=True)

	for article in articles:
	# 处理特殊编号格式，确保文件名有效
	file_id = article["id"].replace(" ", "").replace("之", "_")

	output_file = os.path.join(output_folder, f"{file_id}.json")
	with open(output_file, 'w', encoding='utf-8') as f:
	json.dump(article, f, ensure_ascii=False, indent=2)

	print(f"共保存了 {len(articles)} 条法条到 {output_folder}")

	if __name__ == "__main__":
	# 爬取法条
	print("开始爬取刑法条款...")
	law_articles = crawl_criminal_law()

	if law_articles:
	print(f"成功爬取了 {len(law_articles)} 条法条")

	# 保存法条
	output_folder = "/data3/xubuqiang/candidate_law_articles"
	save_law_articles(law_articles, output_folder)

	# 输出几个示例
	for i in range(min(5, len(law_articles))):
	print(f"条款 {law_articles[i]['id']}: {law_articles[i]['fact'][:50]}...")
	else:
	print("未能爬取到法条内容，请检查网站结构或网络连接")