File size: 4,339 Bytes
0ffdbad | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | import pandas as pd
import os
import glob
import json
import random
import re
from tqdm import tqdm
try:
from opencc import OpenCC
cc = OpenCC('s2t') # 簡體轉繁體
except ImportError:
cc = None
# =================配置區=================
DATA_DIR = "raw_data"
OUTPUT_FILE = "data/v16_training_data.txt"
MIN_LINE_LEN = 15
MAX_LINE_LEN = 3000
DEDUPLICATE = True
# ========================================
def clean_text(text):
if not isinstance(text, str) or not text.strip():
return ""
if cc:
text = cc.convert(text)
text = re.sub(r'\s+', ' ', text)
text = text.strip()
return text
def process_factory_v16():
all_lines = set() if DEDUPLICATE else []
files = glob.glob(os.path.join(DATA_DIR, "*.parquet")) + \
glob.glob(os.path.join(DATA_DIR, "*.jsonl")) + \
glob.glob(os.path.join(DATA_DIR, "*.txt"))
print(f"🏭 數據工廠 V16.2 啟動!偵測到 {len(files)} 個檔案")
total_extracted = 0
for file_path in files:
file_name = os.path.basename(file_path)
print(f"📦 處理中: {file_name}")
file_count = 0
# --- 1. 處理 Parquet (針對 Wikipedia 優化) ---
if file_path.endswith('.parquet'):
df = pd.read_parquet(file_path)
# 優先級:markdown > text > html
col = 'markdown' if 'markdown' in df.columns else \
('text' if 'text' in df.columns else \
('html' if 'html' in df.columns else df.columns[0]))
print(f" 🎯 使用欄位: [{col}]")
for full_content in df[col].dropna().astype(str):
paragraphs = full_content.split('\n')
for p in paragraphs:
p = re.sub(r'^#+\s+', '', p) # 移除 Markdown 標題符號
cleaned = clean_text(p)
if MIN_LINE_LEN < len(cleaned) < MAX_LINE_LEN:
if DEDUPLICATE: all_lines.add(cleaned)
else: all_lines.append(cleaned)
file_count += 1
# --- 2. 處理 JSONL (針對 CSC 或其他數據) ---
elif file_path.endswith('.jsonl'):
with open(file_path, 'r', encoding='utf-8') as f:
for line in f:
try:
data = json.loads(line)
content = data.get('text') or data.get('content') or data.get('target')
if content:
cleaned = clean_text(content)
if MIN_LINE_LEN < len(cleaned) < MAX_LINE_LEN:
if DEDUPLICATE: all_lines.add(cleaned)
else: all_lines.append(cleaned)
file_count += 1
except: continue
# --- 3. 處理普通 TXT (小說、代碼等) ---
elif file_path.endswith('.txt'):
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
content = f.read()
# 這裡改用單換行切割,增加段落密度
paragraphs = content.split('\n')
for p in paragraphs:
cleaned = clean_text(p)
if MIN_LINE_LEN < len(cleaned) < MAX_LINE_LEN:
if DEDUPLICATE: all_lines.add(cleaned)
else: all_lines.append(cleaned)
file_count += 1
print(f"✅ {file_name} 完成,提取了 {file_count} 條優質段落")
total_extracted += file_count
# --- 結束迴圈後的處理 ---
final_data = list(all_lines)
print(f"🌀 正在隨機洗牌(Shuffle)... 最終不重複條目: {len(final_data)}")
random.shuffle(final_data)
print(f"💾 正在寫入: {OUTPUT_FILE}")
os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
for line in tqdm(final_data):
f.write(line + "\n\n")
print(f"🎉 任務完成!V16 訓練資料已就緒。")
if __name__ == "__main__":
process_factory_v16() |