File size: 4,339 Bytes
0ffdbad
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
import pandas as pd
import os
import glob
import json
import random
import re
from tqdm import tqdm

try:
    from opencc import OpenCC
    cc = OpenCC('s2t') # 簡體轉繁體
except ImportError:
    cc = None

# =================配置區=================
DATA_DIR = "raw_data"      
OUTPUT_FILE = "data/v16_training_data.txt"
MIN_LINE_LEN = 15          
MAX_LINE_LEN = 3000        
DEDUPLICATE = True         
# ========================================

def clean_text(text):
    if not isinstance(text, str) or not text.strip(): 
        return ""
    if cc: 
        text = cc.convert(text)
    text = re.sub(r'\s+', ' ', text)
    text = text.strip()
    return text

def process_factory_v16():
    all_lines = set() if DEDUPLICATE else []
    files = glob.glob(os.path.join(DATA_DIR, "*.parquet")) + \
            glob.glob(os.path.join(DATA_DIR, "*.jsonl")) + \
            glob.glob(os.path.join(DATA_DIR, "*.txt"))
    
    print(f"🏭 數據工廠 V16.2 啟動!偵測到 {len(files)} 個檔案")
    total_extracted = 0
    
    for file_path in files:
        file_name = os.path.basename(file_path)
        print(f"📦 處理中: {file_name}")
        file_count = 0
        
        # --- 1. 處理 Parquet (針對 Wikipedia 優化) ---
        if file_path.endswith('.parquet'):
            df = pd.read_parquet(file_path)
            # 優先級:markdown > text > html
            col = 'markdown' if 'markdown' in df.columns else \
                  ('text' if 'text' in df.columns else \
                  ('html' if 'html' in df.columns else df.columns[0]))
            
            print(f"  🎯 使用欄位: [{col}]")
            for full_content in df[col].dropna().astype(str):
                paragraphs = full_content.split('\n')
                for p in paragraphs:
                    p = re.sub(r'^#+\s+', '', p) # 移除 Markdown 標題符號
                    cleaned = clean_text(p)
                    if MIN_LINE_LEN < len(cleaned) < MAX_LINE_LEN:
                        if DEDUPLICATE: all_lines.add(cleaned)
                        else: all_lines.append(cleaned)
                        file_count += 1

        # --- 2. 處理 JSONL (針對 CSC 或其他數據) ---
        elif file_path.endswith('.jsonl'):
            with open(file_path, 'r', encoding='utf-8') as f:
                for line in f:
                    try:
                        data = json.loads(line)
                        content = data.get('text') or data.get('content') or data.get('target')
                        if content:
                            cleaned = clean_text(content)
                            if MIN_LINE_LEN < len(cleaned) < MAX_LINE_LEN:
                                if DEDUPLICATE: all_lines.add(cleaned)
                                else: all_lines.append(cleaned)
                                file_count += 1
                    except: continue

        # --- 3. 處理普通 TXT (小說、代碼等) ---
        elif file_path.endswith('.txt'):
            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
                content = f.read()
                # 這裡改用單換行切割,增加段落密度
                paragraphs = content.split('\n')
                for p in paragraphs:
                    cleaned = clean_text(p)
                    if MIN_LINE_LEN < len(cleaned) < MAX_LINE_LEN:
                        if DEDUPLICATE: all_lines.add(cleaned)
                        else: all_lines.append(cleaned)
                        file_count += 1
        
        print(f"✅ {file_name} 完成,提取了 {file_count} 條優質段落")
        total_extracted += file_count

    # --- 結束迴圈後的處理 ---
    final_data = list(all_lines)
    print(f"🌀 正在隨機洗牌(Shuffle)... 最終不重複條目: {len(final_data)}")
    random.shuffle(final_data)

    print(f"💾 正在寫入: {OUTPUT_FILE}")
    os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
    with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
        for line in tqdm(final_data):
            f.write(line + "\n\n") 

    print(f"🎉 任務完成!V16 訓練資料已就緒。")

if __name__ == "__main__":
    process_factory_v16()