Resonance-Bottleneck-LLM / prepare_data.py
ddmmbb's picture
Upload 5 files
0ffdbad verified
Raw History Blame Contribute Delete
2.4 kB
import os
import glob
import numpy as np
import re # 🆕 引入正則表達式
from tqdm import tqdm
from tokenizers import Tokenizer
# 配置
data_dir = "data"
output_file = "corpus_v15_clean.bin" # 🆕 改個名字,保留舊的對比
vocab_file = "bpe_tokenizer_v12.json"
tokenizer = Tokenizer.from_file(vocab_file)
txt_files = glob.glob(os.path.join(data_dir, "*.txt"))
# 🆕 資料清洗函數:移除模型不需要的雜訊
def clean_text(text):
# 1. 移除維基百科的連結與檔案格式 (例如 [[File:xxx]], [[Category:xxx]])
text = re.sub(r'\[\[(?:File|Category|Image|Special|User):.*?\]\]', '', text)
# 2. 移除過長的亂碼字串 (例如長度超過 10 的純字母數字組合,通常是 ID)
text = re.sub(r'\b[A-Za-z0-9]{10,}\b', '', text)
# 3. 移除重複的符號 (例如大量的 ......... 或 --------)
text = re.sub(r'[\.\-\=\_\*]{4,}', ' ', text)
# 4. 移除常見的垃圾 HTML 標籤
text = re.sub(r'<.*?>', '', text)
# 5. 只保留合理的內容:過短的行(例如只有 1-2 個字)通常是噪聲
lines = text.split('\n')
filtered_lines = [line.strip() for line in lines if len(line.strip()) > 5]
return "\n".join(filtered_lines)
print(f"🚀 開始「精煉」處理 {len(txt_files)} 個文件...")
with open(output_file, "ab") as bin_file:
for fpath in txt_files:
fname = os.path.basename(fpath)
print(f"正在清洗並轉換: {fname}")
with open(fpath, 'r', encoding='utf-8', errors='ignore') as f:
chunk_text = ""
for i, line in enumerate(f):
chunk_text += line
if i % 10000 == 0 and i > 0:
# 🆕 在編碼前先執行清洗
refined_text = clean_text(chunk_text)
tokens = tokenizer.encode(refined_text).ids
if tokens:
np.array(tokens, dtype=np.uint16).tofile(bin_file)
chunk_text = ""
if chunk_text:
refined_text = clean_text(chunk_text)
tokens = tokenizer.encode(refined_text).ids
if tokens:
np.array(tokens, dtype=np.uint16).tofile(bin_file)
print(f"✅ 精煉完成!高品質資料已存至: {output_file}")