fdbw commited on
Commit
a7a37f8
·
verified ·
1 Parent(s): f6e00e7

Create prepare_data.py

Browse files
Files changed (1) hide show
  1. prepare_data.py +60 -0
prepare_data.py ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # prepare_data.py
2
+ import re
3
+ import json
4
+ from config import DATA_FILE, TRAIN_DATA_FILE
5
+
6
+ def extract_entities_and_relations(text):
7
+ """
8
+ 解析《冯氏家谱.txt》中的典型句式,生成问答对。
9
+ 支持格式示例:
10
+ - 冯大山(1890-1965)生于山东济南。
11
+ - 冯二河是冯大山之子。
12
+ - 冯秀英是冯大山之女。
13
+ - 冯三江配李氏。
14
+ - 冯四海与冯五湖为兄弟。
15
+ """
16
+ lines = [line.strip() for line in text.split('\n') if line.strip() and not line.startswith('#')]
17
+ qa_pairs = []
18
+
19
+ for line in lines:
20
+ # 匹配:姓名(生卒年)生于地点
21
+ match_birth = re.search(r'([\u4e00-\u9fa5]+)(?:(|\()(\d{4})[^\d]*\d{0,4}(?:)|\))生于([\u4e00-\u9fa5]+)', line)
22
+ if match_birth:
23
+ name, birth_year, place = match_birth.groups()
24
+ qa_pairs.append({"instruction": f"{name}出生于哪一年?", "output": birth_year})
25
+ qa_pairs.append({"instruction": f"{name}的出生地是哪里?", "output": place})
26
+
27
+ # 匹配:A是B之子/女
28
+ match_parent = re.search(r'([\u4e00-\u9fa5]+)是([\u4e00-\u9fa5]+)之(子|女)', line)
29
+ if match_parent:
30
+ child, parent, rel = match_parent.groups()
31
+ rel_word = "儿子" if rel == "子" else "女儿"
32
+ qa_pairs.append({"instruction": f"{child}的父亲或母亲是谁?", "output": parent})
33
+ qa_pairs.append({"instruction": f"{parent}的{rel_word}是谁?", "output": child})
34
+
35
+ # 匹配:A配B氏(配偶)
36
+ match_spouse = re.search(r'([\u4e00-\u9fa5]+)配([\u4e00-\u9fa5]+)氏', line)
37
+ if match_spouse:
38
+ person, spouse_surname = match_spouse.groups()
39
+ qa_pairs.append({"instruction": f"{person}的妻子姓什么?", "output": spouse_surname})
40
+ qa_pairs.append({"instruction": f"{person}的配偶是谁?", "output": f"{spouse_surname}氏"})
41
+
42
+ # 匹配:A与B为兄弟/姐妹/兄妹
43
+ match_sibling = re.search(r'([\u4e00-\u9fa5]+)与([\u4e00-\u9fa5]+)为(兄弟|姐妹|兄妹)', line)
44
+ if match_sibling:
45
+ p1, p2, rel_type = match_sibling.groups()
46
+ qa_pairs.append({"instruction": f"{p1}的{rel_type}是谁?", "output": p2})
47
+ qa_pairs.append({"instruction": f"{p2}的{rel_type}是谁?", "output": p1})
48
+
49
+ return qa_pairs
50
+
51
+ def main():
52
+ with open(DATA_FILE, 'r', encoding='utf-8') as f:
53
+ content = f.read()
54
+ qa_data = extract_entities_and_relations(content)
55
+ with open(TRAIN_DATA_FILE, 'w', encoding='utf-8') as f:
56
+ json.dump(qa_data, f, ensure_ascii=False, indent=2)
57
+ print(f"✅ 成功从《{DATA_FILE}》提取 {len(qa_data)} 条训练样本,保存至 {TRAIN_DATA_FILE}")
58
+
59
+ if __name__ == "__main__":
60
+ main()