ogx786 commited on
Commit
2a86916
·
verified ·
1 Parent(s): d2eaf99

Create clean_iamhblchunks.py

Browse files
Files changed (1) hide show
  1. clean_iamhblchunks.py +160 -0
clean_iamhblchunks.py ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import pickle
3
+ import hashlib
4
+ from pathlib import Path
5
+ from collections import Counter
6
+
7
+ INPUT_FILE = "data/iamhbl_chunks.json"
8
+ OUTPUT_FILE = "output/iamhbl_clean_chunks.pkl"
9
+
10
+ print("Loading chunks...")
11
+ with open(INPUT_FILE, "r", encoding="utf-8") as f:
12
+ raw_chunks = json.load(f)
13
+
14
+ print("Original chunks:", len(raw_chunks))
15
+
16
+ TYPE_MAP = {
17
+ "general": "general",
18
+ "policy": "management",
19
+ "circular": "management",
20
+ "rate_sheet": "charges",
21
+ "charges": "charges",
22
+ "form": "form",
23
+ "kfs": "kfs",
24
+ "product": "product"
25
+ }
26
+
27
+ def normalize_chunk(c, idx):
28
+ meta = c.get("metadata", {})
29
+ dtype = meta.get("document_type", "general").lower()
30
+
31
+ new_meta = {
32
+ **meta,
33
+ "document_type": TYPE_MAP.get(dtype, "general"),
34
+ "source_type": "pdf",
35
+ "corpus": "internal_iamhbl",
36
+ "access_level": "internal"
37
+ }
38
+
39
+ return {
40
+ "id": f"iamhbl_{idx}",
41
+ "title": meta.get("filename", ""),
42
+ "text": c.get("text", ""),
43
+ "source": meta.get("filename", ""),
44
+ "source_type": "pdf",
45
+ "metadata": new_meta
46
+ }
47
+
48
+ chunks = [
49
+ normalize_chunk(c, i)
50
+ for i, c in enumerate(raw_chunks)
51
+ ]
52
+
53
+ print("Normalized:", len(chunks))
54
+
55
+ def clean_text(text):
56
+ if not text:
57
+ return ""
58
+ return " ".join(
59
+ text.replace("\x00", " ")
60
+ .replace("\ufeff", " ")
61
+ .split()
62
+ )
63
+
64
+ for c in chunks:
65
+ c["text"] = clean_text(c["text"])
66
+
67
+ BAD_PATTERNS = [
68
+ "cookie statement",
69
+ "enable javascript",
70
+ "javascript libraries",
71
+ "privacy policy",
72
+ "accept cookies",
73
+ "all rights reserved"
74
+ ]
75
+
76
+ def valid_chunk(c):
77
+ text = c["text"].lower()
78
+
79
+ if len(text) < 100:
80
+ return False
81
+
82
+ hits = sum(
83
+ x in text
84
+ for x in BAD_PATTERNS
85
+ )
86
+
87
+ if hits >= 2:
88
+ return False
89
+
90
+ return True
91
+
92
+ before = len(chunks)
93
+
94
+ chunks = [
95
+ c for c in chunks
96
+ if valid_chunk(c)
97
+ ]
98
+
99
+ print("Noise removed:", before - len(chunks))
100
+
101
+ before = len(chunks)
102
+
103
+ chunks = [
104
+ c for c in chunks
105
+ if not c["metadata"].get("is_form", False)
106
+ ]
107
+
108
+ print("Forms removed:", before - len(chunks))
109
+
110
+ def text_hash(text):
111
+ return hashlib.md5(
112
+ text.lower()
113
+ .strip()
114
+ .encode("utf-8")
115
+ ).hexdigest()
116
+
117
+ seen = set()
118
+ unique = []
119
+
120
+ for c in chunks:
121
+ h = text_hash(c["text"])
122
+
123
+ if h not in seen:
124
+ seen.add(h)
125
+ c["metadata"]["chunk_hash"] = h
126
+ unique.append(c)
127
+
128
+ print("Duplicates removed:", len(chunks) - len(unique))
129
+
130
+ chunks = unique
131
+
132
+ print("\nFinal chunks:", len(chunks))
133
+
134
+ print(
135
+ Counter(
136
+ c["metadata"]["document_type"]
137
+ for c in chunks
138
+ )
139
+ )
140
+
141
+ Path("output").mkdir(exist_ok=True)
142
+
143
+ with open(OUTPUT_FILE, "wb") as f:
144
+ pickle.dump(chunks, f)
145
+
146
+ print("\nSaved:", OUTPUT_FILE)
147
+
148
+ print("\n========== SAMPLE ==========")
149
+
150
+ sample = chunks[0]
151
+
152
+ print("ID:", sample["id"])
153
+ print("TITLE:", sample["title"])
154
+ print("SOURCE:", sample["source"])
155
+
156
+ print("\nMETADATA:")
157
+ print(sample["metadata"])
158
+
159
+ print("\nTEXT:")
160
+ print(sample["text"][:1500])