ogx786 commited on
Commit
d7b8c4f
·
verified ·
1 Parent(s): 1101ce2

Update error

Browse files
Files changed (1) hide show
  1. error +10 -185
error CHANGED
@@ -1,185 +1,10 @@
1
- import pandas as pd
2
- import random
3
- import csv
4
-
5
-
6
- # ==============================
7
- # FILE PATHS
8
- # ==============================
9
-
10
- NORMAL_FILE = "urdu_roman_137k.csv"
11
- PIPE_FILE = "pipe_pairs.csv"
12
-
13
- OUTPUT_FILE = "hbl_v4_training.csv"
14
-
15
-
16
- # ==============================
17
- # LOAD NORMAL DATA
18
- # ==============================
19
-
20
- print("Loading normal dataset...")
21
-
22
- normal_df = pd.read_csv(
23
- NORMAL_FILE,
24
- encoding="utf-8"
25
- )
26
-
27
- # Ensure correct column names
28
- normal_df.columns = [
29
- "urdu",
30
- "roman"
31
- ]
32
-
33
- print(
34
- "Normal rows:",
35
- len(normal_df)
36
- )
37
-
38
-
39
- # ==============================
40
- # LOAD PIPE DATA
41
- # ==============================
42
-
43
- print("Loading pipe dataset...")
44
-
45
-
46
- pipe_rows = []
47
-
48
- with open(
49
- PIPE_FILE,
50
- "r",
51
- encoding="utf-8"
52
- ) as f:
53
-
54
- for line in f:
55
-
56
- line = line.strip()
57
-
58
- # Skip empty lines
59
- if not line:
60
- continue
61
-
62
- pipe_rows.append(line)
63
-
64
-
65
-
66
- pipe_data = []
67
-
68
- for row in pipe_rows:
69
-
70
- # CSV parser handles commas inside quotes
71
- values = next(
72
- csv.reader([row])
73
- )
74
-
75
- if len(values) != 2:
76
- continue
77
-
78
- pipe_data.append(
79
- {
80
- "urdu": values[0],
81
- "roman": values[1]
82
- }
83
- )
84
-
85
-
86
- pipe_df = pd.DataFrame(pipe_data)
87
-
88
-
89
- print(
90
- "Pipe rows:",
91
- len(pipe_df)
92
- )
93
-
94
-
95
-
96
- # ==============================
97
- # CLEAN ROMAN TEXT
98
- # ==============================
99
-
100
- def clean_roman(text):
101
-
102
- if pd.isna(text):
103
- return ""
104
-
105
- text = str(text)
106
-
107
- # Normalize spaces
108
- text = " ".join(
109
- text.split()
110
- )
111
-
112
- # Uppercase
113
- text = text.upper()
114
-
115
- return text
116
-
117
-
118
-
119
- normal_df["roman"] = normal_df["roman"].apply(
120
- clean_roman
121
- )
122
-
123
- pipe_df["roman"] = pipe_df["roman"].apply(
124
- clean_roman
125
- )
126
-
127
-
128
-
129
- # ==============================
130
- # COMBINE
131
- # ==============================
132
-
133
- print("Combining datasets...")
134
-
135
- combined = pd.concat(
136
- [
137
- normal_df,
138
- pipe_df
139
- ],
140
- ignore_index=True
141
- )
142
-
143
-
144
-
145
- # Remove empty rows
146
-
147
- combined = combined.dropna()
148
-
149
- combined = combined[
150
- (combined["urdu"].str.strip() != "") &
151
- (combined["roman"].str.strip() != "")
152
- ]
153
-
154
-
155
-
156
- # ==============================
157
- # SHUFFLE
158
- # ==============================
159
-
160
- combined = combined.sample(
161
- frac=1,
162
- random_state=42
163
- ).reset_index(
164
- drop=True
165
- )
166
-
167
-
168
-
169
- # ==============================
170
- # SAVE
171
- # ==============================
172
-
173
- combined.to_csv(
174
- OUTPUT_FILE,
175
- index=False,
176
- encoding="utf-8-sig",
177
- quoting=csv.QUOTE_ALL
178
- )
179
-
180
-
181
- print("="*50)
182
- print("DONE")
183
- print("Total rows:", len(combined))
184
- print("Saved:", OUTPUT_FILE)
185
- print("="*50)
 
1
+ "urdu","roman"
2
+ "نور گل","NOOR GUL"
3
+ "عامرحیات خان|محمدامان اللہ خان|محمدامان اللہ خان|مکان نمبر 10، سٹریٹ نمبر 32/1،سیکٹر F-8/1،اسلام آباد|مکان نمبر E/A 334،محلہ سول لائن،میانوالی|میانوالی,میانوالی","AMIR HAYAT KHAN|MUHAMMAD AMAAN ULLAH KHAN|MUHAMMAD AMAAN ULLAH KHAN|HOUSE NUMBER 10 GALI NUMBER 32/1 SECTOR F-8/1 ISLAMABAD|HOUSE NUMBER E/A 334 MOHALA CIVIL LINE MIANWALI|MIANWALI MIANWALI"
4
+ "باسط سعید|محمد سعید|محمد سعید|مکان نمبر 9/56،رنگ پورہ روڈ،محلہ اسلام پورہ،سیالکوٹ|مکان نمبر 9/56،رنگ پورہ روڈ،محلہ اسلام پورہ،سیالکوٹ|سیالکوٹ,سیالکوٹ","BASIT SAEED|MUHAMMAD SYED|MUHAMMAD SYED|HOUSE NUMBER 9/56 RANG PURA ROAD MOHALA ISLAM PURA SIALKOT|HOUSE NUMBER 9/56 RANG PURA ROAD MOHALA ISLAM PURA SIALKOT|SIALKOT SIALKOT"
5
+ "مکان نمبر 9-A،محلہ جوہر سٹریٹ نیو کیمپئس روڈ،لاہور","HOUSE NUMBER 9-A MOHALA JOHAR GALI NEW KEEMPEES ROAD LAHORE"
6
+ "سٹریٹ نمبر 6،محلہ جناح کالونی،جلال پور جٹاں،تحصیل/ضلع گجرات","GALI NUMBER 6 MOHALA JINNAH COLONY JALAL PUR JATAN TEHSIL / DISTRICT GUJRAT"
7
+ "ماہ نور|محمد یعقوب|محمد یعقوب|ڈاک خانہ خاص،چٹی شیخاں،تحصیل سیالکوٹ،ضلع سیالکوٹ|ڈاک خانہ خاص،چٹی شیخاں،تحصیل سیالکوٹ،ضلع سیالکوٹ|سیالکوٹ,سیالکوٹ","MAH NOOR|MUHAMMAD YAQOOB|MUHAMMAD YAQOOB|POST OFFICE KHAS CHITI SHEIKHAAN TEHSIL SIALKOT DISTRICT SIALKOT|POST OFFICE KHAS CHITI SHEIKHAAN TEHSIL SIALKOT DISTRICT SIALKOT|SIALKOT SIALKOT"
8
+ "Bochi Khan Kali, PO Tangi,Hisara Nehri , Tehsil Tangi, District Charsadda , Khyber-Pakhtunkhwa , Pakistan","BOCHI KHAN KALI PO TANGI HISARA NEHRI TEHSIL TANGI DISTRICT CHARSADDA KHYBER-PAKHTUNKHWA PAKISTAN"
9
+ "امانکوٹ،محلہ حاجی آباد،مینگورہ،تحصیل بابوزئی،ضلع سوات","AMANKOT MOHALA HAJI ABAD MINGORA TEHSIL BABU ZAI DISTRICT SAWAT"
10
+ "سعد لطیف","SAAD LATIF"