Update finetune_for_v3.py
Browse files- finetune_for_v3.py +87 -5
finetune_for_v3.py
CHANGED
|
@@ -57,6 +57,7 @@ How to use
|
|
| 57 |
import os
|
| 58 |
import gc
|
| 59 |
import re
|
|
|
|
| 60 |
import random
|
| 61 |
import traceback
|
| 62 |
import unicodedata
|
|
@@ -244,6 +245,80 @@ URDU_COL = "urdu"
|
|
| 244 |
ROMAN_COL = "roman"
|
| 245 |
|
| 246 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 247 |
def split_master_dataset():
|
| 248 |
"""Create train/val/test CSVs from MASTER_CSV if they don't already
|
| 249 |
exist. If they exist, leave them untouched -- re-running the script
|
|
@@ -271,7 +346,7 @@ def split_master_dataset():
|
|
| 271 |
)
|
| 272 |
|
| 273 |
print(f"Loading master dataset: {MASTER_CSV}")
|
| 274 |
-
df =
|
| 275 |
|
| 276 |
missing_cols = {URDU_COL, ROMAN_COL} - set(df.columns)
|
| 277 |
if missing_cols:
|
|
@@ -294,9 +369,16 @@ def split_master_dataset():
|
|
| 294 |
test_df = df.iloc[n_train + n_val:]
|
| 295 |
|
| 296 |
os.makedirs(DATA_DIR, exist_ok=True)
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 300 |
|
| 301 |
print(f"Master dataset split ({n:,} rows total):")
|
| 302 |
print(f" train: {len(train_df):,} rows -> {TRAIN_CSV}")
|
|
@@ -312,7 +394,7 @@ def load_and_validate_csv(path: str, limit_rows=None) -> pd.DataFrame:
|
|
| 312 |
if not os.path.exists(path):
|
| 313 |
raise FileNotFoundError(f"Expected dataset file not found: {path}")
|
| 314 |
|
| 315 |
-
df =
|
| 316 |
|
| 317 |
missing_cols = {URDU_COL, ROMAN_COL} - set(df.columns)
|
| 318 |
if missing_cols:
|
|
|
|
| 57 |
import os
|
| 58 |
import gc
|
| 59 |
import re
|
| 60 |
+
import csv
|
| 61 |
import random
|
| 62 |
import traceback
|
| 63 |
import unicodedata
|
|
|
|
| 245 |
ROMAN_COL = "roman"
|
| 246 |
|
| 247 |
|
| 248 |
+
def robust_read_csv(path: str) -> pd.DataFrame:
|
| 249 |
+
"""Read a CSV with the fast C engine first. If that fails because of
|
| 250 |
+
malformed rows -- e.g. an unquoted comma embedded inside a field, which
|
| 251 |
+
is a known risk here since the last pipe-separated segment is literally
|
| 252 |
+
"city,district" -- fall back to a slower, bad-line-tolerant read and
|
| 253 |
+
report exactly which raw rows were dropped, instead of crashing the
|
| 254 |
+
whole pipeline over a handful of bad rows out of ~2.95M.
|
| 255 |
+
|
| 256 |
+
Reads everything as raw strings (dtype=str, keep_default_na=False,
|
| 257 |
+
na_filter=False) so pandas never silently reinterprets a field's
|
| 258 |
+
content -- e.g. a numeric-looking value being cast to int/float, or an
|
| 259 |
+
empty field being turned into NaN instead of "". Downstream
|
| 260 |
+
preprocessing (clean_field / preprocess_df) expects real strings.
|
| 261 |
+
"""
|
| 262 |
+
read_kwargs = dict(dtype=str, keep_default_na=False, na_filter=False)
|
| 263 |
+
try:
|
| 264 |
+
return pd.read_csv(path, **read_kwargs)
|
| 265 |
+
except pd.errors.ParserError as e:
|
| 266 |
+
print(f"WARNING: fast CSV read of {path} failed ({e}).")
|
| 267 |
+
print("Falling back to a slower, bad-line-tolerant read...")
|
| 268 |
+
|
| 269 |
+
bad_lines = []
|
| 270 |
+
|
| 271 |
+
def _collect_bad_line(bad_line):
|
| 272 |
+
# Called once per malformed row with its raw split fields.
|
| 273 |
+
# Returning None drops the row; we just record it for reporting.
|
| 274 |
+
bad_lines.append(bad_line)
|
| 275 |
+
return None
|
| 276 |
+
|
| 277 |
+
df = pd.read_csv(
|
| 278 |
+
path, engine="python", on_bad_lines=_collect_bad_line, **read_kwargs
|
| 279 |
+
)
|
| 280 |
+
|
| 281 |
+
if bad_lines:
|
| 282 |
+
print(f"Dropped {len(bad_lines):,} malformed row(s) from {path}:")
|
| 283 |
+
for bl in bad_lines[:20]:
|
| 284 |
+
print(f" {bl}")
|
| 285 |
+
if len(bad_lines) > 20:
|
| 286 |
+
print(f" ... and {len(bad_lines) - 20:,} more")
|
| 287 |
+
print(
|
| 288 |
+
"These rows had more/fewer fields than the header expects -- "
|
| 289 |
+
"usually an unquoted comma or stray quote character inside a "
|
| 290 |
+
"field (the trailing 'city,district' segment is the likely "
|
| 291 |
+
"culprit). They are excluded from this run."
|
| 292 |
+
)
|
| 293 |
+
|
| 294 |
+
# Self-heal: rewrite this file in fully-quoted format so future
|
| 295 |
+
# reads hit the fast path instead of re-triggering this fallback
|
| 296 |
+
# every single time. The original is backed up first so nothing
|
| 297 |
+
# is silently lost.
|
| 298 |
+
backup_path = path + ".bak_malformed"
|
| 299 |
+
try:
|
| 300 |
+
if not os.path.exists(backup_path):
|
| 301 |
+
import shutil
|
| 302 |
+
shutil.copy2(path, backup_path)
|
| 303 |
+
df.to_csv(path, index=False, quoting=csv.QUOTE_ALL)
|
| 304 |
+
print(
|
| 305 |
+
f"Rewrote {path} in fully-quoted format (quoting=csv.QUOTE_ALL) "
|
| 306 |
+
f"with the malformed rows already excluded -- future reads of "
|
| 307 |
+
f"this file should hit the fast path. Original backed up to "
|
| 308 |
+
f"{backup_path}."
|
| 309 |
+
)
|
| 310 |
+
except Exception as rewrite_err:
|
| 311 |
+
print(
|
| 312 |
+
f"Could not rewrite {path} in cleaned format ({rewrite_err}). "
|
| 313 |
+
f"Training will still proceed with the in-memory cleaned data "
|
| 314 |
+
f"for this run, but the on-disk file remains as-is."
|
| 315 |
+
)
|
| 316 |
+
else:
|
| 317 |
+
print(f"Bad-line-tolerant read succeeded with no rows dropped from {path}.")
|
| 318 |
+
|
| 319 |
+
return df
|
| 320 |
+
|
| 321 |
+
|
| 322 |
def split_master_dataset():
|
| 323 |
"""Create train/val/test CSVs from MASTER_CSV if they don't already
|
| 324 |
exist. If they exist, leave them untouched -- re-running the script
|
|
|
|
| 346 |
)
|
| 347 |
|
| 348 |
print(f"Loading master dataset: {MASTER_CSV}")
|
| 349 |
+
df = robust_read_csv(MASTER_CSV)
|
| 350 |
|
| 351 |
missing_cols = {URDU_COL, ROMAN_COL} - set(df.columns)
|
| 352 |
if missing_cols:
|
|
|
|
| 369 |
test_df = df.iloc[n_train + n_val:]
|
| 370 |
|
| 371 |
os.makedirs(DATA_DIR, exist_ok=True)
|
| 372 |
+
# quoting=csv.QUOTE_ALL wraps every field in quotes on write, regardless
|
| 373 |
+
# of content. This is the fix for the "expected 2 fields, saw 3" class
|
| 374 |
+
# of error: any comma (or other character) inside a field can no longer
|
| 375 |
+
# be mistaken for the column delimiter on a future read, by this script
|
| 376 |
+
# or anything else that reads these files. Slightly larger files, but
|
| 377 |
+
# for ~2.95M rows of short text fields the overhead is negligible next
|
| 378 |
+
# to the reliability gain.
|
| 379 |
+
train_df.to_csv(TRAIN_CSV, index=False, quoting=csv.QUOTE_ALL)
|
| 380 |
+
val_df.to_csv(VAL_CSV, index=False, quoting=csv.QUOTE_ALL)
|
| 381 |
+
test_df.to_csv(TEST_CSV, index=False, quoting=csv.QUOTE_ALL)
|
| 382 |
|
| 383 |
print(f"Master dataset split ({n:,} rows total):")
|
| 384 |
print(f" train: {len(train_df):,} rows -> {TRAIN_CSV}")
|
|
|
|
| 394 |
if not os.path.exists(path):
|
| 395 |
raise FileNotFoundError(f"Expected dataset file not found: {path}")
|
| 396 |
|
| 397 |
+
df = robust_read_csv(path)
|
| 398 |
|
| 399 |
missing_cols = {URDU_COL, ROMAN_COL} - set(df.columns)
|
| 400 |
if missing_cols:
|