ogx786 commited on
Commit
de187a0
·
verified ·
1 Parent(s): f45cf61

Update finetune_for_v3.py

Browse files
Files changed (1) hide show
  1. finetune_for_v3.py +87 -5
finetune_for_v3.py CHANGED
@@ -57,6 +57,7 @@ How to use
57
  import os
58
  import gc
59
  import re
 
60
  import random
61
  import traceback
62
  import unicodedata
@@ -244,6 +245,80 @@ URDU_COL = "urdu"
244
  ROMAN_COL = "roman"
245
 
246
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
247
  def split_master_dataset():
248
  """Create train/val/test CSVs from MASTER_CSV if they don't already
249
  exist. If they exist, leave them untouched -- re-running the script
@@ -271,7 +346,7 @@ def split_master_dataset():
271
  )
272
 
273
  print(f"Loading master dataset: {MASTER_CSV}")
274
- df = pd.read_csv(MASTER_CSV)
275
 
276
  missing_cols = {URDU_COL, ROMAN_COL} - set(df.columns)
277
  if missing_cols:
@@ -294,9 +369,16 @@ def split_master_dataset():
294
  test_df = df.iloc[n_train + n_val:]
295
 
296
  os.makedirs(DATA_DIR, exist_ok=True)
297
- train_df.to_csv(TRAIN_CSV, index=False)
298
- val_df.to_csv(VAL_CSV, index=False)
299
- test_df.to_csv(TEST_CSV, index=False)
 
 
 
 
 
 
 
300
 
301
  print(f"Master dataset split ({n:,} rows total):")
302
  print(f" train: {len(train_df):,} rows -> {TRAIN_CSV}")
@@ -312,7 +394,7 @@ def load_and_validate_csv(path: str, limit_rows=None) -> pd.DataFrame:
312
  if not os.path.exists(path):
313
  raise FileNotFoundError(f"Expected dataset file not found: {path}")
314
 
315
- df = pd.read_csv(path)
316
 
317
  missing_cols = {URDU_COL, ROMAN_COL} - set(df.columns)
318
  if missing_cols:
 
57
  import os
58
  import gc
59
  import re
60
+ import csv
61
  import random
62
  import traceback
63
  import unicodedata
 
245
  ROMAN_COL = "roman"
246
 
247
 
248
+ def robust_read_csv(path: str) -> pd.DataFrame:
249
+ """Read a CSV with the fast C engine first. If that fails because of
250
+ malformed rows -- e.g. an unquoted comma embedded inside a field, which
251
+ is a known risk here since the last pipe-separated segment is literally
252
+ "city,district" -- fall back to a slower, bad-line-tolerant read and
253
+ report exactly which raw rows were dropped, instead of crashing the
254
+ whole pipeline over a handful of bad rows out of ~2.95M.
255
+
256
+ Reads everything as raw strings (dtype=str, keep_default_na=False,
257
+ na_filter=False) so pandas never silently reinterprets a field's
258
+ content -- e.g. a numeric-looking value being cast to int/float, or an
259
+ empty field being turned into NaN instead of "". Downstream
260
+ preprocessing (clean_field / preprocess_df) expects real strings.
261
+ """
262
+ read_kwargs = dict(dtype=str, keep_default_na=False, na_filter=False)
263
+ try:
264
+ return pd.read_csv(path, **read_kwargs)
265
+ except pd.errors.ParserError as e:
266
+ print(f"WARNING: fast CSV read of {path} failed ({e}).")
267
+ print("Falling back to a slower, bad-line-tolerant read...")
268
+
269
+ bad_lines = []
270
+
271
+ def _collect_bad_line(bad_line):
272
+ # Called once per malformed row with its raw split fields.
273
+ # Returning None drops the row; we just record it for reporting.
274
+ bad_lines.append(bad_line)
275
+ return None
276
+
277
+ df = pd.read_csv(
278
+ path, engine="python", on_bad_lines=_collect_bad_line, **read_kwargs
279
+ )
280
+
281
+ if bad_lines:
282
+ print(f"Dropped {len(bad_lines):,} malformed row(s) from {path}:")
283
+ for bl in bad_lines[:20]:
284
+ print(f" {bl}")
285
+ if len(bad_lines) > 20:
286
+ print(f" ... and {len(bad_lines) - 20:,} more")
287
+ print(
288
+ "These rows had more/fewer fields than the header expects -- "
289
+ "usually an unquoted comma or stray quote character inside a "
290
+ "field (the trailing 'city,district' segment is the likely "
291
+ "culprit). They are excluded from this run."
292
+ )
293
+
294
+ # Self-heal: rewrite this file in fully-quoted format so future
295
+ # reads hit the fast path instead of re-triggering this fallback
296
+ # every single time. The original is backed up first so nothing
297
+ # is silently lost.
298
+ backup_path = path + ".bak_malformed"
299
+ try:
300
+ if not os.path.exists(backup_path):
301
+ import shutil
302
+ shutil.copy2(path, backup_path)
303
+ df.to_csv(path, index=False, quoting=csv.QUOTE_ALL)
304
+ print(
305
+ f"Rewrote {path} in fully-quoted format (quoting=csv.QUOTE_ALL) "
306
+ f"with the malformed rows already excluded -- future reads of "
307
+ f"this file should hit the fast path. Original backed up to "
308
+ f"{backup_path}."
309
+ )
310
+ except Exception as rewrite_err:
311
+ print(
312
+ f"Could not rewrite {path} in cleaned format ({rewrite_err}). "
313
+ f"Training will still proceed with the in-memory cleaned data "
314
+ f"for this run, but the on-disk file remains as-is."
315
+ )
316
+ else:
317
+ print(f"Bad-line-tolerant read succeeded with no rows dropped from {path}.")
318
+
319
+ return df
320
+
321
+
322
  def split_master_dataset():
323
  """Create train/val/test CSVs from MASTER_CSV if they don't already
324
  exist. If they exist, leave them untouched -- re-running the script
 
346
  )
347
 
348
  print(f"Loading master dataset: {MASTER_CSV}")
349
+ df = robust_read_csv(MASTER_CSV)
350
 
351
  missing_cols = {URDU_COL, ROMAN_COL} - set(df.columns)
352
  if missing_cols:
 
369
  test_df = df.iloc[n_train + n_val:]
370
 
371
  os.makedirs(DATA_DIR, exist_ok=True)
372
+ # quoting=csv.QUOTE_ALL wraps every field in quotes on write, regardless
373
+ # of content. This is the fix for the "expected 2 fields, saw 3" class
374
+ # of error: any comma (or other character) inside a field can no longer
375
+ # be mistaken for the column delimiter on a future read, by this script
376
+ # or anything else that reads these files. Slightly larger files, but
377
+ # for ~2.95M rows of short text fields the overhead is negligible next
378
+ # to the reliability gain.
379
+ train_df.to_csv(TRAIN_CSV, index=False, quoting=csv.QUOTE_ALL)
380
+ val_df.to_csv(VAL_CSV, index=False, quoting=csv.QUOTE_ALL)
381
+ test_df.to_csv(TEST_CSV, index=False, quoting=csv.QUOTE_ALL)
382
 
383
  print(f"Master dataset split ({n:,} rows total):")
384
  print(f" train: {len(train_df):,} rows -> {TRAIN_CSV}")
 
394
  if not os.path.exists(path):
395
  raise FileNotFoundError(f"Expected dataset file not found: {path}")
396
 
397
+ df = robust_read_csv(path)
398
 
399
  missing_cols = {URDU_COL, ROMAN_COL} - set(df.columns)
400
  if missing_cols: