Tonic commited on
Commit
23c5952
·
verified ·
1 Parent(s): 9162b77

fix concatenate_datasets

Browse files
Files changed (1) hide show
  1. job_train.py +3 -5
job_train.py CHANGED
@@ -119,7 +119,7 @@ def main():
119
  f"cuda={torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'none'}", flush=True)
120
 
121
  # ---------- data ----------
122
- from datasets import load_dataset
123
  t0 = time.time()
124
  ds = load_dataset(DATASET, "windows_5s")
125
  print(f"[data] loaded in {time.time()-t0:.0f}s: { {k: len(v) for k, v in ds.items()} }", flush=True)
@@ -141,10 +141,8 @@ def main():
141
  # drowned out by the (easy) external corpora
142
  c5_train = ds["train"].filter(lambda r: r["source"] == "challenge5", num_proc=4)
143
  rep = max(1, min(200, (args.max_train_rows // 20) // max(1, len(c5_train))))
144
- c5_up = c5_train
145
- for _ in range(rep):
146
- c5_up = c5_up.concatenate(c5_train)
147
- train_d = train_d.concatenate(c5_up)
148
  print(f"[data] challenge5 train oversampled x{rep} -> +{len(c5_up)} rows; train total={len(train_d)}", flush=True)
149
  val_d = to_xy("validation", cap=args.val_rows)
150
  c5_valid = to_xy("validation", source="challenge5", labeled=True)
 
119
  f"cuda={torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'none'}", flush=True)
120
 
121
  # ---------- data ----------
122
+ from datasets import load_dataset, concatenate_datasets
123
  t0 = time.time()
124
  ds = load_dataset(DATASET, "windows_5s")
125
  print(f"[data] loaded in {time.time()-t0:.0f}s: { {k: len(v) for k, v in ds.items()} }", flush=True)
 
141
  # drowned out by the (easy) external corpora
142
  c5_train = ds["train"].filter(lambda r: r["source"] == "challenge5", num_proc=4)
143
  rep = max(1, min(200, (args.max_train_rows // 20) // max(1, len(c5_train))))
144
+ c5_up = concatenate_datasets([c5_train] * (rep + 1))
145
+ train_d = concatenate_datasets([train_d, c5_up])
 
 
146
  print(f"[data] challenge5 train oversampled x{rep} -> +{len(c5_up)} rows; train total={len(train_d)}", flush=True)
147
  val_d = to_xy("validation", cap=args.val_rows)
148
  c5_valid = to_xy("validation", source="challenge5", labeled=True)