fix concatenate_datasets
Browse files- job_train.py +3 -5
job_train.py
CHANGED
|
@@ -119,7 +119,7 @@ def main():
|
|
| 119 |
f"cuda={torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'none'}", flush=True)
|
| 120 |
|
| 121 |
# ---------- data ----------
|
| 122 |
-
from datasets import load_dataset
|
| 123 |
t0 = time.time()
|
| 124 |
ds = load_dataset(DATASET, "windows_5s")
|
| 125 |
print(f"[data] loaded in {time.time()-t0:.0f}s: { {k: len(v) for k, v in ds.items()} }", flush=True)
|
|
@@ -141,10 +141,8 @@ def main():
|
|
| 141 |
# drowned out by the (easy) external corpora
|
| 142 |
c5_train = ds["train"].filter(lambda r: r["source"] == "challenge5", num_proc=4)
|
| 143 |
rep = max(1, min(200, (args.max_train_rows // 20) // max(1, len(c5_train))))
|
| 144 |
-
c5_up = c5_train
|
| 145 |
-
|
| 146 |
-
c5_up = c5_up.concatenate(c5_train)
|
| 147 |
-
train_d = train_d.concatenate(c5_up)
|
| 148 |
print(f"[data] challenge5 train oversampled x{rep} -> +{len(c5_up)} rows; train total={len(train_d)}", flush=True)
|
| 149 |
val_d = to_xy("validation", cap=args.val_rows)
|
| 150 |
c5_valid = to_xy("validation", source="challenge5", labeled=True)
|
|
|
|
| 119 |
f"cuda={torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'none'}", flush=True)
|
| 120 |
|
| 121 |
# ---------- data ----------
|
| 122 |
+
from datasets import load_dataset, concatenate_datasets
|
| 123 |
t0 = time.time()
|
| 124 |
ds = load_dataset(DATASET, "windows_5s")
|
| 125 |
print(f"[data] loaded in {time.time()-t0:.0f}s: { {k: len(v) for k, v in ds.items()} }", flush=True)
|
|
|
|
| 141 |
# drowned out by the (easy) external corpora
|
| 142 |
c5_train = ds["train"].filter(lambda r: r["source"] == "challenge5", num_proc=4)
|
| 143 |
rep = max(1, min(200, (args.max_train_rows // 20) // max(1, len(c5_train))))
|
| 144 |
+
c5_up = concatenate_datasets([c5_train] * (rep + 1))
|
| 145 |
+
train_d = concatenate_datasets([train_d, c5_up])
|
|
|
|
|
|
|
| 146 |
print(f"[data] challenge5 train oversampled x{rep} -> +{len(c5_up)} rows; train total={len(train_d)}", flush=True)
|
| 147 |
val_d = to_xy("validation", cap=args.val_rows)
|
| 148 |
c5_valid = to_xy("validation", source="challenge5", labeled=True)
|