Sync model/scripts/train.py with adult erotica stories priority integration
Browse files- model/scripts/train.py +2 -2
model/scripts/train.py
CHANGED
|
@@ -630,11 +630,11 @@ def main():
|
|
| 630 |
for f in _raw_parquet:
|
| 631 |
_cat_buckets[f.split("/")[0]].append(f)
|
| 632 |
|
| 633 |
-
PRIORITY_PATTERNS = ("real_indian_history_pib", "real_cricket", "real_industrial_qms", "conversational")
|
| 634 |
_rng_files = _r.Random(int(C("seed", 42)) + hf_round)
|
| 635 |
for cat in _cat_buckets:
|
| 636 |
_rng_files.shuffle(_cat_buckets[cat])
|
| 637 |
-
pri_newest = [f for f in _cat_buckets[cat] if any(p in f for p in ("real_indian_history_pib", "real_cricket"))]
|
| 638 |
pri_other = [f for f in _cat_buckets[cat] if any(p in f for p in PRIORITY_PATTERNS) and f not in pri_newest]
|
| 639 |
rest = [f for f in _cat_buckets[cat] if f not in pri_newest and f not in pri_other]
|
| 640 |
_cat_buckets[cat] = pri_newest + pri_other + rest
|
|
|
|
| 630 |
for f in _raw_parquet:
|
| 631 |
_cat_buckets[f.split("/")[0]].append(f)
|
| 632 |
|
| 633 |
+
PRIORITY_PATTERNS = ("real_indian_history_pib", "real_cricket", "erotica_adult_stories", "real_industrial_qms", "conversational")
|
| 634 |
_rng_files = _r.Random(int(C("seed", 42)) + hf_round)
|
| 635 |
for cat in _cat_buckets:
|
| 636 |
_rng_files.shuffle(_cat_buckets[cat])
|
| 637 |
+
pri_newest = [f for f in _cat_buckets[cat] if any(p in f for p in ("real_indian_history_pib", "real_cricket", "erotica_adult_stories"))]
|
| 638 |
pri_other = [f for f in _cat_buckets[cat] if any(p in f for p in PRIORITY_PATTERNS) and f not in pri_newest]
|
| 639 |
rest = [f for f in _cat_buckets[cat] if f not in pri_newest and f not in pri_other]
|
| 640 |
_cat_buckets[cat] = pri_newest + pri_other + rest
|