ViuAI commited on
Commit
35480b2
·
verified ·
1 Parent(s): 97bba8b

Sync model/scripts/train.py with adult erotica stories priority integration

Browse files
Files changed (1) hide show
  1. model/scripts/train.py +2 -2
model/scripts/train.py CHANGED
@@ -630,11 +630,11 @@ def main():
630
  for f in _raw_parquet:
631
  _cat_buckets[f.split("/")[0]].append(f)
632
 
633
- PRIORITY_PATTERNS = ("real_indian_history_pib", "real_cricket", "erotica_adult_stories", "real_industrial_qms", "conversational")
634
  _rng_files = _r.Random(int(C("seed", 42)) + hf_round)
635
  for cat in _cat_buckets:
636
  _rng_files.shuffle(_cat_buckets[cat])
637
- pri_newest = [f for f in _cat_buckets[cat] if any(p in f for p in ("real_indian_history_pib", "real_cricket", "erotica_adult_stories"))]
638
  pri_other = [f for f in _cat_buckets[cat] if any(p in f for p in PRIORITY_PATTERNS) and f not in pri_newest]
639
  rest = [f for f in _cat_buckets[cat] if f not in pri_newest and f not in pri_other]
640
  _cat_buckets[cat] = pri_newest + pri_other + rest
 
630
  for f in _raw_parquet:
631
  _cat_buckets[f.split("/")[0]].append(f)
632
 
633
+ PRIORITY_PATTERNS = ("real_indian_history_pib", "real_cricket", "erotica_adult_stories", "real_cinema", "real_mythology", "real_finance", "real_industrial_qms", "conversational")
634
  _rng_files = _r.Random(int(C("seed", 42)) + hf_round)
635
  for cat in _cat_buckets:
636
  _rng_files.shuffle(_cat_buckets[cat])
637
+ pri_newest = [f for f in _cat_buckets[cat] if any(p in f for p in ("real_indian_history_pib", "real_cricket", "erotica_adult_stories", "real_cinema", "real_mythology", "real_finance"))]
638
  pri_other = [f for f in _cat_buckets[cat] if any(p in f for p in PRIORITY_PATTERNS) and f not in pri_newest]
639
  rest = [f for f in _cat_buckets[cat] if f not in pri_newest and f not in pri_other]
640
  _cat_buckets[cat] = pri_newest + pri_other + rest