Upload code/build_sft_v20_dataset.py with huggingface_hub
Browse files
code/build_sft_v20_dataset.py
CHANGED
|
@@ -609,7 +609,7 @@ def build_domain_reasoning():
|
|
| 609 |
if len(pool) < cap:
|
| 610 |
print(f" Loading open-thoughts/OpenThoughts (need {cap - len(pool)} more)...")
|
| 611 |
try:
|
| 612 |
-
ds_ot = load_dataset("open-thoughts/
|
| 613 |
CODE_KWS = {"python", "code", "function", "sql", "algorithm", "class ", "def "}
|
| 614 |
MATH_KWS = {"calculate", "solve", "equation", "integer", "prime", "probability"}
|
| 615 |
for row in ds_ot:
|
|
|
|
| 609 |
if len(pool) < cap:
|
| 610 |
print(f" Loading open-thoughts/OpenThoughts (need {cap - len(pool)} more)...")
|
| 611 |
try:
|
| 612 |
+
ds_ot = load_dataset("open-thoughts/Open-Thoughts-114k", split="train", streaming=True)
|
| 613 |
CODE_KWS = {"python", "code", "function", "sql", "algorithm", "class ", "def "}
|
| 614 |
MATH_KWS = {"calculate", "solve", "equation", "integer", "prime", "probability"}
|
| 615 |
for row in ds_ot:
|