| """ |
| Merge all data sources into a single training corpus. |
| Combines: train.txt, llms-full.txt, and all text files from repos_cloned/ |
| """ |
| import sys; sys.path.insert(0, '.') |
| from pathlib import Path |
|
|
| data_dir = Path("data") |
| out_path = data_dir / "corpus.txt" |
|
|
| extensions = {".py", ".rst", ".md", ".html", ".txt", ".cfg", ".ini", |
| ".toml", ".yaml", ".yml", ".json", ".css", ".js", ".bat", |
| "Makefile", "dockerignore", "gitignore"} |
|
|
| segments = [] |
|
|
| |
| p = data_dir / "train.txt" |
| if p.exists(): |
| segments.append(("train.txt", p.read_text(encoding="utf-8"))) |
|
|
| |
| p = data_dir / "llms-full.txt" |
| if p.exists(): |
| segments.append(("llms-full.txt", p.read_text(encoding="utf-8"))) |
|
|
| |
| repos = data_dir / "repos_cloned" |
| if repos.exists(): |
| for f in sorted(repos.rglob("*")): |
| if not f.is_file(): |
| continue |
| if f.suffix in extensions or f.name in extensions: |
| try: |
| text = f.read_text(encoding="utf-8", errors="replace") |
| if len(text) > 50: |
| segments.append((f.relative_to(data_dir).as_posix(), text)) |
| except Exception: |
| pass |
|
|
| print(f"Found {len(segments)} file segments") |
|
|
| |
| lines = [] |
| for name, text in segments: |
| lines.append(f"<|file|>{name}") |
| lines.append(text) |
| lines.append("") |
|
|
| combined = "\n".join(lines) |
| out_path.write_text(combined, encoding="utf-8") |
| size_mb = len(combined) / 1e6 |
| print(f"Written: {out_path} ({size_mb:.1f} MB, {len(combined):,} chars)") |
|
|
