Download download_data.py from sage002/angstrom: direct link, hf CLI and curl.
- Browser
- Download file 8.13 kB
-
https://huggingface.co/sage002/angstrom/resolve/main/download_data.py
- Command line
-
hf download hf://sage002/angstrom/download_data.py
-
curl -L -o download_data.py https://huggingface.co/sage002/angstrom/resolve/main/download_data.py
8.13 kB
| """ | |
| Download ~500M characters of training data from Wikipedia, coding, and cybersecurity sources. | |
| Saves to data/corpus.txt | |
| """ | |
| import os | |
| import sys | |
| import json | |
| import random | |
| import hashlib | |
| import urllib.request | |
| from pathlib import Path | |
| from concurrent.futures import ThreadPoolExecutor, as_completed | |
| DATA_DIR = Path("data") | |
| DATA_DIR.mkdir(exist_ok=True) | |
| OUT_PATH = DATA_DIR / "corpus.txt" | |
| TARGET_CHARS = 500_000_000 # 500M characters | |
| # ββ Wikipedia via HuggingFace datasets ββββββββββββββββββββββββββββββ | |
| def download_wikipedia(target_chars: int) -> str: | |
| """Download Wikipedia articles via HuggingFace streaming.""" | |
| print("[wiki] Loading Wikipedia from HuggingFace...") | |
| try: | |
| from datasets import load_dataset | |
| except ImportError: | |
| print("[wiki] 'datasets' not installed, skipping Wikipedia") | |
| return "" | |
| ds = load_dataset("wikimedia/wikipedia", "20231101.en", split="train", streaming=True) | |
| chunks = [] | |
| total = 0 | |
| for article in ds: | |
| text = article.get("text", "") | |
| if len(text) < 100: | |
| continue | |
| chunks.append(text) | |
| total += len(text) | |
| if total >= target_chars: | |
| break | |
| if len(chunks) % 10000 == 0: | |
| print(f" wiki: {total/1e6:.1f}M chars from {len(chunks)} articles") | |
| print(f" wiki: {total/1e6:.1f}M chars from {len(chunks)} articles") | |
| return "\n\n".join(chunks) | |
| # ββ StackOverflow via HuggingFace βββββββββββββββββββββββββββββββββββ | |
| def download_stackoverflow(target_chars: int) -> str: | |
| """Download StackOverflow Q&A.""" | |
| print("[so] Loading StackOverflow from HuggingFace...") | |
| try: | |
| from datasets import load_dataset | |
| except ImportError: | |
| return "" | |
| try: | |
| ds = load_dataset("lcornell/stackoverflow", split="train", streaming=True) | |
| except Exception: | |
| try: | |
| ds = load_dataset("HuggingFaceFW/stackoverflow", split="train", streaming=True) | |
| except Exception: | |
| print("[so] Could not load StackOverflow dataset, skipping") | |
| return "" | |
| chunks = [] | |
| total = 0 | |
| for item in ds: | |
| title = item.get("title", "") | |
| body = item.get("body", "") | |
| tags = item.get("tags", "") | |
| text = f"Title: {title}\nTags: {tags}\n{body}" | |
| if len(text) < 50: | |
| continue | |
| chunks.append(text) | |
| total += len(text) | |
| if total >= target_chars: | |
| break | |
| if len(chunks) % 10000 == 0: | |
| print(f" so: {total/1e6:.1f}M chars from {len(chunks)} items") | |
| print(f" so: {total/1e6:.1f}M chars from {len(chunks)} items") | |
| return "\n\n".join(chunks) | |
| # ββ GitHub code via HuggingFace βββββββββββββββββββββββββββββββββββββ | |
| def download_github_code(target_chars: int) -> str: | |
| """Download GitHub code snippets.""" | |
| print("[code] Loading GitHub code from HuggingFace...") | |
| try: | |
| from datasets import load_dataset | |
| except ImportError: | |
| return "" | |
| try: | |
| ds = load_dataset("codeparrot/github-code", split="train", streaming=True) | |
| except Exception: | |
| try: | |
| ds = load_dataset("HuggingFaceFW/fineweb", name="sample-10BT", split="train", streaming=True) | |
| except Exception: | |
| print("[code] Could not load code dataset, skipping") | |
| return "" | |
| chunks = [] | |
| total = 0 | |
| for item in ds: | |
| text = item.get("text", "") or item.get("content", "") | |
| if len(text) < 50: | |
| continue | |
| chunks.append(text) | |
| total += len(text) | |
| if total >= target_chars: | |
| break | |
| if len(chunks) % 10000 == 0: | |
| print(f" code: {total/1e6:.1f}M chars from {len(chunks)} files") | |
| print(f" code: {total/1e6:.1f}M chars from {len(chunks)} files") | |
| return "\n\n".join(chunks) | |
| # ββ Cybersecurity text via direct URLs ββββββββββββββββββββββββββββββ | |
| CYBERSEC_URLS = [ | |
| # NIST publications and guidelines | |
| "https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.29.pdf", | |
| # Common security resources (plain text mirrors) | |
| "https://raw.githubusercontent.com/enaqx/awesome-pentest/master/README.md", | |
| "https://raw.githubusercontent.com/Hack-with-Github/Awesome-Hacking/master/README.md", | |
| "https://raw.githubusercontent.com/awslabs/aws-security-reference-architecture-examples/main/README.md", | |
| ] | |
| def download_cybersecurity(target_chars: int) -> str: | |
| """Download cybersecurity reference material.""" | |
| print("[cyber] Downloading cybersecurity resources...") | |
| chunks = [] | |
| total = 0 | |
| for url in CYBERSEC_URLS: | |
| try: | |
| print(f" cyber: fetching {url[:60]}...") | |
| req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) | |
| with urllib.request.urlopen(req, timeout=30) as resp: | |
| data = resp.read().decode("utf-8", errors="replace") | |
| if len(data) > 100: | |
| chunks.append(data) | |
| total += len(data) | |
| except Exception as e: | |
| print(f" cyber: failed {url[:50]}: {e}") | |
| if total >= target_chars: | |
| break | |
| # Also try cybersecurity books/guides from Project Gutenberg | |
| gutenberg_urls = [ | |
| "https://www.gutenberg.org/files/27734/27734-0.txt", # Cybersecurity-related | |
| ] | |
| for url in gutenberg_urls: | |
| try: | |
| print(f" cyber: fetching gutenberg {url[-20:]}...") | |
| req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) | |
| with urllib.request.urlopen(req, timeout=30) as resp: | |
| data = resp.read().decode("utf-8", errors="replace") | |
| if len(data) > 100: | |
| chunks.append(data) | |
| total += len(data) | |
| except Exception as e: | |
| print(f" cyber: failed: {e}") | |
| if total >= target_chars: | |
| break | |
| print(f" cyber: {total/1e6:.1f}M chars from {len(chunks)} sources") | |
| return "\n\n".join(chunks) | |
| # ββ Merge and write βββββββββββββββββββββββββββββββββββββββββββββββββ | |
| def merge_and_write(sources: list[tuple[str, str]], out_path: Path, target: int): | |
| """Merge sources proportionally and write to file.""" | |
| total_chars = sum(len(s) for _, s in sources if s) | |
| if total_chars == 0: | |
| print("ERROR: No data downloaded") | |
| sys.exit(1) | |
| print(f"\nTotal available: {total_chars/1e6:.1f}M chars") | |
| print(f"Target: {target/1e6:.1f}M chars") | |
| # Build output with file markers | |
| lines = [] | |
| written = 0 | |
| for name, text in sources: | |
| if not text: | |
| continue | |
| # Take proportionally to fill target | |
| proportion = len(text) / total_chars | |
| take_chars = int(target * proportion) | |
| taken = text[:take_chars] | |
| lines.append(f"<|file|>{name}") | |
| lines.append(taken) | |
| lines.append("") | |
| written += len(taken) | |
| output = "\n".join(lines) | |
| out_path.write_text(output, encoding="utf-8") | |
| print(f"Written: {out_path} ({len(output)/1e6:.1f}M chars, {len(output):,} chars)") | |
| # ββ Main ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| if __name__ == "__main__": | |
| print(f"Target: {TARGET_CHARS/1e6:.0f}M characters\n") | |
| wiki = download_wikipedia(TARGET_CHARS // 2) | |
| so = download_stackoverflow(TARGET_CHARS // 4) | |
| code = download_github_code(TARGET_CHARS // 4) | |
| cyber = download_cybersecurity(TARGET_CHARS // 8) | |
| sources = [ | |
| ("wikipedia", wiki), | |
| ("stackoverflow", so), | |
| ("github_code", code), | |
| ("cybersecurity", cyber), | |
| ] | |
| merge_and_write(sources, OUT_PATH, TARGET_CHARS) | |
| print("\nDONE β data ready at data/corpus.txt") | |