MiniSEAL / scripts /download_tiny_corpus.py
aaronr43210's picture
Upload scripts
f3ab765 verified
Raw History Blame Contribute Delete
1.3 kB
"""
Generates a tiny local corpus for debugging.
Saves to data/raw/tiny_corpus.txt
"""
import os
def generate_local_corpus(filepath: str, size_mb: float = 1.0):
print(f"Generating {size_mb}MB local tiny corpus to {filepath}...")
os.makedirs(os.path.dirname(filepath), exist_ok=True)
base_text = (
"The future of artificial intelligence is full of exciting possibilities. "
"Generative AI models like Transformers use self-attention mechanisms to understand language. "
"Byte-Pair Encoding (BPE) helps tokenizers manage vocabulary effectively. "
"Distributed training across multiple GPUs allows us to scale these models significantly. "
"Techniques like Rotary Positional Embeddings (RoPE), RMSNorm, and SwiGLU improve performance and stability. "
)
target_bytes = int(size_mb * 1024 * 1024)
current_bytes = 0
with open(filepath, 'w', encoding='utf-8') as f:
while current_bytes < target_bytes:
f.write(base_text + "\n")
current_bytes += len(base_text.encode('utf-8')) + 1
print("Generation complete!")
if __name__ == "__main__":
out_path = os.path.join(os.path.dirname(__file__), "../data/raw/tiny_corpus.txt")
generate_local_corpus(out_path, size_mb=1.0)