File size: 2,918 Bytes
205a575 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | """
Add Complete English Wikipedia (wikimedia/wikipedia - 20231101.en)
to ViuAI/viu-mini-raw-pretrain Hub via fast Server-Side Hub Copy.
Total: 41 Parquet Shards, ~10.83 GB (~5 Billion Tokens).
"""
import sys
import time
from huggingface_hub import HfApi, CommitOperationCopy
sys.stdout.reconfigure(encoding='utf-8')
def main():
api = HfApi()
source_repo = "wikimedia/wikipedia"
target_repo = "ViuAI/viu-mini-raw-pretrain"
subfolder = "20231101.en"
target_folder = "wikipedia"
print("=== Adding Complete English Wikipedia to Pretraining Corpus ===")
print(f"Source: {source_repo} ({subfolder})")
print(f"Target: {target_repo}/{target_folder}/")
# List all files in 20231101.en
repo_files = api.list_repo_files(source_repo, repo_type="dataset")
wiki_files = sorted([f for f in repo_files if f.startswith(f"{subfolder}/") and f.endswith(".parquet")])
print(f"Total English Wikipedia Parquet shards found: {len(wiki_files)}")
# Check already copied files in target repo
target_files = set(api.list_repo_files(target_repo, repo_type="dataset"))
operations = []
skipped = 0
for src_path in wiki_files:
filename = src_path.split("/")[-1]
dst_path = f"{target_folder}/{filename}"
if dst_path in target_files:
print(f" [Already Present] {dst_path}")
skipped += 1
continue
operations.append(
CommitOperationCopy(
src_path_in_repo=src_path,
path_in_repo=dst_path,
src_repo_id=source_repo,
src_repo_type="dataset"
)
)
print(f"\nSkipped: {skipped} files | To Copy: {len(operations)} files")
if not operations:
print("[ok] All English Wikipedia shards are already present in target repo!")
return
# Commit in batches of 10 to ensure stability and smooth progress
batch_size = 10
total_batches = (len(operations) + batch_size - 1) // batch_size
print(f"Committing {len(operations)} server-side copies across {total_batches} batches...")
for i in range(0, len(operations), batch_size):
batch = operations[i:i+batch_size]
batch_idx = (i // batch_size) + 1
print(f"\n[Batch {batch_idx}/{total_batches}] Committing {len(batch)} shards to Hub...")
start_t = time.time()
api.create_commit(
repo_id=target_repo,
repo_type="dataset",
operations=batch,
commit_message=f"feat(wikipedia): ingest English Wikipedia 20231101.en shards batch {batch_idx}/{total_batches}"
)
elapsed = time.time() - start_t
print(f" [ok] Batch {batch_idx}/{total_batches} committed successfully in {elapsed:.2f}s!")
print("\n[SUCCESS] All 41 English Wikipedia shards ingested into ViuAI/viu-mini-raw-pretrain!")
if __name__ == "__main__":
main()
|