#!/usr/bin/env python3 """ Quillan Hugging Face Multi-Dataset & Model Sync Script ====================================================== Uploads: 1. CrashOverrideX/quillan-lyrics-corpus (Dataset: Lyrics, poems, markdown prompts) 2. CrashOverrideX/quillan-audio-media (Dataset: MP3/FLAC/WAV audio & artwork) 3. CrashOverrideX/Quillan-Ronin (Model/Repo: Core architecture, code, and papers) """ import os import shutil import time from pathlib import Path from huggingface_hub import HfApi, create_repo, HfFolder # Explicitly set request timeout env for Hugging Face Hub if available os.environ["HF_HUB_DOWNLOAD_TIMEOUT"] = "120" TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or HfFolder.get_token() api = HfApi(token=TOKEN) user_info = api.whoami() username = user_info.get("name", user_info.get("username", "CrashOverrideX")) print(f"Authenticated with Hugging Face as: {username}") BASE_DIR = Path(__file__).resolve().parent.parent def upload_with_retry(api_instance, **kwargs): max_retries = 3 for attempt in range(1, max_retries + 1): try: api_instance.upload_folder(**kwargs) return except Exception as e: if attempt == max_retries: raise print(f"[WARN] Upload failed (attempt {attempt}/{max_retries}): {e}. Retrying in {attempt * 5}s...") time.sleep(attempt * 5) # ── 1. Create and Upload: Lyrics & Text Dataset ────────────────────────────── print("\n[1/3] Uploading Lyrics & Text Dataset...") lyrics_repo = f"{username}/quillan-lyrics-corpus" create_repo(repo_id=lyrics_repo, repo_type="dataset", token=TOKEN, exist_ok=True) # Prepare clean staging folder for lyrics stage_lyrics = BASE_DIR / "scratch" / "hf_stage_lyrics" if stage_lyrics.exists(): shutil.rmtree(stage_lyrics) stage_lyrics.mkdir(parents=True, exist_ok=True) # Copy Audio Engineer files audio_eng = BASE_DIR / "Audio Engineer" if audio_eng.exists(): for item in audio_eng.rglob("*"): if item.is_file() and not item.name.endswith(".wav") and not item.name.endswith(".mp3"): rel = item.relative_to(audio_eng) dest = stage_lyrics / "lyrics_and_notes" / rel dest.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(item, dest) # Add dataset card README (stage_lyrics / "README.md").write_text("""--- license: apache-2.0 task_categories: - text-generation language: - en tags: - lyrics - music - creative-writing - quillan-ronin size_categories: - n<1K --- # Quillan-Ronin: Sovereign Lyric & Prompt Corpus Curated lyrical dataset, song architectures, conceptual prompts, and rhythmic markdown sheets generated and structured by Quillan-Ronin. """, encoding="utf-8") upload_with_retry( api_instance=api, folder_path=str(stage_lyrics), repo_id=lyrics_repo, repo_type="dataset", commit_message="feat: upload complete Quillan lyrics and prompt dataset", token=TOKEN ) print(f"[SUCCESS] Lyrics Dataset live at: https://huggingface.co/datasets/{lyrics_repo}") print("\n[2/3] Uploading Audio & Media Dataset...") media_repo = f"{username}/quillan-audio-media" create_repo(repo_id=media_repo, repo_type="dataset", token=TOKEN, exist_ok=True) # Prepare clean staging folder for media stage_media = BASE_DIR / "scratch" / "hf_stage_media" if stage_media.exists(): shutil.rmtree(stage_media) stage_media.mkdir(parents=True, exist_ok=True) # Copy audio tracks (excluding soulframe, msi, and NFT designs) audio_dir = BASE_DIR / "06_Media" / "audio" if audio_dir.exists(): for item in audio_dir.rglob("*"): if item.is_file(): # Skip blacklisted items if any(k in item.name.lower() for k in ["soulframe", ".msi", ".exe", ".mp4", ".mkv", ".avi", ".mov"]): continue rel = item.relative_to(audio_dir) dest = stage_media / "audio" / rel dest.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(item, dest) # Copy artwork (excluding NFT designs) main_img_dir = BASE_DIR / "Main images" if main_img_dir.exists(): for item in main_img_dir.rglob("*"): if item.is_file(): rel = item.relative_to(main_img_dir) dest = stage_media / "artwork" / rel dest.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(item, dest) # Add media dataset card README (stage_media / "README.md").write_text("""--- license: apache-2.0 task_categories: - audio-to-audio - text-to-audio language: - en tags: - audio - music-generation - flac - mp3 - quillan-ronin --- # Quillan-Ronin: Multimodal Audio & Media Dataset Full multimodal audio and visual collection produced and designed by Quillan-Ronin: - **Lossless FLAC Master Recordings** (The Sound of Alchemy, singles) - **High-Bitrate MP3 Releases** (Draming of the Sky, Rock Album, ai beats, stems) - **Visual Concepts & Model Diagrams** """, encoding="utf-8") upload_with_retry( api_instance=api, folder_path=str(stage_media), repo_id=media_repo, repo_type="dataset", commit_message="feat: upload audio master tracks, stems, and album artwork", token=TOKEN ) print(f"[SUCCESS] Audio & Media Dataset live at: https://huggingface.co/datasets/{media_repo}") # ── 3. Sync Main Model/Codebase Repo ───────────────────────────────────────── print("\n[3/3] Uploading Quillan-Ronin Main Model Repository...") model_repo = f"{username}/Quillan-Ronin" create_repo(repo_id=model_repo, repo_type="model", token=TOKEN, exist_ok=True) # Prepare sanitized clean staging folder for model architecture stage_model = BASE_DIR / "scratch" / "hf_stage_model" if stage_model.exists(): shutil.rmtree(stage_model) stage_model.mkdir(parents=True, exist_ok=True) # Copy only core code, docs, and papers (strictly excluding all profiles, envs, keys) safe_dirs = ["oni", "docs", "templates", "Papers", "configs"] for sd in safe_dirs: src_p = BASE_DIR / sd if src_p.exists(): dst_p = stage_model / sd shutil.copytree(src_p, dst_p, ignore=shutil.ignore_patterns("*.pyc", "__pycache__", "*profile*", "*.token", "*.key", "*.env*")) for root_file in ["README.md", "AGENTS.md", "LINEAGE.md", "index.html"]: rf = BASE_DIR / root_file if rf.exists(): shutil.copy2(rf, stage_model / root_file) upload_with_retry( api_instance=api, folder_path=str(stage_model), repo_id=model_repo, repo_type="model", commit_message="v5.4.0-oni: synchronize clean architecture, papers, and configs", token=TOKEN ) print(f"[SUCCESS] Main Model Repository live at: https://huggingface.co/{model_repo}") print("\n[COMPLETE] All 3 Hugging Face repositories successfully synchronized!")