Upload commit_review_model_to_public_repo.ipynb

gemma4_training/commit_review_model_to_public_repo.ipynb

@@ -1,117 +1 @@
-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    }
-  },
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2KiA3HOvxFB0"
-      },
-      "outputs": [],
-      "source": [
-        "\n",
-        "import os\n",
-        "from pathlib import Path\n",
-        "from huggingface_hub import snapshot_download, HfApi, get_token\n",
-        "\n",
-        "# Configuration\n",
-        "SOURCE_REPO = \"codeShare/kromoss-lora-exports\"\n",
-        "SOURCE_FOLDER = \"gemma4_lora_to_review/95_percent\"\n",
-        "\n",
-        "DEST_REPO = \"codeShare/gemma-4-E2B-it-heretic-image-caption-adapter\"\n",
-        "\n",
-        "# 1. Authenticate with Hugging Face\n",
-        "HF_TOKEN = os.environ.get(\"HF_TOKEN\") or get_token()\n",
-        "\n",
-        "if not HF_TOKEN:\n",
-        "    try:\n",
-        "        from google.colab import userdata\n",
-        "        HF_TOKEN = userdata.get(\"HF_TOKEN\")\n",
-        "    except Exception:\n",
-        "        pass\n",
-        "\n",
-        "if not HF_TOKEN:\n",
-        "    raise RuntimeError(\"Hugging Face API token is required.\")\n",
-        "\n",
-        "api = HfApi(token=HF_TOKEN)\n",
-        "\n",
-        "# 2. Download ALL files from the source folder\n",
-        "print(f\"Downloading {SOURCE_FOLDER} from {SOURCE_REPO}...\")\n",
-        "\n",
-        "local_download_dir = Path(\"./downloaded_adapter_95\")\n",
-        "\n",
-        "snapshot_download(\n",
-        "    repo_id=SOURCE_REPO,\n",
-        "    repo_type=\"model\",\n",
-        "    allow_patterns=[\n",
-        "        f\"{SOURCE_FOLDER}/**\",\n",
-        "        f\"{SOURCE_FOLDER}/*\"\n",
-        "    ],\n",
-        "    local_dir=str(local_download_dir),\n",
-        "    token=HF_TOKEN\n",
-        ")\n",
-        "\n",
-        "actual_folder = local_download_dir / SOURCE_FOLDER\n",
-        "\n",
-        "if not actual_folder.exists():\n",
-        "    raise FileNotFoundError(\n",
-        "        f\"Could not locate downloaded folder: {actual_folder}\"\n",
-        "    )\n",
-        "\n",
-        "# 3. Collect EVERY file recursively\n",
-        "files_to_upload = [\n",
-        "    p for p in actual_folder.rglob(\"*\")\n",
-        "    if p.is_file()\n",
-        "]\n",
-        "\n",
-        "if not files_to_upload:\n",
-        "    raise RuntimeError(\"No files found to upload!\")\n",
-        "\n",
-        "print(f\"\\nFound {len(files_to_upload)} files:\")\n",
-        "\n",
-        "for p in files_to_upload:\n",
-        "    print(f\"  {p.relative_to(actual_folder)}\")\n",
-        "\n",
-        "# 4. Ensure destination repository exists\n",
-        "api.create_repo(\n",
-        "    repo_id=DEST_REPO,\n",
-        "    repo_type=\"model\",\n",
-        "    private=False,\n",
-        "    exist_ok=True\n",
-        ")\n",
-        "\n",
-        "# 5. Upload ALL files and their subdirectories\n",
-        "print(f\"\\nUploading {len(files_to_upload)} files to {DEST_REPO}...\")\n",
-        "\n",
-        "upload_response = api.upload_folder(\n",
-        "    repo_id=DEST_REPO,\n",
-        "    repo_type=\"model\",\n",
-        "    folder_path=str(actual_folder),\n",
-        "    path_in_repo=\"\",\n",
-        "    commit_message=(\n",
-        "        \"Upload all files from gemma4_lora_to_review/95_percent\"\n",
-        "    ),\n",
-        "    ignore_patterns=[]\n",
-        ")\n",
-        "\n",
-        "print(\"\\nUpload completed successfully!\")\n",
-        "print(\"Destination:\", f\"https://huggingface.co/{DEST_REPO}\")\n",
-        "print(\"Files uploaded:\", len(files_to_upload))\n",
-        "print(\"Response:\", upload_response)\n"
-      ]
-    }
-  ]
-}
+{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[{"file_id":"https://huggingface.co/codeShare/kromoss-lora-exports/blob/main/gemma4_training/commit_review_model_to_public_repo.ipynb","timestamp":1794083665426}]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# Update public lora"],"metadata":{"id":"lBL9DszbuMuZ"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"2KiA3HOvxFB0"},"outputs":[],"source":["\n","import os\n","from pathlib import Path\n","from huggingface_hub import snapshot_download, HfApi, get_token\n","\n","# Configuration\n","SOURCE_REPO = \"codeShare/kromoss-lora-exports\"\n","SOURCE_FOLDER = \"gemma4_lora_to_review/95_percent\"\n","\n","DEST_REPO = \"codeShare/gemma-4-E2B-it-heretic-image-caption-adapter\"\n","\n","# 1. Authenticate with Hugging Face\n","HF_TOKEN = os.environ.get(\"HF_TOKEN\") or get_token()\n","\n","if not HF_TOKEN:\n","    try:\n","        from google.colab import userdata\n","        HF_TOKEN = userdata.get(\"HF_TOKEN\")\n","    except Exception:\n","        pass\n","\n","if not HF_TOKEN:\n","    raise RuntimeError(\"Hugging Face API token is required.\")\n","\n","api = HfApi(token=HF_TOKEN)\n","\n","# 2. Download ALL files from the source folder\n","print(f\"Downloading {SOURCE_FOLDER} from {SOURCE_REPO}...\")\n","\n","local_download_dir = Path(\"./downloaded_adapter_95\")\n","\n","snapshot_download(\n","    repo_id=SOURCE_REPO,\n","    repo_type=\"model\",\n","    allow_patterns=[\n","        f\"{SOURCE_FOLDER}/**\",\n","        f\"{SOURCE_FOLDER}/*\"\n","    ],\n","    local_dir=str(local_download_dir),\n","    token=HF_TOKEN\n",")\n","\n","actual_folder = local_download_dir / SOURCE_FOLDER\n","\n","if not actual_folder.exists():\n","    raise FileNotFoundError(\n","        f\"Could not locate downloaded folder: {actual_folder}\"\n","    )\n","\n","# 3. Collect EVERY file recursively\n","files_to_upload = [\n","    p for p in actual_folder.rglob(\"*\")\n","    if p.is_file()\n","]\n","\n","if not files_to_upload:\n","    raise RuntimeError(\"No files found to upload!\")\n","\n","print(f\"\\nFound {len(files_to_upload)} files:\")\n","\n","for p in files_to_upload:\n","    print(f\"  {p.relative_to(actual_folder)}\")\n","\n","# 4. Ensure destination repository exists\n","api.create_repo(\n","    repo_id=DEST_REPO,\n","    repo_type=\"model\",\n","    private=False,\n","    exist_ok=True\n",")\n","\n","# 5. Upload ALL files and their subdirectories\n","print(f\"\\nUploading {len(files_to_upload)} files to {DEST_REPO}...\")\n","\n","upload_response = api.upload_folder(\n","    repo_id=DEST_REPO,\n","    repo_type=\"model\",\n","    folder_path=str(actual_folder),\n","    path_in_repo=\"\",\n","    commit_message=(\n","        \"Upload all files from gemma4_lora_to_review/95_percent\"\n","    ),\n","    ignore_patterns=[]\n",")\n","\n","print(\"\\nUpload completed successfully!\")\n","print(\"Destination:\", f\"https://huggingface.co/{DEST_REPO}\")\n","print(\"Files uploaded:\", len(files_to_upload))\n","print(\"Response:\", upload_response)\n"]},{"cell_type":"markdown","source":["# Update continue lora"],"metadata":{"id":"7lZuLbvpuFVW"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":421,"referenced_widgets":["1abcbac011e84714b3aa1cccfbb4769d","88ce7c41d18a49a2b9714fe4e0e98de0","a691527845f840c78c735f7bcfe01de1","7c37f9d664bc4978ab2617551abd5e47","ecc49b2c71ee4c40b6f41540a67408dc","a17a534ea3ec46cf91c080cdba18697b","a323a2d76e1e4a29a3fa6c44402d6a03","b1608b4094264244a4cf071ee07f0bae","d3a4a7a0544e4e4697fac1b555462cec","9c77e156326a492aa8c20eb9563626a8","ea26129b1bd341c7bd4c61079e20d466","04e9c410e7c8472c8b9e514191114e3d","7cb8e22e32704f9cbcc7955954cab957","45ad434cde8d49289f24a25139d1dd6c","04524ef40e374bd68a6d975143ad5c9f","6473e417593f40e48a1ca8d5e7685940","661b499a6b6a4b128eabaf63793b48f3","202c33b989224d03814917863438edc5","cf8e2379ecb440b7b2c07c69d822a472","991a7802f4b84308acef1359242d890c","a02abb98273a4fa986cc481d2f2382e1","2a43ef920c9e434c866ee776239ec856","69f5b612ac8f4ff0994870e502861298","841795b7c9004457a0ffcbe6cba6dfaa","b53ee3efd6ab43699b7b587973aad756","98a1a3f03be345dabe2c7ecb14be754a","4195850835c349c780372a7bf96207e3","c05b926f16a04d379b31429b96a8ca28","c8a77da80dd94bb6abeb28ca7141cf8d","821e87b393c54cc9b742aa17ca1e2e7d","854237809d384b7eb20379b064bf893d","8780318cc75d46e7996362da743a9102","6929e03e070849a3949dd61cf5f71539"]},"executionInfo":{"status":"ok","timestamp":1794083622643,"user_tz":-60,"elapsed":3794,"user":{"displayName":"","userId":""}},"outputId":"9d91583c-da3e-4f31-930a-1dd364356b21","id":"p8bX6VyzuJdV"},"outputs":[{"output_type":"stream","name":"stdout","text":["Downloading gemma4_lora_to_review/95_percent from codeShare/kromoss-lora-exports...\n"]},{"output_type":"display_data","data":{"text/plain":["Downloading bytes:           |  0.00B            "],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"1abcbac011e84714b3aa1cccfbb4769d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Reconstructing (incomplete total...): |          |  0.00B /  0.00B            "],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"04e9c410e7c8472c8b9e514191114e3d"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Fetching 6 files:   0%|          | 0/6 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"69f5b612ac8f4ff0994870e502861298"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["\n","Found 6 files:\n","  adapter_model.safetensors\n","  adapter_config.json\n","  processor/chat_template.jinja\n","  processor/processor_config.json\n","  processor/tokenizer_config.json\n","  processor/tokenizer.json\n","\n","Uploading 6 files to codeShare/kromoss-lora-exports/gemma4_lora...\n","\n","Upload completed successfully!\n","Destination: https://huggingface.co/codeShare/kromoss-lora-exports/tree/main/gemma4_lora\n","Files uploaded: 6\n","Response: https://huggingface.co/codeShare/kromoss-lora-exports/commit/8816f362c66362e8dc0200e02af07913a10efc63\n"]}],"source":["import os\n","from pathlib import Path\n","from huggingface_hub import snapshot_download, HfApi, get_token\n","\n","# Configuration\n","SOURCE_REPO = \"codeShare/kromoss-lora-exports\"\n","SOURCE_FOLDER = \"gemma4_lora_to_review/95_percent\"\n","\n","# Destination Configuration\n","DEST_REPO = \"codeShare/kromoss-lora-exports\"\n","DEST_FOLDER = \"gemma4_lora\"\n","\n","# 1. Authenticate with Hugging Face\n","HF_TOKEN = os.environ.get(\"HF_TOKEN\") or get_token()\n","\n","if not HF_TOKEN:\n","    try:\n","        from google.colab import userdata\n","        HF_TOKEN = userdata.get(\"HF_TOKEN\")\n","    except Exception:\n","        pass\n","\n","if not HF_TOKEN:\n","    raise RuntimeError(\"Hugging Face API token is required.\")\n","\n","api = HfApi(token=HF_TOKEN)\n","\n","# 2. Download ALL files from the source folder\n","print(f\"Downloading {SOURCE_FOLDER} from {SOURCE_REPO}...\")\n","\n","local_download_dir = Path(\"./downloaded_adapter_95\")\n","\n","snapshot_download(\n","    repo_id=SOURCE_REPO,\n","    repo_type=\"model\",\n","    allow_patterns=[\n","        f\"{SOURCE_FOLDER}/**\",\n","        f\"{SOURCE_FOLDER}/*\"\n","    ],\n","    local_dir=str(local_download_dir),\n","    token=HF_TOKEN\n",")\n","\n","actual_folder = local_download_dir / SOURCE_FOLDER\n","\n","if not actual_folder.exists():\n","    raise FileNotFoundError(\n","        f\"Could not locate downloaded folder: {actual_folder}\"\n","    )\n","\n","# 3. Collect EVERY file recursively\n","files_to_upload = [\n","    p for p in actual_folder.rglob(\"*\")\n","    if p.is_file()\n","]\n","\n","if not files_to_upload:\n","    raise RuntimeError(\"No files found to upload!\")\n","\n","print(f\"\\nFound {len(files_to_upload)} files:\")\n","\n","for p in files_to_upload:\n","    print(f\"  {p.relative_to(actual_folder)}\")\n","\n","# 4. Ensure destination repository exists (since you already own this repository, this step ensures it exists)\n","api.create_repo(\n","    repo_id=DEST_REPO,\n","    repo_type=\"model\",\n","    private=False,\n","    exist_ok=True\n",")\n","\n","# 5. Upload ALL files directly to the subfolder in the destination repository\n","print(f\"\\nUploading {len(files_to_upload)} files to {DEST_REPO}/{DEST_FOLDER}...\")\n","\n","upload_response = api.upload_folder(\n","    repo_id=DEST_REPO,\n","    repo_type=\"model\",\n","    folder_path=str(actual_folder),\n","    path_in_repo=DEST_FOLDER,\n","    commit_message=(\n","        f\"Upload all files from {SOURCE_FOLDER} into {DEST_FOLDER}\"\n","    ),\n","    ignore_patterns=[]\n",")\n","\n","print(\"\\nUpload completed successfully!\")\n","print(\"Destination:\", f\"https://huggingface.co/{DEST_REPO}/tree/main/{DEST_FOLDER}\")\n","print(\"Files uploaded:\", len(files_to_upload))\n","print(\"Response:\", upload_response)"]}]}