add LoRA metadata v2

jupyter_notebooks/Section_2-4_Figure_9_ectract_LoRA_metadata_v2.ipynb

@@ -0,0 +1,400 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "f36422c8",
+   "metadata": {},
+   "source": [
+    "# LoRA metadata"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "8a2feb6e",
+   "metadata": {},
+   "source": [
+    "LoRA Metadata Processing Workflow\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Load CSV File        │ --> │ Read adapter metadata CSV file.                   │\n",
+    "│ Read Model Versions  │     │ Extract model version IDs and relevant data.      │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Download Adapter     │ --> │ Use stored download URLs to fetch adapter files   │\n",
+    "│ Files Using API      │     │ using rotating API keys.                          │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Parse Metadata       │ --> │ Extract safetensors metadata, such as training    │\n",
+    "│ from SafeTensor      │     │ images, model type, and architecture.             │\n",
+    "│ Files                │     │                                                   │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Store Parsed         │ --> │ Save extracted metadata into structured JSON      │\n",
+    "│ Metadata as JSON     │     │ files for later analysis.                         │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Process JSON Files   │ --> │ Read saved JSON metadata, extract relevant        │\n",
+    "│ for Consolidation    │     │ details, and filter necessary attributes.         │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Extract Training     │ --> │ Identify most frequent training tags, architectures│\n",
+    "│ Tags & Model Info    │     │ and systems used for model creation.               │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Save Consolidated    │ --> │ Store all processed metadata in a structured CSV  │\n",
+    "│ Metadata to CSV      │     │ format for final analysis.                        ���\n",
+    "└──────────────────────┘     └───────────────────────────────────────────────────┘\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "efc9939d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import re\n",
+    "import json\n",
+    "import csv\n",
+    "import struct\n",
+    "import requests\n",
+    "from pathlib import Path\n",
+    "import pandas as pd\n",
+    "from collections import Counter\n",
+    "from concurrent.futures import ProcessPoolExecutor\n",
+    "from pathlib import Path\n",
+    "import matplotlib.pyplot as plt\n",
+    "from matplotlib.font_manager import FontProperties\n",
+    "from matplotlib import font_manager\n",
+    "import pandas as pd\n",
+    "from collections import Counter\n",
+    "from concurrent.futures import ProcessPoolExecutor\n",
+    "\n",
+    "# Define the current directory and important file paths\n",
+    "current_dir = Path.cwd()\n",
+    "\n",
+    "# Define frequently used directories\n",
+    "\n",
+    "data_dir = current_dir.parent / 'data/csv/adapters.csv'\n",
+    "fonts_dir = current_dir.parent / 'misc/assets/fonts'\n",
+    "plots_dir = current_dir.parent / 'results/plots'\n",
+    "raw_data_dir = current_dir.parent / 'data/adapter_metadata/lora'  ### location of the LoRA metadata (JSON)\n",
+    "temp_dir = current_dir.parent / 'data/raw/adapters_safetensors'\n",
+    "misc_dir = current_dir.parent / 'misc'\n",
+    "\n",
+    "# File paths\n",
+    "adapters_csv = current_dir.parent / 'data/csv/adapters.csv'\n",
+    "output_json_dir = raw_data_dir\n",
+    "api_keys_file = misc_dir / 'credentials/civit.txt'\n",
+    "\n",
+    "# Ensure directories exist\n",
+    "os.makedirs(output_json_dir, exist_ok=True)\n",
+    "os.makedirs(temp_dir, exist_ok=True)\n",
+    "\n",
+    "\n",
+    "# Load fonts into Matplotlib\n",
+    "for font_path in font_paths:\n",
+    "    font_manager.fontManager.addfont(font_path)\n",
+    "\n",
+    "# Set default font family for plots\n",
+    "plt.rcParams['font.family'] = ['Noto Sans JP', 'Noto Sans SC', 'sans-serif']\n",
+    "\n",
+    "print('Paths and fonts initialized successfully.')\n",
+    "\n",
+    "print('Paths initialized successfully.')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "87a58593",
+   "metadata": {},
+   "source": [
+    "## Step 2: Download LoRA and extract *.safetensors metadata\n",
+    "This script downloads LoRA adapters from the filtered Civiverse-Models dataset and extracts the metadata found within the *.safetensors' data structure"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "abd3a0bc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "import csv\n",
+    "import json\n",
+    "import struct\n",
+    "import time\n",
+    "import requests\n",
+    "import signal\n",
+    "import contextlib\n",
+    "from pathlib import Path\n",
+    "import re\n",
+    "\n",
+    "# === Paste your API keys here ===\n",
+    "API_KEYS = [\n",
+    "    \"REDACTED-CIVITAI-API-KEY-1\",  #DISCORD\n",
+    "    \"REDACTED-CIVITAI-API-KEY-2\", #ASDD 1\n",
+    "    \"REDACTED-CIVITAI-API-KEY-3\", #ASDD 2\n",
+    "    \"REDACTED-CIVITAI-API-KEY-4\", #BSDD \n",
+    "    \"REDACTED-CIVITAI-API-KEY-5\"\n",
+    "]\n",
+    "if not API_KEYS or any(not isinstance(k, str) or not k.strip() for k in API_KEYS):\n",
+    "    raise ValueError(\"Please paste at least one valid API key into API_KEYS.\")\n",
+    "\n",
+    "# === Config (adjust paths as needed) ===\n",
+    "current_dir = Path.cwd()\n",
+    "output_json_dir = current_dir.parent / \"data/adapter_metadata/lora\"        # where JSON outputs go\n",
+    "temp_dir = current_dir.parent / \"data/raw/adapters_safetensors\"                            # where downloads go\n",
+    "csv_path = current_dir.parent / \"data/csv/adapters_poi_false_sfw.csv\"\n",
+    "\n",
+    "os.makedirs(output_json_dir, exist_ok=True)\n",
+    "os.makedirs(temp_dir, exist_ok=True)\n",
+    "\n",
+    "# === API key state ===\n",
+    "current_key_index = 0\n",
+    "\n",
+    "\n",
+    "\n",
+    "def safe_filename(name: str, max_length: int = 100) -> str:\n",
+    "    # Replace unsafe chars\n",
+    "    sanitized = re.sub(r'[^a-zA-Z0-9_\\-]', '_', name)\n",
+    "    # Truncate if too long\n",
+    "    if len(sanitized) > max_length:\n",
+    "        sanitized = sanitized[:max_length]\n",
+    "    return sanitized\n",
+    "\n",
+    "\n",
+    "def get_headers():\n",
+    "    global current_key_index\n",
+    "    return {\n",
+    "        \"Accept\": \"application/json\",\n",
+    "        \"Authorization\": f\"Bearer {API_KEYS[current_key_index].strip()}\"\n",
+    "    }\n",
+    "\n",
+    "def rotate_api_key():\n",
+    "    global current_key_index\n",
+    "    if current_key_index < len(API_KEYS) - 1:\n",
+    "        current_key_index += 1\n",
+    "        print(f\"🔁 Rotated to API key #{current_key_index + 1}\")\n",
+    "    else:\n",
+    "        raise Exception(\"All API keys have been exhausted.\")\n",
+    "\n",
+    "# === Utilities ===\n",
+    "def save_json(data, filename):\n",
+    "    with open(filename, 'w', encoding=\"utf-8\") as f:\n",
+    "        json.dump(data, f, indent=4, ensure_ascii=False)\n",
+    "\n",
+    "def parse_safetensors(file_path):\n",
+    "    # Minimal, tolerant metadata reader; returns {} on failure.\n",
+    "    try:\n",
+    "        with open(file_path, 'rb') as f:\n",
+    "            file_data = f.read()\n",
+    "        # Many safetensors use 8-byte header length; this code follows your original logic\n",
+    "        # (4-byte) but keeps the 8-byte skip. Keep if it's working in your dataset.\n",
+    "        metadata_size = struct.unpack('<I', file_data[:4])[0]\n",
+    "        metadata_bytes = file_data[8:8 + metadata_size]\n",
+    "        metadata_str = metadata_bytes.decode('utf-8', errors='replace')\n",
+    "        metadata = json.loads(metadata_str)\n",
+    "        return metadata.get('__metadata__', {})\n",
+    "    except Exception as e:\n",
+    "        print(f\"Error parsing safetensors file: {e}\")\n",
+    "        return {}\n",
+    "\n",
+    "# === Timeout context ===\n",
+    "class TimeoutException(Exception):\n",
+    "    pass\n",
+    "\n",
+    "@contextlib.contextmanager\n",
+    "def time_limit(seconds):\n",
+    "    def signal_handler(signum, frame):\n",
+    "        raise TimeoutException(f\"Timed out after {seconds} seconds\")\n",
+    "    # Note: SIGALRM works on Unix-like OS; on Windows this will be a no-op.\n",
+    "    try:\n",
+    "        signal.signal(signal.SIGALRM, signal_handler)\n",
+    "        signal.alarm(seconds)\n",
+    "    except Exception:\n",
+    "        # Fallback: no hard alarm on non-Unix systems\n",
+    "        pass\n",
+    "    try:\n",
+    "        yield\n",
+    "    finally:\n",
+    "        try:\n",
+    "            signal.alarm(0)\n",
+    "        except Exception:\n",
+    "            pass\n",
+    "\n",
+    "# === Download with timeout, retries, backoff, and key rotation ===\n",
+    "def download_file(url, output_folder, timeout=30, overall_timeout=120, max_retries=3):\n",
+    "    filename = url.split(\"/\")[-1]\n",
+    "    output_path = os.path.join(output_folder, filename)\n",
+    "\n",
+    "    global current_key_index\n",
+    "    retries = 0\n",
+    "    backoff = 2\n",
+    "\n",
+    "    while current_key_index < len(API_KEYS):\n",
+    "        try:\n",
+    "            with time_limit(overall_timeout):  # global cap per download\n",
+    "                #print(f\"➡️  GET {url} using key #{current_key_index + 1}\")\n",
+    "                resp = requests.get(\n",
+    "                    url,\n",
+    "                    headers=get_headers(),\n",
+    "                    stream=True,\n",
+    "                    timeout=(10, timeout),  # (connect timeout, per-chunk read timeout)\n",
+    "                )\n",
+    "\n",
+    "                # Auth errors → rotate key\n",
+    "                if resp.status_code in (401, 403):\n",
+    "                    print(f\"❌ Auth {resp.status_code} with key #{current_key_index + 1}. Rotating.\")\n",
+    "                    rotate_api_key()\n",
+    "                    retries = 0\n",
+    "                    backoff = 2\n",
+    "                    continue\n",
+    "\n",
+    "                # Not found → bubble up as FileNotFoundError (do not rotate)\n",
+    "                if resp.status_code == 404:\n",
+    "                    raise FileNotFoundError(f\"Model not found at {url}\")\n",
+    "\n",
+    "                # Rate limit → either rotate or wait/backoff\n",
+    "                if resp.status_code == 429:\n",
+    "                    print(\"⏳ Rate limited (429).\", end=\" \")\n",
+    "                    if current_key_index < len(API_KEYS) - 1:\n",
+    "                        print(\"Rotating key.\")\n",
+    "                        rotate_api_key()\n",
+    "                        retries = 0\n",
+    "                        backoff = 2\n",
+    "                        continue\n",
+    "                    else:\n",
+    "                        print(f\"Waiting {backoff}s (no other keys).\")\n",
+    "                        time.sleep(backoff)\n",
+    "                        backoff = min(backoff * 2, 60)\n",
+    "                        continue\n",
+    "\n",
+    "                # Other HTTP errors → raise to RequestException path\n",
+    "                resp.raise_for_status()\n",
+    "\n",
+    "                # Save file\n",
+    "                with open(output_path, 'wb') as fh:\n",
+    "                    for chunk in resp.iter_content(chunk_size=8192):\n",
+    "                        if chunk:\n",
+    "                            fh.write(chunk)\n",
+    "\n",
+    "                return output_path, filename\n",
+    "\n",
+    "        except TimeoutException as e:\n",
+    "            # Hard overall timeout → propagate\n",
+    "            raise e\n",
+    "        except requests.exceptions.RequestException as e:\n",
+    "            # Network-ish errors: retry same key with backoff up to max_retries\n",
+    "            retries += 1\n",
+    "            if retries <= max_retries:\n",
+    "                print(f\"🌐 Network error (try {retries}/{max_retries}) with key #{current_key_index + 1}: {e}\")\n",
+    "                time.sleep(backoff)\n",
+    "                backoff = min(backoff * 2, 60)\n",
+    "                continue\n",
+    "            else:\n",
+    "                raise Exception(f\"Failed to download {url} after {max_retries} retries: {e}\")\n",
+    "\n",
+    "    # If we exit the loop, we truly ran out\n",
+    "    raise Exception(\"All API keys have been exhausted or failed.\")\n",
+    "\n",
+    "# === Main processing ===\n",
+    "def process_csv(csv_path):\n",
+    "    with open(csv_path, newline='', encoding='utf-8') as csvfile:\n",
+    "        reader = csv.DictReader(csvfile)\n",
+    "        for index, row in enumerate(reader):\n",
+    "            # Collect up to 20 version IDs; use the most recent\n",
+    "            version_ids = []\n",
+    "            for i in range(1, 21):\n",
+    "                k = f'version_id_{i}'\n",
+    "                if k in row and row[k]:\n",
+    "                    try:\n",
+    "                        version_ids.append(int(float(row[k])))\n",
+    "                    except ValueError:\n",
+    "                        print(f\"Invalid version_id value '{row[k]}' in row: {row}\")\n",
+    "\n",
+    "            if not version_ids:\n",
+    "                print(f\"No valid version IDs found in row: {row}\")\n",
+    "                continue\n",
+    "\n",
+    "            most_recent_version_id = str(max(version_ids))\n",
+    "            name = row.get('name', 'unknown')\n",
+    "            sanitized_name = safe_filename(name, max_length=100)\n",
+    "            new_json_file = os.path.join(\n",
+    "                output_json_dir,\n",
+    "                f\"{index:08d}_{most_recent_version_id}_{sanitized_name}.json\"\n",
+    "            )\n",
+    "\n",
+    "            # Skip if JSON already exists\n",
+    "            if os.path.exists(new_json_file):\n",
+    "                #print(f\"↩️  Skipping versionID {most_recent_version_id} (JSON already exists)\")\n",
+    "                continue\n",
+    "\n",
+    "            try:\n",
+    "                adapter_file, fname = download_file(\n",
+    "                    row['downloadUrl'], str(temp_dir),\n",
+    "                    timeout=30, overall_timeout=180\n",
+    "                )\n",
+    "                metadata = parse_safetensors(adapter_file)\n",
+    "\n",
+    "                civitaidata = {\n",
+    "                    k: (int(v) if str(v).isdigit() else v)\n",
+    "                    for k, v in row.items()\n",
+    "                }\n",
+    "                new_json_data = {\n",
+    "                    \"civitaidata\": civitaidata,\n",
+    "                    \"metadata\": metadata,\n",
+    "                    \"versionID\": most_recent_version_id\n",
+    "                }\n",
+    "                save_json(new_json_data, new_json_file)\n",
+    "                #print(f\"✅ Created JSON for versionID {most_recent_version_id} with file {fname}\")\n",
+    "\n",
+    "            except FileNotFoundError as e:\n",
+    "                print(f\"⚠️ {e} — saving empty metadata.\")\n",
+    "                civitaidata = {\n",
+    "                    k: (int(v) if str(v).isdigit() else v)\n",
+    "                    for k, v in row.items()\n",
+    "                }\n",
+    "                empty_json = {\n",
+    "                    \"civitaidata\": civitaidata,\n",
+    "                    \"metadata\": {},\n",
+    "                    \"versionID\": most_recent_version_id,\n",
+    "                    \"error\": \"Model not found (404)\"\n",
+    "                }\n",
+    "                save_json(empty_json, new_json_file)\n",
+    "            except Exception as e:\n",
+    "                print(f\"⚠️ Error processing versionID {most_recent_version_id}: {e}\")\n",
+    "                civitaidata = {\n",
+    "                    k: (int(v) if str(v).isdigit() else v)\n",
+    "                    for k, v in row.items()\n",
+    "                }\n",
+    "                empty_json = {\n",
+    "                    \"civitaidata\": civitaidata,\n",
+    "                    \"metadata\": {},\n",
+    "                    \"versionID\": most_recent_version_id,\n",
+    "                    \"error\": str(e)\n",
+    "                }\n",
+    "                save_json(empty_json, new_json_file)\n",
+    "                print(f\"💾 Saved empty JSON for versionID {most_recent_version_id} due to failure.\")\n",
+    "\n",
+    "# === Run ===\n",
+    "if __name__ == \"__main__\":\n",
+    "    process_csv(csv_path)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}