added qwen query code

jupyter_notebooks/.ipynb_checkpoints/GEMMA_3-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

jupyter_notebooks/.ipynb_checkpoints/MISTRAL-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

jupyter_notebooks/.ipynb_checkpoints/QWEN-checkpoint.ipynb

@@ -0,0 +1,465 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "0543d9c4-055b-49a3-a8d0-9dfb622b2b8c",
+   "metadata": {},
+   "source": [
+    "# QWEN CODE"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fe6ba282-896b-4272-b82b-ef24810732fb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "import torch\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "import signal\n",
+    "from contextlib import contextmanager\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
+    "professions_mapped_file = current_dir.parent / \"misc/lists/professions_mapped.csv\"\n",
+    "# === PROCESS DATA ===\n",
+    "\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = False\n",
+    "TEST_SIZE = 100\n",
+    "MAX_ROWS = 50862\n",
+    "SAVE_INTERVAL = 10\n",
+    "\n",
+    "\n",
+    "index_file = current_dir.parent / \"misc/query_indicies/qwen_local_query_index.txt\"\n",
+    "output_file = current_dir.parent / f\"data/CSV/qwen_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "\n",
+    "# Model settings\n",
+    "MODEL_NAME =  \"Qwen/Qwen3-235B-A22B-Instruct-2507-FP8\"\n",
+    "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
+    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
+    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Define the SPECIFIC profession categories\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\",\n",
+    "    \"adult performer\",\n",
+    "    \"singer/musician\",\n",
+    "    \"model\",\n",
+    "    \"online personality\",\n",
+    "    \"public figure\",\n",
+    "    \"voice actor/ASMR\",\n",
+    "    \"sports professional\",\n",
+    "    \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD MODEL ===\n",
+    "print(f\"Loading model: {MODEL_NAME}\")\n",
+    "print(f\"Cache directory: {CACHE_DIR}\")\n",
+    "print(f\"This may take a while on first run (~65GB download)...\\n\")\n",
+    "\n",
+    "# Check GPU availability\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "print(f\"Device: {device}\")\n",
+    "\n",
+    "if device == \"cpu\":\n",
+    "    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
+    "    print(\"   Consider using a GPU or reducing model size.\")\n",
+    "\n",
+    "# Load tokenizer\n",
+    "print(\"Loading tokenizer...\")\n",
+    "try:\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=True\n",
+    "    )\n",
+    "except Exception as e:\n",
+    "    print(f\"Failed with use_fast=True, trying use_fast=False...\")\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=False\n",
+    "    )\n",
+    "\n",
+    "# Ensure pad token is set\n",
+    "if tokenizer.pad_token is None:\n",
+    "    tokenizer.pad_token = tokenizer.eos_token\n",
+    "\n",
+    "print(\"✅ Tokenizer loaded\")\n",
+    "\n",
+    "# Load model with optimizations\n",
+    "print(\"Loading model (this may take several minutes)...\")\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    MODEL_NAME,\n",
+    "    cache_dir=str(CACHE_DIR),\n",
+    "    torch_dtype=torch.bfloat16,\n",
+    "    device_map=\"cuda:0\",  # Force single GPU instead of \"auto\"\n",
+    "    trust_remote_code=False,\n",
+    "    offload_folder=str(CACHE_DIR / \"offload\"),  # Explicit offload folder\n",
+    "    offload_state_dict=False  # Try to keep everything on GPU\n",
+    ")\n",
+    "model.eval()\n",
+    "print(\"✅ Model loaded\")\n",
+    "\n",
+    "# Load model with optimizations\n",
+    "print(\"Loading model (this may take several minutes)...\")\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    MODEL_NAME,\n",
+    "    cache_dir=str(CACHE_DIR),\n",
+    "    torch_dtype=torch.bfloat16,\n",
+    "    device_map=\"auto\",\n",
+    "    trust_remote_code=False\n",
+    ")\n",
+    "model.eval()\n",
+    "\n",
+    "# FORCE COMPLETE LOADING\n",
+    "print(\"Ensuring model is fully loaded to device...\")\n",
+    "for name, param in tqdm(model.named_parameters(), desc=\"Loading parameters\"):\n",
+    "    if param.device.type == \"meta\":\n",
+    "        # This parameter is on meta device and needs to be materialized\n",
+    "        print(f\"Warning: {name} is on meta device\")\n",
+    "\n",
+    "# Try to trigger full loading with a dummy forward pass\n",
+    "print(\"Running dummy forward pass to ensure full loading...\")\n",
+    "dummy_input = torch.tensor([[1]], device=\"cuda\")\n",
+    "with torch.no_grad():\n",
+    "    try:\n",
+    "        _ = model(dummy_input)\n",
+    "    except Exception as e:\n",
+    "        print(f\"Dummy forward pass failed (expected): {e}\")\n",
+    "\n",
+    "print(\"✅ Model fully loaded and ready\")\n",
+    "\n",
+    "\n",
+    "# Check VRAM usage\n",
+    "if torch.cuda.is_available():\n",
+    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
+    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "if output_file.exists():\n",
+    "    print(\"Loading annotated CSV...\")\n",
+    "    df = pd.read_csv(output_file)\n",
+    "else:\n",
+    "    print(\"Loading raw input CSV...\")\n",
+    "    df = pd.read_csv(input_file)\n",
+    "\n",
+    "\n",
+    "# Try to load profession mapping files\n",
+    "try:\n",
+    "    professions_df = pd.read_csv(professions_file)\n",
+    "    print(f\"✅ Loaded professions.csv\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions.csv not found\")\n",
+    "\n",
+    "try:\n",
+    "    prof_mapped_df = pd.read_csv(professions_mapped_file)\n",
+    "    print(f\"✅ Loaded profession mapping with {len(prof_mapped_df)} categories\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions_mapped.csv not found, using default categories\")\n",
+    "\n",
+    "profession_str = \", \".join(PROFESSION_CATEGORIES)\n",
+    "\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "print(f\"\\nProfession categories ({len(PROFESSION_CATEGORIES)}):\")\n",
+    "for cat in PROFESSION_CATEGORIES:\n",
+    "    print(f\"  - {cat}\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"\\nRunning in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
+    "\n",
+    "# === CREATE PROMPTS (DEEPSEEK STYLE) ===\n",
+    "def create_prompt(row):\n",
+    "    \"\"\"Create prompt for Qwen annotation with specific profession categories.\"\"\"\n",
+    "    name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
+    "    \n",
+    "    # Gather hints\n",
+    "    hints = []\n",
+    "    if pd.notna(row.get('likely_profession')):\n",
+    "        hints.append(str(row['likely_profession']))\n",
+    "    if pd.notna(row.get('likely_nationality')):\n",
+    "        hints.append(str(row['likely_nationality']))\n",
+    "    if pd.notna(row.get('likely_country')):\n",
+    "        hints.append(str(row['likely_country']))\n",
+    "    \n",
+    "    # Add tags if we don't have enough hints\n",
+    "    if len(hints) < 3:\n",
+    "        for i in range(1, 8):\n",
+    "            tag_col = f'tag_{i}'\n",
+    "            if tag_col in row and pd.notna(row[tag_col]):\n",
+    "                tag_val = str(row[tag_col])\n",
+    "                if tag_val not in hints:\n",
+    "                    hints.append(tag_val)\n",
+    "                if len(hints) >= 5:\n",
+    "                    break\n",
+    "    \n",
+    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
+    "    \n",
+    "    return f\"\"\"Given '{name}' ({hint_text}), provide:\n",
+    "1. Full legal name (Western order if non-latin script)\n",
+    "2. Any stage names/aliases (comma separated)\n",
+    "3. Gender (Male/Female/Other/Unknown)\n",
+    "4. Top 3 most likely professions from ONLY these categories:\n",
+    "   - actor\n",
+    "   - adult performer\n",
+    "   - singer/musician\n",
+    "   - model\n",
+    "   - online personality (includes streamers, cosplayers, influencers)\n",
+    "   - public figure (includes politicians, activists, journalists, authors)\n",
+    "   - voice actor/ASMR\n",
+    "   - sports professional\n",
+    "   - tv personality (includes hosts, presenters, reality TV)\n",
+    "\n",
+    "5. Primary country associated\n",
+    "\n",
+    "IMPORTANT:\n",
+    "- Choose professions ONLY from the 9 categories above\n",
+    "- Provide up to 3 professions, comma-separated, ordered by relevance\n",
+    "- Be SPECIFIC: choose the most accurate category for each role\n",
+    "- \"online personality\" includes: streamers, cosplayers, YouTubers, influencers, content creators\n",
+    "- Use 'Unknown' when uncertain or for fictional characters/places\n",
+    "- For multi-role people, list all relevant categories (e.g., \"actor, singer/musician, online personality\")\n",
+    "- For country respond with one word only, for example China or Columbia\n",
+    "- actress = actor\n",
+    "\n",
+    "Respond with exactly 5 numbered lines.\"\"\"\n",
+    "\n",
+    "# Create prompts\n",
+    "print(\"\\nCreating prompts...\")\n",
+    "df['prompt'] = df.apply(create_prompt, axis=1)\n",
+    "print(\"✅ Prompts created\")\n",
+    "\n",
+    "@contextmanager\n",
+    "def timeout(duration):\n",
+    "    def handler(signum, frame):\n",
+    "        raise TimeoutError(\"Operation timed out\")\n",
+    "    \n",
+    "    # Set the signal handler and alarm\n",
+    "    signal.signal(signal.SIGALRM, handler)\n",
+    "    signal.alarm(duration)\n",
+    "    try:\n",
+    "        yield\n",
+    "    finally:\n",
+    "        signal.alarm(0)  # Disable the alarm\n",
+    "\n",
+    "\n",
+    "def query_qwen_local(prompt: str) -> str:\n",
+    "    \"\"\"Query Qwen locally via transformers.\"\"\"\n",
+    "    try:\n",
+    "        with timeout(60):  # 60 second timeout\n",
+    "            # Your existing generation code here\n",
+    "            ...\n",
+    "    except TimeoutError:\n",
+    "        print(f\"[ERROR] Generation timed out after 60 seconds\")\n",
+    "        return None\n",
+    "    except Exception as e:\n",
+    "        print(f\"Generation error: {e}\")\n",
+    "        return None\n",
+    "        \n",
+    "        # Format as chat message for Qwen\n",
+    "        messages = [\n",
+    "            {\"role\": \"system\", \"content\": \"You are an assistant that extracts key data on a person based on the name. Respond with exactly 5 numbered lines. For professions, choose ONLY from these categories: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality.\"},\n",
+    "            {\"role\": \"user\", \"content\": prompt}\n",
+    "        ]\n",
+    "        \n",
+    "        print(f\"[DEBUG] Applying chat template...\")\n",
+    "        # Tokenize\n",
+    "        if hasattr(tokenizer, 'apply_chat_template'):\n",
+    "            text = tokenizer.apply_chat_template(\n",
+    "                messages,\n",
+    "                tokenize=False,\n",
+    "                add_generation_prompt=True\n",
+    "            )\n",
+    "        else:\n",
+    "            # Fallback for older tokenizers\n",
+    "            text = f\"[INST] {prompt} [/INST]\"\n",
+    "        \n",
+    "        print(f\"[DEBUG] Tokenizing input...\")\n",
+    "        inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
+    "        \n",
+    "        print(f\"[DEBUG] Input shape: {inputs['input_ids'].shape}\")\n",
+    "        print(f\"[DEBUG] Starting generation...\")\n",
+    "        \n",
+    "        # Generate\n",
+    "        with torch.no_grad():\n",
+    "            outputs = model.generate(\n",
+    "                **inputs,\n",
+    "                max_new_tokens=100,\n",
+    "                temperature=0.1,\n",
+    "                do_sample=False,\n",
+    "                #top_p=0.9,\n",
+    "                pad_token_id=tokenizer.eos_token_id\n",
+    "            )\n",
+    "        \n",
+    "        print(f\"[DEBUG] Generation complete, decoding...\")\n",
+    "        \n",
+    "        # Decode\n",
+    "        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
+    "        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
+    "        \n",
+    "        print(f\"[DEBUG] Response: {response[:100]}...\")  # First 100 chars\n",
+    "        \n",
+    "        return response.strip()\n",
+    "        \n",
+    "    except Exception as e:\n",
+    "        print(f\"Generation error: {e}\")\n",
+    "        import traceback\n",
+    "        traceback.print_exc()\n",
+    "        return None\n",
+    "\n",
+    "#output_file = current_dir.parent / f\"data/CSV/qwen_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "#index_file = current_dir.parent / \"misc/query_indicies/qwen_local_query_index.txt\"\n",
+    "\n",
+    "\n",
+    "    \n",
+    "# === PARSE RESPONSE (DEEPSEEK STYLE) ===\n",
+    "def parse_response(response):\n",
+    "    \"\"\"Parse Qwen response into structured fields.\"\"\"\n",
+    "    if not response:\n",
+    "        return {\n",
+    "            'full_name': 'Unknown',\n",
+    "            'aliases': 'Unknown',\n",
+    "            'gender': 'Unknown',\n",
+    "            'profession_llm': 'Unknown',\n",
+    "            'country': 'Unknown'\n",
+    "        }\n",
+    "    \n",
+    "    # Split into lines and clean\n",
+    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
+    "    \n",
+    "    # Initialize with Unknown values\n",
+    "    fields = {\n",
+    "        'full_name': 'Unknown',\n",
+    "        'aliases': 'Unknown',\n",
+    "        'gender': 'Unknown',\n",
+    "        'profession_llm': 'Unknown',\n",
+    "        'country': 'Unknown'\n",
+    "    }\n",
+    "    \n",
+    "    # Extract information from each numbered line\n",
+    "    for line in lines:\n",
+    "        if line.startswith('1.'):\n",
+    "            fields['full_name'] = line[2:].strip()\n",
+    "        elif line.startswith('2.'):\n",
+    "            fields['aliases'] = line[2:].strip()\n",
+    "        elif line.startswith('3.'):\n",
+    "            fields['gender'] = line[2:].strip()\n",
+    "        elif line.startswith('4.'):\n",
+    "            fields['profession_llm'] = line[2:].strip()\n",
+    "        elif line.startswith('5.'):\n",
+    "            fields['country'] = line[2:].strip()\n",
+    "    \n",
+    "    return fields\n",
+    "\n",
+    "# === PROCESS DATA ===\n",
+    "index_file.parent.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Load index\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    try:\n",
+    "        current_index = int(index_file.read_text().strip())\n",
+    "    except:\n",
+    "        current_index = 0\n",
+    "\n",
+    "print(f\"Resuming from index {current_index}\")\n",
+    "\n",
+    "start_time = time.time()\n",
+    "\n",
+    "for i in tqdm(range(current_index, len(df)), desc=\"Qwen Local\"):\n",
+    "\n",
+    "    prompt = df.at[i, \"prompt\"]\n",
+    "\n",
+    "    # -------- MODEL QUERY WITH RETRIES --------\n",
+    "    response = None\n",
+    "    for attempt in range(3):\n",
+    "        response = query_qwen_local(prompt)\n",
+    "        \n",
+    "        # Valid response?\n",
+    "        if response and len(response.strip()) > 10:\n",
+    "            break\n",
+    "        \n",
+    "        print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
+    "        time.sleep(0.5)\n",
+    "\n",
+    "    # If still invalid → DO NOT overwrite previous data\n",
+    "    if not response or len(response.strip()) <= 10:\n",
+    "        print(f\"❌ Row {i}: failed after retries, not writing, not advancing index\")\n",
+    "        continue\n",
+    "\n",
+    "    parsed = parse_response(response)\n",
+    "\n",
+    "    # Additional safety: skip rows that parsed as all 'Unknown'\n",
+    "    if all(v == \"Unknown\" for v in parsed.values()):\n",
+    "        print(f\"❌ Row {i}: parsed as all Unknown (likely model crash); skipping.\")\n",
+    "        continue\n",
+    "\n",
+    "    # -------- WRITE PARSED FIELDS SAFELY --------\n",
+    "    for key, value in parsed.items():\n",
+    "        df.at[i, key] = value\n",
+    "\n",
+    "    # Advance progress ONLY after successful write\n",
+    "    current_index = i + 1\n",
+    "\n",
+    "    # -------- GPU MEMORY CLEANUP --------\n",
+    "    if torch.cuda.is_available():\n",
+    "        torch.cuda.empty_cache()\n",
+    "        torch.cuda.synchronize()\n",
+    "\n",
+    "    # -------- SAVE LIKE YOUR DEEPSEEK VERSION --------\n",
+    "    if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "        df.to_csv(output_file, index=False)\n",
+    "        with open(index_file, \"w\") as f:\n",
+    "            f.write(str(current_index))\n",
+    "        print(f\"💾 Progress saved after row {i+1}\")\n",
+    "\n",
+    "# Final save\n",
+    "df.to_csv(output_file, index=False)\n",
+    "index_file.write_text(str(current_index))\n",
+    "print(\"✅ Finished full dataset.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d9c7deb9-847a-472d-8055-f93dbfa6aa2e",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "pm-paper",
+   "language": "python",
+   "name": "pm-paper"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

jupyter_notebooks/.ipynb_checkpoints/Section_2-2-2_Figure_4_Demographic_patterns_in_gen_images-checkpoint.ipynb

@@ -0,0 +1,1795 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d8161f06-4fd2-436d-9e59-3f68b5a67f2c",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T18:30:16.974712Z",
+     "iopub.status.busy": "2025-02-06T18:30:16.974296Z",
+     "iopub.status.idle": "2025-02-06T18:30:16.976909Z",
+     "shell.execute_reply": "2025-02-06T18:30:16.976526Z",
+     "shell.execute_reply.started": "2025-02-06T18:30:16.974692Z"
+    }
+   },
+   "source": [
+    "# Section 6.2:  Age and Gender Estimation using MiVOLO"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "64aaedec-ef56-4a62-b61e-12de2675a1ae",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T19:52:51.171282Z",
+     "iopub.status.busy": "2025-02-06T19:52:51.170711Z",
+     "iopub.status.idle": "2025-02-06T19:52:55.405039Z",
+     "shell.execute_reply": "2025-02-06T19:52:55.404308Z",
+     "shell.execute_reply.started": "2025-02-06T19:52:51.171245Z"
+    }
+   },
+   "source": [
+    "![Alt text](../plots/mivolo.svg)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "4293f307-44fd-455e-90fe-6e6928be9af5",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T21:59:21.970807Z",
+     "iopub.status.busy": "2025-02-08T21:59:21.969931Z",
+     "iopub.status.idle": "2025-02-08T22:00:09.724295Z",
+     "shell.execute_reply": "2025-02-08T22:00:09.723583Z",
+     "shell.execute_reply.started": "2025-02-08T21:59:21.970784Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/lauwag/data/conda/envs/horde/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import csv\n",
+    "from pathlib import Path\n",
+    "import logging\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import requests\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "import cv2\n",
+    "from io import BytesIO\n",
+    "from PIL import Image, UnidentifiedImageError\n",
+    "from datetime import datetime, timedelta\n",
+    "from dateutil.relativedelta import relativedelta\n",
+    "from mivolo.predictor import Predictor\n",
+    "import matplotlib.pyplot as plt\n",
+    "import matplotlib.patches as mpatches"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "63a54f5f-900c-48dd-8932-2632e56c5670",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T22:00:09.726069Z",
+     "iopub.status.busy": "2025-02-08T22:00:09.725699Z",
+     "iopub.status.idle": "2025-02-08T22:00:09.730626Z",
+     "shell.execute_reply": "2025-02-08T22:00:09.730099Z",
+     "shell.execute_reply.started": "2025-02-08T22:00:09.726050Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "current_dir = Path.cwd()\n",
+    "mini = current_dir.parent / 'data/CSV/image_subsets/Civiverse-mini.csv'\n",
+    "mivolo_in = current_dir.parent / 'data/CSV/image_subsets/Civiverse-mini-by-month/'\n",
+    "(current_dir.parent /  'data/CSV/image_subsets/Civiverse-mini-by-month/').mkdir(parents=True, exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1fdfb89a-6094-4382-8755-fae213221ea5",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T22:00:09.731540Z",
+     "iopub.status.busy": "2025-02-08T22:00:09.731359Z",
+     "iopub.status.idle": "2025-02-08T22:00:09.825738Z",
+     "shell.execute_reply": "2025-02-08T22:00:09.825258Z",
+     "shell.execute_reply.started": "2025-02-08T22:00:09.731524Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "def split_by_month(input_path, output_dir):\n",
+    "    # Load the dataset\n",
+    "    df = pd.read_csv(input_path)\n",
+    "    \n",
+    "    # Convert the 'createdAt' column to datetime\n",
+    "    df['createdAt'] = pd.to_datetime(df['createdAt'], errors='coerce')\n",
+    "    \n",
+    "    # Extract year and month\n",
+    "    df['year_month'] = df['createdAt'].dt.to_period('M')\n",
+    "    \n",
+    "    # Group the data by year and month and save each group as a CSV file\n",
+    "    unique_months = df['year_month'].unique()\n",
+    "\n",
+    "    for month in unique_months:\n",
+    "        # Filter data for the specific month\n",
+    "        df_month = df[df['year_month'] == month]\n",
+    "        \n",
+    "        # Define the file name based on the year and month\n",
+    "        file_name = f'{output_dir}/Civiverse-{month}.csv'\n",
+    "        \n",
+    "        # Save the file\n",
+    "        df_month.to_csv(file_name, index=False)\n",
+    "\n",
+    "    print(f\"Data has been split and saved to {output_dir}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "2c909d7c-7d16-4dc7-8364-7f1c0784414c",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T22:00:09.827095Z",
+     "iopub.status.busy": "2025-02-08T22:00:09.826919Z",
+     "iopub.status.idle": "2025-02-08T22:00:10.479484Z",
+     "shell.execute_reply": "2025-02-08T22:00:10.478777Z",
+     "shell.execute_reply.started": "2025-02-08T22:00:09.827079Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/sctmp/lauwag/ipykernel_1497673/1825509207.py:9: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
+      "  df['year_month'] = df['createdAt'].dt.to_period('M')\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Data has been split and saved to /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month\n"
+     ]
+    }
+   ],
+   "source": [
+    "split_by_month(mini, mivolo_in)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "4c543306-ffc8-4b9c-a3df-b49b2271caa9",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T22:00:10.480505Z",
+     "iopub.status.busy": "2025-02-08T22:00:10.480310Z",
+     "iopub.status.idle": "2025-02-08T22:00:10.483961Z",
+     "shell.execute_reply": "2025-02-08T22:00:10.483400Z",
+     "shell.execute_reply.started": "2025-02-08T22:00:10.480486Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "mivolo_out = current_dir.parent / 'data/CSV/image_subsets/MiVOLO-results/'\n",
+    "mivolo_out.mkdir(parents=True, exist_ok=True)  # Create the output directory if it doesn't exist"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ffb7dd23",
+   "metadata": {},
+   "source": [
+    "## MiVOLO gender and age inference"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "304ed12f-c7b6-4129-b24d-7ccc793a62c7",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-08T22:00:10.484802Z",
+     "iopub.status.busy": "2025-02-08T22:00:10.484639Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/lauwag/data/conda/envs/horde/lib/python3.12/site-packages/ultralytics/nn/tasks.py:634: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
+      "  return torch.load(file, map_location=\"cpu\"), file  # load\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model summary (fused): 268 layers, 68125494 parameters, 0 gradients, 257.4 GFLOPs\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[W208 23:00:15.738708520 NNPACK.cpp:61] Could not initialize NNPACK! Reason: Unsupported hardware.\n",
+      "/home/lauwag/data/conda/envs/horde/lib/python3.12/site-packages/mivolo/model/mi_volo.py:33: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
+      "  state = torch.load(ckpt_path, map_location=\"cpu\")\n",
+      "INFO:MiVOLO:Model meta:\n",
+      "min_age: 1, max_age: 95, avg_age: 48.0, num_classes: 3, in_chans: 6, with_persons_model: True, disable_faces: False, use_persons: True, only_age: False, num_classes_gender: 2, input_size: 224, use_person_crops: True, use_face_crops: True\n",
+      "/home/lauwag/data/conda/envs/horde/lib/python3.12/site-packages/timm/models/_helpers.py:39: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
+      "  checkpoint = torch.load(checkpoint_path, map_location='cpu')\n",
+      "INFO:timm.models._helpers:Loaded state_dict from checkpoint '/shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/ext/MiVOLO/models/mivolo_imbd.pth.tar'\n",
+      "INFO:MiVOLO:Model mivolo_d1_224 created, param count: 27432414\n",
+      "INFO:timm.data.config:Data processing configuration for current model + dataset:\n",
+      "INFO:timm.data.config:\tinput_size: (3, 224, 224)\n",
+      "INFO:timm.data.config:\tinterpolation: bicubic\n",
+      "INFO:timm.data.config:\tmean: (0.485, 0.456, 0.406)\n",
+      "INFO:timm.data.config:\tstd: (0.229, 0.224, 0.225)\n",
+      "INFO:timm.data.config:\tcrop_pct: 0.96\n",
+      "INFO:timm.data.config:\tcrop_mode: center\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2022-11.csv\n",
+      "\n",
+      "0: 640x640 (no detections), 723.9ms\n",
+      "Speed: 12.1ms preprocess, 723.9ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "Processed and saved 1 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2022-11.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2022-12.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-01.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-02.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-03.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-04.csv\n",
+      "File not found: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-05.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-06.csv\n",
+      "\n",
+      "0: 416x640 1 person, 455.1ms\n",
+      "Speed: 3.5ms preprocess, 455.1ms inference, 33.5ms postprocess per image at shape (1, 3, 416, 640)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 32.89\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-06.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-07.csv\n",
+      "\n",
+      "0: 640x320 1 person, 395.7ms\n",
+      "Speed: 2.9ms preprocess, 395.7ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 320)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 33.49\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 478.5ms\n",
+      "Speed: 1.9ms preprocess, 478.5ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 17.81\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 2 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-07.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-08.csv\n",
+      "\n",
+      "0: 640x448 1 person, 478.0ms\n",
+      "Speed: 2.9ms preprocess, 478.0ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 40.62\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-08.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-09.csv\n",
+      "\n",
+      "0: 416x640 (no detections), 567.1ms\n",
+      "Speed: 2.4ms preprocess, 567.1ms inference, 0.4ms postprocess per image at shape (1, 3, 416, 640)\n",
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 320x640 (no detections), 393.6ms\n",
+      "Speed: 1.7ms preprocess, 393.6ms inference, 0.4ms postprocess per image at shape (1, 3, 320, 640)\n",
+      "\n",
+      "0: 640x640 (no detections), 711.9ms\n",
+      "Speed: 3.4ms preprocess, 711.9ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x640 (no detections), 699.8ms\n",
+      "Speed: 2.3ms preprocess, 699.8ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x576 1 person, 629.6ms\n",
+      "Speed: 2.4ms preprocess, 629.6ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 576)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 28.65\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 598.3ms\n",
+      "Speed: 2.1ms preprocess, 598.3ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 25.85\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n",
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/98848c97-3d1e-4b52-9967-aeeca354a30e/width=656/98848c97-3d1e-4b52-9967-aeeca354a30e.mp4: cannot identify image file <_io.BytesIO object at 0x14cb00133740>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-09.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-10.csv\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/e6469288-b487-4a06-99c1-59e7ac22fa77/width=1024/e6469288-b487-4a06-99c1-59e7ac22fa77.mp4: cannot identify image file <_io.BytesIO object at 0x14cb00ecbdd0>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 448x640 (no detections), 536.6ms\n",
+      "Speed: 10.1ms preprocess, 536.6ms inference, 0.4ms postprocess per image at shape (1, 3, 448, 640)\n",
+      "Processed and saved 2 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-10.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-11.csv\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 662.9ms\n",
+      "Speed: 2.6ms preprocess, 662.9ms inference, 0.9ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 17.0\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x384 1 person, 895.9ms\n",
+      "Speed: 2.0ms preprocess, 895.9ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 384)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 43.33\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 529.4ms\n",
+      "Speed: 2.6ms preprocess, 529.4ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 1 person, 539.3ms\n",
+      "Speed: 2.8ms preprocess, 539.3ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 39.15\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 708.6ms\n",
+      "Speed: 2.5ms preprocess, 708.6ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 29.64\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n",
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/ad4b5481-0a58-4196-a3e8-fca2fe22a3cc/width=1080/ad4b5481-0a58-4196-a3e8-fca2fe22a3cc.mp4: cannot identify image file <_io.BytesIO object at 0x14cb010c24d0>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-11.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2023-12.csv\n",
+      "\n",
+      "0: 640x384 1 person, 1 face, 461.0ms\n",
+      "Speed: 2.4ms preprocess, 461.0ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 384)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 19.61\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 501.3ms\n",
+      "Speed: 3.1ms preprocess, 501.3ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 22.58\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n",
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/3004b5fa-af81-4de7-829d-1d809d70b878/width=512/3004b5fa-af81-4de7-829d-1d809d70b878.mp4: cannot identify image file <_io.BytesIO object at 0x14cb0020ad40>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x640 (no detections), 842.5ms\n",
+      "Speed: 4.5ms preprocess, 842.5ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x416 (no detections), 446.8ms\n",
+      "Speed: 2.5ms preprocess, 446.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 416)\n",
+      "Processed and saved 5 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2023-12.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-01.csv\n",
+      "\n",
+      "0: 640x448 (no detections), 638.5ms\n",
+      "Speed: 2.3ms preprocess, 638.5ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x416 (no detections), 441.7ms\n",
+      "Speed: 2.5ms preprocess, 441.7ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 416)\n",
+      "\n",
+      "0: 640x448 (no detections), 470.3ms\n",
+      "Speed: 2.3ms preprocess, 470.3ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 (no detections), 693.9ms\n",
+      "Speed: 2.5ms preprocess, 693.9ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x512 1 person, 1 face, 808.6ms\n",
+      "Speed: 3.2ms preprocess, 808.6ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 512)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 15.01\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x320 1 person, 1 face, 345.6ms\n",
+      "Speed: 2.0ms preprocess, 345.6ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 320)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 20.86\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 6 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2024-01.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-02.csv\n",
+      "\n",
+      "0: 640x384 1 person, 1 face, 387.8ms\n",
+      "Speed: 1.9ms preprocess, 387.8ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 384)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 17.31\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x480 1 person, 1 face, 540.4ms\n",
+      "Speed: 2.5ms preprocess, 540.4ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 480)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 17.47\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x640 1 person, 1 face, 713.1ms\n",
+      "Speed: 3.8ms preprocess, 713.1ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 640)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 17.85\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x640 (no detections), 778.8ms\n",
+      "Speed: 28.7ms preprocess, 778.8ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 528.2ms\n",
+      "Speed: 2.3ms preprocess, 528.2ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 21.63\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 518.4ms\n",
+      "Speed: 3.9ms preprocess, 518.4ms inference, 0.9ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 18.25\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 470.7ms\n",
+      "Speed: 2.5ms preprocess, 470.7ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 20.51\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x480 1 person, 1 face, 647.1ms\n",
+      "Speed: 2.4ms preprocess, 647.1ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 480)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 58.87\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 469.8ms\n",
+      "Speed: 2.6ms preprocess, 469.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 477.5ms\n",
+      "Speed: 2.3ms preprocess, 477.5ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 23.79\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 10 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2024-02.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-03.csv\n",
+      "\n",
+      "0: 640x448 1 face, 511.4ms\n",
+      "Speed: 2.5ms preprocess, 511.4ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 24.87\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x544 (no detections), 576.5ms\n",
+      "Speed: 2.9ms preprocess, 576.5ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 544)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 687.1ms\n",
+      "Speed: 9.9ms preprocess, 687.1ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 25.76\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 498.3ms\n",
+      "Speed: 2.3ms preprocess, 498.3ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x512 (no detections), 573.2ms\n",
+      "Speed: 3.0ms preprocess, 573.2ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 512)\n",
+      "Processed and saved 5 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2024-03.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-04.csv\n",
+      "\n",
+      "0: 640x384 (no detections), 518.2ms\n",
+      "Speed: 2.7ms preprocess, 518.2ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 384)\n",
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x512 (no detections), 707.7ms\n",
+      "Speed: 3.6ms preprocess, 707.7ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 512)\n",
+      "\n",
+      "0: 640x416 (no detections), 453.7ms\n",
+      "Speed: 2.4ms preprocess, 453.7ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 416)\n",
+      "\n",
+      "0: 640x384 (no detections), 391.0ms\n",
+      "Speed: 2.0ms preprocess, 391.0ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 384)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 449.8ms\n",
+      "Speed: 2.3ms preprocess, 449.8ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 22.39\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 618.4ms\n",
+      "Speed: 2.3ms preprocess, 618.4ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 631.0ms\n",
+      "Speed: 2.2ms preprocess, 631.0ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 24.05\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x512 1 person, 496.4ms\n",
+      "Speed: 2.6ms preprocess, 496.4ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 512)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 22.81\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 442.8ms\n",
+      "Speed: 2.3ms preprocess, 442.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 477.7ms\n",
+      "Speed: 2.4ms preprocess, 477.7ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 21.62\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 447.0ms\n",
+      "Speed: 2.2ms preprocess, 447.0ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 54.31\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x640 (no detections), 819.0ms\n",
+      "Speed: 3.6ms preprocess, 819.0ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 478.2ms\n",
+      "Speed: 1.8ms preprocess, 478.2ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 20.56\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 471.2ms\n",
+      "Speed: 2.7ms preprocess, 471.2ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 21.31\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 484.0ms\n",
+      "Speed: 2.2ms preprocess, 484.0ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x640 (no detections), 832.6ms\n",
+      "Speed: 3.0ms preprocess, 832.6ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 640)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 508.9ms\n",
+      "Speed: 2.5ms preprocess, 508.9ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 27.19\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n",
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6879c7b9-5cb3-42db-b409-30b4e2f71945/width=1080/6879c7b9-5cb3-42db-b409-30b4e2f71945.mp4: cannot identify image file <_io.BytesIO object at 0x14cb0020ad40>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 9 persons, 461.8ms\n",
+      "Speed: 2.4ms preprocess, 461.8ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([9, 3, 224, 224]), person_input: torch.Size([9, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 30.4\n",
+      "INFO:MiVOLO:\tgender: male [55%]\n",
+      "INFO:MiVOLO:\tage: 28.89\n",
+      "INFO:MiVOLO:\tgender: female [63%]\n",
+      "INFO:MiVOLO:\tage: 30.31\n",
+      "INFO:MiVOLO:\tgender: female [68%]\n",
+      "INFO:MiVOLO:\tage: 31.62\n",
+      "INFO:MiVOLO:\tgender: female [53%]\n",
+      "INFO:MiVOLO:\tage: 35.17\n",
+      "INFO:MiVOLO:\tgender: male [53%]\n",
+      "INFO:MiVOLO:\tage: 33.02\n",
+      "INFO:MiVOLO:\tgender: male [95%]\n",
+      "INFO:MiVOLO:\tage: 35.17\n",
+      "INFO:MiVOLO:\tgender: male [53%]\n",
+      "INFO:MiVOLO:\tage: 35.17\n",
+      "INFO:MiVOLO:\tgender: male [53%]\n",
+      "INFO:MiVOLO:\tage: 35.17\n",
+      "INFO:MiVOLO:\tgender: male [53%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 19 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2024-04.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-05.csv\n",
+      "\n",
+      "0: 640x448 1 person, 455.5ms\n",
+      "Speed: 2.2ms preprocess, 455.5ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 37.57\n",
+      "INFO:MiVOLO:\tgender: male [95%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 438.7ms\n",
+      "Speed: 2.2ms preprocess, 438.7ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 15.62\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 444.8ms\n",
+      "Speed: 2.3ms preprocess, 444.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "ERROR:inference:Unidentified image error for URL https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/6032bd70-6d53-4007-9e89-e69d4748efb5/width=528/6032bd70-6d53-4007-9e89-e69d4748efb5.mp4: cannot identify image file <_io.BytesIO object at 0x14cb00ed0950>\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 453.9ms\n",
+      "Speed: 2.3ms preprocess, 453.9ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 475.0ms\n",
+      "Speed: 1.6ms preprocess, 475.0ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 22.5\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 447.6ms\n",
+      "Speed: 2.5ms preprocess, 447.6ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 23.46\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x512 (no detections), 528.5ms\n",
+      "Speed: 3.2ms preprocess, 528.5ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 512)\n",
+      "\n",
+      "0: 640x448 1 person, 1 face, 449.8ms\n",
+      "Speed: 2.3ms preprocess, 449.8ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 29.32\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 1 face, 617.7ms\n",
+      "Speed: 2.4ms preprocess, 617.7ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 21.32\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 609.1ms\n",
+      "Speed: 2.3ms preprocess, 609.1ms inference, 0.5ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 (no detections), 436.2ms\n",
+      "Speed: 2.5ms preprocess, 436.2ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x512 1 person, 1 face, 585.6ms\n",
+      "Speed: 3.1ms preprocess, 585.6ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 512)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 20.5\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 457.3ms\n",
+      "Speed: 2.1ms preprocess, 457.3ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 25.19\n",
+      "INFO:MiVOLO:\tgender: male [81%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed and saved 14 images so far.\n",
+      "Processed and saved to: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/MiVOLO-results/2024-05.csv\n",
+      "Processing: /shares/weddigen.ki.uzh/laura_wagner/Civitai_page_analysis/Civitai_visualizations/data/CSV/image_subsets/Civiverse-mini-by-month/Civiverse-2024-06.csv\n",
+      "\n",
+      "0: 640x448 (no detections), 484.5ms\n",
+      "Speed: 2.8ms preprocess, 484.5ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "Processed and saved 1 images so far.\n",
+      "\n",
+      "0: 640x512 (no detections), 524.8ms\n",
+      "Speed: 2.9ms preprocess, 524.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 512)\n",
+      "\n",
+      "0: 640x480 1 person, 478.0ms\n",
+      "Speed: 2.6ms preprocess, 478.0ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 480)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 39.4\n",
+      "INFO:MiVOLO:\tgender: male [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x512 1 person, 1 face, 539.8ms\n",
+      "Speed: 2.6ms preprocess, 539.8ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 512)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 21.33\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 1 person, 2 faces, 446.7ms\n",
+      "Speed: 2.4ms preprocess, 446.7ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([2, 3, 224, 224]), person_input: torch.Size([2, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 20.65\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n",
+      "INFO:MiVOLO:\tage: 20.53\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x640 1 person, 1 face, 655.0ms\n",
+      "Speed: 3.3ms preprocess, 655.0ms inference, 0.8ms postprocess per image at shape (1, 3, 640, 640)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 26.34\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x384 (no detections), 400.6ms\n",
+      "Speed: 2.1ms preprocess, 400.6ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 384)\n",
+      "\n",
+      "0: 640x448 1 person, 587.9ms\n",
+      "Speed: 2.2ms preprocess, 587.9ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 448)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 30.4\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 610.3ms\n",
+      "Speed: 2.3ms preprocess, 610.3ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 (no detections), 453.6ms\n",
+      "Speed: 2.3ms preprocess, 453.6ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x512 1 person, 1 face, 511.3ms\n",
+      "Speed: 2.8ms preprocess, 511.3ms inference, 0.7ms postprocess per image at shape (1, 3, 640, 512)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 34.28\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 441.2ms\n",
+      "Speed: 2.3ms preprocess, 441.2ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 (no detections), 586.3ms\n",
+      "Speed: 2.3ms preprocess, 586.3ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 640x448 (no detections), 437.5ms\n",
+      "Speed: 2.4ms preprocess, 437.5ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 448x640 1 person, 1 face, 437.4ms\n",
+      "Speed: 2.4ms preprocess, 437.4ms inference, 0.7ms postprocess per image at shape (1, 3, 448, 640)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:MiVOLO:faces_input: torch.Size([1, 3, 224, 224]), person_input: torch.Size([1, 3, 224, 224])\n",
+      "INFO:MiVOLO:\tage: 22.81\n",
+      "INFO:MiVOLO:\tgender: female [99%]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "0: 640x448 (no detections), 436.8ms\n",
+      "Speed: 2.6ms preprocess, 436.8ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n",
+      "0: 448x640 (no detections), 433.0ms\n",
+      "Speed: 1.9ms preprocess, 433.0ms inference, 0.4ms postprocess per image at shape (1, 3, 448, 640)\n",
+      "\n",
+      "0: 640x448 (no detections), 599.7ms\n",
+      "Speed: 2.5ms preprocess, 599.7ms inference, 0.4ms postprocess per image at shape (1, 3, 640, 448)\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Set up logging\n",
+    "detector_weights = current_dir.parent / 'ext/MiVOLO/models/yolov8x_person_face.pt'\n",
+    "checkpoint = current_dir.parent / 'ext/MiVOLO/models/mivolo_imbd.pth.tar'\n",
+    "\n",
+    "_logger = logging.getLogger(\"inference\")\n",
+    "logging.basicConfig(level=logging.INFO)\n",
+    "\n",
+    "# Placeholder configuration and predictor initialization for MiVOLO\n",
+    "class Config:\n",
+    "    def __init__(self, detector_weights, checkpoint, device, with_persons=True, disable_faces=False, draw=False):\n",
+    "        self.detector_weights = detector_weights\n",
+    "        self.checkpoint = checkpoint\n",
+    "        self.device = device\n",
+    "        self.with_persons = with_persons\n",
+    "        self.disable_faces = disable_faces\n",
+    "        self.draw = draw\n",
+    "\n",
+    "\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "config = Config(detector_weights=detector_weights, checkpoint=checkpoint, device=device)\n",
+    "predictor = Predictor(config, verbose=True)\n",
+    "\n",
+    "def download_image(url):\n",
+    "    try:\n",
+    "        response = requests.get(url)\n",
+    "        response.raise_for_status()\n",
+    "        return Image.open(BytesIO(response.content)).convert(\"RGB\")\n",
+    "    except requests.RequestException as e:\n",
+    "        _logger.error(f\"Failed to download image from {url}: {e}\")\n",
+    "        return None\n",
+    "    except UnidentifiedImageError as e:\n",
+    "        _logger.error(f\"Unidentified image error for URL {url}: {e}\")\n",
+    "        return None\n",
+    "\n",
+    "def process_images_with_progress(data, predictor, output_file, start_idx=0):\n",
+    "    results = []\n",
+    "    total_images = len(data)\n",
+    "\n",
+    "    for idx, row in data.iterrows():\n",
+    "        if idx < start_idx:\n",
+    "            continue\n",
+    "\n",
+    "        img_url = row[\"url\"]\n",
+    "        pil_image = download_image(img_url)\n",
+    "        if pil_image is None:\n",
+    "            continue\n",
+    "\n",
+    "        np_image = np.array(pil_image)\n",
+    "        np_image = cv2.cvtColor(np_image, cv2.COLOR_RGB2BGR)\n",
+    "        detected_objects, _ = predictor.recognize(np_image)\n",
+    "\n",
+    "        row_result = row.to_dict()  # Start with the original row's data\n",
+    "\n",
+    "        if detected_objects and detected_objects.ages:\n",
+    "            for i in range(len(detected_objects.ages)):\n",
+    "                age = detected_objects.ages[i]\n",
+    "                gender = detected_objects.genders[i]\n",
+    "                gender_confidence = detected_objects.gender_scores[i]\n",
+    "\n",
+    "                if gender_confidence >= 0.83:\n",
+    "                    detection = {\n",
+    "                        \"detection_type\": 'face' if i in detected_objects.face_to_person_map else 'person',\n",
+    "                        \"gender\": gender,\n",
+    "                        \"gender_confidence\": gender_confidence,\n",
+    "                        \"age\": age,\n",
+    "                        \"n_persons\": detected_objects.n_persons,\n",
+    "                        \"n_faces\": detected_objects.n_faces,\n",
+    "                        \"detected\": True\n",
+    "                    }\n",
+    "                else:\n",
+    "                    detection = {\n",
+    "                        \"detection_type\": \"N/A\",\n",
+    "                        \"gender\": \"N/A\",\n",
+    "                        \"gender_confidence\": 0,\n",
+    "                        \"age\": 0,\n",
+    "                        \"n_persons\": detected_objects.n_persons if detected_objects else 0,\n",
+    "                        \"n_faces\": detected_objects.n_faces if detected_objects else 0,\n",
+    "                        \"detected\": False\n",
+    "                    }\n",
+    "\n",
+    "                results.append({**row_result, **detection})\n",
+    "        else:\n",
+    "            detection = {\n",
+    "                \"detection_type\": \"N/A\",\n",
+    "                \"gender\": \"N/A\",\n",
+    "                \"gender_confidence\": 0,\n",
+    "                \"age\": 0,\n",
+    "                \"n_persons\": detected_objects.n_persons if detected_objects else 0,\n",
+    "                \"n_faces\": detected_objects.n_faces if detected_objects else 0,\n",
+    "                \"detected\": False\n",
+    "            }\n",
+    "            results.append({**row_result, **detection})\n",
+    "\n",
+    "        if idx % 100 == 0 or idx == total_images - 1:\n",
+    "            df = pd.DataFrame(results)\n",
+    "            if os.path.exists(output_file):\n",
+    "                df.to_csv(output_file, mode='a', header=False, index=False)\n",
+    "            else:\n",
+    "                df.to_csv(output_file, mode='w', header=True, index=False)\n",
+    "            results = []\n",
+    "            print(f\"Processed and saved {idx + 1} images so far.\")\n",
+    "\n",
+    "def generate_months(start, end):\n",
+    "    start_date = datetime.strptime(start, '%Y-%m')\n",
+    "    end_date = datetime.strptime(end, '%Y-%m')\n",
+    "    while start_date <= end_date:\n",
+    "        yield start_date.strftime('%Y-%m')\n",
+    "        start_date += relativedelta(months=1)  # Increment by calendar months\n",
+    "\n",
+    "\n",
+    "start_month = '2022-11'\n",
+    "end_month = '2024-12'\n",
+    "\n",
+    "for month in generate_months(start_month, end_month):\n",
+    "    input_file_path = mivolo_in / f'Civiverse-{month}.csv'\n",
+    "    output_file_path = mivolo_out / f'{month}.csv'\n",
+    "\n",
+    "    if input_file_path.exists():\n",
+    "        print(f\"Processing: {input_file_path}\")\n",
+    "\n",
+    "        data = pd.read_csv(input_file_path)\n",
+    "        start_index = 0\n",
+    "        process_images_with_progress(data, predictor, output_file_path, start_idx=start_index)\n",
+    "\n",
+    "        print(f\"Processed and saved to: {output_file_path}\")\n",
+    "    else:\n",
+    "        print(f\"File not found: {input_file_path}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "26aeeef7",
+   "metadata": {},
+   "source": [
+    "## Visualization code"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "88ec896a-bf9b-4cc6-a787-c1343f8acb41",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import matplotlib.patches as mpatches\n",
+    "\n",
+    "input_dir = current_dir.parent / 'data/CSV/image_subsets/MiVOLO-results/'\n",
+    "plot_dir = current_dir.parent / 'plots/'\n",
+    "\n",
+    "\n",
+    "all_data = pd.DataFrame()\n",
+    "for file_path in input_dir.glob('*.csv'):  # Reads all CSV files in the folder\n",
+    "    #print(f\"Loading: {file_path}\")\n",
+    "    data = pd.read_csv(file_path)\n",
+    "    all_data = pd.concat([all_data, data], ignore_index=True)\n",
+    "\n",
+    "# Filter rows where detection_type equals \"person\"\n",
+    "person_data = all_data[all_data['detection_type'] == 'person']\n",
+    "\n",
+    "# Count unique images and categorize by persons detected\n",
+    "n_images = all_data['id'].nunique()\n",
+    "images_with_zero_persons = all_data[all_data['n_persons'] == 0]['id'].nunique()\n",
+    "images_with_one_or_more_persons = all_data[all_data['n_persons'] > 0]['id'].nunique()\n",
+    "\n",
+    "n_persons_detected = person_data['id'].nunique()  # Unique images with at least one detected person\n",
+    "total_persons_detected = person_data.shape[0]  # Total number of persons detected\n",
+    "\n",
+    "\n",
+    "\n",
+    "n_total_female = person_data[person_data['gender'] == 'female']['id'].nunique()\n",
+    "n_total_male = person_data[person_data['gender'] == 'male']['id'].nunique()\n",
+    "\n",
+    "# Filter the data further for non-missing age and gender\n",
+    "filtered_data = person_data.dropna(subset=['age', 'gender'])\n",
+    "\n",
+    "# Round ages for consistent plotting\n",
+    "filtered_data['rounded_age'] = np.round(filtered_data['age'] * 4) / 4\n",
+    "\n",
+    "# Map browsingLevel to colors\n",
+    "def get_browsing_color(browsing_level):\n",
+    "    color_mapping = {\n",
+    "        1: 'silver',\n",
+    "        2: 'rosybrown',\n",
+    "        4: 'coral',\n",
+    "        8: 'crimson',\n",
+    "        16: 'blueviolet'\n",
+    "    }\n",
+    "    return color_mapping.get(browsing_level, 'black')  # Default to black for unknown values\n",
+    "\n",
+    "filtered_data['color'] = filtered_data['browsingLevel'].apply(get_browsing_color)\n",
+    "\n",
+    "# Aggregate data for plotting\n",
+    "aggregated_data = (\n",
+    "    filtered_data.groupby(['rounded_age', 'gender', 'color'])\n",
+    "    .size()\n",
+    "    .unstack(fill_value=0)\n",
+    ")\n",
+    "\n",
+    "# Define NSFW colors\n",
+    "nsfw_colors = ['blueviolet', 'crimson', 'coral', 'rosybrown', 'silver']\n",
+    "\n",
+    "# Plotting function\n",
+    "def plot_gender_data(ax, data, gender_label):\n",
+    "    ages = data.index\n",
+    "    bottom = np.zeros(len(ages))\n",
+    "    \n",
+    "    for color in nsfw_colors:\n",
+    "        counts = data[color] if color in data.columns else np.zeros(len(ages))\n",
+    "        ax.bar(\n",
+    "            ages,\n",
+    "            counts,\n",
+    "            color=color,\n",
+    "            edgecolor=color,\n",
+    "            linewidth=1,\n",
+    "            width=0.2,\n",
+    "            bottom=bottom,\n",
+    "            alpha=0.5\n",
+    "        )\n",
+    "        bottom += counts\n",
+    "\n",
+    "    x_min = 5\n",
+    "    x_max = filtered_data['rounded_age'].max()\n",
+    "    ax.set_xticks(np.arange(x_min, x_max + 0.5, 5))\n",
+    "    ax.set_xticklabels([f'{int(age)}' for age in np.arange(x_min, x_max + 0.5, 5)], fontsize=12, fontweight='bold')\n",
+    "    ax.set_xticks(np.arange(x_min, x_max + 0.5, 0.5), minor=True)\n",
+    "\n",
+    "    y_min = 0\n",
+    "    y_max = bottom.max() + 100\n",
+    "    y_ticks = np.arange(y_min, y_max + 1, 100)  # Fine-grained steps of 100\n",
+    "    ax.set_yticks(y_ticks)\n",
+    "    ax.set_yticklabels([str(int(y)) for y in y_ticks], fontsize=12, fontweight='bold')\n",
+    "\n",
+    "    ax.grid(True, which='major', color='lightgrey', linestyle='-', linewidth=0.5)\n",
+    "    ax.grid(True, which='minor', color='lightgrey', linestyle=':', linewidth=0.5)\n",
+    "\n",
+    "    ax.spines['top'].set_visible(False)\n",
+    "    ax.spines['right'].set_visible(False)\n",
+    "    ax.spines['left'].set_visible(False)\n",
+    "    ax.spines['bottom'].set_visible(False)\n",
+    "    \n",
+    "    ax.set_xlabel('Age', fontsize=12, fontweight='bold')\n",
+    "    if gender_label == 'Female':\n",
+    "        ax.set_ylabel('Number of Subjects', fontsize=14, fontweight='bold')\n",
+    "    ax.set_title(f'{gender_label} Read', fontsize=14, fontweight='bold')\n",
+    "\n",
+    "# Set up the subplots\n",
+    "fig, axes = plt.subplots(1, 2, figsize=(14, 6.5), sharey=True)\n",
+    "\n",
+    "plot_gender_data(axes[0], aggregated_data.xs('male', level='gender'), 'Male')\n",
+    "plot_gender_data(axes[1], aggregated_data.xs('female', level='gender'), 'Female')\n",
+    "\n",
+    "legend_patches = [\n",
+    "    mpatches.Patch(facecolor='blueviolet', edgecolor='blueviolet', linewidth=2, label='Level 16: XXX'),\n",
+    "    mpatches.Patch(facecolor='crimson', edgecolor='crimson', linewidth=2, label='Level 8: X'),\n",
+    "    mpatches.Patch(facecolor='coral', edgecolor='coral', linewidth=2, label='Level 4: Mature'),\n",
+    "    mpatches.Patch(facecolor='rosybrown', edgecolor='rosybrown', linewidth=2, label='Level 2: Soft'),\n",
+    "    mpatches.Patch(facecolor='silver', edgecolor='silver', linewidth=2, label='Level 1: SFW'),\n",
+    "    mpatches.Patch(facecolor='none', edgecolor='none', label=f'n images: {n_images}', alpha=0),\n",
+    "    mpatches.Patch(facecolor='none', edgecolor='none', label=f'Total persons detected: {total_persons_detected}', alpha=0),\n",
+    "    mpatches.Patch(facecolor='none', edgecolor='none', label=f'Unique images containing persons: {n_persons_detected}', alpha=0),\n",
+    "]\n",
+    "\n",
+    "axes[0].legend(handles=legend_patches, title=\"Browsing Levels\", loc='upper left', fontsize=12, title_fontsize=12, frameon=True)\n",
+    "plt.savefig(f'{plot_dir}/mivolo.svg', format='svg', bbox_inches='tight')\n",
+    "plt.tight_layout()\n",
+    "\n",
+    "# Count images with at least one person\n",
+    "images_with_one_or_more_persons = all_data[all_data['n_persons'] > 0]['id'].nunique()\n",
+    "\n",
+    "# Count unique images in `person_data`\n",
+    "n_persons_detected = person_data['id'].nunique()\n",
+    "\n",
+    "# Count total persons detected\n",
+    "total_persons = person_data.shape[0]  # This counts all detected persons\n",
+    "\n",
+    "# Display potential inconsistencies\n",
+    "print(f\"Total images: {n_images}\")\n",
+    "print(f\"Images with at least one person: {images_with_one_or_more_persons}\")\n",
+    "print(f\"Unique images in `person_data`: {n_persons_detected}\")\n",
+    "print(f\"Total number of persons detected: {total_persons}\")\n",
+    "\n",
+    "\n",
+    "\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "42b2a557-b8f4-4d0f-8907-98e3012a1b34",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-02-06T20:01:54.848400Z",
+     "iopub.status.busy": "2025-02-06T20:01:54.847713Z",
+     "iopub.status.idle": "2025-02-06T20:01:54.851533Z",
+     "shell.execute_reply": "2025-02-06T20:01:54.851102Z",
+     "shell.execute_reply.started": "2025-02-06T20:01:54.848376Z"
+    }
+   },
+   "source": [
+    "### Latex Table"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3e506c41-6497-4ece-99f3-73f09fe1129e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import pandas as pd\n",
+    "\n",
+    "# Define the directory containing CSV files\n",
+    "directory_path = current_dir.parent / 'data/CSV/image_subsets/MiVOLO-results/'  # Update this path with the actual directory path\n",
+    "\n",
+    "# Prepare data for LaTeX table\n",
+    "table_rows = []\n",
+    "\n",
+    "# Loop through each file in the directory\n",
+    "for file_name in os.listdir(directory_path):\n",
+    "    if file_name.endswith('.csv'):\n",
+    "        file_path = os.path.join(directory_path, file_name)\n",
+    "        print(f\"Processing file: {file_name}\")\n",
+    "\n",
+    "        # Load the data\n",
+    "        data = pd.read_csv(file_path)\n",
+    "\n",
+    "        # Total images analyzed\n",
+    "        total_images = data['id'].nunique()\n",
+    "\n",
+    "        # Count of images with no persons detected\n",
+    "        images_no_persons = data[data['n_persons'] == 0]['id'].nunique()\n",
+    "\n",
+    "        # Total persons detected (only using \"person\" detection type)\n",
+    "        total_persons_count = data[data['detection_type'] == 'person'].shape[0]\n",
+    "\n",
+    "        # Average age and standard deviation for male and female individuals\n",
+    "        male_age_stats = data[data['gender'] == 'male']['age'].agg(['mean', 'std']).fillna(0)\n",
+    "        female_age_stats = data[data['gender'] == 'female']['age'].agg(['mean', 'std']).fillna(0)\n",
+    "\n",
+    "        # Count of female and male subjects\n",
+    "        female_images_count = data[data['gender'] == 'female']['id'].nunique()\n",
+    "        male_images_count = data[data['gender'] == 'male']['id'].nunique()\n",
+    "\n",
+    "        # Female to male ratio\n",
+    "        female_to_male_ratio = female_images_count / male_images_count if male_images_count else None\n",
+    "\n",
+    "        # Browsing level analysis for females\n",
+    "        female_browsing_level_1 = data[(data['gender'] == 'female') & (data['browsingLevel'] == 1)]['id'].nunique()\n",
+    "        female_browsing_level_2_16 = data[(data['gender'] == 'female') & (data['browsingLevel'].isin([2, 4, 8, 16]))]['id'].nunique()\n",
+    "        \n",
+    "        female_browsing_level_1_percentage = (female_browsing_level_1 / female_images_count * 100) if female_images_count else 0\n",
+    "        female_browsing_level_2_16_percentage = (female_browsing_level_2_16 / female_images_count * 100) if female_images_count else 0\n",
+    "\n",
+    "        # Browsing level analysis for males\n",
+    "        male_browsing_level_1 = data[(data['gender'] == 'male') & (data['browsingLevel'] == 1)]['id'].nunique()\n",
+    "        male_browsing_level_2_16 = data[(data['gender'] == 'male') & (data['browsingLevel'].isin([2, 4, 8, 16]))]['id'].nunique()\n",
+    "\n",
+    "        male_browsing_level_1_percentage = (male_browsing_level_1 / male_images_count * 100) if male_images_count else 0\n",
+    "        male_browsing_level_2_16_percentage = (male_browsing_level_2_16 / male_images_count * 100) if male_images_count else 0\n",
+    "\n",
+    "        # Add row to table data\n",
+    "        table_rows.append([\n",
+    "            file_name.replace('.csv', ''),  # Remove file extension\n",
+    "            total_images,\n",
+    "            total_persons_count,\n",
+    "            images_no_persons,\n",
+    "            f\"{female_browsing_level_1_percentage:.2f}\",\n",
+    "            f\"{female_browsing_level_2_16_percentage:.2f}\",\n",
+    "            f\"{male_browsing_level_1_percentage:.2f}\",\n",
+    "            f\"{male_browsing_level_2_16_percentage:.2f}\",\n",
+    "            f\"{female_to_male_ratio:.2f}\" if female_to_male_ratio is not None else \"N/A\",\n",
+    "            f\"{female_age_stats['mean']:.2f} ({female_age_stats['std']:.2f})\",\n",
+    "            f\"{male_age_stats['mean']:.2f} ({male_age_stats['std']:.2f})\"\n",
+    "        ])\n",
+    "\n",
+    "# Sort table rows by the filename (assumes filenames are formatted with sortable dates)\n",
+    "table_rows = sorted(table_rows, key=lambda x: x[0])\n",
+    "\n",
+    "# Generate LaTeX table\n",
+    "latex_table = r\"\"\"\n",
+    "\\begin{table}[H]\n",
+    "\\centering\n",
+    "\\scriptsize\n",
+    "\\renewcommand{\\arraystretch}{0.9}\n",
+    "\\caption{Summary of Image Classification for 2023-2024}\n",
+    "\\label{table:image_classification_2023_2024}\n",
+    "\\begin{tabular}{lrrrrrrrrrr}\n",
+    "\\toprule\n",
+    "File Name & Total Images & Total Persons & No Persons & \\multicolumn{2}{c}{Female (\\%)} & \\multicolumn{2}{c}{Male (\\%)} & Female:Male & Female Age (Mean ± SD) & Male Age (Mean ± SD) \\\\\n",
+    " & & & & L1 & L2-16 & L1 & L2-16 & & & \\\\\n",
+    "\\midrule\n",
+    "\"\"\"\n",
+    "for row in table_rows:\n",
+    "    latex_table += \" & \".join(map(str, row)) + r\" \\\\\\\\\\n\"\n",
+    "\n",
+    "latex_table += r\"\"\"\n",
+    "\\bottomrule\n",
+    "\\end{tabular}\n",
+    "\\vspace{1em}\n",
+    "\\noindent\n",
+    "\\textbf{Disclaimer:} \\(\\female\\) and \\(\\male\\) refer to female-read and male-read classifications as determined by the MiVOLO system's weights. \n",
+    "We acknowledge the complexities of gender presentations and stress that these terms do not necessarily correspond to biological sex.\n",
+    "\\end{table}\n",
+    "\"\"\"\n",
+    "\n",
+    "# Output LaTeX table\n",
+    "print(\"\\nGenerated LaTeX Table:\")\n",
+    "print(latex_table)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ef428be-856b-4c4c-b1b0-a052d181d03b",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "HORDE",
+   "language": "python",
+   "name": "horde"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

jupyter_notebooks/MISTRAL.ipynb

@@ -0,0 +1,3006 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "924fc9f0-95c1-44d5-871a-76d655306607",
+   "metadata": {},
+   "source": [
+    "# MISTRAL QUERY"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cfba5e1e-fe51-4fb3-876e-53e3909e87cc",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-11-29T10:08:31.471857Z",
+     "iopub.status.busy": "2025-11-29T10:08:31.471665Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading model: mistralai/Mistral-7B-Instruct-v0.3\n",
+      "Cache directory: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/models\n",
+      "This may take a while on first run (~65GB download)...\n",
+      "\n",
+      "Device: cuda\n",
+      "Loading tokenizer...\n",
+      "✅ Tokenizer loaded\n",
+      "Loading model (this may take several minutes)...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading checkpoint shards: 100%|██████████| 3/3 [00:47<00:00, 15.76s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Model loaded\n",
+      "VRAM used: 13.50 GB\n",
+      "\n",
+      "Loading annotated CSV...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/sctmp/lauwag/ipykernel_1207447/1621707066.py:102: DtypeWarning: Columns (53,54,55,56,57) have mixed types. Specify dtype option on import or set low_memory=False.\n",
+      "  df = pd.read_csv(output_file)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Loaded professions.csv\n",
+      "✅ Loaded profession mapping with 9 categories\n",
+      "Loaded 50861 rows\n",
+      "\n",
+      "Profession categories (9):\n",
+      "  - actor\n",
+      "  - adult performer\n",
+      "  - singer/musician\n",
+      "  - model\n",
+      "  - online personality\n",
+      "  - public figure\n",
+      "  - voice actor/ASMR\n",
+      "  - sports professional\n",
+      "  - tv personality\n",
+      "\n",
+      "Creating prompts...\n",
+      "✅ Prompts created\n",
+      "Resuming from index 8790\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 10/42071 [00:22<31:33:16,  2.70s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8800\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 20/42071 [00:45<30:10:33,  2.58s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8810\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 30/42071 [01:04<28:01:14,  2.40s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8820\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 40/42071 [01:24<27:18:50,  2.34s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8830\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 50/42071 [01:45<27:35:09,  2.36s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8840\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 60/42071 [02:03<26:10:45,  2.24s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8850\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 70/42071 [02:23<28:07:48,  2.41s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8860\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 80/42071 [02:44<28:23:23,  2.43s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8870\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 90/42071 [03:05<29:35:43,  2.54s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8880\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 100/42071 [03:26<29:13:37,  2.51s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8890\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 110/42071 [03:47<28:09:35,  2.42s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8900\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 120/42071 [04:08<29:11:22,  2.50s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8910\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 130/42071 [04:31<35:26:12,  3.04s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8920\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 140/42071 [04:54<31:01:14,  2.66s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8930\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 150/42071 [05:14<28:32:42,  2.45s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8940\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 160/42071 [05:36<28:37:48,  2.46s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8950\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 170/42071 [05:56<29:47:28,  2.56s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8960\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 180/42071 [06:14<24:59:54,  2.15s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8970\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 190/42071 [06:39<36:14:58,  3.12s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8980\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 200/42071 [07:01<27:25:15,  2.36s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 8990\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   0%|          | 210/42071 [07:20<29:50:27,  2.57s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9000\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 220/42071 [07:39<28:15:01,  2.43s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9010\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 230/42071 [08:00<29:22:05,  2.53s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9020\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 240/42071 [08:21<30:00:58,  2.58s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9030\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 250/42071 [08:41<26:54:22,  2.32s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9040\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 260/42071 [09:03<32:51:56,  2.83s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9050\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 270/42071 [09:23<28:50:25,  2.48s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9060\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 280/42071 [09:43<25:37:13,  2.21s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9070\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 290/42071 [10:04<28:16:26,  2.44s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9080\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 300/42071 [10:26<29:36:56,  2.55s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9090\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 310/42071 [10:47<27:13:03,  2.35s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9100\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 320/42071 [11:08<27:56:12,  2.41s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9110\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 330/42071 [11:26<28:24:37,  2.45s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9120\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 340/42071 [11:46<28:02:33,  2.42s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9130\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 350/42071 [12:10<30:51:36,  2.66s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9140\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 360/42071 [12:30<27:10:53,  2.35s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9150\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 370/42071 [12:51<27:30:42,  2.38s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9160\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 380/42071 [13:10<26:18:09,  2.27s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9170\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 390/42071 [13:31<29:09:59,  2.52s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9180\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 400/42071 [13:51<28:30:21,  2.46s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9190\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 410/42071 [14:11<28:16:34,  2.44s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9200\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 420/42071 [14:31<28:40:48,  2.48s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9210\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 430/42071 [14:50<27:36:54,  2.39s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9220\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 440/42071 [15:12<29:55:01,  2.59s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9230\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 450/42071 [15:34<29:02:10,  2.51s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9240\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 460/42071 [15:55<30:21:48,  2.63s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9250\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 470/42071 [16:14<25:51:25,  2.24s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9260\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 480/42071 [16:34<28:45:04,  2.49s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9270\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 490/42071 [16:55<31:28:55,  2.73s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9280\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 500/42071 [17:15<27:51:29,  2.41s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9290\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 510/42071 [17:35<25:41:11,  2.22s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9300\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|          | 520/42071 [17:54<26:36:23,  2.31s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9310\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|▏         | 530/42071 [18:13<26:31:03,  2.30s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9320\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|▏         | 540/42071 [18:33<29:41:27,  2.57s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9330\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|▏         | 550/42071 [18:53<29:34:09,  2.56s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9340\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|▏         | 560/42071 [19:15<30:45:00,  2.67s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9350\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|▏         | 570/42071 [19:35<28:44:33,  2.49s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9360\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|▏         | 580/42071 [19:54<28:12:08,  2.45s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9370\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|▏         | 590/42071 [20:19<36:03:11,  3.13s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9380\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|▏         | 600/42071 [20:40<30:45:13,  2.67s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9390\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|▏         | 610/42071 [20:59<26:18:00,  2.28s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9400\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|▏         | 620/42071 [21:20<27:15:41,  2.37s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9410\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   1%|▏         | 630/42071 [21:42<28:20:07,  2.46s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9420\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 640/42071 [22:03<34:02:41,  2.96s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9430\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 650/42071 [22:25<29:27:42,  2.56s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9440\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 660/42071 [22:44<30:01:06,  2.61s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9450\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 670/42071 [23:07<28:55:03,  2.51s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9460\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 680/42071 [23:28<29:29:57,  2.57s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9470\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 690/42071 [23:48<27:56:29,  2.43s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9480\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 700/42071 [24:09<28:46:20,  2.50s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9490\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 710/42071 [24:29<26:39:32,  2.32s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9500\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 720/42071 [24:51<28:42:54,  2.50s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9510\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 730/42071 [25:13<30:13:50,  2.63s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9520\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 740/42071 [25:32<25:45:45,  2.24s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9530\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 750/42071 [25:52<27:09:24,  2.37s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9540\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 760/42071 [26:11<25:37:54,  2.23s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9550\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 770/42071 [26:31<27:00:30,  2.35s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9560\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 780/42071 [26:49<24:59:54,  2.18s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9570\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 790/42071 [27:10<30:40:18,  2.67s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9580\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 800/42071 [27:31<30:25:14,  2.65s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9590\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 810/42071 [27:51<28:53:15,  2.52s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9600\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 820/42071 [28:11<28:19:57,  2.47s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9610\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 830/42071 [28:31<26:49:31,  2.34s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9620\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 840/42071 [28:53<31:16:04,  2.73s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9630\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 850/42071 [29:14<29:41:08,  2.59s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9640\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 860/42071 [29:33<27:24:08,  2.39s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9650\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 870/42071 [29:55<29:04:09,  2.54s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9660\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 880/42071 [30:15<28:15:42,  2.47s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9670\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 890/42071 [30:35<27:52:10,  2.44s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9680\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 900/42071 [30:56<28:28:41,  2.49s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9690\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 910/42071 [31:19<27:07:56,  2.37s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9700\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 920/42071 [31:53<39:26:22,  3.45s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9710\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 930/42071 [32:18<36:31:49,  3.20s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9720\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 940/42071 [32:39<29:56:19,  2.62s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9730\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 950/42071 [32:59<28:41:17,  2.51s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9740\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 960/42071 [33:17<26:33:39,  2.33s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9750\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 970/42071 [33:36<27:48:54,  2.44s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9760\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 980/42071 [33:55<27:49:27,  2.44s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9770\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 990/42071 [34:17<28:25:06,  2.49s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9780\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 1000/42071 [34:38<29:39:06,  2.60s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9790\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 1010/42071 [35:00<26:58:20,  2.36s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9800\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 1020/42071 [35:21<28:52:02,  2.53s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9810\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 1030/42071 [35:42<27:38:16,  2.42s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9820\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 1040/42071 [36:04<28:46:01,  2.52s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9830\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   2%|▏         | 1050/42071 [36:27<29:11:36,  2.56s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9840\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1060/42071 [36:50<32:38:56,  2.87s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9850\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1070/42071 [37:13<34:34:54,  3.04s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9860\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1080/42071 [37:33<24:58:04,  2.19s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9870\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1090/42071 [37:54<26:10:04,  2.30s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9880\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1100/42071 [38:14<27:58:06,  2.46s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9890\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1110/42071 [38:33<26:39:34,  2.34s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9900\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1120/42071 [38:52<27:28:26,  2.42s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9910\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1130/42071 [39:14<29:49:35,  2.62s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9920\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1140/42071 [39:33<26:01:58,  2.29s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9930\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1150/42071 [39:55<27:55:10,  2.46s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9940\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1160/42071 [40:17<29:34:52,  2.60s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9950\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1170/42071 [40:36<26:16:52,  2.31s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9960\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1180/42071 [40:56<26:20:08,  2.32s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9970\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1190/42071 [41:15<27:37:16,  2.43s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9980\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1200/42071 [41:37<29:51:54,  2.63s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 9990\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1210/42071 [42:00<30:40:11,  2.70s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10000\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1220/42071 [42:25<39:44:45,  3.50s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10010\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1230/42071 [42:45<27:05:26,  2.39s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10020\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1240/42071 [43:07<28:36:46,  2.52s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10030\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1250/42071 [43:26<27:06:25,  2.39s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10040\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1260/42071 [43:45<27:13:15,  2.40s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10050\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1270/42071 [44:04<27:38:57,  2.44s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10060\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1280/42071 [44:23<25:33:36,  2.26s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10070\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1290/42071 [44:43<28:33:36,  2.52s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10080\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1300/42071 [45:03<27:43:28,  2.45s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10090\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1310/42071 [45:23<28:06:46,  2.48s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10100\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1320/42071 [45:45<29:54:07,  2.64s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10110\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1330/42071 [46:07<28:03:02,  2.48s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10120\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1340/42071 [46:29<29:45:30,  2.63s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10130\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1350/42071 [46:51<25:53:02,  2.29s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10140\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1360/42071 [47:11<25:51:37,  2.29s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10150\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1370/42071 [47:33<32:36:57,  2.88s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10160\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1380/42071 [47:52<25:32:45,  2.26s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10170\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1390/42071 [48:10<26:03:37,  2.31s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10180\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1400/42071 [48:31<29:37:43,  2.62s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10190\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1410/42071 [48:53<26:32:48,  2.35s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10200\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1420/42071 [49:12<25:33:11,  2.26s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10210\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1430/42071 [49:32<26:27:44,  2.34s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10220\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1440/42071 [49:52<26:42:31,  2.37s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10230\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1450/42071 [50:14<31:28:17,  2.79s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10240\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1460/42071 [50:34<28:30:12,  2.53s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10250\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   3%|▎         | 1470/42071 [50:53<25:55:24,  2.30s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10260\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▎         | 1480/42071 [51:13<26:01:28,  2.31s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10270\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▎         | 1490/42071 [51:31<26:06:37,  2.32s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10280\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▎         | 1500/42071 [51:50<25:25:45,  2.26s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10290\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▎         | 1510/42071 [52:11<27:24:03,  2.43s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10300\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▎         | 1520/42071 [52:31<27:24:13,  2.43s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10310\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▎         | 1530/42071 [52:53<25:51:08,  2.30s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10320\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▎         | 1540/42071 [53:15<27:53:16,  2.48s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10330\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▎         | 1550/42071 [53:35<28:24:31,  2.52s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10340\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▎         | 1560/42071 [53:53<24:07:14,  2.14s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10350\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▎         | 1570/42071 [54:14<29:19:36,  2.61s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10360\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1580/42071 [54:35<27:11:59,  2.42s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10370\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1590/42071 [54:56<27:59:16,  2.49s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10380\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1600/42071 [55:15<27:38:15,  2.46s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10390\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1610/42071 [55:39<29:13:06,  2.60s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10400\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1620/42071 [55:58<27:14:23,  2.42s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10410\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1630/42071 [56:20<29:34:50,  2.63s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10420\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1640/42071 [56:41<28:50:48,  2.57s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10430\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1650/42071 [57:01<27:25:49,  2.44s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10440\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1660/42071 [57:24<29:58:02,  2.67s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10450\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1670/42071 [57:45<30:44:36,  2.74s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10460\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1680/42071 [58:05<27:03:06,  2.41s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10470\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1690/42071 [58:26<27:52:00,  2.48s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10480\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1700/42071 [58:47<28:32:42,  2.55s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10490\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1710/42071 [59:08<29:30:02,  2.63s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10500\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1720/42071 [59:30<29:16:09,  2.61s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10510\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1730/42071 [59:50<28:16:11,  2.52s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10520\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1740/42071 [1:00:11<28:16:09,  2.52s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10530\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1750/42071 [1:00:32<28:29:43,  2.54s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10540\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1760/42071 [1:00:55<30:12:43,  2.70s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10550\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1770/42071 [1:01:15<27:06:18,  2.42s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10560\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1780/42071 [1:01:34<26:16:20,  2.35s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10570\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1790/42071 [1:01:59<27:50:05,  2.49s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10580\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1800/42071 [1:02:20<26:56:05,  2.41s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10590\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1810/42071 [1:02:39<26:25:22,  2.36s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10600\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Mistral Local:   4%|▍         | 1816/42071 [1:02:51<21:34:08,  1.93s/it]"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "import torch\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
+    "professions_mapped_file = current_dir.parent / \"misc/lists/professions_mapped.csv\"\n",
+    "# === PROCESS DATA ===\n",
+    "\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = False\n",
+    "TEST_SIZE = 100\n",
+    "MAX_ROWS = 50862\n",
+    "SAVE_INTERVAL = 10\n",
+    "\n",
+    "output_file = current_dir.parent / f\"data/CSV/mistral_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/query_indicies/mistral_local_query_index.txt\"\n",
+    "\n",
+    "\n",
+    "# Model settings\n",
+    "#MODEL_NAME = \"mistralai/Mistral-Small-3.1-24B-Instruct-2503\"\n",
+    "MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
+    "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
+    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
+    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Define the SPECIFIC profession categories\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\",\n",
+    "    \"adult performer\",\n",
+    "    \"singer/musician\",\n",
+    "    \"model\",\n",
+    "    \"online personality\",\n",
+    "    \"public figure\",\n",
+    "    \"voice actor/ASMR\",\n",
+    "    \"sports professional\",\n",
+    "    \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD MODEL ===\n",
+    "print(f\"Loading model: {MODEL_NAME}\")\n",
+    "print(f\"Cache directory: {CACHE_DIR}\")\n",
+    "print(f\"This may take a while on first run (~65GB download)...\\n\")\n",
+    "\n",
+    "# Check GPU availability\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "print(f\"Device: {device}\")\n",
+    "\n",
+    "if device == \"cpu\":\n",
+    "    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
+    "    print(\"   Consider using a GPU or reducing model size.\")\n",
+    "\n",
+    "# Load tokenizer\n",
+    "print(\"Loading tokenizer...\")\n",
+    "try:\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=True\n",
+    "    )\n",
+    "except Exception as e:\n",
+    "    print(f\"Failed with use_fast=True, trying use_fast=False...\")\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=False\n",
+    "    )\n",
+    "\n",
+    "# Ensure pad token is set\n",
+    "if tokenizer.pad_token is None:\n",
+    "    tokenizer.pad_token = tokenizer.eos_token\n",
+    "\n",
+    "print(\"✅ Tokenizer loaded\")\n",
+    "\n",
+    "# Load model with optimizations\n",
+    "print(\"Loading model (this may take several minutes)...\")\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    MODEL_NAME,\n",
+    "    cache_dir=str(CACHE_DIR),\n",
+    "    torch_dtype=torch.bfloat16,\n",
+    "    device_map=\"auto\",\n",
+    "    trust_remote_code=False\n",
+    ")\n",
+    "model.eval()\n",
+    "print(\"✅ Model loaded\")\n",
+    "\n",
+    "# Check VRAM usage\n",
+    "if torch.cuda.is_available():\n",
+    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
+    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "if output_file.exists():\n",
+    "    print(\"Loading annotated CSV...\")\n",
+    "    df = pd.read_csv(output_file)\n",
+    "else:\n",
+    "    print(\"Loading raw input CSV...\")\n",
+    "    df = pd.read_csv(input_file)\n",
+    "\n",
+    "\n",
+    "# Try to load profession mapping files\n",
+    "try:\n",
+    "    professions_df = pd.read_csv(professions_file)\n",
+    "    print(f\"✅ Loaded professions.csv\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions.csv not found\")\n",
+    "\n",
+    "try:\n",
+    "    prof_mapped_df = pd.read_csv(professions_mapped_file)\n",
+    "    print(f\"✅ Loaded profession mapping with {len(prof_mapped_df)} categories\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions_mapped.csv not found, using default categories\")\n",
+    "\n",
+    "profession_str = \", \".join(PROFESSION_CATEGORIES)\n",
+    "\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "print(f\"\\nProfession categories ({len(PROFESSION_CATEGORIES)}):\")\n",
+    "for cat in PROFESSION_CATEGORIES:\n",
+    "    print(f\"  - {cat}\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"\\nRunning in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
+    "\n",
+    "# === CREATE PROMPTS (DEEPSEEK STYLE) ===\n",
+    "def create_prompt(row):\n",
+    "    \"\"\"Create prompt for Mistral annotation with specific profession categories.\"\"\"\n",
+    "    name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
+    "    \n",
+    "    # Gather hints\n",
+    "    hints = []\n",
+    "    if pd.notna(row.get('likely_profession')):\n",
+    "        hints.append(str(row['likely_profession']))\n",
+    "    if pd.notna(row.get('likely_nationality')):\n",
+    "        hints.append(str(row['likely_nationality']))\n",
+    "    if pd.notna(row.get('likely_country')):\n",
+    "        hints.append(str(row['likely_country']))\n",
+    "    \n",
+    "    # Add tags if we don't have enough hints\n",
+    "    if len(hints) < 3:\n",
+    "        for i in range(1, 8):\n",
+    "            tag_col = f'tag_{i}'\n",
+    "            if tag_col in row and pd.notna(row[tag_col]):\n",
+    "                tag_val = str(row[tag_col])\n",
+    "                if tag_val not in hints:\n",
+    "                    hints.append(tag_val)\n",
+    "                if len(hints) >= 5:\n",
+    "                    break\n",
+    "    \n",
+    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
+    "    \n",
+    "    return f\"\"\"Given '{name}' ({hint_text}), provide:\n",
+    "1. Full legal name (Western order if non-latin script)\n",
+    "2. Any stage names/aliases (comma separated)\n",
+    "3. Gender (Male/Female/Other/Unknown)\n",
+    "4. Top 3 most likely professions from ONLY these categories:\n",
+    "   - actor\n",
+    "   - adult performer\n",
+    "   - singer/musician\n",
+    "   - model\n",
+    "   - online personality (includes streamers, cosplayers, influencers)\n",
+    "   - public figure (includes politicians, activists, journalists, authors)\n",
+    "   - voice actor/ASMR\n",
+    "   - sports professional\n",
+    "   - tv personality (includes hosts, presenters, reality TV)\n",
+    "\n",
+    "5. Primary country associated\n",
+    "\n",
+    "IMPORTANT:\n",
+    "- Choose professions ONLY from the 9 categories above\n",
+    "- Provide up to 3 professions, comma-separated, ordered by relevance\n",
+    "- Be SPECIFIC: choose the most accurate category for each role\n",
+    "- \"online personality\" includes: streamers, cosplayers, YouTubers, influencers, content creators\n",
+    "- Use 'Unknown' when uncertain or for fictional characters/places\n",
+    "- For multi-role people, list all relevant categories (e.g., \"actor, singer/musician, online personality\")\n",
+    "- For country respond with one word only, for example China or Columbia\n",
+    "- actress = actor\n",
+    "\n",
+    "Respond with exactly 5 numbered lines.\"\"\"\n",
+    "\n",
+    "# Create prompts\n",
+    "print(\"\\nCreating prompts...\")\n",
+    "df['prompt'] = df.apply(create_prompt, axis=1)\n",
+    "print(\"✅ Prompts created\")\n",
+    "\n",
+    "# === QUERY MISTRAL LOCAL ===\n",
+    "def query_mistral_local(prompt: str) -> str:\n",
+    "    \"\"\"Query Mistral locally via transformers.\"\"\"\n",
+    "    try:\n",
+    "        # Format as chat message for Mistral\n",
+    "        messages = [\n",
+    "            {\"role\": \"system\", \"content\": \"You are an assistant that extracts key data on a person based on the name. Respond with exactly 5 numbered lines. For professions, choose ONLY from these categories: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality.\"},\n",
+    "            {\"role\": \"user\", \"content\": prompt}\n",
+    "        ]\n",
+    "        \n",
+    "        # Tokenize\n",
+    "        if hasattr(tokenizer, 'apply_chat_template'):\n",
+    "            text = tokenizer.apply_chat_template(\n",
+    "                messages,\n",
+    "                tokenize=False,\n",
+    "                add_generation_prompt=True\n",
+    "            )\n",
+    "        else:\n",
+    "            # Fallback for older tokenizers\n",
+    "            text = f\"[INST] {prompt} [/INST]\"\n",
+    "        \n",
+    "        inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
+    "        \n",
+    "        # Generate\n",
+    "        with torch.no_grad():\n",
+    "            outputs = model.generate(\n",
+    "                **inputs,\n",
+    "                max_new_tokens=512,\n",
+    "                temperature=0.05,\n",
+    "                do_sample=True,\n",
+    "                top_p=0.8,\n",
+    "                pad_token_id=tokenizer.pad_token_id if tokenizer.pad_token_id else tokenizer.eos_token_id\n",
+    "            )\n",
+    "        \n",
+    "        # Decode\n",
+    "        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
+    "        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
+    "        \n",
+    "        return response.strip()\n",
+    "        \n",
+    "    except Exception as e:\n",
+    "        print(f\"Generation error: {e}\")\n",
+    "        return None\n",
+    "\n",
+    "# === PARSE RESPONSE (DEEPSEEK STYLE) ===\n",
+    "def parse_response(response):\n",
+    "    \"\"\"Parse Mistral response into structured fields.\"\"\"\n",
+    "    if not response:\n",
+    "        return {\n",
+    "            'full_name': 'Unknown',\n",
+    "            'aliases': 'Unknown',\n",
+    "            'gender': 'Unknown',\n",
+    "            'profession_llm': 'Unknown',\n",
+    "            'country': 'Unknown'\n",
+    "        }\n",
+    "    \n",
+    "    # Split into lines and clean\n",
+    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
+    "    \n",
+    "    # Initialize with Unknown values\n",
+    "    fields = {\n",
+    "        'full_name': 'Unknown',\n",
+    "        'aliases': 'Unknown',\n",
+    "        'gender': 'Unknown',\n",
+    "        'profession_llm': 'Unknown',\n",
+    "        'country': 'Unknown'\n",
+    "    }\n",
+    "    \n",
+    "    # Extract information from each numbered line\n",
+    "    for line in lines:\n",
+    "        if line.startswith('1.'):\n",
+    "            fields['full_name'] = line[2:].strip()\n",
+    "        elif line.startswith('2.'):\n",
+    "            fields['aliases'] = line[2:].strip()\n",
+    "        elif line.startswith('3.'):\n",
+    "            fields['gender'] = line[2:].strip()\n",
+    "        elif line.startswith('4.'):\n",
+    "            fields['profession_llm'] = line[2:].strip()\n",
+    "        elif line.startswith('5.'):\n",
+    "            fields['country'] = line[2:].strip()\n",
+    "    \n",
+    "    return fields\n",
+    "\n",
+    "# === PROCESS DATA ===\n",
+    "output_file = current_dir.parent / f\"data/CSV/mistral_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/query_indicies/mistral_local_query_index.txt\"\n",
+    "\n",
+    "index_file.parent.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Load index\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    try:\n",
+    "        current_index = int(index_file.read_text().strip())\n",
+    "    except:\n",
+    "        current_index = 0\n",
+    "\n",
+    "print(f\"Resuming from index {current_index}\")\n",
+    "\n",
+    "start_time = time.time()\n",
+    "\n",
+    "for i in tqdm(range(current_index, len(df)), desc=\"Mistral Local\"):\n",
+    "\n",
+    "    prompt = df.at[i, \"prompt\"]\n",
+    "\n",
+    "    # -------- MODEL QUERY WITH RETRIES --------\n",
+    "    response = None\n",
+    "    for attempt in range(3):\n",
+    "        response = query_mistral_local(prompt)\n",
+    "        \n",
+    "        # Valid response?\n",
+    "        if response and len(response.strip()) > 10:\n",
+    "            break\n",
+    "        \n",
+    "        print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
+    "        time.sleep(0.5)\n",
+    "\n",
+    "    # If still invalid → DO NOT overwrite previous data\n",
+    "    if not response or len(response.strip()) <= 10:\n",
+    "        print(f\"❌ Row {i}: failed after retries, not writing, not advancing index\")\n",
+    "        continue\n",
+    "\n",
+    "    parsed = parse_response(response)\n",
+    "\n",
+    "    # Additional safety: skip rows that parsed as all 'Unknown'\n",
+    "    if all(v == \"Unknown\" for v in parsed.values()):\n",
+    "        print(f\"❌ Row {i}: parsed as all Unknown (likely model crash); skipping.\")\n",
+    "        continue\n",
+    "\n",
+    "    # -------- WRITE PARSED FIELDS SAFELY --------\n",
+    "    for key, value in parsed.items():\n",
+    "        df.at[i, key] = value\n",
+    "\n",
+    "    # Advance progress ONLY after successful write\n",
+    "    current_index = i + 1\n",
+    "\n",
+    "    # -------- GPU MEMORY CLEANUP --------\n",
+    "    if torch.cuda.is_available():\n",
+    "        torch.cuda.empty_cache()\n",
+    "        torch.cuda.synchronize()\n",
+    "\n",
+    "    # -------- SAVE LIKE YOUR DEEPSEEK VERSION --------\n",
+    "    if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "        df.to_csv(output_file, index=False)\n",
+    "        with open(index_file, \"w\") as f:\n",
+    "            f.write(str(current_index))\n",
+    "        print(f\"💾 Progress saved after row {i+1}\")\n",
+    "\n",
+    "# Final save\n",
+    "df.to_csv(output_file, index=False)\n",
+    "index_file.write_text(str(current_index))\n",
+    "print(\"✅ Finished full dataset.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8ad48edb-b136-46a4-9277-c1de58f00093",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "pm-paper",
+   "language": "python",
+   "name": "pm-paper"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

jupyter_notebooks/Section_2-2-2_Figure_4_Demographic_patterns_in_gen_images.ipynb

@@ -1773,9 +1773,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "HORDE",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "horde"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -1787,7 +1787,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.4"
+   "version": "3.12.10"
   }
  },
  "nbformat": 4,

jupyter_notebooks/Section_2-3-4_Figure_8_deepfake_adapters.ipynb

@@ -4294,12 +4294,586 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
+   "id": "601d840a-45f3-4679-be84-ebdb7951c2a8",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-11-28T21:37:19.929686Z",
+     "iopub.status.busy": "2025-11-28T21:37:19.928739Z",
+     "iopub.status.idle": "2025-11-28T21:43:25.222061Z",
+     "shell.execute_reply": "2025-11-28T21:43:25.221487Z",
+     "shell.execute_reply.started": "2025-11-28T21:37:19.929296Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading model: google/gemma-3-27b-it\n",
+      "Cache directory: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/models\n",
+      "This may take a while on first run (~65GB download)...\n",
+      "\n",
+      "Device: cuda\n",
+      "Loading tokenizer...\n",
+      "✅ Tokenizer loaded\n",
+      "Loading model (this may take several minutes)...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading checkpoint shards: 100%|██████████| 12/12 [01:13<00:00,  6.12s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Model loaded\n",
+      "VRAM used: 76.27 GB\n",
+      "\n",
+      "Loading raw input CSV...\n",
+      "✅ Loaded professions.csv\n",
+      "✅ Loaded profession mapping with 9 categories\n",
+      "Loaded 50861 rows\n",
+      "\n",
+      "Profession categories (9):\n",
+      "  - actor\n",
+      "  - adult performer\n",
+      "  - singer/musician\n",
+      "  - model\n",
+      "  - online personality\n",
+      "  - public figure\n",
+      "  - voice actor/ASMR\n",
+      "  - sports professional\n",
+      "  - tv personality\n",
+      "\n",
+      "Running in TEST MODE with 100 samples\n",
+      "\n",
+      "Creating prompts...\n",
+      "✅ Prompts created\n",
+      "Resuming from index 0\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local:  10%|█         | 10/100 [00:27<03:51,  2.57s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 10\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local:  20%|██        | 20/100 [00:55<03:46,  2.83s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 20\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local:  24%|██▍       | 24/100 [01:08<04:01,  3.18s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "❌ Row 23: parsed as all Unknown (likely model crash); skipping.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local:  30%|███       | 30/100 [01:24<03:01,  2.60s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 30\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local:  40%|████      | 40/100 [01:50<02:50,  2.84s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 40\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local:  50%|█████     | 50/100 [02:22<02:34,  3.10s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 50\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local:  60%|██████    | 60/100 [02:54<02:04,  3.11s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 60\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local:  70%|███████   | 70/100 [03:20<01:23,  2.77s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 70\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local:  80%|████████  | 80/100 [03:49<00:56,  2.83s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 80\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local:  90%|█████████ | 90/100 [04:17<00:27,  2.75s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 90\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local: 100%|██████████| 100/100 [04:48<00:00,  2.88s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "💾 Progress saved after row 100\n",
+      "✅ Finished full dataset.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "import torch\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
+    "professions_mapped_file = current_dir.parent / \"misc/lists/professions_mapped.csv\"\n",
+    "# === PROCESS DATA ===\n",
+    "output_file = current_dir.parent / f\"data/CSV/gemma_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/query_indicies/gemma_local_query_index.txt\"\n",
+    "\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = True\n",
+    "TEST_SIZE = 100\n",
+    "MAX_ROWS = 50862\n",
+    "SAVE_INTERVAL = 10\n",
+    "\n",
+    "# Model settings\n",
+    "MODEL_NAME =  MODEL_NAME = \"google/gemma-3-27b-it\"\n",
+    "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
+    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
+    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Define the SPECIFIC profession categories\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\",\n",
+    "    \"adult performer\",\n",
+    "    \"singer/musician\",\n",
+    "    \"model\",\n",
+    "    \"online personality\",\n",
+    "    \"public figure\",\n",
+    "    \"voice actor/ASMR\",\n",
+    "    \"sports professional\",\n",
+    "    \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD MODEL ===\n",
+    "print(f\"Loading model: {MODEL_NAME}\")\n",
+    "print(f\"Cache directory: {CACHE_DIR}\")\n",
+    "print(f\"This may take a while on first run (~65GB download)...\\n\")\n",
+    "\n",
+    "# Check GPU availability\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "print(f\"Device: {device}\")\n",
+    "\n",
+    "if device == \"cpu\":\n",
+    "    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
+    "    print(\"   Consider using a GPU or reducing model size.\")\n",
+    "\n",
+    "# Load tokenizer\n",
+    "print(\"Loading tokenizer...\")\n",
+    "try:\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=True\n",
+    "    )\n",
+    "except Exception as e:\n",
+    "    print(f\"Failed with use_fast=True, trying use_fast=False...\")\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=False\n",
+    "    )\n",
+    "\n",
+    "# Ensure pad token is set\n",
+    "if tokenizer.pad_token is None:\n",
+    "    tokenizer.pad_token = tokenizer.eos_token\n",
+    "\n",
+    "print(\"✅ Tokenizer loaded\")\n",
+    "\n",
+    "# Load model with optimizations\n",
+    "print(\"Loading model (this may take several minutes)...\")\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    MODEL_NAME,\n",
+    "    cache_dir=str(CACHE_DIR),\n",
+    "    torch_dtype=torch.bfloat16,\n",
+    "    device_map=\"auto\",\n",
+    "    trust_remote_code=False\n",
+    ")\n",
+    "model.eval()\n",
+    "print(\"✅ Model loaded\")\n",
+    "\n",
+    "# Check VRAM usage\n",
+    "if torch.cuda.is_available():\n",
+    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
+    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "if output_file.exists():\n",
+    "    print(\"Loading annotated CSV...\")\n",
+    "    df = pd.read_csv(output_file)\n",
+    "else:\n",
+    "    print(\"Loading raw input CSV...\")\n",
+    "    df = pd.read_csv(input_file)\n",
+    "\n",
+    "\n",
+    "# Try to load profession mapping files\n",
+    "try:\n",
+    "    professions_df = pd.read_csv(professions_file)\n",
+    "    print(f\"✅ Loaded professions.csv\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions.csv not found\")\n",
+    "\n",
+    "try:\n",
+    "    prof_mapped_df = pd.read_csv(professions_mapped_file)\n",
+    "    print(f\"✅ Loaded profession mapping with {len(prof_mapped_df)} categories\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions_mapped.csv not found, using default categories\")\n",
+    "\n",
+    "profession_str = \", \".join(PROFESSION_CATEGORIES)\n",
+    "\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "print(f\"\\nProfession categories ({len(PROFESSION_CATEGORIES)}):\")\n",
+    "for cat in PROFESSION_CATEGORIES:\n",
+    "    print(f\"  - {cat}\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"\\nRunning in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
+    "\n",
+    "# === CREATE PROMPTS ===\n",
+    "def create_prompt(row):\n",
+    "    \"\"\"Create prompt for Gemma annotation with specific profession categories.\"\"\"\n",
+    "    name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
+    "    \n",
+    "    # Gather hints\n",
+    "    hints = []\n",
+    "    if pd.notna(row.get('likely_profession')):\n",
+    "        hints.append(str(row['likely_profession']))\n",
+    "    if pd.notna(row.get('likely_nationality')):\n",
+    "        hints.append(str(row['likely_nationality']))\n",
+    "    if pd.notna(row.get('likely_country')):\n",
+    "        hints.append(str(row['likely_country']))\n",
+    "    \n",
+    "    # Add tags if we don't have enough hints\n",
+    "    if len(hints) < 3:\n",
+    "        for i in range(1, 8):\n",
+    "            tag_col = f'tag_{i}'\n",
+    "            if tag_col in row and pd.notna(row[tag_col]):\n",
+    "                tag_val = str(row[tag_col])\n",
+    "                if tag_val not in hints:\n",
+    "                    hints.append(tag_val)\n",
+    "                if len(hints) >= 5:\n",
+    "                    break\n",
+    "    \n",
+    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
+    "    \n",
+    "    return f\"\"\"Given '{name}' ({hint_text}), provide:\n",
+    "1. Full legal name (Western order if non-latin script)\n",
+    "2. Any stage names/aliases (comma separated)\n",
+    "3. Gender (Male/Female/Other/Unknown)\n",
+    "4. Top 3 most likely professions from ONLY these categories:\n",
+    "   - actor\n",
+    "   - adult performer\n",
+    "   - singer/musician\n",
+    "   - model\n",
+    "   - online personality (includes streamers, cosplayers, influencers)\n",
+    "   - public figure (includes politicians, activists, journalists, authors)\n",
+    "   - voice actor/ASMR\n",
+    "   - sports professional\n",
+    "   - tv personality (includes hosts, presenters, reality TV)\n",
+    "\n",
+    "5. Primary country associated\n",
+    "\n",
+    "IMPORTANT:\n",
+    "- Choose professions ONLY from the 9 categories above\n",
+    "- Provide up to 3 professions, comma-separated, ordered by relevance\n",
+    "- Be SPECIFIC: choose the most accurate category for each role\n",
+    "- \"online personality\" includes: streamers, cosplayers, YouTubers, influencers, content creators\n",
+    "- Use 'Unknown' when uncertain or for fictional characters/places\n",
+    "- For multi-role people, list all relevant categories (e.g., \"actor, singer/musician, online personality\")\n",
+    "- For country respond with one word only, for example China or Columbia\n",
+    "- actress = actor\n",
+    "\n",
+    "Respond with exactly 5 numbered lines.\"\"\"\n",
+    "\n",
+    "# Create prompts\n",
+    "print(\"\\nCreating prompts...\")\n",
+    "df['prompt'] = df.apply(create_prompt, axis=1)\n",
+    "print(\"✅ Prompts created\")\n",
+    "\n",
+    "# === QUERY Gemma LOCAL ===\n",
+    "def query_gemma_local(prompt: str) -> str:\n",
+    "    \"\"\"Query Gemma locally via transformers.\"\"\"\n",
+    "    try:\n",
+    "        # Format as chat message for Qwen\n",
+    "        messages = [\n",
+    "            {\"role\": \"system\", \"content\": \"You are an assistant that extracts key data on a person based on the name. Respond with exactly 5 numbered lines. For professions, choose ONLY from these categories: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality.\"},\n",
+    "            {\"role\": \"user\", \"content\": prompt}\n",
+    "        ]\n",
+    "        \n",
+    "        # Tokenize\n",
+    "        if hasattr(tokenizer, 'apply_chat_template'):\n",
+    "            text = tokenizer.apply_chat_template(\n",
+    "                messages,\n",
+    "                tokenize=False,\n",
+    "                add_generation_prompt=True\n",
+    "            )\n",
+    "        else:\n",
+    "            # Fallback for older tokenizers\n",
+    "            text = f\"[INST] {prompt} [/INST]\"\n",
+    "        \n",
+    "        inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
+    "        \n",
+    "        # Generate\n",
+    "        with torch.no_grad():\n",
+    "            outputs = model.generate(\n",
+    "                **inputs,\n",
+    "                max_new_tokens=512,\n",
+    "                temperature=0.1,\n",
+    "                do_sample=True,\n",
+    "                top_p=0.9,\n",
+    "                pad_token_id=tokenizer.eos_token_id\n",
+    "            )\n",
+    "        \n",
+    "        # Decode\n",
+    "        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
+    "        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
+    "        \n",
+    "        return response.strip()\n",
+    "        \n",
+    "    except Exception as e:\n",
+    "        print(f\"Generation error: {e}\")\n",
+    "        return None\n",
+    "\n",
+    "# === PARSE RESPONSE ===\n",
+    "def parse_response(response):\n",
+    "    \"\"\"Parse Gemma response into structured fields.\"\"\"\n",
+    "    if not response:\n",
+    "        return {\n",
+    "            'full_name': 'Unknown',\n",
+    "            'aliases': 'Unknown',\n",
+    "            'gender': 'Unknown',\n",
+    "            'profession_llm': 'Unknown',\n",
+    "            'country': 'Unknown'\n",
+    "        }\n",
+    "    \n",
+    "    # Split into lines and clean\n",
+    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
+    "    \n",
+    "    # Initialize with Unknown values\n",
+    "    fields = {\n",
+    "        'full_name': 'Unknown',\n",
+    "        'aliases': 'Unknown',\n",
+    "        'gender': 'Unknown',\n",
+    "        'profession_llm': 'Unknown',\n",
+    "        'country': 'Unknown'\n",
+    "    }\n",
+    "    \n",
+    "    # Extract information from each numbered line\n",
+    "    for line in lines:\n",
+    "        if line.startswith('1.'):\n",
+    "            fields['full_name'] = line[2:].strip()\n",
+    "        elif line.startswith('2.'):\n",
+    "            fields['aliases'] = line[2:].strip()\n",
+    "        elif line.startswith('3.'):\n",
+    "            fields['gender'] = line[2:].strip()\n",
+    "        elif line.startswith('4.'):\n",
+    "            fields['profession_llm'] = line[2:].strip()\n",
+    "        elif line.startswith('5.'):\n",
+    "            fields['country'] = line[2:].strip()\n",
+    "    \n",
+    "    return fields\n",
+    "\n",
+    "# === PROCESS DATA ===\n",
+    "index_file.parent.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Load index\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    try:\n",
+    "        current_index = int(index_file.read_text().strip())\n",
+    "    except:\n",
+    "        current_index = 0\n",
+    "\n",
+    "print(f\"Resuming from index {current_index}\")\n",
+    "\n",
+    "start_time = time.time()\n",
+    "\n",
+    "for i in tqdm(range(current_index, len(df)), desc=\"Gemma Local\"):\n",
+    "\n",
+    "    prompt = df.at[i, \"prompt\"]\n",
+    "\n",
+    "    # -------- MODEL QUERY WITH RETRIES --------\n",
+    "    response = None\n",
+    "    for attempt in range(3):\n",
+    "        response = query_gemma_local(prompt)\n",
+    "        \n",
+    "        # Valid response?\n",
+    "        if response and len(response.strip()) > 10:\n",
+    "            break\n",
+    "        \n",
+    "        print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
+    "        time.sleep(0.5)\n",
+    "\n",
+    "    # If still invalid → DO NOT overwrite previous data\n",
+    "    if not response or len(response.strip()) <= 10:\n",
+    "        print(f\"❌ Row {i}: failed after retries, not writing, not advancing index\")\n",
+    "        continue\n",
+    "\n",
+    "    parsed = parse_response(response)\n",
+    "\n",
+    "    # Additional safety: skip rows that parsed as all 'Unknown'\n",
+    "    if all(v == \"Unknown\" for v in parsed.values()):\n",
+    "        print(f\"❌ Row {i}: parsed as all Unknown (likely model crash); skipping.\")\n",
+    "        continue\n",
+    "\n",
+    "    # -------- WRITE PARSED FIELDS SAFELY --------\n",
+    "    for key, value in parsed.items():\n",
+    "        df.at[i, key] = value\n",
+    "\n",
+    "    # Advance progress ONLY after successful write\n",
+    "    current_index = i + 1\n",
+    "\n",
+    "    # -------- GPU MEMORY CLEANUP --------\n",
+    "    if torch.cuda.is_available():\n",
+    "        torch.cuda.empty_cache()\n",
+    "        torch.cuda.synchronize()\n",
+    "\n",
+    "    # -------- SAVE LIKE YOUR DEEPSEEK VERSION --------\n",
+    "    if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "        df.to_csv(output_file, index=False)\n",
+    "        with open(index_file, \"w\") as f:\n",
+    "            f.write(str(current_index))\n",
+    "        print(f\"💾 Progress saved after row {i+1}\")\n",
+    "\n",
+    "# Final save\n",
+    "df.to_csv(output_file, index=False)\n",
+    "index_file.write_text(str(current_index))\n",
+    "print(\"✅ Finished full dataset.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
    "id": "4gb2vta3w5g",
    "metadata": {
     "execution": {
      "iopub.execute_input": "2025-11-28T21:20:20.130202Z",
-     "iopub.status.busy": "2025-11-28T21:20:20.130008Z"
+     "iopub.status.busy": "2025-11-28T21:20:20.130008Z",
+     "iopub.status.idle": "2025-11-28T21:24:01.683721Z",
+     "shell.execute_reply": "2025-11-28T21:24:01.683179Z",
+     "shell.execute_reply.started": "2025-11-28T21:20:20.130186Z"
     }
    },
    "outputs": [
@@ -4321,7 +4895,64 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Loading checkpoint shards:  25%|██▌       | 3/12 [00:46<02:18, 15.38s/it]"
+      "Loading checkpoint shards: 100%|██████████| 12/12 [02:51<00:00, 14.26s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Model loaded\n",
+      "VRAM used: 64.66 GB\n",
+      "\n",
+      "Loaded 50861 rows\n",
+      "Running in TEST MODE with 10 samples\n",
+      "\n",
+      "Annotating with GEMMA-3-27B LOCAL - rows 0 to 10...\n",
+      "Model: google/gemma-3-27b-it\n",
+      "This may take a while (~250-600 samples/hour on A100)\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Gemma Local: 100%|██████████| 10/10 [00:45<00:00,  4.59s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Saved after 10 rows (~0.0 samples/hour)\n",
+      "\n",
+      "✅ Done! Results: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/CSV/gemma_local_annotated_POI_test.csv\n",
+      "Total time: 0.8 minutes\n",
+      "Average speed: 784.9 samples/hour\n",
+      "\n",
+      "Top professions:\n",
+      "profession_llm\n",
+      "singer/musician, actor, model                            1\n",
+      "model, online personality, adult performer               1\n",
+      "singer/musician, voice actor/ASMR, online personality    1\n",
+      "model, online personality, public figure                 1\n",
+      "singer/musician, model, public figure                    1\n",
+      "model, adult performer, online personality               1\n",
+      "actor, singer/musician, model                            1\n",
+      "actress, idol, model                                     1\n",
+      "actor, public figure, model                              1\n",
+      "actor, model, public figure                              1\n",
+      "Name: count, dtype: int64\n",
+      "\n",
+      "Final VRAM usage: 64.79 GB\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
      ]
     }
    ],

misc/query_indicies/gemma_local_query_index.txt

@@ -0,0 +1 @@
+22720

misc/query_indicies/mistral_local_query_index.txt

@@ -1 +1 @@
-8790
+25820