refactored model inference code

jupyter_notebooks/GEMMA_3.ipynb

@@ -1,400 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "471e0cef-678e-4403-8eca-f8e1991d86de",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import json\n",
-    "import time\n",
-    "import re\n",
-    "from pathlib import Path\n",
-    "from tqdm import tqdm\n",
-    "import torch\n",
-    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
-    "\n",
-    "current_dir = Path.cwd()\n",
-    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
-    "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
-    "professions_mapped_file = current_dir.parent / \"misc/lists/professions_mapped.csv\"\n",
-    "# === PROCESS DATA ===\n",
-    "\n",
-    "\n",
-    "# === CONFIGURATION ===\n",
-    "TEST_MODE = False\n",
-    "TEST_SIZE = 100\n",
-    "MAX_ROWS = 50862\n",
-    "SAVE_INTERVAL = 10\n",
-    "\n",
-    "output_file = current_dir.parent / f\"data/CSV/gemma_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
-    "index_file = current_dir.parent / \"misc/query_indicies/gemma_local_query_index.txt\"\n",
-    "\n",
-    "# Model settings\n",
-    "MODEL_NAME =  MODEL_NAME = \"google/gemma-3-27b-it\"\n",
-    "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
-    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
-    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
-    "\n",
-    "# Define the SPECIFIC profession categories\n",
-    "PROFESSION_CATEGORIES = [\n",
-    "    \"actor\",\n",
-    "    \"adult performer\",\n",
-    "    \"singer/musician\",\n",
-    "    \"model\",\n",
-    "    \"online personality\",\n",
-    "    \"public figure\",\n",
-    "    \"voice actor/ASMR\",\n",
-    "    \"sports professional\",\n",
-    "    \"tv personality\"\n",
-    "]\n",
-    "\n",
-    "# === LOAD MODEL ===\n",
-    "print(f\"Loading model: {MODEL_NAME}\")\n",
-    "print(f\"Cache directory: {CACHE_DIR}\")\n",
-    "print(f\"This may take a while on first run (~65GB download)...\\n\")\n",
-    "\n",
-    "# Check GPU availability\n",
-    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
-    "print(f\"Device: {device}\")\n",
-    "\n",
-    "if device == \"cpu\":\n",
-    "    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
-    "    print(\"   Consider using a GPU or reducing model size.\")\n",
-    "\n",
-    "# Load tokenizer\n",
-    "print(\"Loading tokenizer...\")\n",
-    "try:\n",
-    "    tokenizer = AutoTokenizer.from_pretrained(\n",
-    "        MODEL_NAME,\n",
-    "        cache_dir=str(CACHE_DIR),\n",
-    "        use_fast=True\n",
-    "    )\n",
-    "except Exception as e:\n",
-    "    print(f\"Failed with use_fast=True, trying use_fast=False...\")\n",
-    "    tokenizer = AutoTokenizer.from_pretrained(\n",
-    "        MODEL_NAME,\n",
-    "        cache_dir=str(CACHE_DIR),\n",
-    "        use_fast=False\n",
-    "    )\n",
-    "\n",
-    "# Ensure pad token is set\n",
-    "if tokenizer.pad_token is None:\n",
-    "    tokenizer.pad_token = tokenizer.eos_token\n",
-    "\n",
-    "print(\"✅ Tokenizer loaded\")\n",
-    "\n",
-    "# Load model with optimizations\n",
-    "print(\"Loading model (this may take several minutes)...\")\n",
-    "model = AutoModelForCausalLM.from_pretrained(\n",
-    "    MODEL_NAME,\n",
-    "    cache_dir=str(CACHE_DIR),\n",
-    "    torch_dtype=torch.bfloat16,\n",
-    "    device_map=\"auto\",\n",
-    "    trust_remote_code=False\n",
-    ")\n",
-    "model.eval()\n",
-    "print(\"✅ Model loaded\")\n",
-    "\n",
-    "# Check VRAM usage\n",
-    "if torch.cuda.is_available():\n",
-    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
-    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
-    "\n",
-    "# === LOAD DATA ===\n",
-    "print(\"Loading raw input CSV...\")\n",
-    "df = pd.read_csv(input_file)  # ALWAYS load the full input\n",
-    "print(f\"Loaded {len(df)} rows from input file\")\n",
-    "\n",
-    "# If we have previous annotations, merge them\n",
-    "if output_file.exists():\n",
-    "    print(\"Found existing annotations, merging...\")\n",
-    "    existing_df = pd.read_csv(output_file)\n",
-    "    print(f\"Existing annotations has {len(existing_df)} rows\")\n",
-    "    \n",
-    "    # Update df with existing annotations\n",
-    "    # Only update the columns that were annotated\n",
-    "    annotation_cols = ['full_name', 'aliases', 'gender', 'profession_llm', 'country']\n",
-    "    for col in annotation_cols:\n",
-    "        if col in existing_df.columns:\n",
-    "            df[col] = existing_df[col][:len(df)]  # Make sure we don't exceed df length\n",
-    "    \n",
-    "    print(f\"Merged annotations, continuing with {len(df)} total rows\")\n",
-    "\n",
-    "\n",
-    "# Try to load profession mapping files\n",
-    "try:\n",
-    "    professions_df = pd.read_csv(professions_file)\n",
-    "    print(f\"✅ Loaded professions.csv\")\n",
-    "except:\n",
-    "    print(\"⚠️ Warning: professions.csv not found\")\n",
-    "\n",
-    "try:\n",
-    "    prof_mapped_df = pd.read_csv(professions_mapped_file)\n",
-    "    print(f\"✅ Loaded profession mapping with {len(prof_mapped_df)} categories\")\n",
-    "except:\n",
-    "    print(\"⚠️ Warning: professions_mapped.csv not found, using default categories\")\n",
-    "\n",
-    "profession_str = \", \".join(PROFESSION_CATEGORIES)\n",
-    "\n",
-    "print(f\"Loaded {len(df)} rows\")\n",
-    "print(f\"\\nProfession categories ({len(PROFESSION_CATEGORIES)}):\")\n",
-    "for cat in PROFESSION_CATEGORIES:\n",
-    "    print(f\"  - {cat}\")\n",
-    "\n",
-    "if TEST_MODE:\n",
-    "    print(f\"\\nRunning in TEST MODE with {TEST_SIZE} samples\")\n",
-    "    df = df.head(TEST_SIZE).copy()\n",
-    "elif MAX_ROWS:\n",
-    "    df = df.head(MAX_ROWS).copy()\n",
-    "\n",
-    "# === CREATE PROMPTS ===\n",
-    "def create_prompt(row):\n",
-    "    \"\"\"Create prompt for Gemma annotation with specific profession categories.\"\"\"\n",
-    "    name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
-    "    \n",
-    "    # Gather hints\n",
-    "    hints = []\n",
-    "    if pd.notna(row.get('likely_profession')):\n",
-    "        hints.append(str(row['likely_profession']))\n",
-    "    if pd.notna(row.get('likely_nationality')):\n",
-    "        hints.append(str(row['likely_nationality']))\n",
-    "    if pd.notna(row.get('likely_country')):\n",
-    "        hints.append(str(row['likely_country']))\n",
-    "    \n",
-    "    # Add tags if we don't have enough hints\n",
-    "    if len(hints) < 3:\n",
-    "        for i in range(1, 8):\n",
-    "            tag_col = f'tag_{i}'\n",
-    "            if tag_col in row and pd.notna(row[tag_col]):\n",
-    "                tag_val = str(row[tag_col])\n",
-    "                if tag_val not in hints:\n",
-    "                    hints.append(tag_val)\n",
-    "                if len(hints) >= 5:\n",
-    "                    break\n",
-    "    \n",
-    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
-    "    \n",
-    "    return f\"\"\"Given '{name}' ({hint_text}), provide:\n",
-    "1. Full legal name (Western order if non-latin script)\n",
-    "2. Any stage names/aliases (comma separated)\n",
-    "3. Gender (Male/Female/Other/Unknown)\n",
-    "4. Top 3 most likely professions from ONLY these categories:\n",
-    "   - actor\n",
-    "   - adult performer\n",
-    "   - singer/musician\n",
-    "   - model\n",
-    "   - online personality (includes streamers, cosplayers, influencers)\n",
-    "   - public figure (includes politicians, activists, journalists, authors)\n",
-    "   - voice actor/ASMR\n",
-    "   - sports professional\n",
-    "   - tv personality (includes hosts, presenters, reality TV)\n",
-    "\n",
-    "5. Primary country associated\n",
-    "\n",
-    "IMPORTANT:\n",
-    "- Choose professions ONLY from the 9 categories above\n",
-    "- Provide up to 3 professions, comma-separated, ordered by relevance\n",
-    "- Be SPECIFIC: choose the most accurate category for each role\n",
-    "- \"online personality\" includes: streamers, cosplayers, YouTubers, influencers, content creators\n",
-    "- Use 'Unknown' when uncertain or for fictional characters/places\n",
-    "- For multi-role people, list all relevant categories (e.g., \"actor, singer/musician, online personality\")\n",
-    "- For country respond with one word only, for example China or Columbia\n",
-    "- actress = actor\n",
-    "\n",
-    "Respond with exactly 5 numbered lines.\"\"\"\n",
-    "\n",
-    "# Create prompts\n",
-    "print(\"\\nCreating prompts...\")\n",
-    "df['prompt'] = df.apply(create_prompt, axis=1)\n",
-    "print(\"✅ Prompts created\")\n",
-    "\n",
-    "# === QUERY Gemma LOCAL ===\n",
-    "def query_gemma_local(prompt: str) -> str:\n",
-    "    \"\"\"Query Gemma locally via transformers.\"\"\"\n",
-    "    try:\n",
-    "        # Format as chat message for GEMMA\n",
-    "        messages = [\n",
-    "            {\"role\": \"system\", \"content\": \"You are an assistant that extracts key data on a person based on the name. Respond with exactly 5 numbered lines. For professions, choose ONLY from these categories: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality.\"},\n",
-    "            {\"role\": \"user\", \"content\": prompt}\n",
-    "        ]\n",
-    "        \n",
-    "        # Tokenize\n",
-    "        if hasattr(tokenizer, 'apply_chat_template'):\n",
-    "            text = tokenizer.apply_chat_template(\n",
-    "                messages,\n",
-    "                tokenize=False,\n",
-    "                add_generation_prompt=True\n",
-    "            )\n",
-    "        else:\n",
-    "            # Fallback for older tokenizers\n",
-    "            text = f\"[INST] {prompt} [/INST]\"\n",
-    "        \n",
-    "        inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
-    "        \n",
-    "        # Generate\n",
-    "        with torch.no_grad():\n",
-    "            outputs = model.generate(\n",
-    "                **inputs,\n",
-    "                max_new_tokens=512,\n",
-    "                temperature=0.1,\n",
-    "                do_sample=True,\n",
-    "                top_p=0.9,\n",
-    "                pad_token_id=tokenizer.eos_token_id\n",
-    "            )\n",
-    "        \n",
-    "        # Decode\n",
-    "        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
-    "        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
-    "        \n",
-    "        return response.strip()\n",
-    "        \n",
-    "    except Exception as e:\n",
-    "        print(f\"Generation error: {e}\")\n",
-    "        return None\n",
-    "\n",
-    "output_file = current_dir.parent / f\"data/CSV/gemma_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
-    "index_file = current_dir.parent / \"misc/query_indicies/gemma_local_query_index.txt\"\n",
-    "\n",
-    "\n",
-    "# === PARSE RESPONSE ===\n",
-    "def parse_response(response):\n",
-    "    \"\"\"Parse Gemma response into structured fields.\"\"\"\n",
-    "    if not response:\n",
-    "        return {\n",
-    "            'full_name': 'Unknown',\n",
-    "            'aliases': 'Unknown',\n",
-    "            'gender': 'Unknown',\n",
-    "            'profession_llm': 'Unknown',\n",
-    "            'country': 'Unknown'\n",
-    "        }\n",
-    "    \n",
-    "    # Split into lines and clean\n",
-    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
-    "    \n",
-    "    # Initialize with Unknown values\n",
-    "    fields = {\n",
-    "        'full_name': 'Unknown',\n",
-    "        'aliases': 'Unknown',\n",
-    "        'gender': 'Unknown',\n",
-    "        'profession_llm': 'Unknown',\n",
-    "        'country': 'Unknown'\n",
-    "    }\n",
-    "    \n",
-    "    # Extract information from each numbered line\n",
-    "    for line in lines:\n",
-    "        if line.startswith('1.'):\n",
-    "            fields['full_name'] = line[2:].strip()\n",
-    "        elif line.startswith('2.'):\n",
-    "            fields['aliases'] = line[2:].strip()\n",
-    "        elif line.startswith('3.'):\n",
-    "            fields['gender'] = line[2:].strip()\n",
-    "        elif line.startswith('4.'):\n",
-    "            fields['profession_llm'] = line[2:].strip()\n",
-    "        elif line.startswith('5.'):\n",
-    "            fields['country'] = line[2:].strip()\n",
-    "    \n",
-    "    return fields\n",
-    "\n",
-    "\n",
-    "# === PROCESS DATA ===\n",
-    "index_file.parent.mkdir(parents=True, exist_ok=True)\n",
-    "\n",
-    "# Load index\n",
-    "current_index = 0\n",
-    "if index_file.exists():\n",
-    "    try:\n",
-    "        current_index = int(index_file.read_text().strip())\n",
-    "    except:\n",
-    "        current_index = 0\n",
-    "\n",
-    "print(f\"Resuming from index {current_index}\")\n",
-    "\n",
-    "start_time = time.time()\n",
-    "\n",
-    "for i in tqdm(range(current_index, len(df)), desc=\"Gemma Local\"):\n",
-    "\n",
-    "    prompt = df.at[i, \"prompt\"]\n",
-    "\n",
-    "    # -------- MODEL QUERY WITH RETRIES --------\n",
-    "    response = None\n",
-    "    for attempt in range(3):\n",
-    "        response = query_gemma_local(prompt)\n",
-    "        \n",
-    "        # Valid response?\n",
-    "        if response and len(response.strip()) > 10:\n",
-    "            break\n",
-    "        \n",
-    "        print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
-    "        time.sleep(0.5)\n",
-    "\n",
-    "    # If still invalid → DO NOT overwrite previous data\n",
-    "    if not response or len(response.strip()) <= 10:\n",
-    "        print(f\"❌ Row {i}: failed after retries, not writing, not advancing index\")\n",
-    "        continue\n",
-    "\n",
-    "    parsed = parse_response(response)\n",
-    "\n",
-    "    # Additional safety: skip rows that parsed as all 'Unknown'\n",
-    "    if all(v == \"Unknown\" for v in parsed.values()):\n",
-    "        print(f\"❌ Row {i}: parsed as all Unknown (likely model crash); skipping.\")\n",
-    "        continue\n",
-    "\n",
-    "    # -------- WRITE PARSED FIELDS SAFELY --------\n",
-    "    for key, value in parsed.items():\n",
-    "        df.at[i, key] = value\n",
-    "\n",
-    "    # Advance progress ONLY after successful write\n",
-    "    current_index = i + 1\n",
-    "\n",
-    "    # -------- GPU MEMORY CLEANUP --------\n",
-    "    if torch.cuda.is_available():\n",
-    "        torch.cuda.empty_cache()\n",
-    "        torch.cuda.synchronize()\n",
-    "\n",
-    "    # -------- SAVE LIKE YOUR DEEPSEEK VERSION --------\n",
-    "    if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
-    "        df.to_csv(output_file, index=False)\n",
-    "        with open(index_file, \"w\") as f:\n",
-    "            f.write(str(current_index))\n",
-    "        print(f\"💾 Progress saved after row {i+1}\")\n",
-    "\n",
-    "# Final save\n",
-    "df.to_csv(output_file, index=False)\n",
-    "index_file.write_text(str(current_index))\n",
-    "print(\"✅ Finished full dataset.\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7a1be7c0-ce54-4445-8534-bf3ab5e70197",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "pm-paper",
-   "language": "python",
-   "name": "pm-paper"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.13"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

jupyter_notebooks/QWEN.ipynb

@@ -1,1341 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "0543d9c4-055b-49a3-a8d0-9dfb622b2b8c",
-   "metadata": {},
-   "source": [
-    "# QWEN 2.5-32B Local Inference\n",
-    "\n",
-    "## Hardware Requirements\n",
-    "- **GPU**: NVIDIA A100 (40GB or 80GB recommended)\n",
-    "- **VRAM Usage**: \n",
-    "  - 8-bit quantization: ~32GB\n",
-    "  - 4-bit quantization: ~16-20GB\n",
-    "  - bfloat16 (no quantization): ~64GB\n",
-    "- **System RAM**: 32GB minimum, 64GB recommended\n",
-    "- **Storage**: ~65GB for model download\n",
-    "\n",
-    "## Configuration\n",
-    "This notebook uses **8-bit quantization** via `bitsandbytes` for optimal performance on A100 GPUs:\n",
-    "- Reduces VRAM usage from 64GB to ~32GB\n",
-    "- Minimal quality degradation\n",
-    "- Faster inference than bfloat16\n",
-    "\n",
-    "## Model Details\n",
-    "- **Model**: Qwen/Qwen2.5-32B-Instruct\n",
-    "- **Task**: Entity annotation and profession classification\n",
-    "- **Quantization**: LLM.int8() (8-bit)\n",
-    "- **Device**: CUDA (auto device mapping)\n",
-    "\n",
-    "## Dependencies\n",
-    "Make sure to install:\n",
-    "```bash\n",
-    "pip install transformers>=4.35.0 bitsandbytes>=0.41.0 accelerate torch pandas tqdm\n",
-    "```"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "fe6ba282-896b-4272-b82b-ef24810732fb",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-12-07T21:04:54.429449Z",
-     "iopub.status.busy": "2025-12-07T21:04:54.429316Z"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Loading model: Qwen/Qwen2.5-32B-Instruct\n",
-      "Cache directory: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/models\n",
-      "This may take a while on first run (~65GB download)...\n",
-      "\n",
-      "Device: cuda\n",
-      "Loading tokenizer...\n",
-      "✅ Tokenizer loaded\n",
-      "Configuring 8-bit quantization...\n",
-      "Loading model with 8-bit quantization (this may take several minutes)...\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Loading checkpoint shards: 100%|██████████| 17/17 [03:55<00:00, 13.84s/it]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ Model loaded with 8-bit quantization\n",
-      "VRAM used: 32.72 GB\n",
-      "\n",
-      "Loading annotated CSV...\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/tmp/ipykernel_1943010/494578941.py:113: DtypeWarning: Columns (53,54,55,56,57) have mixed types. Specify dtype option on import or set low_memory=False.\n",
-      "  df = pd.read_csv(output_file)\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ Loaded professions.csv\n",
-      "✅ Loaded profession mapping with 9 categories\n",
-      "Loaded 50861 rows\n",
-      "\n",
-      "Profession categories (9):\n",
-      "  - actor\n",
-      "  - adult performer\n",
-      "  - singer/musician\n",
-      "  - model\n",
-      "  - online personality\n",
-      "  - public figure\n",
-      "  - voice actor/ASMR\n",
-      "  - sports professional\n",
-      "  - tv personality\n",
-      "\n",
-      "Creating prompts...\n",
-      "✅ Prompts created\n",
-      "Resuming from index 48500\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   0%|          | 0/2361 [00:00<?, ?it/s]The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details.\n",
-      "Qwen Local:   0%|          | 10/2361 [01:07<4:34:08,  7.00s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48510\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   1%|          | 20/2361 [02:16<4:45:58,  7.33s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48520\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   1%|▏         | 30/2361 [03:24<4:52:57,  7.54s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48530\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   2%|▏         | 40/2361 [04:37<5:05:46,  7.90s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48540\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   2%|▏         | 50/2361 [05:48<4:44:32,  7.39s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48550\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   3%|▎         | 60/2361 [06:58<4:41:41,  7.35s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48560\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   3%|▎         | 70/2361 [08:12<4:52:28,  7.66s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48570\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   3%|▎         | 80/2361 [09:19<4:36:29,  7.27s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48580\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   4%|▍         | 90/2361 [10:30<4:47:42,  7.60s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48590\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   4%|▍         | 100/2361 [11:32<4:16:12,  6.80s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48600\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   5%|▍         | 110/2361 [12:41<4:31:59,  7.25s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48610\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   5%|▌         | 120/2361 [13:48<4:39:07,  7.47s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48620\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   6%|▌         | 130/2361 [14:54<4:20:42,  7.01s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48630\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   6%|▌         | 140/2361 [15:57<4:10:07,  6.76s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48640\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   6%|▋         | 150/2361 [17:06<4:33:11,  7.41s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48650\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   7%|▋         | 160/2361 [18:17<4:50:10,  7.91s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48660\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   7%|▋         | 170/2361 [19:24<4:12:48,  6.92s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48670\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   8%|▊         | 180/2361 [20:32<4:45:13,  7.85s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48680\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   8%|▊         | 190/2361 [21:42<4:17:34,  7.12s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48690\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   8%|▊         | 200/2361 [22:51<4:22:23,  7.29s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48700\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   9%|▉         | 210/2361 [23:58<4:16:32,  7.16s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48710\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:   9%|▉         | 220/2361 [25:11<4:30:09,  7.57s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48720\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  10%|▉         | 230/2361 [26:13<3:56:48,  6.67s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48730\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  10%|█         | 240/2361 [27:21<4:19:15,  7.33s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48740\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  11%|█         | 250/2361 [28:30<4:25:08,  7.54s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48750\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  11%|█         | 260/2361 [29:33<3:52:41,  6.65s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48760\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  11%|█▏        | 270/2361 [30:42<4:35:01,  7.89s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48770\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  12%|█▏        | 280/2361 [31:51<4:24:56,  7.64s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48780\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  12%|█▏        | 290/2361 [33:06<4:36:14,  8.00s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48790\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  13%|█▎        | 300/2361 [34:16<4:15:00,  7.42s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48800\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  13%|█▎        | 310/2361 [35:25<4:02:07,  7.08s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48810\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  14%|█▎        | 320/2361 [36:33<4:10:07,  7.35s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48820\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  14%|█▍        | 330/2361 [37:40<4:02:47,  7.17s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48830\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  14%|█▍        | 340/2361 [38:55<4:12:49,  7.51s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48840\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  15%|█▍        | 350/2361 [40:06<4:13:06,  7.55s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48850\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  15%|█▌        | 360/2361 [41:18<4:18:31,  7.75s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48860\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  16%|█▌        | 370/2361 [42:27<4:25:14,  7.99s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48870\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  16%|█▌        | 380/2361 [43:37<3:58:31,  7.22s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48880\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  17%|█▋        | 390/2361 [44:44<3:59:45,  7.30s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48890\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  17%|█▋        | 400/2361 [45:53<4:12:26,  7.72s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48900\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  17%|█▋        | 410/2361 [47:02<4:07:58,  7.63s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48910\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  18%|█▊        | 420/2361 [48:14<4:12:44,  7.81s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48920\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  18%|█▊        | 430/2361 [49:24<4:00:51,  7.48s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48930\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  19%|█▊        | 440/2361 [50:37<4:26:00,  8.31s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48940\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  19%|█▉        | 450/2361 [51:42<3:43:29,  7.02s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48950\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  19%|█▉        | 460/2361 [52:49<3:33:21,  6.73s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48960\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  20%|█▉        | 470/2361 [54:00<3:42:24,  7.06s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48970\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  20%|██        | 480/2361 [55:08<3:55:25,  7.51s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48980\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  21%|██        | 490/2361 [56:15<3:37:41,  6.98s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 48990\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  21%|██        | 500/2361 [57:22<3:48:29,  7.37s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 49000\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  22%|██▏       | 510/2361 [58:30<3:45:27,  7.31s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 49010\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  22%|██▏       | 520/2361 [59:42<3:56:27,  7.71s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 49020\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  22%|██▏       | 530/2361 [1:00:50<3:42:52,  7.30s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 49030\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  23%|██▎       | 540/2361 [1:01:57<3:33:27,  7.03s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 49040\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  23%|██▎       | 550/2361 [1:03:06<3:39:19,  7.27s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 49050\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  24%|██▎       | 560/2361 [1:04:13<3:30:51,  7.02s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 49060\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  24%|██▍       | 570/2361 [1:05:23<3:43:28,  7.49s/it]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "💾 Progress saved after row 49070\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Qwen Local:  24%|██▍       | 571/2361 [1:05:30<3:36:42,  7.26s/it]"
-     ]
-    }
-   ],
-   "source": [
-    "import pandas as pd\n",
-    "import json\n",
-    "import time\n",
-    "import re\n",
-    "from pathlib import Path\n",
-    "from tqdm import tqdm\n",
-    "import torch\n",
-    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
-    "import signal\n",
-    "from contextlib import contextmanager\n",
-    "\n",
-    "current_dir = Path.cwd()\n",
-    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
-    "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
-    "professions_mapped_file = current_dir.parent / \"misc/lists/professions_mapped.csv\"\n",
-    "# === PROCESS DATA ===\n",
-    "\n",
-    "\n",
-    "# === CONFIGURATION ===\n",
-    "TEST_MODE = False\n",
-    "TEST_SIZE = 100\n",
-    "MAX_ROWS = 50862\n",
-    "SAVE_INTERVAL = 10\n",
-    "\n",
-    "\n",
-    "index_file = current_dir.parent / \"misc/query_indicies/qwen_local_query_index.txt\"\n",
-    "output_file = current_dir.parent / f\"data/CSV/qwen_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
-    "\n",
-    "# Model settings\n",
-    "MODEL_NAME = \"Qwen/Qwen2.5-32B-Instruct\"\n",
-    "#MODEL_NAME = \"Qwen/Qwen2.5-14B-Instruct\"\n",
-    "#MODEL_NAME =  \"Qwen/Qwen3-235B-A22B-Instruct-2507-FP8\"\n",
-    "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
-    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
-    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
-    "\n",
-    "# Define the SPECIFIC profession categories\n",
-    "PROFESSION_CATEGORIES = [\n",
-    "    \"actor\",\n",
-    "    \"adult performer\",\n",
-    "    \"singer/musician\",\n",
-    "    \"model\",\n",
-    "    \"online personality\",\n",
-    "    \"public figure\",\n",
-    "    \"voice actor/ASMR\",\n",
-    "    \"sports professional\",\n",
-    "    \"tv personality\"\n",
-    "]\n",
-    "\n",
-    "# === LOAD MODEL ===\n",
-    "print(f\"Loading model: {MODEL_NAME}\")\n",
-    "print(f\"Cache directory: {CACHE_DIR}\")\n",
-    "print(f\"This may take a while on first run (~65GB download)...\\n\")\n",
-    "\n",
-    "# Check GPU availability\n",
-    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
-    "print(f\"Device: {device}\")\n",
-    "\n",
-    "if device == \"cpu\":\n",
-    "    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
-    "    print(\"   Consider using a GPU or reducing model size.\")\n",
-    "\n",
-    "# Load tokenizer\n",
-    "print(\"Loading tokenizer...\")\n",
-    "try:\n",
-    "    tokenizer = AutoTokenizer.from_pretrained(\n",
-    "        MODEL_NAME,\n",
-    "        cache_dir=str(CACHE_DIR),\n",
-    "        use_fast=True\n",
-    "    )\n",
-    "except Exception as e:\n",
-    "    print(f\"Failed with use_fast=True, trying use_fast=False...\")\n",
-    "    tokenizer = AutoTokenizer.from_pretrained(\n",
-    "        MODEL_NAME,\n",
-    "        cache_dir=str(CACHE_DIR),\n",
-    "        use_fast=False\n",
-    "    )\n",
-    "\n",
-    "# Ensure pad token is set\n",
-    "if tokenizer.pad_token is None:\n",
-    "    tokenizer.pad_token = tokenizer.eos_token\n",
-    "\n",
-    "print(\"✅ Tokenizer loaded\")\n",
-    "\n",
-    "# Configure 8-bit quantization for A100\n",
-    "print(\"Configuring 8-bit quantization...\")\n",
-    "quantization_config = BitsAndBytesConfig(\n",
-    "    load_in_8bit=True,\n",
-    "    llm_int8_threshold=6.0,\n",
-    "    llm_int8_has_fp16_weight=False\n",
-    ")\n",
-    "\n",
-    "# Load model with 8-bit quantization\n",
-    "print(\"Loading model with 8-bit quantization (this may take several minutes)...\")\n",
-    "model = AutoModelForCausalLM.from_pretrained(\n",
-    "    MODEL_NAME,\n",
-    "    cache_dir=str(CACHE_DIR),\n",
-    "    quantization_config=quantization_config,\n",
-    "    device_map=\"auto\",\n",
-    "    trust_remote_code=False\n",
-    ")\n",
-    "model.eval()\n",
-    "print(\"✅ Model loaded with 8-bit quantization\")\n",
-    "\n",
-    "# Check VRAM usage\n",
-    "if torch.cuda.is_available():\n",
-    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
-    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
-    "\n",
-    "# === LOAD DATA ===\n",
-    "if output_file.exists():\n",
-    "    print(\"Loading annotated CSV...\")\n",
-    "    df = pd.read_csv(output_file)\n",
-    "else:\n",
-    "    print(\"Loading raw input CSV...\")\n",
-    "    df = pd.read_csv(input_file)\n",
-    "\n",
-    "\n",
-    "# Try to load profession mapping files\n",
-    "try:\n",
-    "    professions_df = pd.read_csv(professions_file)\n",
-    "    print(f\"✅ Loaded professions.csv\")\n",
-    "except:\n",
-    "    print(\"⚠️ Warning: professions.csv not found\")\n",
-    "\n",
-    "try:\n",
-    "    prof_mapped_df = pd.read_csv(professions_mapped_file)\n",
-    "    print(f\"✅ Loaded profession mapping with {len(prof_mapped_df)} categories\")\n",
-    "except:\n",
-    "    print(\"⚠️ Warning: professions_mapped.csv not found, using default categories\")\n",
-    "\n",
-    "profession_str = \", \".join(PROFESSION_CATEGORIES)\n",
-    "\n",
-    "print(f\"Loaded {len(df)} rows\")\n",
-    "print(f\"\\nProfession categories ({len(PROFESSION_CATEGORIES)}):\")\n",
-    "for cat in PROFESSION_CATEGORIES:\n",
-    "    print(f\"  - {cat}\")\n",
-    "\n",
-    "if TEST_MODE:\n",
-    "    print(f\"\\nRunning in TEST MODE with {TEST_SIZE} samples\")\n",
-    "    df = df.head(TEST_SIZE).copy()\n",
-    "elif MAX_ROWS:\n",
-    "    df = df.head(MAX_ROWS).copy()\n",
-    "\n",
-    "# === CREATE PROMPTS (OPTIMIZED FOR CLEAN OUTPUTS) ===\n",
-    "def create_prompt(row):\n",
-    "    \"\"\"Create prompt for Qwen annotation with strict formatting requirements.\"\"\"\n",
-    "    name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
-    "    \n",
-    "    # Gather hints\n",
-    "    hints = []\n",
-    "    if pd.notna(row.get('likely_profession')):\n",
-    "        hints.append(str(row['likely_profession']))\n",
-    "    if pd.notna(row.get('likely_nationality')):\n",
-    "        hints.append(str(row['likely_nationality']))\n",
-    "    if pd.notna(row.get('likely_country')):\n",
-    "        hints.append(str(row['likely_country']))\n",
-    "    \n",
-    "    # Add tags if we don't have enough hints\n",
-    "    if len(hints) < 3:\n",
-    "        for i in range(1, 8):\n",
-    "            tag_col = f'tag_{i}'\n",
-    "            if tag_col in row and pd.notna(row[tag_col]):\n",
-    "                tag_val = str(row[tag_col])\n",
-    "                if tag_val not in hints:\n",
-    "                    hints.append(tag_val)\n",
-    "                if len(hints) >= 5:\n",
-    "                    break\n",
-    "    \n",
-    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
-    "    \n",
-    "    return f\"\"\"Extract information about '{name}' ({hint_text}).\n",
-    "\n",
-    "Respond with EXACTLY 5 numbered lines. Each line must contain ONLY the value, no labels or extra text.\n",
-    "\n",
-    "FORMAT REQUIREMENTS:\n",
-    "1. Full legal name in Western order (first last). VALUE ONLY.\n",
-    "2. Stage names/aliases, comma-separated. If none, write \"None\". VALUE ONLY.\n",
-    "3. Gender: MUST be exactly one word: Male, Female, Other, or Unknown. VALUE ONLY.\n",
-    "4. Professions: Choose up to 3 from this list ONLY: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality. Comma-separated. VALUE ONLY.\n",
-    "5. Primary country: Country name only (e.g., \"China\", \"United States\", \"Colombia\"). VALUE ONLY.\n",
-    "\n",
-    "RULES:\n",
-    "- Professions MUST match the exact categories listed (actress = actor)\n",
-    "- \"online personality\" includes streamers, cosplayers, YouTubers, influencers\n",
-    "- \"public figure\" includes politicians, activists, journalists, authors\n",
-    "- Use \"Unknown\" when uncertain or for fictional characters\n",
-    "- NO explanatory text, NO labels like \"Gender:\", NO prefixes\n",
-    "- For multi-role people, list up to 3 categories by relevance\n",
-    "\n",
-    "EXAMPLE FORMAT:\n",
-    "1. Taylor Swift\n",
-    "2. None\n",
-    "3. Female\n",
-    "4. singer/musician, public figure\n",
-    "5. United States\"\"\"\n",
-    "\n",
-    "# Create prompts\n",
-    "print(\"\\nCreating prompts...\")\n",
-    "df['prompt'] = df.apply(create_prompt, axis=1)\n",
-    "print(\"✅ Prompts created\")\n",
-    "\n",
-    "@contextmanager\n",
-    "def timeout(duration):\n",
-    "    def handler(signum, frame):\n",
-    "        raise TimeoutError(\"Operation timed out\")\n",
-    "    \n",
-    "    # Set the signal handler and alarm\n",
-    "    signal.signal(signal.SIGALRM, handler)\n",
-    "    signal.alarm(duration)\n",
-    "    try:\n",
-    "        yield\n",
-    "    finally:\n",
-    "        signal.alarm(0)  # Disable the alarm\n",
-    "\n",
-    "\n",
-    "def query_qwen_local(prompt: str) -> str:\n",
-    "    \"\"\"Query Qwen locally via transformers.\"\"\"\n",
-    "    try:\n",
-    "        # Format as chat message for Qwen with strict system prompt\n",
-    "        messages = [\n",
-    "            {\"role\": \"system\", \"content\": \"You are a data extraction assistant. Respond with exactly 5 numbered lines containing ONLY values. No labels, no explanations, no prefixes. Follow the format precisely.\"},\n",
-    "            {\"role\": \"user\", \"content\": prompt}\n",
-    "        ]\n",
-    "        \n",
-    "        # Tokenize\n",
-    "        if hasattr(tokenizer, 'apply_chat_template'):\n",
-    "            text = tokenizer.apply_chat_template(\n",
-    "                messages,\n",
-    "                tokenize=False,\n",
-    "                add_generation_prompt=True\n",
-    "            )\n",
-    "        else:\n",
-    "            # Fallback for older tokenizers\n",
-    "            text = f\"[INST] {prompt} [/INST]\"\n",
-    "        \n",
-    "        inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
-    "        \n",
-    "        # Generate with timeout\n",
-    "        with timeout(60):\n",
-    "            with torch.no_grad():\n",
-    "                outputs = model.generate(\n",
-    "                    **inputs,\n",
-    "                    max_new_tokens=100,\n",
-    "                    temperature=0.1,\n",
-    "                    do_sample=False,\n",
-    "                    pad_token_id=tokenizer.eos_token_id\n",
-    "                )\n",
-    "        \n",
-    "        # Decode\n",
-    "        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
-    "        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
-    "        \n",
-    "        return response.strip()\n",
-    "        \n",
-    "    except TimeoutError:\n",
-    "        print(f\"[ERROR] Generation timed out after 60 seconds\")\n",
-    "        return None\n",
-    "    except Exception as e:\n",
-    "        print(f\"Generation error: {e}\")\n",
-    "        import traceback\n",
-    "        traceback.print_exc()\n",
-    "        return None\n",
-    "\n",
-    "    \n",
-    "# === PARSE RESPONSE WITH CLEANING ===\n",
-    "def parse_response(response):\n",
-    "    \"\"\"Parse Qwen response into structured fields with cleaning.\"\"\"\n",
-    "    if not response:\n",
-    "        return {\n",
-    "            'full_name': 'Unknown',\n",
-    "            'aliases': 'Unknown',\n",
-    "            'gender': 'Unknown',\n",
-    "            'profession_llm': 'Unknown',\n",
-    "            'country': 'Unknown'\n",
-    "        }\n",
-    "    \n",
-    "    # Split into lines and clean\n",
-    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
-    "    \n",
-    "    # Initialize with Unknown values\n",
-    "    fields = {\n",
-    "        'full_name': 'Unknown',\n",
-    "        'aliases': 'Unknown',\n",
-    "        'gender': 'Unknown',\n",
-    "        'profession_llm': 'Unknown',\n",
-    "        'country': 'Unknown'\n",
-    "    }\n",
-    "    \n",
-    "    # Extract information from each numbered line\n",
-    "    for line in lines:\n",
-    "        if line.startswith('1.'):\n",
-    "            fields['full_name'] = line[2:].strip()\n",
-    "        elif line.startswith('2.'):\n",
-    "            fields['aliases'] = line[2:].strip()\n",
-    "        elif line.startswith('3.'):\n",
-    "            # Clean gender field - remove any labels\n",
-    "            gender_raw = line[2:].strip()\n",
-    "            # Remove common prefixes\n",
-    "            gender_raw = re.sub(r'^(Gender:|gender:)\\s*', '', gender_raw, flags=re.IGNORECASE)\n",
-    "            # Extract just the gender word\n",
-    "            gender_match = re.search(r'\\b(Male|Female|Other|Unknown)\\b', gender_raw, re.IGNORECASE)\n",
-    "            fields['gender'] = gender_match.group(1).capitalize() if gender_match else gender_raw\n",
-    "        elif line.startswith('4.'):\n",
-    "            fields['profession_llm'] = line[2:].strip()\n",
-    "        elif line.startswith('5.'):\n",
-    "            # Clean country field - remove any labels\n",
-    "            country_raw = line[2:].strip()\n",
-    "            # Remove common prefixes like \"Primary country:\", \"Country:\", etc.\n",
-    "            country_raw = re.sub(r'^(Primary\\s+)?(associated\\s+)?country:\\s*', '', country_raw, flags=re.IGNORECASE)\n",
-    "            fields['country'] = country_raw\n",
-    "    \n",
-    "    return fields\n",
-    "\n",
-    "# === PROCESS DATA ===\n",
-    "index_file.parent.mkdir(parents=True, exist_ok=True)\n",
-    "\n",
-    "# Load index\n",
-    "current_index = 0\n",
-    "if index_file.exists():\n",
-    "    try:\n",
-    "        current_index = int(index_file.read_text().strip())\n",
-    "    except:\n",
-    "        current_index = 0\n",
-    "\n",
-    "print(f\"Resuming from index {current_index}\")\n",
-    "\n",
-    "start_time = time.time()\n",
-    "\n",
-    "for i in tqdm(range(current_index, len(df)), desc=\"Qwen Local\"):\n",
-    "\n",
-    "    prompt = df.at[i, \"prompt\"]\n",
-    "\n",
-    "    # -------- MODEL QUERY WITH RETRIES --------\n",
-    "    response = None\n",
-    "    for attempt in range(3):\n",
-    "        response = query_qwen_local(prompt)\n",
-    "        \n",
-    "        # Valid response?\n",
-    "        if response and len(response.strip()) > 10:\n",
-    "            break\n",
-    "        \n",
-    "        print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
-    "        time.sleep(0.5)\n",
-    "\n",
-    "    # If still invalid → DO NOT overwrite previous data\n",
-    "    if not response or len(response.strip()) <= 10:\n",
-    "        print(f\"❌ Row {i}: failed after retries, not writing, not advancing index\")\n",
-    "        continue\n",
-    "\n",
-    "    parsed = parse_response(response)\n",
-    "\n",
-    "    # Additional safety: skip rows that parsed as all 'Unknown'\n",
-    "    if all(v == \"Unknown\" for v in parsed.values()):\n",
-    "        print(f\"❌ Row {i}: parsed as all Unknown (likely model crash); skipping.\")\n",
-    "        continue\n",
-    "\n",
-    "    # -------- WRITE PARSED FIELDS SAFELY --------\n",
-    "    for key, value in parsed.items():\n",
-    "        df.at[i, key] = value\n",
-    "\n",
-    "    # Advance progress ONLY after successful write\n",
-    "    current_index = i + 1\n",
-    "\n",
-    "    # -------- GPU MEMORY CLEANUP --------\n",
-    "    if torch.cuda.is_available():\n",
-    "        torch.cuda.empty_cache()\n",
-    "        torch.cuda.synchronize()\n",
-    "\n",
-    "    # -------- SAVE LIKE YOUR DEEPSEEK VERSION --------\n",
-    "    if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
-    "        df.to_csv(output_file, index=False)\n",
-    "        with open(index_file, \"w\") as f:\n",
-    "            f.write(str(current_index))\n",
-    "        print(f\"💾 Progress saved after row {i+1}\")\n",
-    "\n",
-    "# Final save\n",
-    "df.to_csv(output_file, index=False)\n",
-    "index_file.write_text(str(current_index))\n",
-    "print(\"✅ Finished full dataset.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d9c7deb9-847a-472d-8055-f93dbfa6aa2e",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "pm-paper",
-   "language": "python",
-   "name": "pm-paper"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.13"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

jupyter_notebooks/Section_2-3-4_Bloomz_query.ipynb

@@ -1,350 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "id": "3b87c378-241e-41ab-be6e-84222594f22f",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-11-28T13:07:47.025752Z",
-     "iopub.status.busy": "2025-11-28T13:07:47.025552Z",
-     "iopub.status.idle": "2025-11-28T13:07:47.128799Z",
-     "shell.execute_reply": "2025-11-28T13:07:47.128306Z",
-     "shell.execute_reply.started": "2025-11-28T13:07:47.025736Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import json\n",
-    "import time\n",
-    "import re\n",
-    "from pathlib import Path\n",
-    "from tqdm import tqdm\n",
-    "import torch\n",
-    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
-    "from datetime import datetime\n",
-    "\n",
-    "# Import is used for pd.notna() and pd.isna() checks\n",
-    "\n",
-    "current_dir = Path.cwd()\n",
-    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
-    "\n",
-    "# === CONFIGURATION ===\n",
-    "TEST_MODE = True\n",
-    "TEST_SIZE = 10\n",
-    "MAX_ROWS = 20000\n",
-    "SAVE_INTERVAL = 20\n",
-    "\n",
-    "# Model settings - BLOOMZ (BigScience - European consortium)\n",
-    "MODEL_NAME = \"bigscience/bloomz-7b1\"  # Largest instruction-tuned BLOOM model\n",
-    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
-    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
-    "\n",
-    "PROFESSION_CATEGORIES = [\n",
-    "    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n",
-    "    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n",
-    "    \"sports professional\", \"tv personality\"\n",
-    "]\n",
-    "\n",
-    "# === LOAD MODEL ===\n",
-    "print(f\"Loading model: {MODEL_NAME}\")\n",
-    "print(f\"Cache directory: {CACHE_DIR}\")\n",
-    "print(f\"This may take a while on first run (~14GB download)...\\n\")\n",
-    "\n",
-    "# Check GPU availability\n",
-    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
-    "print(f\"Device: {device}\")\n",
-    "\n",
-    "if device == \"cpu\":\n",
-    "    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
-    "    print(\"   Consider using a GPU or reducing model size.\")\n",
-    "\n",
-    "# Load tokenizer\n",
-    "print(\"Loading tokenizer...\")\n",
-    "try:\n",
-    "    tokenizer = AutoTokenizer.from_pretrained(\n",
-    "        MODEL_NAME,\n",
-    "        cache_dir=str(CACHE_DIR)\n",
-    "    )\n",
-    "    print(\"✅ Tokenizer loaded\")\n",
-    "except Exception as e:\n",
-    "    print(f\"❌ Error loading tokenizer: {e}\")\n",
-    "    raise\n",
-    "\n",
-    "# Ensure pad token is set\n",
-    "if tokenizer.pad_token is None:\n",
-    "    tokenizer.pad_token = tokenizer.eos_token\n",
-    "    print(f\"Set pad_token to eos_token: {tokenizer.eos_token}\")\n",
-    "\n",
-    "# Load model with optimizations\n",
-    "print(\"Loading model (this may take several minutes)...\")\n",
-    "try:\n",
-    "    model = AutoModelForCausalLM.from_pretrained(\n",
-    "        MODEL_NAME,\n",
-    "        cache_dir=str(CACHE_DIR),\n",
-    "        torch_dtype=torch.bfloat16,  # Use BF16 for efficiency\n",
-    "        device_map=\"auto\",  # Automatically distribute across GPUs\n",
-    "        low_cpu_mem_usage=True  # Optimize memory usage\n",
-    "    )\n",
-    "    model.eval()  # Set to evaluation mode\n",
-    "    print(\"✅ Model loaded\")\n",
-    "except Exception as e:\n",
-    "    print(f\"❌ Error loading model: {e}\")\n",
-    "    raise\n",
-    "\n",
-    "# Check VRAM usage\n",
-    "if torch.cuda.is_available():\n",
-    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
-    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
-    "\n",
-    "# === LOAD DATA ===\n",
-    "df = pd.read_csv(input_file)\n",
-    "print(f\"Loaded {len(df)} rows\")\n",
-    "\n",
-    "if TEST_MODE:\n",
-    "    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n",
-    "    df = df.head(TEST_SIZE).copy()\n",
-    "elif MAX_ROWS:\n",
-    "    df = df.head(MAX_ROWS).copy()\n",
-    "\n",
-    "# === CREATE PROMPT (Exact DeepSeek style) ===\n",
-    "# === CREATE PROMPT (BLOOMZ-optimized - Complete format) ===\n",
-    "def create_prompt(row):\n",
-    "    \"\"\"Create prompt optimized for BLOOMZ with complete examples.\"\"\"\n",
-    "    name = row.get('real_name', row.get('name', ''))\n",
-    "    if pd.isna(name):\n",
-    "        name = row.get('name', '')\n",
-    "\n",
-    "    # Gather hints\n",
-    "    hints = []\n",
-    "    if pd.notna(row.get('likely_profession')):\n",
-    "        hints.append(str(row['likely_profession']))\n",
-    "    if pd.notna(row.get('likely_nationality')):\n",
-    "        hints.append(str(row['likely_nationality']))\n",
-    "    if pd.notna(row.get('likely_country')):\n",
-    "        hints.append(str(row['likely_country']))\n",
-    "\n",
-    "    # Add tags if we don't have enough hints\n",
-    "    if len(hints) < 3:\n",
-    "        for i in range(1, 8):\n",
-    "            tag_col = f'tag_{i}'\n",
-    "            if tag_col in row and pd.notna(row[tag_col]):\n",
-    "                tag_val = str(row[tag_col])\n",
-    "                if tag_val not in hints:\n",
-    "                    hints.append(tag_val)\n",
-    "                if len(hints) >= 5:\n",
-    "                    break\n",
-    "\n",
-    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
-    "\n",
-    "    # BLOOMZ needs complete format shown - don't end with \"1.\"\n",
-    "    return f\"\"\"Task: Extract person information in 5 numbered lines.\n",
-    "\n",
-    "Format:\n",
-    "1. Full legal name\n",
-    "2. Stage names/aliases\n",
-    "3. Gender\n",
-    "4. Professions (from: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality)\n",
-    "5. Country\n",
-    "\n",
-    "Example:\n",
-    "Person: Taylor Swift (singer, American, pop music)\n",
-    "Answer:\n",
-    "1. Taylor Alison Swift\n",
-    "2. Taylor Swift\n",
-    "3. Female\n",
-    "4. singer/musician, online personality\n",
-    "5. United States\n",
-    "\n",
-    "Person: {name} ({hint_text})\n",
-    "Answer:\n",
-    "\"\"\"\n",
-    "\n",
-    "# === QUERY BLOOMZ LOCAL (Fixed generation parameters) ===\n",
-    "def query_bloomz_local(prompt: str) -> str:\n",
-    "    \"\"\"Query BLOOMZ-7B1 locally via transformers, return raw response string.\"\"\"\n",
-    "    try:\n",
-    "        # Store full prompt for logging\n",
-    "        query_bloomz_local.last_full_prompt = prompt\n",
-    "\n",
-    "        # Tokenize WITHOUT truncation\n",
-    "        inputs = tokenizer(\n",
-    "            prompt,\n",
-    "            return_tensors=\"pt\",\n",
-    "            truncation=False\n",
-    "        ).to(device)\n",
-    "        \n",
-    "        # Store the input length for proper extraction later\n",
-    "        input_length = inputs['input_ids'].shape[1]\n",
-    "\n",
-    "        # Generate with parameters optimized for BLOOMZ completion\n",
-    "        with torch.no_grad():\n",
-    "            outputs = model.generate(\n",
-    "                **inputs,\n",
-    "                max_new_tokens=300,  # Increased to allow full completion\n",
-    "                min_new_tokens=50,   # Force minimum generation length\n",
-    "                temperature=0.5,     # Balanced creativity\n",
-    "                do_sample=True,\n",
-    "                top_p=0.9,\n",
-    "                top_k=50,\n",
-    "                repetition_penalty=1.15,\n",
-    "                pad_token_id=tokenizer.eos_token_id,\n",
-    "                eos_token_id=tokenizer.eos_token_id,\n",
-    "                early_stopping=False,  # Changed to False - don't stop early!\n",
-    "                num_beams=1  # Greedy-like but with sampling\n",
-    "            )\n",
-    "\n",
-    "        # Extract only the NEW generated tokens\n",
-    "        generated_ids = outputs[0][input_length:]\n",
-    "        generated_text = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
-    "\n",
-    "        # Debug output (show first 5 now to see more patterns)\n",
-    "        if not hasattr(query_bloomz_local, 'debug_count'):\n",
-    "            query_bloomz_local.debug_count = 0\n",
-    "\n",
-    "        if query_bloomz_local.debug_count < 5:\n",
-    "            print(f\"\\n📝 BLOOMZ Debug #{query_bloomz_local.debug_count + 1}:\")\n",
-    "            print(f\"Prompt tokens: {input_length}\")\n",
-    "            print(f\"Generated tokens: {len(generated_ids)}\")\n",
-    "            print(f\"Full generation:\\n{generated_text}\")\n",
-    "            print(f\"{'='*60}\\n\")\n",
-    "            query_bloomz_local.debug_count += 1\n",
-    "\n",
-    "        return generated_text.strip()\n",
-    "\n",
-    "    except Exception as e:\n",
-    "        print(f\"Error querying BLOOMZ: {e}\")\n",
-    "        import traceback\n",
-    "        traceback.print_exc()\n",
-    "        query_bloomz_local.last_full_prompt = f\"ERROR: {e}\"\n",
-    "        return None\n",
-    "\n",
-    "        \n",
-    "# === PROCESS ===\n",
-    "output_file = current_dir.parent / f\"data/CSV/bloomz_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
-    "index_file = current_dir.parent / \"misc/bloomz_query_index.txt\"\n",
-    "\n",
-    "current_index = 0\n",
-    "if index_file.exists():\n",
-    "    with open(index_file) as f:\n",
-    "        current_index = int(f.read().strip())\n",
-    "    print(f\"Resuming from index {current_index}\")\n",
-    "\n",
-    "# Initialize columns (same as DeepSeek)\n",
-    "for col in ['full_name', 'gender', 'profession_llm', 'country', 'aliases']:\n",
-    "    if col not in df.columns:\n",
-    "        df[col] = 'Unknown'\n",
-    "\n",
-    "# Create prompts for all rows (same as DeepSeek)\n",
-    "print(\"Creating prompts...\")\n",
-    "df['prompt'] = df.apply(create_prompt, axis=1)\n",
-    "\n",
-    "print(f\"\\nAnnotating with BLOOMZ-7B1 LOCAL - rows {current_index} to {len(df)}...\")\n",
-    "print(f\"Model: {MODEL_NAME}\")\n",
-    "print(f\"This may take a while...\\n\")\n",
-    "\n",
-    "try:\n",
-    "    start_time = time.time()\n",
-    "\n",
-    "    for i in tqdm(range(current_index, len(df)), desc=\"Annotating\"):\n",
-    "        row = df.iloc[i]\n",
-    "\n",
-    "        # Query BLOOMZ (equivalent to DeepSeek query)\n",
-    "        response = query_bloomz_local(row['prompt'])\n",
-    "        parsed_data = parse_response(response)\n",
-    "\n",
-    "        # Log the complete interaction for debugging\n",
-    "        log_response(\n",
-    "            idx=i,\n",
-    "            name=row.get('real_name', row.get('name', 'Unknown')),\n",
-    "            prompt=row['prompt'],\n",
-    "            full_prompt=query_bloomz_local.last_full_prompt if hasattr(query_bloomz_local, 'last_full_prompt') else 'N/A',\n",
-    "            raw_response=response if response else 'None',\n",
-    "            parsed_data=parsed_data\n",
-    "        )\n",
-    "\n",
-    "        # Update dataframe\n",
-    "        for key, value in parsed_data.items():\n",
-    "            df.at[i, key] = value\n",
-    "\n",
-    "        current_index = i + 1\n",
-    "\n",
-    "        # Save progress at intervals\n",
-    "        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
-    "            df.to_csv(output_file, index=False)\n",
-    "            with open(index_file, 'w') as f:\n",
-    "                f.write(str(current_index))\n",
-    "            print(f\"✅ Progress saved after {i+1} rows\")\n",
-    "\n",
-    "        # Optional: Add small delay to prevent overheating (not needed for rate limiting like DeepSeek)\n",
-    "        # time.sleep(0.1)\n",
-    "\n",
-    "    elapsed_total = time.time() - start_time\n",
-    "    print(f\"\\n✅ Done! Final results saved to {output_file}\")\n",
-    "\n",
-    "    # Summary statistics (same as DeepSeek)\n",
-    "    print(\"\\n=== Summary Statistics ===\")\n",
-    "    print(f\"Total processed: {len(df)}\")\n",
-    "    print(f\"\\nGender distribution:\")\n",
-    "    print(df['gender'].value_counts())\n",
-    "    print(f\"\\nTop 10 profession combinations:\")\n",
-    "    print(df['profession_llm'].value_counts().head(10))\n",
-    "    print(f\"\\nTop 10 countries:\")\n",
-    "    print(df['country'].value_counts().head(10))\n",
-    "\n",
-    "    # Sample results\n",
-    "    print(\"\\n=== Sample Results ===\")\n",
-    "    display_cols = ['real_name', 'full_name', 'gender', 'profession_llm', 'country']\n",
-    "    available_cols = [col for col in display_cols if col in df.columns]\n",
-    "    print(df[available_cols].head(10).to_string(index=False))\n",
-    "\n",
-    "    # Additional info for local model\n",
-    "    print(f\"\\nTotal time: {elapsed_total/60:.1f} minutes\")\n",
-    "    print(f\"Average speed: {len(df)/(elapsed_total/3600):.1f} samples/hour\")\n",
-    "    if torch.cuda.is_available():\n",
-    "        print(f\"Final VRAM usage: {torch.cuda.max_memory_allocated() / 1024**3:.2f} GB\")\n",
-    "\n",
-    "except Exception as e:\n",
-    "    print(f\"⚠️ Error encountered: {e}\")\n",
-    "    print(f\"⚠️ Last processed index: {current_index}\")\n",
-    "\n",
-    "    # Save progress before exiting\n",
-    "    df.to_csv(output_file, index=False)\n",
-    "    with open(index_file, 'w') as f:\n",
-    "        f.write(str(current_index))\n",
-    "\n",
-    "    print(f\"⚠️ Progress saved up to row {current_index}\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c458d50f-0cb3-421e-99ab-62795f88242c",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "pm-paper",
-   "language": "python",
-   "name": "pm-paper"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.13"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

jupyter_notebooks/Section_2-3-4_Figure_8_Step_1_LLM_annotation.ipynb

@@ -0,0 +1,509 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "23d0ae58",
+   "metadata": {},
+   "source": [
+    "# Deepfake Adapter Dataset - LLM Annotation Pipeline"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Unified Model Loading & Inference\nAbstracted interface for loading and querying Mistral, Gemma, and Qwen models."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "import torch\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
+    "import signal\n",
+    "from contextlib import contextmanager\n",
+    "\n",
+    "# Configuration\n",
+    "current_dir = Path.cwd()\n",
+    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
+    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Model configurations\n",
+    "MODEL_CONFIGS = {\n",
+    "    'mistral': {\n",
+    "        'name': 'mistralai/Mistral-7B-Instruct-v0.3',\n",
+    "        'dtype': torch.bfloat16,\n",
+    "        'quantization': None,\n",
+    "        'generation_params': {\n",
+    "            'max_new_tokens': 512,\n",
+    "            'temperature': 0.05,\n",
+    "            'do_sample': True,\n",
+    "            'top_p': 0.8,\n",
+    "        }\n",
+    "    },\n",
+    "    'gemma': {\n",
+    "        'name': 'google/gemma-3-27b-it',\n",
+    "        'dtype': torch.bfloat16,\n",
+    "        'quantization': None,\n",
+    "        'generation_params': {\n",
+    "            'max_new_tokens': 512,\n",
+    "            'temperature': 0.1,\n",
+    "            'do_sample': True,\n",
+    "            'top_p': 0.9,\n",
+    "        }\n",
+    "    },\n",
+    "    'qwen': {\n",
+    "        'name': 'Qwen/Qwen2.5-32B-Instruct',\n",
+    "        'dtype': None,  # Will use quantization\n",
+    "        'quantization': BitsAndBytesConfig(\n",
+    "            load_in_8bit=True,\n",
+    "            llm_int8_threshold=6.0,\n",
+    "            llm_int8_has_fp16_weight=False\n",
+    "        ),\n",
+    "        'generation_params': {\n",
+    "            'max_new_tokens': 100,\n",
+    "            'temperature': 0.1,\n",
+    "            'do_sample': False,\n",
+    "        }\n",
+    "    }\n",
+    "}\n",
+    "\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\",\n",
+    "    \"adult performer\",\n",
+    "    \"singer/musician\",\n",
+    "    \"model\",\n",
+    "    \"online personality\",\n",
+    "    \"public figure\",\n",
+    "    \"voice actor/ASMR\",\n",
+    "    \"sports professional\",\n",
+    "    \"tv personality\"\n",
+    "]\n"
+   ],
+   "execution_count": null,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "source": [
+    "def load_model(model_type='mistral'):\n",
+    "    \"\"\"\n",
+    "    Load model and tokenizer based on type.\n",
+    "    \n",
+    "    Args:\n",
+    "        model_type: 'mistral', 'gemma', or 'qwen'\n",
+    "    \n",
+    "    Returns:\n",
+    "        tuple: (model, tokenizer, config)\n",
+    "    \"\"\"\n",
+    "    if model_type not in MODEL_CONFIGS:\n",
+    "        raise ValueError(f\"Unknown model type: {model_type}. Choose from {list(MODEL_CONFIGS.keys())}\")\n",
+    "    \n",
+    "    config = MODEL_CONFIGS[model_type]\n",
+    "    model_name = config['name']\n",
+    "    \n",
+    "    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "    print(f\"Loading model: {model_name}\")\n",
+    "    print(f\"Cache directory: {CACHE_DIR}\")\n",
+    "    print(f\"Device: {device}\\n\")\n",
+    "    \n",
+    "    if device == \"cpu\":\n",
+    "        print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
+    "    \n",
+    "    # Load tokenizer\n",
+    "    try:\n",
+    "        tokenizer = AutoTokenizer.from_pretrained(\n",
+    "            model_name,\n",
+    "            cache_dir=str(CACHE_DIR),\n",
+    "            use_fast=True\n",
+    "        )\n",
+    "    except:\n",
+    "        tokenizer = AutoTokenizer.from_pretrained(\n",
+    "            model_name,\n",
+    "            cache_dir=str(CACHE_DIR),\n",
+    "            use_fast=False\n",
+    "        )\n",
+    "    \n",
+    "    if tokenizer.pad_token is None:\n",
+    "        tokenizer.pad_token = tokenizer.eos_token\n",
+    "    \n",
+    "    # Load model\n",
+    "    model_kwargs = {\n",
+    "        'cache_dir': str(CACHE_DIR),\n",
+    "        'device_map': 'auto',\n",
+    "        'trust_remote_code': False\n",
+    "    }\n",
+    "    \n",
+    "    if config['quantization']:\n",
+    "        model_kwargs['quantization_config'] = config['quantization']\n",
+    "    else:\n",
+    "        model_kwargs['torch_dtype'] = config['dtype']\n",
+    "    \n",
+    "    model = AutoModelForCausalLM.from_pretrained(model_name, **model_kwargs)\n",
+    "    model.eval()\n",
+    "    \n",
+    "    # Check VRAM\n",
+    "    if torch.cuda.is_available():\n",
+    "        vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
+    "        print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
+    "    \n",
+    "    return model, tokenizer, config\n"
+   ],
+   "execution_count": null,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "source": [
+    "@contextmanager\n",
+    "def timeout(duration):\n",
+    "    \"\"\"Context manager for timeout.\"\"\"\n",
+    "    def handler(signum, frame):\n",
+    "        raise TimeoutError(\"Operation timed out\")\n",
+    "    \n",
+    "    signal.signal(signal.SIGALRM, handler)\n",
+    "    signal.alarm(duration)\n",
+    "    try:\n",
+    "        yield\n",
+    "    finally:\n",
+    "        signal.alarm(0)\n",
+    "\n",
+    "def query_model(prompt, model, tokenizer, config, use_timeout=False):\n",
+    "    \"\"\"\n",
+    "    Query model with given prompt.\n",
+    "    \n",
+    "    Args:\n",
+    "        prompt: Input prompt string\n",
+    "        model: Loaded model\n",
+    "        tokenizer: Loaded tokenizer\n",
+    "        config: Model configuration dict\n",
+    "        use_timeout: Whether to use 60s timeout (for Qwen)\n",
+    "    \n",
+    "    Returns:\n",
+    "        str: Model response or None on error\n",
+    "    \"\"\"\n",
+    "    try:\n",
+    "        device = next(model.parameters()).device\n",
+    "        \n",
+    "        # Format as chat message\n",
+    "        messages = [\n",
+    "            {\"role\": \"system\", \"content\": \"You are a data extraction assistant. Respond with exactly 5 numbered lines containing ONLY values. No labels, no explanations, no prefixes. Follow the format precisely.\"},\n",
+    "            {\"role\": \"user\", \"content\": prompt}\n",
+    "        ]\n",
+    "        \n",
+    "        # Tokenize\n",
+    "        if hasattr(tokenizer, 'apply_chat_template'):\n",
+    "            text = tokenizer.apply_chat_template(\n",
+    "                messages,\n",
+    "                tokenize=False,\n",
+    "                add_generation_prompt=True\n",
+    "            )\n",
+    "        else:\n",
+    "            text = f\"[INST] {prompt} [/INST]\"\n",
+    "        \n",
+    "        inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
+    "        \n",
+    "        # Generation parameters\n",
+    "        gen_kwargs = config['generation_params'].copy()\n",
+    "        gen_kwargs['pad_token_id'] = tokenizer.eos_token_id\n",
+    "        \n",
+    "        # Generate\n",
+    "        generation_fn = lambda: model.generate(**inputs, **gen_kwargs)\n",
+    "        \n",
+    "        if use_timeout:\n",
+    "            with timeout(60):\n",
+    "                with torch.no_grad():\n",
+    "                    outputs = generation_fn()\n",
+    "        else:\n",
+    "            with torch.no_grad():\n",
+    "                outputs = generation_fn()\n",
+    "        \n",
+    "        # Decode\n",
+    "        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
+    "        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
+    "        \n",
+    "        return response.strip()\n",
+    "        \n",
+    "    except TimeoutError:\n",
+    "        print(f\"[ERROR] Generation timed out after 60 seconds\")\n",
+    "        return None\n",
+    "    except Exception as e:\n",
+    "        print(f\"[ERROR] Generation failed: {e}\")\n",
+    "        return None\n"
+   ],
+   "execution_count": null,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "source": [
+    "def create_prompt(row):\n",
+    "    \"\"\"Create annotation prompt from row data.\"\"\"\n",
+    "    name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
+    "    \n",
+    "    # Gather hints\n",
+    "    hints = []\n",
+    "    if pd.notna(row.get('likely_profession')):\n",
+    "        hints.append(str(row['likely_profession']))\n",
+    "    if pd.notna(row.get('likely_nationality')):\n",
+    "        hints.append(str(row['likely_nationality']))\n",
+    "    if pd.notna(row.get('likely_country')):\n",
+    "        hints.append(str(row['likely_country']))\n",
+    "    \n",
+    "    # Add tags if needed\n",
+    "    if len(hints) < 3:\n",
+    "        for i in range(1, 8):\n",
+    "            tag_col = f'tag_{i}'\n",
+    "            if tag_col in row and pd.notna(row[tag_col]):\n",
+    "                tag_val = str(row[tag_col])\n",
+    "                if tag_val not in hints:\n",
+    "                    hints.append(tag_val)\n",
+    "                if len(hints) >= 5:\n",
+    "                    break\n",
+    "    \n",
+    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
+    "    \n",
+    "    return f\"\"\"Extract information about '{name}' ({hint_text}).\n",
+    "\n",
+    "Respond with EXACTLY 5 numbered lines. Each line must contain ONLY the value, no labels or extra text.\n",
+    "\n",
+    "FORMAT REQUIREMENTS:\n",
+    "1. Full legal name in Western order (first last). VALUE ONLY.\n",
+    "2. Stage names/aliases, comma-separated. If none, write \"None\". VALUE ONLY.\n",
+    "3. Gender: MUST be exactly one word: Male, Female, Other, or Unknown. VALUE ONLY.\n",
+    "4. Professions: Choose up to 3 from this list ONLY: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality. Comma-separated. VALUE ONLY.\n",
+    "5. Primary country: Country name only (e.g., \"China\", \"United States\", \"Colombia\"). VALUE ONLY.\n",
+    "\n",
+    "RULES:\n",
+    "- Professions MUST match the exact categories listed (actress = actor)\n",
+    "- \"online personality\" includes streamers, cosplayers, YouTubers, influencers\n",
+    "- \"public figure\" includes politicians, activists, journalists, authors\n",
+    "- Use \"Unknown\" when uncertain or for fictional characters\n",
+    "- NO explanatory text, NO labels like \"Gender:\", NO prefixes\n",
+    "- For multi-role people, list up to 3 categories by relevance\n",
+    "\n",
+    "EXAMPLE FORMAT:\n",
+    "1. Taylor Swift\n",
+    "2. None\n",
+    "3. Female\n",
+    "4. singer/musician, public figure\n",
+    "5. United States\"\"\"\n"
+   ],
+   "execution_count": null,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "source": [
+    "def parse_response(response):\n",
+    "    \"\"\"Parse model response into structured fields.\"\"\"\n",
+    "    if not response:\n",
+    "        return {\n",
+    "            'full_name': 'Unknown',\n",
+    "            'aliases': 'Unknown',\n",
+    "            'gender': 'Unknown',\n",
+    "            'profession_llm': 'Unknown',\n",
+    "            'country': 'Unknown'\n",
+    "        }\n",
+    "    \n",
+    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
+    "    \n",
+    "    fields = {\n",
+    "        'full_name': 'Unknown',\n",
+    "        'aliases': 'Unknown',\n",
+    "        'gender': 'Unknown',\n",
+    "        'profession_llm': 'Unknown',\n",
+    "        'country': 'Unknown'\n",
+    "    }\n",
+    "    \n",
+    "    for line in lines:\n",
+    "        if line.startswith('1.'):\n",
+    "            fields['full_name'] = line[2:].strip()\n",
+    "        elif line.startswith('2.'):\n",
+    "            fields['aliases'] = line[2:].strip()\n",
+    "        elif line.startswith('3.'):\n",
+    "            gender_raw = line[2:].strip()\n",
+    "            gender_raw = re.sub(r'^(Gender:|gender:)\\s*', '', gender_raw, flags=re.IGNORECASE)\n",
+    "            gender_match = re.search(r'\\b(Male|Female|Other|Unknown)\\b', gender_raw, re.IGNORECASE)\n",
+    "            fields['gender'] = gender_match.group(1).capitalize() if gender_match else gender_raw\n",
+    "        elif line.startswith('4.'):\n",
+    "            fields['profession_llm'] = line[2:].strip()\n",
+    "        elif line.startswith('5.'):\n",
+    "            country_raw = line[2:].strip()\n",
+    "            country_raw = re.sub(r'^(Primary\\s+)?(associated\\s+)?country:\\s*', '', country_raw, flags=re.IGNORECASE)\n",
+    "            fields['country'] = country_raw\n",
+    "    \n",
+    "    return fields\n"
+   ],
+   "execution_count": null,
+   "outputs": []
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "source": [
+    "def annotate_dataset(model_type='mistral', test_mode=False, test_size=100, max_rows=50862, save_interval=10):\n",
+    "    \"\"\"\n",
+    "    Annotate dataset using specified model.\n",
+    "    \n",
+    "    Args:\n",
+    "        model_type: 'mistral', 'gemma', or 'qwen'\n",
+    "        test_mode: If True, only process test_size rows\n",
+    "        test_size: Number of rows to process in test mode\n",
+    "        max_rows: Maximum rows to process\n",
+    "        save_interval: Save progress every N rows\n",
+    "    \"\"\"\n",
+    "    # Setup paths\n",
+    "    input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "    output_file = current_dir.parent / f\"data/CSV/{model_type}_local_annotated_POI{'_test' if test_mode else ''}.csv\"\n",
+    "    index_file = current_dir.parent / f\"misc/query_indicies/{model_type}_local_query_index.txt\"\n",
+    "    index_file.parent.mkdir(parents=True, exist_ok=True)\n",
+    "    \n",
+    "    # Load model\n",
+    "    model, tokenizer, config = load_model(model_type)\n",
+    "    \n",
+    "    # Load data\n",
+    "    print(f\"Loaded {len(df)} rows from input file\")\n",
+    "    df = pd.read_csv(input_file)\n",
+    "    \n",
+    "    # Merge existing annotations if available\n",
+    "    if output_file.exists():\n",
+    "        existing_df = pd.read_csv(output_file)\n",
+    "        annotation_cols = ['full_name', 'aliases', 'gender', 'profession_llm', 'country']\n",
+    "        for col in annotation_cols:\n",
+    "            if col in existing_df.columns:\n",
+    "                df[col] = existing_df[col][:len(df)]\n",
+    "    \n",
+    "    # Apply limits\n",
+    "    if test_mode:\n",
+    "        df = df.head(test_size).copy()\n",
+    "    elif max_rows:\n",
+    "        df = df.head(max_rows).copy()\n",
+    "    \n",
+    "    # Create prompts\n",
+    "    df['prompt'] = df.apply(create_prompt, axis=1)\n",
+    "    \n",
+    "    # Load progress index\n",
+    "    current_index = 0\n",
+    "    if index_file.exists():\n",
+    "        try:\n",
+    "            current_index = int(index_file.read_text().strip())\n",
+    "        except:\n",
+    "            current_index = 0\n",
+    "    \n",
+    "    print(f\"Resuming from index {current_index}\")\n",
+    "    \n",
+    "    # Process rows\n",
+    "    use_timeout = (model_type == 'qwen')\n",
+    "    \n",
+    "    for i in tqdm(range(current_index, len(df)), desc=f\"{model_type.capitalize()} Annotation\"):\n",
+    "        prompt = df.at[i, \"prompt\"]\n",
+    "        \n",
+    "        # Query with retries\n",
+    "        response = None\n",
+    "        for attempt in range(3):\n",
+    "            response = query_model(prompt, model, tokenizer, config, use_timeout)\n",
+    "            \n",
+    "            if response and len(response.strip()) > 10:\n",
+    "                break\n",
+    "            \n",
+    "            print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
+    "            time.sleep(0.5)\n",
+    "        \n",
+    "        # Skip if invalid\n",
+    "        if not response or len(response.strip()) <= 10:\n",
+    "            print(f\"❌ Row {i}: failed after retries, skipping\")\n",
+    "            continue\n",
+    "        \n",
+    "        # Parse and validate\n",
+    "        parsed = parse_response(response)\n",
+    "        \n",
+    "        if all(v == \"Unknown\" for v in parsed.values()):\n",
+    "            print(f\"❌ Row {i}: parsed as all Unknown, skipping\")\n",
+    "            continue\n",
+    "        \n",
+    "        # Write fields\n",
+    "        for key, value in parsed.items():\n",
+    "            df.at[i, key] = value\n",
+    "        \n",
+    "        current_index = i + 1\n",
+    "        \n",
+    "        # GPU cleanup\n",
+    "        if torch.cuda.is_available():\n",
+    "            torch.cuda.empty_cache()\n",
+    "            torch.cuda.synchronize()\n",
+    "        \n",
+    "        # Save progress\n",
+    "        if (i + 1) % save_interval == 0 or (i + 1) == len(df):\n",
+    "            df.to_csv(output_file, index=False)\n",
+    "            index_file.write_text(str(current_index))\n",
+    "            print(f\"💾 Progress saved after row {i+1}\")\n",
+    "    \n",
+    "    # Final save\n",
+    "    df.to_csv(output_file, index=False)\n",
+    "    index_file.write_text(str(current_index))\n",
+    "    print(f\"✓ Finished annotation with {model_type}\")\n"
+   ],
+   "execution_count": null,
+   "outputs": []
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Usage Examples\nRun annotation with your chosen model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "source": [
+    "# Example 1: Annotate with Mistral (13.5 GB VRAM)\n",
+    "# annotate_dataset(model_type='mistral', test_mode=False)\n",
+    "\n",
+    "# Example 2: Annotate with Gemma (56.3 GB VRAM)\n",
+    "# annotate_dataset(model_type='gemma', test_mode=False)\n",
+    "\n",
+    "# Example 3: Annotate with Qwen (32.7 GB VRAM, 8-bit)\n",
+    "# annotate_dataset(model_type='qwen', test_mode=False)\n",
+    "\n",
+    "# Test mode (first 100 rows)\n",
+    "# annotate_dataset(model_type='mistral', test_mode=True, test_size=100)\n"
+   ],
+   "execution_count": null,
+   "outputs": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "latm",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

jupyter_notebooks/Section_2-3-4__Figure_8_Deepfake_victims.ipynb

@@ -1,724 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "06763fde",
-   "metadata": {},
-   "source": [
-    "# LLM annotation of Deepfake adapters"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b773045a",
-   "metadata": {},
-   "source": [
-    "## Step 01 Data cleaning NER using spaCy"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "234a55e5",
-   "metadata": {},
-   "source": [
-    "#### Here we clean leetspeak and architectre specifiers from the names as a preprossesing step for the named entity recognition NER below"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "id": "f177df11",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-11-21T10:32:27.999485Z",
-     "iopub.status.busy": "2025-11-21T10:32:27.999298Z",
-     "iopub.status.idle": "2025-11-21T10:32:28.001776Z",
-     "shell.execute_reply": "2025-11-21T10:32:28.001287Z",
-     "shell.execute_reply.started": "2025-11-21T10:32:27.999469Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import spacy\n",
-    "import re\n",
-    "import torch\n",
-    "from pathlib import Path\n",
-    "import unicodedata\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "id": "a2383f32",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-11-21T10:32:28.388500Z",
-     "iopub.status.busy": "2025-11-21T10:32:28.388376Z",
-     "iopub.status.idle": "2025-11-21T10:32:30.176781Z",
-     "shell.execute_reply": "2025-11-21T10:32:30.176262Z",
-     "shell.execute_reply.started": "2025-11-21T10:32:28.388488Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "current_dir = Path.cwd()\n",
-    "poi_models_dir = current_dir.parent / \"data/CSV/model_adapter/real_person_adapters.csv\"  ### POI models dataset\n",
-    "output = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_01_NER.csv\"  ### Output file"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 15,
-   "id": "66fc691f",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-11-21T10:23:43.615765Z",
-     "iopub.status.busy": "2025-11-21T10:23:43.615607Z",
-     "iopub.status.idle": "2025-11-21T10:23:47.279107Z",
-     "shell.execute_reply": "2025-11-21T10:23:47.278503Z",
-     "shell.execute_reply.started": "2025-11-21T10:23:43.615751Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "nlp = spacy.load(\"en_core_web_sm\")  # or another model of your choice"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "id": "8348c4a4",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-11-21T10:23:47.279657Z",
-     "iopub.status.busy": "2025-11-21T10:23:47.279527Z",
-     "iopub.status.idle": "2025-11-21T10:24:34.043631Z",
-     "shell.execute_reply": "2025-11-21T10:24:34.042977Z",
-     "shell.execute_reply.started": "2025-11-21T10:23:47.279644Z"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Done! Saved to NER_poi_step_01.csv\n"
-     ]
-    }
-   ],
-   "source": [
-    "def preprocess_name(name):\n",
-    "    name = str(name)\n",
-    "\n",
-    "    # Normalize unicode characters (e.g., fancy fonts)\n",
-    "    name = unicodedata.normalize(\"NFKD\", name)\n",
-    "\n",
-    "    # Lowercase everything\n",
-    "    name = name.lower()\n",
-    "\n",
-    "    # Remove special keywords and patterns\n",
-    "    junk_words = [\n",
-    "        'jav', 'jp', 'lora', 'locon', 'lycoris', 'requested', 'japanese', 'model',\n",
-    "        'flux', 'flux1.d', 'pony', 'realistic'\n",
-    "    ]\n",
-    "    for word in junk_words:\n",
-    "        name = re.sub(rf'\\b{re.escape(word)}\\b', '', name, flags=re.IGNORECASE)\n",
-    "\n",
-    "    # Remove versions like v1, v2.0, etc.\n",
-    "    name = re.sub(r'v\\.?\\d+(\\.\\d+)?', '', name)\n",
-    "\n",
-    "    # Remove 'not' followed by a word\n",
-    "    name = re.sub(r'\\bnot\\s+\\w+', '', name)\n",
-    "\n",
-    "    # Replace underscores and pipes with spaces\n",
-    "    name = re.sub(r'[_|]', ' ', name)\n",
-    "\n",
-    "    # Remove parentheses and content within\n",
-    "    name = re.sub(r'\\(.*?\\)', '', name)\n",
-    "\n",
-    "    # Remove excess whitespace\n",
-    "    name = re.sub(r'\\s+', ' ', name).strip()\n",
-    "\n",
-    "    return name\n",
-    "\n",
-    "# -------------------------\n",
-    "# Fallback Extractor\n",
-    "# -------------------------\n",
-    "def fallback_extract(text):\n",
-    "    words = text.split()\n",
-    "    capitalized = [w for w in words if w and w[0].isalpha()]\n",
-    "    if len(capitalized) >= 2:\n",
-    "        return \" \".join(capitalized[:2])\n",
-    "    elif capitalized:\n",
-    "        return capitalized[0]\n",
-    "    return None\n",
-    "\n",
-    "# -------------------------\n",
-    "# Full Extraction Logic\n",
-    "# -------------------------\n",
-    "def extract_real_name(raw_name):\n",
-    "    cleaned = preprocess_name(raw_name)\n",
-    "    doc = nlp(cleaned)\n",
-    "    persons = [ent.text for ent in doc.ents if ent.label_ == \"PERSON\"]\n",
-    "    if persons:\n",
-    "        return persons[0]\n",
-    "    return fallback_extract(cleaned)\n",
-    "\n",
-    "# -------------------------\n",
-    "# Load Data and Apply\n",
-    "# -------------------------\n",
-    "df = pd.read_csv(poi_models_dir)  # Or your own path\n",
-    "\n",
-    "# Apply the full extractor\n",
-    "texts = df['name'].astype(str).tolist()\n",
-    "docs = list(nlp.pipe([preprocess_name(t) for t in texts], batch_size=32))\n",
-    "\n",
-    "def extract_from_doc(doc, raw_text):\n",
-    "    persons = [ent.text for ent in doc.ents if ent.label_ == \"PERSON\"]\n",
-    "    if persons:\n",
-    "        return persons[0]\n",
-    "    return fallback_extract(preprocess_name(raw_text))\n",
-    "\n",
-    "df['real_name'] = [extract_from_doc(doc, raw_text) for doc, raw_text in zip(docs, texts)]\n",
-    "\n",
-    "# Save the result\n",
-    "df.to_csv(output, index=False)\n",
-    "print(\"Done! Saved to NER_poi_step_01.csv\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "a40b17fe",
-   "metadata": {},
-   "source": [
-    "## Step 2: compare country and profession with lists"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "id": "414954ed",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-11-21T10:27:41.992098Z",
-     "iopub.status.busy": "2025-11-21T10:27:41.991889Z",
-     "iopub.status.idle": "2025-11-21T10:27:49.779590Z",
-     "shell.execute_reply": "2025-11-21T10:27:49.778889Z",
-     "shell.execute_reply.started": "2025-11-21T10:27:41.992081Z"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "      real_name likely_country likely_nationality likely_profession\n",
-      "0            iu                      South Korean         celebrity\n",
-      "1    super pose                                                    \n",
-      "2         liyuu                                                    \n",
-      "3         irene                      South Korean         celebrity\n",
-      "4  aespa karina                      South Korean         celebrity\n"
-     ]
-    }
-   ],
-   "source": [
-    "import pandas as pd\n",
-    "from pathlib import Path\n",
-    "\n",
-    "# Set up paths\n",
-    "current_dir = Path.cwd()\n",
-    "countries = current_dir.parent / \"misc/lists/countries.csv\"\n",
-    "professions = current_dir.parent / \"misc/lists/professions.csv\"\n",
-    "inputNER = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_01_NER.csv\"\n",
-    "outfile = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
-    "\n",
-    "# Load datasets\n",
-    "poi_df = pd.read_csv(inputNER)\n",
-    "countries_df = pd.read_csv(countries)\n",
-    "professions_df = pd.read_csv(professions)\n",
-    "\n",
-    "# Step 1: Combine tags into one lowercase list\n",
-    "def combine_tags(row):\n",
-    "    return [str(row[f\"tag_{i}\"]).strip().lower() for i in range(1, 8) if pd.notna(row.get(f\"tag_{i}\"))]\n",
-    "\n",
-    "poi_df[\"tags\"] = poi_df.apply(combine_tags, axis=1)\n",
-    "\n",
-    "# Step 2: Build tag → (country, nationality) mapping\n",
-    "tag_to_country_nationality = {}\n",
-    "\n",
-    "for _, row in countries_df.iterrows():\n",
-    "    country = str(row[\"en_short_name\"]).strip()\n",
-    "    nationality = str(row[\"nationality\"]).strip()\n",
-    "\n",
-    "    country_lc = country.lower()\n",
-    "    nationality_lc = nationality.lower()\n",
-    "\n",
-    "    # Add variations to the mapping\n",
-    "    tag_to_country_nationality[country_lc] = (country, \"\")\n",
-    "    tag_to_country_nationality[nationality_lc] = (\"\", nationality)\n",
-    "    tag_to_country_nationality[country_lc.replace(\" \", \"\")] = (country, \"\")\n",
-    "    tag_to_country_nationality[nationality_lc.replace(\" \", \"\")] = (\"\", nationality)\n",
-    "\n",
-    "    for part in country_lc.split():\n",
-    "        tag_to_country_nationality[part] = (country, \"\")\n",
-    "    for part in nationality_lc.split():\n",
-    "        tag_to_country_nationality[part] = (\"\", nationality)\n",
-    "\n",
-    "# Step 3: Infer likely_country and likely_nationality\n",
-    "# Step 3: Infer likely_country and likely_nationality\n",
-    "def infer_country_and_nationality(tags):\n",
-    "    for tag in tags:\n",
-    "        cleaned = tag.replace(\" \", \"\").lower()\n",
-    "        if cleaned in tag_to_country_nationality:\n",
-    "            country, nationality = tag_to_country_nationality[cleaned]\n",
-    "            # Special case: skip if country is \"Isle of Man\"\n",
-    "            if country == \"Isle of Man\":\n",
-    "                country = \"\"\n",
-    "            return pd.Series([country, nationality])\n",
-    "    return pd.Series([\"\", \"\"])\n",
-    "\n",
-    "\n",
-    "poi_df[[\"likely_country\", \"likely_nationality\"]] = poi_df[\"tags\"].apply(infer_country_and_nationality)\n",
-    "\n",
-    "# Step 4: Build tag → profession mapping\n",
-    "profession_alias_map = {}\n",
-    "\n",
-    "for _, row in professions_df.iterrows():\n",
-    "    canonical = str(row['profession']).strip().lower()\n",
-    "    profession_alias_map[canonical] = canonical\n",
-    "    for alias_col in ['alias_1', 'alias_2', 'alias_3']:\n",
-    "        alias = row.get(alias_col)\n",
-    "        if pd.notna(alias):\n",
-    "            profession_alias_map[str(alias).strip().lower()] = canonical\n",
-    "\n",
-    "# Step 5: Infer likely profession from tags\n",
-    "def infer_profession_from_tags(tags):\n",
-    "    matched = []\n",
-    "    for tag in tags:\n",
-    "        cleaned = tag.strip().lower()\n",
-    "        if cleaned in profession_alias_map:\n",
-    "            matched.append(profession_alias_map[cleaned])\n",
-    "\n",
-    "    if not matched:\n",
-    "        return \"\"\n",
-    "    if \"celebrity\" in matched and len(set(matched)) > 1:\n",
-    "        # Drop 'celebrity' if other professions are present\n",
-    "        matched = [m for m in matched if m != \"celebrity\"]\n",
-    "\n",
-    "    return matched[0]  # Return the first specific match\n",
-    "\n",
-    "\n",
-    "poi_df[\"likely_profession\"] = poi_df[\"tags\"].apply(infer_profession_from_tags)\n",
-    "\n",
-    "# Step 6: Save enriched dataset\n",
-    "poi_df.to_csv(outfile, index=False)\n",
-    "\n",
-    "# Optional: Preview\n",
-    "print(poi_df[[\"real_name\", \"likely_country\", \"likely_nationality\", \"likely_profession\"]].head())\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 18,
-   "id": "054f230b",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-11-21T10:27:58.250798Z",
-     "iopub.status.busy": "2025-11-21T10:27:58.250588Z",
-     "iopub.status.idle": "2025-11-21T10:27:58.253063Z",
-     "shell.execute_reply": "2025-11-21T10:27:58.252570Z",
-     "shell.execute_reply.started": "2025-11-21T10:27:58.250780Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "#!pip install transformers torch\n",
-    "#!python -m spacy download en_core_web_trf\n",
-    "#pip install openai"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "59185461",
-   "metadata": {},
-   "source": [
-    "## Step 3: Query Deepseek-v3 with NAME and HINTS"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "504b970f",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-11-21T10:02:17.101296Z",
-     "iopub.status.busy": "2025-11-21T10:02:17.101174Z",
-     "iopub.status.idle": "2025-11-21T10:02:17.103118Z",
-     "shell.execute_reply": "2025-11-21T10:02:17.102724Z",
-     "shell.execute_reply.started": "2025-11-21T10:02:17.101283Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "#!pip install openpyxl"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "id": "7c209115",
-   "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-11-21T10:29:28.176565Z",
-     "iopub.status.busy": "2025-11-21T10:29:28.176373Z",
-     "iopub.status.idle": "2025-11-21T10:30:01.404034Z",
-     "shell.execute_reply": "2025-11-21T10:30:01.403269Z",
-     "shell.execute_reply.started": "2025-11-21T10:29:28.176550Z"
-    }
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Row 1/4...\n"
-     ]
-    },
-    {
-     "ename": "ModuleNotFoundError",
-     "evalue": "No module named 'openpyxl'",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
-      "\u001b[31mModuleNotFoundError\u001b[39m                       Traceback (most recent call last)",
-      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[19]\u001b[39m\u001b[32m, line 112\u001b[39m\n\u001b[32m    110\u001b[39m out_df.to_csv(OUTPUT_CSV, index=\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[32m    111\u001b[39m \u001b[38;5;66;03m# Excel\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m112\u001b[39m \u001b[43mout_df\u001b[49m\u001b[43m.\u001b[49m\u001b[43mto_excel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mOUTPUT_XLSX\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindex\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[32m    113\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(INDEX_FILE, \u001b[33m'\u001b[39m\u001b[33mw\u001b[39m\u001b[33m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[32m    114\u001b[39m     f.write(\u001b[38;5;28mstr\u001b[39m(current_index))\n",
-      "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/pandas/util/_decorators.py:333\u001b[39m, in \u001b[36mdeprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m    327\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(args) > num_allow_args:\n\u001b[32m    328\u001b[39m     warnings.warn(\n\u001b[32m    329\u001b[39m         msg.format(arguments=_format_argument_list(allow_args)),\n\u001b[32m    330\u001b[39m         \u001b[38;5;167;01mFutureWarning\u001b[39;00m,\n\u001b[32m    331\u001b[39m         stacklevel=find_stack_level(),\n\u001b[32m    332\u001b[39m     )\n\u001b[32m--> \u001b[39m\u001b[32m333\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
-      "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/pandas/core/generic.py:2439\u001b[39m, in \u001b[36mNDFrame.to_excel\u001b[39m\u001b[34m(self, excel_writer, sheet_name, na_rep, float_format, columns, header, index, index_label, startrow, startcol, engine, merge_cells, inf_rep, freeze_panes, storage_options, engine_kwargs)\u001b[39m\n\u001b[32m   2426\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpandas\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mio\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mformats\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mexcel\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m ExcelFormatter\n\u001b[32m   2428\u001b[39m formatter = ExcelFormatter(\n\u001b[32m   2429\u001b[39m     df,\n\u001b[32m   2430\u001b[39m     na_rep=na_rep,\n\u001b[32m   (...)\u001b[39m\u001b[32m   2437\u001b[39m     inf_rep=inf_rep,\n\u001b[32m   2438\u001b[39m )\n\u001b[32m-> \u001b[39m\u001b[32m2439\u001b[39m \u001b[43mformatter\u001b[49m\u001b[43m.\u001b[49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m   2440\u001b[39m \u001b[43m    \u001b[49m\u001b[43mexcel_writer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2441\u001b[39m \u001b[43m    \u001b[49m\u001b[43msheet_name\u001b[49m\u001b[43m=\u001b[49m\u001b[43msheet_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2442\u001b[39m \u001b[43m    \u001b[49m\u001b[43mstartrow\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstartrow\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2443\u001b[39m \u001b[43m    \u001b[49m\u001b[43mstartcol\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstartcol\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2444\u001b[39m \u001b[43m    \u001b[49m\u001b[43mfreeze_panes\u001b[49m\u001b[43m=\u001b[49m\u001b[43mfreeze_panes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2445\u001b[39m \u001b[43m    \u001b[49m\u001b[43mengine\u001b[49m\u001b[43m=\u001b[49m\u001b[43mengine\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2446\u001b[39m \u001b[43m    \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2447\u001b[39m \u001b[43m    \u001b[49m\u001b[43mengine_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mengine_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m   2448\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
-      "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/pandas/io/formats/excel.py:943\u001b[39m, in \u001b[36mExcelFormatter.write\u001b[39m\u001b[34m(self, writer, sheet_name, startrow, startcol, freeze_panes, engine, storage_options, engine_kwargs)\u001b[39m\n\u001b[32m    941\u001b[39m     need_save = \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[32m    942\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m943\u001b[39m     writer = \u001b[43mExcelWriter\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m    944\u001b[39m \u001b[43m        \u001b[49m\u001b[43mwriter\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    945\u001b[39m \u001b[43m        \u001b[49m\u001b[43mengine\u001b[49m\u001b[43m=\u001b[49m\u001b[43mengine\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    946\u001b[39m \u001b[43m        \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    947\u001b[39m \u001b[43m        \u001b[49m\u001b[43mengine_kwargs\u001b[49m\u001b[43m=\u001b[49m\u001b[43mengine_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m    948\u001b[39m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m    949\u001b[39m     need_save = \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m    951\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n",
-      "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/pandas/io/excel/_openpyxl.py:57\u001b[39m, in \u001b[36mOpenpyxlWriter.__init__\u001b[39m\u001b[34m(self, path, engine, date_format, datetime_format, mode, storage_options, if_sheet_exists, engine_kwargs, **kwargs)\u001b[39m\n\u001b[32m     44\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m__init__\u001b[39m(\n\u001b[32m     45\u001b[39m     \u001b[38;5;28mself\u001b[39m,\n\u001b[32m     46\u001b[39m     path: FilePath | WriteExcelBuffer | ExcelWriter,\n\u001b[32m   (...)\u001b[39m\u001b[32m     55\u001b[39m ) -> \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m     56\u001b[39m     \u001b[38;5;66;03m# Use the openpyxl module as the Excel writer.\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m57\u001b[39m     \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mopenpyxl\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mworkbook\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Workbook\n\u001b[32m     59\u001b[39m     engine_kwargs = combine_kwargs(engine_kwargs, kwargs)\n\u001b[32m     61\u001b[39m     \u001b[38;5;28msuper\u001b[39m().\u001b[34m__init__\u001b[39m(\n\u001b[32m     62\u001b[39m         path,\n\u001b[32m     63\u001b[39m         mode=mode,\n\u001b[32m   (...)\u001b[39m\u001b[32m     66\u001b[39m         engine_kwargs=engine_kwargs,\n\u001b[32m     67\u001b[39m     )\n",
-      "\u001b[31mModuleNotFoundError\u001b[39m: No module named 'openpyxl'"
-     ]
-    }
-   ],
-   "source": [
-    "import pandas as pd\n",
-    "import openai\n",
-    "import time\n",
-    "import os\n",
-    "from pathlib import Path\n",
-    "from openai import OpenAI  # Add this import\n",
-    "\n",
-    "# === PATHS & CONFIG ===\n",
-    "current_dir = Path.cwd()\n",
-    "inputCSV = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
-    "api_key_file = current_dir.parent / \"misc/credentials/deepseek_api_key.txt\" #store your API key under misc/credentials/deepseek_api_key.txt\n",
-    "\n",
-    "# Output both CSV and Excel for compatibility\n",
-    "OUTPUT_CSV = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_03_deepseek.csv\"\n",
-    "OUTPUT_XLSX = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_03_deepseek.xlsx\"\n",
-    "INDEX_FILE = current_dir.parent / \"misc/query-indicies/deepseek_query_index.txt\"\n",
-    "SAVE_INTERVAL = 1  # Save every N rows\n",
-    "START_ROW = 1      # Row index to start from (0-based)\n",
-    "END_ROW = 5        # Row index to end (exclusive)\n",
-    "\n",
-    "# === LOAD API KEY & CLIENT ===\n",
-    "with open(api_key_file) as f:\n",
-    "    api_key = f.read().strip()\n",
-    "\n",
-    "client = OpenAI(\n",
-    "    api_key=api_key,\n",
-    "    base_url=\"https://api.deepseek.com/v1\"\n",
-    ")\n",
-    "\n",
-    "# === LOAD DATA ===\n",
-    "df = pd.read_csv(inputCSV)\n",
-    "df = df.iloc[START_ROW:END_ROW].reset_index(drop=True)\n",
-    "\n",
-    "# === PREPARE PROMPTS ===\n",
-    "def create_prompt(row):\n",
-    "    name = row['real_name'] if pd.notna(row['real_name']) else row['name']\n",
-    "    hints = []\n",
-    "    for col in ('likely_profession','likely_nationality','likely_country'):\n",
-    "        if pd.notna(row.get(col, None)):\n",
-    "            hints.append(row[col])\n",
-    "    if not hints:\n",
-    "        tags = [row[f'tag_{i}'] for i in range(1,8) if pd.notna(row.get(f'tag_{i}', None))]\n",
-    "        hints.extend(tags[:3])\n",
-    "    hint_text = \", \".join(hints[:5])\n",
-    "    return (\n",
-    "        f\"Given '{name}' ({hint_text}), provide:\\n\"\n",
-    "        \"1. Full legal name (Western order if non-latin script)\\n\"\n",
-    "        \"2. Any stage names/aliases (comma separated)\\n\"\n",
-    "        \"3. Gender\\n\"\n",
-    "        \"4. Top 3 most specific, factual professions (use industry-standard terms, no euphemisms)\\n\"\n",
-    "        \"5. Primary country associated\\n\"\n",
-    "        \"Use 'Unknown' when uncertain or you encounter a fictional character or place. \"\n",
-    "        \"For entertainment fields, specify sub-genres when known (kpop, adult industry, etc.).\"\n",
-    "    )\n",
-    "\n",
-    "prompts = df.apply(create_prompt, axis=1).tolist()\n",
-    "\n",
-    "# === CHECK FOR EXISTING OUTPUT ===\n",
-    "if os.path.exists(INDEX_FILE):\n",
-    "    with open(INDEX_FILE, 'r') as f:\n",
-    "        current_index = int(f.read().strip())\n",
-    "else:\n",
-    "    current_index = 0\n",
-    "\n",
-    "results = []\n",
-    "if os.path.exists(OUTPUT_XLSX):\n",
-    "    existing = pd.read_excel(OUTPUT_XLSX)\n",
-    "    if {'full_name','aliases','gender','profession_llm','country'}.issubset(existing.columns):\n",
-    "        results = existing[['full_name','aliases','gender','profession_llm','country']].values.tolist()\n",
-    "\n",
-    "# === QUERY & PARSE ===\n",
-    "def query_deepseek(prompt):\n",
-    "    try:\n",
-    "        resp = client.chat.completions.create(\n",
-    "            model=\"deepseek-chat\",\n",
-    "            messages=[\n",
-    "                {\"role\":\"system\",\"content\":\"You extract key data on a person; respond with exactly 5 numbered lines.\"},\n",
-    "                {\"role\":\"user\",\"content\":prompt}\n",
-    "            ],\n",
-    "            temperature=0.05, top_p=0.8\n",
-    "        )\n",
-    "        return resp.choices[0].message.content.strip()\n",
-    "    except Exception as e:\n",
-    "        print(\"API error:\", e)\n",
-    "        return \"\"\n",
-    "\n",
-    "def parse_response(resp):\n",
-    "    lines = [l.strip() for l in resp.split('\\n') if l.strip()]\n",
-    "    out = [\"Unknown\"]*5\n",
-    "    for l in lines:\n",
-    "        if l.startswith('1.'): out[0] = l[2:].strip()\n",
-    "        elif l.startswith('2.'): out[1] = l[2:].strip()\n",
-    "        elif l.startswith('3.'): out[2] = l[2:].strip()\n",
-    "        elif l.startswith('4.'): out[3] = l[2:].strip()\n",
-    "        elif l.startswith('5.'): out[4] = l[2:].strip()\n",
-    "    return out\n",
-    "\n",
-    "# === PROCESS & SAVE ===\n",
-    "for i in range(current_index, len(df)):\n",
-    "    print(f\"Row {i+1}/{len(df)}...\")\n",
-    "    data = parse_response(query_deepseek(prompts[i]))\n",
-    "    if i < len(results): results[i] = data\n",
-    "    else: results.append(data)\n",
-    "    current_index = i+1\n",
-    "\n",
-    "    if current_index % SAVE_INTERVAL == 0 or current_index == len(df):\n",
-    "        out_df = df.iloc[:current_index].copy()\n",
-    "        out_df[['full_name','aliases','gender','profession_llm','country']] = pd.DataFrame(results[:current_index])\n",
-    "        # CSV\n",
-    "        out_df.to_csv(OUTPUT_CSV, index=False)\n",
-    "        # Excel\n",
-    "        out_df.to_excel(OUTPUT_XLSX, index=False)\n",
-    "        with open(INDEX_FILE, 'w') as f:\n",
-    "            f.write(str(current_index))\n",
-    "        print(\"Saved up to row\", current_index)\n",
-    "    time.sleep(1)\n",
-    "\n",
-    "print(\"All done! Files:\", OUTPUT_CSV, OUTPUT_XLSX)\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "9d377005",
-   "metadata": {},
-   "source": [
-    "# Aggregate by individual names"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "d2e75354",
-   "metadata": {},
-   "source": [
-    " ##### e.g. Emma Watson [model1, model2, model3] etc."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "747c3a2f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import re\n",
-    "from pathlib import Path\n",
-    "current_dir = Path.cwd()\n",
-    "\n",
-    "profession_map = current_dir.parent / \"misc/lists/mapped_professions.csv\"\n",
-    "\n",
-    "poi_df = current_dir.parent / \"data/CSV/Deepseek_annotated_POI.csv\"\n",
-    "\n",
-    "output = current_dir.parent / \"data/CSV/Deepseek_annotated_POI_aggregated.csv\"\n",
-    "\n",
-    "countries_csv = current_dir.parent / \"misc/lists/countries.csv\"\n",
-    "countries_df = pd.read_csv(countries_csv)\n",
-    "\n",
-    "# Extract valid country names (strip whitespace)\n",
-    "valid_countries = set(countries_df['en_short_name'].str.strip())\n",
-    "\n",
-    "\n",
-    "# Load the dataset\n",
-    "df = pd.read_csv(poi_df)  # Update path if needed\n",
-    "\n",
-    "# Step 1: Group by 'full_name' and aggregate required information\n",
-    "grouped_df = df.groupby('full_name').agg(\n",
-    "    No_of_models=('id', 'count'),\n",
-    "    modelIDs=('id', lambda x: list(x)),\n",
-    "    combinedDownloadCount=('downloadCount', 'sum')\n",
-    ").reset_index()\n",
-    "\n",
-    "# Step 2: Keep representative info for each person\n",
-    "# Keep representative info (including aliases)\n",
-    "additional_columns = df.groupby('full_name').agg(\n",
-    "    country=('country', 'first'),\n",
-    "    profession_llm=('profession_llm', 'first'),\n",
-    "    gender=('gender', 'first'),\n",
-    "    aliases=('aliases', 'first')  # ✅ Add this line\n",
-    ").reset_index()\n",
-    "\n",
-    "\n",
-    "\n",
-    "def standardize_country(country):\n",
-    "    if not isinstance(country, str):\n",
-    "        return \"Unknown\"\n",
-    "\n",
-    "    country_clean = country.strip()\n",
-    "    lowered = country_clean.lower()\n",
-    "\n",
-    "    # Handle fictional or fantasy countries\n",
-    "    fictional_keywords = [\"fictional\", \"westeros\", \"asgard\", \"middle-earth\", \"naboo\", \"middle earth\", \"latveria\"]\n",
-    "    if any(keyword in lowered for keyword in fictional_keywords):\n",
-    "        return \"Unknown\"\n",
-    "\n",
-    "    # Handle known region-based adjustments\n",
-    "    if \"macau\" in lowered:\n",
-    "        return \"Macau\"\n",
-    "    elif \"hong kong\" in lowered:\n",
-    "        return \"Hong Kong\"\n",
-    "    elif \"taiwan\" in lowered:\n",
-    "        return \"Taiwan\"\n",
-    "\n",
-    "    # Normalize complex or alternate country names\n",
-    "    lowered = lowered.replace(\"United Kingdom of Great Britain and Northern Ireland\", \"united kingdom\")\n",
-    "    lowered = lowered.replace(\"england\", \"united kingdom\")\n",
-    "    lowered = lowered.replace(\"united states of america\", \"united states\")\n",
-    "\n",
-    "    # Remove anything in brackets and after commas\n",
-    "    country_clean = re.sub(r\"\\(.*?\\)\", \"\", country_clean)\n",
-    "    country_clean = country_clean.split(',')[0].strip().lower()\n",
-    "\n",
-    "    # Manual overrides\n",
-    "    replacements = {\n",
-    "        \"united kingdom\": \"UK\",\n",
-    "        \"united kingdom of great britain and northern ireland\": \"UK\",\n",
-    "        \"french southern territories\": \"Other\",\n",
-    "        \"united states\": \"US\",\n",
-    "        \"united states of america\": \"US\",\n",
-    "        \"turkey\": \"Türkiye\",\n",
-    "        \"czech republic\": \"Czechia\"\n",
-    "    }\n",
-    "\n",
-    "    if country_clean in replacements:\n",
-    "        return replacements[country_clean]\n",
-    "\n",
-    "    # Final check against valid country list (case-insensitive)\n",
-    "    for valid in valid_countries:\n",
-    "        if country_clean == valid.lower():\n",
-    "            return valid\n",
-    "\n",
-    "    return \"Unknown\"\n",
-    "\n",
-    "\n",
-    "# Updated function to fully remove anything in brackets (complete or not)\n",
-    "def get_profession_short(profession):\n",
-    "    if isinstance(profession, str):\n",
-    "        # Get first part before comma\n",
-    "        first_prof = profession.split(',')[0].strip()\n",
-    "        # Remove all bracketed content, even malformed\n",
-    "        first_prof = re.sub(r\"[\\[].∗?[\\[].*?[\\]]\", \"\", first_prof)  # removes properly closed\n",
-    "        first_prof = re.sub(r\"[\\(\\[].*\", \"\", first_prof)         # removes malformed\n",
-    "        cleaned = first_prof.strip()\n",
-    "        # Normalize 'Actress' to 'Actor'\n",
-    "        if cleaned.lower() == \"actress\":\n",
-    "            return \"Actor\"\n",
-    "        return cleaned\n",
-    "    return None\n",
-    "\n",
-    "# Load your mapping file\n",
-    "mapping_df = pd.read_csv(profession_map, on_bad_lines='skip')  # or 'warn'\n",
-    "\n",
-    "\n",
-    "# Ensure the mapping columns are named correctly\n",
-    "# (Assuming columns are: 'profession_llm' or 'profession_short', and 'category' or 'mapped_profession')\n",
-    "# Adjust these as needed\n",
-    "mapping_df.columns = [col.strip().lower() for col in mapping_df.columns]\n",
-    "\n",
-    "# Rename for clarity and consistency\n",
-    "if 'profession_llm' in mapping_df.columns:\n",
-    "    mapping_df = mapping_df.rename(columns={'profession_llm': 'profession_short'})\n",
-    "if 'category' in mapping_df.columns:\n",
-    "    mapping_df = mapping_df.rename(columns={'category': 'mapped_profession'})\n",
-    "\n",
-    "# Merge the mapped profession into final_df\n",
-    "#final_df = final_df.merge(mapping_df[['profession_short', 'mapped_profession']], on='profession_short', how='left')\n",
-    "\n",
-    "\n",
-    "additional_columns = df.groupby('full_name').agg(\n",
-    "    country=('country', 'first'),\n",
-    "    profession_llm=('profession_llm', 'first'),\n",
-    "    gender=('gender', 'first'),\n",
-    "    aliases=('aliases', 'first')  # <-- Added this line\n",
-    ").reset_index()\n",
-    "\n",
-    "\n",
-    "# Step 3: Merge the aggregated info with the representative info\n",
-    "final_df = pd.merge(grouped_df, additional_columns, on='full_name', how='left')\n",
-    "\n",
-    "# Step 4: Clean and transform columns\n",
-    "final_df['profession_short'] = final_df['profession_llm'].apply(get_profession_short)\n",
-    "final_df['country'] = final_df['country'].apply(standardize_country)\n",
-    "\n",
-    "# Step 5: Merge with profession mapping\n",
-    "final_df = final_df.merge(mapping_df[['profession_short', 'mapped_profession']], on='profession_short', how='left')\n",
-    "\n",
-    "# Optional: Save the result to a CSV file\n",
-    "final_df.to_csv(output, index=False)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "704e5246",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "pm-paper",
-   "language": "python",
-   "name": "pm-paper"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.13"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

jupyter_notebooks/Section_3-3-4_deepfakes.ipynb

@@ -1,124 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "a58d08a9",
-   "metadata": {},
-   "source": [
-    "# LLM annotation of Deepfake adapter dataset"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "38c3a6a6",
-   "metadata": {},
-   "source": [
-    "## Step 1 NER"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ad478ed2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "from pathlib import Path\n",
-    "\n",
-    "# Set up paths\n",
-    "current_dir = Path.cwd()\n",
-    "countries = current_dir.parent / \"misc/lists/countries.csv\"\n",
-    "professions = current_dir.parent / \"misc/lists/professions.csv\"\n",
-    "inputNER = current_dir.parent / \"data/CSV/NER_POI_step01_pre_annotation.csv\"\n",
-    "outfile = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\n",
-    "\n",
-    "# Load datasets\n",
-    "poi_df = pd.read_csv(inputNER)\n",
-    "countries_df = pd.read_csv(countries)\n",
-    "professions_df = pd.read_csv(professions)\n",
-    "\n",
-    "# Step 1: Combine tags into one lowercase list\n",
-    "def combine_tags(row):\n",
-    "    return [str(row[f\"tag_{i}\"]).strip().lower() for i in range(1, 8) if pd.notna(row.get(f\"tag_{i}\"))]\n",
-    "\n",
-    "poi_df[\"tags\"] = poi_df.apply(combine_tags, axis=1)\n",
-    "\n",
-    "# Step 2: Build tag → (country, nationality) mapping\n",
-    "tag_to_country_nationality = {}\n",
-    "\n",
-    "for _, row in countries_df.iterrows():\n",
-    "    country = str(row[\"en_short_name\"]).strip()\n",
-    "    nationality = str(row[\"nationality\"]).strip()\n",
-    "\n",
-    "    country_lc = country.lower()\n",
-    "    nationality_lc = nationality.lower()\n",
-    "\n",
-    "    # Add variations to the mapping\n",
-    "    tag_to_country_nationality[country_lc] = (country, \"\")\n",
-    "    tag_to_country_nationality[nationality_lc] = (\"\", nationality)\n",
-    "    tag_to_country_nationality[country_lc.replace(\" \", \"\")] = (country, \"\")\n",
-    "    tag_to_country_nationality[nationality_lc.replace(\" \", \"\")] = (\"\", nationality)\n",
-    "\n",
-    "    for part in country_lc.split():\n",
-    "        tag_to_country_nationality[part] = (country, \"\")\n",
-    "    for part in nationality_lc.split():\n",
-    "        tag_to_country_nationality[part] = (\"\", nationality)\n",
-    "\n",
-    "# Step 3: Infer likely_country and likely_nationality\n",
-    "def infer_country_and_nationality(tags):\n",
-    "    for tag in tags:\n",
-    "        cleaned = tag.replace(\" \", \"\").lower()\n",
-    "        if cleaned in tag_to_country_nationality:\n",
-    "            country, nationality = tag_to_country_nationality[cleaned]\n",
-    "            return pd.Series([country, nationality])\n",
-    "    return pd.Series([\"\", \"\"])\n",
-    "\n",
-    "poi_df[[\"likely_country\", \"likely_nationality\"]] = poi_df[\"tags\"].apply(infer_country_and_nationality)\n",
-    "\n",
-    "# Step 4: Build tag → profession mapping\n",
-    "profession_alias_map = {}\n",
-    "\n",
-    "for _, row in professions_df.iterrows():\n",
-    "    canonical = str(row['profession']).strip().lower()\n",
-    "    profession_alias_map[canonical] = canonical\n",
-    "    for alias_col in ['alias_1', 'alias_2', 'alias_3']:\n",
-    "        alias = row.get(alias_col)\n",
-    "        if pd.notna(alias):\n",
-    "            profession_alias_map[str(alias).strip().lower()] = canonical\n",
-    "\n",
-    "# Step 5: Infer likely profession from tags\n",
-    "def infer_profession_from_tags(tags):\n",
-    "    matched = []\n",
-    "    for tag in tags:\n",
-    "        cleaned = tag.strip().lower()\n",
-    "        if cleaned in profession_alias_map:\n",
-    "            matched.append(profession_alias_map[cleaned])\n",
-    "\n",
-    "    if not matched:\n",
-    "        return \"\"\n",
-    "    if \"celebrity\" in matched and len(set(matched)) > 1:\n",
-    "        # Drop 'celebrity' if other professions are present\n",
-    "        matched = [m for m in matched if m != \"celebrity\"]\n",
-    "\n",
-    "    return matched[0]  # Return the first specific match\n",
-    "\n",
-    "\n",
-    "poi_df[\"likely_profession\"] = poi_df[\"tags\"].apply(infer_profession_from_tags)\n",
-    "\n",
-    "# Step 6: Save enriched dataset\n",
-    "poi_df.to_csv(outfile, index=False)\n",
-    "\n",
-    "# Optional: Preview\n",
-    "print(poi_df[[\"real_name\", \"likely_country\", \"likely_nationality\", \"likely_profession\"]].head())\n"
-   ]
-  }
- ],
- "metadata": {
-  "language_info": {
-   "name": "python"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}