added mistral 24b code

jupyter_notebooks/.ipynb_checkpoints/Section_2-3-4_Figure_8_Step_1_LLM_annotation-checkpoint.ipynb

@@ -0,0 +1,1451 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "23d0ae58",
+   "metadata": {},
+   "source": [
+    "# Deepfake Adapter Dataset - LLM Annotation Pipeline"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e4407358",
+   "metadata": {},
+   "source": [
+    "### Unified Model Loading & Inference\n",
+    "Code for querying Mistral, Gemma, and Qwen models."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1a1b9d0e",
+   "metadata": {},
+   "source": [
+    "## CLEANING & PREPROCESSING"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3df42c46",
+   "metadata": {},
+   "source": [
+    "#### Named Entity Recognitition (NER) using SpaCy "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a287eef4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "import emoji\n",
+    "import spacy\n",
+    "\n",
+    "# Load spaCy model\n",
+    "# You may need to download it first: python -m spacy download en_core_web_sm\n",
+    "try:\n",
+    "    nlp = spacy.load(\"en_core_web_sm\")\n",
+    "    print(\"✅ spaCy model loaded: en_core_web_sm\")\n",
+    "except OSError:\n",
+    "    print(\"❌ spaCy model not found. Downloading...\")\n",
+    "    import subprocess\n",
+    "    subprocess.run([\"python\", \"-m\", \"spacy\", \"download\", \"en_core_web_sm\"])\n",
+    "    nlp = spacy.load(\"en_core_web_sm\")\n",
+    "    print(\"✅ spaCy model downloaded and loaded\")\n",
+    "\n",
+    "# Set up paths\n",
+    "current_dir = Path.cwd()\n",
+    "#input_file = current_dir.parent / \"data/CSV/real_person_adapters.csv\"\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter.csv\"\n",
+    "\n",
+    "# Load dataset\n",
+    "df = pd.read_csv(input_file)\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "\n",
+    "def translate_leetspeak(text: str) -> str:\n",
+    "    \"\"\"\n",
+    "    Translate common leetspeak patterns to normal letters.\n",
+    "    Examples: 4kira -> Akira, 3mma -> Emma, 1rene -> Irene\n",
+    "    \"\"\"\n",
+    "    if not text:\n",
+    "        return text\n",
+    "    \n",
+    "    # Common leetspeak mappings (order matters!)\n",
+    "    leetspeak_map = {\n",
+    "        '4': 'a',\n",
+    "        '3': 'e', \n",
+    "        '1': 'i',\n",
+    "        '0': 'o',\n",
+    "        '7': 't',\n",
+    "        '5': 's',\n",
+    "        '8': 'b',\n",
+    "        '9': 'g',\n",
+    "        '@': 'a',\n",
+    "        '$': 's',\n",
+    "        '!': 'i',\n",
+    "    }\n",
+    "    \n",
+    "    result = text\n",
+    "    # Apply mappings at word boundaries or start of string\n",
+    "    for leet, normal in leetspeak_map.items():\n",
+    "        # Replace at start of word\n",
+    "        result = re.sub(rf'\\b{re.escape(leet)}', normal, result, flags=re.IGNORECASE)\n",
+    "        # Replace standalone numbers that look like letters in context\n",
+    "        result = re.sub(rf'(?<=[a-z]){re.escape(leet)}(?=[a-z])', normal, result, flags=re.IGNORECASE)\n",
+    "    \n",
+    "    return result\n",
+    "\n",
+    "def preprocess_for_ner(name: str) -> str:\n",
+    "    \"\"\"\n",
+    "    Preprocess the name before spaCy NER.\n",
+    "    Remove noise but keep the actual name parts.\n",
+    "    \"\"\"\n",
+    "    if pd.isna(name):\n",
+    "        return \"\"\n",
+    "    \n",
+    "    name = str(name)\n",
+    "    \n",
+    "    # FIRST: Translate leetspeak\n",
+    "    name = translate_leetspeak(name)\n",
+    "    \n",
+    "    # Remove emoji\n",
+    "    name = emoji.replace_emoji(name, replace=' ')\n",
+    "    \n",
+    "    # Remove version indicators (v1, v2, v1.0, etc.)\n",
+    "    name = re.sub(r'\\s*[vV]\\d+(\\.\\d+)?\\s*', ' ', name)\n",
+    "    \n",
+    "    # Remove LoRA-related terms (case insensitive)\n",
+    "    lora_terms = ['lora', 'loha', 'lycoris', 'controlnet', 'textual inversion', \n",
+    "                  'embedding', 'ti', 'checkpoint', 'model', 'adapter', 'pony', 'sdxl', 'flux', 'illustrious', 'sd14', 'sd14', 'sd2', 'sd3', 'diffusion', 'stable', 'hunyuan']\n",
+    "    for term in lora_terms:\n",
+    "        name = re.sub(rf'\\b{term}\\b', '', name, flags=re.IGNORECASE)\n",
+    "    \n",
+    "    # Remove content in parentheses or brackets (often metadata)\n",
+    "    name = re.sub(r'\\([^)]*\\)', '', name)\n",
+    "    name = re.sub(r'\\[[^\\]]*\\]', '', name)\n",
+    "    \n",
+    "    # Remove special characters like 「」\n",
+    "    name = re.sub(r'[「」『』【】〈〉《》]', '', name)\n",
+    "    \n",
+    "    # Handle pipe - keep first part\n",
+    "    if '|' in name:\n",
+    "        name = name.split('|')[0]\n",
+    "    \n",
+    "    # Handle forward slash - keep first part\n",
+    "    if '/' in name:\n",
+    "        name = name.split('/')[0]\n",
+    "    \n",
+    "    # Replace underscores with spaces\n",
+    "    name = name.replace('_', ' ')\n",
+    "    \n",
+    "    # Remove multiple spaces\n",
+    "    name = re.sub(r'\\s+', ' ', name)\n",
+    "    \n",
+    "    # Strip\n",
+    "    name = name.strip()\n",
+    "    \n",
+    "    return name\n",
+    "\n",
+    "def extract_person_name(text: str) -> str:\n",
+    "    \"\"\"\n",
+    "    Use spaCy NER to extract person names from text.\n",
+    "    Falls back to cleaned text if no PERSON entity found.\n",
+    "    \"\"\"\n",
+    "    if not text:\n",
+    "        return \"\"\n",
+    "    \n",
+    "    # Run spaCy NER\n",
+    "    doc = nlp(text)\n",
+    "    \n",
+    "    # Extract PERSON entities\n",
+    "    person_entities = [ent.text for ent in doc.ents if ent.label_ == \"PERSON\"]\n",
+    "    \n",
+    "    if person_entities:\n",
+    "        # Return the first (usually longest/best) person name\n",
+    "        return person_entities[0].strip()\n",
+    "    \n",
+    "    # If no PERSON entity found, try to extract capitalized words (likely names)\n",
+    "    # This helps with names spaCy might miss\n",
+    "    words = text.split()\n",
+    "    capitalized_words = [w for w in words if w and w[0].isupper() and len(w) > 1]\n",
+    "    \n",
+    "    if capitalized_words:\n",
+    "        # Join first few capitalized words (likely the name)\n",
+    "        return ' '.join(capitalized_words[:3]).strip()\n",
+    "    \n",
+    "    # Last resort: return cleaned text\n",
+    "    return text.strip()\n",
+    "\n",
+    "def clean_name_with_spacy(name: str) -> str:\n",
+    "    \"\"\"\n",
+    "    Complete name cleaning pipeline with spaCy NER.\n",
+    "    \n",
+    "    Pipeline:\n",
+    "    1. Translate leetspeak (4→a, 3→e, 1→i, etc.)\n",
+    "    2. Remove noise (emoji, version tags, LoRA terms)\n",
+    "    3. Use spaCy to extract PERSON entities\n",
+    "    4. Fallback to capitalized words or cleaned text\n",
+    "    \"\"\"\n",
+    "    # Step 1 & 2: Preprocess (leetspeak + noise removal)\n",
+    "    preprocessed = preprocess_for_ner(name)\n",
+    "    \n",
+    "    if not preprocessed:\n",
+    "        return \"\"\n",
+    "    \n",
+    "    # Step 3: Extract person name using spaCy NER\n",
+    "    person_name = extract_person_name(preprocessed)\n",
+    "    \n",
+    "    return person_name\n",
+    "\n",
+    "# Apply name cleaning with spaCy\n",
+    "print(\"\\n🔄 Processing names with spaCy NER...\")\n",
+    "df['real_name'] = df['name'].apply(clean_name_with_spacy)\n",
+    "\n",
+    "# Show examples with detailed comparison\n",
+    "print(\"\\n📊 Name cleaning examples (with spaCy NER):\")\n",
+    "print(\"=\" * 100)\n",
+    "print(f\"{'Original Name':<50} | {'Cleaned Name':<30}\")\n",
+    "print(\"=\" * 100)\n",
+    "\n",
+    "examples = df[['name', 'real_name']].head(30)\n",
+    "shown = 0\n",
+    "for idx, row in examples.iterrows():\n",
+    "    if row['name'] != row['real_name'] and shown < 20:\n",
+    "        print(f\"{row['name']:<50} | {row['real_name']:<30}\")\n",
+    "        shown += 1\n",
+    "\n",
+    "print(\"=\" * 100)\n",
+    "\n",
+    "# Show specific test cases\n",
+    "print(\"\\n🧪 Leetspeak translation examples:\")\n",
+    "test_names = ['4kira LoRA', '3mma Watson v2', '1rene LORA', 'L3vi Ackerman']\n",
+    "for test in test_names:\n",
+    "    result = clean_name_with_spacy(test)\n",
+    "    print(f\"  {test:<30} -> {result}\")\n",
+    "\n",
+    "# Statistics\n",
+    "print(f\"\\n📈 Statistics:\")\n",
+    "print(f\"  Total rows: {len(df)}\")\n",
+    "print(f\"  Non-empty names: {(df['real_name'] != '').sum()}\")\n",
+    "print(f\"  Empty names: {(df['real_name'] == '').sum()}\")\n",
+    "\n",
+    "# Show some examples of what spaCy identified\n",
+    "print(\"\\n🎯 Sample spaCy NER results:\")\n",
+    "sample_names = df['real_name'].head(20).tolist()\n",
+    "for i, name in enumerate(sample_names[:10], 1):\n",
+    "    if name:\n",
+    "        print(f\"  {i}. {name}\")\n",
+    "\n",
+    "print(f\"\\n✅ Cleaned {len(df)} names using spaCy NER\")\n",
+    "\n",
+    "# Save intermediate result\n",
+    "output_step1 = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_01_NER.csv\"\n",
+    "df.to_csv(output_step1, index=False)\n",
+    "print(f\"💾 Saved to {output_step1}\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "64687c72",
+   "metadata": {},
+   "source": [
+    "#### STEP 02: Nationality tag to Country hint\n",
+    "here tags related to nationality gets converted to the country equivalent."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d6eaef5b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from pathlib import Path\n",
+    "\n",
+    "# Set up paths\n",
+    "current_dir = Path.cwd()\n",
+    "countries_file = current_dir.parent / \"misc/lists/countries.csv\"\n",
+    "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_01_NER.csv\"\n",
+    "output_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "\n",
+    "# Load datasets\n",
+    "poi_df = pd.read_csv(input_file)\n",
+    "countries_df = pd.read_csv(countries_file)\n",
+    "professions_df = pd.read_csv(professions_file)\n",
+    "\n",
+    "# Define uninhabited or non-relevant territories to exclude\n",
+    "excluded_territories = {\n",
+    "    'isle of man', 'bouvet island', 'heard island and mcdonald islands',\n",
+    "    'french southern territories', 'south georgia and the south sandwich islands',\n",
+    "    'svalbard and jan mayen', 'british indian ocean territory', 'antarctica',\n",
+    "    'christmas island', 'cocos (keeling) islands', 'norfolk island',\n",
+    "    'pitcairn', 'tokelau', 'united states minor outlying islands',\n",
+    "    'wallis and futuna', 'western sahara'\n",
+    "}\n",
+    "\n",
+    "# Step 1: Combine tags into one lowercase list\n",
+    "def combine_tags(row):\n",
+    "    return [str(row[f\"tag_{i}\"]).strip().lower() for i in range(1, 8) if pd.notna(row.get(f\"tag_{i}\"))]\n",
+    "\n",
+    "poi_df[\"tags\"] = poi_df.apply(combine_tags, axis=1)\n",
+    "\n",
+    "# Step 2: Build tag → (country, nationality) mapping with PRIORITIES\n",
+    "tag_to_country_nationality = {}\n",
+    "# We'll use a priority score: direct country name = 3, nationality = 2, word parts = 1\n",
+    "\n",
+    "for _, row in countries_df.iterrows():\n",
+    "    country = str(row[\"en_short_name\"]).strip()\n",
+    "    nationality = str(row[\"nationality\"]).strip()\n",
+    "    \n",
+    "    # Skip excluded territories\n",
+    "    if country.lower() in excluded_territories:\n",
+    "        continue\n",
+    "\n",
+    "    country_lc = country.lower()\n",
+    "    nationality_lc = nationality.lower()\n",
+    "\n",
+    "    # Store as (country, nationality, priority)\n",
+    "    # Exact country name match = highest priority\n",
+    "    if country_lc not in tag_to_country_nationality:\n",
+    "        tag_to_country_nationality[country_lc] = (country, \"\", 3)\n",
+    "    \n",
+    "    # Exact nationality match = medium priority  \n",
+    "    if nationality_lc not in tag_to_country_nationality:\n",
+    "        tag_to_country_nationality[nationality_lc] = (\"\", nationality, 2)\n",
+    "    \n",
+    "    # No-space versions\n",
+    "    country_no_space = country_lc.replace(\" \", \"\")\n",
+    "    nationality_no_space = nationality_lc.replace(\" \", \"\")\n",
+    "    \n",
+    "    if country_no_space not in tag_to_country_nationality:\n",
+    "        tag_to_country_nationality[country_no_space] = (country, \"\", 3)\n",
+    "    if nationality_no_space not in tag_to_country_nationality:\n",
+    "        tag_to_country_nationality[nationality_no_space] = (\"\", nationality, 2)\n",
+    "\n",
+    "    # Word parts = lowest priority (only for longer words to avoid false matches)\n",
+    "    for part in country_lc.split():\n",
+    "        if len(part) > 4:  # Only words longer than 4 chars\n",
+    "            if part not in tag_to_country_nationality:\n",
+    "                tag_to_country_nationality[part] = (country, \"\", 1)\n",
+    "    for part in nationality_lc.split():\n",
+    "        if len(part) > 4:\n",
+    "            if part not in tag_to_country_nationality:\n",
+    "                tag_to_country_nationality[part] = (\"\", nationality, 1)\n",
+    "\n",
+    "print(f\"Built country/nationality mapping with {len(tag_to_country_nationality)} entries\")\n",
+    "\n",
+    "# Step 3: Infer likely_country and likely_nationality by checking ALL tags\n",
+    "def infer_country_and_nationality(tags):\n",
+    "    \"\"\"\n",
+    "    Check ALL tags and return the best match based on priority.\n",
+    "    Priority: exact country name > nationality > word parts\n",
+    "    \"\"\"\n",
+    "    best_match = None\n",
+    "    best_priority = 0\n",
+    "    \n",
+    "    for tag in tags:\n",
+    "        # Try cleaned version (no spaces)\n",
+    "        cleaned = tag.replace(\" \", \"\").lower()\n",
+    "        \n",
+    "        # Check cleaned version\n",
+    "        if cleaned in tag_to_country_nationality:\n",
+    "            country, nationality, priority = tag_to_country_nationality[cleaned]\n",
+    "            if priority > best_priority and country and country.lower() not in excluded_territories:\n",
+    "                best_match = (country, nationality)\n",
+    "                best_priority = priority\n",
+    "        \n",
+    "        # Also check original tag\n",
+    "        if tag in tag_to_country_nationality:\n",
+    "            country, nationality, priority = tag_to_country_nationality[tag]\n",
+    "            if priority > best_priority and country and country.lower() not in excluded_territories:\n",
+    "                best_match = (country, nationality)\n",
+    "                best_priority = priority\n",
+    "    \n",
+    "    if best_match:\n",
+    "        return pd.Series(best_match)\n",
+    "    return pd.Series([\"\", \"\"])\n",
+    "\n",
+    "poi_df[[\"likely_country\", \"likely_nationality\"]] = poi_df[\"tags\"].apply(infer_country_and_nationality)\n",
+    "\n",
+    "# Step 4: Build tag → profession mapping\n",
+    "profession_alias_map = {}\n",
+    "\n",
+    "for _, row in professions_df.iterrows():\n",
+    "    canonical = str(row['profession']).strip().lower()\n",
+    "    profession_alias_map[canonical] = canonical\n",
+    "    for alias_col in ['alias_1', 'alias_2', 'alias_3']:\n",
+    "        alias = row.get(alias_col)\n",
+    "        if pd.notna(alias):\n",
+    "            profession_alias_map[str(alias).strip().lower()] = canonical\n",
+    "\n",
+    "# Step 5: Infer likely profession from tags\n",
+    "def infer_profession_from_tags(tags):\n",
+    "    matched = []\n",
+    "    for tag in tags:\n",
+    "        cleaned = tag.strip().lower()\n",
+    "        if cleaned in profession_alias_map:\n",
+    "            matched.append(profession_alias_map[cleaned])\n",
+    "\n",
+    "    if not matched:\n",
+    "        return \"\"\n",
+    "    if \"celebrity\" in matched and len(set(matched)) > 1:\n",
+    "        # Drop 'celebrity' if other professions are present\n",
+    "        matched = [m for m in matched if m != \"celebrity\"]\n",
+    "\n",
+    "    return matched[0]  # Return the first specific match\n",
+    "\n",
+    "\n",
+    "poi_df[\"likely_profession\"] = poi_df[\"tags\"].apply(infer_profession_from_tags)\n",
+    "\n",
+    "# Step 6: Save enriched dataset\n",
+    "poi_df.to_csv(output_file, index=False)\n",
+    "\n",
+    "# Preview results\n",
+    "print(f\"\\nProcessed {len(poi_df)} rows\")\n",
+    "print(f\"Rows with country: {(poi_df['likely_country'] != '').sum()}\")\n",
+    "print(f\"Rows with nationality: {(poi_df['likely_nationality'] != '').sum()}\")\n",
+    "print(f\"Rows with profession: {(poi_df['likely_profession'] != '').sum()}\")\n",
+    "\n",
+    "print(f\"\\nTop 10 countries:\")\n",
+    "print(poi_df[poi_df['likely_country'] != '']['likely_country'].value_counts().head(10))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4a4a58b3",
+   "metadata": {},
+   "source": [
+    "## LLM ANNOTATION"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b298844d",
+   "metadata": {},
+   "source": [
+    "#### Model Configurations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "39f3d65e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "import torch\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
+    "import signal\n",
+    "from contextlib import contextmanager\n",
+    "\n",
+    "# Configuration\n",
+    "current_dir = Path.cwd()\n",
+    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
+    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Model configurations\n",
+    "MODEL_CONFIGS = {\n",
+    "    'mistral': {\n",
+    "        'name': 'mistralai/Mistral-7B-Instruct-v0.3',\n",
+    "        'dtype': torch.bfloat16,\n",
+    "        'quantization': None,\n",
+    "        'generation_params': {\n",
+    "            'max_new_tokens': 512,\n",
+    "            'temperature': 0.05,\n",
+    "            'do_sample': True,\n",
+    "            'top_p': 0.8,\n",
+    "        }\n",
+    "    },\n",
+    "    'gemma': {\n",
+    "        'name': 'google/gemma-3-27b-it',\n",
+    "        'dtype': torch.bfloat16,\n",
+    "        'quantization': None,\n",
+    "        'generation_params': {\n",
+    "            'max_new_tokens': 512,\n",
+    "            'temperature': 0.1,\n",
+    "            'do_sample': True,\n",
+    "            'top_p': 0.9,\n",
+    "        }\n",
+    "    },\n",
+    "    'qwen': {\n",
+    "        'name': 'Qwen/Qwen2.5-32B-Instruct',\n",
+    "        'dtype': None,  # Will use quantization\n",
+    "        'quantization': BitsAndBytesConfig(\n",
+    "            load_in_8bit=True,\n",
+    "            llm_int8_threshold=6.0,\n",
+    "            llm_int8_has_fp16_weight=False\n",
+    "        ),\n",
+    "        'generation_params': {\n",
+    "            'max_new_tokens': 100,\n",
+    "            'temperature': 0.1,\n",
+    "            'do_sample': False,\n",
+    "        }\n",
+    "    }\n",
+    "}\n",
+    "\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\",\n",
+    "    \"adult performer\",\n",
+    "    \"singer/musician\",\n",
+    "    \"model\",\n",
+    "    \"online personality\",\n",
+    "    \"public figure\",\n",
+    "    \"voice actor/ASMR\",\n",
+    "    \"sports professional\",\n",
+    "    \"tv personality\"\n",
+    "]\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c215b38c",
+   "metadata": {},
+   "source": [
+    "#### Load Model Function"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cfb5b13e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def load_model(model_type='mistral'):\n",
+    "    \"\"\"\n",
+    "    Load model and tokenizer based on type.\n",
+    "    \n",
+    "    Args:\n",
+    "        model_type: 'mistral', 'gemma', or 'qwen'\n",
+    "    \n",
+    "    Returns:\n",
+    "        tuple: (model, tokenizer, config)\n",
+    "    \"\"\"\n",
+    "    if model_type not in MODEL_CONFIGS:\n",
+    "        raise ValueError(f\"Unknown model type: {model_type}. Choose from {list(MODEL_CONFIGS.keys())}\")\n",
+    "    \n",
+    "    config = MODEL_CONFIGS[model_type]\n",
+    "    model_name = config['name']\n",
+    "    \n",
+    "    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "    print(f\"Loading model: {model_name}\")\n",
+    "    print(f\"Cache directory: {CACHE_DIR}\")\n",
+    "    print(f\"Device: {device}\\n\")\n",
+    "    \n",
+    "    if device == \"cpu\":\n",
+    "        print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
+    "    \n",
+    "    # Load tokenizer\n",
+    "    try:\n",
+    "        tokenizer = AutoTokenizer.from_pretrained(\n",
+    "            model_name,\n",
+    "            cache_dir=str(CACHE_DIR),\n",
+    "            use_fast=True\n",
+    "        )\n",
+    "    except:\n",
+    "        tokenizer = AutoTokenizer.from_pretrained(\n",
+    "            model_name,\n",
+    "            cache_dir=str(CACHE_DIR),\n",
+    "            use_fast=False\n",
+    "        )\n",
+    "    \n",
+    "    if tokenizer.pad_token is None:\n",
+    "        tokenizer.pad_token = tokenizer.eos_token\n",
+    "    \n",
+    "    # Load model\n",
+    "    model_kwargs = {\n",
+    "        'cache_dir': str(CACHE_DIR),\n",
+    "        'device_map': 'auto',\n",
+    "        'trust_remote_code': False\n",
+    "    }\n",
+    "    \n",
+    "    if config['quantization']:\n",
+    "        model_kwargs['quantization_config'] = config['quantization']\n",
+    "    else:\n",
+    "        model_kwargs['torch_dtype'] = config['dtype']\n",
+    "    \n",
+    "    model = AutoModelForCausalLM.from_pretrained(model_name, **model_kwargs)\n",
+    "    model.eval()\n",
+    "    \n",
+    "    # Check VRAM\n",
+    "    if torch.cuda.is_available():\n",
+    "        vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
+    "        print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
+    "    \n",
+    "    return model, tokenizer, config\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "11b2221a",
+   "metadata": {},
+   "source": [
+    "#### Inference Code"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "229f96bd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@contextmanager\n",
+    "def timeout(duration):\n",
+    "    \"\"\"Context manager for timeout.\"\"\"\n",
+    "    def handler(signum, frame):\n",
+    "        raise TimeoutError(\"Operation timed out\")\n",
+    "    \n",
+    "    signal.signal(signal.SIGALRM, handler)\n",
+    "    signal.alarm(duration)\n",
+    "    try:\n",
+    "        yield\n",
+    "    finally:\n",
+    "        signal.alarm(0)\n",
+    "\n",
+    "def query_model(prompt, model, tokenizer, config, use_timeout=False):\n",
+    "    \"\"\"\n",
+    "    Query model with given prompt.\n",
+    "    \n",
+    "    Args:\n",
+    "        prompt: Input prompt string\n",
+    "        model: Loaded model\n",
+    "        tokenizer: Loaded tokenizer\n",
+    "        config: Model configuration dict\n",
+    "        use_timeout: Whether to use 60s timeout (for Qwen)\n",
+    "    \n",
+    "    Returns:\n",
+    "        str: Model response or None on error\n",
+    "    \"\"\"\n",
+    "    try:\n",
+    "        device = next(model.parameters()).device\n",
+    "        \n",
+    "        # Format as chat message\n",
+    "        messages = [\n",
+    "            {\"role\": \"system\", \"content\": \"You are a data extraction assistant. Respond with exactly 5 numbered lines containing ONLY values. No labels, no explanations, no prefixes. Follow the format precisely.\"},\n",
+    "            {\"role\": \"user\", \"content\": prompt}\n",
+    "        ]\n",
+    "        \n",
+    "        # Tokenize\n",
+    "        if hasattr(tokenizer, 'apply_chat_template'):\n",
+    "            text = tokenizer.apply_chat_template(\n",
+    "                messages,\n",
+    "                tokenize=False,\n",
+    "                add_generation_prompt=True\n",
+    "            )\n",
+    "        else:\n",
+    "            text = f\"[INST] {prompt} [/INST]\"\n",
+    "        \n",
+    "        inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
+    "        \n",
+    "        # Generation parameters\n",
+    "        gen_kwargs = config['generation_params'].copy()\n",
+    "        gen_kwargs['pad_token_id'] = tokenizer.eos_token_id\n",
+    "        \n",
+    "        # Generate\n",
+    "        generation_fn = lambda: model.generate(**inputs, **gen_kwargs)\n",
+    "        \n",
+    "        if use_timeout:\n",
+    "            with timeout(60):\n",
+    "                with torch.no_grad():\n",
+    "                    outputs = generation_fn()\n",
+    "        else:\n",
+    "            with torch.no_grad():\n",
+    "                outputs = generation_fn()\n",
+    "        \n",
+    "        # Decode\n",
+    "        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
+    "        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
+    "        \n",
+    "        return response.strip()\n",
+    "        \n",
+    "    except TimeoutError:\n",
+    "        print(f\"[ERROR] Generation timed out after 60 seconds\")\n",
+    "        return None\n",
+    "    except Exception as e:\n",
+    "        print(f\"[ERROR] Generation failed: {e}\")\n",
+    "        return None\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "88f005f8",
+   "metadata": {},
+   "source": [
+    "#### Prompt creation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dfe05463",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_prompt(row):\n",
+    "    \"\"\"Create annotation prompt from row data.\"\"\"\n",
+    "    name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
+    "    \n",
+    "    # Gather hints\n",
+    "    hints = []\n",
+    "    if pd.notna(row.get('likely_profession')):\n",
+    "        hints.append(str(row['likely_profession']))\n",
+    "    if pd.notna(row.get('likely_nationality')):\n",
+    "        hints.append(str(row['likely_nationality']))\n",
+    "    if pd.notna(row.get('likely_country')):\n",
+    "        hints.append(str(row['likely_country']))\n",
+    "    \n",
+    "    # Add tags if needed\n",
+    "    if len(hints) < 3:\n",
+    "        for i in range(1, 8):\n",
+    "            tag_col = f'tag_{i}'\n",
+    "            if tag_col in row and pd.notna(row[tag_col]):\n",
+    "                tag_val = str(row[tag_col])\n",
+    "                if tag_val not in hints:\n",
+    "                    hints.append(tag_val)\n",
+    "                if len(hints) >= 5:\n",
+    "                    break\n",
+    "    \n",
+    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
+    "    \n",
+    "    return f\"\"\"Extract information about '{name}' ({hint_text}).\n",
+    "\n",
+    "Respond with EXACTLY 5 numbered lines. Each line must contain ONLY the value, no labels or extra text.\n",
+    "\n",
+    "FORMAT REQUIREMENTS:\n",
+    "1. Full legal name in Western order (first last). VALUE ONLY.\n",
+    "2. Stage names/aliases, comma-separated. If none, write \"None\". VALUE ONLY.\n",
+    "3. Gender: MUST be exactly one word: Male, Female, Other, or Unknown. VALUE ONLY.\n",
+    "4. Professions: Choose up to 3 from this list ONLY: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality. Comma-separated. VALUE ONLY.\n",
+    "5. Primary country: Country name only (e.g., \"China\", \"United States\", \"Colombia\"). VALUE ONLY.\n",
+    "\n",
+    "RULES:\n",
+    "- Professions MUST match the exact categories listed (actress = actor)\n",
+    "- \"online personality\" includes streamers, cosplayers, YouTubers, influencers\n",
+    "- \"public figure\" includes politicians, activists, journalists, authors\n",
+    "- Use \"Unknown\" when uncertain or for fictional characters\n",
+    "- NO explanatory text, NO labels like \"Gender:\", NO prefixes\n",
+    "- For multi-role people, list up to 3 categories by relevance\n",
+    "\n",
+    "EXAMPLE FORMAT:\n",
+    "1. Taylor Swift\n",
+    "2. None\n",
+    "3. Female\n",
+    "4. singer/musician, public figure\n",
+    "5. United States\"\"\"\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "854fa668",
+   "metadata": {},
+   "source": [
+    "#### Response parsing code"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1a4be2ee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def parse_response(response):\n",
+    "    \"\"\"Parse model response into structured fields.\"\"\"\n",
+    "    if not response:\n",
+    "        return {\n",
+    "            'full_name': 'Unknown',\n",
+    "            'aliases': 'Unknown',\n",
+    "            'gender': 'Unknown',\n",
+    "            'profession_llm': 'Unknown',\n",
+    "            'country': 'Unknown'\n",
+    "        }\n",
+    "    \n",
+    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
+    "    \n",
+    "    fields = {\n",
+    "        'full_name': 'Unknown',\n",
+    "        'aliases': 'Unknown',\n",
+    "        'gender': 'Unknown',\n",
+    "        'profession_llm': 'Unknown',\n",
+    "        'country': 'Unknown'\n",
+    "    }\n",
+    "    \n",
+    "    for line in lines:\n",
+    "        if line.startswith('1.'):\n",
+    "            fields['full_name'] = line[2:].strip()\n",
+    "        elif line.startswith('2.'):\n",
+    "            fields['aliases'] = line[2:].strip()\n",
+    "        elif line.startswith('3.'):\n",
+    "            gender_raw = line[2:].strip()\n",
+    "            gender_raw = re.sub(r'^(Gender:|gender:)\\s*', '', gender_raw, flags=re.IGNORECASE)\n",
+    "            gender_match = re.search(r'\\b(Male|Female|Other|Unknown)\\b', gender_raw, re.IGNORECASE)\n",
+    "            fields['gender'] = gender_match.group(1).capitalize() if gender_match else gender_raw\n",
+    "        elif line.startswith('4.'):\n",
+    "            fields['profession_llm'] = line[2:].strip()\n",
+    "        elif line.startswith('5.'):\n",
+    "            country_raw = line[2:].strip()\n",
+    "            country_raw = re.sub(r'^(Primary\\s+)?(associated\\s+)?country:\\s*', '', country_raw, flags=re.IGNORECASE)\n",
+    "            fields['country'] = country_raw\n",
+    "    \n",
+    "    return fields\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7e2f7a86",
+   "metadata": {},
+   "source": [
+    "#### CSV annotation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5f3dd5d6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def annotate_dataset(model_type='mistral', test_mode=False, test_size=100, max_rows=50862, save_interval=10):\n",
+    "    \"\"\"\n",
+    "    Annotate dataset using specified model.\n",
+    "    \n",
+    "    Args:\n",
+    "        model_type: 'mistral', 'gemma', or 'qwen'\n",
+    "        test_mode: If True, only process test_size rows\n",
+    "        test_size: Number of rows to process in test mode\n",
+    "        max_rows: Maximum rows to process\n",
+    "        save_interval: Save progress every N rows\n",
+    "    \"\"\"\n",
+    "    # Setup paths\n",
+    "    input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "    output_file = current_dir.parent / f\"data/CSV/{model_type}_local_annotated_POI{'_test' if test_mode else ''}.csv\"\n",
+    "    index_file = current_dir.parent / f\"misc/query_indicies/{model_type}_local_query_index.txt\"\n",
+    "    index_file.parent.mkdir(parents=True, exist_ok=True)\n",
+    "    \n",
+    "    # Load model\n",
+    "    model, tokenizer, config = load_model(model_type)\n",
+    "    \n",
+    "    # Load data\n",
+    "    print(f\"Loaded {len(df)} rows from input file\")\n",
+    "    df = pd.read_csv(input_file)\n",
+    "    \n",
+    "    # Merge existing annotations if available\n",
+    "    if output_file.exists():\n",
+    "        existing_df = pd.read_csv(output_file)\n",
+    "        annotation_cols = ['full_name', 'aliases', 'gender', 'profession_llm', 'country']\n",
+    "        for col in annotation_cols:\n",
+    "            if col in existing_df.columns:\n",
+    "                df[col] = existing_df[col][:len(df)]\n",
+    "    \n",
+    "    # Apply limits\n",
+    "    if test_mode:\n",
+    "        df = df.head(test_size).copy()\n",
+    "    elif max_rows:\n",
+    "        df = df.head(max_rows).copy()\n",
+    "    \n",
+    "    # Create prompts\n",
+    "    df['prompt'] = df.apply(create_prompt, axis=1)\n",
+    "    \n",
+    "    # Load progress index\n",
+    "    current_index = 0\n",
+    "    if index_file.exists():\n",
+    "        try:\n",
+    "            current_index = int(index_file.read_text().strip())\n",
+    "        except:\n",
+    "            current_index = 0\n",
+    "    \n",
+    "    print(f\"Resuming from index {current_index}\")\n",
+    "    \n",
+    "    # Process rows\n",
+    "    use_timeout = (model_type == 'qwen')\n",
+    "    \n",
+    "    for i in tqdm(range(current_index, len(df)), desc=f\"{model_type.capitalize()} Annotation\"):\n",
+    "        prompt = df.at[i, \"prompt\"]\n",
+    "        \n",
+    "        # Query with retries\n",
+    "        response = None\n",
+    "        for attempt in range(3):\n",
+    "            response = query_model(prompt, model, tokenizer, config, use_timeout)\n",
+    "            \n",
+    "            if response and len(response.strip()) > 10:\n",
+    "                break\n",
+    "            \n",
+    "            print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
+    "            time.sleep(0.5)\n",
+    "        \n",
+    "        # Skip if invalid\n",
+    "        if not response or len(response.strip()) <= 10:\n",
+    "            print(f\"❌ Row {i}: failed after retries, skipping\")\n",
+    "            continue\n",
+    "        \n",
+    "        # Parse and validate\n",
+    "        parsed = parse_response(response)\n",
+    "        \n",
+    "        if all(v == \"Unknown\" for v in parsed.values()):\n",
+    "            print(f\"❌ Row {i}: parsed as all Unknown, skipping\")\n",
+    "            continue\n",
+    "        \n",
+    "        # Write fields\n",
+    "        for key, value in parsed.items():\n",
+    "            df.at[i, key] = value\n",
+    "        \n",
+    "        current_index = i + 1\n",
+    "        \n",
+    "        # GPU cleanup\n",
+    "        if torch.cuda.is_available():\n",
+    "            torch.cuda.empty_cache()\n",
+    "            torch.cuda.synchronize()\n",
+    "        \n",
+    "        # Save progress\n",
+    "        if (i + 1) % save_interval == 0 or (i + 1) == len(df):\n",
+    "            df.to_csv(output_file, index=False)\n",
+    "            index_file.write_text(str(current_index))\n",
+    "            print(f\"💾 Progress saved after row {i+1}\")\n",
+    "    \n",
+    "    # Final save\n",
+    "    df.to_csv(output_file, index=False)\n",
+    "    index_file.write_text(str(current_index))\n",
+    "    print(f\"✓ Finished annotation with {model_type}\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "55da2f4c",
+   "metadata": {},
+   "source": [
+    "### Usage Examples\n",
+    "Run annotation with your chosen model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "351ea40c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Example 1: Annotate with Mistral (13.5 GB VRAM)\n",
+    "# annotate_dataset(model_type='mistral', test_mode=False)\n",
+    "\n",
+    "# Example 2: Annotate with Gemma (56.3 GB VRAM)\n",
+    "# annotate_dataset(model_type='gemma', test_mode=False)\n",
+    "\n",
+    "# Example 3: Annotate with Qwen (32.7 GB VRAM, 8-bit)\n",
+    "# annotate_dataset(model_type='qwen', test_mode=False)\n",
+    "\n",
+    "# Test mode (first 100 rows)\n",
+    "# annotate_dataset(model_type='mistral', test_mode=True, test_size=100)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e8203abc-e7c3-4cb6-aaeb-fdc6933981fc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "import torch\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
+    "import signal\n",
+    "from contextlib import contextmanager\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
+    "professions_mapped_file = current_dir.parent / \"misc/lists/professions_mapped.csv\"\n",
+    "# === PROCESS DATA ===\n",
+    "\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = False\n",
+    "TEST_SIZE = 100\n",
+    "MAX_ROWS = 50862\n",
+    "SAVE_INTERVAL = 10\n",
+    "\n",
+    "\n",
+    "index_file = current_dir.parent / \"misc/query_indicies/eurollm_local_query_index.txt\"\n",
+    "output_file = current_dir.parent / f\"data/CSV/eurollm_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "\n",
+    "# Model settings\n",
+    "MODEL_NAME = \"utter-project/EuroLLM-9B\"\n",
+    "#MODEL_NAME = \"Qwen/Qwen2.5-32B-Instruct\"\n",
+    "#MODEL_NAME = \"Qwen/Qwen2.5-14B-Instruct\"\n",
+    "#MODEL_NAME =  \"Qwen/Qwen3-235B-A22B-Instruct-2507-FP8\"\n",
+    "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
+    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
+    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Define the SPECIFIC profession categories\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\",\n",
+    "    \"adult performer\",\n",
+    "    \"singer/musician\",\n",
+    "    \"model\",\n",
+    "    \"online personality\",\n",
+    "    \"public figure\",\n",
+    "    \"voice actor/ASMR\",\n",
+    "    \"sports professional\",\n",
+    "    \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD MODEL ===\n",
+    "print(f\"Loading model: {MODEL_NAME}\")\n",
+    "print(f\"Cache directory: {CACHE_DIR}\")\n",
+    "print(f\"This may take a while on first run...\\n\")\n",
+    "\n",
+    "# Check GPU availability\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "print(f\"Device: {device}\")\n",
+    "\n",
+    "if device == \"cpu\":\n",
+    "    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
+    "    print(\"   Consider using a GPU or reducing model size.\")\n",
+    "\n",
+    "# Get HF token from credentials file\n",
+    "import os\n",
+    "credentials_dir = current_dir.parent / \"misc/credentials\"\n",
+    "hf_token_file = credentials_dir / \"hf_token.txt\"\n",
+    "\n",
+    "HF_TOKEN = None\n",
+    "if hf_token_file.exists():\n",
+    "    HF_TOKEN = hf_token_file.read_text().strip()\n",
+    "    print(\"✅ HF token loaded from credentials file\")\n",
+    "else:\n",
+    "    print(\"⚠️  HF token file not found at:\", hf_token_file)\n",
+    "    print(\"    The script will try to use cached credentials from 'huggingface-cli login'\")\n",
+    "    print(\"    Or create the file: misc/credentials/hf_token.txt with your token\")\n",
+    "    HF_TOKEN = None  # Will use cached token if available\n",
+    "\n",
+    "# Load tokenizer\n",
+    "print(\"Loading tokenizer...\")\n",
+    "try:\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=True,\n",
+    "        token=HF_TOKEN\n",
+    "    )\n",
+    "except Exception as e:\n",
+    "    print(f\"Failed with use_fast=True, trying use_fast=False...\")\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=False,\n",
+    "        token=HF_TOKEN\n",
+    "    )\n",
+    "\n",
+    "# Ensure pad token is set\n",
+    "if tokenizer.pad_token is None:\n",
+    "    tokenizer.pad_token = tokenizer.eos_token\n",
+    "\n",
+    "print(\"✅ Tokenizer loaded\")\n",
+    "\n",
+    "# Configure 8-bit quantization for A100\n",
+    "print(\"Configuring 8-bit quantization...\")\n",
+    "quantization_config = BitsAndBytesConfig(\n",
+    "    load_in_8bit=True,\n",
+    "    llm_int8_threshold=6.0,\n",
+    "    llm_int8_has_fp16_weight=False\n",
+    ")\n",
+    "\n",
+    "# Load model with 8-bit quantization\n",
+    "print(\"Loading model with 8-bit quantization (this may take several minutes)...\")\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    MODEL_NAME,\n",
+    "    cache_dir=str(CACHE_DIR),\n",
+    "    quantization_config=quantization_config,\n",
+    "    device_map=\"auto\",\n",
+    "    trust_remote_code=False,\n",
+    "    token=HF_TOKEN\n",
+    ")\n",
+    "model.eval()\n",
+    "print(\"✅ Model loaded with 8-bit quantization\")\n",
+    "\n",
+    "# Check VRAM usage\n",
+    "if torch.cuda.is_available():\n",
+    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
+    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "if output_file.exists():\n",
+    "    print(\"Loading annotated CSV...\")\n",
+    "    df = pd.read_csv(output_file)\n",
+    "else:\n",
+    "    print(\"Loading raw input CSV...\")\n",
+    "    df = pd.read_csv(input_file)\n",
+    "\n",
+    "\n",
+    "# Try to load profession mapping files\n",
+    "try:\n",
+    "    professions_df = pd.read_csv(professions_file)\n",
+    "    print(f\"✅ Loaded professions.csv\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions.csv not found\")\n",
+    "\n",
+    "try:\n",
+    "    prof_mapped_df = pd.read_csv(professions_mapped_file)\n",
+    "    print(f\"✅ Loaded profession mapping with {len(prof_mapped_df)} categories\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions_mapped.csv not found, using default categories\")\n",
+    "\n",
+    "profession_str = \", \".join(PROFESSION_CATEGORIES)\n",
+    "\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "print(f\"\\nProfession categories ({len(PROFESSION_CATEGORIES)}):\")\n",
+    "for cat in PROFESSION_CATEGORIES:\n",
+    "    print(f\"  - {cat}\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"\\nRunning in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
+    "\n",
+    "# === CREATE PROMPTS (OPTIMIZED FOR CLEAN OUTPUTS) ===\n",
+    "def create_prompt(row):\n",
+    "    \"\"\"Create prompt for EuroLLM annotation with strict formatting requirements.\"\"\"\n",
+    "    name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
+    "    \n",
+    "    # Gather hints\n",
+    "    hints = []\n",
+    "    if pd.notna(row.get('likely_profession')):\n",
+    "        hints.append(str(row['likely_profession']))\n",
+    "    if pd.notna(row.get('likely_nationality')):\n",
+    "        hints.append(str(row['likely_nationality']))\n",
+    "    if pd.notna(row.get('likely_country')):\n",
+    "        hints.append(str(row['likely_country']))\n",
+    "    \n",
+    "    # Add tags if we don't have enough hints\n",
+    "    if len(hints) < 3:\n",
+    "        for i in range(1, 8):\n",
+    "            tag_col = f'tag_{i}'\n",
+    "            if tag_col in row and pd.notna(row[tag_col]):\n",
+    "                tag_val = str(row[tag_col])\n",
+    "                if tag_val not in hints:\n",
+    "                    hints.append(tag_val)\n",
+    "                if len(hints) >= 5:\n",
+    "                    break\n",
+    "    \n",
+    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
+    "    \n",
+    "    return f\"\"\"Extract information about '{name}'. \n",
+    "Context hints (DO NOT copy these as professions): {hint_text}\n",
+    "\n",
+    "Respond with EXACTLY 5 numbered lines. Each line must contain ONLY the value, no labels or extra text.\n",
+    "\n",
+    "FORMAT REQUIREMENTS:\n",
+    "1. Full legal name in Western order (first last). VALUE ONLY.\n",
+    "2. Stage names/aliases, comma-separated. If none, write \"None\". VALUE ONLY.\n",
+    "3. Gender: MUST be exactly one word: Male, Female, Other, or Unknown. VALUE ONLY.\n",
+    "4. Professions: Choose up to 3 from this list ONLY: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality. Comma-separated. VALUE ONLY.\n",
+    "5. Primary country: Country name only (e.g., \"China\", \"United States\", \"Colombia\"). VALUE ONLY.\n",
+    "6. If uncertain about an item, write \"Unknown\"\n",
+    "\n",
+    "CRITICAL RULES FOR PROFESSIONS (Line 4):\n",
+    "- ONLY use the exact profession categories listed above\n",
+    "- DO NOT use descriptive words like \"sexy\", \"photorealistic\", \"celebrity\"\n",
+    "- DO NOT copy the hint words as professions\n",
+    "- If uncertain write \"Unknown\"\n",
+    "- Valid professions are ONLY: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality\n",
+    "- Actress = actor, streamer = online personality, YouTuber = online personality\n",
+    "\n",
+    "OTHER RULES:\n",
+    "- Use \"Unknown\" when uncertain or for fictional characters\n",
+    "- NO explanatory text, NO labels like \"Gender:\", NO prefixes\n",
+    "- For multi-role people, list up to 3 categories by relevance\n",
+    "\n",
+    "EXAMPLE FORMAT:\n",
+    "1. Taylor Swift\n",
+    "2. None\n",
+    "3. Female\n",
+    "4. singer/musician, public figure\n",
+    "5. United States\"\"\"\n",
+    "\n",
+    "# Create prompts\n",
+    "print(\"\\nCreating prompts...\")\n",
+    "df['prompt'] = df.apply(create_prompt, axis=1)\n",
+    "print(\"✅ Prompts created\")\n",
+    "\n",
+    "@contextmanager\n",
+    "def timeout(duration):\n",
+    "    def handler(signum, frame):\n",
+    "        raise TimeoutError(\"Operation timed out\")\n",
+    "    \n",
+    "    # Set the signal handler and alarm\n",
+    "    signal.signal(signal.SIGALRM, handler)\n",
+    "    signal.alarm(duration)\n",
+    "    try:\n",
+    "        yield\n",
+    "    finally:\n",
+    "        signal.alarm(0)  # Disable the alarm\n",
+    "\n",
+    "\n",
+    "def query_eurollm_local(prompt: str) -> str:\n",
+    "    \"\"\"Query EuroLLM locally via transformers with very low temperature.\"\"\"\n",
+    "    try:\n",
+    "        # Format as chat message for EuroLLM with strict system prompt\n",
+    "        messages = [\n",
+    "            {\"role\": \"system\", \"content\": \"You are a data extraction assistant. Respond with exactly 5 numbered lines containing ONLY values. No labels, no explanations, no prefixes. Follow the format precisely.\"},\n",
+    "            {\"role\": \"user\", \"content\": prompt}\n",
+    "        ]\n",
+    "        \n",
+    "        # Tokenize\n",
+    "        if hasattr(tokenizer, 'apply_chat_template') and tokenizer.chat_template is not None:\n",
+    "            text = tokenizer.apply_chat_template(\n",
+    "                messages,\n",
+    "                tokenize=False,\n",
+    "                add_generation_prompt=True\n",
+    "            )\n",
+    "        else:\n",
+    "            # Fallback for models without chat template\n",
+    "            text = f\"[INST] {prompt} [/INST]\"\n",
+    "        \n",
+    "        inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
+    "        \n",
+    "        # Generate with timeout and very low temperature\n",
+    "        with timeout(60):\n",
+    "            with torch.no_grad():\n",
+    "                outputs = model.generate(\n",
+    "                    **inputs,\n",
+    "                    max_new_tokens=100,\n",
+    "                    temperature=0.01,  # Very low temperature for more deterministic outputs\n",
+    "                    do_sample=True,     # Must be True when temperature is set\n",
+    "                    pad_token_id=tokenizer.eos_token_id\n",
+    "                )\n",
+    "        \n",
+    "        # Decode\n",
+    "        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
+    "        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
+    "        \n",
+    "        return response.strip()\n",
+    "        \n",
+    "    except TimeoutError:\n",
+    "        print(f\"[ERROR] Generation timed out after 60 seconds\")\n",
+    "        return None\n",
+    "    except Exception as e:\n",
+    "        print(f\"Generation error: {e}\")\n",
+    "        import traceback\n",
+    "        traceback.print_exc()\n",
+    "        return None\n",
+    "\n",
+    "    \n",
+    "# === PARSE RESPONSE WITH CLEANING ===\n",
+    "def parse_response(response):\n",
+    "    \"\"\"Parse EuroLLM response into structured fields with cleaning.\"\"\"\n",
+    "    if not response:\n",
+    "        return {\n",
+    "            'full_name': 'Unknown',\n",
+    "            'aliases': 'Unknown',\n",
+    "            'gender': 'Unknown',\n",
+    "            'profession_llm': 'Unknown',\n",
+    "            'country': 'Unknown'\n",
+    "        }\n",
+    "    \n",
+    "    # Valid profession categories\n",
+    "    VALID_PROFESSIONS = {\n",
+    "        \"actor\", \"adult performer\", \"singer/musician\", \"model\", \n",
+    "        \"online personality\", \"public figure\", \"voice actor/asmr\", \n",
+    "        \"sports professional\", \"tv personality\"\n",
+    "    }\n",
+    "    \n",
+    "    # Split into lines and clean\n",
+    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
+    "    \n",
+    "    # Initialize with Unknown values\n",
+    "    fields = {\n",
+    "        'full_name': 'Unknown',\n",
+    "        'aliases': 'Unknown',\n",
+    "        'gender': 'Unknown',\n",
+    "        'profession_llm': 'Unknown',\n",
+    "        'country': 'Unknown'\n",
+    "    }\n",
+    "    \n",
+    "    # Extract information from each numbered line\n",
+    "    for line in lines:\n",
+    "        if line.startswith('1.'):\n",
+    "            fields['full_name'] = line[2:].strip()\n",
+    "        elif line.startswith('2.'):\n",
+    "            fields['aliases'] = line[2:].strip()\n",
+    "        elif line.startswith('3.'):\n",
+    "            # Clean gender field - remove any labels\n",
+    "            gender_raw = line[2:].strip()\n",
+    "            # Remove common prefixes\n",
+    "            gender_raw = re.sub(r'^(Gender:|gender:)\\s*', '', gender_raw, flags=re.IGNORECASE)\n",
+    "            # Extract just the gender word\n",
+    "            gender_match = re.search(r'\\b(Male|Female|Other|Unknown)\\b', gender_raw, re.IGNORECASE)\n",
+    "            fields['gender'] = gender_match.group(1).capitalize() if gender_match else gender_raw\n",
+    "        elif line.startswith('4.'):\n",
+    "            # Clean and validate profession field\n",
+    "            profession_raw = line[2:].strip()\n",
+    "            \n",
+    "            # Split by comma and validate each profession\n",
+    "            professions = [p.strip().lower() for p in profession_raw.split(',')]\n",
+    "            valid_profs = []\n",
+    "            \n",
+    "            for prof in professions:\n",
+    "                # Check if it's a valid profession\n",
+    "                if prof in VALID_PROFESSIONS:\n",
+    "                    valid_profs.append(prof)\n",
+    "                # Check for common invalid entries\n",
+    "                elif prof in ['unknown', '']:\n",
+    "                    continue\n",
+    "                # Reject descriptive words that aren't professions\n",
+    "                elif prof in ['sexy', 'photorealistic', 'celebrity', 'famous', 'popular', \n",
+    "                            'beautiful', 'attractive', 'hot', 'gorgeous']:\n",
+    "                    continue\n",
+    "                # If it looks like it might be close to a valid profession, keep it\n",
+    "                elif any(valid in prof for valid in VALID_PROFESSIONS):\n",
+    "                    # Try to extract the valid part\n",
+    "                    for valid in VALID_PROFESSIONS:\n",
+    "                        if valid in prof:\n",
+    "                            valid_profs.append(valid)\n",
+    "                            break\n",
+    "            \n",
+    "            # Set the cleaned professions or Unknown if none are valid\n",
+    "            if valid_profs:\n",
+    "                fields['profession_llm'] = ', '.join(valid_profs)\n",
+    "            else:\n",
+    "                fields['profession_llm'] = 'Unknown'\n",
+    "                \n",
+    "        elif line.startswith('5.'):\n",
+    "            # Clean country field - remove any labels\n",
+    "            country_raw = line[2:].strip()\n",
+    "            # Remove common prefixes like \"Primary country:\", \"Country:\", etc.\n",
+    "            country_raw = re.sub(r'^(Primary\\s+)?(associated\\s+)?country:\\s*', '', country_raw, flags=re.IGNORECASE)\n",
+    "            fields['country'] = country_raw\n",
+    "    \n",
+    "    return fields\n",
+    "\n",
+    "# === PROCESS DATA ===\n",
+    "index_file.parent.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Load index\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    try:\n",
+    "        current_index = int(index_file.read_text().strip())\n",
+    "    except:\n",
+    "        current_index = 0\n",
+    "\n",
+    "print(f\"Resuming from index {current_index}\")\n",
+    "\n",
+    "start_time = time.time()\n",
+    "\n",
+    "for i in tqdm(range(current_index, len(df)), desc=\"EuroLLM Local\"):\n",
+    "\n",
+    "    prompt = df.at[i, \"prompt\"]\n",
+    "\n",
+    "    # -------- MODEL QUERY WITH RETRIES --------\n",
+    "    response = None\n",
+    "    for attempt in range(3):\n",
+    "        response = query_eurollm_local(prompt)\n",
+    "        \n",
+    "        # DEBUG: Print first few responses to see what's happening\n",
+    "        if i < 5:\n",
+    "            print(f\"\\n=== DEBUG Row {i}, Attempt {attempt+1} ===\")\n",
+    "            print(f\"Response length: {len(response) if response else 0}\")\n",
+    "            print(f\"Response: {response[:500] if response else 'None'}\")\n",
+    "            print(\"=\" * 50)\n",
+    "        \n",
+    "        # Valid response?\n",
+    "        if response and len(response.strip()) > 10:\n",
+    "            break\n",
+    "        \n",
+    "        print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
+    "        time.sleep(0.5)\n",
+    "\n",
+    "    # If still invalid → DO NOT overwrite previous data\n",
+    "    if not response or len(response.strip()) <= 10:\n",
+    "        print(f\"❌ Row {i}: failed after retries, not writing, not advancing index\")\n",
+    "        continue\n",
+    "\n",
+    "    parsed = parse_response(response)\n",
+    "\n",
+    "    # DEBUG: Print first few parsed results\n",
+    "    if i < 5:\n",
+    "        print(f\"\\n=== PARSED Row {i} ===\")\n",
+    "        for key, value in parsed.items():\n",
+    "            print(f\"  {key}: {value}\")\n",
+    "        print(\"=\" * 50)\n",
+    "\n",
+    "    # Additional safety: skip rows that parsed as all 'Unknown'\n",
+    "    if all(v == \"Unknown\" for v in parsed.values()):\n",
+    "        print(f\"❌ Row {i}: parsed as all Unknown (likely model crash); skipping.\")\n",
+    "        continue\n",
+    "\n",
+    "    # -------- WRITE PARSED FIELDS SAFELY --------\n",
+    "    for key, value in parsed.items():\n",
+    "        df.at[i, key] = value\n",
+    "\n",
+    "    # Advance progress ONLY after successful write\n",
+    "    current_index = i + 1\n",
+    "\n",
+    "    # -------- GPU MEMORY CLEANUP --------\n",
+    "    if torch.cuda.is_available():\n",
+    "        torch.cuda.empty_cache()\n",
+    "        torch.cuda.synchronize()\n",
+    "\n",
+    "    # -------- SAVE LIKE YOUR DEEPSEEK VERSION --------\n",
+    "    if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "        df.to_csv(output_file, index=False)\n",
+    "        with open(index_file, \"w\") as f:\n",
+    "            f.write(str(current_index))\n",
+    "        print(f\"💾 Progress saved after row {i+1}\")\n",
+    "\n",
+    "# Final save\n",
+    "df.to_csv(output_file, index=False)\n",
+    "index_file.write_text(str(current_index))\n",
+    "print(\"✅ Finished full dataset.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a55a5e30-83f3-4f7c-a537-b1216d4e8a07",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "pm-paper",
+   "language": "python",
+   "name": "pm-paper"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

jupyter_notebooks/.ipynb_checkpoints/Section_2-4_Figure_9_ectract_LoRA_metadata_v2-checkpoint.ipynb

@@ -0,0 +1,400 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "f36422c8",
+   "metadata": {},
+   "source": [
+    "# LoRA metadata"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "8a2feb6e",
+   "metadata": {},
+   "source": [
+    "LoRA Metadata Processing Workflow\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Load CSV File        │ --> │ Read adapter metadata CSV file.                   │\n",
+    "│ Read Model Versions  │     │ Extract model version IDs and relevant data.      │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Download Adapter     │ --> │ Use stored download URLs to fetch adapter files   │\n",
+    "│ Files Using API      │     │ using rotating API keys.                          │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Parse Metadata       │ --> │ Extract safetensors metadata, such as training    │\n",
+    "│ from SafeTensor      │     │ images, model type, and architecture.             │\n",
+    "│ Files                │     │                                                   │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Store Parsed         │ --> │ Save extracted metadata into structured JSON      │\n",
+    "│ Metadata as JSON     │     │ files for later analysis.                         │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Process JSON Files   │ --> │ Read saved JSON metadata, extract relevant        │\n",
+    "│ for Consolidation    │     │ details, and filter necessary attributes.         │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Extract Training     │ --> │ Identify most frequent training tags, architectures│\n",
+    "│ Tags & Model Info    │     │ and systems used for model creation.               │\n",
+    "└─────────┬────────────┘     └───────────────────────────────────────────────────┘\n",
+    "          ↓\n",
+    "┌──────────────────────┐     ┌───────────────────────────────────────────────────┐\n",
+    "│ Save Consolidated    │ --> │ Store all processed metadata in a structured CSV  │\n",
+    "│ Metadata to CSV      │     │ format for final analysis.                        ���\n",
+    "└──────────────────────┘     └───────────────────────────────────────────────────┘\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "efc9939d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import re\n",
+    "import json\n",
+    "import csv\n",
+    "import struct\n",
+    "import requests\n",
+    "from pathlib import Path\n",
+    "import pandas as pd\n",
+    "from collections import Counter\n",
+    "from concurrent.futures import ProcessPoolExecutor\n",
+    "from pathlib import Path\n",
+    "import matplotlib.pyplot as plt\n",
+    "from matplotlib.font_manager import FontProperties\n",
+    "from matplotlib import font_manager\n",
+    "import pandas as pd\n",
+    "from collections import Counter\n",
+    "from concurrent.futures import ProcessPoolExecutor\n",
+    "\n",
+    "# Define the current directory and important file paths\n",
+    "current_dir = Path.cwd()\n",
+    "\n",
+    "# Define frequently used directories\n",
+    "\n",
+    "data_dir = current_dir.parent / 'data/csv/adapters.csv'\n",
+    "fonts_dir = current_dir.parent / 'misc/assets/fonts'\n",
+    "plots_dir = current_dir.parent / 'results/plots'\n",
+    "raw_data_dir = current_dir.parent / 'data/adapter_metadata/lora'  ### location of the LoRA metadata (JSON)\n",
+    "temp_dir = current_dir.parent / 'data/raw/adapters_safetensors'\n",
+    "misc_dir = current_dir.parent / 'misc'\n",
+    "\n",
+    "# File paths\n",
+    "adapters_csv = current_dir.parent / 'data/csv/adapters.csv'\n",
+    "output_json_dir = raw_data_dir\n",
+    "api_keys_file = misc_dir / 'credentials/civit.txt'\n",
+    "\n",
+    "# Ensure directories exist\n",
+    "os.makedirs(output_json_dir, exist_ok=True)\n",
+    "os.makedirs(temp_dir, exist_ok=True)\n",
+    "\n",
+    "\n",
+    "# Load fonts into Matplotlib\n",
+    "for font_path in font_paths:\n",
+    "    font_manager.fontManager.addfont(font_path)\n",
+    "\n",
+    "# Set default font family for plots\n",
+    "plt.rcParams['font.family'] = ['Noto Sans JP', 'Noto Sans SC', 'sans-serif']\n",
+    "\n",
+    "print('Paths and fonts initialized successfully.')\n",
+    "\n",
+    "print('Paths initialized successfully.')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "87a58593",
+   "metadata": {},
+   "source": [
+    "## Step 2: Download LoRA and extract *.safetensors metadata\n",
+    "This script downloads LoRA adapters from the filtered Civiverse-Models dataset and extracts the metadata found within the *.safetensors' data structure"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "abd3a0bc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "import csv\n",
+    "import json\n",
+    "import struct\n",
+    "import time\n",
+    "import requests\n",
+    "import signal\n",
+    "import contextlib\n",
+    "from pathlib import Path\n",
+    "import re\n",
+    "\n",
+    "# === Paste your API keys here ===\n",
+    "API_KEYS = [\n",
+    "    \"REDACTED-CIVITAI-API-KEY-1\",  #DISCORD\n",
+    "    \"REDACTED-CIVITAI-API-KEY-2\", #ASDD 1\n",
+    "    \"REDACTED-CIVITAI-API-KEY-3\", #ASDD 2\n",
+    "    \"REDACTED-CIVITAI-API-KEY-4\", #BSDD \n",
+    "    \"REDACTED-CIVITAI-API-KEY-5\"\n",
+    "]\n",
+    "if not API_KEYS or any(not isinstance(k, str) or not k.strip() for k in API_KEYS):\n",
+    "    raise ValueError(\"Please paste at least one valid API key into API_KEYS.\")\n",
+    "\n",
+    "# === Config (adjust paths as needed) ===\n",
+    "current_dir = Path.cwd()\n",
+    "output_json_dir = current_dir.parent / \"data/adapter_metadata/lora\"        # where JSON outputs go\n",
+    "temp_dir = current_dir.parent / \"data/raw/adapters_safetensors\"                            # where downloads go\n",
+    "csv_path = current_dir.parent / \"data/csv/adapters_poi_false_sfw.csv\"\n",
+    "\n",
+    "os.makedirs(output_json_dir, exist_ok=True)\n",
+    "os.makedirs(temp_dir, exist_ok=True)\n",
+    "\n",
+    "# === API key state ===\n",
+    "current_key_index = 0\n",
+    "\n",
+    "\n",
+    "\n",
+    "def safe_filename(name: str, max_length: int = 100) -> str:\n",
+    "    # Replace unsafe chars\n",
+    "    sanitized = re.sub(r'[^a-zA-Z0-9_\\-]', '_', name)\n",
+    "    # Truncate if too long\n",
+    "    if len(sanitized) > max_length:\n",
+    "        sanitized = sanitized[:max_length]\n",
+    "    return sanitized\n",
+    "\n",
+    "\n",
+    "def get_headers():\n",
+    "    global current_key_index\n",
+    "    return {\n",
+    "        \"Accept\": \"application/json\",\n",
+    "        \"Authorization\": f\"Bearer {API_KEYS[current_key_index].strip()}\"\n",
+    "    }\n",
+    "\n",
+    "def rotate_api_key():\n",
+    "    global current_key_index\n",
+    "    if current_key_index < len(API_KEYS) - 1:\n",
+    "        current_key_index += 1\n",
+    "        print(f\"🔁 Rotated to API key #{current_key_index + 1}\")\n",
+    "    else:\n",
+    "        raise Exception(\"All API keys have been exhausted.\")\n",
+    "\n",
+    "# === Utilities ===\n",
+    "def save_json(data, filename):\n",
+    "    with open(filename, 'w', encoding=\"utf-8\") as f:\n",
+    "        json.dump(data, f, indent=4, ensure_ascii=False)\n",
+    "\n",
+    "def parse_safetensors(file_path):\n",
+    "    # Minimal, tolerant metadata reader; returns {} on failure.\n",
+    "    try:\n",
+    "        with open(file_path, 'rb') as f:\n",
+    "            file_data = f.read()\n",
+    "        # Many safetensors use 8-byte header length; this code follows your original logic\n",
+    "        # (4-byte) but keeps the 8-byte skip. Keep if it's working in your dataset.\n",
+    "        metadata_size = struct.unpack('<I', file_data[:4])[0]\n",
+    "        metadata_bytes = file_data[8:8 + metadata_size]\n",
+    "        metadata_str = metadata_bytes.decode('utf-8', errors='replace')\n",
+    "        metadata = json.loads(metadata_str)\n",
+    "        return metadata.get('__metadata__', {})\n",
+    "    except Exception as e:\n",
+    "        print(f\"Error parsing safetensors file: {e}\")\n",
+    "        return {}\n",
+    "\n",
+    "# === Timeout context ===\n",
+    "class TimeoutException(Exception):\n",
+    "    pass\n",
+    "\n",
+    "@contextlib.contextmanager\n",
+    "def time_limit(seconds):\n",
+    "    def signal_handler(signum, frame):\n",
+    "        raise TimeoutException(f\"Timed out after {seconds} seconds\")\n",
+    "    # Note: SIGALRM works on Unix-like OS; on Windows this will be a no-op.\n",
+    "    try:\n",
+    "        signal.signal(signal.SIGALRM, signal_handler)\n",
+    "        signal.alarm(seconds)\n",
+    "    except Exception:\n",
+    "        # Fallback: no hard alarm on non-Unix systems\n",
+    "        pass\n",
+    "    try:\n",
+    "        yield\n",
+    "    finally:\n",
+    "        try:\n",
+    "            signal.alarm(0)\n",
+    "        except Exception:\n",
+    "            pass\n",
+    "\n",
+    "# === Download with timeout, retries, backoff, and key rotation ===\n",
+    "def download_file(url, output_folder, timeout=30, overall_timeout=120, max_retries=3):\n",
+    "    filename = url.split(\"/\")[-1]\n",
+    "    output_path = os.path.join(output_folder, filename)\n",
+    "\n",
+    "    global current_key_index\n",
+    "    retries = 0\n",
+    "    backoff = 2\n",
+    "\n",
+    "    while current_key_index < len(API_KEYS):\n",
+    "        try:\n",
+    "            with time_limit(overall_timeout):  # global cap per download\n",
+    "                #print(f\"➡️  GET {url} using key #{current_key_index + 1}\")\n",
+    "                resp = requests.get(\n",
+    "                    url,\n",
+    "                    headers=get_headers(),\n",
+    "                    stream=True,\n",
+    "                    timeout=(10, timeout),  # (connect timeout, per-chunk read timeout)\n",
+    "                )\n",
+    "\n",
+    "                # Auth errors → rotate key\n",
+    "                if resp.status_code in (401, 403):\n",
+    "                    print(f\"❌ Auth {resp.status_code} with key #{current_key_index + 1}. Rotating.\")\n",
+    "                    rotate_api_key()\n",
+    "                    retries = 0\n",
+    "                    backoff = 2\n",
+    "                    continue\n",
+    "\n",
+    "                # Not found → bubble up as FileNotFoundError (do not rotate)\n",
+    "                if resp.status_code == 404:\n",
+    "                    raise FileNotFoundError(f\"Model not found at {url}\")\n",
+    "\n",
+    "                # Rate limit → either rotate or wait/backoff\n",
+    "                if resp.status_code == 429:\n",
+    "                    print(\"⏳ Rate limited (429).\", end=\" \")\n",
+    "                    if current_key_index < len(API_KEYS) - 1:\n",
+    "                        print(\"Rotating key.\")\n",
+    "                        rotate_api_key()\n",
+    "                        retries = 0\n",
+    "                        backoff = 2\n",
+    "                        continue\n",
+    "                    else:\n",
+    "                        print(f\"Waiting {backoff}s (no other keys).\")\n",
+    "                        time.sleep(backoff)\n",
+    "                        backoff = min(backoff * 2, 60)\n",
+    "                        continue\n",
+    "\n",
+    "                # Other HTTP errors → raise to RequestException path\n",
+    "                resp.raise_for_status()\n",
+    "\n",
+    "                # Save file\n",
+    "                with open(output_path, 'wb') as fh:\n",
+    "                    for chunk in resp.iter_content(chunk_size=8192):\n",
+    "                        if chunk:\n",
+    "                            fh.write(chunk)\n",
+    "\n",
+    "                return output_path, filename\n",
+    "\n",
+    "        except TimeoutException as e:\n",
+    "            # Hard overall timeout → propagate\n",
+    "            raise e\n",
+    "        except requests.exceptions.RequestException as e:\n",
+    "            # Network-ish errors: retry same key with backoff up to max_retries\n",
+    "            retries += 1\n",
+    "            if retries <= max_retries:\n",
+    "                print(f\"🌐 Network error (try {retries}/{max_retries}) with key #{current_key_index + 1}: {e}\")\n",
+    "                time.sleep(backoff)\n",
+    "                backoff = min(backoff * 2, 60)\n",
+    "                continue\n",
+    "            else:\n",
+    "                raise Exception(f\"Failed to download {url} after {max_retries} retries: {e}\")\n",
+    "\n",
+    "    # If we exit the loop, we truly ran out\n",
+    "    raise Exception(\"All API keys have been exhausted or failed.\")\n",
+    "\n",
+    "# === Main processing ===\n",
+    "def process_csv(csv_path):\n",
+    "    with open(csv_path, newline='', encoding='utf-8') as csvfile:\n",
+    "        reader = csv.DictReader(csvfile)\n",
+    "        for index, row in enumerate(reader):\n",
+    "            # Collect up to 20 version IDs; use the most recent\n",
+    "            version_ids = []\n",
+    "            for i in range(1, 21):\n",
+    "                k = f'version_id_{i}'\n",
+    "                if k in row and row[k]:\n",
+    "                    try:\n",
+    "                        version_ids.append(int(float(row[k])))\n",
+    "                    except ValueError:\n",
+    "                        print(f\"Invalid version_id value '{row[k]}' in row: {row}\")\n",
+    "\n",
+    "            if not version_ids:\n",
+    "                print(f\"No valid version IDs found in row: {row}\")\n",
+    "                continue\n",
+    "\n",
+    "            most_recent_version_id = str(max(version_ids))\n",
+    "            name = row.get('name', 'unknown')\n",
+    "            sanitized_name = safe_filename(name, max_length=100)\n",
+    "            new_json_file = os.path.join(\n",
+    "                output_json_dir,\n",
+    "                f\"{index:08d}_{most_recent_version_id}_{sanitized_name}.json\"\n",
+    "            )\n",
+    "\n",
+    "            # Skip if JSON already exists\n",
+    "            if os.path.exists(new_json_file):\n",
+    "                #print(f\"↩️  Skipping versionID {most_recent_version_id} (JSON already exists)\")\n",
+    "                continue\n",
+    "\n",
+    "            try:\n",
+    "                adapter_file, fname = download_file(\n",
+    "                    row['downloadUrl'], str(temp_dir),\n",
+    "                    timeout=30, overall_timeout=180\n",
+    "                )\n",
+    "                metadata = parse_safetensors(adapter_file)\n",
+    "\n",
+    "                civitaidata = {\n",
+    "                    k: (int(v) if str(v).isdigit() else v)\n",
+    "                    for k, v in row.items()\n",
+    "                }\n",
+    "                new_json_data = {\n",
+    "                    \"civitaidata\": civitaidata,\n",
+    "                    \"metadata\": metadata,\n",
+    "                    \"versionID\": most_recent_version_id\n",
+    "                }\n",
+    "                save_json(new_json_data, new_json_file)\n",
+    "                #print(f\"✅ Created JSON for versionID {most_recent_version_id} with file {fname}\")\n",
+    "\n",
+    "            except FileNotFoundError as e:\n",
+    "                print(f\"⚠️ {e} — saving empty metadata.\")\n",
+    "                civitaidata = {\n",
+    "                    k: (int(v) if str(v).isdigit() else v)\n",
+    "                    for k, v in row.items()\n",
+    "                }\n",
+    "                empty_json = {\n",
+    "                    \"civitaidata\": civitaidata,\n",
+    "                    \"metadata\": {},\n",
+    "                    \"versionID\": most_recent_version_id,\n",
+    "                    \"error\": \"Model not found (404)\"\n",
+    "                }\n",
+    "                save_json(empty_json, new_json_file)\n",
+    "            except Exception as e:\n",
+    "                print(f\"⚠️ Error processing versionID {most_recent_version_id}: {e}\")\n",
+    "                civitaidata = {\n",
+    "                    k: (int(v) if str(v).isdigit() else v)\n",
+    "                    for k, v in row.items()\n",
+    "                }\n",
+    "                empty_json = {\n",
+    "                    \"civitaidata\": civitaidata,\n",
+    "                    \"metadata\": {},\n",
+    "                    \"versionID\": most_recent_version_id,\n",
+    "                    \"error\": str(e)\n",
+    "                }\n",
+    "                save_json(empty_json, new_json_file)\n",
+    "                print(f\"💾 Saved empty JSON for versionID {most_recent_version_id} due to failure.\")\n",
+    "\n",
+    "# === Run ===\n",
+    "if __name__ == \"__main__\":\n",
+    "    process_csv(csv_path)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

jupyter_notebooks/Section_2-3-4_Figure_8_Step_1_LLM_annotation.ipynb

@@ -35,71 +35,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "a287eef4",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "✅ spaCy model loaded: en_core_web_sm\n",
-      "Loaded 50861 rows\n",
-      "\n",
-      "🔄 Processing names with spaCy NER...\n",
-      "\n",
-      "📊 Name cleaning examples (with spaCy NER):\n",
-      "====================================================================================================\n",
-      "Original Name                                      | Cleaned Name                  \n",
-      "====================================================================================================\n",
-      "Super Pose Book Vol.1 - ControlNet                 | Super Pose Book               \n",
-      "Liyuu LoRA                                         | Liyuu                         \n",
-      "HashimotoKanna/ 橋本環奈 _JP_Actress                   | HashimotoKanna                \n",
-      "Emma Watson (JG)                                   | Emma Watson                   \n",
-      "Gal Gadot「LoRa」                                    | Gal Gadot                     \n",
-      "Scarlett Johansson「LoRa」                           | Scarlett Johansson            \n",
-      "Gakki | Aragaki Yui | 新垣結衣                         | Gakki                         \n",
-      "Actress Satomi_石原○○                                | Actress Satomi                \n",
-      "Game of Thrones Cast                               | Game Thrones Cast             \n",
-      "Natalie Portman「LoRa」                              | Natalie Portman               \n",
-      "Emma Watson LoRA                                   | Emma Watson                   \n",
-      "Karina Makina Lora                                 | Karina Makina                 \n",
-      "WRAV YUA_三xx亜                                      | WRAV YUA                      \n",
-      "Dilraba Dilmurat 迪丽热巴                              | Dilraba Dilmurat              \n",
-      "MIMI,大幂幂                                           | MIMI                          \n",
-      "Chinese Idol - YangMi杨幂                            | Chinese Idol YangMi杨幂         \n",
-      "Xiaorouseeu / 小柔SeeU - Chinese cosplayer and influencer | Xiaorouseeu                   \n",
-      "Jennifer Connelly (80s/90s)                        | Jennifer Connelly             \n",
-      "====================================================================================================\n",
-      "\n",
-      "🧪 Leetspeak translation examples:\n",
-      "  4kira LoRA                     -> akira\n",
-      "  3mma Watson v2                 -> Watson\n",
-      "  1rene LORA                     -> irene\n",
-      "  L3vi Ackerman                  -> Levi Ackerman\n",
-      "\n",
-      "📈 Statistics:\n",
-      "  Total rows: 50861\n",
-      "  Non-empty names: 50858\n",
-      "  Empty names: 3\n",
-      "\n",
-      "🎯 Sample spaCy NER results:\n",
-      "  1. IU\n",
-      "  2. Super Pose Book\n",
-      "  3. Liyuu\n",
-      "  4. Irene\n",
-      "  5. AESPA Karina\n",
-      "  6. Saika Kawakita\n",
-      "  7. Liu Yifei\n",
-      "  8. HashimotoKanna\n",
-      "  9. Emma Watson\n",
-      "  10. Gal Gadot\n",
-      "\n",
-      "✅ Cleaned 50861 names using spaCy NER\n",
-      "💾 Saved to /home/lauhp/000_PHD/000_010_PUBLICATION/CODE/pm-paper/data/CSV/model_adapter/real_person_adapter_step_01_NER.csv\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import pandas as pd\n",
     "import re\n",
@@ -1011,13 +950,928 @@
     "# Test mode (first 100 rows)\n",
     "# annotate_dataset(model_type='mistral', test_mode=True, test_size=100)\n"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6431d347-d80c-4e8b-83a7-531e5df95a72",
+   "metadata": {},
+   "source": [
+    "## EuroLLM-9B-Instruct"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e8203abc-e7c3-4cb6-aaeb-fdc6933981fc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "import torch\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
+    "import signal\n",
+    "from contextlib import contextmanager\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
+    "professions_mapped_file = current_dir.parent / \"misc/lists/professions_mapped.csv\"\n",
+    "# === PROCESS DATA ===\n",
+    "\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = False\n",
+    "TEST_SIZE = 100\n",
+    "MAX_ROWS = 50862\n",
+    "SAVE_INTERVAL = 10\n",
+    "\n",
+    "\n",
+    "index_file = current_dir.parent / \"misc/query_indicies/eurollm_local_query_index.txt\"\n",
+    "output_file = current_dir.parent / f\"data/CSV/eurollm_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "\n",
+    "# Model settings\n",
+    "MODEL_NAME = \"utter-project/EuroLLM-9B-Instruct\"\n",
+    "#MODEL_NAME = \"Qwen/Qwen2.5-32B-Instruct\"\n",
+    "#MODEL_NAME = \"Qwen/Qwen2.5-14B-Instruct\"\n",
+    "#MODEL_NAME =  \"Qwen/Qwen3-235B-A22B-Instruct-2507-FP8\"\n",
+    "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
+    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
+    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Define the SPECIFIC profession categories\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\",\n",
+    "    \"adult performer\",\n",
+    "    \"singer/musician\",\n",
+    "    \"model\",\n",
+    "    \"online personality\",\n",
+    "    \"public figure\",\n",
+    "    \"voice actor/ASMR\",\n",
+    "    \"sports professional\",\n",
+    "    \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD MODEL ===\n",
+    "print(f\"Loading model: {MODEL_NAME}\")\n",
+    "print(f\"Cache directory: {CACHE_DIR}\")\n",
+    "print(f\"This may take a while on first run...\\n\")\n",
+    "\n",
+    "# Check GPU availability\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "print(f\"Device: {device}\")\n",
+    "\n",
+    "if device == \"cpu\":\n",
+    "    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
+    "    print(\"   Consider using a GPU or reducing model size.\")\n",
+    "\n",
+    "# Get HF token from credentials file\n",
+    "import os\n",
+    "credentials_dir = current_dir.parent / \"misc/credentials\"\n",
+    "hf_token_file = credentials_dir / \"hf_token.txt\"\n",
+    "\n",
+    "HF_TOKEN = None\n",
+    "if hf_token_file.exists():\n",
+    "    HF_TOKEN = hf_token_file.read_text().strip()\n",
+    "    print(\"✅ HF token loaded from credentials file\")\n",
+    "else:\n",
+    "    print(\"⚠️  HF token file not found at:\", hf_token_file)\n",
+    "    print(\"    The script will try to use cached credentials from 'huggingface-cli login'\")\n",
+    "    print(\"    Or create the file: misc/credentials/hf_token.txt with your token\")\n",
+    "    HF_TOKEN = None  # Will use cached token if available\n",
+    "\n",
+    "# Load tokenizer\n",
+    "print(\"Loading tokenizer...\")\n",
+    "try:\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=True,\n",
+    "        token=HF_TOKEN\n",
+    "    )\n",
+    "except Exception as e:\n",
+    "    print(f\"Failed with use_fast=True, trying use_fast=False...\")\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=False,\n",
+    "        token=HF_TOKEN\n",
+    "    )\n",
+    "\n",
+    "# Ensure pad token is set\n",
+    "if tokenizer.pad_token is None:\n",
+    "    tokenizer.pad_token = tokenizer.eos_token\n",
+    "\n",
+    "print(\"✅ Tokenizer loaded\")\n",
+    "\n",
+    "# Configure 8-bit quantization for A100\n",
+    "print(\"Configuring 8-bit quantization...\")\n",
+    "quantization_config = BitsAndBytesConfig(\n",
+    "    load_in_8bit=True,\n",
+    "    llm_int8_threshold=6.0,\n",
+    "    llm_int8_has_fp16_weight=False\n",
+    ")\n",
+    "\n",
+    "# Load model with 8-bit quantization\n",
+    "print(\"Loading model with 8-bit quantization (this may take several minutes)...\")\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    MODEL_NAME,\n",
+    "    cache_dir=str(CACHE_DIR),\n",
+    "    quantization_config=quantization_config,\n",
+    "    device_map=\"auto\",\n",
+    "    trust_remote_code=False,\n",
+    "    token=HF_TOKEN\n",
+    ")\n",
+    "model.eval()\n",
+    "print(\"✅ Model loaded with 8-bit quantization\")\n",
+    "\n",
+    "# Check VRAM usage\n",
+    "if torch.cuda.is_available():\n",
+    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
+    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "if output_file.exists():\n",
+    "    print(\"Loading annotated CSV...\")\n",
+    "    df = pd.read_csv(output_file)\n",
+    "else:\n",
+    "    print(\"Loading raw input CSV...\")\n",
+    "    df = pd.read_csv(input_file)\n",
+    "\n",
+    "\n",
+    "# Try to load profession mapping files\n",
+    "try:\n",
+    "    professions_df = pd.read_csv(professions_file)\n",
+    "    print(f\"✅ Loaded professions.csv\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions.csv not found\")\n",
+    "\n",
+    "try:\n",
+    "    prof_mapped_df = pd.read_csv(professions_mapped_file)\n",
+    "    print(f\"✅ Loaded profession mapping with {len(prof_mapped_df)} categories\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions_mapped.csv not found, using default categories\")\n",
+    "\n",
+    "profession_str = \", \".join(PROFESSION_CATEGORIES)\n",
+    "\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "print(f\"\\nProfession categories ({len(PROFESSION_CATEGORIES)}):\")\n",
+    "for cat in PROFESSION_CATEGORIES:\n",
+    "    print(f\"  - {cat}\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"\\nRunning in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
+    "\n",
+    "# === CREATE PROMPTS (OPTIMIZED FOR CLEAN OUTPUTS) ===\n",
+    "def create_prompt(row):\n",
+    "    \"\"\"Create prompt for EuroLLM annotation with strict formatting requirements.\"\"\"\n",
+    "    name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
+    "    \n",
+    "    # Gather hints\n",
+    "    hints = []\n",
+    "    if pd.notna(row.get('likely_profession')):\n",
+    "        hints.append(str(row['likely_profession']))\n",
+    "    if pd.notna(row.get('likely_nationality')):\n",
+    "        hints.append(str(row['likely_nationality']))\n",
+    "    if pd.notna(row.get('likely_country')):\n",
+    "        hints.append(str(row['likely_country']))\n",
+    "    \n",
+    "    # Add tags if we don't have enough hints\n",
+    "    if len(hints) < 3:\n",
+    "        for i in range(1, 8):\n",
+    "            tag_col = f'tag_{i}'\n",
+    "            if tag_col in row and pd.notna(row[tag_col]):\n",
+    "                tag_val = str(row[tag_col])\n",
+    "                if tag_val not in hints:\n",
+    "                    hints.append(tag_val)\n",
+    "                if len(hints) >= 5:\n",
+    "                    break\n",
+    "    \n",
+    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
+    "    \n",
+    "    return f\"\"\"Extract information about '{name}'. \n",
+    "Context hints (DO NOT copy these as professions): {hint_text}\n",
+    "\n",
+    "Respond with EXACTLY 5 numbered lines. Each line must contain ONLY the value, no labels or extra text.\n",
+    "\n",
+    "FORMAT REQUIREMENTS:\n",
+    "1. Full legal name in Western order (first last). VALUE ONLY.\n",
+    "2. Stage names/aliases, comma-separated. If none, write \"None\". VALUE ONLY.\n",
+    "3. Gender: MUST be exactly one word: Male, Female, Other, or Unknown. VALUE ONLY.\n",
+    "4. Professions: Choose up to 3 from this list ONLY: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality. Comma-separated. VALUE ONLY.\n",
+    "5. Primary country: Country name only (e.g., \"China\", \"United States\", \"Colombia\"). VALUE ONLY.\n",
+    "\n",
+    "CRITICAL RULES FOR PROFESSIONS (Line 4):\n",
+    "- ONLY use the exact profession categories listed above\n",
+    "- DO NOT use descriptive words like \"sexy\", \"photorealistic\", \"celebrity\"\n",
+    "- DO NOT copy the hint words as professions\n",
+    "- If uncertain about profession, write \"Unknown\"\n",
+    "- Valid professions are ONLY: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality\n",
+    "- Actress = actor, streamer = online personality, YouTuber = online personality\n",
+    "\n",
+    "OTHER RULES:\n",
+    "- Use \"Unknown\" when uncertain or for fictional characters\n",
+    "- NO explanatory text, NO labels like \"Gender:\", NO prefixes\n",
+    "- For multi-role people, list up to 3 categories by relevance\"\"\"\n",
+    "\n",
+    "# Create prompts\n",
+    "print(\"\\nCreating prompts...\")\n",
+    "df['prompt'] = df.apply(create_prompt, axis=1)\n",
+    "print(\"✅ Prompts created\")\n",
+    "\n",
+    "@contextmanager\n",
+    "def timeout(duration):\n",
+    "    def handler(signum, frame):\n",
+    "        raise TimeoutError(\"Operation timed out\")\n",
+    "    \n",
+    "    # Set the signal handler and alarm\n",
+    "    signal.signal(signal.SIGALRM, handler)\n",
+    "    signal.alarm(duration)\n",
+    "    try:\n",
+    "        yield\n",
+    "    finally:\n",
+    "        signal.alarm(0)  # Disable the alarm\n",
+    "\n",
+    "\n",
+    "def query_eurollm_local(prompt: str) -> str:\n",
+    "    \"\"\"Query EuroLLM locally via transformers with very low temperature.\"\"\"\n",
+    "    try:\n",
+    "        # Format as chat message for EuroLLM with strict system prompt\n",
+    "        messages = [\n",
+    "            {\"role\": \"system\", \"content\": \"You are a data extraction assistant. Respond with exactly 5 numbered lines containing ONLY values. No labels, no explanations, no prefixes. Follow the format precisely.\"},\n",
+    "            {\"role\": \"user\", \"content\": prompt}\n",
+    "        ]\n",
+    "        \n",
+    "        # Tokenize\n",
+    "        if hasattr(tokenizer, 'apply_chat_template') and tokenizer.chat_template is not None:\n",
+    "            text = tokenizer.apply_chat_template(\n",
+    "                messages,\n",
+    "                tokenize=False,\n",
+    "                add_generation_prompt=True\n",
+    "            )\n",
+    "        else:\n",
+    "            # Fallback for models without chat template\n",
+    "            text = f\"[INST] {prompt} [/INST]\"\n",
+    "        \n",
+    "        inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
+    "        \n",
+    "        # Generate with timeout and very low temperature\n",
+    "        with timeout(60):\n",
+    "            with torch.no_grad():\n",
+    "                outputs = model.generate(\n",
+    "                    **inputs,\n",
+    "                    max_new_tokens=100,\n",
+    "                    temperature=0.01,  # Very low temperature for more deterministic outputs\n",
+    "                    do_sample=True,     # Must be True when temperature is set\n",
+    "                    pad_token_id=tokenizer.eos_token_id\n",
+    "                )\n",
+    "        \n",
+    "        # Decode\n",
+    "        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
+    "        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
+    "        \n",
+    "        return response.strip()\n",
+    "        \n",
+    "    except TimeoutError:\n",
+    "        print(f\"[ERROR] Generation timed out after 60 seconds\")\n",
+    "        return None\n",
+    "    except Exception as e:\n",
+    "        print(f\"Generation error: {e}\")\n",
+    "        import traceback\n",
+    "        traceback.print_exc()\n",
+    "        return None\n",
+    "\n",
+    "    \n",
+    "# === PARSE RESPONSE WITH CLEANING ===\n",
+    "def parse_response(response):\n",
+    "    \"\"\"Parse EuroLLM response into structured fields with cleaning.\"\"\"\n",
+    "    if not response:\n",
+    "        return {\n",
+    "            'full_name': 'Unknown',\n",
+    "            'aliases': 'Unknown',\n",
+    "            'gender': 'Unknown',\n",
+    "            'profession_llm': 'Unknown',\n",
+    "            'country': 'Unknown'\n",
+    "        }\n",
+    "    \n",
+    "    # Valid profession categories\n",
+    "    VALID_PROFESSIONS = {\n",
+    "        \"actor\", \"adult performer\", \"singer/musician\", \"model\", \n",
+    "        \"online personality\", \"public figure\", \"voice actor/asmr\", \n",
+    "        \"sports professional\", \"tv personality\"\n",
+    "    }\n",
+    "    \n",
+    "    # Split into lines and clean\n",
+    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
+    "    \n",
+    "    # Initialize with Unknown values\n",
+    "    fields = {\n",
+    "        'full_name': 'Unknown',\n",
+    "        'aliases': 'Unknown',\n",
+    "        'gender': 'Unknown',\n",
+    "        'profession_llm': 'Unknown',\n",
+    "        'country': 'Unknown'\n",
+    "    }\n",
+    "    \n",
+    "    # Extract information from each numbered line\n",
+    "    for line in lines:\n",
+    "        if line.startswith('1.'):\n",
+    "            fields['full_name'] = line[2:].strip()\n",
+    "        elif line.startswith('2.'):\n",
+    "            fields['aliases'] = line[2:].strip()\n",
+    "        elif line.startswith('3.'):\n",
+    "            # Clean gender field - remove any labels\n",
+    "            gender_raw = line[2:].strip()\n",
+    "            # Remove common prefixes\n",
+    "            gender_raw = re.sub(r'^(Gender:|gender:)\\s*', '', gender_raw, flags=re.IGNORECASE)\n",
+    "            # Extract just the gender word\n",
+    "            gender_match = re.search(r'\\b(Male|Female|Other|Unknown)\\b', gender_raw, re.IGNORECASE)\n",
+    "            fields['gender'] = gender_match.group(1).capitalize() if gender_match else gender_raw\n",
+    "        elif line.startswith('4.'):\n",
+    "            # Clean and validate profession field\n",
+    "            profession_raw = line[2:].strip()\n",
+    "            \n",
+    "            # Split by comma and validate each profession\n",
+    "            professions = [p.strip().lower() for p in profession_raw.split(',')]\n",
+    "            valid_profs = []\n",
+    "            \n",
+    "            for prof in professions:\n",
+    "                # Check if it's a valid profession\n",
+    "                if prof in VALID_PROFESSIONS:\n",
+    "                    valid_profs.append(prof)\n",
+    "                # Check for common invalid entries\n",
+    "                elif prof in ['unknown', '']:\n",
+    "                    continue\n",
+    "                # Reject descriptive words that aren't professions\n",
+    "                elif prof in ['sexy', 'photorealistic', 'celebrity', 'famous', 'popular', \n",
+    "                            'beautiful', 'attractive', 'hot', 'gorgeous']:\n",
+    "                    continue\n",
+    "                # If it looks like it might be close to a valid profession, keep it\n",
+    "                elif any(valid in prof for valid in VALID_PROFESSIONS):\n",
+    "                    # Try to extract the valid part\n",
+    "                    for valid in VALID_PROFESSIONS:\n",
+    "                        if valid in prof:\n",
+    "                            valid_profs.append(valid)\n",
+    "                            break\n",
+    "            \n",
+    "            # Set the cleaned professions or Unknown if none are valid\n",
+    "            if valid_profs:\n",
+    "                fields['profession_llm'] = ', '.join(valid_profs)\n",
+    "            else:\n",
+    "                fields['profession_llm'] = 'Unknown'\n",
+    "                \n",
+    "        elif line.startswith('5.'):\n",
+    "            # Clean country field - remove any labels\n",
+    "            country_raw = line[2:].strip()\n",
+    "            # Remove common prefixes like \"Primary country:\", \"Country:\", etc.\n",
+    "            country_raw = re.sub(r'^(Primary\\s+)?(associated\\s+)?country:\\s*', '', country_raw, flags=re.IGNORECASE)\n",
+    "            fields['country'] = country_raw\n",
+    "    \n",
+    "    return fields\n",
+    "\n",
+    "# === PROCESS DATA ===\n",
+    "index_file.parent.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Load index\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    try:\n",
+    "        current_index = int(index_file.read_text().strip())\n",
+    "    except:\n",
+    "        current_index = 0\n",
+    "\n",
+    "print(f\"Resuming from index {current_index}\")\n",
+    "\n",
+    "start_time = time.time()\n",
+    "\n",
+    "for i in tqdm(range(current_index, len(df)), desc=\"EuroLLM Local\"):\n",
+    "\n",
+    "    prompt = df.at[i, \"prompt\"]\n",
+    "\n",
+    "    # -------- MODEL QUERY WITH RETRIES --------\n",
+    "    response = None\n",
+    "    for attempt in range(3):\n",
+    "        response = query_eurollm_local(prompt)\n",
+    "        \n",
+    "        # DEBUG: Print first few responses to see what's happening\n",
+    "        if i < 5:\n",
+    "            print(f\"\\n=== DEBUG Row {i}, Attempt {attempt+1} ===\")\n",
+    "            print(f\"Response length: {len(response) if response else 0}\")\n",
+    "            print(f\"Response: {response[:500] if response else 'None'}\")\n",
+    "            print(\"=\" * 50)\n",
+    "        \n",
+    "        # Valid response?\n",
+    "        if response and len(response.strip()) > 10:\n",
+    "            break\n",
+    "        \n",
+    "        print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
+    "        time.sleep(0.5)\n",
+    "\n",
+    "    # If still invalid → DO NOT overwrite previous data\n",
+    "    if not response or len(response.strip()) <= 10:\n",
+    "        print(f\"❌ Row {i}: failed after retries, not writing, not advancing index\")\n",
+    "        continue\n",
+    "\n",
+    "    parsed = parse_response(response)\n",
+    "\n",
+    "    # DEBUG: Print first few parsed results\n",
+    "    if i < 5:\n",
+    "        print(f\"\\n=== PARSED Row {i} ===\")\n",
+    "        for key, value in parsed.items():\n",
+    "            print(f\"  {key}: {value}\")\n",
+    "        print(\"=\" * 50)\n",
+    "\n",
+    "    # Additional safety: skip rows that parsed as all 'Unknown'\n",
+    "    if all(v == \"Unknown\" for v in parsed.values()):\n",
+    "        print(f\"❌ Row {i}: parsed as all Unknown (likely model crash); skipping.\")\n",
+    "        continue\n",
+    "\n",
+    "    # -------- WRITE PARSED FIELDS SAFELY --------\n",
+    "    for key, value in parsed.items():\n",
+    "        df.at[i, key] = value\n",
+    "\n",
+    "    # Advance progress ONLY after successful write\n",
+    "    current_index = i + 1\n",
+    "\n",
+    "    # -------- GPU MEMORY CLEANUP --------\n",
+    "    if torch.cuda.is_available():\n",
+    "        torch.cuda.empty_cache()\n",
+    "        torch.cuda.synchronize()\n",
+    "\n",
+    "    # -------- SAVE LIKE YOUR DEEPSEEK VERSION --------\n",
+    "    if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "        df.to_csv(output_file, index=False)\n",
+    "        with open(index_file, \"w\") as f:\n",
+    "            f.write(str(current_index))\n",
+    "        print(f\"💾 Progress saved after row {i+1}\")\n",
+    "\n",
+    "# Final save\n",
+    "df.to_csv(output_file, index=False)\n",
+    "index_file.write_text(str(current_index))\n",
+    "print(\"✅ Finished full dataset.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "472e5ac2-ec04-4bfa-8a67-116277238c15",
+   "metadata": {},
+   "source": [
+    "## Mistral 24b instruct"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "a55a5e30-83f3-4f7c-a537-b1216d4e8a07",
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-12-08T23:57:35.685431Z",
+     "iopub.status.busy": "2025-12-08T23:57:35.685314Z",
+     "iopub.status.idle": "2025-12-08T23:59:48.656498Z",
+     "shell.execute_reply": "2025-12-08T23:59:48.655927Z",
+     "shell.execute_reply.started": "2025-12-08T23:57:35.685419Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading model: mistralai/Mistral-Small-Instruct-2409\n",
+      "Cache directory: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/models\n",
+      "This may take a while on first run (~65GB download)...\n",
+      "\n",
+      "Device: cuda\n",
+      "Loading tokenizer...\n",
+      "✅ Tokenizer loaded\n"
+     ]
+    },
+    {
+     "ename": "NameError",
+     "evalue": "name 'BitsAndBytesConfig' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
+      "\u001b[31mNameError\u001b[39m                                 Traceback (most recent call last)",
+      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 82\u001b[39m\n\u001b[32m     78\u001b[39m     tokenizer.pad_token = tokenizer.eos_token\n\u001b[32m     80\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33m✅ Tokenizer loaded\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m---> \u001b[39m\u001b[32m82\u001b[39m quantization_config = \u001b[43mBitsAndBytesConfig\u001b[49m(\n\u001b[32m     83\u001b[39m     load_in_8bit=\u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m     84\u001b[39m )\n\u001b[32m     87\u001b[39m \u001b[38;5;66;03m# Load model with optimizations\u001b[39;00m\n\u001b[32m     88\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33mLoading model (this may take several minutes)...\u001b[39m\u001b[33m\"\u001b[39m)\n",
+      "\u001b[31mNameError\u001b[39m: name 'BitsAndBytesConfig' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "import torch\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
+    "professions_mapped_file = current_dir.parent / \"misc/lists/professions_mapped.csv\"\n",
+    "# === PROCESS DATA ===\n",
+    "\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = False\n",
+    "TEST_SIZE = 100\n",
+    "MAX_ROWS = 50862\n",
+    "SAVE_INTERVAL = 10\n",
+    "\n",
+    "output_file = current_dir.parent / f\"data/CSV/mistral24_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/query_indicies/mistral24_local_query_index.txt\"\n",
+    "\n",
+    "\n",
+    "# Model settings\n",
+    "#MODEL_NAME = \"mistralai/Mistral-Small-3.1-24B-Instruct-2503\"\n",
+    "MODEL_NAME = \"mistralai/Mistral-Small-Instruct-2409\"\n",
+    "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
+    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
+    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Define the SPECIFIC profession categories\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\",\n",
+    "    \"adult performer\",\n",
+    "    \"singer/musician\",\n",
+    "    \"model\",\n",
+    "    \"online personality\",\n",
+    "    \"public figure\",\n",
+    "    \"voice actor/ASMR\",\n",
+    "    \"sports professional\",\n",
+    "    \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD MODEL ===\n",
+    "print(f\"Loading model: {MODEL_NAME}\")\n",
+    "print(f\"Cache directory: {CACHE_DIR}\")\n",
+    "print(f\"This may take a while on first run (~65GB download)...\\n\")\n",
+    "\n",
+    "# Check GPU availability\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "print(f\"Device: {device}\")\n",
+    "\n",
+    "if device == \"cpu\":\n",
+    "    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
+    "    print(\"   Consider using a GPU or reducing model size.\")\n",
+    "\n",
+    "# Load tokenizer\n",
+    "print(\"Loading tokenizer...\")\n",
+    "try:\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=True\n",
+    "    )\n",
+    "except Exception as e:\n",
+    "    print(f\"Failed with use_fast=True, trying use_fast=False...\")\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        use_fast=False\n",
+    "    )\n",
+    "\n",
+    "# Ensure pad token is set\n",
+    "if tokenizer.pad_token is None:\n",
+    "    tokenizer.pad_token = tokenizer.eos_token\n",
+    "\n",
+    "print(\"✅ Tokenizer loaded\")\n",
+    "\n",
+    "quantization_config = BitsAndBytesConfig(\n",
+    "    load_in_8bit=True\n",
+    ")\n",
+    "\n",
+    "\n",
+    "# Load model with optimizations\n",
+    "print(\"Loading model (this may take several minutes)...\")\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    MODEL_NAME,\n",
+    "    cache_dir=str(CACHE_DIR),\n",
+    "    torch_dtype=torch.bfloat16,\n",
+    "    quantization_config=quantization_config,\n",
+    "    device_map=\"auto\",\n",
+    "    trust_remote_code=False\n",
+    ")\n",
+    "model.eval()\n",
+    "print(\"✅ Model loaded\")\n",
+    "\n",
+    "# Check VRAM usage\n",
+    "if torch.cuda.is_available():\n",
+    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
+    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "print(\"Loading raw input CSV...\")\n",
+    "df = pd.read_csv(input_file)  # ALWAYS load the full input\n",
+    "print(f\"Loaded {len(df)} rows from input file\")\n",
+    "\n",
+    "# If we have previous annotations, merge them\n",
+    "if output_file.exists():\n",
+    "    print(\"Found existing annotations, merging...\")\n",
+    "    existing_df = pd.read_csv(output_file)\n",
+    "    print(f\"Existing annotations has {len(existing_df)} rows\")\n",
+    "    \n",
+    "    # Update df with existing annotations\n",
+    "    # Only update the columns that were annotated\n",
+    "    annotation_cols = ['full_name', 'aliases', 'gender', 'profession_llm', 'country']\n",
+    "    for col in annotation_cols:\n",
+    "        if col in existing_df.columns:\n",
+    "            df[col] = existing_df[col][:len(df)]  # Make sure we don't exceed df length\n",
+    "    \n",
+    "    print(f\"Merged annotations, continuing with {len(df)} total rows\")\n",
+    "\n",
+    "\n",
+    "# Try to load profession mapping files\n",
+    "try:\n",
+    "    professions_df = pd.read_csv(professions_file)\n",
+    "    print(f\"✅ Loaded professions.csv\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions.csv not found\")\n",
+    "\n",
+    "try:\n",
+    "    prof_mapped_df = pd.read_csv(professions_mapped_file)\n",
+    "    print(f\"✅ Loaded profession mapping with {len(prof_mapped_df)} categories\")\n",
+    "except:\n",
+    "    print(\"⚠️ Warning: professions_mapped.csv not found, using default categories\")\n",
+    "\n",
+    "profession_str = \", \".join(PROFESSION_CATEGORIES)\n",
+    "\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "print(f\"\\nProfession categories ({len(PROFESSION_CATEGORIES)}):\")\n",
+    "for cat in PROFESSION_CATEGORIES:\n",
+    "    print(f\"  - {cat}\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"\\nRunning in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
+    "\n",
+    "# === CREATE PROMPTS (DEEPSEEK STYLE) ===\n",
+    "def create_prompt(row):\n",
+    "    \"\"\"Create prompt for Mistral annotation with specific profession categories.\"\"\"\n",
+    "    name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
+    "    \n",
+    "    # Gather hints\n",
+    "    hints = []\n",
+    "    if pd.notna(row.get('likely_profession')):\n",
+    "        hints.append(str(row['likely_profession']))\n",
+    "    if pd.notna(row.get('likely_nationality')):\n",
+    "        hints.append(str(row['likely_nationality']))\n",
+    "    if pd.notna(row.get('likely_country')):\n",
+    "        hints.append(str(row['likely_country']))\n",
+    "    \n",
+    "    # Add tags if we don't have enough hints\n",
+    "    if len(hints) < 3:\n",
+    "        for i in range(1, 8):\n",
+    "            tag_col = f'tag_{i}'\n",
+    "            if tag_col in row and pd.notna(row[tag_col]):\n",
+    "                tag_val = str(row[tag_col])\n",
+    "                if tag_val not in hints:\n",
+    "                    hints.append(tag_val)\n",
+    "                if len(hints) >= 5:\n",
+    "                    break\n",
+    "    \n",
+    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
+    "    \n",
+    "    return f\"\"\"Given '{name}' ({hint_text}), provide:\n",
+    "1. Full legal name (Western order if non-latin script)\n",
+    "2. Any stage names/aliases (comma separated)\n",
+    "3. Gender (Male/Female/Other/Unknown)\n",
+    "4. Top 3 most likely professions from ONLY these categories:\n",
+    "   - actor\n",
+    "   - adult performer\n",
+    "   - singer/musician\n",
+    "   - model\n",
+    "   - online personality (includes streamers, cosplayers, influencers)\n",
+    "   - public figure (includes politicians, activists, journalists, authors)\n",
+    "   - voice actor/ASMR\n",
+    "   - sports professional\n",
+    "   - tv personality (includes hosts, presenters, reality TV)\n",
+    "\n",
+    "5. Primary country associated\n",
+    "\n",
+    "IMPORTANT:\n",
+    "- Choose professions ONLY from the 9 categories above\n",
+    "- Provide up to 3 professions, comma-separated, ordered by relevance\n",
+    "- Be SPECIFIC: choose the most accurate category for each role\n",
+    "- \"online personality\" includes: streamers, cosplayers, YouTubers, influencers, content creators\n",
+    "- Use 'Unknown' when uncertain or for fictional characters/places\n",
+    "- For multi-role people, list all relevant categories (e.g., \"actor, singer/musician, online personality\")\n",
+    "- For country respond with one word only, for example China or Columbia\n",
+    "- actress = actor\n",
+    "\n",
+    "Respond with exactly 5 numbered lines.\"\"\"\n",
+    "\n",
+    "# Create prompts\n",
+    "print(\"\\nCreating prompts...\")\n",
+    "df['prompt'] = df.apply(create_prompt, axis=1)\n",
+    "print(\"✅ Prompts created\")\n",
+    "\n",
+    "# === QUERY MISTRAL LOCAL ===\n",
+    "def query_mistral_local(prompt: str) -> str:\n",
+    "    \"\"\"Query Mistral locally via transformers.\"\"\"\n",
+    "    try:\n",
+    "        # Format as chat message for Mistral\n",
+    "        messages = [\n",
+    "            {\"role\": \"system\", \"content\": \"You are an assistant that extracts key data on a person based on the name. Respond with exactly 5 numbered lines. For professions, choose ONLY from these categories: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality.\"},\n",
+    "            {\"role\": \"user\", \"content\": prompt}\n",
+    "        ]\n",
+    "        \n",
+    "        # Tokenize\n",
+    "        if hasattr(tokenizer, 'apply_chat_template'):\n",
+    "            text = tokenizer.apply_chat_template(\n",
+    "                messages,\n",
+    "                tokenize=False,\n",
+    "                add_generation_prompt=True\n",
+    "            )\n",
+    "        else:\n",
+    "            # Fallback for older tokenizers\n",
+    "            text = f\"[INST] {prompt} [/INST]\"\n",
+    "        \n",
+    "        inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
+    "        \n",
+    "        # Generate\n",
+    "        with torch.no_grad():\n",
+    "            outputs = model.generate(\n",
+    "                **inputs,\n",
+    "                max_new_tokens=512,\n",
+    "                temperature=0.05,\n",
+    "                do_sample=True,\n",
+    "                top_p=0.8,\n",
+    "                pad_token_id=tokenizer.pad_token_id if tokenizer.pad_token_id else tokenizer.eos_token_id\n",
+    "            )\n",
+    "        \n",
+    "        # Decode\n",
+    "        generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
+    "        response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
+    "        \n",
+    "        return response.strip()\n",
+    "        \n",
+    "    except Exception as e:\n",
+    "        print(f\"Generation error: {e}\")\n",
+    "        return None\n",
+    "\n",
+    "# === PARSE RESPONSE (DEEPSEEK STYLE) ===\n",
+    "def parse_response(response):\n",
+    "    \"\"\"Parse Mistral response into structured fields.\"\"\"\n",
+    "    if not response:\n",
+    "        return {\n",
+    "            'full_name': 'Unknown',\n",
+    "            'aliases': 'Unknown',\n",
+    "            'gender': 'Unknown',\n",
+    "            'profession_llm': 'Unknown',\n",
+    "            'country': 'Unknown'\n",
+    "        }\n",
+    "    \n",
+    "    # Split into lines and clean\n",
+    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
+    "    \n",
+    "    # Initialize with Unknown values\n",
+    "    fields = {\n",
+    "        'full_name': 'Unknown',\n",
+    "        'aliases': 'Unknown',\n",
+    "        'gender': 'Unknown',\n",
+    "        'profession_llm': 'Unknown',\n",
+    "        'country': 'Unknown'\n",
+    "    }\n",
+    "    \n",
+    "    # Extract information from each numbered line\n",
+    "    for line in lines:\n",
+    "        if line.startswith('1.'):\n",
+    "            fields['full_name'] = line[2:].strip()\n",
+    "        elif line.startswith('2.'):\n",
+    "            fields['aliases'] = line[2:].strip()\n",
+    "        elif line.startswith('3.'):\n",
+    "            fields['gender'] = line[2:].strip()\n",
+    "        elif line.startswith('4.'):\n",
+    "            fields['profession_llm'] = line[2:].strip()\n",
+    "        elif line.startswith('5.'):\n",
+    "            fields['country'] = line[2:].strip()\n",
+    "    \n",
+    "    return fields\n",
+    "\n",
+    "# === PROCESS DATA ===\n",
+    "output_file = current_dir.parent / f\"data/CSV/mistral24_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/query_indicies/mistral24_local_query_index.txt\"\n",
+    "\n",
+    "index_file.parent.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "# Load index\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    try:\n",
+    "        current_index = int(index_file.read_text().strip())\n",
+    "    except:\n",
+    "        current_index = 0\n",
+    "\n",
+    "print(f\"Resuming from index {current_index}\")\n",
+    "\n",
+    "start_time = time.time()\n",
+    "\n",
+    "for i in tqdm(range(current_index, len(df)), desc=\"Mistral Local\"):\n",
+    "\n",
+    "    prompt = df.at[i, \"prompt\"]\n",
+    "\n",
+    "    # -------- MODEL QUERY WITH RETRIES --------\n",
+    "    response = None\n",
+    "    for attempt in range(3):\n",
+    "        response = query_mistral_local(prompt)\n",
+    "        \n",
+    "        # Valid response?\n",
+    "        if response and len(response.strip()) > 10:\n",
+    "            break\n",
+    "        \n",
+    "        print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
+    "        time.sleep(0.5)\n",
+    "\n",
+    "    # If still invalid → DO NOT overwrite previous data\n",
+    "    if not response or len(response.strip()) <= 10:\n",
+    "        print(f\"❌ Row {i}: failed after retries, not writing, not advancing index\")\n",
+    "        continue\n",
+    "\n",
+    "    parsed = parse_response(response)\n",
+    "\n",
+    "    # Additional safety: skip rows that parsed as all 'Unknown'\n",
+    "    if all(v == \"Unknown\" for v in parsed.values()):\n",
+    "        print(f\"❌ Row {i}: parsed as all Unknown (likely model crash); skipping.\")\n",
+    "        continue\n",
+    "\n",
+    "    # -------- WRITE PARSED FIELDS SAFELY --------\n",
+    "    for key, value in parsed.items():\n",
+    "        df.at[i, key] = value\n",
+    "\n",
+    "    # Advance progress ONLY after successful write\n",
+    "    current_index = i + 1\n",
+    "\n",
+    "    # -------- GPU MEMORY CLEANUP --------\n",
+    "    if torch.cuda.is_available():\n",
+    "        torch.cuda.empty_cache()\n",
+    "        torch.cuda.synchronize()\n",
+    "\n",
+    "    # -------- SAVE LIKE YOUR DEEPSEEK VERSION --------\n",
+    "    if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "        df.to_csv(output_file, index=False)\n",
+    "        with open(index_file, \"w\") as f:\n",
+    "            f.write(str(current_index))\n",
+    "        print(f\"💾 Progress saved after row {i+1}\")\n",
+    "\n",
+    "# Final save\n",
+    "df.to_csv(output_file, index=False)\n",
+    "index_file.write_text(str(current_index))\n",
+    "print(\"✅ Finished full dataset.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d7212e75-0ff6-45a0-8695-c4a3d3e02818",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import transformers\n",
+    "print(f\"Transformers version: {transformers.__version__}\")\n",
+    "\n",
+    "# Check if Mistral3 is available\n",
+    "try:\n",
+    "    from transformers import Mistral3ForCausalLM\n",
+    "    print(\"✅ Mistral3 is available\")\n",
+    "except ImportError:\n",
+    "    print(\"❌ Mistral3 not available in this transformers version\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a6ab032e-246e-4c4e-9776-ff0bfbf6fd9c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "latm",
+   "display_name": "pm-paper",
    "language": "python",
-   "name": "python3"
+   "name": "pm-paper"
   },
   "language_info": {
    "codemirror_mode": {
@@ -1029,7 +1883,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.11.13"
   }
  },
  "nbformat": 4,

jupyter_notebooks/Section_2-4_Figure_9_ectract_LoRA_metadata_v2.ipynb

@@ -391,8 +391,22 @@
   }
  ],
  "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
   "language_info": {
-   "name": "python"
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.9"
   }
  },
  "nbformat": 4,

misc/query_indicies/eurollm_local_query_index.txt

@@ -0,0 +1 @@
+80

misc/query_indicies/mistral24_local_query_index.txt

@@ -0,0 +1 @@
+3890