corrected LLM query code

.gitignore

@@ -8,4 +8,5 @@ misc/credentials
 scripts/ARCHIVE
 scripts/CEMETARY
 cemetary
-.venv
+.venv
+logs

jupyter_notebooks/.ipynb_checkpoints/Section_2-3_Figure_5_co-occurence_promotional_tags-checkpoint.ipynb

@@ -0,0 +1,314 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create *.json for figure 5 (Co-occurence network of Tags)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing: america\n",
+      "  ✅ Saved to /home/lauhp/000_PHD/000_010_PUBLICATION/2025_SAGE/CODE/pm-paper_uzh_gitlab/pm-paper/public/json/tags_america.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from itertools import combinations\n",
+    "from collections import Counter, defaultdict\n",
+    "import json\n",
+    "import re\n",
+    "import os\n",
+    "\n",
+    "from pathlib import Path\n",
+    "current_dir = Path.cwd()\n",
+    "\n",
+    "# === CONFIG ===\n",
+    "file_path = current_dir.parent / \"data/CSV/Models/Civi_models.csv\"\n",
+    "output_dir = current_dir.parent / \"public/json/\"\n",
+    "#target_terms = [\"asian\", \"indian\", \"man\", \"woman\", \"german\", \"korean\", \"american\", \"russian\", \"style\", \"japanese\", \"chinese\"]  # Add any tags you want to process\n",
+    "#target_terms = [\"character\", \"instagram\", \"youtuber\", \"actor\", \"actress\", \"celebrity\", \"vtuber\", \"kpop\"]  # Add any tags you want to process\n",
+    "target_terms = [\"america\"]  # Add any tags you want to process\n",
+    "min_connections = 1   # minimum number of link connections per node\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "df = pd.read_csv(file_path)\n",
+    "tag_columns = [f\"tag_{i}\" for i in range(1, 8)]\n",
+    "df_tags = df[tag_columns]\n",
+    "\n",
+    "# === MAIN LOOP ===\n",
+    "for target_term in target_terms:\n",
+    "    print(f\"Processing: {target_term}\")\n",
+    "    \n",
+    "    pattern = re.compile(rf'\\b{re.escape(target_term)}\\b', flags=re.IGNORECASE)\n",
+    "    df_filtered = df_tags[df_tags.apply(\n",
+    "        lambda row: row.astype(str).apply(lambda x: bool(pattern.search(x))).any(),\n",
+    "        axis=1\n",
+    "    )]\n",
+    "\n",
+    "    # Skip if no data matches\n",
+    "    if df_filtered.empty:\n",
+    "        print(f\"  ⚠️ No matches for '{target_term}', skipping.\")\n",
+    "        continue\n",
+    "\n",
+    "    # === COUNT INDIVIDUAL TAGS ===\n",
+    "    all_tags = df_filtered.values.flatten()\n",
+    "    all_tags = [tag for tag in all_tags if pd.notna(tag)]\n",
+    "    tag_counts = Counter(all_tags)\n",
+    "\n",
+    "    # === CO-OCCURRENCE ===\n",
+    "    co_occurrences = defaultdict(int)\n",
+    "    for tags in df_filtered.itertuples(index=False, name=None):\n",
+    "        tags = [tag for tag in tags if pd.notna(tag)]\n",
+    "        for tag1, tag2 in combinations(tags, 2):\n",
+    "            co_occurrences[frozenset([tag1, tag2])] += 1\n",
+    "\n",
+    "    edges = [(list(pair)[0], list(pair)[1], weight) for pair, weight in co_occurrences.items()]\n",
+    "\n",
+    "    # === FILTER BY CONNECTIONS ===\n",
+    "    connected_tags = Counter()\n",
+    "    for tag1, tag2, _ in edges:\n",
+    "        connected_tags[tag1] += 1\n",
+    "        connected_tags[tag2] += 1\n",
+    "\n",
+    "    nodes = [{\"id\": tag, \"size\": tag_counts[tag]} for tag in tag_counts if connected_tags[tag] >= min_connections]\n",
+    "    valid_ids = set(node[\"id\"] for node in nodes)\n",
+    "    links = [{\"source\": tag1, \"target\": tag2, \"value\": weight}\n",
+    "             for tag1, tag2, weight in edges\n",
+    "             if tag1 in valid_ids and tag2 in valid_ids]\n",
+    "\n",
+    "    if not nodes or not links:\n",
+    "        print(f\"  ⚠️ Not enough connections for '{target_term}', skipping.\")\n",
+    "        continue\n",
+    "\n",
+    "    # === EXPORT ===\n",
+    "    d3_data = {\"nodes\": nodes, \"links\": links}\n",
+    "    safe_term = re.sub(r'\\W+', '_', target_term.lower())\n",
+    "    output_file = os.path.join(output_dir, f\"tags_{safe_term}.json\")\n",
+    "    \n",
+    "    with open(output_file, \"w\") as f:\n",
+    "        json.dump(d3_data, f, indent=4)\n",
+    "    \n",
+    "    print(f\"  ✅ Saved to {output_file}\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Different Countries"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_68582/2797381217.py:15: DtypeWarning: Columns (50,51,54,55,56,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,458) have mixed types. Specify dtype option on import or set low_memory=False.\n",
+      "  df = pd.read_csv(file_path)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing: united states\n",
+      "  ⚠️ No matches for 'united states', skipping.\n",
+      "Processing: korea\n",
+      "  ✅ Saved to data/json/tags_korea_poi.json\n",
+      "Processing: uk\n",
+      "  ✅ Saved to data/json/tags_uk_poi.json\n",
+      "Processing: russia\n",
+      "  ✅ Saved to data/json/tags_russia_poi.json\n",
+      "Processing: china\n",
+      "  ✅ Saved to data/json/tags_china_poi.json\n",
+      "Processing: canada\n",
+      "  ✅ Saved to data/json/tags_canada_poi.json\n",
+      "Processing: India\n",
+      "  ✅ Saved to data/json/tags_india_poi.json\n",
+      "Processing: germany\n",
+      "  ✅ Saved to data/json/tags_germany_poi.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from itertools import combinations\n",
+    "from collections import Counter, defaultdict\n",
+    "import json\n",
+    "import re\n",
+    "import os\n",
+    "\n",
+    "# === CONFIG ===\n",
+    "file_path = \"data/model_subsets/all_models_poi_true.csv\"\n",
+    "output_dir = \"data/json/\"\n",
+    "target_terms = [\"united states\", \"korea\", \"uk\", \"russia\", \"china\", \"canada\", \"India\", \"germany\"] # Add any tags you want to process\n",
+    "min_connections = 1  # minimum number of link connections per node\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "df = pd.read_csv(file_path)\n",
+    "tag_columns = [f\"tag_{i}\" for i in range(1, 8)]\n",
+    "df_tags = df[tag_columns]\n",
+    "\n",
+    "# === MAIN LOOP ===\n",
+    "for target_term in target_terms:\n",
+    "    print(f\"Processing: {target_term}\")\n",
+    "    \n",
+    "    pattern = re.compile(rf'\\b{re.escape(target_term)}\\b', flags=re.IGNORECASE)\n",
+    "    df_filtered = df_tags[df_tags.apply(\n",
+    "        lambda row: row.astype(str).apply(lambda x: bool(pattern.search(x))).any(),\n",
+    "        axis=1\n",
+    "    )]\n",
+    "\n",
+    "    # Skip if no data matches\n",
+    "    if df_filtered.empty:\n",
+    "        print(f\"  ⚠️ No matches for '{target_term}', skipping.\")\n",
+    "        continue\n",
+    "\n",
+    "    # === COUNT INDIVIDUAL TAGS ===\n",
+    "    all_tags = df_filtered.values.flatten()\n",
+    "    all_tags = [tag for tag in all_tags if pd.notna(tag)]\n",
+    "    tag_counts = Counter(all_tags)\n",
+    "\n",
+    "    # === CO-OCCURRENCE ===\n",
+    "    co_occurrences = defaultdict(int)\n",
+    "    for tags in df_filtered.itertuples(index=False, name=None):\n",
+    "        tags = [tag for tag in tags if pd.notna(tag)]\n",
+    "        for tag1, tag2 in combinations(tags, 2):\n",
+    "            co_occurrences[frozenset([tag1, tag2])] += 1\n",
+    "\n",
+    "    edges = [(list(pair)[0], list(pair)[1], weight) for pair, weight in co_occurrences.items()]\n",
+    "\n",
+    "    # === FILTER BY CONNECTIONS ===\n",
+    "    connected_tags = Counter()\n",
+    "    for tag1, tag2, _ in edges:\n",
+    "        connected_tags[tag1] += 1\n",
+    "        connected_tags[tag2] += 1\n",
+    "\n",
+    "    nodes = [{\"id\": tag, \"size\": tag_counts[tag]} for tag in tag_counts if connected_tags[tag] >= min_connections]\n",
+    "    valid_ids = set(node[\"id\"] for node in nodes)\n",
+    "    links = [{\"source\": tag1, \"target\": tag2, \"value\": weight}\n",
+    "             for tag1, tag2, weight in edges\n",
+    "             if tag1 in valid_ids and tag2 in valid_ids]\n",
+    "\n",
+    "    if not nodes or not links:\n",
+    "        print(f\"  ⚠️ Not enough connections for '{target_term}', skipping.\")\n",
+    "        continue\n",
+    "\n",
+    "    # === EXPORT ===\n",
+    "    d3_data = {\"nodes\": nodes, \"links\": links}\n",
+    "    safe_term = re.sub(r'\\W+', '_', target_term.lower())\n",
+    "    output_file = os.path.join(output_dir, f\"tags_{safe_term}_poi.json\")\n",
+    "    \n",
+    "    with open(output_file, \"w\") as f:\n",
+    "        json.dump(d3_data, f, indent=4)\n",
+    "    \n",
+    "    print(f\"  ✅ Saved to {output_file}\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_79893/1420003123.py:13: DtypeWarning: Columns (50,51,54,55,56,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,458) have mixed types. Specify dtype option on import or set low_memory=False.\n",
+      "  df = pd.read_csv(file_path)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ Exported 60330 nodes and 16921 links to public/json/nodes_all.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from itertools import combinations\n",
+    "from collections import Counter, defaultdict\n",
+    "import json\n",
+    "import os\n",
+    "\n",
+    "# === CONFIG ===\n",
+    "file_path = \"data/model_subsets/all_models_poi_false.csv\"\n",
+    "output_file = \"public/json/nodes_all.json\"\n",
+    "min_link_threshold = 10  # Only keep edges with co-occurrence >= this\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "df = pd.read_csv(file_path)\n",
+    "tag_columns = [f\"tag_{i}\" for i in range(1, 8)]\n",
+    "df_tags = df[tag_columns]\n",
+    "\n",
+    "# === COUNT INDIVIDUAL TAGS ===\n",
+    "all_tags = df_tags.values.flatten()\n",
+    "all_tags = [tag for tag in all_tags if pd.notna(tag)]\n",
+    "tag_counts = Counter(all_tags)\n",
+    "\n",
+    "# === CO-OCCURRENCE ===\n",
+    "co_occurrences = defaultdict(int)\n",
+    "for tags in df_tags.itertuples(index=False, name=None):\n",
+    "    tags = [tag for tag in tags if pd.notna(tag)]\n",
+    "    for tag1, tag2 in combinations(tags, 2):\n",
+    "        co_occurrences[frozenset([tag1, tag2])] += 1\n",
+    "\n",
+    "# === Build Edges (Filtered by co-occurrence threshold)\n",
+    "edges = [\n",
+    "    {\"source\": list(pair)[0], \"target\": list(pair)[1], \"value\": weight}\n",
+    "    for pair, weight in co_occurrences.items()\n",
+    "    if weight >= min_link_threshold\n",
+    "]\n",
+    "\n",
+    "# === Build Nodes (All tags that appear, regardless of links)\n",
+    "nodes = [{\"id\": tag, \"size\": tag_counts[tag]} for tag in tag_counts]\n",
+    "\n",
+    "# === EXPORT ===\n",
+    "d3_data = {\"nodes\": nodes, \"links\": edges}\n",
+    "\n",
+    "os.makedirs(os.path.dirname(output_file), exist_ok=True)\n",
+    "with open(output_file, \"w\") as f:\n",
+    "    json.dump(d3_data, f, indent=4)\n",
+    "\n",
+    "print(f\"✅ Exported {len(nodes)} nodes and {len(edges)} links to {output_file}\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "latm",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

jupyter_notebooks/Section_2-3-4_Bloomz_query.ipynb

@@ -2,414 +2,328 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "id": "3b87c378-241e-41ab-be6e-84222594f22f",
-   "metadata": {},
+   "metadata": {
+    "execution": {
+     "iopub.execute_input": "2025-11-28T13:07:47.025752Z",
+     "iopub.status.busy": "2025-11-28T13:07:47.025552Z",
+     "iopub.status.idle": "2025-11-28T13:07:47.128799Z",
+     "shell.execute_reply": "2025-11-28T13:07:47.128306Z",
+     "shell.execute_reply.started": "2025-11-28T13:07:47.025736Z"
+    }
+   },
    "outputs": [],
    "source": [
-    "import pandas as pd",
-    "import json",
-    "import time",
-    "import re",
-    "from pathlib import Path",
-    "from tqdm import tqdm",
-    "import torch",
-    "from transformers import AutoModelForCausalLM, AutoTokenizer",
-    "from datetime import datetime",
-    "",
-    "# Import is used for pd.notna() and pd.isna() checks",
-    "",
-    "current_dir = Path.cwd())",
-    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"",
-    "",
-    "# === CONFIGURATION ===",
-    "TEST_MODE = True",
-    "TEST_SIZE = 10",
-    "MAX_ROWS = 20000",
-    "SAVE_INTERVAL = 10",
-    "",
-    "# Model settings - BLOOMZ (BigScience - European consortium)",
-    "MODEL_NAME = \"bigscience/bloomz-7b1\"  # Largest instruction-tuned BLOOM model",
-    "CACHE_DIR = current_dir.parent / \"data/models\"",
-    "CACHE_DIR.mkdir(parents=True, exist_ok=True)",
-    "",
-    "PROFESSION_CATEGORIES = [",
-    "    \"actor\", \"adult performer\", \"singer/musician\", \"model\",",
-    "    \"online personality\", \"public figure\", \"voice actor/ASMR\",",
-    "    \"sports professional\", \"tv personality\"",
-    "]",
-    "",
-    "# === LOAD MODEL ===",
-    "print(f\"Loading model: {MODEL_NAME}\")",
-    "print(f\"Cache directory: {CACHE_DIR}\")",
-    "print(f\"This may take a while on first run (~14GB download)...\\n\")",
-    "",
-    "# Check GPU availability",
-    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"",
-    "print(f\"Device: {device}\")",
-    "",
-    "if device == \"cpu\":",
-    "    print(\"\u26a0\ufe0f  WARNING: No GPU detected! Inference will be VERY slow.\")",
-    "    print(\"   Consider using a GPU or reducing model size.\")",
-    "",
-    "# Load tokenizer",
-    "print(\"Loading tokenizer...\")",
-    "try:",
-    "    tokenizer = AutoTokenizer.from_pretrained(",
-    "        MODEL_NAME,",
-    "        cache_dir=str(CACHE_DIR)",
-    "    )",
-    "    print(\"\u2705 Tokenizer loaded\")",
-    "except Exception as e:",
-    "    print(f\"\u274c Error loading tokenizer: {e}\")",
-    "    raise",
-    "",
-    "# Ensure pad token is set",
-    "if tokenizer.pad_token is None:",
-    "    tokenizer.pad_token = tokenizer.eos_token",
-    "    print(f\"Set pad_token to eos_token: {tokenizer.eos_token}\")",
-    "",
-    "# Load model with optimizations",
-    "print(\"Loading model (this may take several minutes)...\")",
-    "try:",
-    "    model = AutoModelForCausalLM.from_pretrained(",
-    "        MODEL_NAME,",
-    "        cache_dir=str(CACHE_DIR),",
-    "        torch_dtype=torch.bfloat16,  # Use BF16 for efficiency",
-    "        device_map=\"auto\",  # Automatically distribute across GPUs",
-    "        low_cpu_mem_usage=True  # Optimize memory usage",
-    "    )",
-    "    model.eval()  # Set to evaluation mode",
-    "    print(\"\u2705 Model loaded\")",
-    "except Exception as e:",
-    "    print(f\"\u274c Error loading model: {e}\")",
-    "    raise",
-    "",
-    "# Check VRAM usage",
-    "if torch.cuda.is_available():",
-    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3",
-    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")",
-    "",
-    "# === LOAD DATA ===",
-    "df = pd.read_csv(input_file)",
-    "print(f\"Loaded {len(df)} rows\")",
-    "",
-    "if TEST_MODE:",
-    "    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")",
-    "    df = df.head(TEST_SIZE).copy()",
-    "elif MAX_ROWS:",
-    "    df = df.head(MAX_ROWS).copy()",
-    "",
-    "# === CREATE PROMPT (Exact DeepSeek style) ===",
-    "def create_prompt(row):",
-    "    \"\"\"Create prompt.\"\"\"",
-    "    name = row.get('real_name', row.get('name', ''))",
-    "    if pd.isna(name):",
-    "        name = row.get('name', '')",
-    "",
-    "    # Gather hints exactly like DeepSeek version",
-    "    hints = []",
-    "    if pd.notna(row.get('likely_profession')):",
-    "        hints.append(str(row['likely_profession']))",
-    "    if pd.notna(row.get('likely_nationality')):",
-    "        hints.append(str(row['likely_nationality']))",
-    "    if pd.notna(row.get('likely_country')):",
-    "        hints.append(str(row['likely_country']))",
-    "",
-    "    # Add tags if we don't have enough hints",
-    "    if len(hints) < 3:",
-    "        for i in range(1, 8):",
-    "            tag_col = f'tag_{i}'",
-    "            if tag_col in row and pd.notna(row[tag_col]):",
-    "                tag_val = str(row[tag_col])",
-    "                if tag_val not in hints:",
-    "                    hints.append(tag_val)",
-    "                if len(hints) >= 5:",
-    "                    break",
-    "",
-    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"",
-    "",
-    "    return f\"\"\"Given '{name}' ({hint_text}), provide:",
-    "1. Full legal name (Western order if non-latin script)",
-    "2. Any stage names/aliases (comma separated)",
-    "3. Gender (Male/Female/Other/Unknown)",
-    "4. Top 3 most likely professions from ONLY these categories:",
-    "   - actor",
-    "   - adult performer",
-    "   - singer/musician",
-    "   - model",
-    "   - online personality (includes streamers, cosplayers, influencers)",
-    "   - public figure (includes politicians, activists, journalists, authors)",
-    "   - voice actor/ASMR",
-    "   - sports professional",
-    "   - tv personality (includes hosts, presenters, reality TV)",
-    "",
-    "5. Primary country associated",
-    "",
-    "IMPORTANT:",
-    "- Choose professions ONLY from the 9 categories above",
-    "- Provide up to 3 professions, comma-separated, ordered by relevance",
-    "- Be SPECIFIC: choose the most accurate category for each role",
-    "- \"online personality\" includes: streamers, cosplayers, YouTubers, influencers, content creators",
-    "- Use 'Unknown' when uncertain or for fictional characters/places",
-    "- For multi-role people, list all relevant categories (e.g., \"actor, singer/musician, online personality\")",
-    "",
-    "Respond with exactly 5 numbered lines.\"\"\"",
-    "",
-    "df['prompt'] = df.apply(create_prompt, axis=1)",
-    "",
-    "# === SETUP LOGGING ===",
-    "log_dir = current_dir.parent / \"logs\"",
-    "log_dir.mkdir(parents=True, exist_ok=True)",
-    "log_file = log_dir / f\"bloomz_responses_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log\"",
-    "",
-    "def log_response(idx, name, prompt, full_prompt, raw_response, parsed_data):",
-    "    \"\"\"Log all BLOOM responses to file for debugging.\"\"\"",
-    "    with open(log_file, 'a', encoding='utf-8') as f:",
-    "        f.write(f\"{'='*80}\\n\")",
-    "        f.write(f\"INDEX: {idx}\\n\")",
-    "        f.write(f\"NAME: {name}\\n\")",
-    "        f.write(f\"TIMESTAMP: {datetime.now().isoformat()}\\n\")",
-    "        f.write(f\"\\n--- ORIGINAL PROMPT ---\\n{prompt}\\n\")",
-    "        f.write(f\"\\n--- FULL PROMPT SENT TO MODEL ---\\n{full_prompt}\\n\")",
-    "        f.write(f\"\\n--- RAW MODEL RESPONSE ---\\n{raw_response}\\n\")",
-    "        f.write(f\"\\n--- PARSED DATA ---\\n{json.dumps(parsed_data, indent=2)}\\n\")",
-    "        f.write(f\"{'='*80}\\n\\n\")",
-    "",
-    "print(f\"\ud83d\udcdd Logging BLOOM responses to: {log_file}\\n\")",
-    "",
-    "# === QUERY BLOOMZ LOCAL (IMPROVED WITH FEW-SHOT) ===",
-    "def query_bloomz_local(prompt: str) -> str:",
-    "    \"\"\"Query BLOOMZ-7B1 locally via transformers, return raw response string.",
-    "",
-    "    Uses few-shot prompting which works much better for BLOOMZ.",
-    "    \"\"\"",
-    "    try:",
-    "        # Extract name and hints from the original prompt",
-    "        # Format: \"Given 'Name' (hints), provide:\"",
-    "        name_match = re.search(r\"Given '([^']+)' \\(([^)]+)\\)\", prompt)",
-    "        if name_match:",
-    "            name = name_match.group(1)",
-    "            hints = name_match.group(2)",
-    "        else:",
-    "            name = \"Unknown\"",
-    "            hints = \"no hints\"",
-    "",
-    "        # BLOOMZ responds MUCH better to few-shot examples than long instructions",
-    "        full_prompt = f\"\"\"Extract person information in 5 numbered lines.",
-    "",
-    "Example 1:",
-    "Name: Scarlett Johansson (actor, model)",
-    "1. Scarlett Ingrid Johansson",
-    "2. ScarJo",
-    "3. Female",
-    "4. actor, model",
-    "5. United States",
-    "",
-    "Example 2:",
-    "Name: Ed Sheeran (singer, musician)",
-    "1. Edward Christopher Sheeran",
-    "2. Ed Sheeran",
-    "3. Male",
-    "4. singer/musician",
-    "5. United Kingdom",
-    "",
-    "Example 3:",
-    "Name: Amouranth (streamer, cosplayer, model)",
-    "1. Kaitlyn Siragusa",
-    "2. Amouranth",
-    "3. Female",
-    "4. online personality, model",
-    "5. United States",
-    "",
-    "Now extract for:",
-    "Name: {name} ({hints})",
-    "1.\"\"\"",
-    "",
-    "        # Store full prompt for logging",
-    "        query_bloomz_local.last_full_prompt = full_prompt",
-    "",
-    "        inputs = tokenizer(",
-    "            full_prompt,",
-    "            return_tensors=\"pt\",",
-    "            truncation=True,",
-    "            max_length=2048",
-    "        ).to(device)",
-    "",
-    "        # Generate with adjusted parameters for BLOOMZ",
-    "        with torch.no_grad():",
-    "            outputs = model.generate(",
-    "                **inputs,",
-    "                max_new_tokens=256,",
-    "                temperature=0.3,  # Increased for more variability",
-    "                do_sample=True,",
-    "                top_p=0.9,",
-    "                top_k=40,",
-    "                repetition_penalty=1.1,",
-    "                pad_token_id=tokenizer.eos_token_id,  # Use EOS as pad token",
-    "                eos_token_id=tokenizer.eos_token_id,",
-    "                early_stopping=True",
-    "            )",
-    "",
-    "        # Decode the entire output to see what's happening",
-    "        full_output = tokenizer.decode(outputs[0], skip_special_tokens=True)",
-    "",
-    "        # Extract only the generated part (after the prompt)",
-    "        generated_text = full_output[len(tokenizer.decode(inputs['input_ids'][0], skip_special_tokens=True)):]",
-    "",
-    "        # Debug output (show first 3)",
-    "        if not hasattr(query_bloomz_local, 'debug_count'):",
-    "            query_bloomz_local.debug_count = 0",
-    "",
-    "        if query_bloomz_local.debug_count < 3:",
-    "            print(f\"\\n\ud83d\udcdd BLOOMZ Debug #{query_bloomz_local.debug_count + 1}:\")",
-    "            print(f\"Input: {name} ({hints})\")",
-    "            print(f\"Generated: {generated_text[:200]}\")",
-    "            print(f\"{'='*60}\\n\")",
-    "            query_bloomz_local.debug_count += 1",
-    "",
-    "        return generated_text.strip()",
-    "",
-    "    except Exception as e:",
-    "        print(f\"Error querying BLOOMZ: {e}\")",
-    "        query_bloomz_local.last_full_prompt = f\"ERROR: {e}\"",
-    "        return None",
-    "",
-    "# === PARSE RESPONSE (Exact DeepSeek format) ===",
-    "def parse_response(response):",
-    "    \"\"\"Parse numbered response into structured fields.\"\"\"",
-    "    if not response:",
-    "        return {",
-    "            'full_name': 'Unknown',",
-    "            'aliases': 'Unknown',",
-    "            'gender': 'Unknown',",
-    "            'profession_llm': 'Unknown',",
-    "            'country': 'Unknown'",
-    "        }",
-    "",
-    "    # Split into lines and clean",
-    "    lines = [line.strip() for line in response.split('\\n') if line.strip()]",
-    "",
-    "    # Initialize with Unknown values",
-    "    fields = {",
-    "        'full_name': 'Unknown',",
-    "        'aliases': 'Unknown',",
-    "        'gender': 'Unknown',",
-    "        'profession_llm': 'Unknown',",
-    "        'country': 'Unknown'",
-    "    }",
-    "",
-    "    # Extract information from each numbered line",
-    "    for line in lines:",
-    "        if line.startswith('1.') or line.startswith('1)'):",
-    "            fields['full_name'] = line[2:].strip()",
-    "        elif line.startswith('2.') or line.startswith('2)'):",
-    "            fields['aliases'] = line[2:].strip()",
-    "        elif line.startswith('3.') or line.startswith('3)'):",
-    "            fields['gender'] = line[2:].strip()",
-    "        elif line.startswith('4.') or line.startswith('4)'):",
-    "            fields['profession_llm'] = line[2:].strip()",
-    "        elif line.startswith('5.') or line.startswith('5)'):",
-    "            fields['country'] = line[2:].strip()",
-    "",
-    "    return fields",
-    "",
-    "# === PROCESS ===",
-    "output_file = current_dir.parent / f\"data/CSV/bloomz_annotated_POI{'_test' if TEST_MODE else ''}.csv\"",
-    "index_file = current_dir.parent / \"misc/bloomz_query_index.txt\"",
-    "",
-    "current_index = 0",
-    "if index_file.exists():",
-    "    with open(index_file) as f:",
-    "        current_index = int(f.read().strip())",
-    "    print(f\"Resuming from index {current_index}\")",
-    "",
-    "# Initialize columns (same as DeepSeek)",
-    "for col in ['full_name', 'gender', 'profession_llm', 'country', 'aliases']:",
-    "    if col not in df.columns:",
-    "        df[col] = 'Unknown'",
-    "",
-    "# Create prompts for all rows (same as DeepSeek)",
-    "print(\"Creating prompts...\")",
-    "df['prompt'] = df.apply(create_prompt, axis=1)",
-    "",
-    "print(f\"\\nAnnotating with BLOOMZ-7B1 LOCAL - rows {current_index} to {len(df)}...\")",
-    "print(f\"Model: {MODEL_NAME}\")",
-    "print(f\"This may take a while...\\n\")",
-    "",
-    "try:",
-    "    start_time = time.time()",
-    "",
-    "    for i in tqdm(range(current_index, len(df)), desc=\"Annotating\"):",
-    "        row = df.iloc[i]",
-    "",
-    "        # Query BLOOMZ (equivalent to DeepSeek query)",
-    "        response = query_bloomz_local(row['prompt'])",
-    "        parsed_data = parse_response(response)",
-    "",
-    "        # Log the complete interaction for debugging",
-    "        log_response(",
-    "            idx=i,",
-    "            name=row.get('real_name', row.get('name', 'Unknown')),",
-    "            prompt=row['prompt'],",
-    "            full_prompt=query_bloomz_local.last_full_prompt if hasattr(query_bloomz_local, 'last_full_prompt') else 'N/A',",
-    "            raw_response=response if response else 'None',",
-    "            parsed_data=parsed_data",
-    "        )",
-    "",
-    "        # Update dataframe",
-    "        for key, value in parsed_data.items():",
-    "            df.at[i, key] = value",
-    "",
-    "        current_index = i + 1",
-    "",
-    "        # Save progress at intervals",
-    "        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):",
-    "            df.to_csv(output_file, index=False)",
-    "            with open(index_file, 'w') as f:",
-    "                f.write(str(current_index))",
-    "            print(f\"\u2705 Progress saved after {i+1} rows\")",
-    "",
-    "        # Optional: Add small delay to prevent overheating (not needed for rate limiting like DeepSeek)",
-    "        # time.sleep(0.1)",
-    "",
-    "    elapsed_total = time.time() - start_time",
-    "    print(f\"\\n\u2705 Done! Final results saved to {output_file}\")",
-    "",
-    "    # Summary statistics (same as DeepSeek)",
-    "    print(\"\\n=== Summary Statistics ===\")",
-    "    print(f\"Total processed: {len(df)}\")",
-    "    print(f\"\\nGender distribution:\")",
-    "    print(df['gender'].value_counts())",
-    "    print(f\"\\nTop 10 profession combinations:\")",
-    "    print(df['profession_llm'].value_counts().head(10))",
-    "    print(f\"\\nTop 10 countries:\")",
-    "    print(df['country'].value_counts().head(10))",
-    "",
-    "    # Sample results",
-    "    print(\"\\n=== Sample Results ===\")",
-    "    display_cols = ['real_name', 'full_name', 'gender', 'profession_llm', 'country']",
-    "    available_cols = [col for col in display_cols if col in df.columns]",
-    "    print(df[available_cols].head(10).to_string(index=False))",
-    "",
-    "    # Additional info for local model",
-    "    print(f\"\\nTotal time: {elapsed_total/60:.1f} minutes\")",
-    "    print(f\"Average speed: {len(df)/(elapsed_total/3600):.1f} samples/hour\")",
-    "    if torch.cuda.is_available():",
-    "        print(f\"Final VRAM usage: {torch.cuda.max_memory_allocated() / 1024**3:.2f} GB\")",
-    "",
-    "except Exception as e:",
-    "    print(f\"\u26a0\ufe0f Error encountered: {e}\")",
-    "    print(f\"\u26a0\ufe0f Last processed index: {current_index}\")",
-    "",
-    "    # Save progress before exiting",
-    "    df.to_csv(output_file, index=False)",
-    "    with open(index_file, 'w') as f:",
-    "        f.write(str(current_index))",
-    "",
-    "    print(f\"\u26a0\ufe0f Progress saved up to row {current_index}\")",
-    ""
+    "import pandas as pd\n",
+    "import json\n",
+    "import time\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "import torch\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "from datetime import datetime\n",
+    "\n",
+    "# Import is used for pd.notna() and pd.isna() checks\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "\n",
+    "# === CONFIGURATION ===\n",
+    "TEST_MODE = True\n",
+    "TEST_SIZE = 10\n",
+    "MAX_ROWS = 20000\n",
+    "SAVE_INTERVAL = 20\n",
+    "\n",
+    "# Model settings - BLOOMZ (BigScience - European consortium)\n",
+    "MODEL_NAME = \"bigscience/bloomz-7b1\"  # Largest instruction-tuned BLOOM model\n",
+    "CACHE_DIR = current_dir.parent / \"data/models\"\n",
+    "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "PROFESSION_CATEGORIES = [\n",
+    "    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n",
+    "    \"online personality\", \"public figure\", \"voice actor/ASMR\",\n",
+    "    \"sports professional\", \"tv personality\"\n",
+    "]\n",
+    "\n",
+    "# === LOAD MODEL ===\n",
+    "print(f\"Loading model: {MODEL_NAME}\")\n",
+    "print(f\"Cache directory: {CACHE_DIR}\")\n",
+    "print(f\"This may take a while on first run (~14GB download)...\\n\")\n",
+    "\n",
+    "# Check GPU availability\n",
+    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "print(f\"Device: {device}\")\n",
+    "\n",
+    "if device == \"cpu\":\n",
+    "    print(\"⚠️  WARNING: No GPU detected! Inference will be VERY slow.\")\n",
+    "    print(\"   Consider using a GPU or reducing model size.\")\n",
+    "\n",
+    "# Load tokenizer\n",
+    "print(\"Loading tokenizer...\")\n",
+    "try:\n",
+    "    tokenizer = AutoTokenizer.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR)\n",
+    "    )\n",
+    "    print(\"✅ Tokenizer loaded\")\n",
+    "except Exception as e:\n",
+    "    print(f\"❌ Error loading tokenizer: {e}\")\n",
+    "    raise\n",
+    "\n",
+    "# Ensure pad token is set\n",
+    "if tokenizer.pad_token is None:\n",
+    "    tokenizer.pad_token = tokenizer.eos_token\n",
+    "    print(f\"Set pad_token to eos_token: {tokenizer.eos_token}\")\n",
+    "\n",
+    "# Load model with optimizations\n",
+    "print(\"Loading model (this may take several minutes)...\")\n",
+    "try:\n",
+    "    model = AutoModelForCausalLM.from_pretrained(\n",
+    "        MODEL_NAME,\n",
+    "        cache_dir=str(CACHE_DIR),\n",
+    "        torch_dtype=torch.bfloat16,  # Use BF16 for efficiency\n",
+    "        device_map=\"auto\",  # Automatically distribute across GPUs\n",
+    "        low_cpu_mem_usage=True  # Optimize memory usage\n",
+    "    )\n",
+    "    model.eval()  # Set to evaluation mode\n",
+    "    print(\"✅ Model loaded\")\n",
+    "except Exception as e:\n",
+    "    print(f\"❌ Error loading model: {e}\")\n",
+    "    raise\n",
+    "\n",
+    "# Check VRAM usage\n",
+    "if torch.cuda.is_available():\n",
+    "    vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
+    "    print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
+    "\n",
+    "# === LOAD DATA ===\n",
+    "df = pd.read_csv(input_file)\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "\n",
+    "if TEST_MODE:\n",
+    "    print(f\"Running in TEST MODE with {TEST_SIZE} samples\")\n",
+    "    df = df.head(TEST_SIZE).copy()\n",
+    "elif MAX_ROWS:\n",
+    "    df = df.head(MAX_ROWS).copy()\n",
+    "\n",
+    "# === CREATE PROMPT (Exact DeepSeek style) ===\n",
+    "# === CREATE PROMPT (BLOOMZ-optimized - Complete format) ===\n",
+    "def create_prompt(row):\n",
+    "    \"\"\"Create prompt optimized for BLOOMZ with complete examples.\"\"\"\n",
+    "    name = row.get('real_name', row.get('name', ''))\n",
+    "    if pd.isna(name):\n",
+    "        name = row.get('name', '')\n",
+    "\n",
+    "    # Gather hints\n",
+    "    hints = []\n",
+    "    if pd.notna(row.get('likely_profession')):\n",
+    "        hints.append(str(row['likely_profession']))\n",
+    "    if pd.notna(row.get('likely_nationality')):\n",
+    "        hints.append(str(row['likely_nationality']))\n",
+    "    if pd.notna(row.get('likely_country')):\n",
+    "        hints.append(str(row['likely_country']))\n",
+    "\n",
+    "    # Add tags if we don't have enough hints\n",
+    "    if len(hints) < 3:\n",
+    "        for i in range(1, 8):\n",
+    "            tag_col = f'tag_{i}'\n",
+    "            if tag_col in row and pd.notna(row[tag_col]):\n",
+    "                tag_val = str(row[tag_col])\n",
+    "                if tag_val not in hints:\n",
+    "                    hints.append(tag_val)\n",
+    "                if len(hints) >= 5:\n",
+    "                    break\n",
+    "\n",
+    "    hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
+    "\n",
+    "    # BLOOMZ needs complete format shown - don't end with \"1.\"\n",
+    "    return f\"\"\"Task: Extract person information in 5 numbered lines.\n",
+    "\n",
+    "Format:\n",
+    "1. Full legal name\n",
+    "2. Stage names/aliases\n",
+    "3. Gender\n",
+    "4. Professions (from: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality)\n",
+    "5. Country\n",
+    "\n",
+    "Example:\n",
+    "Person: Taylor Swift (singer, American, pop music)\n",
+    "Answer:\n",
+    "1. Taylor Alison Swift\n",
+    "2. Taylor Swift\n",
+    "3. Female\n",
+    "4. singer/musician, online personality\n",
+    "5. United States\n",
+    "\n",
+    "Person: {name} ({hint_text})\n",
+    "Answer:\n",
+    "\"\"\"\n",
+    "\n",
+    "# === QUERY BLOOMZ LOCAL (Fixed generation parameters) ===\n",
+    "def query_bloomz_local(prompt: str) -> str:\n",
+    "    \"\"\"Query BLOOMZ-7B1 locally via transformers, return raw response string.\"\"\"\n",
+    "    try:\n",
+    "        # Store full prompt for logging\n",
+    "        query_bloomz_local.last_full_prompt = prompt\n",
+    "\n",
+    "        # Tokenize WITHOUT truncation\n",
+    "        inputs = tokenizer(\n",
+    "            prompt,\n",
+    "            return_tensors=\"pt\",\n",
+    "            truncation=False\n",
+    "        ).to(device)\n",
+    "        \n",
+    "        # Store the input length for proper extraction later\n",
+    "        input_length = inputs['input_ids'].shape[1]\n",
+    "\n",
+    "        # Generate with parameters optimized for BLOOMZ completion\n",
+    "        with torch.no_grad():\n",
+    "            outputs = model.generate(\n",
+    "                **inputs,\n",
+    "                max_new_tokens=300,  # Increased to allow full completion\n",
+    "                min_new_tokens=50,   # Force minimum generation length\n",
+    "                temperature=0.5,     # Balanced creativity\n",
+    "                do_sample=True,\n",
+    "                top_p=0.9,\n",
+    "                top_k=50,\n",
+    "                repetition_penalty=1.15,\n",
+    "                pad_token_id=tokenizer.eos_token_id,\n",
+    "                eos_token_id=tokenizer.eos_token_id,\n",
+    "                early_stopping=False,  # Changed to False - don't stop early!\n",
+    "                num_beams=1  # Greedy-like but with sampling\n",
+    "            )\n",
+    "\n",
+    "        # Extract only the NEW generated tokens\n",
+    "        generated_ids = outputs[0][input_length:]\n",
+    "        generated_text = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
+    "\n",
+    "        # Debug output (show first 5 now to see more patterns)\n",
+    "        if not hasattr(query_bloomz_local, 'debug_count'):\n",
+    "            query_bloomz_local.debug_count = 0\n",
+    "\n",
+    "        if query_bloomz_local.debug_count < 5:\n",
+    "            print(f\"\\n📝 BLOOMZ Debug #{query_bloomz_local.debug_count + 1}:\")\n",
+    "            print(f\"Prompt tokens: {input_length}\")\n",
+    "            print(f\"Generated tokens: {len(generated_ids)}\")\n",
+    "            print(f\"Full generation:\\n{generated_text}\")\n",
+    "            print(f\"{'='*60}\\n\")\n",
+    "            query_bloomz_local.debug_count += 1\n",
+    "\n",
+    "        return generated_text.strip()\n",
+    "\n",
+    "    except Exception as e:\n",
+    "        print(f\"Error querying BLOOMZ: {e}\")\n",
+    "        import traceback\n",
+    "        traceback.print_exc()\n",
+    "        query_bloomz_local.last_full_prompt = f\"ERROR: {e}\"\n",
+    "        return None\n",
+    "\n",
+    "        \n",
+    "# === PROCESS ===\n",
+    "output_file = current_dir.parent / f\"data/CSV/bloomz_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
+    "index_file = current_dir.parent / \"misc/bloomz_query_index.txt\"\n",
+    "\n",
+    "current_index = 0\n",
+    "if index_file.exists():\n",
+    "    with open(index_file) as f:\n",
+    "        current_index = int(f.read().strip())\n",
+    "    print(f\"Resuming from index {current_index}\")\n",
+    "\n",
+    "# Initialize columns (same as DeepSeek)\n",
+    "for col in ['full_name', 'gender', 'profession_llm', 'country', 'aliases']:\n",
+    "    if col not in df.columns:\n",
+    "        df[col] = 'Unknown'\n",
+    "\n",
+    "# Create prompts for all rows (same as DeepSeek)\n",
+    "print(\"Creating prompts...\")\n",
+    "df['prompt'] = df.apply(create_prompt, axis=1)\n",
+    "\n",
+    "print(f\"\\nAnnotating with BLOOMZ-7B1 LOCAL - rows {current_index} to {len(df)}...\")\n",
+    "print(f\"Model: {MODEL_NAME}\")\n",
+    "print(f\"This may take a while...\\n\")\n",
+    "\n",
+    "try:\n",
+    "    start_time = time.time()\n",
+    "\n",
+    "    for i in tqdm(range(current_index, len(df)), desc=\"Annotating\"):\n",
+    "        row = df.iloc[i]\n",
+    "\n",
+    "        # Query BLOOMZ (equivalent to DeepSeek query)\n",
+    "        response = query_bloomz_local(row['prompt'])\n",
+    "        parsed_data = parse_response(response)\n",
+    "\n",
+    "        # Log the complete interaction for debugging\n",
+    "        log_response(\n",
+    "            idx=i,\n",
+    "            name=row.get('real_name', row.get('name', 'Unknown')),\n",
+    "            prompt=row['prompt'],\n",
+    "            full_prompt=query_bloomz_local.last_full_prompt if hasattr(query_bloomz_local, 'last_full_prompt') else 'N/A',\n",
+    "            raw_response=response if response else 'None',\n",
+    "            parsed_data=parsed_data\n",
+    "        )\n",
+    "\n",
+    "        # Update dataframe\n",
+    "        for key, value in parsed_data.items():\n",
+    "            df.at[i, key] = value\n",
+    "\n",
+    "        current_index = i + 1\n",
+    "\n",
+    "        # Save progress at intervals\n",
+    "        if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
+    "            df.to_csv(output_file, index=False)\n",
+    "            with open(index_file, 'w') as f:\n",
+    "                f.write(str(current_index))\n",
+    "            print(f\"✅ Progress saved after {i+1} rows\")\n",
+    "\n",
+    "        # Optional: Add small delay to prevent overheating (not needed for rate limiting like DeepSeek)\n",
+    "        # time.sleep(0.1)\n",
+    "\n",
+    "    elapsed_total = time.time() - start_time\n",
+    "    print(f\"\\n✅ Done! Final results saved to {output_file}\")\n",
+    "\n",
+    "    # Summary statistics (same as DeepSeek)\n",
+    "    print(\"\\n=== Summary Statistics ===\")\n",
+    "    print(f\"Total processed: {len(df)}\")\n",
+    "    print(f\"\\nGender distribution:\")\n",
+    "    print(df['gender'].value_counts())\n",
+    "    print(f\"\\nTop 10 profession combinations:\")\n",
+    "    print(df['profession_llm'].value_counts().head(10))\n",
+    "    print(f\"\\nTop 10 countries:\")\n",
+    "    print(df['country'].value_counts().head(10))\n",
+    "\n",
+    "    # Sample results\n",
+    "    print(\"\\n=== Sample Results ===\")\n",
+    "    display_cols = ['real_name', 'full_name', 'gender', 'profession_llm', 'country']\n",
+    "    available_cols = [col for col in display_cols if col in df.columns]\n",
+    "    print(df[available_cols].head(10).to_string(index=False))\n",
+    "\n",
+    "    # Additional info for local model\n",
+    "    print(f\"\\nTotal time: {elapsed_total/60:.1f} minutes\")\n",
+    "    print(f\"Average speed: {len(df)/(elapsed_total/3600):.1f} samples/hour\")\n",
+    "    if torch.cuda.is_available():\n",
+    "        print(f\"Final VRAM usage: {torch.cuda.max_memory_allocated() / 1024**3:.2f} GB\")\n",
+    "\n",
+    "except Exception as e:\n",
+    "    print(f\"⚠️ Error encountered: {e}\")\n",
+    "    print(f\"⚠️ Last processed index: {current_index}\")\n",
+    "\n",
+    "    # Save progress before exiting\n",
+    "    df.to_csv(output_file, index=False)\n",
+    "    with open(index_file, 'w') as f:\n",
+    "        f.write(str(current_index))\n",
+    "\n",
+    "    print(f\"⚠️ Progress saved up to row {current_index}\")\n"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c458d50f-0cb3-421e-99ab-62795f88242c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -433,4 +347,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

jupyter_notebooks/Section_2-3_Figure_5_co-occurence_promotional_tags.ipynb

@@ -292,7 +292,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "latm",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -306,9 +306,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.12.10"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

misc/bloomz_query_index.txt

@@ -1 +0,0 @@
-10

misc/query_indicies/deepseek_query_index.txt

@@ -1 +0,0 @@
-10

misc/query_indicies/mistral_local_query_index.txt

@@ -0,0 +1 @@
+8790