Hugging Face's logo

latentcanon
/
code

Model card Files
xet
 Community
laura.wagner commited on
Commit
b5829da
·
1 Parent(s): 04a2e0a

added llm results comparison

Browse files
Files changed (12) hide show
  1. jupyter_notebooks/.ipynb_checkpoints/GEMMA_3-checkpoint.ipynb +396 -2
  2. jupyter_notebooks/.ipynb_checkpoints/QWEN-checkpoint.ipynb +290 -110
  3. jupyter_notebooks/.ipynb_checkpoints/Section_2-3-4_compare-models-checkpoint.ipynb +237 -0
  4. jupyter_notebooks/GEMMA_3.ipynb +0 -0
  5. jupyter_notebooks/MISTRAL.ipynb +0 -0
  6. jupyter_notebooks/QWEN.ipynb +0 -0
  7. jupyter_notebooks/Section_2-3-4_Figure_8_deepfake_adapters.ipynb +3 -3
  8. jupyter_notebooks/Section_2-3-4_compare-models.ipynb +1680 -0
  9. misc/query_indicies/.ipynb_checkpoints/gemma_local_query_index-checkpoint.txt +1 -0
  10. misc/query_indicies/gemma_local_query_index.txt +1 -1
  11. misc/query_indicies/mistral_local_query_index.txt +1 -1
  12. misc/query_indicies/qwen_local_query_index.txt +1 -0
jupyter_notebooks/.ipynb_checkpoints/GEMMA_3-checkpoint.ipynb CHANGED
@@ -1,6 +1,400 @@
1
  {
2
- "cells": [],
3
- "metadata": {},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
  "nbformat": 4,
5
  "nbformat_minor": 5
6
  }
 
1
  {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": null,
6
+ "id": "471e0cef-678e-4403-8eca-f8e1991d86de",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "import pandas as pd\n",
11
+ "import json\n",
12
+ "import time\n",
13
+ "import re\n",
14
+ "from pathlib import Path\n",
15
+ "from tqdm import tqdm\n",
16
+ "import torch\n",
17
+ "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
18
+ "\n",
19
+ "current_dir = Path.cwd()\n",
20
+ "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
21
+ "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
22
+ "professions_mapped_file = current_dir.parent / \"misc/lists/professions_mapped.csv\"\n",
23
+ "# === PROCESS DATA ===\n",
24
+ "\n",
25
+ "\n",
26
+ "# === CONFIGURATION ===\n",
27
+ "TEST_MODE = False\n",
28
+ "TEST_SIZE = 100\n",
29
+ "MAX_ROWS = 50862\n",
30
+ "SAVE_INTERVAL = 10\n",
31
+ "\n",
32
+ "output_file = current_dir.parent / f\"data/CSV/gemma_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
33
+ "index_file = current_dir.parent / \"misc/query_indicies/gemma_local_query_index.txt\"\n",
34
+ "\n",
35
+ "# Model settings\n",
36
+ "MODEL_NAME = MODEL_NAME = \"google/gemma-3-27b-it\"\n",
37
+ "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
38
+ "CACHE_DIR = current_dir.parent / \"data/models\"\n",
39
+ "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
40
+ "\n",
41
+ "# Define the SPECIFIC profession categories\n",
42
+ "PROFESSION_CATEGORIES = [\n",
43
+ " \"actor\",\n",
44
+ " \"adult performer\",\n",
45
+ " \"singer/musician\",\n",
46
+ " \"model\",\n",
47
+ " \"online personality\",\n",
48
+ " \"public figure\",\n",
49
+ " \"voice actor/ASMR\",\n",
50
+ " \"sports professional\",\n",
51
+ " \"tv personality\"\n",
52
+ "]\n",
53
+ "\n",
54
+ "# === LOAD MODEL ===\n",
55
+ "print(f\"Loading model: {MODEL_NAME}\")\n",
56
+ "print(f\"Cache directory: {CACHE_DIR}\")\n",
57
+ "print(f\"This may take a while on first run (~65GB download)...\\n\")\n",
58
+ "\n",
59
+ "# Check GPU availability\n",
60
+ "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
61
+ "print(f\"Device: {device}\")\n",
62
+ "\n",
63
+ "if device == \"cpu\":\n",
64
+ " print(\"⚠️ WARNING: No GPU detected! Inference will be VERY slow.\")\n",
65
+ " print(\" Consider using a GPU or reducing model size.\")\n",
66
+ "\n",
67
+ "# Load tokenizer\n",
68
+ "print(\"Loading tokenizer...\")\n",
69
+ "try:\n",
70
+ " tokenizer = AutoTokenizer.from_pretrained(\n",
71
+ " MODEL_NAME,\n",
72
+ " cache_dir=str(CACHE_DIR),\n",
73
+ " use_fast=True\n",
74
+ " )\n",
75
+ "except Exception as e:\n",
76
+ " print(f\"Failed with use_fast=True, trying use_fast=False...\")\n",
77
+ " tokenizer = AutoTokenizer.from_pretrained(\n",
78
+ " MODEL_NAME,\n",
79
+ " cache_dir=str(CACHE_DIR),\n",
80
+ " use_fast=False\n",
81
+ " )\n",
82
+ "\n",
83
+ "# Ensure pad token is set\n",
84
+ "if tokenizer.pad_token is None:\n",
85
+ " tokenizer.pad_token = tokenizer.eos_token\n",
86
+ "\n",
87
+ "print(\"✅ Tokenizer loaded\")\n",
88
+ "\n",
89
+ "# Load model with optimizations\n",
90
+ "print(\"Loading model (this may take several minutes)...\")\n",
91
+ "model = AutoModelForCausalLM.from_pretrained(\n",
92
+ " MODEL_NAME,\n",
93
+ " cache_dir=str(CACHE_DIR),\n",
94
+ " torch_dtype=torch.bfloat16,\n",
95
+ " device_map=\"auto\",\n",
96
+ " trust_remote_code=False\n",
97
+ ")\n",
98
+ "model.eval()\n",
99
+ "print(\"✅ Model loaded\")\n",
100
+ "\n",
101
+ "# Check VRAM usage\n",
102
+ "if torch.cuda.is_available():\n",
103
+ " vram_gb = torch.cuda.max_memory_allocated() / 1024**3\n",
104
+ " print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
105
+ "\n",
106
+ "# === LOAD DATA ===\n",
107
+ "print(\"Loading raw input CSV...\")\n",
108
+ "df = pd.read_csv(input_file) # ALWAYS load the full input\n",
109
+ "print(f\"Loaded {len(df)} rows from input file\")\n",
110
+ "\n",
111
+ "# If we have previous annotations, merge them\n",
112
+ "if output_file.exists():\n",
113
+ " print(\"Found existing annotations, merging...\")\n",
114
+ " existing_df = pd.read_csv(output_file)\n",
115
+ " print(f\"Existing annotations has {len(existing_df)} rows\")\n",
116
+ " \n",
117
+ " # Update df with existing annotations\n",
118
+ " # Only update the columns that were annotated\n",
119
+ " annotation_cols = ['full_name', 'aliases', 'gender', 'profession_llm', 'country']\n",
120
+ " for col in annotation_cols:\n",
121
+ " if col in existing_df.columns:\n",
122
+ " df[col] = existing_df[col][:len(df)] # Make sure we don't exceed df length\n",
123
+ " \n",
124
+ " print(f\"Merged annotations, continuing with {len(df)} total rows\")\n",
125
+ "\n",
126
+ "\n",
127
+ "# Try to load profession mapping files\n",
128
+ "try:\n",
129
+ " professions_df = pd.read_csv(professions_file)\n",
130
+ " print(f\"✅ Loaded professions.csv\")\n",
131
+ "except:\n",
132
+ " print(\"⚠️ Warning: professions.csv not found\")\n",
133
+ "\n",
134
+ "try:\n",
135
+ " prof_mapped_df = pd.read_csv(professions_mapped_file)\n",
136
+ " print(f\"✅ Loaded profession mapping with {len(prof_mapped_df)} categories\")\n",
137
+ "except:\n",
138
+ " print(\"⚠️ Warning: professions_mapped.csv not found, using default categories\")\n",
139
+ "\n",
140
+ "profession_str = \", \".join(PROFESSION_CATEGORIES)\n",
141
+ "\n",
142
+ "print(f\"Loaded {len(df)} rows\")\n",
143
+ "print(f\"\\nProfession categories ({len(PROFESSION_CATEGORIES)}):\")\n",
144
+ "for cat in PROFESSION_CATEGORIES:\n",
145
+ " print(f\" - {cat}\")\n",
146
+ "\n",
147
+ "if TEST_MODE:\n",
148
+ " print(f\"\\nRunning in TEST MODE with {TEST_SIZE} samples\")\n",
149
+ " df = df.head(TEST_SIZE).copy()\n",
150
+ "elif MAX_ROWS:\n",
151
+ " df = df.head(MAX_ROWS).copy()\n",
152
+ "\n",
153
+ "# === CREATE PROMPTS ===\n",
154
+ "def create_prompt(row):\n",
155
+ " \"\"\"Create prompt for Gemma annotation with specific profession categories.\"\"\"\n",
156
+ " name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
157
+ " \n",
158
+ " # Gather hints\n",
159
+ " hints = []\n",
160
+ " if pd.notna(row.get('likely_profession')):\n",
161
+ " hints.append(str(row['likely_profession']))\n",
162
+ " if pd.notna(row.get('likely_nationality')):\n",
163
+ " hints.append(str(row['likely_nationality']))\n",
164
+ " if pd.notna(row.get('likely_country')):\n",
165
+ " hints.append(str(row['likely_country']))\n",
166
+ " \n",
167
+ " # Add tags if we don't have enough hints\n",
168
+ " if len(hints) < 3:\n",
169
+ " for i in range(1, 8):\n",
170
+ " tag_col = f'tag_{i}'\n",
171
+ " if tag_col in row and pd.notna(row[tag_col]):\n",
172
+ " tag_val = str(row[tag_col])\n",
173
+ " if tag_val not in hints:\n",
174
+ " hints.append(tag_val)\n",
175
+ " if len(hints) >= 5:\n",
176
+ " break\n",
177
+ " \n",
178
+ " hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
179
+ " \n",
180
+ " return f\"\"\"Given '{name}' ({hint_text}), provide:\n",
181
+ "1. Full legal name (Western order if non-latin script)\n",
182
+ "2. Any stage names/aliases (comma separated)\n",
183
+ "3. Gender (Male/Female/Other/Unknown)\n",
184
+ "4. Top 3 most likely professions from ONLY these categories:\n",
185
+ " - actor\n",
186
+ " - adult performer\n",
187
+ " - singer/musician\n",
188
+ " - model\n",
189
+ " - online personality (includes streamers, cosplayers, influencers)\n",
190
+ " - public figure (includes politicians, activists, journalists, authors)\n",
191
+ " - voice actor/ASMR\n",
192
+ " - sports professional\n",
193
+ " - tv personality (includes hosts, presenters, reality TV)\n",
194
+ "\n",
195
+ "5. Primary country associated\n",
196
+ "\n",
197
+ "IMPORTANT:\n",
198
+ "- Choose professions ONLY from the 9 categories above\n",
199
+ "- Provide up to 3 professions, comma-separated, ordered by relevance\n",
200
+ "- Be SPECIFIC: choose the most accurate category for each role\n",
201
+ "- \"online personality\" includes: streamers, cosplayers, YouTubers, influencers, content creators\n",
202
+ "- Use 'Unknown' when uncertain or for fictional characters/places\n",
203
+ "- For multi-role people, list all relevant categories (e.g., \"actor, singer/musician, online personality\")\n",
204
+ "- For country respond with one word only, for example China or Columbia\n",
205
+ "- actress = actor\n",
206
+ "\n",
207
+ "Respond with exactly 5 numbered lines.\"\"\"\n",
208
+ "\n",
209
+ "# Create prompts\n",
210
+ "print(\"\\nCreating prompts...\")\n",
211
+ "df['prompt'] = df.apply(create_prompt, axis=1)\n",
212
+ "print(\"✅ Prompts created\")\n",
213
+ "\n",
214
+ "# === QUERY Gemma LOCAL ===\n",
215
+ "def query_gemma_local(prompt: str) -> str:\n",
216
+ " \"\"\"Query Gemma locally via transformers.\"\"\"\n",
217
+ " try:\n",
218
+ " # Format as chat message for GEMMA\n",
219
+ " messages = [\n",
220
+ " {\"role\": \"system\", \"content\": \"You are an assistant that extracts key data on a person based on the name. Respond with exactly 5 numbered lines. For professions, choose ONLY from these categories: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality.\"},\n",
221
+ " {\"role\": \"user\", \"content\": prompt}\n",
222
+ " ]\n",
223
+ " \n",
224
+ " # Tokenize\n",
225
+ " if hasattr(tokenizer, 'apply_chat_template'):\n",
226
+ " text = tokenizer.apply_chat_template(\n",
227
+ " messages,\n",
228
+ " tokenize=False,\n",
229
+ " add_generation_prompt=True\n",
230
+ " )\n",
231
+ " else:\n",
232
+ " # Fallback for older tokenizers\n",
233
+ " text = f\"[INST] {prompt} [/INST]\"\n",
234
+ " \n",
235
+ " inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
236
+ " \n",
237
+ " # Generate\n",
238
+ " with torch.no_grad():\n",
239
+ " outputs = model.generate(\n",
240
+ " **inputs,\n",
241
+ " max_new_tokens=512,\n",
242
+ " temperature=0.1,\n",
243
+ " do_sample=True,\n",
244
+ " top_p=0.9,\n",
245
+ " pad_token_id=tokenizer.eos_token_id\n",
246
+ " )\n",
247
+ " \n",
248
+ " # Decode\n",
249
+ " generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
250
+ " response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
251
+ " \n",
252
+ " return response.strip()\n",
253
+ " \n",
254
+ " except Exception as e:\n",
255
+ " print(f\"Generation error: {e}\")\n",
256
+ " return None\n",
257
+ "\n",
258
+ "output_file = current_dir.parent / f\"data/CSV/gemma_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
259
+ "index_file = current_dir.parent / \"misc/query_indicies/gemma_local_query_index.txt\"\n",
260
+ "\n",
261
+ "\n",
262
+ "# === PARSE RESPONSE ===\n",
263
+ "def parse_response(response):\n",
264
+ " \"\"\"Parse Gemma response into structured fields.\"\"\"\n",
265
+ " if not response:\n",
266
+ " return {\n",
267
+ " 'full_name': 'Unknown',\n",
268
+ " 'aliases': 'Unknown',\n",
269
+ " 'gender': 'Unknown',\n",
270
+ " 'profession_llm': 'Unknown',\n",
271
+ " 'country': 'Unknown'\n",
272
+ " }\n",
273
+ " \n",
274
+ " # Split into lines and clean\n",
275
+ " lines = [line.strip() for line in response.split('\\n') if line.strip()]\n",
276
+ " \n",
277
+ " # Initialize with Unknown values\n",
278
+ " fields = {\n",
279
+ " 'full_name': 'Unknown',\n",
280
+ " 'aliases': 'Unknown',\n",
281
+ " 'gender': 'Unknown',\n",
282
+ " 'profession_llm': 'Unknown',\n",
283
+ " 'country': 'Unknown'\n",
284
+ " }\n",
285
+ " \n",
286
+ " # Extract information from each numbered line\n",
287
+ " for line in lines:\n",
288
+ " if line.startswith('1.'):\n",
289
+ " fields['full_name'] = line[2:].strip()\n",
290
+ " elif line.startswith('2.'):\n",
291
+ " fields['aliases'] = line[2:].strip()\n",
292
+ " elif line.startswith('3.'):\n",
293
+ " fields['gender'] = line[2:].strip()\n",
294
+ " elif line.startswith('4.'):\n",
295
+ " fields['profession_llm'] = line[2:].strip()\n",
296
+ " elif line.startswith('5.'):\n",
297
+ " fields['country'] = line[2:].strip()\n",
298
+ " \n",
299
+ " return fields\n",
300
+ "\n",
301
+ "\n",
302
+ "# === PROCESS DATA ===\n",
303
+ "index_file.parent.mkdir(parents=True, exist_ok=True)\n",
304
+ "\n",
305
+ "# Load index\n",
306
+ "current_index = 0\n",
307
+ "if index_file.exists():\n",
308
+ " try:\n",
309
+ " current_index = int(index_file.read_text().strip())\n",
310
+ " except:\n",
311
+ " current_index = 0\n",
312
+ "\n",
313
+ "print(f\"Resuming from index {current_index}\")\n",
314
+ "\n",
315
+ "start_time = time.time()\n",
316
+ "\n",
317
+ "for i in tqdm(range(current_index, len(df)), desc=\"Gemma Local\"):\n",
318
+ "\n",
319
+ " prompt = df.at[i, \"prompt\"]\n",
320
+ "\n",
321
+ " # -------- MODEL QUERY WITH RETRIES --------\n",
322
+ " response = None\n",
323
+ " for attempt in range(3):\n",
324
+ " response = query_gemma_local(prompt)\n",
325
+ " \n",
326
+ " # Valid response?\n",
327
+ " if response and len(response.strip()) > 10:\n",
328
+ " break\n",
329
+ " \n",
330
+ " print(f\"⚠️ Row {i}: Empty or invalid response, retry {attempt+1}/3\")\n",
331
+ " time.sleep(0.5)\n",
332
+ "\n",
333
+ " # If still invalid → DO NOT overwrite previous data\n",
334
+ " if not response or len(response.strip()) <= 10:\n",
335
+ " print(f\"❌ Row {i}: failed after retries, not writing, not advancing index\")\n",
336
+ " continue\n",
337
+ "\n",
338
+ " parsed = parse_response(response)\n",
339
+ "\n",
340
+ " # Additional safety: skip rows that parsed as all 'Unknown'\n",
341
+ " if all(v == \"Unknown\" for v in parsed.values()):\n",
342
+ " print(f\"❌ Row {i}: parsed as all Unknown (likely model crash); skipping.\")\n",
343
+ " continue\n",
344
+ "\n",
345
+ " # -------- WRITE PARSED FIELDS SAFELY --------\n",
346
+ " for key, value in parsed.items():\n",
347
+ " df.at[i, key] = value\n",
348
+ "\n",
349
+ " # Advance progress ONLY after successful write\n",
350
+ " current_index = i + 1\n",
351
+ "\n",
352
+ " # -------- GPU MEMORY CLEANUP --------\n",
353
+ " if torch.cuda.is_available():\n",
354
+ " torch.cuda.empty_cache()\n",
355
+ " torch.cuda.synchronize()\n",
356
+ "\n",
357
+ " # -------- SAVE LIKE YOUR DEEPSEEK VERSION --------\n",
358
+ " if (i + 1) % SAVE_INTERVAL == 0 or (i + 1) == len(df):\n",
359
+ " df.to_csv(output_file, index=False)\n",
360
+ " with open(index_file, \"w\") as f:\n",
361
+ " f.write(str(current_index))\n",
362
+ " print(f\"💾 Progress saved after row {i+1}\")\n",
363
+ "\n",
364
+ "# Final save\n",
365
+ "df.to_csv(output_file, index=False)\n",
366
+ "index_file.write_text(str(current_index))\n",
367
+ "print(\"✅ Finished full dataset.\")\n"
368
+ ]
369
+ },
370
+ {
371
+ "cell_type": "code",
372
+ "execution_count": null,
373
+ "id": "7a1be7c0-ce54-4445-8534-bf3ab5e70197",
374
+ "metadata": {},
375
+ "outputs": [],
376
+ "source": []
377
+ }
378
+ ],
379
+ "metadata": {
380
+ "kernelspec": {
381
+ "display_name": "pm-paper",
382
+ "language": "python",
383
+ "name": "pm-paper"
384
+ },
385
+ "language_info": {
386
+ "codemirror_mode": {
387
+ "name": "ipython",
388
+ "version": 3
389
+ },
390
+ "file_extension": ".py",
391
+ "mimetype": "text/x-python",
392
+ "name": "python",
393
+ "nbconvert_exporter": "python",
394
+ "pygments_lexer": "ipython3",
395
+ "version": "3.11.13"
396
+ }
397
+ },
398
  "nbformat": 4,
399
  "nbformat_minor": 5
400
  }
jupyter_notebooks/.ipynb_checkpoints/QWEN-checkpoint.ipynb CHANGED
@@ -5,15 +5,230 @@
5
  "id": "0543d9c4-055b-49a3-a8d0-9dfb622b2b8c",
6
  "metadata": {},
7
  "source": [
8
- "# QWEN CODE"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  ]
10
  },
11
  {
12
  "cell_type": "code",
13
- "execution_count": null,
14
  "id": "fe6ba282-896b-4272-b82b-ef24810732fb",
15
- "metadata": {},
16
- "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  "source": [
18
  "import pandas as pd\n",
19
  "import json\n",
@@ -22,7 +237,7 @@
22
  "from pathlib import Path\n",
23
  "from tqdm import tqdm\n",
24
  "import torch\n",
25
- "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
26
  "import signal\n",
27
  "from contextlib import contextmanager\n",
28
  "\n",
@@ -44,7 +259,9 @@
44
  "output_file = current_dir.parent / f\"data/CSV/qwen_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
45
  "\n",
46
  "# Model settings\n",
47
- "MODEL_NAME = \"Qwen/Qwen3-235B-A22B-Instruct-2507-FP8\"\n",
 
 
48
  "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
49
  "CACHE_DIR = current_dir.parent / \"data/models\"\n",
50
  "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
@@ -97,49 +314,25 @@
97
  "\n",
98
  "print(\"✅ Tokenizer loaded\")\n",
99
  "\n",
100
- "# Load model with optimizations\n",
101
- "print(\"Loading model (this may take several minutes)...\")\n",
102
- "model = AutoModelForCausalLM.from_pretrained(\n",
103
- " MODEL_NAME,\n",
104
- " cache_dir=str(CACHE_DIR),\n",
105
- " torch_dtype=torch.bfloat16,\n",
106
- " device_map=\"cuda:0\", # Force single GPU instead of \"auto\"\n",
107
- " trust_remote_code=False,\n",
108
- " offload_folder=str(CACHE_DIR / \"offload\"), # Explicit offload folder\n",
109
- " offload_state_dict=False # Try to keep everything on GPU\n",
110
  ")\n",
111
- "model.eval()\n",
112
- "print(\"✅ Model loaded\")\n",
113
  "\n",
114
- "# Load model with optimizations\n",
115
- "print(\"Loading model (this may take several minutes)...\")\n",
116
  "model = AutoModelForCausalLM.from_pretrained(\n",
117
  " MODEL_NAME,\n",
118
  " cache_dir=str(CACHE_DIR),\n",
119
- " torch_dtype=torch.bfloat16,\n",
120
  " device_map=\"auto\",\n",
121
  " trust_remote_code=False\n",
122
  ")\n",
123
  "model.eval()\n",
124
- "\n",
125
- "# FORCE COMPLETE LOADING\n",
126
- "print(\"Ensuring model is fully loaded to device...\")\n",
127
- "for name, param in tqdm(model.named_parameters(), desc=\"Loading parameters\"):\n",
128
- " if param.device.type == \"meta\":\n",
129
- " # This parameter is on meta device and needs to be materialized\n",
130
- " print(f\"Warning: {name} is on meta device\")\n",
131
- "\n",
132
- "# Try to trigger full loading with a dummy forward pass\n",
133
- "print(\"Running dummy forward pass to ensure full loading...\")\n",
134
- "dummy_input = torch.tensor([[1]], device=\"cuda\")\n",
135
- "with torch.no_grad():\n",
136
- " try:\n",
137
- " _ = model(dummy_input)\n",
138
- " except Exception as e:\n",
139
- " print(f\"Dummy forward pass failed (expected): {e}\")\n",
140
- "\n",
141
- "print(\"✅ Model fully loaded and ready\")\n",
142
- "\n",
143
  "\n",
144
  "# Check VRAM usage\n",
145
  "if torch.cuda.is_available():\n",
@@ -181,9 +374,9 @@
181
  "elif MAX_ROWS:\n",
182
  " df = df.head(MAX_ROWS).copy()\n",
183
  "\n",
184
- "# === CREATE PROMPTS (DEEPSEEK STYLE) ===\n",
185
  "def create_prompt(row):\n",
186
- " \"\"\"Create prompt for Qwen annotation with specific profession categories.\"\"\"\n",
187
  " name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
188
  " \n",
189
  " # Gather hints\n",
@@ -208,34 +401,31 @@
208
  " \n",
209
  " hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
210
  " \n",
211
- " return f\"\"\"Given '{name}' ({hint_text}), provide:\n",
212
- "1. Full legal name (Western order if non-latin script)\n",
213
- "2. Any stage names/aliases (comma separated)\n",
214
- "3. Gender (Male/Female/Other/Unknown)\n",
215
- "4. Top 3 most likely professions from ONLY these categories:\n",
216
- " - actor\n",
217
- " - adult performer\n",
218
- " - singer/musician\n",
219
- " - model\n",
220
- " - online personality (includes streamers, cosplayers, influencers)\n",
221
- " - public figure (includes politicians, activists, journalists, authors)\n",
222
- " - voice actor/ASMR\n",
223
- " - sports professional\n",
224
- " - tv personality (includes hosts, presenters, reality TV)\n",
225
- "\n",
226
- "5. Primary country associated\n",
227
- "\n",
228
- "IMPORTANT:\n",
229
- "- Choose professions ONLY from the 9 categories above\n",
230
- "- Provide up to 3 professions, comma-separated, ordered by relevance\n",
231
- "- Be SPECIFIC: choose the most accurate category for each role\n",
232
- "- \"online personality\" includes: streamers, cosplayers, YouTubers, influencers, content creators\n",
233
- "- Use 'Unknown' when uncertain or for fictional characters/places\n",
234
- "- For multi-role people, list all relevant categories (e.g., \"actor, singer/musician, online personality\")\n",
235
- "- For country respond with one word only, for example China or Columbia\n",
236
- "- actress = actor\n",
237
- "\n",
238
- "Respond with exactly 5 numbered lines.\"\"\"\n",
239
  "\n",
240
  "# Create prompts\n",
241
  "print(\"\\nCreating prompts...\")\n",
@@ -259,23 +449,12 @@
259
  "def query_qwen_local(prompt: str) -> str:\n",
260
  " \"\"\"Query Qwen locally via transformers.\"\"\"\n",
261
  " try:\n",
262
- " with timeout(60): # 60 second timeout\n",
263
- " # Your existing generation code here\n",
264
- " ...\n",
265
- " except TimeoutError:\n",
266
- " print(f\"[ERROR] Generation timed out after 60 seconds\")\n",
267
- " return None\n",
268
- " except Exception as e:\n",
269
- " print(f\"Generation error: {e}\")\n",
270
- " return None\n",
271
- " \n",
272
- " # Format as chat message for Qwen\n",
273
  " messages = [\n",
274
- " {\"role\": \"system\", \"content\": \"You are an assistant that extracts key data on a person based on the name. Respond with exactly 5 numbered lines. For professions, choose ONLY from these categories: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality.\"},\n",
275
  " {\"role\": \"user\", \"content\": prompt}\n",
276
  " ]\n",
277
  " \n",
278
- " print(f\"[DEBUG] Applying chat template...\")\n",
279
  " # Tokenize\n",
280
  " if hasattr(tokenizer, 'apply_chat_template'):\n",
281
  " text = tokenizer.apply_chat_template(\n",
@@ -287,47 +466,38 @@
287
  " # Fallback for older tokenizers\n",
288
  " text = f\"[INST] {prompt} [/INST]\"\n",
289
  " \n",
290
- " print(f\"[DEBUG] Tokenizing input...\")\n",
291
  " inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
292
  " \n",
293
- " print(f\"[DEBUG] Input shape: {inputs['input_ids'].shape}\")\n",
294
- " print(f\"[DEBUG] Starting generation...\")\n",
295
- " \n",
296
- " # Generate\n",
297
- " with torch.no_grad():\n",
298
- " outputs = model.generate(\n",
299
- " **inputs,\n",
300
- " max_new_tokens=100,\n",
301
- " temperature=0.1,\n",
302
- " do_sample=False,\n",
303
- " #top_p=0.9,\n",
304
- " pad_token_id=tokenizer.eos_token_id\n",
305
- " )\n",
306
- " \n",
307
- " print(f\"[DEBUG] Generation complete, decoding...\")\n",
308
  " \n",
309
  " # Decode\n",
310
  " generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
311
  " response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
312
  " \n",
313
- " print(f\"[DEBUG] Response: {response[:100]}...\") # First 100 chars\n",
314
- " \n",
315
  " return response.strip()\n",
316
  " \n",
 
 
 
317
  " except Exception as e:\n",
318
  " print(f\"Generation error: {e}\")\n",
319
  " import traceback\n",
320
  " traceback.print_exc()\n",
321
  " return None\n",
322
  "\n",
323
- "#output_file = current_dir.parent / f\"data/CSV/qwen_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
324
- "#index_file = current_dir.parent / \"misc/query_indicies/qwen_local_query_index.txt\"\n",
325
- "\n",
326
- "\n",
327
  " \n",
328
- "# === PARSE RESPONSE (DEEPSEEK STYLE) ===\n",
329
  "def parse_response(response):\n",
330
- " \"\"\"Parse Qwen response into structured fields.\"\"\"\n",
331
  " if not response:\n",
332
  " return {\n",
333
  " 'full_name': 'Unknown',\n",
@@ -356,11 +526,21 @@
356
  " elif line.startswith('2.'):\n",
357
  " fields['aliases'] = line[2:].strip()\n",
358
  " elif line.startswith('3.'):\n",
359
- " fields['gender'] = line[2:].strip()\n",
 
 
 
 
 
 
360
  " elif line.startswith('4.'):\n",
361
  " fields['profession_llm'] = line[2:].strip()\n",
362
  " elif line.startswith('5.'):\n",
363
- " fields['country'] = line[2:].strip()\n",
 
 
 
 
364
  " \n",
365
  " return fields\n",
366
  "\n",
@@ -429,7 +609,7 @@
429
  "# Final save\n",
430
  "df.to_csv(output_file, index=False)\n",
431
  "index_file.write_text(str(current_index))\n",
432
- "print(\"✅ Finished full dataset.\")\n"
433
  ]
434
  },
435
  {
 
5
  "id": "0543d9c4-055b-49a3-a8d0-9dfb622b2b8c",
6
  "metadata": {},
7
  "source": [
8
+ "# QWEN 2.5-32B Local Inference\n",
9
+ "\n",
10
+ "## Hardware Requirements\n",
11
+ "- **GPU**: NVIDIA A100 (40GB or 80GB recommended)\n",
12
+ "- **VRAM Usage**: \n",
13
+ " - 8-bit quantization: ~32GB\n",
14
+ " - 4-bit quantization: ~16-20GB\n",
15
+ " - bfloat16 (no quantization): ~64GB\n",
16
+ "- **System RAM**: 32GB minimum, 64GB recommended\n",
17
+ "- **Storage**: ~65GB for model download\n",
18
+ "\n",
19
+ "## Configuration\n",
20
+ "This notebook uses **8-bit quantization** via `bitsandbytes` for optimal performance on A100 GPUs:\n",
21
+ "- Reduces VRAM usage from 64GB to ~32GB\n",
22
+ "- Minimal quality degradation\n",
23
+ "- Faster inference than bfloat16\n",
24
+ "\n",
25
+ "## Model Details\n",
26
+ "- **Model**: Qwen/Qwen2.5-32B-Instruct\n",
27
+ "- **Task**: Entity annotation and profession classification\n",
28
+ "- **Quantization**: LLM.int8() (8-bit)\n",
29
+ "- **Device**: CUDA (auto device mapping)\n",
30
+ "\n",
31
+ "## Dependencies\n",
32
+ "Make sure to install:\n",
33
+ "```bash\n",
34
+ "pip install transformers>=4.35.0 bitsandbytes>=0.41.0 accelerate torch pandas tqdm\n",
35
+ "```"
36
  ]
37
  },
38
  {
39
  "cell_type": "code",
40
+ "execution_count": 1,
41
  "id": "fe6ba282-896b-4272-b82b-ef24810732fb",
42
+ "metadata": {
43
+ "execution": {
44
+ "iopub.execute_input": "2025-11-29T20:14:34.671159Z",
45
+ "iopub.status.busy": "2025-11-29T20:14:34.671015Z",
46
+ "iopub.status.idle": "2025-11-29T20:26:40.952267Z",
47
+ "shell.execute_reply": "2025-11-29T20:26:40.951486Z",
48
+ "shell.execute_reply.started": "2025-11-29T20:14:34.671146Z"
49
+ }
50
+ },
51
+ "outputs": [
52
+ {
53
+ "name": "stderr",
54
+ "output_type": "stream",
55
+ "text": [
56
+ "/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
57
+ " from .autonotebook import tqdm as notebook_tqdm\n"
58
+ ]
59
+ },
60
+ {
61
+ "name": "stdout",
62
+ "output_type": "stream",
63
+ "text": [
64
+ "Loading model: Qwen/Qwen2.5-32B-Instruct\n",
65
+ "Cache directory: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/models\n",
66
+ "This may take a while on first run (~65GB download)...\n",
67
+ "\n",
68
+ "Device: cuda\n",
69
+ "Loading tokenizer...\n",
70
+ "✅ Tokenizer loaded\n",
71
+ "Configuring 8-bit quantization...\n",
72
+ "Loading model with 8-bit quantization (this may take several minutes)...\n"
73
+ ]
74
+ },
75
+ {
76
+ "name": "stderr",
77
+ "output_type": "stream",
78
+ "text": [
79
+ "Loading checkpoint shards: 100%|██████████| 17/17 [03:49<00:00, 13.50s/it]\n"
80
+ ]
81
+ },
82
+ {
83
+ "name": "stdout",
84
+ "output_type": "stream",
85
+ "text": [
86
+ "✅ Model loaded with 8-bit quantization\n",
87
+ "VRAM used: 32.72 GB\n",
88
+ "\n",
89
+ "Loading raw input CSV...\n",
90
+ "✅ Loaded professions.csv\n",
91
+ "✅ Loaded profession mapping with 9 categories\n",
92
+ "Loaded 50861 rows\n",
93
+ "\n",
94
+ "Profession categories (9):\n",
95
+ " - actor\n",
96
+ " - adult performer\n",
97
+ " - singer/musician\n",
98
+ " - model\n",
99
+ " - online personality\n",
100
+ " - public figure\n",
101
+ " - voice actor/ASMR\n",
102
+ " - sports professional\n",
103
+ " - tv personality\n",
104
+ "\n",
105
+ "Creating prompts...\n",
106
+ "✅ Prompts created\n",
107
+ "Resuming from index 0\n"
108
+ ]
109
+ },
110
+ {
111
+ "name": "stderr",
112
+ "output_type": "stream",
113
+ "text": [
114
+ "Qwen Local: 0%| | 0/50861 [00:00<?, ?it/s]The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details.\n",
115
+ "Qwen Local: 0%| | 10/50861 [01:35<115:51:08, 8.20s/it]"
116
+ ]
117
+ },
118
+ {
119
+ "name": "stdout",
120
+ "output_type": "stream",
121
+ "text": [
122
+ "💾 Progress saved after row 10\n"
123
+ ]
124
+ },
125
+ {
126
+ "name": "stderr",
127
+ "output_type": "stream",
128
+ "text": [
129
+ "Qwen Local: 0%| | 20/50861 [03:01<143:54:04, 10.19s/it]"
130
+ ]
131
+ },
132
+ {
133
+ "name": "stdout",
134
+ "output_type": "stream",
135
+ "text": [
136
+ "💾 Progress saved after row 20\n"
137
+ ]
138
+ },
139
+ {
140
+ "name": "stderr",
141
+ "output_type": "stream",
142
+ "text": [
143
+ "Qwen Local: 0%| | 24/50861 [03:52<151:28:05, 10.73s/it]"
144
+ ]
145
+ },
146
+ {
147
+ "name": "stdout",
148
+ "output_type": "stream",
149
+ "text": [
150
+ "❌ Row 23: parsed as all Unknown (likely model crash); skipping.\n"
151
+ ]
152
+ },
153
+ {
154
+ "name": "stderr",
155
+ "output_type": "stream",
156
+ "text": [
157
+ "Qwen Local: 0%| | 30/50861 [04:36<116:38:03, 8.26s/it]"
158
+ ]
159
+ },
160
+ {
161
+ "name": "stdout",
162
+ "output_type": "stream",
163
+ "text": [
164
+ "💾 Progress saved after row 30\n"
165
+ ]
166
+ },
167
+ {
168
+ "name": "stderr",
169
+ "output_type": "stream",
170
+ "text": [
171
+ "Qwen Local: 0%| | 40/50861 [05:55<142:16:40, 10.08s/it]"
172
+ ]
173
+ },
174
+ {
175
+ "name": "stdout",
176
+ "output_type": "stream",
177
+ "text": [
178
+ "💾 Progress saved after row 40\n"
179
+ ]
180
+ },
181
+ {
182
+ "name": "stderr",
183
+ "output_type": "stream",
184
+ "text": [
185
+ "Qwen Local: 0%| | 45/50861 [06:45<127:20:20, 9.02s/it]\n"
186
+ ]
187
+ },
188
+ {
189
+ "ename": "KeyboardInterrupt",
190
+ "evalue": "",
191
+ "output_type": "error",
192
+ "traceback": [
193
+ "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
194
+ "\u001b[31mKeyboardInterrupt\u001b[39m Traceback (most recent call last)",
195
+ "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 330\u001b[39m\n\u001b[32m 328\u001b[39m response = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 329\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m attempt \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[32m3\u001b[39m):\n\u001b[32m--> \u001b[39m\u001b[32m330\u001b[39m response = \u001b[43mquery_qwen_local\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprompt\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 332\u001b[39m \u001b[38;5;66;03m# Valid response?\u001b[39;00m\n\u001b[32m 333\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m response \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(response.strip()) > \u001b[32m10\u001b[39m:\n",
196
+ "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 245\u001b[39m, in \u001b[36mquery_qwen_local\u001b[39m\u001b[34m(prompt)\u001b[39m\n\u001b[32m 243\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m timeout(\u001b[32m60\u001b[39m):\n\u001b[32m 244\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m torch.no_grad():\n\u001b[32m--> \u001b[39m\u001b[32m245\u001b[39m outputs = \u001b[43mmodel\u001b[49m\u001b[43m.\u001b[49m\u001b[43mgenerate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 246\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 247\u001b[39m \u001b[43m \u001b[49m\u001b[43mmax_new_tokens\u001b[49m\u001b[43m=\u001b[49m\u001b[32;43m100\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 248\u001b[39m \u001b[43m \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[43m=\u001b[49m\u001b[32;43m0.1\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 249\u001b[39m \u001b[43m \u001b[49m\u001b[43mdo_sample\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[32m 250\u001b[39m \u001b[43m \u001b[49m\u001b[43mpad_token_id\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtokenizer\u001b[49m\u001b[43m.\u001b[49m\u001b[43meos_token_id\u001b[49m\n\u001b[32m 251\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 253\u001b[39m \u001b[38;5;66;03m# Decode\u001b[39;00m\n\u001b[32m 254\u001b[39m generated_ids = outputs[\u001b[32m0\u001b[39m][inputs[\u001b[33m'\u001b[39m\u001b[33minput_ids\u001b[39m\u001b[33m'\u001b[39m].shape[\u001b[32m1\u001b[39m]:]\n",
197
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/utils/_contextlib.py:120\u001b[39m, in \u001b[36mcontext_decorator.<locals>.decorate_context\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 117\u001b[39m \u001b[38;5;129m@functools\u001b[39m.wraps(func)\n\u001b[32m 118\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mdecorate_context\u001b[39m(*args, **kwargs):\n\u001b[32m 119\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m ctx_factory():\n\u001b[32m--> \u001b[39m\u001b[32m120\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
198
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:2564\u001b[39m, in \u001b[36mGenerationMixin.generate\u001b[39m\u001b[34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, use_model_defaults, custom_generate, **kwargs)\u001b[39m\n\u001b[32m 2561\u001b[39m model_kwargs[\u001b[33m\"\u001b[39m\u001b[33muse_cache\u001b[39m\u001b[33m\"\u001b[39m] = generation_config.use_cache\n\u001b[32m 2563\u001b[39m \u001b[38;5;66;03m# 9. Call generation mode\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m2564\u001b[39m result = \u001b[43mdecoding_method\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 2565\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 2566\u001b[39m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2567\u001b[39m \u001b[43m \u001b[49m\u001b[43mlogits_processor\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprepared_logits_processor\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2568\u001b[39m \u001b[43m \u001b[49m\u001b[43mstopping_criteria\u001b[49m\u001b[43m=\u001b[49m\u001b[43mprepared_stopping_criteria\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2569\u001b[39m \u001b[43m \u001b[49m\u001b[43mgeneration_config\u001b[49m\u001b[43m=\u001b[49m\u001b[43mgeneration_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2570\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mgeneration_mode_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2571\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mmodel_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 2572\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 2574\u001b[39m \u001b[38;5;66;03m# Convert to legacy cache format if requested\u001b[39;00m\n\u001b[32m 2575\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[32m 2576\u001b[39m generation_config.return_legacy_cache \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[32m 2577\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(result, \u001b[33m\"\u001b[39m\u001b[33mpast_key_values\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 2578\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mgetattr\u001b[39m(result.past_key_values, \u001b[33m\"\u001b[39m\u001b[33mto_legacy_cache\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 2579\u001b[39m ):\n",
199
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/transformers/generation/utils.py:2787\u001b[39m, in \u001b[36mGenerationMixin._sample\u001b[39m\u001b[34m(self, input_ids, logits_processor, stopping_criteria, generation_config, synced_gpus, streamer, **model_kwargs)\u001b[39m\n\u001b[32m 2785\u001b[39m is_prefill = \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[32m 2786\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m2787\u001b[39m outputs = \u001b[43mmodel_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mmodel_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[32m 2789\u001b[39m \u001b[38;5;66;03m# synced_gpus: don't waste resources running the code we don't need; kwargs must be updated before skipping\u001b[39;00m\n\u001b[32m 2790\u001b[39m model_kwargs = \u001b[38;5;28mself\u001b[39m._update_model_kwargs_for_generation(\n\u001b[32m 2791\u001b[39m outputs,\n\u001b[32m 2792\u001b[39m model_kwargs,\n\u001b[32m 2793\u001b[39m is_encoder_decoder=\u001b[38;5;28mself\u001b[39m.config.is_encoder_decoder,\n\u001b[32m 2794\u001b[39m )\n",
200
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1775\u001b[39m, in \u001b[36mModule._wrapped_call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1773\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._compiled_call_impl(*args, **kwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[32m 1774\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1775\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
201
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1786\u001b[39m, in \u001b[36mModule._call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1781\u001b[39m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[32m 1782\u001b[39m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[32m 1783\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m._backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_pre_hooks\n\u001b[32m 1784\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[32m 1785\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[32m-> \u001b[39m\u001b[32m1786\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1788\u001b[39m result = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 1789\u001b[39m called_always_called_hooks = \u001b[38;5;28mset\u001b[39m()\n",
202
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/transformers/utils/generic.py:918\u001b[39m, in \u001b[36mcan_return_tuple.<locals>.wrapper\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 916\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m return_dict_passed \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 917\u001b[39m return_dict = return_dict_passed\n\u001b[32m--> \u001b[39m\u001b[32m918\u001b[39m output = \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 919\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m return_dict \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(output, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[32m 920\u001b[39m output = output.to_tuple()\n",
203
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/transformers/models/qwen2/modeling_qwen2.py:449\u001b[39m, in \u001b[36mQwen2ForCausalLM.forward\u001b[39m\u001b[34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, labels, use_cache, cache_position, logits_to_keep, **kwargs)\u001b[39m\n\u001b[32m 417\u001b[39m \u001b[38;5;129m@can_return_tuple\u001b[39m\n\u001b[32m 418\u001b[39m \u001b[38;5;129m@auto_docstring\u001b[39m\n\u001b[32m 419\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mforward\u001b[39m(\n\u001b[32m (...)\u001b[39m\u001b[32m 430\u001b[39m **kwargs: Unpack[TransformersKwargs],\n\u001b[32m 431\u001b[39m ) -> CausalLMOutputWithPast:\n\u001b[32m 432\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33mr\u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 433\u001b[39m \u001b[33;03m Example:\u001b[39;00m\n\u001b[32m 434\u001b[39m \n\u001b[32m (...)\u001b[39m\u001b[32m 447\u001b[39m \u001b[33;03m \"Hey, are you conscious? Can you talk to me?\\nI'm not conscious, but I can talk to you.\"\u001b[39;00m\n\u001b[32m 448\u001b[39m \u001b[33;03m ```\"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m449\u001b[39m outputs: BaseModelOutputWithPast = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 450\u001b[39m \u001b[43m \u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m=\u001b[49m\u001b[43minput_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 451\u001b[39m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m=\u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 452\u001b[39m \u001b[43m \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[43m=\u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 453\u001b[39m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 454\u001b[39m \u001b[43m \u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[43m=\u001b[49m\u001b[43minputs_embeds\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 455\u001b[39m \u001b[43m \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[43m=\u001b[49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 456\u001b[39m \u001b[43m \u001b[49m\u001b[43mcache_position\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcache_position\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 457\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 458\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 460\u001b[39m hidden_states = outputs.last_hidden_state\n\u001b[32m 461\u001b[39m \u001b[38;5;66;03m# Only compute necessary logits, and do not upcast them to float if we are not computing the loss\u001b[39;00m\n",
204
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1775\u001b[39m, in \u001b[36mModule._wrapped_call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1773\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._compiled_call_impl(*args, **kwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[32m 1774\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1775\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
205
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1786\u001b[39m, in \u001b[36mModule._call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1781\u001b[39m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[32m 1782\u001b[39m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[32m 1783\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m._backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_pre_hooks\n\u001b[32m 1784\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[32m 1785\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[32m-> \u001b[39m\u001b[32m1786\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1788\u001b[39m result = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 1789\u001b[39m called_always_called_hooks = \u001b[38;5;28mset\u001b[39m()\n",
206
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/transformers/utils/generic.py:1064\u001b[39m, in \u001b[36mcheck_model_inputs.<locals>.wrapper\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1061\u001b[39m monkey_patched_layers.append((module, original_forward))\n\u001b[32m 1063\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1064\u001b[39m outputs = \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1065\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m original_exception:\n\u001b[32m 1066\u001b[39m \u001b[38;5;66;03m# If we get a TypeError, it's possible that the model is not receiving the recordable kwargs correctly.\u001b[39;00m\n\u001b[32m 1067\u001b[39m \u001b[38;5;66;03m# Get a TypeError even after removing the recordable kwargs -> re-raise the original exception\u001b[39;00m\n\u001b[32m 1068\u001b[39m \u001b[38;5;66;03m# Otherwise -> we're probably missing `**kwargs` in the decorated function\u001b[39;00m\n\u001b[32m 1069\u001b[39m kwargs_without_recordable = {k: v \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m kwargs.items() \u001b[38;5;28;01mif\u001b[39;00m k \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m recordable_keys}\n",
207
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/transformers/models/qwen2/modeling_qwen2.py:384\u001b[39m, in \u001b[36mQwen2Model.forward\u001b[39m\u001b[34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, use_cache, cache_position, **kwargs)\u001b[39m\n\u001b[32m 381\u001b[39m position_embeddings = \u001b[38;5;28mself\u001b[39m.rotary_emb(hidden_states, position_ids)\n\u001b[32m 383\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m decoder_layer \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.layers[: \u001b[38;5;28mself\u001b[39m.config.num_hidden_layers]:\n\u001b[32m--> \u001b[39m\u001b[32m384\u001b[39m hidden_states = \u001b[43mdecoder_layer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 385\u001b[39m \u001b[43m \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 386\u001b[39m \u001b[43m \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcausal_mask_mapping\u001b[49m\u001b[43m[\u001b[49m\u001b[43mdecoder_layer\u001b[49m\u001b[43m.\u001b[49m\u001b[43mattention_type\u001b[49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 387\u001b[39m \u001b[43m \u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[43m=\u001b[49m\u001b[43mposition_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 388\u001b[39m \u001b[43m \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 389\u001b[39m \u001b[43m \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[43m=\u001b[49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 390\u001b[39m \u001b[43m \u001b[49m\u001b[43mcache_position\u001b[49m\u001b[43m=\u001b[49m\u001b[43mcache_position\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 391\u001b[39m \u001b[43m \u001b[49m\u001b[43mposition_embeddings\u001b[49m\u001b[43m=\u001b[49m\u001b[43mposition_embeddings\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 392\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 393\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 395\u001b[39m hidden_states = \u001b[38;5;28mself\u001b[39m.norm(hidden_states)\n\u001b[32m 396\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m BaseModelOutputWithPast(\n\u001b[32m 397\u001b[39m last_hidden_state=hidden_states,\n\u001b[32m 398\u001b[39m past_key_values=past_key_values \u001b[38;5;28;01mif\u001b[39;00m use_cache \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m 399\u001b[39m )\n",
208
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/transformers/modeling_layers.py:94\u001b[39m, in \u001b[36mGradientCheckpointingLayer.__call__\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 91\u001b[39m logger.warning_once(message)\n\u001b[32m 93\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._gradient_checkpointing_func(partial(\u001b[38;5;28msuper\u001b[39m().\u001b[34m__call__\u001b[39m, **kwargs), *args)\n\u001b[32m---> \u001b[39m\u001b[32m94\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[34;43m__call__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
209
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1775\u001b[39m, in \u001b[36mModule._wrapped_call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1773\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._compiled_call_impl(*args, **kwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[32m 1774\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1775\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
210
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1786\u001b[39m, in \u001b[36mModule._call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1781\u001b[39m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[32m 1782\u001b[39m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[32m 1783\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m._backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_pre_hooks\n\u001b[32m 1784\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[32m 1785\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[32m-> \u001b[39m\u001b[32m1786\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1788\u001b[39m result = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 1789\u001b[39m called_always_called_hooks = \u001b[38;5;28mset\u001b[39m()\n",
211
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/transformers/utils/deprecation.py:172\u001b[39m, in \u001b[36mdeprecate_kwarg.<locals>.wrapper.<locals>.wrapped_func\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 168\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m minimum_action \u001b[38;5;129;01min\u001b[39;00m (Action.NOTIFY, Action.NOTIFY_ALWAYS) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torchdynamo_compiling():\n\u001b[32m 169\u001b[39m \u001b[38;5;66;03m# DeprecationWarning is ignored by default, so we use FutureWarning instead\u001b[39;00m\n\u001b[32m 170\u001b[39m warnings.warn(message, \u001b[38;5;167;01mFutureWarning\u001b[39;00m, stacklevel=\u001b[32m2\u001b[39m)\n\u001b[32m--> \u001b[39m\u001b[32m172\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
212
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/transformers/models/qwen2/modeling_qwen2.py:249\u001b[39m, in \u001b[36mQwen2DecoderLayer.forward\u001b[39m\u001b[34m(self, hidden_states, attention_mask, position_ids, past_key_values, use_cache, cache_position, position_embeddings, **kwargs)\u001b[39m\n\u001b[32m 247\u001b[39m residual = hidden_states\n\u001b[32m 248\u001b[39m hidden_states = \u001b[38;5;28mself\u001b[39m.post_attention_layernorm(hidden_states)\n\u001b[32m--> \u001b[39m\u001b[32m249\u001b[39m hidden_states = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mmlp\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 250\u001b[39m hidden_states = residual + hidden_states\n\u001b[32m 251\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m hidden_states\n",
213
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1775\u001b[39m, in \u001b[36mModule._wrapped_call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1773\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._compiled_call_impl(*args, **kwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[32m 1774\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1775\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
214
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1786\u001b[39m, in \u001b[36mModule._call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1781\u001b[39m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[32m 1782\u001b[39m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[32m 1783\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m._backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_pre_hooks\n\u001b[32m 1784\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[32m 1785\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[32m-> \u001b[39m\u001b[32m1786\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1788\u001b[39m result = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 1789\u001b[39m called_always_called_hooks = \u001b[38;5;28mset\u001b[39m()\n",
215
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/transformers/models/qwen2/modeling_qwen2.py:46\u001b[39m, in \u001b[36mQwen2MLP.forward\u001b[39m\u001b[34m(self, x)\u001b[39m\n\u001b[32m 45\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x):\n\u001b[32m---> \u001b[39m\u001b[32m46\u001b[39m down_proj = \u001b[38;5;28mself\u001b[39m.down_proj(\u001b[38;5;28mself\u001b[39m.act_fn(\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mgate_proj\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m) * \u001b[38;5;28mself\u001b[39m.up_proj(x))\n\u001b[32m 47\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m down_proj\n",
216
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1775\u001b[39m, in \u001b[36mModule._wrapped_call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1773\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._compiled_call_impl(*args, **kwargs) \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[32m 1774\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1775\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
217
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/nn/modules/module.py:1786\u001b[39m, in \u001b[36mModule._call_impl\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 1781\u001b[39m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[32m 1782\u001b[39m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[32m 1783\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m._backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m._forward_pre_hooks\n\u001b[32m 1784\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[32m 1785\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[32m-> \u001b[39m\u001b[32m1786\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1788\u001b[39m result = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 1789\u001b[39m called_always_called_hooks = \u001b[38;5;28mset\u001b[39m()\n",
218
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/bitsandbytes/nn/modules.py:1071\u001b[39m, in \u001b[36mLinear8bitLt.forward\u001b[39m\u001b[34m(self, x)\u001b[39m\n\u001b[32m 1068\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m.bias \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m.bias.dtype != x.dtype:\n\u001b[32m 1069\u001b[39m \u001b[38;5;28mself\u001b[39m.bias.data = \u001b[38;5;28mself\u001b[39m.bias.data.to(x.dtype)\n\u001b[32m-> \u001b[39m\u001b[32m1071\u001b[39m out = \u001b[43mbnb\u001b[49m\u001b[43m.\u001b[49m\u001b[43mmatmul\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstate\u001b[49m\u001b[43m=\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1073\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m.state.has_fp16_weights \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m.state.CB \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m 1074\u001b[39m \u001b[38;5;28mself\u001b[39m.weight.data = \u001b[38;5;28mself\u001b[39m.state.CB\n",
219
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/bitsandbytes/autograd/_functions.py:424\u001b[39m, in \u001b[36mmatmul\u001b[39m\u001b[34m(A, B, out, state, threshold, bias)\u001b[39m\n\u001b[32m 422\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m A.device.type \u001b[38;5;129;01min\u001b[39;00m (\u001b[33m\"\u001b[39m\u001b[33mcpu\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mxpu\u001b[39m\u001b[33m\"\u001b[39m):\n\u001b[32m 423\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m MatMul8bitFp.apply(A, B, out, bias, state)\n\u001b[32m--> \u001b[39m\u001b[32m424\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mMatMul8bitLt\u001b[49m\u001b[43m.\u001b[49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\u001b[43mA\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mB\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbias\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstate\u001b[49m\u001b[43m)\u001b[49m\n",
220
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/autograd/function.py:581\u001b[39m, in \u001b[36mFunction.apply\u001b[39m\u001b[34m(cls, *args, **kwargs)\u001b[39m\n\u001b[32m 578\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m torch._C._are_functorch_transforms_active():\n\u001b[32m 579\u001b[39m \u001b[38;5;66;03m# See NOTE: [functorch vjp and autograd interaction]\u001b[39;00m\n\u001b[32m 580\u001b[39m args = _functorch.utils.unwrap_dead_wrappers(args)\n\u001b[32m--> \u001b[39m\u001b[32m581\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[32m 583\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_setup_ctx_defined:\n\u001b[32m 584\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[32m 585\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mIn order to use an autograd.Function with functorch transforms \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 586\u001b[39m \u001b[33m\"\u001b[39m\u001b[33m(vmap, grad, jvp, jacrev, ...), it must override the setup_context \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 587\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mstaticmethod. For more details, please see \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 588\u001b[39m \u001b[33m\"\u001b[39m\u001b[33mhttps://pytorch.org/docs/main/notes/extending.func.html\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 589\u001b[39m )\n",
221
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/bitsandbytes/autograd/_functions.py:192\u001b[39m, in \u001b[36mMatMul8bitLt.forward\u001b[39m\u001b[34m(ctx, A, B, out, bias, state)\u001b[39m\n\u001b[32m 189\u001b[39m CA, CAt, SCA, SCAt, outlier_cols = F.int8_double_quant(A.to(torch.float16), threshold=state.threshold)\n\u001b[32m 190\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 191\u001b[39m \u001b[38;5;66;03m# Fast path\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m192\u001b[39m CA, SCA, outlier_cols = \u001b[43mF\u001b[49m\u001b[43m.\u001b[49m\u001b[43mint8_vectorwise_quant\u001b[49m\u001b[43m(\u001b[49m\u001b[43mA\u001b[49m\u001b[43m.\u001b[49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtorch\u001b[49m\u001b[43m.\u001b[49m\u001b[43mfloat16\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mthreshold\u001b[49m\u001b[43m=\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m.\u001b[49m\u001b[43mthreshold\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 193\u001b[39m CAt = SCAt = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m 195\u001b[39m has_grad = \u001b[38;5;28;01mFalse\u001b[39;00m\n",
222
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/bitsandbytes/functional.py:2058\u001b[39m, in \u001b[36mint8_vectorwise_quant\u001b[39m\u001b[34m(A, threshold)\u001b[39m\n\u001b[32m 2040\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mint8_vectorwise_quant\u001b[39m(A: torch.Tensor, threshold=\u001b[32m0.0\u001b[39m):\n\u001b[32m 2041\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"Quantizes a tensor with dtype `torch.float16` to `torch.int8` in accordance to the `LLM.int8()` algorithm.\u001b[39;00m\n\u001b[32m 2042\u001b[39m \n\u001b[32m 2043\u001b[39m \u001b[33;03m For more information, see the [LLM.int8() paper](https://arxiv.org/abs/2208.07339).\u001b[39;00m\n\u001b[32m (...)\u001b[39m\u001b[32m 2056\u001b[39m \u001b[33;03m - `torch.Tensor` with dtype `torch.int32`, *optional*: A list of column indices which contain outlier features.\u001b[39;00m\n\u001b[32m 2057\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m2058\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[43m.\u001b[49m\u001b[43mops\u001b[49m\u001b[43m.\u001b[49m\u001b[43mbitsandbytes\u001b[49m\u001b[43m.\u001b[49m\u001b[43mint8_vectorwise_quant\u001b[49m\u001b[43m.\u001b[49m\u001b[43mdefault\u001b[49m\u001b[43m(\u001b[49m\u001b[43mA\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mthreshold\u001b[49m\u001b[43m)\u001b[49m\n",
223
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/_ops.py:841\u001b[39m, in \u001b[36mOpOverload.__call__\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m 840\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, /, *args: _P.args, **kwargs: _P.kwargs) -> _T:\n\u001b[32m--> \u001b[39m\u001b[32m841\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_op\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
224
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/_compile.py:53\u001b[39m, in \u001b[36m_disable_dynamo.<locals>.inner\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 50\u001b[39m disable_fn = torch._dynamo.disable(fn, recursive, wrapping=\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[32m 51\u001b[39m fn.__dynamo_disable = disable_fn \u001b[38;5;66;03m# type: ignore[attr-defined]\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m53\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mdisable_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
225
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/_dynamo/eval_frame.py:1044\u001b[39m, in \u001b[36mDisableContext.__call__.<locals>._fn\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 1042\u001b[39m _maybe_set_eval_frame(_callback_from_stance(\u001b[38;5;28mself\u001b[39m.callback))\n\u001b[32m 1043\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m-> \u001b[39m\u001b[32m1044\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1045\u001b[39m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[32m 1046\u001b[39m set_eval_frame(\u001b[38;5;28;01mNone\u001b[39;00m)\n",
226
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/torch/library.py:732\u001b[39m, in \u001b[36m_impl.<locals>.register_.<locals>.func_no_dynamo\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 730\u001b[39m \u001b[38;5;129m@torch\u001b[39m._disable_dynamo\n\u001b[32m 731\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mfunc_no_dynamo\u001b[39m(*args, **kwargs):\n\u001b[32m--> \u001b[39m\u001b[32m732\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
227
+ "\u001b[36mFile \u001b[39m\u001b[32m/shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/.venv/lib/python3.11/site-packages/bitsandbytes/backends/cuda/ops.py:148\u001b[39m, in \u001b[36m_\u001b[39m\u001b[34m(A, threshold)\u001b[39m\n\u001b[32m 145\u001b[39m outlier_cols = torch.argwhere(outliers.any(dim=\u001b[32m0\u001b[39m)).view(-\u001b[32m1\u001b[39m)\n\u001b[32m 146\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 147\u001b[39m \u001b[38;5;66;03m# Needed for torch.compile support.\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m148\u001b[39m outlier_cols = \u001b[43mtorch\u001b[49m\u001b[43m.\u001b[49m\u001b[43mempty\u001b[49m\u001b[43m(\u001b[49m\u001b[32;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m=\u001b[49m\u001b[43mA\u001b[49m\u001b[43m.\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdtype\u001b[49m\u001b[43m=\u001b[49m\u001b[43mtorch\u001b[49m\u001b[43m.\u001b[49m\u001b[43mint64\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 150\u001b[39m \u001b[38;5;28;01mwith\u001b[39;00m _cuda_device_of(A):\n\u001b[32m 151\u001b[39m lib.cint8_vector_quant(\n\u001b[32m 152\u001b[39m get_ptr(A),\n\u001b[32m 153\u001b[39m get_ptr(out_row),\n\u001b[32m (...)\u001b[39m\u001b[32m 158\u001b[39m _get_tensor_stream(A),\n\u001b[32m 159\u001b[39m )\n",
228
+ "\u001b[31mKeyboardInterrupt\u001b[39m: "
229
+ ]
230
+ }
231
+ ],
232
  "source": [
233
  "import pandas as pd\n",
234
  "import json\n",
 
237
  "from pathlib import Path\n",
238
  "from tqdm import tqdm\n",
239
  "import torch\n",
240
+ "from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n",
241
  "import signal\n",
242
  "from contextlib import contextmanager\n",
243
  "\n",
 
259
  "output_file = current_dir.parent / f\"data/CSV/qwen_local_annotated_POI{'_test' if TEST_MODE else ''}.csv\"\n",
260
  "\n",
261
  "# Model settings\n",
262
+ "MODEL_NAME = \"Qwen/Qwen2.5-32B-Instruct\"\n",
263
+ "#MODEL_NAME = \"Qwen/Qwen2.5-14B-Instruct\"\n",
264
+ "#MODEL_NAME = \"Qwen/Qwen3-235B-A22B-Instruct-2507-FP8\"\n",
265
  "#MODEL_NAME = \"mistralai/Mistral-7B-Instruct-v0.3\"\n",
266
  "CACHE_DIR = current_dir.parent / \"data/models\"\n",
267
  "CACHE_DIR.mkdir(parents=True, exist_ok=True)\n",
 
314
  "\n",
315
  "print(\"✅ Tokenizer loaded\")\n",
316
  "\n",
317
+ "# Configure 8-bit quantization for A100\n",
318
+ "print(\"Configuring 8-bit quantization...\")\n",
319
+ "quantization_config = BitsAndBytesConfig(\n",
320
+ " load_in_8bit=True,\n",
321
+ " llm_int8_threshold=6.0,\n",
322
+ " llm_int8_has_fp16_weight=False\n",
 
 
 
 
323
  ")\n",
 
 
324
  "\n",
325
+ "# Load model with 8-bit quantization\n",
326
+ "print(\"Loading model with 8-bit quantization (this may take several minutes)...\")\n",
327
  "model = AutoModelForCausalLM.from_pretrained(\n",
328
  " MODEL_NAME,\n",
329
  " cache_dir=str(CACHE_DIR),\n",
330
+ " quantization_config=quantization_config,\n",
331
  " device_map=\"auto\",\n",
332
  " trust_remote_code=False\n",
333
  ")\n",
334
  "model.eval()\n",
335
+ "print(\"✅ Model loaded with 8-bit quantization\")\n",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
336
  "\n",
337
  "# Check VRAM usage\n",
338
  "if torch.cuda.is_available():\n",
 
374
  "elif MAX_ROWS:\n",
375
  " df = df.head(MAX_ROWS).copy()\n",
376
  "\n",
377
+ "# === CREATE PROMPTS (OPTIMIZED FOR CLEAN OUTPUTS) ===\n",
378
  "def create_prompt(row):\n",
379
+ " \"\"\"Create prompt for Qwen annotation with strict formatting requirements.\"\"\"\n",
380
  " name = row['real_name'] if pd.notna(row.get('real_name')) else row.get('name', '')\n",
381
  " \n",
382
  " # Gather hints\n",
 
401
  " \n",
402
  " hint_text = \", \".join(hints[:5]) if hints else \"none\"\n",
403
  " \n",
404
+ " return f\"\"\"Extract information about '{name}' ({hint_text}).\n",
405
+ "\n",
406
+ "Respond with EXACTLY 5 numbered lines. Each line must contain ONLY the value, no labels or extra text.\n",
407
+ "\n",
408
+ "FORMAT REQUIREMENTS:\n",
409
+ "1. Full legal name in Western order (first last). VALUE ONLY.\n",
410
+ "2. Stage names/aliases, comma-separated. If none, write \"None\". VALUE ONLY.\n",
411
+ "3. Gender: MUST be exactly one word: Male, Female, Other, or Unknown. VALUE ONLY.\n",
412
+ "4. Professions: Choose up to 3 from this list ONLY: actor, adult performer, singer/musician, model, online personality, public figure, voice actor/ASMR, sports professional, tv personality. Comma-separated. VALUE ONLY.\n",
413
+ "5. Primary country: Country name only (e.g., \"China\", \"United States\", \"Colombia\"). VALUE ONLY.\n",
414
+ "\n",
415
+ "RULES:\n",
416
+ "- Professions MUST match the exact categories listed (actress = actor)\n",
417
+ "- \"online personality\" includes streamers, cosplayers, YouTubers, influencers\n",
418
+ "- \"public figure\" includes politicians, activists, journalists, authors\n",
419
+ "- Use \"Unknown\" when uncertain or for fictional characters\n",
420
+ "- NO explanatory text, NO labels like \"Gender:\", NO prefixes\n",
421
+ "- For multi-role people, list up to 3 categories by relevance\n",
422
+ "\n",
423
+ "EXAMPLE FORMAT:\n",
424
+ "1. Taylor Swift\n",
425
+ "2. None\n",
426
+ "3. Female\n",
427
+ "4. singer/musician, public figure\n",
428
+ "5. United States\"\"\"\n",
 
 
 
429
  "\n",
430
  "# Create prompts\n",
431
  "print(\"\\nCreating prompts...\")\n",
 
449
  "def query_qwen_local(prompt: str) -> str:\n",
450
  " \"\"\"Query Qwen locally via transformers.\"\"\"\n",
451
  " try:\n",
452
+ " # Format as chat message for Qwen with strict system prompt\n",
 
 
 
 
 
 
 
 
 
 
453
  " messages = [\n",
454
+ " {\"role\": \"system\", \"content\": \"You are a data extraction assistant. Respond with exactly 5 numbered lines containing ONLY values. No labels, no explanations, no prefixes. Follow the format precisely.\"},\n",
455
  " {\"role\": \"user\", \"content\": prompt}\n",
456
  " ]\n",
457
  " \n",
 
458
  " # Tokenize\n",
459
  " if hasattr(tokenizer, 'apply_chat_template'):\n",
460
  " text = tokenizer.apply_chat_template(\n",
 
466
  " # Fallback for older tokenizers\n",
467
  " text = f\"[INST] {prompt} [/INST]\"\n",
468
  " \n",
 
469
  " inputs = tokenizer([text], return_tensors=\"pt\", padding=True).to(device)\n",
470
  " \n",
471
+ " # Generate with timeout\n",
472
+ " with timeout(60):\n",
473
+ " with torch.no_grad():\n",
474
+ " outputs = model.generate(\n",
475
+ " **inputs,\n",
476
+ " max_new_tokens=100,\n",
477
+ " temperature=0.1,\n",
478
+ " do_sample=False,\n",
479
+ " pad_token_id=tokenizer.eos_token_id\n",
480
+ " )\n",
 
 
 
 
 
481
  " \n",
482
  " # Decode\n",
483
  " generated_ids = outputs[0][inputs['input_ids'].shape[1]:]\n",
484
  " response = tokenizer.decode(generated_ids, skip_special_tokens=True)\n",
485
  " \n",
 
 
486
  " return response.strip()\n",
487
  " \n",
488
+ " except TimeoutError:\n",
489
+ " print(f\"[ERROR] Generation timed out after 60 seconds\")\n",
490
+ " return None\n",
491
  " except Exception as e:\n",
492
  " print(f\"Generation error: {e}\")\n",
493
  " import traceback\n",
494
  " traceback.print_exc()\n",
495
  " return None\n",
496
  "\n",
 
 
 
 
497
  " \n",
498
+ "# === PARSE RESPONSE WITH CLEANING ===\n",
499
  "def parse_response(response):\n",
500
+ " \"\"\"Parse Qwen response into structured fields with cleaning.\"\"\"\n",
501
  " if not response:\n",
502
  " return {\n",
503
  " 'full_name': 'Unknown',\n",
 
526
  " elif line.startswith('2.'):\n",
527
  " fields['aliases'] = line[2:].strip()\n",
528
  " elif line.startswith('3.'):\n",
529
+ " # Clean gender field - remove any labels\n",
530
+ " gender_raw = line[2:].strip()\n",
531
+ " # Remove common prefixes\n",
532
+ " gender_raw = re.sub(r'^(Gender:|gender:)\\s*', '', gender_raw, flags=re.IGNORECASE)\n",
533
+ " # Extract just the gender word\n",
534
+ " gender_match = re.search(r'\\b(Male|Female|Other|Unknown)\\b', gender_raw, re.IGNORECASE)\n",
535
+ " fields['gender'] = gender_match.group(1).capitalize() if gender_match else gender_raw\n",
536
  " elif line.startswith('4.'):\n",
537
  " fields['profession_llm'] = line[2:].strip()\n",
538
  " elif line.startswith('5.'):\n",
539
+ " # Clean country field - remove any labels\n",
540
+ " country_raw = line[2:].strip()\n",
541
+ " # Remove common prefixes like \"Primary country:\", \"Country:\", etc.\n",
542
+ " country_raw = re.sub(r'^(Primary\\s+)?(associated\\s+)?country:\\s*', '', country_raw, flags=re.IGNORECASE)\n",
543
+ " fields['country'] = country_raw\n",
544
  " \n",
545
  " return fields\n",
546
  "\n",
 
609
  "# Final save\n",
610
  "df.to_csv(output_file, index=False)\n",
611
  "index_file.write_text(str(current_index))\n",
612
+ "print(\"✅ Finished full dataset.\")"
613
  ]
614
  },
615
  {
jupyter_notebooks/.ipynb_checkpoints/Section_2-3-4_compare-models-checkpoint.ipynb ADDED
@@ -0,0 +1,237 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": null,
6
+ "id": "6cbaef9d-3058-4a59-a8ee-32fcc2062ed6",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "# Jupyter Notebook Cell for Country Name Standardization\n",
11
+ "# Copy and paste this entire cell into your Jupyter notebook\n",
12
+ "\n",
13
+ "import pandas as pd\n",
14
+ "import os\n",
15
+ "import re\n",
16
+ "\n",
17
+ "# Define country standardization mappings\n",
18
+ "COUNTRY_MAPPINGS = {\n",
19
+ " # USA variations\n",
20
+ " 'united states': 'USA',\n",
21
+ " 'united states of america': 'USA',\n",
22
+ " 'america': 'USA',\n",
23
+ " 'us': 'USA',\n",
24
+ " 'u.s.': 'USA',\n",
25
+ " 'u.s.a.': 'USA',\n",
26
+ " 'states': 'USA',\n",
27
+ " \n",
28
+ " # UK variations\n",
29
+ " 'united kingdom': 'UK',\n",
30
+ " 'england': 'UK',\n",
31
+ " 'britain': 'UK',\n",
32
+ " 'great britain': 'UK',\n",
33
+ " 'uk': 'UK',\n",
34
+ " 'u.k.': 'UK',\n",
35
+ " \n",
36
+ " # Turkey -> Türkiye\n",
37
+ " 'turkey': 'Türkiye',\n",
38
+ " \n",
39
+ " # Czech Republic -> Czechia\n",
40
+ " 'czech republic': 'Czechia',\n",
41
+ " 'czechoslovakia': 'Czechia',\n",
42
+ " 'czechoslowakia': 'Czechia', # Common misspelling\n",
43
+ " \n",
44
+ " # USSR -> Russia\n",
45
+ " 'ussr': 'Russia',\n",
46
+ " 'udssr': 'Russia',\n",
47
+ " 'soviet union': 'Russia',\n",
48
+ " \n",
49
+ " # Korea variations (maintain distinction between North and South)\n",
50
+ " 'korea': 'South Korea', # If just \"Korea\", assume South Korea\n",
51
+ " 'south korea': 'South Korea',\n",
52
+ " 'republic of korea': 'South Korea',\n",
53
+ " 'north korea': 'North Korea',\n",
54
+ " 'dprk': 'North Korea',\n",
55
+ " \n",
56
+ " # China variations\n",
57
+ " 'china': 'China',\n",
58
+ " 'people\\'s republic of china': 'China',\n",
59
+ " 'prc': 'China',\n",
60
+ " 'mainland china': 'China',\n",
61
+ " \n",
62
+ " # Common standardizations\n",
63
+ " 'holland': 'Netherlands',\n",
64
+ " 'the netherlands': 'Netherlands',\n",
65
+ " 'deutschland': 'Germany',\n",
66
+ " 'nippon': 'Japan',\n",
67
+ " 'espana': 'Spain',\n",
68
+ " 'españa': 'Spain',\n",
69
+ " \n",
70
+ " # Keep these as-is but included for completeness\n",
71
+ " 'usa': 'USA',\n",
72
+ " 'russia': 'Russia',\n",
73
+ "}\n",
74
+ "\n",
75
+ "def standardize_country(country_value):\n",
76
+ " \"\"\"\n",
77
+ " Standardize a single country name based on the mapping.\n",
78
+ " \"\"\"\n",
79
+ " if pd.isna(country_value):\n",
80
+ " return country_value\n",
81
+ " \n",
82
+ " # Convert to string and strip whitespace\n",
83
+ " country_str = str(country_value).strip()\n",
84
+ " \n",
85
+ " # Return if empty or 'Unknown'\n",
86
+ " if not country_str or country_str.lower() == 'unknown':\n",
87
+ " return country_str\n",
88
+ " \n",
89
+ " # Convert to lowercase for matching\n",
90
+ " country_lower = country_str.lower()\n",
91
+ " \n",
92
+ " # Check if it matches any of our mappings\n",
93
+ " for pattern, replacement in COUNTRY_MAPPINGS.items():\n",
94
+ " if country_lower == pattern:\n",
95
+ " return replacement\n",
96
+ " \n",
97
+ " # If no exact match found, return original with proper capitalization\n",
98
+ " # This preserves countries not in our mapping\n",
99
+ " return country_str\n",
100
+ "\n",
101
+ "def process_csv_file(input_file, output_file):\n",
102
+ " \"\"\"\n",
103
+ " Process a CSV file to standardize country names.\n",
104
+ " \"\"\"\n",
105
+ " print(f\"Processing: {input_file}\")\n",
106
+ " \n",
107
+ " # For mistral.csv which has no header, we need special handling\n",
108
+ " if 'mistral' in input_file.lower():\n",
109
+ " # Read without header\n",
110
+ " df = pd.read_csv(input_file, header=None)\n",
111
+ " \n",
112
+ " # Check if the last column contains country data\n",
113
+ " # Based on the structure, country should be in the last column\n",
114
+ " last_col = df.columns[-1]\n",
115
+ " \n",
116
+ " # Apply standardization to the last column\n",
117
+ " df[last_col] = df[last_col].apply(standardize_country)\n",
118
+ " \n",
119
+ " print(f\" - Standardized column {last_col} (assumed to be country)\")\n",
120
+ " print(f\" - Sample values after standardization: {df[last_col].dropna().head(10).tolist()}\")\n",
121
+ " else:\n",
122
+ " # Normal CSV with header\n",
123
+ " df = pd.read_csv(input_file)\n",
124
+ " \n",
125
+ " # Check if 'country' column exists\n",
126
+ " if 'country' in df.columns:\n",
127
+ " # Apply standardization\n",
128
+ " df['country'] = df['country'].apply(standardize_country)\n",
129
+ " \n",
130
+ " print(f\" - Found and standardized 'country' column\")\n",
131
+ " print(f\" - Unique countries after standardization: {sorted(df['country'].dropna().unique())}\")\n",
132
+ " else:\n",
133
+ " print(f\" - Warning: No 'country' column found in {input_file}\")\n",
134
+ " \n",
135
+ " # Save the processed file\n",
136
+ " df.to_csv(output_file, index=False)\n",
137
+ " print(f\" - Saved to: {output_file}\\n\")\n",
138
+ " \n",
139
+ " return df\n",
140
+ "\n",
141
+ "# ============================================================\n",
142
+ "# MAIN EXECUTION - Adjust paths as needed for your environment\n",
143
+ "# ============================================================\n",
144
+ "\n",
145
+ "# Input files - adjust these paths to match your file locations\n",
146
+ "input_files = [\n",
147
+ " 'gemma.csv',\n",
148
+ " 'mistral.csv',\n",
149
+ " 'qwen.csv'\n",
150
+ "]\n",
151
+ "\n",
152
+ "# Create a results dictionary to store processed dataframes\n",
153
+ "results = {}\n",
154
+ "\n",
155
+ "for filename in input_files:\n",
156
+ " # Adjust these paths based on where your files are located\n",
157
+ " # For example, if files are in current directory, use: input_path = filename\n",
158
+ " input_path = filename # Modify this based on your file location\n",
159
+ " \n",
160
+ " # Create output filename\n",
161
+ " base_name = filename.replace('.csv', '')\n",
162
+ " output_path = f'{base_name}_standardized_country.csv'\n",
163
+ " \n",
164
+ " # Check if input file exists\n",
165
+ " if os.path.exists(input_path):\n",
166
+ " df = process_csv_file(input_path, output_path)\n",
167
+ " results[base_name] = df\n",
168
+ " else:\n",
169
+ " print(f\"Warning: {input_path} not found! Please check the file path.\\n\")\n",
170
+ "\n",
171
+ "# Display summary statistics\n",
172
+ "print(\"=\" * 60)\n",
173
+ "print(\"SUMMARY OF COUNTRY STANDARDIZATION\")\n",
174
+ "print(\"=\" * 60)\n",
175
+ "\n",
176
+ "for name, df in results.items():\n",
177
+ " print(f\"\\n{name.upper()}:\")\n",
178
+ " print(f\" Total rows: {len(df)}\")\n",
179
+ " \n",
180
+ " # Find the country column\n",
181
+ " if name == 'mistral':\n",
182
+ " # For mistral, assume last column is country\n",
183
+ " country_col = df.columns[-1]\n",
184
+ " else:\n",
185
+ " country_col = 'country' if 'country' in df.columns else None\n",
186
+ " \n",
187
+ " if country_col is not None:\n",
188
+ " country_counts = df[country_col].value_counts()\n",
189
+ " print(f\" Top 10 countries:\")\n",
190
+ " for country, count in country_counts.head(10).items():\n",
191
+ " print(f\" - {country}: {count}\")\n",
192
+ "\n",
193
+ "print(\"\\n\" + \"=\" * 60)\n",
194
+ "print(\"Processing complete!\")\n",
195
+ "print(\"Standardized files have been created with '_standardized_country.csv' suffix\")\n",
196
+ "print(\"=\" * 60)\n",
197
+ "\n",
198
+ "# Optional: Display before/after comparison for a sample\n",
199
+ "print(\"\\n\" + \"=\" * 60)\n",
200
+ "print(\"EXAMPLE TRANSFORMATIONS APPLIED:\")\n",
201
+ "print(\"=\" * 60)\n",
202
+ "print(\"• 'United States' → 'USA'\")\n",
203
+ "print(\"• 'United States of America' → 'USA'\")\n",
204
+ "print(\"• 'States' → 'USA'\")\n",
205
+ "print(\"• 'England' → 'UK'\")\n",
206
+ "print(\"• 'United Kingdom' → 'UK'\")\n",
207
+ "print(\"• 'Britain' → 'UK'\")\n",
208
+ "print(\"• 'Turkey' → 'Türkiye'\")\n",
209
+ "print(\"• 'Czech Republic' → 'Czechia'\")\n",
210
+ "print(\"• 'Korea' → 'South Korea'\")\n",
211
+ "print(\"• 'UDSSR' → 'Russia'\")\n",
212
+ "print(\"=\" * 60)"
213
+ ]
214
+ }
215
+ ],
216
+ "metadata": {
217
+ "kernelspec": {
218
+ "display_name": "Python 3 (ipykernel)",
219
+ "language": "python",
220
+ "name": "python3"
221
+ },
222
+ "language_info": {
223
+ "codemirror_mode": {
224
+ "name": "ipython",
225
+ "version": 3
226
+ },
227
+ "file_extension": ".py",
228
+ "mimetype": "text/x-python",
229
+ "name": "python",
230
+ "nbconvert_exporter": "python",
231
+ "pygments_lexer": "ipython3",
232
+ "version": "3.12.10"
233
+ }
234
+ },
235
+ "nbformat": 4,
236
+ "nbformat_minor": 5
237
+ }
jupyter_notebooks/GEMMA_3.ipynb CHANGED
The diff for this file is too large to render. See raw diff
 
jupyter_notebooks/MISTRAL.ipynb CHANGED
The diff for this file is too large to render. See raw diff
 
jupyter_notebooks/QWEN.ipynb CHANGED
The diff for this file is too large to render. See raw diff
 
jupyter_notebooks/Section_2-3-4_Figure_8_deepfake_adapters.ipynb CHANGED
@@ -5204,9 +5204,9 @@
5204
  ],
5205
  "metadata": {
5206
  "kernelspec": {
5207
- "display_name": "pm-paper",
5208
  "language": "python",
5209
- "name": "pm-paper"
5210
  },
5211
  "language_info": {
5212
  "codemirror_mode": {
@@ -5218,7 +5218,7 @@
5218
  "name": "python",
5219
  "nbconvert_exporter": "python",
5220
  "pygments_lexer": "ipython3",
5221
- "version": "3.11.13"
5222
  }
5223
  },
5224
  "nbformat": 4,
 
5204
  ],
5205
  "metadata": {
5206
  "kernelspec": {
5207
+ "display_name": "Python 3 (ipykernel)",
5208
  "language": "python",
5209
+ "name": "python3"
5210
  },
5211
  "language_info": {
5212
  "codemirror_mode": {
 
5218
  "name": "python",
5219
  "nbconvert_exporter": "python",
5220
  "pygments_lexer": "ipython3",
5221
+ "version": "3.12.10"
5222
  }
5223
  },
5224
  "nbformat": 4,
jupyter_notebooks/Section_2-3-4_compare-models.ipynb ADDED
@@ -0,0 +1,1680 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 24,
6
+ "id": "6cbaef9d-3058-4a59-a8ee-32fcc2062ed6",
7
+ "metadata": {
8
+ "execution": {
9
+ "iopub.execute_input": "2025-12-07T15:53:19.934286Z",
10
+ "iopub.status.busy": "2025-12-07T15:53:19.934073Z",
11
+ "iopub.status.idle": "2025-12-07T15:53:37.185333Z",
12
+ "shell.execute_reply": "2025-12-07T15:53:37.184777Z",
13
+ "shell.execute_reply.started": "2025-12-07T15:53:19.934269Z"
14
+ }
15
+ },
16
+ "outputs": [
17
+ {
18
+ "name": "stdout",
19
+ "output_type": "stream",
20
+ "text": [
21
+ "Processing: gemma_local_annotated_POI.csv\n"
22
+ ]
23
+ },
24
+ {
25
+ "name": "stderr",
26
+ "output_type": "stream",
27
+ "text": [
28
+ "/tmp/ipykernel_1771575/3158489896.py:277: DtypeWarning: Columns (52,53,54,55,56) have mixed types. Specify dtype option on import or set low_memory=False.\n",
29
+ " df = pd.read_csv(input_path)\n"
30
+ ]
31
+ },
32
+ {
33
+ "name": "stdout",
34
+ "output_type": "stream",
35
+ "text": [
36
+ " - Found and standardized 'country' column\n",
37
+ " - Updated 'name' column for 4 fictional entries\n",
38
+ " - Updated 'real_name' column for 4 fictional entries\n",
39
+ " - Updated 'full_name' column for 4 fictional entries\n",
40
+ " - Changes: 8458 standardized, 4 fictional→Unknown\n",
41
+ " - Unique countries after standardization: ['Afghanistan', 'Albania', 'Algeria', 'American Samoa', 'Angola', 'Argentina', 'Armenia', 'Australia', 'Austria', 'Azerbaijan', 'Bahamas', 'Bangladesh', 'Barbados', 'Belarus', 'Belgium', 'Benin', 'Bolivia', 'Brazil', 'Bulgaria', 'Cambodia', 'Canada', 'Central African Republic', 'Chile', 'China', 'Colombia', 'Costa Rica', 'Croatia', 'Cuba', 'Cyprus', 'Czechia', 'Denmark', 'Dominican Republic', 'Egypt', 'El Salvador', 'Estonia', 'Ethiopia', 'Finland', 'France', 'French Polynesia', 'Georgia', 'Germany', 'Ghana', 'Greece', 'Guatemala', 'Guyana', 'Hong Kong', 'Hungary', 'Iceland', 'India', 'Indonesia', 'Iran', 'Iraq', 'Ireland', 'Israel', 'Italy', 'Jamaica', 'Japan', 'Jordan', 'Kazakhstan', 'Kenya', 'Kosovo', 'Kuwait', 'Kyrgyzstan', 'Latvia', 'Lebanon', 'Lithuania', 'Macau', 'Malaysia', 'Malta', 'Mexico', 'Moldova', 'Monaco', 'Mongolia', 'Morocco', 'Myanmar', 'Namibia', 'Nepal', 'Netherlands', 'New Zealand', 'Nicaragua', 'Nigeria', 'North Korea', 'North Macedonia', 'Norway', 'Pakistan', 'Palestine', 'Paraguay', 'Peru', 'Philippines', 'Poland', 'Portugal', 'Puerto Rico', 'Republic of the Congo', 'Romania', 'Russia', 'Samoa', 'Saudi Arabia', 'Senegal', 'Serbia', 'Singapore', 'Slovakia', 'Slovenia', 'Somalia', 'South Africa', 'South Korea', 'Spain', 'Sri Lanka', 'Sudan', 'Sweden', 'Switzerland', 'Syria', 'Taiwan', 'Tanzania', 'Thailand', 'Trinidad and Tobago', 'Tunisia', 'Türkiye', 'UK', 'USA', 'Uganda', 'Ukraine', 'Unknown', 'Uruguay', 'Uzbekistan', 'Venezuela', 'Vietnam', 'Zambia', 'Zimbabwe']\n",
42
+ " - Saved to: gemma_standardized_country.csv\n",
43
+ " - Full path: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/CSV/gemma_standardized_country.csv\n",
44
+ "\n",
45
+ "Processing: mistral_local_annotated_POI.csv\n"
46
+ ]
47
+ },
48
+ {
49
+ "name": "stderr",
50
+ "output_type": "stream",
51
+ "text": [
52
+ "/tmp/ipykernel_1771575/3158489896.py:247: DtypeWarning: Columns (52,53,54,55,56) have mixed types. Specify dtype option on import or set low_memory=False.\n",
53
+ " df = pd.read_csv(input_path)\n"
54
+ ]
55
+ },
56
+ {
57
+ "name": "stdout",
58
+ "output_type": "stream",
59
+ "text": [
60
+ " - Found and standardized 'country' column\n",
61
+ " - Updated 'name' column for fictional entries\n",
62
+ " - Saved to: mistral_standardized_country.csv\n",
63
+ " - Full path: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/CSV/mistral_standardized_country.csv\n",
64
+ "\n",
65
+ "Processing: qwen_local_annotated_POI.csv\n"
66
+ ]
67
+ },
68
+ {
69
+ "name": "stderr",
70
+ "output_type": "stream",
71
+ "text": [
72
+ "/tmp/ipykernel_1771575/3158489896.py:277: DtypeWarning: Columns (53,54,55,56,57) have mixed types. Specify dtype option on import or set low_memory=False.\n",
73
+ " df = pd.read_csv(input_path)\n"
74
+ ]
75
+ },
76
+ {
77
+ "name": "stdout",
78
+ "output_type": "stream",
79
+ "text": [
80
+ " - Found and standardized 'country' column\n",
81
+ " - Updated 'name' column for 13 fictional entries\n",
82
+ " - Updated 'real_name' column for 13 fictional entries\n",
83
+ " - Updated 'full_name' column for 13 fictional entries\n",
84
+ " - Changes: 30873 standardized, 13 fictional→Unknown\n",
85
+ " - Unique countries after standardization: ['Afghanistan', 'Albania', 'Algeria', 'American Samoa', 'Argentina', 'Armenia', 'Australia', 'Austria', 'Azerbaijan', 'Bangladesh', 'Barbados', 'Belarus', 'Belgium', 'Benin', 'Bolivia', 'Bosnia and Herzegovina', 'Brazil', 'Bulgaria', 'Canada', 'Central African Republic', 'Chile', 'China', 'Colombia', 'Costa Rica', 'Croatia', 'Cuba', 'Czechia', 'Denmark', 'Discworld', 'Dominican Republic', 'Ecuador', 'Egypt', 'El Salvador', 'Estonia', 'Finland', 'France', 'Georgia', 'Germany', 'Ghana', 'Greece', 'Guatemala', 'Haiti', 'Hong Kong', 'Hungary', 'Iceland', 'India', 'Indonesia', 'Iran', 'Iraq', 'Ireland', 'Israel', 'Italy', 'Jamaica', 'Japan', 'Jordan', 'Kazakhstan', 'Kenya', 'Kuwait', 'Kyrgyzstan', 'Laos', 'Latvia', 'Lebanon', 'Lithuania', 'Malaysia', 'Malta', 'Mexico', 'Moldova', 'Monaco', 'Morocco', 'Myanmar', 'Nepal', 'Netherlands', 'New Zealand', 'Nicaragua', 'Nigeria', 'North Korea', 'North Macedonia', 'Norway', 'Pakistan', 'Palestine', 'Papua New Guinea', 'Paraguay', 'Peru', 'Philippines', 'Poland', 'Portugal', 'Puerto Rico', 'Romania', 'Rome', 'Russia', 'Samoa', 'Saudi Arabia', 'Senegal', 'Serbia', 'Singapore', 'Slovakia', 'Slovenia', 'South Africa', 'South Korea', 'South Sudan', 'Spain', 'Sri Lanka', 'Sudan', 'Sweden', 'Switzerland', 'Syria', 'Taiwan', 'Thailand', 'Tunisia', 'Türkiye', 'UK', 'USA', 'Uganda', 'Ukraine', 'United Arab Emirates', 'Unknown', 'Uruguay', 'Uzbekistan', 'Vatican City', 'Venezuela', 'Vietnam', 'Zimbabwe']\n",
86
+ " - Saved to: qwen_standardized_country.csv\n",
87
+ " - Full path: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/CSV/qwen_standardized_country.csv\n",
88
+ "\n",
89
+ "============================================================\n",
90
+ "SUMMARY OF COUNTRY STANDARDIZATION\n",
91
+ "============================================================\n",
92
+ "\n",
93
+ "GEMMA:\n",
94
+ " Total rows: 50861\n",
95
+ " Top 10 countries:\n",
96
+ " - USA: 18879\n",
97
+ " - Unknown: 5639\n",
98
+ " - Japan: 3911\n",
99
+ " - UK: 2792\n",
100
+ " - South Korea: 2058\n",
101
+ " - China: 1721\n",
102
+ " - Canada: 1118\n",
103
+ " - India: 1081\n",
104
+ " - Russia: 973\n",
105
+ " - France: 782\n",
106
+ "\n",
107
+ "MISTRAL:\n",
108
+ " Total rows: 50861\n",
109
+ " Top 10 countries:\n",
110
+ " - Unknown: 10787\n",
111
+ " - USA: 6622\n",
112
+ " - Japan: 2498\n",
113
+ " - South Korea: 1650\n",
114
+ " - UK: 1609\n",
115
+ " - China: 836\n",
116
+ " - Canada: 542\n",
117
+ " - India: 405\n",
118
+ " - Russia: 358\n",
119
+ " - Australia: 336\n",
120
+ "\n",
121
+ "QWEN:\n",
122
+ " Total rows: 50861\n",
123
+ " Top 10 countries:\n",
124
+ " - USA: 19841\n",
125
+ " - Japan: 3924\n",
126
+ " - UK: 2787\n",
127
+ " - South Korea: 2178\n",
128
+ " - Unknown: 2093\n",
129
+ " - China: 2046\n",
130
+ " - India: 932\n",
131
+ " - Russia: 840\n",
132
+ " - Canada: 730\n",
133
+ " - France: 656\n",
134
+ "\n",
135
+ "============================================================\n",
136
+ "Processing complete!\n",
137
+ "\n",
138
+ "Output files created:\n",
139
+ " - gemma_standardized_country.csv\n",
140
+ " - mistral_standardized_country.csv\n",
141
+ " - qwen_standardized_country.csv\n",
142
+ "============================================================\n"
143
+ ]
144
+ }
145
+ ],
146
+ "source": [
147
+ "# Comprehensive Country Name Standardization with Fictional Place Detection\n",
148
+ "# Saves files as modelname_standardized_country.csv\n",
149
+ "\n",
150
+ "import pandas as pd\n",
151
+ "from pathlib import Path\n",
152
+ "\n",
153
+ "# Define fictional places, companies, and invalid entries\n",
154
+ "FICTIONAL_PLACES = {\n",
155
+ " 'arrakis', 'asgard', 'krypton', 'gotham', 'metropolis', 'wakanda', \n",
156
+ " 'middle earth', 'hogwarts', 'westeros', 'narnia', 'mordor', 'gondor',\n",
157
+ " 'atlantis', 'shangri-la', 'eldorado', 'camelot', 'avalon', 'valhalla',\n",
158
+ " 'pandora', 'tatooine', 'coruscant', 'naboo', 'alderaan', 'endor',\n",
159
+ " 'rivendell', 'shire', 'rohan', 'isengard', 'minas tirith',\n",
160
+ " 'springfield', 'quahog', 'south park', 'bikini bottom',\n",
161
+ " 'emerald city', 'oz', 'neverland', 'wonderland'\n",
162
+ "}\n",
163
+ "\n",
164
+ "# Companies and brands that shouldn't be countries\n",
165
+ "COMPANIES_BRANDS = {\n",
166
+ " 'nintendo', 'hollywood', 'disney', 'pixar', 'marvel', 'dc',\n",
167
+ " 'sony', 'microsoft', 'apple', 'google', 'facebook', 'meta',\n",
168
+ " 'amazon', 'netflix', 'hbo', 'warner bros', 'paramount'\n",
169
+ "}\n",
170
+ "\n",
171
+ "# Historical entities that no longer exist\n",
172
+ "HISTORICAL_ENTITIES = {\n",
173
+ " 'roman republic', 'roman empire', 'byzantine empire', 'ottoman empire',\n",
174
+ " 'austro-hungarian empire', 'yugoslavia', 'east germany', 'west germany',\n",
175
+ " 'rhodesia', 'zaire', 'persia', 'prussia', 'mesopotamia', 'babylon'\n",
176
+ "}\n",
177
+ "\n",
178
+ "# Define country standardization mappings\n",
179
+ "COUNTRY_MAPPINGS = {\n",
180
+ " # USA variations\n",
181
+ " 'united states': 'USA',\n",
182
+ " 'united states of america': 'USA',\n",
183
+ " 'america': 'USA',\n",
184
+ " 'american': 'USA', # Added based on your data\n",
185
+ " 'us': 'USA',\n",
186
+ " 'u.s.': 'USA',\n",
187
+ " 'u.s.a.': 'USA',\n",
188
+ " 'states': 'USA',\n",
189
+ " 'unitedstates': 'USA',\n",
190
+ " \n",
191
+ " # UK variations\n",
192
+ " 'united kingdom': 'UK',\n",
193
+ " 'unitedkingdom': 'UK',\n",
194
+ " 'england': 'UK',\n",
195
+ " 'britain': 'UK',\n",
196
+ " 'great britain': 'UK',\n",
197
+ " 'uk': 'UK',\n",
198
+ " 'u.k.': 'UK',\n",
199
+ " 'scotland': 'UK', # Part of UK\n",
200
+ " 'wales': 'UK', # Part of UK\n",
201
+ " 'northern ireland': 'UK', # Part of UK\n",
202
+ " 'northernireland': 'UK',\n",
203
+ " \n",
204
+ " # Turkey -> Türkiye\n",
205
+ " 'turkey': 'Türkiye',\n",
206
+ " \n",
207
+ " # Czech Republic -> Czechia\n",
208
+ " 'czech republic': 'Czechia',\n",
209
+ " 'czechoslovakia': 'Czechia',\n",
210
+ " 'czechoslowakia': 'Czechia',\n",
211
+ " \n",
212
+ " # USSR -> Russia\n",
213
+ " 'ussr': 'Russia',\n",
214
+ " 'udssr': 'Russia',\n",
215
+ " 'soviet union': 'Russia',\n",
216
+ " \n",
217
+ " # Korea variations\n",
218
+ " 'korea': 'South Korea',\n",
219
+ " 'south korea': 'South Korea',\n",
220
+ " 'southkorea': 'South Korea',\n",
221
+ " 'republic of korea': 'South Korea',\n",
222
+ " 'north korea': 'North Korea',\n",
223
+ " 'northkorea': 'North Korea',\n",
224
+ " 'dprk': 'North Korea',\n",
225
+ " 'democratic people\\'s republic of korea': 'North Korea',\n",
226
+ " 'korea (democratic people\\'s republic of)': 'North Korea',\n",
227
+ " \n",
228
+ " # China variations\n",
229
+ " 'china': 'China',\n",
230
+ " 'people\\'s republic of china': 'China',\n",
231
+ " 'prc': 'China',\n",
232
+ " 'mainland china': 'China',\n",
233
+ " \n",
234
+ " # Hong Kong variations\n",
235
+ " 'hong kong': 'Hong Kong',\n",
236
+ " 'hongkong': 'Hong Kong',\n",
237
+ " \n",
238
+ " # Colombia/Columbia confusion\n",
239
+ " 'columbia': 'Colombia', # Common misspelling\n",
240
+ " 'colombia': 'Colombia',\n",
241
+ " \n",
242
+ " # Dominican Republic variations\n",
243
+ " 'dominican': 'Dominican Republic',\n",
244
+ " 'dominican republic': 'Dominican Republic',\n",
245
+ " 'dominicanrepublic': 'Dominican Republic',\n",
246
+ " \n",
247
+ " # Puerto Rico variations\n",
248
+ " 'puerto rico': 'Puerto Rico',\n",
249
+ " 'puertorico': 'Puerto Rico',\n",
250
+ " \n",
251
+ " # El Salvador variations\n",
252
+ " 'el salvador': 'El Salvador',\n",
253
+ " 'elsalvador': 'El Salvador',\n",
254
+ " \n",
255
+ " # South Africa variations\n",
256
+ " 'south africa': 'South Africa',\n",
257
+ " 'southafrica': 'South Africa',\n",
258
+ " \n",
259
+ " # Sri Lanka variations\n",
260
+ " 'sri lanka': 'Sri Lanka',\n",
261
+ " 'srilanka': 'Sri Lanka',\n",
262
+ " \n",
263
+ " # American Samoa variations\n",
264
+ " 'american samoa': 'American Samoa',\n",
265
+ " 'americansamoa': 'American Samoa',\n",
266
+ " \n",
267
+ " # North Macedonia variations\n",
268
+ " 'macedonia': 'North Macedonia',\n",
269
+ " 'northmacedonia': 'North Macedonia',\n",
270
+ " 'north macedonia': 'North Macedonia',\n",
271
+ " \n",
272
+ " # Central African Republic variations\n",
273
+ " 'central african republic': 'Central African Republic',\n",
274
+ " 'centralafricanrepublic': 'Central African Republic',\n",
275
+ " \n",
276
+ " # Republic of the Congo variations\n",
277
+ " 'republic of the congo': 'Republic of the Congo',\n",
278
+ " 'republic_of_the_congo': 'Republic of the Congo',\n",
279
+ " \n",
280
+ " # Common standardizations\n",
281
+ " 'holland': 'Netherlands',\n",
282
+ " 'the netherlands': 'Netherlands',\n",
283
+ " 'deutschland': 'Germany',\n",
284
+ " 'nippon': 'Japan',\n",
285
+ " 'espana': 'Spain',\n",
286
+ " 'españa': 'Spain',\n",
287
+ " \n",
288
+ " # Additional standardizations\n",
289
+ " 'vatican': 'Vatican City',\n",
290
+ " 'uae': 'United Arab Emirates',\n",
291
+ " 'emirates': 'United Arab Emirates',\n",
292
+ " 'bosnia': 'Bosnia and Herzegovina',\n",
293
+ " 'papua': 'Papua New Guinea',\n",
294
+ " 'png': 'Papua New Guinea',\n",
295
+ " 'trinidad': 'Trinidad and Tobago',\n",
296
+ "}\n",
297
+ "\n",
298
+ "def is_fictional_or_invalid(country_str):\n",
299
+ " \"\"\"\n",
300
+ " Check if a country string is fictional, a company, or historically invalid.\n",
301
+ " \"\"\"\n",
302
+ " country_lower = country_str.lower().strip()\n",
303
+ " \n",
304
+ " # Check for fictional places\n",
305
+ " if country_lower in FICTIONAL_PLACES:\n",
306
+ " return True\n",
307
+ " \n",
308
+ " # Check for companies/brands\n",
309
+ " if country_lower in COMPANIES_BRANDS:\n",
310
+ " return True\n",
311
+ " \n",
312
+ " # Check for historical entities\n",
313
+ " if country_lower in HISTORICAL_ENTITIES:\n",
314
+ " return True\n",
315
+ " \n",
316
+ " # Check for entries that contain \"Unknown\" with additional text\n",
317
+ " if 'unknown' in country_lower and len(country_lower) > 7:\n",
318
+ " return True\n",
319
+ " \n",
320
+ " # Check for entries that explicitly say \"fictional\"\n",
321
+ " if 'fictional' in country_lower:\n",
322
+ " return True\n",
323
+ " \n",
324
+ " # Check for entries with multiple countries (containing comma)\n",
325
+ " if ',' in country_str and country_lower != 'trinidad and tobago':\n",
326
+ " return True\n",
327
+ " \n",
328
+ " return False\n",
329
+ "\n",
330
+ "def standardize_country(country_value):\n",
331
+ " \"\"\"\n",
332
+ " Standardize a single country name based on the mapping.\n",
333
+ " \"\"\"\n",
334
+ " if pd.isna(country_value):\n",
335
+ " return country_value\n",
336
+ " \n",
337
+ " # Convert to string and strip whitespace\n",
338
+ " country_str = str(country_value).strip()\n",
339
+ " \n",
340
+ " # Return if empty or already 'Unknown'\n",
341
+ " if not country_str or country_str == 'Unknown':\n",
342
+ " return country_str\n",
343
+ " \n",
344
+ " # Check if it's fictional or invalid\n",
345
+ " if is_fictional_or_invalid(country_str):\n",
346
+ " return 'Unknown'\n",
347
+ " \n",
348
+ " # Convert to lowercase for matching\n",
349
+ " country_lower = country_str.lower()\n",
350
+ " \n",
351
+ " # Check if it matches any of our mappings\n",
352
+ " for pattern, replacement in COUNTRY_MAPPINGS.items():\n",
353
+ " if country_lower == pattern:\n",
354
+ " return replacement\n",
355
+ " \n",
356
+ " # If no exact match found, return original with proper capitalization\n",
357
+ " # This preserves valid countries not in our mapping\n",
358
+ " return country_str\n",
359
+ "\n",
360
+ "def extract_model_name(file_path):\n",
361
+ " \"\"\"\n",
362
+ " Extract the model name (gemma, mistral, qwen) from the file path.\n",
363
+ " \"\"\"\n",
364
+ " file_name = Path(file_path).stem.lower()\n",
365
+ " \n",
366
+ " if 'gemma' in file_name:\n",
367
+ " return 'gemma'\n",
368
+ " elif 'mistral' in file_name:\n",
369
+ " return 'mistral'\n",
370
+ " elif 'qwen' in file_name:\n",
371
+ " return 'qwen'\n",
372
+ " else:\n",
373
+ " # Fallback to using the full stem if model name not found\n",
374
+ " return Path(file_path).stem\n",
375
+ "\n",
376
+ "def process_csv_file(input_file, output_file):\n",
377
+ " \"\"\"\n",
378
+ " Process a CSV file to standardize country names and handle fictional places.\n",
379
+ " \"\"\"\n",
380
+ " # Convert to Path objects for consistent handling\n",
381
+ " input_path = Path(input_file)\n",
382
+ " output_path = Path(output_file)\n",
383
+ " \n",
384
+ " print(f\"Processing: {input_path.name}\")\n",
385
+ " \n",
386
+ " # Track changes\n",
387
+ " changes_made = {'standardized': 0, 'fictional_to_unknown': 0}\n",
388
+ " \n",
389
+ " # For mistral.csv which might have no header, we need special handling\n",
390
+ " if 'mistral' in input_path.name.lower():\n",
391
+ " # Try to read normally first\n",
392
+ " try:\n",
393
+ " df = pd.read_csv(input_path)\n",
394
+ " # Check if 'country' column exists\n",
395
+ " if 'country' in df.columns:\n",
396
+ " original_values = df['country'].copy()\n",
397
+ " df['country'] = df['country'].apply(standardize_country)\n",
398
+ " \n",
399
+ " # Count changes\n",
400
+ " changes_made['standardized'] = (original_values != df['country']).sum()\n",
401
+ " changes_made['fictional_to_unknown'] = ((df['country'] == 'Unknown') & (original_values != 'Unknown')).sum()\n",
402
+ " \n",
403
+ " print(f\" - Found and standardized 'country' column\")\n",
404
+ " \n",
405
+ " # Also update 'name' column for fictional entries\n",
406
+ " if 'name' in df.columns:\n",
407
+ " fictional_mask = df['country'] == 'Unknown'\n",
408
+ " df.loc[fictional_mask & (original_values != 'Unknown'), 'name'] = 'Unknown'\n",
409
+ " print(f\" - Updated 'name' column for fictional entries\")\n",
410
+ " else:\n",
411
+ " # If no country column, assume last column\n",
412
+ " last_col = df.columns[-1]\n",
413
+ " df[last_col] = df[last_col].apply(standardize_country)\n",
414
+ " print(f\" - Standardized column '{last_col}' (assumed to be country)\")\n",
415
+ " except:\n",
416
+ " # If normal reading fails, try without header\n",
417
+ " df = pd.read_csv(input_path, header=None)\n",
418
+ " last_col = df.columns[-1]\n",
419
+ " df[last_col] = df[last_col].apply(standardize_country)\n",
420
+ " print(f\" - Standardized column {last_col} (assumed to be country, no header)\")\n",
421
+ " else:\n",
422
+ " # Normal CSV with header\n",
423
+ " df = pd.read_csv(input_path)\n",
424
+ " \n",
425
+ " # Check if 'country' column exists\n",
426
+ " if 'country' in df.columns:\n",
427
+ " original_values = df['country'].copy()\n",
428
+ " df['country'] = df['country'].apply(standardize_country)\n",
429
+ " \n",
430
+ " # Count changes\n",
431
+ " changes_made['standardized'] = (original_values != df['country']).sum()\n",
432
+ " changes_made['fictional_to_unknown'] = ((df['country'] == 'Unknown') & (original_values != 'Unknown')).sum()\n",
433
+ " \n",
434
+ " print(f\" - Found and standardized 'country' column\")\n",
435
+ " \n",
436
+ " # Also update 'name' and 'real_name' columns for fictional entries\n",
437
+ " for col in ['name', 'real_name', 'full_name']:\n",
438
+ " if col in df.columns:\n",
439
+ " # Set to Unknown where country was changed to Unknown due to being fictional\n",
440
+ " fictional_mask = (df['country'] == 'Unknown') & (original_values != 'Unknown')\n",
441
+ " df.loc[fictional_mask, col] = 'Unknown'\n",
442
+ " if fictional_mask.any():\n",
443
+ " print(f\" - Updated '{col}' column for {fictional_mask.sum()} fictional entries\")\n",
444
+ " \n",
445
+ " print(f\" - Changes: {changes_made['standardized']} standardized, {changes_made['fictional_to_unknown']} fictional→Unknown\")\n",
446
+ " print(f\" - Unique countries after standardization: {sorted(df['country'].dropna().unique())}\")\n",
447
+ " else:\n",
448
+ " print(f\" - Warning: No 'country' column found in {input_path.name}\")\n",
449
+ " \n",
450
+ " # Save the processed file\n",
451
+ " df.to_csv(output_path, index=False)\n",
452
+ " print(f\" - Saved to: {output_path.name}\")\n",
453
+ " print(f\" - Full path: {output_path.absolute()}\\n\")\n",
454
+ " \n",
455
+ " return df\n",
456
+ "\n",
457
+ "# ============================================================\n",
458
+ "# MAIN EXECUTION\n",
459
+ "# ============================================================\n",
460
+ "\n",
461
+ "# Get current directory (assuming you're running from a notebook)\n",
462
+ "current_dir = Path.cwd()\n",
463
+ "\n",
464
+ "# Define your input files using Path objects\n",
465
+ "input_files = [\n",
466
+ " current_dir.parent / \"data/CSV/gemma_local_annotated_POI.csv\",\n",
467
+ " current_dir.parent / \"data/CSV/mistral_local_annotated_POI.csv\",\n",
468
+ " current_dir.parent / \"data/CSV/qwen_local_annotated_POI.csv\"\n",
469
+ "]\n",
470
+ "\n",
471
+ "# Create a results dictionary to store processed dataframes\n",
472
+ "results = {}\n",
473
+ "\n",
474
+ "for input_path in input_files:\n",
475
+ " # Convert to Path object if it isn't already\n",
476
+ " input_path = Path(input_path)\n",
477
+ " \n",
478
+ " # Extract model name and create standardized output filename\n",
479
+ " model_name = extract_model_name(input_path)\n",
480
+ " output_path = input_path.parent / f\"{model_name}_standardized_country.csv\"\n",
481
+ " \n",
482
+ " # Check if input file exists\n",
483
+ " if input_path.exists():\n",
484
+ " df = process_csv_file(input_path, output_path)\n",
485
+ " # Use model name as key\n",
486
+ " results[model_name] = df\n",
487
+ " else:\n",
488
+ " print(f\"Warning: {input_path} not found! Please check the file path.\")\n",
489
+ " print(f\" Absolute path checked: {input_path.absolute()}\\n\")\n",
490
+ "\n",
491
+ "# Display summary statistics\n",
492
+ "print(\"=\" * 60)\n",
493
+ "print(\"SUMMARY OF COUNTRY STANDARDIZATION\")\n",
494
+ "print(\"=\" * 60)\n",
495
+ "\n",
496
+ "for name, df in results.items():\n",
497
+ " print(f\"\\n{name.upper()}:\")\n",
498
+ " print(f\" Total rows: {len(df)}\")\n",
499
+ " \n",
500
+ " # Find the country column\n",
501
+ " if 'mistral' in name.lower():\n",
502
+ " # For mistral, might need to check last column\n",
503
+ " if 'country' in df.columns:\n",
504
+ " country_col = 'country'\n",
505
+ " else:\n",
506
+ " country_col = df.columns[-1]\n",
507
+ " else:\n",
508
+ " country_col = 'country' if 'country' in df.columns else None\n",
509
+ " \n",
510
+ " if country_col is not None:\n",
511
+ " country_counts = df[country_col].value_counts()\n",
512
+ " print(f\" Top 10 countries:\")\n",
513
+ " for country, count in country_counts.head(10).items():\n",
514
+ " print(f\" - {country}: {count}\")\n",
515
+ "\n",
516
+ "print(\"\\n\" + \"=\" * 60)\n",
517
+ "print(\"Processing complete!\")\n",
518
+ "print(\"\\nOutput files created:\")\n",
519
+ "for model_name in results.keys():\n",
520
+ " print(f\" - {model_name}_standardized_country.csv\")\n",
521
+ "print(\"=\" * 60)"
522
+ ]
523
+ },
524
+ {
525
+ "cell_type": "markdown",
526
+ "id": "b5e06b76-da90-4cff-b8dd-a2b27f56cc94",
527
+ "metadata": {},
528
+ "source": [
529
+ "## Create a combined CSV for visual comparison line by line"
530
+ ]
531
+ },
532
+ {
533
+ "cell_type": "code",
534
+ "execution_count": 25,
535
+ "id": "7f2fe3fb-5863-45ec-9df3-9fa64751936b",
536
+ "metadata": {
537
+ "execution": {
538
+ "iopub.execute_input": "2025-12-07T15:54:25.968287Z",
539
+ "iopub.status.busy": "2025-12-07T15:54:25.968073Z",
540
+ "iopub.status.idle": "2025-12-07T15:54:32.371854Z",
541
+ "shell.execute_reply": "2025-12-07T15:54:32.371349Z",
542
+ "shell.execute_reply.started": "2025-12-07T15:54:25.968269Z"
543
+ }
544
+ },
545
+ "outputs": [
546
+ {
547
+ "name": "stdout",
548
+ "output_type": "stream",
549
+ "text": [
550
+ "Reading base file from gemma: gemma_standardized_country.csv\n"
551
+ ]
552
+ },
553
+ {
554
+ "name": "stderr",
555
+ "output_type": "stream",
556
+ "text": [
557
+ "/tmp/ipykernel_1771575/428075483.py:48: DtypeWarning: Columns (52,53,54,55,56) have mixed types. Specify dtype option on import or set low_memory=False.\n",
558
+ " base_df = pd.read_csv(first_path)\n"
559
+ ]
560
+ },
561
+ {
562
+ "name": "stdout",
563
+ "output_type": "stream",
564
+ "text": [
565
+ " - Shape: (50861, 58)\n",
566
+ " - Columns: ['id', 'name', 'type', 'baseModel', 'downloadCount', 'nsfwLevel', 'modelVersions', 'publishedAt', 'usernameHash', 'downloadUrl']...\n",
567
+ "\n",
568
+ "Processing gemma: gemma_standardized_country.csv\n"
569
+ ]
570
+ },
571
+ {
572
+ "name": "stderr",
573
+ "output_type": "stream",
574
+ "text": [
575
+ "/tmp/ipykernel_1771575/428075483.py:79: DtypeWarning: Columns (52,53,54,55,56) have mixed types. Specify dtype option on import or set low_memory=False.\n",
576
+ " df = pd.read_csv(file_path)\n"
577
+ ]
578
+ },
579
+ {
580
+ "name": "stdout",
581
+ "output_type": "stream",
582
+ "text": [
583
+ " - Added gemma_full_name\n",
584
+ " - Added gemma_aliases\n",
585
+ " - Added gemma_gender\n",
586
+ " - Added gemma_profession_llm\n",
587
+ " - Added gemma_country\n",
588
+ "\n",
589
+ "Processing mistral: mistral_standardized_country.csv\n"
590
+ ]
591
+ },
592
+ {
593
+ "name": "stderr",
594
+ "output_type": "stream",
595
+ "text": [
596
+ "/tmp/ipykernel_1771575/428075483.py:79: DtypeWarning: Columns (52,53,54,55,56) have mixed types. Specify dtype option on import or set low_memory=False.\n",
597
+ " df = pd.read_csv(file_path)\n"
598
+ ]
599
+ },
600
+ {
601
+ "name": "stdout",
602
+ "output_type": "stream",
603
+ "text": [
604
+ " - Added mistral_full_name\n",
605
+ " - Added mistral_aliases\n",
606
+ " - Added mistral_gender\n",
607
+ " - Added mistral_profession_llm\n",
608
+ " - Added mistral_country\n",
609
+ "\n",
610
+ "Processing qwen: qwen_standardized_country.csv\n"
611
+ ]
612
+ },
613
+ {
614
+ "name": "stderr",
615
+ "output_type": "stream",
616
+ "text": [
617
+ "/tmp/ipykernel_1771575/428075483.py:79: DtypeWarning: Columns (53,54,55,56,57) have mixed types. Specify dtype option on import or set low_memory=False.\n",
618
+ " df = pd.read_csv(file_path)\n"
619
+ ]
620
+ },
621
+ {
622
+ "name": "stdout",
623
+ "output_type": "stream",
624
+ "text": [
625
+ " - Added qwen_full_name\n",
626
+ " - Added qwen_aliases\n",
627
+ " - Added qwen_gender\n",
628
+ " - Added qwen_profession_llm\n",
629
+ " - Added qwen_country\n",
630
+ "\n",
631
+ "Reordering columns to group by field type...\n",
632
+ " - Added 3 columns for full_name\n",
633
+ " - Added 3 columns for aliases\n",
634
+ " - Added 3 columns for gender\n",
635
+ " - Added 3 columns for profession_llm\n",
636
+ " - Added 3 columns for country\n",
637
+ "\n",
638
+ "Combined CSV saved to: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/CSV/combined_llm_annotations.csv\n",
639
+ "Final shape: (50861, 67)\n",
640
+ "Column order: likely_country, gemma_full_name, mistral_full_name, qwen_full_name, gemma_aliases, mistral_aliases, qwen_aliases, gemma_gender, mistral_gender, qwen_gender, gemma_profession_llm, mistral_profession_llm, qwen_profession_llm, gemma_country, mistral_country...\n",
641
+ "\n",
642
+ "============================================================\n",
643
+ "SAMPLE COMPARISON ACROSS MODELS (first 5 rows)\n",
644
+ "============================================================\n",
645
+ "\n",
646
+ "FULL_NAME Comparison:\n",
647
+ "----------------------------------------\n",
648
+ " name gemma mistral qwen\n",
649
+ "0 IU Lee Ji-eun Lee Ji-eun (Lee, Ji-eun) Lee Ji-eun\n",
650
+ "1 Super Pose Book Vol.1 - ControlNet Unknown Unknown Super Pose Book\n",
651
+ "2 Liyuu LoRA Li Yuchan Liyuu (Full legal name unknown) Li Yuxiao\n",
652
+ "3 Irene Irene Kim Irene (Full legal name unknown) Irene Kim\n",
653
+ "4 AESPA Karina Yoo Jimin Karina (Kim Jung-yeon) Kim Karina\n",
654
+ "\n",
655
+ "COUNTRY Comparison:\n",
656
+ "----------------------------------------\n",
657
+ " name gemma mistral qwen\n",
658
+ "0 IU South Korea South Korea South Korea\n",
659
+ "1 Super Pose Book Vol.1 - ControlNet Unknown Unknown Unknown\n",
660
+ "2 Liyuu LoRA China Unknown China\n",
661
+ "3 Irene South Korea Unknown South Korea\n",
662
+ "4 AESPA Karina South Korea South Korea South Korea\n",
663
+ "\n",
664
+ "PROFESSION_LLM Comparison:\n",
665
+ "----------------------------------------\n",
666
+ " name gemma mistral qwen\n",
667
+ "0 IU singer/musician, actor, tv personality singer/musician, tv personality, actress singer/musician, model, public figure\n",
668
+ "1 Super Pose Book Vol.1 - ControlNet model, adult performer, online personality model, online personality, actress model, online personality, artist\n",
669
+ "2 Liyuu LoRA singer/musician, actor, model model, online personality, artist model, online personality\n",
670
+ "3 Irene model, online personality, actor actor, model, tv personality singer/musician, model, public figure\n",
671
+ "4 AESPA Karina singer/musician, tv personality, online personality singer/musician, tv personality, public figure singer/musician, model, online personality\n",
672
+ "\n",
673
+ "============================================================\n",
674
+ "SUMMARY STATISTICS\n",
675
+ "============================================================\n",
676
+ "\n",
677
+ "GEMMA Country Distribution:\n",
678
+ "gemma_country\n",
679
+ "USA 18879\n",
680
+ "Unknown 5639\n",
681
+ "Japan 3911\n",
682
+ "UK 2792\n",
683
+ "South Korea 2058\n",
684
+ "Name: count, dtype: int64\n",
685
+ "\n",
686
+ "MISTRAL Country Distribution:\n",
687
+ "mistral_country\n",
688
+ "Unknown 10787\n",
689
+ "USA 6622\n",
690
+ "Japan 2498\n",
691
+ "South Korea 1650\n",
692
+ "UK 1609\n",
693
+ "Name: count, dtype: int64\n",
694
+ "\n",
695
+ "QWEN Country Distribution:\n",
696
+ "qwen_country\n",
697
+ "USA 19841\n",
698
+ "Japan 3924\n",
699
+ "UK 2787\n",
700
+ "South Korea 2178\n",
701
+ "Unknown 2093\n",
702
+ "Name: count, dtype: int64\n",
703
+ "\n",
704
+ "============================================================\n",
705
+ "FINAL COLUMNS IN COMBINED FILE (GROUPED BY TYPE):\n",
706
+ "============================================================\n",
707
+ "\n",
708
+ "Common columns:\n",
709
+ " - id\n",
710
+ " - name\n",
711
+ " - type\n",
712
+ " - baseModel\n",
713
+ " - downloadCount\n",
714
+ " - nsfwLevel\n",
715
+ " - modelVersions\n",
716
+ " - publishedAt\n",
717
+ " - usernameHash\n",
718
+ " - downloadUrl\n",
719
+ " - firstImageUrl\n",
720
+ " - latestImageUrl\n",
721
+ " - poi\n",
722
+ " - AutoV1\n",
723
+ " - AutoV2\n",
724
+ " - AutoV3\n",
725
+ " - SHA256\n",
726
+ " - CRC32\n",
727
+ " - BLAKE3\n",
728
+ " - previewImage\n",
729
+ " ... and 32 more\n",
730
+ "\n",
731
+ "Full name columns:\n",
732
+ " - gemma_full_name\n",
733
+ " - mistral_full_name\n",
734
+ " - qwen_full_name\n",
735
+ "\n",
736
+ "Aliases columns:\n",
737
+ " - gemma_aliases\n",
738
+ " - mistral_aliases\n",
739
+ " - qwen_aliases\n",
740
+ "\n",
741
+ "Gender columns:\n",
742
+ " - gemma_gender\n",
743
+ " - mistral_gender\n",
744
+ " - qwen_gender\n",
745
+ "\n",
746
+ "Profession llm columns:\n",
747
+ " - gemma_profession_llm\n",
748
+ " - mistral_profession_llm\n",
749
+ " - qwen_profession_llm\n",
750
+ "\n",
751
+ "Country columns:\n",
752
+ " - likely_country\n",
753
+ " - gemma_country\n",
754
+ " - mistral_country\n",
755
+ " - qwen_country\n",
756
+ "\n",
757
+ "============================================================\n",
758
+ "✓ Combined file saved to: combined_llm_annotations.csv\n",
759
+ " Total rows: 50861\n",
760
+ " Total columns: 67\n",
761
+ "============================================================\n"
762
+ ]
763
+ }
764
+ ],
765
+ "source": [
766
+ "# Script to combine LLM-annotated CSVs with model-specific columns\n",
767
+ "# Creates a single CSV with columns grouped by field type\n",
768
+ "\n",
769
+ "import pandas as pd\n",
770
+ "from pathlib import Path\n",
771
+ "\n",
772
+ "def combine_llm_annotations(input_files, output_file):\n",
773
+ " \"\"\"\n",
774
+ " Combine multiple LLM-annotated CSVs into one file with model-specific columns.\n",
775
+ " Columns are grouped by field type (full_name, aliases, etc.) across models.\n",
776
+ " \n",
777
+ " Args:\n",
778
+ " input_files: Dictionary with model names as keys and file paths as values\n",
779
+ " output_file: Path to save the combined CSV\n",
780
+ " \"\"\"\n",
781
+ " \n",
782
+ " # LLM-specific columns that will be prefixed with model name\n",
783
+ " llm_columns = ['full_name', 'aliases', 'gender', 'profession_llm', 'country']\n",
784
+ " \n",
785
+ " # Columns that should be kept from the first file (common across all)\n",
786
+ " # Adjust this list based on your actual columns\n",
787
+ " common_columns = [\n",
788
+ " 'id', 'name', 'type', 'baseModel', 'downloadCount', 'nsfwLevel', \n",
789
+ " 'modelVersions', 'publishedAt', 'usernameHash', 'downloadUrl',\n",
790
+ " 'firstImageUrl', 'latestImageUrl', 'poi', 'AutoV1', 'AutoV2', 'AutoV3',\n",
791
+ " 'SHA256', 'CRC32', 'BLAKE3', 'previewImage',\n",
792
+ " 'version_id_1', 'version_id_2', 'version_id_3', 'version_id_4', 'version_id_5',\n",
793
+ " 'version_id_6', 'version_id_7', 'version_id_8', 'version_id_9', 'version_id_10',\n",
794
+ " 'version_id_11', 'version_id_12', 'version_id_13', 'version_id_14', 'version_id_15',\n",
795
+ " 'version_id_16', 'version_id_17', 'version_id_18', 'version_id_19', 'version_id_20',\n",
796
+ " 'tag_1', 'tag_2', 'tag_3', 'tag_4', 'tag_5', 'tag_6', 'tag_7',\n",
797
+ " 'real_name', 'tags', 'likely_country', 'likely_nationality', 'likely_profession'\n",
798
+ " ]\n",
799
+ " \n",
800
+ " # Read the first file to get the base dataframe\n",
801
+ " first_model = list(input_files.keys())[0]\n",
802
+ " first_path = input_files[first_model]\n",
803
+ " \n",
804
+ " print(f\"Reading base file from {first_model}: {first_path.name}\")\n",
805
+ " \n",
806
+ " # Read the first file\n",
807
+ " if 'mistral' in first_model and not pd.read_csv(first_path, nrows=1).columns.str.contains('id').any():\n",
808
+ " # Special handling for mistral if it has no header\n",
809
+ " base_df = pd.read_csv(first_path, header=None)\n",
810
+ " # We'll need to map columns manually\n",
811
+ " print(\" - Note: Mistral file appears to have no header, handling specially\")\n",
812
+ " else:\n",
813
+ " base_df = pd.read_csv(first_path)\n",
814
+ " \n",
815
+ " print(f\" - Shape: {base_df.shape}\")\n",
816
+ " print(f\" - Columns: {base_df.columns.tolist()[:10]}...\")\n",
817
+ " \n",
818
+ " # Start with common columns that exist in the base dataframe\n",
819
+ " existing_common_cols = [col for col in common_columns if col in base_df.columns]\n",
820
+ " result_df = base_df[existing_common_cols].copy()\n",
821
+ " \n",
822
+ " # Dictionary to collect columns by type\n",
823
+ " columns_by_type = {col_type: [] for col_type in llm_columns}\n",
824
+ " \n",
825
+ " # Process each model's data\n",
826
+ " for model_name, file_path in input_files.items():\n",
827
+ " print(f\"\\nProcessing {model_name}: {file_path.name}\")\n",
828
+ " \n",
829
+ " # Read the model's data\n",
830
+ " if 'mistral' in model_name and not pd.read_csv(file_path, nrows=1).columns.str.contains('id').any():\n",
831
+ " df = pd.read_csv(file_path, header=None)\n",
832
+ " # Map columns based on position (you may need to adjust this)\n",
833
+ " # Assuming the last 5 columns are the LLM annotations\n",
834
+ " num_cols = len(df.columns)\n",
835
+ " llm_start_idx = num_cols - 5\n",
836
+ " \n",
837
+ " for i, col_name in enumerate(llm_columns):\n",
838
+ " if llm_start_idx + i < num_cols:\n",
839
+ " new_col_name = f'{model_name}_{col_name}'\n",
840
+ " result_df[new_col_name] = df.iloc[:, llm_start_idx + i]\n",
841
+ " columns_by_type[col_name].append(new_col_name)\n",
842
+ " print(f\" - Added {new_col_name} from column {llm_start_idx + i}\")\n",
843
+ " else:\n",
844
+ " df = pd.read_csv(file_path)\n",
845
+ " \n",
846
+ " # Add LLM-specific columns with model prefix\n",
847
+ " for col in llm_columns:\n",
848
+ " if col in df.columns:\n",
849
+ " new_col_name = f'{model_name}_{col}'\n",
850
+ " result_df[new_col_name] = df[col]\n",
851
+ " columns_by_type[col].append(new_col_name)\n",
852
+ " print(f\" - Added {new_col_name}\")\n",
853
+ " else:\n",
854
+ " print(f\" - Warning: Column '{col}' not found in {model_name}\")\n",
855
+ " \n",
856
+ " # Reorder columns to group by type\n",
857
+ " print(\"\\nReordering columns to group by field type...\")\n",
858
+ " \n",
859
+ " # Get the final column order\n",
860
+ " final_columns = existing_common_cols.copy()\n",
861
+ " \n",
862
+ " # Add columns grouped by type\n",
863
+ " for col_type in llm_columns:\n",
864
+ " if columns_by_type[col_type]:\n",
865
+ " final_columns.extend(columns_by_type[col_type])\n",
866
+ " print(f\" - Added {len(columns_by_type[col_type])} columns for {col_type}\")\n",
867
+ " \n",
868
+ " # Reorder the dataframe\n",
869
+ " result_df = result_df[final_columns]\n",
870
+ " \n",
871
+ " # Save the combined dataframe\n",
872
+ " result_df.to_csv(output_file, index=False)\n",
873
+ " print(f\"\\nCombined CSV saved to: {output_file}\")\n",
874
+ " print(f\"Final shape: {result_df.shape}\")\n",
875
+ " print(f\"Column order: {', '.join([col for col in result_df.columns if any(col.endswith(f'_{ctype}') for ctype in llm_columns)][:15])}...\")\n",
876
+ " \n",
877
+ " return result_df\n",
878
+ "\n",
879
+ "def get_column_comparison(df, models):\n",
880
+ " \"\"\"\n",
881
+ " Create a comparison of values across models for inspection.\n",
882
+ " \"\"\"\n",
883
+ " print(\"\\n\" + \"=\"*60)\n",
884
+ " print(\"SAMPLE COMPARISON ACROSS MODELS (first 5 rows)\")\n",
885
+ " print(\"=\"*60)\n",
886
+ " \n",
887
+ " # Show a few examples comparing across models\n",
888
+ " comparison_cols = ['full_name', 'country', 'profession_llm']\n",
889
+ " \n",
890
+ " for col in comparison_cols:\n",
891
+ " print(f\"\\n{col.upper()} Comparison:\")\n",
892
+ " print(\"-\" * 40)\n",
893
+ " \n",
894
+ " # Create a comparison dataframe\n",
895
+ " comp_data = {'name': df['name'].iloc[:5] if 'name' in df.columns else df.index[:5]}\n",
896
+ " \n",
897
+ " for model in models:\n",
898
+ " col_name = f'{model}_{col}'\n",
899
+ " if col_name in df.columns:\n",
900
+ " comp_data[model] = df[col_name].iloc[:5]\n",
901
+ " \n",
902
+ " comp_df = pd.DataFrame(comp_data)\n",
903
+ " print(comp_df.to_string())\n",
904
+ "\n",
905
+ "# ============================================================\n",
906
+ "# MAIN EXECUTION\n",
907
+ "# ============================================================\n",
908
+ "\n",
909
+ "# Get current directory\n",
910
+ "current_dir = Path.cwd()\n",
911
+ "\n",
912
+ "# Define input files - these should be your standardized files\n",
913
+ "# Adjust the paths as needed\n",
914
+ "input_files = {\n",
915
+ " 'gemma': current_dir.parent / \"data/CSV/gemma_standardized_country.csv\",\n",
916
+ " 'mistral': current_dir.parent / \"data/CSV/mistral_standardized_country.csv\",\n",
917
+ " 'qwen': current_dir.parent / \"data/CSV/qwen_standardized_country.csv\"\n",
918
+ "}\n",
919
+ "\n",
920
+ "# Check if all files exist\n",
921
+ "all_exist = True\n",
922
+ "for model, path in input_files.items():\n",
923
+ " if not path.exists():\n",
924
+ " print(f\"Warning: {path} does not exist!\")\n",
925
+ " all_exist = False\n",
926
+ "\n",
927
+ "if all_exist:\n",
928
+ " # Define output file\n",
929
+ " output_file = current_dir.parent / \"data/CSV/combined_llm_annotations.csv\"\n",
930
+ " \n",
931
+ " # Combine the files\n",
932
+ " combined_df = combine_llm_annotations(input_files, output_file)\n",
933
+ " \n",
934
+ " # Show comparison\n",
935
+ " get_column_comparison(combined_df, list(input_files.keys()))\n",
936
+ " \n",
937
+ " # Show summary statistics\n",
938
+ " print(\"\\n\" + \"=\"*60)\n",
939
+ " print(\"SUMMARY STATISTICS\")\n",
940
+ " print(\"=\"*60)\n",
941
+ " \n",
942
+ " for model in input_files.keys():\n",
943
+ " country_col = f'{model}_country'\n",
944
+ " if country_col in combined_df.columns:\n",
945
+ " print(f\"\\n{model.upper()} Country Distribution:\")\n",
946
+ " print(combined_df[country_col].value_counts().head(5))\n",
947
+ " \n",
948
+ " print(\"\\n\" + \"=\"*60)\n",
949
+ " print(\"FINAL COLUMNS IN COMBINED FILE (GROUPED BY TYPE):\")\n",
950
+ " print(\"=\"*60)\n",
951
+ " \n",
952
+ " # Group columns by type for better display\n",
953
+ " common_cols = [col for col in combined_df.columns if not any(col.startswith(m + '_') for m in input_files.keys())]\n",
954
+ " \n",
955
+ " print(\"\\nCommon columns:\")\n",
956
+ " for col in common_cols[:20]: # Show first 20 common columns\n",
957
+ " print(f\" - {col}\")\n",
958
+ " if len(common_cols) > 20:\n",
959
+ " print(f\" ... and {len(common_cols)-20} more\")\n",
960
+ " \n",
961
+ " # Show columns grouped by type\n",
962
+ " llm_columns = ['full_name', 'aliases', 'gender', 'profession_llm', 'country']\n",
963
+ " for col_type in llm_columns:\n",
964
+ " type_cols = [col for col in combined_df.columns if col.endswith(f'_{col_type}')]\n",
965
+ " if type_cols:\n",
966
+ " print(f\"\\n{col_type.capitalize().replace('_', ' ')} columns:\")\n",
967
+ " for col in type_cols:\n",
968
+ " print(f\" - {col}\")\n",
969
+ " \n",
970
+ " print(\"\\n\" + \"=\"*60)\n",
971
+ " print(f\"✓ Combined file saved to: {output_file.name}\")\n",
972
+ " print(f\" Total rows: {len(combined_df)}\")\n",
973
+ " print(f\" Total columns: {len(combined_df.columns)}\")\n",
974
+ " print(\"=\"*60)\n",
975
+ "\n",
976
+ "else:\n",
977
+ " print(\"\\nPlease ensure all standardized files exist before running this script.\")\n",
978
+ " print(\"Expected files:\")\n",
979
+ " for model, path in input_files.items():\n",
980
+ " print(f\" - {path}\")"
981
+ ]
982
+ },
983
+ {
984
+ "cell_type": "markdown",
985
+ "id": "e0f8f331-a6f3-49e8-9734-ee7ffafebabc",
986
+ "metadata": {},
987
+ "source": [
988
+ "# ANALYSIS OF MODEL AGREEMENT WITH REFINED NAME MATCHING\n",
989
+ "\n",
990
+ "RULES FOR VALIDATION:\n",
991
+ "1. A row is considered \"valid\" if at least two models agree on:\n",
992
+ " - Country: Exact match required\n",
993
+ " - Gender: Exact match required \n",
994
+ " - Profession: Values are comma-separated lists; considered matching if:\n",
995
+ " a) They have at least two professions in common (order doesn'tmatter)\n",
996
+ "\n",
997
+ "2. For names (full_name) - REFINED RULES:\n",
998
+ " - Case-insensitive comparison\n",
999
+ " - Remove content in parentheses/brackets before comparison\n",
1000
+ " - Normalize special characters and spacing\n",
1001
+ " - Split into tokens and apply flexible matching:\n",
1002
+ " * Minimum threshold: At least 50% of tokens must match\n",
1003
+ " * Allow variations like \"Olivia Rose Holt\", \"Olivia Hastings Holt\", \"Olivia Holt\"\n",
1004
+ " * Handle name variations like \"Kim Na-eun\" vs \"Kim Naeun\"\n",
1005
+ " * Consider partial matches when most tokens align\n",
1006
+ "\n",
1007
+ "3. All Three Models Agreement:\n",
1008
+ " - All three models must agree on country, gender, and have matching professions\n",
1009
+ " - For names: All three must meet the refined matching criteria with each other"
1010
+ ]
1011
+ },
1012
+ {
1013
+ "cell_type": "code",
1014
+ "execution_count": 26,
1015
+ "id": "fe38cf23-771d-4888-a5ed-f2b8beb17017",
1016
+ "metadata": {
1017
+ "execution": {
1018
+ "iopub.execute_input": "2025-12-07T15:54:36.393391Z",
1019
+ "iopub.status.busy": "2025-12-07T15:54:36.393240Z",
1020
+ "iopub.status.idle": "2025-12-07T15:54:49.470031Z",
1021
+ "shell.execute_reply": "2025-12-07T15:54:49.469484Z",
1022
+ "shell.execute_reply.started": "2025-12-07T15:54:36.393376Z"
1023
+ }
1024
+ },
1025
+ "outputs": [
1026
+ {
1027
+ "name": "stdout",
1028
+ "output_type": "stream",
1029
+ "text": [
1030
+ "Loading combined data from: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/CSV/combined_llm_annotations.csv\n"
1031
+ ]
1032
+ },
1033
+ {
1034
+ "name": "stderr",
1035
+ "output_type": "stream",
1036
+ "text": [
1037
+ "/tmp/ipykernel_1771575/114048965.py:455: DtypeWarning: Columns (52,53,54,55,56,57,58,59,60,61,62,63,64,65,66) have mixed types. Specify dtype option on import or set low_memory=False.\n",
1038
+ " df_combined = pd.read_csv(combined_file)\n"
1039
+ ]
1040
+ },
1041
+ {
1042
+ "name": "stdout",
1043
+ "output_type": "stream",
1044
+ "text": [
1045
+ "Loaded 50,861 rows with 67 columns\n",
1046
+ "Analyzing agreement between models: gemma, qwen, mistral\n",
1047
+ "Total rows to analyze: 50861\n",
1048
+ "\n",
1049
+ "============================================================\n",
1050
+ "AGREEMENT ANALYSIS RESULTS (WITH REFINED NAME MATCHING)\n",
1051
+ "============================================================\n",
1052
+ "\n",
1053
+ "Total rows analyzed: 50,861\n",
1054
+ "Rows with at least 2 models MEANINGFULLY agreeing (valid): 33,517 (65.9%)\n",
1055
+ "Rows with all 3 models MEANINGFULLY agreeing: 14,179 (27.9%)\n",
1056
+ "Rows with all-unknown values for any field: 6,489 (12.8%)\n",
1057
+ "\n",
1058
+ "Field-wise ANY agreement (including unknown matches):\n",
1059
+ " - Country: 37,685 rows (74.1%)\n",
1060
+ " - Gender: 42,147 rows (82.9%)\n",
1061
+ " - Profession: 42,356 rows (83.3%)\n",
1062
+ " - Name: 36,859 rows (72.5%)\n",
1063
+ "\n",
1064
+ "Field-wise MEANINGFUL agreement (excluding unknown matches):\n",
1065
+ " - Country: 34,120 rows (67.1%)\n",
1066
+ " - Gender: 41,753 rows (82.1%)\n",
1067
+ " - Profession: 42,356 rows (83.3%)\n",
1068
+ " - Name: 36,859 rows (72.5%)\n",
1069
+ "\n",
1070
+ "Examples of successful name matches with variations:\n",
1071
+ " Row 0:\n",
1072
+ " gemma: 'Lee Ji-eun' → cleaned: 'Lee Ji-eun'\n",
1073
+ " qwen: 'Lee Ji-eun' → cleaned: 'Lee Ji-eun'\n",
1074
+ " mistral: 'Lee Ji-eun (Lee, Ji-eun)' → cleaned: 'Lee Ji-eun'\n",
1075
+ " Meaningful name agreements: 3/3\n",
1076
+ " Row 2:\n",
1077
+ " gemma: 'Li Yuchan' → cleaned: 'Li Yuchan'\n",
1078
+ " qwen: 'Li Yuxiao' → cleaned: 'Li Yuxiao'\n",
1079
+ " mistral: 'Liyuu (Full legal name unknown)' → cleaned: 'Liyuu'\n",
1080
+ " Meaningful name agreements: 1/3\n",
1081
+ " Row 3:\n",
1082
+ " gemma: 'Irene Kim' → cleaned: 'Irene Kim'\n",
1083
+ " qwen: 'Irene Kim' → cleaned: 'Irene Kim'\n",
1084
+ " mistral: 'Irene (Full legal name unknown)' → cleaned: 'Irene'\n",
1085
+ " Meaningful name agreements: 3/3\n",
1086
+ "\n",
1087
+ "Detailed MEANINGFUL Agreement Patterns:\n",
1088
+ "\n",
1089
+ "Number of MEANINGFUL agreeing pairs per field (out of 3 possible pairs):\n",
1090
+ "\n",
1091
+ "Country:\n",
1092
+ " - 0 meaningful agreeing pairs: 16,741 rows (32.9%)\n",
1093
+ " - 1 meaningful agreeing pairs: 18,697 rows (36.8%)\n",
1094
+ " - 3 meaningful agreeing pairs: 15,423 rows (30.3%)\n",
1095
+ "\n",
1096
+ "Gender:\n",
1097
+ " - 0 meaningful agreeing pairs: 9,108 rows (17.9%)\n",
1098
+ " - 1 meaningful agreeing pairs: 14,756 rows (29.0%)\n",
1099
+ " - 3 meaningful agreeing pairs: 26,997 rows (53.1%)\n",
1100
+ "\n",
1101
+ "Profession:\n",
1102
+ " - 0 meaningful agreeing pairs: 8,505 rows (16.7%)\n",
1103
+ " - 1 meaningful agreeing pairs: 14,318 rows (28.2%)\n",
1104
+ " - 2 meaningful agreeing pairs: 1,538 rows (3.0%)\n",
1105
+ " - 3 meaningful agreeing pairs: 26,500 rows (52.1%)\n",
1106
+ "\n",
1107
+ "Name:\n",
1108
+ " - 0 meaningful agreeing pairs: 14,002 rows (27.5%)\n",
1109
+ " - 1 meaningful agreeing pairs: 14,494 rows (28.5%)\n",
1110
+ " - 2 meaningful agreeing pairs: 500 rows (1.0%)\n",
1111
+ " - 3 meaningful agreeing pairs: 21,865 rows (43.0%)\n",
1112
+ "\n",
1113
+ "Saved analyzed data to: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/CSV/analyzed_llm_agreement.csv\n",
1114
+ "Saved valid rows (33,517) to: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/CSV/analyzed_llm_agreement_valid.csv\n",
1115
+ "Saved MEANINGFUL consensus rows (14,179) to: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/CSV/analyzed_llm_agreement_consensus.csv\n",
1116
+ "Saved all-unknown rows (6,489) to: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/CSV/analyzed_llm_agreement_all_unknown.csv\n",
1117
+ "\n",
1118
+ "============================================================\n",
1119
+ "SAMPLE OF MEANINGFUL CONSENSUS ROWS (first 2 rows)\n",
1120
+ "============================================================\n",
1121
+ "\n",
1122
+ "Consensus Row 0:\n",
1123
+ " gemma:\n",
1124
+ " Name: Lee Ji-eun\n",
1125
+ " Country: South Korea\n",
1126
+ " Gender: Female\n",
1127
+ " Profession: singer/musician, actor, tv personality\n",
1128
+ " qwen:\n",
1129
+ " Name: Lee Ji-eun\n",
1130
+ " Country: South Korea\n",
1131
+ " Gender: Female\n",
1132
+ " Profession: singer/musician, model, public figure\n",
1133
+ " mistral:\n",
1134
+ " Name: Lee Ji-eun (Lee, Ji-eun)\n",
1135
+ " Country: South Korea\n",
1136
+ " Gender: Female\n",
1137
+ " Profession: singer/musician, tv personality, actress\n",
1138
+ "\n",
1139
+ "Consensus Row 4:\n",
1140
+ " gemma:\n",
1141
+ " Name: Yoo Jimin\n",
1142
+ " Country: South Korea\n",
1143
+ " Gender: Female\n",
1144
+ " Profession: singer/musician, tv personality, online personality\n",
1145
+ " qwen:\n",
1146
+ " Name: Kim Karina\n",
1147
+ " Country: South Korea\n",
1148
+ " Gender: Female\n",
1149
+ " Profession: singer/musician, model, online personality\n",
1150
+ " mistral:\n",
1151
+ " Name: Karina (Kim Jung-yeon)\n",
1152
+ " Country: South Korea\n",
1153
+ " Gender: Female\n",
1154
+ " Profession: singer/musician, tv personality, public figure\n"
1155
+ ]
1156
+ }
1157
+ ],
1158
+ "source": [
1159
+ "import pandas as pd\n",
1160
+ "import numpy as np\n",
1161
+ "import re\n",
1162
+ "from typing import List, Set, Tuple\n",
1163
+ "\n",
1164
+ "def is_unknown_value(value) -> bool:\n",
1165
+ " \"\"\"Check if a value is considered 'unknown' or empty.\"\"\"\n",
1166
+ " if pd.isna(value):\n",
1167
+ " return True\n",
1168
+ " \n",
1169
+ " value_str = str(value).strip().lower()\n",
1170
+ " \n",
1171
+ " # Common unknown/empty indicators\n",
1172
+ " unknown_patterns = [\n",
1173
+ " 'unknown',\n",
1174
+ " 'n/a',\n",
1175
+ " 'na',\n",
1176
+ " 'none',\n",
1177
+ " '',\n",
1178
+ " 'null',\n",
1179
+ " 'not specified',\n",
1180
+ " 'not available',\n",
1181
+ " 'unspecified',\n",
1182
+ " '?',\n",
1183
+ " '??'\n",
1184
+ " ]\n",
1185
+ " \n",
1186
+ " return value_str in unknown_patterns or len(value_str) == 0\n",
1187
+ "\n",
1188
+ "def clean_name_text(name: str) -> str:\n",
1189
+ " \"\"\"Clean and normalize name text for comparison.\"\"\"\n",
1190
+ " if pd.isna(name):\n",
1191
+ " return \"\"\n",
1192
+ " \n",
1193
+ " name_str = str(name)\n",
1194
+ " \n",
1195
+ " # Remove content in parentheses, brackets, and their variations\n",
1196
+ " name_str = re.sub(r'\\([^)]*\\)', '', name_str) # Remove (legal name unknown)\n",
1197
+ " name_str = re.sub(r'\\[[^\\]]*\\]', '', name_str) # Remove [anything]\n",
1198
+ " name_str = re.sub(r'\\{[^}]*\\}', '', name_str) # Remove {anything}\n",
1199
+ " \n",
1200
+ " # Remove common explanatory phrases\n",
1201
+ " explanatory_phrases = [\n",
1202
+ " r'legal name unknown',\n",
1203
+ " r'full name',\n",
1204
+ " r'also known as',\n",
1205
+ " r'aka',\n",
1206
+ " r'birth name',\n",
1207
+ " r'real name',\n",
1208
+ " r'stage name'\n",
1209
+ " ]\n",
1210
+ " \n",
1211
+ " for phrase in explanatory_phrases:\n",
1212
+ " name_str = re.sub(phrase, '', name_str, flags=re.IGNORECASE)\n",
1213
+ " \n",
1214
+ " # Normalize special characters\n",
1215
+ " name_str = name_str.strip()\n",
1216
+ " \n",
1217
+ " # Replace multiple spaces with single space\n",
1218
+ " name_str = re.sub(r'\\s+', ' ', name_str)\n",
1219
+ " \n",
1220
+ " return name_str\n",
1221
+ "\n",
1222
+ "def normalize_name(name: str) -> List[str]:\n",
1223
+ " \"\"\"Normalize name by cleaning, converting to lowercase, and splitting into tokens.\"\"\"\n",
1224
+ " if pd.isna(name):\n",
1225
+ " return []\n",
1226
+ " \n",
1227
+ " # Clean the name text first\n",
1228
+ " cleaned_name = clean_name_text(name)\n",
1229
+ " \n",
1230
+ " if not cleaned_name:\n",
1231
+ " return []\n",
1232
+ " \n",
1233
+ " # Check if it's an unknown name after cleaning\n",
1234
+ " if is_unknown_value(cleaned_name):\n",
1235
+ " return []\n",
1236
+ " \n",
1237
+ " # Convert to lowercase\n",
1238
+ " name_lower = cleaned_name.lower()\n",
1239
+ " \n",
1240
+ " # Normalize hyphenated names and special characters\n",
1241
+ " # Convert \"na-eun\" to \"naeun\" for better matching\n",
1242
+ " name_lower = re.sub(r'([a-z])-([a-z])', r'\\1\\2', name_lower)\n",
1243
+ " \n",
1244
+ " # Split into tokens\n",
1245
+ " tokens = name_lower.split()\n",
1246
+ " \n",
1247
+ " # Remove common prefixes/suffixes that don't affect identity\n",
1248
+ " common_elements = {'jr', 'sr', 'ii', 'iii', 'iv', 'v', 'dr', 'mr', 'mrs', 'ms', 'prof'}\n",
1249
+ " tokens = [t for t in tokens if t not in common_elements]\n",
1250
+ " \n",
1251
+ " return tokens\n",
1252
+ "\n",
1253
+ "def names_match(name1: str, name2: str, threshold: float = 0.5) -> bool:\n",
1254
+ " \"\"\"Check if two names match with refined partial matching.\"\"\"\n",
1255
+ " # Check if either value is unknown\n",
1256
+ " if is_unknown_value(name1) or is_unknown_value(name2):\n",
1257
+ " return False\n",
1258
+ " \n",
1259
+ " tokens1 = normalize_name(name1)\n",
1260
+ " tokens2 = normalize_name(name2)\n",
1261
+ " \n",
1262
+ " # If either is empty after normalization (was unknown), no match\n",
1263
+ " if not tokens1 or not tokens2:\n",
1264
+ " return False\n",
1265
+ " \n",
1266
+ " # Special case: if one name is clearly a subset of the other\n",
1267
+ " # e.g., \"Olivia Holt\" vs \"Olivia Rose Holt\"\n",
1268
+ " set1 = set(tokens1)\n",
1269
+ " set2 = set(tokens2)\n",
1270
+ " \n",
1271
+ " # Check if one is subset of the other\n",
1272
+ " if set1.issubset(set2) or set2.issubset(set1):\n",
1273
+ " return True\n",
1274
+ " \n",
1275
+ " # Calculate token overlap\n",
1276
+ " overlap = len(set1.intersection(set2))\n",
1277
+ " \n",
1278
+ " # Special handling for Asian names with/without hyphens\n",
1279
+ " # Create versions without hyphens for comparison\n",
1280
+ " tokens1_no_hyphen = [t.replace('-', '') for t in tokens1]\n",
1281
+ " tokens2_no_hyphen = [t.replace('-', '') for t in tokens2]\n",
1282
+ " \n",
1283
+ " set1_no_hyphen = set(tokens1_no_hyphen)\n",
1284
+ " set2_no_hyphen = set(tokens2_no_hyphen)\n",
1285
+ " \n",
1286
+ " overlap_no_hyphen = len(set1_no_hyphen.intersection(set2_no_hyphen))\n",
1287
+ " \n",
1288
+ " # Use the better overlap\n",
1289
+ " best_overlap = max(overlap, overlap_no_hyphen)\n",
1290
+ " \n",
1291
+ " # Calculate match ratio based on the shorter name\n",
1292
+ " min_length = min(len(tokens1), len(tokens2))\n",
1293
+ " match_ratio = best_overlap / min_length if min_length > 0 else 0\n",
1294
+ " \n",
1295
+ " return match_ratio >= threshold\n",
1296
+ "\n",
1297
+ "def normalize_professions(prof_str: str) -> Set[str]:\n",
1298
+ " \"\"\"Normalize profession strings to sets for comparison.\"\"\"\n",
1299
+ " if pd.isna(prof_str) or is_unknown_value(prof_str):\n",
1300
+ " return set()\n",
1301
+ " \n",
1302
+ " # Split by comma, strip whitespace, convert to lowercase\n",
1303
+ " professions = [p.strip().lower() for p in str(prof_str).split(',') if p.strip() and not is_unknown_value(p)]\n",
1304
+ " return set(professions)\n",
1305
+ "\n",
1306
+ "def professions_match(prof1: str, prof2: str) -> bool:\n",
1307
+ " \"\"\"Check if two profession strings match (order doesn't matter).\"\"\"\n",
1308
+ " set1 = normalize_professions(prof1)\n",
1309
+ " set2 = normalize_professions(prof2)\n",
1310
+ " \n",
1311
+ " # Both empty or unknown - not a meaningful match\n",
1312
+ " if not set1 and not set2:\n",
1313
+ " return False\n",
1314
+ " \n",
1315
+ " # One is empty/unknown, the other has content - not a match\n",
1316
+ " if (not set1 and set2) or (set1 and not set2):\n",
1317
+ " return False\n",
1318
+ " \n",
1319
+ " # Match if they share at least one profession\n",
1320
+ " return len(set1.intersection(set2)) > 0\n",
1321
+ "\n",
1322
+ "def check_model_agreement(row: pd.Series, models: List[str]) -> dict:\n",
1323
+ " \"\"\"\n",
1324
+ " Check agreement between models for a single row.\n",
1325
+ " \n",
1326
+ " Returns:\n",
1327
+ " Dictionary with agreement results for this row\n",
1328
+ " \"\"\"\n",
1329
+ " results = {\n",
1330
+ " 'row_valid': False,\n",
1331
+ " 'all_agree': False,\n",
1332
+ " 'agreements': {\n",
1333
+ " 'country': 0,\n",
1334
+ " 'gender': 0,\n",
1335
+ " 'profession': 0,\n",
1336
+ " 'name': 0\n",
1337
+ " },\n",
1338
+ " 'meaningful_agreements': {\n",
1339
+ " 'country': 0, # Agreements that are NOT all unknown\n",
1340
+ " 'gender': 0,\n",
1341
+ " 'profession': 0,\n",
1342
+ " 'name': 0\n",
1343
+ " },\n",
1344
+ " 'name_variations': [], # Store name variations for debugging\n",
1345
+ " 'has_unknown_values': False # Track if row has unknown values\n",
1346
+ " }\n",
1347
+ " \n",
1348
+ " # Collect values from each model\n",
1349
+ " countries = [row.get(f'{model}_country', None) for model in models]\n",
1350
+ " genders = [row.get(f'{model}_gender', None) for model in models]\n",
1351
+ " professions = [row.get(f'{model}_profession_llm', None) for model in models]\n",
1352
+ " raw_names = [row.get(f'{model}_full_name', None) for model in models]\n",
1353
+ " cleaned_names = [clean_name_text(name) if pd.notna(name) else \"\" for name in raw_names]\n",
1354
+ " \n",
1355
+ " # Check if any field has all unknown values\n",
1356
+ " all_countries_unknown = all(is_unknown_value(c) for c in countries)\n",
1357
+ " all_genders_unknown = all(is_unknown_value(g) for g in genders)\n",
1358
+ " all_professions_unknown = all(is_unknown_value(p) for p in professions)\n",
1359
+ " all_names_unknown = all(is_unknown_value(n) for n in raw_names)\n",
1360
+ " \n",
1361
+ " results['has_unknown_values'] = (all_countries_unknown or all_genders_unknown or \n",
1362
+ " all_professions_unknown or all_names_unknown)\n",
1363
+ " \n",
1364
+ " # Store name variations for debugging\n",
1365
+ " results['name_variations'] = list(zip(models, raw_names, cleaned_names))\n",
1366
+ " \n",
1367
+ " # Count pairwise agreements\n",
1368
+ " country_agreements = 0\n",
1369
+ " gender_agreements = 0\n",
1370
+ " profession_agreements = 0\n",
1371
+ " name_agreements = 0\n",
1372
+ " \n",
1373
+ " # Count meaningful agreements (excluding unknown matches)\n",
1374
+ " meaningful_country_agreements = 0\n",
1375
+ " meaningful_gender_agreements = 0\n",
1376
+ " meaningful_profession_agreements = 0\n",
1377
+ " meaningful_name_agreements = 0\n",
1378
+ " \n",
1379
+ " # Check all pairs of models\n",
1380
+ " n_models = len(models)\n",
1381
+ " for i in range(n_models):\n",
1382
+ " for j in range(i + 1, n_models):\n",
1383
+ " # Country agreement (exact match)\n",
1384
+ " if pd.notna(countries[i]) and pd.notna(countries[j]) and countries[i] == countries[j]:\n",
1385
+ " country_agreements += 1\n",
1386
+ " # Check if it's a meaningful agreement (not unknown)\n",
1387
+ " if not is_unknown_value(countries[i]) and not is_unknown_value(countries[j]):\n",
1388
+ " meaningful_country_agreements += 1\n",
1389
+ " \n",
1390
+ " # Gender agreement (exact match)\n",
1391
+ " if pd.notna(genders[i]) and pd.notna(genders[j]) and genders[i] == genders[j]:\n",
1392
+ " gender_agreements += 1\n",
1393
+ " # Check if it's a meaningful agreement (not unknown)\n",
1394
+ " if not is_unknown_value(genders[i]) and not is_unknown_value(genders[j]):\n",
1395
+ " meaningful_gender_agreements += 1\n",
1396
+ " \n",
1397
+ " # Profession agreement (set-based with overlap)\n",
1398
+ " if pd.notna(professions[i]) and pd.notna(professions[j]) and professions_match(professions[i], professions[j]):\n",
1399
+ " profession_agreements += 1\n",
1400
+ " # Check if it's a meaningful agreement (not unknown)\n",
1401
+ " if not is_unknown_value(professions[i]) and not is_unknown_value(professions[j]):\n",
1402
+ " meaningful_profession_agreements += 1\n",
1403
+ " \n",
1404
+ " # Name agreement (refined partial match)\n",
1405
+ " if pd.notna(raw_names[i]) and pd.notna(raw_names[j]) and names_match(raw_names[i], raw_names[j]):\n",
1406
+ " name_agreements += 1\n",
1407
+ " # Check if it's a meaningful agreement (not unknown)\n",
1408
+ " if not is_unknown_value(raw_names[i]) and not is_unknown_value(raw_names[j]):\n",
1409
+ " meaningful_name_agreements += 1\n",
1410
+ " \n",
1411
+ " # Store agreement counts\n",
1412
+ " results['agreements']['country'] = country_agreements\n",
1413
+ " results['agreements']['gender'] = gender_agreements\n",
1414
+ " results['agreements']['profession'] = profession_agreements\n",
1415
+ " results['agreements']['name'] = name_agreements\n",
1416
+ " \n",
1417
+ " results['meaningful_agreements']['country'] = meaningful_country_agreements\n",
1418
+ " results['meaningful_agreements']['gender'] = meaningful_gender_agreements\n",
1419
+ " results['meaningful_agreements']['profession'] = meaningful_profession_agreements\n",
1420
+ " results['meaningful_agreements']['name'] = meaningful_name_agreements\n",
1421
+ " \n",
1422
+ " # Check if row is valid (at least 2 models agree on all three fields)\n",
1423
+ " # We need at least one MEANINGFUL agreement pair for country, gender, AND profession\n",
1424
+ " valid_country = meaningful_country_agreements >= 1\n",
1425
+ " valid_gender = meaningful_gender_agreements >= 1\n",
1426
+ " valid_profession = meaningful_profession_agreements >= 1\n",
1427
+ " \n",
1428
+ " results['row_valid'] = valid_country and valid_gender and valid_profession\n",
1429
+ " \n",
1430
+ " # Check if all three models agree on everything MEANINGFULLY\n",
1431
+ " # Need all possible pairs (3 pairs for 3 models) to have meaningful agreement\n",
1432
+ " # AND we must have at least one non-unknown value per model for each field\n",
1433
+ " all_agree_country = meaningful_country_agreements == 3 # 3 meaningful pairs\n",
1434
+ " all_agree_gender = meaningful_gender_agreements == 3\n",
1435
+ " all_agree_profession = meaningful_profession_agreements == 3\n",
1436
+ " \n",
1437
+ " # For names, we need all pairs to have meaningful agreement\n",
1438
+ " all_agree_name = meaningful_name_agreements == 3\n",
1439
+ " \n",
1440
+ " # All three models must agree meaningfully on country, gender, and profession\n",
1441
+ " results['all_agree'] = (all_agree_country and all_agree_gender and \n",
1442
+ " all_agree_profession and not results['has_unknown_values'])\n",
1443
+ " \n",
1444
+ " return results\n",
1445
+ "\n",
1446
+ "def analyze_model_agreement(df: pd.DataFrame, models: List[str]) -> pd.DataFrame:\n",
1447
+ " \"\"\"\n",
1448
+ " Analyze agreement between models and add agreement columns.\n",
1449
+ " \n",
1450
+ " Returns:\n",
1451
+ " DataFrame with added agreement analysis columns\n",
1452
+ " \"\"\"\n",
1453
+ " print(f\"Analyzing agreement between models: {', '.join(models)}\")\n",
1454
+ " print(f\"Total rows to analyze: {len(df)}\")\n",
1455
+ " \n",
1456
+ " # Prepare results storage\n",
1457
+ " analysis_results = []\n",
1458
+ " \n",
1459
+ " # Track overall statistics\n",
1460
+ " total_rows = len(df)\n",
1461
+ " valid_rows = 0\n",
1462
+ " all_agree_rows = 0\n",
1463
+ " unknown_only_rows = 0 # Rows where all models say unknown for any field\n",
1464
+ " field_agreements = {'country': 0, 'gender': 0, 'profession': 0, 'name': 0}\n",
1465
+ " meaningful_field_agreements = {'country': 0, 'gender': 0, 'profession': 0, 'name': 0}\n",
1466
+ " \n",
1467
+ " # Track name matching examples for debugging\n",
1468
+ " name_match_examples = []\n",
1469
+ " name_mismatch_examples = []\n",
1470
+ " \n",
1471
+ " # Analyze each row\n",
1472
+ " for idx, row in df.iterrows():\n",
1473
+ " results = check_model_agreement(row, models)\n",
1474
+ " analysis_results.append(results)\n",
1475
+ " \n",
1476
+ " # Update counters\n",
1477
+ " if results['row_valid']:\n",
1478
+ " valid_rows += 1\n",
1479
+ " if results['all_agree']:\n",
1480
+ " all_agree_rows += 1\n",
1481
+ " if results['has_unknown_values']:\n",
1482
+ " unknown_only_rows += 1\n",
1483
+ " \n",
1484
+ " # Track field agreements (rows with at least one agreement pair)\n",
1485
+ " if results['agreements']['country'] > 0:\n",
1486
+ " field_agreements['country'] += 1\n",
1487
+ " if results['agreements']['gender'] > 0:\n",
1488
+ " field_agreements['gender'] += 1\n",
1489
+ " if results['agreements']['profession'] > 0:\n",
1490
+ " field_agreements['profession'] += 1\n",
1491
+ " if results['agreements']['name'] > 0:\n",
1492
+ " field_agreements['name'] += 1\n",
1493
+ " \n",
1494
+ " # Track meaningful field agreements\n",
1495
+ " if results['meaningful_agreements']['country'] > 0:\n",
1496
+ " meaningful_field_agreements['country'] += 1\n",
1497
+ " if results['meaningful_agreements']['gender'] > 0:\n",
1498
+ " meaningful_field_agreements['gender'] += 1\n",
1499
+ " if results['meaningful_agreements']['profession'] > 0:\n",
1500
+ " meaningful_field_agreements['profession'] += 1\n",
1501
+ " if results['meaningful_agreements']['name'] > 0:\n",
1502
+ " meaningful_field_agreements['name'] += 1\n",
1503
+ " \n",
1504
+ " # Collect name examples for debugging\n",
1505
+ " if len(name_match_examples) < 5 and results['meaningful_agreements']['name'] > 0:\n",
1506
+ " name_variations = results['name_variations']\n",
1507
+ " if any(n1 != n2 for _, n1, _ in name_variations for _, n2, _ in name_variations if n1 != n2):\n",
1508
+ " name_match_examples.append({\n",
1509
+ " 'row_idx': idx,\n",
1510
+ " 'raw_names': [n for _, n, _ in name_variations],\n",
1511
+ " 'cleaned_names': [c for _, _, c in name_variations],\n",
1512
+ " 'agreements': results['meaningful_agreements']['name']\n",
1513
+ " })\n",
1514
+ " \n",
1515
+ " # Convert results to DataFrame\n",
1516
+ " results_df = pd.DataFrame(analysis_results)\n",
1517
+ " \n",
1518
+ " # Add agreement columns to original dataframe\n",
1519
+ " df_analyzed = df.copy()\n",
1520
+ " df_analyzed['row_valid'] = results_df['row_valid']\n",
1521
+ " df_analyzed['all_models_agree'] = results_df['all_agree']\n",
1522
+ " df_analyzed['has_unknown_values'] = results_df['has_unknown_values']\n",
1523
+ " df_analyzed['country_agreements'] = results_df['agreements'].apply(lambda x: x['country'])\n",
1524
+ " df_analyzed['gender_agreements'] = results_df['agreements'].apply(lambda x: x['gender'])\n",
1525
+ " df_analyzed['profession_agreements'] = results_df['agreements'].apply(lambda x: x['profession'])\n",
1526
+ " df_analyzed['name_agreements'] = results_df['agreements'].apply(lambda x: x['name'])\n",
1527
+ " df_analyzed['meaningful_country_agreements'] = results_df['meaningful_agreements'].apply(lambda x: x['country'])\n",
1528
+ " df_analyzed['meaningful_gender_agreements'] = results_df['meaningful_agreements'].apply(lambda x: x['gender'])\n",
1529
+ " df_analyzed['meaningful_profession_agreements'] = results_df['meaningful_agreements'].apply(lambda x: x['profession'])\n",
1530
+ " df_analyzed['meaningful_name_agreements'] = results_df['meaningful_agreements'].apply(lambda x: x['name'])\n",
1531
+ " \n",
1532
+ " # Calculate overall statistics\n",
1533
+ " valid_percentage = (valid_rows / total_rows) * 100\n",
1534
+ " all_agree_percentage = (all_agree_rows / total_rows) * 100\n",
1535
+ " unknown_percentage = (unknown_only_rows / total_rows) * 100\n",
1536
+ " \n",
1537
+ " print(\"\\n\" + \"=\"*60)\n",
1538
+ " print(\"AGREEMENT ANALYSIS RESULTS (WITH REFINED NAME MATCHING)\")\n",
1539
+ " print(\"=\"*60)\n",
1540
+ " print(f\"\\nTotal rows analyzed: {total_rows:,}\")\n",
1541
+ " print(f\"Rows with at least 2 models MEANINGFULLY agreeing (valid): {valid_rows:,} ({valid_percentage:.1f}%)\")\n",
1542
+ " print(f\"Rows with all 3 models MEANINGFULLY agreeing: {all_agree_rows:,} ({all_agree_percentage:.1f}%)\")\n",
1543
+ " print(f\"Rows with all-unknown values for any field: {unknown_only_rows:,} ({unknown_percentage:.1f}%)\")\n",
1544
+ " \n",
1545
+ " print(f\"\\nField-wise ANY agreement (including unknown matches):\")\n",
1546
+ " for field, count in field_agreements.items():\n",
1547
+ " percentage = (count / total_rows) * 100\n",
1548
+ " print(f\" - {field.capitalize()}: {count:,} rows ({percentage:.1f}%)\")\n",
1549
+ " \n",
1550
+ " print(f\"\\nField-wise MEANINGFUL agreement (excluding unknown matches):\")\n",
1551
+ " for field, count in meaningful_field_agreements.items():\n",
1552
+ " percentage = (count / total_rows) * 100\n",
1553
+ " print(f\" - {field.capitalize()}: {count:,} rows ({percentage:.1f}%)\")\n",
1554
+ " \n",
1555
+ " # Show name matching examples\n",
1556
+ " if name_match_examples:\n",
1557
+ " print(f\"\\nExamples of successful name matches with variations:\")\n",
1558
+ " for example in name_match_examples[:3]:\n",
1559
+ " print(f\" Row {example['row_idx']}:\")\n",
1560
+ " for model, raw_name, cleaned_name in zip(models, example['raw_names'], example['cleaned_names']):\n",
1561
+ " print(f\" {model}: '{raw_name}' → cleaned: '{cleaned_name}'\")\n",
1562
+ " print(f\" Meaningful name agreements: {example['agreements']}/3\")\n",
1563
+ " \n",
1564
+ " # Detailed breakdown of agreement patterns\n",
1565
+ " print(f\"\\nDetailed MEANINGFUL Agreement Patterns:\")\n",
1566
+ " \n",
1567
+ " # Count different meaningful agreement levels\n",
1568
+ " print(f\"\\nNumber of MEANINGFUL agreeing pairs per field (out of 3 possible pairs):\")\n",
1569
+ " for field in ['country', 'gender', 'profession', 'name']:\n",
1570
+ " col_name = f'meaningful_{field}_agreements'\n",
1571
+ " print(f\"\\n{field.capitalize()}:\")\n",
1572
+ " for i in range(4): # 0, 1, 2, or 3 agreeing pairs\n",
1573
+ " count = (df_analyzed[col_name] == i).sum()\n",
1574
+ " if count > 0:\n",
1575
+ " percentage = (count / total_rows) * 100\n",
1576
+ " print(f\" - {i} meaningful agreeing pairs: {count:,} rows ({percentage:.1f}%)\")\n",
1577
+ " \n",
1578
+ " return df_analyzed\n",
1579
+ "\n",
1580
+ "def save_analysis_results(df_analyzed: pd.DataFrame, output_path: Path):\n",
1581
+ " \"\"\"Save the analyzed dataframe with agreement columns.\"\"\"\n",
1582
+ " # Save full analyzed dataset\n",
1583
+ " df_analyzed.to_csv(output_path, index=False)\n",
1584
+ " print(f\"\\nSaved analyzed data to: {output_path}\")\n",
1585
+ " \n",
1586
+ " # Save only valid rows (at least 2 models meaningfully agree)\n",
1587
+ " valid_path = output_path.parent / f\"{output_path.stem}_valid{output_path.suffix}\"\n",
1588
+ " valid_df = df_analyzed[df_analyzed['row_valid']]\n",
1589
+ " valid_df.to_csv(valid_path, index=False)\n",
1590
+ " print(f\"Saved valid rows ({len(valid_df):,}) to: {valid_path}\")\n",
1591
+ " \n",
1592
+ " # Save rows where all 3 models MEANINGFULLY agree (excluding unknown agreements)\n",
1593
+ " consensus_path = output_path.parent / f\"{output_path.stem}_consensus{output_path.suffix}\"\n",
1594
+ " consensus_df = df_analyzed[df_analyzed['all_models_agree']]\n",
1595
+ " consensus_df.to_csv(consensus_path, index=False)\n",
1596
+ " print(f\"Saved MEANINGFUL consensus rows ({len(consensus_df):,}) to: {consensus_path}\")\n",
1597
+ " \n",
1598
+ " # Also save rows that were excluded due to all-unknown values\n",
1599
+ " unknown_path = output_path.parent / f\"{output_path.stem}_all_unknown{output_path.suffix}\"\n",
1600
+ " unknown_df = df_analyzed[df_analyzed['has_unknown_values']]\n",
1601
+ " unknown_df.to_csv(unknown_path, index=False)\n",
1602
+ " print(f\"Saved all-unknown rows ({len(unknown_df):,}) to: {unknown_path}\")\n",
1603
+ "\n",
1604
+ "# ============================================================\n",
1605
+ "# RUN THE ANALYSIS\n",
1606
+ "# ============================================================\n",
1607
+ "\n",
1608
+ "# Load the combined data\n",
1609
+ "combined_file = current_dir.parent / \"data/CSV/combined_llm_annotations.csv\"\n",
1610
+ "print(f\"Loading combined data from: {combined_file}\")\n",
1611
+ "\n",
1612
+ "if combined_file.exists():\n",
1613
+ " df_combined = pd.read_csv(combined_file)\n",
1614
+ " print(f\"Loaded {len(df_combined):,} rows with {len(df_combined.columns)} columns\")\n",
1615
+ " \n",
1616
+ " # Define the models\n",
1617
+ " models = ['gemma', 'qwen', 'mistral']\n",
1618
+ " \n",
1619
+ " # Run the agreement analysis\n",
1620
+ " df_analyzed = analyze_model_agreement(df_combined, models)\n",
1621
+ " \n",
1622
+ " # Save the results\n",
1623
+ " output_file = current_dir.parent / \"data/CSV/analyzed_llm_agreement.csv\"\n",
1624
+ " save_analysis_results(df_analyzed, output_file)\n",
1625
+ " \n",
1626
+ " # Show sample of consensus rows\n",
1627
+ " print(\"\\n\" + \"=\"*60)\n",
1628
+ " print(\"SAMPLE OF MEANINGFUL CONSENSUS ROWS (first 2 rows)\")\n",
1629
+ " print(\"=\"*60)\n",
1630
+ " \n",
1631
+ " consensus_rows = df_analyzed[df_analyzed['all_models_agree']].head(2)\n",
1632
+ " \n",
1633
+ " if len(consensus_rows) > 0:\n",
1634
+ " for idx, row in consensus_rows.iterrows():\n",
1635
+ " print(f\"\\nConsensus Row {idx}:\")\n",
1636
+ " for model in models:\n",
1637
+ " name_col = f'{model}_full_name'\n",
1638
+ " country_col = f'{model}_country'\n",
1639
+ " gender_col = f'{model}_gender'\n",
1640
+ " prof_col = f'{model}_profession_llm'\n",
1641
+ " \n",
1642
+ " if name_col in row:\n",
1643
+ " print(f\" {model}:\")\n",
1644
+ " print(f\" Name: {row[name_col]}\")\n",
1645
+ " print(f\" Country: {row[country_col]}\")\n",
1646
+ " print(f\" Gender: {row[gender_col]}\")\n",
1647
+ " print(f\" Profession: {row[prof_col]}\")\n"
1648
+ ]
1649
+ },
1650
+ {
1651
+ "cell_type": "code",
1652
+ "execution_count": null,
1653
+ "id": "5df51c5d-9a40-4742-9916-cc4863770d56",
1654
+ "metadata": {},
1655
+ "outputs": [],
1656
+ "source": []
1657
+ }
1658
+ ],
1659
+ "metadata": {
1660
+ "kernelspec": {
1661
+ "display_name": "Python 3 (ipykernel)",
1662
+ "language": "python",
1663
+ "name": "python3"
1664
+ },
1665
+ "language_info": {
1666
+ "codemirror_mode": {
1667
+ "name": "ipython",
1668
+ "version": 3
1669
+ },
1670
+ "file_extension": ".py",
1671
+ "mimetype": "text/x-python",
1672
+ "name": "python",
1673
+ "nbconvert_exporter": "python",
1674
+ "pygments_lexer": "ipython3",
1675
+ "version": "3.12.10"
1676
+ }
1677
+ },
1678
+ "nbformat": 4,
1679
+ "nbformat_minor": 5
1680
+ }
misc/query_indicies/.ipynb_checkpoints/gemma_local_query_index-checkpoint.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 42867
misc/query_indicies/gemma_local_query_index.txt CHANGED
@@ -1 +1 @@
1
- 22720
 
1
+ 49840
misc/query_indicies/mistral_local_query_index.txt CHANGED
@@ -1 +1 @@
1
- 25820
 
1
+ 30550
misc/query_indicies/qwen_local_query_index.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 44060