added deepfake analysis script

scripts/Section_3-3-4_deepfakes.ipynb

@@ -0,0 +1,124 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "a58d08a9",
+   "metadata": {},
+   "source": [
+    "# LLM annotation of Deepfake adapter dataset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "38c3a6a6",
+   "metadata": {},
+   "source": [
+    "## Step 1 NER"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ad478ed2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from pathlib import Path\n",
+    "\n",
+    "# Set up paths\n",
+    "current_dir = Path.cwd()\n",
+    "countries = current_dir.parent / \"misc/lists/countries.csv\"\n",
+    "professions = current_dir.parent / \"misc/lists/professions.csv\"\n",
+    "inputNER = current_dir.parent / \"data/CSV/NER_POI_step01_pre_annotation.csv\"\n",
+    "outfile = current_dir.parent / \"data/CSV/NER_POI_step02_annotated.csv\"\n",
+    "\n",
+    "# Load datasets\n",
+    "poi_df = pd.read_csv(inputNER)\n",
+    "countries_df = pd.read_csv(countries)\n",
+    "professions_df = pd.read_csv(professions)\n",
+    "\n",
+    "# Step 1: Combine tags into one lowercase list\n",
+    "def combine_tags(row):\n",
+    "    return [str(row[f\"tag_{i}\"]).strip().lower() for i in range(1, 8) if pd.notna(row.get(f\"tag_{i}\"))]\n",
+    "\n",
+    "poi_df[\"tags\"] = poi_df.apply(combine_tags, axis=1)\n",
+    "\n",
+    "# Step 2: Build tag → (country, nationality) mapping\n",
+    "tag_to_country_nationality = {}\n",
+    "\n",
+    "for _, row in countries_df.iterrows():\n",
+    "    country = str(row[\"en_short_name\"]).strip()\n",
+    "    nationality = str(row[\"nationality\"]).strip()\n",
+    "\n",
+    "    country_lc = country.lower()\n",
+    "    nationality_lc = nationality.lower()\n",
+    "\n",
+    "    # Add variations to the mapping\n",
+    "    tag_to_country_nationality[country_lc] = (country, \"\")\n",
+    "    tag_to_country_nationality[nationality_lc] = (\"\", nationality)\n",
+    "    tag_to_country_nationality[country_lc.replace(\" \", \"\")] = (country, \"\")\n",
+    "    tag_to_country_nationality[nationality_lc.replace(\" \", \"\")] = (\"\", nationality)\n",
+    "\n",
+    "    for part in country_lc.split():\n",
+    "        tag_to_country_nationality[part] = (country, \"\")\n",
+    "    for part in nationality_lc.split():\n",
+    "        tag_to_country_nationality[part] = (\"\", nationality)\n",
+    "\n",
+    "# Step 3: Infer likely_country and likely_nationality\n",
+    "def infer_country_and_nationality(tags):\n",
+    "    for tag in tags:\n",
+    "        cleaned = tag.replace(\" \", \"\").lower()\n",
+    "        if cleaned in tag_to_country_nationality:\n",
+    "            country, nationality = tag_to_country_nationality[cleaned]\n",
+    "            return pd.Series([country, nationality])\n",
+    "    return pd.Series([\"\", \"\"])\n",
+    "\n",
+    "poi_df[[\"likely_country\", \"likely_nationality\"]] = poi_df[\"tags\"].apply(infer_country_and_nationality)\n",
+    "\n",
+    "# Step 4: Build tag → profession mapping\n",
+    "profession_alias_map = {}\n",
+    "\n",
+    "for _, row in professions_df.iterrows():\n",
+    "    canonical = str(row['profession']).strip().lower()\n",
+    "    profession_alias_map[canonical] = canonical\n",
+    "    for alias_col in ['alias_1', 'alias_2', 'alias_3']:\n",
+    "        alias = row.get(alias_col)\n",
+    "        if pd.notna(alias):\n",
+    "            profession_alias_map[str(alias).strip().lower()] = canonical\n",
+    "\n",
+    "# Step 5: Infer likely profession from tags\n",
+    "def infer_profession_from_tags(tags):\n",
+    "    matched = []\n",
+    "    for tag in tags:\n",
+    "        cleaned = tag.strip().lower()\n",
+    "        if cleaned in profession_alias_map:\n",
+    "            matched.append(profession_alias_map[cleaned])\n",
+    "\n",
+    "    if not matched:\n",
+    "        return \"\"\n",
+    "    if \"celebrity\" in matched and len(set(matched)) > 1:\n",
+    "        # Drop 'celebrity' if other professions are present\n",
+    "        matched = [m for m in matched if m != \"celebrity\"]\n",
+    "\n",
+    "    return matched[0]  # Return the first specific match\n",
+    "\n",
+    "\n",
+    "poi_df[\"likely_profession\"] = poi_df[\"tags\"].apply(infer_profession_from_tags)\n",
+    "\n",
+    "# Step 6: Save enriched dataset\n",
+    "poi_df.to_csv(outfile, index=False)\n",
+    "\n",
+    "# Optional: Preview\n",
+    "print(poi_df[[\"real_name\", \"likely_country\", \"likely_nationality\", \"likely_profession\"]].head())\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}