change visualization to bar chart

jupyter_notebooks/Section_2-3-4_Figure_8_Step_1_LLM_annotation.ipynb

@@ -10,14 +10,496 @@
   },
   {
    "cell_type": "markdown",
+   "id": "e4407358",
    "metadata": {},
    "source": [
-    "### Unified Model Loading & Inference\nAbstracted interface for loading and querying Mistral, Gemma, and Qwen models."
+    "### Unified Model Loading & Inference\n",
+    "Code for querying Mistral, Gemma, and Qwen models."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1a1b9d0e",
+   "metadata": {},
+   "source": [
+    "## CLEANING & PREPROCESSING"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3df42c46",
+   "metadata": {},
+   "source": [
+    "#### Named Entity Recognitition (NER) using SpaCy "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a287eef4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✅ spaCy model loaded: en_core_web_sm\n",
+      "Loaded 50861 rows\n",
+      "\n",
+      "🔄 Processing names with spaCy NER...\n",
+      "\n",
+      "📊 Name cleaning examples (with spaCy NER):\n",
+      "====================================================================================================\n",
+      "Original Name                                      | Cleaned Name                  \n",
+      "====================================================================================================\n",
+      "Super Pose Book Vol.1 - ControlNet                 | Super Pose Book               \n",
+      "Liyuu LoRA                                         | Liyuu                         \n",
+      "HashimotoKanna/ 橋本環奈 _JP_Actress                   | HashimotoKanna                \n",
+      "Emma Watson (JG)                                   | Emma Watson                   \n",
+      "Gal Gadot「LoRa」                                    | Gal Gadot                     \n",
+      "Scarlett Johansson「LoRa」                           | Scarlett Johansson            \n",
+      "Gakki | Aragaki Yui | 新垣結衣                         | Gakki                         \n",
+      "Actress Satomi_石原○○                                | Actress Satomi                \n",
+      "Game of Thrones Cast                               | Game Thrones Cast             \n",
+      "Natalie Portman「LoRa」                              | Natalie Portman               \n",
+      "Emma Watson LoRA                                   | Emma Watson                   \n",
+      "Karina Makina Lora                                 | Karina Makina                 \n",
+      "WRAV YUA_三xx亜                                      | WRAV YUA                      \n",
+      "Dilraba Dilmurat 迪丽热巴                              | Dilraba Dilmurat              \n",
+      "MIMI,大幂幂                                           | MIMI                          \n",
+      "Chinese Idol - YangMi杨幂                            | Chinese Idol YangMi杨幂         \n",
+      "Xiaorouseeu / 小柔SeeU - Chinese cosplayer and influencer | Xiaorouseeu                   \n",
+      "Jennifer Connelly (80s/90s)                        | Jennifer Connelly             \n",
+      "====================================================================================================\n",
+      "\n",
+      "🧪 Leetspeak translation examples:\n",
+      "  4kira LoRA                     -> akira\n",
+      "  3mma Watson v2                 -> Watson\n",
+      "  1rene LORA                     -> irene\n",
+      "  L3vi Ackerman                  -> Levi Ackerman\n",
+      "\n",
+      "📈 Statistics:\n",
+      "  Total rows: 50861\n",
+      "  Non-empty names: 50858\n",
+      "  Empty names: 3\n",
+      "\n",
+      "🎯 Sample spaCy NER results:\n",
+      "  1. IU\n",
+      "  2. Super Pose Book\n",
+      "  3. Liyuu\n",
+      "  4. Irene\n",
+      "  5. AESPA Karina\n",
+      "  6. Saika Kawakita\n",
+      "  7. Liu Yifei\n",
+      "  8. HashimotoKanna\n",
+      "  9. Emma Watson\n",
+      "  10. Gal Gadot\n",
+      "\n",
+      "✅ Cleaned 50861 names using spaCy NER\n",
+      "💾 Saved to /home/lauhp/000_PHD/000_010_PUBLICATION/CODE/pm-paper/data/CSV/model_adapter/real_person_adapter_step_01_NER.csv\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import re\n",
+    "from pathlib import Path\n",
+    "import emoji\n",
+    "import spacy\n",
+    "\n",
+    "# Load spaCy model\n",
+    "# You may need to download it first: python -m spacy download en_core_web_sm\n",
+    "try:\n",
+    "    nlp = spacy.load(\"en_core_web_sm\")\n",
+    "    print(\"✅ spaCy model loaded: en_core_web_sm\")\n",
+    "except OSError:\n",
+    "    print(\"❌ spaCy model not found. Downloading...\")\n",
+    "    import subprocess\n",
+    "    subprocess.run([\"python\", \"-m\", \"spacy\", \"download\", \"en_core_web_sm\"])\n",
+    "    nlp = spacy.load(\"en_core_web_sm\")\n",
+    "    print(\"✅ spaCy model downloaded and loaded\")\n",
+    "\n",
+    "# Set up paths\n",
+    "current_dir = Path.cwd()\n",
+    "#input_file = current_dir.parent / \"data/CSV/real_person_adapters.csv\"\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter.csv\"\n",
+    "\n",
+    "# Load dataset\n",
+    "df = pd.read_csv(input_file)\n",
+    "print(f\"Loaded {len(df)} rows\")\n",
+    "\n",
+    "def translate_leetspeak(text: str) -> str:\n",
+    "    \"\"\"\n",
+    "    Translate common leetspeak patterns to normal letters.\n",
+    "    Examples: 4kira -> Akira, 3mma -> Emma, 1rene -> Irene\n",
+    "    \"\"\"\n",
+    "    if not text:\n",
+    "        return text\n",
+    "    \n",
+    "    # Common leetspeak mappings (order matters!)\n",
+    "    leetspeak_map = {\n",
+    "        '4': 'a',\n",
+    "        '3': 'e', \n",
+    "        '1': 'i',\n",
+    "        '0': 'o',\n",
+    "        '7': 't',\n",
+    "        '5': 's',\n",
+    "        '8': 'b',\n",
+    "        '9': 'g',\n",
+    "        '@': 'a',\n",
+    "        '$': 's',\n",
+    "        '!': 'i',\n",
+    "    }\n",
+    "    \n",
+    "    result = text\n",
+    "    # Apply mappings at word boundaries or start of string\n",
+    "    for leet, normal in leetspeak_map.items():\n",
+    "        # Replace at start of word\n",
+    "        result = re.sub(rf'\\b{re.escape(leet)}', normal, result, flags=re.IGNORECASE)\n",
+    "        # Replace standalone numbers that look like letters in context\n",
+    "        result = re.sub(rf'(?<=[a-z]){re.escape(leet)}(?=[a-z])', normal, result, flags=re.IGNORECASE)\n",
+    "    \n",
+    "    return result\n",
+    "\n",
+    "def preprocess_for_ner(name: str) -> str:\n",
+    "    \"\"\"\n",
+    "    Preprocess the name before spaCy NER.\n",
+    "    Remove noise but keep the actual name parts.\n",
+    "    \"\"\"\n",
+    "    if pd.isna(name):\n",
+    "        return \"\"\n",
+    "    \n",
+    "    name = str(name)\n",
+    "    \n",
+    "    # FIRST: Translate leetspeak\n",
+    "    name = translate_leetspeak(name)\n",
+    "    \n",
+    "    # Remove emoji\n",
+    "    name = emoji.replace_emoji(name, replace=' ')\n",
+    "    \n",
+    "    # Remove version indicators (v1, v2, v1.0, etc.)\n",
+    "    name = re.sub(r'\\s*[vV]\\d+(\\.\\d+)?\\s*', ' ', name)\n",
+    "    \n",
+    "    # Remove LoRA-related terms (case insensitive)\n",
+    "    lora_terms = ['lora', 'loha', 'lycoris', 'controlnet', 'textual inversion', \n",
+    "                  'embedding', 'ti', 'checkpoint', 'model', 'adapter', 'pony', 'sdxl', 'flux', 'illustrious', 'sd14', 'sd14', 'sd2', 'sd3', 'diffusion', 'stable', 'hunyuan']\n",
+    "    for term in lora_terms:\n",
+    "        name = re.sub(rf'\\b{term}\\b', '', name, flags=re.IGNORECASE)\n",
+    "    \n",
+    "    # Remove content in parentheses or brackets (often metadata)\n",
+    "    name = re.sub(r'\\([^)]*\\)', '', name)\n",
+    "    name = re.sub(r'\\[[^\\]]*\\]', '', name)\n",
+    "    \n",
+    "    # Remove special characters like 「」\n",
+    "    name = re.sub(r'[「」『』【】〈〉《》]', '', name)\n",
+    "    \n",
+    "    # Handle pipe - keep first part\n",
+    "    if '|' in name:\n",
+    "        name = name.split('|')[0]\n",
+    "    \n",
+    "    # Handle forward slash - keep first part\n",
+    "    if '/' in name:\n",
+    "        name = name.split('/')[0]\n",
+    "    \n",
+    "    # Replace underscores with spaces\n",
+    "    name = name.replace('_', ' ')\n",
+    "    \n",
+    "    # Remove multiple spaces\n",
+    "    name = re.sub(r'\\s+', ' ', name)\n",
+    "    \n",
+    "    # Strip\n",
+    "    name = name.strip()\n",
+    "    \n",
+    "    return name\n",
+    "\n",
+    "def extract_person_name(text: str) -> str:\n",
+    "    \"\"\"\n",
+    "    Use spaCy NER to extract person names from text.\n",
+    "    Falls back to cleaned text if no PERSON entity found.\n",
+    "    \"\"\"\n",
+    "    if not text:\n",
+    "        return \"\"\n",
+    "    \n",
+    "    # Run spaCy NER\n",
+    "    doc = nlp(text)\n",
+    "    \n",
+    "    # Extract PERSON entities\n",
+    "    person_entities = [ent.text for ent in doc.ents if ent.label_ == \"PERSON\"]\n",
+    "    \n",
+    "    if person_entities:\n",
+    "        # Return the first (usually longest/best) person name\n",
+    "        return person_entities[0].strip()\n",
+    "    \n",
+    "    # If no PERSON entity found, try to extract capitalized words (likely names)\n",
+    "    # This helps with names spaCy might miss\n",
+    "    words = text.split()\n",
+    "    capitalized_words = [w for w in words if w and w[0].isupper() and len(w) > 1]\n",
+    "    \n",
+    "    if capitalized_words:\n",
+    "        # Join first few capitalized words (likely the name)\n",
+    "        return ' '.join(capitalized_words[:3]).strip()\n",
+    "    \n",
+    "    # Last resort: return cleaned text\n",
+    "    return text.strip()\n",
+    "\n",
+    "def clean_name_with_spacy(name: str) -> str:\n",
+    "    \"\"\"\n",
+    "    Complete name cleaning pipeline with spaCy NER.\n",
+    "    \n",
+    "    Pipeline:\n",
+    "    1. Translate leetspeak (4→a, 3→e, 1→i, etc.)\n",
+    "    2. Remove noise (emoji, version tags, LoRA terms)\n",
+    "    3. Use spaCy to extract PERSON entities\n",
+    "    4. Fallback to capitalized words or cleaned text\n",
+    "    \"\"\"\n",
+    "    # Step 1 & 2: Preprocess (leetspeak + noise removal)\n",
+    "    preprocessed = preprocess_for_ner(name)\n",
+    "    \n",
+    "    if not preprocessed:\n",
+    "        return \"\"\n",
+    "    \n",
+    "    # Step 3: Extract person name using spaCy NER\n",
+    "    person_name = extract_person_name(preprocessed)\n",
+    "    \n",
+    "    return person_name\n",
+    "\n",
+    "# Apply name cleaning with spaCy\n",
+    "print(\"\\n🔄 Processing names with spaCy NER...\")\n",
+    "df['real_name'] = df['name'].apply(clean_name_with_spacy)\n",
+    "\n",
+    "# Show examples with detailed comparison\n",
+    "print(\"\\n📊 Name cleaning examples (with spaCy NER):\")\n",
+    "print(\"=\" * 100)\n",
+    "print(f\"{'Original Name':<50} | {'Cleaned Name':<30}\")\n",
+    "print(\"=\" * 100)\n",
+    "\n",
+    "examples = df[['name', 'real_name']].head(30)\n",
+    "shown = 0\n",
+    "for idx, row in examples.iterrows():\n",
+    "    if row['name'] != row['real_name'] and shown < 20:\n",
+    "        print(f\"{row['name']:<50} | {row['real_name']:<30}\")\n",
+    "        shown += 1\n",
+    "\n",
+    "print(\"=\" * 100)\n",
+    "\n",
+    "# Show specific test cases\n",
+    "print(\"\\n🧪 Leetspeak translation examples:\")\n",
+    "test_names = ['4kira LoRA', '3mma Watson v2', '1rene LORA', 'L3vi Ackerman']\n",
+    "for test in test_names:\n",
+    "    result = clean_name_with_spacy(test)\n",
+    "    print(f\"  {test:<30} -> {result}\")\n",
+    "\n",
+    "# Statistics\n",
+    "print(f\"\\n📈 Statistics:\")\n",
+    "print(f\"  Total rows: {len(df)}\")\n",
+    "print(f\"  Non-empty names: {(df['real_name'] != '').sum()}\")\n",
+    "print(f\"  Empty names: {(df['real_name'] == '').sum()}\")\n",
+    "\n",
+    "# Show some examples of what spaCy identified\n",
+    "print(\"\\n🎯 Sample spaCy NER results:\")\n",
+    "sample_names = df['real_name'].head(20).tolist()\n",
+    "for i, name in enumerate(sample_names[:10], 1):\n",
+    "    if name:\n",
+    "        print(f\"  {i}. {name}\")\n",
+    "\n",
+    "print(f\"\\n✅ Cleaned {len(df)} names using spaCy NER\")\n",
+    "\n",
+    "# Save intermediate result\n",
+    "output_step1 = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_01_NER.csv\"\n",
+    "df.to_csv(output_step1, index=False)\n",
+    "print(f\"💾 Saved to {output_step1}\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "64687c72",
+   "metadata": {},
+   "source": [
+    "#### STEP 02: Nationality tag to Country hint\n",
+    "here tags related to nationality gets converted to the country equivalent."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d6eaef5b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from pathlib import Path\n",
+    "\n",
+    "# Set up paths\n",
+    "current_dir = Path.cwd()\n",
+    "countries_file = current_dir.parent / \"misc/lists/countries.csv\"\n",
+    "professions_file = current_dir.parent / \"misc/lists/professions.csv\"\n",
+    "input_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_01_NER.csv\"\n",
+    "output_file = current_dir.parent / \"data/CSV/model_adapter/real_person_adapter_step_02_NER.csv\"\n",
+    "\n",
+    "# Load datasets\n",
+    "poi_df = pd.read_csv(input_file)\n",
+    "countries_df = pd.read_csv(countries_file)\n",
+    "professions_df = pd.read_csv(professions_file)\n",
+    "\n",
+    "# Define uninhabited or non-relevant territories to exclude\n",
+    "excluded_territories = {\n",
+    "    'isle of man', 'bouvet island', 'heard island and mcdonald islands',\n",
+    "    'french southern territories', 'south georgia and the south sandwich islands',\n",
+    "    'svalbard and jan mayen', 'british indian ocean territory', 'antarctica',\n",
+    "    'christmas island', 'cocos (keeling) islands', 'norfolk island',\n",
+    "    'pitcairn', 'tokelau', 'united states minor outlying islands',\n",
+    "    'wallis and futuna', 'western sahara'\n",
+    "}\n",
+    "\n",
+    "# Step 1: Combine tags into one lowercase list\n",
+    "def combine_tags(row):\n",
+    "    return [str(row[f\"tag_{i}\"]).strip().lower() for i in range(1, 8) if pd.notna(row.get(f\"tag_{i}\"))]\n",
+    "\n",
+    "poi_df[\"tags\"] = poi_df.apply(combine_tags, axis=1)\n",
+    "\n",
+    "# Step 2: Build tag → (country, nationality) mapping with PRIORITIES\n",
+    "tag_to_country_nationality = {}\n",
+    "# We'll use a priority score: direct country name = 3, nationality = 2, word parts = 1\n",
+    "\n",
+    "for _, row in countries_df.iterrows():\n",
+    "    country = str(row[\"en_short_name\"]).strip()\n",
+    "    nationality = str(row[\"nationality\"]).strip()\n",
+    "    \n",
+    "    # Skip excluded territories\n",
+    "    if country.lower() in excluded_territories:\n",
+    "        continue\n",
+    "\n",
+    "    country_lc = country.lower()\n",
+    "    nationality_lc = nationality.lower()\n",
+    "\n",
+    "    # Store as (country, nationality, priority)\n",
+    "    # Exact country name match = highest priority\n",
+    "    if country_lc not in tag_to_country_nationality:\n",
+    "        tag_to_country_nationality[country_lc] = (country, \"\", 3)\n",
+    "    \n",
+    "    # Exact nationality match = medium priority  \n",
+    "    if nationality_lc not in tag_to_country_nationality:\n",
+    "        tag_to_country_nationality[nationality_lc] = (\"\", nationality, 2)\n",
+    "    \n",
+    "    # No-space versions\n",
+    "    country_no_space = country_lc.replace(\" \", \"\")\n",
+    "    nationality_no_space = nationality_lc.replace(\" \", \"\")\n",
+    "    \n",
+    "    if country_no_space not in tag_to_country_nationality:\n",
+    "        tag_to_country_nationality[country_no_space] = (country, \"\", 3)\n",
+    "    if nationality_no_space not in tag_to_country_nationality:\n",
+    "        tag_to_country_nationality[nationality_no_space] = (\"\", nationality, 2)\n",
+    "\n",
+    "    # Word parts = lowest priority (only for longer words to avoid false matches)\n",
+    "    for part in country_lc.split():\n",
+    "        if len(part) > 4:  # Only words longer than 4 chars\n",
+    "            if part not in tag_to_country_nationality:\n",
+    "                tag_to_country_nationality[part] = (country, \"\", 1)\n",
+    "    for part in nationality_lc.split():\n",
+    "        if len(part) > 4:\n",
+    "            if part not in tag_to_country_nationality:\n",
+    "                tag_to_country_nationality[part] = (\"\", nationality, 1)\n",
+    "\n",
+    "print(f\"Built country/nationality mapping with {len(tag_to_country_nationality)} entries\")\n",
+    "\n",
+    "# Step 3: Infer likely_country and likely_nationality by checking ALL tags\n",
+    "def infer_country_and_nationality(tags):\n",
+    "    \"\"\"\n",
+    "    Check ALL tags and return the best match based on priority.\n",
+    "    Priority: exact country name > nationality > word parts\n",
+    "    \"\"\"\n",
+    "    best_match = None\n",
+    "    best_priority = 0\n",
+    "    \n",
+    "    for tag in tags:\n",
+    "        # Try cleaned version (no spaces)\n",
+    "        cleaned = tag.replace(\" \", \"\").lower()\n",
+    "        \n",
+    "        # Check cleaned version\n",
+    "        if cleaned in tag_to_country_nationality:\n",
+    "            country, nationality, priority = tag_to_country_nationality[cleaned]\n",
+    "            if priority > best_priority and country and country.lower() not in excluded_territories:\n",
+    "                best_match = (country, nationality)\n",
+    "                best_priority = priority\n",
+    "        \n",
+    "        # Also check original tag\n",
+    "        if tag in tag_to_country_nationality:\n",
+    "            country, nationality, priority = tag_to_country_nationality[tag]\n",
+    "            if priority > best_priority and country and country.lower() not in excluded_territories:\n",
+    "                best_match = (country, nationality)\n",
+    "                best_priority = priority\n",
+    "    \n",
+    "    if best_match:\n",
+    "        return pd.Series(best_match)\n",
+    "    return pd.Series([\"\", \"\"])\n",
+    "\n",
+    "poi_df[[\"likely_country\", \"likely_nationality\"]] = poi_df[\"tags\"].apply(infer_country_and_nationality)\n",
+    "\n",
+    "# Step 4: Build tag → profession mapping\n",
+    "profession_alias_map = {}\n",
+    "\n",
+    "for _, row in professions_df.iterrows():\n",
+    "    canonical = str(row['profession']).strip().lower()\n",
+    "    profession_alias_map[canonical] = canonical\n",
+    "    for alias_col in ['alias_1', 'alias_2', 'alias_3']:\n",
+    "        alias = row.get(alias_col)\n",
+    "        if pd.notna(alias):\n",
+    "            profession_alias_map[str(alias).strip().lower()] = canonical\n",
+    "\n",
+    "# Step 5: Infer likely profession from tags\n",
+    "def infer_profession_from_tags(tags):\n",
+    "    matched = []\n",
+    "    for tag in tags:\n",
+    "        cleaned = tag.strip().lower()\n",
+    "        if cleaned in profession_alias_map:\n",
+    "            matched.append(profession_alias_map[cleaned])\n",
+    "\n",
+    "    if not matched:\n",
+    "        return \"\"\n",
+    "    if \"celebrity\" in matched and len(set(matched)) > 1:\n",
+    "        # Drop 'celebrity' if other professions are present\n",
+    "        matched = [m for m in matched if m != \"celebrity\"]\n",
+    "\n",
+    "    return matched[0]  # Return the first specific match\n",
+    "\n",
+    "\n",
+    "poi_df[\"likely_profession\"] = poi_df[\"tags\"].apply(infer_profession_from_tags)\n",
+    "\n",
+    "# Step 6: Save enriched dataset\n",
+    "poi_df.to_csv(output_file, index=False)\n",
+    "\n",
+    "# Preview results\n",
+    "print(f\"\\nProcessed {len(poi_df)} rows\")\n",
+    "print(f\"Rows with country: {(poi_df['likely_country'] != '').sum()}\")\n",
+    "print(f\"Rows with nationality: {(poi_df['likely_nationality'] != '').sum()}\")\n",
+    "print(f\"Rows with profession: {(poi_df['likely_profession'] != '').sum()}\")\n",
+    "\n",
+    "print(f\"\\nTop 10 countries:\")\n",
+    "print(poi_df[poi_df['likely_country'] != '']['likely_country'].value_counts().head(10))\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4a4a58b3",
+   "metadata": {},
+   "source": [
+    "## LLM ANNOTATION"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b298844d",
+   "metadata": {},
+   "source": [
+    "#### Model Configurations"
    ]
   },
   {
    "cell_type": "code",
+   "execution_count": null,
+   "id": "39f3d65e",
    "metadata": {},
+   "outputs": [],
    "source": [
     "import pandas as pd\n",
     "import json\n",
@@ -86,13 +568,22 @@
     "    \"sports professional\",\n",
     "    \"tv personality\"\n",
     "]\n"
-   ],
-   "execution_count": null,
-   "outputs": []
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c215b38c",
+   "metadata": {},
+   "source": [
+    "#### Load Model Function"
+   ]
   },
   {
    "cell_type": "code",
+   "execution_count": null,
+   "id": "cfb5b13e",
    "metadata": {},
+   "outputs": [],
    "source": [
     "def load_model(model_type='mistral'):\n",
     "    \"\"\"\n",
@@ -156,13 +647,22 @@
     "        print(f\"VRAM used: {vram_gb:.2f} GB\\n\")\n",
     "    \n",
     "    return model, tokenizer, config\n"
-   ],
-   "execution_count": null,
-   "outputs": []
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "11b2221a",
+   "metadata": {},
+   "source": [
+    "#### Inference Code"
+   ]
   },
   {
    "cell_type": "code",
+   "execution_count": null,
+   "id": "229f96bd",
    "metadata": {},
+   "outputs": [],
    "source": [
     "@contextmanager\n",
     "def timeout(duration):\n",
@@ -239,13 +739,22 @@
     "    except Exception as e:\n",
     "        print(f\"[ERROR] Generation failed: {e}\")\n",
     "        return None\n"
-   ],
-   "execution_count": null,
-   "outputs": []
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "88f005f8",
+   "metadata": {},
+   "source": [
+    "#### Prompt creation"
+   ]
   },
   {
    "cell_type": "code",
+   "execution_count": null,
+   "id": "dfe05463",
    "metadata": {},
+   "outputs": [],
    "source": [
     "def create_prompt(row):\n",
     "    \"\"\"Create annotation prompt from row data.\"\"\"\n",
@@ -298,13 +807,22 @@
     "3. Female\n",
     "4. singer/musician, public figure\n",
     "5. United States\"\"\"\n"
-   ],
-   "execution_count": null,
-   "outputs": []
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "854fa668",
+   "metadata": {},
+   "source": [
+    "#### Response parsing code"
+   ]
   },
   {
    "cell_type": "code",
+   "execution_count": null,
+   "id": "1a4be2ee",
    "metadata": {},
+   "outputs": [],
    "source": [
     "def parse_response(response):\n",
     "    \"\"\"Parse model response into structured fields.\"\"\"\n",
@@ -345,13 +863,22 @@
     "            fields['country'] = country_raw\n",
     "    \n",
     "    return fields\n"
-   ],
-   "execution_count": null,
-   "outputs": []
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7e2f7a86",
+   "metadata": {},
+   "source": [
+    "#### CSV annotation"
+   ]
   },
   {
    "cell_type": "code",
+   "execution_count": null,
+   "id": "5f3dd5d6",
    "metadata": {},
+   "outputs": [],
    "source": [
     "def annotate_dataset(model_type='mistral', test_mode=False, test_size=100, max_rows=50862, save_interval=10):\n",
     "    \"\"\"\n",
@@ -454,20 +981,23 @@
     "    df.to_csv(output_file, index=False)\n",
     "    index_file.write_text(str(current_index))\n",
     "    print(f\"✓ Finished annotation with {model_type}\")\n"
-   ],
-   "execution_count": null,
-   "outputs": []
+   ]
   },
   {
    "cell_type": "markdown",
+   "id": "55da2f4c",
    "metadata": {},
    "source": [
-    "### Usage Examples\nRun annotation with your chosen model."
+    "### Usage Examples\n",
+    "Run annotation with your chosen model."
    ]
   },
   {
    "cell_type": "code",
+   "execution_count": null,
+   "id": "351ea40c",
    "metadata": {},
+   "outputs": [],
    "source": [
     "# Example 1: Annotate with Mistral (13.5 GB VRAM)\n",
     "# annotate_dataset(model_type='mistral', test_mode=False)\n",
@@ -480,9 +1010,7 @@
     "\n",
     "# Test mode (first 100 rows)\n",
     "# annotate_dataset(model_type='mistral', test_mode=True, test_size=100)\n"
-   ],
-   "execution_count": null,
-   "outputs": []
+   ]
   }
  ],
  "metadata": {
@@ -506,4 +1034,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}

jupyter_notebooks/Section_2-3-4_Figure_8_Step_2_response_comparison_and_consensus_extraction.ipynb

@@ -1031,7 +1031,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "id": "ca58ae0f-a3e1-44c9-9645-412997f1777d",
    "metadata": {
     "execution": {
@@ -1047,171 +1047,66 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "============================================================\n",
-      "STRICT CONSENSUS FILTER\n",
-      "============================================================\n",
-      "Reading: /shares/weddigen.ki.uzh/laura_wagner/phase_01/pm-paper/data/CSV/combined_llm_annotations.csv\n",
+      "================================================================================\n",
+      "IMPROVED CONSENSUS CREATION\n",
+      "================================================================================\n",
+      "Method: hybrid\n",
+      "Reading: /home/lauhp/000_PHD/000_010_PUBLICATION/CODE/pm-paper/data/CSV/combined_llm_annotations.csv\n",
       "Input shape: (50861, 67)\n",
-      "Models to check: gemma, mistral, qwen\n",
+      "Models: gemma, mistral, qwen\n",
       "\n",
       "Processing rows...\n",
       "  Processed 50000/50861 rows...\n",
       "  Processed 50861 rows.           \n",
       "\n",
-      "============================================================\n",
-      "FILTERING RESULTS\n",
-      "============================================================\n",
+      "================================================================================\n",
+      "RESULTS\n",
+      "================================================================================\n",
       "Total input rows:          50,861\n",
-      "Rows passing all criteria: 4,181 (8.2%)\n",
-      "\n",
-      "Failure reasons (rows can fail multiple):\n",
-      "  - Country disagreement:    25,292 (49.7%)\n",
-      "  - Gender disagreement:     6,783 (13.3%)\n",
-      "  - Profession disagreement: 43,698 (85.9%)\n",
-      "  - Contains 'Unknown':      46,680 (91.8%)\n",
-      "\n",
-      "============================================================\n",
-      "CONSENSUS DISTRIBUTIONS\n",
-      "============================================================\n",
+      "Rows passing all criteria: 23,084 (45.4%)\n",
       "\n",
-      "Country (top 10):\n",
-      "consensus_country\n",
-      "usa            2186\n",
-      "uk              405\n",
-      "japan           337\n",
-      "south korea     237\n",
-      "india           146\n",
-      "china           101\n",
-      "canada           79\n",
-      "russia           74\n",
-      "brazil           73\n",
-      "france           69\n",
-      "Name: count, dtype: int64\n",
+      "Consensus method usage:\n",
+      "  - weighted: 43,445 (188.2%)\n",
+      "  - adult_special: 5,622 (24.4%)\n",
+      "  - adult_any_position: 1,784 (7.7%)\n",
+      "  - no_consensus: 10 (0.0%)\n",
       "\n",
-      "Gender:\n",
-      "consensus_gender\n",
-      "female    3622\n",
-      "male       559\n",
-      "Name: count, dtype: int64\n",
+      "================================================================================\n",
+      "PROFESSION DISTRIBUTION\n",
+      "================================================================================\n",
       "\n",
-      "Profession (top 10):\n",
+      "Top 20 professions:\n",
       "consensus_profession\n",
-      "actor, tv personality, public figure              1154\n",
-      "actor, tv personality, online personality          633\n",
-      "singer/musician, model, online personality         341\n",
-      "actor, singer/musician, tv personality             322\n",
-      "actor, tv personality, model                       267\n",
-      "actor, model, online personality                   161\n",
-      "actor, model, tv personality                       125\n",
-      "model, adult performer, online personality          80\n",
-      "adult performer, model, online personality          75\n",
-      "singer/musician, tv personality, public figure      71\n",
-      "Name: count, dtype: int64\n",
-      "\n",
-      "Profession agreement level:\n",
-      "profession_agreement_count\n",
-      "2    4096\n",
-      "3      85\n",
+      "actor                  11046\n",
+      "singer/musician         3939\n",
+      "model                   3017\n",
+      "online personality      1651\n",
+      "adult performer         1206\n",
+      "public figure            986\n",
+      "sports professional      458\n",
+      "voice actor/asmr         376\n",
+      "tv personality           373\n",
+      "wrestler                  10\n",
+      "comedian                   8\n",
+      "cheerleader                2\n",
+      "actress                    2\n",
+      "dancer                     2\n",
+      "architect                  1\n",
+      "entrepreneur               1\n",
+      "basketball player          1\n",
+      "podcaster                  1\n",
+      "artist                     1\n",
+      "gymnast                    1\n",
       "Name: count, dtype: int64\n",
       "\n",
-      "============================================================\n",
-      "✓ Strict consensus file saved to: strict_consensus.csv\n",
-      "  Total rows: 4,181\n",
-      "  Total columns: 71\n",
-      "============================================================\n",
-      "\n",
-      "============================================================\n",
-      "SAMPLE COMPARISONS (first 5 rows)\n",
-      "============================================================\n",
+      "🎯 Adult performer variants: 1206 (5.22%)\n",
       "\n",
-      "--- Row 1: Liu Yifei ---\n",
-      "Country:\n",
-      "  Consensus: china\n",
-      "    gemma: China\n",
-      "    mistral: China\n",
-      "    qwen: China\n",
-      "Gender:\n",
-      "  Consensus: female\n",
-      "    gemma: Female\n",
-      "    mistral: Female\n",
-      "    qwen: Female\n",
-      "Profession (agreement: 2/3):\n",
-      "  Consensus: actor, model, singer/musician\n",
-      "    gemma: actor, model, singer/musician\n",
-      "    mistral: actor, tv personality, model\n",
-      "    qwen: actor, model, singer/musician\n",
-      "\n",
-      "--- Row 2: Emma Watson (JG) ---\n",
-      "Country:\n",
-      "  Consensus: uk\n",
-      "    gemma: UK\n",
-      "    mistral: UK\n",
-      "    qwen: UK\n",
-      "Gender:\n",
-      "  Consensus: female\n",
-      "    gemma: Female\n",
-      "    mistral: Female\n",
-      "    qwen: Female\n",
-      "Profession (agreement: 2/3):\n",
-      "  Consensus: actor, public figure, model\n",
-      "    gemma: actor, public figure, model\n",
-      "    mistral: actor, tv personality, public figure\n",
-      "    qwen: actor, public figure, model\n",
-      "\n",
-      "--- Row 3: Gal Gadot「LoRa」 ---\n",
-      "Country:\n",
-      "  Consensus: israel\n",
-      "    gemma: Israel\n",
-      "    mistral: Israel\n",
-      "    qwen: Israel\n",
-      "Gender:\n",
-      "  Consensus: female\n",
-      "    gemma: Female\n",
-      "    mistral: Female\n",
-      "    qwen: Female\n",
-      "Profession (agreement: 2/3):\n",
-      "  Consensus: actor, model, tv personality\n",
-      "    gemma: actor, model, tv personality\n",
-      "    mistral: actor, model, tv personality\n",
-      "    qwen: actor, model, public figure\n",
-      "\n",
-      "--- Row 4: Game of Thrones Cast ---\n",
-      "Country:\n",
-      "  Consensus: uk\n",
-      "    gemma: UK\n",
-      "    mistral: UK\n",
-      "    qwen: UK\n",
-      "Gender:\n",
-      "  Consensus: female\n",
-      "    gemma: Female\n",
-      "    mistral: Female\n",
-      "    qwen: Female\n",
-      "Profession (agreement: 2/3):\n",
-      "  Consensus: actor, tv personality, online personality\n",
-      "    gemma: actor, tv personality, online personality\n",
-      "    mistral: actor, tv personality, online personality\n",
-      "    qwen: actor, model\n",
-      "\n",
-      "--- Row 5: Karina Makina Lora ---\n",
-      "Country:\n",
-      "  Consensus: south korea\n",
-      "    gemma: South Korea\n",
-      "    mistral: South Korea\n",
-      "    qwen: South Korea\n",
-      "Gender:\n",
-      "  Consensus: female\n",
-      "    gemma: Female\n",
-      "    mistral: Female\n",
-      "    qwen: Female\n",
-      "Profession (agreement: 2/3):\n",
-      "  Consensus: singer/musician, model, online personality\n",
-      "    gemma: singer/musician, model, online personality\n",
-      "    mistral: singer/musician, tv personality, kpop idol\n",
-      "    qwen: singer/musician, model, online personality\n",
+      "================================================================================\n",
+      "✓ Improved consensus saved to: improved_consensus.csv\n",
+      "  Total rows: 23,084\n",
+      "================================================================================\n",
       "\n",
-      "============================================================\n",
-      "COMPLETE!\n",
-      "============================================================\n"
+      "✅ Complete!\n"
      ]
     }
    ],
@@ -2338,6 +2233,602 @@
     "        print(\"Please adjust the path in the script to point to your data file.\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "2d761d7f",
+   "metadata": {},
+   "source": [
+    "# Improved Consensus Script with Position-Aware Logic\n",
+    "# Specifically addresses the \"adult performer\" underrepresentation problem"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2aac8386",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "================================================================================\n",
+      "IMPROVED CONSENSUS CREATION\n",
+      "================================================================================\n",
+      "Method: hybrid\n",
+      "Reading: /home/lauhp/000_PHD/000_010_PUBLICATION/CODE/pm-paper/data/CSV/combined_llm_annotations.csv\n",
+      "Input shape: (50861, 67)\n",
+      "Models: gemma, mistral, qwen\n",
+      "\n",
+      "Processing rows...\n",
+      "  Processed 50000/50861 rows...\n",
+      "  Processed 50861 rows.           \n",
+      "\n",
+      "================================================================================\n",
+      "RESULTS\n",
+      "================================================================================\n",
+      "Total input rows:          50,861\n",
+      "Rows passing all criteria: 23,084 (45.4%)\n",
+      "\n",
+      "Consensus method usage:\n",
+      "  - weighted: 43,445 (188.2%)\n",
+      "  - adult_special: 5,622 (24.4%)\n",
+      "  - adult_any_position: 1,784 (7.7%)\n",
+      "  - no_consensus: 10 (0.0%)\n",
+      "\n",
+      "================================================================================\n",
+      "PROFESSION DISTRIBUTION\n",
+      "================================================================================\n",
+      "\n",
+      "Top 20 professions:\n",
+      "consensus_profession\n",
+      "actor                  11046\n",
+      "singer/musician         3939\n",
+      "model                   3017\n",
+      "online personality      1651\n",
+      "adult performer         1206\n",
+      "public figure            986\n",
+      "sports professional      458\n",
+      "voice actor/asmr         376\n",
+      "tv personality           373\n",
+      "wrestler                  10\n",
+      "comedian                   8\n",
+      "cheerleader                2\n",
+      "actress                    2\n",
+      "dancer                     2\n",
+      "architect                  1\n",
+      "entrepreneur               1\n",
+      "basketball player          1\n",
+      "podcaster                  1\n",
+      "artist                     1\n",
+      "gymnast                    1\n",
+      "Name: count, dtype: int64\n",
+      "\n",
+      "🎯 Adult performer variants: 1206 (5.22%)\n",
+      "\n",
+      "================================================================================\n",
+      "✓ Improved consensus saved to: improved_consensus.csv\n",
+      "  Total rows: 23,084\n",
+      "================================================================================\n",
+      "\n",
+      "✅ Complete!\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "\n",
+    "import pandas as pd\n",
+    "from pathlib import Path\n",
+    "from collections import Counter\n",
+    "import re\n",
+    "\n",
+    "# ============================================================\n",
+    "# CONFIGURATION\n",
+    "# ============================================================\n",
+    "\n",
+    "# Semantic grouping for professions\n",
+    "PROFESSION_GROUPS = {\n",
+    "    'adult_entertainment': [\n",
+    "        'adult performer', 'adult film', 'pornstar', 'av actress', \n",
+    "        'av idol', 'jav idol', 'adult model', 'adult entertainer'\n",
+    "    ],\n",
+    "    'mainstream_model': [\n",
+    "        'model', 'fashion model', 'instagram model', 'supermodel',\n",
+    "        'runway model', 'commercial model'\n",
+    "    ],\n",
+    "    'actor': [\n",
+    "        'actor', 'actress', 'film actor', 'tv actor', 'television actor'\n",
+    "    ],\n",
+    "    'musician': [\n",
+    "        'singer', 'musician', 'singer/musician', 'music artist', 'vocalist'\n",
+    "    ],\n",
+    "    'online_personality': [\n",
+    "        'online personality', 'influencer', 'content creator', \n",
+    "        'youtuber', 'streamer', 'social media personality'\n",
+    "    ],\n",
+    "    'tv_personality': [\n",
+    "        'tv personality', 'television personality', 'tv host', 'presenter'\n",
+    "    ]\n",
+    "}\n",
+    "\n",
+    "# Position weights (1st mention = most important)\n",
+    "POSITION_WEIGHTS = {\n",
+    "    0: 3.0,  # First position\n",
+    "    1: 2.0,  # Second position\n",
+    "    2: 1.0   # Third position\n",
+    "}\n",
+    "\n",
+    "# Model reliability weights (based on analysis)\n",
+    "MODEL_WEIGHTS = {\n",
+    "    'gemma': 1.0,\n",
+    "    'mistral': 0.85,  # Slightly lower due to 76% detection rate vs 92%\n",
+    "    'qwen': 1.0\n",
+    "}\n",
+    "\n",
+    "# ============================================================\n",
+    "# UTILITY FUNCTIONS\n",
+    "# ============================================================\n",
+    "\n",
+    "def normalize_value(value):\n",
+    "    \"\"\"Normalize values for comparison (handle NaN, whitespace, case)\"\"\"\n",
+    "    if pd.isna(value):\n",
+    "        return None\n",
+    "    return str(value).strip().lower()\n",
+    "\n",
+    "def is_unknown_value(value):\n",
+    "    \"\"\"Check if a value represents 'unknown' or similar non-informative values\"\"\"\n",
+    "    if value is None:\n",
+    "        return True\n",
+    "    \n",
+    "    value_str = str(value).strip().lower()\n",
+    "    \n",
+    "    unknown_patterns = [\n",
+    "        'unknown', 'n/a', 'na', 'none', 'not specified', \n",
+    "        'not available', 'unclear', 'uncertain', '', 'null'\n",
+    "    ]\n",
+    "    \n",
+    "    return value_str in unknown_patterns\n",
+    "\n",
+    "def parse_profession_list(profession_str):\n",
+    "    \"\"\"Parse comma-separated profession list into normalized list\"\"\"\n",
+    "    if pd.isna(profession_str):\n",
+    "        return []\n",
+    "    \n",
+    "    professions = [p.strip().lower() for p in str(profession_str).split(',')]\n",
+    "    return [p for p in professions if p and not is_unknown_value(p)]\n",
+    "\n",
+    "def find_profession_group(profession, groups=PROFESSION_GROUPS):\n",
+    "    \"\"\"Find which semantic group a profession belongs to\"\"\"\n",
+    "    profession_lower = profession.lower()\n",
+    "    \n",
+    "    for group_name, terms in groups.items():\n",
+    "        if profession_lower in terms:\n",
+    "            return group_name\n",
+    "        # Partial match for compound terms\n",
+    "        if any(term in profession_lower for term in terms):\n",
+    "            return group_name\n",
+    "    \n",
+    "    return profession_lower  # Return as-is if no group found\n",
+    "\n",
+    "# ============================================================\n",
+    "# CONSENSUS ALGORITHMS\n",
+    "# ============================================================\n",
+    "\n",
+    "def get_position_weighted_consensus(profession_lists, model_names, \n",
+    "                                   weights=POSITION_WEIGHTS, \n",
+    "                                   model_weights=MODEL_WEIGHTS):\n",
+    "    \"\"\"\n",
+    "    Get consensus using position-based weighting.\n",
+    "    \n",
+    "    Professions mentioned first get more weight than those mentioned second or third.\n",
+    "    Different models can have different reliability weights.\n",
+    "    \n",
+    "    Returns: (consensus_profession, weighted_score, breakdown_dict)\n",
+    "    \"\"\"\n",
+    "    scores = {}\n",
+    "    breakdown = {}\n",
+    "    \n",
+    "    for model, prof_list in zip(model_names, profession_lists):\n",
+    "        professions = parse_profession_list(prof_list)\n",
+    "        model_weight = model_weights.get(model, 1.0)\n",
+    "        \n",
+    "        for position, profession in enumerate(professions[:3]):  # Only top 3\n",
+    "            pos_weight = weights.get(position, 0)\n",
+    "            score = pos_weight * model_weight\n",
+    "            \n",
+    "            scores[profession] = scores.get(profession, 0) + score\n",
+    "            \n",
+    "            if profession not in breakdown:\n",
+    "                breakdown[profession] = []\n",
+    "            breakdown[profession].append({\n",
+    "                'model': model,\n",
+    "                'position': position + 1,\n",
+    "                'weight': score\n",
+    "            })\n",
+    "    \n",
+    "    if not scores:\n",
+    "        return None, 0, {}\n",
+    "    \n",
+    "    best_profession = max(scores.items(), key=lambda x: x[1])\n",
+    "    return best_profession[0], best_profession[1], breakdown\n",
+    "\n",
+    "def get_semantic_consensus(profession_lists, model_names, required_agreement=2):\n",
+    "    \"\"\"\n",
+    "    Get consensus by grouping semantically similar professions.\n",
+    "    \n",
+    "    First votes on profession categories (e.g., \"adult entertainment\"),\n",
+    "    then picks the most common specific term within the winning category.\n",
+    "    \n",
+    "    Returns: (consensus_profession, agreement_count, category)\n",
+    "    \"\"\"\n",
+    "    category_votes = {}\n",
+    "    profession_within_category = {}\n",
+    "    \n",
+    "    for model, prof_list in zip(model_names, profession_lists):\n",
+    "        professions = parse_profession_list(prof_list)\n",
+    "        \n",
+    "        if not professions:\n",
+    "            continue\n",
+    "        \n",
+    "        # Use first profession from each model for category voting\n",
+    "        first_prof = professions[0]\n",
+    "        category = find_profession_group(first_prof)\n",
+    "        \n",
+    "        category_votes[category] = category_votes.get(category, 0) + 1\n",
+    "        \n",
+    "        if category not in profession_within_category:\n",
+    "            profession_within_category[category] = []\n",
+    "        profession_within_category[category].append(first_prof)\n",
+    "    \n",
+    "    if not category_votes:\n",
+    "        return None, 0, None\n",
+    "    \n",
+    "    # Find winning category\n",
+    "    winning_category, count = max(category_votes.items(), key=lambda x: x[1])\n",
+    "    \n",
+    "    if count < required_agreement:\n",
+    "        return None, count, winning_category\n",
+    "    \n",
+    "    # Pick most common specific term within winning category\n",
+    "    specific_terms = profession_within_category[winning_category]\n",
+    "    most_common_term = Counter(specific_terms).most_common(1)[0][0]\n",
+    "    \n",
+    "    return most_common_term, count, winning_category\n",
+    "\n",
+    "def get_any_position_consensus(profession_lists, target_profession, \n",
+    "                               required_agreement=2):\n",
+    "    \"\"\"\n",
+    "    Check if target profession appears ANYWHERE in the lists.\n",
+    "    \n",
+    "    Useful for professions that are consistently mentioned but not always first.\n",
+    "    \n",
+    "    Returns: (found, agreement_count, positions_found)\n",
+    "    \"\"\"\n",
+    "    count = 0\n",
+    "    positions = []\n",
+    "    \n",
+    "    for prof_list in profession_lists:\n",
+    "        professions = parse_profession_list(prof_list)\n",
+    "        \n",
+    "        for i, prof in enumerate(professions):\n",
+    "            if target_profession.lower() in prof.lower():\n",
+    "                count += 1\n",
+    "                positions.append(i + 1)\n",
+    "                break\n",
+    "    \n",
+    "    return count >= required_agreement, count, positions\n",
+    "\n",
+    "def get_hybrid_consensus(profession_lists, model_names):\n",
+    "    \"\"\"\n",
+    "    Hybrid consensus strategy that tries multiple approaches.\n",
+    "    \n",
+    "    Strategy priority:\n",
+    "    1. Check for \"adult performer\" anywhere in lists (special case)\n",
+    "    2. Position-weighted consensus\n",
+    "    3. Semantic consensus\n",
+    "    4. Fallback to most common first profession\n",
+    "    \n",
+    "    Returns: (consensus_profession, method_used, confidence_score)\n",
+    "    \"\"\"\n",
+    "    # Special case: Check for adult entertainment professions\n",
+    "    adult_terms = PROFESSION_GROUPS['adult_entertainment']\n",
+    "    for term in adult_terms:\n",
+    "        found, count, positions = get_any_position_consensus(profession_lists, term, required_agreement=2)\n",
+    "        if found:\n",
+    "            # Use weighted consensus to pick the exact term\n",
+    "            weighted_prof, score, _ = get_position_weighted_consensus(profession_lists, model_names)\n",
+    "            \n",
+    "            # Check if the weighted winner is an adult entertainment term\n",
+    "            if any(term in weighted_prof for term in adult_terms):\n",
+    "                return weighted_prof, 'adult_special', score\n",
+    "            \n",
+    "            # If weighted winner isn't adult term, but 2+ models mentioned it, use it\n",
+    "            return term, 'adult_any_position', count * 2.0\n",
+    "    \n",
+    "    # Try position-weighted consensus\n",
+    "    weighted_prof, score, breakdown = get_position_weighted_consensus(profession_lists, model_names)\n",
+    "    \n",
+    "    if score >= 3.0:  # Reasonable threshold (e.g., 2 models first position)\n",
+    "        return weighted_prof, 'weighted', score\n",
+    "    \n",
+    "    # Try semantic consensus\n",
+    "    semantic_prof, count, category = get_semantic_consensus(profession_lists, model_names)\n",
+    "    \n",
+    "    if count >= 2:\n",
+    "        return semantic_prof, 'semantic', count * 1.5\n",
+    "    \n",
+    "    # Fallback: most common first profession\n",
+    "    first_professions = []\n",
+    "    for prof_list in profession_lists:\n",
+    "        professions = parse_profession_list(prof_list)\n",
+    "        if professions:\n",
+    "            first_professions.append(professions[0])\n",
+    "    \n",
+    "    if first_professions:\n",
+    "        most_common = Counter(first_professions).most_common(1)[0]\n",
+    "        if most_common[1] >= 2:\n",
+    "            return most_common[0], 'simple_majority', most_common[1]\n",
+    "    \n",
+    "    return None, 'no_consensus', 0\n",
+    "\n",
+    "# ============================================================\n",
+    "# ORIGINAL CONSENSUS (for comparison)\n",
+    "# ============================================================\n",
+    "\n",
+    "def get_consensus_value_original(values, required_agreement=2):\n",
+    "    \"\"\"Original consensus method - compares entire strings\"\"\"\n",
+    "    normalized = [normalize_value(v) for v in values]\n",
+    "    valid_values = [v for v in normalized if v is not None]\n",
+    "    \n",
+    "    if not valid_values:\n",
+    "        return None, 0\n",
+    "    \n",
+    "    value_counts = Counter(valid_values)\n",
+    "    most_common_value, count = value_counts.most_common(1)[0]\n",
+    "    \n",
+    "    if count >= required_agreement:\n",
+    "        return most_common_value, count\n",
+    "    else:\n",
+    "        return None, count\n",
+    "\n",
+    "# ============================================================\n",
+    "# MAIN CONSENSUS CREATION\n",
+    "# ============================================================\n",
+    "\n",
+    "def create_improved_consensus(input_file, output_file, \n",
+    "                            models=['gemma', 'mistral', 'qwen'],\n",
+    "                            consensus_method='hybrid'):\n",
+    "    \"\"\"\n",
+    "    Create improved consensus CSV with better profession detection.\n",
+    "    \n",
+    "    Parameters:\n",
+    "    - input_file: Path to combined_llm_annotations.csv\n",
+    "    - output_file: Path to save improved consensus\n",
+    "    - models: List of model names\n",
+    "    - consensus_method: 'hybrid', 'weighted', 'semantic', or 'original'\n",
+    "    \"\"\"\n",
+    "    print(\"=\"*80)\n",
+    "    print(\"IMPROVED CONSENSUS CREATION\")\n",
+    "    print(\"=\"*80)\n",
+    "    print(f\"Method: {consensus_method}\")\n",
+    "    print(f\"Reading: {input_file}\")\n",
+    "    \n",
+    "    df = pd.read_csv(input_file)\n",
+    "    \n",
+    "    print(f\"Input shape: {df.shape}\")\n",
+    "    print(f\"Models: {', '.join(models)}\")\n",
+    "    \n",
+    "    consensus_data = []\n",
+    "    stats = {\n",
+    "        'total_rows': len(df),\n",
+    "        'country_fail': 0,\n",
+    "        'gender_fail': 0,\n",
+    "        'profession_fail': 0,\n",
+    "        'unknown_values': 0,\n",
+    "        'all_pass': 0,\n",
+    "        'method_counts': {}\n",
+    "    }\n",
+    "    \n",
+    "    print(\"\\nProcessing rows...\")\n",
+    "    \n",
+    "    for idx, row in df.iterrows():\n",
+    "        if idx % 1000 == 0:\n",
+    "            print(f\"  Processed {idx}/{len(df)} rows...\", end='\\r')\n",
+    "        \n",
+    "        # Get values for each field\n",
+    "        countries = [row[f'{model}_country'] for model in models]\n",
+    "        genders = [row[f'{model}_gender'] for model in models]\n",
+    "        professions = [row[f'{model}_profession_llm'] for model in models]\n",
+    "        \n",
+    "        # Country consensus (strict: all 3 must agree)\n",
+    "        country_consensus, country_count = get_consensus_value_original(countries, required_agreement=3)\n",
+    "        \n",
+    "        # Gender consensus (strict: all 3 must agree)\n",
+    "        gender_consensus, gender_count = get_consensus_value_original(genders, required_agreement=3)\n",
+    "        \n",
+    "        # Profession consensus (IMPROVED)\n",
+    "        if consensus_method == 'hybrid':\n",
+    "            profession_consensus, method, prof_score = get_hybrid_consensus(professions, models)\n",
+    "            prof_count = int(prof_score / 1.5)  # Rough conversion to count\n",
+    "        elif consensus_method == 'weighted':\n",
+    "            profession_consensus, prof_score, _ = get_position_weighted_consensus(professions, models)\n",
+    "            prof_count = int(prof_score / 2)\n",
+    "            method = 'weighted'\n",
+    "        elif consensus_method == 'semantic':\n",
+    "            profession_consensus, prof_count, _ = get_semantic_consensus(professions, models)\n",
+    "            method = 'semantic'\n",
+    "            prof_score = prof_count\n",
+    "        else:  # original\n",
+    "            # Get first profession from each model\n",
+    "            first_profs = [parse_profession_list(p)[0] if parse_profession_list(p) else None \n",
+    "                          for p in professions]\n",
+    "            profession_consensus, prof_count = get_consensus_value_original(first_profs, required_agreement=2)\n",
+    "            method = 'original'\n",
+    "            prof_score = prof_count\n",
+    "        \n",
+    "        # Track method usage\n",
+    "        stats['method_counts'][method] = stats['method_counts'].get(method, 0) + 1\n",
+    "        \n",
+    "        # Determine if row passes\n",
+    "        country_pass = country_count == 3\n",
+    "        gender_pass = gender_count == 3\n",
+    "        profession_pass = profession_consensus is not None\n",
+    "        \n",
+    "        has_unknown = (\n",
+    "            is_unknown_value(country_consensus) or \n",
+    "            is_unknown_value(gender_consensus) or \n",
+    "            is_unknown_value(profession_consensus)\n",
+    "        )\n",
+    "        \n",
+    "        if not country_pass:\n",
+    "            stats['country_fail'] += 1\n",
+    "        if not gender_pass:\n",
+    "            stats['gender_fail'] += 1\n",
+    "        if not profession_pass:\n",
+    "            stats['profession_fail'] += 1\n",
+    "        if has_unknown:\n",
+    "            stats['unknown_values'] += 1\n",
+    "        \n",
+    "        if country_pass and gender_pass and profession_pass and not has_unknown:\n",
+    "            stats['all_pass'] += 1\n",
+    "            \n",
+    "            consensus_data.append({\n",
+    "                'row_index': idx,\n",
+    "                'consensus_country': country_consensus,\n",
+    "                'consensus_gender': gender_consensus,\n",
+    "                'consensus_profession': profession_consensus,\n",
+    "                'profession_method': method,\n",
+    "                'profession_confidence': prof_score\n",
+    "            })\n",
+    "    \n",
+    "    print(f\"\\n  Processed {len(df)} rows.           \")\n",
+    "    \n",
+    "    # Create result dataframe\n",
+    "    if consensus_data:\n",
+    "        result_df = df.iloc[[c['row_index'] for c in consensus_data]].copy().reset_index(drop=True)\n",
+    "        \n",
+    "        # Add consensus columns\n",
+    "        for key in ['consensus_country', 'consensus_gender', 'consensus_profession', \n",
+    "                   'profession_method', 'profession_confidence']:\n",
+    "            result_df[key] = [c[key] for c in consensus_data]\n",
+    "        \n",
+    "        # Reorder columns (consensus columns first)\n",
+    "        consensus_cols = ['consensus_country', 'consensus_gender', 'consensus_profession',\n",
+    "                         'profession_method', 'profession_confidence']\n",
+    "        other_cols = [c for c in result_df.columns if c not in consensus_cols]\n",
+    "        result_df = result_df[consensus_cols + other_cols]\n",
+    "        \n",
+    "        # Save\n",
+    "        result_df.to_csv(output_file, index=False)\n",
+    "        \n",
+    "        print(\"\\n\" + \"=\"*80)\n",
+    "        print(\"RESULTS\")\n",
+    "        print(\"=\"*80)\n",
+    "        print(f\"Total input rows:          {stats['total_rows']:,}\")\n",
+    "        print(f\"Rows passing all criteria: {stats['all_pass']:,} ({stats['all_pass']/stats['total_rows']*100:.1f}%)\")\n",
+    "        \n",
+    "        print(f\"\\nConsensus method usage:\")\n",
+    "        for method, count in sorted(stats['method_counts'].items(), key=lambda x: -x[1]):\n",
+    "            print(f\"  - {method}: {count:,} ({count/stats['all_pass']*100:.1f}%)\")\n",
+    "        \n",
+    "        print(\"\\n\" + \"=\"*80)\n",
+    "        print(\"PROFESSION DISTRIBUTION\")\n",
+    "        print(\"=\"*80)\n",
+    "        print(\"\\nTop 20 professions:\")\n",
+    "        print(result_df['consensus_profession'].value_counts().head(20))\n",
+    "        \n",
+    "        # Specifically check adult performer\n",
+    "        adult_count = result_df['consensus_profession'].apply(\n",
+    "            lambda x: 'adult' in str(x).lower() if pd.notna(x) else False\n",
+    "        ).sum()\n",
+    "        print(f\"\\n🎯 Adult performer variants: {adult_count} ({adult_count/len(result_df)*100:.2f}%)\")\n",
+    "        \n",
+    "        print(\"\\n\" + \"=\"*80)\n",
+    "        print(f\"✓ Improved consensus saved to: {output_file.name}\")\n",
+    "        print(f\"  Total rows: {len(result_df):,}\")\n",
+    "        print(\"=\"*80)\n",
+    "        \n",
+    "        return result_df\n",
+    "    else:\n",
+    "        print(\"\\n⚠ WARNING: No rows passed all criteria!\")\n",
+    "        return pd.DataFrame()\n",
+    "\n",
+    "# ============================================================\n",
+    "# COMPARISON FUNCTION\n",
+    "# ============================================================\n",
+    "\n",
+    "def compare_consensus_methods(input_file, models=['gemma', 'mistral', 'qwen']):\n",
+    "    \"\"\"Compare different consensus methods side by side\"\"\"\n",
+    "    \n",
+    "    print(\"=\"*80)\n",
+    "    print(\"CONSENSUS METHOD COMPARISON\")\n",
+    "    print(\"=\"*80)\n",
+    "    \n",
+    "    methods = ['original', 'weighted', 'semantic', 'hybrid']\n",
+    "    results = {}\n",
+    "    \n",
+    "    for method in methods:\n",
+    "        print(f\"\\n--- Testing {method} method ---\")\n",
+    "        \n",
+    "        output_file = Path(input_file).parent / f\"consensus_{method}.csv\"\n",
+    "        result_df = create_improved_consensus(input_file, output_file, models, method)\n",
+    "        \n",
+    "        if len(result_df) > 0:\n",
+    "            adult_count = result_df['consensus_profession'].apply(\n",
+    "                lambda x: 'adult' in str(x).lower() if pd.notna(x) else False\n",
+    "            ).sum()\n",
+    "            \n",
+    "            results[method] = {\n",
+    "                'total_rows': len(result_df),\n",
+    "                'adult_performer_count': adult_count,\n",
+    "                'adult_performer_pct': adult_count / len(result_df) * 100\n",
+    "            }\n",
+    "    \n",
+    "    print(\"\\n\" + \"=\"*80)\n",
+    "    print(\"COMPARISON SUMMARY\")\n",
+    "    print(\"=\"*80)\n",
+    "    \n",
+    "    print(f\"\\n{'Method':<15} {'Total Rows':<12} {'Adult Performer':<16} {'% Adult':<10}\")\n",
+    "    print(\"-\" * 65)\n",
+    "    \n",
+    "    for method, stats in results.items():\n",
+    "        print(f\"{method:<15} {stats['total_rows']:<12,} {stats['adult_performer_count']:<16,} {stats['adult_performer_pct']:<10.2f}%\")\n",
+    "    \n",
+    "    if 'original' in results and 'hybrid' in results:\n",
+    "        improvement = results['hybrid']['adult_performer_count'] - results['original']['adult_performer_count']\n",
+    "        pct_improvement = improvement / results['original']['adult_performer_count'] * 100\n",
+    "        \n",
+    "        print(f\"\\n✨ Hybrid method improvement over original:\")\n",
+    "        print(f\"   +{improvement} adult performer cases (+{pct_improvement:.1f}%)\")\n",
+    "\n",
+    "# ============================================================\n",
+    "# MAIN EXECUTION\n",
+    "# ============================================================\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    current_dir = Path.cwd()\n",
+    "    \n",
+    "    input_file = current_dir.parent / \"data/CSV/combined_llm_annotations.csv\"\n",
+    "    output_file = current_dir.parent / \"data/CSV/improved_consensus.csv\"\n",
+    "    \n",
+    "    if not input_file.exists():\n",
+    "        print(f\"Error: Input file not found: {input_file}\")\n",
+    "    else:\n",
+    "        # Run comparison (comment out if you just want hybrid)\n",
+    "        # compare_consensus_methods(input_file)\n",
+    "        \n",
+    "        # Or run single method (hybrid recommended)\n",
+    "        result_df = create_improved_consensus(\n",
+    "            input_file, \n",
+    "            output_file, \n",
+    "            models=['gemma', 'mistral', 'qwen'],\n",
+    "            consensus_method='hybrid'\n",
+    "        )\n",
+    "        \n",
+    "        print(\"\\n✅ Complete!\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -2349,7 +2840,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "latm",
    "language": "python",
    "name": "python3"
   },
@@ -2363,7 +2854,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.10"
+   "version": "3.10.15"
   }
  },
  "nbformat": 4,

jupyter_notebooks/Section_2-3-4__Figure_8a_sunburst_gender.ipynb

@@ -9,14 +9,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "8a.json\n"
+      "✓ Saved 8a.json\n"
      ]
     }
    ],
@@ -29,9 +29,10 @@
     "current_dir = Path.cwd()\n",
     "sunburst_json = current_dir.parent / \"public/json/8a.json\"\n",
     "\n",
+    "# Load consensus CSV\n",
+    "consensus_file = current_dir.parent / \"data/CSV/analyzed_llm_agreement_consensus.csv\"\n",
+    "df = pd.read_csv(consensus_file)\n",
     "\n",
-    "aggregated_poi = current_dir.parent / \"data/CSV/Deepseek_annotated_POI_aggregated.csv\"\n",
-    "df = pd.read_csv(aggregated_poi)\n",
     "# ---- Normalize Gender (group Non-binary and Unknown into 'Other') ----\n",
     "def normalize_gender(g):\n",
     "    g = str(g).strip().lower()\n",
@@ -42,29 +43,30 @@
     "    else:\n",
     "        return \"Other\"\n",
     "\n",
-    "df['gender_normalized'] = df['gender'].apply(normalize_gender)\n",
+    "df['gender_normalized'] = df['consensus_gender'].apply(normalize_gender)\n",
     "\n",
     "# ---- Step 1: Limit to top 10 countries ----\n",
-    "df['country_cleaned'] = df['country'].apply(lambda x: x if x not in ['Unknown', '', None] else 'Other')\n",
+    "df['country_cleaned'] = df['consensus_country'].apply(lambda x: x if x not in ['Unknown', '', None] else 'Other')\n",
     "top_countries = df['country_cleaned'].value_counts().nlargest(12).index.tolist()\n",
     "df['country_limited'] = df['country_cleaned'].apply(lambda x: x if x in top_countries else 'Other')\n",
     "\n",
     "# ---- Step 2: Normalize and limit professions ----\n",
     "valid_categories = [\n",
-    "    \"Actor\", \"Adult Performer\", \"Singer, Musician\", \"Model\",\n",
-    "    \"Online Personality\", \"TV Personality\", \"Voice Actor\",\"Public Figure\", \"Sports Professional\"\n",
+    "    \"actor\", \"adult performer\", \"singer/musician\", \"model\",\n",
+    "    \"online personality\", \"tv personality\", \"voice actor/asmr\", \"public figure\", \"sports professional\"\n",
     "]\n",
     "\n",
     "def remap_profession(profession):\n",
-    "    if profession == 'Unknown' or profession not in valid_categories:\n",
+    "    profession_lower = str(profession).strip().lower()\n",
+    "    if profession_lower == 'unknown' or profession_lower not in valid_categories:\n",
     "        return 'Other'\n",
-    "    elif profession == 'Fictional Character':\n",
-    "        return 'Actor'\n",
-    "    elif profession == 'Voice actor':\n",
-    "        return 'Voice Actor'\n",
-    "    return profession\n",
+    "    elif profession_lower == 'fictional character':\n",
+    "        return 'actor'\n",
+    "    elif profession_lower in ['voice actor', 'voice actor/asmr']:\n",
+    "        return 'voice actor/ASMR'\n",
+    "    return profession_lower\n",
     "\n",
-    "df['profession_limited'] = df['mapped_profession'].apply(remap_profession)\n",
+    "df['profession_limited'] = df['consensus_primary_profession'].apply(remap_profession)\n",
     "\n",
     "# ---- Step 3: Group by gender and profession ----\n",
     "sunburst_data = df.groupby(['gender_normalized', 'profession_limited']).size().reset_index(name='count')\n",
@@ -92,7 +94,7 @@
     "with open(sunburst_json, \"w\", encoding='utf-8') as f:\n",
     "    json.dump(sunburst_dict, f, ensure_ascii=False, indent=2)\n",
     "\n",
-    "print(\"8a.json\")\n"
+    "print(\"✓ Saved 8a.json\")\n"
    ]
   },
   {

jupyter_notebooks/Section_2-3-4__Figure_8b_sunburst_profession.ipynb

@@ -9,14 +9,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "✅ Sunburst data saved to sunburst_data.json\n"
+      "✓ Saved sunburst_countries_A.json\n"
      ]
     }
    ],
@@ -30,60 +30,273 @@
     "\n",
     "sunburst_path = current_dir.parent / \"public/json/sunburst_countries_A.json\"\n",
     "\n",
+    "# Load consensus CSV\n",
+    "consensus_file = current_dir.parent / \"data/CSV/improved_consensus.csv\"\n",
+    "df = pd.read_csv(consensus_file)\n",
     "\n",
-    "aggregated_poi = current_dir.parent / \"data/CSV/Deepseek_annotated_POI_aggregated.csv\"\n",
+    "# ============================================================\n",
+    "# 1. NORMALIZE COUNTRIES\n",
+    "# ============================================================\n",
     "\n",
-    "df = pd.read_csv(aggregated_poi)\n",
-    "df['country_cleaned'] = df['country'].apply(lambda x: x if x not in ['Unknown', '', None] else 'Other')\n",
+    "def normalize_country(x: str):\n",
+    "    if not isinstance(x, str) or x.strip() == \"\" or x.lower() == \"unknown\":\n",
+    "        return \"Other\"\n",
+    "    x = x.strip()\n",
     "\n",
-    "# Now get top countries excluding what was forced into 'Other'\n",
-    "top_countries = df['country_cleaned'].value_counts().nlargest(15).index.tolist()\n",
+    "    # Ensure matching with your HTML manualOrder\n",
+    "    replacements = {\n",
+    "        \"USA\": \"United States\",\n",
+    "        \"US\": \"United States\",\n",
+    "        \"U.S.\": \"United States\",\n",
+    "        \"UK\": \"United Kingdom\",\n",
+    "        \"U.K.\": \"United Kingdom\"\n",
+    "    }\n",
+    "    return replacements.get(x, x)\n",
     "\n",
-    "# Final limited country column\n",
-    "df['country_limited'] = df['country_cleaned'].apply(lambda x: x if x in top_countries else 'Other')\n",
+    "df[\"country_clean\"] = df[\"consensus_country\"].apply(normalize_country)\n",
     "\n",
-    "# ---- Step 2: Limit to top 7 professions and combine Unknown and Sports Professional with Other ----\n",
-    "top_professions = df['mapped_profession'].value_counts().nlargest(7).index.tolist()\n",
+    "# Limit to the top 15 countries, merge the rest into \"Other\"\n",
+    "top_countries = df[\"country_clean\"].value_counts().nlargest(25).index.tolist()\n",
     "\n",
-    "# Explicitly remove 'Unknown' and 'Sports Professional' even if they are in the top 7\n",
-    "top_professions = [p for p in top_professions if p not in ['Unknown', 'Sports Professional']]\n",
+    "df[\"country_limited\"] = df[\"country_clean\"].apply(\n",
+    "    lambda x: x if x in top_countries else \"Other\"\n",
+    ")\n",
+    "\n",
+    "# ============================================================\n",
+    "# 2. NORMALIZE PROFESSIONS\n",
+    "# ============================================================\n",
+    "\n",
+    "def normalize_profession(x: str):\n",
+    "    if not isinstance(x, str) or x.strip() == \"\" or x.lower() == \"unknown\":\n",
+    "        return \"Other\"\n",
+    "    x = x.strip().lower()\n",
+    "    \n",
+    "    mapping = {\n",
+    "        \"actor\": \"Actor\",\n",
+    "        \"model\": \"Model\",\n",
+    "        \"adult performer\": \"Adult Performer\",\n",
+    "        \"singer/musician\": \"Singer, Musician\",\n",
+    "        \"online personality\": \"Online Personality\",\n",
+    "        \"sports professional\": \"Sports Professional\",\n",
+    "        \"voice actor/asmr\": \"Voice Actor\",        # ← fixed key\n",
+    "        \"public figure\": \"Public Figure\",          # ← now its own category\n",
+    "        \"tv personality\": \"Other\",\n",
+    "    }\n",
+    "    return mapping.get(x, \"Other\")\n",
+    "\n",
+    "df[\"profession_clean\"] = df[\"consensus_profession\"].apply(normalize_profession)\n",
     "\n",
-    "# Normalize 'Unknown' and '\n",
-    "# Sports Professional' into 'Other'\n",
-    "# Normalize 'Unknown' and 'Sports Professional' into 'Other'\n",
-    "df['profession_limited'] = df['mapped_profession'].apply(\n",
-    "    lambda x: 'Other' if x in ['Unknown', 'Sports Professional'] or x not in top_professions else x\n",
+    "# Only keep the top 7 professions (excluding Other)\n",
+    "top_prof = (\n",
+    "    df[df[\"profession_clean\"] != \"Other\"][\"profession_clean\"]\n",
+    "    .value_counts()\n",
+    "    .nlargest(7)\n",
+    "    .index.tolist()\n",
     ")\n",
     "\n",
+    "# Re-limit professions, everything else → Other\n",
+    "df[\"profession_limited\"] = df[\"profession_clean\"].apply(\n",
+    "    lambda x: x if x in top_prof else \"Other\"\n",
+    ")\n",
     "\n",
+    "# ============================================================\n",
+    "# 3. GROUP INTO SUNBURST STRUCTURE\n",
+    "# ============================================================\n",
     "\n",
-    "# ---- Step 3: Group by the limited country and profession ----\n",
-    "sunburst_data = df.groupby(['country_limited', 'profession_limited']).size().reset_index(name='count')\n",
+    "sunburst_data = (\n",
+    "    df.groupby([\"country_limited\", \"profession_limited\"])\n",
+    "    .size()\n",
+    "    .reset_index(name=\"count\")\n",
+    ")\n",
     "\n",
-    "# ---- Step 4: Create a nested structure for D3.js ----\n",
     "sunburst_dict = {\"name\": \"root\", \"children\": []}\n",
     "country_map = defaultdict(list)\n",
     "\n",
     "for _, row in sunburst_data.iterrows():\n",
-    "    country = row['country_limited']\n",
-    "    profession = row['profession_limited']\n",
-    "    count = int(row['count'])\n",
-    "    country_map[country].append({\"name\": profession, \"value\": count})\n",
+    "    c = row[\"country_limited\"]\n",
+    "    p = row[\"profession_limited\"]\n",
+    "    v = int(row[\"count\"])\n",
+    "\n",
+    "    country_map[c].append({\"name\": p, \"value\": v})\n",
+    "\n",
+    "# Sort professions inside each country so \"Other\" is last\n",
+    "for c, profs in country_map.items():\n",
+    "    profs_sorted = sorted(profs, key=lambda d: (d[\"name\"] == \"Other\", d[\"name\"]))\n",
+    "    country_map[c] = profs_sorted\n",
     "\n",
-    "# For each country, sort the profession list so that \"Other\" appears at the end\n",
-    "for country, professions in country_map.items():\n",
-    "    professions_sorted = sorted(professions, key=lambda d: (d[\"name\"] == \"Other\", d[\"name\"]))\n",
-    "    country_map[country] = professions_sorted\n",
+    "# Calculate total datapoints per country and sort\n",
+    "country_totals = []\n",
+    "for c, profs in country_map.items():\n",
+    "    total = sum(p[\"value\"] for p in profs)\n",
+    "    country_totals.append((c, total, profs))\n",
     "\n",
-    "for country, professions in country_map.items():\n",
-    "    sunburst_dict[\"children\"].append({\"name\": country, \"children\": professions})\n",
+    "# Sort by total (descending), but put \"Other\" last\n",
+    "country_totals.sort(key=lambda x: (x[0] == \"Other\", -x[1]))\n",
     "\n",
-    "# ---- Step 5: Save to a JSON file ----\n",
-    "with open(sunburst_path, \"w\", encoding='utf-8') as f:\n",
+    "# Build final JSON with sorted countries\n",
+    "for c, total, profs in country_totals:\n",
+    "    sunburst_dict[\"children\"].append({\"name\": c, \"children\": profs})\n",
+    "\n",
+    "# ============================================================\n",
+    "# 4. SAVE JSON\n",
+    "# ============================================================\n",
+    "\n",
+    "with open(sunburst_path, \"w\", encoding=\"utf-8\") as f:\n",
     "    json.dump(sunburst_dict, f, ensure_ascii=False, indent=2)\n",
     "\n",
+    "print(\"✓ Saved sunburst_countries_A.json\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# version that only considers data up until dec 31st 2024"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "✓ Filtered to 17356 records published on or before December 31, 2024\n",
+      "✓ Saved sunburst_countries_A.json (2024 data only)\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "from collections import defaultdict\n",
+    "import json\n",
+    "from pathlib import Path\n",
+    "\n",
+    "current_dir = Path.cwd()\n",
+    "sunburst_path = current_dir.parent / \"public/json/sunburst_countries_A.json\"\n",
+    "\n",
+    "# Load consensus CSV\n",
+    "consensus_file = current_dir.parent / \"data/CSV/improved_consensus.csv\"\n",
+    "df = pd.read_csv(consensus_file)\n",
+    "\n",
+    "# ============================================================\n",
+    "# FILTER DATA UP TO DECEMBER 31, 2024\n",
+    "# ============================================================\n",
+    "# Convert publishedAt to datetime\n",
+    "df[\"publishedAt\"] = pd.to_datetime(df[\"publishedAt\"], errors=\"coerce\", utc=True)\n",
+    "\n",
+    "# Filter to only include data up to December 31, 2024\n",
+    "# Make cutoff_date timezone-aware (UTC) to match publishedAt\n",
+    "cutoff_date = pd.Timestamp(\"2024-12-31 23:59:59\", tz=\"UTC\")\n",
+    "df = df[df[\"publishedAt\"] <= cutoff_date]\n",
+    "\n",
+    "print(f\"✓ Filtered to {len(df)} records published on or before December 31, 2024\")\n",
+    "\n",
+    "# ============================================================\n",
+    "# 1. NORMALIZE COUNTRIES\n",
+    "# ============================================================\n",
+    "def normalize_country(x: str):\n",
+    "    if not isinstance(x, str) or x.strip() == \"\" or x.lower() == \"unknown\":\n",
+    "        return \"Other\"\n",
+    "    x = x.strip()\n",
+    "    # Ensure matching with your HTML manualOrder\n",
+    "    replacements = {\n",
+    "        \"USA\": \"United States\",\n",
+    "        \"US\": \"United States\",\n",
+    "        \"U.S.\": \"United States\",\n",
+    "        \"UK\": \"United Kingdom\",\n",
+    "        \"U.K.\": \"United Kingdom\"\n",
+    "    }\n",
+    "    return replacements.get(x, x)\n",
+    "\n",
+    "df[\"country_clean\"] = df[\"consensus_country\"].apply(normalize_country)\n",
+    "\n",
+    "# Limit to the top 15 countries, merge the rest into \"Other\"\n",
+    "top_countries = df[\"country_clean\"].value_counts().nlargest(25).index.tolist()\n",
+    "df[\"country_limited\"] = df[\"country_clean\"].apply(\n",
+    "    lambda x: x if x in top_countries else \"Other\"\n",
+    ")\n",
+    "\n",
+    "# ============================================================\n",
+    "# 2. NORMALIZE PROFESSIONS\n",
+    "# ============================================================\n",
+    "def normalize_profession(x: str):\n",
+    "    if not isinstance(x, str) or x.strip() == \"\" or x.lower() == \"unknown\":\n",
+    "        return \"Other\"\n",
+    "    x = x.strip().lower()\n",
+    "    mapping = {\n",
+    "        \"actor\": \"Actor\",\n",
+    "        \"model\": \"Model\",\n",
+    "        \"adult performer\": \"Adult Performer\",\n",
+    "        \"singer/musician\": \"Singer, Musician\",\n",
+    "        \"online personality\": \"Online Personality\",\n",
+    "        \"sports professional\": \"Sports Professional\",\n",
+    "        \"voice actor/asmr\": \"Voice Actor\",        # ← fixed key\n",
+    "        \"public figure\": \"Public Figure\",          # ← now its own category\n",
+    "        \"tv personality\": \"Other\",\n",
+    "    }\n",
+    "    return mapping.get(x, \"Other\")\n",
+    "\n",
+    "df[\"profession_clean\"] = df[\"consensus_profession\"].apply(normalize_profession)\n",
+    "\n",
+    "# Only keep the top 7 professions (excluding Other)\n",
+    "top_prof = (\n",
+    "    df[df[\"profession_clean\"] != \"Other\"][\"profession_clean\"]\n",
+    "        .value_counts()\n",
+    "        .nlargest(7)\n",
+    "        .index.tolist()\n",
+    ")\n",
+    "\n",
+    "# Re-limit professions, everything else → Other\n",
+    "df[\"profession_limited\"] = df[\"profession_clean\"].apply(\n",
+    "    lambda x: x if x in top_prof else \"Other\"\n",
+    ")\n",
+    "\n",
+    "# ============================================================\n",
+    "# 3. GROUP INTO SUNBURST STRUCTURE\n",
+    "# ============================================================\n",
+    "sunburst_data = (\n",
+    "    df.groupby([\"country_limited\", \"profession_limited\"])\n",
+    "        .size()\n",
+    "        .reset_index(name=\"count\")\n",
+    ")\n",
+    "\n",
+    "sunburst_dict = {\"name\": \"root\", \"children\": []}\n",
+    "country_map = defaultdict(list)\n",
+    "\n",
+    "for _, row in sunburst_data.iterrows():\n",
+    "    c = row[\"country_limited\"]\n",
+    "    p = row[\"profession_limited\"]\n",
+    "    v = int(row[\"count\"])\n",
+    "    country_map[c].append({\"name\": p, \"value\": v})\n",
+    "\n",
+    "# Sort professions inside each country so \"Other\" is last\n",
+    "for c, profs in country_map.items():\n",
+    "    profs_sorted = sorted(profs, key=lambda d: (d[\"name\"] == \"Other\", d[\"name\"]))\n",
+    "    country_map[c] = profs_sorted\n",
+    "\n",
+    "# Calculate total datapoints per country and sort\n",
+    "country_totals = []\n",
+    "for c, profs in country_map.items():\n",
+    "    total = sum(p[\"value\"] for p in profs)\n",
+    "    country_totals.append((c, total, profs))\n",
+    "\n",
+    "# Sort by total (descending), but put \"Other\" last\n",
+    "country_totals.sort(key=lambda x: (x[0] == \"Other\", -x[1]))\n",
+    "\n",
+    "# Build final JSON with sorted countries\n",
+    "for c, total, profs in country_totals:\n",
+    "    sunburst_dict[\"children\"].append({\"name\": c, \"children\": profs})\n",
+    "\n",
+    "# ============================================================\n",
+    "# 4. SAVE JSON\n",
+    "# ============================================================\n",
+    "with open(sunburst_path, \"w\", encoding=\"utf-8\") as f:\n",
+    "    json.dump(sunburst_dict, f, ensure_ascii=False, indent=2)\n",
     "\n",
-    "print(\"✅ Sunburst data saved to sunburst_data.json\")\n"
+    "print(\"✓ Saved sunburst_countries_A.json (2024 data only)\")"
    ]
   },
   {
@@ -101,7 +314,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "latm",
    "language": "python",
    "name": "python3"
   },
@@ -115,7 +328,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.10"
+   "version": "3.10.15"
   }
  },
  "nbformat": 4,

public/Figure_8a_barchart.html

@@ -0,0 +1,244 @@
+<!DOCTYPE html>
+<meta charset="utf-8">
+<style>
+  body {
+    font-family: "Segoe UI", sans-serif;
+    margin: 20px;
+  }
+
+  svg {
+    font: 12px sans-serif;
+  }
+
+  .bar {
+    cursor: pointer;
+  }
+
+  .bar:hover {
+    opacity: 0.8;
+  }
+
+  .axis text {
+    font-size: 11px;
+  }
+
+  .legend {
+    font-size: 12px;
+  }
+
+  .legend rect {
+    stroke-width: 1;
+    stroke: #000;
+  }
+
+  h2 {
+    text-align: center;
+  }
+
+  #downloadBtn {
+    display: block;
+    margin: 0 auto 20px;
+  }
+</style>
+<body>
+<h2>Bar Chart: Gender → Profession</h2>
+<button id="downloadBtn">Download SVG</button>
+<svg id="chart"></svg>
+<script src="https://d3js.org/d3.v6.min.js"></script>
+<script>
+const colorMap = {
+  "Adult Performer": "#8A2BE2",
+  "Model": "#DC143C",
+  "Actor": "#FF7F50",
+  "Performer": "magenta",
+  "Singer, Musician": "wheat",
+  "TV Personality": "#708090",
+  "Sports Professional": "gold",
+  "Public Figure": "maroon",
+  "Voice Actor": "lightgreen",
+  "Online Personality": "#4682B4",
+  "Other": "#ccc"
+};
+
+const professionOrder = [
+  "Adult Performer", "Actor", "Singer, Musician", "Model", "Online Personality", "Public Figure", "Sports Professional", "Voice Actor", "TV Personality", "Other"
+];
+
+const genderOrder = ["Female", "Male", "Other"];
+
+d3.json("json/sunburst_gender_profession.json").then(data => {
+  // Transform hierarchical data into flat array
+  const flatData = [];
+
+  data.children.forEach(gender => {
+    if (gender.children) {
+      gender.children.forEach(profession => {
+        flatData.push({
+          gender: gender.name,
+          profession: profession.name,
+          value: profession.value
+        });
+      });
+    }
+  });
+
+  // Group by gender and calculate totals
+  const genderData = d3.rollup(
+    flatData,
+    v => ({
+      total: d3.sum(v, d => d.value),
+      professions: v
+    }),
+    d => d.gender
+  );
+
+  // Convert to array and sort by gender order
+  const genders = Array.from(genderData.keys()).sort((a, b) => {
+    const aIndex = genderOrder.indexOf(a);
+    const bIndex = genderOrder.indexOf(b);
+    return (aIndex === -1 ? Infinity : aIndex) - (bIndex === -1 ? Infinity : bIndex);
+  });
+
+  // Set up dimensions - responsive to window size
+  const margin = {top: 40, right: 200, bottom: 150, left: 80};
+  const width = Math.max(800, window.innerWidth - 100) - margin.left - margin.right;
+  const height = 800 - margin.top - margin.bottom;
+
+  const svg = d3.select("#chart")
+    .attr("width", width + margin.left + margin.right)
+    .attr("height", height + margin.top + margin.bottom)
+    .append("g")
+    .attr("transform", `translate(${margin.left},${margin.top})`);
+
+  // Create stacked data
+  const stack = d3.stack()
+    .keys(professionOrder)
+    .value((d, key) => {
+      const prof = d[1].professions.find(p => p.profession === key);
+      return prof ? prof.value : 0;
+    });
+
+  const series = stack(Array.from(genderData));
+
+  // Scales
+  const x = d3.scaleBand()
+    .domain(genders)
+    .range([0, width])
+    .padding(0.3);
+
+  const y = d3.scaleLinear()
+    .domain([0, d3.max(Array.from(genderData.values()), d => d.total)])
+    .nice()
+    .range([height, 0]);
+
+  // X axis
+  svg.append("g")
+    .attr("class", "axis")
+    .attr("transform", `translate(0,${height})`)
+    .call(d3.axisBottom(x))
+    .selectAll("text")
+    .attr("transform", "rotate(-90)")
+    .style("text-anchor", "end")
+    .style("font-size", "11px")
+    .style("font-weight", "bold")
+    .attr("dx", "-0.5em")
+    .attr("dy", "-0.5em");
+
+  // Y axis
+  svg.append("g")
+    .attr("class", "axis")
+    .call(d3.axisLeft(y));
+
+  // Y axis label
+  svg.append("text")
+    .attr("transform", "rotate(-90)")
+    .attr("y", 0 - margin.left + 20)
+    .attr("x", 0 - (height / 2))
+    .attr("dy", "1em")
+    .style("text-anchor", "middle")
+    .style("font-size", "14px")
+    .style("font-weight", "bold")
+    .text("Count");
+
+  // Draw bars
+  svg.append("g")
+    .selectAll("g")
+    .data(series)
+    .join("g")
+    .attr("fill", d => colorMap[d.key])
+    .selectAll("rect")
+    .data(d => d)
+    .join("rect")
+    .attr("class", "bar")
+    .attr("x", d => x(d.data[0]))
+    .attr("y", d => y(d[1]))
+    .attr("height", d => y(d[0]) - y(d[1]))
+    .attr("width", x.bandwidth())
+    .append("title")
+    .text(d => {
+      const profession = series.find(s => s.find(item => item === d))?.key;
+      return `${d.data[0]} - ${profession}: ${d[1] - d[0]}`;
+    });
+
+  // Count total values per profession
+  const professionCounts = {};
+  flatData.forEach(d => {
+    professionCounts[d.profession] = (professionCounts[d.profession] || 0) + d.value;
+  });
+
+  // Legend
+  const legend = svg.append("g")
+    .attr("class", "legend")
+    .attr("transform", `translate(${width + 20}, 0)`);
+
+  professionOrder.forEach((prof, i) => {
+    const row = legend.append("g")
+      .attr("transform", `translate(0,${i * 22})`);
+
+    row.append("rect")
+      .attr("width", 18)
+      .attr("height", 18)
+      .attr("fill", colorMap[prof]);
+
+    row.append("text")
+      .attr("x", 24)
+      .attr("y", 9)
+      .attr("dy", "0.35em")
+      .style("font-size", "12px")
+      .text(`${prof} (${professionCounts[prof] || 0})`);
+  });
+
+  // Download button functionality
+  document.getElementById("downloadBtn").addEventListener("click", () => {
+    const svgNode = document.querySelector("#chart");
+    const clonedSvg = svgNode.cloneNode(true);
+
+    clonedSvg.setAttribute("xmlns", "http://www.w3.org/2000/svg");
+
+    // Inline styles
+    const allElements = clonedSvg.querySelectorAll("*");
+    allElements.forEach(el => {
+      const style = window.getComputedStyle(el);
+      el.setAttribute("style", `
+        font: ${style.font};
+        fill: ${style.fill};
+        stroke: ${style.stroke};
+        stroke-width: ${style.strokeWidth};
+      `);
+    });
+
+    const svgData = new XMLSerializer().serializeToString(clonedSvg);
+    const svgBlob = new Blob([svgData], { type: "image/svg+xml;charset=utf-8" });
+    const url = URL.createObjectURL(svgBlob);
+    const a = document.createElement("a");
+    a.href = url;
+    a.download = "bar_chart_gender.svg";
+    document.body.appendChild(a);
+    a.click();
+    document.body.removeChild(a);
+    URL.revokeObjectURL(url);
+  });
+});
+</script>
+</body>
+</html>

public/Figure_8b_barchart.html

@@ -46,11 +46,14 @@
 <svg id="chart"></svg>
 <script src="https://d3js.org/d3.v6.min.js"></script>
 <script>
+/* --------------------------------------------------------
+   1. PROFESSION COLORS + ORDER
+   -------------------------------------------------------- */
 const colorMap = {
   "Adult Performer": "#8A2BE2",
   "Model": "#DC143C",
   "Actor": "#FF7F50",
-  "Performer": "magenta",
+  "Public Figure": "#20B2AA",  // ← add this (or pick your color)
   "Singer, Musician": "wheat",
   "Sports Professional": "gold",
   "Voice Actor": "lightgreen",
@@ -58,10 +61,25 @@ const colorMap = {
   "Other": "#ccc"
 };
 
+// IMPORTANT → this now matches exactly the categories
+// that your updated Python script produces.
 const professionOrder = [
-  "Actor", "Adult Performer", "Singer, Musician", "Model", "Online Personality", "Sports Professional", "Voice Actor", "Other"
+  "Actor",
+  "Adult Performer",
+  "Singer, Musician",
+  "Model",
+  "Online Personality",
+  "Sports Professional",
+  "Voice Actor",
+  "Public Figure",
+  "Other"
 ];
 
+//#DC143C
+
+/* --------------------------------------------------------
+   2. COUNTRY ORDER (unchanged)
+   -------------------------------------------------------- */
 const manualOrder = [
   "United States",
   "Japan",
@@ -92,27 +110,29 @@ const manualOrder = [
   "Philippines",
   "Hong Kong",
   "Macau",
-  "British Virgin Islands",
   "Other"
 ];
 
+/* --------------------------------------------------------
+   3. LOAD JSON + TRANSFORM IT
+   -------------------------------------------------------- */
 d3.json("json/sunburst_countries_A.json").then(data => {
-  // Transform hierarchical data into flat array
+
   const flatData = [];
 
   data.children.forEach(country => {
     if (country.children) {
-      country.children.forEach(profession => {
+      country.children.forEach(prof => {
         flatData.push({
           country: country.name,
-          profession: profession.name,
-          value: profession.value
+          profession: prof.name,
+          value: prof.value
         });
       });
     }
   });
 
-  // Group by country and calculate totals
+  // Group per country
   const countryData = d3.rollup(
     flatData,
     v => ({
@@ -122,14 +142,16 @@ d3.json("json/sunburst_countries_A.json").then(data => {
     d => d.country
   );
 
-  // Convert to array and sort by manual order
+  // Sort according to manual ordering
   const countries = Array.from(countryData.keys()).sort((a, b) => {
-    const aIndex = manualOrder.indexOf(a);
-    const bIndex = manualOrder.indexOf(b);
-    return (aIndex === -1 ? Infinity : aIndex) - (bIndex === -1 ? Infinity : bIndex);
+    const ai = manualOrder.indexOf(a);
+    const bi = manualOrder.indexOf(b);
+    return (ai === -1 ? Infinity : ai) - (bi === -1 ? Infinity : bi);
   });
 
-  // Set up dimensions - responsive to window size
+  /* --------------------------------------------------------
+     4. SVG SETUP
+     -------------------------------------------------------- */
   const margin = {top: 40, right: 200, bottom: 150, left: 80};
   const width = Math.max(800, window.innerWidth - 100) - margin.left - margin.right;
   const height = 800 - margin.top - margin.bottom;
@@ -140,7 +162,9 @@ d3.json("json/sunburst_countries_A.json").then(data => {
     .append("g")
     .attr("transform", `translate(${margin.left},${margin.top})`);
 
-  // Create stacked data
+  /* --------------------------------------------------------
+     5. STACKED DATA
+     -------------------------------------------------------- */
   const stack = d3.stack()
     .keys(professionOrder)
     .value((d, key) => {
@@ -150,11 +174,13 @@ d3.json("json/sunburst_countries_A.json").then(data => {
 
   const series = stack(Array.from(countryData));
 
-  // Scales - only show top 25 countries
-  const top25Countries = countries.slice(0, 25);
+  /* --------------------------------------------------------
+     6. AXES
+     -------------------------------------------------------- */
+  const topCountries = countries.slice(0, 25);
 
   const x = d3.scaleBand()
-    .domain(top25Countries)
+    .domain(topCountries)
     .range([0, width])
     .padding(0.3);
 
@@ -163,36 +189,29 @@ d3.json("json/sunburst_countries_A.json").then(data => {
     .nice()
     .range([height, 0]);
 
-  // X axis
   svg.append("g")
-    .attr("class", "axis")
     .attr("transform", `translate(0,${height})`)
     .call(d3.axisBottom(x))
     .selectAll("text")
     .attr("transform", "rotate(-90)")
     .style("text-anchor", "end")
-    .style("font-size", "11px")
     .style("font-weight", "bold")
     .attr("dx", "-0.5em")
     .attr("dy", "-0.5em");
 
-  // Y axis
   svg.append("g")
-    .attr("class", "axis")
     .call(d3.axisLeft(y));
 
-  // Y axis label
   svg.append("text")
     .attr("transform", "rotate(-90)")
     .attr("y", 0 - margin.left + 20)
-    .attr("x", 0 - (height / 2))
-    .attr("dy", "1em")
-    .style("text-anchor", "middle")
-    .style("font-size", "14px")
+    .attr("x", 0 - height / 2)
     .style("font-weight", "bold")
     .text("Count");
 
-  // Draw bars
+  /* --------------------------------------------------------
+     7. DRAW BARS
+     -------------------------------------------------------- */
   svg.append("g")
     .selectAll("g")
     .data(series)
@@ -208,24 +227,23 @@ d3.json("json/sunburst_countries_A.json").then(data => {
     .attr("width", x.bandwidth())
     .append("title")
     .text(d => {
-      const profession = series.find(s => s.find(item => item === d))?.key;
-      return `${d.data[0]} - ${profession}: ${d[1] - d[0]}`;
+      const profKey = series.find(s => s.includes(d))?.key;
+      return `${d.data[0]} – ${profKey}: ${d[1] - d[0]}`;
     });
 
-  // Count total values per profession
+  /* --------------------------------------------------------
+     8. LEGEND
+     -------------------------------------------------------- */
   const professionCounts = {};
   flatData.forEach(d => {
     professionCounts[d.profession] = (professionCounts[d.profession] || 0) + d.value;
   });
 
-  // Legend
   const legend = svg.append("g")
-    .attr("class", "legend")
     .attr("transform", `translate(${width + 20}, 0)`);
 
   professionOrder.forEach((prof, i) => {
-    const row = legend.append("g")
-      .attr("transform", `translate(0,${i * 22})`);
+    const row = legend.append("g").attr("transform", `translate(0,${i * 22})`);
 
     row.append("rect")
       .attr("width", 18)
@@ -236,41 +254,37 @@ d3.json("json/sunburst_countries_A.json").then(data => {
       .attr("x", 24)
       .attr("y", 9)
       .attr("dy", "0.35em")
-      .style("font-size", "12px")
       .text(`${prof} (${professionCounts[prof] || 0})`);
   });
 
-  // Download button functionality
+  /* --------------------------------------------------------
+     9. DOWNLOAD SVG BUTTON
+     -------------------------------------------------------- */
   document.getElementById("downloadBtn").addEventListener("click", () => {
     const svgNode = document.querySelector("#chart");
-    const clonedSvg = svgNode.cloneNode(true);
+    const clone = svgNode.cloneNode(true);
 
-    clonedSvg.setAttribute("xmlns", "http://www.w3.org/2000/svg");
+    clone.setAttribute("xmlns", "http://www.w3.org/2000/svg");
 
-    // Inline styles
-    const allElements = clonedSvg.querySelectorAll("*");
-    allElements.forEach(el => {
+    const all = clone.querySelectorAll("*");
+    all.forEach(el => {
       const style = window.getComputedStyle(el);
-      el.setAttribute("style", `
-        font: ${style.font};
-        fill: ${style.fill};
-        stroke: ${style.stroke};
-        stroke-width: ${style.strokeWidth};
-      `);
+      el.setAttribute("style", `font:${style.font}; fill:${style.fill}; stroke:${style.stroke};`);
     });
 
-    const svgData = new XMLSerializer().serializeToString(clonedSvg);
-    const svgBlob = new Blob([svgData], { type: "image/svg+xml;charset=utf-8" });
-    const url = URL.createObjectURL(svgBlob);
+    const svgData = new XMLSerializer().serializeToString(clone);
+    const blob = new Blob([svgData], { type: "image/svg+xml;charset=utf-8" });
+    const url = URL.createObjectURL(blob);
     const a = document.createElement("a");
+
     a.href = url;
     a.download = "bar_chart.svg";
-    document.body.appendChild(a);
     a.click();
-    document.body.removeChild(a);
     URL.revokeObjectURL(url);
   });
+
 });
 </script>
+
 </body>
 </html>

public/Figure_8b_sunburst.html

@@ -37,26 +37,23 @@ const width = 800;
 const radius = width / 3;
 
 const colorMap = {
-  "Adult Performer": "#8A2BE2",
-  "Model": "#DC143C",
-  "Actor": "#FF7F50",
-  "Performer": "magenta",
-  "Singer, Musician": "wheat",
-  //"Fictional Character": "#708090",
-  "Sports Professional": "gold",
-  "Voice Actor": "lightgreen",
-  "Online Personality": "#4682B4",
+  "actor": "#FF7F50",
+  "adult performer": "#8A2BE2",
+  "singer/musician": "#FFD700",
+  "model": "#DC143C",
+  "online personality": "#4682B4",
+  "public figure": "#9370DB",
   "Other": "#ccc"
 };
 
 const professionOrder = [
-  "Actor", "Adult Performer", "Singer, Musician", "Model", "Online Personality", "Sports Professional", "Voice Actor",  "Other"
+  "actor", "adult performer", "singer/musician", "model", "online personality", "public figure", "Other"
 ];
 
 const manualOrder = [
-  "United States",
+  "USA",
   "Japan",
-  "United Kingdom",
+  "UK",
   "South Korea",
   "Russia",
   "India",
@@ -69,21 +66,6 @@ const manualOrder = [
   "Australia",
   "Spain",
   "Ukraine",
-  "Turkey",
-  "Netherlands",
-  "Indonesia",
-  "Poland",
-  "Czech Republic",
-  "Argentina",
-  "Sweden",
-  "Mexico",
-  "Taiwan",
-  "Thailand",
-  "Ireland",
-  "Philippines",
-  "Hong Kong",
-  "Macau",
-  "British Virgin Islands",
   "Other"
 ];
 

public/json/8a.json

@@ -5,28 +5,44 @@
       "name": "Female",
       "children": [
         {
-          "name": "Actor",
-          "value": 8
+          "name": "actor",
+          "value": 9397
         },
         {
-          "name": "Adult Performer",
-          "value": 2
+          "name": "adult performer",
+          "value": 718
         },
         {
-          "name": "Online Personality",
-          "value": 4
+          "name": "model",
+          "value": 2715
         },
         {
-          "name": "Singer, Musician",
-          "value": 2
+          "name": "online personality",
+          "value": 1348
         },
         {
-          "name": "Sports Professional",
-          "value": 1
+          "name": "public figure",
+          "value": 406
         },
         {
-          "name": "Voice Actor",
-          "value": 1
+          "name": "singer/musician",
+          "value": 3324
+        },
+        {
+          "name": "sports professional",
+          "value": 166
+        },
+        {
+          "name": "tv personality",
+          "value": 247
+        },
+        {
+          "name": "voice actor/ASMR",
+          "value": 343
+        },
+        {
+          "name": "Other",
+          "value": 15
         }
       ]
     },
@@ -34,8 +50,44 @@
       "name": "Male",
       "children": [
         {
-          "name": "Voice Actor",
-          "value": 1
+          "name": "actor",
+          "value": 1373
+        },
+        {
+          "name": "adult performer",
+          "value": 12
+        },
+        {
+          "name": "model",
+          "value": 43
+        },
+        {
+          "name": "online personality",
+          "value": 144
+        },
+        {
+          "name": "public figure",
+          "value": 522
+        },
+        {
+          "name": "singer/musician",
+          "value": 341
+        },
+        {
+          "name": "sports professional",
+          "value": 221
+        },
+        {
+          "name": "tv personality",
+          "value": 45
+        },
+        {
+          "name": "voice actor/ASMR",
+          "value": 4
+        },
+        {
+          "name": "Other",
+          "value": 15
         }
       ]
     },
@@ -43,8 +95,8 @@
       "name": "Other",
       "children": [
         {
-          "name": "Other",
-          "value": 2
+          "name": "actor",
+          "value": 1
         }
       ]
     }

public/json/sunburst_countries_A.json

@@ -2,126 +2,856 @@
   "name": "root",
   "children": [
     {
-      "name": "Australia",
+      "name": "usa",
       "children": [
         {
           "name": "Actor",
+          "value": 3912
+        },
+        {
+          "name": "Adult Performer",
+          "value": 262
+        },
+        {
+          "name": "Model",
+          "value": 356
+        },
+        {
+          "name": "Online Personality",
+          "value": 255
+        },
+        {
+          "name": "Public Figure",
+          "value": 225
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 733
+        },
+        {
+          "name": "Voice Actor",
+          "value": 5
+        },
+        {
+          "name": "Other",
+          "value": 231
+        }
+      ]
+    },
+    {
+      "name": "japan",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 357
+        },
+        {
+          "name": "Adult Performer",
+          "value": 392
+        },
+        {
+          "name": "Model",
+          "value": 616
+        },
+        {
+          "name": "Online Personality",
+          "value": 136
+        },
+        {
+          "name": "Public Figure",
+          "value": 30
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 503
+        },
+        {
+          "name": "Voice Actor",
+          "value": 360
+        },
+        {
+          "name": "Other",
+          "value": 31
+        }
+      ]
+    },
+    {
+      "name": "south korea",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 182
+        },
+        {
+          "name": "Model",
+          "value": 119
+        },
+        {
+          "name": "Online Personality",
+          "value": 45
+        },
+        {
+          "name": "Public Figure",
+          "value": 2
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 1093
+        },
+        {
+          "name": "Other",
+          "value": 9
+        }
+      ]
+    },
+    {
+      "name": "uk",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 1012
+        },
+        {
+          "name": "Adult Performer",
+          "value": 12
+        },
+        {
+          "name": "Model",
+          "value": 111
+        },
+        {
+          "name": "Online Personality",
+          "value": 26
+        },
+        {
+          "name": "Public Figure",
+          "value": 46
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 152
+        },
+        {
+          "name": "Voice Actor",
+          "value": 2
+        },
+        {
+          "name": "Other",
+          "value": 67
+        }
+      ]
+    },
+    {
+      "name": "china",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 229
+        },
+        {
+          "name": "Adult Performer",
+          "value": 34
+        },
+        {
+          "name": "Model",
+          "value": 257
+        },
+        {
+          "name": "Online Personality",
+          "value": 220
+        },
+        {
+          "name": "Public Figure",
+          "value": 13
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 63
+        },
+        {
+          "name": "Other",
+          "value": 6
+        }
+      ]
+    },
+    {
+      "name": "india",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 691
+        },
+        {
+          "name": "Adult Performer",
+          "value": 4
+        },
+        {
+          "name": "Model",
+          "value": 30
+        },
+        {
+          "name": "Online Personality",
+          "value": 14
+        },
+        {
+          "name": "Public Figure",
+          "value": 7
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 6
+        },
+        {
+          "name": "Other",
+          "value": 6
+        }
+      ]
+    },
+    {
+      "name": "france",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 204
+        },
+        {
+          "name": "Adult Performer",
+          "value": 27
+        },
+        {
+          "name": "Model",
+          "value": 40
+        },
+        {
+          "name": "Online Personality",
+          "value": 23
+        },
+        {
+          "name": "Public Figure",
+          "value": 60
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 38
+        },
+        {
+          "name": "Other",
+          "value": 28
+        }
+      ]
+    },
+    {
+      "name": "russia",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 36
+        },
+        {
+          "name": "Adult Performer",
+          "value": 57
+        },
+        {
+          "name": "Model",
+          "value": 118
+        },
+        {
+          "name": "Online Personality",
+          "value": 89
+        },
+        {
+          "name": "Public Figure",
+          "value": 46
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 24
+        },
+        {
+          "name": "Other",
+          "value": 17
+        }
+      ]
+    },
+    {
+      "name": "canada",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 246
+        },
+        {
+          "name": "Adult Performer",
+          "value": 4
+        },
+        {
+          "name": "Model",
+          "value": 19
+        },
+        {
+          "name": "Online Personality",
+          "value": 11
+        },
+        {
+          "name": "Public Figure",
+          "value": 18
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 66
+        },
+        {
+          "name": "Voice Actor",
           "value": 1
+        },
+        {
+          "name": "Other",
+          "value": 11
+        }
+      ]
+    },
+    {
+      "name": "brazil",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 90
+        },
+        {
+          "name": "Adult Performer",
+          "value": 21
+        },
+        {
+          "name": "Model",
+          "value": 95
+        },
+        {
+          "name": "Online Personality",
+          "value": 34
+        },
+        {
+          "name": "Public Figure",
+          "value": 15
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 39
+        },
+        {
+          "name": "Other",
+          "value": 27
+        }
+      ]
+    },
+    {
+      "name": "australia",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 172
+        },
+        {
+          "name": "Adult Performer",
+          "value": 6
+        },
+        {
+          "name": "Model",
+          "value": 37
+        },
+        {
+          "name": "Online Personality",
+          "value": 16
+        },
+        {
+          "name": "Public Figure",
+          "value": 3
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 25
+        },
+        {
+          "name": "Other",
+          "value": 7
+        }
+      ]
+    },
+    {
+      "name": "germany",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 51
+        },
+        {
+          "name": "Adult Performer",
+          "value": 7
+        },
+        {
+          "name": "Model",
+          "value": 52
+        },
+        {
+          "name": "Online Personality",
+          "value": 22
+        },
+        {
+          "name": "Public Figure",
+          "value": 47
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 27
+        },
+        {
+          "name": "Other",
+          "value": 22
         }
       ]
     },
     {
-      "name": "China",
+      "name": "italy",
       "children": [
         {
           "name": "Actor",
+          "value": 74
+        },
+        {
+          "name": "Adult Performer",
+          "value": 7
+        },
+        {
+          "name": "Model",
+          "value": 37
+        },
+        {
+          "name": "Online Personality",
+          "value": 29
+        },
+        {
+          "name": "Public Figure",
+          "value": 20
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 28
+        },
+        {
+          "name": "Voice Actor",
           "value": 1
+        },
+        {
+          "name": "Other",
+          "value": 26
         }
       ]
     },
     {
-      "name": "Colombia",
+      "name": "ukraine",
       "children": [
         {
           "name": "Actor",
-          "value": 1
+          "value": 11
+        },
+        {
+          "name": "Adult Performer",
+          "value": 49
+        },
+        {
+          "name": "Model",
+          "value": 44
+        },
+        {
+          "name": "Online Personality",
+          "value": 48
+        },
+        {
+          "name": "Public Figure",
+          "value": 13
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 6
+        },
+        {
+          "name": "Other",
+          "value": 4
         }
       ]
     },
     {
-      "name": "Czechia",
+      "name": "türkiye",
       "children": [
         {
-          "name": "Voice Actor",
-          "value": 1
+          "name": "Actor",
+          "value": 82
+        },
+        {
+          "name": "Model",
+          "value": 32
+        },
+        {
+          "name": "Online Personality",
+          "value": 14
+        },
+        {
+          "name": "Public Figure",
+          "value": 15
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 10
+        },
+        {
+          "name": "Other",
+          "value": 9
         }
       ]
     },
     {
-      "name": "Denmark",
+      "name": "thailand",
       "children": [
         {
           "name": "Actor",
+          "value": 25
+        },
+        {
+          "name": "Adult Performer",
+          "value": 12
+        },
+        {
+          "name": "Model",
+          "value": 19
+        },
+        {
+          "name": "Online Personality",
+          "value": 43
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 41
+        },
+        {
+          "name": "Other",
           "value": 1
         }
       ]
     },
     {
-      "name": "Germany",
+      "name": "spain",
       "children": [
+        {
+          "name": "Actor",
+          "value": 60
+        },
+        {
+          "name": "Adult Performer",
+          "value": 6
+        },
+        {
+          "name": "Model",
+          "value": 11
+        },
         {
           "name": "Online Personality",
-          "value": 1
+          "value": 17
+        },
+        {
+          "name": "Public Figure",
+          "value": 11
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 12
+        },
+        {
+          "name": "Other",
+          "value": 22
         }
       ]
     },
     {
-      "name": "India",
+      "name": "netherlands",
       "children": [
         {
           "name": "Actor",
-          "value": 1
+          "value": 27
+        },
+        {
+          "name": "Adult Performer",
+          "value": 2
+        },
+        {
+          "name": "Model",
+          "value": 24
+        },
+        {
+          "name": "Online Personality",
+          "value": 17
+        },
+        {
+          "name": "Public Figure",
+          "value": 34
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 10
+        },
+        {
+          "name": "Other",
+          "value": 13
         }
       ]
     },
     {
-      "name": "Italy",
+      "name": "ireland",
       "children": [
+        {
+          "name": "Actor",
+          "value": 61
+        },
+        {
+          "name": "Online Personality",
+          "value": 2
+        },
+        {
+          "name": "Public Figure",
+          "value": 28
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 10
+        },
         {
           "name": "Other",
-          "value": 1
+          "value": 10
         }
       ]
     },
     {
-      "name": "Japan",
+      "name": "indonesia",
       "children": [
         {
-          "name": "Voice Actor",
+          "name": "Actor",
+          "value": 14
+        },
+        {
+          "name": "Adult Performer",
+          "value": 2
+        },
+        {
+          "name": "Model",
+          "value": 26
+        },
+        {
+          "name": "Online Personality",
+          "value": 30
+        },
+        {
+          "name": "Public Figure",
+          "value": 4
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 28
+        },
+        {
+          "name": "Other",
           "value": 1
         }
       ]
     },
     {
-      "name": "Other",
+      "name": "sweden",
       "children": [
+        {
+          "name": "Actor",
+          "value": 28
+        },
+        {
+          "name": "Model",
+          "value": 15
+        },
+        {
+          "name": "Online Personality",
+          "value": 9
+        },
+        {
+          "name": "Public Figure",
+          "value": 21
+        },
         {
           "name": "Singer, Musician",
-          "value": 1
+          "value": 23
         },
         {
           "name": "Other",
-          "value": 2
+          "value": 9
+        }
+      ]
+    },
+    {
+      "name": "poland",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 25
+        },
+        {
+          "name": "Adult Performer",
+          "value": 6
+        },
+        {
+          "name": "Model",
+          "value": 30
+        },
+        {
+          "name": "Online Personality",
+          "value": 14
+        },
+        {
+          "name": "Public Figure",
+          "value": 9
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 8
+        },
+        {
+          "name": "Other",
+          "value": 10
         }
       ]
     },
     {
-      "name": "Poland",
+      "name": "argentina",
       "children": [
         {
           "name": "Actor",
+          "value": 12
+        },
+        {
+          "name": "Adult Performer",
           "value": 1
+        },
+        {
+          "name": "Model",
+          "value": 5
+        },
+        {
+          "name": "Online Personality",
+          "value": 3
+        },
+        {
+          "name": "Public Figure",
+          "value": 23
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 30
+        },
+        {
+          "name": "Other",
+          "value": 15
         }
       ]
     },
     {
-      "name": "US",
+      "name": "taiwan",
       "children": [
         {
           "name": "Actor",
-          "value": 2
+          "value": 14
         },
         {
           "name": "Adult Performer",
+          "value": 3
+        },
+        {
+          "name": "Model",
+          "value": 18
+        },
+        {
+          "name": "Online Personality",
+          "value": 31
+        },
+        {
+          "name": "Public Figure",
+          "value": 2
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 15
+        },
+        {
+          "name": "Other",
           "value": 2
+        }
+      ]
+    },
+    {
+      "name": "mexico",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 57
+        },
+        {
+          "name": "Adult Performer",
+          "value": 1
+        },
+        {
+          "name": "Model",
+          "value": 9
         },
         {
           "name": "Online Personality",
+          "value": 2
+        },
+        {
+          "name": "Public Figure",
+          "value": 2
+        },
+        {
+          "name": "Singer, Musician",
+          "value": 7
+        },
+        {
+          "name": "Other",
           "value": 3
+        }
+      ]
+    },
+    {
+      "name": "Other",
+      "children": [
+        {
+          "name": "Actor",
+          "value": 255
+        },
+        {
+          "name": "Adult Performer",
+          "value": 39
+        },
+        {
+          "name": "Model",
+          "value": 204
+        },
+        {
+          "name": "Online Personality",
+          "value": 95
+        },
+        {
+          "name": "Public Figure",
+          "value": 87
         },
         {
           "name": "Singer, Musician",
-          "value": 1
+          "value": 110
+        },
+        {
+          "name": "Other",
+          "value": 62
         }
       ]
     }