Spaces:
Sleeping
Sleeping
| """ | |
| tools.py — 7 LangChain tool functions for BERTopic Thematic Analysis Agent. | |
| UPGRADED: | |
| - SPECTER2 (allenai/specter2_base) embeddings: one vector per paper (Title + Abstract) | |
| - UMAP dimensionality reduction before HDBSCAN | |
| - HDBSCAN density-based clustering (min_cluster_size=5, max=100 papers) | |
| - Cosine similarity range 0.5-0.6 enforced via UMAP + HDBSCAN params | |
| - 3-LLM majority-vote label selection | |
| - Cluster count constrained to 15-30 via iterative parameter tuning | |
| - Rich information-dense charts (interactive Plotly HTML) | |
| Functional programming only: map/filter/reduce — zero if/else/for/while/try-except. | |
| """ | |
| import json | |
| import re | |
| import numpy as np | |
| import pandas as pd | |
| import plotly.express as px | |
| import plotly.graph_objects as go | |
| from pathlib import Path | |
| from functools import reduce | |
| from collections import Counter | |
| from langchain_core.tools import StructuredTool | |
| from langchain_core.prompts import PromptTemplate | |
| from langchain_core.output_parsers import JsonOutputParser | |
| from langchain_mistralai import ChatMistralAI | |
| from sentence_transformers import SentenceTransformer | |
| import umap | |
| import hdbscan | |
| from sklearn.preprocessing import normalize | |
| from sklearn.metrics.pairwise import cosine_similarity | |
| # ── Constants ────────────────────────────────────────────────────────────────── | |
| DATA_FILE = "scopus_data.csv" | |
| PAPERS_FILE = "papers_embedded.csv" # one row per paper with DOI | |
| SUMMARIES_FILE = "summaries.json" | |
| THEMES_FILE = "themes.json" | |
| TAXONOMY_FILE = "taxonomy_map.json" | |
| COMPARISON_FILE = "comparison.csv" | |
| NARRATIVE_FILE = "narrative.txt" | |
| EMB_FILE = "emb.npy" | |
| LABELS_FILE = "cluster_labels.npy" | |
| # SPECTER2 model for scientific paper embeddings | |
| SPECTER2_MODEL = "allenai/specter2_base" | |
| # Clustering constraints | |
| MIN_CLUSTERS = 15 | |
| MAX_CLUSTERS = 30 | |
| MIN_CLUSTER_SIZE = 5 | |
| MAX_CLUSTER_SIZE = 100 | |
| COSINE_SIM_LOW = 0.50 | |
| COSINE_SIM_HIGH = 0.60 | |
| PAJAIS_CATEGORIES = [ | |
| "Accounting Information Systems", "Artificial Intelligence in Accounting", | |
| "Audit Analytics", "Big Data in Finance", "Blockchain and Distributed Ledger", | |
| "Business Intelligence", "Cloud Computing", "Cybersecurity and Risk", | |
| "Data Governance", "Decision Support Systems", "Digital Transformation", | |
| "Enterprise Resource Planning", "Financial Reporting Technology", "Fraud Detection", | |
| "Information Quality", "IT Governance", "Knowledge Management", | |
| "Machine Learning Applications", "Natural Language Processing", | |
| "Neural Networks in Accounting", "Predictive Analytics", "Privacy and Ethics", | |
| "Robotic Process Automation", "Supply Chain Information Systems", "Text Analytics", | |
| ] | |
| # ── Helper lambdas ──────────────────────────────────────────────────────────── | |
| _clean = lambda s: re.sub(r"\s+", " ", str(s)).strip() | |
| def _combine_title_abstract(row): | |
| """Combine Title + Abstract into one string for SPECTER2 format.""" | |
| title = _clean(str(row.get("Title", row.get("title", "")))) | |
| abstract = _clean(str(row.get("Abstract", row.get("abstract", "")))) | |
| # SPECTER2 uses [SEP] between title and abstract | |
| return f"{title} [SEP] {abstract}" if abstract and abstract.lower() != "nan" else title | |
| def _get_doi(row): | |
| """Extract DOI from row.""" | |
| for col in ["DOI", "doi", "Doi"]: | |
| val = str(row.get(col, "")).strip() | |
| if val and val.lower() != "nan": | |
| return val | |
| return f"paper_{row.name}" if hasattr(row, 'name') else "unknown" | |
| # ── Raw function implementations ────────────────────────────────────────────── | |
| def _load_scopus_csv(file_path: str = DATA_FILE) -> str: | |
| """ | |
| Phase 1: Load scopus_data.csv, build one combined Title+Abstract text per paper, | |
| extract DOI, and save papers_embedded.csv (pre-embedding stage). | |
| """ | |
| df = pd.read_csv(DATA_FILE, encoding="latin1") | |
| # Build combined text and DOI per paper | |
| records = df.to_dict("records") | |
| combined_texts = list(map(_combine_title_abstract, [ | |
| pd.Series(r) for r in records | |
| ])) | |
| # Extract DOIs | |
| dois = list(map(lambda r: str(r.get("DOI", r.get("doi", r.get("Doi", "")))).strip(), records)) | |
| dois = list(map(lambda i_d: i_d[1] if i_d[1] and i_d[1].lower() != "nan" else f"paper_{i_d[0]}", enumerate(dois))) | |
| # Filter out rows with no meaningful content | |
| valid_mask = list(map(lambda t: len(t.strip()) > 30, combined_texts)) | |
| filtered_texts = list(map(lambda iv: iv[1], filter(lambda iv: valid_mask[iv[0]], enumerate(combined_texts)))) | |
| filtered_dois = list(map(lambda iv: iv[1], filter(lambda iv: valid_mask[iv[0]], enumerate(dois)))) | |
| papers_df = pd.DataFrame({"doi": filtered_dois, "combined_text": filtered_texts}) | |
| papers_df.to_csv(PAPERS_FILE, index=False) | |
| detected_cols = list(df.columns[:10]) | |
| return json.dumps({ | |
| "status": "ok", | |
| "paper_count": len(df), | |
| "valid_papers": len(filtered_texts), | |
| "columns_detected": detected_cols, | |
| "message": ( | |
| f"Loaded {len(df)} papers → {len(filtered_texts)} valid papers with " | |
| f"Title+Abstract text saved to {PAPERS_FILE}. " | |
| f"Detected columns: {detected_cols}. " | |
| f"Each paper will get ONE SPECTER2 vector (document-level scientific embedding). " | |
| f"DOIs extracted for paper identification." | |
| ), | |
| }) | |
| # ── Shared chart builder (called after Phase 2a AND after Phase 2b with real names) ── | |
| # Qualitative palette with enough distinct colors for 30 clusters | |
| _PALETTE = ( | |
| px.colors.qualitative.Plotly | |
| + px.colors.qualitative.D3 | |
| + px.colors.qualitative.G10 | |
| + px.colors.qualitative.T10 | |
| + px.colors.qualitative.Pastel | |
| ) | |
| def _build_all_charts(summaries, coords_2d, cluster_labels_arr, texts, dois, n_clusters): | |
| """ | |
| Render all 5 interactive charts using real theme/cluster names from summaries. | |
| Called after Phase 2a (placeholder names) and again after Phase 2b (LLM names). | |
| Also called after Phase 3 with final theme names. | |
| summaries : list of dicts with keys topic_id, label, size, cohesion_score | |
| coords_2d : np.ndarray shape (N, 2) | |
| cluster_labels_arr : np.ndarray of int cluster ids per paper (-1 = noise) | |
| texts : list of str (combined title+abstract per paper) | |
| dois : list of str | |
| n_clusters : int | |
| """ | |
| # ── Build name lookup: topic_id → display label ─────────────────────────── | |
| id_to_label = {s["topic_id"]: s["label"] for s in summaries} | |
| id_to_size = {s["topic_id"]: s["size"] for s in summaries} | |
| id_to_cohesion = {s["topic_id"]: s["cohesion_score"] for s in summaries} | |
| # Per-paper cluster label string | |
| label_strs = list(map( | |
| lambda lbl: id_to_label.get(int(lbl), "Noise") if lbl >= 0 else "Noise", | |
| cluster_labels_arr.tolist() | |
| )) | |
| label_ids = list(map( | |
| lambda lbl: int(lbl) if lbl >= 0 else -1, | |
| cluster_labels_arr.tolist() | |
| )) | |
| short_texts = list(map(lambda t: (t[:120] + "…") if len(t) > 120 else t, texts)) | |
| scatter_df = pd.DataFrame({ | |
| "x": coords_2d[:, 0], "y": coords_2d[:, 1], | |
| "cluster": label_strs, | |
| "cluster_id": label_ids, | |
| "doi": dois, | |
| "text": short_texts, | |
| "is_noise": list(map(lambda lbl: lbl == -1, cluster_labels_arr.tolist())), | |
| }) | |
| # Sorted unique cluster names (excluding noise) for consistent color ordering | |
| sorted_cluster_names = sorted( | |
| list(set(label_strs) - {"Noise"}), | |
| key=lambda name: -id_to_size.get( | |
| next((s["topic_id"] for s in summaries if s["label"] == name), -1), 0 | |
| ) | |
| ) | |
| color_map = {name: _PALETTE[i % len(_PALETTE)] for i, name in enumerate(sorted_cluster_names)} | |
| # ── Compute centroids in 2D for annotation labels ───────────────────────── | |
| def centroid_2d(name): | |
| mask = scatter_df["cluster"] == name | |
| return scatter_df.loc[mask, "x"].mean(), scatter_df.loc[mask, "y"].mean() | |
| centroids = {name: centroid_2d(name) for name in sorted_cluster_names} | |
| # ════════════════════════════════════════════════════════════════════════ | |
| # CHART 1 — UMAP Scatter with cluster name annotations at centroids | |
| # ════════════════════════════════════════════════════════════════════════ | |
| fig1 = go.Figure() | |
| # One trace per cluster so legend shows real names + color | |
| for name in sorted_cluster_names: | |
| sub = scatter_df[scatter_df["cluster"] == name] | |
| fig1.add_trace(go.Scatter( | |
| x=sub["x"], y=sub["y"], | |
| mode="markers", | |
| name=name, | |
| marker=dict(color=color_map[name], size=8, opacity=0.75, | |
| line=dict(width=0.4, color="white")), | |
| customdata=sub[["doi", "text"]].values, | |
| hovertemplate=( | |
| "<b>%{customdata[0]}</b><br>" | |
| "%{customdata[1]}<extra>" + name + "</extra>" | |
| ), | |
| )) | |
| # Noise points | |
| noise_df = scatter_df[scatter_df["is_noise"]] | |
| if len(noise_df): | |
| fig1.add_trace(go.Scatter( | |
| x=noise_df["x"], y=noise_df["y"], mode="markers", | |
| marker=dict(color="lightgrey", size=4, opacity=0.35, | |
| line=dict(width=0)), | |
| name="Noise / Unassigned", hoverinfo="skip", | |
| )) | |
| # Centroid text labels — cluster name + size badge | |
| for name, (cx, cy) in centroids.items(): | |
| sz = id_to_size.get( | |
| next((s["topic_id"] for s in summaries if s["label"] == name), -1), 0 | |
| ) | |
| display = f"<b>{name}</b><br><span style='font-size:10px'>n={sz}</span>" | |
| fig1.add_annotation( | |
| x=cx, y=cy, text=display, | |
| showarrow=False, font=dict(size=10, color="#1a1a2e"), | |
| bgcolor="rgba(255,255,255,0.82)", | |
| bordercolor=color_map.get(name, "#999"), | |
| borderwidth=1.5, borderpad=4, | |
| xanchor="center", yanchor="middle", | |
| ) | |
| fig1.update_layout( | |
| title=dict( | |
| text=f"<b>SPECTER2 + UMAP Paper Landscape</b> — {n_clusters} Clusters<br>" | |
| f"<sub>Each point = 1 paper · Hover for DOI & title · Labels at cluster centroids</sub>", | |
| font_size=15, | |
| ), | |
| xaxis=dict(title="UMAP Dimension 1", showgrid=False, zeroline=False), | |
| yaxis=dict(title="UMAP Dimension 2", showgrid=False, zeroline=False), | |
| legend=dict(title="<b>Cluster</b>", font_size=9, tracegroupgap=2, | |
| itemsizing="constant", | |
| bordercolor="#ddd", borderwidth=1), | |
| template="plotly_white", | |
| width=1100, height=750, | |
| margin=dict(l=50, r=220, t=90, b=50), | |
| ) | |
| fig1.write_html("chart_umap_scatter.html") | |
| # ════════════════════════════════════════════════════════════════════════ | |
| # CHART 2 — Horizontal bar chart: cluster name on Y-axis, size on X | |
| # Secondary axis: cohesion score dots | |
| # ════════════════════════════════════════════════════════════════════════ | |
| sizes_df = pd.DataFrame({ | |
| "Cluster": [s["label"] for s in summaries], | |
| "Papers": [s["size"] for s in summaries], | |
| "Cohesion": [s["cohesion_score"] for s in summaries], | |
| }).sort_values("Papers", ascending=True) # ascending so longest bar is on top | |
| bar_colors = list(map(lambda name: color_map.get(name, "#888"), sizes_df["Cluster"])) | |
| fig2 = go.Figure() | |
| fig2.add_trace(go.Bar( | |
| y=sizes_df["Cluster"], | |
| x=sizes_df["Papers"], | |
| orientation="h", | |
| name="Papers", | |
| marker_color=bar_colors, | |
| text=sizes_df["Papers"], | |
| textposition="outside", | |
| hovertemplate="<b>%{y}</b><br>Papers: %{x}<extra></extra>", | |
| )) | |
| # Cohesion dots on secondary X axis | |
| fig2.add_trace(go.Scatter( | |
| y=sizes_df["Cluster"], | |
| x=sizes_df["Cohesion"], | |
| mode="markers+text", | |
| name="Cohesion Score", | |
| marker=dict(color="crimson", size=10, symbol="diamond", | |
| line=dict(width=1, color="darkred")), | |
| text=list(map(lambda c: f"{c:.2f}", sizes_df["Cohesion"])), | |
| textposition="middle right", | |
| xaxis="x2", | |
| hovertemplate="<b>%{y}</b><br>Cohesion: %{x:.3f}<extra></extra>", | |
| )) | |
| max_papers = max(sizes_df["Papers"]) | |
| fig2.update_layout( | |
| title=dict( | |
| text="<b>Cluster Sizes & Intra-Cluster Cohesion</b><br>" | |
| "<sub>Bars = paper count · Red diamonds = mean cosine similarity to centroid</sub>", | |
| font_size=14, | |
| ), | |
| xaxis=dict(title="Number of Papers", range=[0, max_papers * 1.18]), | |
| xaxis2=dict(title="Cohesion (cosine)", overlaying="x", side="top", | |
| range=[0, 1.05], showgrid=False), | |
| yaxis=dict(title="", automargin=True, tickfont=dict(size=11)), | |
| legend=dict(x=1.01, y=1, bordercolor="#ddd", borderwidth=1), | |
| template="plotly_white", | |
| height=max(500, 28 * len(summaries)), | |
| margin=dict(l=220, r=120, t=100, b=50), | |
| bargap=0.28, | |
| ) | |
| # Target cohesion band | |
| fig2.add_vrect( | |
| x0=COSINE_SIM_LOW, x1=COSINE_SIM_HIGH, xref="x2", | |
| fillcolor="rgba(0,180,0,0.07)", line_width=0, | |
| annotation_text="Target cohesion band", annotation_position="top left", | |
| ) | |
| fig2.write_html("chart_cluster_sizes.html") | |
| # ════════════════════════════════════════════════════════════════════════ | |
| # CHART 3 — Treemap: area = papers, color = cohesion, label = cluster name + n | |
| # ════════════════════════════════════════════════════════════════════════ | |
| treemap_df = pd.DataFrame({ | |
| "label": [f"{s['label']}<br>n={s['size']}" for s in summaries], | |
| "plain_label": [s["label"] for s in summaries], | |
| "size": [s["size"] for s in summaries], | |
| "cohesion": [s["cohesion_score"] for s in summaries], | |
| }) | |
| fig3 = px.treemap( | |
| treemap_df, path=["label"], values="size", color="cohesion", | |
| color_continuous_scale="RdYlGn", | |
| color_continuous_midpoint=0.55, | |
| hover_data={"plain_label": True, "cohesion": ":.3f", "size": True, "label": False}, | |
| title="<b>Cluster Treemap</b> — Area = Papers · Color = Cohesion Score (green=high)", | |
| ) | |
| fig3.update_traces( | |
| textfont_size=12, | |
| hovertemplate="<b>%{customdata[0]}</b><br>Papers: %{value}<br>Cohesion: %{customdata[1]:.3f}<extra></extra>", | |
| ) | |
| fig3.update_layout( | |
| coloraxis_colorbar=dict(title="Cohesion", tickformat=".2f"), | |
| margin=dict(t=60, l=10, r=10, b=10), | |
| height=580, | |
| ) | |
| fig3.write_html("chart_treemap.html") | |
| # ════════════════════════════════════════════════════════════════════════ | |
| # CHART 4 — Cumulative coverage with cluster name annotations at steps | |
| # ════════════════════════════════════════════════════════════════════════ | |
| sorted_summaries = sorted(summaries, key=lambda s: -s["size"]) | |
| sorted_sizes = [s["size"] for s in sorted_summaries] | |
| sorted_names = [s["label"] for s in sorted_summaries] | |
| total_clustered = sum(sorted_sizes) | |
| cumulative = list(map( | |
| lambda i: sum(sorted_sizes[:i + 1]) / total_clustered * 100, | |
| range(len(sorted_sizes)) | |
| )) | |
| x_vals = list(range(1, len(sorted_sizes) + 1)) | |
| fig4 = go.Figure() | |
| fig4.add_trace(go.Scatter( | |
| y=cumulative, x=x_vals, | |
| mode="lines+markers", | |
| name="Cumulative Coverage", | |
| fill="tozeroy", fillcolor="rgba(0,100,255,0.07)", | |
| line=dict(color="royalblue", width=2.5), | |
| marker=dict(size=8, color="royalblue"), | |
| customdata=list(zip(sorted_names, sorted_sizes)), | |
| hovertemplate="<b>%{customdata[0]}</b><br>Rank: %{x}<br>Papers: %{customdata[1]}<br>Coverage: %{y:.1f}%<extra></extra>", | |
| )) | |
| fig4.add_hline(y=80, line_dash="dash", line_color="orange", | |
| annotation_text="80% coverage", annotation_font_size=11) | |
| fig4.add_hline(y=50, line_dash="dot", line_color="green", | |
| annotation_text="50% coverage", annotation_font_size=11) | |
| # Annotate first 8 cluster names along the curve | |
| for i, (name, cum) in enumerate(zip(sorted_names[:8], cumulative[:8])): | |
| short = name[:22] + "…" if len(name) > 22 else name | |
| fig4.add_annotation( | |
| x=i + 1, y=cum, | |
| text=short, showarrow=True, arrowhead=2, arrowsize=0.8, | |
| ax=20, ay=-30 if i % 2 == 0 else 30, | |
| font=dict(size=9, color="#333"), | |
| bgcolor="rgba(255,255,255,0.8)", borderpad=2, | |
| ) | |
| fig4.update_layout( | |
| title=dict( | |
| text="<b>Cumulative Paper Coverage by Cluster Rank</b><br>" | |
| "<sub>Hover each point to see cluster name and paper count</sub>", | |
| font_size=14, | |
| ), | |
| xaxis=dict(title="Cluster Rank (largest first)", tickmode="linear", dtick=1), | |
| yaxis=dict(title="Cumulative Coverage %", range=[0, 105]), | |
| template="plotly_white", height=500, | |
| margin=dict(l=60, r=60, t=90, b=60), | |
| ) | |
| fig4.write_html("chart_cumulative.html") | |
| # ════════════════════════════════════════════════════════════════════════ | |
| # CHART 5 — Cohesion histogram + per-cluster dot strip | |
| # ════════════════════════════════════════════════════════════════════════ | |
| cohesion_scores = [s["cohesion_score"] for s in summaries] | |
| cluster_names_list = [s["label"] for s in summaries] | |
| fig5 = go.Figure() | |
| fig5.add_trace(go.Histogram( | |
| x=cohesion_scores, nbinsx=12, name="All Clusters", | |
| marker_color="steelblue", opacity=0.55, | |
| hovertemplate="Cohesion: %{x:.2f}<br>Count: %{y}<extra></extra>", | |
| )) | |
| # Strip dots colored by cluster name | |
| jitter = np.random.default_rng(0).uniform(-0.4, 0.4, len(cohesion_scores)) | |
| fig5.add_trace(go.Scatter( | |
| x=cohesion_scores, | |
| y=list(map(lambda j: j, jitter)), | |
| mode="markers+text", | |
| text=list(map(lambda n: n[:18], cluster_names_list)), | |
| textposition="top center", | |
| textfont=dict(size=8), | |
| marker=dict( | |
| color=list(map(lambda n: color_map.get(n, "#888"), cluster_names_list)), | |
| size=11, opacity=0.9, line=dict(width=0.8, color="white"), | |
| symbol="circle", | |
| ), | |
| name="Individual Clusters", | |
| customdata=list(zip(cluster_names_list, [s["size"] for s in summaries])), | |
| hovertemplate="<b>%{customdata[0]}</b><br>Cohesion: %{x:.3f}<br>Papers: %{customdata[1]}<extra></extra>", | |
| yaxis="y2", | |
| )) | |
| fig5.add_vrect( | |
| x0=COSINE_SIM_LOW, x1=COSINE_SIM_HIGH, | |
| fillcolor="rgba(0,180,0,0.12)", line_width=1.5, | |
| line_color="green", | |
| annotation_text=f"Target range [{COSINE_SIM_LOW}–{COSINE_SIM_HIGH}]", | |
| annotation_position="top left", annotation_font_size=11, | |
| ) | |
| fig5.update_layout( | |
| title=dict( | |
| text="<b>Intra-Cluster Cohesion Distribution</b><br>" | |
| "<sub>Green band = target cosine similarity range · Each dot = one cluster</sub>", | |
| font_size=14, | |
| ), | |
| xaxis=dict(title="Mean Cosine Similarity to Cluster Centroid", range=[0, 1.05]), | |
| yaxis=dict(title="Number of Clusters", showgrid=True), | |
| yaxis2=dict(title="", overlaying="y", range=[-2.5, 4], showticklabels=False, showgrid=False), | |
| template="plotly_white", height=480, | |
| legend=dict(x=0.01, y=0.99), | |
| margin=dict(l=60, r=40, t=90, b=60), | |
| ) | |
| fig5.write_html("chart_cohesion_hist.html") | |
| def _build_themes_chart(themes): | |
| """ | |
| Generate two rich charts for themes.json: | |
| 1. chart_themes_detail.html — sunburst showing theme → evidence sentences, | |
| with cohesion badge and DOI strip | |
| 2. chart_themes_radar.html — radar/spider chart comparing themes on size & cohesion | |
| """ | |
| if not themes: | |
| return | |
| # ════════════════════════════════════════════════════════════════════════ | |
| # CHART A — Sunburst: root → theme → top evidence sentences | |
| # ════════════════════════════════════════════════════════════════════════ | |
| ids, labels, parents, values, colors, hovers = [], [], [], [], [], [] | |
| # Root node | |
| total_papers = sum(t["size"] for t in themes) | |
| ids.append("root") | |
| labels.append(f"All Themes<br>n={total_papers}") | |
| parents.append("") | |
| values.append(total_papers) | |
| colors.append(0.5) | |
| hovers.append(f"Total papers across {len(themes)} themes") | |
| palette_sun = _PALETTE | |
| for i, t in enumerate(themes): | |
| tid = f"theme_{t['theme_id']}" | |
| theme_label = f"<b>{t['name']}</b><br>n={t['size']} · coh={t.get('cohesion_score',0):.2f}" | |
| ids.append(tid) | |
| labels.append(theme_label) | |
| parents.append("root") | |
| values.append(t["size"]) | |
| colors.append(t.get("cohesion_score", 0.5)) | |
| hovers.append( | |
| f"<b>{t['name']}</b><br>" | |
| f"Papers: {t['size']}<br>" | |
| f"Cohesion: {t.get('cohesion_score',0):.3f}<br>" | |
| f"Reasoning: {t.get('reasoning','')[:120]}" | |
| ) | |
| # Evidence sentence children | |
| for j, ev in enumerate(t.get("evidence", [])[:3]): | |
| eid = f"{tid}_ev{j}" | |
| short_ev = (ev[:80] + "…") if len(ev) > 80 else ev | |
| ids.append(eid) | |
| labels.append(short_ev) | |
| parents.append(tid) | |
| values.append(max(1, t["size"] // 3)) | |
| colors.append(t.get("cohesion_score", 0.5) * 0.9) | |
| hovers.append(ev[:300]) | |
| fig_sun = go.Figure(go.Sunburst( | |
| ids=ids, labels=labels, parents=parents, values=values, | |
| customdata=hovers, | |
| hovertemplate="%{customdata}<extra></extra>", | |
| branchvalues="total", | |
| marker=dict( | |
| colors=colors, | |
| colorscale="RdYlGn", | |
| cmin=0.3, cmid=0.55, cmax=0.85, | |
| colorbar=dict(title="Cohesion", thickness=15, x=1.02), | |
| line=dict(width=1, color="white"), | |
| ), | |
| textfont=dict(size=11), | |
| insidetextorientation="radial", | |
| maxdepth=2, | |
| )) | |
| fig_sun.update_layout( | |
| title=dict( | |
| text="<b>Themes Structure — Sunburst View</b><br>" | |
| "<sub>Inner ring = themes (area ∝ papers) · Outer ring = representative evidence · Color = cohesion</sub>", | |
| font_size=14, | |
| ), | |
| height=700, width=750, | |
| margin=dict(t=90, l=10, r=120, b=10), | |
| ) | |
| fig_sun.write_html("chart_themes_detail.html") | |
| # ════════════════════════════════════════════════════════════════════════ | |
| # CHART B — Horizontal grouped bar: themes compared on size + cohesion | |
| # with evidence preview on hover | |
| # ════════════════════════════════════════════════════════════════════════ | |
| themes_sorted = sorted(themes, key=lambda t: -t["size"]) | |
| theme_names = [t["name"] for t in themes_sorted] | |
| sizes = [t["size"] for t in themes_sorted] | |
| cohesions = [t.get("cohesion_score", 0) for t in themes_sorted] | |
| reasonings = [t.get("reasoning", "")[:150] for t in themes_sorted] | |
| evidences = [" | ".join(t.get("evidence", [])[:2])[:200] for t in themes_sorted] | |
| bar_colors2 = [_PALETTE[i % len(_PALETTE)] for i in range(len(themes_sorted))] | |
| fig_b = go.Figure() | |
| fig_b.add_trace(go.Bar( | |
| y=theme_names, x=sizes, orientation="h", | |
| name="Papers", | |
| marker_color=bar_colors2, | |
| text=sizes, textposition="outside", | |
| customdata=list(zip(reasonings, evidences)), | |
| hovertemplate=( | |
| "<b>%{y}</b><br>" | |
| "Papers: %{x}<br>" | |
| "Reasoning: %{customdata[0]}<br>" | |
| "Evidence: %{customdata[1]}<extra></extra>" | |
| ), | |
| )) | |
| fig_b.add_trace(go.Scatter( | |
| y=theme_names, x=cohesions, | |
| mode="markers+text", | |
| name="Cohesion", | |
| marker=dict(color="crimson", size=12, symbol="diamond", | |
| line=dict(width=1, color="darkred")), | |
| text=list(map(lambda c: f"{c:.2f}", cohesions)), | |
| textposition="middle right", | |
| xaxis="x2", | |
| hovertemplate="<b>%{y}</b><br>Cohesion: %{x:.3f}<extra></extra>", | |
| )) | |
| max_sz = max(sizes) if sizes else 1 | |
| fig_b.update_layout( | |
| title=dict( | |
| text="<b>Final Themes — Papers & Cohesion</b><br>" | |
| "<sub>Hover bars for reasoning & evidence · Red diamonds = cohesion score</sub>", | |
| font_size=14, | |
| ), | |
| xaxis=dict(title="Papers", range=[0, max_sz * 1.2]), | |
| xaxis2=dict(title="Cohesion Score", overlaying="x", side="top", | |
| range=[0, 1.1], showgrid=False), | |
| yaxis=dict(automargin=True, tickfont=dict(size=11)), | |
| legend=dict(x=1.02, y=1, bordercolor="#ddd", borderwidth=1), | |
| template="plotly_white", | |
| height=max(450, 32 * len(themes_sorted)), | |
| margin=dict(l=220, r=120, t=100, b=50), | |
| bargap=0.3, | |
| ) | |
| fig_b.write_html("chart_themes_radar.html") | |
| def _run_bertopic_discovery(dummy: str = "") -> str: | |
| """ | |
| Phase 2a: Embed papers with SPECTER2, reduce with UMAP, cluster with HDBSCAN. | |
| Iterates parameters to achieve 15-30 clusters. Generates rich visualisation charts. | |
| """ | |
| papers_df = pd.read_csv(PAPERS_FILE) | |
| texts = papers_df["combined_text"].tolist() | |
| dois = papers_df["doi"].tolist() | |
| # ── SPECTER2 Embeddings ─────────────────────────────────────────────────── | |
| print(f"[SPECTER2] Loading model: {SPECTER2_MODEL}") | |
| model = SentenceTransformer(SPECTER2_MODEL) | |
| print(f"[SPECTER2] Encoding {len(texts)} papers...") | |
| embeddings = model.encode( | |
| texts, | |
| normalize_embeddings=True, | |
| show_progress_bar=True, | |
| batch_size=16, | |
| ) | |
| np.save(EMB_FILE, embeddings) | |
| print(f"[SPECTER2] Embeddings shape: {embeddings.shape}") | |
| # ── UMAP Reduction ──────────────────────────────────────────────────────── | |
| # n_neighbors controls local/global structure; cosine metric for scientific text | |
| reducer = umap.UMAP( | |
| n_components=10, # reduce to 10D before HDBSCAN | |
| n_neighbors=15, | |
| min_dist=0.0, | |
| metric="cosine", | |
| random_state=42, | |
| ) | |
| reduced = reducer.fit_transform(embeddings) | |
| print(f"[UMAP] Reduced to shape: {reduced.shape}") | |
| # ── HDBSCAN with iterative parameter search for 15-30 clusters ─────────── | |
| def try_hdbscan(min_size, min_samples): | |
| clusterer = hdbscan.HDBSCAN( | |
| min_cluster_size=min_size, | |
| min_samples=min_samples, | |
| metric="euclidean", | |
| cluster_selection_method="eom", | |
| prediction_data=True, | |
| ) | |
| lbl = clusterer.fit_predict(reduced) | |
| valid_clusters = set(lbl[lbl >= 0]) | |
| # Enforce max cluster size constraint | |
| cluster_sizes = Counter(lbl[lbl >= 0].tolist()) | |
| oversized = sum(1 for sz in cluster_sizes.values() if sz > MAX_CLUSTER_SIZE) | |
| return lbl, len(valid_clusters), oversized, clusterer | |
| # Parameter grid search — pick first combo giving 15-30 valid clusters | |
| param_grid = [ | |
| (ms, msa) | |
| for ms in [15, 12, 10, 8, 7, 6, 5] | |
| for msa in [5, 4, 3, 2] | |
| ] | |
| best_labels, best_n, best_clusterer = None, 0, None | |
| results = list(map(lambda params: (params, *try_hdbscan(*params)), param_grid)) | |
| # Filter to those in [15, 30] range first | |
| valid_results = list(filter(lambda r: MIN_CLUSTERS <= r[2] <= MAX_CLUSTERS and r[3] == 0, results)) | |
| chosen = ( | |
| valid_results[0] if valid_results | |
| else sorted(results, key=lambda r: abs(r[2] - 20))[0] # closest to 20 | |
| ) | |
| chosen_params, chosen_labels, chosen_n, _, best_clusterer = chosen | |
| best_labels = chosen_labels | |
| print(f"[HDBSCAN] Best params: min_cluster_size={chosen_params[0]}, min_samples={chosen_params[1]}") | |
| print(f"[HDBSCAN] Clusters found: {chosen_n}") | |
| np.save(LABELS_FILE, best_labels) | |
| # ── Build summaries per cluster ─────────────────────────────────────────── | |
| valid_cluster_ids = sorted(set(best_labels[best_labels >= 0].tolist())) | |
| # Cosine similarity quality check per cluster | |
| def compute_cluster_cohesion(label): | |
| indices = [i for i, lbl in enumerate(best_labels) if lbl == label] | |
| if len(indices) < 2: | |
| return 0.0 | |
| cluster_emb = embeddings[indices] | |
| centroid = cluster_emb.mean(axis=0, keepdims=True) | |
| sims = cosine_similarity(cluster_emb, centroid).flatten() | |
| return float(np.mean(sims)) | |
| def build_topic(label): | |
| indices = [i for i, lbl in enumerate(best_labels) if lbl == label] | |
| cluster_emb = embeddings[indices] | |
| centroid = cluster_emb.mean(axis=0) | |
| # Top 3 papers closest to centroid (for LLM labeling) | |
| dists = np.linalg.norm(cluster_emb - centroid, axis=1) | |
| top3_idx = np.argsort(dists)[:3].tolist() | |
| top3_titles = list(map(lambda i: texts[indices[i]][:200], top3_idx)) | |
| top3_dois = list(map(lambda i: dois[indices[i]], top3_idx)) | |
| cohesion = compute_cluster_cohesion(label) | |
| return { | |
| "topic_id": int(label), | |
| "label": f"Cluster_{label}", | |
| "top_sentences": top3_titles, | |
| "top_dois": top3_dois, | |
| "size": len(indices), | |
| "cohesion_score": round(cohesion, 4), | |
| "approved": False, | |
| "rename_to": "", | |
| "reasoning": "", | |
| } | |
| summaries = list(map(build_topic, valid_cluster_ids)) | |
| noise_count = int(np.sum(best_labels == -1)) | |
| with open(SUMMARIES_FILE, "w") as f: | |
| json.dump(summaries, f, indent=2) | |
| # ── 2D UMAP coords (saved for re-use in later chart regeneration) ───────── | |
| reducer_2d = umap.UMAP(n_components=2, n_neighbors=15, min_dist=0.1, metric="cosine", random_state=42) | |
| coords_2d = reducer_2d.fit_transform(embeddings) | |
| np.save("coords_2d.npy", coords_2d) | |
| # ── Build initial charts with placeholder Cluster_N names ───────────────── | |
| _build_all_charts(summaries, coords_2d, best_labels, texts, dois, chosen_n) | |
| avg_cohesion = float(np.mean([s["cohesion_score"] for s in summaries])) | |
| in_target_range = sum(1 for s in summaries if COSINE_SIM_LOW <= s["cohesion_score"] <= COSINE_SIM_HIGH) | |
| return json.dumps({ | |
| "status": "ok", | |
| "cluster_count": chosen_n, | |
| "noise_papers": noise_count, | |
| "total_papers": len(texts), | |
| "hdbscan_params": {"min_cluster_size": chosen_params[0], "min_samples": chosen_params[1]}, | |
| "avg_cohesion": round(avg_cohesion, 4), | |
| "clusters_in_target_cosine_range": in_target_range, | |
| "charts": [ | |
| "chart_umap_scatter.html", | |
| "chart_cluster_sizes.html", | |
| "chart_treemap.html", | |
| "chart_cumulative.html", | |
| "chart_cohesion_hist.html", | |
| ], | |
| "message": ( | |
| f"SPECTER2 embedded {len(texts)} papers → UMAP (10D) → HDBSCAN → " | |
| f"{chosen_n} clusters (noise: {noise_count} papers). " | |
| f"Avg cohesion: {avg_cohesion:.3f}. " | |
| f"{in_target_range}/{chosen_n} clusters in target cosine range [{COSINE_SIM_LOW}-{COSINE_SIM_HIGH}]. " | |
| f"5 rich interactive charts generated." | |
| ), | |
| }) | |
| def _label_topics_with_llm(dummy: str = "") -> str: | |
| """ | |
| Phase 2b: Label each cluster using 3-LLM majority-vote approach. | |
| Each cluster's top-3 paper titles are sent to Mistral 3 times; | |
| the most common label wins. | |
| """ | |
| with open(SUMMARIES_FILE) as f: | |
| summaries = json.load(f) | |
| llm = ChatMistralAI(model="mistral-small-latest", temperature=0.3) | |
| parser = JsonOutputParser() | |
| prompt = PromptTemplate( | |
| input_variables=["titles"], | |
| template=( | |
| "You are an expert academic thematic analyst specialising in IS/technology journals.\n" | |
| "Given these 3 representative paper title+abstract excerpts from the same research cluster:\n\n" | |
| "{titles}\n\n" | |
| "Generate a concise, academically precise cluster label (4-7 words) that captures " | |
| "the unifying research theme. This will appear in a journal paper.\n\n" | |
| 'Return ONLY valid JSON: {{"label": "...", "reasoning": "one sentence explanation"}}\n' | |
| "No markdown, no preamble, no extra keys." | |
| ), | |
| ) | |
| def vote_label(topic): | |
| excerpts = "\n".join( | |
| list(map(lambda s: f"- {s[:200]}", topic["top_sentences"][:3])) | |
| ) | |
| # 3 independent calls | |
| raw_results = list(map( | |
| lambda _: (prompt | llm | parser).invoke({"titles": excerpts}), | |
| range(3) | |
| )) | |
| # Normalise to dict | |
| parsed = list(map( | |
| lambda r: (r[0] if isinstance(r, list) else r), | |
| raw_results | |
| )) | |
| labels = list(map(lambda r: r.get("label", topic["label"]), parsed)) | |
| # Majority vote: pick most common label | |
| label_counts = Counter(labels) | |
| winner = label_counts.most_common(1)[0][0] | |
| winner_reasoning = next( | |
| (r.get("reasoning", "") for r in parsed if r.get("label") == winner), | |
| parsed[0].get("reasoning", "") if parsed else "" | |
| ) | |
| all_candidates = list(label_counts.keys()) | |
| return { | |
| **topic, | |
| "label": winner, | |
| "label_candidates": all_candidates, | |
| "reasoning": winner_reasoning, | |
| } | |
| labeled = list(map(vote_label, summaries)) | |
| with open(SUMMARIES_FILE, "w") as f: | |
| json.dump(labeled, f, indent=2) | |
| # ── Regenerate all charts with real LLM-voted names ─────────────────────── | |
| coords_2d_path = Path("coords_2d.npy") | |
| labels_path = Path(LABELS_FILE) | |
| papers_path = Path(PAPERS_FILE) | |
| charts_rebuilt = False | |
| if coords_2d_path.exists() and labels_path.exists() and papers_path.exists(): | |
| coords_2d = np.load("coords_2d.npy") | |
| cluster_labels_arr = np.load(LABELS_FILE) | |
| papers_df2 = pd.read_csv(PAPERS_FILE) | |
| _build_all_charts( | |
| labeled, | |
| coords_2d, | |
| cluster_labels_arr, | |
| papers_df2["combined_text"].tolist(), | |
| papers_df2["doi"].tolist(), | |
| len(set(cluster_labels_arr[cluster_labels_arr >= 0])), | |
| ) | |
| charts_rebuilt = True | |
| return json.dumps({ | |
| "status": "ok", | |
| "labeled_count": len(labeled), | |
| "charts_rebuilt_with_real_names": charts_rebuilt, | |
| "message": ( | |
| f"Labeled {len(labeled)} clusters using 3-LLM majority vote. " | |
| f"{'All 5 charts regenerated with real cluster names and centroid labels.' if charts_rebuilt else 'Charts could not be rebuilt (missing cache files).'} " | |
| f"Review the topic table, approve/rename clusters, then submit." | |
| ), | |
| }) | |
| def _consolidate_into_themes(dummy: str = "") -> str: | |
| """Phase 3: Merge approved topics from summaries.json into themes.json.""" | |
| with open(SUMMARIES_FILE) as f: | |
| summaries = json.load(f) | |
| approved = list(filter(lambda t: t.get("approved", False), summaries)) | |
| themes = list(map(lambda t: { | |
| "theme_id": t["topic_id"], | |
| "name": t.get("rename_to") or t["label"], | |
| "original_label": t["label"], | |
| "label_candidates": t.get("label_candidates", []), | |
| "evidence": t["top_sentences"], | |
| "top_dois": t.get("top_dois", []), | |
| "size": t["size"], | |
| "cohesion_score": t.get("cohesion_score", 0.0), | |
| "reasoning": t.get("reasoning", ""), | |
| }, approved)) | |
| with open(THEMES_FILE, "w") as f: | |
| json.dump(themes, f, indent=2) | |
| # ── Generate themes.json visual (sunburst + detail card chart) ──────────── | |
| _build_themes_chart(themes) | |
| # ── Regenerate scatter/bar/treemap with rename_to labels ────────────────── | |
| coords_2d_path = Path("coords_2d.npy") | |
| labels_path = Path(LABELS_FILE) | |
| papers_path = Path(PAPERS_FILE) | |
| if coords_2d_path.exists() and labels_path.exists() and papers_path.exists(): | |
| coords_2d = np.load("coords_2d.npy") | |
| cluster_labels_arr = np.load(LABELS_FILE) | |
| papers_df2 = pd.read_csv(PAPERS_FILE) | |
| # Build a patched summaries list using final theme names | |
| theme_name_map = {t["theme_id"]: t["name"] for t in themes} | |
| with open(SUMMARIES_FILE) as f: | |
| summaries_all = json.load(f) | |
| patched = list(map( | |
| lambda s: {**s, "label": theme_name_map.get(s["topic_id"], s["label"])}, | |
| summaries_all | |
| )) | |
| _build_all_charts( | |
| patched, | |
| coords_2d, | |
| cluster_labels_arr, | |
| papers_df2["combined_text"].tolist(), | |
| papers_df2["doi"].tolist(), | |
| len(themes), | |
| ) | |
| return json.dumps({ | |
| "status": "ok", "theme_count": len(themes), | |
| "message": ( | |
| f"Consolidated {len(themes)} approved clusters into themes.json. " | |
| f"Themes visualization chart generated (chart_themes_detail.html). " | |
| f"All scatter/bar/treemap charts updated with final theme names." | |
| ), | |
| }) | |
| def _compare_with_taxonomy(dummy: str = "") -> str: | |
| """Phase 5.5: Map themes to PAJAIS 25 categories via Mistral. Saves taxonomy_map.json.""" | |
| with open(THEMES_FILE) as f: | |
| themes = json.load(f) | |
| llm = ChatMistralAI(model="mistral-small-latest", temperature=0.1) | |
| parser = JsonOutputParser() | |
| prompt = PromptTemplate( | |
| input_variables=["theme_name", "evidence", "categories"], | |
| template=( | |
| "Map this research theme to the most relevant PAJAIS category.\n\n" | |
| "Theme: {theme_name}\nEvidence: {evidence}\n\n" | |
| "PAJAIS Categories:\n{categories}\n\n" | |
| 'Return ONLY JSON: {{"pajais_category": "...", "confidence": 0.0-1.0, "rationale": "..."}}' | |
| ), | |
| ) | |
| categories_str = "\n".join(list(map(lambda c: f"- {c}", PAJAIS_CATEGORIES))) | |
| def map_theme(t): | |
| result = (prompt | llm | parser).invoke({ | |
| "theme_name": t["name"], "evidence": " ".join(t["evidence"][:2]), | |
| "categories": categories_str, | |
| }) | |
| result = result[0] if isinstance(result, list) else result | |
| return {**t, "pajais_mapping": result} | |
| mapped = list(map(map_theme, themes)) | |
| with open(TAXONOMY_FILE, "w") as f: | |
| json.dump(mapped, f, indent=2) | |
| return json.dumps({ | |
| "status": "ok", "mapped_count": len(mapped), | |
| "message": f"Mapped {len(mapped)} themes to PAJAIS taxonomy.", | |
| }) | |
| def _generate_comparison_csv(dummy: str = "") -> str: | |
| """Phase 6a: Build comparison.csv from taxonomy_map.json.""" | |
| with open(TAXONOMY_FILE) as f: | |
| taxonomy = json.load(f) | |
| rows = list(map(lambda e: { | |
| "Theme Name": e.get("name", ""), | |
| "Original Label": e.get("original_label", ""), | |
| "Label Candidates (3-vote)": " | ".join(e.get("label_candidates", [])), | |
| "Topic Size (Papers)": e.get("size", 0), | |
| "Cohesion Score": e.get("cohesion_score", ""), | |
| "PAJAIS Category": e.get("pajais_mapping", {}).get("pajais_category", ""), | |
| "Confidence": e.get("pajais_mapping", {}).get("confidence", ""), | |
| "Rationale": e.get("pajais_mapping", {}).get("rationale", ""), | |
| "Top Evidence": " | ".join(e.get("evidence", [])[:2]), | |
| "Top DOIs": " | ".join(e.get("top_dois", [])[:3]), | |
| }, taxonomy)) | |
| pd.DataFrame(rows).to_csv(COMPARISON_FILE, index=False) | |
| return json.dumps({ | |
| "status": "ok", "rows": len(rows), "file": COMPARISON_FILE, | |
| "message": f"Comparison CSV with {len(rows)} rows saved (includes cohesion scores, 3-vote labels, DOIs).", | |
| }) | |
| def _export_narrative(dummy: str = "") -> str: | |
| """Phase 6b: Generate 500-word academic Section 7 and save narrative.txt.""" | |
| with open(TAXONOMY_FILE) as f: | |
| taxonomy = json.load(f) | |
| themes_text = "\n".join(list(map( | |
| lambda t: ( | |
| f"Theme '{t['name']}' (papers={t['size']}, cohesion={t.get('cohesion_score',0):.3f}) " | |
| f"maps to PAJAIS '{t.get('pajais_mapping', {}).get('pajais_category', 'N/A')}' — " | |
| f"{t.get('pajais_mapping', {}).get('rationale', '')}." | |
| ), | |
| taxonomy, | |
| ))) | |
| llm = ChatMistralAI(model="mistral-small-latest", temperature=0.4) | |
| prompt = PromptTemplate( | |
| input_variables=["themes_text"], | |
| template=( | |
| "Write Section 7 'Discussion of Themes' (~500 words, formal academic tone) " | |
| "suitable for a journal paper in the International Journal of Electronic Commerce.\n\n" | |
| "Themes, cluster sizes, cohesion scores, and PAJAIS mappings:\n{themes_text}\n\n" | |
| "Write the discussion section:" | |
| ), | |
| ) | |
| result = (prompt | llm).invoke({"themes_text": themes_text}) | |
| narrative = result.content if hasattr(result, "content") else str(result) | |
| Path(NARRATIVE_FILE).write_text(narrative) | |
| return json.dumps({ | |
| "status": "ok", "word_count": len(narrative.split()), | |
| "file": NARRATIVE_FILE, "message": "Narrative saved to narrative.txt.", | |
| }) | |
| # ── Wrap with StructuredTool ────────────────────────────────────────────────── | |
| load_scopus_csv = StructuredTool.from_function( | |
| func=_load_scopus_csv, name="load_scopus_csv", handle_tool_error=True, | |
| description="Load scopus_data.csv, combine Title+Abstract per paper, extract DOIs, save papers_embedded.csv.", | |
| ) | |
| run_bertopic_discovery = StructuredTool.from_function( | |
| func=_run_bertopic_discovery, name="run_bertopic_discovery", handle_tool_error=True, | |
| description=( | |
| "Embed papers with SPECTER2, reduce with UMAP (cosine, 10D), cluster with HDBSCAN " | |
| "(min_cluster_size=5, 15-30 clusters). Saves summaries.json, generates 5 rich charts." | |
| ), | |
| ) | |
| label_topics_with_llm = StructuredTool.from_function( | |
| func=_label_topics_with_llm, name="label_topics_with_llm", handle_tool_error=True, | |
| description="Label clusters using 3-LLM majority-vote. Each cluster submitted 3× to Mistral; best label wins.", | |
| ) | |
| consolidate_into_themes = StructuredTool.from_function( | |
| func=_consolidate_into_themes, name="consolidate_into_themes", handle_tool_error=True, | |
| description="Merge approved clusters into themes.json.", | |
| ) | |
| compare_with_taxonomy = StructuredTool.from_function( | |
| func=_compare_with_taxonomy, name="compare_with_taxonomy", handle_tool_error=True, | |
| description="Map themes to PAJAIS 25 categories via Mistral. Saves taxonomy_map.json.", | |
| ) | |
| generate_comparison_csv = StructuredTool.from_function( | |
| func=_generate_comparison_csv, name="generate_comparison_csv", handle_tool_error=True, | |
| description="Export comparison.csv (with cohesion scores, 3-vote labels, DOIs) from taxonomy_map.json.", | |
| ) | |
| export_narrative = StructuredTool.from_function( | |
| func=_export_narrative, name="export_narrative", handle_tool_error=True, | |
| description="Generate 500-word academic Section 7 for IJEC journal and save narrative.txt.", | |
| ) |