| r"""Re-run the split-language BERTopic pipeline locally and export backend artifacts. |
| |
| Ported from Model/clustering_experiments.ipynb (scenario S3: split language), |
| adapted to run on CPU/Windows and write directly into backend/models + backend/data. |
| |
| Pipeline: |
| 1. Load input CSV, audit abstract quality, drop low-context rows, dedup -> unique publications. |
| 2. Detect language (langdetect) -> lang_group in {en, id}. |
| 3. S1: encode with paraphrase-multilingual-MiniLM-L12-v2, grid-search UMAP+HDBSCAN, |
| pick best UMAP/HDBSCAN config by silhouette (best_up/best_hp). |
| 4. S3: split by language; cluster EN with all-mpnet-base-v2 (best_up/best_hp) and |
| ID with firqaaa/indo-sentence-bert-base (fixed params); reduce_outliers per language. |
| 5. Export embeddings.npy/_en/_id, publication_ids.npy, bt_en/bt_id, topic_info.json, |
| publications.csv/metadata.csv, publication_lecturers.csv, publication_topics.csv, |
| lecturer_profiles.csv, lecturer_embeddings.npy/lecturer_names.npy. |
| |
| Usage (from backend/ with the project venv): |
| ..\.venv\Scripts\python.exe scripts\rerun_bertopic.py |
| ..\.venv\Scripts\python.exe scripts\rerun_bertopic.py --input "C:\Kuliah\Skripsi\Work\Model\metadata.csv" |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import hashlib |
| import json |
| import os |
| import re |
| import shutil |
| import time |
| from collections import defaultdict |
| from itertools import product |
|
|
| import numpy as np |
| import pandas as pd |
|
|
| BACKEND_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| DEFAULT_INPUT = r"C:\Kuliah\Skripsi\Work\Model\metadata.csv" |
| DEFAULT_MODELS_DIR = os.path.join(BACKEND_DIR, "models") |
| DEFAULT_DATA_DIR = os.path.join(BACKEND_DIR, "data") |
|
|
| MINILM_NAME = "paraphrase-multilingual-MiniLM-L12-v2" |
| EN_MODEL_NAME = "all-mpnet-base-v2" |
| ID_MODEL_NAME = "firqaaa/indo-sentence-bert-base" |
|
|
| |
| MIN_ABSTRACT_WORDS = 30 |
| MIN_ABSTRACT_UNIQUE_WORDS = 10 |
| MIN_ABSTRACT_UNIQUE_RATIO = 0.25 |
| MAX_ABSTRACT_TOKEN_RATIO = 0.30 |
| FILTER_LOW_CONTEXT_ABSTRACTS = True |
|
|
| UMAP_RANDOM_STATE = 42 |
| OUTLIER_STRATEGY = "distributions" |
|
|
| ABSTRACT_PLACEHOLDERS = { |
| "", "-", "\u2013", "\u2014", ".", "..", "...", "na", "n/a", "none", "null", |
| "no abstract", "no abstract available", "abstract not available", "not available", |
| "tidak ada", "tidak tersedia", "belum tersedia", "abstrak tidak tersedia", |
| "abstract", "abstrak", |
| } |
| ABSTRACT_PLACEHOLDER_PREFIXES = ( |
| "no abstract available", "abstract not available", "this article has no abstract", |
| "tidak tersedia", "abstrak tidak tersedia", |
| ) |
| ABSTRACT_TEMPLATE_PATTERNS = ( |
| "abstract is written in english or indonesian", |
| "keywords: 3-5 words", |
| "kata kunci: 3-5 kata", |
| ) |
|
|
| UMAP_GRID = [ |
| {"n_neighbors": 10, "n_components": 5, "min_dist": 0.0}, |
| {"n_neighbors": 15, "n_components": 5, "min_dist": 0.0}, |
| {"n_neighbors": 15, "n_components": 10, "min_dist": 0.0}, |
| {"n_neighbors": 25, "n_components": 5, "min_dist": 0.1}, |
| ] |
| HDBSCAN_GRID = [ |
| {"min_cluster_size": 5, "min_samples": 3}, |
| {"min_cluster_size": 10, "min_samples": 5}, |
| {"min_cluster_size": 15, "min_samples": 5}, |
| {"min_cluster_size": 15, "min_samples": 10}, |
| ] |
|
|
|
|
| |
| |
| |
| def normalize_audit_text(value) -> str: |
| return re.sub(r"\s+", " ", str(value).lower()).strip() |
|
|
|
|
| def abstract_tokens(value): |
| return re.findall(r"[A-Za-z\u00c0-\u017f0-9]+(?:[-'][A-Za-z\u00c0-\u017f0-9]+)?", str(value).lower()) |
|
|
|
|
| def max_token_ratio(tokens) -> float: |
| if not tokens: |
| return 0.0 |
| counts = defaultdict(int) |
| for token in tokens: |
| counts[token] += 1 |
| return max(counts.values()) / len(tokens) |
|
|
|
|
| def normalize_key_text(value) -> str: |
| return re.sub(r"\s+", " ", str(value).lower()).strip() |
|
|
|
|
| def normalize_key_year(value) -> str: |
| if pd.isna(value): |
| return "" |
| try: |
| v = float(value) |
| return str(int(v)) if v.is_integer() else str(value) |
| except Exception: |
| return str(value) |
|
|
|
|
| def split_lecturers(cell) -> list[str]: |
| if cell is None or (isinstance(cell, float) and pd.isna(cell)): |
| return [] |
| return [p.strip() for p in re.split(r"[;,/]| dan ", str(cell)) if p.strip()] |
|
|
|
|
| |
| |
| |
| def preprocess(input_path: str): |
| df_rows = pd.read_csv(input_path) |
| print(f"File: {input_path}") |
| print(f"Shape awal: {df_rows.shape}") |
|
|
| rename_map = {"nama_dosen": "dosen", "judul_penelitian": "judul_publikasi", "tahun_terbit": "tahun"} |
| df_rows = df_rows.rename(columns={k: v for k, v in rename_map.items() if k in df_rows.columns}) |
|
|
| required = ["judul_publikasi", "dosen", "abstrak", "tahun"] |
| missing = [c for c in required if c not in df_rows.columns] |
| if missing: |
| raise ValueError(f"Kolom wajib tidak ditemukan: {missing}") |
|
|
| df_rows = df_rows.copy() |
| df_rows["source_row_id"] = np.arange(len(df_rows)) |
| df_rows["judul_publikasi"] = df_rows["judul_publikasi"].fillna("").astype(str).str.strip() |
| df_rows["abstrak"] = df_rows["abstrak"].fillna("").astype(str).str.strip() |
| df_rows["dosen"] = df_rows["dosen"].fillna("").astype(str).str.strip() |
| df_rows["tahun"] = pd.to_numeric(df_rows["tahun"], errors="coerce") |
| df_rows["Judul + Deskripsi"] = (df_rows["judul_publikasi"] + " " + df_rows["abstrak"]).str.strip() |
|
|
| token_lists = df_rows["abstrak"].apply(abstract_tokens) |
| df_rows["abstract_norm"] = df_rows["abstrak"].apply(normalize_audit_text) |
| df_rows["abstract_char_count"] = df_rows["abstrak"].str.len() |
| df_rows["abstract_word_count"] = token_lists.apply(len) |
| df_rows["abstract_unique_word_count"] = token_lists.apply(lambda v: len(set(v))) |
| df_rows["abstract_unique_ratio"] = np.where( |
| df_rows["abstract_word_count"] > 0, |
| df_rows["abstract_unique_word_count"] / df_rows["abstract_word_count"], 0.0, |
| ) |
| df_rows["abstract_max_token_ratio"] = token_lists.apply(max_token_ratio) |
| df_rows["abstract_is_placeholder"] = ( |
| df_rows["abstract_norm"].isin(ABSTRACT_PLACEHOLDERS) |
| | df_rows["abstract_norm"].apply(lambda t: any(t.startswith(p) for p in ABSTRACT_PLACEHOLDER_PREFIXES)) |
| ) |
| df_rows["abstract_is_template"] = df_rows["abstract_norm"].apply( |
| lambda t: any(p in t for p in ABSTRACT_TEMPLATE_PATTERNS) |
| ) |
| df_rows["abstract_has_repeated_punct"] = df_rows["abstrak"].str.contains( |
| r"(?:\.{8,}|_{5,}|-{5,})", regex=True, na=False |
| ) |
|
|
| def quality_reasons(row): |
| reasons = [] |
| if not str(row["Judul + Deskripsi"]).strip(): |
| reasons.append("empty_text") |
| if row["abstract_is_placeholder"]: |
| reasons.append("empty_or_placeholder_abstract") |
| else: |
| if row["abstract_word_count"] < MIN_ABSTRACT_WORDS: |
| reasons.append(f"abstract_words<{MIN_ABSTRACT_WORDS}") |
| if row["abstract_unique_word_count"] < MIN_ABSTRACT_UNIQUE_WORDS: |
| reasons.append(f"unique_words<{MIN_ABSTRACT_UNIQUE_WORDS}") |
| if row["abstract_is_template"]: |
| reasons.append("template_abstract") |
| if row["abstract_has_repeated_punct"] and row["abstract_word_count"] < 80: |
| reasons.append("repeated_punctuation") |
| if row["abstract_word_count"] >= MIN_ABSTRACT_WORDS and row["abstract_unique_ratio"] < MIN_ABSTRACT_UNIQUE_RATIO: |
| reasons.append(f"unique_ratio<{MIN_ABSTRACT_UNIQUE_RATIO}") |
| if row["abstract_word_count"] >= MIN_ABSTRACT_WORDS and row["abstract_max_token_ratio"] > MAX_ABSTRACT_TOKEN_RATIO: |
| reasons.append(f"max_token_ratio>{MAX_ABSTRACT_TOKEN_RATIO}") |
| return "; ".join(dict.fromkeys(reasons)) if reasons else "keep" |
|
|
| df_rows["abstract_quality_reason"] = df_rows.apply(quality_reasons, axis=1) |
| df_rows["abstract_quality_status"] = np.where(df_rows["abstract_quality_reason"].eq("keep"), "keep", "reject") |
|
|
| quality_reject = df_rows["abstract_quality_status"].eq("reject") |
| empty_text = df_rows["Judul + Deskripsi"].str.len().eq(0) |
| actual_reject = empty_text | (FILTER_LOW_CONTEXT_ABSTRACTS & quality_reject) |
|
|
| rows_before = len(df_rows) |
| df_rejected = df_rows[actual_reject].copy().reset_index(drop=True) |
| df_rows = df_rows[~actual_reject].copy().reset_index(drop=True) |
| if df_rows.empty: |
| raise ValueError("Semua baris terhapus oleh preprocessing.") |
|
|
| preprocessing_summary = { |
| "filter_low_context_abstracts": bool(FILTER_LOW_CONTEXT_ABSTRACTS), |
| "min_abstract_words": int(MIN_ABSTRACT_WORDS), |
| "min_abstract_unique_words": int(MIN_ABSTRACT_UNIQUE_WORDS), |
| "min_abstract_unique_ratio": float(MIN_ABSTRACT_UNIQUE_RATIO), |
| "max_abstract_token_ratio": float(MAX_ABSTRACT_TOKEN_RATIO), |
| "n_rows_before_preprocessing": int(rows_before), |
| "n_rows_after_preprocessing": int(len(df_rows)), |
| "n_rows_removed_preprocessing": int(len(df_rejected)), |
| "n_rows_removed_empty_text": int(empty_text.sum()), |
| "n_rows_removed_low_context": int((actual_reject & quality_reject).sum()), |
| "n_rows_empty_or_placeholder_abstract": int(df_rejected["abstract_is_placeholder"].sum()) if len(df_rejected) else 0, |
| } |
| print("\nPreprocessing summary:") |
| print(json.dumps(preprocessing_summary, indent=2)) |
|
|
| |
| def make_key(row): |
| return "|".join([ |
| normalize_key_text(row["judul_publikasi"]), |
| normalize_key_text(row["abstrak"]), |
| normalize_key_year(row["tahun"]), |
| ]) |
|
|
| df_rows["publication_key"] = df_rows.apply(make_key, axis=1) |
| df_rows["publication_id"] = df_rows["publication_key"].apply( |
| lambda v: hashlib.sha1(v.encode("utf-8", errors="ignore")).hexdigest()[:16] |
| ) |
|
|
| |
| edge_rows = [] |
| for pid, cell in zip(df_rows["publication_id"], df_rows["dosen"]): |
| for name in split_lecturers(cell): |
| edge_rows.append((pid, name)) |
| df_publication_lecturers = pd.DataFrame(edge_rows, columns=["publication_id", "dosen"]) |
| df_publication_lecturers = df_publication_lecturers[df_publication_lecturers["dosen"].str.len() > 0] |
| df_publication_lecturers = df_publication_lecturers.drop_duplicates(["publication_id", "dosen"]).reset_index(drop=True) |
|
|
| dosen_grouped = df_publication_lecturers.groupby("publication_id")["dosen"].apply(lambda s: sorted(set(s))) |
| df_pub = df_rows.drop_duplicates("publication_id", keep="first").copy().reset_index(drop=True) |
| df_pub["dosen_list"] = df_pub["publication_id"].map( |
| dosen_grouped.apply(lambda v: json.dumps(v, ensure_ascii=False)) |
| ).fillna("[]") |
| df_pub["dosen"] = df_pub["publication_id"].map(dosen_grouped.apply(lambda v: "; ".join(v))).fillna("") |
| df_pub["jumlah_dosen"] = df_pub["publication_id"].map(dosen_grouped.apply(len)).fillna(0).astype(int) |
|
|
| df_original_rows = df_rows.copy() |
| df = df_pub.reset_index(drop=True) |
|
|
| print(f"\nPublikasi unik untuk model: {len(df):,}") |
| print(f"Relasi publikasi-dosen: {len(df_publication_lecturers):,}") |
| print(f"Baris duplikat publikasi yang digabung: {len(df_original_rows) - len(df):,}") |
| return df, df_original_rows, df_rejected, df_publication_lecturers, preprocessing_summary |
|
|
|
|
| |
| |
| |
| _sentence_models = {} |
|
|
|
|
| def get_model(name): |
| from sentence_transformers import SentenceTransformer |
| if name not in _sentence_models: |
| print(f"Loading {name}...") |
| _sentence_models[name] = SentenceTransformer(name, device="cpu") |
| return _sentence_models[name] |
|
|
|
|
| def encode(name, texts, batch_size=32): |
| model = get_model(name) |
| print(f"Encoding {len(texts)} dokumen dengan {name}...") |
| t0 = time.time() |
| emb = model.encode( |
| [str(t) for t in texts], |
| show_progress_bar=True, |
| batch_size=batch_size, |
| convert_to_numpy=True, |
| normalize_embeddings=False, |
| ).astype(np.float32) |
| print(f" shape {emb.shape} ({time.time() - t0:.1f}s)") |
| return emb |
|
|
|
|
| def make_umap(params): |
| from umap import UMAP |
| kwargs = dict(params) |
| kwargs["metric"] = "cosine" |
| kwargs["random_state"] = UMAP_RANDOM_STATE |
| return UMAP(**kwargs) |
|
|
|
|
| def make_hdbscan(params): |
| from hdbscan import HDBSCAN |
| kwargs = dict(params) |
| kwargs.setdefault("prediction_data", False) |
| kwargs.setdefault("core_dist_n_jobs", -1) |
| return HDBSCAN(**kwargs) |
|
|
|
|
| def fit_bertopic(name, docs_input, embeddings, model_name, umap_params, hdbscan_params): |
| from bertopic import BERTopic |
| t0 = time.time() |
| topic_model = BERTopic( |
| umap_model=make_umap(umap_params), |
| hdbscan_model=make_hdbscan(hdbscan_params), |
| embedding_model=get_model(model_name), |
| calculate_probabilities=False, |
| verbose=False, |
| ) |
| topics, probs = topic_model.fit_transform(docs_input, embeddings) |
| print(f"{name}: fit selesai ({time.time() - t0:.1f}s)") |
| return topic_model, topics, probs |
|
|
|
|
| def grid_search_best(docs): |
| from sklearn.metrics import silhouette_score |
| emb = encode(MINILM_NAME, docs) |
| best_score, best_up, best_hp = -1.0, None, None |
| print(f"\nGrid search: {len(UMAP_GRID)} x {len(HDBSCAN_GRID)} = {len(UMAP_GRID) * len(HDBSCAN_GRID)} kombinasi") |
| for i, (up, hp) in enumerate(product(UMAP_GRID, HDBSCAN_GRID)): |
| try: |
| _, topics, _ = fit_bertopic(f"Grid {i+1}", docs, emb, MINILM_NAME, up, hp) |
| labels = np.array(topics) |
| mask = labels != -1 |
| n_topics = len(set(labels)) - (1 if -1 in labels else 0) |
| noise_pct = (labels == -1).sum() / len(labels) * 100 |
| if mask.sum() > 1 and len(set(labels[mask])) > 1: |
| sil = float(silhouette_score(emb[mask], labels[mask])) |
| else: |
| sil = -1.0 |
| print(f"[{i+1:2d}] topics={n_topics:2d} noise={noise_pct:5.1f}% sil={sil:.4f} umap={up} hdb={hp}") |
| if sil > best_score: |
| best_score, best_up, best_hp = sil, up, hp |
| except Exception as e: |
| print(f"[{i+1:2d}] ERROR: {e}") |
| if best_up is None: |
| raise RuntimeError("Grid search gagal.") |
| print(f"\nBest config sil={best_score:.4f} UMAP={best_up} HDBSCAN={best_hp}") |
| return best_up, best_hp |
|
|
|
|
| def backup_path(path): |
| if os.path.exists(path): |
| bak = f"{path}.bak.{int(time.time())}" |
| if os.path.isdir(path): |
| shutil.copytree(path, bak) |
| else: |
| shutil.copy2(path, bak) |
| print(f" backup: {os.path.basename(bak)}") |
|
|
|
|
| |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument("--input", default=DEFAULT_INPUT) |
| parser.add_argument("--models-dir", default=DEFAULT_MODELS_DIR) |
| parser.add_argument("--data-dir", default=DEFAULT_DATA_DIR) |
| parser.add_argument("--batch-size", type=int, default=32) |
| parser.add_argument("--no-backup", action="store_true") |
| args = parser.parse_args() |
|
|
| from langdetect import detect, DetectorFactory |
| DetectorFactory.seed = 42 |
|
|
| models_dir, data_dir = args.models_dir, args.data_dir |
| os.makedirs(models_dir, exist_ok=True) |
| os.makedirs(data_dir, exist_ok=True) |
|
|
| df, df_original_rows, df_rejected_rows, df_publication_lecturers, preprocessing_summary = preprocess(args.input) |
|
|
| |
| def detect_lang(text): |
| try: |
| return detect(str(text)) |
| except Exception: |
| return "unknown" |
|
|
| print("\nMendeteksi bahasa...") |
| df["lang"] = df["abstrak"].apply(detect_lang) |
| df["lang_group"] = df["lang"].apply(lambda x: "id" if x == "id" else "en") |
| print(df["lang_group"].value_counts().to_dict()) |
|
|
| docs = df["Judul + Deskripsi"].fillna("").tolist() |
|
|
| |
| best_up, best_hp = grid_search_best(docs) |
|
|
| |
| df_en = df[df["lang_group"] == "en"].reset_index(drop=True) |
| df_id = df[df["lang_group"] == "id"].reset_index(drop=True) |
| docs_en = df_en["Judul + Deskripsi"].fillna("").tolist() |
| docs_id = df_id["Judul + Deskripsi"].fillna("").tolist() |
| print(f"\nDokumen EN: {len(docs_en)} | ID: {len(docs_id)}") |
|
|
| print("\n=== EN Clustering ===") |
| embeddings_en = encode(EN_MODEL_NAME, docs_en, args.batch_size) |
| bt_en, topics_en, _ = fit_bertopic("S3_EN", docs_en, embeddings_en, EN_MODEL_NAME, best_up, best_hp) |
|
|
| print("\n=== ID Clustering ===") |
| embeddings_id = encode(ID_MODEL_NAME, docs_id, args.batch_size) |
| id_up = {"n_neighbors": min(10, max(2, len(docs_id) - 1)), "n_components": 5, "min_dist": 0.0} |
| id_hp = {"min_cluster_size": 5, "min_samples": 3} |
| bt_id, topics_id, _ = fit_bertopic("S3_ID", docs_id, embeddings_id, ID_MODEL_NAME, id_up, id_hp) |
|
|
| print("\nReduce outliers...") |
| topics_en_final = bt_en.reduce_outliers(docs_en, topics_en, strategy=OUTLIER_STRATEGY) |
| topics_id_final = bt_id.reduce_outliers(docs_id, topics_id, strategy=OUTLIER_STRATEGY) |
|
|
| bt_en.update_topics(docs_en, topics=topics_en_final) |
| bt_id.update_topics(docs_id, topics=topics_id_final) |
|
|
| n_en_final = len(set(topics_en_final)) - (1 if -1 in topics_en_final else 0) |
| n_id_final = len(set(topics_id_final)) - (1 if -1 in topics_id_final else 0) |
| max_en = max((t for t in topics_en_final if t != -1), default=-1) |
| id_offset = max_en + 1 |
| print(f"\nEN: {n_en_final} topik (max_id={max_en}) | ID: {n_id_final} topik (offset={id_offset})") |
|
|
| |
| en_idx = df[df["lang_group"] == "en"].index.tolist() |
| id_idx = df[df["lang_group"] == "id"].index.tolist() |
| dim = embeddings_en.shape[1] |
| combined = np.zeros((len(df), dim), dtype=np.float32) |
| for i, idx in enumerate(en_idx): |
| combined[idx] = embeddings_en[i] |
| for i, idx in enumerate(id_idx): |
| combined[idx] = embeddings_id[i] |
|
|
| print("\nMenyimpan artefak (dengan backup)...") |
| for fname, arr in [ |
| ("embeddings_en.npy", embeddings_en), |
| ("embeddings_id.npy", embeddings_id), |
| ("embeddings.npy", combined), |
| ("publication_ids.npy", df["publication_id"].astype(str).values), |
| ]: |
| p = os.path.join(models_dir, fname) |
| if not args.no_backup: |
| backup_path(p) |
| np.save(p, arr) |
| print(f" saved {fname}") |
|
|
| |
| for sub, model, mname in [("bt_en", bt_en, EN_MODEL_NAME), ("bt_id", bt_id, ID_MODEL_NAME)]: |
| p = os.path.join(models_dir, sub) |
| if not args.no_backup: |
| backup_path(p) |
| if os.path.exists(p): |
| shutil.rmtree(p) |
| model.save(p, serialization="safetensors", save_ctfidf=True, save_embedding_model=mname) |
| print(f" saved {sub}") |
|
|
| |
| rejected_reason_counts = ( |
| {str(k): int(v) for k, v in df_rejected_rows["abstract_quality_reason"].value_counts().items()} |
| if len(df_rejected_rows) else {} |
| ) |
| topic_info = { |
| "en": {"model_name": EN_MODEL_NAME, "n_topics": n_en_final, "max_topic_id": int(max_en)}, |
| "id": {"model_name": ID_MODEL_NAME, "n_topics": n_id_final, "id_topic_offset": int(id_offset)}, |
| "data": { |
| "unit": "unique_publication", |
| "n_source_rows": int(len(df_original_rows)), |
| "n_unique_publications": int(len(df)), |
| "n_publication_lecturer_edges": int(len(df_publication_lecturers)), |
| "duplicate_publication_rows_merged": int(len(df_original_rows) - len(df)), |
| }, |
| "preprocessing": preprocessing_summary, |
| "rejected_reason_counts": rejected_reason_counts, |
| } |
| p = os.path.join(models_dir, "topic_info.json") |
| if not args.no_backup: |
| backup_path(p) |
| with open(p, "w", encoding="utf-8") as f: |
| json.dump(topic_info, f, indent=2) |
| print(" saved topic_info.json") |
| print(json.dumps(topic_info, indent=2)) |
|
|
| |
| base_cols = ["publication_id", "judul_publikasi", "abstrak", "tahun", "dosen", |
| "dosen_list", "jumlah_dosen", "lang_group"] |
| quality_cols = [c for c in ["abstract_word_count", "abstract_unique_word_count", |
| "abstract_unique_ratio", "abstract_max_token_ratio", |
| "abstract_quality_reason"] if c in df.columns] |
| pub = df[base_cols + quality_cols].copy() |
| pub["embedding_index"] = np.arange(len(pub)) |
| pub["topic_id"] = -1 |
| pub["topic_id_raw"] = -1 |
| for i, idx in enumerate(en_idx): |
| pub.loc[idx, "topic_id"] = int(topics_en_final[i]) |
| pub.loc[idx, "topic_id_raw"] = int(topics_en[i]) |
| for i, idx in enumerate(id_idx): |
| ft, rt = topics_id_final[i], topics_id[i] |
| pub.loc[idx, "topic_id"] = int(ft + id_offset) if ft != -1 else -1 |
| pub.loc[idx, "topic_id_raw"] = int(rt + id_offset) if rt != -1 else -1 |
| pub["lang"] = pub["lang_group"] |
| pub["is_noise"] = pub["topic_id"] == -1 |
|
|
| publication_lecturers = df_publication_lecturers.merge( |
| pub[["publication_id", "embedding_index", "topic_id", "lang", "judul_publikasi", "tahun"]], |
| on="publication_id", how="left", |
| ) |
|
|
| publications_export = pub.drop(columns=["lang_group"]).copy() |
| publication_topics_export = pub[["publication_id", "embedding_index", "topic_id", "topic_id_raw", |
| "lang", "is_noise"] + quality_cols].copy() |
|
|
| |
| lec_counts = publication_lecturers.groupby("dosen")["publication_id"].nunique().rename("jumlah_publikasi_unik") |
| lec_topic = ( |
| publication_lecturers[publication_lecturers["topic_id"] != -1] |
| .groupby(["dosen", "topic_id"]).size().rename("jumlah").reset_index() |
| ) |
| if lec_topic.empty: |
| lec_top = pd.Series(dtype=str, name="top_topics") |
| else: |
| lec_top = ( |
| lec_topic.sort_values(["dosen", "jumlah"], ascending=[True, False]) |
| .groupby("dosen").head(5).groupby("dosen")[["topic_id", "jumlah"]] |
| .apply(lambda g: json.dumps([{"topic_id": int(r.topic_id), "count": int(r.jumlah)} |
| for r in g.itertuples(index=False)], ensure_ascii=False)) |
| .rename("top_topics") |
| ) |
| lecturer_profiles = lec_counts.to_frame().join(lec_top, how="left").reset_index() |
| lecturer_profiles["top_topics"] = lecturer_profiles["top_topics"].fillna("[]") |
|
|
| pid_to_pos = {pid: i for i, pid in enumerate(df["publication_id"].astype(str))} |
| lec_emb, lec_names = [], [] |
| for dosen, group in publication_lecturers.groupby("dosen"): |
| positions = [pid_to_pos[str(pid)] for pid in group["publication_id"].unique() if str(pid) in pid_to_pos] |
| if positions: |
| lec_names.append(dosen) |
| lec_emb.append(combined[positions].mean(axis=0)) |
| lec_emb = np.vstack(lec_emb).astype(np.float32) if lec_emb else np.empty((0, dim), dtype=np.float32) |
| lec_index = {n: i for i, n in enumerate(lec_names)} |
| lecturer_profiles["embedding_index"] = lecturer_profiles["dosen"].map(lec_index).fillna(-1).astype(int) |
|
|
| for fname, arr in [("lecturer_embeddings.npy", lec_emb), |
| ("lecturer_names.npy", np.array(lec_names, dtype=str))]: |
| p = os.path.join(models_dir, fname) |
| if not args.no_backup: |
| backup_path(p) |
| np.save(p, arr) |
| print(f" saved {fname}") |
|
|
| for fname, frame in [ |
| ("publications.csv", publications_export), |
| ("metadata.csv", publications_export), |
| ("publication_lecturers.csv", publication_lecturers), |
| ("publication_topics.csv", publication_topics_export), |
| ("lecturer_profiles.csv", lecturer_profiles), |
| ]: |
| p = os.path.join(data_dir, fname) |
| if not args.no_backup: |
| backup_path(p) |
| frame.to_csv(p, index=False) |
| print(f" saved {fname} ({len(frame)} rows)") |
|
|
| print("\nTopic distribution:") |
| print(publications_export["topic_id"].value_counts().sort_index().head(20)) |
| print("\nSelesai. embeddings rows =", combined.shape[0], "| publications rows =", len(publications_export)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
