r"""Re-run the split-language BERTopic pipeline locally and export backend artifacts. Ported from Model/clustering_experiments.ipynb (scenario S3: split language), adapted to run on CPU/Windows and write directly into backend/models + backend/data. Pipeline: 1. Load input CSV, audit abstract quality, drop low-context rows, dedup -> unique publications. 2. Detect language (langdetect) -> lang_group in {en, id}. 3. S1: encode with paraphrase-multilingual-MiniLM-L12-v2, grid-search UMAP+HDBSCAN, pick best UMAP/HDBSCAN config by silhouette (best_up/best_hp). 4. S3: split by language; cluster EN with all-mpnet-base-v2 (best_up/best_hp) and ID with firqaaa/indo-sentence-bert-base (fixed params); reduce_outliers per language. 5. Export embeddings.npy/_en/_id, publication_ids.npy, bt_en/bt_id, topic_info.json, publications.csv/metadata.csv, publication_lecturers.csv, publication_topics.csv, lecturer_profiles.csv, lecturer_embeddings.npy/lecturer_names.npy. Usage (from backend/ with the project venv): ..\.venv\Scripts\python.exe scripts\rerun_bertopic.py ..\.venv\Scripts\python.exe scripts\rerun_bertopic.py --input "C:\Kuliah\Skripsi\Work\Model\metadata.csv" """ from __future__ import annotations import argparse import hashlib import json import os import re import shutil import time from collections import defaultdict from itertools import product import numpy as np import pandas as pd BACKEND_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DEFAULT_INPUT = r"C:\Kuliah\Skripsi\Work\Model\metadata.csv" DEFAULT_MODELS_DIR = os.path.join(BACKEND_DIR, "models") DEFAULT_DATA_DIR = os.path.join(BACKEND_DIR, "data") MINILM_NAME = "paraphrase-multilingual-MiniLM-L12-v2" EN_MODEL_NAME = "all-mpnet-base-v2" ID_MODEL_NAME = "firqaaa/indo-sentence-bert-base" # Preprocessing thresholds (match notebook defaults) MIN_ABSTRACT_WORDS = 30 MIN_ABSTRACT_UNIQUE_WORDS = 10 MIN_ABSTRACT_UNIQUE_RATIO = 0.25 MAX_ABSTRACT_TOKEN_RATIO = 0.30 FILTER_LOW_CONTEXT_ABSTRACTS = True UMAP_RANDOM_STATE = 42 OUTLIER_STRATEGY = "distributions" ABSTRACT_PLACEHOLDERS = { "", "-", "\u2013", "\u2014", ".", "..", "...", "na", "n/a", "none", "null", "no abstract", "no abstract available", "abstract not available", "not available", "tidak ada", "tidak tersedia", "belum tersedia", "abstrak tidak tersedia", "abstract", "abstrak", } ABSTRACT_PLACEHOLDER_PREFIXES = ( "no abstract available", "abstract not available", "this article has no abstract", "tidak tersedia", "abstrak tidak tersedia", ) ABSTRACT_TEMPLATE_PATTERNS = ( "abstract is written in english or indonesian", "keywords: 3-5 words", "kata kunci: 3-5 kata", ) UMAP_GRID = [ {"n_neighbors": 10, "n_components": 5, "min_dist": 0.0}, {"n_neighbors": 15, "n_components": 5, "min_dist": 0.0}, {"n_neighbors": 15, "n_components": 10, "min_dist": 0.0}, {"n_neighbors": 25, "n_components": 5, "min_dist": 0.1}, ] HDBSCAN_GRID = [ {"min_cluster_size": 5, "min_samples": 3}, {"min_cluster_size": 10, "min_samples": 5}, {"min_cluster_size": 15, "min_samples": 5}, {"min_cluster_size": 15, "min_samples": 10}, ] # -------------------------------------------------------------------------------------- # Text / key helpers # -------------------------------------------------------------------------------------- def normalize_audit_text(value) -> str: return re.sub(r"\s+", " ", str(value).lower()).strip() def abstract_tokens(value): return re.findall(r"[A-Za-z\u00c0-\u017f0-9]+(?:[-'][A-Za-z\u00c0-\u017f0-9]+)?", str(value).lower()) def max_token_ratio(tokens) -> float: if not tokens: return 0.0 counts = defaultdict(int) for token in tokens: counts[token] += 1 return max(counts.values()) / len(tokens) def normalize_key_text(value) -> str: return re.sub(r"\s+", " ", str(value).lower()).strip() def normalize_key_year(value) -> str: if pd.isna(value): return "" try: v = float(value) return str(int(v)) if v.is_integer() else str(value) except Exception: return str(value) def split_lecturers(cell) -> list[str]: if cell is None or (isinstance(cell, float) and pd.isna(cell)): return [] return [p.strip() for p in re.split(r"[;,/]| dan ", str(cell)) if p.strip()] # -------------------------------------------------------------------------------------- # Preprocessing (notebook cell s1) # -------------------------------------------------------------------------------------- def preprocess(input_path: str): df_rows = pd.read_csv(input_path) print(f"File: {input_path}") print(f"Shape awal: {df_rows.shape}") rename_map = {"nama_dosen": "dosen", "judul_penelitian": "judul_publikasi", "tahun_terbit": "tahun"} df_rows = df_rows.rename(columns={k: v for k, v in rename_map.items() if k in df_rows.columns}) required = ["judul_publikasi", "dosen", "abstrak", "tahun"] missing = [c for c in required if c not in df_rows.columns] if missing: raise ValueError(f"Kolom wajib tidak ditemukan: {missing}") df_rows = df_rows.copy() df_rows["source_row_id"] = np.arange(len(df_rows)) df_rows["judul_publikasi"] = df_rows["judul_publikasi"].fillna("").astype(str).str.strip() df_rows["abstrak"] = df_rows["abstrak"].fillna("").astype(str).str.strip() df_rows["dosen"] = df_rows["dosen"].fillna("").astype(str).str.strip() df_rows["tahun"] = pd.to_numeric(df_rows["tahun"], errors="coerce") df_rows["Judul + Deskripsi"] = (df_rows["judul_publikasi"] + " " + df_rows["abstrak"]).str.strip() token_lists = df_rows["abstrak"].apply(abstract_tokens) df_rows["abstract_norm"] = df_rows["abstrak"].apply(normalize_audit_text) df_rows["abstract_char_count"] = df_rows["abstrak"].str.len() df_rows["abstract_word_count"] = token_lists.apply(len) df_rows["abstract_unique_word_count"] = token_lists.apply(lambda v: len(set(v))) df_rows["abstract_unique_ratio"] = np.where( df_rows["abstract_word_count"] > 0, df_rows["abstract_unique_word_count"] / df_rows["abstract_word_count"], 0.0, ) df_rows["abstract_max_token_ratio"] = token_lists.apply(max_token_ratio) df_rows["abstract_is_placeholder"] = ( df_rows["abstract_norm"].isin(ABSTRACT_PLACEHOLDERS) | df_rows["abstract_norm"].apply(lambda t: any(t.startswith(p) for p in ABSTRACT_PLACEHOLDER_PREFIXES)) ) df_rows["abstract_is_template"] = df_rows["abstract_norm"].apply( lambda t: any(p in t for p in ABSTRACT_TEMPLATE_PATTERNS) ) df_rows["abstract_has_repeated_punct"] = df_rows["abstrak"].str.contains( r"(?:\.{8,}|_{5,}|-{5,})", regex=True, na=False ) def quality_reasons(row): reasons = [] if not str(row["Judul + Deskripsi"]).strip(): reasons.append("empty_text") if row["abstract_is_placeholder"]: reasons.append("empty_or_placeholder_abstract") else: if row["abstract_word_count"] < MIN_ABSTRACT_WORDS: reasons.append(f"abstract_words<{MIN_ABSTRACT_WORDS}") if row["abstract_unique_word_count"] < MIN_ABSTRACT_UNIQUE_WORDS: reasons.append(f"unique_words<{MIN_ABSTRACT_UNIQUE_WORDS}") if row["abstract_is_template"]: reasons.append("template_abstract") if row["abstract_has_repeated_punct"] and row["abstract_word_count"] < 80: reasons.append("repeated_punctuation") if row["abstract_word_count"] >= MIN_ABSTRACT_WORDS and row["abstract_unique_ratio"] < MIN_ABSTRACT_UNIQUE_RATIO: reasons.append(f"unique_ratio<{MIN_ABSTRACT_UNIQUE_RATIO}") if row["abstract_word_count"] >= MIN_ABSTRACT_WORDS and row["abstract_max_token_ratio"] > MAX_ABSTRACT_TOKEN_RATIO: reasons.append(f"max_token_ratio>{MAX_ABSTRACT_TOKEN_RATIO}") return "; ".join(dict.fromkeys(reasons)) if reasons else "keep" df_rows["abstract_quality_reason"] = df_rows.apply(quality_reasons, axis=1) df_rows["abstract_quality_status"] = np.where(df_rows["abstract_quality_reason"].eq("keep"), "keep", "reject") quality_reject = df_rows["abstract_quality_status"].eq("reject") empty_text = df_rows["Judul + Deskripsi"].str.len().eq(0) actual_reject = empty_text | (FILTER_LOW_CONTEXT_ABSTRACTS & quality_reject) rows_before = len(df_rows) df_rejected = df_rows[actual_reject].copy().reset_index(drop=True) df_rows = df_rows[~actual_reject].copy().reset_index(drop=True) if df_rows.empty: raise ValueError("Semua baris terhapus oleh preprocessing.") preprocessing_summary = { "filter_low_context_abstracts": bool(FILTER_LOW_CONTEXT_ABSTRACTS), "min_abstract_words": int(MIN_ABSTRACT_WORDS), "min_abstract_unique_words": int(MIN_ABSTRACT_UNIQUE_WORDS), "min_abstract_unique_ratio": float(MIN_ABSTRACT_UNIQUE_RATIO), "max_abstract_token_ratio": float(MAX_ABSTRACT_TOKEN_RATIO), "n_rows_before_preprocessing": int(rows_before), "n_rows_after_preprocessing": int(len(df_rows)), "n_rows_removed_preprocessing": int(len(df_rejected)), "n_rows_removed_empty_text": int(empty_text.sum()), "n_rows_removed_low_context": int((actual_reject & quality_reject).sum()), "n_rows_empty_or_placeholder_abstract": int(df_rejected["abstract_is_placeholder"].sum()) if len(df_rejected) else 0, } print("\nPreprocessing summary:") print(json.dumps(preprocessing_summary, indent=2)) # Build publication identity + lecturer edges def make_key(row): return "|".join([ normalize_key_text(row["judul_publikasi"]), normalize_key_text(row["abstrak"]), normalize_key_year(row["tahun"]), ]) df_rows["publication_key"] = df_rows.apply(make_key, axis=1) df_rows["publication_id"] = df_rows["publication_key"].apply( lambda v: hashlib.sha1(v.encode("utf-8", errors="ignore")).hexdigest()[:16] ) # Explode multi-lecturer cells (metadata.csv joins dosen with '; ') edge_rows = [] for pid, cell in zip(df_rows["publication_id"], df_rows["dosen"]): for name in split_lecturers(cell): edge_rows.append((pid, name)) df_publication_lecturers = pd.DataFrame(edge_rows, columns=["publication_id", "dosen"]) df_publication_lecturers = df_publication_lecturers[df_publication_lecturers["dosen"].str.len() > 0] df_publication_lecturers = df_publication_lecturers.drop_duplicates(["publication_id", "dosen"]).reset_index(drop=True) dosen_grouped = df_publication_lecturers.groupby("publication_id")["dosen"].apply(lambda s: sorted(set(s))) df_pub = df_rows.drop_duplicates("publication_id", keep="first").copy().reset_index(drop=True) df_pub["dosen_list"] = df_pub["publication_id"].map( dosen_grouped.apply(lambda v: json.dumps(v, ensure_ascii=False)) ).fillna("[]") df_pub["dosen"] = df_pub["publication_id"].map(dosen_grouped.apply(lambda v: "; ".join(v))).fillna("") df_pub["jumlah_dosen"] = df_pub["publication_id"].map(dosen_grouped.apply(len)).fillna(0).astype(int) df_original_rows = df_rows.copy() df = df_pub.reset_index(drop=True) print(f"\nPublikasi unik untuk model: {len(df):,}") print(f"Relasi publikasi-dosen: {len(df_publication_lecturers):,}") print(f"Baris duplikat publikasi yang digabung: {len(df_original_rows) - len(df):,}") return df, df_original_rows, df_rejected, df_publication_lecturers, preprocessing_summary # -------------------------------------------------------------------------------------- # Modeling helpers # -------------------------------------------------------------------------------------- _sentence_models = {} def get_model(name): from sentence_transformers import SentenceTransformer if name not in _sentence_models: print(f"Loading {name}...") _sentence_models[name] = SentenceTransformer(name, device="cpu") return _sentence_models[name] def encode(name, texts, batch_size=32): model = get_model(name) print(f"Encoding {len(texts)} dokumen dengan {name}...") t0 = time.time() emb = model.encode( [str(t) for t in texts], show_progress_bar=True, batch_size=batch_size, convert_to_numpy=True, normalize_embeddings=False, ).astype(np.float32) print(f" shape {emb.shape} ({time.time() - t0:.1f}s)") return emb def make_umap(params): from umap import UMAP kwargs = dict(params) kwargs["metric"] = "cosine" kwargs["random_state"] = UMAP_RANDOM_STATE return UMAP(**kwargs) def make_hdbscan(params): from hdbscan import HDBSCAN kwargs = dict(params) kwargs.setdefault("prediction_data", False) kwargs.setdefault("core_dist_n_jobs", -1) return HDBSCAN(**kwargs) def fit_bertopic(name, docs_input, embeddings, model_name, umap_params, hdbscan_params): from bertopic import BERTopic t0 = time.time() topic_model = BERTopic( umap_model=make_umap(umap_params), hdbscan_model=make_hdbscan(hdbscan_params), embedding_model=get_model(model_name), calculate_probabilities=False, verbose=False, ) topics, probs = topic_model.fit_transform(docs_input, embeddings) print(f"{name}: fit selesai ({time.time() - t0:.1f}s)") return topic_model, topics, probs def grid_search_best(docs): from sklearn.metrics import silhouette_score emb = encode(MINILM_NAME, docs) best_score, best_up, best_hp = -1.0, None, None print(f"\nGrid search: {len(UMAP_GRID)} x {len(HDBSCAN_GRID)} = {len(UMAP_GRID) * len(HDBSCAN_GRID)} kombinasi") for i, (up, hp) in enumerate(product(UMAP_GRID, HDBSCAN_GRID)): try: _, topics, _ = fit_bertopic(f"Grid {i+1}", docs, emb, MINILM_NAME, up, hp) labels = np.array(topics) mask = labels != -1 n_topics = len(set(labels)) - (1 if -1 in labels else 0) noise_pct = (labels == -1).sum() / len(labels) * 100 if mask.sum() > 1 and len(set(labels[mask])) > 1: sil = float(silhouette_score(emb[mask], labels[mask])) else: sil = -1.0 print(f"[{i+1:2d}] topics={n_topics:2d} noise={noise_pct:5.1f}% sil={sil:.4f} umap={up} hdb={hp}") if sil > best_score: best_score, best_up, best_hp = sil, up, hp except Exception as e: print(f"[{i+1:2d}] ERROR: {e}") if best_up is None: raise RuntimeError("Grid search gagal.") print(f"\nBest config sil={best_score:.4f} UMAP={best_up} HDBSCAN={best_hp}") return best_up, best_hp def backup_path(path): if os.path.exists(path): bak = f"{path}.bak.{int(time.time())}" if os.path.isdir(path): shutil.copytree(path, bak) else: shutil.copy2(path, bak) print(f" backup: {os.path.basename(bak)}") # -------------------------------------------------------------------------------------- # Main # -------------------------------------------------------------------------------------- def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input", default=DEFAULT_INPUT) parser.add_argument("--models-dir", default=DEFAULT_MODELS_DIR) parser.add_argument("--data-dir", default=DEFAULT_DATA_DIR) parser.add_argument("--batch-size", type=int, default=32) parser.add_argument("--no-backup", action="store_true") args = parser.parse_args() from langdetect import detect, DetectorFactory DetectorFactory.seed = 42 models_dir, data_dir = args.models_dir, args.data_dir os.makedirs(models_dir, exist_ok=True) os.makedirs(data_dir, exist_ok=True) df, df_original_rows, df_rejected_rows, df_publication_lecturers, preprocessing_summary = preprocess(args.input) # Language detection def detect_lang(text): try: return detect(str(text)) except Exception: return "unknown" print("\nMendeteksi bahasa...") df["lang"] = df["abstrak"].apply(detect_lang) df["lang_group"] = df["lang"].apply(lambda x: "id" if x == "id" else "en") print(df["lang_group"].value_counts().to_dict()) docs = df["Judul + Deskripsi"].fillna("").tolist() # S1 grid search -> best UMAP/HDBSCAN best_up, best_hp = grid_search_best(docs) # S3 split-language clustering df_en = df[df["lang_group"] == "en"].reset_index(drop=True) df_id = df[df["lang_group"] == "id"].reset_index(drop=True) docs_en = df_en["Judul + Deskripsi"].fillna("").tolist() docs_id = df_id["Judul + Deskripsi"].fillna("").tolist() print(f"\nDokumen EN: {len(docs_en)} | ID: {len(docs_id)}") print("\n=== EN Clustering ===") embeddings_en = encode(EN_MODEL_NAME, docs_en, args.batch_size) bt_en, topics_en, _ = fit_bertopic("S3_EN", docs_en, embeddings_en, EN_MODEL_NAME, best_up, best_hp) print("\n=== ID Clustering ===") embeddings_id = encode(ID_MODEL_NAME, docs_id, args.batch_size) id_up = {"n_neighbors": min(10, max(2, len(docs_id) - 1)), "n_components": 5, "min_dist": 0.0} id_hp = {"min_cluster_size": 5, "min_samples": 3} bt_id, topics_id, _ = fit_bertopic("S3_ID", docs_id, embeddings_id, ID_MODEL_NAME, id_up, id_hp) print("\nReduce outliers...") topics_en_final = bt_en.reduce_outliers(docs_en, topics_en, strategy=OUTLIER_STRATEGY) topics_id_final = bt_id.reduce_outliers(docs_id, topics_id, strategy=OUTLIER_STRATEGY) bt_en.update_topics(docs_en, topics=topics_en_final) bt_id.update_topics(docs_id, topics=topics_id_final) n_en_final = len(set(topics_en_final)) - (1 if -1 in topics_en_final else 0) n_id_final = len(set(topics_id_final)) - (1 if -1 in topics_id_final else 0) max_en = max((t for t in topics_en_final if t != -1), default=-1) id_offset = max_en + 1 print(f"\nEN: {n_en_final} topik (max_id={max_en}) | ID: {n_id_final} topik (offset={id_offset})") # ---- Export embeddings ---- en_idx = df[df["lang_group"] == "en"].index.tolist() id_idx = df[df["lang_group"] == "id"].index.tolist() dim = embeddings_en.shape[1] combined = np.zeros((len(df), dim), dtype=np.float32) for i, idx in enumerate(en_idx): combined[idx] = embeddings_en[i] for i, idx in enumerate(id_idx): combined[idx] = embeddings_id[i] print("\nMenyimpan artefak (dengan backup)...") for fname, arr in [ ("embeddings_en.npy", embeddings_en), ("embeddings_id.npy", embeddings_id), ("embeddings.npy", combined), ("publication_ids.npy", df["publication_id"].astype(str).values), ]: p = os.path.join(models_dir, fname) if not args.no_backup: backup_path(p) np.save(p, arr) print(f" saved {fname}") # ---- Save BERTopic models ---- for sub, model, mname in [("bt_en", bt_en, EN_MODEL_NAME), ("bt_id", bt_id, ID_MODEL_NAME)]: p = os.path.join(models_dir, sub) if not args.no_backup: backup_path(p) if os.path.exists(p): shutil.rmtree(p) model.save(p, serialization="safetensors", save_ctfidf=True, save_embedding_model=mname) print(f" saved {sub}") # ---- topic_info.json ---- rejected_reason_counts = ( {str(k): int(v) for k, v in df_rejected_rows["abstract_quality_reason"].value_counts().items()} if len(df_rejected_rows) else {} ) topic_info = { "en": {"model_name": EN_MODEL_NAME, "n_topics": n_en_final, "max_topic_id": int(max_en)}, "id": {"model_name": ID_MODEL_NAME, "n_topics": n_id_final, "id_topic_offset": int(id_offset)}, "data": { "unit": "unique_publication", "n_source_rows": int(len(df_original_rows)), "n_unique_publications": int(len(df)), "n_publication_lecturer_edges": int(len(df_publication_lecturers)), "duplicate_publication_rows_merged": int(len(df_original_rows) - len(df)), }, "preprocessing": preprocessing_summary, "rejected_reason_counts": rejected_reason_counts, } p = os.path.join(models_dir, "topic_info.json") if not args.no_backup: backup_path(p) with open(p, "w", encoding="utf-8") as f: json.dump(topic_info, f, indent=2) print(" saved topic_info.json") print(json.dumps(topic_info, indent=2)) # ---- Backend data tables ---- base_cols = ["publication_id", "judul_publikasi", "abstrak", "tahun", "dosen", "dosen_list", "jumlah_dosen", "lang_group"] quality_cols = [c for c in ["abstract_word_count", "abstract_unique_word_count", "abstract_unique_ratio", "abstract_max_token_ratio", "abstract_quality_reason"] if c in df.columns] pub = df[base_cols + quality_cols].copy() pub["embedding_index"] = np.arange(len(pub)) pub["topic_id"] = -1 pub["topic_id_raw"] = -1 for i, idx in enumerate(en_idx): pub.loc[idx, "topic_id"] = int(topics_en_final[i]) pub.loc[idx, "topic_id_raw"] = int(topics_en[i]) for i, idx in enumerate(id_idx): ft, rt = topics_id_final[i], topics_id[i] pub.loc[idx, "topic_id"] = int(ft + id_offset) if ft != -1 else -1 pub.loc[idx, "topic_id_raw"] = int(rt + id_offset) if rt != -1 else -1 pub["lang"] = pub["lang_group"] pub["is_noise"] = pub["topic_id"] == -1 publication_lecturers = df_publication_lecturers.merge( pub[["publication_id", "embedding_index", "topic_id", "lang", "judul_publikasi", "tahun"]], on="publication_id", how="left", ) publications_export = pub.drop(columns=["lang_group"]).copy() publication_topics_export = pub[["publication_id", "embedding_index", "topic_id", "topic_id_raw", "lang", "is_noise"] + quality_cols].copy() # lecturer profiles + embeddings lec_counts = publication_lecturers.groupby("dosen")["publication_id"].nunique().rename("jumlah_publikasi_unik") lec_topic = ( publication_lecturers[publication_lecturers["topic_id"] != -1] .groupby(["dosen", "topic_id"]).size().rename("jumlah").reset_index() ) if lec_topic.empty: lec_top = pd.Series(dtype=str, name="top_topics") else: lec_top = ( lec_topic.sort_values(["dosen", "jumlah"], ascending=[True, False]) .groupby("dosen").head(5).groupby("dosen")[["topic_id", "jumlah"]] .apply(lambda g: json.dumps([{"topic_id": int(r.topic_id), "count": int(r.jumlah)} for r in g.itertuples(index=False)], ensure_ascii=False)) .rename("top_topics") ) lecturer_profiles = lec_counts.to_frame().join(lec_top, how="left").reset_index() lecturer_profiles["top_topics"] = lecturer_profiles["top_topics"].fillna("[]") pid_to_pos = {pid: i for i, pid in enumerate(df["publication_id"].astype(str))} lec_emb, lec_names = [], [] for dosen, group in publication_lecturers.groupby("dosen"): positions = [pid_to_pos[str(pid)] for pid in group["publication_id"].unique() if str(pid) in pid_to_pos] if positions: lec_names.append(dosen) lec_emb.append(combined[positions].mean(axis=0)) lec_emb = np.vstack(lec_emb).astype(np.float32) if lec_emb else np.empty((0, dim), dtype=np.float32) lec_index = {n: i for i, n in enumerate(lec_names)} lecturer_profiles["embedding_index"] = lecturer_profiles["dosen"].map(lec_index).fillna(-1).astype(int) for fname, arr in [("lecturer_embeddings.npy", lec_emb), ("lecturer_names.npy", np.array(lec_names, dtype=str))]: p = os.path.join(models_dir, fname) if not args.no_backup: backup_path(p) np.save(p, arr) print(f" saved {fname}") for fname, frame in [ ("publications.csv", publications_export), ("metadata.csv", publications_export), ("publication_lecturers.csv", publication_lecturers), ("publication_topics.csv", publication_topics_export), ("lecturer_profiles.csv", lecturer_profiles), ]: p = os.path.join(data_dir, fname) if not args.no_backup: backup_path(p) frame.to_csv(p, index=False) print(f" saved {fname} ({len(frame)} rows)") print("\nTopic distribution:") print(publications_export["topic_id"].value_counts().sort_index().head(20)) print("\nSelesai. embeddings rows =", combined.shape[0], "| publications rows =", len(publications_export)) if __name__ == "__main__": main()