""" Clasificacion de papers por METODOLOGIA (la tecnica que usa el paper). Enfoque HIBRIDO (lexico + SPECTER): - Cada familia metodologica tiene (a) un texto-ancla estilo paper que se embebe con SPECTER y (b) un lexico de terminos regex. - Para cada paper calculamos: sem_f = cos(v_paper, v_ancla_f) # similitud semantica lex_f = #matches del lexico de f en titulo+abstract - Regla de asignacion (multi-tag, hasta MAX_TAGS): se asigna la familia f si sem_f >= TAU_HIGH y sem_f >= s_max - MARGIN (semantico fuerte) o lex_f >= 1 y sem_f >= TAU_LEX_FLOOR (lexico + piso semantico) fallback: si nada pasa pero s_max >= TAU_FALLBACK, se toma el top-1. El lexico aporta precision (menciones explicitas) y el piso semantico evita falsos positivos del lexico (ej. "we do NOT use GARCH" en un paper no-GARCH igual tendria sem bajo y no pasaria el piso). La similitud se calcula con producto punto porque los vectores ya estan normalizados L2 (cos == dot). Para todo el corpus: matrix @ anchors.T. """ from __future__ import annotations import re import sqlite3 from dataclasses import dataclass import numpy as np # ------------------------------------------------------------------ # Umbrales (calibrables; ver scripts/classify_methodology.py --dry-run) # # NOTA: SPECTER comprime todo el espacio "paper de finanzas", asi que la # similitud ABSOLUTA a las anclas es alta para todas las familias y no # discrimina. Por eso trabajamos con z-scores POR FAMILIA: # z_ij = (sim_ij - mu_j) / sigma_j # o sea, cuanto MAS se parece el paper i a la metodologia j que el paper # tipico. Esto elimina el sesgo de anclas centrales y balancea frecuencias. # ------------------------------------------------------------------ TAU_Z = 1.50 # semantico: familia destaca (z alto) sin necesidad de lexico LEX_Z_FLOOR = -0.25 # un match de lexico vale, salvo que el paper sea muy atipico TAU_FALLBACK_Z = 0.80 # si nada pasa, toma el top-1 por z solo si supera esto MAX_TAGS = 3 # ------------------------------------------------------------------ # Taxonomia: 15 familias. anchor = pseudo-paper (title [SEP] abstract). # ------------------------------------------------------------------ TAXONOMY: list[dict] = [ { "name": "Deep Learning", "anchor": "Deep learning for financial prediction [SEP] We apply deep " "neural networks including LSTM, GRU, CNN, transformers, attention " "mechanisms, autoencoders and graph neural networks to learn " "representations and forecast asset prices and financial time series.", "terms": [ "deep learning", "neural network", "neural networks", "lstm", "gru", "recurrent neural", "convolutional", "cnn", "transformer", "attention mechanism", "autoencoder", "deep neural", "feedforward", "mlp", "resnet", "graph neural", "gnn", "representation learning", "deep model", ], }, { "name": "Reinforcement Learning", "anchor": "Reinforcement learning for trading and execution [SEP] We " "formulate the problem as a Markov decision process and train agents " "with Q-learning, deep Q-networks, policy gradient and actor-critic " "methods to maximize cumulative reward.", "terms": [ "reinforcement learning", "q-learning", "deep q-network", "dqn", "policy gradient", "actor-critic", "actor critic", "markov decision process", "reward function", "ppo", "deep rl", ], }, { "name": "Classical ML", "anchor": "Machine learning with classical models [SEP] We use random " "forests, gradient boosting (XGBoost, LightGBM), support vector " "machines, decision trees, clustering and feature selection to predict " "and classify financial outcomes.", "terms": [ "random forest", "gradient boosting", "xgboost", "lightgbm", "support vector", "svm", "decision tree", "k-means", "clustering", "k-nearest", "knn", "ensemble learning", "feature selection", "naive bayes", "principal component", "pca", ], }, { "name": "NLP & LLMs", "anchor": "Natural language processing of financial text [SEP] We apply " "sentiment analysis, BERT, GPT and large language models to news, " "filings and social media to extract trading signals from text.", "terms": [ "natural language processing", "nlp", "sentiment analysis", "text mining", "bert", "gpt", "large language model", "large language models", "llm", "llms", "word embedding", "topic model", "news sentiment", "financial text", "language model", ], }, { "name": "Stochastic Processes & SDEs", "anchor": "Continuous-time stochastic modeling [SEP] We model asset " "dynamics with stochastic differential equations, stochastic volatility " "(Heston, rough volatility), jump-diffusion, Levy processes and Hawkes " "processes.", "terms": [ "stochastic differential equation", "sde", "stochastic volatility", "heston", "rough volatility", "jump diffusion", "jump-diffusion", "levy process", "brownian motion", "ornstein-uhlenbeck", "geometric brownian", "fractional brownian", "hawkes process", "stochastic process", ], }, { "name": "PDE & Numerical Methods", "anchor": "Numerical methods for pricing and stochastic control [SEP] We " "solve partial differential equations and backward stochastic " "differential equations with finite-difference, finite-element, deep " "BSDE, deep Galerkin and physics-informed neural network schemes.", "terms": [ "partial differential equation", "pde", "finite difference", "finite element", "deep bsde", "bsde", "physics-informed", "pinn", "deep galerkin", "numerical scheme", "free boundary", "hamilton-jacobi", "hjb", "lattice method", "tree method", ], }, { "name": "Time-Series Econometrics", "anchor": "Time-series econometric modeling [SEP] We estimate GARCH, " "ARIMA, ARMA, vector autoregression (VAR) and cointegration models to " "capture volatility dynamics, realized volatility and serial dependence.", "terms": [ "garch", "arch model", "arima", "arma", "vector autoregression", "var model", "cointegration", "egarch", "realized volatility", "unit root", "heteroskedasticity", "ewma", "volatility model", ], }, { "name": "Panel & Causal Econometrics", "anchor": "Causal inference with panel data [SEP] We use fixed effects, " "difference-in-differences, instrumental variables, two-stage least " "squares and regression discontinuity to identify treatment effects in " "economics and finance.", "terms": [ "panel data", "fixed effect", "fixed effects", "random effects", "difference-in-differences", "difference in differences", "instrumental variable", "two-stage least squares", "2sls", "regression discontinuity", "propensity score", "causal inference", "natural experiment", "ppml", "treatment effect", ], }, { "name": "Bayesian & Filtering", "anchor": "Bayesian inference and filtering [SEP] We perform posterior " "estimation with MCMC, Gibbs sampling, variational inference, particle " "filters and Kalman filtering for latent states in financial models.", "terms": [ "bayesian", "mcmc", "markov chain monte carlo", "gibbs sampling", "particle filter", "sequential monte carlo", "posterior distribution", "prior distribution", "variational inference", "hidden markov", "kalman filter", "state estimation", ], }, { "name": "Optimization & Portfolio", "anchor": "Portfolio optimization and allocation [SEP] We solve " "mean-variance (Markowitz), convex, quadratic and robust optimization " "problems to construct portfolios and allocate capital under " "constraints.", "terms": [ "portfolio optimization", "mean-variance", "mean variance", "markowitz", "convex optimization", "quadratic programming", "robust optimization", "efficient frontier", "risk parity", "utility maximization", "linear programming", "stochastic optimization", "asset allocation", ], }, { "name": "Monte Carlo & Simulation", "anchor": "Monte Carlo simulation [SEP] We use Monte Carlo and " "quasi-Monte Carlo simulation, scenario generation, path simulation and " "bootstrap resampling to estimate prices, risk measures and " "distributions.", "terms": [ "monte carlo", "quasi-monte carlo", "scenario generation", "bootstrap", "resampling", "nested simulation", "path simulation", "simulation study", ], }, { "name": "Networks & Graphs", "anchor": "Network and graph analysis of financial systems [SEP] We " "study interconnectedness, contagion and systemic risk using complex " "networks, centrality measures, correlation networks and graph topology.", "terms": [ "financial network", "systemic risk network", "network analysis", "graph theory", "centrality", "contagion", "interconnectedness", "network topology", "complex network", "correlation network", "minimum spanning tree", ], }, { "name": "Factor Models & Asset Pricing", "anchor": "Empirical asset pricing with factor models [SEP] We study the " "cross-section of returns, risk premia and anomalies using Fama-French " "factors, the CAPM, the arbitrage pricing theory and stochastic discount " "factors.", "terms": [ "factor model", "fama-french", "fama french", "asset pricing", "cross-section of returns", "cross-sectional returns", "risk premium", "risk premia", "capm", "arbitrage pricing", "multi-factor", "stochastic discount factor", "characteristic-sorted", ], }, { "name": "Copulas, Dependence & EVT", "anchor": "Modeling dependence and tail risk [SEP] We use copulas, vine " "copulas and extreme value theory to capture tail dependence and the " "joint distribution of financial returns.", "terms": [ "copula", "vine copula", "tail dependence", "extreme value theory", "extreme value", "generalized pareto", "peaks over threshold", "dependence structure", ], }, { "name": "Agent-Based & Microstructure", "anchor": "Agent-based modeling of markets [SEP] We simulate " "heterogeneous agents and the limit order book to study market " "microstructure, order flow, market making and price formation.", "terms": [ "agent-based model", "agent based model", "multi-agent", "limit order book", "market microstructure", "order flow", "market making", "heterogeneous agents", "artificial market", ], }, ] FAMILIES: list[str] = [t["name"] for t in TAXONOMY] # ------------------------------------------------------------------ # Lexico compilado # ------------------------------------------------------------------ def _compile_lexicons() -> dict[str, re.Pattern]: """Un regex por familia: \\b(term1|term2|...)\\b, case-insensitive.""" out: dict[str, re.Pattern] = {} for t in TAXONOMY: # ordenar por largo desc para que el alternation prefiera el match largo terms = sorted(t["terms"], key=len, reverse=True) pat = r"\b(" + "|".join(re.escape(x) for x in terms) + r")\b" out[t["name"]] = re.compile(pat, re.IGNORECASE) return out LEXICONS = _compile_lexicons() def lexicon_hits(text: str) -> dict[str, int]: """Cuenta matches del lexico de cada familia en el texto.""" if not text: return {f: 0 for f in FAMILIES} return {f: len(LEXICONS[f].findall(text)) for f in FAMILIES} # ------------------------------------------------------------------ # Anclas (requiere el modelo SPECTER) # ------------------------------------------------------------------ def build_anchor_matrix(model) -> np.ndarray: """ Embebe los textos-ancla con SPECTER. Output (n_fam, dim), L2-normalizado. `model` es un embeddings.EmbeddingModel. """ anchors = [t["anchor"] for t in TAXONOMY] return model.encode(anchors, show_progress=False) # ------------------------------------------------------------------ # Clasificacion (compute puro, sin DB) # ------------------------------------------------------------------ @dataclass class PaperTags: arxiv_id: str tags: list[dict] # [{"family","sem","z","lex"}], ordenado por confianza def _assign(sims_row: np.ndarray, z_row: np.ndarray, lex: dict[str, int]) -> list[dict]: """ Regla hibrida lexico-primario + semantica z-scoreada para un paper. - familia con match de lexico (y z no muy atipico) -> tag de alta confianza - familia sin lexico pero con z >= TAU_Z -> tag semantico (capta parafrasis) - ranking: lexico primero (por #hits, luego z), luego semanticos por z - fallback: si nada pasa, top-1 por z si z >= TAU_FALLBACK_Z """ lex_cand: list[tuple[str, float, float, int]] = [] # (fam, sem, z, lx) sem_cand: list[tuple[str, float, float, int]] = [] for j, fam in enumerate(FAMILIES): sem = float(sims_row[j]) z = float(z_row[j]) lx = lex.get(fam, 0) if lx >= 1 and z >= LEX_Z_FLOOR: lex_cand.append((fam, sem, z, lx)) elif lx == 0 and z >= TAU_Z: sem_cand.append((fam, sem, z, lx)) lex_cand.sort(key=lambda c: (-c[3], -c[2])) # mas hits, luego mas z sem_cand.sort(key=lambda c: -c[2]) # mas z cand = (lex_cand + sem_cand)[:MAX_TAGS] if not cand: j = int(z_row.argmax()) if float(z_row[j]) >= TAU_FALLBACK_Z: cand = [(FAMILIES[j], float(sims_row[j]), float(z_row[j]), lex.get(FAMILIES[j], 0))] return [{"family": f, "sem": round(s, 3), "z": round(zz, 2), "lex": int(l)} for f, s, zz, l in cand] def classify_all( ids: list[str], matrix: np.ndarray, # (n_papers, dim) L2-normalizado anchors: np.ndarray, # (n_fam, dim) L2-normalizado texts: dict[str, str], # arxiv_id -> titulo+abstract (para lexico) ) -> list[PaperTags]: """Clasifica todo el corpus en memoria. No toca DB.""" sims = matrix @ anchors.T # (n_papers, n_fam) mu = sims.mean(axis=0) sigma = sims.std(axis=0) + 1e-6 zmat = (sims - mu) / sigma # z-score por familia results: list[PaperTags] = [] for i, aid in enumerate(ids): lex = lexicon_hits(texts.get(aid, "")) tags = _assign(sims[i], zmat[i], lex) results.append(PaperTags(arxiv_id=aid, tags=tags)) return results # ------------------------------------------------------------------ # Pipeline completo: clasifica y persiste # ------------------------------------------------------------------ def _load_texts(conn: sqlite3.Connection) -> dict[str, str]: """Mapa arxiv_id -> 'title. abstract' para el lexico.""" out: dict[str, str] = {} for r in conn.execute( "SELECT arxiv_id, title, abstract, openalex_abstract FROM papers" ): title = r["title"] or "" abstract = r["openalex_abstract"] or r["abstract"] or "" out[r["arxiv_id"]] = f"{title}. {abstract}" return out def classify_corpus( conn: sqlite3.Connection, model, index, # embeddings.EmbeddingIndex commit_every: int = 2000, write: bool = True, ) -> list[PaperTags]: """ Clasifica todos los papers con embedding y persiste methodology_json. Devuelve la lista de PaperTags (util para diagnostico aunque write=False). """ from src import db anchors = build_anchor_matrix(model) texts = _load_texts(conn) results = classify_all(index.ids, index.matrix, anchors, texts) if write: for k, pt in enumerate(results): db.update_methodology(conn, pt.arxiv_id, pt.tags or None) if (k + 1) % commit_every == 0: conn.commit() conn.commit() return results