Spaces:

PabloLE
/

Busqueda-Arxiv

Sleeping

Update: deteccion de metodologias (UI + columna methodology_json + DB refrescada)

739cf68 verified 5 months ago

16.9 kB

"""
Clasificacion de papers por METODOLOGIA (la tecnica que usa el paper).
Enfoque HIBRIDO (lexico + SPECTER):
  - Cada familia metodologica tiene (a) un texto-ancla estilo paper que se
    embebe con SPECTER y (b) un lexico de terminos regex.
  - Para cada paper calculamos:
        sem_f  = cos(v_paper, v_ancla_f)        # similitud semantica
        lex_f  = #matches del lexico de f en titulo+abstract
  - Regla de asignacion (multi-tag, hasta MAX_TAGS):
        se asigna la familia f si
            sem_f >= TAU_HIGH  y  sem_f >= s_max - MARGIN     (semantico fuerte)
          o lex_f >= 1         y  sem_f >= TAU_LEX_FLOOR       (lexico + piso semantico)
        fallback: si nada pasa pero s_max >= TAU_FALLBACK, se toma el top-1.
  El lexico aporta precision (menciones explicitas) y el piso semantico evita
  falsos positivos del lexico (ej. "we do NOT use GARCH" en un paper no-GARCH
  igual tendria sem bajo y no pasaria el piso).
La similitud se calcula con producto punto porque los vectores ya estan
normalizados L2 (cos == dot). Para todo el corpus: matrix @ anchors.T.
"""

from __future__ import annotations

import re
import sqlite3
from dataclasses import dataclass

import numpy as np

# ------------------------------------------------------------------
# Umbrales (calibrables; ver scripts/classify_methodology.py --dry-run)
#
# NOTA: SPECTER comprime todo el espacio "paper de finanzas", asi que la
# similitud ABSOLUTA a las anclas es alta para todas las familias y no
# discrimina. Por eso trabajamos con z-scores POR FAMILIA:
#     z_ij = (sim_ij - mu_j) / sigma_j
# o sea, cuanto MAS se parece el paper i a la metodologia j que el paper
# tipico. Esto elimina el sesgo de anclas centrales y balancea frecuencias.
# ------------------------------------------------------------------
TAU_Z = 1.50           # semantico: familia destaca (z alto) sin necesidad de lexico
LEX_Z_FLOOR = -0.25    # un match de lexico vale, salvo que el paper sea muy atipico
TAU_FALLBACK_Z = 0.80  # si nada pasa, toma el top-1 por z solo si supera esto
MAX_TAGS = 3


# ------------------------------------------------------------------
# Taxonomia: 15 familias. anchor = pseudo-paper (title [SEP] abstract).
# ------------------------------------------------------------------
TAXONOMY: list[dict] = [
    {
        "name": "Deep Learning",
        "anchor": "Deep learning for financial prediction [SEP] We apply deep "
        "neural networks including LSTM, GRU, CNN, transformers, attention "
        "mechanisms, autoencoders and graph neural networks to learn "
        "representations and forecast asset prices and financial time series.",
        "terms": [
            "deep learning", "neural network", "neural networks", "lstm", "gru",
            "recurrent neural", "convolutional", "cnn", "transformer",
            "attention mechanism", "autoencoder", "deep neural", "feedforward",
            "mlp", "resnet", "graph neural", "gnn", "representation learning",
            "deep model",
        ],
    },
    {
        "name": "Reinforcement Learning",
        "anchor": "Reinforcement learning for trading and execution [SEP] We "
        "formulate the problem as a Markov decision process and train agents "
        "with Q-learning, deep Q-networks, policy gradient and actor-critic "
        "methods to maximize cumulative reward.",
        "terms": [
            "reinforcement learning", "q-learning", "deep q-network", "dqn",
            "policy gradient", "actor-critic", "actor critic",
            "markov decision process", "reward function", "ppo", "deep rl",
        ],
    },
    {
        "name": "Classical ML",
        "anchor": "Machine learning with classical models [SEP] We use random "
        "forests, gradient boosting (XGBoost, LightGBM), support vector "
        "machines, decision trees, clustering and feature selection to predict "
        "and classify financial outcomes.",
        "terms": [
            "random forest", "gradient boosting", "xgboost", "lightgbm",
            "support vector", "svm", "decision tree", "k-means", "clustering",
            "k-nearest", "knn", "ensemble learning", "feature selection",
            "naive bayes", "principal component", "pca",
        ],
    },
    {
        "name": "NLP & LLMs",
        "anchor": "Natural language processing of financial text [SEP] We apply "
        "sentiment analysis, BERT, GPT and large language models to news, "
        "filings and social media to extract trading signals from text.",
        "terms": [
            "natural language processing", "nlp", "sentiment analysis",
            "text mining", "bert", "gpt", "large language model",
            "large language models", "llm", "llms", "word embedding",
            "topic model", "news sentiment", "financial text", "language model",
        ],
    },
    {
        "name": "Stochastic Processes & SDEs",
        "anchor": "Continuous-time stochastic modeling [SEP] We model asset "
        "dynamics with stochastic differential equations, stochastic volatility "
        "(Heston, rough volatility), jump-diffusion, Levy processes and Hawkes "
        "processes.",
        "terms": [
            "stochastic differential equation", "sde", "stochastic volatility",
            "heston", "rough volatility", "jump diffusion", "jump-diffusion",
            "levy process", "brownian motion", "ornstein-uhlenbeck",
            "geometric brownian", "fractional brownian", "hawkes process",
            "stochastic process",
        ],
    },
    {
        "name": "PDE & Numerical Methods",
        "anchor": "Numerical methods for pricing and stochastic control [SEP] We "
        "solve partial differential equations and backward stochastic "
        "differential equations with finite-difference, finite-element, deep "
        "BSDE, deep Galerkin and physics-informed neural network schemes.",
        "terms": [
            "partial differential equation", "pde", "finite difference",
            "finite element", "deep bsde", "bsde", "physics-informed", "pinn",
            "deep galerkin", "numerical scheme", "free boundary",
            "hamilton-jacobi", "hjb", "lattice method", "tree method",
        ],
    },
    {
        "name": "Time-Series Econometrics",
        "anchor": "Time-series econometric modeling [SEP] We estimate GARCH, "
        "ARIMA, ARMA, vector autoregression (VAR) and cointegration models to "
        "capture volatility dynamics, realized volatility and serial dependence.",
        "terms": [
            "garch", "arch model", "arima", "arma", "vector autoregression",
            "var model", "cointegration", "egarch", "realized volatility",
            "unit root", "heteroskedasticity", "ewma", "volatility model",
        ],
    },
    {
        "name": "Panel & Causal Econometrics",
        "anchor": "Causal inference with panel data [SEP] We use fixed effects, "
        "difference-in-differences, instrumental variables, two-stage least "
        "squares and regression discontinuity to identify treatment effects in "
        "economics and finance.",
        "terms": [
            "panel data", "fixed effect", "fixed effects", "random effects",
            "difference-in-differences", "difference in differences",
            "instrumental variable", "two-stage least squares", "2sls",
            "regression discontinuity", "propensity score", "causal inference",
            "natural experiment", "ppml", "treatment effect",
        ],
    },
    {
        "name": "Bayesian & Filtering",
        "anchor": "Bayesian inference and filtering [SEP] We perform posterior "
        "estimation with MCMC, Gibbs sampling, variational inference, particle "
        "filters and Kalman filtering for latent states in financial models.",
        "terms": [
            "bayesian", "mcmc", "markov chain monte carlo", "gibbs sampling",
            "particle filter", "sequential monte carlo", "posterior distribution",
            "prior distribution", "variational inference", "hidden markov",
            "kalman filter", "state estimation",
        ],
    },
    {
        "name": "Optimization & Portfolio",
        "anchor": "Portfolio optimization and allocation [SEP] We solve "
        "mean-variance (Markowitz), convex, quadratic and robust optimization "
        "problems to construct portfolios and allocate capital under "
        "constraints.",
        "terms": [
            "portfolio optimization", "mean-variance", "mean variance",
            "markowitz", "convex optimization", "quadratic programming",
            "robust optimization", "efficient frontier", "risk parity",
            "utility maximization", "linear programming", "stochastic optimization",
            "asset allocation",
        ],
    },
    {
        "name": "Monte Carlo & Simulation",
        "anchor": "Monte Carlo simulation [SEP] We use Monte Carlo and "
        "quasi-Monte Carlo simulation, scenario generation, path simulation and "
        "bootstrap resampling to estimate prices, risk measures and "
        "distributions.",
        "terms": [
            "monte carlo", "quasi-monte carlo", "scenario generation",
            "bootstrap", "resampling", "nested simulation", "path simulation",
            "simulation study",
        ],
    },
    {
        "name": "Networks & Graphs",
        "anchor": "Network and graph analysis of financial systems [SEP] We "
        "study interconnectedness, contagion and systemic risk using complex "
        "networks, centrality measures, correlation networks and graph topology.",
        "terms": [
            "financial network", "systemic risk network", "network analysis",
            "graph theory", "centrality", "contagion", "interconnectedness",
            "network topology", "complex network", "correlation network",
            "minimum spanning tree",
        ],
    },
    {
        "name": "Factor Models & Asset Pricing",
        "anchor": "Empirical asset pricing with factor models [SEP] We study the "
        "cross-section of returns, risk premia and anomalies using Fama-French "
        "factors, the CAPM, the arbitrage pricing theory and stochastic discount "
        "factors.",
        "terms": [
            "factor model", "fama-french", "fama french", "asset pricing",
            "cross-section of returns", "cross-sectional returns", "risk premium",
            "risk premia", "capm", "arbitrage pricing", "multi-factor",
            "stochastic discount factor", "characteristic-sorted",
        ],
    },
    {
        "name": "Copulas, Dependence & EVT",
        "anchor": "Modeling dependence and tail risk [SEP] We use copulas, vine "
        "copulas and extreme value theory to capture tail dependence and the "
        "joint distribution of financial returns.",
        "terms": [
            "copula", "vine copula", "tail dependence", "extreme value theory",
            "extreme value", "generalized pareto", "peaks over threshold",
            "dependence structure",
        ],
    },
    {
        "name": "Agent-Based & Microstructure",
        "anchor": "Agent-based modeling of markets [SEP] We simulate "
        "heterogeneous agents and the limit order book to study market "
        "microstructure, order flow, market making and price formation.",
        "terms": [
            "agent-based model", "agent based model", "multi-agent",
            "limit order book", "market microstructure", "order flow",
            "market making", "heterogeneous agents", "artificial market",
        ],
    },
]

FAMILIES: list[str] = [t["name"] for t in TAXONOMY]


# ------------------------------------------------------------------
# Lexico compilado
# ------------------------------------------------------------------

def _compile_lexicons() -> dict[str, re.Pattern]:
    """Un regex por familia: \\b(term1|term2|...)\\b, case-insensitive."""
    out: dict[str, re.Pattern] = {}
    for t in TAXONOMY:
        # ordenar por largo desc para que el alternation prefiera el match largo
        terms = sorted(t["terms"], key=len, reverse=True)
        pat = r"\b(" + "|".join(re.escape(x) for x in terms) + r")\b"
        out[t["name"]] = re.compile(pat, re.IGNORECASE)
    return out


LEXICONS = _compile_lexicons()


def lexicon_hits(text: str) -> dict[str, int]:
    """Cuenta matches del lexico de cada familia en el texto."""
    if not text:
        return {f: 0 for f in FAMILIES}
    return {f: len(LEXICONS[f].findall(text)) for f in FAMILIES}


# ------------------------------------------------------------------
# Anclas (requiere el modelo SPECTER)
# ------------------------------------------------------------------

def build_anchor_matrix(model) -> np.ndarray:
    """
    Embebe los textos-ancla con SPECTER. Output (n_fam, dim), L2-normalizado.
    `model` es un embeddings.EmbeddingModel.
    """
    anchors = [t["anchor"] for t in TAXONOMY]
    return model.encode(anchors, show_progress=False)


# ------------------------------------------------------------------
# Clasificacion (compute puro, sin DB)
# ------------------------------------------------------------------

@dataclass
class PaperTags:
    arxiv_id: str
    tags: list[dict]   # [{"family","sem","z","lex"}], ordenado por confianza


def _assign(sims_row: np.ndarray, z_row: np.ndarray,
            lex: dict[str, int]) -> list[dict]:
    """
    Regla hibrida lexico-primario + semantica z-scoreada para un paper.
      - familia con match de lexico (y z no muy atipico) -> tag de alta confianza
      - familia sin lexico pero con z >= TAU_Z -> tag semantico (capta parafrasis)
      - ranking: lexico primero (por #hits, luego z), luego semanticos por z
      - fallback: si nada pasa, top-1 por z si z >= TAU_FALLBACK_Z
    """
    lex_cand: list[tuple[str, float, float, int]] = []   # (fam, sem, z, lx)
    sem_cand: list[tuple[str, float, float, int]] = []
    for j, fam in enumerate(FAMILIES):
        sem = float(sims_row[j])
        z = float(z_row[j])
        lx = lex.get(fam, 0)
        if lx >= 1 and z >= LEX_Z_FLOOR:
            lex_cand.append((fam, sem, z, lx))
        elif lx == 0 and z >= TAU_Z:
            sem_cand.append((fam, sem, z, lx))
    lex_cand.sort(key=lambda c: (-c[3], -c[2]))   # mas hits, luego mas z
    sem_cand.sort(key=lambda c: -c[2])            # mas z
    cand = (lex_cand + sem_cand)[:MAX_TAGS]
    if not cand:
        j = int(z_row.argmax())
        if float(z_row[j]) >= TAU_FALLBACK_Z:
            cand = [(FAMILIES[j], float(sims_row[j]), float(z_row[j]),
                     lex.get(FAMILIES[j], 0))]
    return [{"family": f, "sem": round(s, 3), "z": round(zz, 2), "lex": int(l)}
            for f, s, zz, l in cand]


def classify_all(
    ids: list[str],
    matrix: np.ndarray,        # (n_papers, dim) L2-normalizado
    anchors: np.ndarray,       # (n_fam, dim) L2-normalizado
    texts: dict[str, str],     # arxiv_id -> titulo+abstract (para lexico)
) -> list[PaperTags]:
    """Clasifica todo el corpus en memoria. No toca DB."""
    sims = matrix @ anchors.T                     # (n_papers, n_fam)
    mu = sims.mean(axis=0)
    sigma = sims.std(axis=0) + 1e-6
    zmat = (sims - mu) / sigma                    # z-score por familia
    results: list[PaperTags] = []
    for i, aid in enumerate(ids):
        lex = lexicon_hits(texts.get(aid, ""))
        tags = _assign(sims[i], zmat[i], lex)
        results.append(PaperTags(arxiv_id=aid, tags=tags))
    return results


# ------------------------------------------------------------------
# Pipeline completo: clasifica y persiste
# ------------------------------------------------------------------

def _load_texts(conn: sqlite3.Connection) -> dict[str, str]:
    """Mapa arxiv_id -> 'title. abstract' para el lexico."""
    out: dict[str, str] = {}
    for r in conn.execute(
        "SELECT arxiv_id, title, abstract, openalex_abstract FROM papers"
    ):
        title = r["title"] or ""
        abstract = r["openalex_abstract"] or r["abstract"] or ""
        out[r["arxiv_id"]] = f"{title}. {abstract}"
    return out


def classify_corpus(
    conn: sqlite3.Connection,
    model,
    index,                     # embeddings.EmbeddingIndex
    commit_every: int = 2000,
    write: bool = True,
) -> list[PaperTags]:
    """
    Clasifica todos los papers con embedding y persiste methodology_json.
    Devuelve la lista de PaperTags (util para diagnostico aunque write=False).
    """
    from src import db

    anchors = build_anchor_matrix(model)
    texts = _load_texts(conn)
    results = classify_all(index.ids, index.matrix, anchors, texts)

    if write:
        for k, pt in enumerate(results):
            db.update_methodology(conn, pt.arxiv_id, pt.tags or None)
            if (k + 1) % commit_every == 0:
                conn.commit()
        conn.commit()
    return results

	"""
	Clasificacion de papers por METODOLOGIA (la tecnica que usa el paper).

	Enfoque HIBRIDO (lexico + SPECTER):
	- Cada familia metodologica tiene (a) un texto-ancla estilo paper que se
	embebe con SPECTER y (b) un lexico de terminos regex.
	- Para cada paper calculamos:
	sem_f = cos(v_paper, v_ancla_f) # similitud semantica
	lex_f = #matches del lexico de f en titulo+abstract
	- Regla de asignacion (multi-tag, hasta MAX_TAGS):
	se asigna la familia f si
	sem_f >= TAU_HIGH y sem_f >= s_max - MARGIN (semantico fuerte)
	o lex_f >= 1 y sem_f >= TAU_LEX_FLOOR (lexico + piso semantico)
	fallback: si nada pasa pero s_max >= TAU_FALLBACK, se toma el top-1.

	El lexico aporta precision (menciones explicitas) y el piso semantico evita
	falsos positivos del lexico (ej. "we do NOT use GARCH" en un paper no-GARCH
	igual tendria sem bajo y no pasaria el piso).

	La similitud se calcula con producto punto porque los vectores ya estan
	normalizados L2 (cos == dot). Para todo el corpus: matrix @ anchors.T.
	"""

	from __future__ import annotations

	import re
	import sqlite3
	from dataclasses import dataclass

	import numpy as np

	# ------------------------------------------------------------------
	# Umbrales (calibrables; ver scripts/classify_methodology.py --dry-run)
	#
	# NOTA: SPECTER comprime todo el espacio "paper de finanzas", asi que la
	# similitud ABSOLUTA a las anclas es alta para todas las familias y no
	# discrimina. Por eso trabajamos con z-scores POR FAMILIA:
	# z_ij = (sim_ij - mu_j) / sigma_j
	# o sea, cuanto MAS se parece el paper i a la metodologia j que el paper
	# tipico. Esto elimina el sesgo de anclas centrales y balancea frecuencias.
	# ------------------------------------------------------------------
	TAU_Z = 1.50 # semantico: familia destaca (z alto) sin necesidad de lexico
	LEX_Z_FLOOR = -0.25 # un match de lexico vale, salvo que el paper sea muy atipico
	TAU_FALLBACK_Z = 0.80 # si nada pasa, toma el top-1 por z solo si supera esto
	MAX_TAGS = 3


	# ------------------------------------------------------------------
	# Taxonomia: 15 familias. anchor = pseudo-paper (title [SEP] abstract).
	# ------------------------------------------------------------------
	TAXONOMY: list[dict] = [
	{
	"name": "Deep Learning",
	"anchor": "Deep learning for financial prediction [SEP] We apply deep "
	"neural networks including LSTM, GRU, CNN, transformers, attention "
	"mechanisms, autoencoders and graph neural networks to learn "
	"representations and forecast asset prices and financial time series.",
	"terms": [
	"deep learning", "neural network", "neural networks", "lstm", "gru",
	"recurrent neural", "convolutional", "cnn", "transformer",
	"attention mechanism", "autoencoder", "deep neural", "feedforward",
	"mlp", "resnet", "graph neural", "gnn", "representation learning",
	"deep model",
	],
	},
	{
	"name": "Reinforcement Learning",
	"anchor": "Reinforcement learning for trading and execution [SEP] We "
	"formulate the problem as a Markov decision process and train agents "
	"with Q-learning, deep Q-networks, policy gradient and actor-critic "
	"methods to maximize cumulative reward.",
	"terms": [
	"reinforcement learning", "q-learning", "deep q-network", "dqn",
	"policy gradient", "actor-critic", "actor critic",
	"markov decision process", "reward function", "ppo", "deep rl",
	],
	},
	{
	"name": "Classical ML",
	"anchor": "Machine learning with classical models [SEP] We use random "
	"forests, gradient boosting (XGBoost, LightGBM), support vector "
	"machines, decision trees, clustering and feature selection to predict "
	"and classify financial outcomes.",
	"terms": [
	"random forest", "gradient boosting", "xgboost", "lightgbm",
	"support vector", "svm", "decision tree", "k-means", "clustering",
	"k-nearest", "knn", "ensemble learning", "feature selection",
	"naive bayes", "principal component", "pca",
	],
	},
	{
	"name": "NLP & LLMs",
	"anchor": "Natural language processing of financial text [SEP] We apply "
	"sentiment analysis, BERT, GPT and large language models to news, "
	"filings and social media to extract trading signals from text.",
	"terms": [
	"natural language processing", "nlp", "sentiment analysis",
	"text mining", "bert", "gpt", "large language model",
	"large language models", "llm", "llms", "word embedding",
	"topic model", "news sentiment", "financial text", "language model",
	],
	},
	{
	"name": "Stochastic Processes & SDEs",
	"anchor": "Continuous-time stochastic modeling [SEP] We model asset "
	"dynamics with stochastic differential equations, stochastic volatility "
	"(Heston, rough volatility), jump-diffusion, Levy processes and Hawkes "
	"processes.",
	"terms": [
	"stochastic differential equation", "sde", "stochastic volatility",
	"heston", "rough volatility", "jump diffusion", "jump-diffusion",
	"levy process", "brownian motion", "ornstein-uhlenbeck",
	"geometric brownian", "fractional brownian", "hawkes process",
	"stochastic process",
	],
	},
	{
	"name": "PDE & Numerical Methods",
	"anchor": "Numerical methods for pricing and stochastic control [SEP] We "
	"solve partial differential equations and backward stochastic "
	"differential equations with finite-difference, finite-element, deep "
	"BSDE, deep Galerkin and physics-informed neural network schemes.",
	"terms": [
	"partial differential equation", "pde", "finite difference",
	"finite element", "deep bsde", "bsde", "physics-informed", "pinn",
	"deep galerkin", "numerical scheme", "free boundary",
	"hamilton-jacobi", "hjb", "lattice method", "tree method",
	],
	},
	{
	"name": "Time-Series Econometrics",
	"anchor": "Time-series econometric modeling [SEP] We estimate GARCH, "
	"ARIMA, ARMA, vector autoregression (VAR) and cointegration models to "
	"capture volatility dynamics, realized volatility and serial dependence.",
	"terms": [
	"garch", "arch model", "arima", "arma", "vector autoregression",
	"var model", "cointegration", "egarch", "realized volatility",
	"unit root", "heteroskedasticity", "ewma", "volatility model",
	],
	},
	{
	"name": "Panel & Causal Econometrics",
	"anchor": "Causal inference with panel data [SEP] We use fixed effects, "
	"difference-in-differences, instrumental variables, two-stage least "
	"squares and regression discontinuity to identify treatment effects in "
	"economics and finance.",
	"terms": [
	"panel data", "fixed effect", "fixed effects", "random effects",
	"difference-in-differences", "difference in differences",
	"instrumental variable", "two-stage least squares", "2sls",
	"regression discontinuity", "propensity score", "causal inference",
	"natural experiment", "ppml", "treatment effect",
	],
	},
	{
	"name": "Bayesian & Filtering",
	"anchor": "Bayesian inference and filtering [SEP] We perform posterior "
	"estimation with MCMC, Gibbs sampling, variational inference, particle "
	"filters and Kalman filtering for latent states in financial models.",
	"terms": [
	"bayesian", "mcmc", "markov chain monte carlo", "gibbs sampling",
	"particle filter", "sequential monte carlo", "posterior distribution",
	"prior distribution", "variational inference", "hidden markov",
	"kalman filter", "state estimation",
	],
	},
	{
	"name": "Optimization & Portfolio",
	"anchor": "Portfolio optimization and allocation [SEP] We solve "
	"mean-variance (Markowitz), convex, quadratic and robust optimization "
	"problems to construct portfolios and allocate capital under "
	"constraints.",
	"terms": [
	"portfolio optimization", "mean-variance", "mean variance",
	"markowitz", "convex optimization", "quadratic programming",
	"robust optimization", "efficient frontier", "risk parity",
	"utility maximization", "linear programming", "stochastic optimization",
	"asset allocation",
	],
	},
	{
	"name": "Monte Carlo & Simulation",
	"anchor": "Monte Carlo simulation [SEP] We use Monte Carlo and "
	"quasi-Monte Carlo simulation, scenario generation, path simulation and "
	"bootstrap resampling to estimate prices, risk measures and "
	"distributions.",
	"terms": [
	"monte carlo", "quasi-monte carlo", "scenario generation",
	"bootstrap", "resampling", "nested simulation", "path simulation",
	"simulation study",
	],
	},
	{
	"name": "Networks & Graphs",
	"anchor": "Network and graph analysis of financial systems [SEP] We "
	"study interconnectedness, contagion and systemic risk using complex "
	"networks, centrality measures, correlation networks and graph topology.",
	"terms": [
	"financial network", "systemic risk network", "network analysis",
	"graph theory", "centrality", "contagion", "interconnectedness",
	"network topology", "complex network", "correlation network",
	"minimum spanning tree",
	],
	},
	{
	"name": "Factor Models & Asset Pricing",
	"anchor": "Empirical asset pricing with factor models [SEP] We study the "
	"cross-section of returns, risk premia and anomalies using Fama-French "
	"factors, the CAPM, the arbitrage pricing theory and stochastic discount "
	"factors.",
	"terms": [
	"factor model", "fama-french", "fama french", "asset pricing",
	"cross-section of returns", "cross-sectional returns", "risk premium",
	"risk premia", "capm", "arbitrage pricing", "multi-factor",
	"stochastic discount factor", "characteristic-sorted",
	],
	},
	{
	"name": "Copulas, Dependence & EVT",
	"anchor": "Modeling dependence and tail risk [SEP] We use copulas, vine "
	"copulas and extreme value theory to capture tail dependence and the "
	"joint distribution of financial returns.",
	"terms": [
	"copula", "vine copula", "tail dependence", "extreme value theory",
	"extreme value", "generalized pareto", "peaks over threshold",
	"dependence structure",
	],
	},
	{
	"name": "Agent-Based & Microstructure",
	"anchor": "Agent-based modeling of markets [SEP] We simulate "
	"heterogeneous agents and the limit order book to study market "
	"microstructure, order flow, market making and price formation.",
	"terms": [
	"agent-based model", "agent based model", "multi-agent",
	"limit order book", "market microstructure", "order flow",
	"market making", "heterogeneous agents", "artificial market",
	],
	},
	]

	FAMILIES: list[str] = [t["name"] for t in TAXONOMY]


	# ------------------------------------------------------------------
	# Lexico compilado
	# ------------------------------------------------------------------

	def _compile_lexicons() -> dict[str, re.Pattern]:
	"""Un regex por familia: \\b(term1\|term2\|...)\\b, case-insensitive."""
	out: dict[str, re.Pattern] = {}
	for t in TAXONOMY:
	# ordenar por largo desc para que el alternation prefiera el match largo
	terms = sorted(t["terms"], key=len, reverse=True)
	pat = r"\b(" + "\|".join(re.escape(x) for x in terms) + r")\b"
	out[t["name"]] = re.compile(pat, re.IGNORECASE)
	return out


	LEXICONS = _compile_lexicons()


	def lexicon_hits(text: str) -> dict[str, int]:
	"""Cuenta matches del lexico de cada familia en el texto."""
	if not text:
	return {f: 0 for f in FAMILIES}
	return {f: len(LEXICONS[f].findall(text)) for f in FAMILIES}


	# ------------------------------------------------------------------
	# Anclas (requiere el modelo SPECTER)
	# ------------------------------------------------------------------

	def build_anchor_matrix(model) -> np.ndarray:
	"""
	Embebe los textos-ancla con SPECTER. Output (n_fam, dim), L2-normalizado.
	`model` es un embeddings.EmbeddingModel.
	"""
	anchors = [t["anchor"] for t in TAXONOMY]
	return model.encode(anchors, show_progress=False)


	# ------------------------------------------------------------------
	# Clasificacion (compute puro, sin DB)
	# ------------------------------------------------------------------

	@dataclass
	class PaperTags:
	arxiv_id: str
	tags: list[dict] # [{"family","sem","z","lex"}], ordenado por confianza


	def _assign(sims_row: np.ndarray, z_row: np.ndarray,
	lex: dict[str, int]) -> list[dict]:
	"""
	Regla hibrida lexico-primario + semantica z-scoreada para un paper.
	- familia con match de lexico (y z no muy atipico) -> tag de alta confianza
	- familia sin lexico pero con z >= TAU_Z -> tag semantico (capta parafrasis)
	- ranking: lexico primero (por #hits, luego z), luego semanticos por z
	- fallback: si nada pasa, top-1 por z si z >= TAU_FALLBACK_Z
	"""
	lex_cand: list[tuple[str, float, float, int]] = [] # (fam, sem, z, lx)
	sem_cand: list[tuple[str, float, float, int]] = []
	for j, fam in enumerate(FAMILIES):
	sem = float(sims_row[j])
	z = float(z_row[j])
	lx = lex.get(fam, 0)
	if lx >= 1 and z >= LEX_Z_FLOOR:
	lex_cand.append((fam, sem, z, lx))
	elif lx == 0 and z >= TAU_Z:
	sem_cand.append((fam, sem, z, lx))
	lex_cand.sort(key=lambda c: (-c[3], -c[2])) # mas hits, luego mas z
	sem_cand.sort(key=lambda c: -c[2]) # mas z
	cand = (lex_cand + sem_cand)[:MAX_TAGS]
	if not cand:
	j = int(z_row.argmax())
	if float(z_row[j]) >= TAU_FALLBACK_Z:
	cand = [(FAMILIES[j], float(sims_row[j]), float(z_row[j]),
	lex.get(FAMILIES[j], 0))]
	return [{"family": f, "sem": round(s, 3), "z": round(zz, 2), "lex": int(l)}
	for f, s, zz, l in cand]


	def classify_all(
	ids: list[str],
	matrix: np.ndarray, # (n_papers, dim) L2-normalizado
	anchors: np.ndarray, # (n_fam, dim) L2-normalizado
	texts: dict[str, str], # arxiv_id -> titulo+abstract (para lexico)
	) -> list[PaperTags]:
	"""Clasifica todo el corpus en memoria. No toca DB."""
	sims = matrix @ anchors.T # (n_papers, n_fam)
	mu = sims.mean(axis=0)
	sigma = sims.std(axis=0) + 1e-6
	zmat = (sims - mu) / sigma # z-score por familia
	results: list[PaperTags] = []
	for i, aid in enumerate(ids):
	lex = lexicon_hits(texts.get(aid, ""))
	tags = _assign(sims[i], zmat[i], lex)
	results.append(PaperTags(arxiv_id=aid, tags=tags))
	return results


	# ------------------------------------------------------------------
	# Pipeline completo: clasifica y persiste
	# ------------------------------------------------------------------

	def _load_texts(conn: sqlite3.Connection) -> dict[str, str]:
	"""Mapa arxiv_id -> 'title. abstract' para el lexico."""
	out: dict[str, str] = {}
	for r in conn.execute(
	"SELECT arxiv_id, title, abstract, openalex_abstract FROM papers"
	):
	title = r["title"] or ""
	abstract = r["openalex_abstract"] or r["abstract"] or ""
	out[r["arxiv_id"]] = f"{title}. {abstract}"
	return out


	def classify_corpus(
	conn: sqlite3.Connection,
	model,
	index, # embeddings.EmbeddingIndex
	commit_every: int = 2000,
	write: bool = True,
	) -> list[PaperTags]:
	"""
	Clasifica todos los papers con embedding y persiste methodology_json.
	Devuelve la lista de PaperTags (util para diagnostico aunque write=False).
	"""
	from src import db

	anchors = build_anchor_matrix(model)
	texts = _load_texts(conn)
	results = classify_all(index.ids, index.matrix, anchors, texts)

	if write:
	for k, pt in enumerate(results):
	db.update_methodology(conn, pt.arxiv_id, pt.tags or None)
	if (k + 1) % commit_every == 0:
	conn.commit()
	conn.commit()
	return results