Delete pipeline

pipeline/exporter.py

@@ -1,202 +0,0 @@
-"""pipeline/exporter.py — Generate markdown, MT5 .set, Julia config strings."""
-import re, json
-from datetime import datetime
-
-
-def slugify(t):
-    t = t.lower().strip()
-    t = re.sub(r"[^\w\s-]","",t)
-    t = re.sub(r"[\s_-]+","-",t)
-    return t[:50]
-
-
-def strategy_md(rec):
-    name    = rec.get("name","Unknown")
-    sources = rec.get("sources",[])
-    layers  = rec.get("layers",[])
-    lines   = [f"# {name}", "",
-               f"> **Category:** {rec.get('category','')}  ",
-               f"> **Sources:** {', '.join(sources)[:80]}  ",
-               f"> **ID:** `{rec.get('canonical_id','')}`",
-               "", "---", "", "## Description", "", rec.get("description",""), ""]
-    for label, key in [("## Entry Rules","entry_rules"),("## Exit Rules","exit_rules"),("## Filters","filters")]:
-        items = rec.get(key,[])
-        if items:
-            lines += [label, ""]
-            for i,r in enumerate(items,1): lines.append(f"{i}. {r}")
-            lines.append("")
-    if rec.get("mathematical_basis"):
-        lines += ["## Mathematical Basis", "", rec["mathematical_basis"], ""]
-    if layers:
-        lines += [f"## Variants ({len(layers)} sources)", ""]
-        for i,l in enumerate(layers,1):
-            d = l.get("data",{}); lines.append(f"### Variant {i} — {l.get('source_file','')}")
-            if d.get("description"): lines.append(d["description"])
-            lines.append("")
-    lines += ["---","","*Generated by Quant Knowledge Extractor — Julia Engine*"]
-    return "\n".join(lines)
-
-
-def formula_md(rec):
-    lines = [f"# {rec.get('name','Unknown')}", "",
-             f"> **Category:** {rec.get('category','')}  ",
-             f"> **Sources:** {', '.join(rec.get('sources',[]))[:80]}",
-             "", "---", ""]
-    if rec.get("latex"):
-        lines += ["## Formula (LaTeX)", "", "$$", rec["latex"], "$$", ""]
-    if rec.get("plain_text"):
-        lines += ["## Plain Text", "", "```", rec["plain_text"], "```", ""]
-    if rec.get("purpose"):
-        lines += ["## Purpose", "", rec["purpose"], ""]
-    if rec.get("variables"):
-        lines += ["## Variables", "", "| Symbol | Description |", "|--------|-------------|"]
-        for s,d in rec["variables"].items(): lines.append(f"| `{s}` | {d} |")
-        lines.append("")
-    lines += ["---","","*Generated by Quant Knowledge Extractor — Julia Engine*"]
-    return "\n".join(lines)
-
-
-def backtest_report_md(result: dict, record: dict) -> str:
-    name   = result.get("strategy","Unknown")
-    sym    = result.get("symbol","")
-    tf     = result.get("timeframe","")
-    viable = result.get("is_viable", False)
-    status = "✅ VIABLE" if viable else "❌ NOT VIABLE"
-    now    = datetime.now().strftime("%Y-%m-%d %H:%M UTC")
-    lines  = [
-        f"# {name}", f"## Backtest Report — {sym} {tf}", "",
-        f"> **Status:** {status}  ",
-        f"> **Robustness:** {result.get('robustness',0):.0f}/100  ",
-        f"> **Engine:** Julia 1.10  ",
-        f"> **Generated:** {now}", "", "---", "", "## Viability", "",
-    ]
-    for r in result.get("reasons",[]): lines.append(f"- {r}")
-    lines += [""]
-    if result.get("optimal_params"):
-        lines += ["## Optimal Parameters (Julia Walk-Forward)", "",
-                  "| Parameter | Value |", "|-----------|-------|"]
-        for k,v in result["optimal_params"].items():
-            lines.append(f"| `{k}` | `{v:.4g}` |")
-        lines += [""]
-    lines += [
-        "## OOS Performance (Walk-Forward Aggregate)", "",
-        "| Metric | Value |", "|--------|-------|",
-        f"| Sharpe (mean±std) | `{result.get('oos_sharpe_mean',0):.3f} ± {result.get('oos_sharpe_std',0):.3f}` |",
-        f"| Win Rate | `{result.get('oos_win_rate',0):.1f}%` |",
-        f"| Max Drawdown (mean) | `{result.get('oos_max_dd',0):.2f}%` |",
-        f"| Profit Factor | `{result.get('oos_pf_mean',0):.3f}` |",
-        f"| Total OOS Trades | `{result.get('oos_trades',0)}` |",
-        f"| WF Efficiency Ratio | `{result.get('wf_efficiency',0):.3f}` |",
-        f"| Robustness Score | `{result.get('robustness',0):.0f}/100` |",
-    ]
-    if record.get("description"):
-        lines += ["", "## Strategy Description", "", record["description"]]
-    lines += ["", "---", "", "*Backtested using Julia 1.10 — BacktestEngine.jl + Optimizer.jl*"]
-    return "\n".join(lines)
-
-
-def optimal_json(result: dict, record: dict) -> dict:
-    return {
-        "metadata": {
-            "strategy": result.get("strategy"), "symbol": result.get("symbol"),
-            "timeframe": result.get("timeframe"),
-            "generated": datetime.now().isoformat(),
-            "engine": "Julia 1.10",
-            "is_viable": result.get("is_viable"), "robustness": result.get("robustness"),
-        },
-        "optimal_parameters": result.get("optimal_params",{}),
-        "performance": {
-            "oos_sharpe_mean": round(result.get("oos_sharpe_mean",0),4),
-            "oos_sharpe_std":  round(result.get("oos_sharpe_std",0),4),
-            "oos_win_rate":    round(result.get("oos_win_rate",0),2),
-            "oos_max_dd_pct":  round(result.get("oos_max_dd",0),2),
-            "oos_pf_mean":     round(result.get("oos_pf_mean",0),4),
-            "oos_total_trades":result.get("oos_trades",0),
-            "wf_efficiency":   round(result.get("wf_efficiency",0),4),
-        },
-        "viability": {"is_viable": result.get("is_viable"), "reasons": result.get("reasons",[])},
-    }
-
-
-def mt5_set(result: dict, record: dict) -> str:
-    name   = result.get("strategy","Unknown")
-    sym    = result.get("symbol",""); tf = result.get("timeframe","")
-    params = result.get("optimal_params",{})
-    lines  = [
-        f"; MT5 Strategy Tester Set File — Generated by Julia Engine",
-        f"; Strategy: {name}", f"; Symbol: {sym}  Timeframe: {tf}",
-        f"; OOS Sharpe: {result.get('oos_sharpe_mean',0):.3f}  Score: {result.get('robustness',0):.0f}/100",
-        f"; Viable: {result.get('is_viable', False)}",
-        f";", f"; 1. Open MT5 → View → Strategy Tester",
-        f"; 2. Select Expert Advisor", f"; 3. Inputs → Open → select this file", "",
-    ]
-    for k, v in params.items():
-        mt5k = "".join(w.capitalize() for w in k.replace("-","_").split("_"))
-        try:
-            fv   = float(v); step = max(1.0, abs(fv)*0.1)
-            lines.append(f"{mt5k}={fv:.4g}||{max(0,fv-step*3):.4g}||{step:.4g}||{fv+step*3:.4g}||1")
-        except: lines.append(f"{mt5k}={v}")
-    lines += ["","RiskPercent=1.0||0.5||0.5||3.0||1","LotSize=0.1||0.01||0.01||1.0||1"]
-    return "\n".join(lines)
-
-
-def julia_config(result: dict) -> str:
-    name   = result.get("strategy","Unknown")
-    sym    = result.get("symbol",""); tf = result.get("timeframe","")
-    struct = "".join(w.capitalize() for w in name.split()[:4])
-    params = result.get("optimal_params",{})
-    plines = "\n    ".join(
-        f'{k.lower().replace("-","_")} = {float(v):.6g}'
-        for k,v in params.items()
-    ) or "# no parameters"
-    return f'''# Optimal Parameters — {name}
-# Engine: Julia 1.10 BacktestEngine.jl + Optimizer.jl
-# Symbol: {sym}  Timeframe: {tf}
-# OOS Sharpe: {result.get("oos_sharpe_mean",0):.3f}  Robustness: {result.get("robustness",0):.0f}/100
-# Viable: {result.get("is_viable",False)}
-# Generated: {datetime.now().strftime("%Y-%m-%d")}
-
-function optimal_params()::{struct}Params
-    return {struct}Params(
-        {plines}
-    )
-end
-
-const BACKTEST_METADATA = (
-    strategy   = "{name}",
-    symbol     = "{sym}",
-    timeframe  = "{tf}",
-    engine     = "Julia 1.10",
-    oos_sharpe = {result.get("oos_sharpe_mean",0):.4f},
-    max_dd_pct = {result.get("oos_max_dd",0):.2f},
-    win_rate   = {result.get("oos_win_rate",0):.1f},
-    wf_eff     = {result.get("wf_efficiency",0):.4f},
-    viable     = {str(result.get("is_viable",False)).lower()},
-)
-'''
-
-
-def index_md(all_results: list) -> str:
-    viable = sorted([r for r in all_results if r.get("is_viable")],
-                    key=lambda r: r.get("oos_sharpe_mean",0), reverse=True)
-    not_v  = [r for r in all_results if not r.get("is_viable")]
-    now    = datetime.now().strftime("%Y-%m-%d %H:%M UTC")
-    lines  = [
-        "# Backtest Results Index — Julia Engine", "",
-        f"> Generated: {now}  Engine: Julia 1.10  ",
-        f"> Total: {len(all_results)}  Viable: {len(viable)}", "", "---", "",
-        "## ✅ Viable Strategies (by OOS Sharpe)", "",
-        "| # | Strategy | Symbol | TF | Sharpe | DD% | Win% | PF | Score |",
-        "|---|----------|--------|----|--------|-----|------|----|-------|",
-    ]
-    for i,r in enumerate(viable,1):
-        lines.append(
-            f"| {i} | {r.get('strategy','')[:35]} | {r.get('symbol','')} | {r.get('timeframe','')} | "
-            f"{r.get('oos_sharpe_mean',0):.2f} | {r.get('oos_max_dd',0):.1f} | "
-            f"{r.get('oos_win_rate',0):.1f} | {r.get('oos_pf_mean',0):.2f} | {r.get('robustness',0):.0f} |")
-    lines += ["", "## ❌ Not Viable", "", "| Strategy | Symbol | TF | Reason |", "|----------|--------|----|--------|"]
-    for r in not_v[:30]:
-        reasons = r.get("reasons",[])
-        reason = reasons[0][:60] if reasons else ""
-        lines.append(f"| {r.get('strategy','')[:35]} | {r.get('symbol','')} | {r.get('timeframe','')} | {reason} |")
-    return "\n".join(lines)

pipeline/extractor.py

@@ -1,153 +0,0 @@
-"""pipeline/extractor.py — Claude API extraction + 3-layer deduplication."""
-import json, time, hashlib
-from typing import Optional
-import numpy as np
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
-import anthropic
-from loguru import logger
-import utils.config as cfg
-
-
-class AIExtractor:
-    MODEL = "claude-sonnet-4-20250514"
-    def __init__(self):
-        self.client = anthropic.Anthropic(api_key=cfg.ANTHROPIC_API_KEY)
-        self.tokens_used = 0
-
-    def extract(self, chunk) -> dict:
-        if chunk.word_count < 20:
-            return {"strategies":[],"formulas":[],"systems":[]}
-        prompt = cfg.EXTRACTION_PROMPT.format(
-            source_file=chunk.source_file, page_start=chunk.page_start,
-            page_end=chunk.page_end, text=chunk.text)
-        raw = self._call(prompt)
-        if not raw: return {"strategies":[],"formulas":[],"systems":[]}
-        return self._parse(raw, chunk)
-
-    def _call(self, prompt, retries=3):
-        delay = 2.0
-        for attempt in range(retries):
-            try:
-                resp = self.client.messages.create(
-                    model=self.MODEL, max_tokens=4096,
-                    messages=[{"role":"user","content":prompt}])
-                self.tokens_used += resp.usage.input_tokens + resp.usage.output_tokens
-                return resp.content[0].text if resp.content else ""
-            except anthropic.RateLimitError:
-                logger.warning(f"Rate limit — {delay}s")
-                time.sleep(delay); delay *= 2
-            except Exception as e:
-                logger.error(f"API: {e}")
-                if attempt == retries-1: return ""
-                time.sleep(delay); delay *= 2
-        return ""
-
-    def _parse(self, raw, chunk):
-        raw = raw.strip()
-        if raw.startswith("```"): raw = "\n".join(raw.split("\n")[1:]).rstrip("`").strip()
-        try: data = json.loads(raw)
-        except:
-            try:
-                s=raw.find("{"); e=raw.rfind("}")
-                data=json.loads(raw[s:e+1]) if s!=-1 else {}
-            except: return {"strategies":[],"formulas":[],"systems":[]}
-        result = {}
-        for kind in ("strategies","formulas","systems"):
-            result[kind] = []
-            for item in data.get(kind,[]):
-                if isinstance(item,dict) and item.get("name"):
-                    item.update({"source_file":chunk.source_file,
-                                 "source_pages":f"{chunk.page_start}-{chunk.page_end}"})
-                    item["content_hash"] = _hash(
-                        item.get("description","") + item.get("plain_text","") +
-                        item.get("entry_system","") + item.get("name",""))
-                    result[kind].append(item)
-        return result
-
-    def compile_strategy_code(self, record: dict) -> str:
-        """Ask Claude to generate Julia signal code for this strategy."""
-        compact = {k: record.get(k) for k in
-            ("name","category","description","entry_rules","exit_rules",
-             "filters","parameters","mathematical_basis")}
-        prompt = cfg.COMPILER_PROMPT.format(
-            strategy_json=json.dumps(compact, indent=2))
-        code = self._call(prompt)
-        if not code: return ""
-        if "```" in code:
-            lines = code.split("\n")
-            in_block = False; out = []
-            for line in lines:
-                if line.strip().startswith("```"): in_block = not in_block; continue
-                if in_block: out.append(line)
-            code = "\n".join(out)
-        return code.strip()
-
-
-class Deduplicator:
-    def __init__(self, threshold=None):
-        self.threshold = threshold or cfg.SIMILARITY_THRESHOLD
-        self._vec = TfidfVectorizer(ngram_range=(1,2), max_features=5000, stop_words="english")
-
-    def process(self, extracted, kb):
-        stats = {k:{"added":0,"merged":0,"skipped":0} for k in ("strategies","formulas","systems")}
-        for kind in ("strategies","formulas","systems"):
-            for item in extracted.get(kind,[]):
-                stats[kind][self._process_one(item, kb[kind], kind)] += 1
-        return stats
-
-    def _process_one(self, item, store, kind):
-        h = item.get("content_hash","")
-        for e in store.values():
-            if e.get("content_hash") == h:
-                self._add_src(item, e); return "skipped"
-        sid = self._similar(item, store, kind)
-        if sid: self._merge(item, store[sid]); return "merged"
-        cid = _cid(item["name"], h, kind)
-        item["canonical_id"] = cid
-        item["sources"]  = [item.get("source_file","")]
-        item["layers"]   = []
-        store[cid]       = item
-        return "added"
-
-    def _similar(self, item, store, kind):
-        if not store: return None
-        texts = [_text(v,kind) for v in store.values()] + [_text(item,kind)]
-        try:
-            mat  = self._vec.fit_transform(texts)
-            sims = cosine_similarity(mat[-1], mat[:-1])[0]
-            idx  = int(np.argmax(sims))
-            if sims[idx] >= self.threshold:
-                return list(store.keys())[idx]
-        except: pass
-        return None
-
-    @staticmethod
-    def _add_src(item, existing):
-        s = item.get("source_file","")
-        if s and s not in existing.get("sources",[]):
-            existing.setdefault("sources",[]).append(s)
-
-    @staticmethod
-    def _merge(item, existing):
-        Deduplicator._add_src(item, existing)
-        layers = existing.setdefault("layers",[])
-        if item.get("content_hash") not in {l.get("content_hash") for l in layers}:
-            layers.append({"source_file":item.get("source_file"),
-                           "content_hash":item.get("content_hash"),
-                           "data":{k:v for k,v in item.items()
-                                   if k not in ("sources","layers","canonical_id")}})
-
-
-def _hash(text):
-    return hashlib.sha256(" ".join(text.lower().split()).encode()).hexdigest()[:16]
-
-def _cid(name, h, kind):
-    return hashlib.md5(f"{kind}_{name}_{h}".encode()).hexdigest()[:12]
-
-def _text(item, kind):
-    if kind=="strategies":
-        return f"{item.get('name','')} {item.get('description','')} {' '.join(item.get('entry_rules',[]))}"
-    if kind=="formulas":
-        return f"{item.get('name','')} {item.get('plain_text','')} {item.get('purpose','')}"
-    return f"{item.get('name','')} {item.get('entry_system','')} {item.get('exit_system','')}"

pipeline/julia_bridge.py

@@ -1,212 +0,0 @@
-"""
-pipeline/julia_bridge.py
-─────────────────────────────────────────────────────
-Python bridge to Julia via juliacall.
-
-Julia is loaded ONCE on first use (lazy init) to avoid
-slowing down Space startup. Subsequent calls are instant.
-
-juliacall converts:
-  Python list/np.ndarray  →  Julia Vector{Float64}
-  Python dict             →  Julia Dict
-  Julia Dict{String,Any}  →  Python dict
-  Julia Vector            →  Python list
-
-All heavy computation — indicators, backtest, optimizer —
-runs in Julia. Python only calls this bridge.
-"""
-
-from __future__ import annotations
-import os
-from pathlib import Path
-from typing import Optional
-import numpy as np
-from loguru import logger
-
-# ── Julia runtime (loaded once) ───────────────────────
-_jl = None
-_QE = None   # QuantEngine module
-
-JULIA_SRC = Path(__file__).parent.parent / "src"
-
-
-def _init_julia():
-    """Lazy-load Julia + QuantEngine on first call."""
-    global _jl, _QE
-    if _jl is not None:
-        return
-
-    logger.info("Initializing Julia runtime…")
-    try:
-        from juliacall import Main as jl
-        _jl = jl
-
-        # Load the QuantEngine module
-        _jl.seval(f'push!(LOAD_PATH, "{JULIA_SRC}")')
-        _jl.seval(f'include("{JULIA_SRC / "QuantEngine.jl"}")')
-        _jl.seval("using .QuantEngine")
-        _QE = _jl.QuantEngine
-        logger.info("Julia QuantEngine loaded ✓")
-    except Exception as e:
-        logger.error(f"Julia init failed: {e}")
-        raise RuntimeError(f"Julia init failed: {e}") from e
-
-
-def _arr(x) -> "jl.Vector":
-    """Convert Python list/ndarray to Julia Vector{Float64}."""
-    _init_julia()
-    import numpy as np
-    arr = np.asarray(x, dtype=np.float64)
-    return _jl.convert(_jl.Vector[_jl.Float64], arr.tolist())
-
-
-def _signals(x) -> "jl.Vector":
-    """Convert signal array to Julia Vector{Int}."""
-    _init_julia()
-    arr = [int(v) for v in x]
-    return _jl.convert(_jl.Vector[_jl.Int], arr)
-
-
-def _pydict(jl_dict) -> dict:
-    """Recursively convert Julia Dict to Python dict."""
-    out = {}
-    for k in jl_dict.keys():
-        v = jl_dict[k]
-        k_py = str(k)
-        if hasattr(v, "keys"):
-            out[k_py] = _pydict(v)
-        elif hasattr(v, "__iter__") and not isinstance(v, str):
-            out[k_py] = list(v)
-        elif isinstance(v, bool):
-            out[k_py] = bool(v)
-        elif hasattr(v, "__float__"):
-            out[k_py] = float(v)
-        elif hasattr(v, "__int__"):
-            out[k_py] = int(v)
-        else:
-            out[k_py] = v
-    return out
-
-
-# ── Public API ────────────────────────────────────────
-
-def full_backtest_pipeline(
-    strategy_code:  str,
-    strategy_name:  str,
-    open_p:         list | np.ndarray,
-    high:           list | np.ndarray,
-    low:            list | np.ndarray,
-    close:          list | np.ndarray,
-    volume:         list | np.ndarray,
-    timeframe:      str,
-    symbol:         str,
-    n_windows:      int   = 5,
-    is_ratio:       float = 0.70,
-    min_trades:     int   = 30,
-    min_sharpe:     float = 0.5,
-    max_combos:     int   = 300,
-    initial_equity: float = 10_000.0,
-    commission_pct: float = 0.0002,
-    risk_per_trade: float = 0.01,
-) -> dict:
-    """
-    Run full Julia backtest pipeline.
-    Compile strategy code → walk-forward optimize → return results dict.
-    """
-    _init_julia()
-    try:
-        result = _QE.full_backtest_pipeline(
-            strategy_code, strategy_name,
-            _arr(open_p), _arr(high), _arr(low),
-            _arr(close),  _arr(volume),
-            timeframe, symbol,
-            n_windows      = n_windows,
-            is_ratio       = is_ratio,
-            min_trades     = min_trades,
-            min_sharpe     = min_sharpe,
-            max_combos     = max_combos,
-            initial_equity = initial_equity,
-            commission_pct = commission_pct,
-            risk_per_trade = risk_per_trade,
-        )
-        return _pydict(result)
-    except Exception as e:
-        logger.error(f"Julia pipeline error [{strategy_name}]: {e}")
-        return {
-            "is_valid": False,
-            "error": str(e),
-            "strategy": strategy_name,
-            "symbol": symbol,
-            "timeframe": timeframe,
-        }
-
-
-def run_backtest_only(
-    open_p: np.ndarray, high: np.ndarray, low: np.ndarray,
-    close: np.ndarray, volume: np.ndarray,
-    signals: list | np.ndarray,
-    timeframe: str = "1h",
-    initial_equity: float = 10_000.0,
-    commission_pct: float = 0.0002,
-    risk_per_trade: float = 0.01,
-) -> dict:
-    """Run a single backtest with pre-computed signals."""
-    _init_julia()
-    try:
-        cfg = _QE.BacktestConfig(
-            initial_equity=initial_equity,
-            commission_pct=commission_pct,
-            risk_per_trade=risk_per_trade,
-        )
-        r = _QE.run_backtest(
-            _arr(open_p), _arr(high), _arr(low),
-            _arr(close),  _arr(volume),
-            _signals(signals), timeframe, cfg,
-        )
-        return {
-            "total_return":  float(r.total_return),
-            "cagr":          float(r.cagr),
-            "sharpe":        float(r.sharpe),
-            "sortino":       float(r.sortino),
-            "calmar":        float(r.calmar),
-            "max_dd":        float(r.max_dd),
-            "n_trades":      int(r.n_trades),
-            "win_rate":      float(r.win_rate),
-            "profit_factor": float(r.profit_factor),
-            "final_equity":  float(r.final_equity),
-            "equity_curve":  list(r.equity_curve),
-            "is_valid":      bool(r.is_valid),
-            "error":         str(r.error_msg),
-        }
-    except Exception as e:
-        return {"is_valid": False, "error": str(e)}
-
-
-def indicator(name: str, *args, **kwargs):
-    """
-    Call any indicator by name from Python.
-    Returns numpy array.
-
-    Example:
-        rsi_values = indicator("rsi", close_array, 14)
-        upper, mid, lower = indicator("bbands", close_array, 20, 2.0)
-    """
-    _init_julia()
-    fn = getattr(_QE, name, None)
-    if fn is None:
-        raise ValueError(f"Unknown indicator: {name}")
-    jl_args = [_arr(a) if isinstance(a, (list, np.ndarray)) else a
-               for a in args]
-    result = fn(*jl_args, **kwargs)
-    if isinstance(result, tuple):
-        return tuple(np.array(list(r)) for r in result)
-    return np.array(list(result))
-
-
-def julia_available() -> bool:
-    """Check if Julia runtime is available."""
-    try:
-        _init_julia()
-        return _QE is not None
-    except Exception:
-        return False

pipeline/pdf_processor.py

@@ -1,88 +0,0 @@
-"""pipeline/pdf_processor.py — PDF text extraction + OCR fallback."""
-import re
-from pathlib import Path
-from typing import Generator
-from dataclasses import dataclass
-import fitz, pdfplumber, pytesseract
-from pdf2image import convert_from_path
-from loguru import logger
-import utils.config as cfg
-
-
-@dataclass
-class Chunk:
-    source_file: str
-    page_start:  int
-    page_end:    int
-    text:        str
-    was_ocr:     bool = False
-
-    @property
-    def word_count(self): return len(self.text.split())
-
-
-class PDFProcessor:
-    MIN_CHARS   = 80
-    CHUNK_WORDS = cfg.MAX_TOKENS_PER_CHUNK // 2
-
-    def process(self, pdf_path: Path) -> Generator[Chunk, None, None]:
-        pages = self._extract_pages(pdf_path)
-        yield from self._chunk(pages, pdf_path.name)
-
-    def _extract_pages(self, path):
-        mu = {}
-        try:
-            doc = fitz.open(str(path))
-            for i, pg in enumerate(doc): mu[i+1] = self._clean(pg.get_text("text"))
-            doc.close()
-        except: pass
-        pl = {}
-        try:
-            with pdfplumber.open(str(path)) as pdf:
-                for i, pg in enumerate(pdf.pages):
-                    try: pl[i+1] = self._clean(pg.extract_text() or "")
-                    except: pl[i+1] = ""
-        except: pass
-        total = max(len(mu), len(pl), 1)
-        results = []; ocr_needed = []
-        for pnum in range(1, total+1):
-            best = mu.get(pnum,"") if len(mu.get(pnum,"")) > len(pl.get(pnum,"")) else pl.get(pnum,"")
-            if len(best) >= self.MIN_CHARS: results.append((pnum, best, False))
-            else: results.append((pnum, best, False)); ocr_needed.append(pnum)
-        if ocr_needed:
-            ocr = self._ocr(path, ocr_needed)
-            for i,(pnum,_,_) in enumerate(results):
-                if pnum in ocr: results[i] = (pnum, ocr[pnum], True)
-        return results
-
-    def _ocr(self, path, pages):
-        out = {}
-        try:
-            imgs = convert_from_path(str(path), dpi=cfg.OCR_DPI,
-                first_page=min(pages), last_page=max(pages))
-            for i, pnum in enumerate(range(min(pages), max(pages)+1)):
-                if pnum in pages and i < len(imgs):
-                    out[pnum] = self._clean(pytesseract.image_to_string(imgs[i],lang="eng",config="--psm 6"))
-        except Exception as e: logger.warning(f"OCR: {e}")
-        return out
-
-    def _chunk(self, pages, source):
-        buf, words, p_start, any_ocr = [], 0, 1, False
-        for pnum, text, ocr in pages:
-            if not text: continue
-            buf.append(text); words += len(text.split())
-            if ocr: any_ocr = True
-            if words >= self.CHUNK_WORDS:
-                yield Chunk(source, p_start, pnum, "\n\n".join(buf), any_ocr)
-                buf, words, p_start, any_ocr = [text], len(text.split()), pnum, ocr
-        if buf:
-            last = pages[-1][0] if pages else p_start
-            yield Chunk(source, p_start, last, "\n\n".join(buf), any_ocr)
-
-    @staticmethod
-    def _clean(text):
-        if not text: return ""
-        text = re.sub(r"(\w)-\n(\w)", r"\1\2", text)
-        text = re.sub(r"\n{3,}", "\n\n", text)
-        text = re.sub(r"[ \t]+", " ", text)
-        return text.strip()