Spaces:
Sleeping
Sleeping
File size: 4,124 Bytes
e53bbe5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | #!/usr/bin/env python3
"""
Incremental refresh orchestrator. Runs the full pipeline in order, using
--since for papers and --upsert for trials to avoid full re-ingestion.
State is tracked in data/.refresh_state.json. On first run (no state file),
defaults to fetching papers since 2024-12-31 (end of the initial corpus window).
Usage:
uv run python scripts/refresh.py # incremental from last state
uv run python scripts/refresh.py --since 2025-01-01
uv run python scripts/refresh.py --dry-run
uv run python scripts/refresh.py --skip-papers # trials + graph only
uv run python scripts/refresh.py --full-rebuild # ignore state, full re-ingest
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from datetime import date
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
from dotenv import load_dotenv
load_dotenv()
from rich.console import Console
from config import REFRESH_STATE_PATH
console = Console()
_SCRIPTS_DIR = Path(__file__).parent
_DEFAULT_SINCE = "2024-12-31"
def _load_state() -> dict:
if REFRESH_STATE_PATH.exists():
return json.loads(REFRESH_STATE_PATH.read_text())
return {}
def _save_state(state: dict) -> None:
REFRESH_STATE_PATH.parent.mkdir(parents=True, exist_ok=True)
REFRESH_STATE_PATH.write_text(json.dumps(state, indent=2))
def _run(script: str, *args: str, dry_run: bool = False) -> None:
cmd = [sys.executable, str(_SCRIPTS_DIR / script), *args]
console.print(f"[cyan]{'(dry-run) ' if dry_run else ''}Running:[/cyan] {' '.join(cmd)}")
if dry_run:
return
result = subprocess.run(cmd, check=False)
if result.returncode != 0:
console.print(f"[red]Script {script} exited with code {result.returncode}[/red]")
sys.exit(result.returncode)
def main() -> None:
parser = argparse.ArgumentParser(description="Incremental pipeline refresh")
parser.add_argument("--since", metavar="YYYY-MM-DD", help="Override since-date for paper ingestion")
parser.add_argument("--dry-run", action="store_true", help="Print what would run without executing")
parser.add_argument("--skip-papers", action="store_true", help="Skip paper ingestion (trials + graph only)")
parser.add_argument("--full-rebuild", action="store_true", help="Full re-ingest, ignoring state")
args = parser.parse_args()
state = _load_state()
today = str(date.today())
if args.full_rebuild:
console.print("[yellow]Full rebuild requested — ignoring refresh state[/yellow]")
since_date = None
elif args.since:
since_date = args.since
else:
since_date = state.get("last_papers_ingest", _DEFAULT_SINCE)
console.print(f"[bold]Candle-fire incremental refresh[/bold] — {today}")
if not args.full_rebuild and not args.skip_papers:
console.print(f" Fetching papers since: {since_date}")
# 1. Papers
if not args.skip_papers:
if args.full_rebuild or since_date is None:
_run("ingest_papers.py", "--skip-fulltext", "--skip-citations", dry_run=args.dry_run)
else:
_run("ingest_papers.py", "--since", since_date, "--skip-fulltext", "--skip-citations", dry_run=args.dry_run)
# 2. Trials
_run("ingest_trials.py", "--upsert", dry_run=args.dry_run)
# 3. Entity extraction (auto-resumes via .progress.json)
_run("extract_entities.py", dry_run=args.dry_run)
# 4. Derive seeds
_run("derive_seeds.py", dry_run=args.dry_run)
# 5. Build graph
_run("build_graph.py", dry_run=args.dry_run)
# 6. Build index (idempotent — skips existing chunks)
_run("build_index.py", dry_run=args.dry_run)
if not args.dry_run:
new_state = {**state, "last_papers_ingest": today, "last_trials_ingest": today}
_save_state(new_state)
console.print(f"\n[bold green]Refresh complete.[/bold green] State written to {REFRESH_STATE_PATH}")
else:
console.print("\n[yellow]Dry run complete — no changes made.[/yellow]")
if __name__ == "__main__":
main()
|