Anna Zhou
Prepare shared GitHub deployment
f1cc362
Raw History Blame Contribute Delete
12 kB
#!/usr/bin/env python3
"""
fetch_papers.py β€” Download source PDFs for a case study using PyPaperBot.
Tries sources in order: Unpaywall (free, open-access) β†’ Anna's Archive API
(if key provided) β†’ SciDB scrape β†’ Sci-Hub.
Downloaded PDFs are saved to data/raw/<CASE_ID>/ and uploaded to the
corresponding Google Drive folder (using drive_folders.json).
Usage:
# Pass DOIs directly:
python3 -m case_study_pipeline.fetch_papers \\
--case-id CS13 \\
--dois 10.1525/elementa.2021.00036 10.1007/s11069-021-04592-1
# Or read from a dois.txt file (one DOI per line):
python3 -m case_study_pipeline.fetch_papers --case-id CS13
# Skip Drive upload:
python3 -m case_study_pipeline.fetch_papers --case-id CS13 --no-upload
# If automated sources fail, open UWaterloo EZproxy in Chrome and wait
# while you complete WatIAM/Duo and download the PDF:
python3 -m case_study_pipeline.fetch_papers --case-id CS38 --manual-ezproxy
Options:
--scihub-mirror Sci-Hub mirror URL (default: https://sci-hub.se)
--aa-api-key Anna's Archive API key (see annas-archive.se/faq#api)
--no-upload Skip Drive upload after download
--manual-ezproxy Open Chrome for browser-assisted UWaterloo access
"""
from __future__ import annotations
import argparse
import json
import re
import sys
import shutil
import subprocess
import webbrowser
from pathlib import Path
from urllib.parse import quote
REPO_ROOT = Path(__file__).resolve().parent.parent
INBOX_DIR = REPO_ROOT / "data" / "raw" / "_inbox"
# Load .env so UWATERLOO_USER/PASS and GOOGLE credentials are available
try:
from dotenv import load_dotenv
load_dotenv(REPO_ROOT / ".env")
except ImportError:
pass
PIPELINE_DIR = Path(__file__).resolve().parent
DRIVE_FOLDERS_FILE = PIPELINE_DIR / "drive_folders.json"
def load_drive_folders() -> dict:
if not DRIVE_FOLDERS_FILE.exists():
return {}
return json.loads(DRIVE_FOLDERS_FILE.read_text())
def get_dois_for_case(case_dir: Path, dois_arg: list[str] | None) -> list[str]:
"""Return DOIs from --dois arg, or from data/raw/CSX/dois.txt if it exists."""
if dois_arg:
return [d.strip() for d in dois_arg if d.strip()]
dois_file = case_dir / "dois.txt"
if dois_file.exists():
lines = dois_file.read_text().splitlines()
return [l.strip() for l in lines if l.strip() and not l.startswith("#")]
return []
def find_case_dir(case_id: str) -> Path | None:
"""Find data/raw/CSxx or data/raw/CSxx-Place_Name style folders."""
exact = REPO_ROOT / "data" / "raw" / case_id
if exact.exists():
return exact
raw_dir = REPO_ROOT / "data" / "raw"
for path in raw_dir.iterdir():
if path.is_dir() and re.match(rf"^{re.escape(case_id)}($|-)", path.name):
return path
return None
def fetch_and_download(
dois: list[str],
dest_dir: Path,
scihub_mirror: str,
aa_api_key: str | None,
) -> list[Path]:
"""
Run PyPaperBot for each DOI. Downloads go into dest_dir.
Returns list of PDF paths that were successfully downloaded.
"""
try:
from PyPaperBot.Crossref import getPapersInfoFromDOIs
from PyPaperBot.Downloader import downloadPapers
except ImportError:
print(
"ERROR: PyPaperBot not found. Install it with:\n"
" pip3 install bibtexparser crossref-commons undetected-chromedriver\n"
" pip3 install -e ~/PyPaperBot/PyPaperBot"
)
sys.exit(1)
dest_dir.mkdir(parents=True, exist_ok=True)
papers = []
for doi in dois:
print(f"\n Fetching metadata for DOI: {doi}")
paper = getPapersInfoFromDOIs(doi, restrict=None)
papers.append(paper)
before = set(dest_dir.glob("*.pdf"))
dwn_dir = str(dest_dir) + "/"
downloadPapers(
papers,
dwn_dir,
num_limit=None,
SciHub_URL=scihub_mirror,
SciDB_URL=None,
AnnasArchive_API_key=aa_api_key,
)
after = set(dest_dir.glob("*.pdf"))
new_pdfs = sorted(after - before)
return new_pdfs
def safe_pdf_filename(title: str, fallback: str = "downloaded_paper") -> str:
"""Return a filesystem-safe PDF name."""
stem = re.sub(r"[^\w\-_. ]", "_", title).strip(" ._")
return f"{stem or fallback}.pdf"
def display_path(path: Path) -> Path:
"""Prefer repo-relative paths in CLI output."""
try:
return path.relative_to(REPO_ROOT)
except ValueError:
return path
def resolve_doi_target_url(doi: str) -> tuple[str, str]:
"""Return a likely publisher PDF/article URL plus title for manual browser access."""
try:
import requests
url = f"https://api.crossref.org/works/{quote(doi, safe='')}"
resp = requests.get(url, timeout=20, headers={"User-Agent": "rfr-rag-fetch-papers/0.1"})
if resp.ok:
msg = resp.json().get("message", {})
title = (msg.get("title") or [""])[0]
links = msg.get("link") or []
for link in links:
link_url = link.get("URL") or ""
if "pdf" in link_url.lower():
return link_url, title
if msg.get("URL"):
return msg["URL"], title
except Exception as exc:
print(f" [Browser] Crossref lookup failed for {doi}: {exc}")
return f"https://doi.org/{doi}", doi
def to_ezproxy_url(url: str) -> str:
from .ezproxy import to_ezproxy_url as convert
return convert(url)
def open_in_chrome(url: str) -> None:
"""Open URL in Google Chrome, falling back to the default browser."""
try:
subprocess.run(["open", "-a", "Google Chrome", url], check=True)
except Exception:
webbrowser.open(url)
def file_browser_downloads(
*,
case_id: str,
case_dir: Path,
before_inbox: set[Path],
dois: list[str],
titles: dict[str, str],
no_upload: bool,
) -> list[Path]:
"""Move newly downloaded inbox PDFs into the case folder and optionally upload them."""
INBOX_DIR.mkdir(parents=True, exist_ok=True)
after_inbox = set(INBOX_DIR.glob("*.pdf"))
new_downloads = sorted(after_inbox - before_inbox)
if not new_downloads:
print(" [Browser] No new PDFs found in data/raw/_inbox/.")
return []
moved: list[Path] = []
for src in new_downloads:
if len(new_downloads) == 1 and len(dois) == 1:
dest_name = safe_pdf_filename(titles.get(dois[0], ""), src.stem)
else:
dest_name = src.name
dest = case_dir / dest_name
n = 1
while dest.exists():
n += 1
dest = case_dir / f"({n}){dest_name}"
shutil.move(str(src), str(dest))
moved.append(dest)
print(f" [Browser] Moved {src.name} -> {display_path(dest)}")
if moved and not no_upload:
folders = load_drive_folders()
folder_id = folders.get(case_id)
if not folder_id:
print(f" [Drive] No folder ID found for {case_id} in drive_folders.json β€” skipping upload.")
else:
from .drive_upload import make_uploader, try_upload
uploader = make_uploader(folder_id=folder_id)
if uploader:
for pdf in moved:
print(f" [Drive] Uploading {pdf.name}...")
try_upload(uploader, pdf)
else:
print(" [Drive] No uploader available β€” skipping upload.")
elif no_upload:
print(" [Drive] Upload skipped (--no-upload).")
return moved
def browser_ezproxy_flow(case_id: str, case_dir: Path, dois: list[str], no_upload: bool) -> list[Path]:
"""Open failed DOI targets through UWaterloo EZproxy and wait for manual download."""
INBOX_DIR.mkdir(parents=True, exist_ok=True)
before_inbox = set(INBOX_DIR.glob("*.pdf"))
titles: dict[str, str] = {}
print("\n" + "=" * 60)
print(" Browser-assisted UWaterloo EZproxy")
print(" Chrome will open each DOI through EZproxy.")
print(" Complete WatIAM/Duo if asked, then click Download PDF.")
print(f" Save location should be: {INBOX_DIR}")
print("=" * 60 + "\n")
for doi in dois:
target_url, title = resolve_doi_target_url(doi)
titles[doi] = title
proxy_url = to_ezproxy_url(target_url)
print(f" DOI: {doi}")
print(f" Opening: {proxy_url}")
open_in_chrome(proxy_url)
input("\nPress Enter after Chrome has finished downloading the PDF(s) into data/raw/_inbox/...")
return file_browser_downloads(
case_id=case_id,
case_dir=case_dir,
before_inbox=before_inbox,
dois=dois,
titles=titles,
no_upload=no_upload,
)
def main():
parser = argparse.ArgumentParser(
description="Download source PDFs for an RFR case study via PyPaperBot."
)
parser.add_argument("--case-id", required=True, help="e.g. CS13")
parser.add_argument(
"--dois", nargs="+", metavar="DOI",
help="One or more DOIs to download. If omitted, reads data/raw/<CASE_ID>/dois.txt",
)
parser.add_argument(
"--scihub-mirror", default="https://sci-hub.se",
help="Sci-Hub mirror URL (default: https://sci-hub.se)",
)
parser.add_argument(
"--aa-api-key", default=None,
help="Anna's Archive API key for paywalled papers",
)
parser.add_argument(
"--no-upload", action="store_true",
help="Skip uploading downloaded PDFs to Google Drive",
)
parser.add_argument(
"--manual-ezproxy",
action="store_true",
help="If automated download finds no PDFs, open UWaterloo EZproxy URLs in Chrome and wait for manual download",
)
args = parser.parse_args()
case_id = args.case_id.upper()
case_dir = find_case_dir(case_id)
if case_dir is None:
print(f"ERROR: Case directory not found for {case_id} under {REPO_ROOT / 'data' / 'raw'}")
sys.exit(1)
dois = get_dois_for_case(case_dir, args.dois)
if not dois:
print(
f"ERROR: No DOIs provided. Either pass --dois or create:\n"
f" {case_dir / 'dois.txt'} (one DOI per line)"
)
sys.exit(1)
print(f"\n{'='*60}")
print(f" fetch_papers β€” {case_id}")
print(f" DOIs to fetch: {len(dois)}")
for d in dois:
print(f" {d}")
print(f" Destination: {case_dir}")
print(f"{'='*60}\n")
new_pdfs = fetch_and_download(
dois=dois,
dest_dir=case_dir,
scihub_mirror=args.scihub_mirror,
aa_api_key=args.aa_api_key,
)
print(f"\n{'='*60}")
if new_pdfs:
print(f" Downloaded {len(new_pdfs)} PDF(s):")
for p in new_pdfs:
print(f" {p.name}")
else:
print(" No new PDFs downloaded.")
print(f"{'='*60}\n")
if not new_pdfs and args.manual_ezproxy:
browser_pdfs = browser_ezproxy_flow(
case_id=case_id,
case_dir=case_dir,
dois=dois,
no_upload=args.no_upload,
)
new_pdfs.extend(browser_pdfs)
# Drive upload
if new_pdfs and not args.no_upload and not args.manual_ezproxy:
folders = load_drive_folders()
folder_id = folders.get(case_id)
if not folder_id:
print(f" [Drive] No folder ID found for {case_id} in drive_folders.json β€” skipping upload.")
else:
from .drive_upload import make_uploader, try_upload
uploader = make_uploader(folder_id=folder_id)
if uploader:
for pdf in new_pdfs:
print(f" [Drive] Uploading {pdf.name}...")
try_upload(uploader, pdf)
else:
print(" [Drive] No uploader available β€” skipping upload.")
elif args.no_upload and not args.manual_ezproxy:
print(" [Drive] Upload skipped (--no-upload).")
print("\nDone.")
if __name__ == "__main__":
main()