Spaces:
Sleeping
Sleeping
Anna Zhou commited on
Commit ·
f1cc362
1
Parent(s): 0504fcc
Prepare shared GitHub deployment
Browse files- .github/workflows/deploy-hf-space.yml +75 -0
- CLAUDE.md +38 -0
- case_study_pipeline/README.md +62 -0
- case_study_pipeline/acquire_sources.py +320 -0
- case_study_pipeline/discover_sources.py +702 -0
- case_study_pipeline/fetch_papers.py +181 -6
- case_study_pipeline/pdf_utils.py +10 -1
- case_study_pipeline/select_best_run.py +21 -5
- case_study_pipeline/sort_inbox.py +5 -3
- data/raw/CS36-Greenville_NC_USA/case_meta.json +2 -2
.github/workflows/deploy-hf-space.yml
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Deploy Hugging Face Space
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch:
|
| 5 |
+
inputs:
|
| 6 |
+
space_id:
|
| 7 |
+
description: "Target Hugging Face Space, e.g. UWRFR/UWaterlooManagedRetreat"
|
| 8 |
+
required: true
|
| 9 |
+
default: "UWRFR/UWaterlooManagedRetreat"
|
| 10 |
+
create_pr:
|
| 11 |
+
description: "Open a Hugging Face PR instead of pushing directly"
|
| 12 |
+
required: true
|
| 13 |
+
type: boolean
|
| 14 |
+
default: false
|
| 15 |
+
push:
|
| 16 |
+
branches:
|
| 17 |
+
- main
|
| 18 |
+
|
| 19 |
+
concurrency:
|
| 20 |
+
group: deploy-hf-space
|
| 21 |
+
cancel-in-progress: true
|
| 22 |
+
|
| 23 |
+
jobs:
|
| 24 |
+
deploy:
|
| 25 |
+
runs-on: ubuntu-latest
|
| 26 |
+
env:
|
| 27 |
+
HF_SPACE_ID: ${{ github.event.inputs.space_id || vars.HF_SPACE_ID || 'UWRFR/UWaterlooManagedRetreat' }}
|
| 28 |
+
CREATE_PR: ${{ github.event.inputs.create_pr || 'false' }}
|
| 29 |
+
steps:
|
| 30 |
+
- name: Check out repository
|
| 31 |
+
uses: actions/checkout@v4
|
| 32 |
+
|
| 33 |
+
- name: Set up Python
|
| 34 |
+
uses: actions/setup-python@v5
|
| 35 |
+
with:
|
| 36 |
+
python-version: "3.11"
|
| 37 |
+
|
| 38 |
+
- name: Install Hugging Face CLI
|
| 39 |
+
run: |
|
| 40 |
+
python -m pip install --upgrade pip
|
| 41 |
+
python -m pip install --upgrade "huggingface_hub[cli]"
|
| 42 |
+
|
| 43 |
+
- name: Validate required files
|
| 44 |
+
run: |
|
| 45 |
+
test -f README.md
|
| 46 |
+
test -f app/app.py
|
| 47 |
+
test -f requirements.txt
|
| 48 |
+
test -f data/extracted/precomputed_embeddings.json
|
| 49 |
+
|
| 50 |
+
- name: Upload Space files
|
| 51 |
+
env:
|
| 52 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
| 53 |
+
run: |
|
| 54 |
+
if [ -z "$HF_TOKEN" ]; then
|
| 55 |
+
echo "Missing HF_TOKEN GitHub secret" >&2
|
| 56 |
+
exit 1
|
| 57 |
+
fi
|
| 58 |
+
|
| 59 |
+
CREATE_PR_FLAG=""
|
| 60 |
+
if [ "$CREATE_PR" = "true" ]; then
|
| 61 |
+
CREATE_PR_FLAG="--create-pr"
|
| 62 |
+
fi
|
| 63 |
+
|
| 64 |
+
hf upload "$HF_SPACE_ID" . \
|
| 65 |
+
--type space \
|
| 66 |
+
--exclude ".git/*" \
|
| 67 |
+
--exclude ".github/*" \
|
| 68 |
+
--exclude ".venv/*" \
|
| 69 |
+
--exclude "venv/*" \
|
| 70 |
+
--exclude "__pycache__/*" \
|
| 71 |
+
--exclude ".streamlit_history/*" \
|
| 72 |
+
--exclude "data/raw/*" \
|
| 73 |
+
--exclude "*.pdf" \
|
| 74 |
+
--commit-message "Deploy from GitHub Actions: $GITHUB_SHA" \
|
| 75 |
+
$CREATE_PR_FLAG
|
CLAUDE.md
CHANGED
|
@@ -39,6 +39,44 @@ The Cowork sandbox is **ephemeral** (installed packages don't persist between se
|
|
| 39 |
this — it documents the exact URL pattern (`ezproxy.py`) and the
|
| 40 |
inbox-sorting + Drive-upload script (`sort_inbox.py`).
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
---
|
| 43 |
|
| 44 |
## Core research questions
|
|
|
|
| 39 |
this — it documents the exact URL pattern (`ezproxy.py`) and the
|
| 40 |
inbox-sorting + Drive-upload script (`sort_inbox.py`).
|
| 41 |
|
| 42 |
+
### Searching for new supplemental papers (consensus search process)
|
| 43 |
+
|
| 44 |
+
When looking for additional papers beyond what's in `sources.txt`, follow this process:
|
| 45 |
+
|
| 46 |
+
**1. Read `sources.txt` first** — check what's already listed so you don't duplicate.
|
| 47 |
+
|
| 48 |
+
**2. Always anchor searches on the location name** — every search must include the specific town/city name in quotes (e.g. `"Cagayan de Oro"`, `"Valmeyer"`). Papers must mention the actual place, not just the general topic.
|
| 49 |
+
|
| 50 |
+
**3. Run 2–3 targeted searches in parallel:**
|
| 51 |
+
- `"[Location]" "[disaster event]" resettlement OR buyout OR relocation journal doi`
|
| 52 |
+
- `"[Location]" flood OR hazard managed retreat academic paper`
|
| 53 |
+
- `"[Location]" "Natural Hazards" OR "IJDRR" OR "Climatic Change" doi`
|
| 54 |
+
|
| 55 |
+
**4. Verify each paper actually covers the location** — fetch the abstract to confirm the location name appears in the paper itself, not just in a reference list or tangentially.
|
| 56 |
+
|
| 57 |
+
**5. Get the DOI** for each confirmed paper so it can be downloaded directly.
|
| 58 |
+
|
| 59 |
+
**6. Flag open access vs paywalled** — open access (PMC, ResearchGate, IOP open) can be downloaded directly; paywalled ones need UWaterloo EZproxy.
|
| 60 |
+
|
| 61 |
+
> **The most common mistake:** finding papers *about* the general topic (flood buyouts, managed retreat in the Philippines, etc.) that don't actually study the specific case study location. The location name check is the critical filter.
|
| 62 |
+
|
| 63 |
+
### Automated high-precision discovery
|
| 64 |
+
|
| 65 |
+
Use the structured discovery command before relying on broad web or Consensus
|
| 66 |
+
search results:
|
| 67 |
+
|
| 68 |
+
```bash
|
| 69 |
+
python3 -m case_study_pipeline.discover_sources --case-id CS36
|
| 70 |
+
python3 -m case_study_pipeline.discover_sources --case-id CS36 --alias "Pitt County" --alias "Hurricane Floyd"
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
This writes `sources.json`, `paper_candidates.json`, and
|
| 74 |
+
`paper_discovery_report.md` in the case folder. Existing `sources.txt` /
|
| 75 |
+
`dois.txt` entries are treated as accepted seed sources. New papers are only
|
| 76 |
+
marked `review_required` unless they pass strict identity checks against the
|
| 77 |
+
case village/city, province/state, country, aliases, and managed-retreat terms.
|
| 78 |
+
Policy/legal context is labelled separately from direct case evidence.
|
| 79 |
+
|
| 80 |
---
|
| 81 |
|
| 82 |
## Core research questions
|
case_study_pipeline/README.md
CHANGED
|
@@ -98,6 +98,68 @@ as `run_case_study.py`. Each writes `<CASE_ID>_Check1_report.{md,pdf}` or
|
|
| 98 |
Requires `OPENAI_API_KEY` (in `rfr-rag/.env` or the environment) -- no Anthropic key
|
| 99 |
needed for these two scripts.
|
| 100 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
## Session notes
|
| 102 |
|
| 103 |
`session_notes/` holds one-off "continue this work" notes written for specific case
|
|
|
|
| 98 |
Requires `OPENAI_API_KEY` (in `rfr-rag/.env` or the environment) -- no Anthropic key
|
| 99 |
needed for these two scripts.
|
| 100 |
|
| 101 |
+
## High-precision paper discovery
|
| 102 |
+
|
| 103 |
+
Use `discover_sources.py` before running the extraction pipeline when a case
|
| 104 |
+
needs source cleanup, DOI lookup, or supplemental paper discovery.
|
| 105 |
+
|
| 106 |
+
```bash
|
| 107 |
+
python3 -m case_study_pipeline.discover_sources --case-id CS36
|
| 108 |
+
python3 -m case_study_pipeline.discover_sources \
|
| 109 |
+
--case-id CS36 \
|
| 110 |
+
--alias "Pitt County" \
|
| 111 |
+
--alias "Hurricane Floyd"
|
| 112 |
+
```
|
| 113 |
+
|
| 114 |
+
The discovery command is deliberately conservative:
|
| 115 |
+
|
| 116 |
+
- Existing `sources.txt` / `dois.txt` entries are treated as accepted seed
|
| 117 |
+
sources, but still resolved through scholarly metadata APIs where possible.
|
| 118 |
+
- New search results are not silently accepted. They are classified as
|
| 119 |
+
`review_required` or `rejected_by_identity_filter`.
|
| 120 |
+
- Direct case evidence must match the core place identity, such as the village,
|
| 121 |
+
city, resettlement name, province/state, and country from the spreadsheet or
|
| 122 |
+
`case_meta.json`.
|
| 123 |
+
- National/provincial/local legal and policy papers can be surfaced as
|
| 124 |
+
`candidate_policy_context`, but they stay separate from direct case evidence.
|
| 125 |
+
|
| 126 |
+
The command writes:
|
| 127 |
+
|
| 128 |
+
```text
|
| 129 |
+
data/raw/<CASE>/
|
| 130 |
+
├── sources.json # structured accepted seed sources
|
| 131 |
+
├── paper_candidates.json # all candidates with evidence/scoring
|
| 132 |
+
└── paper_discovery_report.md # human-readable review report
|
| 133 |
+
```
|
| 134 |
+
|
| 135 |
+
Recommended review rule: only use `accepted_seed` sources and manually approved
|
| 136 |
+
`review_required` candidates in the LLM extraction. Do not use rejected/background
|
| 137 |
+
papers unless a human reclassifies them after reading the full text.
|
| 138 |
+
|
| 139 |
+
## Automated source acquisition
|
| 140 |
+
|
| 141 |
+
After discovery, run `acquire_sources.py` to populate the case folder with source
|
| 142 |
+
files the extraction pipeline can read:
|
| 143 |
+
|
| 144 |
+
```bash
|
| 145 |
+
python3 -m case_study_pipeline.acquire_sources --case-id CS39
|
| 146 |
+
python3 -m case_study_pipeline.acquire_sources \
|
| 147 |
+
--case-id CS38 \
|
| 148 |
+
--extra-url "https://www.academia.edu/37237146/Heritage_and_Postdisaster_Recovery_Indigenous_Community_Resilience"
|
| 149 |
+
```
|
| 150 |
+
|
| 151 |
+
This command prefers publisher/open-access PDFs, then falls back to a
|
| 152 |
+
provenance-labelled `.md` capture when the PDF is blocked but the article/report
|
| 153 |
+
text is publicly reachable. That fallback is intentional: the extraction
|
| 154 |
+
pipeline reads `.pdf`, `.txt`, and `.md`, so a full-text markdown capture is
|
| 155 |
+
better than stopping the pipeline on a missing PDF. Use the generated
|
| 156 |
+
`source_acquisition_report.json` to see what was acquired.
|
| 157 |
+
|
| 158 |
+
If a publisher PDF is paywalled and no public full-text route is available, the
|
| 159 |
+
script will not bypass access controls. In that case, the non-manual route is to
|
| 160 |
+
add an authorized/open URL with `--extra-url`, or use the EZproxy/browser flow
|
| 161 |
+
below for institutional access.
|
| 162 |
+
|
| 163 |
## Session notes
|
| 164 |
|
| 165 |
`session_notes/` holds one-off "continue this work" notes written for specific case
|
case_study_pipeline/acquire_sources.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Acquire source evidence for a case study without manual browser downloads.
|
| 4 |
+
|
| 5 |
+
The extraction pipeline can read PDFs, .txt, and .md files. This command
|
| 6 |
+
therefore prefers PDFs, but falls back to provenance-labelled markdown captures
|
| 7 |
+
when a publisher blocks automated PDF access.
|
| 8 |
+
|
| 9 |
+
Usage:
|
| 10 |
+
python3 -m case_study_pipeline.acquire_sources --case-id CS39
|
| 11 |
+
python3 -m case_study_pipeline.acquire_sources --case-id CS38 \
|
| 12 |
+
--extra-url https://www.academia.edu/37237146/Heritage_and_Postdisaster_Recovery_Indigenous_Community_Resilience
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
import argparse
|
| 18 |
+
import json
|
| 19 |
+
import re
|
| 20 |
+
from dataclasses import dataclass
|
| 21 |
+
from datetime import datetime, timezone
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
from typing import Any
|
| 24 |
+
from urllib.parse import quote, urlparse
|
| 25 |
+
|
| 26 |
+
import requests
|
| 27 |
+
from bs4 import BeautifulSoup
|
| 28 |
+
|
| 29 |
+
from .fetch_papers import find_case_dir, get_dois_for_case
|
| 30 |
+
|
| 31 |
+
REPO_ROOT = Path(__file__).resolve().parent.parent
|
| 32 |
+
URL_RE = re.compile(r"https?://[^\s;,)]+", re.I)
|
| 33 |
+
|
| 34 |
+
HEADERS = {
|
| 35 |
+
"User-Agent": "rfr-rag-source-acquirer/0.1 (mailto:a74zhou@uwaterloo.ca)",
|
| 36 |
+
"Accept": "text/html,application/pdf;q=0.9,*/*;q=0.8",
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@dataclass
|
| 41 |
+
class Candidate:
|
| 42 |
+
doi: str = ""
|
| 43 |
+
title: str = ""
|
| 44 |
+
url: str = ""
|
| 45 |
+
pdf_url: str = ""
|
| 46 |
+
source: str = ""
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _slug(value: str, fallback: str = "source") -> str:
|
| 50 |
+
value = re.sub(r"[^A-Za-z0-9]+", "_", value).strip("_")
|
| 51 |
+
return (value[:90] or fallback).strip("_")
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _normalize_doi(doi: str) -> str:
|
| 55 |
+
return doi.strip().rstrip(".,);]").lower()
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _safe_get(url: str, timeout: int = 30) -> requests.Response | None:
|
| 59 |
+
try:
|
| 60 |
+
resp = requests.get(url, headers=HEADERS, timeout=timeout, allow_redirects=True)
|
| 61 |
+
return resp
|
| 62 |
+
except requests.RequestException as exc:
|
| 63 |
+
print(f" GET failed: {url} ({exc})")
|
| 64 |
+
return None
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _is_pdf_response(resp: requests.Response) -> bool:
|
| 68 |
+
content_type = resp.headers.get("content-type", "").lower()
|
| 69 |
+
return "application/pdf" in content_type or resp.content.startswith(b"%PDF")
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def _download_pdf(url: str, dest: Path, force: bool = False) -> Path | None:
|
| 73 |
+
if dest.exists() and not force:
|
| 74 |
+
print(f" PDF exists: {dest.name}")
|
| 75 |
+
return dest
|
| 76 |
+
resp = _safe_get(url)
|
| 77 |
+
if not resp or not resp.ok:
|
| 78 |
+
return None
|
| 79 |
+
if not _is_pdf_response(resp):
|
| 80 |
+
print(f" Not a PDF: {url} ({resp.headers.get('content-type', 'unknown')})")
|
| 81 |
+
return None
|
| 82 |
+
dest.write_bytes(resp.content)
|
| 83 |
+
print(f" Saved PDF: {dest.name} ({dest.stat().st_size:,} bytes)")
|
| 84 |
+
return dest
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _html_to_text(html: str) -> str:
|
| 88 |
+
soup = BeautifulSoup(html, "html.parser")
|
| 89 |
+
for tag in soup(["script", "style", "noscript", "svg", "form", "nav", "footer"]):
|
| 90 |
+
tag.decompose()
|
| 91 |
+
lines: list[str] = []
|
| 92 |
+
last = None
|
| 93 |
+
for raw in soup.get_text("\n").splitlines():
|
| 94 |
+
line = re.sub(r"\s+", " ", raw).strip()
|
| 95 |
+
if line and line != last:
|
| 96 |
+
lines.append(line)
|
| 97 |
+
last = line
|
| 98 |
+
return "\n".join(lines)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _capture_markdown(url: str, dest: Path, title: str, doi: str = "", force: bool = False) -> Path | None:
|
| 102 |
+
if dest.exists() and not force:
|
| 103 |
+
print(f" Markdown exists: {dest.name}")
|
| 104 |
+
return dest
|
| 105 |
+
|
| 106 |
+
resp = _safe_get(url)
|
| 107 |
+
body = ""
|
| 108 |
+
capture_url = url
|
| 109 |
+
if resp and resp.ok and "text/html" in resp.headers.get("content-type", "").lower():
|
| 110 |
+
body = _html_to_text(resp.text)
|
| 111 |
+
|
| 112 |
+
# Some sites block direct scripted capture but are readable through Jina's
|
| 113 |
+
# text mirror. This is not a paywall bypass; it is a text rendering proxy for
|
| 114 |
+
# publicly reachable pages.
|
| 115 |
+
if len(body) < 2_000:
|
| 116 |
+
jina_url = "https://r.jina.ai/http://r.jina.ai/http://" + url
|
| 117 |
+
jina = _safe_get(jina_url)
|
| 118 |
+
if jina and jina.ok and len(jina.text) > len(body):
|
| 119 |
+
body = jina.text
|
| 120 |
+
capture_url = jina_url
|
| 121 |
+
|
| 122 |
+
if len(body) < 1_000:
|
| 123 |
+
print(f" Markdown capture too thin ({len(body)} chars): {url}")
|
| 124 |
+
return None
|
| 125 |
+
|
| 126 |
+
header = (
|
| 127 |
+
f"# {title or url}\n\n"
|
| 128 |
+
f"Source URL: {url}\n"
|
| 129 |
+
f"Capture URL: {capture_url}\n"
|
| 130 |
+
f"DOI: {doi}\n"
|
| 131 |
+
f"Captured: {datetime.now(timezone.utc).isoformat()}\n"
|
| 132 |
+
f"Reliability note: Automated source capture; prefer publisher PDF when available.\n\n"
|
| 133 |
+
)
|
| 134 |
+
dest.write_text(header + body + "\n", encoding="utf-8")
|
| 135 |
+
print(f" Saved markdown: {dest.name} ({len(body):,} chars)")
|
| 136 |
+
return dest
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _request_json(url: str, params: dict[str, Any] | None = None) -> dict[str, Any] | None:
|
| 140 |
+
try:
|
| 141 |
+
resp = requests.get(url, params=params or {}, headers=HEADERS, timeout=25)
|
| 142 |
+
if not resp.ok:
|
| 143 |
+
return None
|
| 144 |
+
return resp.json()
|
| 145 |
+
except (requests.RequestException, ValueError):
|
| 146 |
+
return None
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def _crossref_candidate(doi: str) -> Candidate:
|
| 150 |
+
data = _request_json(f"https://api.crossref.org/works/{quote(doi, safe='')}")
|
| 151 |
+
if not data:
|
| 152 |
+
return Candidate(doi=doi, url=f"https://doi.org/{doi}", source="doi")
|
| 153 |
+
msg = data.get("message", {})
|
| 154 |
+
title = (msg.get("title") or [""])[0]
|
| 155 |
+
links = msg.get("link") or []
|
| 156 |
+
pdf_url = ""
|
| 157 |
+
for link in links:
|
| 158 |
+
if "pdf" in (link.get("content-type") or "").lower():
|
| 159 |
+
pdf_url = link.get("URL") or ""
|
| 160 |
+
break
|
| 161 |
+
return Candidate(
|
| 162 |
+
doi=doi,
|
| 163 |
+
title=title,
|
| 164 |
+
url=msg.get("URL") or f"https://doi.org/{doi}",
|
| 165 |
+
pdf_url=pdf_url,
|
| 166 |
+
source="crossref",
|
| 167 |
+
)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def _unpaywall_pdf(doi: str, email: str = "a74zhou@uwaterloo.ca") -> str:
|
| 171 |
+
data = _request_json(f"https://api.unpaywall.org/v2/{doi}", {"email": email})
|
| 172 |
+
if not data:
|
| 173 |
+
return ""
|
| 174 |
+
best = data.get("best_oa_location") or {}
|
| 175 |
+
return best.get("url_for_pdf") or best.get("url") or ""
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _openalex_pdf(doi: str) -> str:
|
| 179 |
+
data = _request_json("https://api.openalex.org/works", {"filter": f"doi:{doi}"})
|
| 180 |
+
if not data or not data.get("results"):
|
| 181 |
+
return ""
|
| 182 |
+
loc = (data["results"][0].get("best_oa_location") or {})
|
| 183 |
+
return loc.get("pdf_url") or loc.get("landing_page_url") or ""
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def _semantic_scholar_pdf(doi: str) -> str:
|
| 187 |
+
url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{quote(doi, safe='')}"
|
| 188 |
+
data = _request_json(url, {"fields": "title,openAccessPdf,url"})
|
| 189 |
+
if not data:
|
| 190 |
+
return ""
|
| 191 |
+
oa = data.get("openAccessPdf") or {}
|
| 192 |
+
return oa.get("url") or ""
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def _publisher_pdf_patterns(doi: str, landing_url: str) -> list[str]:
|
| 196 |
+
parsed = urlparse(landing_url)
|
| 197 |
+
urls = []
|
| 198 |
+
if "springer" in parsed.netloc:
|
| 199 |
+
urls.append(f"https://link.springer.com/content/pdf/{doi}.pdf")
|
| 200 |
+
if "mdpi.com" in parsed.netloc:
|
| 201 |
+
urls.append(landing_url.rstrip("/") + "/pdf")
|
| 202 |
+
return urls
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def _load_candidates(case_dir: Path, cli_dois: list[str] | None, extra_urls: list[str]) -> list[Candidate]:
|
| 206 |
+
candidates: list[Candidate] = []
|
| 207 |
+
seen: set[str] = set()
|
| 208 |
+
|
| 209 |
+
def add(candidate: Candidate) -> None:
|
| 210 |
+
key = candidate.doi or candidate.url or candidate.pdf_url or candidate.title
|
| 211 |
+
key = key.lower()
|
| 212 |
+
if key and key not in seen:
|
| 213 |
+
seen.add(key)
|
| 214 |
+
candidates.append(candidate)
|
| 215 |
+
|
| 216 |
+
for doi in get_dois_for_case(case_dir, cli_dois):
|
| 217 |
+
add(_crossref_candidate(_normalize_doi(doi)))
|
| 218 |
+
|
| 219 |
+
sources_json = case_dir / "sources.json"
|
| 220 |
+
if sources_json.exists():
|
| 221 |
+
data = json.loads(sources_json.read_text(encoding="utf-8"))
|
| 222 |
+
for src in data.get("sources", []):
|
| 223 |
+
doi = _normalize_doi(src.get("doi", ""))
|
| 224 |
+
add(Candidate(
|
| 225 |
+
doi=doi,
|
| 226 |
+
title=src.get("title", ""),
|
| 227 |
+
url=src.get("url", "") or (f"https://doi.org/{doi}" if doi else ""),
|
| 228 |
+
pdf_url=src.get("pdf_url", ""),
|
| 229 |
+
source="sources.json",
|
| 230 |
+
))
|
| 231 |
+
|
| 232 |
+
sources_txt = case_dir / "sources.txt"
|
| 233 |
+
if sources_txt.exists():
|
| 234 |
+
for url in URL_RE.findall(sources_txt.read_text(encoding="utf-8")):
|
| 235 |
+
add(Candidate(url=url, title=Path(urlparse(url).path).stem or url, source="sources.txt"))
|
| 236 |
+
|
| 237 |
+
for url in extra_urls:
|
| 238 |
+
add(Candidate(url=url, title=Path(urlparse(url).path).stem or url, source="extra-url"))
|
| 239 |
+
|
| 240 |
+
return candidates
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def acquire_candidate(case_dir: Path, candidate: Candidate, force: bool = False) -> list[Path]:
|
| 244 |
+
title = candidate.title or candidate.doi or candidate.url
|
| 245 |
+
stem = _slug(title)
|
| 246 |
+
print(f"\n Source: {title}")
|
| 247 |
+
if candidate.doi:
|
| 248 |
+
print(f" DOI: {candidate.doi}")
|
| 249 |
+
|
| 250 |
+
outputs: list[Path] = []
|
| 251 |
+
pdf_urls = [candidate.pdf_url]
|
| 252 |
+
if candidate.doi:
|
| 253 |
+
pdf_urls.extend([
|
| 254 |
+
_unpaywall_pdf(candidate.doi),
|
| 255 |
+
_semantic_scholar_pdf(candidate.doi),
|
| 256 |
+
_openalex_pdf(candidate.doi),
|
| 257 |
+
])
|
| 258 |
+
pdf_urls.extend(_publisher_pdf_patterns(candidate.doi, candidate.url))
|
| 259 |
+
if candidate.url.lower().endswith(".pdf"):
|
| 260 |
+
pdf_urls.append(candidate.url)
|
| 261 |
+
|
| 262 |
+
seen_urls: set[str] = set()
|
| 263 |
+
for pdf_url in [u for u in pdf_urls if u]:
|
| 264 |
+
if pdf_url in seen_urls:
|
| 265 |
+
continue
|
| 266 |
+
seen_urls.add(pdf_url)
|
| 267 |
+
out = _download_pdf(pdf_url, case_dir / f"{stem}.pdf", force=force)
|
| 268 |
+
if out:
|
| 269 |
+
outputs.append(out)
|
| 270 |
+
return outputs
|
| 271 |
+
|
| 272 |
+
# Fall back to text capture from DOI landing page or explicit source URL.
|
| 273 |
+
capture_urls = [candidate.url]
|
| 274 |
+
if candidate.doi:
|
| 275 |
+
capture_urls.append(f"https://doi.org/{candidate.doi}")
|
| 276 |
+
for url in [u for u in capture_urls if u]:
|
| 277 |
+
out = _capture_markdown(url, case_dir / f"{stem}.md", title=title, doi=candidate.doi, force=force)
|
| 278 |
+
if out:
|
| 279 |
+
outputs.append(out)
|
| 280 |
+
return outputs
|
| 281 |
+
|
| 282 |
+
print(" Could not acquire source automatically.")
|
| 283 |
+
return outputs
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def main(argv: list[str] | None = None) -> None:
|
| 287 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 288 |
+
parser.add_argument("--case-id", required=True, help="e.g. CS38")
|
| 289 |
+
parser.add_argument("--dois", nargs="+", metavar="DOI")
|
| 290 |
+
parser.add_argument("--extra-url", action="append", default=[], help="Additional article/report URL to capture")
|
| 291 |
+
parser.add_argument("--force", action="store_true", help="Overwrite existing acquired files")
|
| 292 |
+
args = parser.parse_args(argv)
|
| 293 |
+
|
| 294 |
+
case_id = args.case_id.upper()
|
| 295 |
+
case_dir = find_case_dir(case_id)
|
| 296 |
+
if case_dir is None:
|
| 297 |
+
raise SystemExit(f"Case directory not found for {case_id}")
|
| 298 |
+
|
| 299 |
+
print(f"Acquiring sources for {case_id}: {case_dir}")
|
| 300 |
+
candidates = _load_candidates(case_dir, args.dois, args.extra_url)
|
| 301 |
+
print(f"Found {len(candidates)} candidate source(s).")
|
| 302 |
+
|
| 303 |
+
acquired: list[Path] = []
|
| 304 |
+
for candidate in candidates:
|
| 305 |
+
acquired.extend(acquire_candidate(case_dir, candidate, force=args.force))
|
| 306 |
+
|
| 307 |
+
report = {
|
| 308 |
+
"case_id": case_id,
|
| 309 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 310 |
+
"acquired_files": [p.name for p in acquired],
|
| 311 |
+
"candidate_count": len(candidates),
|
| 312 |
+
}
|
| 313 |
+
report_path = case_dir / "source_acquisition_report.json"
|
| 314 |
+
report_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
|
| 315 |
+
print(f"\nAcquired {len(acquired)} file(s).")
|
| 316 |
+
print(f"Wrote {report_path}")
|
| 317 |
+
|
| 318 |
+
|
| 319 |
+
if __name__ == "__main__":
|
| 320 |
+
main()
|
case_study_pipeline/discover_sources.py
ADDED
|
@@ -0,0 +1,702 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
High-precision paper discovery for one managed-retreat case study.
|
| 4 |
+
|
| 5 |
+
This command is intentionally conservative. It auto-records sources already
|
| 6 |
+
listed in the case folder (sources.txt / dois.txt) as seed sources, then searches
|
| 7 |
+
scholarly APIs for additional candidates and labels them with evidence. New
|
| 8 |
+
discoveries are not silently accepted as case evidence; they are written to a
|
| 9 |
+
review report for human approval.
|
| 10 |
+
|
| 11 |
+
Usage:
|
| 12 |
+
python3 -m case_study_pipeline.discover_sources --case-id CS36
|
| 13 |
+
python3 -m case_study_pipeline.discover_sources --case-id CS36 --max-results 8
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
from __future__ import annotations
|
| 17 |
+
|
| 18 |
+
import argparse
|
| 19 |
+
import json
|
| 20 |
+
import re
|
| 21 |
+
import textwrap
|
| 22 |
+
from dataclasses import dataclass, field
|
| 23 |
+
from datetime import datetime, timezone
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
from typing import Any
|
| 26 |
+
|
| 27 |
+
import requests
|
| 28 |
+
|
| 29 |
+
REPO_ROOT = Path(__file__).resolve().parent.parent
|
| 30 |
+
DATA_RAW = REPO_ROOT / "data" / "raw"
|
| 31 |
+
|
| 32 |
+
DOI_RE = re.compile(r"\b10\.\d{4,9}/[-._;()/:A-Z0-9]+\b", re.I)
|
| 33 |
+
WORD_RE = re.compile(r"[A-Za-z][A-Za-z.'-]*")
|
| 34 |
+
|
| 35 |
+
CASE_EVIDENCE_TERMS = (
|
| 36 |
+
"buyout",
|
| 37 |
+
"buy-out",
|
| 38 |
+
"acquisition",
|
| 39 |
+
"relocation",
|
| 40 |
+
"resettlement",
|
| 41 |
+
"retreat",
|
| 42 |
+
"managed realignment",
|
| 43 |
+
"floodplain",
|
| 44 |
+
"hazard mitigation",
|
| 45 |
+
"disaster recovery",
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
POLICY_TERMS = (
|
| 49 |
+
"policy",
|
| 50 |
+
"law",
|
| 51 |
+
"legal",
|
| 52 |
+
"program",
|
| 53 |
+
"governance",
|
| 54 |
+
"funding",
|
| 55 |
+
"fema",
|
| 56 |
+
"hud",
|
| 57 |
+
"cdbg",
|
| 58 |
+
"hm gp",
|
| 59 |
+
"hazard mitigation grant",
|
| 60 |
+
"national flood insurance",
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
STOPWORDS = {
|
| 64 |
+
"the",
|
| 65 |
+
"and",
|
| 66 |
+
"of",
|
| 67 |
+
"for",
|
| 68 |
+
"in",
|
| 69 |
+
"on",
|
| 70 |
+
"at",
|
| 71 |
+
"to",
|
| 72 |
+
"from",
|
| 73 |
+
"with",
|
| 74 |
+
"a",
|
| 75 |
+
"an",
|
| 76 |
+
"city",
|
| 77 |
+
"village",
|
| 78 |
+
"district",
|
| 79 |
+
"county",
|
| 80 |
+
"province",
|
| 81 |
+
"state",
|
| 82 |
+
"region",
|
| 83 |
+
"new",
|
| 84 |
+
"north",
|
| 85 |
+
"south",
|
| 86 |
+
"east",
|
| 87 |
+
"west",
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
@dataclass
|
| 92 |
+
class CaseIdentity:
|
| 93 |
+
case_id: str
|
| 94 |
+
name: str = ""
|
| 95 |
+
location: str = ""
|
| 96 |
+
country: str = ""
|
| 97 |
+
aliases: list[str] = field(default_factory=list)
|
| 98 |
+
|
| 99 |
+
@property
|
| 100 |
+
def core_place_terms(self) -> list[str]:
|
| 101 |
+
terms = []
|
| 102 |
+
for value in [self.name, (self.location.split(",")[0] if self.location else "")]:
|
| 103 |
+
for part in re.split(r"[,;/()]+", value or ""):
|
| 104 |
+
part = re.sub(r"\s+", " ", part).strip()
|
| 105 |
+
if len(part) >= 3 and part.lower() not in STOPWORDS:
|
| 106 |
+
terms.append(part)
|
| 107 |
+
return _dedupe(terms)
|
| 108 |
+
|
| 109 |
+
@property
|
| 110 |
+
def alias_terms(self) -> list[str]:
|
| 111 |
+
terms = []
|
| 112 |
+
for value in self.aliases:
|
| 113 |
+
for part in re.split(r"[,;/()]+", value or ""):
|
| 114 |
+
part = re.sub(r"\s+", " ", part).strip()
|
| 115 |
+
if len(part) >= 3 and part.lower() not in STOPWORDS:
|
| 116 |
+
terms.append(part)
|
| 117 |
+
return _dedupe(terms)
|
| 118 |
+
|
| 119 |
+
@property
|
| 120 |
+
def admin_terms(self) -> list[str]:
|
| 121 |
+
pieces = []
|
| 122 |
+
location_parts = [p.strip() for p in re.split(r"[,;/()]+", self.location or "") if p.strip()]
|
| 123 |
+
for part in location_parts[1:]:
|
| 124 |
+
if len(part) >= 3:
|
| 125 |
+
pieces.append(part)
|
| 126 |
+
if self.country:
|
| 127 |
+
pieces.append(self.country)
|
| 128 |
+
return _dedupe(pieces)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _dedupe(values: list[str]) -> list[str]:
|
| 132 |
+
seen = set()
|
| 133 |
+
out = []
|
| 134 |
+
for value in values:
|
| 135 |
+
key = _norm(value)
|
| 136 |
+
if key and key not in seen:
|
| 137 |
+
seen.add(key)
|
| 138 |
+
out.append(value)
|
| 139 |
+
return out
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def _norm(value: str) -> str:
|
| 143 |
+
value = value.lower()
|
| 144 |
+
value = value.replace("&", " and ")
|
| 145 |
+
value = re.sub(r"[^a-z0-9]+", " ", value)
|
| 146 |
+
return re.sub(r"\s+", " ", value).strip()
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def _title_key(title: str) -> str:
|
| 150 |
+
words = [w for w in _norm(title).split() if len(w) > 2 and w not in STOPWORDS]
|
| 151 |
+
return " ".join(words[:18])
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def find_case_dir(case_id: str) -> Path | None:
|
| 155 |
+
exact = DATA_RAW / case_id
|
| 156 |
+
if exact.exists():
|
| 157 |
+
return exact
|
| 158 |
+
for path in DATA_RAW.iterdir():
|
| 159 |
+
if path.is_dir() and re.match(rf"^{re.escape(case_id)}($|-)", path.name):
|
| 160 |
+
return path
|
| 161 |
+
return None
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def load_case_identity(case_dir: Path, aliases: list[str]) -> CaseIdentity:
|
| 165 |
+
meta_path = case_dir / "case_meta.json"
|
| 166 |
+
if not meta_path.exists():
|
| 167 |
+
raise FileNotFoundError(f"Missing case_meta.json in {case_dir}")
|
| 168 |
+
meta = json.loads(meta_path.read_text())
|
| 169 |
+
return CaseIdentity(
|
| 170 |
+
case_id=meta.get("case_id", case_dir.name.split("-")[0]),
|
| 171 |
+
name=meta.get("name", ""),
|
| 172 |
+
location=meta.get("location", ""),
|
| 173 |
+
country=meta.get("country", ""),
|
| 174 |
+
aliases=aliases,
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def parse_sources_txt(case_dir: Path) -> list[dict[str, Any]]:
|
| 179 |
+
path = case_dir / "sources.txt"
|
| 180 |
+
if not path.exists():
|
| 181 |
+
return []
|
| 182 |
+
|
| 183 |
+
text = path.read_text(encoding="utf-8")
|
| 184 |
+
primary = ""
|
| 185 |
+
supplemental = ""
|
| 186 |
+
if "PRIMARY SOURCE:" in text:
|
| 187 |
+
after_primary = text.split("PRIMARY SOURCE:", 1)[1]
|
| 188 |
+
if "SUPPLEMENTAL SOURCES:" in after_primary:
|
| 189 |
+
primary, supplemental = after_primary.split("SUPPLEMENTAL SOURCES:", 1)
|
| 190 |
+
else:
|
| 191 |
+
primary = after_primary
|
| 192 |
+
else:
|
| 193 |
+
supplemental = text
|
| 194 |
+
|
| 195 |
+
entries = []
|
| 196 |
+
for role, block in (("primary_case_evidence", primary), ("secondary_case_evidence", supplemental)):
|
| 197 |
+
for raw in _split_citation_block(block):
|
| 198 |
+
doi = _extract_doi(raw)
|
| 199 |
+
entries.append({
|
| 200 |
+
"role": role,
|
| 201 |
+
"status": "accepted_seed",
|
| 202 |
+
"citation": raw,
|
| 203 |
+
"doi": doi,
|
| 204 |
+
"source": "sources.txt",
|
| 205 |
+
})
|
| 206 |
+
return entries
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def parse_dois_txt(case_dir: Path) -> list[str]:
|
| 210 |
+
path = case_dir / "dois.txt"
|
| 211 |
+
if not path.exists():
|
| 212 |
+
return []
|
| 213 |
+
return [_extract_doi(line) for line in path.read_text().splitlines() if _extract_doi(line)]
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def _split_citation_block(block: str) -> list[str]:
|
| 217 |
+
block = re.sub(r"\bAND\b", "\n\n", block)
|
| 218 |
+
chunks = [re.sub(r"\s+", " ", c).strip() for c in re.split(r"\n\s*\n", block)]
|
| 219 |
+
return [c for c in chunks if len(c) > 20 and not c.lower().startswith(("journal page:", "local pdf:", "findable "))]
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def _extract_doi(text: str) -> str:
|
| 223 |
+
match = DOI_RE.search(text or "")
|
| 224 |
+
if not match:
|
| 225 |
+
return ""
|
| 226 |
+
return match.group(0).rstrip(".,);]").lower()
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def request_json(url: str, params: dict[str, Any], timeout: int = 20) -> dict[str, Any]:
|
| 230 |
+
headers = {"User-Agent": "rfr-rag-paper-discovery/0.1 (mailto:a74zhou@uwaterloo.ca)"}
|
| 231 |
+
resp = requests.get(url, params=params, headers=headers, timeout=timeout)
|
| 232 |
+
resp.raise_for_status()
|
| 233 |
+
return resp.json()
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def resolve_crossref_by_doi(doi: str) -> dict[str, Any] | None:
|
| 237 |
+
if not doi:
|
| 238 |
+
return None
|
| 239 |
+
try:
|
| 240 |
+
data = request_json(f"https://api.crossref.org/works/{doi}", {})
|
| 241 |
+
item = data.get("message", {})
|
| 242 |
+
return _crossref_item_to_candidate(item)
|
| 243 |
+
except Exception:
|
| 244 |
+
return None
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def search_crossref(query: str, rows: int) -> list[dict[str, Any]]:
|
| 248 |
+
try:
|
| 249 |
+
data = request_json(
|
| 250 |
+
"https://api.crossref.org/works",
|
| 251 |
+
{"query.bibliographic": query, "rows": rows, "select": "DOI,title,author,issued,container-title,URL,abstract,type"},
|
| 252 |
+
)
|
| 253 |
+
return [_crossref_item_to_candidate(item) for item in data.get("message", {}).get("items", [])]
|
| 254 |
+
except Exception:
|
| 255 |
+
return []
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def _crossref_item_to_candidate(item: dict[str, Any]) -> dict[str, Any]:
|
| 259 |
+
authors = []
|
| 260 |
+
for author in item.get("author", [])[:8]:
|
| 261 |
+
name = " ".join(part for part in [author.get("given", ""), author.get("family", "")] if part)
|
| 262 |
+
if name:
|
| 263 |
+
authors.append(name)
|
| 264 |
+
year = None
|
| 265 |
+
parts = item.get("issued", {}).get("date-parts", [])
|
| 266 |
+
if parts and parts[0]:
|
| 267 |
+
year = parts[0][0]
|
| 268 |
+
title = " ".join(item.get("title", [])[:1])
|
| 269 |
+
venue = " ".join(item.get("container-title", [])[:1])
|
| 270 |
+
doi = (item.get("DOI") or "").lower()
|
| 271 |
+
return {
|
| 272 |
+
"title": title,
|
| 273 |
+
"authors": authors,
|
| 274 |
+
"year": year,
|
| 275 |
+
"doi": doi,
|
| 276 |
+
"venue": venue,
|
| 277 |
+
"url": item.get("URL") or (f"https://doi.org/{doi}" if doi else ""),
|
| 278 |
+
"abstract": _strip_tags(item.get("abstract", "")),
|
| 279 |
+
"source_api": "crossref",
|
| 280 |
+
"type": item.get("type", ""),
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
|
| 284 |
+
def search_semantic_scholar(query: str, limit: int) -> list[dict[str, Any]]:
|
| 285 |
+
try:
|
| 286 |
+
data = request_json(
|
| 287 |
+
"https://api.semanticscholar.org/graph/v1/paper/search",
|
| 288 |
+
{
|
| 289 |
+
"query": query,
|
| 290 |
+
"limit": limit,
|
| 291 |
+
"fields": "title,authors,year,venue,abstract,externalIds,openAccessPdf,url",
|
| 292 |
+
},
|
| 293 |
+
)
|
| 294 |
+
return [_semantic_item_to_candidate(item) for item in data.get("data", [])]
|
| 295 |
+
except Exception:
|
| 296 |
+
return []
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def _semantic_item_to_candidate(item: dict[str, Any]) -> dict[str, Any]:
|
| 300 |
+
external = item.get("externalIds", {}) or {}
|
| 301 |
+
doi = (external.get("DOI") or "").lower()
|
| 302 |
+
pdf = item.get("openAccessPdf") or {}
|
| 303 |
+
return {
|
| 304 |
+
"title": item.get("title", ""),
|
| 305 |
+
"authors": [a.get("name", "") for a in item.get("authors", [])[:8] if a.get("name")],
|
| 306 |
+
"year": item.get("year"),
|
| 307 |
+
"doi": doi,
|
| 308 |
+
"venue": item.get("venue", ""),
|
| 309 |
+
"url": item.get("url", "") or (f"https://doi.org/{doi}" if doi else ""),
|
| 310 |
+
"pdf_url": pdf.get("url", "") if isinstance(pdf, dict) else "",
|
| 311 |
+
"abstract": item.get("abstract", "") or "",
|
| 312 |
+
"source_api": "semantic_scholar",
|
| 313 |
+
"type": "paper",
|
| 314 |
+
}
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
def search_openalex(query: str, per_page: int) -> list[dict[str, Any]]:
|
| 318 |
+
try:
|
| 319 |
+
data = request_json(
|
| 320 |
+
"https://api.openalex.org/works",
|
| 321 |
+
{"search": query, "per-page": per_page, "mailto": "a74zhou@uwaterloo.ca"},
|
| 322 |
+
)
|
| 323 |
+
return [_openalex_item_to_candidate(item) for item in data.get("results", [])]
|
| 324 |
+
except Exception:
|
| 325 |
+
return []
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def _openalex_item_to_candidate(item: dict[str, Any]) -> dict[str, Any]:
|
| 329 |
+
doi_url = item.get("doi") or ""
|
| 330 |
+
doi = doi_url.replace("https://doi.org/", "").lower()
|
| 331 |
+
authors = []
|
| 332 |
+
for auth in item.get("authorships", [])[:8]:
|
| 333 |
+
name = (auth.get("author") or {}).get("display_name", "")
|
| 334 |
+
if name:
|
| 335 |
+
authors.append(name)
|
| 336 |
+
location = item.get("primary_location") or {}
|
| 337 |
+
source = location.get("source") or {}
|
| 338 |
+
oa = item.get("open_access") or {}
|
| 339 |
+
return {
|
| 340 |
+
"title": item.get("display_name", ""),
|
| 341 |
+
"authors": authors,
|
| 342 |
+
"year": item.get("publication_year"),
|
| 343 |
+
"doi": doi,
|
| 344 |
+
"venue": source.get("display_name", ""),
|
| 345 |
+
"url": item.get("id", "") or doi_url,
|
| 346 |
+
"pdf_url": location.get("pdf_url") or oa.get("oa_url") or "",
|
| 347 |
+
"abstract": _openalex_abstract(item.get("abstract_inverted_index") or {}),
|
| 348 |
+
"source_api": "openalex",
|
| 349 |
+
"type": item.get("type", ""),
|
| 350 |
+
}
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def _openalex_abstract(index: dict[str, list[int]]) -> str:
|
| 354 |
+
if not index:
|
| 355 |
+
return ""
|
| 356 |
+
words = []
|
| 357 |
+
for word, positions in index.items():
|
| 358 |
+
for pos in positions:
|
| 359 |
+
words.append((pos, word))
|
| 360 |
+
return " ".join(word for _, word in sorted(words))
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
def _strip_tags(text: str) -> str:
|
| 364 |
+
text = re.sub(r"<[^>]+>", " ", text or "")
|
| 365 |
+
return re.sub(r"\s+", " ", text).strip()
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
def build_queries(identity: CaseIdentity, seed_entries: list[dict[str, Any]], max_seed_titles: int = 3) -> list[str]:
|
| 369 |
+
queries = []
|
| 370 |
+
place = identity.name or identity.location
|
| 371 |
+
location = identity.location or identity.name
|
| 372 |
+
country = identity.country
|
| 373 |
+
if place and country:
|
| 374 |
+
queries.extend([
|
| 375 |
+
f'"{place}" "{country}" buyout relocation resettlement',
|
| 376 |
+
f'"{place}" "{country}" floodplain acquisition managed retreat',
|
| 377 |
+
])
|
| 378 |
+
if location and location != place:
|
| 379 |
+
queries.append(f'"{location}" buyout relocation flood')
|
| 380 |
+
|
| 381 |
+
for entry in seed_entries[:max_seed_titles]:
|
| 382 |
+
title = infer_title_from_citation(entry.get("citation", ""))
|
| 383 |
+
if title:
|
| 384 |
+
queries.append(title)
|
| 385 |
+
|
| 386 |
+
return _dedupe(queries)
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def infer_title_from_citation(citation: str) -> str:
|
| 390 |
+
quoted = re.findall(r"[“\"]([^”\"]{12,180})[”\"]", citation)
|
| 391 |
+
if quoted:
|
| 392 |
+
return quoted[0]
|
| 393 |
+
# Fallback: text between year and journal-ish punctuation.
|
| 394 |
+
match = re.search(r"\(\d{4}\)\.?\s*([^\.]{20,180})\.", citation)
|
| 395 |
+
if match:
|
| 396 |
+
return match.group(1).strip()
|
| 397 |
+
return ""
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def merge_candidates(candidates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
| 401 |
+
merged: dict[str, dict[str, Any]] = {}
|
| 402 |
+
for cand in candidates:
|
| 403 |
+
title = cand.get("title", "")
|
| 404 |
+
doi = cand.get("doi", "")
|
| 405 |
+
key = f"doi:{doi}" if doi else f"title:{_title_key(title)}"
|
| 406 |
+
if not key.endswith(":"):
|
| 407 |
+
existing = merged.get(key)
|
| 408 |
+
if existing:
|
| 409 |
+
existing["source_api"] = ", ".join(_dedupe([existing.get("source_api", ""), cand.get("source_api", "")]))
|
| 410 |
+
if not existing.get("abstract") and cand.get("abstract"):
|
| 411 |
+
existing["abstract"] = cand["abstract"]
|
| 412 |
+
if not existing.get("pdf_url") and cand.get("pdf_url"):
|
| 413 |
+
existing["pdf_url"] = cand["pdf_url"]
|
| 414 |
+
else:
|
| 415 |
+
merged[key] = cand
|
| 416 |
+
return list(merged.values())
|
| 417 |
+
|
| 418 |
+
|
| 419 |
+
def classify_candidate(candidate: dict[str, Any], identity: CaseIdentity, seed_dois: set[str], seed_titles: set[str]) -> dict[str, Any]:
|
| 420 |
+
text = " ".join([
|
| 421 |
+
candidate.get("title", ""),
|
| 422 |
+
candidate.get("abstract", ""),
|
| 423 |
+
candidate.get("venue", ""),
|
| 424 |
+
])
|
| 425 |
+
norm_text = _norm(text)
|
| 426 |
+
|
| 427 |
+
found_core_place = [term for term in identity.core_place_terms if _norm(term) and _norm(term) in norm_text]
|
| 428 |
+
found_alias = [term for term in identity.alias_terms if _norm(term) and _norm(term) in norm_text]
|
| 429 |
+
found_admin = [term for term in identity.admin_terms if _norm(term) and _norm(term) in norm_text]
|
| 430 |
+
found_case_terms = [term for term in CASE_EVIDENCE_TERMS if _norm(term) in norm_text]
|
| 431 |
+
found_policy_terms = [term for term in POLICY_TERMS if _norm(term) in norm_text]
|
| 432 |
+
|
| 433 |
+
doi = candidate.get("doi", "")
|
| 434 |
+
title_key = _title_key(candidate.get("title", ""))
|
| 435 |
+
is_seed = bool((doi and doi in seed_dois) or (title_key and title_key in seed_titles))
|
| 436 |
+
|
| 437 |
+
score = 0
|
| 438 |
+
reasons = []
|
| 439 |
+
if is_seed:
|
| 440 |
+
score += 60
|
| 441 |
+
reasons.append("matches DOI/title already listed in local source notes")
|
| 442 |
+
if found_core_place:
|
| 443 |
+
score += 25
|
| 444 |
+
reasons.append(f"core case place term found: {', '.join(found_core_place[:3])}")
|
| 445 |
+
if found_alias:
|
| 446 |
+
score += 8
|
| 447 |
+
reasons.append(f"case alias/event term found: {', '.join(found_alias[:3])}")
|
| 448 |
+
if found_admin:
|
| 449 |
+
score += 15
|
| 450 |
+
reasons.append(f"administrative/country term found: {', '.join(found_admin[:3])}")
|
| 451 |
+
if found_case_terms:
|
| 452 |
+
score += 15
|
| 453 |
+
reasons.append(f"retreat/recovery term found: {', '.join(found_case_terms[:3])}")
|
| 454 |
+
if found_policy_terms:
|
| 455 |
+
score += 8
|
| 456 |
+
reasons.append(f"policy/legal term found: {', '.join(found_policy_terms[:3])}")
|
| 457 |
+
if candidate.get("pdf_url"):
|
| 458 |
+
score += 3
|
| 459 |
+
reasons.append("open PDF URL available")
|
| 460 |
+
|
| 461 |
+
if is_seed:
|
| 462 |
+
source_class = "primary_or_secondary_seed"
|
| 463 |
+
status = "accepted_seed"
|
| 464 |
+
elif found_core_place and (found_admin or len(identity.admin_terms) <= 1) and found_case_terms:
|
| 465 |
+
source_class = "candidate_case_evidence"
|
| 466 |
+
status = "review_required"
|
| 467 |
+
elif found_admin and found_policy_terms and found_case_terms:
|
| 468 |
+
source_class = "candidate_policy_context"
|
| 469 |
+
status = "review_required"
|
| 470 |
+
elif found_core_place and found_admin:
|
| 471 |
+
source_class = "weak_location_candidate"
|
| 472 |
+
status = "rejected_by_identity_filter"
|
| 473 |
+
reasons.append("mentions the place but lacks managed-retreat/recovery terms")
|
| 474 |
+
else:
|
| 475 |
+
source_class = "reject_or_background"
|
| 476 |
+
status = "rejected_by_identity_filter"
|
| 477 |
+
reasons.append("does not mention required case identity terms in title/abstract metadata")
|
| 478 |
+
|
| 479 |
+
candidate["identity_evidence"] = {
|
| 480 |
+
"core_place_terms_found": found_core_place,
|
| 481 |
+
"alias_terms_found": found_alias,
|
| 482 |
+
"admin_terms_found": found_admin,
|
| 483 |
+
"case_terms_found": found_case_terms,
|
| 484 |
+
"policy_terms_found": found_policy_terms,
|
| 485 |
+
}
|
| 486 |
+
candidate["confidence_score"] = min(score, 100)
|
| 487 |
+
candidate["source_class"] = source_class
|
| 488 |
+
candidate["status"] = status
|
| 489 |
+
candidate["acceptance_reason"] = "; ".join(reasons)
|
| 490 |
+
return candidate
|
| 491 |
+
|
| 492 |
+
|
| 493 |
+
def local_pdf_matches(case_dir: Path, candidate: dict[str, Any]) -> str:
|
| 494 |
+
title_key = _title_key(candidate.get("title", ""))
|
| 495 |
+
doi = candidate.get("doi", "")
|
| 496 |
+
for pdf in case_dir.glob("*.pdf"):
|
| 497 |
+
name_key = _title_key(pdf.stem)
|
| 498 |
+
if title_key and title_key[:40] in name_key:
|
| 499 |
+
return pdf.name
|
| 500 |
+
if doi:
|
| 501 |
+
# Some downloaded files include DOI fragments poorly; title match is the main route.
|
| 502 |
+
continue
|
| 503 |
+
return ""
|
| 504 |
+
|
| 505 |
+
|
| 506 |
+
def write_sources_json(case_dir: Path, identity: CaseIdentity, candidates: list[dict[str, Any]]) -> None:
|
| 507 |
+
accepted = [c for c in candidates if c.get("status") == "accepted_seed"]
|
| 508 |
+
payload = {
|
| 509 |
+
"schema_version": "0.1",
|
| 510 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 511 |
+
"case_identity": {
|
| 512 |
+
"case_id": identity.case_id,
|
| 513 |
+
"name": identity.name,
|
| 514 |
+
"location": identity.location,
|
| 515 |
+
"country": identity.country,
|
| 516 |
+
"aliases": identity.aliases,
|
| 517 |
+
},
|
| 518 |
+
"acceptance_policy": {
|
| 519 |
+
"accepted_seed": "Existing spreadsheet/local sources are accepted but still carry evidence metadata.",
|
| 520 |
+
"review_required": "Newly discovered candidate sources must be manually approved before use.",
|
| 521 |
+
"rejected_by_identity_filter": "Candidate did not pass strict case identity matching.",
|
| 522 |
+
},
|
| 523 |
+
"sources": accepted,
|
| 524 |
+
}
|
| 525 |
+
(case_dir / "sources.json").write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n")
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def write_candidates(case_dir: Path, candidates: list[dict[str, Any]]) -> None:
|
| 529 |
+
(case_dir / "paper_candidates.json").write_text(json.dumps(candidates, indent=2, ensure_ascii=False) + "\n")
|
| 530 |
+
|
| 531 |
+
|
| 532 |
+
def write_report(case_dir: Path, identity: CaseIdentity, queries: list[str], candidates: list[dict[str, Any]]) -> None:
|
| 533 |
+
review = [c for c in candidates if c.get("status") == "review_required"]
|
| 534 |
+
accepted = [c for c in candidates if c.get("status") == "accepted_seed"]
|
| 535 |
+
rejected = [c for c in candidates if c.get("status") == "rejected_by_identity_filter"]
|
| 536 |
+
|
| 537 |
+
lines = [
|
| 538 |
+
f"# Paper Discovery Report — {identity.case_id}",
|
| 539 |
+
"",
|
| 540 |
+
f"Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}",
|
| 541 |
+
"",
|
| 542 |
+
"## Case Identity",
|
| 543 |
+
"",
|
| 544 |
+
f"- Name: {identity.name}",
|
| 545 |
+
f"- Location: {identity.location}",
|
| 546 |
+
f"- Country: {identity.country}",
|
| 547 |
+
f"- Aliases: {', '.join(identity.aliases) if identity.aliases else 'None'}",
|
| 548 |
+
"",
|
| 549 |
+
"## Queries",
|
| 550 |
+
"",
|
| 551 |
+
]
|
| 552 |
+
lines.extend(f"- `{q}`" for q in queries)
|
| 553 |
+
lines.extend([
|
| 554 |
+
"",
|
| 555 |
+
"## Accepted Seed Sources",
|
| 556 |
+
"",
|
| 557 |
+
])
|
| 558 |
+
if accepted:
|
| 559 |
+
for c in sorted(accepted, key=lambda x: x.get("confidence_score", 0), reverse=True):
|
| 560 |
+
lines.extend(_candidate_md(c))
|
| 561 |
+
else:
|
| 562 |
+
lines.append("No accepted seed sources found.")
|
| 563 |
+
|
| 564 |
+
lines.extend([
|
| 565 |
+
"",
|
| 566 |
+
"## Review Required",
|
| 567 |
+
"",
|
| 568 |
+
"These are not automatically accepted. Approve only if the full text truly covers the case or governing policy.",
|
| 569 |
+
"",
|
| 570 |
+
])
|
| 571 |
+
if review:
|
| 572 |
+
for c in sorted(review, key=lambda x: x.get("confidence_score", 0), reverse=True):
|
| 573 |
+
lines.extend(_candidate_md(c))
|
| 574 |
+
else:
|
| 575 |
+
lines.append("No new candidates passed the review threshold.")
|
| 576 |
+
|
| 577 |
+
lines.extend([
|
| 578 |
+
"",
|
| 579 |
+
"## Rejected / Background",
|
| 580 |
+
"",
|
| 581 |
+
f"{len(rejected)} candidates did not pass strict identity matching.",
|
| 582 |
+
])
|
| 583 |
+
(case_dir / "paper_discovery_report.md").write_text("\n".join(lines) + "\n")
|
| 584 |
+
|
| 585 |
+
|
| 586 |
+
def _candidate_md(c: dict[str, Any]) -> list[str]:
|
| 587 |
+
title = c.get("title") or "(untitled)"
|
| 588 |
+
authors = ", ".join(c.get("authors", [])[:4])
|
| 589 |
+
if len(c.get("authors", [])) > 4:
|
| 590 |
+
authors += ", et al."
|
| 591 |
+
abstract = c.get("abstract", "")
|
| 592 |
+
abstract = textwrap.shorten(re.sub(r"\s+", " ", abstract), width=500, placeholder=" ...") if abstract else ""
|
| 593 |
+
lines = [
|
| 594 |
+
f"### {title}",
|
| 595 |
+
"",
|
| 596 |
+
f"- Status: `{c.get('status')}`",
|
| 597 |
+
f"- Class: `{c.get('source_class')}`",
|
| 598 |
+
f"- Score: {c.get('confidence_score')}",
|
| 599 |
+
f"- Authors: {authors or 'Unknown'}",
|
| 600 |
+
f"- Year: {c.get('year') or 'Unknown'}",
|
| 601 |
+
f"- Venue: {c.get('venue') or 'Unknown'}",
|
| 602 |
+
f"- DOI: {c.get('doi') or 'None'}",
|
| 603 |
+
f"- URL: {c.get('url') or 'None'}",
|
| 604 |
+
f"- PDF URL: {c.get('pdf_url') or 'None'}",
|
| 605 |
+
f"- Local PDF: {c.get('local_pdf') or 'None'}",
|
| 606 |
+
f"- Evidence: {c.get('acceptance_reason') or 'None'}",
|
| 607 |
+
]
|
| 608 |
+
if abstract:
|
| 609 |
+
lines.extend(["", f"> {abstract}"])
|
| 610 |
+
lines.append("")
|
| 611 |
+
return lines
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
def discover(case_id: str, aliases: list[str], max_results: int) -> tuple[Path, list[dict[str, Any]]]:
|
| 615 |
+
case_dir = find_case_dir(case_id.upper())
|
| 616 |
+
if not case_dir:
|
| 617 |
+
raise FileNotFoundError(f"No case folder found for {case_id}")
|
| 618 |
+
|
| 619 |
+
identity = load_case_identity(case_dir, aliases)
|
| 620 |
+
seed_entries = parse_sources_txt(case_dir)
|
| 621 |
+
doi_entries = parse_dois_txt(case_dir)
|
| 622 |
+
for doi in doi_entries:
|
| 623 |
+
if doi and not any(e.get("doi") == doi for e in seed_entries):
|
| 624 |
+
seed_entries.append({
|
| 625 |
+
"role": "source_from_dois_txt",
|
| 626 |
+
"status": "accepted_seed",
|
| 627 |
+
"citation": "",
|
| 628 |
+
"doi": doi,
|
| 629 |
+
"source": "dois.txt",
|
| 630 |
+
})
|
| 631 |
+
|
| 632 |
+
seed_candidates = []
|
| 633 |
+
for entry in seed_entries:
|
| 634 |
+
resolved = resolve_crossref_by_doi(entry.get("doi", "")) if entry.get("doi") else None
|
| 635 |
+
if not resolved and entry.get("citation"):
|
| 636 |
+
hits = search_crossref(entry["citation"], rows=1)
|
| 637 |
+
resolved = hits[0] if hits else None
|
| 638 |
+
candidate = resolved or {
|
| 639 |
+
"title": infer_title_from_citation(entry.get("citation", "")),
|
| 640 |
+
"authors": [],
|
| 641 |
+
"year": None,
|
| 642 |
+
"doi": entry.get("doi", ""),
|
| 643 |
+
"venue": "",
|
| 644 |
+
"url": f"https://doi.org/{entry.get('doi')}" if entry.get("doi") else "",
|
| 645 |
+
"abstract": "",
|
| 646 |
+
"source_api": "local_sources",
|
| 647 |
+
"type": "unknown",
|
| 648 |
+
}
|
| 649 |
+
candidate["seed_role"] = entry.get("role")
|
| 650 |
+
candidate["seed_citation"] = entry.get("citation")
|
| 651 |
+
candidate["seed_source"] = entry.get("source")
|
| 652 |
+
seed_candidates.append(candidate)
|
| 653 |
+
|
| 654 |
+
queries = build_queries(identity, seed_entries)
|
| 655 |
+
discovered = []
|
| 656 |
+
for query in queries:
|
| 657 |
+
discovered.extend(search_semantic_scholar(query, limit=max_results))
|
| 658 |
+
discovered.extend(search_openalex(query, per_page=max_results))
|
| 659 |
+
discovered.extend(search_crossref(query, rows=max_results))
|
| 660 |
+
|
| 661 |
+
all_candidates = merge_candidates(seed_candidates + discovered)
|
| 662 |
+
seed_dois = {e.get("doi") for e in seed_entries if e.get("doi")}
|
| 663 |
+
seed_titles = {_title_key(infer_title_from_citation(e.get("citation", ""))) for e in seed_entries if e.get("citation")}
|
| 664 |
+
seed_titles = {t for t in seed_titles if t}
|
| 665 |
+
|
| 666 |
+
classified = []
|
| 667 |
+
for candidate in all_candidates:
|
| 668 |
+
candidate = classify_candidate(candidate, identity, seed_dois, seed_titles)
|
| 669 |
+
local_pdf = local_pdf_matches(case_dir, candidate)
|
| 670 |
+
if local_pdf:
|
| 671 |
+
candidate["local_pdf"] = local_pdf
|
| 672 |
+
classified.append(candidate)
|
| 673 |
+
|
| 674 |
+
classified.sort(key=lambda c: (c.get("status") != "accepted_seed", -c.get("confidence_score", 0), c.get("title", "")))
|
| 675 |
+
write_sources_json(case_dir, identity, classified)
|
| 676 |
+
write_candidates(case_dir, classified)
|
| 677 |
+
write_report(case_dir, identity, queries, classified)
|
| 678 |
+
return case_dir, classified
|
| 679 |
+
|
| 680 |
+
|
| 681 |
+
def main(argv: list[str] | None = None) -> None:
|
| 682 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 683 |
+
parser.add_argument("--case-id", required=True, help="Case ID, e.g. CS36")
|
| 684 |
+
parser.add_argument("--alias", action="append", default=[], help="Additional accepted location/program alias. Repeatable.")
|
| 685 |
+
parser.add_argument("--max-results", type=int, default=5, help="Results per query per API (default: 5)")
|
| 686 |
+
args = parser.parse_args(argv)
|
| 687 |
+
|
| 688 |
+
case_dir, candidates = discover(args.case_id, args.alias, args.max_results)
|
| 689 |
+
accepted = sum(1 for c in candidates if c.get("status") == "accepted_seed")
|
| 690 |
+
review = sum(1 for c in candidates if c.get("status") == "review_required")
|
| 691 |
+
rejected = sum(1 for c in candidates if c.get("status") == "rejected_by_identity_filter")
|
| 692 |
+
print(f"Discovery complete for {args.case_id.upper()} -> {case_dir}")
|
| 693 |
+
print(f" accepted seed sources: {accepted}")
|
| 694 |
+
print(f" review-required candidates: {review}")
|
| 695 |
+
print(f" rejected/background candidates: {rejected}")
|
| 696 |
+
print(f" wrote: {case_dir / 'sources.json'}")
|
| 697 |
+
print(f" wrote: {case_dir / 'paper_candidates.json'}")
|
| 698 |
+
print(f" wrote: {case_dir / 'paper_discovery_report.md'}")
|
| 699 |
+
|
| 700 |
+
|
| 701 |
+
if __name__ == "__main__":
|
| 702 |
+
main()
|
case_study_pipeline/fetch_papers.py
CHANGED
|
@@ -20,22 +20,31 @@ Usage:
|
|
| 20 |
# Skip Drive upload:
|
| 21 |
python3 -m case_study_pipeline.fetch_papers --case-id CS13 --no-upload
|
| 22 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
Options:
|
| 24 |
--scihub-mirror Sci-Hub mirror URL (default: https://sci-hub.se)
|
| 25 |
--aa-api-key Anna's Archive API key (see annas-archive.se/faq#api)
|
| 26 |
--no-upload Skip Drive upload after download
|
|
|
|
| 27 |
"""
|
| 28 |
|
| 29 |
from __future__ import annotations
|
| 30 |
|
| 31 |
import argparse
|
| 32 |
import json
|
|
|
|
| 33 |
import sys
|
| 34 |
-
import tempfile
|
| 35 |
import shutil
|
|
|
|
|
|
|
| 36 |
from pathlib import Path
|
|
|
|
| 37 |
|
| 38 |
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
| 39 |
|
| 40 |
# Load .env so UWATERLOO_USER/PASS and GOOGLE credentials are available
|
| 41 |
try:
|
|
@@ -64,6 +73,19 @@ def get_dois_for_case(case_dir: Path, dois_arg: list[str] | None) -> list[str]:
|
|
| 64 |
return []
|
| 65 |
|
| 66 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
def fetch_and_download(
|
| 68 |
dois: list[str],
|
| 69 |
dest_dir: Path,
|
|
@@ -110,6 +132,145 @@ def fetch_and_download(
|
|
| 110 |
return new_pdfs
|
| 111 |
|
| 112 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
def main():
|
| 114 |
parser = argparse.ArgumentParser(
|
| 115 |
description="Download source PDFs for an RFR case study via PyPaperBot."
|
|
@@ -131,13 +292,18 @@ def main():
|
|
| 131 |
"--no-upload", action="store_true",
|
| 132 |
help="Skip uploading downloaded PDFs to Google Drive",
|
| 133 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
args = parser.parse_args()
|
| 135 |
|
| 136 |
case_id = args.case_id.upper()
|
| 137 |
-
case_dir =
|
| 138 |
|
| 139 |
-
if
|
| 140 |
-
print(f"ERROR: Case directory not found
|
| 141 |
sys.exit(1)
|
| 142 |
|
| 143 |
dois = get_dois_for_case(case_dir, args.dois)
|
|
@@ -172,8 +338,17 @@ def main():
|
|
| 172 |
print(" No new PDFs downloaded.")
|
| 173 |
print(f"{'='*60}\n")
|
| 174 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
# Drive upload
|
| 176 |
-
if new_pdfs and not args.no_upload:
|
| 177 |
folders = load_drive_folders()
|
| 178 |
folder_id = folders.get(case_id)
|
| 179 |
if not folder_id:
|
|
@@ -187,7 +362,7 @@ def main():
|
|
| 187 |
try_upload(uploader, pdf)
|
| 188 |
else:
|
| 189 |
print(" [Drive] No uploader available — skipping upload.")
|
| 190 |
-
elif args.no_upload:
|
| 191 |
print(" [Drive] Upload skipped (--no-upload).")
|
| 192 |
|
| 193 |
print("\nDone.")
|
|
|
|
| 20 |
# Skip Drive upload:
|
| 21 |
python3 -m case_study_pipeline.fetch_papers --case-id CS13 --no-upload
|
| 22 |
|
| 23 |
+
# If automated sources fail, open UWaterloo EZproxy in Chrome and wait
|
| 24 |
+
# while you complete WatIAM/Duo and download the PDF:
|
| 25 |
+
python3 -m case_study_pipeline.fetch_papers --case-id CS38 --manual-ezproxy
|
| 26 |
+
|
| 27 |
Options:
|
| 28 |
--scihub-mirror Sci-Hub mirror URL (default: https://sci-hub.se)
|
| 29 |
--aa-api-key Anna's Archive API key (see annas-archive.se/faq#api)
|
| 30 |
--no-upload Skip Drive upload after download
|
| 31 |
+
--manual-ezproxy Open Chrome for browser-assisted UWaterloo access
|
| 32 |
"""
|
| 33 |
|
| 34 |
from __future__ import annotations
|
| 35 |
|
| 36 |
import argparse
|
| 37 |
import json
|
| 38 |
+
import re
|
| 39 |
import sys
|
|
|
|
| 40 |
import shutil
|
| 41 |
+
import subprocess
|
| 42 |
+
import webbrowser
|
| 43 |
from pathlib import Path
|
| 44 |
+
from urllib.parse import quote
|
| 45 |
|
| 46 |
REPO_ROOT = Path(__file__).resolve().parent.parent
|
| 47 |
+
INBOX_DIR = REPO_ROOT / "data" / "raw" / "_inbox"
|
| 48 |
|
| 49 |
# Load .env so UWATERLOO_USER/PASS and GOOGLE credentials are available
|
| 50 |
try:
|
|
|
|
| 73 |
return []
|
| 74 |
|
| 75 |
|
| 76 |
+
def find_case_dir(case_id: str) -> Path | None:
|
| 77 |
+
"""Find data/raw/CSxx or data/raw/CSxx-Place_Name style folders."""
|
| 78 |
+
exact = REPO_ROOT / "data" / "raw" / case_id
|
| 79 |
+
if exact.exists():
|
| 80 |
+
return exact
|
| 81 |
+
|
| 82 |
+
raw_dir = REPO_ROOT / "data" / "raw"
|
| 83 |
+
for path in raw_dir.iterdir():
|
| 84 |
+
if path.is_dir() and re.match(rf"^{re.escape(case_id)}($|-)", path.name):
|
| 85 |
+
return path
|
| 86 |
+
return None
|
| 87 |
+
|
| 88 |
+
|
| 89 |
def fetch_and_download(
|
| 90 |
dois: list[str],
|
| 91 |
dest_dir: Path,
|
|
|
|
| 132 |
return new_pdfs
|
| 133 |
|
| 134 |
|
| 135 |
+
def safe_pdf_filename(title: str, fallback: str = "downloaded_paper") -> str:
|
| 136 |
+
"""Return a filesystem-safe PDF name."""
|
| 137 |
+
stem = re.sub(r"[^\w\-_. ]", "_", title).strip(" ._")
|
| 138 |
+
return f"{stem or fallback}.pdf"
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def display_path(path: Path) -> Path:
|
| 142 |
+
"""Prefer repo-relative paths in CLI output."""
|
| 143 |
+
try:
|
| 144 |
+
return path.relative_to(REPO_ROOT)
|
| 145 |
+
except ValueError:
|
| 146 |
+
return path
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def resolve_doi_target_url(doi: str) -> tuple[str, str]:
|
| 150 |
+
"""Return a likely publisher PDF/article URL plus title for manual browser access."""
|
| 151 |
+
try:
|
| 152 |
+
import requests
|
| 153 |
+
|
| 154 |
+
url = f"https://api.crossref.org/works/{quote(doi, safe='')}"
|
| 155 |
+
resp = requests.get(url, timeout=20, headers={"User-Agent": "rfr-rag-fetch-papers/0.1"})
|
| 156 |
+
if resp.ok:
|
| 157 |
+
msg = resp.json().get("message", {})
|
| 158 |
+
title = (msg.get("title") or [""])[0]
|
| 159 |
+
links = msg.get("link") or []
|
| 160 |
+
for link in links:
|
| 161 |
+
link_url = link.get("URL") or ""
|
| 162 |
+
if "pdf" in link_url.lower():
|
| 163 |
+
return link_url, title
|
| 164 |
+
if msg.get("URL"):
|
| 165 |
+
return msg["URL"], title
|
| 166 |
+
except Exception as exc:
|
| 167 |
+
print(f" [Browser] Crossref lookup failed for {doi}: {exc}")
|
| 168 |
+
|
| 169 |
+
return f"https://doi.org/{doi}", doi
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def to_ezproxy_url(url: str) -> str:
|
| 173 |
+
from .ezproxy import to_ezproxy_url as convert
|
| 174 |
+
|
| 175 |
+
return convert(url)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def open_in_chrome(url: str) -> None:
|
| 179 |
+
"""Open URL in Google Chrome, falling back to the default browser."""
|
| 180 |
+
try:
|
| 181 |
+
subprocess.run(["open", "-a", "Google Chrome", url], check=True)
|
| 182 |
+
except Exception:
|
| 183 |
+
webbrowser.open(url)
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def file_browser_downloads(
|
| 187 |
+
*,
|
| 188 |
+
case_id: str,
|
| 189 |
+
case_dir: Path,
|
| 190 |
+
before_inbox: set[Path],
|
| 191 |
+
dois: list[str],
|
| 192 |
+
titles: dict[str, str],
|
| 193 |
+
no_upload: bool,
|
| 194 |
+
) -> list[Path]:
|
| 195 |
+
"""Move newly downloaded inbox PDFs into the case folder and optionally upload them."""
|
| 196 |
+
INBOX_DIR.mkdir(parents=True, exist_ok=True)
|
| 197 |
+
after_inbox = set(INBOX_DIR.glob("*.pdf"))
|
| 198 |
+
new_downloads = sorted(after_inbox - before_inbox)
|
| 199 |
+
|
| 200 |
+
if not new_downloads:
|
| 201 |
+
print(" [Browser] No new PDFs found in data/raw/_inbox/.")
|
| 202 |
+
return []
|
| 203 |
+
|
| 204 |
+
moved: list[Path] = []
|
| 205 |
+
for src in new_downloads:
|
| 206 |
+
if len(new_downloads) == 1 and len(dois) == 1:
|
| 207 |
+
dest_name = safe_pdf_filename(titles.get(dois[0], ""), src.stem)
|
| 208 |
+
else:
|
| 209 |
+
dest_name = src.name
|
| 210 |
+
|
| 211 |
+
dest = case_dir / dest_name
|
| 212 |
+
n = 1
|
| 213 |
+
while dest.exists():
|
| 214 |
+
n += 1
|
| 215 |
+
dest = case_dir / f"({n}){dest_name}"
|
| 216 |
+
shutil.move(str(src), str(dest))
|
| 217 |
+
moved.append(dest)
|
| 218 |
+
print(f" [Browser] Moved {src.name} -> {display_path(dest)}")
|
| 219 |
+
|
| 220 |
+
if moved and not no_upload:
|
| 221 |
+
folders = load_drive_folders()
|
| 222 |
+
folder_id = folders.get(case_id)
|
| 223 |
+
if not folder_id:
|
| 224 |
+
print(f" [Drive] No folder ID found for {case_id} in drive_folders.json — skipping upload.")
|
| 225 |
+
else:
|
| 226 |
+
from .drive_upload import make_uploader, try_upload
|
| 227 |
+
|
| 228 |
+
uploader = make_uploader(folder_id=folder_id)
|
| 229 |
+
if uploader:
|
| 230 |
+
for pdf in moved:
|
| 231 |
+
print(f" [Drive] Uploading {pdf.name}...")
|
| 232 |
+
try_upload(uploader, pdf)
|
| 233 |
+
else:
|
| 234 |
+
print(" [Drive] No uploader available — skipping upload.")
|
| 235 |
+
elif no_upload:
|
| 236 |
+
print(" [Drive] Upload skipped (--no-upload).")
|
| 237 |
+
|
| 238 |
+
return moved
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def browser_ezproxy_flow(case_id: str, case_dir: Path, dois: list[str], no_upload: bool) -> list[Path]:
|
| 242 |
+
"""Open failed DOI targets through UWaterloo EZproxy and wait for manual download."""
|
| 243 |
+
INBOX_DIR.mkdir(parents=True, exist_ok=True)
|
| 244 |
+
before_inbox = set(INBOX_DIR.glob("*.pdf"))
|
| 245 |
+
titles: dict[str, str] = {}
|
| 246 |
+
|
| 247 |
+
print("\n" + "=" * 60)
|
| 248 |
+
print(" Browser-assisted UWaterloo EZproxy")
|
| 249 |
+
print(" Chrome will open each DOI through EZproxy.")
|
| 250 |
+
print(" Complete WatIAM/Duo if asked, then click Download PDF.")
|
| 251 |
+
print(f" Save location should be: {INBOX_DIR}")
|
| 252 |
+
print("=" * 60 + "\n")
|
| 253 |
+
|
| 254 |
+
for doi in dois:
|
| 255 |
+
target_url, title = resolve_doi_target_url(doi)
|
| 256 |
+
titles[doi] = title
|
| 257 |
+
proxy_url = to_ezproxy_url(target_url)
|
| 258 |
+
print(f" DOI: {doi}")
|
| 259 |
+
print(f" Opening: {proxy_url}")
|
| 260 |
+
open_in_chrome(proxy_url)
|
| 261 |
+
|
| 262 |
+
input("\nPress Enter after Chrome has finished downloading the PDF(s) into data/raw/_inbox/...")
|
| 263 |
+
|
| 264 |
+
return file_browser_downloads(
|
| 265 |
+
case_id=case_id,
|
| 266 |
+
case_dir=case_dir,
|
| 267 |
+
before_inbox=before_inbox,
|
| 268 |
+
dois=dois,
|
| 269 |
+
titles=titles,
|
| 270 |
+
no_upload=no_upload,
|
| 271 |
+
)
|
| 272 |
+
|
| 273 |
+
|
| 274 |
def main():
|
| 275 |
parser = argparse.ArgumentParser(
|
| 276 |
description="Download source PDFs for an RFR case study via PyPaperBot."
|
|
|
|
| 292 |
"--no-upload", action="store_true",
|
| 293 |
help="Skip uploading downloaded PDFs to Google Drive",
|
| 294 |
)
|
| 295 |
+
parser.add_argument(
|
| 296 |
+
"--manual-ezproxy",
|
| 297 |
+
action="store_true",
|
| 298 |
+
help="If automated download finds no PDFs, open UWaterloo EZproxy URLs in Chrome and wait for manual download",
|
| 299 |
+
)
|
| 300 |
args = parser.parse_args()
|
| 301 |
|
| 302 |
case_id = args.case_id.upper()
|
| 303 |
+
case_dir = find_case_dir(case_id)
|
| 304 |
|
| 305 |
+
if case_dir is None:
|
| 306 |
+
print(f"ERROR: Case directory not found for {case_id} under {REPO_ROOT / 'data' / 'raw'}")
|
| 307 |
sys.exit(1)
|
| 308 |
|
| 309 |
dois = get_dois_for_case(case_dir, args.dois)
|
|
|
|
| 338 |
print(" No new PDFs downloaded.")
|
| 339 |
print(f"{'='*60}\n")
|
| 340 |
|
| 341 |
+
if not new_pdfs and args.manual_ezproxy:
|
| 342 |
+
browser_pdfs = browser_ezproxy_flow(
|
| 343 |
+
case_id=case_id,
|
| 344 |
+
case_dir=case_dir,
|
| 345 |
+
dois=dois,
|
| 346 |
+
no_upload=args.no_upload,
|
| 347 |
+
)
|
| 348 |
+
new_pdfs.extend(browser_pdfs)
|
| 349 |
+
|
| 350 |
# Drive upload
|
| 351 |
+
if new_pdfs and not args.no_upload and not args.manual_ezproxy:
|
| 352 |
folders = load_drive_folders()
|
| 353 |
folder_id = folders.get(case_id)
|
| 354 |
if not folder_id:
|
|
|
|
| 362 |
try_upload(uploader, pdf)
|
| 363 |
else:
|
| 364 |
print(" [Drive] No uploader available — skipping upload.")
|
| 365 |
+
elif args.no_upload and not args.manual_ezproxy:
|
| 366 |
print(" [Drive] Upload skipped (--no-upload).")
|
| 367 |
|
| 368 |
print("\nDone.")
|
case_study_pipeline/pdf_utils.py
CHANGED
|
@@ -14,6 +14,11 @@ import pdfplumber
|
|
| 14 |
from fpdf import FPDF
|
| 15 |
|
| 16 |
SOURCE_EXTENSIONS = {".pdf", ".txt", ".md"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
# Roughly maps "smart" unicode punctuation (and a few common symbols) to ASCII so
|
| 19 |
# the core PDF fonts (latin-1 only) can render them without crashing.
|
|
@@ -61,7 +66,11 @@ def extract_source_documents(folder: Path, max_chars_per_file: int | None = None
|
|
| 61 |
|
| 62 |
files = sorted(
|
| 63 |
p for p in folder.iterdir()
|
| 64 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
)
|
| 66 |
if not files:
|
| 67 |
raise FileNotFoundError(
|
|
|
|
| 14 |
from fpdf import FPDF
|
| 15 |
|
| 16 |
SOURCE_EXTENSIONS = {".pdf", ".txt", ".md"}
|
| 17 |
+
SOURCE_METADATA_FILENAMES = {
|
| 18 |
+
"dois.txt",
|
| 19 |
+
"paper_discovery_report.md",
|
| 20 |
+
"sources.txt",
|
| 21 |
+
}
|
| 22 |
|
| 23 |
# Roughly maps "smart" unicode punctuation (and a few common symbols) to ASCII so
|
| 24 |
# the core PDF fonts (latin-1 only) can render them without crashing.
|
|
|
|
| 66 |
|
| 67 |
files = sorted(
|
| 68 |
p for p in folder.iterdir()
|
| 69 |
+
if (
|
| 70 |
+
p.is_file()
|
| 71 |
+
and p.suffix.lower() in SOURCE_EXTENSIONS
|
| 72 |
+
and p.name not in SOURCE_METADATA_FILENAMES
|
| 73 |
+
)
|
| 74 |
)
|
| 75 |
if not files:
|
| 76 |
raise FileNotFoundError(
|
case_study_pipeline/select_best_run.py
CHANGED
|
@@ -36,10 +36,21 @@ _RUN_DIR_RE = re.compile(r"^run_\d+", re.IGNORECASE)
|
|
| 36 |
# ---------------------------------------------------------------------------
|
| 37 |
|
| 38 |
def _all_case_ids() -> list[str]:
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
if p.is_dir()
|
| 42 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
|
| 45 |
def _grade_run(run_dir: Path, case_id: str) -> int:
|
|
@@ -80,7 +91,12 @@ def _write_selected(
|
|
| 80 |
# ---------------------------------------------------------------------------
|
| 81 |
|
| 82 |
def select_for_case(case_id: str, force: bool = False) -> None:
|
| 83 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
|
| 85 |
if not pipeline_output.exists():
|
| 86 |
print(f" [{case_id}] No pipeline_output/ — skipping.")
|
|
|
|
| 36 |
# ---------------------------------------------------------------------------
|
| 37 |
|
| 38 |
def _all_case_ids() -> list[str]:
|
| 39 |
+
ids = []
|
| 40 |
+
for p in DATA_RAW.iterdir():
|
| 41 |
+
if p.is_dir():
|
| 42 |
+
m = re.match(r"^(CS\d+)", p.name)
|
| 43 |
+
if m:
|
| 44 |
+
ids.append(m.group(1))
|
| 45 |
+
return sorted(set(ids), key=lambda x: int(x[2:]))
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _find_case_folder(case_id: str) -> Path | None:
|
| 49 |
+
"""Find a case folder named either CS37 or CS37-Kabyaza_RWA."""
|
| 50 |
+
for p in DATA_RAW.iterdir():
|
| 51 |
+
if p.is_dir() and re.match(rf"^{re.escape(case_id)}($|-)", p.name):
|
| 52 |
+
return p
|
| 53 |
+
return None
|
| 54 |
|
| 55 |
|
| 56 |
def _grade_run(run_dir: Path, case_id: str) -> int:
|
|
|
|
| 91 |
# ---------------------------------------------------------------------------
|
| 92 |
|
| 93 |
def select_for_case(case_id: str, force: bool = False) -> None:
|
| 94 |
+
case_folder = _find_case_folder(case_id)
|
| 95 |
+
if case_folder is None:
|
| 96 |
+
print(f" [{case_id}] No folder found in data/raw/. Skipping.")
|
| 97 |
+
return
|
| 98 |
+
|
| 99 |
+
pipeline_output = case_folder / "pipeline_output"
|
| 100 |
|
| 101 |
if not pipeline_output.exists():
|
| 102 |
print(f" [{case_id}] No pipeline_output/ — skipping.")
|
case_study_pipeline/sort_inbox.py
CHANGED
|
@@ -41,6 +41,8 @@ try:
|
|
| 41 |
except ImportError:
|
| 42 |
pass
|
| 43 |
|
|
|
|
|
|
|
| 44 |
|
| 45 |
def load_drive_folders() -> dict:
|
| 46 |
if not DRIVE_FOLDERS_FILE.exists():
|
|
@@ -81,9 +83,9 @@ def main():
|
|
| 81 |
sys.exit(1)
|
| 82 |
|
| 83 |
case_id = args.case_id.upper()
|
| 84 |
-
dest_dir =
|
| 85 |
-
if not dest_dir
|
| 86 |
-
print(f"ERROR: case directory not found
|
| 87 |
sys.exit(1)
|
| 88 |
|
| 89 |
dest_name = args.rename if args.rename else args.filename
|
|
|
|
| 41 |
except ImportError:
|
| 42 |
pass
|
| 43 |
|
| 44 |
+
from .fetch_papers import find_case_dir
|
| 45 |
+
|
| 46 |
|
| 47 |
def load_drive_folders() -> dict:
|
| 48 |
if not DRIVE_FOLDERS_FILE.exists():
|
|
|
|
| 83 |
sys.exit(1)
|
| 84 |
|
| 85 |
case_id = args.case_id.upper()
|
| 86 |
+
dest_dir = find_case_dir(case_id)
|
| 87 |
+
if not dest_dir:
|
| 88 |
+
print(f"ERROR: case directory not found for {case_id} under {REPO_ROOT / 'data' / 'raw'}")
|
| 89 |
sys.exit(1)
|
| 90 |
|
| 91 |
dest_name = args.rename if args.rename else args.filename
|
data/raw/CS36-Greenville_NC_USA/case_meta.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"case_id": "CS36",
|
| 3 |
"name": "Greenville",
|
| 4 |
-
"location": "Greenville, North
|
| 5 |
"country": "United States"
|
| 6 |
-
}
|
|
|
|
| 1 |
{
|
| 2 |
"case_id": "CS36",
|
| 3 |
"name": "Greenville",
|
| 4 |
+
"location": "Greenville, North Carolina",
|
| 5 |
"country": "United States"
|
| 6 |
+
}
|