Spaces:
Sleeping
Sleeping
File size: 2,891 Bytes
e960612 951bd56 9676219 e960612 4d1b66a c69f364 e960612 17f5452 4b31614 c69f364 e960612 c69f364 e960612 17f5452 4b31614 17f5452 4b31614 17f5452 e960612 c69f364 e960612 4b31614 e960612 4b31614 e960612 951bd56 e960612 c69f364 e7e92a4 4b31614 e7e92a4 c69f364 e7e92a4 c69f364 e7e92a4 9676219 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | import logging
import shutil
import tempfile
import zipfile
from pathlib import Path
import country_converter as coco
import polars as pl
import pyalex
import requests
from config import (
BEACON_PATH,
DATAVERSE_API_KEY,
OPENALEX_API_KEY,
ROR_PATH,
)
logger = logging.getLogger(__name__)
DATAVERSE_URL = 'https://dataverse.harvard.edu/api/access/dataset/:persistentId/?persistentId=doi:10.7910/DVN/OCZNVY'
pyalex.config.api_key = OPENALEX_API_KEY
def download_beacon_dataset() -> None:
"""Download beacon.csv from Harvard Dataverse."""
logger.info('Downloading beacon.csv')
response = requests.get(
DATAVERSE_URL,
headers={'X-Dataverse-key': DATAVERSE_API_KEY},
stream=True,
timeout=60,
)
BEACON_PATH.parent.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir_path = Path(tmpdir)
zip_path = tmpdir_path / 'dataset.zip'
zip_path.write_bytes(response.content)
with zipfile.ZipFile(zip_path, 'r') as z:
z.extractall(tmpdir_path)
shutil.move(tmpdir_path / 'beacon.tab', BEACON_PATH)
(
pl.scan_csv(BEACON_PATH, separator='\t')
# TODO: filter out journals with less than X articles?
# .filter(pl.col('record_count_2025') > 0)
.with_columns(pl.col('country').str.replace_all('NA', 'none'))
.with_columns(
pl.col('country')
.alias('country_code')
.map_batches(
lambda x: pl.Series(
[c[0] for c in coco.convert(x, to='ISO2', enforce_list=True)]
),
),
)
.sink_csv(BEACON_PATH)
)
logger.info('Download completed')
def download_ror_dataset() -> None:
"""Download ROR csv dataset to ror_data.csv."""
logger.info('Starting to download ROR entities.')
r = requests.get('https://zenodo.org/api/records/18985120', timeout=10).json()
r = requests.get(r['links']['files'], timeout=10).json()
r = requests.get(r['entries'][0]['links']['content'], timeout=60)
logger.info('Download complete. Starting to unpack and update database.')
ROR_PATH.parent.mkdir(exist_ok=True)
with tempfile.TemporaryDirectory() as tmp:
tmppath = Path(tmp)
zippath = tmppath / 'resp.zip'
extractpath = tmppath / 'content'
with zippath.open('wb') as f:
f.write(r.content)
with zipfile.ZipFile(zippath, 'r') as zip_ref:
zip_ref.extractall(extractpath)
csvpath = next(extractpath.glob('*csv'))
shutil.move(csvpath, ROR_PATH)
logger.info('Done.')
def get_journals() -> pl.LazyFrame:
"""Get a dataframe of all OJS journals in the beacon.csv"""
logger.info('Reading OJS journals.')
if not BEACON_PATH.exists():
download_beacon_dataset()
return pl.scan_csv(BEACON_PATH)
|