Download fetch_from_openalex.py from acl-crown-analysis/acl-crown-analysis-code: direct link, hf CLI and curl.
- Browser
- Download file 8.43 kB
-
https://huggingface.co/acl-crown-analysis/acl-crown-analysis-code/resolve/main/fetch_from_openalex.py
- Command line
-
hf download hf://acl-crown-analysis/acl-crown-analysis-code/fetch_from_openalex.py
-
curl -L -o fetch_from_openalex.py https://huggingface.co/acl-crown-analysis/acl-crown-analysis-code/resolve/main/fetch_from_openalex.py
8.43 kB
| import os | |
| import csv | |
| import yaml | |
| import time | |
| import requests | |
| import concurrent.futures | |
| from urllib.parse import quote | |
| from difflib import SequenceMatcher | |
| from pathlib import Path | |
| BASE_DIR = Path(__file__).resolve().parent.parent | |
| DATA_DIR = BASE_DIR / "data" | |
| DATA_FALSE_DIR = BASE_DIR / "data-false" | |
| CONFIG_FILE = BASE_DIR / "config" / "venues_top.yaml" | |
| HEADERS = { | |
| "User-Agent": "mailto:your_email@example.com" | |
| } | |
| def load_config(): | |
| if not CONFIG_FILE.exists(): | |
| print(f"Config file not found: {CONFIG_FILE}") | |
| return set(), set() | |
| with open(CONFIG_FILE, 'r', encoding='utf-8') as f: | |
| config = yaml.safe_load(f) | |
| top_confs = set(conf.lower() for conf in config.get('top_conferences', [])) | |
| top_journals = set(jour.lower() for jour in config.get('top_journals', [])) | |
| return top_confs, top_journals | |
| def is_top_venue(venue_name, top_list): | |
| if not venue_name: | |
| return False | |
| v = venue_name.lower() | |
| for top in top_list: | |
| if top in v: | |
| return True | |
| return False | |
| def calculate_similarity(a, b): | |
| if not a or not b: | |
| return 0.0 | |
| return SequenceMatcher(None, a.lower(), b.lower()).ratio() | |
| def search_openalex(title): | |
| encoded_title = quote(title) | |
| url = f"https://api.openalex.org/works?filter=title.search:{encoded_title}&per-page=5" | |
| try: | |
| response = requests.get(url, headers=HEADERS, timeout=15) | |
| if response.status_code == 200: | |
| data = response.json() | |
| results = data.get('results', []) | |
| if not results: | |
| return None | |
| best_match = None | |
| highest_similarity = 0.0 | |
| threshold = 0.85 | |
| for result in results: | |
| result_title = result.get('display_name', '') | |
| similarity = calculate_similarity(title, result_title) | |
| if similarity > highest_similarity: | |
| highest_similarity = similarity | |
| best_match = result | |
| if highest_similarity >= threshold: | |
| return best_match | |
| else: | |
| pass | |
| except Exception as e: | |
| print(f"Error searching OpenAlex for '{title}': {e}") | |
| return None | |
| def get_citations_from_openalex(work_id): | |
| citations_data = [] | |
| short_id = work_id.split('/')[-1] | |
| page = 1 | |
| per_page = 200 | |
| while True: | |
| url = f"https://api.openalex.org/works?filter=cites:{short_id}&per-page={per_page}&page={page}&select=publication_year,primary_location" | |
| try: | |
| response = requests.get(url, headers=HEADERS, timeout=20) | |
| if response.status_code == 200: | |
| data = response.json() | |
| results = data.get('results', []) | |
| if not results: | |
| break | |
| citations_data.extend(results) | |
| meta = data.get('meta', {}) | |
| count = meta.get('count', 0) | |
| if len(citations_data) >= count: | |
| break | |
| page += 1 | |
| time.sleep(0.5) | |
| else: | |
| print(f"Error fetching citations: {response.status_code}") | |
| break | |
| except Exception as e: | |
| print(f"Error fetching citations: {e}") | |
| break | |
| return citations_data | |
| def process_single_paper(title, top_confs, top_journals): | |
| paper = search_openalex(title) | |
| if not paper: | |
| return None | |
| work_id = paper.get('id') | |
| citation_count = paper.get('cited_by_count', 0) | |
| if citation_count == 0: | |
| return { | |
| 'paperId': work_id, | |
| 'citationCount': 0, | |
| 'top_conf_citations': 0, | |
| 'top_journal_citations': 0, | |
| 'year_counts': {y: 0 for y in range(2014, 2025)} | |
| } | |
| citing_works = get_citations_from_openalex(work_id) | |
| top_conf_count = 0 | |
| top_journal_count = 0 | |
| year_counts = {y: 0 for y in range(2014, 2025)} | |
| for work in citing_works: | |
| year = work.get('publication_year') | |
| if year and 2014 <= year <= 2024: | |
| year_counts[year] += 1 | |
| loc = work.get('primary_location') or {} | |
| source = loc.get('source') or {} | |
| display_name = source.get('display_name') | |
| if display_name: | |
| if is_top_venue(display_name, top_confs): | |
| top_conf_count += 1 | |
| elif is_top_venue(display_name, top_journals): | |
| top_journal_count += 1 | |
| return { | |
| 'paperId': work_id, | |
| 'citationCount': citation_count, | |
| 'top_conf_citations': top_conf_count, | |
| 'top_journal_citations': top_journal_count, | |
| 'year_counts': year_counts | |
| } | |
| def process_false_files(): | |
| top_confs, top_journals = load_config() | |
| if not DATA_FALSE_DIR.exists(): | |
| print(f"False data directory not found: {DATA_FALSE_DIR}") | |
| return | |
| txt_files = [f for f in os.listdir(DATA_FALSE_DIR) if f.endswith('.txt')] | |
| for txt_file in txt_files: | |
| print(f"\nProcessing false file: {txt_file}") | |
| txt_path = DATA_FALSE_DIR / txt_file | |
| csv_file = txt_file.replace('.txt', '.csv') | |
| csv_path = DATA_DIR / csv_file | |
| if not csv_path.exists(): | |
| print(f"Warning: Corresponding CSV file {csv_file} not found.") | |
| continue | |
| with open(txt_path, 'r', encoding='utf-8') as f: | |
| titles_to_search = [line.strip() for line in f.readlines() if line.strip()] | |
| if not titles_to_search: | |
| print("No titles to search in this file.") | |
| continue | |
| print(f"Found {len(titles_to_search)} titles to retry with OpenAlex.") | |
| rows = [] | |
| fieldnames = [] | |
| with open(csv_path, 'r', encoding='utf-8') as f: | |
| reader = csv.DictReader(f) | |
| fieldnames = reader.fieldnames | |
| rows = list(reader) | |
| titles_found = [] | |
| max_workers = 5 | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: | |
| future_to_title = { | |
| executor.submit(process_single_paper, title, top_confs, top_journals): title | |
| for title in titles_to_search | |
| } | |
| for future in concurrent.futures.as_completed(future_to_title): | |
| title = future_to_title[future] | |
| try: | |
| result = future.result() | |
| if result: | |
| print(f" [✓] Found in OpenAlex: {title[:40]}...") | |
| titles_found.append(title) | |
| for row in rows: | |
| if row['title'].strip() == title.strip(): | |
| row['paperId'] = result['paperId'] | |
| row['citationCount'] = result['citationCount'] | |
| row['top_conf_citations'] = result['top_conf_citations'] | |
| row['top_journal_citations'] = result['top_journal_citations'] | |
| for year, count in result['year_counts'].items(): | |
| row[f'citations_{year}'] = count | |
| break | |
| else: | |
| print(f" [x] Not found in OpenAlex: {title[:40]}...") | |
| except Exception as e: | |
| print(f"Error processing {title}: {e}") | |
| if titles_found: | |
| print(f"Updating {csv_file} with {len(titles_found)} new records...") | |
| with open(csv_path, 'w', encoding='utf-8', newline='') as f: | |
| writer = csv.DictWriter(f, fieldnames=fieldnames) | |
| writer.writeheader() | |
| writer.writerows(rows) | |
| remaining_titles = [t for t in titles_to_search if t not in titles_found] | |
| with open(txt_path, 'w', encoding='utf-8') as f: | |
| for t in remaining_titles: | |
| f.write(f"{t}\n") | |
| print(f"Removed found titles from {txt_file}. Remaining: {len(remaining_titles)}") | |
| else: | |
| print("No new papers found.") | |
| if __name__ == "__main__": | |
| process_false_files() | |