LeoMjl commited on
Commit
7eac020
·
verified ·
1 Parent(s): e85cc8b

Upload 2 files

Browse files
Files changed (2) hide show
  1. fetch_from_Ss.py +257 -0
  2. fetch_from_openalex.py +241 -0
fetch_from_Ss.py ADDED
@@ -0,0 +1,257 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import csv
3
+ import yaml
4
+ import time
5
+ import requests
6
+ import concurrent.futures
7
+ from pathlib import Path
8
+
9
+ API_KEY = "YOUR_API_KEY_HERE"
10
+
11
+ BASE_DIR = Path(__file__).resolve().parent.parent
12
+ DATA_DIR = BASE_DIR / "data"
13
+ DATA_FALSE_DIR = BASE_DIR / "data-false"
14
+ CONFIG_FILE = BASE_DIR / "config" / "venues_top.yaml"
15
+
16
+ HEADERS = {"x-api-key": API_KEY}
17
+
18
+ def request_with_retry(url, params=None, max_retries=5):
19
+ for i in range(max_retries):
20
+ try:
21
+ response = requests.get(url, params=params, headers=HEADERS, timeout=15)
22
+ if response.status_code == 200:
23
+ return response
24
+ elif response.status_code == 429:
25
+ wait_time = 5 * (i + 1)
26
+ print(f"Rate limit exceeded (429). Waiting {wait_time}s... (Attempt {i+1}/{max_retries})")
27
+ time.sleep(wait_time)
28
+ continue
29
+ elif 500 <= response.status_code < 600:
30
+ print(f"Server error ({response.status_code}). Retrying... (Attempt {i+1}/{max_retries})")
31
+ time.sleep(2)
32
+ continue
33
+ else:
34
+ return response
35
+ except requests.exceptions.RequestException as e:
36
+ print(f"Network error: {e}. Retrying... (Attempt {i+1}/{max_retries})")
37
+ time.sleep(2)
38
+
39
+ print(f"Failed to fetch {url} after {max_retries} attempts.")
40
+ return None
41
+
42
+ def load_config():
43
+ if not CONFIG_FILE.exists():
44
+ print(f"Config file not found: {CONFIG_FILE}")
45
+ return set(), set()
46
+
47
+ with open(CONFIG_FILE, 'r', encoding='utf-8') as f:
48
+ config = yaml.safe_load(f)
49
+
50
+ top_confs = set(conf.lower() for conf in config.get('top_conferences', []))
51
+ top_journals = set(jour.lower() for jour in config.get('top_journals', []))
52
+
53
+ return top_confs, top_journals
54
+
55
+ def search_paper(title):
56
+ url = "https://api.semanticscholar.org/graph/v1/paper/search"
57
+ params = {
58
+ "query": title,
59
+ "limit": 1,
60
+ "fields": "paperId,title,year"
61
+ }
62
+
63
+ response = request_with_retry(url, params)
64
+
65
+ if response and response.status_code == 200:
66
+ data = response.json()
67
+ if data.get('data'):
68
+ return data['data'][0]
69
+ elif response:
70
+ print(f"Error searching paper: {response.status_code} - {response.text}")
71
+
72
+ return None
73
+
74
+ def get_citations(paper_id):
75
+ citations = []
76
+ offset = 0
77
+ limit = 1000
78
+ total = 0
79
+
80
+ detail_url = f"https://api.semanticscholar.org/graph/v1/paper/{paper_id}"
81
+ r = request_with_retry(detail_url, params={"fields": "citationCount"})
82
+
83
+ if r and r.status_code == 200:
84
+ total = r.json().get('citationCount', 0)
85
+ else:
86
+ return [], 0
87
+
88
+ if total == 0:
89
+ return [], 0
90
+
91
+ print(f"Fetching {total} citations for paper {paper_id}...")
92
+
93
+ while True:
94
+ url = f"https://api.semanticscholar.org/graph/v1/paper/{paper_id}/citations"
95
+ params = {
96
+ "fields": "year,venue",
97
+ "offset": offset,
98
+ "limit": limit
99
+ }
100
+
101
+ response = request_with_retry(url, params)
102
+
103
+ if response and response.status_code == 200:
104
+ data = response.json()
105
+ batch = data.get('data', [])
106
+ if not batch:
107
+ break
108
+
109
+ citations.extend(batch)
110
+ offset += len(batch)
111
+
112
+ if offset >= total or len(batch) < limit:
113
+ break
114
+
115
+ time.sleep(1.0)
116
+ else:
117
+ print(f"Error fetching citations page.")
118
+ break
119
+
120
+ return citations, total
121
+
122
+ def is_top_venue(venue_name, top_list):
123
+ if not venue_name:
124
+ return False
125
+ v = venue_name.lower()
126
+ for top in top_list:
127
+ if top in v:
128
+ return True
129
+ return False
130
+
131
+ def process_single_row(row, top_confs, top_journals):
132
+ title = row.get('title')
133
+ if not title:
134
+ return None, None
135
+
136
+ paper_info = search_paper(title)
137
+
138
+ if paper_info:
139
+ paper_id = paper_info['paperId']
140
+ row['paperId'] = paper_id
141
+
142
+ citations, count = get_citations(paper_id)
143
+ row['citationCount'] = count
144
+
145
+ top_conf_count = 0
146
+ top_journal_count = 0
147
+ year_counts = {y: 0 for y in range(2014, 2025)}
148
+
149
+ for cit in citations:
150
+ citing_paper = cit.get('citingPaper', {})
151
+ if not citing_paper:
152
+ continue
153
+
154
+ year = citing_paper.get('year')
155
+ if year and 2014 <= year <= 2024:
156
+ year_counts[year] += 1
157
+
158
+ venue = citing_paper.get('venue')
159
+ if is_top_venue(venue, top_confs):
160
+ top_conf_count += 1
161
+ elif is_top_venue(venue, top_journals):
162
+ top_journal_count += 1
163
+
164
+ row['top_conf_citations'] = top_conf_count
165
+ row['top_journal_citations'] = top_journal_count
166
+ for year in range(2014, 2025):
167
+ row[f'citations_{year}'] = year_counts[year]
168
+
169
+ return row, True
170
+ else:
171
+ return row, False
172
+
173
+ def process_files():
174
+ top_confs, top_journals = load_config()
175
+
176
+ if not DATA_DIR.exists():
177
+ print(f"Data directory not found: {DATA_DIR}")
178
+ return
179
+
180
+ files = [f for f in os.listdir(DATA_DIR) if f.endswith('.csv')]
181
+
182
+ for filename in files:
183
+ file_path = DATA_DIR / filename
184
+ false_file_path = DATA_FALSE_DIR / filename.replace('.csv', '.txt')
185
+
186
+ DATA_FALSE_DIR.mkdir(parents=True, exist_ok=True)
187
+
188
+ print(f"\nProcessing file: {filename}")
189
+
190
+ not_found_papers = []
191
+ fieldnames = []
192
+
193
+ with open(file_path, 'r', encoding='utf-8') as f:
194
+ reader = csv.DictReader(f)
195
+ fieldnames = reader.fieldnames
196
+ rows = list(reader)
197
+
198
+ if false_file_path.exists():
199
+ with open(false_file_path, 'r', encoding='utf-8') as f:
200
+ not_found_papers = [line.strip() for line in f.readlines()]
201
+
202
+ total_rows = len(rows)
203
+ batch_size = 10
204
+ max_workers = 3
205
+
206
+ indices_to_process = []
207
+ for i, row in enumerate(rows):
208
+ if row.get('title') and not (row.get('paperId') and row.get('citationCount')):
209
+ indices_to_process.append(i)
210
+
211
+ print(f"Total rows: {total_rows}, Rows to process: {len(indices_to_process)}")
212
+
213
+ for i in range(0, len(indices_to_process), batch_size):
214
+ batch_indices = indices_to_process[i:i + batch_size]
215
+
216
+ print(f"Processing batch {i//batch_size + 1}/{(len(indices_to_process) + batch_size - 1)//batch_size} (Rows {batch_indices[0]+1}-{batch_indices[-1]+1})...")
217
+
218
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
219
+ future_to_index = {
220
+ executor.submit(process_single_row, rows[idx], top_confs, top_journals): idx
221
+ for idx in batch_indices
222
+ }
223
+
224
+ for future in concurrent.futures.as_completed(future_to_index):
225
+ idx = future_to_index[future]
226
+ try:
227
+ updated_row, found = future.result()
228
+ if updated_row:
229
+ rows[idx] = updated_row
230
+ title = updated_row.get('title')
231
+
232
+ if found:
233
+ print(f" [✓] Found: {title[:40]}...")
234
+ if title in not_found_papers:
235
+ not_found_papers.remove(title)
236
+ else:
237
+ print(f" [x] Not Found: {title[:40]}...")
238
+ if title not in not_found_papers:
239
+ not_found_papers.append(title)
240
+ except Exception as exc:
241
+ print(f"Row {idx} generated an exception: {exc}")
242
+
243
+ print(f"Saving progress after batch...")
244
+ with open(file_path, 'w', encoding='utf-8', newline='') as f:
245
+ writer = csv.DictWriter(f, fieldnames=fieldnames)
246
+ writer.writeheader()
247
+ writer.writerows(rows)
248
+
249
+ if not_found_papers:
250
+ with open(false_file_path, 'w', encoding='utf-8') as f:
251
+ for t in not_found_papers:
252
+ f.write(f"{t}\n")
253
+
254
+ time.sleep(1)
255
+
256
+ if __name__ == "__main__":
257
+ process_files()
fetch_from_openalex.py ADDED
@@ -0,0 +1,241 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import csv
3
+ import yaml
4
+ import time
5
+ import requests
6
+ import concurrent.futures
7
+ from urllib.parse import quote
8
+ from difflib import SequenceMatcher
9
+ from pathlib import Path
10
+
11
+ BASE_DIR = Path(__file__).resolve().parent.parent
12
+ DATA_DIR = BASE_DIR / "data"
13
+ DATA_FALSE_DIR = BASE_DIR / "data-false"
14
+ CONFIG_FILE = BASE_DIR / "config" / "venues_top.yaml"
15
+
16
+ HEADERS = {
17
+ "User-Agent": "mailto:your_email@example.com"
18
+ }
19
+
20
+ def load_config():
21
+ if not CONFIG_FILE.exists():
22
+ print(f"Config file not found: {CONFIG_FILE}")
23
+ return set(), set()
24
+
25
+ with open(CONFIG_FILE, 'r', encoding='utf-8') as f:
26
+ config = yaml.safe_load(f)
27
+
28
+ top_confs = set(conf.lower() for conf in config.get('top_conferences', []))
29
+ top_journals = set(jour.lower() for jour in config.get('top_journals', []))
30
+
31
+ return top_confs, top_journals
32
+
33
+ def is_top_venue(venue_name, top_list):
34
+ if not venue_name:
35
+ return False
36
+ v = venue_name.lower()
37
+ for top in top_list:
38
+ if top in v:
39
+ return True
40
+ return False
41
+
42
+ def calculate_similarity(a, b):
43
+ if not a or not b:
44
+ return 0.0
45
+ return SequenceMatcher(None, a.lower(), b.lower()).ratio()
46
+
47
+ def search_openalex(title):
48
+ encoded_title = quote(title)
49
+ url = f"https://api.openalex.org/works?filter=title.search:{encoded_title}&per-page=5"
50
+
51
+ try:
52
+ response = requests.get(url, headers=HEADERS, timeout=15)
53
+ if response.status_code == 200:
54
+ data = response.json()
55
+ results = data.get('results', [])
56
+
57
+ if not results:
58
+ return None
59
+
60
+ best_match = None
61
+ highest_similarity = 0.0
62
+ threshold = 0.85
63
+
64
+ for result in results:
65
+ result_title = result.get('display_name', '')
66
+ similarity = calculate_similarity(title, result_title)
67
+
68
+ if similarity > highest_similarity:
69
+ highest_similarity = similarity
70
+ best_match = result
71
+
72
+ if highest_similarity >= threshold:
73
+ return best_match
74
+ else:
75
+ pass
76
+
77
+ except Exception as e:
78
+ print(f"Error searching OpenAlex for '{title}': {e}")
79
+
80
+ return None
81
+
82
+ def get_citations_from_openalex(work_id):
83
+ citations_data = []
84
+ short_id = work_id.split('/')[-1]
85
+ page = 1
86
+ per_page = 200
87
+
88
+ while True:
89
+ url = f"https://api.openalex.org/works?filter=cites:{short_id}&per-page={per_page}&page={page}&select=publication_year,primary_location"
90
+
91
+ try:
92
+ response = requests.get(url, headers=HEADERS, timeout=20)
93
+ if response.status_code == 200:
94
+ data = response.json()
95
+ results = data.get('results', [])
96
+ if not results:
97
+ break
98
+
99
+ citations_data.extend(results)
100
+ meta = data.get('meta', {})
101
+ count = meta.get('count', 0)
102
+ if len(citations_data) >= count:
103
+ break
104
+
105
+ page += 1
106
+ time.sleep(0.5)
107
+ else:
108
+ print(f"Error fetching citations: {response.status_code}")
109
+ break
110
+ except Exception as e:
111
+ print(f"Error fetching citations: {e}")
112
+ break
113
+
114
+ return citations_data
115
+
116
+ def process_single_paper(title, top_confs, top_journals):
117
+ paper = search_openalex(title)
118
+
119
+ if not paper:
120
+ return None
121
+ work_id = paper.get('id')
122
+ citation_count = paper.get('cited_by_count', 0)
123
+ if citation_count == 0:
124
+ return {
125
+ 'paperId': work_id,
126
+ 'citationCount': 0,
127
+ 'top_conf_citations': 0,
128
+ 'top_journal_citations': 0,
129
+ 'year_counts': {y: 0 for y in range(2014, 2025)}
130
+ }
131
+
132
+ citing_works = get_citations_from_openalex(work_id)
133
+
134
+ top_conf_count = 0
135
+ top_journal_count = 0
136
+ year_counts = {y: 0 for y in range(2014, 2025)}
137
+
138
+ for work in citing_works:
139
+ year = work.get('publication_year')
140
+ if year and 2014 <= year <= 2024:
141
+ year_counts[year] += 1
142
+
143
+ loc = work.get('primary_location') or {}
144
+ source = loc.get('source') or {}
145
+ display_name = source.get('display_name')
146
+
147
+ if display_name:
148
+ if is_top_venue(display_name, top_confs):
149
+ top_conf_count += 1
150
+ elif is_top_venue(display_name, top_journals):
151
+ top_journal_count += 1
152
+
153
+ return {
154
+ 'paperId': work_id,
155
+ 'citationCount': citation_count,
156
+ 'top_conf_citations': top_conf_count,
157
+ 'top_journal_citations': top_journal_count,
158
+ 'year_counts': year_counts
159
+ }
160
+
161
+ def process_false_files():
162
+ top_confs, top_journals = load_config()
163
+
164
+ if not DATA_FALSE_DIR.exists():
165
+ print(f"False data directory not found: {DATA_FALSE_DIR}")
166
+ return
167
+
168
+ txt_files = [f for f in os.listdir(DATA_FALSE_DIR) if f.endswith('.txt')]
169
+
170
+ for txt_file in txt_files:
171
+ print(f"\nProcessing false file: {txt_file}")
172
+ txt_path = DATA_FALSE_DIR / txt_file
173
+ csv_file = txt_file.replace('.txt', '.csv')
174
+ csv_path = DATA_DIR / csv_file
175
+
176
+ if not csv_path.exists():
177
+ print(f"Warning: Corresponding CSV file {csv_file} not found.")
178
+ continue
179
+
180
+ with open(txt_path, 'r', encoding='utf-8') as f:
181
+ titles_to_search = [line.strip() for line in f.readlines() if line.strip()]
182
+
183
+ if not titles_to_search:
184
+ print("No titles to search in this file.")
185
+ continue
186
+
187
+ print(f"Found {len(titles_to_search)} titles to retry with OpenAlex.")
188
+
189
+ rows = []
190
+ fieldnames = []
191
+ with open(csv_path, 'r', encoding='utf-8') as f:
192
+ reader = csv.DictReader(f)
193
+ fieldnames = reader.fieldnames
194
+ rows = list(reader)
195
+
196
+ titles_found = []
197
+ max_workers = 5
198
+
199
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
200
+ future_to_title = {
201
+ executor.submit(process_single_paper, title, top_confs, top_journals): title
202
+ for title in titles_to_search
203
+ }
204
+
205
+ for future in concurrent.futures.as_completed(future_to_title):
206
+ title = future_to_title[future]
207
+ try:
208
+ result = future.result()
209
+ if result:
210
+ print(f" [✓] Found in OpenAlex: {title[:40]}...")
211
+ titles_found.append(title)
212
+ for row in rows:
213
+ if row['title'].strip() == title.strip():
214
+ row['paperId'] = result['paperId']
215
+ row['citationCount'] = result['citationCount']
216
+ row['top_conf_citations'] = result['top_conf_citations']
217
+ row['top_journal_citations'] = result['top_journal_citations']
218
+ for year, count in result['year_counts'].items():
219
+ row[f'citations_{year}'] = count
220
+ break
221
+ else:
222
+ print(f" [x] Not found in OpenAlex: {title[:40]}...")
223
+ except Exception as e:
224
+ print(f"Error processing {title}: {e}")
225
+
226
+ if titles_found:
227
+ print(f"Updating {csv_file} with {len(titles_found)} new records...")
228
+ with open(csv_path, 'w', encoding='utf-8', newline='') as f:
229
+ writer = csv.DictWriter(f, fieldnames=fieldnames)
230
+ writer.writeheader()
231
+ writer.writerows(rows)
232
+ remaining_titles = [t for t in titles_to_search if t not in titles_found]
233
+ with open(txt_path, 'w', encoding='utf-8') as f:
234
+ for t in remaining_titles:
235
+ f.write(f"{t}\n")
236
+ print(f"Removed found titles from {txt_file}. Remaining: {len(remaining_titles)}")
237
+ else:
238
+ print("No new papers found.")
239
+
240
+ if __name__ == "__main__":
241
+ process_false_files()