derangedsaturnian commited on
Commit
7ce7bc2
·
verified ·
1 Parent(s): f35aa24

Create scripts/download_pdfs.py

Browse files
Files changed (1) hide show
  1. scripts/download_pdfs.py +228 -0
scripts/download_pdfs.py ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ import requests
3
+ import os
4
+ import re
5
+ import time
6
+ import json
7
+ from pathlib import Path
8
+ from xml.etree import ElementTree as ET
9
+
10
+
11
+ CSV_PATH = "./data/SB_publication_PMC.csv"
12
+ OUTPUT_DIR = "./data/processed_articles"
13
+ METADATA_FILE = "./data/articles_metadata.json"
14
+
15
+ os.makedirs(OUTPUT_DIR, exist_ok=True)
16
+
17
+ df = pd.read_csv(CSV_PATH)
18
+ print(f"Total articles to process: {len(df)}")
19
+
20
+ def sanitize_filename(title):
21
+ """title clean"""
22
+ if pd.isna(title):
23
+ return "untitled"
24
+ clean = re.sub(r'[\\/*?:"<>|]', "", str(title))
25
+ clean = clean.strip().replace(" ", "_")
26
+ return clean[:150] # Limit length
27
+
28
+ def extract_pmc_content(pmc_id):
29
+ pmc_id_num = pmc_id.replace('PMC', '')
30
+
31
+ url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
32
+ params = {
33
+ 'db': 'pmc',
34
+ 'id': pmc_id_num,
35
+ 'rettype': 'xml',
36
+ 'retmode': 'xml'
37
+ }
38
+ try:
39
+ response = requests.get(url, params=params, timeout=60)
40
+ if response.status_code != 200:
41
+ return None, f"HTTP {response.status_code}"
42
+
43
+ root = ET.fromstring(response.content)
44
+
45
+ article_data = {
46
+ 'pmc_id': pmc_id,
47
+ 'title': '',
48
+ 'abstract': '',
49
+ 'introduction': '',
50
+ 'methods': '',
51
+ 'results': '',
52
+ 'discussion': '',
53
+ 'conclusion': '',
54
+ 'full_text': '',
55
+ 'authors': [],
56
+ 'keywords': [],
57
+ 'journal': '',
58
+ 'pub_date': ''
59
+ }
60
+
61
+ title_elem = root.find('.//article-title')
62
+ if title_elem is not None:
63
+ article_data['title'] = ET.tostring(title_elem, encoding='unicode', method='text').strip()
64
+
65
+ for author in root.findall('.//contrib[@contrib-type="author"]'):
66
+ surname = author.find('.//surname')
67
+ given = author.find('.//given-names')
68
+ if surname is not None:
69
+ name = surname.text
70
+ if given is not None:
71
+ name = f"{given.text} {name}"
72
+ article_data['authors'].append(name)
73
+
74
+ journal_elem = root.find('.//journal-title')
75
+ if journal_elem is not None:
76
+ article_data['journal'] = journal_elem.text
77
+
78
+ pub_date = root.find('.//pub-date')
79
+ if pub_date is not None:
80
+ year = pub_date.find('year')
81
+ month = pub_date.find('month')
82
+ if year is not None:
83
+ article_data['pub_date'] = year.text
84
+ if month is not None:
85
+ article_data['pub_date'] = f"{month.text}/{year.text}"
86
+ for kwd in root.findall('.//kwd'):
87
+ if kwd.text:
88
+ article_data['keywords'].append(kwd.text.strip())
89
+
90
+ abstract = root.find('.//abstract')
91
+ if abstract is not None:
92
+ article_data['abstract'] = ET.tostring(abstract, encoding='unicode', method='text').strip()
93
+
94
+ body = root.find('.//body')
95
+ if body is not None:
96
+ for sec in body.findall('.//sec'):
97
+ title_elem = sec.find('.//title')
98
+ if title_elem is not None:
99
+ section_title = title_elem.text.lower() if title_elem.text else ''
100
+ section_text = ET.tostring(sec, encoding='unicode', method='text').strip()
101
+
102
+ if 'introduction' in section_title or 'background' in section_title:
103
+ article_data['introduction'] += section_text + "\n\n"
104
+ elif 'method' in section_title or 'material' in section_title:
105
+ article_data['methods'] += section_text + "\n\n"
106
+ elif 'result' in section_title:
107
+ article_data['results'] += section_text + "\n\n"
108
+ elif 'discussion' in section_title:
109
+ article_data['discussion'] += section_text + "\n\n"
110
+ elif 'conclusion' in section_title:
111
+ article_data['conclusion'] += section_text + "\n\n"
112
+ else:
113
+
114
+ article_data['full_text'] += section_text + "\n\n"
115
+
116
+ full_text_parts = [
117
+ f"Title: {article_data['title']}",
118
+ f"\nAuthors: {', '.join(article_data['authors'])}",
119
+ f"\nJournal: {article_data['journal']} ({article_data['pub_date']})",
120
+ f"\nKeywords: {', '.join(article_data['keywords'])}",
121
+ f"\n\nABSTRACT:\n{article_data['abstract']}",
122
+ f"\n\nINTRODUCTION:\n{article_data['introduction']}",
123
+ f"\n\nMETHODS:\n{article_data['methods']}",
124
+ f"\n\nRESULTS:\n{article_data['results']}",
125
+ f"\n\nDISCUSSION:\n{article_data['discussion']}",
126
+ f"\n\nCONCLUSION:\n{article_data['conclusion']}",
127
+ f"\n\nADDITIONAL CONTENT:\n{article_data['full_text']}"
128
+ ]
129
+ article_data['full_text'] = "\n".join(filter(None, full_text_parts))
130
+
131
+ return article_data, None
132
+ except ET.ParseError as e:
133
+ return None, f"XML parse error: {str(e)}"
134
+ except Exception as e:
135
+ return None, f"Error: {str(e)}"
136
+
137
+ print("\n" + "="*60)
138
+ print("starting article extraction...")
139
+ print("usig NCBI E-utilities API (rate: 3 req/sec)")
140
+ print("="*60 + "\n")
141
+
142
+ successful = 0
143
+ failed = 0
144
+ metadata_list = []
145
+
146
+ for idx, row in df.iterrows():
147
+ title = row['Title']
148
+ url = row['Link']
149
+
150
+ if pd.isna(url):
151
+ print(f"[{idx+1}/{len(df)}] Skipping - no URL")
152
+ failed += 1
153
+ continue
154
+ pmc_match = re.search(r'PMC\d+', url)
155
+ if not pmc_match:
156
+ print(f"[{idx+1}/{len(df)}] Skipping - no PMC ID in {url}")
157
+ failed += 1
158
+ continue
159
+
160
+ pmc_id = pmc_match.group(0)
161
+
162
+ output_file = f"{OUTPUT_DIR}/{pmc_id}.txt"
163
+ json_file = f"{OUTPUT_DIR}/{pmc_id}.json"
164
+
165
+ if os.path.exists(output_file) and os.path.exists(json_file):
166
+ print(f"[{idx+1}/{len(df)}] {pmc_id} - Already processed, skipping")
167
+ successful += 1
168
+ continue
169
+
170
+ print(f"[{idx+1}/{len(df)}] {pmc_id}: {title[:50]}...")
171
+
172
+
173
+ article_data, error = extract_pmc_content(pmc_id)
174
+
175
+ if article_data:
176
+
177
+ with open(output_file, 'w', encoding='utf-8') as f:
178
+ f.write(article_data['full_text'])
179
+
180
+ article_meta = {
181
+ 'pmc_id': pmc_id,
182
+ 'title': article_data['title'],
183
+ 'authors': article_data['authors'],
184
+ 'journal': article_data['journal'],
185
+ 'pub_date': article_data['pub_date'],
186
+ 'keywords': article_data['keywords'],
187
+ 'abstract': article_data['abstract'],
188
+ 'url': url,
189
+ 'sections': {
190
+ 'introduction_length': len(article_data['introduction']),
191
+ 'methods_length': len(article_data['methods']),
192
+ 'results_length': len(article_data['results']),
193
+ 'discussion_length': len(article_data['discussion']),
194
+ 'conclusion_length': len(article_data['conclusion'])
195
+ }
196
+ }
197
+
198
+ with open(json_file, 'w', encoding='utf-8') as f:
199
+ json.dump(article_data, f, indent=2)
200
+
201
+ metadata_list.append(article_meta)
202
+
203
+ print(f"saved ({len(article_data['full_text'])} chars)")
204
+ successful += 1
205
+ else:
206
+ print(f"failed: {error}")
207
+ failed += 1
208
+ time.sleep(0.35)
209
+
210
+ if (idx + 1) % 50 == 0:
211
+ with open(METADATA_FILE, 'w', encoding='utf-8') as f:
212
+ json.dump(metadata_list, f, indent=2)
213
+ print(f"\n progress saved: {successful} successful, {failed} failed\n")
214
+
215
+ with open(METADATA_FILE, 'w', encoding='utf-8') as f:
216
+ json.dump(metadata_list, f, indent=2)
217
+
218
+ print("\n" + "="*60)
219
+ print("Extraction complete!")
220
+ print(f"Successful: {successful}")
221
+ print(f"Failed: {failed}")
222
+ print(f"Total: {successful + failed}")
223
+ print(f"\nOutput:")
224
+ print(f" - Full texts: {OUTPUT_DIR}/PMC*.txt")
225
+ print(f" - Structured JSON: {OUTPUT_DIR}/PMC*.json")
226
+ print(f" - Metadata index: {METADATA_FILE}")
227
+ print("="*60)
228
+ print("####################")