LeoMjl commited on
Commit
c36ef00
·
verified ·
1 Parent(s): 71cd1f1

Upload classify_papers.py

Browse files
Files changed (1) hide show
  1. classify_papers.py +176 -0
classify_papers.py ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pandas as pd
2
+ from openai import OpenAI
3
+ import time
4
+ import json
5
+ import os
6
+ import glob
7
+
8
+ # Configuration for API Key and Base URL
9
+ # Please replace 'YOUR_API_KEY' with your actual API key
10
+ API_KEY = "YOUR_API_KEY"
11
+ BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
12
+
13
+ client = OpenAI(
14
+ api_key=API_KEY,
15
+ base_url=BASE_URL,
16
+ )
17
+
18
+ # Define categories
19
+ CATEGORIES = [
20
+ "Machine Learning (including Deep Learning)",
21
+ "Representation Learning and Optimization",
22
+ "Probability, Statistics, and Inference",
23
+ "Natural Language Processing",
24
+ "Computer Vision",
25
+ "Multimodal Learning",
26
+ "AI Foundations and Theory",
27
+ "Reinforcement Learning and Decision Making",
28
+ "Interpretability, Fairness, and Applied Systems"
29
+ ]
30
+
31
+ def clean_title(title):
32
+ if not isinstance(title, str):
33
+ return str(title)
34
+ return title.replace('\\', ' ').replace('"', "'").replace('\n', ' ').strip()
35
+
36
+ def get_classifications(titles):
37
+ cleaned_titles = [clean_title(t) for t in titles]
38
+
39
+ prompt = f"""
40
+ As an AI expert, please classify the following papers into one of the categories below.
41
+ Select strictly from the provided list:
42
+ {', '.join(CATEGORIES)}
43
+
44
+ Return the result in JSON format as follows:
45
+ {{
46
+ "results": [
47
+ {{"title": "Paper Title 1", "category": "Category Name"}},
48
+ ...
49
+ ]
50
+ }}
51
+
52
+ Note:
53
+ 1. The "title" field in the JSON must match the provided title exactly (including symbols).
54
+ 2. If the title contains LaTeX formulas or special characters, keep them as is. Do not escape or modify them to ensure valid JSON.
55
+
56
+ Papers to classify:
57
+ """
58
+ for i, title in enumerate(cleaned_titles):
59
+ prompt += f"{i+1}. {title}\n"
60
+
61
+ max_retries = 3
62
+ for attempt in range(max_retries):
63
+ try:
64
+ completion = client.chat.completions.create(
65
+ model="deepseek-v3",
66
+ messages=[
67
+ {'role': 'user', 'content': prompt}
68
+ ],
69
+ temperature=0.1,
70
+ response_format={"type": "json_object"}
71
+ )
72
+
73
+ content = completion.choices[0].message.content
74
+
75
+ try:
76
+ if "```json" in content:
77
+ content = content.split("```json")[1].split("```")[0]
78
+ elif "```" in content:
79
+ content = content.split("```")[1].split("```")[0]
80
+
81
+ parsed_data = json.loads(content)
82
+ return parsed_data.get("results", [])
83
+ except json.JSONDecodeError:
84
+ print(f"JSON parsing failed (Attempt {attempt + 1}/{max_retries}), raw content: {content[:100]}...")
85
+
86
+ except Exception as e:
87
+ print(f"API call error (Attempt {attempt + 1}/{max_retries}): {e}")
88
+
89
+ if attempt < max_retries - 1:
90
+ wait_time = 2 * (attempt + 1)
91
+ print(f"Waiting {wait_time} seconds before retrying...")
92
+ time.sleep(wait_time)
93
+
94
+ return []
95
+
96
+ def process_file(file_path):
97
+ print(f"\n{'='*50}")
98
+ print(f"Processing file: {os.path.basename(file_path)}")
99
+ print(f"{'='*50}")
100
+
101
+ print(f"Reading file {file_path}...")
102
+ df = pd.read_csv(file_path)
103
+
104
+ if 'ai_category' not in df.columns:
105
+ df['ai_category'] = None
106
+
107
+ pending_mask = df['ai_category'].isna()
108
+ pending_indices = df[pending_mask].index.tolist()
109
+
110
+ total_pending = len(pending_indices)
111
+ print(f"Total papers pending classification: {total_pending}")
112
+
113
+ if total_pending == 0:
114
+ print("All papers in this file have been classified!")
115
+ return
116
+
117
+ batch_size = 5
118
+ save_interval = 10
119
+
120
+ for i in range(0, total_pending, batch_size):
121
+ current_batch_indices = pending_indices[i : i + batch_size]
122
+ batch_titles = df.loc[current_batch_indices, 'title'].tolist()
123
+ print(f"Processing {i+1} to {min(i+batch_size, total_pending)} (Progress: {i}/{total_pending})...")
124
+
125
+ results = get_classifications(batch_titles)
126
+
127
+ if results and len(results) > 0:
128
+ title_to_category = {item['title']: item['category'] for item in results}
129
+
130
+ for idx, title in zip(current_batch_indices, batch_titles):
131
+ cleaned_title = clean_title(title)
132
+ category = title_to_category.get(title)
133
+
134
+ if not category:
135
+ category = title_to_category.get(cleaned_title)
136
+
137
+ if not category:
138
+ try:
139
+ relative_index = current_batch_indices.index(idx)
140
+ if relative_index < len(results):
141
+ category = results[relative_index].get('category')
142
+ except ValueError:
143
+ pass
144
+
145
+ if category:
146
+ df.at[idx, 'ai_category'] = category
147
+ else:
148
+ print(f"Warning: No valid results obtained for this batch.")
149
+
150
+ if (i // batch_size + 1) % save_interval == 0:
151
+ print("Saving intermediate results to source file...")
152
+ df.to_csv(file_path, index=False)
153
+
154
+ time.sleep(1)
155
+
156
+ df.to_csv(file_path, index=False)
157
+ print(f"File processing complete! Results updated in: {file_path}")
158
+
159
+ def main():
160
+ data_dir = "./data"
161
+
162
+ if not os.path.exists(data_dir):
163
+ print(f"Data directory '{data_dir}' does not exist. Please create it and add your CSV files.")
164
+ return
165
+
166
+ csv_files = glob.glob(os.path.join(data_dir, "*.final.csv"))
167
+
168
+ print(f"Found {len(csv_files)} conference files to check/process...")
169
+
170
+ for file_path in csv_files:
171
+ process_file(file_path)
172
+
173
+ print("\nAll files processed!")
174
+
175
+ if __name__ == "__main__":
176
+ main()