Spaces:
Runtime error
Runtime error
| import streamlit as st | |
| import requests | |
| import pandas as pd | |
| import xml.etree.ElementTree as ET | |
| import unicodedata | |
| import re | |
| from collections import Counter | |
| def clean_text(text): | |
| text = text.replace('"', '\"') # Escape double quotes | |
| text = text.replace('\n', ' ') # Replace newlines with space | |
| text = unicodedata.normalize("NFKD", text) # Normalize Unicode characters | |
| text = re.sub(r'\s+', ' ', text).strip() # Remove extra whitespace | |
| return text | |
| def fetch_arxiv_data(search_term, max_results=50): | |
| url = f"http://export.arxiv.org/api/query?search_query={search_term}&start=0&max_results={max_results}" | |
| response = requests.get(url) | |
| response.raise_for_status() | |
| root = ET.fromstring(response.content) | |
| columns = ["ID", "Title", "Authors", "Summary", "Published Date", "Categories", "DOI", "Journal Reference", "Primary Category", "Link", "Comments", "Subjects", "Version", "PDF Link"] | |
| df = pd.DataFrame(columns=columns) | |
| for entry in root.findall("{http://www.w3.org/2005/Atom}entry"): | |
| arxiv_id = entry.find("{http://www.w3.org/2005/Atom}id").text | |
| title = clean_text(entry.find("{http://www.w3.org/2005/Atom}title").text) | |
| summary = clean_text(entry.find("{http://www.w3.org/2005/Atom}summary").text) | |
| published = entry.find("{http://www.w3.org/2005/Atom}published").text | |
| authors = [author.find("{http://www.w3.org/2005/Atom}name").text for author in entry.findall("{http://www.w3.org/2005/Atom}author")] | |
| authors_list = ", ".join(authors) | |
| categories = [category.attrib["term"] for category in entry.findall("{http://www.w3.org/2005/Atom}category")] | |
| categories_list = ", ".join(categories) | |
| doi = "N/A" | |
| for link in entry.findall("{http://www.w3.org/2005/Atom}link"): | |
| if link.attrib.get("title") == "doi": | |
| doi = link.attrib.get("href") | |
| break | |
| journal_ref = entry.find("{http://arxiv.org/schemas/atom}journal_ref") | |
| journal_ref_text = clean_text(journal_ref.text) if journal_ref is not None else "N/A" | |
| primary_category = entry.find("{http://arxiv.org/schemas/atom}primary_category").attrib.get("term", "N/A") | |
| link = entry.find("{http://www.w3.org/2005/Atom}link").attrib.get("href", "N/A") | |
| # Extract comments if available | |
| comments = entry.find("{http://arxiv.org/schemas/atom}comment") | |
| comments_text = clean_text(comments.text) if comments is not None else "N/A" | |
| # Extract subjects if available | |
| subjects = categories_list | |
| # Extract submission history to get the current version | |
| version = "v1" # Default version | |
| for link in entry.findall("{http://www.w3.org/2005/Atom}link"): | |
| if link.attrib.get("title") == "pdf": | |
| pdf_link = link.attrib.get("href") | |
| new_row = { | |
| "ID": arxiv_id, | |
| "Title": title, | |
| "Authors": authors_list, | |
| "Summary": summary, | |
| "Published Date": published, | |
| "Categories": categories_list, | |
| "DOI": doi, | |
| "Journal Reference": journal_ref_text, | |
| "Primary Category": primary_category, | |
| "Link": link, | |
| "Comments": comments_text, | |
| "Subjects": subjects, | |
| "Version": version, | |
| "PDF Link": pdf_link | |
| } | |
| df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True) | |
| # Sort the DataFrame by Published Date with latest on top | |
| df["Published Date"] = pd.to_datetime(df["Published Date"]) | |
| df = df.sort_values(by="Published Date", ascending=False).reset_index(drop=True) | |
| return df | |
| # Streamlit UI | |
| st.title("arXiv Search App") | |
| st.write("Enter a search term to find papers on arXiv.") | |
| # User input | |
| search_term = st.text_input("Search Term", "machine learning") | |
| max_results = st.slider("Number of Results", min_value=1, max_value=100, value=10) | |
| if st.button("Fetch Papers"): | |
| # Fetch arXiv data | |
| try: | |
| df_arxiv = fetch_arxiv_data(search_term, max_results) | |
| st.write(f"Total Articles Found: {len(df_arxiv)}") | |
| # Displaying the data in the table | |
| st.dataframe(df_arxiv) | |
| # Download as CSV | |
| csv = df_arxiv.to_csv(index=False).encode('utf-8') | |
| st.download_button( | |
| label="Download data as CSV", | |
| data=csv, | |
| file_name='arxiv_data.csv', | |
| mime='text/csv', | |
| ) | |
| except Exception as e: | |
| st.error(f"An error occurred: {e}") |