Spaces:
Runtime error
Runtime error
File size: 4,524 Bytes
b61cfc5 026eac1 b61cfc5 026eac1 b61cfc5 026eac1 b61cfc5 026eac1 b61cfc5 026eac1 b61cfc5 026eac1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | import streamlit as st
import requests
import pandas as pd
import xml.etree.ElementTree as ET
import unicodedata
import re
from collections import Counter
def clean_text(text):
text = text.replace('"', '\"') # Escape double quotes
text = text.replace('\n', ' ') # Replace newlines with space
text = unicodedata.normalize("NFKD", text) # Normalize Unicode characters
text = re.sub(r'\s+', ' ', text).strip() # Remove extra whitespace
return text
def fetch_arxiv_data(search_term, max_results=50):
url = f"http://export.arxiv.org/api/query?search_query={search_term}&start=0&max_results={max_results}"
response = requests.get(url)
response.raise_for_status()
root = ET.fromstring(response.content)
columns = ["ID", "Title", "Authors", "Summary", "Published Date", "Categories", "DOI", "Journal Reference", "Primary Category", "Link", "Comments", "Subjects", "Version", "PDF Link"]
df = pd.DataFrame(columns=columns)
for entry in root.findall("{http://www.w3.org/2005/Atom}entry"):
arxiv_id = entry.find("{http://www.w3.org/2005/Atom}id").text
title = clean_text(entry.find("{http://www.w3.org/2005/Atom}title").text)
summary = clean_text(entry.find("{http://www.w3.org/2005/Atom}summary").text)
published = entry.find("{http://www.w3.org/2005/Atom}published").text
authors = [author.find("{http://www.w3.org/2005/Atom}name").text for author in entry.findall("{http://www.w3.org/2005/Atom}author")]
authors_list = ", ".join(authors)
categories = [category.attrib["term"] for category in entry.findall("{http://www.w3.org/2005/Atom}category")]
categories_list = ", ".join(categories)
doi = "N/A"
for link in entry.findall("{http://www.w3.org/2005/Atom}link"):
if link.attrib.get("title") == "doi":
doi = link.attrib.get("href")
break
journal_ref = entry.find("{http://arxiv.org/schemas/atom}journal_ref")
journal_ref_text = clean_text(journal_ref.text) if journal_ref is not None else "N/A"
primary_category = entry.find("{http://arxiv.org/schemas/atom}primary_category").attrib.get("term", "N/A")
link = entry.find("{http://www.w3.org/2005/Atom}link").attrib.get("href", "N/A")
# Extract comments if available
comments = entry.find("{http://arxiv.org/schemas/atom}comment")
comments_text = clean_text(comments.text) if comments is not None else "N/A"
# Extract subjects if available
subjects = categories_list
# Extract submission history to get the current version
version = "v1" # Default version
for link in entry.findall("{http://www.w3.org/2005/Atom}link"):
if link.attrib.get("title") == "pdf":
pdf_link = link.attrib.get("href")
new_row = {
"ID": arxiv_id,
"Title": title,
"Authors": authors_list,
"Summary": summary,
"Published Date": published,
"Categories": categories_list,
"DOI": doi,
"Journal Reference": journal_ref_text,
"Primary Category": primary_category,
"Link": link,
"Comments": comments_text,
"Subjects": subjects,
"Version": version,
"PDF Link": pdf_link
}
df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True)
# Sort the DataFrame by Published Date with latest on top
df["Published Date"] = pd.to_datetime(df["Published Date"])
df = df.sort_values(by="Published Date", ascending=False).reset_index(drop=True)
return df
# Streamlit UI
st.title("arXiv Search App")
st.write("Enter a search term to find papers on arXiv.")
# User input
search_term = st.text_input("Search Term", "machine learning")
max_results = st.slider("Number of Results", min_value=1, max_value=100, value=10)
if st.button("Fetch Papers"):
# Fetch arXiv data
try:
df_arxiv = fetch_arxiv_data(search_term, max_results)
st.write(f"Total Articles Found: {len(df_arxiv)}")
# Displaying the data in the table
st.dataframe(df_arxiv)
# Download as CSV
csv = df_arxiv.to_csv(index=False).encode('utf-8')
st.download_button(
label="Download data as CSV",
data=csv,
file_name='arxiv_data.csv',
mime='text/csv',
)
except Exception as e:
st.error(f"An error occurred: {e}") |