import streamlit as st import requests import pandas as pd import xml.etree.ElementTree as ET import unicodedata import re from collections import Counter def clean_text(text): text = text.replace('"', '\"') # Escape double quotes text = text.replace('\n', ' ') # Replace newlines with space text = unicodedata.normalize("NFKD", text) # Normalize Unicode characters text = re.sub(r'\s+', ' ', text).strip() # Remove extra whitespace return text def fetch_arxiv_data(search_term, max_results=50): url = f"http://export.arxiv.org/api/query?search_query={search_term}&start=0&max_results={max_results}" response = requests.get(url) response.raise_for_status() root = ET.fromstring(response.content) columns = ["ID", "Title", "Authors", "Summary", "Published Date", "Categories", "DOI", "Journal Reference", "Primary Category", "Link", "Comments", "Subjects", "Version", "PDF Link"] df = pd.DataFrame(columns=columns) for entry in root.findall("{http://www.w3.org/2005/Atom}entry"): arxiv_id = entry.find("{http://www.w3.org/2005/Atom}id").text title = clean_text(entry.find("{http://www.w3.org/2005/Atom}title").text) summary = clean_text(entry.find("{http://www.w3.org/2005/Atom}summary").text) published = entry.find("{http://www.w3.org/2005/Atom}published").text authors = [author.find("{http://www.w3.org/2005/Atom}name").text for author in entry.findall("{http://www.w3.org/2005/Atom}author")] authors_list = ", ".join(authors) categories = [category.attrib["term"] for category in entry.findall("{http://www.w3.org/2005/Atom}category")] categories_list = ", ".join(categories) doi = "N/A" for link in entry.findall("{http://www.w3.org/2005/Atom}link"): if link.attrib.get("title") == "doi": doi = link.attrib.get("href") break journal_ref = entry.find("{http://arxiv.org/schemas/atom}journal_ref") journal_ref_text = clean_text(journal_ref.text) if journal_ref is not None else "N/A" primary_category = entry.find("{http://arxiv.org/schemas/atom}primary_category").attrib.get("term", "N/A") link = entry.find("{http://www.w3.org/2005/Atom}link").attrib.get("href", "N/A") # Extract comments if available comments = entry.find("{http://arxiv.org/schemas/atom}comment") comments_text = clean_text(comments.text) if comments is not None else "N/A" # Extract subjects if available subjects = categories_list # Extract submission history to get the current version version = "v1" # Default version for link in entry.findall("{http://www.w3.org/2005/Atom}link"): if link.attrib.get("title") == "pdf": pdf_link = link.attrib.get("href") new_row = { "ID": arxiv_id, "Title": title, "Authors": authors_list, "Summary": summary, "Published Date": published, "Categories": categories_list, "DOI": doi, "Journal Reference": journal_ref_text, "Primary Category": primary_category, "Link": link, "Comments": comments_text, "Subjects": subjects, "Version": version, "PDF Link": pdf_link } df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True) # Sort the DataFrame by Published Date with latest on top df["Published Date"] = pd.to_datetime(df["Published Date"]) df = df.sort_values(by="Published Date", ascending=False).reset_index(drop=True) return df # Streamlit UI st.title("arXiv Search App") st.write("Enter a search term to find papers on arXiv.") # User input search_term = st.text_input("Search Term", "machine learning") max_results = st.slider("Number of Results", min_value=1, max_value=100, value=10) if st.button("Fetch Papers"): # Fetch arXiv data try: df_arxiv = fetch_arxiv_data(search_term, max_results) st.write(f"Total Articles Found: {len(df_arxiv)}") # Displaying the data in the table st.dataframe(df_arxiv) # Download as CSV csv = df_arxiv.to_csv(index=False).encode('utf-8') st.download_button( label="Download data as CSV", data=csv, file_name='arxiv_data.csv', mime='text/csv', ) except Exception as e: st.error(f"An error occurred: {e}")