File size: 4,524 Bytes
b61cfc5
 
 
 
 
 
 
 
 
026eac1
b61cfc5
 
 
 
 
 
 
 
 
 
 
026eac1
b61cfc5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
026eac1
 
 
 
 
 
 
 
 
 
 
 
 
b61cfc5
 
 
 
 
 
 
 
 
 
026eac1
 
 
 
 
b61cfc5
 
 
026eac1
 
 
 
b61cfc5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
026eac1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
import streamlit as st
import requests
import pandas as pd
import xml.etree.ElementTree as ET
import unicodedata
import re
from collections import Counter

def clean_text(text):
    text = text.replace('"', '\"')  # Escape double quotes
    text = text.replace('\n', ' ')  # Replace newlines with space
    text = unicodedata.normalize("NFKD", text)  # Normalize Unicode characters
    text = re.sub(r'\s+', ' ', text).strip()  # Remove extra whitespace
    return text

def fetch_arxiv_data(search_term, max_results=50):
    url = f"http://export.arxiv.org/api/query?search_query={search_term}&start=0&max_results={max_results}"
    response = requests.get(url)
    response.raise_for_status()
    root = ET.fromstring(response.content)

    columns = ["ID", "Title", "Authors", "Summary", "Published Date", "Categories", "DOI", "Journal Reference", "Primary Category", "Link", "Comments", "Subjects", "Version", "PDF Link"]
    df = pd.DataFrame(columns=columns)

    for entry in root.findall("{http://www.w3.org/2005/Atom}entry"):
        arxiv_id = entry.find("{http://www.w3.org/2005/Atom}id").text
        title = clean_text(entry.find("{http://www.w3.org/2005/Atom}title").text)
        summary = clean_text(entry.find("{http://www.w3.org/2005/Atom}summary").text)
        published = entry.find("{http://www.w3.org/2005/Atom}published").text

        authors = [author.find("{http://www.w3.org/2005/Atom}name").text for author in entry.findall("{http://www.w3.org/2005/Atom}author")]
        authors_list = ", ".join(authors)

        categories = [category.attrib["term"] for category in entry.findall("{http://www.w3.org/2005/Atom}category")]
        categories_list = ", ".join(categories)

        doi = "N/A"
        for link in entry.findall("{http://www.w3.org/2005/Atom}link"):
            if link.attrib.get("title") == "doi":
                doi = link.attrib.get("href")
                break

        journal_ref = entry.find("{http://arxiv.org/schemas/atom}journal_ref")
        journal_ref_text = clean_text(journal_ref.text) if journal_ref is not None else "N/A"

        primary_category = entry.find("{http://arxiv.org/schemas/atom}primary_category").attrib.get("term", "N/A")

        link = entry.find("{http://www.w3.org/2005/Atom}link").attrib.get("href", "N/A")

        # Extract comments if available
        comments = entry.find("{http://arxiv.org/schemas/atom}comment")
        comments_text = clean_text(comments.text) if comments is not None else "N/A"

        # Extract subjects if available
        subjects = categories_list

        # Extract submission history to get the current version
        version = "v1"  # Default version
        for link in entry.findall("{http://www.w3.org/2005/Atom}link"):
            if link.attrib.get("title") == "pdf":
                pdf_link = link.attrib.get("href")

        new_row = {
            "ID": arxiv_id,
            "Title": title,
            "Authors": authors_list,
            "Summary": summary,
            "Published Date": published,
            "Categories": categories_list,
            "DOI": doi,
            "Journal Reference": journal_ref_text,
            "Primary Category": primary_category,
            "Link": link,
            "Comments": comments_text,
            "Subjects": subjects,
            "Version": version,
            "PDF Link": pdf_link
        }
        df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True)

    # Sort the DataFrame by Published Date with latest on top
    df["Published Date"] = pd.to_datetime(df["Published Date"])
    df = df.sort_values(by="Published Date", ascending=False).reset_index(drop=True)

    return df

# Streamlit UI
st.title("arXiv Search App")
st.write("Enter a search term to find papers on arXiv.")

# User input
search_term = st.text_input("Search Term", "machine learning")
max_results = st.slider("Number of Results", min_value=1, max_value=100, value=10)

if st.button("Fetch Papers"):
    # Fetch arXiv data
    try:
        df_arxiv = fetch_arxiv_data(search_term, max_results)
        st.write(f"Total Articles Found: {len(df_arxiv)}")

        # Displaying the data in the table
        st.dataframe(df_arxiv)

        # Download as CSV
        csv = df_arxiv.to_csv(index=False).encode('utf-8')
        st.download_button(
            label="Download data as CSV",
            data=csv,
            file_name='arxiv_data.csv',
            mime='text/csv',
        )
    except Exception as e:
        st.error(f"An error occurred: {e}")