Spaces:
Sleeping
Sleeping
import requests
import pandas as pd
import time
from datetime import datetime
import gradio as gr
import os
import re
import bibtexparser
from bibtexparser.bparser import BibTexParser
from bibtexparser.customization import convert_to_unicode
from difflib import SequenceMatcher
import numpy as np
from collections import Counter
import math
# ============================================
# CLEAN RESPONSIVE CSS - (same as your original)
# ============================================
custom_css = """
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap');
:root {
--primary: #2563eb;
--secondary: #7c3aed;
--border: #d1d5db;
--bg-soft: #e5e7eb !important;
--text-main: #111827;
}
/* ================================================= */
/* GENERAL LAYOUT */
/* ================================================= */
body, .gradio-container {
font-family: 'Inter', sans_serif !important;
background-color: #e5e7eb !important;
color: var(--text-main) !important;
}
.gradio-container {
max-width: 1600px !important;
margin: auto !important;
}
.white-card {
background: #ffffff !important;
border: 1px solid var(--border) !important;
border-radius: 12px !important;
padding: 2rem !important;
box-shadow: 0 4px 6px rgba(0, 0, 0, 0.05) !important;
}
h1 span {
background: linear-gradient(135deg, var(--primary), var(--secondary));
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
}
.subtitle-text {
color: #2563eb !important;
font-weight: 500 !important;
}
/* ================================================= */
/* METRICS */
/* ================================================= */
.metric-value {
font-size: 2rem !important;
font-weight: 700 !important;
color: var(--primary) !important;
}
/* ================================================= */
/* BUTTONS */
/* ================================================= */
button, .gr-button {
color: #ffffff !important;
font-weight: 600 !important;
}
button.variant-primary {
background-color: var(--primary) !important;
}
button.variant-secondary {
background-color: #6b7280 !important;
}
/* ================================================= */
/* TABS */
/* ================================================= */
.tab-nav {
background-color: #f3f4f6 !important;
border-radius: 8px !important;
padding: 4px !important;
margin-bottom: 20px !important;
}
.tab-nav button {
border-radius: 6px !important;
padding: 8px 16px !important;
font-weight: 500 !important;
color: #4b5563 !important;
}
.tab-nav button.selected {
background-color: var(--primary) !important;
color: white !important;
}
/* ================================================= */
/* SLIDER COMMENT */
/* ================================================= */
.slider-comment {
color: #ffffff !important;
font-size: 0.9rem !important;
margin-top: 8px !important;
margin-bottom: 8px !important;
font-style: italic !important;
background: #1f2937 !important;
padding: 8px 12px !important;
border-radius: 6px !important;
border-left: 4px solid var(--primary) !important;
font-weight: 500 !important;
}
/* ================================================= */
/* DROPDOWN BUTTON (Closed State) */
/* ================================================= */
.gr-dropdown, .gr-dropdown select, select {
background-color: #374151 !important;
color: #ffffff !important;
border: 1px solid #4b5563 !important;
border-radius: 6px !important;
}
/* ========================================= */
/* FORCE BUTTON TEXT TO ALWAYS BE WHITE */
/* ========================================= */
/* All buttons */
button,
.gr-button {
color: #ffffff !important;
}
/* Primary button */
button.variant-primary {
background-color: #2563eb !important;
color: #ffffff !important;
}
/* Secondary button */
button.variant-secondary {
background-color: #6b7280 !important;
color: #ffffff !important;
}
/* Keep text white even on hover */
button:hover {
color: #ffffff !important;
}
/* ================================================= */
/* METRIC LABEL BLACK BACKGROUND */
/* ================================================= */
.metric-label {
display: inline-block !important;
background-color: #111827 !important;
color: #ffffff !important;
padding: 6px 12px !important;
border-radius: 6px !important;
font-weight: 600 !important;
}
/* ================================================= */
/* METRIC VALUE BLACK BACKGROUND FIX */
/* ================================================= */
.metric-value {
display: inline-block !important;
background-color: #111827 !important;
color: #ffffff !important;
padding: 6px 12px !important;
border-radius: 6px !important;
font-weight: 700 !important;
}
/* Remove default white background from strong */
.metric-value strong {
background: transparent !important;
color: #ffffff !important;
}
/* ================================================= */
/* DROPDOWN POPUP PAGE - WHITE BACKGROUND */
/* ================================================= */
/* The popup menu that appears when clicking dropdown */
.gr-dropdown-list,
.dropdown-list,
div[role="listbox"],
.options-container,
select option,
.gr-dropdown option,
.gradio-dropdown option,
div.gr-box select option {
background-color: #ffffff !important; /* Pure white background */
color: #111827 !important; /* Dark text */
}
/* Individual options */
select option,
.gr-dropdown option,
.gradio-dropdown option {
background-color: #ffffff !important;
color: #111827 !important;
padding: 8px 12px !important;
}
/* Hover effect */
select option:hover,
.gr-dropdown option:hover,
.gradio-dropdown option:hover {
background-color: #f3f4f6 !important; /* Light gray on hover */
color: #111827 !important;
}
/* Selected option */
select option:checked,
.gr-dropdown option:checked,
.gradio-dropdown option:checked {
background-color: #2563eb !important;
color: white !important;
}
/* ================================================= */
/* PDF LINK STYLING */
/* ================================================= */
.pdf-link {
color: #2563eb !important;
text-decoration: none !important;
font-weight: 500 !important;
}
.pdf-link:hover {
text-decoration: underline !important;
color: #1d4ed8 !important;
}
.no-pdf {
color: #6b7280 !important;
font-style: italic !important;
}
/* ================================================= */
/* REFERENCE CHECKER TABLE STYLING */
/* ================================================= */
.ref-valid {
color: #10b981 !important;
font-weight: 600 !important;
}
.ref-invalid {
color: #ef4444 !important;
font-weight: 600 !important;
}
.ref-partial {
color: #f59e0b !important;
font-weight: 600 !important;
}
.comparison-table {
border-collapse: collapse !important;
width: 100% !important;
}
.comparison-table th {
background-color: #2563eb !important;
color: white !important;
padding: 10px !important;
text-align: left !important;
}
.comparison-table td {
padding: 8px !important;
border-bottom: 1px solid #e5e7eb !important;
vertical-align: top !important;
}
.comparison-table tr:hover {
background-color: #f9fafb !important;
}
.original-ref {
background-color: #fee2e2 !important;
padding: 4px 8px !important;
border-radius: 4px !important;
border-left: 3px solid #ef4444 !important;
}
.corrected-ref {
background-color: #dcfce7 !important;
padding: 4px 8px !important;
border-radius: 4px !important;
border-left: 3px solid #10b981 !important;
}
"""
# ============================================
# SEARCH ENGINE - ENHANCED WITH GOOGLE SCHOLAR LIKE SENSITIVITY
# ============================================
class PaperSearchPro:
def __init__(self):
self.base_url = "https://api.openalex.org/works"
self.per_page = 200
# Common stop words to ignore in matching
self.stop_words = set(['the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'be',
'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will',
'would', 'should', 'could', 'may', 'might', 'must', 'can'])
def calculate_text_similarity(self, text1, text2):
"""Calculate similarity between two texts using multiple methods"""
if not text1 or not text2:
return 0
text1 = text1.lower()
text2 = text2.lower()
# Method 1: Sequence matching (good for phrases)
seq_similarity = SequenceMatcher(None, text1, text2).ratio()
# Method 2: Word overlap (good for keywords)
words1 = set(re.findall(r'\w+', text1))
words2 = set(re.findall(r'\w+', text2))
# Remove stop words for better matching
words1 = words1 - self.stop_words
words2 = words2 - self.stop_words
if not words1 or not words2:
return seq_similarity
intersection = words1.intersection(words2)
union = words1.union(words2)
word_similarity = len(intersection) / len(union) if union else 0
# Method 3: Partial word matching (for stem variations)
partial_matches = 0
for w1 in words1:
for w2 in words2:
if len(w1) > 3 and len(w2) > 3:
if w1 in w2 or w2 in w1 or SequenceMatcher(None, w1, w2).ratio() > 0.8:
partial_matches += 1
break
partial_ratio = partial_matches / len(words1) if words1 else 0
# Weighted combination
return (seq_similarity * 0.3 + word_similarity * 0.5 + partial_ratio * 0.2)
def extract_keywords_from_phrase(self, phrase):
"""Extract meaningful keywords from a phrase"""
words = re.findall(r'\w+', phrase.lower())
# Remove stop words and short words
keywords = [w for w in words if w not in self.stop_words and len(w) > 2]
# Also include bigrams if phrase has multiple words
if len(words) > 1:
bigrams = [' '.join(words[i:i+2]) for i in range(len(words)-1)]
keywords.extend(bigrams)
return list(set(keywords)) # Remove duplicates
def search_papers(self, query, keywords, authors, max_papers, year_from, year_to, open_access, sort_by, sort_order):
filters = []
if year_from:
filters.append(f"publication_year:>{int(year_from)-1}")
if year_to:
filters.append(f"publication_year:<{int(year_to)+1}")
if open_access:
filters.append("open_access.is_oa:true")
# Add author filter if provided
if authors and authors.strip():
# Clean and format author names for filtering
author_list = [a.strip() for a in authors.split(",") if a.strip()]
if author_list:
# OpenAlex uses author.search filter
author_filter = "|".join(author_list)
filters.append(f"author.search:{author_filter}")
filter_str = ",".join(filters) if filters else None
# Map sort options to API parameters
sort_field_map = {
"Relevance": "relevance_score",
"Year": "publication_date",
"Citations": "cited_by_count"
}
sort_field = sort_field_map.get(sort_by, "relevance_score")
sort_direction = "desc" if sort_order == "Descending" else "asc"
sort_param = f"{sort_field}:{sort_direction}"
all_papers = []
page = 1
while len(all_papers) < max_papers:
params = {
"search": query,
"per-page": self.per_page,
"page": page,
"sort": sort_param
}
if filter_str:
params["filter"] = filter_str
try:
response = requests.get(self.base_url, params=params, timeout=20)
data = response.json()
results = data.get("results", [])
if not results:
break
for work in results:
paper = self.extract_info(work)
all_papers.append(paper)
page += 1
time.sleep(0.2)
except Exception as e:
print(f"Error: {e}")
break
# ===============================
# ENHANCED KEYWORD FILTERING - GOOGLE SCHOLAR LIKE SENSITIVITY
# Supports unlimited keywords with fuzzy matching and relevance scoring
# ===============================
keyword_phrases = [k.strip() for k in keywords.split(",") if k.strip()]
if len(keyword_phrases) >= 1: # At least one keyword required
filtered = []
# Pre-process each keyword phrase into search terms
keyword_search_terms = []
for phrase in keyword_phrases:
# Extract meaningful keywords from each phrase
terms = self.extract_keywords_from_phrase(phrase)
keyword_search_terms.append({
'original': phrase,
'terms': terms,
'weight': 1.0
})
# Calculate IDF-like weights for terms (rarer terms get higher weight)
all_terms = []
for kst in keyword_search_terms:
all_terms.extend(kst['terms'])
term_freq = Counter(all_terms)
total_terms = len(all_terms)
for kst in keyword_search_terms:
# Adjust weight based on term rarity
avg_freq = sum(term_freq[t] for t in kst['terms']) / len(kst['terms']) if kst['terms'] else 1
# Rarer terms (lower frequency) get higher weight
kst['weight'] = 1.0 / (avg_freq / total_terms + 0.1) if avg_freq > 0 else 1.0
for paper in all_papers:
# Combine all text fields for searching
combined_text = f"{paper['Title']} {paper['Abstract']} {paper['Authors']}".lower()
# Calculate relevance score using multiple factors
score = 0
matched_terms = []
matched_phrases = []
# Check each keyword phrase
for i, kst in enumerate(keyword_search_terms):
phrase_score = 0
phrase_matched_terms = []
# Check each term in the phrase
for term in kst['terms']:
# Exact match (highest score)
if term in combined_text:
term_score = 1.0 * kst['weight']
phrase_matched_terms.append(term)
else:
# Fuzzy/partial matching
# Check if term appears as part of a word
words = re.findall(r'\w+', combined_text)
best_partial = 0
for word in words:
if len(word) > 3 and len(term) > 3:
# Check if term is a substring of word
if term in word:
best_partial = max(best_partial, 0.8)
# Check word similarity
elif SequenceMatcher(None, term, word).ratio() > 0.7:
sim = SequenceMatcher(None, term, word).ratio()
best_partial = max(best_partial, sim * 0.7)
if best_partial > 0:
term_score = best_partial * kst['weight']
phrase_matched_terms.append(f"{term}~")
else:
term_score = 0
if term_score > 0:
phrase_score += term_score
# Normalize phrase score by number of terms
if kst['terms']:
phrase_score = phrase_score / len(kst['terms'])
# First two phrases are required (must have at least partial match)
if i < 2:
if phrase_score > 0.3: # Partial match threshold for required phrases
score += phrase_score * 2 # Higher weight for first two
matched_phrases.append(kst['original'])
matched_terms.extend(phrase_matched_terms)
else:
# Required phrase not matched well enough - skip this paper
score = 0
break
else:
# Additional phrases boost score
if phrase_score > 0:
score += phrase_score
matched_phrases.append(kst['original'])
matched_terms.extend(phrase_matched_terms)
if score > 0:
# Additional boost factors
# 1. Title match boost (keywords in title are more important)
title_text = paper['Title'].lower()
title_boost = 0
for term in set(matched_terms):
term_clean = term.replace('~', '')
if term_clean in title_text:
title_boost += 0.2
score += title_boost
# 2. Recent papers boost (slightly)
if paper['Year'] != 'N/A' and paper['Year'] >= 2020:
score += 0.1
# 3. Citation boost (log scale)
if paper['Citations'] > 0:
citation_boost = math.log10(paper['Citations'] + 1) * 0.05
score += min(citation_boost, 0.3)
paper['Relevance'] = round(score, 2)
paper['Matched Keywords'] = f"Phrases: {', '.join(matched_phrases[:5])}" + \
(f" (+{len(matched_phrases)-5} more)" if len(matched_phrases) > 5 else "") + \
f" | Terms: {', '.join(matched_terms[:10])}" + \
(f" (+{len(matched_terms)-10} more)" if len(matched_terms) > 10 else "")
filtered.append(paper)
# Sort by relevance score (descending)
filtered.sort(key=lambda x: x['Relevance'], reverse=True)
all_papers = filtered[:max_papers]
# ===============================
# AUTHOR FILTERING (if no keywords)
# ===============================
elif authors and authors.strip() and not keyword_phrases:
# If we have author filter but no keyword hierarchy
author_list = [a.strip().lower() for a in authors.split(",") if a.strip()]
filtered = []
for p in all_papers:
all_authors_lower = p['All Authors'].lower()
match_all = True
matched_authors = []
for author in author_list:
# Split author name into parts for partial matching
author_parts = author.split()
author_matched = False
# Check if any part of the author name appears
for part in author_parts:
if len(part) > 2 and part in all_authors_lower: # Only check significant parts
author_matched = True
matched_authors.append(part)
break
if not author_matched:
# Fallback: check if the whole author string appears
if author in all_authors_lower:
author_matched = True
matched_authors.append(author)
else:
match_all = False
break
if match_all:
p["Relevance"] = len(matched_authors)
p["Matched Keywords"] = f"Authors: {', '.join(matched_authors)}"
filtered.append(p)
all_papers = filtered[:max_papers]
total = len(all_papers)
avg_rel = sum(p.get("Relevance", 0) for p in all_papers) / total if total else 0
years = [p["Year"] for p in all_papers if p["Year"] != "N/A"]
year_range = f"{min(years)} - {max(years)}" if years else "N/A"
df = pd.DataFrame(all_papers)
# Reorder columns to include BibTeX and remove All Authors from display
column_order = ['Title', 'Authors', 'Year', 'Citations', 'Journal', 'PDF_URL', 'Abstract',
'BibTeX', 'Relevance', 'Matched Keywords']
df = df[column_order]
return df, all_papers, total, avg_rel, year_range
def search_specific_paper(self, search_type, search_value):
"""Search for a specific paper by title, author, or DOI"""
all_papers = []
if search_type == "DOI":
# Clean DOI (remove URL if present)
if "doi.org/" in search_value:
search_value = search_value.split("doi.org/")[-1]
# Direct lookup by DOI
url = f"https://api.openalex.org/works/doi/{search_value}"
try:
response = requests.get(url, timeout=15)
if response.status_code == 200:
work = response.json()
paper = self.extract_info(work)
all_papers.append(paper)
except Exception as e:
print(f"Error fetching DOI: {e}")
elif search_type == "Exact Title":
# Search by exact title with fuzzy matching
params = {
"search": search_value,
"per-page": 20,
"sort": "relevance_score:desc"
}
try:
response = requests.get(self.base_url, params=params, timeout=15)
data = response.json()
results = data.get("results", [])
best_match = None
best_score = 0
for work in results:
title = work.get("title", "")
if title:
# Calculate similarity score
similarity = self.calculate_text_similarity(search_value, title)
if similarity > best_score and similarity > 0.4: # Threshold for acceptance
best_score = similarity
best_match = work
if best_match:
paper = self.extract_info(best_match)
paper['Match_Score'] = round(best_score, 2)
all_papers.append(paper)
except Exception as e:
print(f"Error searching title: {e}")
elif search_type == "Specific Author":
# Search for papers by a specific author
params = {
"search": search_value,
"filter": "authorships.author.display_name.search:" + search_value,
"per-page": 50,
"sort": "cited_by_count:desc"
}
try:
response = requests.get(self.base_url, params=params, timeout=15)
data = response.json()
results = data.get("results", [])
for work in results[:20]: # Limit to 20 papers
paper = self.extract_info(work)
all_papers.append(paper)
except Exception as e:
print(f"Error searching author: {e}")
if all_papers:
for i, p in enumerate(all_papers):
p['Rank'] = i + 1
df = pd.DataFrame(all_papers)
column_order = ['Title', 'Authors', 'Year', 'Citations', 'Journal', 'PDF_URL', 'Abstract', 'BibTeX']
if 'Match_Score' in df.columns:
column_order.insert(1, 'Match_Score')
# Only include columns that exist
available_cols = [col for col in column_order if col in df.columns]
df = df[available_cols]
summary = f"✅ Found **{len(all_papers)}** paper(s) matching your query"
return df, all_papers, summary
return pd.DataFrame(), [], "❌ No papers found"
def extract_info(self, work):
# Rebuild abstract
abstract = ""
if work.get("abstract_inverted_index"):
words = []
for word, positions in work["abstract_inverted_index"].items():
for pos in positions:
words.append((pos, word))
words.sort()
abstract = " ".join([w[1] for w in words])
else:
abstract = "No abstract available"
# Get all authors - FULL LIST (no et al.)
all_authors = []
for authorship in work.get("authorships", []):
# FIRST PRIORITY: Use raw_author_name (full name as published)
raw_name = authorship.get("raw_author_name", "")
display_name = authorship.get("author", {}).get("display_name", "")
# Choose the best name
if raw_name and len(raw_name) > 2:
# Raw name is usually more complete (e.g., "Mamoru SHOJI" vs "M. Shoji")
author_name = raw_name
elif display_name:
author_name = display_name
else:
author_name = ""
if author_name:
all_authors.append(author_name)
# Show ALL authors, no truncation
authors_str = ", ".join(all_authors)
# Get DOI for reference
doi = work.get("doi", "")
if doi:
doi = doi.replace("https://doi.org/", "")
# Get PDF URL if available
pdf_url = ""
if work.get("open_access", {}).get("oa_url"):
pdf_url = work["open_access"]["oa_url"]
elif work.get("primary_location", {}).get("pdf_url"):
pdf_url = work["primary_location"]["pdf_url"]
else:
pdf_url = "No free PDF available"
# Generate BibTeX entry
bibtex = self.generate_bibtex(work, all_authors)
return {
"Title": work.get("title", "No title"),
"Authors": authors_str,
"All Authors": ", ".join(all_authors), # Hidden field for filtering
"Year": work.get("publication_year", "N/A"),
"Citations": work.get("cited_by_count", 0),
"Journal": (work.get("primary_location", {}).get("source") or {}).get("display_name", "Unknown"),
"DOI": doi,
"PDF_URL": pdf_url,
"Abstract": abstract,
"BibTeX": bibtex,
"Relevance": 0,
"Matched Keywords": ""
}
def generate_bibtex(self, work, authors):
"""Generate BibTeX entry for a paper"""
# Create citation key
first_author = authors[0].split()[-1] if authors else "Unknown"
year = work.get("publication_year", "0000")
title_words = work.get("title", "Untitled").split()[:3]
title_short = "".join([w[0] for w in title_words if w]) if title_words else "Paper"
citation_key = f"{first_author}{year}{title_short}"
# Clean the citation key (remove special characters)
citation_key = ''.join(c for c in citation_key if c.isalnum())
# Get DOI or URL
doi = work.get("doi", "")
url = work.get("primary_location", {}).get("landing_page_url", "")
if not url and work.get("open_access", {}).get("oa_url"):
url = work.get("open_access", {}).get("oa_url")
# Determine entry type (article, book, etc.)
pub_type = "article"
if work.get("type") == "book":
pub_type = "book"
elif work.get("type") == "book-chapter":
pub_type = "incollection"
# Build BibTeX string
bibtex_lines = [
f"@{pub_type}{{{citation_key},",
f" title = {{{work.get('title', 'Untitled')}}},",
f" author = {{{' and '.join(authors)}}},",
f" year = {{{work.get('publication_year', '0000')}}},"
]
# Add journal/booktitle
source = (work.get("primary_location", {}).get("source") or {})
if pub_type == "article":
journal = source.get("display_name", "Unknown Journal")
bibtex_lines.append(f" journal = {{{journal}}},")
else:
booktitle = source.get("display_name", "Unknown Publication")
bibtex_lines.append(f" booktitle = {{{booktitle}}},")
# Add volume, issue if available
if work.get("biblio", {}).get("volume"):
bibtex_lines.append(f" volume = {{{work['biblio']['volume']}}},")
if work.get("biblio", {}).get("issue"):
bibtex_lines.append(f" number = {{{work['biblio']['issue']}}},")
# Add pages if available
if work.get("biblio", {}).get("first_page") and work.get("biblio", {}).get("last_page"):
pages = f"{work['biblio']['first_page']}--{work['biblio']['last_page']}"
bibtex_lines.append(f" pages = {{{pages}}},")
# Add DOI/URL
if doi:
bibtex_lines.append(f" doi = {{{doi}}},")
elif url:
bibtex_lines.append(f" url = {{{url}}},")
# Add abstract if available
if work.get("abstract_inverted_index"):
# Rebuild abstract for BibTeX
words = []
for word, positions in work["abstract_inverted_index"].items():
for pos in positions:
words.append((pos, word))
words.sort()
abstract = " ".join([w[1] for w in words])
if abstract:
bibtex_lines.append(f" abstract = {{{abstract}}},")
# Close the entry
bibtex_lines.append("}")
return "\n".join(bibtex_lines)
# ============================================
# BIBTEX CHECKER FEATURE
# ============================================
def parse_bibtex_file(self, bibtex_file):
"""Parse a BibTeX file and extract entries"""
entries = []
try:
with open(bibtex_file, 'r', encoding='utf-8') as f:
bibtex_str = f.read()
# Parse BibTeX
parser = BibTexParser(common_strings=True)
parser.ignore_nonstandard_types = False
parser.homogenize_fields = True
bib_database = bibtexparser.loads(bibtex_str, parser=parser)
for entry in bib_database.entries:
# Extract key fields
bib_entry = {
'id': entry.get('ID', ''),
'title': entry.get('title', ''),
'author': entry.get('author', ''),
'year': entry.get('year', ''),
'journal': entry.get('journal', entry.get('booktitle', '')),
'doi': entry.get('doi', ''),
'volume': entry.get('volume', ''),
'number': entry.get('number', ''),
'pages': entry.get('pages', ''),
'publisher': entry.get('publisher', ''),
'entry_type': entry.get('ENTRYTYPE', ''),
'raw_bibtex': self._reconstruct_bibtex(entry)
}
entries.append(bib_entry)
return entries
except Exception as e:
print(f"Error parsing BibTeX: {e}")
return []
def _reconstruct_bibtex(self, entry):
"""Reconstruct BibTeX string from entry"""
bib_lines = [f"@{entry['ENTRYTYPE']}{{{entry['ID']},"]
for field, value in entry.items():
if field not in ['ENTRYTYPE', 'ID'] and value:
bib_lines.append(f" {field} = {{{value}}},")
bib_lines.append("}")
return "\n".join(bib_lines)
def check_bibtex_entry(self, entry):
"""Check a BibTeX entry against OpenAlex database"""
# Try DOI first if available
if entry.get('doi'):
doi = entry['doi'].strip()
url = f"https://api.openalex.org/works/doi/{doi}"
try:
response = requests.get(url, timeout=10)
if response.status_code == 200:
work = response.json()
corrected = self.extract_info(work)
# Generate corrected BibTeX
corrected_bibtex = self.generate_bibtex(work, corrected['Authors'].split(', '))
return {
'original': entry,
'corrected': corrected,
'corrected_bibtex': corrected_bibtex,
'status': 'valid',
'confidence': 1.0,
'method': 'DOI'
}
except:
pass
# Try title search with enhanced matching
if entry.get('title'):
params = {
'search': entry['title'],
'per-page': 10,
'sort': 'relevance_score:desc'
}
try:
response = requests.get(self.base_url, params=params, timeout=10)
data = response.json()
results = data.get('results', [])
if results:
best_match = None
best_confidence = 0
for work in results:
if work.get('title'):
# Calculate similarity
similarity = self.calculate_text_similarity(entry['title'], work['title'])
confidence = similarity * 0.6 # Base confidence from title match
# Check year
if entry.get('year') and work.get('publication_year'):
if str(work['publication_year']) == entry['year']:
confidence += 0.25
# Check author
if entry.get('author'):
work_authors = []
for auth in work.get('authorships', []):
name = auth.get('raw_author_name', '') or auth.get('author', {}).get('display_name', '')
if name:
work_authors.append(name.lower())
work_authors_str = ' '.join(work_authors)
entry_authors = entry['author'].lower()
# Check if any author matches
for author in entry_authors.split(' and '):
author_parts = author.strip().split()
for part in author_parts:
if len(part) > 2 and part in work_authors_str:
confidence += 0.15
break
confidence = min(confidence, 1.0)
if confidence > best_confidence:
best_confidence = confidence
best_match = work
if best_match and best_confidence > 0.4:
corrected = self.extract_info(best_match)
corrected_bibtex = self.generate_bibtex(best_match, corrected['Authors'].split(', '))
status = 'valid' if best_confidence > 0.8 else 'partial' if best_confidence > 0.5 else 'not_found'
return {
'original': entry,
'corrected': corrected,
'corrected_bibtex': corrected_bibtex,
'status': status,
'confidence': best_confidence,
'method': 'title search'
}
except:
pass
# No match found
return {
'original': entry,
'corrected': None,
'corrected_bibtex': None,
'status': 'not_found',
'confidence': 0,
'method': 'none'
}
def check_bibtex_file(self, bibtex_file):
"""Main function to check BibTeX file"""
# Parse BibTeX file
entries = self.parse_bibtex_file(bibtex_file)
if not entries:
return None, None, "❌ No valid BibTeX entries found"
# Check each entry
results = []
valid_count = 0
partial_count = 0
invalid_count = 0
for i, entry in enumerate(entries):
result = self.check_bibtex_entry(entry)
results.append(result)
if result['status'] == 'valid':
valid_count += 1
elif result['status'] == 'partial':
partial_count += 1
else:
invalid_count += 1
# Small delay to avoid rate limiting
time.sleep(0.2)
# Create comparison HTML
html_output = f"""
<div style="margin-bottom: 20px; padding: 15px; background-color: #f3f4f6; border-radius: 8px;">
<h3 style="color: #2563eb; margin-bottom: 10px;">📊 BibTeX Check Summary</h3>
<p><span style="color: #10b981; font-weight: 600;">✅ Valid: {valid_count}</span> |
<span style="color: #f59e0b; font-weight: 600;">⚠️ Partial: {partial_count}</span> |
<span style="color: #ef4444; font-weight: 600;">❌ Not Found: {invalid_count}</span></p>
<p>Total entries checked: {len(entries)}</p>
</div>
<table class="comparison-table">
<tr>
<th>#</th>
<th>Original BibTeX Key</th>
<th>Original Title</th>
<th>Corrected Title</th>
<th>Confidence</th>
<th>Status</th>
</tr>
"""
for i, result in enumerate(results):
status_class = {
'valid': 'ref-valid',
'partial': 'ref-partial',
'not_found': 'ref-invalid'
}.get(result['status'], '')
status_text = {
'valid': '✅ VALID',
'partial': '⚠️ PARTIAL',
'not_found': '❌ NOT FOUND'
}.get(result['status'], '')
original = result['original']
original_title = original['title'][:50] + "..." if len(original['title']) > 50 else original['title']
if result['corrected']:
corrected_title = result['corrected']['Title'][:50] + "..." if len(result['corrected']['Title']) > 50 else result['corrected']['Title']
confidence = f"{result['confidence']:.0%}"
else:
corrected_title = "No match found"
confidence = "0%"
html_output += f"""
<tr>
<td>{i+1}</td>
<td><code>{original['id']}</code></td>
<td><div class="original-ref">{original_title}</div></td>
<td><div class="corrected-ref">{corrected_title}</div></td>
<td>{confidence}</td>
<td class="{status_class}">{status_text}</td>
</tr>
"""
html_output += "</table>"
# Create downloadable files
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
# Create corrected BibTeX file
corrected_bib_path = f"corrected_bibliography_{timestamp}.bib"
with open(corrected_bib_path, 'w', encoding='utf-8') as f:
for result in results:
if result['corrected_bibtex']:
f.write(result['corrected_bibtex'] + "\n\n")
else:
# Keep original if no correction found
f.write(result['original']['raw_bibtex'] + "\n\n")
# Create comparison CSV
comparison_data = []
for result in results:
if result['corrected']:
corr = result['corrected']
comparison_data.append({
'Entry_Key': result['original']['id'],
'Original_Title': result['original']['title'],
'Status': result['status'],
'Confidence': result['confidence'],
'Found_By': result['method'],
'Corrected_Title': corr['Title'],
'Corrected_Authors': corr['Authors'],
'Corrected_Year': corr['Year'],
'Corrected_Journal': corr['Journal'],
'Corrected_DOI': corr.get('DOI', ''),
'PDF_URL': corr.get('PDF_URL', '')
})
else:
comparison_data.append({
'Entry_Key': result['original']['id'],
'Original_Title': result['original']['title'],
'Status': result['status'],
'Confidence': result['confidence'],
'Found_By': result['method'],
'Corrected_Title': 'NOT FOUND',
'Corrected_Authors': '',
'Corrected_Year': '',
'Corrected_Journal': '',
'Corrected_DOI': '',
'PDF_URL': ''
})
df = pd.DataFrame(comparison_data)
csv_path = f"bibtex_comparison_{timestamp}.csv"
df.to_csv(csv_path, index=False)
return corrected_bib_path, csv_path, html_output
# ============================================
# INTERFACE
# ============================================
def create_interface():
# Define the theme here to pass to launch later
default_theme = gr.themes.Default(
primary_hue="blue",
neutral_hue="gray"
)
with gr.Blocks(title="PaperSearchPro", css=custom_css, theme=default_theme) as demo:
gr.HTML("""
<div style='text-align:center; padding:3rem 0'>
<h1><span>📱XYZ📱PaperInfo</span></h1>
<p class='subtitle-text'>Easy Academic Paper/journal details by Paul</p>
</div>
""")
# Create tabs for different search modes
with gr.Tabs(elem_classes="tab-nav") as tabs_component:
# Tab 1: General Search
with gr.TabItem("🔍 General Search", id=0):
with gr.Column(elem_classes="white-card"):
query = gr.Textbox(
label="Search Topic",
value="photoacoustic imaging",
placeholder="Enter research topic or keywords..."
)
with gr.Row():
with gr.Column():
keywords = gr.Textbox(
label="Required Keywords (unlimited, first 2 required for match)",
value="blood, photoacoustics, imaging, vascular, hemoglobin",
placeholder="e.g., blood, photoacoustics, imaging, vascular (comma-separated)"
)
gr.Markdown("""
<small style="color: #6b7280;">✨ **Enhanced matching**: Supports unlimited keywords with fuzzy/partial matching like Google Scholar</small>
""")
with gr.Column():
authors = gr.Textbox(
label="Filter by Authors (optional)",
value="",
placeholder="e.g., Smith, Zhang, Wang (comma-separated)"
)
# ==========================
# MAX PAPERS SLIDER WITH BLACK REGION COMMENT
# ==========================
max_papers = gr.Slider(1000, 5000000, value=10000, step=1000, label="Max Papers")
# Add black region comment below slider
gr.HTML("""
<div class="slider-comment">
⚡ Tip: Set this number as high as possible or more than 5000 for better scanning of all available papers
</div>
""")
with gr.Row():
year_from = gr.Number(value=2000, label="Year From", precision=0)
year_to = gr.Number(value=2026, label="Year To", precision=0)
open_access = gr.Checkbox(label="Open Access Only")
# ==========================
# SORTING OPTIONS
# ==========================
with gr.Row():
sort_by = gr.Dropdown(
label="Sort By",
choices=["Relevance", "Year", "Citations"],
value="Relevance"
)
sort_order = gr.Dropdown(
label="Sort Order",
choices=["Descending", "Ascending"],
value="Descending"
)
search_btn = gr.Button("🔍 SEARCH PAPERS", variant="primary", size="lg")
# Tab 2: Specific Paper Search
with gr.TabItem("🎯 Find Specific Paper", id=1):
with gr.Column(elem_classes="white-card"):
gr.Markdown("""
### Find a specific paper by:
- **DOI** (e.g., 10.1038/nature12373)
- **Exact Title** (e.g., "Photoacoustic imaging in biomedicine")
- **Author** (find papers by a specific researcher)
✨ **Enhanced**: Title matching now uses fuzzy similarity scoring
""")
with gr.Row():
specific_search_type = gr.Dropdown(
label="Search by",
choices=["DOI", "Exact Title", "Specific Author"],
value="DOI"
)
specific_search_value = gr.Textbox(
label="Enter value",
placeholder="e.g., 10.1038/nature12373 or exact paper title or author name",
scale=2
)
specific_search_btn = gr.Button("🔍 FIND PAPER", variant="primary")
specific_summary = gr.Markdown("Ready")
# Tab 3: BibTeX Checker
with gr.TabItem("📚 Check BibTeX", id=2):
with gr.Column(elem_classes="white-card"):
gr.Markdown("""
### 📋 Upload a BibTeX file to check its entries
This tool will:
1. Parse all BibTeX entries from your file
2. Check each entry against the OpenAlex database
3. Show you which entries are correct and which need fixing
4. Generate a corrected BibTeX file
5. Provide a comparison CSV with all details
✨ **Enhanced**: Better title matching with fuzzy similarity scoring
""")
bibtex_file = gr.File(
label="Upload BibTeX File (.bib)",
file_types=[".bib"],
type="filepath"
)
check_bibtex_btn = gr.Button("🔍 CHECK BIBTEX", variant="primary", size="lg")
bibtex_results = gr.HTML(label="BibTeX Comparison")
with gr.Row():
corrected_bibtex_download = gr.File(label="Download Corrected BibTeX")
comparison_csv_download = gr.File(label="Download Comparison CSV")
# Metrics section (common for first two tabs)
with gr.Row(visible=True) as metrics_row:
total_display = gr.Markdown("**0**", elem_classes="metric-value")
avg_display = gr.Markdown("**0.0**", elem_classes="metric-value")
year_display = gr.Markdown("**N/A**", elem_classes="metric-value")
# Add labels for metrics
with gr.Row(visible=True) as metrics_labels_row:
gr.Markdown("**Total Papers**", elem_classes="metric-label")
gr.Markdown("**Avg Relevance**", elem_classes="metric-label")
gr.Markdown("**Year Range**", elem_classes="metric-label")
# Results section (common for first two tabs)
results = gr.Dataframe(
label="Results",
row_count=15,
column_widths=["25%", "15%", "5%", "5%", "10%", "15%", "15%", "5%", "5%"],
wrap=True,
visible=True
)
state = gr.State([])
# ==========================
# EXPORT SECTION (for first two tabs)
# ==========================
with gr.Column(visible=True) as export_section:
gr.HTML("""
<h3 style="color: #2563eb !important; font-weight: 600; margin-bottom: 16px; margin-top: 24px;">📥 Export Results</h3>
""")
with gr.Row():
filename = gr.Textbox(
label="Filename",
value="papers_results",
placeholder="Enter base filename without extension"
)
format_type = gr.Dropdown(
label="Format",
choices=["TXT", "CSV", "Excel", "BibTeX"],
value="TXT"
)
export_btn = gr.Button("📥 Download Results", variant="secondary")
export_status = gr.Markdown("")
download_file = gr.File(visible=False)
# Move searcher instantiation and click bindings inside the Blocks context
searcher = PaperSearchPro()
def search_wrapper(q, k, a, m, yf, yt, oa, sb, so):
try:
df, papers, total, avg, yr = searcher.search_papers(q, k, a, m, yf, yt, oa, sb, so)
return df, papers, f"**{total}**", f"**{avg:.2f}**", f"**{yr}**"
except Exception as e:
empty_df = pd.DataFrame(columns=['Title', 'Authors', 'Year', 'Citations', 'Journal', 'PDF_URL', 'Abstract', 'BibTeX', 'Relevance', 'Matched Keywords'])
return empty_df, [], "**0**", "**0.0**", "**N/A**"
def specific_search_wrapper(search_type, search_value):
try:
df, papers, summary = searcher.search_specific_paper(search_type, search_value)
if not papers:
return df, papers, summary, "**0**", "**0.0**", "**N/A**"
total = len(papers)
avg_rel = sum(p.get('Match_Score', 0) for p in papers) / total if total else 0
years = [p["Year"] for p in papers if p["Year"] != "N/A"]
year_range = f"{min(years)} - {max(years)}" if years else "N/A"
return df, papers, summary, f"**{total}**", f"**{avg_rel:.2f}**", f"**{year_range}**"
except Exception as e:
empty_df = pd.DataFrame(columns=['Title', 'Authors', 'Year', 'Citations', 'Journal', 'PDF_URL', 'Abstract', 'BibTeX'])
return empty_df, [], f"❌ Error: {str(e)}", "**0**", "**0.0**", "**N/A**"
def check_bibtex_wrapper(bibtex_file):
if bibtex_file is None:
return None, None, "❌ Please upload a BibTeX file", None, None
try:
corrected_bib_path, csv_path, html_output = searcher.check_bibtex_file(bibtex_file.name)
return corrected_bib_path, csv_path, html_output, corrected_bib_path, csv_path
except Exception as e:
return None, None, f"❌ Error: {str(e)}", None, None
search_btn.click(
search_wrapper,
inputs=[query, keywords, authors, max_papers, year_from, year_to, open_access, sort_by, sort_order],
outputs=[results, state, total_display, avg_display, year_display]
)
specific_search_btn.click(
specific_search_wrapper,
inputs=[specific_search_type, specific_search_value],
outputs=[results, state, specific_summary, total_display, avg_display, year_display]
)
check_bibtex_btn.click(
check_bibtex_wrapper,
inputs=[bibtex_file],
outputs=[corrected_bibtex_download, comparison_csv_download, bibtex_results, corrected_bibtex_download, comparison_csv_download]
)
def export_results(papers, fname, fmt):
if not papers:
return None, "❌ No papers to export", gr.File(visible=False)
df = pd.DataFrame(papers)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
if 'All Authors' in df.columns:
df = df.drop(columns=['All Authors'])
if fmt == "CSV":
path = f"{fname}_{timestamp}.csv"
df.to_csv(path, index=False)
elif fmt == "Excel":
path = f"{fname}_{timestamp}.xlsx"
df.to_excel(path, index=False, engine='openpyxl')
elif fmt == "BibTeX":
path = f"{fname}_{timestamp}.bib"
with open(path, "w", encoding="utf-8") as f:
for _, row in df.iterrows():
if row['BibTeX'] and row['BibTeX'] != "":
f.write(row['BibTeX'] + "\n\n")
else: # TXT format
path = f"{fname}_{timestamp}.txt"
with open(path, "w", encoding="utf-8") as f:
f.write("PAPER SEARCH PRO RESULTS\n")
f.write("="*80 + "\n")
f.write(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
f.write(f"Total Papers: {len(df)}\n")
f.write("="*80 + "\n\n")
for i, row in df.iterrows():
f.write(f"PAPER #{i+1}\n")
f.write("-"*40 + "\n")
for col in df.columns:
if col == "BibTeX":
f.write(f"\n{col}:\n{row[col]}\n")
elif col == "PDF_URL":
f.write(f"{col}: {row[col]}\n")
else:
f.write(f"{col}: {row[col]}\n")
f.write("\n" + "="*80 + "\n\n")
return path, f"✅ File ready: {os.path.basename(path)}", gr.File(visible=True)
export_btn.click(
export_results,
inputs=[state, filename, format_type],
outputs=[download_file, export_status, download_file]
)
# Show/hide elements based on active tab
def on_tab_select(evt: gr.SelectData):
if evt.value == "📚 Check BibTeX":
return {
metrics_row: gr.update(visible=False),
metrics_labels_row: gr.update(visible=False),
results: gr.update(visible=False),
export_section: gr.update(visible=False)
}
else:
return {
metrics_row: gr.update(visible=True),
metrics_labels_row: gr.update(visible=True),
results: gr.update(visible=True),
export_section: gr.update(visible=True)
}
tabs_component.select(
on_tab_select,
inputs=None,
outputs=[metrics_row, metrics_labels_row, results, export_section],
queue=False
)
return demo
# ============================================
# MAIN
# ============================================
if __name__ == "__main__":
# Install required package if not present
try:
import bibtexparser
except ImportError:
os.system("pip install bibtexparser")
import bibtexparser
demo = create_interface()
demo.launch(
inbrowser=True,
height=1000,
server_name="0.0.0.0", # Makes it accessible on network
share=False
)
import requests
import pandas as pd
import time
from datetime import datetime
import gradio as gr
import os
import re
import bibtexparser
from bibtexparser.bparser import BibTexParser
from bibtexparser.customization import convert_to_unicode
from difflib import SequenceMatcher
import numpy as np
from collections import Counter
import math
# ============================================
# CLEAN RESPONSIVE CSS - (same as your original)
# ============================================
custom_css = """
@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap');
:root {
--primary: #2563eb;
--secondary: #7c3aed;
--border: #d1d5db;
--bg-soft: #e5e7eb !important;
--text-main: #111827;
}
/* ================================================= */
/* GENERAL LAYOUT */
/* ================================================= */
body, .gradio-container {
font-family: 'Inter', sans_serif !important;
background-color: #e5e7eb !important;
color: var(--text-main) !important;
}
.gradio-container {
max-width: 1600px !important;
margin: auto !important;
}
.white-card {
background: #ffffff !important;
border: 1px solid var(--border) !important;
border-radius: 12px !important;
padding: 2rem !important;
box-shadow: 0 4px 6px rgba(0, 0, 0, 0.05) !important;
}
h1 span {
background: linear-gradient(135deg, var(--primary), var(--secondary));
-webkit-background-clip: text;
-webkit-text-fill-color: transparent;
}
.subtitle-text {
color: #2563eb !important;
font-weight: 500 !important;
}
/* ================================================= */
/* METRICS */
/* ================================================= */
.metric-value {
font-size: 2rem !important;
font-weight: 700 !important;
color: var(--primary) !important;
}
/* ================================================= */
/* BUTTONS */
/* ================================================= */
button, .gr-button {
color: #ffffff !important;
font-weight: 600 !important;
}
button.variant-primary {
background-color: var(--primary) !important;
}
button.variant-secondary {
background-color: #6b7280 !important;
}
/* ================================================= */
/* TABS */
/* ================================================= */
.tab-nav {
background-color: #f3f4f6 !important;
border-radius: 8px !important;
padding: 4px !important;
margin-bottom: 20px !important;
}