anik-jha commited on
Commit
62cb8f8
Β·
0 Parent(s):

proxy setup

Browse files
Files changed (6) hide show
  1. .github/workflows/daily-update.yml +74 -0
  2. .gitignore +50 -0
  3. Dockerfile +55 -0
  4. README.md +31 -0
  5. app.py +157 -0
  6. requirements.txt +13 -0
.github/workflows/daily-update.yml ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: Daily Paper Update
2
+
3
+ on:
4
+ schedule:
5
+ - cron: '0 0 * * *' # Runs at midnight UTC every day
6
+ workflow_dispatch: # Allows manual triggering
7
+
8
+ jobs:
9
+ update-papers:
10
+ runs-on: ubuntu-latest
11
+
12
+ steps:
13
+ - name: Checkout repository
14
+ uses: actions/checkout@v4
15
+ with:
16
+ token: ${{ secrets.GITHUB_TOKEN }}
17
+
18
+ - name: Set up Docker Buildx
19
+ uses: docker/setup-buildx-action@v3
20
+
21
+ - name: Build and start services
22
+ run: docker-compose up --build -d
23
+
24
+ - name: Wait for service to be ready
25
+ run: |
26
+ timeout 300 bash -c 'until curl -f http://localhost:8080/; do sleep 5; done'
27
+
28
+ - name: Trigger paper fetch
29
+ run: curl -X GET http://localhost:8080/api/papers
30
+
31
+ - name: Stop services
32
+ run: docker-compose down
33
+
34
+ - name: Check for changes
35
+ id: check_changes
36
+ run: |
37
+ if git diff --quiet data/ papers/; then
38
+ echo "No changes detected"
39
+ echo "changes=false" >> $GITHUB_OUTPUT
40
+ else
41
+ echo "Changes detected in data/ or papers/"
42
+ echo "changes=true" >> $GITHUB_OUTPUT
43
+ fi
44
+
45
+ - name: Commit and push changes
46
+ if: steps.check_changes.outputs.changes == 'true'
47
+ run: |
48
+ git config --local user.email "action@github.com"
49
+ git config --local user.name "GitHub Action"
50
+ git add data/ papers/
51
+ git commit -m "Daily paper update: $(date +'%Y-%m-%d')"
52
+ git push
53
+
54
+ - name: Sync to public repository
55
+ if: steps.check_changes.outputs.changes == 'true'
56
+ run: |
57
+ # Clone public repo
58
+ git clone https://x-access-token:${{ secrets.PUBLIC_REPO_PAT }}@github.com/YOUR_USERNAME/daily-paper-reader-public.git public-repo
59
+ cd public-repo
60
+
61
+ # Copy updated directories from private repo
62
+ cp -r ../data/* ./data/ 2>/dev/null || mkdir -p ./data && cp -r ../data/* ./data/
63
+ cp -r ../papers/* ./papers/ 2>/dev/null || mkdir -p ./papers && cp -r ../papers/* ./papers/
64
+
65
+ # Check if there are changes
66
+ if git diff --quiet; then
67
+ echo "No changes to sync to public repo"
68
+ else
69
+ git config --local user.email "action@github.com"
70
+ git config --local user.name "GitHub Action"
71
+ git add data/ papers/
72
+ git commit -m "Daily paper update: $(date +'%Y-%m-%d')"
73
+ git push
74
+ fi
.gitignore ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Environment files
2
+ .env
3
+ *.env
4
+
5
+ # Python
6
+ __pycache__/
7
+ *.py[cod]
8
+ *$py.class
9
+ *.so
10
+ .Python
11
+ venv/
12
+ .venv/
13
+ ENV/
14
+ env/
15
+
16
+ # Application specific - DO NOT COMMIT ACTUAL CODE FILES
17
+ # The actual code should be in daily_paper_reader_code Space (private)
18
+ # Only the loader app.py should be in this proxy repo
19
+ *.py
20
+ !app.py
21
+ *.ipynb
22
+ templates/
23
+ utils/
24
+ data/
25
+ papers/
26
+ reference/
27
+ cache.json
28
+ *.sh
29
+ *.md
30
+ !README.md
31
+ SETUP_INSTRUCTIONS.md
32
+ PROXY_SETUP_SUMMARY.md
33
+ QUICK_REFERENCE.md
34
+ *.png
35
+
36
+ # GitHub workflow files (optional - keep if needed)
37
+ .github/conversation*
38
+ .github/objective*
39
+ .github/project*
40
+
41
+ # IDE
42
+ .vscode/
43
+ .idea/
44
+ *.swp
45
+ *.swo
46
+ *~
47
+
48
+ # OS
49
+ .DS_Store
50
+ Thumbs.db
Dockerfile ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Dockerfile for PUBLIC Space
2
+ # This loads and runs code from a private HF Space
3
+
4
+ FROM python:3.12-slim
5
+
6
+ # Set working directory
7
+ WORKDIR /app
8
+
9
+ # Set Python to run in unbuffered mode (see logs immediately)
10
+ ENV PYTHONUNBUFFERED=1
11
+
12
+ # Install system dependencies
13
+ RUN apt-get update && apt-get install -y --no-install-recommends \
14
+ curl \
15
+ build-essential \
16
+ gcc \
17
+ libffi-dev \
18
+ libxml2-dev \
19
+ libxslt1-dev \
20
+ poppler-utils \
21
+ && apt-get clean \
22
+ && rm -rf /var/lib/apt/lists/*
23
+
24
+ # Install Python packages needed for loading from private space
25
+ RUN pip install --no-cache-dir --upgrade pip && \
26
+ pip install --no-cache-dir \
27
+ huggingface-hub \
28
+ Flask
29
+
30
+ # Copy requirements and install additional dependencies
31
+ COPY requirements.txt .
32
+ RUN pip install --no-cache-dir -r requirements.txt
33
+
34
+ # Create a non-root user
35
+ RUN useradd -m -u 1000 appuser && \
36
+ chown -R appuser:appuser /app
37
+
38
+ # Copy only the loader script
39
+ COPY app.py .
40
+
41
+ # Switch to non-root user
42
+ USER appuser
43
+
44
+ # Set environment variables
45
+ ENV PORT=5000
46
+
47
+ # Expose port
48
+ EXPOSE 5000
49
+
50
+ # Health check
51
+ HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
52
+ CMD curl -f http://localhost:5000/ || exit 1
53
+
54
+ # Run the loader which will download and execute the private space code
55
+ CMD ["python", "app.py"]
README.md ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Daily Paper Reader
3
+ emoji: πŸ“š
4
+ colorFrom: blue
5
+ colorTo: purple
6
+ sdk: docker
7
+ sdk_version: "3.12"
8
+ app_file: app.py
9
+ pinned: false
10
+ ---
11
+
12
+ # Daily Paper Reader
13
+
14
+ A simple Python-based application that fetches, ranks, and summarizes the top ML/CV research papers from arXiv using HuggingFace models.
15
+
16
+ ## Features
17
+
18
+ - πŸ“š **Automatic Paper Fetching**: Retrieves recent papers from arXiv (cs.CV and cs.LG)
19
+ - πŸ† **Smart Ranking**: Ranks papers using author h-index, GitHub stars, and recency
20
+ - πŸ€– **AI Summaries**: Generates structured summaries using HuggingFace LLMs
21
+ - πŸ’Ύ **LRU Caching**: Caches summaries to save API calls
22
+ - πŸ”„ **Cohere Reranking**: Optional reranking with Cohere API
23
+ - 🎨 **Clean UI**: Simple, responsive web interface
24
+
25
+ ## Setup
26
+
27
+ Visit the Space to use the application directly, or deploy your own instance using Docker.
28
+
29
+ ## Architecture
30
+
31
+ The application automatically fetches and processes papers from arXiv, providing intelligent ranking and AI-powered summaries.
app.py ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Public-facing loader app for Daily Paper Reader.
3
+ This app downloads and executes the main application from a private HF Space.
4
+ """
5
+ import os
6
+ import sys
7
+ import importlib.util
8
+ from pathlib import Path
9
+ from huggingface_hub import snapshot_download
10
+
11
+ # Configuration - UPDATE THIS WITH YOUR PRIVATE SPACE REPO ID
12
+ PRIVATE_SPACE_REPO = "ml-stuff01/daily_paper_reader_code" # e.g., "ml-stuff01/daily-paper-reader-code"
13
+ CACHE_DIR = Path("/tmp/private_space_cache")
14
+
15
+ def setup_cache_directory():
16
+ """Setup and return cache directory for private space files."""
17
+ CACHE_DIR.mkdir(exist_ok=True)
18
+ print(f"βœ“ Cache directory created: {CACHE_DIR}")
19
+ return CACHE_DIR
20
+
21
+ def download_private_space():
22
+ """Download the entire private space repository."""
23
+ try:
24
+ # Check for HF token
25
+ hf_token = os.getenv("HF_TOKEN")
26
+ if not hf_token:
27
+ print("❌ HF_TOKEN environment variable is not set!")
28
+ print("To fix this:")
29
+ print(" 1. Go to https://huggingface.co/settings/tokens")
30
+ print(" 2. Create a token with 'read' access")
31
+ print(" 3. Add it as a secret named 'HF_TOKEN' in your Space settings")
32
+ print(" 4. Or set it locally: export HF_TOKEN='your_token_here'")
33
+ return False
34
+
35
+ print(f"πŸ“₯ Downloading private space from {PRIVATE_SPACE_REPO}...")
36
+ print(f"πŸ”‘ Using authentication token (length: {len(hf_token)} chars)")
37
+
38
+ snapshot_download(
39
+ repo_id=PRIVATE_SPACE_REPO,
40
+ repo_type="space",
41
+ local_dir=CACHE_DIR,
42
+ token=hf_token
43
+ )
44
+ print("βœ“ Private space downloaded successfully!")
45
+ return True
46
+ except Exception as e:
47
+ print(f"❌ Error downloading private space: {str(e)}")
48
+ print("\nTroubleshooting:")
49
+ print(f" 1. Verify the space '{PRIVATE_SPACE_REPO}' exists and is spelled correctly")
50
+ print(" 2. Ensure your HF token has access to this private space")
51
+ print(" 3. Check that the token has 'read' permissions")
52
+ print(f" 4. Try accessing the space directly: https://huggingface.co/spaces/{PRIVATE_SPACE_REPO}")
53
+ return False
54
+
55
+ def install_requirements():
56
+ """Install requirements from the private space if requirements.txt exists."""
57
+ requirements_path = CACHE_DIR / "requirements.txt"
58
+
59
+ if not requirements_path.exists():
60
+ print("ℹ️ No requirements.txt found in private space, skipping installation")
61
+ return True
62
+
63
+ try:
64
+ print(f"πŸ“¦ Installing dependencies from {requirements_path}...")
65
+ import subprocess
66
+ result = subprocess.run(
67
+ [sys.executable, "-m", "pip", "install", "-r", str(requirements_path)],
68
+ capture_output=True,
69
+ text=True,
70
+ timeout=300 # 5 minute timeout
71
+ )
72
+
73
+ if result.returncode == 0:
74
+ print("βœ“ Dependencies installed successfully!")
75
+ return True
76
+ else:
77
+ print(f"⚠️ Warning during dependency installation:")
78
+ print(result.stderr)
79
+ print("Continuing anyway - some dependencies might be missing...")
80
+ return True # Continue even if some packages fail
81
+
82
+ except subprocess.TimeoutExpired:
83
+ print("⚠️ Dependency installation timed out. Continuing anyway...")
84
+ return True
85
+ except Exception as e:
86
+ print(f"⚠️ Error installing dependencies: {str(e)}")
87
+ print("Continuing anyway - some dependencies might be missing...")
88
+ return True # Continue even on error
89
+
90
+ def load_and_run_app():
91
+ """Load the main application from the private space and run it."""
92
+ try:
93
+ # Path to the main app file in the private space cache
94
+ app_path = CACHE_DIR / "app.py"
95
+
96
+ if not app_path.exists():
97
+ print(f"❌ App file not found at {app_path}")
98
+ return False
99
+
100
+ print(f"πŸš€ Loading application from {app_path}...")
101
+
102
+ # Add the cache directory to Python path so imports work
103
+ sys.path.insert(0, str(CACHE_DIR))
104
+
105
+ # Change to the cache directory so relative paths work
106
+ os.chdir(str(CACHE_DIR))
107
+
108
+ # Load the module dynamically
109
+ spec = importlib.util.spec_from_file_location("app_main", app_path)
110
+ app_module = importlib.util.module_from_spec(spec)
111
+
112
+ # Execute the module - this will start the Flask server
113
+ # The exec_module will block if the app runs a server
114
+ spec.loader.exec_module(app_module)
115
+
116
+ # If we get here, the module loaded but didn't start a blocking server
117
+ # Try to find and run the Flask app
118
+ if hasattr(app_module, 'app'):
119
+ print("βœ“ Flask app found, starting server...")
120
+ port = int(os.getenv('PORT', 5000))
121
+ app_module.app.run(host='0.0.0.0', port=port, debug=False)
122
+ else:
123
+ print("βœ“ Application module loaded successfully!")
124
+
125
+ return True
126
+
127
+ except Exception as e:
128
+ print(f"❌ Error loading application: {str(e)}")
129
+ import traceback
130
+ traceback.print_exc()
131
+ return False
132
+
133
+ if __name__ == "__main__":
134
+ print("=" * 60)
135
+ print("πŸ“š Daily Paper Reader")
136
+ print("=" * 60)
137
+
138
+ # Step 1: Setup cache directory
139
+ setup_cache_directory()
140
+
141
+ # Step 2: Download private space
142
+ if not download_private_space():
143
+ print("\n❌ Failed to download private space. Exiting.")
144
+ sys.exit(1)
145
+
146
+ # Step 3: Install requirements from private space
147
+ if not install_requirements():
148
+ print("\n❌ Failed to install requirements. Exiting.")
149
+ sys.exit(1)
150
+
151
+ # Step 4: Load and run the app (this should block if it starts a server)
152
+ if not load_and_run_app():
153
+ print("\n❌ Failed to load application. Exiting.")
154
+ sys.exit(1)
155
+
156
+ # If we reach here, the app finished without error
157
+ # This shouldn't normally happen if the app runs a server
requirements.txt ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Minimal requirements for the proxy loader
2
+ # The actual app dependencies will be installed from the private space
3
+
4
+ # Core dependencies for downloading and running private space
5
+ huggingface-hub>=0.20.0
6
+ Flask>=3.0.0
7
+ python-dotenv>=1.0.0
8
+
9
+ # Common dependencies that the private space app will likely need
10
+ # These are pre-installed to speed up the loading process
11
+ requests>=2.31.0
12
+ beautifulsoup4>=4.12.0
13
+ PyPDF2>=3.0.0