Spaces:
Sleeping
Sleeping
Deploy BytePulse AI Backend v1.0
Browse files- README.md +26 -0
- app.py +21 -0
- app/api/v1/endpoints/digests.py +169 -0
- app/api/v1/endpoints/notifications.py +33 -0
- app/api/v1/endpoints/search.py +38 -0
- app/api/v1/endpoints/trigger.py +24 -0
- app/api/v1/router.py +9 -0
- app/config.py +30 -0
- app/db/database.py +23 -0
- app/db/models.py +42 -0
- app/scheduler/cron.py +166 -0
- app/scrapers/aggregator.py +59 -0
- app/scrapers/arstechnica.py +39 -0
- app/scrapers/devto.py +38 -0
- app/scrapers/duckduckgo.py +46 -0
- app/scrapers/github_trending.py +65 -0
- app/scrapers/hackernews.py +38 -0
- app/scrapers/huggingface.py +39 -0
- app/scrapers/reddit.py +57 -0
- app/scrapers/techcrunch.py +37 -0
- app/scrapers/twitter_x.py +62 -0
- app/services/encryption.py +56 -0
- app/services/langchain_tools.py +77 -0
- app/services/notifications.py +54 -0
- app/services/summarizer.py +272 -0
- app/vectorstore/chroma_store.py +71 -0
- main.py +114 -0
- requirements.txt +22 -0
README.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: BytePulse AI Backend
|
| 3 |
+
emoji: ⚡
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: indigo
|
| 6 |
+
sdk: gradio
|
| 7 |
+
sdk_version: 6.20.0
|
| 8 |
+
app_file: app.py
|
| 9 |
+
pinned: false
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# 🚀 BytePulse AI - Daily Tech Digest Backend
|
| 13 |
+
|
| 14 |
+
Enterprise AI Tech Digest Aggregator powered by **FastAPI**, **Gemini 2.5 Flash**, **ChromaDB Vector Store**, and **AES-128 Fernet Encryption**.
|
| 15 |
+
|
| 16 |
+
## 🛡️ Environment Secrets
|
| 17 |
+
Set in **Space Settings → Repository Secrets**:
|
| 18 |
+
|
| 19 |
+
- `GOOGLE_API_KEY`: Gemini 2.5 Flash API Key
|
| 20 |
+
- `API_SECRET_KEY`: `bytepulse_secure_app_key_2026`
|
| 21 |
+
- `ENCRYPTION_KEY`: 32-byte Fernet key
|
| 22 |
+
|
| 23 |
+
## 🌐 API Endpoints
|
| 24 |
+
- `GET /health` → `{"status": "healthy"}`
|
| 25 |
+
- `GET /docs` → Swagger UI
|
| 26 |
+
- `POST /api/v1/digest/trigger` → Trigger aggregation
|
app.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import gradio as gr
|
| 2 |
+
from main import app as fastapi_app
|
| 3 |
+
|
| 4 |
+
# Minimal Gradio landing page
|
| 5 |
+
with gr.Blocks(title="BytePulse AI Backend", theme=gr.themes.Soft()) as demo:
|
| 6 |
+
gr.Markdown(
|
| 7 |
+
"""
|
| 8 |
+
# ⚡ BytePulse AI — Daily Tech Digest API
|
| 9 |
+
|
| 10 |
+
> Backend is **live and healthy!** 🚀
|
| 11 |
+
|
| 12 |
+
| Endpoint | Description |
|
| 13 |
+
|----------|-------------|
|
| 14 |
+
| `/health` | Health check |
|
| 15 |
+
| `/docs` | Swagger API Docs |
|
| 16 |
+
| `/api/v1` | API Base |
|
| 17 |
+
"""
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
# Mount FastAPI app inside Gradio (Gradio 6 compatible)
|
| 21 |
+
app = gr.mount_gradio_app(fastapi_app, demo, path="/gradio")
|
app/api/v1/endpoints/digests.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
| 2 |
+
from slowapi import Limiter
|
| 3 |
+
from slowapi.util import get_remote_address
|
| 4 |
+
from sqlalchemy.orm import Session
|
| 5 |
+
from typing import List, Optional
|
| 6 |
+
from pydantic import BaseModel
|
| 7 |
+
from datetime import datetime
|
| 8 |
+
|
| 9 |
+
from app.db.database import get_db
|
| 10 |
+
from app.db.models import DailyDigest, DigestItem
|
| 11 |
+
|
| 12 |
+
router = APIRouter()
|
| 13 |
+
limiter = Limiter(key_func=get_remote_address)
|
| 14 |
+
|
| 15 |
+
# Pydantic Schemas
|
| 16 |
+
class DigestItemSchema(BaseModel):
|
| 17 |
+
id: int
|
| 18 |
+
daily_digest_id: Optional[int]
|
| 19 |
+
title: str
|
| 20 |
+
url: str
|
| 21 |
+
source: str
|
| 22 |
+
category: str
|
| 23 |
+
summary: str
|
| 24 |
+
key_takeaway: Optional[str]
|
| 25 |
+
importance_score: float
|
| 26 |
+
created_at: datetime
|
| 27 |
+
|
| 28 |
+
class Config:
|
| 29 |
+
from_attributes = True
|
| 30 |
+
|
| 31 |
+
class DailyDigestSchema(BaseModel):
|
| 32 |
+
id: int
|
| 33 |
+
date_str: str
|
| 34 |
+
title: str
|
| 35 |
+
overall_summary: str
|
| 36 |
+
top_highlights: Optional[str]
|
| 37 |
+
created_at: datetime
|
| 38 |
+
items: List[DigestItemSchema] = []
|
| 39 |
+
|
| 40 |
+
class Config:
|
| 41 |
+
from_attributes = True
|
| 42 |
+
|
| 43 |
+
from app.services.encryption import decrypt_text
|
| 44 |
+
|
| 45 |
+
def format_decrypted_digest(digest: DailyDigest):
|
| 46 |
+
if not digest:
|
| 47 |
+
return None
|
| 48 |
+
digest.overall_summary = decrypt_text(digest.overall_summary)
|
| 49 |
+
if digest.items:
|
| 50 |
+
for it in digest.items:
|
| 51 |
+
it.summary = decrypt_text(it.summary)
|
| 52 |
+
it.key_takeaway = decrypt_text(it.key_takeaway)
|
| 53 |
+
return digest
|
| 54 |
+
|
| 55 |
+
@router.get("/latest", response_model=DailyDigestSchema)
|
| 56 |
+
@limiter.limit("30/minute")
|
| 57 |
+
def get_latest_digest(request: Request, db: Session = Depends(get_db)):
|
| 58 |
+
"""
|
| 59 |
+
Returns the most recent daily tech digest with all summarized news items.
|
| 60 |
+
"""
|
| 61 |
+
digest = db.query(DailyDigest).order_by(DailyDigest.created_at.desc()).first()
|
| 62 |
+
if not digest:
|
| 63 |
+
raise HTTPException(status_code=404, detail="No digests found yet. Run trigger endpoint to generate one.")
|
| 64 |
+
return format_decrypted_digest(digest)
|
| 65 |
+
|
| 66 |
+
@router.get("", response_model=List[DailyDigestSchema])
|
| 67 |
+
@limiter.limit("30/minute")
|
| 68 |
+
def list_digests(
|
| 69 |
+
request: Request,
|
| 70 |
+
skip: int = Query(0, ge=0),
|
| 71 |
+
limit: int = Query(10, ge=1, le=50),
|
| 72 |
+
db: Session = Depends(get_db)
|
| 73 |
+
):
|
| 74 |
+
"""
|
| 75 |
+
Get a paginated list of past daily digests.
|
| 76 |
+
"""
|
| 77 |
+
digests = db.query(DailyDigest).order_by(DailyDigest.created_at.desc()).offset(skip).limit(limit).all()
|
| 78 |
+
return [format_decrypted_digest(d) for d in digests]
|
| 79 |
+
|
| 80 |
+
@router.get("/{digest_id}", response_model=DailyDigestSchema)
|
| 81 |
+
@limiter.limit("30/minute")
|
| 82 |
+
def get_digest_by_id(request: Request, digest_id: int, db: Session = Depends(get_db)):
|
| 83 |
+
"""
|
| 84 |
+
Get a specific daily digest by ID.
|
| 85 |
+
"""
|
| 86 |
+
digest = db.query(DailyDigest).filter(DailyDigest.id == digest_id).first()
|
| 87 |
+
if not digest:
|
| 88 |
+
raise HTTPException(status_code=404, detail=f"Digest with ID {digest_id} not found")
|
| 89 |
+
return format_decrypted_digest(digest)
|
| 90 |
+
|
| 91 |
+
@router.get("/items/categories", response_model=List[str])
|
| 92 |
+
@limiter.limit("60/minute")
|
| 93 |
+
def list_categories(request: Request, db: Session = Depends(get_db)):
|
| 94 |
+
"""
|
| 95 |
+
Get distinct categories of tech updates available in the database.
|
| 96 |
+
"""
|
| 97 |
+
categories = db.query(DigestItem.category).distinct().all()
|
| 98 |
+
return [c[0] for c in categories if c[0]]
|
| 99 |
+
|
| 100 |
+
@router.get("/items/{item_id}/full")
|
| 101 |
+
@limiter.limit("15/minute")
|
| 102 |
+
async def get_full_article_detail(request: Request, item_id: int, db: Session = Depends(get_db)):
|
| 103 |
+
"""
|
| 104 |
+
Returns full details for a tech news update, including all text chunks from
|
| 105 |
+
RecursiveCharacterTextSplitter and an in-depth AI Technical Analysis from Gemini.
|
| 106 |
+
"""
|
| 107 |
+
from app.services.summarizer import generate_deep_ai_analysis
|
| 108 |
+
from app.vectorstore.chroma_store import vector_store
|
| 109 |
+
|
| 110 |
+
item = db.query(DigestItem).filter(DigestItem.id == item_id).first()
|
| 111 |
+
if not item:
|
| 112 |
+
raise HTTPException(status_code=404, detail=f"Digest item with ID {item_id} not found")
|
| 113 |
+
|
| 114 |
+
# Fetch text chunks from ChunkTextStore
|
| 115 |
+
item_chunks = [rec["chunk_text"] for rec in vector_store.chunk_records if rec.get("item_id") == item_id]
|
| 116 |
+
if not item_chunks:
|
| 117 |
+
item_chunks = [item.summary]
|
| 118 |
+
|
| 119 |
+
# Generate deep AI analysis
|
| 120 |
+
deep_analysis = await generate_deep_ai_analysis(
|
| 121 |
+
title=item.title,
|
| 122 |
+
source=item.source,
|
| 123 |
+
summary=item.summary,
|
| 124 |
+
text_chunks=item_chunks
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
return {
|
| 128 |
+
"item": {
|
| 129 |
+
"id": item.id,
|
| 130 |
+
"daily_digest_id": item.daily_digest_id,
|
| 131 |
+
"title": item.title,
|
| 132 |
+
"url": item.url,
|
| 133 |
+
"source": item.source,
|
| 134 |
+
"category": item.category,
|
| 135 |
+
"summary": item.summary,
|
| 136 |
+
"key_takeaway": item.key_takeaway,
|
| 137 |
+
"importance_score": item.importance_score,
|
| 138 |
+
"created_at": item.created_at
|
| 139 |
+
},
|
| 140 |
+
"text_chunks": item_chunks,
|
| 141 |
+
"deep_ai_analysis": deep_analysis
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
class SocialPostRequest(BaseModel):
|
| 145 |
+
title: str
|
| 146 |
+
source: str = "Tech Update"
|
| 147 |
+
summary: str
|
| 148 |
+
key_takeaway: Optional[str] = ""
|
| 149 |
+
|
| 150 |
+
@router.post("/generate-social-post")
|
| 151 |
+
@limiter.limit("10/minute")
|
| 152 |
+
async def generate_social_post_endpoint(request: Request, req: SocialPostRequest):
|
| 153 |
+
"""
|
| 154 |
+
Generates ready-to-share social media post ideas (LinkedIn, Twitter/X thread, Newsletter, Reddit)
|
| 155 |
+
using Gemini 2.5 Flash LLM.
|
| 156 |
+
"""
|
| 157 |
+
from app.services.summarizer import generate_social_media_posts
|
| 158 |
+
posts = await generate_social_media_posts(
|
| 159 |
+
title=req.title,
|
| 160 |
+
source=req.source,
|
| 161 |
+
summary=req.summary,
|
| 162 |
+
key_takeaway=req.key_takeaway or ""
|
| 163 |
+
)
|
| 164 |
+
return {
|
| 165 |
+
"status": "success",
|
| 166 |
+
"posts": posts
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
|
app/api/v1/endpoints/notifications.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Depends, HTTPException
|
| 2 |
+
from sqlalchemy.orm import Session
|
| 3 |
+
from pydantic import BaseModel
|
| 4 |
+
from datetime import datetime
|
| 5 |
+
|
| 6 |
+
from app.db.database import get_db
|
| 7 |
+
from app.db.models import DeviceToken
|
| 8 |
+
|
| 9 |
+
router = APIRouter()
|
| 10 |
+
|
| 11 |
+
class DeviceTokenRequest(BaseModel):
|
| 12 |
+
token: str
|
| 13 |
+
device_type: str = "android" # android | ios
|
| 14 |
+
|
| 15 |
+
@router.post("/register")
|
| 16 |
+
def register_device_token(req: DeviceTokenRequest, db: Session = Depends(get_db)):
|
| 17 |
+
"""
|
| 18 |
+
Registers or updates an FCM token for mobile push notifications.
|
| 19 |
+
"""
|
| 20 |
+
if not req.token:
|
| 21 |
+
raise HTTPException(status_code=400, detail="Token cannot be empty")
|
| 22 |
+
|
| 23 |
+
existing = db.query(DeviceToken).filter(DeviceToken.token == req.token).first()
|
| 24 |
+
if existing:
|
| 25 |
+
existing.device_type = req.device_type
|
| 26 |
+
existing.updated_at = datetime.utcnow()
|
| 27 |
+
db.commit()
|
| 28 |
+
return {"status": "updated", "message": "Device token updated successfully"}
|
| 29 |
+
|
| 30 |
+
new_token = DeviceToken(token=req.token, device_type=req.device_type)
|
| 31 |
+
db.add(new_token)
|
| 32 |
+
db.commit()
|
| 33 |
+
return {"status": "registered", "message": "Device token registered successfully"}
|
app/api/v1/endpoints/search.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Query
|
| 2 |
+
from typing import List, Dict, Any
|
| 3 |
+
from app.vectorstore.chroma_store import vector_store
|
| 4 |
+
from app.scrapers.duckduckgo import search_duckduckgo_tech_news
|
| 5 |
+
|
| 6 |
+
router = APIRouter()
|
| 7 |
+
|
| 8 |
+
@router.get("")
|
| 9 |
+
def search_text_chunks(
|
| 10 |
+
q: str = Query(..., description="Query string for text search across tech updates", min_length=2),
|
| 11 |
+
top_k: int = Query(5, ge=1, le=20)
|
| 12 |
+
):
|
| 13 |
+
"""
|
| 14 |
+
Search tech updates across stored text chunks generated by LangChain TextSplitter.
|
| 15 |
+
"""
|
| 16 |
+
results = vector_store.search_similar(query=q, top_k=top_k)
|
| 17 |
+
return {
|
| 18 |
+
"query": q,
|
| 19 |
+
"total_results": len(results),
|
| 20 |
+
"results": results
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
@router.get("/live")
|
| 24 |
+
async def live_duckduckgo_search(
|
| 25 |
+
q: str = Query(..., description="Real-time web search query via DuckDuckGo", min_length=2),
|
| 26 |
+
limit: int = Query(5, ge=1, le=10)
|
| 27 |
+
):
|
| 28 |
+
"""
|
| 29 |
+
Performs real-time web search using DuckDuckGo and returns live search results.
|
| 30 |
+
"""
|
| 31 |
+
results = await search_duckduckgo_tech_news(query=q, limit=limit)
|
| 32 |
+
return {
|
| 33 |
+
"query": q,
|
| 34 |
+
"source": "DuckDuckGo Real-Time Search",
|
| 35 |
+
"total_results": len(results),
|
| 36 |
+
"results": results
|
| 37 |
+
}
|
| 38 |
+
|
app/api/v1/endpoints/trigger.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter, BackgroundTasks
|
| 2 |
+
from app.scheduler.cron import run_daily_tech_digest_pipeline
|
| 3 |
+
|
| 4 |
+
router = APIRouter()
|
| 5 |
+
|
| 6 |
+
@router.post("")
|
| 7 |
+
async def trigger_manual_scrape(background_tasks: BackgroundTasks):
|
| 8 |
+
"""
|
| 9 |
+
Manually triggers the tech update scraping, LangChain AI summarization, ChromaDB indexing, and FCM push notification pipeline.
|
| 10 |
+
Runs asynchronously in background tasks.
|
| 11 |
+
"""
|
| 12 |
+
background_tasks.add_task(run_daily_tech_digest_pipeline)
|
| 13 |
+
return {
|
| 14 |
+
"status": "triggered",
|
| 15 |
+
"message": "Daily tech update pipeline started in background. Check /api/v1/digests/latest shortly."
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
@router.post("/sync")
|
| 19 |
+
async def trigger_manual_scrape_sync():
|
| 20 |
+
"""
|
| 21 |
+
Synchronously triggers the pipeline and returns the result immediately.
|
| 22 |
+
"""
|
| 23 |
+
result = await run_daily_tech_digest_pipeline()
|
| 24 |
+
return result
|
app/api/v1/router.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from fastapi import APIRouter
|
| 2 |
+
from app.api.v1.endpoints import digests, search, notifications, trigger
|
| 3 |
+
|
| 4 |
+
api_router = APIRouter()
|
| 5 |
+
|
| 6 |
+
api_router.include_router(digests.router, prefix="/digests", tags=["Tech Digests"])
|
| 7 |
+
api_router.include_router(search.router, prefix="/search", tags=["Semantic Search"])
|
| 8 |
+
api_router.include_router(notifications.router, prefix="/notifications", tags=["Push Notifications"])
|
| 9 |
+
api_router.include_router(trigger.router, prefix="/trigger-scrape", tags=["Pipeline Trigger"])
|
app/config.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
class Settings(BaseSettings):
|
| 5 |
+
APP_NAME: str = "BytePulse AI - Daily Tech Briefing"
|
| 6 |
+
VERSION: str = "1.0.0"
|
| 7 |
+
DEBUG: bool = True
|
| 8 |
+
|
| 9 |
+
# Environment Secrets (Must be set in Hugging Face Space Settings or .env)
|
| 10 |
+
API_SECRET_KEY: str = "bytepulse_secure_app_key_2026"
|
| 11 |
+
ENCRYPTION_KEY: Optional[str] = None
|
| 12 |
+
|
| 13 |
+
HOST: str = "0.0.0.0"
|
| 14 |
+
PORT: int = 8000
|
| 15 |
+
|
| 16 |
+
DATABASE_URL: str = "sqlite:///./tech_updates.db"
|
| 17 |
+
CHROMA_PERSIST_DIRECTORY: str = "./chroma_db"
|
| 18 |
+
|
| 19 |
+
GOOGLE_API_KEY: Optional[str] = None
|
| 20 |
+
OPENAI_API_KEY: Optional[str] = None
|
| 21 |
+
PREFERRED_LLM_PROVIDER: str = "gemini"
|
| 22 |
+
|
| 23 |
+
FIREBASE_CREDENTIALS_FILE: Optional[str] = "./firebase-credentials.json"
|
| 24 |
+
|
| 25 |
+
DAILY_CRON_HOUR: int = 8
|
| 26 |
+
DAILY_CRON_MINUTE: int = 0
|
| 27 |
+
|
| 28 |
+
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
|
| 29 |
+
|
| 30 |
+
settings = Settings()
|
app/db/database.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from sqlalchemy import create_engine
|
| 2 |
+
from sqlalchemy.orm import sessionmaker, declarative_base
|
| 3 |
+
from app.config import settings
|
| 4 |
+
|
| 5 |
+
# Check if SQLite and configure connect_args
|
| 6 |
+
connect_args = {"check_same_thread": False} if settings.DATABASE_URL.startswith("sqlite") else {}
|
| 7 |
+
|
| 8 |
+
engine = create_engine(
|
| 9 |
+
settings.DATABASE_URL,
|
| 10 |
+
connect_args=connect_args,
|
| 11 |
+
echo=settings.DEBUG
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
| 15 |
+
|
| 16 |
+
Base = declarative_base()
|
| 17 |
+
|
| 18 |
+
def get_db():
|
| 19 |
+
db = SessionLocal()
|
| 20 |
+
try:
|
| 21 |
+
yield db
|
| 22 |
+
finally:
|
| 23 |
+
db.close()
|
app/db/models.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from sqlalchemy import Column, Integer, String, Text, DateTime, ForeignKey, Float
|
| 2 |
+
from sqlalchemy.orm import relationship
|
| 3 |
+
from datetime import datetime
|
| 4 |
+
from app.db.database import Base
|
| 5 |
+
|
| 6 |
+
class DailyDigest(Base):
|
| 7 |
+
__tablename__ = "daily_digests"
|
| 8 |
+
|
| 9 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 10 |
+
date_str = Column(String(50), unique=True, index=True) # e.g., "2026-07-22"
|
| 11 |
+
title = Column(String(255), nullable=False)
|
| 12 |
+
overall_summary = Column(Text, nullable=False)
|
| 13 |
+
top_highlights = Column(Text, nullable=True) # JSON or markdown string of top 3 highlights
|
| 14 |
+
created_at = Column(DateTime, default=datetime.utcnow)
|
| 15 |
+
|
| 16 |
+
items = relationship("DigestItem", back_populates="daily_digest", cascade="all, delete-orphan")
|
| 17 |
+
|
| 18 |
+
class DigestItem(Base):
|
| 19 |
+
__tablename__ = "digest_items"
|
| 20 |
+
|
| 21 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 22 |
+
daily_digest_id = Column(Integer, ForeignKey("daily_digests.id"), nullable=True)
|
| 23 |
+
title = Column(String(500), nullable=False)
|
| 24 |
+
url = Column(String(1000), nullable=False)
|
| 25 |
+
source = Column(String(100), nullable=False) # HackerNews, TechCrunch, GitHub Trends
|
| 26 |
+
category = Column(String(100), default="Tech") # AI, Web, Security, Mobile, Open Source, etc.
|
| 27 |
+
summary = Column(Text, nullable=False)
|
| 28 |
+
key_takeaway = Column(Text, nullable=True)
|
| 29 |
+
importance_score = Column(Float, default=5.0)
|
| 30 |
+
published_at = Column(DateTime, default=datetime.utcnow)
|
| 31 |
+
created_at = Column(DateTime, default=datetime.utcnow)
|
| 32 |
+
|
| 33 |
+
daily_digest = relationship("DailyDigest", back_populates="items")
|
| 34 |
+
|
| 35 |
+
class DeviceToken(Base):
|
| 36 |
+
__tablename__ = "device_tokens"
|
| 37 |
+
|
| 38 |
+
id = Column(Integer, primary_key=True, index=True)
|
| 39 |
+
token = Column(String(500), unique=True, index=True, nullable=False)
|
| 40 |
+
device_type = Column(String(50), default="android") # android | ios
|
| 41 |
+
created_at = Column(DateTime, default=datetime.utcnow)
|
| 42 |
+
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
app/scheduler/cron.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import asyncio
|
| 3 |
+
from datetime import datetime
|
| 4 |
+
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
| 5 |
+
from sqlalchemy.orm import Session
|
| 6 |
+
|
| 7 |
+
from app.config import settings
|
| 8 |
+
from app.db.database import SessionLocal, engine, Base
|
| 9 |
+
from app.db.models import DailyDigest, DigestItem, DeviceToken
|
| 10 |
+
from app.scrapers.aggregator import aggregate_all_sources
|
| 11 |
+
from app.services.summarizer import process_and_summarize_updates
|
| 12 |
+
from app.vectorstore.chroma_store import vector_store
|
| 13 |
+
from app.services.notifications import send_daily_digest_notification
|
| 14 |
+
|
| 15 |
+
logger = logging.getLogger(__name__)
|
| 16 |
+
scheduler = AsyncIOScheduler()
|
| 17 |
+
|
| 18 |
+
async def run_daily_tech_digest_pipeline():
|
| 19 |
+
"""
|
| 20 |
+
Main background job:
|
| 21 |
+
1. Scrapes latest tech news
|
| 22 |
+
2. Summarizes with LangChain
|
| 23 |
+
3. Saves to Relational DB & Vector DB
|
| 24 |
+
4. Sends FCM Push Notification
|
| 25 |
+
"""
|
| 26 |
+
today_str = datetime.utcnow().strftime("%Y-%m-%d")
|
| 27 |
+
logger.info(f"--- Starting Daily Tech Digest Pipeline for {today_str} ---")
|
| 28 |
+
|
| 29 |
+
db: Session = SessionLocal()
|
| 30 |
+
try:
|
| 31 |
+
# Step 1: Check if digest already exists for today
|
| 32 |
+
existing = db.query(DailyDigest).filter(DailyDigest.date_str == today_str).first()
|
| 33 |
+
if existing:
|
| 34 |
+
logger.info(f"Digest for {today_str} already exists (ID: {existing.id}). Overwriting with fresh updates.")
|
| 35 |
+
db.delete(existing)
|
| 36 |
+
db.commit()
|
| 37 |
+
|
| 38 |
+
# Step 2: Fetch raw items from scrapers
|
| 39 |
+
raw_items = await aggregate_all_sources(limit_per_source=5)
|
| 40 |
+
if not raw_items:
|
| 41 |
+
logger.warning("No news items retrieved from scrapers. Aborting pipeline.")
|
| 42 |
+
return {"status": "no_data"}
|
| 43 |
+
|
| 44 |
+
# Step 3: Summarize via LangChain
|
| 45 |
+
digest_meta, processed_items = await process_and_summarize_updates(raw_items)
|
| 46 |
+
|
| 47 |
+
from app.services.encryption import encrypt_text
|
| 48 |
+
|
| 49 |
+
# Step 4: Save Encrypted Data to Database
|
| 50 |
+
new_digest = DailyDigest(
|
| 51 |
+
date_str=today_str,
|
| 52 |
+
title=digest_meta["title"],
|
| 53 |
+
overall_summary=encrypt_text(digest_meta["overall_summary"]),
|
| 54 |
+
top_highlights=digest_meta["top_highlights"]
|
| 55 |
+
)
|
| 56 |
+
db.add(new_digest)
|
| 57 |
+
db.flush() # get new_digest.id
|
| 58 |
+
|
| 59 |
+
db_items = []
|
| 60 |
+
for item in processed_items:
|
| 61 |
+
db_item = DigestItem(
|
| 62 |
+
daily_digest_id=new_digest.id,
|
| 63 |
+
title=item["title"],
|
| 64 |
+
url=item["url"],
|
| 65 |
+
source=item["source"],
|
| 66 |
+
category=item["category"],
|
| 67 |
+
summary=encrypt_text(item["summary"]),
|
| 68 |
+
key_takeaway=encrypt_text(item["key_takeaway"]),
|
| 69 |
+
importance_score=item["importance_score"]
|
| 70 |
+
)
|
| 71 |
+
db.add(db_item)
|
| 72 |
+
db_items.append(db_item)
|
| 73 |
+
|
| 74 |
+
db.commit()
|
| 75 |
+
logger.info(f"Successfully saved Encrypted Daily Digest #{new_digest.id} with {len(db_items)} items to database.")
|
| 76 |
+
|
| 77 |
+
# Step 5: Index in ChromaDB Vector Store
|
| 78 |
+
chroma_items = []
|
| 79 |
+
for item in db_items:
|
| 80 |
+
chroma_items.append({
|
| 81 |
+
"id": item.id,
|
| 82 |
+
"title": item.title,
|
| 83 |
+
"url": item.url,
|
| 84 |
+
"source": item.source,
|
| 85 |
+
"category": item.category,
|
| 86 |
+
"summary": item.summary,
|
| 87 |
+
"key_takeaway": item.key_takeaway
|
| 88 |
+
})
|
| 89 |
+
vector_count = vector_store.add_digest_items(chroma_items)
|
| 90 |
+
logger.info(f"Indexed {vector_count} items into ChromaDB vector store.")
|
| 91 |
+
|
| 92 |
+
# Step 6: Send FCM Push Notification to registered devices
|
| 93 |
+
tokens = [t.token for t in db.query(DeviceToken).all()]
|
| 94 |
+
notif_res = await send_daily_digest_notification(
|
| 95 |
+
tokens=tokens,
|
| 96 |
+
digest_title=new_digest.title,
|
| 97 |
+
summary_snippet=new_digest.overall_summary
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
return {
|
| 101 |
+
"status": "success",
|
| 102 |
+
"digest_id": new_digest.id,
|
| 103 |
+
"date": today_str,
|
| 104 |
+
"items_count": len(db_items),
|
| 105 |
+
"vector_indexed": vector_count,
|
| 106 |
+
"notification_status": notif_res
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
except Exception as e:
|
| 110 |
+
logger.error(f"Error executing Daily Tech Digest Pipeline: {e}", exc_info=True)
|
| 111 |
+
db.rollback()
|
| 112 |
+
return {"status": "error", "message": str(e)}
|
| 113 |
+
finally:
|
| 114 |
+
db.close()
|
| 115 |
+
|
| 116 |
+
async def keep_alive_self_ping():
|
| 117 |
+
"""
|
| 118 |
+
Self-ping background worker to keep Hugging Face Spaces 100% active and prevent idle sleep mode.
|
| 119 |
+
"""
|
| 120 |
+
import httpx
|
| 121 |
+
target_url = getattr(settings, "SPACE_URL", None) or f"http://127.0.0.1:{settings.PORT}/health"
|
| 122 |
+
try:
|
| 123 |
+
async with httpx.AsyncClient(timeout=5.0) as client:
|
| 124 |
+
res = await client.get(target_url)
|
| 125 |
+
logger.info(f"Keep-Alive Self-Ping sent to {target_url} -> Status {res.status_code}")
|
| 126 |
+
except Exception as e:
|
| 127 |
+
logger.warning(f"Keep-Alive Self-Ping failed: {e}")
|
| 128 |
+
|
| 129 |
+
def start_scheduler():
|
| 130 |
+
"""
|
| 131 |
+
Registers twice-daily cron jobs (7:00 AM Morning & 7:00 PM Evening UTC)
|
| 132 |
+
and 10-minute keep-alive pings to prevent Hugging Face Space sleep mode.
|
| 133 |
+
"""
|
| 134 |
+
# 1. Morning Digest Job at 7:00 AM
|
| 135 |
+
scheduler.add_job(
|
| 136 |
+
run_daily_tech_digest_pipeline,
|
| 137 |
+
trigger="cron",
|
| 138 |
+
hour=7,
|
| 139 |
+
minute=0,
|
| 140 |
+
id="morning_tech_digest_job",
|
| 141 |
+
replace_existing=True
|
| 142 |
+
)
|
| 143 |
+
# 2. Evening Digest Job at 7:00 PM (19:00)
|
| 144 |
+
scheduler.add_job(
|
| 145 |
+
run_daily_tech_digest_pipeline,
|
| 146 |
+
trigger="cron",
|
| 147 |
+
hour=19,
|
| 148 |
+
minute=0,
|
| 149 |
+
id="evening_tech_digest_job",
|
| 150 |
+
replace_existing=True
|
| 151 |
+
)
|
| 152 |
+
# 3. Keep-Alive Self-Ping every 10 minutes (prevents Hugging Face 48h idle pause)
|
| 153 |
+
scheduler.add_job(
|
| 154 |
+
keep_alive_self_ping,
|
| 155 |
+
trigger="interval",
|
| 156 |
+
minutes=10,
|
| 157 |
+
id="hf_keep_alive_ping_job",
|
| 158 |
+
replace_existing=True
|
| 159 |
+
)
|
| 160 |
+
scheduler.start()
|
| 161 |
+
logger.info("APScheduler started. Scheduled twice daily digests (07:00 & 19:00 UTC) and 10-minute HF Keep-Alive self-pings.")
|
| 162 |
+
|
| 163 |
+
def stop_scheduler():
|
| 164 |
+
if scheduler.running:
|
| 165 |
+
scheduler.shutdown()
|
| 166 |
+
logger.info("APScheduler stopped.")
|
app/scrapers/aggregator.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import logging
|
| 3 |
+
from typing import List, Dict, Any
|
| 4 |
+
from app.scrapers.hackernews import fetch_hackernews_top
|
| 5 |
+
from app.scrapers.techcrunch import fetch_techcrunch_news
|
| 6 |
+
from app.scrapers.github_trending import fetch_github_trending
|
| 7 |
+
from app.scrapers.duckduckgo import search_duckduckgo_tech_news
|
| 8 |
+
from app.scrapers.reddit import fetch_reddit_tech_news
|
| 9 |
+
from app.scrapers.twitter_x import fetch_twitter_x_tech_news
|
| 10 |
+
from app.scrapers.huggingface import fetch_huggingface_news
|
| 11 |
+
from app.scrapers.devto import fetch_devto_top_articles
|
| 12 |
+
from app.scrapers.arstechnica import fetch_arstechnica_news
|
| 13 |
+
|
| 14 |
+
logger = logging.getLogger(__name__)
|
| 15 |
+
|
| 16 |
+
async def aggregate_all_sources(limit_per_source: int = 4) -> List[Dict[str, Any]]:
|
| 17 |
+
"""
|
| 18 |
+
Scrapes all 8 authoritative tech sources concurrently:
|
| 19 |
+
1. HackerNews
|
| 20 |
+
2. TechCrunch RSS
|
| 21 |
+
3. GitHub Trending
|
| 22 |
+
4. DuckDuckGo Real-Time Search
|
| 23 |
+
5. Reddit (r/technology, r/programming, r/MachineLearning, r/LocalLLaMA)
|
| 24 |
+
6. X (Twitter) Targeted Tech Search
|
| 25 |
+
7. Hugging Face AI Blog & Model Releases
|
| 26 |
+
8. Dev.to Top Developer Tutorials
|
| 27 |
+
9. Ars Technica Verified Tech News
|
| 28 |
+
"""
|
| 29 |
+
logger.info("Starting 8-channel aggregated fetch across top tech platforms...")
|
| 30 |
+
|
| 31 |
+
hn_task = fetch_hackernews_top(limit=limit_per_source)
|
| 32 |
+
tc_task = fetch_techcrunch_news(limit=limit_per_source)
|
| 33 |
+
gh_task = fetch_github_trending(limit=limit_per_source)
|
| 34 |
+
ddg_task = search_duckduckgo_tech_news(query="latest AI software developer tech news updates", limit=limit_per_source)
|
| 35 |
+
reddit_task = fetch_reddit_tech_news(limit_per_sub=2)
|
| 36 |
+
twitter_task = fetch_twitter_x_tech_news(max_results=limit_per_source)
|
| 37 |
+
hf_task = fetch_huggingface_news(limit=limit_per_source)
|
| 38 |
+
devto_task = fetch_devto_top_articles(limit=limit_per_source)
|
| 39 |
+
ars_task = fetch_arstechnica_news(limit=limit_per_source)
|
| 40 |
+
|
| 41 |
+
results = await asyncio.gather(
|
| 42 |
+
hn_task, tc_task, gh_task, ddg_task, reddit_task, twitter_task, hf_task, devto_task, ars_task,
|
| 43 |
+
return_exceptions=True
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
all_items = []
|
| 47 |
+
source_names = [
|
| 48 |
+
"HackerNews", "TechCrunch", "GitHub Trends", "DuckDuckGo Web Search",
|
| 49 |
+
"Reddit", "X (Twitter)", "Hugging Face", "Dev.to", "Ars Technica"
|
| 50 |
+
]
|
| 51 |
+
|
| 52 |
+
for idx, res in enumerate(results):
|
| 53 |
+
if isinstance(res, Exception):
|
| 54 |
+
logger.error(f"Failed to fetch from {source_names[idx]}: {res}")
|
| 55 |
+
elif isinstance(res, list):
|
| 56 |
+
logger.info(f"Fetched {len(res)} items from {source_names[idx]}")
|
| 57 |
+
all_items.extend(res)
|
| 58 |
+
|
| 59 |
+
return all_items
|
app/scrapers/arstechnica.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import httpx
|
| 2 |
+
import xml.etree.ElementTree as ET
|
| 3 |
+
import logging
|
| 4 |
+
from typing import List, Dict, Any
|
| 5 |
+
|
| 6 |
+
logger = logging.getLogger(__name__)
|
| 7 |
+
|
| 8 |
+
ARS_RSS = "https://feeds.arstechnica.com/arstechnica/technology-lab"
|
| 9 |
+
|
| 10 |
+
async def fetch_arstechnica_news(limit: int = 5) -> List[Dict[str, Any]]:
|
| 11 |
+
"""
|
| 12 |
+
Scrapes Ars Technica official RSS feed for high-accuracy tech journalism.
|
| 13 |
+
"""
|
| 14 |
+
items = []
|
| 15 |
+
headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) BytePulseAI/1.0" }
|
| 16 |
+
|
| 17 |
+
try:
|
| 18 |
+
async with httpx.AsyncClient(headers=headers, timeout=10.0, follow_redirects=True) as client:
|
| 19 |
+
res = await client.get(ARS_RSS)
|
| 20 |
+
if res.status_code == 200:
|
| 21 |
+
root = ET.fromstring(res.text)
|
| 22 |
+
for entry in root.findall(".//item")[:limit]:
|
| 23 |
+
title = entry.findtext("title", "")
|
| 24 |
+
url = entry.findtext("link", "")
|
| 25 |
+
description = entry.findtext("description", "") or title
|
| 26 |
+
pub_date = entry.findtext("pubDate", "")
|
| 27 |
+
|
| 28 |
+
items.append({
|
| 29 |
+
"title": f"[ArsTechnica] {title}",
|
| 30 |
+
"url": url,
|
| 31 |
+
"source": "Ars Technica",
|
| 32 |
+
"raw_content": f"Title: {title}\nURL: {url}\nSummary: {description[:300]}",
|
| 33 |
+
"published_at": pub_date
|
| 34 |
+
})
|
| 35 |
+
except Exception as err:
|
| 36 |
+
logger.warning(f"Ars Technica scraper failed: {err}")
|
| 37 |
+
|
| 38 |
+
logger.info(f"Ars Technica scraper returned {len(items)} items.")
|
| 39 |
+
return items
|
app/scrapers/devto.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import httpx
|
| 2 |
+
import logging
|
| 3 |
+
from typing import List, Dict, Any
|
| 4 |
+
|
| 5 |
+
logger = logging.getLogger(__name__)
|
| 6 |
+
|
| 7 |
+
DEVTO_API_URL = "https://dev.to/api/articles?top=1&per_page=5"
|
| 8 |
+
|
| 9 |
+
async def fetch_devto_top_articles(limit: int = 5) -> List[Dict[str, Any]]:
|
| 10 |
+
"""
|
| 11 |
+
Scrapes Dev.to official REST API for top trending developer articles and tutorials.
|
| 12 |
+
"""
|
| 13 |
+
items = []
|
| 14 |
+
headers = { "User-Agent": "BytePulseAI/1.0" }
|
| 15 |
+
|
| 16 |
+
try:
|
| 17 |
+
async with httpx.AsyncClient(headers=headers, timeout=10.0, follow_redirects=True) as client:
|
| 18 |
+
res = await client.get(DEVTO_API_URL)
|
| 19 |
+
if res.status_code == 200:
|
| 20 |
+
data = res.json()
|
| 21 |
+
for article in data[:limit]:
|
| 22 |
+
title = article.get("title", "")
|
| 23 |
+
url = article.get("url", "")
|
| 24 |
+
description = article.get("description", "")
|
| 25 |
+
tag_list = ", ".join(article.get("tag_list", []))
|
| 26 |
+
|
| 27 |
+
items.append({
|
| 28 |
+
"title": f"[Dev.to] {title}",
|
| 29 |
+
"url": url,
|
| 30 |
+
"source": "Dev.to",
|
| 31 |
+
"raw_content": f"Title: {title}\nTags: {tag_list}\nSummary: {description}",
|
| 32 |
+
"published_at": article.get("published_at")
|
| 33 |
+
})
|
| 34 |
+
except Exception as err:
|
| 35 |
+
logger.warning(f"Dev.to scraper failed: {err}")
|
| 36 |
+
|
| 37 |
+
logger.info(f"Dev.to scraper returned {len(items)} items.")
|
| 38 |
+
return items
|
app/scrapers/duckduckgo.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import httpx
|
| 2 |
+
from bs4 import BeautifulSoup
|
| 3 |
+
import logging
|
| 4 |
+
from typing import List, Dict, Any
|
| 5 |
+
|
| 6 |
+
logger = logging.getLogger(__name__)
|
| 7 |
+
|
| 8 |
+
DUCKDUCKGO_HTML_URL = "https://html.duckduckgo.com/html/"
|
| 9 |
+
|
| 10 |
+
async def search_duckduckgo_tech_news(query: str = "latest artificial intelligence tech developments news", limit: int = 5) -> List[Dict[str, Any]]:
|
| 11 |
+
"""
|
| 12 |
+
Performs real-time web search using DuckDuckGo to find live tech updates.
|
| 13 |
+
"""
|
| 14 |
+
results = []
|
| 15 |
+
headers = {
|
| 16 |
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
| 17 |
+
}
|
| 18 |
+
data = {"q": query}
|
| 19 |
+
|
| 20 |
+
try:
|
| 21 |
+
async with httpx.AsyncClient(timeout=10.0, follow_redirects=True) as client:
|
| 22 |
+
res = await client.post(DUCKDUCKGO_HTML_URL, data=data, headers=headers)
|
| 23 |
+
if res.status_code == 200:
|
| 24 |
+
soup = BeautifulSoup(res.text, "html.parser")
|
| 25 |
+
rows = soup.select(".result")
|
| 26 |
+
|
| 27 |
+
for row in rows[:limit]:
|
| 28 |
+
title_elem = row.select_one(".result__title a")
|
| 29 |
+
snippet_elem = row.select_one(".result__snippet")
|
| 30 |
+
|
| 31 |
+
if title_elem:
|
| 32 |
+
title = title_elem.get_text(strip=True)
|
| 33 |
+
link = title_elem.get("href", "")
|
| 34 |
+
snippet = snippet_elem.get_text(strip=True) if snippet_elem else "Real-time web update"
|
| 35 |
+
|
| 36 |
+
results.append({
|
| 37 |
+
"title": title,
|
| 38 |
+
"url": link,
|
| 39 |
+
"source": "DuckDuckGo Web Search",
|
| 40 |
+
"score": 85,
|
| 41 |
+
"raw_content": f"Query: {query}\nTitle: {title}\nSnippet: {snippet}"
|
| 42 |
+
})
|
| 43 |
+
except Exception as e:
|
| 44 |
+
logger.error(f"Error executing DuckDuckGo real-time web search for '{query}': {e}")
|
| 45 |
+
|
| 46 |
+
return results
|
app/scrapers/github_trending.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import httpx
|
| 2 |
+
from bs4 import BeautifulSoup
|
| 3 |
+
import logging
|
| 4 |
+
from typing import List, Dict, Any
|
| 5 |
+
|
| 6 |
+
logger = logging.getLogger(__name__)
|
| 7 |
+
|
| 8 |
+
GITHUB_TRENDING_URL = "https://github.com/trending"
|
| 9 |
+
GITHUB_SEARCH_API = "https://api.github.com/search/repositories?q=created:>2026-01-01&sort=stars&order=desc"
|
| 10 |
+
|
| 11 |
+
async def fetch_github_trending(limit: int = 10) -> List[Dict[str, Any]]:
|
| 12 |
+
"""
|
| 13 |
+
Fetches trending GitHub repositories.
|
| 14 |
+
First tries HTML scraping of github.com/trending, falls back to REST search API if blocked.
|
| 15 |
+
"""
|
| 16 |
+
repos = []
|
| 17 |
+
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}
|
| 18 |
+
|
| 19 |
+
try:
|
| 20 |
+
async with httpx.AsyncClient(timeout=12.0, follow_redirects=True) as client:
|
| 21 |
+
res = await client.get(GITHUB_TRENDING_URL, headers=headers)
|
| 22 |
+
if res.status_code == 200:
|
| 23 |
+
soup = BeautifulSoup(res.text, "html.parser")
|
| 24 |
+
rows = soup.select("article.Box-row")
|
| 25 |
+
for row in rows[:limit]:
|
| 26 |
+
title_elem = row.select_one("h2 a")
|
| 27 |
+
desc_elem = row.select_one("p")
|
| 28 |
+
lang_elem = row.select_one("span[itemprop='programmingLanguage']")
|
| 29 |
+
|
| 30 |
+
repo_name = title_elem.get_text(strip=True).replace(" ", "") if title_elem else "Unknown Repo"
|
| 31 |
+
repo_link = f"https://github.com{title_elem['href']}" if title_elem and 'href' in title_elem.attrs else ""
|
| 32 |
+
desc = desc_elem.get_text(strip=True) if desc_elem else "No description provided."
|
| 33 |
+
lang = lang_elem.get_text(strip=True) if lang_elem else "Multi-language"
|
| 34 |
+
|
| 35 |
+
repos.append({
|
| 36 |
+
"title": f"GitHub Trending: {repo_name}",
|
| 37 |
+
"url": repo_link,
|
| 38 |
+
"source": "GitHub Trends",
|
| 39 |
+
"score": 90,
|
| 40 |
+
"language": lang,
|
| 41 |
+
"raw_content": f"Repository: {repo_name}\nLanguage: {lang}\nDescription: {desc}"
|
| 42 |
+
})
|
| 43 |
+
except Exception as e:
|
| 44 |
+
logger.warning(f"Error scraping GitHub trending HTML ({e}), trying GitHub API fallback...")
|
| 45 |
+
|
| 46 |
+
# Fallback to GitHub Search API if scraping returns no items
|
| 47 |
+
if not repos:
|
| 48 |
+
try:
|
| 49 |
+
async with httpx.AsyncClient(timeout=10.0) as client:
|
| 50 |
+
api_res = await client.get("https://api.github.com/search/repositories?q=stars:>500&sort=updated&order=desc", headers=headers)
|
| 51 |
+
if api_res.status_code == 200:
|
| 52 |
+
items = api_res.json().get("items", [])[:limit]
|
| 53 |
+
for item in items:
|
| 54 |
+
repos.append({
|
| 55 |
+
"title": f"GitHub Repo: {item.get('full_name')}",
|
| 56 |
+
"url": item.get("html_url", ""),
|
| 57 |
+
"source": "GitHub Trends",
|
| 58 |
+
"score": item.get("stargazers_count", 0),
|
| 59 |
+
"language": item.get("language", "Unknown"),
|
| 60 |
+
"raw_content": f"Repository: {item.get('full_name')}\nStars: {item.get('stargazers_count')}\nDescription: {item.get('description', '')}"
|
| 61 |
+
})
|
| 62 |
+
except Exception as api_err:
|
| 63 |
+
logger.error(f"GitHub API fallback failed: {api_err}")
|
| 64 |
+
|
| 65 |
+
return repos
|
app/scrapers/hackernews.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import httpx
|
| 2 |
+
import logging
|
| 3 |
+
from typing import List, Dict, Any
|
| 4 |
+
|
| 5 |
+
logger = logging.getLogger(__name__)
|
| 6 |
+
|
| 7 |
+
HN_TOP_STORIES_URL = "https://hacker-news.firebaseio.com/v0/topstories.json"
|
| 8 |
+
HN_ITEM_URL = "https://hacker-news.firebaseio.com/v0/item/{item_id}.json"
|
| 9 |
+
|
| 10 |
+
async def fetch_hackernews_top(limit: int = 10) -> List[Dict[str, Any]]:
|
| 11 |
+
"""
|
| 12 |
+
Fetches top stories from HackerNews API.
|
| 13 |
+
"""
|
| 14 |
+
articles = []
|
| 15 |
+
try:
|
| 16 |
+
async with httpx.AsyncClient(timeout=10.0) as client:
|
| 17 |
+
res = await client.get(HN_TOP_STORIES_URL)
|
| 18 |
+
if res.status_code != 200:
|
| 19 |
+
logger.error(f"HackerNews API returned status {res.status_code}")
|
| 20 |
+
return articles
|
| 21 |
+
|
| 22 |
+
story_ids = res.json()[:limit]
|
| 23 |
+
for s_id in story_ids:
|
| 24 |
+
item_res = await client.get(HN_ITEM_URL.format(item_id=s_id))
|
| 25 |
+
if item_res.status_code == 200:
|
| 26 |
+
data = item_res.json()
|
| 27 |
+
if data and data.get("type") == "story" and "url" in data:
|
| 28 |
+
articles.append({
|
| 29 |
+
"title": data.get("title", "No Title"),
|
| 30 |
+
"url": data.get("url", f"https://news.ycombinator.com/item?id={s_id}"),
|
| 31 |
+
"source": "HackerNews",
|
| 32 |
+
"score": data.get("score", 0),
|
| 33 |
+
"by": data.get("by", "unknown"),
|
| 34 |
+
"raw_content": f"Title: {data.get('title')}. Score: {data.get('score')}. By: {data.get('by')}"
|
| 35 |
+
})
|
| 36 |
+
except Exception as e:
|
| 37 |
+
logger.error(f"Error fetching HackerNews updates: {e}")
|
| 38 |
+
return articles
|
app/scrapers/huggingface.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import httpx
|
| 2 |
+
import xml.etree.ElementTree as ET
|
| 3 |
+
import logging
|
| 4 |
+
from typing import List, Dict, Any
|
| 5 |
+
|
| 6 |
+
logger = logging.getLogger(__name__)
|
| 7 |
+
|
| 8 |
+
HUGGINGFACE_RSS = "https://huggingface.co/blog/feed.xml"
|
| 9 |
+
|
| 10 |
+
async def fetch_huggingface_news(limit: int = 5) -> List[Dict[str, Any]]:
|
| 11 |
+
"""
|
| 12 |
+
Scrapes Hugging Face official RSS feed for AI/LLM research releases and blog posts.
|
| 13 |
+
"""
|
| 14 |
+
items = []
|
| 15 |
+
headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) BytePulseAI/1.0" }
|
| 16 |
+
|
| 17 |
+
try:
|
| 18 |
+
async with httpx.AsyncClient(headers=headers, timeout=10.0, follow_redirects=True) as client:
|
| 19 |
+
res = await client.get(HUGGINGFACE_RSS)
|
| 20 |
+
if res.status_code == 200:
|
| 21 |
+
root = ET.fromstring(res.text)
|
| 22 |
+
for entry in root.findall(".//item")[:limit]:
|
| 23 |
+
title = entry.findtext("title", "")
|
| 24 |
+
url = entry.findtext("link", "")
|
| 25 |
+
description = entry.findtext("description", "") or title
|
| 26 |
+
pub_date = entry.findtext("pubDate", "")
|
| 27 |
+
|
| 28 |
+
items.append({
|
| 29 |
+
"title": f"[HuggingFace] {title}",
|
| 30 |
+
"url": url,
|
| 31 |
+
"source": "Hugging Face",
|
| 32 |
+
"raw_content": f"Title: {title}\nURL: {url}\nSummary: {description[:300]}",
|
| 33 |
+
"published_at": pub_date
|
| 34 |
+
})
|
| 35 |
+
except Exception as err:
|
| 36 |
+
logger.warning(f"Hugging Face scraper failed: {err}")
|
| 37 |
+
|
| 38 |
+
logger.info(f"Hugging Face scraper returned {len(items)} items.")
|
| 39 |
+
return items
|
app/scrapers/reddit.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import httpx
|
| 2 |
+
import logging
|
| 3 |
+
from typing import List, Dict, Any
|
| 4 |
+
|
| 5 |
+
logger = logging.getLogger(__name__)
|
| 6 |
+
|
| 7 |
+
SUBREDDITS = ["technology", "programming", "MachineLearning", "ArtificialInteligence", "LocalLLaMA"]
|
| 8 |
+
|
| 9 |
+
async def fetch_reddit_tech_news(limit_per_sub: int = 3) -> List[Dict[str, Any]]:
|
| 10 |
+
"""
|
| 11 |
+
Scrapes top hot tech posts from Reddit public JSON feeds.
|
| 12 |
+
Returns: List of standardized item dicts.
|
| 13 |
+
"""
|
| 14 |
+
items = []
|
| 15 |
+
headers = {
|
| 16 |
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) BytePulseAI/1.0 (Tech Digest Aggregator)"
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
async with httpx.AsyncClient(headers=headers, timeout=10.0, follow_redirects=True) as client:
|
| 20 |
+
for sub in SUBREDDITS:
|
| 21 |
+
try:
|
| 22 |
+
url = f"https://www.reddit.com/r/{sub}/hot.json?limit={limit_per_sub}"
|
| 23 |
+
res = await client.get(url)
|
| 24 |
+
if res.status_code == 200:
|
| 25 |
+
data = res.json()
|
| 26 |
+
children = data.get("data", {}).get("children", [])
|
| 27 |
+
for child in children:
|
| 28 |
+
post = child.get("data", {})
|
| 29 |
+
if post.get("stickied"):
|
| 30 |
+
continue # skip pinned posts
|
| 31 |
+
|
| 32 |
+
title = post.get("title", "")
|
| 33 |
+
permalink = f"https://www.reddit.com{post.get('permalink')}"
|
| 34 |
+
post_url = post.get("url", permalink)
|
| 35 |
+
selftext = post.get("selftext", "")
|
| 36 |
+
score = post.get("score", 0)
|
| 37 |
+
num_comments = post.get("num_comments", 0)
|
| 38 |
+
|
| 39 |
+
content_snippet = selftext[:300] if selftext else title
|
| 40 |
+
|
| 41 |
+
items.append({
|
| 42 |
+
"title": f"[r/{sub}] {title}",
|
| 43 |
+
"url": post_url,
|
| 44 |
+
"source": f"Reddit (r/{sub})",
|
| 45 |
+
"raw_content": f"Title: {title}\nScore: {score} upvotes, {num_comments} comments\n\nContent: {content_snippet}",
|
| 46 |
+
"published_at": None,
|
| 47 |
+
"metadata": {
|
| 48 |
+
"upvotes": score,
|
| 49 |
+
"comments": num_comments,
|
| 50 |
+
"subreddit": sub
|
| 51 |
+
}
|
| 52 |
+
})
|
| 53 |
+
except Exception as err:
|
| 54 |
+
logger.warning(f"Reddit scraper failed for r/{sub}: {err}")
|
| 55 |
+
|
| 56 |
+
logger.info(f"Reddit Scraper fetched {len(items)} posts from r/{','.join(SUBREDDITS)}.")
|
| 57 |
+
return items
|
app/scrapers/techcrunch.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import httpx
|
| 2 |
+
import feedparser
|
| 3 |
+
import logging
|
| 4 |
+
from bs4 import BeautifulSoup
|
| 5 |
+
from typing import List, Dict, Any
|
| 6 |
+
|
| 7 |
+
logger = logging.getLogger(__name__)
|
| 8 |
+
|
| 9 |
+
TECHCRUNCH_RSS_URL = "https://techcrunch.com/feed/"
|
| 10 |
+
|
| 11 |
+
async def fetch_techcrunch_news(limit: int = 10) -> List[Dict[str, Any]]:
|
| 12 |
+
"""
|
| 13 |
+
Fetches latest tech articles from TechCrunch RSS feed.
|
| 14 |
+
"""
|
| 15 |
+
articles = []
|
| 16 |
+
try:
|
| 17 |
+
async with httpx.AsyncClient(timeout=12.0, follow_redirects=True) as client:
|
| 18 |
+
res = await client.get(TECHCRUNCH_RSS_URL, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"})
|
| 19 |
+
if res.status_code == 200:
|
| 20 |
+
feed = feedparser.parse(res.text)
|
| 21 |
+
for entry in feed.entries[:limit]:
|
| 22 |
+
# Clean HTML description snippet using BeautifulSoup
|
| 23 |
+
raw_summary = entry.get("summary", "") or entry.get("description", "")
|
| 24 |
+
soup = BeautifulSoup(raw_summary, "html.parser")
|
| 25 |
+
clean_text = soup.get_text(strip=True)
|
| 26 |
+
|
| 27 |
+
articles.append({
|
| 28 |
+
"title": entry.get("title", "No Title"),
|
| 29 |
+
"url": entry.get("link", ""),
|
| 30 |
+
"source": "TechCrunch",
|
| 31 |
+
"score": 100,
|
| 32 |
+
"published_parsed": entry.get("published", ""),
|
| 33 |
+
"raw_content": f"Title: {entry.get('title')}\nSnippet: {clean_text}"
|
| 34 |
+
})
|
| 35 |
+
except Exception as e:
|
| 36 |
+
logger.error(f"Error fetching TechCrunch news: {e}")
|
| 37 |
+
return articles
|
app/scrapers/twitter_x.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import httpx
|
| 2 |
+
import logging
|
| 3 |
+
from typing import List, Dict, Any
|
| 4 |
+
|
| 5 |
+
logger = logging.getLogger(__name__)
|
| 6 |
+
|
| 7 |
+
async def fetch_twitter_x_tech_news(query: str = "site:x.com (AI OR LLM OR OpenAI OR Gemini OR React OR Python OR Anthropic)", max_results: int = 5) -> List[Dict[str, Any]]:
|
| 8 |
+
"""
|
| 9 |
+
Scrapes real-time trending tech tweets & announcements from X (Twitter) via targeted search & fallback.
|
| 10 |
+
"""
|
| 11 |
+
items = []
|
| 12 |
+
|
| 13 |
+
# Method 1: Try DDGS search
|
| 14 |
+
try:
|
| 15 |
+
try:
|
| 16 |
+
from ddgs import DDGS
|
| 17 |
+
except ImportError:
|
| 18 |
+
from duckduckgo_search import DDGS
|
| 19 |
+
|
| 20 |
+
with DDGS() as ddgs:
|
| 21 |
+
results = list(ddgs.text(query, max_results=max_results))
|
| 22 |
+
for res in results:
|
| 23 |
+
title = res.get("title", "")
|
| 24 |
+
url = res.get("href", "")
|
| 25 |
+
snippet = res.get("body", "")
|
| 26 |
+
|
| 27 |
+
items.append({
|
| 28 |
+
"title": f"[X/Twitter] {title.replace(' on X: ', ': ')}",
|
| 29 |
+
"url": url,
|
| 30 |
+
"source": "X (Twitter)",
|
| 31 |
+
"raw_content": f"Tweet/Post Title: {title}\nURL: {url}\nContent Snippet: {snippet}",
|
| 32 |
+
"published_at": None
|
| 33 |
+
})
|
| 34 |
+
if items:
|
| 35 |
+
logger.info(f"Twitter/X Scraper fetched {len(items)} posts via DDGS.")
|
| 36 |
+
return items
|
| 37 |
+
except Exception as err:
|
| 38 |
+
logger.warning(f"DuckDuckGo search for X/Twitter failed: {err}")
|
| 39 |
+
|
| 40 |
+
# Method 2: Fallback Nitter / Tech Twitter Public RSS Feed
|
| 41 |
+
try:
|
| 42 |
+
headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) BytePulseAI/1.0" }
|
| 43 |
+
async with httpx.AsyncClient(headers=headers, timeout=8.0, follow_redirects=True) as client:
|
| 44 |
+
res = await client.get("https://nitter.net/search/rss?f=tweets&q=AI+OR+LLM+OR+OpenAI")
|
| 45 |
+
if res.status_code == 200:
|
| 46 |
+
import xml.etree.ElementTree as ET
|
| 47 |
+
root = ET.fromstring(res.text)
|
| 48 |
+
for item in root.findall(".//item")[:max_results]:
|
| 49 |
+
title = item.findtext("title", "")
|
| 50 |
+
link = item.findtext("link", "")
|
| 51 |
+
items.append({
|
| 52 |
+
"title": f"[X/Twitter] {title[:100]}",
|
| 53 |
+
"url": link,
|
| 54 |
+
"source": "X (Twitter)",
|
| 55 |
+
"raw_content": f"X Tweet: {title}\nURL: {link}",
|
| 56 |
+
"published_at": None
|
| 57 |
+
})
|
| 58 |
+
except Exception as e:
|
| 59 |
+
logger.warning(f"Nitter fallback failed: {e}")
|
| 60 |
+
|
| 61 |
+
logger.info(f"Twitter/X Scraper returned {len(items)} posts.")
|
| 62 |
+
return items
|
app/services/encryption.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import base64
|
| 3 |
+
import logging
|
| 4 |
+
from cryptography.fernet import Fernet
|
| 5 |
+
from app.config import settings
|
| 6 |
+
|
| 7 |
+
logger = logging.getLogger(__name__)
|
| 8 |
+
|
| 9 |
+
def get_or_create_fernet_key() -> bytes:
|
| 10 |
+
"""
|
| 11 |
+
Retrieves ENCRYPTION_KEY from environment/settings or generates a deterministic fallback key based on API_SECRET_KEY.
|
| 12 |
+
"""
|
| 13 |
+
env_key = getattr(settings, "ENCRYPTION_KEY", None) or os.environ.get("ENCRYPTION_KEY")
|
| 14 |
+
if env_key:
|
| 15 |
+
try:
|
| 16 |
+
return env_key.encode('utf-8')
|
| 17 |
+
except Exception as e:
|
| 18 |
+
logger.warning(f"Invalid ENCRYPTION_KEY in env: {e}")
|
| 19 |
+
|
| 20 |
+
# Fallback key derived deterministically from API_SECRET_KEY
|
| 21 |
+
seed_str = (settings.API_SECRET_KEY + "bytepulse_salt_2026_aes_key").encode('utf-8')
|
| 22 |
+
key_32bytes = (seed_str * 4)[:32]
|
| 23 |
+
return base64.urlsafe_b64encode(key_32bytes)
|
| 24 |
+
|
| 25 |
+
_fernet_key = get_or_create_fernet_key()
|
| 26 |
+
_cipher_suite = Fernet(_fernet_key)
|
| 27 |
+
|
| 28 |
+
def encrypt_text(plain_text: str) -> str:
|
| 29 |
+
"""
|
| 30 |
+
Encrypts plain text into AES Fernet Ciphertext string.
|
| 31 |
+
Returns: Ciphertext string (e.g. gAAAAABm...)
|
| 32 |
+
"""
|
| 33 |
+
if not plain_text:
|
| 34 |
+
return ""
|
| 35 |
+
try:
|
| 36 |
+
encrypted_bytes = _cipher_suite.encrypt(plain_text.encode('utf-8'))
|
| 37 |
+
return encrypted_bytes.decode('utf-8')
|
| 38 |
+
except Exception as e:
|
| 39 |
+
logger.error(f"Error encrypting text: {e}")
|
| 40 |
+
return plain_text
|
| 41 |
+
|
| 42 |
+
def decrypt_text(cipher_text: str) -> str:
|
| 43 |
+
"""
|
| 44 |
+
Decrypts AES Fernet Ciphertext back to plain text.
|
| 45 |
+
If input is not encrypted or decryption fails, returns original text gracefully.
|
| 46 |
+
"""
|
| 47 |
+
if not cipher_text:
|
| 48 |
+
return ""
|
| 49 |
+
if not cipher_text.startswith("gAAAAA"):
|
| 50 |
+
return cipher_text # Not encrypted
|
| 51 |
+
try:
|
| 52 |
+
decrypted_bytes = _cipher_suite.decrypt(cipher_text.encode('utf-8'))
|
| 53 |
+
return decrypted_bytes.decode('utf-8')
|
| 54 |
+
except Exception as e:
|
| 55 |
+
logger.warning(f"Error decrypting ciphertext: {e}")
|
| 56 |
+
return cipher_text
|
app/services/langchain_tools.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import asyncio
|
| 3 |
+
from typing import List, Dict, Any
|
| 4 |
+
from langchain_core.tools import Tool
|
| 5 |
+
|
| 6 |
+
from app.scrapers.hackernews import fetch_hackernews_top
|
| 7 |
+
from app.scrapers.techcrunch import fetch_techcrunch_news
|
| 8 |
+
from app.scrapers.github_trending import fetch_github_trending
|
| 9 |
+
from app.scrapers.duckduckgo import search_duckduckgo_tech_news
|
| 10 |
+
from app.scrapers.reddit import fetch_reddit_tech_news
|
| 11 |
+
from app.scrapers.twitter_x import fetch_twitter_x_tech_news
|
| 12 |
+
|
| 13 |
+
logger = logging.getLogger(__name__)
|
| 14 |
+
|
| 15 |
+
def get_hackernews_tool_func(query: str = "") -> str:
|
| 16 |
+
"""Synchronous wrapper for HackerNews tool."""
|
| 17 |
+
items = asyncio.run(fetch_hackernews_top(limit=5))
|
| 18 |
+
return str([{ "title": i["title"], "url": i["url"], "snippet": i["raw_content"][:150] } for i in items])
|
| 19 |
+
|
| 20 |
+
def get_reddit_tool_func(query: str = "") -> str:
|
| 21 |
+
"""Synchronous wrapper for Reddit Tech Scraper tool."""
|
| 22 |
+
items = asyncio.run(fetch_reddit_tech_news(limit_per_sub=2))
|
| 23 |
+
return str([{ "title": i["title"], "url": i["url"], "snippet": i["raw_content"][:150] } for i in items])
|
| 24 |
+
|
| 25 |
+
def get_twitter_tool_func(query: str = "") -> str:
|
| 26 |
+
"""Synchronous wrapper for Twitter/X Search tool."""
|
| 27 |
+
items = asyncio.run(fetch_twitter_x_tech_news(max_results=5))
|
| 28 |
+
return str([{ "title": i["title"], "url": i["url"], "snippet": i["raw_content"][:150] } for i in items])
|
| 29 |
+
|
| 30 |
+
def get_github_tool_func(query: str = "") -> str:
|
| 31 |
+
"""Synchronous wrapper for GitHub Trending tool."""
|
| 32 |
+
items = asyncio.run(fetch_github_trending(limit=5))
|
| 33 |
+
return str([{ "title": i["title"], "url": i["url"], "snippet": i["raw_content"][:150] } for i in items])
|
| 34 |
+
|
| 35 |
+
def get_duckduckgo_tool_func(query: str) -> str:
|
| 36 |
+
"""Synchronous wrapper for DuckDuckGo Live Search tool."""
|
| 37 |
+
items = asyncio.run(search_duckduckgo_tech_news(query=query if query else "latest AI tech updates", limit=5))
|
| 38 |
+
return str([{ "title": i["title"], "url": i["url"], "snippet": i["raw_content"][:150] } for i in items])
|
| 39 |
+
|
| 40 |
+
# Export LangChain Tools
|
| 41 |
+
HackerNewsTool = Tool(
|
| 42 |
+
name="HackerNewsScraper",
|
| 43 |
+
func=get_hackernews_tool_func,
|
| 44 |
+
description="Fetches top developer stories and tech discussions from Y Combinator Hacker News."
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
RedditTechTool = Tool(
|
| 48 |
+
name="RedditTechScraper",
|
| 49 |
+
func=get_reddit_tool_func,
|
| 50 |
+
description="Fetches trending tech posts from r/technology, r/programming, r/MachineLearning, and r/LocalLLaMA."
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
TwitterXTechTool = Tool(
|
| 54 |
+
name="TwitterXTechSearch",
|
| 55 |
+
func=get_twitter_tool_func,
|
| 56 |
+
description="Fetches real-time viral tweets and tech announcements from X (Twitter)."
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
GitHubTrendingTool = Tool(
|
| 60 |
+
name="GitHubTrendingScraper",
|
| 61 |
+
func=get_github_tool_func,
|
| 62 |
+
description="Fetches top trending open-source repositories and software projects on GitHub."
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
DuckDuckGoSearchTool = Tool(
|
| 66 |
+
name="DuckDuckGoLiveSearch",
|
| 67 |
+
func=get_duckduckgo_tool_func,
|
| 68 |
+
description="Performs real-time web search for any tech news query, framework, or AI model."
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
ALL_LANGCHAIN_TOOLS = [
|
| 72 |
+
HackerNewsTool,
|
| 73 |
+
RedditTechTool,
|
| 74 |
+
TwitterXTechTool,
|
| 75 |
+
GitHubTrendingTool,
|
| 76 |
+
DuckDuckGoSearchTool
|
| 77 |
+
]
|
app/services/notifications.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import logging
|
| 3 |
+
from typing import List
|
| 4 |
+
from app.config import settings
|
| 5 |
+
|
| 6 |
+
logger = logging.getLogger(__name__)
|
| 7 |
+
|
| 8 |
+
# Initialize Firebase Admin SDK if available
|
| 9 |
+
_firebase_initialized = False
|
| 10 |
+
try:
|
| 11 |
+
if settings.FIREBASE_CREDENTIALS_FILE and os.path.exists(settings.FIREBASE_CREDENTIALS_FILE):
|
| 12 |
+
import firebase_admin
|
| 13 |
+
from firebase_admin import credentials, messaging
|
| 14 |
+
cred = credentials.Certificate(settings.FIREBASE_CREDENTIALS_FILE)
|
| 15 |
+
firebase_admin.initialize_app(cred)
|
| 16 |
+
_firebase_initialized = True
|
| 17 |
+
logger.info("Firebase Admin SDK initialized successfully.")
|
| 18 |
+
else:
|
| 19 |
+
logger.info("Firebase credentials file not found. Running in mock push notification mode.")
|
| 20 |
+
except Exception as e:
|
| 21 |
+
logger.warning(f"Failed to initialize Firebase Admin SDK: {e}")
|
| 22 |
+
|
| 23 |
+
async def send_daily_digest_notification(tokens: List[str], digest_title: str, summary_snippet: str) -> dict:
|
| 24 |
+
"""
|
| 25 |
+
Sends FCM Push Notification to registered mobile devices.
|
| 26 |
+
"""
|
| 27 |
+
if not tokens:
|
| 28 |
+
logger.info("No device tokens registered. Skipping notification dispatch.")
|
| 29 |
+
return {"sent_count": 0, "status": "no_tokens"}
|
| 30 |
+
|
| 31 |
+
if _firebase_initialized:
|
| 32 |
+
try:
|
| 33 |
+
from firebase_admin import messaging
|
| 34 |
+
message = messaging.MulticastMessage(
|
| 35 |
+
notification=messaging.Notification(
|
| 36 |
+
title=f"🚀 {digest_title}",
|
| 37 |
+
body=summary_snippet[:150] + "..." if len(summary_snippet) > 150 else summary_snippet
|
| 38 |
+
),
|
| 39 |
+
data={
|
| 40 |
+
"type": "daily_digest",
|
| 41 |
+
"click_action": "FLUTTER_NOTIFICATION_CLICK"
|
| 42 |
+
},
|
| 43 |
+
tokens=tokens
|
| 44 |
+
)
|
| 45 |
+
response = messaging.send_multicast(message)
|
| 46 |
+
logger.info(f"FCM Notification sent. Success: {response.success_count}, Failures: {response.failure_count}")
|
| 47 |
+
return {"sent_count": response.success_count, "failure_count": response.failure_count, "status": "success"}
|
| 48 |
+
except Exception as err:
|
| 49 |
+
logger.error(f"Error dispatching FCM notification: {err}")
|
| 50 |
+
return {"sent_count": 0, "error": str(err), "status": "failed"}
|
| 51 |
+
|
| 52 |
+
# Mock mode output
|
| 53 |
+
logger.info(f"[MOCK PUSH NOTIFICATION] Title: '🚀 {digest_title}' | Body: '{summary_snippet[:100]}...' | Recipients: {len(tokens)} devices")
|
| 54 |
+
return {"sent_count": len(tokens), "status": "mock_success"}
|
app/services/summarizer.py
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import json
|
| 3 |
+
import re
|
| 4 |
+
from typing import List, Dict, Any, Tuple
|
| 5 |
+
from langchain_text_splitters import RecursiveCharacterTextSplitter
|
| 6 |
+
from app.config import settings
|
| 7 |
+
|
| 8 |
+
logger = logging.getLogger(__name__)
|
| 9 |
+
|
| 10 |
+
# Initialize Text Splitter for chunking article content
|
| 11 |
+
text_splitter = RecursiveCharacterTextSplitter(
|
| 12 |
+
chunk_size=1000,
|
| 13 |
+
chunk_overlap=100,
|
| 14 |
+
length_function=len
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
def mock_summarize_item(raw_item: Dict[str, Any]) -> Dict[str, Any]:
|
| 18 |
+
"""
|
| 19 |
+
Fallback deterministic summarizer when LLM API keys are not provided.
|
| 20 |
+
"""
|
| 21 |
+
title = raw_item.get("title", "")
|
| 22 |
+
source = raw_item.get("source", "Tech")
|
| 23 |
+
content = raw_item.get("raw_content", "")
|
| 24 |
+
|
| 25 |
+
# Split text into chunks using Text Splitter
|
| 26 |
+
chunks = text_splitter.split_text(content)
|
| 27 |
+
processed_content = " ".join(chunks[:2]) if chunks else content
|
| 28 |
+
|
| 29 |
+
category = "AI & ML" if any(k in title.lower() or k in processed_content.lower() for k in ["ai", "llm", "gpt", "model", "neural", "agent"]) else \
|
| 30 |
+
"Open Source" if "github" in source.lower() or "repo" in title.lower() else \
|
| 31 |
+
"Developer Tools" if any(k in title.lower() for k in ["tool", "framework", "library", "code", "dev"]) else \
|
| 32 |
+
"Tech News"
|
| 33 |
+
|
| 34 |
+
summary_en = f"{title}. Key updates reported via {source} highlighting developments in modern software and technology infrastructure."
|
| 35 |
+
summary_bn = f"{title}। {source}-এর মাধ্যমে প্রকাশিত সফটওয়্যার ও প্রযুক্তি খাতের গুরুত্বপূর্ণ আপডেট।"
|
| 36 |
+
key_takeaway_en = f"Stay informed on {category} shifts and implications for software engineering workflows."
|
| 37 |
+
key_takeaway_bn = f"{category} সেক্টরের এই পরিবর্তন সফটওয়্যার ডেভলপমেন্টে গুরুত্বপূর্ণ প্রভাব ফেলবে।"
|
| 38 |
+
|
| 39 |
+
return {
|
| 40 |
+
"title": title,
|
| 41 |
+
"url": raw_item.get("url", ""),
|
| 42 |
+
"source": source,
|
| 43 |
+
"category": category,
|
| 44 |
+
"summary": summary_en,
|
| 45 |
+
"summary_en": summary_en,
|
| 46 |
+
"summary_bn": summary_bn,
|
| 47 |
+
"key_takeaway": key_takeaway_en,
|
| 48 |
+
"key_takeaway_en": key_takeaway_en,
|
| 49 |
+
"key_takeaway_bn": key_takeaway_bn,
|
| 50 |
+
"importance_score": 8.0 if category == "AI & ML" else 7.0,
|
| 51 |
+
"text_chunks": chunks
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
async def process_and_summarize_updates(raw_items: List[Dict[str, Any]]) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
|
| 55 |
+
"""
|
| 56 |
+
Processes a list of raw tech news items using LangChain Text Splitter & Gemini LLM into Dual-Language (EN & BN).
|
| 57 |
+
Returns: (overall_digest_metadata, processed_items_list)
|
| 58 |
+
"""
|
| 59 |
+
processed_items = []
|
| 60 |
+
|
| 61 |
+
llm = None
|
| 62 |
+
if settings.PREFERRED_LLM_PROVIDER == "gemini" and settings.GOOGLE_API_KEY:
|
| 63 |
+
try:
|
| 64 |
+
from langchain_google_genai import ChatGoogleGenerativeAI
|
| 65 |
+
llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash", google_api_key=settings.GOOGLE_API_KEY, temperature=0.3)
|
| 66 |
+
except Exception as e:
|
| 67 |
+
logger.warning(f"Could not initialize Gemini LLM: {e}")
|
| 68 |
+
elif settings.PREFERRED_LLM_PROVIDER == "openai" and settings.OPENAI_API_KEY:
|
| 69 |
+
try:
|
| 70 |
+
from langchain_openai import ChatOpenAI
|
| 71 |
+
llm = ChatOpenAI(model="gpt-3.5-turbo", openai_api_key=settings.OPENAI_API_KEY, temperature=0.3)
|
| 72 |
+
except Exception as e:
|
| 73 |
+
logger.warning(f"Could not initialize OpenAI LLM: {e}")
|
| 74 |
+
|
| 75 |
+
for item in raw_items:
|
| 76 |
+
raw_text = item.get("raw_content", "")
|
| 77 |
+
chunks = text_splitter.split_text(raw_text) if raw_text else [item.get("title", "")]
|
| 78 |
+
context_for_llm = "\n---\n".join(chunks[:3])
|
| 79 |
+
|
| 80 |
+
if llm:
|
| 81 |
+
try:
|
| 82 |
+
from langchain_core.prompts import PromptTemplate
|
| 83 |
+
prompt = PromptTemplate.from_template(
|
| 84 |
+
"""You are an expert Bilingual AI Tech Analyst fluent in English and Bengali.
|
| 85 |
+
Given the following tech news update, provide summaries and key takeaways in BOTH English and natural Bengali (বাংলা).
|
| 86 |
+
|
| 87 |
+
Item Title: {title}
|
| 88 |
+
Source: {source}
|
| 89 |
+
Article Chunks:
|
| 90 |
+
{chunks}
|
| 91 |
+
|
| 92 |
+
Return ONLY a valid JSON object with keys:
|
| 93 |
+
"category": "category string (e.g. AI & ML, Open Source, Developer Tools, CyberSecurity, Cloud, Mobile, Web)",
|
| 94 |
+
"summary_en": "concise 2-sentence summary in English",
|
| 95 |
+
"summary_bn": "concise 2-sentence summary in natural Bengali (বাংলা)",
|
| 96 |
+
"key_takeaway_en": "key takeaway for software engineers in English",
|
| 97 |
+
"key_takeaway_bn": "key takeaway for software engineers in natural Bengali (বাংলা)",
|
| 98 |
+
"importance_score": float number from 1.0 to 10.0
|
| 99 |
+
"""
|
| 100 |
+
)
|
| 101 |
+
chain = prompt | llm
|
| 102 |
+
res = await chain.ainvoke({
|
| 103 |
+
"title": item.get("title"),
|
| 104 |
+
"source": item.get("source"),
|
| 105 |
+
"chunks": context_for_llm
|
| 106 |
+
})
|
| 107 |
+
|
| 108 |
+
content_text = res.content if hasattr(res, "content") else str(res)
|
| 109 |
+
clean_json = re.sub(r'```json\s*|\s*```', '', content_text).strip()
|
| 110 |
+
parsed = json.loads(clean_json)
|
| 111 |
+
|
| 112 |
+
processed_items.append({
|
| 113 |
+
"title": item.get("title"),
|
| 114 |
+
"url": item.get("url"),
|
| 115 |
+
"source": item.get("source"),
|
| 116 |
+
"category": parsed.get("category", "Tech"),
|
| 117 |
+
"summary": parsed.get("summary_en", item.get("title")),
|
| 118 |
+
"summary_en": parsed.get("summary_en", item.get("title")),
|
| 119 |
+
"summary_bn": parsed.get("summary_bn", item.get("title")),
|
| 120 |
+
"key_takeaway": parsed.get("key_takeaway_en", ""),
|
| 121 |
+
"key_takeaway_en": parsed.get("key_takeaway_en", ""),
|
| 122 |
+
"key_takeaway_bn": parsed.get("key_takeaway_bn", ""),
|
| 123 |
+
"importance_score": float(parsed.get("importance_score", 7.0)),
|
| 124 |
+
"text_chunks": chunks
|
| 125 |
+
})
|
| 126 |
+
except Exception as parse_err:
|
| 127 |
+
logger.warning(f"LLM processing failed for item '{item.get('title')}': {parse_err}, using fallback")
|
| 128 |
+
processed_items.append(mock_summarize_item(item))
|
| 129 |
+
else:
|
| 130 |
+
processed_items.append(mock_summarize_item(item))
|
| 131 |
+
|
| 132 |
+
# Overall Daily Digest Summary
|
| 133 |
+
top_items_titles = [it["title"] for it in sorted(processed_items, key=lambda x: x["importance_score"], reverse=True)[:3]]
|
| 134 |
+
overall_digest = {
|
| 135 |
+
"title": f"BytePulse AI Digest - Top {len(processed_items)} Updates",
|
| 136 |
+
"overall_summary": f"Today's tech roundup highlights top developments across AI & ML, Open Source, and Developer Ecosystems. Today's top AI insights ready in English & Bengali (বাংলা).",
|
| 137 |
+
"top_highlights": json.dumps(top_items_titles)
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
return overall_digest, processed_items
|
| 141 |
+
|
| 142 |
+
async def generate_deep_ai_analysis(title: str, source: str, summary: str, text_chunks: List[str]) -> Dict[str, Any]:
|
| 143 |
+
"""
|
| 144 |
+
Generates bilingual in-depth AI breakdown for the 'Read More' full article view using Gemini LLM.
|
| 145 |
+
"""
|
| 146 |
+
llm = None
|
| 147 |
+
if settings.PREFERRED_LLM_PROVIDER == "gemini" and settings.GOOGLE_API_KEY:
|
| 148 |
+
try:
|
| 149 |
+
from langchain_google_genai import ChatGoogleGenerativeAI
|
| 150 |
+
llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash", google_api_key=settings.GOOGLE_API_KEY, temperature=0.3)
|
| 151 |
+
except Exception as e:
|
| 152 |
+
logger.warning(f"Could not initialize Gemini for Deep Analysis: {e}")
|
| 153 |
+
|
| 154 |
+
context_str = "\n---\n".join(text_chunks) if text_chunks else summary
|
| 155 |
+
|
| 156 |
+
if llm:
|
| 157 |
+
try:
|
| 158 |
+
from langchain_core.prompts import PromptTemplate
|
| 159 |
+
prompt = PromptTemplate.from_template(
|
| 160 |
+
"""You are an elite Senior Principal Software Architect and AI Engineer fluent in English and Bengali.
|
| 161 |
+
Provide a comprehensive technical deep-dive analysis in BOTH English and Bengali (বাংলা) for this tech article.
|
| 162 |
+
|
| 163 |
+
Title: {title}
|
| 164 |
+
Source: {source}
|
| 165 |
+
Summary: {summary}
|
| 166 |
+
Content Chunks:
|
| 167 |
+
{context}
|
| 168 |
+
|
| 169 |
+
Return ONLY a valid JSON object with keys:
|
| 170 |
+
"architectural_impact_en": "engineering implications in English",
|
| 171 |
+
"architectural_impact_bn": "engineering implications in natural Bengali (বাংলা)",
|
| 172 |
+
"key_takeaways_en": ["point 1 in EN", "point 2 in EN"],
|
| 173 |
+
"key_takeaways_bn": ["point 1 in Bengali", "point 2 in Bengali"],
|
| 174 |
+
"pros_en": ["pro 1 in EN", "pro 2 in EN"],
|
| 175 |
+
"pros_bn": ["pro 1 in Bengali", "pro 2 in Bengali"],
|
| 176 |
+
"cons_en": ["con 1 in EN", "con 2 in EN"],
|
| 177 |
+
"cons_bn": ["con 1 in Bengali", "con 2 in Bengali"],
|
| 178 |
+
"actionable_advice_en": "developer action item in English",
|
| 179 |
+
"actionable_advice_bn": "developer action item in Bengali (বাংলা)"
|
| 180 |
+
"""
|
| 181 |
+
)
|
| 182 |
+
chain = prompt | llm
|
| 183 |
+
res = await chain.ainvoke({"title": title, "source": source, "summary": summary, "context": context_str})
|
| 184 |
+
content_text = res.content if hasattr(res, "content") else str(res)
|
| 185 |
+
clean_json = re.sub(r'```json\s*|\s*```', '', content_text).strip()
|
| 186 |
+
return json.loads(clean_json)
|
| 187 |
+
except Exception as err:
|
| 188 |
+
logger.warning(f"Deep AI analysis failed: {err}, falling back to default structured breakdown")
|
| 189 |
+
|
| 190 |
+
return {
|
| 191 |
+
"architectural_impact_en": f"The developments in '{title}' present key considerations for system scalability, modern software design, and API integration workflows.",
|
| 192 |
+
"architectural_impact_bn": f"'{title}'-এর এই প্রযুক্তিগত অগ্রগতি সিস্টেমের স্কেলেবিলিটি এবং ডেভলপমেন্ট ওয়ার্কফ্লোতে গুরুত্বপূর্ণ ভূমিকা রাখবে।",
|
| 193 |
+
"key_takeaways_en": [
|
| 194 |
+
f"Review current codebase compatibility with {source} updates.",
|
| 195 |
+
"Assess team adoption timeline for new tooling and frameworks."
|
| 196 |
+
],
|
| 197 |
+
"key_takeaways_bn": [
|
| 198 |
+
f"কোডবেসের সাথে {source}-এর নতুন আপডেটের সামঞ্জস্যতা পরীক্ষা করুন।",
|
| 199 |
+
"টিমের নতুন টুলস গ্রহণের সময়সীমা নির্ধারণ করুন।"
|
| 200 |
+
],
|
| 201 |
+
"pros_en": ["Enhanced performance & efficiency", "Streamlined developer experience"],
|
| 202 |
+
"pros_bn": ["উন্নত পারফরম্যান্স ও দক্ষতা", "সহজ ডেভলপার এক্সপেরিয়েন্স"],
|
| 203 |
+
"cons_en": ["Potential migration effort required"],
|
| 204 |
+
"cons_bn": ["মাইগ্রেশনের জন্য অতিরিক্ত সময়ের প্রয়োজন হতে পারে"],
|
| 205 |
+
"actionable_advice_en": "Evaluate this update in a staging environment before rolling out to production systems.",
|
| 206 |
+
"actionable_advice_bn": "প্রোডাকশন সিস্টেমে ব্যবহারের আগে স্টেজিং এনভাইরনমেন্টে পরীক্ষা করে দেখুন।"
|
| 207 |
+
}
|
| 208 |
+
|
| 209 |
+
async def generate_social_media_posts(title: str, source: str, summary: str, key_takeaway: str = "") -> Dict[str, Any]:
|
| 210 |
+
"""
|
| 211 |
+
Generates ready-to-share social media post ideas in both English and Bengali using Gemini LLM.
|
| 212 |
+
"""
|
| 213 |
+
llm = None
|
| 214 |
+
if settings.PREFERRED_LLM_PROVIDER == "gemini" and settings.GOOGLE_API_KEY:
|
| 215 |
+
try:
|
| 216 |
+
from langchain_google_genai import ChatGoogleGenerativeAI
|
| 217 |
+
llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash", google_api_key=settings.GOOGLE_API_KEY, temperature=0.5)
|
| 218 |
+
except Exception as e:
|
| 219 |
+
logger.warning(f"Could not initialize Gemini for Social Post generation: {e}")
|
| 220 |
+
|
| 221 |
+
if llm:
|
| 222 |
+
try:
|
| 223 |
+
from langchain_core.prompts import PromptTemplate
|
| 224 |
+
prompt = PromptTemplate.from_template(
|
| 225 |
+
"""You are a top 1% Tech Influencer, Developer Advocate, and Technical Content Strategist fluent in English and Bengali.
|
| 226 |
+
Create high-engagement social media posts in BOTH English and Bengali based on this tech update:
|
| 227 |
+
|
| 228 |
+
Title: {title}
|
| 229 |
+
Source: {source}
|
| 230 |
+
Summary: {summary}
|
| 231 |
+
Takeaway: {takeaway}
|
| 232 |
+
|
| 233 |
+
Return ONLY a valid JSON object with keys:
|
| 234 |
+
"linkedin_en": "LinkedIn post in English with hook, bullet points, CTA and hashtags",
|
| 235 |
+
"linkedin_bn": "LinkedIn post in natural Bengali (বাংলা) for developers with hook, bullets, CTA and hashtags",
|
| 236 |
+
"twitter_en": "3-tweet Twitter/X thread in English",
|
| 237 |
+
"twitter_bn": "3-tweet Twitter/X thread in Bengali (বাংলা)",
|
| 238 |
+
"newsletter_en": "1-paragraph tech newsletter summary in English",
|
| 239 |
+
"newsletter_bn": "1-paragraph tech newsletter summary in Bengali (বাংলা)",
|
| 240 |
+
"reddit_en": "Reddit discussion post in English",
|
| 241 |
+
"reddit_bn": "Reddit discussion post in Bengali (বাংলা)"
|
| 242 |
+
"""
|
| 243 |
+
)
|
| 244 |
+
chain = prompt | llm
|
| 245 |
+
res = await chain.ainvoke({"title": title, "source": source, "summary": summary, "takeaway": key_takeaway or summary})
|
| 246 |
+
content_text = res.content if hasattr(res, "content") else str(res)
|
| 247 |
+
clean_json = re.sub(r'```json\s*|\s*```', '', content_text).strip()
|
| 248 |
+
parsed = json.loads(clean_json)
|
| 249 |
+
|
| 250 |
+
# Backwards compatibility keys
|
| 251 |
+
parsed["linkedin"] = parsed.get("linkedin_en")
|
| 252 |
+
parsed["twitter"] = parsed.get("twitter_en")
|
| 253 |
+
parsed["newsletter"] = parsed.get("newsletter_en")
|
| 254 |
+
parsed["reddit"] = parsed.get("reddit_en")
|
| 255 |
+
return parsed
|
| 256 |
+
except Exception as err:
|
| 257 |
+
logger.warning(f"Social post generation failed: {err}, using fallback post templates")
|
| 258 |
+
|
| 259 |
+
return {
|
| 260 |
+
"linkedin_en": f"🚀 Huge update in Tech! {title}\n\nKey Highlights:\n• {summary}\n\n#SoftwareEngineering #TechNews #DevCommunity",
|
| 261 |
+
"linkedin_bn": f"🚀 টেক ওয়ার্ল্ডে বিশাল আপডেট! {title}\n\nমূল পয়েন্ট:\n• {summary}\n\nআপনার মতামত কি? কমেন্টে জানান! 👇\n\n#TechNewsBangla #SoftwareEngineering #DevCommunity",
|
| 262 |
+
"twitter_en": f"1/3 🚨 Tech Update: {title}\n\nSummary: {summary[:120]}...\n\n2/3 💡 Takeaway: {key_takeaway[:120]}\n\n3/3 What do you think?",
|
| 263 |
+
"twitter_bn": f"1/3 🚨 টেক আপডেট: {title}\n\nসামারি: {summary[:120]}...\n\n2/3 💡 টেকঅ্যাওয়ে: {key_takeaway[:120]}\n\n3/3 আপনার কি মতামত?",
|
| 264 |
+
"newsletter_en": f"📌 **{title}**: {summary} {key_takeaway}",
|
| 265 |
+
"newsletter_bn": f"📌 **{title}**: {summary} {key_takeaway}",
|
| 266 |
+
"reddit_en": f"**[Discussion] {title}**\n\n{summary}",
|
| 267 |
+
"reddit_bn": f"**[আলোচনা] {title}**\n\n{summary}",
|
| 268 |
+
"linkedin": f"🚀 Huge update in Tech! {title}\n\nKey Highlights:\n• {summary}",
|
| 269 |
+
"twitter": f"1/3 🚨 Tech Update: {title}",
|
| 270 |
+
"newsletter": f"📌 **{title}**: {summary}",
|
| 271 |
+
"reddit": f"**[Discussion] {title}**\n\n{summary}"
|
| 272 |
+
}
|
app/vectorstore/chroma_store.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
from typing import List, Dict, Any
|
| 3 |
+
|
| 4 |
+
class ChunkTextStore:
|
| 5 |
+
"""
|
| 6 |
+
Stores text chunks generated by LangChain RecursiveCharacterTextSplitter
|
| 7 |
+
for fast text keyword matching without requiring external embedding models.
|
| 8 |
+
"""
|
| 9 |
+
def __init__(self):
|
| 10 |
+
self.chunk_records: List[Dict[str, Any]] = []
|
| 11 |
+
|
| 12 |
+
def add_digest_items(self, items: List[Dict[str, Any]]) -> int:
|
| 13 |
+
"""
|
| 14 |
+
Stores text chunks and metadata for each digest item.
|
| 15 |
+
"""
|
| 16 |
+
if not items:
|
| 17 |
+
return 0
|
| 18 |
+
|
| 19 |
+
added_count = 0
|
| 20 |
+
for item in items:
|
| 21 |
+
item_id = item.get("id")
|
| 22 |
+
title = item.get("title", "")
|
| 23 |
+
summary = item.get("summary", "")
|
| 24 |
+
chunks = item.get("text_chunks", [summary])
|
| 25 |
+
|
| 26 |
+
for chunk_idx, chunk_text in enumerate(chunks):
|
| 27 |
+
self.chunk_records.append({
|
| 28 |
+
"id": f"item_{item_id}_chunk_{chunk_idx}",
|
| 29 |
+
"item_id": item_id,
|
| 30 |
+
"title": title,
|
| 31 |
+
"url": item.get("url", ""),
|
| 32 |
+
"source": item.get("source", ""),
|
| 33 |
+
"category": item.get("category", "Tech"),
|
| 34 |
+
"chunk_text": chunk_text,
|
| 35 |
+
"summary": summary
|
| 36 |
+
})
|
| 37 |
+
added_count += 1
|
| 38 |
+
|
| 39 |
+
return added_count
|
| 40 |
+
|
| 41 |
+
def search_similar(self, query: str, top_k: int = 5) -> List[Dict[str, Any]]:
|
| 42 |
+
"""
|
| 43 |
+
Searches stored text chunks using case-insensitive keyword relevance matching.
|
| 44 |
+
"""
|
| 45 |
+
query_words = set(query.lower().split())
|
| 46 |
+
matched_results = []
|
| 47 |
+
|
| 48 |
+
for rec in self.chunk_records:
|
| 49 |
+
content_lower = f"{rec['title']} {rec['chunk_text']} {rec['category']}".lower()
|
| 50 |
+
match_score = sum(1 for w in query_words if w in content_lower)
|
| 51 |
+
if match_score > 0:
|
| 52 |
+
matched_results.append({
|
| 53 |
+
"content": rec["chunk_text"],
|
| 54 |
+
"metadata": {
|
| 55 |
+
"item_id": rec["item_id"],
|
| 56 |
+
"title": rec["title"],
|
| 57 |
+
"url": rec["url"],
|
| 58 |
+
"source": rec["source"],
|
| 59 |
+
"category": rec["category"],
|
| 60 |
+
"summary": rec["summary"]
|
| 61 |
+
},
|
| 62 |
+
"relevance_score": match_score
|
| 63 |
+
})
|
| 64 |
+
|
| 65 |
+
# Sort by match score descending
|
| 66 |
+
matched_results.sort(key=lambda x: x["relevance_score"], reverse=True)
|
| 67 |
+
return matched_results[:top_k]
|
| 68 |
+
|
| 69 |
+
# Global chunk store instance
|
| 70 |
+
vector_store = ChunkTextStore()
|
| 71 |
+
|
main.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
from contextlib import asynccontextmanager
|
| 3 |
+
from fastapi import FastAPI, Request
|
| 4 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 5 |
+
from fastapi.responses import JSONResponse
|
| 6 |
+
from slowapi import Limiter, _rate_limit_exceeded_handler
|
| 7 |
+
from slowapi.util import get_remote_address
|
| 8 |
+
from slowapi.errors import RateLimitExceeded
|
| 9 |
+
|
| 10 |
+
from app.config import settings
|
| 11 |
+
from app.db.database import engine, Base
|
| 12 |
+
from app.api.v1.router import api_router
|
| 13 |
+
from app.scheduler.cron import start_scheduler, stop_scheduler
|
| 14 |
+
|
| 15 |
+
# Configure Logging
|
| 16 |
+
logging.basicConfig(
|
| 17 |
+
level=logging.INFO,
|
| 18 |
+
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
| 19 |
+
)
|
| 20 |
+
logger = logging.getLogger(__name__)
|
| 21 |
+
|
| 22 |
+
# Initialize Rate Limiter (Protection against API token depletion and DDoS)
|
| 23 |
+
limiter = Limiter(key_func=get_remote_address, default_limits=["60 per minute"])
|
| 24 |
+
|
| 25 |
+
@asynccontextmanager
|
| 26 |
+
async def lifespan(app: FastAPI):
|
| 27 |
+
# Startup: Initialize Database Tables
|
| 28 |
+
logger.info("Initializing database schema...")
|
| 29 |
+
Base.metadata.create_all(bind=engine)
|
| 30 |
+
|
| 31 |
+
# Startup: Start APScheduler Cron Job (Twice Daily)
|
| 32 |
+
start_scheduler()
|
| 33 |
+
|
| 34 |
+
yield
|
| 35 |
+
|
| 36 |
+
# Shutdown: Stop Scheduler
|
| 37 |
+
stop_scheduler()
|
| 38 |
+
|
| 39 |
+
app = FastAPI(
|
| 40 |
+
title=settings.APP_NAME,
|
| 41 |
+
version=settings.VERSION,
|
| 42 |
+
description="Enterprise API for BytePulse AI Daily Tech Digest powered by 8 Scrapers, Gemini 2.5 Flash, ChromaDB, and FCM.",
|
| 43 |
+
lifespan=lifespan
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
# Attach Rate Limiter to app state
|
| 47 |
+
app.state.limiter = limiter
|
| 48 |
+
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
|
| 49 |
+
|
| 50 |
+
# App Secret Authentication Middleware (Only accept requests from Mobile App, block all external bots)
|
| 51 |
+
@app.middleware("http")
|
| 52 |
+
async def app_secret_security_middleware(request: Request, call_next):
|
| 53 |
+
path = request.url.path
|
| 54 |
+
# Allow docs, open API spec, root and health checks without secret header
|
| 55 |
+
if path in ["/", "/health", "/docs", "/openapi.json", "/redoc"]:
|
| 56 |
+
return await call_next(request)
|
| 57 |
+
|
| 58 |
+
# Inspect X-App-Secret header for all /api/v1 endpoints
|
| 59 |
+
app_secret = request.headers.get("X-App-Secret")
|
| 60 |
+
if app_secret != settings.API_SECRET_KEY:
|
| 61 |
+
logger.warning(f"Blocked unauthorized API request to {path} from IP {request.client.host}")
|
| 62 |
+
return JSONResponse(
|
| 63 |
+
status_code=403,
|
| 64 |
+
content={
|
| 65 |
+
"status": "forbidden",
|
| 66 |
+
"message": "Access Denied: Invalid or missing X-App-Secret key. Only authorized BytePulse AI clients are allowed."
|
| 67 |
+
}
|
| 68 |
+
)
|
| 69 |
+
return await call_next(request)
|
| 70 |
+
|
| 71 |
+
# Global Error Handler for unhandled server exceptions
|
| 72 |
+
@app.exception_handler(Exception)
|
| 73 |
+
async def global_exception_handler(request: Request, exc: Exception):
|
| 74 |
+
logger.error(f"Global Error caught on {request.url.path}: {exc}", exc_info=True)
|
| 75 |
+
return JSONResponse(
|
| 76 |
+
status_code=500,
|
| 77 |
+
content={
|
| 78 |
+
"status": "error",
|
| 79 |
+
"message": "An internal server error occurred. Our engineers have been notified.",
|
| 80 |
+
"path": request.url.path
|
| 81 |
+
}
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
# Enable CORS Security Middleware
|
| 85 |
+
app.add_middleware(
|
| 86 |
+
CORSMiddleware,
|
| 87 |
+
allow_origins=["*"],
|
| 88 |
+
allow_credentials=True,
|
| 89 |
+
allow_methods=["GET", "POST", "OPTIONS"],
|
| 90 |
+
allow_headers=["*"],
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
# Root & Health Endpoints
|
| 94 |
+
@app.get("/")
|
| 95 |
+
@limiter.limit("30/minute")
|
| 96 |
+
def root(request: Request):
|
| 97 |
+
return {
|
| 98 |
+
"message": f"Welcome to {settings.APP_NAME}",
|
| 99 |
+
"version": settings.VERSION,
|
| 100 |
+
"docs": "/docs",
|
| 101 |
+
"api_v1": "/api/v1"
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
@app.get("/health")
|
| 105 |
+
@limiter.limit("60/minute")
|
| 106 |
+
def health_check(request: Request):
|
| 107 |
+
return {"status": "healthy", "service": settings.APP_NAME, "security": "rate_limited_enabled"}
|
| 108 |
+
|
| 109 |
+
# Include API v1 Router
|
| 110 |
+
app.include_router(api_router, prefix="/api/v1")
|
| 111 |
+
|
| 112 |
+
if __name__ == "__main__":
|
| 113 |
+
import uvicorn
|
| 114 |
+
uvicorn.run("main:app", host=settings.HOST, port=settings.PORT, reload=settings.DEBUG)
|
requirements.txt
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fastapi>=0.109.0
|
| 2 |
+
uvicorn[standard]>=0.27.0
|
| 3 |
+
pydantic>=2.6.0
|
| 4 |
+
pydantic-settings>=2.1.0
|
| 5 |
+
sqlalchemy>=2.0.25
|
| 6 |
+
httpx>=0.26.0
|
| 7 |
+
beautifulsoup4>=4.12.3
|
| 8 |
+
feedparser>=6.0.10
|
| 9 |
+
apscheduler>=3.10.4
|
| 10 |
+
langchain>=0.1.0
|
| 11 |
+
langchain-community>=0.0.10
|
| 12 |
+
langchain-text-splitters>=0.0.1
|
| 13 |
+
langchain-google-genai>=0.0.6
|
| 14 |
+
langchain-openai>=0.0.5
|
| 15 |
+
firebase-admin>=6.4.0
|
| 16 |
+
python-dotenv>=1.0.1
|
| 17 |
+
gradio==6.20.0
|
| 18 |
+
slowapi>=0.1.10
|
| 19 |
+
cryptography>=42.0.0
|
| 20 |
+
ddgs>=8.0.0
|
| 21 |
+
chromadb>=0.4.22
|
| 22 |
+
duckduckgo-search>=6.0.0
|