dookieMonster03
Update app.py
7428bc4
Raw History Blame Contribute Delete
2.94 kB
import bs4
import uuid
import requests
from typing import Dict, List, Any
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
app = FastAPI(docs_url=None, openapi_url=None, redoc_url=None)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
LANGUAGES = ["python", "javascript", "html", "typescript", "java", "cpp", "php", "go", "css", "c"]
def fetch_github_page(**kwargs) -> str:
try:
params = '&'.join([f'{k}={v}' for k, v in kwargs.items()])
url = 'https://github.com/topics/hacktoberfest?{}'.format(params)
headers = {
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'User-Agent': str(uuid.uuid4()),
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Cache-Control': 'max-age=0',
}
response = requests.get(url, headers)
response.raise_for_status()
return response.text
except requests.RequestException as e:
raise HTTPException(status_code=502, detail=f"Error fetching data: {e}")
def parse_repositories(page_content: str) -> List[Dict[str, Any]]:
soup = bs4.BeautifulSoup(page_content, 'html.parser')
articles = soup.find_all("article")
repositories = []
for article in articles:
repo_info = [a_tag.text.strip() for a_tag in article.find_all("a") if a_tag.text.strip()]
if len(repo_info) >= 3 and "Star" in repo_info[2]:
owner = repo_info[0]
repo_name = repo_info[1]
stars = repo_info[2][6:-1] if repo_info[2].startswith("Star\n") else repo_info[2]
repositories.append({
"user": owner,
"repo": repo_name,
"stars": stars,
})
return repositories
@app.get("/")
def index():
return {"name": "github repo scraper", "author": "arpy8"}
@app.get("/all")
def get_all_repos():
payload: Dict[str, Any] = {}
for lang in LANGUAGES:
page_content = fetch_github_page(l=lang)
repositories = parse_repositories(page_content)
payload[lang] = repositories
return payload
@app.get("/data")
def get_repos(language: str = "python"):
page_content = fetch_github_page(l=language)
repositories = parse_repositories(page_content)
return repositories
if __name__=="__main__":
import uvicorn
uvicorn.run(app)