Spaces:
Runtime error
Runtime error
File size: 2,941 Bytes
95e759d 69d5551 95e759d 69d5551 6dce24d 95e759d 69d5551 508d9ec 6dce24d 95e759d 21fff8b 6dce24d 95e759d a4601b6 503d4d2 69d5551 503d4d2 95e759d 6dce24d 95e759d 6dce24d 95e759d 6dce24d 95e759d 6dce24d 95e759d ea53b0e 95e759d 6dce24d 21fff8b 6dce24d 21fff8b 6dce24d 95e759d 8cee853 95e759d 6dce24d 95e759d 6dce24d 358550f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 | import bs4
import uuid
import requests
from typing import Dict, List, Any
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
app = FastAPI(docs_url=None, openapi_url=None, redoc_url=None)
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
LANGUAGES = ["python", "javascript", "html", "typescript", "java", "cpp", "php", "go", "css", "c"]
def fetch_github_page(**kwargs) -> str:
try:
params = '&'.join([f'{k}={v}' for k, v in kwargs.items()])
url = 'https://github.com/topics/hacktoberfest?{}'.format(params)
headers = {
# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'User-Agent': str(uuid.uuid4()),
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Cache-Control': 'max-age=0',
}
response = requests.get(url, headers)
response.raise_for_status()
return response.text
except requests.RequestException as e:
raise HTTPException(status_code=502, detail=f"Error fetching data: {e}")
def parse_repositories(page_content: str) -> List[Dict[str, Any]]:
soup = bs4.BeautifulSoup(page_content, 'html.parser')
articles = soup.find_all("article")
repositories = []
for article in articles:
repo_info = [a_tag.text.strip() for a_tag in article.find_all("a") if a_tag.text.strip()]
if len(repo_info) >= 3 and "Star" in repo_info[2]:
owner = repo_info[0]
repo_name = repo_info[1]
stars = repo_info[2][6:-1] if repo_info[2].startswith("Star\n") else repo_info[2]
repositories.append({
"user": owner,
"repo": repo_name,
"stars": stars,
})
return repositories
@app.get("/")
def index():
return {"name": "github repo scraper", "author": "arpy8"}
@app.get("/all")
def get_all_repos():
payload: Dict[str, Any] = {}
for lang in LANGUAGES:
page_content = fetch_github_page(l=lang)
repositories = parse_repositories(page_content)
payload[lang] = repositories
return payload
@app.get("/data")
def get_repos(language: str = "python"):
page_content = fetch_github_page(l=language)
repositories = parse_repositories(page_content)
return repositories
if __name__=="__main__":
import uvicorn
uvicorn.run(app)
|