File size: 2,941 Bytes
95e759d
69d5551
95e759d
69d5551
6dce24d
 
95e759d
69d5551
508d9ec
6dce24d
 
 
 
 
 
 
 
95e759d
21fff8b
 
6dce24d
95e759d
a4601b6
 
503d4d2
69d5551
 
503d4d2
 
 
 
 
 
 
 
 
 
 
 
95e759d
6dce24d
 
 
95e759d
6dce24d
95e759d
6dce24d
95e759d
 
 
 
 
 
 
 
 
 
6dce24d
95e759d
 
ea53b0e
95e759d
 
 
 
 
 
6dce24d
 
 
 
 
21fff8b
6dce24d
 
21fff8b
 
 
 
 
6dce24d
95e759d
8cee853
 
 
95e759d
6dce24d
 
95e759d
6dce24d
 
358550f
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
import bs4
import uuid
import requests
from typing import Dict, List, Any
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware


app = FastAPI(docs_url=None, openapi_url=None, redoc_url=None)

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

LANGUAGES = ["python", "javascript", "html", "typescript", "java", "cpp", "php", "go", "css", "c"]

def fetch_github_page(**kwargs) -> str:
    try:
        params = '&'.join([f'{k}={v}' for k, v in kwargs.items()])
        url = 'https://github.com/topics/hacktoberfest?{}'.format(params)
        headers = {
            # 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
            'User-Agent': str(uuid.uuid4()),
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Sec-Fetch-User': '?1',
            'Cache-Control': 'max-age=0',
        }
        response = requests.get(url, headers)
        response.raise_for_status()
        
        return response.text
    
    except requests.RequestException as e:
        raise HTTPException(status_code=502, detail=f"Error fetching data: {e}")

def parse_repositories(page_content: str) -> List[Dict[str, Any]]:
    soup = bs4.BeautifulSoup(page_content, 'html.parser')
    articles = soup.find_all("article")
    repositories = []

    for article in articles:
        repo_info = [a_tag.text.strip() for a_tag in article.find_all("a") if a_tag.text.strip()]

        if len(repo_info) >= 3 and "Star" in repo_info[2]:
            owner = repo_info[0]
            repo_name = repo_info[1]
            stars = repo_info[2][6:-1] if repo_info[2].startswith("Star\n") else repo_info[2]

            repositories.append({
                "user": owner,
                "repo": repo_name,
                "stars": stars,
            })

    return repositories

@app.get("/")
def index():
    return {"name": "github repo scraper", "author": "arpy8"}

@app.get("/all")
def get_all_repos():
    payload: Dict[str, Any] = {}

    for lang in LANGUAGES:
        page_content = fetch_github_page(l=lang)
        repositories = parse_repositories(page_content)
        payload[lang] = repositories

    return payload

@app.get("/data")
def get_repos(language: str = "python"):
    page_content = fetch_github_page(l=language)
    repositories = parse_repositories(page_content)

    return repositories

if __name__=="__main__":
    import uvicorn
    uvicorn.run(app)