Spaces:
Sleeping
Sleeping
Commit ·
82b2e56
1
Parent(s): 9209fc6
fix: load full reel library, unblock server during extraction, bulk drop
Browse files- Dashboard fetched only the newest 500 reels; page through Supabase so
the whole library loads (754 rows today). db.list_reels pages too.
- /extract/url, /extract/file and /extract/text were async but ran the
blocking pipeline, freezing every other request. Make them sync so
FastAPI runs them in its threadpool.
- Bulk import drop zone called the audio drop handler and rejected
saved_posts.json. Give it its own handler.
- Drop the unused /reels search param, which interpolated raw text into
a PostgREST filter.
Claude-Session: https://claude.ai/code/session_0166W7S7XvHg5P2uyJtP4GiC
- db.py +12 -8
- main.py +6 -10
- verify_pipeline.py +1 -1
db.py
CHANGED
|
@@ -74,14 +74,18 @@ def insert_reel(url, title, raw_transcript, post_caption, extracted_json,
|
|
| 74 |
return row_to_record(row)
|
| 75 |
|
| 76 |
|
| 77 |
-
def list_reels(limit: int = 20
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
#
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
|
| 86 |
|
| 87 |
def get_reel_details(reel_id: str) -> Optional[dict]:
|
|
|
|
| 74 |
return row_to_record(row)
|
| 75 |
|
| 76 |
|
| 77 |
+
def list_reels(limit: int = 20) -> list:
|
| 78 |
+
rows = []
|
| 79 |
+
while len(rows) < limit:
|
| 80 |
+
# PostgREST caps rows per request, so page until the limit is reached.
|
| 81 |
+
page = (get_client().table(TABLE)
|
| 82 |
+
.select("id, url, title, extracted_json, created_at, cluster, status, error")
|
| 83 |
+
.order("created_at", desc=True).order("id")
|
| 84 |
+
.range(len(rows), limit - 1).execute().data or [])
|
| 85 |
+
if not page:
|
| 86 |
+
break
|
| 87 |
+
rows.extend(page)
|
| 88 |
+
return [row_to_record(r) for r in rows]
|
| 89 |
|
| 90 |
|
| 91 |
def get_reel_details(reel_id: str) -> Optional[dict]:
|
main.py
CHANGED
|
@@ -673,13 +673,10 @@ def health_check():
|
|
| 673 |
return {"status": "ok", "message": "Extractor service is awake and running."}
|
| 674 |
|
| 675 |
@app.get("/reels")
|
| 676 |
-
def list_reels(
|
| 677 |
-
limit: int = Query(20, description="Max number of items to return"),
|
| 678 |
-
search: Optional[str] = Query(None, description="Search across title/transcript/caption"),
|
| 679 |
-
):
|
| 680 |
"""Retrieve saved reels from Supabase."""
|
| 681 |
try:
|
| 682 |
-
return db.list_reels(limit=limit
|
| 683 |
except Exception as e:
|
| 684 |
logger.error(f"Error fetching reels: {str(e)}")
|
| 685 |
raise HTTPException(status_code=500, detail=f"Failed to fetch reels: {str(e)}")
|
|
@@ -859,7 +856,7 @@ def process_reel_url(url: str) -> dict:
|
|
| 859 |
|
| 860 |
|
| 861 |
@app.post("/extract/url", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
|
| 862 |
-
|
| 863 |
"""Accepts an Instagram Reel URL and runs the full extraction pipeline."""
|
| 864 |
url = payload.get("url")
|
| 865 |
if not url:
|
|
@@ -961,7 +958,7 @@ def _worker_loop(idle_interval: float = 20.0) -> None:
|
|
| 961 |
|
| 962 |
|
| 963 |
@app.post("/extract/file", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
|
| 964 |
-
|
| 965 |
file: UploadFile = File(...),
|
| 966 |
title: str = Form("Uploaded Audio File"),
|
| 967 |
caption: Optional[str] = Form(None)
|
|
@@ -980,8 +977,7 @@ async def extract_file(
|
|
| 980 |
temp_file_path = temp_file.name
|
| 981 |
|
| 982 |
try:
|
| 983 |
-
|
| 984 |
-
temp_file.write(content)
|
| 985 |
temp_file.close()
|
| 986 |
|
| 987 |
# 1. Transcribe
|
|
@@ -1008,7 +1004,7 @@ async def extract_file(
|
|
| 1008 |
os.remove(temp_file_path)
|
| 1009 |
|
| 1010 |
@app.post("/extract/text", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
|
| 1011 |
-
|
| 1012 |
"""
|
| 1013 |
Accepts raw transcript text and/or captions directly to skip scraping/transcribing
|
| 1014 |
and extract JSON highlights. Saves to database.
|
|
|
|
| 673 |
return {"status": "ok", "message": "Extractor service is awake and running."}
|
| 674 |
|
| 675 |
@app.get("/reels")
|
| 676 |
+
def list_reels(limit: int = Query(20, description="Max number of items to return")):
|
|
|
|
|
|
|
|
|
|
| 677 |
"""Retrieve saved reels from Supabase."""
|
| 678 |
try:
|
| 679 |
+
return db.list_reels(limit=limit)
|
| 680 |
except Exception as e:
|
| 681 |
logger.error(f"Error fetching reels: {str(e)}")
|
| 682 |
raise HTTPException(status_code=500, detail=f"Failed to fetch reels: {str(e)}")
|
|
|
|
| 856 |
|
| 857 |
|
| 858 |
@app.post("/extract/url", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
|
| 859 |
+
def extract_url(payload: dict):
|
| 860 |
"""Accepts an Instagram Reel URL and runs the full extraction pipeline."""
|
| 861 |
url = payload.get("url")
|
| 862 |
if not url:
|
|
|
|
| 958 |
|
| 959 |
|
| 960 |
@app.post("/extract/file", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
|
| 961 |
+
def extract_file(
|
| 962 |
file: UploadFile = File(...),
|
| 963 |
title: str = Form("Uploaded Audio File"),
|
| 964 |
caption: Optional[str] = Form(None)
|
|
|
|
| 977 |
temp_file_path = temp_file.name
|
| 978 |
|
| 979 |
try:
|
| 980 |
+
temp_file.write(file.file.read())
|
|
|
|
| 981 |
temp_file.close()
|
| 982 |
|
| 983 |
# 1. Transcribe
|
|
|
|
| 1004 |
os.remove(temp_file_path)
|
| 1005 |
|
| 1006 |
@app.post("/extract/text", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
|
| 1007 |
+
def extract_text(payload: dict):
|
| 1008 |
"""
|
| 1009 |
Accepts raw transcript text and/or captions directly to skip scraping/transcribing
|
| 1010 |
and extract JSON highlights. Saves to database.
|
verify_pipeline.py
CHANGED
|
@@ -377,7 +377,7 @@ def test_reels_include_cluster():
|
|
| 377 |
print("Testing GET /reels includes cluster (db mocked)...")
|
| 378 |
import main
|
| 379 |
orig = main.db.list_reels
|
| 380 |
-
main.db.list_reels = lambda limit=20
|
| 381 |
"id": "x", "url": None, "title": "t", "raw_transcript": None,
|
| 382 |
"post_caption": None, "extracted_json": {"core_topic": "c"},
|
| 383 |
"created_at": "2026-06-06T00:00:00Z", "cluster": "Unclustered"}]
|
|
|
|
| 377 |
print("Testing GET /reels includes cluster (db mocked)...")
|
| 378 |
import main
|
| 379 |
orig = main.db.list_reels
|
| 380 |
+
main.db.list_reels = lambda limit=20: [{
|
| 381 |
"id": "x", "url": None, "title": "t", "raw_transcript": None,
|
| 382 |
"post_caption": None, "extracted_json": {"core_topic": "c"},
|
| 383 |
"created_at": "2026-06-06T00:00:00Z", "cluster": "Unclustered"}]
|