WhoIsLuwah commited on
Commit
82b2e56
·
1 Parent(s): 9209fc6

fix: load full reel library, unblock server during extraction, bulk drop

Browse files

- Dashboard fetched only the newest 500 reels; page through Supabase so
the whole library loads (754 rows today). db.list_reels pages too.
- /extract/url, /extract/file and /extract/text were async but ran the
blocking pipeline, freezing every other request. Make them sync so
FastAPI runs them in its threadpool.
- Bulk import drop zone called the audio drop handler and rejected
saved_posts.json. Give it its own handler.
- Drop the unused /reels search param, which interpolated raw text into
a PostgREST filter.

Claude-Session: https://claude.ai/code/session_0166W7S7XvHg5P2uyJtP4GiC

Files changed (3) hide show
  1. db.py +12 -8
  2. main.py +6 -10
  3. verify_pipeline.py +1 -1
db.py CHANGED
@@ -74,14 +74,18 @@ def insert_reel(url, title, raw_transcript, post_caption, extracted_json,
74
  return row_to_record(row)
75
 
76
 
77
- def list_reels(limit: int = 20, search: Optional[str] = None) -> list:
78
- q = get_client().table(TABLE).select("id, url, title, extracted_json, created_at, cluster, status, error").order("created_at", desc=True).limit(limit)
79
- if search:
80
- # UI filters in-memory; this server search is a coarse fallback over text cols.
81
- like = f"%{search}%"
82
- q = q.or_(f"title.ilike.{like},raw_transcript.ilike.{like},post_caption.ilike.{like}")
83
- res = q.execute()
84
- return [row_to_record(r) for r in (res.data or [])]
 
 
 
 
85
 
86
 
87
  def get_reel_details(reel_id: str) -> Optional[dict]:
 
74
  return row_to_record(row)
75
 
76
 
77
+ def list_reels(limit: int = 20) -> list:
78
+ rows = []
79
+ while len(rows) < limit:
80
+ # PostgREST caps rows per request, so page until the limit is reached.
81
+ page = (get_client().table(TABLE)
82
+ .select("id, url, title, extracted_json, created_at, cluster, status, error")
83
+ .order("created_at", desc=True).order("id")
84
+ .range(len(rows), limit - 1).execute().data or [])
85
+ if not page:
86
+ break
87
+ rows.extend(page)
88
+ return [row_to_record(r) for r in rows]
89
 
90
 
91
  def get_reel_details(reel_id: str) -> Optional[dict]:
main.py CHANGED
@@ -673,13 +673,10 @@ def health_check():
673
  return {"status": "ok", "message": "Extractor service is awake and running."}
674
 
675
  @app.get("/reels")
676
- def list_reels(
677
- limit: int = Query(20, description="Max number of items to return"),
678
- search: Optional[str] = Query(None, description="Search across title/transcript/caption"),
679
- ):
680
  """Retrieve saved reels from Supabase."""
681
  try:
682
- return db.list_reels(limit=limit, search=search)
683
  except Exception as e:
684
  logger.error(f"Error fetching reels: {str(e)}")
685
  raise HTTPException(status_code=500, detail=f"Failed to fetch reels: {str(e)}")
@@ -859,7 +856,7 @@ def process_reel_url(url: str) -> dict:
859
 
860
 
861
  @app.post("/extract/url", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
862
- async def extract_url(payload: dict):
863
  """Accepts an Instagram Reel URL and runs the full extraction pipeline."""
864
  url = payload.get("url")
865
  if not url:
@@ -961,7 +958,7 @@ def _worker_loop(idle_interval: float = 20.0) -> None:
961
 
962
 
963
  @app.post("/extract/file", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
964
- async def extract_file(
965
  file: UploadFile = File(...),
966
  title: str = Form("Uploaded Audio File"),
967
  caption: Optional[str] = Form(None)
@@ -980,8 +977,7 @@ async def extract_file(
980
  temp_file_path = temp_file.name
981
 
982
  try:
983
- content = await file.read()
984
- temp_file.write(content)
985
  temp_file.close()
986
 
987
  # 1. Transcribe
@@ -1008,7 +1004,7 @@ async def extract_file(
1008
  os.remove(temp_file_path)
1009
 
1010
  @app.post("/extract/text", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
1011
- async def extract_text(payload: dict):
1012
  """
1013
  Accepts raw transcript text and/or captions directly to skip scraping/transcribing
1014
  and extract JSON highlights. Saves to database.
 
673
  return {"status": "ok", "message": "Extractor service is awake and running."}
674
 
675
  @app.get("/reels")
676
+ def list_reels(limit: int = Query(20, description="Max number of items to return")):
 
 
 
677
  """Retrieve saved reels from Supabase."""
678
  try:
679
+ return db.list_reels(limit=limit)
680
  except Exception as e:
681
  logger.error(f"Error fetching reels: {str(e)}")
682
  raise HTTPException(status_code=500, detail=f"Failed to fetch reels: {str(e)}")
 
856
 
857
 
858
  @app.post("/extract/url", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
859
+ def extract_url(payload: dict):
860
  """Accepts an Instagram Reel URL and runs the full extraction pipeline."""
861
  url = payload.get("url")
862
  if not url:
 
958
 
959
 
960
  @app.post("/extract/file", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
961
+ def extract_file(
962
  file: UploadFile = File(...),
963
  title: str = Form("Uploaded Audio File"),
964
  caption: Optional[str] = Form(None)
 
977
  temp_file_path = temp_file.name
978
 
979
  try:
980
+ temp_file.write(file.file.read())
 
981
  temp_file.close()
982
 
983
  # 1. Transcribe
 
1004
  os.remove(temp_file_path)
1005
 
1006
  @app.post("/extract/text", response_model=ExtractionResponse, dependencies=[Depends(verify_api_key)])
1007
+ def extract_text(payload: dict):
1008
  """
1009
  Accepts raw transcript text and/or captions directly to skip scraping/transcribing
1010
  and extract JSON highlights. Saves to database.
verify_pipeline.py CHANGED
@@ -377,7 +377,7 @@ def test_reels_include_cluster():
377
  print("Testing GET /reels includes cluster (db mocked)...")
378
  import main
379
  orig = main.db.list_reels
380
- main.db.list_reels = lambda limit=20, search=None: [{
381
  "id": "x", "url": None, "title": "t", "raw_transcript": None,
382
  "post_caption": None, "extracted_json": {"core_topic": "c"},
383
  "created_at": "2026-06-06T00:00:00Z", "cluster": "Unclustered"}]
 
377
  print("Testing GET /reels includes cluster (db mocked)...")
378
  import main
379
  orig = main.db.list_reels
380
+ main.db.list_reels = lambda limit=20: [{
381
  "id": "x", "url": None, "title": "t", "raw_transcript": None,
382
  "post_caption": None, "extracted_json": {"core_topic": "c"},
383
  "created_at": "2026-06-06T00:00:00Z", "cluster": "Unclustered"}]