Update app.py

app.py

@@ -146,19 +146,52 @@ def run_pipeline():
                 if state.stop_event.is_set(): break
                 
                 state.add_log(f"Processing {anime} Episode {episode}...")
-                captions_data = []
                 episode_json_path = OUTPUT_DIR / f"{anime}_ep{episode}.json"
                 
+                # 1. Try to download existing JSON from TARGET_REPO if it exists
+                existing_captions_map = {}
+                highest_clip_num = 0
+                remote_json_path = f"{anime}/{anime}_ep{episode}.json"
+                
+                try:
+                    state.add_log(f"Checking existing JSON in {TARGET_REPO} at {remote_json_path}...")
+                    downloaded_file = api.hf_hub_download(
+                        repo_id=TARGET_REPO,
+                        filename=remote_json_path,
+                        repo_type="dataset"
+                    )
+                    with open(downloaded_file, "r", encoding="utf-8") as f:
+                        data = json.load(f)
+                        # Support both lists or dict structures
+                        captions_list = data.get("captions", []) if isinstance(data, dict) else data
+                        for item in captions_list:
+                            c_num = item.get("clip_number")
+                            if c_num is not None:
+                                existing_captions_map[c_num] = item
+                                if c_num > highest_clip_num:
+                                    highest_clip_num = c_num
+                    state.add_log(f"Found existing JSON! Highest processed clip number: {highest_clip_num}")
+                except Exception as e:
+                    state.add_log(f"No existing JSON found or error downloading: {e}. Starting fresh for this episode.")
+
                 def get_clip_num(f):
                     m = CLIP_RE.search(f)
                     return int(m.group(2)) if m else 0
+                
                 clips.sort(key=get_clip_num)
 
                 token_limit_hit = False
+                new_captions_processed = 0
+
                 for clip_path in clips:
                     if state.stop_event.is_set(): break
                     filename = Path(clip_path).name
                     clip_num = get_clip_num(filename)
+                    
+                    # 2. Skip clips that have already been processed (<= highest_clip_num)
+                    if clip_num <= highest_clip_num:
+                        continue
+
                     local_video = DOWNLOAD_DIR / filename
                     video_url = f"https://huggingface.co/datasets/{SOURCE_REPO}/resolve/main/{clip_path}"
                     
@@ -167,8 +200,13 @@ def run_pipeline():
                         frames = extract_frames(str(local_video))
                         if frames:
                             caption = get_caption(client, frames)
-                            captions_data.append({"clip_number": clip_num, "filename": filename, "caption": caption})
-                            state.add_log(f"Captioned clip {clip_num}")
+                            existing_captions_map[clip_num] = {
+                                "clip_number": clip_num,
+                                "filename": filename,
+                                "caption": caption
+                            }
+                            new_captions_processed += 1
+                            state.add_log(f"Captioned new clip {clip_num}")
                         if local_video.exists(): local_video.unlink()
                     except ResourceWarning as rw:
                         state.add_log(f"LIMIT: {rw}")
@@ -183,10 +221,19 @@ def run_pipeline():
                         state.add_log(f"Error on clip {clip_num}: {e}")
                         continue
 
-                if captions_data:
-                    with open(episode_json_path, "w") as f:
-                        json.dump({"anime": anime, "episode": episode, "captions": captions_data}, f, indent=2)
-                    upload_to_hf(api, episode_json_path, f"{anime}/{anime}_ep{episode}.json", TARGET_REPO)
+                # 3. Save combined/merged captions back to JSON and upload
+                if new_captions_processed > 0 or existing_captions_map:
+                    # Sort captions by clip_number before saving
+                    sorted_captions = sorted(existing_captions_map.values(), key=lambda x: x["clip_number"])
+                    output_data = {
+                        "anime": anime,
+                        "episode": episode,
+                        "captions": sorted_captions
+                    }
+                    with open(episode_json_path, "w", encoding="utf-8") as f:
+                        json.dump(output_data, f, indent=2, ensure_ascii=False)
+                    upload_to_hf(api, episode_json_path, remote_json_path, TARGET_REPO)
+                    state.add_log(f"Successfully updated and uploaded JSON for {anime} Ep {episode} (Total captions: {len(sorted_captions)})")
                 
                 if token_limit_hit:
                     state.add_log("Stopping due to token limit.")
@@ -224,7 +271,7 @@ async def index():
     </head>
     <body>
         <div class="container">
-            <h1>AniClips Captioner</h1>
+            <h1>AniClips Captioner (Resume Enabled)</h1>
             <div class="status">Status: <span id="status-text">Checking...</span></div>
             <div class="controls">
                 <button class="start" onclick="startPipeline()">Start Pipeline</button>