openreels-deploy commited on
Commit
e506bae
·
1 Parent(s): 5aabd18

fix: script budget, closing card, stuck-job recovery

Browse files
CHANGELOG.md CHANGED
@@ -18,6 +18,14 @@ All notable changes to Pitlane will be documented in this file.
18
  the clip options, and the job page shows the four clip stages plus one playable vertical per
19
  clip with its hook, download and copyable title and caption.
20
 
 
 
 
 
 
 
 
 
21
  ### Fixed
22
  - **Scripts can no longer run to 56 seconds**: the director's system prompt set a 72-88 word
23
  budget, but the user message injected a competing 100-140 word budget from a stale pacing
 
18
  the clip options, and the job page shows the four clip stages plus one playable vertical per
19
  clip with its hook, download and copyable title and caption.
20
 
21
+ - **The last shot carries the answer**: a model kept putting the winner's name on the closing card,
22
+ turning the closing question into a paragraph and leaving the last shot as filler. A guard now
23
+ moves any closing statement onto the last footage scene and keeps only the question on the card.
24
+
25
+ - **A restart can no longer leave a job "running" forever**: jobs whose queue entry was dropped
26
+ but whose metadata still said `running` are failed on boot, so they stop blocking the single
27
+ worker slot.
28
+
29
  ### Fixed
30
  - **Scripts can no longer run to 56 seconds**: the director's system prompt set a 72-88 word
31
  budget, but the user message injected a competing 100-140 word budget from a stale pacing
src/pipeline/orchestrator.ts CHANGED
@@ -276,6 +276,14 @@ function prepareScore(
276
  prepared = capped.score;
277
  }
278
 
 
 
 
 
 
 
 
 
279
  if (noAiAssets) {
280
  const { score: sanitized, adjustments } = sanitizeScoreForNoAiAssets(prepared);
281
  if (adjustments.length > 0) {
 
276
  prepared = capped.score;
277
  }
278
 
279
+ // The answer belongs to the last shot, not to the card that asks the question.
280
+ const closing = moveClosingConclusionToFootage(prepared);
281
+ if (closing.moved.length > 0) {
282
+ cb.onLog?.("[director] Moved the closing answer onto the last shot");
283
+ cb.onProgress?.("director", { type: "closing_answer_on_footage" });
284
+ prepared = closing.score;
285
+ }
286
+
287
  if (noAiAssets) {
288
  const { score: sanitized, adjustments } = sanitizeScoreForNoAiAssets(prepared);
289
  if (adjustments.length > 0) {
src/pipeline/score-guards.test.ts CHANGED
@@ -5,6 +5,7 @@ import {
5
  enforceFootageOpening,
6
  enforceMotionContinuity,
7
  enforceTransitionDiscipline,
 
8
  } from "./score-guards.js";
9
 
10
  const card = (script: string, prompt = "A heading"): Scene => ({
@@ -172,6 +173,42 @@ describe("enforceTransitionDiscipline", () => {
172
  });
173
  });
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  describe("enforceFootageOpening", () => {
176
  it("moves the opening card's line onto the first shot", () => {
177
  const score = scoreOf([card("The car with no clutch."), shot("It had pedals.")]);
 
5
  enforceFootageOpening,
6
  enforceMotionContinuity,
7
  enforceTransitionDiscipline,
8
+ moveClosingConclusionToFootage,
9
  } from "./score-guards.js";
10
 
11
  const card = (script: string, prompt = "A heading"): Scene => ({
 
173
  });
174
  });
175
 
176
+ describe("moveClosingConclusionToFootage", () => {
177
+ it("puts the answer on the last shot and leaves only the question on the card", () => {
178
+ const score = scoreOf([
179
+ shot("The RS200 was built for domination."),
180
+ shot("At Rally Portugal it ploughed into the crowd."),
181
+ card("The Ford RS200 carried the highest death toll. Which car was the deadliest? Comment below."),
182
+ ]);
183
+
184
+ const result = moveClosingConclusionToFootage(score);
185
+ expect(result.moved).toEqual([1]);
186
+ expect(result.score.scenes[1]!.script_line).toContain("The Ford RS200 carried the highest death toll.");
187
+ expect(result.score.scenes[2]!.script_line).toBe(
188
+ "Which car was the deadliest? Comment below.",
189
+ );
190
+ // The caller's score must not be edited in place.
191
+ expect(score.scenes[2]!.script_line).toContain("The Ford RS200");
192
+ });
193
+
194
+ it("leaves a card that only asks alone", () => {
195
+ const score = scoreOf([shot("A car."), card("Which one would you pick? Comment below.")]);
196
+ expect(moveClosingConclusionToFootage(score).moved).toEqual([]);
197
+ });
198
+
199
+ it("never empties the card when every sentence is a statement", () => {
200
+ const score = scoreOf([shot("A car."), card("It was fast. It was also lethal.")]);
201
+ const result = moveClosingConclusionToFootage(score);
202
+ expect(result.score.scenes[1]!.script_line).toBe("It was also lethal.");
203
+ expect(result.score.scenes[0]!.script_line).toContain("It was fast.");
204
+ });
205
+
206
+ it("does nothing when the film does not end on a card", () => {
207
+ const score = scoreOf([card("A heading."), shot("The answer.")]);
208
+ expect(moveClosingConclusionToFootage(score).moved).toEqual([]);
209
+ });
210
+ });
211
+
212
  describe("enforceFootageOpening", () => {
213
  it("moves the opening card's line onto the first shot", () => {
214
  const score = scoreOf([card("The car with no clutch."), shot("It had pedals.")]);
src/pipeline/score-guards.ts CHANGED
@@ -141,8 +141,7 @@ export function moveClosingConclusionToFootage(score: DirectorScore): ClosingCar
141
  // A direct ask stays on the card even when it is an imperative rather than
142
  // a question: "Comment below." is part of the call to action, not an answer.
143
  const isCallToAction = (part: string) =>
144
- /^(comment|drop|tell|share|subscribe|follow|post|vote|reply|let)\b/i.test(part) ||
145
- part.split(/\s+/).length <= 4;
146
 
147
  const onCard = parts.filter((part) => isQuestion(part) || isCallToAction(part));
148
  const answer = parts.filter((part) => !isQuestion(part) && !isCallToAction(part));
 
141
  // A direct ask stays on the card even when it is an imperative rather than
142
  // a question: "Comment below." is part of the call to action, not an answer.
143
  const isCallToAction = (part: string) =>
144
+ /^(comment|drop|tell|share|subscribe|follow|post|vote|reply|let)\b/i.test(part);
 
145
 
146
  const onCard = parts.filter((part) => isQuestion(part) || isCallToAction(part));
147
  const answer = parts.filter((part) => !isQuestion(part) && !isCallToAction(part));
src/queue/orphan-recovery.test.ts CHANGED
@@ -1,5 +1,6 @@
1
  import { describe, expect, it, vi } from "vitest";
2
  import {
 
3
  isOrphaned,
4
  planQueueSweep,
5
  selectOrphanedJobs,
@@ -83,6 +84,30 @@ describe("removeFromQueue", () => {
83
  });
84
  });
85
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86
  describe("planQueueSweep", () => {
87
  const entry = (over: Partial<Parameters<typeof planQueueSweep>[0][number]>) => ({
88
  id: "job",
 
1
  import { describe, expect, it, vi } from "vitest";
2
  import {
3
+ findStaleLocalJobs,
4
  isOrphaned,
5
  planQueueSweep,
6
  selectOrphanedJobs,
 
84
  });
85
  });
86
 
87
+ describe("findStaleLocalJobs", () => {
88
+ it("fails a running job that has no queue entry", () => {
89
+ const result = findStaleLocalJobs(
90
+ [
91
+ { id: "stuck", status: "running" },
92
+ { id: "waiting", status: "queued" },
93
+ { id: "done", status: "completed" },
94
+ ],
95
+ new Set(["waiting"]),
96
+ );
97
+ expect(result.map((job) => job.id)).toEqual(["stuck"]);
98
+ expect(result[0]!.reason).toContain("Retry it");
99
+ });
100
+
101
+ it("leaves every in-flight job alone when the queue still owns it", () => {
102
+ expect(
103
+ findStaleLocalJobs(
104
+ [{ id: "a", status: "running" }, { id: "b", status: "queued" }],
105
+ new Set(["a", "b"]),
106
+ ),
107
+ ).toEqual([]);
108
+ });
109
+ });
110
+
111
  describe("planQueueSweep", () => {
112
  const entry = (over: Partial<Parameters<typeof planQueueSweep>[0][number]>) => ({
113
  id: "job",
src/queue/orphan-recovery.ts CHANGED
@@ -117,9 +117,29 @@ export async function listActiveJobs(
117
  return candidates;
118
  }
119
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
120
  /** One entry in the queue, whatever list it is currently sitting in. */
121
- export interface QueueEntry {
122
- id: string;
123
  /**
124
  * `completed` and `failed` are only read at boot: they are the queue's own
125
  * verdict on a job whose metadata can still say `running` after a restart.
 
117
  return candidates;
118
  }
119
 
120
+ /**
121
+ * Local jobs that still claim to be in flight but have no queue entry at all.
122
+ *
123
+ * The queue sweep only sees jobs BullMQ still knows about. A restart can leave
124
+ * a job directory whose metadata says `running` after its queue entry was
125
+ * dropped, and that job then reports "running" forever while no worker owns
126
+ * it. With one worker slot this eventually blocks every new request.
127
+ */
128
+ export function findStaleLocalJobs(
129
+ metas: Array<{ id: string; status: string }>,
130
+ queuedIds: Set<string>,
131
+ ): OrphanedJob[] {
132
+ return metas
133
+ .filter((meta) => meta && (meta.status === "running" || meta.status === "queued"))
134
+ .filter((meta) => !queuedIds.has(meta.id))
135
+ .map((meta) => ({
136
+ id: meta.id,
137
+ reason: "The Space restarted before this job finished. Retry it.",
138
+ }));
139
+ }
140
+
141
  /** One entry in the queue, whatever list it is currently sitting in. */
142
+ export interface QueueEntry { id: string;
 
143
  /**
144
  * `completed` and `failed` are only read at boot: they are the queue's own
145
  * verdict on a job whose metadata can still say `running` after a restart.
src/worker.ts CHANGED
@@ -11,6 +11,7 @@ import type { Attribution } from "./schema/providers.js";
11
  import {
12
  buildJobLinks,
13
  buildPlaceholderMeta,
 
14
  readJobMeta,
15
  STAGE_NAMES,
16
  writeJobMeta,
@@ -22,7 +23,9 @@ import { runClipPipeline } from "./clip/pipeline.js";
22
  import { normalizeClipSpec } from "./clip/spec.js";
23
  import { CLIP_STAGE_NAMES, type ClipStageName } from "./clip/stages.js";
24
  import {
 
25
  listQueueEntries,
 
26
  planQueueSweep,
27
  removeFromQueue,
28
  } from "./queue/orphan-recovery.js";
@@ -164,6 +167,22 @@ async function recoverOrphanedJobs(force = false): Promise<void> {
164
  : entries,
165
  );
166
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
  for (const id of sweep.remove) {
168
  // The job already finished. A queue entry left over from the restart
169
  // would re-run it and overwrite the render that was published.
@@ -171,7 +190,7 @@ async function recoverOrphanedJobs(force = false): Promise<void> {
171
  await removeFromQueue(queue, redis, id);
172
  }
173
 
174
- for (const orphan of sweep.fail) {
175
  console.warn(`[worker] Failing orphaned job ${orphan.id}: ${orphan.reason}`);
176
  const meta = metaOf(orphan.id);
177
  if (meta) {
 
11
  import {
12
  buildJobLinks,
13
  buildPlaceholderMeta,
14
+ listJobMetas,
15
  readJobMeta,
16
  STAGE_NAMES,
17
  writeJobMeta,
 
23
  import { normalizeClipSpec } from "./clip/spec.js";
24
  import { CLIP_STAGE_NAMES, type ClipStageName } from "./clip/stages.js";
25
  import {
26
+ findStaleLocalJobs,
27
  listQueueEntries,
28
+ type OrphanedJob,
29
  planQueueSweep,
30
  removeFromQueue,
31
  } from "./queue/orphan-recovery.js";
 
167
  : entries,
168
  );
169
 
170
+ // A restart can also leave a job directory whose metadata still says
171
+ // "running" after its queue entry was dropped. BullMQ has nothing to
172
+ // sweep in that case, and the job would report running forever.
173
+ const queuedIds = new Set(entries.map((entry) => entry.id));
174
+ const localCandidates = findStaleLocalJobs(
175
+ listJobMetas(JOBS_DIR).map(({ id, meta }) => ({ id, status: meta.status })),
176
+ queuedIds,
177
+ );
178
+ const staleLocals: OrphanedJob[] = [];
179
+ for (const candidate of localCandidates) {
180
+ // The queue listing is paged; a real entry past the first page must not
181
+ // be mistaken for a missing one.
182
+ if (!(await queue.getJob(candidate.id))) staleLocals.push(candidate);
183
+ }
184
+ const toFail = [...sweep.fail, ...staleLocals];
185
+
186
  for (const id of sweep.remove) {
187
  // The job already finished. A queue entry left over from the restart
188
  // would re-run it and overwrite the render that was published.
 
190
  await removeFromQueue(queue, redis, id);
191
  }
192
 
193
+ for (const orphan of toFail) {
194
  console.warn(`[worker] Failing orphaned job ${orphan.id}: ${orphan.reason}`);
195
  const meta = metaOf(orphan.id);
196
  if (meta) {