openreels-deploy commited on
Commit ·
e506bae
1
Parent(s): 5aabd18
fix: script budget, closing card, stuck-job recovery
Browse files- CHANGELOG.md +8 -0
- src/pipeline/orchestrator.ts +8 -0
- src/pipeline/score-guards.test.ts +37 -0
- src/pipeline/score-guards.ts +1 -2
- src/queue/orphan-recovery.test.ts +25 -0
- src/queue/orphan-recovery.ts +22 -2
- src/worker.ts +20 -1
CHANGELOG.md
CHANGED
|
@@ -18,6 +18,14 @@ All notable changes to Pitlane will be documented in this file.
|
|
| 18 |
the clip options, and the job page shows the four clip stages plus one playable vertical per
|
| 19 |
clip with its hook, download and copyable title and caption.
|
| 20 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
### Fixed
|
| 22 |
- **Scripts can no longer run to 56 seconds**: the director's system prompt set a 72-88 word
|
| 23 |
budget, but the user message injected a competing 100-140 word budget from a stale pacing
|
|
|
|
| 18 |
the clip options, and the job page shows the four clip stages plus one playable vertical per
|
| 19 |
clip with its hook, download and copyable title and caption.
|
| 20 |
|
| 21 |
+
- **The last shot carries the answer**: a model kept putting the winner's name on the closing card,
|
| 22 |
+
turning the closing question into a paragraph and leaving the last shot as filler. A guard now
|
| 23 |
+
moves any closing statement onto the last footage scene and keeps only the question on the card.
|
| 24 |
+
|
| 25 |
+
- **A restart can no longer leave a job "running" forever**: jobs whose queue entry was dropped
|
| 26 |
+
but whose metadata still said `running` are failed on boot, so they stop blocking the single
|
| 27 |
+
worker slot.
|
| 28 |
+
|
| 29 |
### Fixed
|
| 30 |
- **Scripts can no longer run to 56 seconds**: the director's system prompt set a 72-88 word
|
| 31 |
budget, but the user message injected a competing 100-140 word budget from a stale pacing
|
src/pipeline/orchestrator.ts
CHANGED
|
@@ -276,6 +276,14 @@ function prepareScore(
|
|
| 276 |
prepared = capped.score;
|
| 277 |
}
|
| 278 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
if (noAiAssets) {
|
| 280 |
const { score: sanitized, adjustments } = sanitizeScoreForNoAiAssets(prepared);
|
| 281 |
if (adjustments.length > 0) {
|
|
|
|
| 276 |
prepared = capped.score;
|
| 277 |
}
|
| 278 |
|
| 279 |
+
// The answer belongs to the last shot, not to the card that asks the question.
|
| 280 |
+
const closing = moveClosingConclusionToFootage(prepared);
|
| 281 |
+
if (closing.moved.length > 0) {
|
| 282 |
+
cb.onLog?.("[director] Moved the closing answer onto the last shot");
|
| 283 |
+
cb.onProgress?.("director", { type: "closing_answer_on_footage" });
|
| 284 |
+
prepared = closing.score;
|
| 285 |
+
}
|
| 286 |
+
|
| 287 |
if (noAiAssets) {
|
| 288 |
const { score: sanitized, adjustments } = sanitizeScoreForNoAiAssets(prepared);
|
| 289 |
if (adjustments.length > 0) {
|
src/pipeline/score-guards.test.ts
CHANGED
|
@@ -5,6 +5,7 @@ import {
|
|
| 5 |
enforceFootageOpening,
|
| 6 |
enforceMotionContinuity,
|
| 7 |
enforceTransitionDiscipline,
|
|
|
|
| 8 |
} from "./score-guards.js";
|
| 9 |
|
| 10 |
const card = (script: string, prompt = "A heading"): Scene => ({
|
|
@@ -172,6 +173,42 @@ describe("enforceTransitionDiscipline", () => {
|
|
| 172 |
});
|
| 173 |
});
|
| 174 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
describe("enforceFootageOpening", () => {
|
| 176 |
it("moves the opening card's line onto the first shot", () => {
|
| 177 |
const score = scoreOf([card("The car with no clutch."), shot("It had pedals.")]);
|
|
|
|
| 5 |
enforceFootageOpening,
|
| 6 |
enforceMotionContinuity,
|
| 7 |
enforceTransitionDiscipline,
|
| 8 |
+
moveClosingConclusionToFootage,
|
| 9 |
} from "./score-guards.js";
|
| 10 |
|
| 11 |
const card = (script: string, prompt = "A heading"): Scene => ({
|
|
|
|
| 173 |
});
|
| 174 |
});
|
| 175 |
|
| 176 |
+
describe("moveClosingConclusionToFootage", () => {
|
| 177 |
+
it("puts the answer on the last shot and leaves only the question on the card", () => {
|
| 178 |
+
const score = scoreOf([
|
| 179 |
+
shot("The RS200 was built for domination."),
|
| 180 |
+
shot("At Rally Portugal it ploughed into the crowd."),
|
| 181 |
+
card("The Ford RS200 carried the highest death toll. Which car was the deadliest? Comment below."),
|
| 182 |
+
]);
|
| 183 |
+
|
| 184 |
+
const result = moveClosingConclusionToFootage(score);
|
| 185 |
+
expect(result.moved).toEqual([1]);
|
| 186 |
+
expect(result.score.scenes[1]!.script_line).toContain("The Ford RS200 carried the highest death toll.");
|
| 187 |
+
expect(result.score.scenes[2]!.script_line).toBe(
|
| 188 |
+
"Which car was the deadliest? Comment below.",
|
| 189 |
+
);
|
| 190 |
+
// The caller's score must not be edited in place.
|
| 191 |
+
expect(score.scenes[2]!.script_line).toContain("The Ford RS200");
|
| 192 |
+
});
|
| 193 |
+
|
| 194 |
+
it("leaves a card that only asks alone", () => {
|
| 195 |
+
const score = scoreOf([shot("A car."), card("Which one would you pick? Comment below.")]);
|
| 196 |
+
expect(moveClosingConclusionToFootage(score).moved).toEqual([]);
|
| 197 |
+
});
|
| 198 |
+
|
| 199 |
+
it("never empties the card when every sentence is a statement", () => {
|
| 200 |
+
const score = scoreOf([shot("A car."), card("It was fast. It was also lethal.")]);
|
| 201 |
+
const result = moveClosingConclusionToFootage(score);
|
| 202 |
+
expect(result.score.scenes[1]!.script_line).toBe("It was also lethal.");
|
| 203 |
+
expect(result.score.scenes[0]!.script_line).toContain("It was fast.");
|
| 204 |
+
});
|
| 205 |
+
|
| 206 |
+
it("does nothing when the film does not end on a card", () => {
|
| 207 |
+
const score = scoreOf([card("A heading."), shot("The answer.")]);
|
| 208 |
+
expect(moveClosingConclusionToFootage(score).moved).toEqual([]);
|
| 209 |
+
});
|
| 210 |
+
});
|
| 211 |
+
|
| 212 |
describe("enforceFootageOpening", () => {
|
| 213 |
it("moves the opening card's line onto the first shot", () => {
|
| 214 |
const score = scoreOf([card("The car with no clutch."), shot("It had pedals.")]);
|
src/pipeline/score-guards.ts
CHANGED
|
@@ -141,8 +141,7 @@ export function moveClosingConclusionToFootage(score: DirectorScore): ClosingCar
|
|
| 141 |
// A direct ask stays on the card even when it is an imperative rather than
|
| 142 |
// a question: "Comment below." is part of the call to action, not an answer.
|
| 143 |
const isCallToAction = (part: string) =>
|
| 144 |
-
/^(comment|drop|tell|share|subscribe|follow|post|vote|reply|let)\b/i.test(part)
|
| 145 |
-
part.split(/\s+/).length <= 4;
|
| 146 |
|
| 147 |
const onCard = parts.filter((part) => isQuestion(part) || isCallToAction(part));
|
| 148 |
const answer = parts.filter((part) => !isQuestion(part) && !isCallToAction(part));
|
|
|
|
| 141 |
// A direct ask stays on the card even when it is an imperative rather than
|
| 142 |
// a question: "Comment below." is part of the call to action, not an answer.
|
| 143 |
const isCallToAction = (part: string) =>
|
| 144 |
+
/^(comment|drop|tell|share|subscribe|follow|post|vote|reply|let)\b/i.test(part);
|
|
|
|
| 145 |
|
| 146 |
const onCard = parts.filter((part) => isQuestion(part) || isCallToAction(part));
|
| 147 |
const answer = parts.filter((part) => !isQuestion(part) && !isCallToAction(part));
|
src/queue/orphan-recovery.test.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
import { describe, expect, it, vi } from "vitest";
|
| 2 |
import {
|
|
|
|
| 3 |
isOrphaned,
|
| 4 |
planQueueSweep,
|
| 5 |
selectOrphanedJobs,
|
|
@@ -83,6 +84,30 @@ describe("removeFromQueue", () => {
|
|
| 83 |
});
|
| 84 |
});
|
| 85 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
describe("planQueueSweep", () => {
|
| 87 |
const entry = (over: Partial<Parameters<typeof planQueueSweep>[0][number]>) => ({
|
| 88 |
id: "job",
|
|
|
|
| 1 |
import { describe, expect, it, vi } from "vitest";
|
| 2 |
import {
|
| 3 |
+
findStaleLocalJobs,
|
| 4 |
isOrphaned,
|
| 5 |
planQueueSweep,
|
| 6 |
selectOrphanedJobs,
|
|
|
|
| 84 |
});
|
| 85 |
});
|
| 86 |
|
| 87 |
+
describe("findStaleLocalJobs", () => {
|
| 88 |
+
it("fails a running job that has no queue entry", () => {
|
| 89 |
+
const result = findStaleLocalJobs(
|
| 90 |
+
[
|
| 91 |
+
{ id: "stuck", status: "running" },
|
| 92 |
+
{ id: "waiting", status: "queued" },
|
| 93 |
+
{ id: "done", status: "completed" },
|
| 94 |
+
],
|
| 95 |
+
new Set(["waiting"]),
|
| 96 |
+
);
|
| 97 |
+
expect(result.map((job) => job.id)).toEqual(["stuck"]);
|
| 98 |
+
expect(result[0]!.reason).toContain("Retry it");
|
| 99 |
+
});
|
| 100 |
+
|
| 101 |
+
it("leaves every in-flight job alone when the queue still owns it", () => {
|
| 102 |
+
expect(
|
| 103 |
+
findStaleLocalJobs(
|
| 104 |
+
[{ id: "a", status: "running" }, { id: "b", status: "queued" }],
|
| 105 |
+
new Set(["a", "b"]),
|
| 106 |
+
),
|
| 107 |
+
).toEqual([]);
|
| 108 |
+
});
|
| 109 |
+
});
|
| 110 |
+
|
| 111 |
describe("planQueueSweep", () => {
|
| 112 |
const entry = (over: Partial<Parameters<typeof planQueueSweep>[0][number]>) => ({
|
| 113 |
id: "job",
|
src/queue/orphan-recovery.ts
CHANGED
|
@@ -117,9 +117,29 @@ export async function listActiveJobs(
|
|
| 117 |
return candidates;
|
| 118 |
}
|
| 119 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
/** One entry in the queue, whatever list it is currently sitting in. */
|
| 121 |
-
export interface QueueEntry {
|
| 122 |
-
id: string;
|
| 123 |
/**
|
| 124 |
* `completed` and `failed` are only read at boot: they are the queue's own
|
| 125 |
* verdict on a job whose metadata can still say `running` after a restart.
|
|
|
|
| 117 |
return candidates;
|
| 118 |
}
|
| 119 |
|
| 120 |
+
/**
|
| 121 |
+
* Local jobs that still claim to be in flight but have no queue entry at all.
|
| 122 |
+
*
|
| 123 |
+
* The queue sweep only sees jobs BullMQ still knows about. A restart can leave
|
| 124 |
+
* a job directory whose metadata says `running` after its queue entry was
|
| 125 |
+
* dropped, and that job then reports "running" forever while no worker owns
|
| 126 |
+
* it. With one worker slot this eventually blocks every new request.
|
| 127 |
+
*/
|
| 128 |
+
export function findStaleLocalJobs(
|
| 129 |
+
metas: Array<{ id: string; status: string }>,
|
| 130 |
+
queuedIds: Set<string>,
|
| 131 |
+
): OrphanedJob[] {
|
| 132 |
+
return metas
|
| 133 |
+
.filter((meta) => meta && (meta.status === "running" || meta.status === "queued"))
|
| 134 |
+
.filter((meta) => !queuedIds.has(meta.id))
|
| 135 |
+
.map((meta) => ({
|
| 136 |
+
id: meta.id,
|
| 137 |
+
reason: "The Space restarted before this job finished. Retry it.",
|
| 138 |
+
}));
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
/** One entry in the queue, whatever list it is currently sitting in. */
|
| 142 |
+
export interface QueueEntry { id: string;
|
|
|
|
| 143 |
/**
|
| 144 |
* `completed` and `failed` are only read at boot: they are the queue's own
|
| 145 |
* verdict on a job whose metadata can still say `running` after a restart.
|
src/worker.ts
CHANGED
|
@@ -11,6 +11,7 @@ import type { Attribution } from "./schema/providers.js";
|
|
| 11 |
import {
|
| 12 |
buildJobLinks,
|
| 13 |
buildPlaceholderMeta,
|
|
|
|
| 14 |
readJobMeta,
|
| 15 |
STAGE_NAMES,
|
| 16 |
writeJobMeta,
|
|
@@ -22,7 +23,9 @@ import { runClipPipeline } from "./clip/pipeline.js";
|
|
| 22 |
import { normalizeClipSpec } from "./clip/spec.js";
|
| 23 |
import { CLIP_STAGE_NAMES, type ClipStageName } from "./clip/stages.js";
|
| 24 |
import {
|
|
|
|
| 25 |
listQueueEntries,
|
|
|
|
| 26 |
planQueueSweep,
|
| 27 |
removeFromQueue,
|
| 28 |
} from "./queue/orphan-recovery.js";
|
|
@@ -164,6 +167,22 @@ async function recoverOrphanedJobs(force = false): Promise<void> {
|
|
| 164 |
: entries,
|
| 165 |
);
|
| 166 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
for (const id of sweep.remove) {
|
| 168 |
// The job already finished. A queue entry left over from the restart
|
| 169 |
// would re-run it and overwrite the render that was published.
|
|
@@ -171,7 +190,7 @@ async function recoverOrphanedJobs(force = false): Promise<void> {
|
|
| 171 |
await removeFromQueue(queue, redis, id);
|
| 172 |
}
|
| 173 |
|
| 174 |
-
for (const orphan of
|
| 175 |
console.warn(`[worker] Failing orphaned job ${orphan.id}: ${orphan.reason}`);
|
| 176 |
const meta = metaOf(orphan.id);
|
| 177 |
if (meta) {
|
|
|
|
| 11 |
import {
|
| 12 |
buildJobLinks,
|
| 13 |
buildPlaceholderMeta,
|
| 14 |
+
listJobMetas,
|
| 15 |
readJobMeta,
|
| 16 |
STAGE_NAMES,
|
| 17 |
writeJobMeta,
|
|
|
|
| 23 |
import { normalizeClipSpec } from "./clip/spec.js";
|
| 24 |
import { CLIP_STAGE_NAMES, type ClipStageName } from "./clip/stages.js";
|
| 25 |
import {
|
| 26 |
+
findStaleLocalJobs,
|
| 27 |
listQueueEntries,
|
| 28 |
+
type OrphanedJob,
|
| 29 |
planQueueSweep,
|
| 30 |
removeFromQueue,
|
| 31 |
} from "./queue/orphan-recovery.js";
|
|
|
|
| 167 |
: entries,
|
| 168 |
);
|
| 169 |
|
| 170 |
+
// A restart can also leave a job directory whose metadata still says
|
| 171 |
+
// "running" after its queue entry was dropped. BullMQ has nothing to
|
| 172 |
+
// sweep in that case, and the job would report running forever.
|
| 173 |
+
const queuedIds = new Set(entries.map((entry) => entry.id));
|
| 174 |
+
const localCandidates = findStaleLocalJobs(
|
| 175 |
+
listJobMetas(JOBS_DIR).map(({ id, meta }) => ({ id, status: meta.status })),
|
| 176 |
+
queuedIds,
|
| 177 |
+
);
|
| 178 |
+
const staleLocals: OrphanedJob[] = [];
|
| 179 |
+
for (const candidate of localCandidates) {
|
| 180 |
+
// The queue listing is paged; a real entry past the first page must not
|
| 181 |
+
// be mistaken for a missing one.
|
| 182 |
+
if (!(await queue.getJob(candidate.id))) staleLocals.push(candidate);
|
| 183 |
+
}
|
| 184 |
+
const toFail = [...sweep.fail, ...staleLocals];
|
| 185 |
+
|
| 186 |
for (const id of sweep.remove) {
|
| 187 |
// The job already finished. A queue entry left over from the restart
|
| 188 |
// would re-run it and overwrite the render that was published.
|
|
|
|
| 190 |
await removeFromQueue(queue, redis, id);
|
| 191 |
}
|
| 192 |
|
| 193 |
+
for (const orphan of toFail) {
|
| 194 |
console.warn(`[worker] Failing orphaned job ${orphan.id}: ${orphan.reason}`);
|
| 195 |
const meta = metaOf(orphan.id);
|
| 196 |
if (meta) {
|