Spaces:
Running
Running
Add action metrics to agent chat and simplify feedback filters
Browse files- src/agent-session.ts +7 -4
- src/server.ts +7 -2
- tests/agent-chat.test.ts +146 -0
src/agent-session.ts
CHANGED
|
@@ -15,9 +15,13 @@ function ageContext(profile: Record<string, unknown>) {
|
|
| 15 |
return { ...profile, ageYears: age };
|
| 16 |
}
|
| 17 |
|
| 18 |
-
async function analysisResults(sessionId: string, userId: string): Promise<SessionResult> {
|
| 19 |
const response = await fetch(new URL(`/api/sessions/${sessionId}`, env.ANALYSIS_API_URL), {
|
| 20 |
-
headers: {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
signal: AbortSignal.timeout(120000),
|
| 22 |
});
|
| 23 |
if (!response.ok) throw new Error(`Unable to load results for session ${sessionId} (${response.status})`);
|
|
@@ -51,10 +55,9 @@ export async function generateSessionFeedback(db: SupabaseClient, sessionId: str
|
|
| 51 |
const { data: examples, error: examplesError } = await db.from("annotations")
|
| 52 |
.select("action_id,score,annotations_notes(note),actions!inner(action_id,session_id,type,sessions!inner(player_id,players!inner(gender,bdate,height,weight,position)))")
|
| 53 |
.eq("actions.type", action.type)
|
| 54 |
-
.eq("actions.sessions.players.gender", player.gender)
|
| 55 |
.in("actions.session_id", ownedSessionIds)
|
| 56 |
.neq("action_id", action.action_id)
|
| 57 |
-
.limit(
|
| 58 |
if (examplesError) throw new Error(`Unable to load human annotations: ${examplesError.message}`);
|
| 59 |
const referenceRows = (examples ?? []).filter((row: any) =>
|
| 60 |
row.score !== null || (row.annotations_notes ?? []).some((note: { note: string }) => typeof note.note === "string" && note.note.trim().length > 0));
|
|
|
|
| 15 |
return { ...profile, ageYears: age };
|
| 16 |
}
|
| 17 |
|
| 18 |
+
export async function analysisResults(sessionId: string, userId: string, isAdmin = false): Promise<SessionResult> {
|
| 19 |
const response = await fetch(new URL(`/api/sessions/${sessionId}`, env.ANALYSIS_API_URL), {
|
| 20 |
+
headers: {
|
| 21 |
+
"x-sporalize-proxy-secret": env.ANALYSIS_PROXY_SECRET,
|
| 22 |
+
"x-sporalize-user-id": userId,
|
| 23 |
+
...(isAdmin ? { "x-sporalize-admin": "true" } : {}),
|
| 24 |
+
},
|
| 25 |
signal: AbortSignal.timeout(120000),
|
| 26 |
});
|
| 27 |
if (!response.ok) throw new Error(`Unable to load results for session ${sessionId} (${response.status})`);
|
|
|
|
| 55 |
const { data: examples, error: examplesError } = await db.from("annotations")
|
| 56 |
.select("action_id,score,annotations_notes(note),actions!inner(action_id,session_id,type,sessions!inner(player_id,players!inner(gender,bdate,height,weight,position)))")
|
| 57 |
.eq("actions.type", action.type)
|
|
|
|
| 58 |
.in("actions.session_id", ownedSessionIds)
|
| 59 |
.neq("action_id", action.action_id)
|
| 60 |
+
.limit(5);
|
| 61 |
if (examplesError) throw new Error(`Unable to load human annotations: ${examplesError.message}`);
|
| 62 |
const referenceRows = (examples ?? []).filter((row: any) =>
|
| 63 |
row.score !== null || (row.annotations_notes ?? []).some((note: { note: string }) => typeof note.note === "string" && note.note.trim().length > 0));
|
src/server.ts
CHANGED
|
@@ -9,7 +9,7 @@ import { pipeline } from "node:stream/promises";
|
|
| 9 |
import { z } from "zod";
|
| 10 |
import { authPublicOrigin, env } from "./config.js";
|
| 11 |
import { analysisRouter } from "./analysis.js";
|
| 12 |
-
import { generateSessionFeedback } from "./agent-session.js";
|
| 13 |
import { authenticatedUser, clearPasswordRecoveryCookie, clearSessionCookies, hasPasswordRecoveryCookie, safeRedirect, setPasswordRecoveryCookie, setSessionCookies } from "./auth.js";
|
| 14 |
import { hashPassword, oauthClient, supabaseAdmin, supabaseAuth, syncProfile, userHasPassword, userPayload } from "./supabase.js";
|
| 15 |
import { registerProfileImageRoutes } from "./profile-images.js";
|
|
@@ -828,7 +828,12 @@ app.post("/api/agent/chat", async (req, res, next) => {
|
|
| 828 |
const { data: feedback, error: feedbackError } = await supabaseAdmin.from("actions_feedbacks")
|
| 829 |
.select("feedback").eq("action_id", input.actionId).order("feedback");
|
| 830 |
if (feedbackError) throw new Error(`Unable to load chat feedback: ${feedbackError.message}`);
|
| 831 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 832 |
}
|
| 833 |
}
|
| 834 |
if (input.playerId && !input.sessionId) {
|
|
|
|
| 9 |
import { z } from "zod";
|
| 10 |
import { authPublicOrigin, env } from "./config.js";
|
| 11 |
import { analysisRouter } from "./analysis.js";
|
| 12 |
+
import { analysisResults, generateSessionFeedback } from "./agent-session.js";
|
| 13 |
import { authenticatedUser, clearPasswordRecoveryCookie, clearSessionCookies, hasPasswordRecoveryCookie, safeRedirect, setPasswordRecoveryCookie, setSessionCookies } from "./auth.js";
|
| 14 |
import { hashPassword, oauthClient, supabaseAdmin, supabaseAuth, syncProfile, userHasPassword, userPayload } from "./supabase.js";
|
| 15 |
import { registerProfileImageRoutes } from "./profile-images.js";
|
|
|
|
| 828 |
const { data: feedback, error: feedbackError } = await supabaseAdmin.from("actions_feedbacks")
|
| 829 |
.select("feedback").eq("action_id", input.actionId).order("feedback");
|
| 830 |
if (feedbackError) throw new Error(`Unable to load chat feedback: ${feedbackError.message}`);
|
| 831 |
+
const results = await analysisResults(input.sessionId, current.user.id, isAdmin);
|
| 832 |
+
const measured = results.actions.find((item) => item.id === input.actionId);
|
| 833 |
+
if (!measured) return res.status(404).json({ error: "Analysis results for this action are unavailable." });
|
| 834 |
+
context.action = { ...action,
|
| 835 |
+
results: { metrics: measured.inActionMetrics ?? [], analytics: measured.rawAnalytics ?? {} },
|
| 836 |
+
feedback: (feedback ?? []).map((row) => row.feedback) };
|
| 837 |
}
|
| 838 |
}
|
| 839 |
if (input.playerId && !input.sessionId) {
|
tests/agent-chat.test.ts
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import assert from "node:assert/strict";
|
| 2 |
+
import { spawn } from "node:child_process";
|
| 3 |
+
import { once } from "node:events";
|
| 4 |
+
import { createServer, type IncomingHttpHeaders } from "node:http";
|
| 5 |
+
import type { AddressInfo } from "node:net";
|
| 6 |
+
import { test } from "node:test";
|
| 7 |
+
|
| 8 |
+
test("action chat sends current measurements on every message and preserves access checks", { timeout: 30_000 }, async () => {
|
| 9 |
+
const owner = "11111111-1111-4111-8111-111111111111";
|
| 10 |
+
const other = "22222222-2222-4222-8222-222222222222";
|
| 11 |
+
const sessionId = "33333333-3333-4333-8333-333333333333";
|
| 12 |
+
const actionId = "44444444-4444-4444-8444-444444444444";
|
| 13 |
+
const otherActionId = "55555555-5555-4555-8555-555555555555";
|
| 14 |
+
const analytics = { foot_region: "Toe", active_foot_speed: 1232.04, standing_foot_contact_seconds: 0,
|
| 15 |
+
minimum_knee_angle: null, future_metric: false };
|
| 16 |
+
const measured = { id: actionId, label: "Pass", rawAnalytics: analytics,
|
| 17 |
+
inActionMetrics: Object.entries(analytics).map(([name, value]) => ({ name, value, unit: name === "active_foot_speed" ? "cm/s" : "" })) };
|
| 18 |
+
const otherMeasured = { id: otherActionId, label: "Shoot", rawAnalytics: { ball_speed: 1303.31 },
|
| 19 |
+
inActionMetrics: [{ name: "Ball Speed", value: 1303.31, unit: "cm/s" }] };
|
| 20 |
+
let role = "user";
|
| 21 |
+
let sessionOwner = owner;
|
| 22 |
+
let resultsAvailable = true;
|
| 23 |
+
let analysisStatus = 200;
|
| 24 |
+
let analysisReads = 0;
|
| 25 |
+
let analysisHeaders: IncomingHttpHeaders = {};
|
| 26 |
+
type ChatPayload = { actions: unknown[]; message: string; history: unknown[];
|
| 27 |
+
context: { action?: { action_id: string; results: { metrics: unknown[]; analytics: unknown }; feedback: string[]; annotations: unknown }; actions?: unknown[] } };
|
| 28 |
+
const chats: ChatPayload[] = [];
|
| 29 |
+
const upstream = createServer(async (req, res) => {
|
| 30 |
+
const url = new URL(req.url!, "http://stub");
|
| 31 |
+
res.setHeader("Content-Type", "application/json");
|
| 32 |
+
if (url.pathname === "/auth/v1/user") return res.end(JSON.stringify({ id: owner, email: "test@example.com" }));
|
| 33 |
+
if (url.pathname === `/api/sessions/${sessionId}`) {
|
| 34 |
+
analysisReads++;
|
| 35 |
+
analysisHeaders = req.headers;
|
| 36 |
+
res.statusCode = analysisStatus;
|
| 37 |
+
return res.end(JSON.stringify({ id: sessionId, actions: resultsAvailable ? [otherMeasured, measured] : [] }));
|
| 38 |
+
}
|
| 39 |
+
if (url.pathname === "/api/chat") {
|
| 40 |
+
let body = "";
|
| 41 |
+
for await (const chunk of req) body += chunk;
|
| 42 |
+
chats.push(JSON.parse(body));
|
| 43 |
+
return res.end(JSON.stringify({ reply: "Measured coaching reply." }));
|
| 44 |
+
}
|
| 45 |
+
const table = url.pathname.split("/").at(-1);
|
| 46 |
+
let rows: Record<string, unknown>[] = table === "users" ? [{ user_id: owner, account_role: role }]
|
| 47 |
+
: table === "users_subscriptions" ? [{ user_id: owner, subscription_id: "plan", startdate: "2000-01-01", enddate: "2099-01-01", plan: { subscription_id: "plan" } }]
|
| 48 |
+
: table === "sessions" ? [{ session_id: sessionId, user_id: sessionOwner, name: "Practice", players: { fname: "Player" } }]
|
| 49 |
+
: table === "actions" ? [actionId, otherActionId].map((id) => ({ action_id: id, session_id: sessionId, type: "Pass", start_frame: 750, end_frame: 780,
|
| 50 |
+
annotations: [{ score: 7, annotations_notes: [{ note: "Coach note." }] }] }))
|
| 51 |
+
: table === "actions_feedbacks" ? [{ action_id: actionId, feedback: "Saved action evaluation." }]
|
| 52 |
+
: table === "sessions_feedbacks" ? [{ session_id: sessionId, feedback: "Saved session evaluation." }] : [];
|
| 53 |
+
for (const [key, filter] of url.searchParams) {
|
| 54 |
+
if (filter.startsWith("eq.")) rows = rows.filter((row) => String(row[key]) === filter.slice(3));
|
| 55 |
+
}
|
| 56 |
+
res.end(JSON.stringify(rows));
|
| 57 |
+
});
|
| 58 |
+
upstream.listen(0, "127.0.0.1");
|
| 59 |
+
await once(upstream, "listening");
|
| 60 |
+
const upstreamOrigin = `http://127.0.0.1:${(upstream.address() as AddressInfo).port}`;
|
| 61 |
+
const portProbe = createServer();
|
| 62 |
+
portProbe.listen(0, "127.0.0.1");
|
| 63 |
+
await once(portProbe, "listening");
|
| 64 |
+
const port = (portProbe.address() as AddressInfo).port;
|
| 65 |
+
await new Promise<void>((resolve) => portProbe.close(() => resolve()));
|
| 66 |
+
const origin = `http://127.0.0.1:${port}`;
|
| 67 |
+
const child = spawn(process.execPath, ["--import", "tsx", "src/server.ts"], {
|
| 68 |
+
env: { ...process.env, NODE_ENV: "test", PORT: String(port), FRONTEND_URL: origin, BACKEND_URL: origin,
|
| 69 |
+
SUPABASE_URL: upstreamOrigin, SUPABASE_ANON_KEY: "test-anon", SUPABASE_SERVICE_ROLE_KEY: "test-service",
|
| 70 |
+
ANALYSIS_API_URL: upstreamOrigin, ANALYSIS_PROXY_SECRET: "x".repeat(32),
|
| 71 |
+
AGENT_API_URL: upstreamOrigin, AGENT_API_KEY: "a".repeat(32), HF_TOKEN: "" },
|
| 72 |
+
stdio: ["ignore", "pipe", "pipe"],
|
| 73 |
+
});
|
| 74 |
+
try {
|
| 75 |
+
let stderr = "";
|
| 76 |
+
child.stderr.on("data", (data) => { stderr += String(data); });
|
| 77 |
+
await new Promise<void>((resolve, reject) => {
|
| 78 |
+
child.stdout.on("data", (data) => { if (String(data).includes("API listening")) resolve(); });
|
| 79 |
+
child.once("exit", (code) => reject(new Error(`API exited: ${code}\n${stderr}`)));
|
| 80 |
+
child.once("error", reject);
|
| 81 |
+
});
|
| 82 |
+
const request = (input: Record<string, unknown>, authenticated = true) => fetch(`${origin}/api/agent/chat`, {
|
| 83 |
+
method: "POST", signal: AbortSignal.timeout(10000),
|
| 84 |
+
headers: { "Content-Type": "application/json", Origin: origin, ...(authenticated ? { Cookie: "sp_access=access" } : {}) },
|
| 85 |
+
body: JSON.stringify({ message: "Explain these measurements.", ...input }),
|
| 86 |
+
});
|
| 87 |
+
const selected = { sessionId, actionId };
|
| 88 |
+
assert.equal((await request(selected, false)).status, 401);
|
| 89 |
+
assert.equal(analysisReads, 0);
|
| 90 |
+
assert.equal((await request(selected)).status, 200);
|
| 91 |
+
assert.equal(analysisReads, 1);
|
| 92 |
+
assert.deepEqual(chats[0].context.action?.results, { metrics: measured.inActionMetrics, analytics });
|
| 93 |
+
assert.deepEqual(chats[0].context.action?.feedback, ["Saved action evaluation."]);
|
| 94 |
+
assert.deepEqual(chats[0].context.action?.annotations, [{ score: 7, annotations_notes: [{ note: "Coach note." }] }]);
|
| 95 |
+
assert.equal(analysisHeaders["x-sporalize-user-id"], owner);
|
| 96 |
+
assert.equal(analysisHeaders["x-sporalize-proxy-secret"], "x".repeat(32));
|
| 97 |
+
assert.equal(analysisHeaders["x-sporalize-admin"], undefined);
|
| 98 |
+
|
| 99 |
+
analytics.active_foot_speed = 1500.123456;
|
| 100 |
+
measured.inActionMetrics.find((metric) => metric.name === "active_foot_speed")!.value = analytics.active_foot_speed;
|
| 101 |
+
const history = [{ role: "user", content: "Explain these measurements." }, { role: "assistant", content: "Previous reply." }];
|
| 102 |
+
assert.equal((await request({ ...selected, message: "What changed?", history })).status, 200);
|
| 103 |
+
assert.equal(analysisReads, 2);
|
| 104 |
+
assert.deepEqual(chats[1].context.action?.results, { metrics: measured.inActionMetrics, analytics });
|
| 105 |
+
assert.deepEqual(chats[1].history, history);
|
| 106 |
+
assert.equal((await request({ sessionId, actionId: otherActionId })).status, 200);
|
| 107 |
+
assert.deepEqual(chats[2].context.action?.results, { metrics: otherMeasured.inActionMetrics, analytics: otherMeasured.rawAnalytics });
|
| 108 |
+
|
| 109 |
+
const chatCount = chats.length;
|
| 110 |
+
const readCount = analysisReads;
|
| 111 |
+
sessionOwner = other;
|
| 112 |
+
assert.equal((await request(selected)).status, 404);
|
| 113 |
+
assert.equal(analysisReads, readCount, "unauthorized requests must not load measurements");
|
| 114 |
+
assert.equal(chats.length, chatCount);
|
| 115 |
+
role = "admin";
|
| 116 |
+
assert.equal((await request(selected)).status, 200);
|
| 117 |
+
assert.equal(analysisHeaders["x-sporalize-admin"], "true");
|
| 118 |
+
assert.equal(analysisHeaders["x-sporalize-user-id"], owner);
|
| 119 |
+
assert.deepEqual(chats.at(-1)?.context.action?.results, { metrics: measured.inActionMetrics, analytics });
|
| 120 |
+
role = "user";
|
| 121 |
+
sessionOwner = owner;
|
| 122 |
+
|
| 123 |
+
const successfulChats = chats.length;
|
| 124 |
+
resultsAvailable = false;
|
| 125 |
+
assert.equal((await request(selected)).status, 404);
|
| 126 |
+
analysisStatus = 503;
|
| 127 |
+
assert.equal((await request(selected)).status, 500);
|
| 128 |
+
assert.equal(chats.length, successfulChats, "missing results must not produce an ungrounded reply");
|
| 129 |
+
const failedReads = analysisReads;
|
| 130 |
+
assert.equal((await request({ actionId })).status, 400);
|
| 131 |
+
assert.equal((await request({ ...selected, actionId: "66666666-6666-4666-8666-666666666666" })).status, 404);
|
| 132 |
+
assert.equal(analysisReads, failedReads);
|
| 133 |
+
assert.equal((await request({ sessionId })).status, 200);
|
| 134 |
+
assert.equal((await request({})).status, 200);
|
| 135 |
+
assert.equal(analysisReads, failedReads, "general and session chats do not need action measurements");
|
| 136 |
+
assert.equal(chats.at(-1)?.context.action, undefined);
|
| 137 |
+
} finally {
|
| 138 |
+
if (child.exitCode === null && child.signalCode === null) {
|
| 139 |
+
const exited = once(child, "exit");
|
| 140 |
+
child.kill();
|
| 141 |
+
await exited;
|
| 142 |
+
}
|
| 143 |
+
upstream.closeAllConnections();
|
| 144 |
+
await new Promise<void>((resolve) => upstream.close(() => resolve()));
|
| 145 |
+
}
|
| 146 |
+
});
|