bhavikprit commited on
Commit
687f7e3
·
1 Parent(s): 6331d61

feat(#160): add Real-Time Workload Signals API endpoint

Browse files

New GET /api/workload endpoint providing system-wide capacity metrics
and throttle recommendations for agent load awareness.

Response sections:
- capacity: active tasks, recent task/error rates, completion throughput
- queue: pending task depth by status/priority, estimated wait time
- agents: online/busy/idle counts, busy ratio, per-agent load distribution
- recommendation: actionable signal (normal|throttle|shed|pause)
with reasons, submit_ok flag, and suggested delay
- thresholds: current threshold configuration for transparency

Recommendation engine factors:
- Queue depth (20/50/100 thresholds)
- Agent saturation ratio (80%/95%)
- Error rate in last 5 minutes (10%/25%)
- Agent availability (pause if none online)

Fixes #160

Files changed (1) hide show
  1. src/app/api/workload/route.ts +294 -0
src/app/api/workload/route.ts ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { NextRequest, NextResponse } from 'next/server';
2
+ import { getDatabase } from '@/lib/db';
3
+ import { requireRole } from '@/lib/auth';
4
+ import { logger } from '@/lib/logger';
5
+
6
+ /**
7
+ * GET /api/workload - Real-Time Workload Signals
8
+ *
9
+ * Provides system-wide capacity metrics and throttle recommendations
10
+ * so agents can make informed decisions about work submission.
11
+ *
12
+ * Response:
13
+ * capacity - Current system capacity metrics
14
+ * queue - Task queue depth and breakdown
15
+ * agents - Agent availability and load distribution
16
+ * recommendation - Actionable signal: normal | throttle | shed | pause
17
+ * thresholds - Current threshold configuration
18
+ *
19
+ * Agents should call this before submitting new work to avoid
20
+ * cascading failures and SLO breaches.
21
+ */
22
+ export async function GET(request: NextRequest) {
23
+ const auth = requireRole(request, 'viewer');
24
+ if ('error' in auth) return NextResponse.json({ error: auth.error }, { status: auth.status });
25
+
26
+ try {
27
+ const db = getDatabase();
28
+ const workspaceId = auth.user.workspace_id ?? 1;
29
+ const now = Math.floor(Date.now() / 1000);
30
+
31
+ // --- Capacity metrics ---
32
+ const capacity = buildCapacityMetrics(db, workspaceId, now);
33
+
34
+ // --- Queue depth ---
35
+ const queue = buildQueueMetrics(db, workspaceId);
36
+
37
+ // --- Agent availability ---
38
+ const agents = buildAgentMetrics(db, workspaceId, now);
39
+
40
+ // --- Recommendation ---
41
+ const recommendation = computeRecommendation(capacity, queue, agents);
42
+
43
+ return NextResponse.json({
44
+ timestamp: now,
45
+ workspace_id: workspaceId,
46
+ capacity,
47
+ queue,
48
+ agents,
49
+ recommendation,
50
+ thresholds: THRESHOLDS,
51
+ });
52
+ } catch (error) {
53
+ logger.error({ err: error }, 'GET /api/workload error');
54
+ return NextResponse.json({ error: 'Failed to fetch workload signals' }, { status: 500 });
55
+ }
56
+ }
57
+
58
+ // Configurable thresholds for recommendation engine
59
+ const THRESHOLDS = {
60
+ queue_depth_normal: 20,
61
+ queue_depth_throttle: 50,
62
+ queue_depth_shed: 100,
63
+ busy_agent_ratio_throttle: 0.8,
64
+ busy_agent_ratio_shed: 0.95,
65
+ error_rate_throttle: 0.1,
66
+ error_rate_shed: 0.25,
67
+ recent_window_seconds: 300, // 5 minutes for recent activity
68
+ };
69
+
70
+ interface CapacityMetrics {
71
+ active_tasks: number;
72
+ tasks_last_5m: number;
73
+ errors_last_5m: number;
74
+ error_rate_5m: number;
75
+ completions_last_hour: number;
76
+ avg_completion_rate_per_hour: number;
77
+ }
78
+
79
+ interface QueueMetrics {
80
+ total_pending: number;
81
+ by_status: Record<string, number>;
82
+ by_priority: Record<string, number>;
83
+ oldest_pending_age_seconds: number | null;
84
+ estimated_wait_seconds: number | null;
85
+ }
86
+
87
+ interface AgentMetrics {
88
+ total: number;
89
+ online: number;
90
+ busy: number;
91
+ idle: number;
92
+ offline: number;
93
+ busy_ratio: number;
94
+ load_distribution: Array<{ agent: string; assigned: number; in_progress: number }>;
95
+ }
96
+
97
+ function buildCapacityMetrics(db: any, workspaceId: number, now: number): CapacityMetrics {
98
+ const recentWindow = now - THRESHOLDS.recent_window_seconds;
99
+ const hourAgo = now - 3600;
100
+
101
+ const activeTasks = (db.prepare(
102
+ `SELECT COUNT(*) as c FROM tasks WHERE workspace_id = ? AND status IN ('assigned', 'in_progress', 'review', 'quality_review')`
103
+ ).get(workspaceId) as any).c;
104
+
105
+ const tasksLast5m = (db.prepare(
106
+ `SELECT COUNT(*) as c FROM activities WHERE workspace_id = ? AND created_at >= ? AND type IN ('task_created', 'task_assigned')`
107
+ ).get(workspaceId, recentWindow) as any).c;
108
+
109
+ const errorsLast5m = (db.prepare(
110
+ `SELECT COUNT(*) as c FROM activities WHERE workspace_id = ? AND created_at >= ? AND (type LIKE '%error%' OR type LIKE '%fail%')`
111
+ ).get(workspaceId, recentWindow) as any).c;
112
+
113
+ const totalLast5m = (db.prepare(
114
+ `SELECT COUNT(*) as c FROM activities WHERE workspace_id = ? AND created_at >= ?`
115
+ ).get(workspaceId, recentWindow) as any).c;
116
+
117
+ const completionsLastHour = (db.prepare(
118
+ `SELECT COUNT(*) as c FROM tasks WHERE workspace_id = ? AND status = 'done' AND updated_at >= ?`
119
+ ).get(workspaceId, hourAgo) as any).c;
120
+
121
+ // Average completion rate over last 24h
122
+ const dayAgo = now - 86400;
123
+ const completionsLastDay = (db.prepare(
124
+ `SELECT COUNT(*) as c FROM tasks WHERE workspace_id = ? AND status = 'done' AND updated_at >= ?`
125
+ ).get(workspaceId, dayAgo) as any).c;
126
+
127
+ return {
128
+ active_tasks: activeTasks,
129
+ tasks_last_5m: tasksLast5m,
130
+ errors_last_5m: errorsLast5m,
131
+ error_rate_5m: totalLast5m > 0 ? Math.round((errorsLast5m / totalLast5m) * 10000) / 10000 : 0,
132
+ completions_last_hour: completionsLastHour,
133
+ avg_completion_rate_per_hour: Math.round((completionsLastDay / 24) * 100) / 100,
134
+ };
135
+ }
136
+
137
+ function buildQueueMetrics(db: any, workspaceId: number): QueueMetrics {
138
+ const now = Math.floor(Date.now() / 1000);
139
+
140
+ const pendingStatuses = ['inbox', 'assigned', 'in_progress', 'review', 'quality_review'];
141
+
142
+ const byStatus = db.prepare(
143
+ `SELECT status, COUNT(*) as count FROM tasks WHERE workspace_id = ? AND status IN (${pendingStatuses.map(() => '?').join(',')}) GROUP BY status`
144
+ ).all(workspaceId, ...pendingStatuses) as Array<{ status: string; count: number }>;
145
+
146
+ const byPriority = db.prepare(
147
+ `SELECT priority, COUNT(*) as count FROM tasks WHERE workspace_id = ? AND status IN (${pendingStatuses.map(() => '?').join(',')}) GROUP BY priority`
148
+ ).all(workspaceId, ...pendingStatuses) as Array<{ priority: string; count: number }>;
149
+
150
+ const totalPending = byStatus.reduce((sum, r) => sum + r.count, 0);
151
+
152
+ const oldest = db.prepare(
153
+ `SELECT MIN(created_at) as oldest FROM tasks WHERE workspace_id = ? AND status IN ('inbox', 'assigned')`
154
+ ).get(workspaceId) as any;
155
+
156
+ const oldestAge = oldest?.oldest ? now - oldest.oldest : null;
157
+
158
+ // Estimate wait: pending tasks / completion rate per hour * 3600
159
+ const hourAgo = now - 3600;
160
+ const completionsLastHour = (db.prepare(
161
+ `SELECT COUNT(*) as c FROM tasks WHERE workspace_id = ? AND status = 'done' AND updated_at >= ?`
162
+ ).get(workspaceId, hourAgo) as any).c;
163
+
164
+ const estimatedWait = completionsLastHour > 0
165
+ ? Math.round((totalPending / completionsLastHour) * 3600)
166
+ : null;
167
+
168
+ return {
169
+ total_pending: totalPending,
170
+ by_status: Object.fromEntries(byStatus.map(r => [r.status, r.count])),
171
+ by_priority: Object.fromEntries(byPriority.map(r => [r.priority, r.count])),
172
+ oldest_pending_age_seconds: oldestAge,
173
+ estimated_wait_seconds: estimatedWait,
174
+ };
175
+ }
176
+
177
+ function buildAgentMetrics(db: any, workspaceId: number, now: number): AgentMetrics {
178
+ const agentStatuses = db.prepare(
179
+ `SELECT status, COUNT(*) as count FROM agents WHERE workspace_id = ? GROUP BY status`
180
+ ).all(workspaceId) as Array<{ status: string; count: number }>;
181
+
182
+ const statusMap: Record<string, number> = {};
183
+ let total = 0;
184
+ for (const row of agentStatuses) {
185
+ statusMap[row.status] = row.count;
186
+ total += row.count;
187
+ }
188
+
189
+ const online = (statusMap['idle'] || 0) + (statusMap['busy'] || 0);
190
+ const busy = statusMap['busy'] || 0;
191
+ const idle = statusMap['idle'] || 0;
192
+ const offline = statusMap['offline'] || 0;
193
+
194
+ // Load distribution per agent
195
+ const loadDist = db.prepare(`
196
+ SELECT a.name as agent,
197
+ SUM(CASE WHEN t.status = 'assigned' THEN 1 ELSE 0 END) as assigned,
198
+ SUM(CASE WHEN t.status = 'in_progress' THEN 1 ELSE 0 END) as in_progress
199
+ FROM agents a
200
+ LEFT JOIN tasks t ON t.assigned_to = a.name AND t.workspace_id = a.workspace_id AND t.status IN ('assigned', 'in_progress')
201
+ WHERE a.workspace_id = ? AND a.status != 'offline'
202
+ GROUP BY a.name
203
+ ORDER BY (assigned + in_progress) DESC
204
+ `).all(workspaceId) as Array<{ agent: string; assigned: number; in_progress: number }>;
205
+
206
+ return {
207
+ total,
208
+ online,
209
+ busy,
210
+ idle,
211
+ offline,
212
+ busy_ratio: online > 0 ? Math.round((busy / online) * 100) / 100 : 0,
213
+ load_distribution: loadDist,
214
+ };
215
+ }
216
+
217
+ type RecommendationLevel = 'normal' | 'throttle' | 'shed' | 'pause';
218
+
219
+ interface Recommendation {
220
+ action: RecommendationLevel;
221
+ reason: string;
222
+ details: string[];
223
+ submit_ok: boolean;
224
+ suggested_delay_ms: number;
225
+ }
226
+
227
+ function computeRecommendation(
228
+ capacity: CapacityMetrics,
229
+ queue: QueueMetrics,
230
+ agents: AgentMetrics
231
+ ): Recommendation {
232
+ const reasons: string[] = [];
233
+ let level: RecommendationLevel = 'normal';
234
+
235
+ // Check error rate
236
+ if (capacity.error_rate_5m >= THRESHOLDS.error_rate_shed) {
237
+ level = escalate(level, 'shed');
238
+ reasons.push(`High error rate: ${(capacity.error_rate_5m * 100).toFixed(1)}%`);
239
+ } else if (capacity.error_rate_5m >= THRESHOLDS.error_rate_throttle) {
240
+ level = escalate(level, 'throttle');
241
+ reasons.push(`Elevated error rate: ${(capacity.error_rate_5m * 100).toFixed(1)}%`);
242
+ }
243
+
244
+ // Check queue depth
245
+ if (queue.total_pending >= THRESHOLDS.queue_depth_shed) {
246
+ level = escalate(level, 'shed');
247
+ reasons.push(`Queue depth critical: ${queue.total_pending} pending tasks`);
248
+ } else if (queue.total_pending >= THRESHOLDS.queue_depth_throttle) {
249
+ level = escalate(level, 'throttle');
250
+ reasons.push(`Queue depth high: ${queue.total_pending} pending tasks`);
251
+ }
252
+
253
+ // Check agent saturation
254
+ if (agents.busy_ratio >= THRESHOLDS.busy_agent_ratio_shed) {
255
+ level = escalate(level, 'shed');
256
+ reasons.push(`Agent saturation critical: ${(agents.busy_ratio * 100).toFixed(0)}% busy`);
257
+ } else if (agents.busy_ratio >= THRESHOLDS.busy_agent_ratio_throttle) {
258
+ level = escalate(level, 'throttle');
259
+ reasons.push(`Agent saturation high: ${(agents.busy_ratio * 100).toFixed(0)}% busy`);
260
+ }
261
+
262
+ // No online agents = pause
263
+ if (agents.online === 0 && agents.total > 0) {
264
+ level = 'pause';
265
+ reasons.push('No agents online');
266
+ }
267
+
268
+ const delayMap: Record<RecommendationLevel, number> = {
269
+ normal: 0,
270
+ throttle: 2000,
271
+ shed: 10000,
272
+ pause: 30000,
273
+ };
274
+
275
+ const actionDescriptions: Record<RecommendationLevel, string> = {
276
+ normal: 'System healthy — submit work freely',
277
+ throttle: 'System under load — reduce submission rate and defer non-critical work',
278
+ shed: 'System overloaded — submit only critical/high-priority work, defer everything else',
279
+ pause: 'System unavailable — hold all submissions until capacity returns',
280
+ };
281
+
282
+ return {
283
+ action: level,
284
+ reason: actionDescriptions[level],
285
+ details: reasons.length > 0 ? reasons : ['All metrics within normal bounds'],
286
+ submit_ok: level === 'normal' || level === 'throttle',
287
+ suggested_delay_ms: delayMap[level],
288
+ };
289
+ }
290
+
291
+ function escalate(current: RecommendationLevel, proposed: RecommendationLevel): RecommendationLevel {
292
+ const order: RecommendationLevel[] = ['normal', 'throttle', 'shed', 'pause'];
293
+ return order.indexOf(proposed) > order.indexOf(current) ? proposed : current;
294
+ }