fsanyoto commited on
Commit
ea2cfaf
·
verified ·
1 Parent(s): 2369935

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "e9719153a",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "ee0d655cf",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- e9719153a
 
1
+ ee0d655cf
api/usage_activity.py CHANGED
@@ -2,9 +2,6 @@
2
 
3
  from calendar import monthrange
4
  from collections import Counter
5
- from concurrent.futures import Future
6
- import contextvars
7
- from functools import partial
8
  from datetime import date, datetime, time, timedelta, timezone
9
  import base64
10
  import binascii
@@ -25,12 +22,6 @@ SOURCE_KINDS = {
25
  "turn_audit": ("ai_turn",),
26
  }
27
  PAGE = 200
28
- # A first prefix this large ends most reads in one round trip. A full prefix still doubles.
29
- FIRST_PREFIX = 1 << 20
30
- # Sources and their per-table reads run concurrently. Five of the eight pooled connections, at most.
31
- READ_WORKERS = 5
32
- # The threaded read hung on the staging Space (no response in 230 s); the request thread reads serially.
33
- CONCURRENT_READS = False
34
  UTC = timezone.utc
35
 
36
 
@@ -69,9 +60,9 @@ def date_range(from_=None, to=None, *, now=None):
69
  return start, end, now
70
 
71
 
72
- def _unlimited_prefix(reader, first=FIRST_PREFIX):
73
  """Expand readers without a cursor until a short result proves exhaustion."""
74
- limit = first
75
  while True:
76
  rows = reader(limit)
77
  if len(rows) < limit:
@@ -124,8 +115,7 @@ def _activity(session, start, stop, now, source):
124
  with activity_log.read_scope(session.runtime):
125
  while True:
126
  page = activity_log.read(session.runtime, can_read_object=lambda obj: True,
127
- since=start, until=min(stop, now), cursor=cursor,
128
- limit=activity_log.MAX_PAGE)
129
  for row in page["records"]:
130
  if _tenant_row(row, session.tenant) and row["kind"] in {"record", "field"}:
131
  yield _timestamp(row["occurred_at"]), row.get("by") or "", row["kind"] + "_edit"
@@ -156,82 +146,41 @@ def _history_tables(rt, now):
156
  return sorted(tables)
157
 
158
 
159
- def _cell_table(session, table, source):
160
- history = user_tables.value_history(table, st=session.runtime,
161
- retention_days=history_rows.RETENTION_DAYS, extended_entitled=True)
162
- for row in history["events"]:
163
- if _tenant_row(row, session.tenant) and row.get("kind") in {"edit", "restore"}:
164
- for _ in row.get("changes", {}):
165
- yield _timestamp(row["at"]), row.get("actor") or "", "cell_edit"
166
-
167
-
168
- def _cells_units(session, start, stop, now, source):
169
- """Return one independent read per history table, in table order."""
170
  source["retention"] = {"days": history_rows.RETENTION_DAYS}
171
- return [partial(_cell_table, session, table) for table in _history_tables(session.runtime, now)
172
- if table != activity_log.DATABASE_KEY]
 
 
 
 
 
 
 
173
 
174
 
175
- def _cells(session, start, stop, now, source):
176
- for unit in _cells_units(session, start, stop, now, source):
177
- yield from unit(source)
178
-
179
-
180
- def _view_table(session, start, table, source):
181
- home = view_audit.home(session.runtime, table)
182
- data = None
183
- if home == view_audit.DOCUMENT:
184
- # Read outside rows(), whose legacy document fallback swallows store failures.
185
- data = session.runtime.get(table) or {}
186
- ring = data.get(view_audit.AUDIT_KEY) or []
187
- if len(ring) >= view_audit.RING_CAP:
188
- held = min(_timestamp(row["ts"]) for row in ring).isoformat()
189
- source["heldFrom"] = max(source["heldFrom"], held)
190
- source["limitations"].append({"code": "view_ring", "cause":
191
- "A view document retains only its newest 2000 audit rows."})
192
- rows = _unlimited_prefix(lambda limit: view_audit.rows(session.runtime,
193
- table_key=table, since=math.nextafter(start.timestamp(), -math.inf), limit=limit, data=data))
194
- for row in rows:
195
- if _tenant_row(row, session.tenant):
196
- yield _timestamp(row["ts"]), row.get("actor") or "", "view_edit"
197
-
198
-
199
- def _prefetch_view_layouts(rt, tables):
200
- """Seed this request's layout memo from one marker query instead of one query per workspace."""
201
- from core import view_settings_rows
202
- if callable(getattr(rt, "view_settings_rows", None)):
203
- return
204
- names = {}
205
- for table in tables:
206
- bound = view_settings_rows.bind(rt, table)
207
- if isinstance(bound, view_settings_rows.PgRows):
208
- names.setdefault(bound.tenant_slug, []).append(bound.name)
209
- if not names:
210
- return
211
- from core import store_pg
212
- layouts = store_pg._layout_memo()
213
- if layouts is None:
214
- return
215
- for slug, keys in names.items():
216
- migrated = store_pg._view_rows_many(keys, slug)
217
- for key in keys:
218
- # Same key and value _view_rows memoises. An entry this request already holds wins.
219
- layouts.setdefault((store_pg._effective_slug(slug), str(key)), key in migrated)
220
-
221
-
222
- def _views_units(session, start, stop, now, source):
223
- """Return one independent read per view workspace, in key order."""
224
  source["retention"] = {"days": None, "documentRowsPerTable": view_audit.RING_CAP}
225
  from core.view_templates import _WS_KEYS
226
- keys = sorted(set(_WS_KEYS.values()) | {key for key in session.runtime.list_keys("ut_")
227
- if key.endswith("_table_workspace")})
228
- _prefetch_view_layouts(session.runtime, keys)
229
- return [partial(_view_table, session, start, table) for table in keys]
230
-
231
-
232
- def _views(session, start, stop, now, source):
233
- for unit in _views_units(session, start, stop, now, source):
234
- yield from unit(source)
 
 
 
 
 
 
 
 
 
 
235
 
236
 
237
  def _turns(session, start, stop, now, source):
@@ -270,135 +219,6 @@ def _turns(session, start, stop, now, source):
270
 
271
  READERS = {"security_events": _security, "activity_events": _activity,
272
  "value_history": _cells, "view_audit": _views, "turn_audit": _turns}
273
- # A source listed here splits into per-table reads that share the request's worker pool.
274
- UNITS = {"value_history": _cells_units, "view_audit": _views_units}
275
-
276
-
277
- def _count(unit, source, start, stop, now, user, kinds):
278
- """Count one read's in-range events, returning the counter or the exception that stopped it."""
279
- from core import request_timing
280
- local = Counter()
281
- try:
282
- with request_timing.thread_scope():
283
- for at, actor, event_kind in unit(source):
284
- if (start <= at < stop and at <= now and event_kind in kinds
285
- and (user is None or actor == user)):
286
- local[(at.date().isoformat(), actor, event_kind)] += 1
287
- except Exception as exc:
288
- return None, exc
289
- return local, None
290
-
291
-
292
- def _plan(splitter, session, start, stop, now, source):
293
- try:
294
- return splitter(session, start, stop, now, source), None
295
- except Exception as exc:
296
- return None, exc
297
-
298
-
299
- def _enlisted():
300
- """Whether a caller transaction is bound, whose one connection must not cross threads."""
301
- import sys
302
- store_pg = sys.modules.get("core.store_pg")
303
- held = getattr(store_pg, "_CALLER_TRANSACTION", None)
304
- return held is not None and held.get() is not None
305
-
306
-
307
- class _Inline:
308
- """Run a submitted call at once, for a caller whose transaction pins one thread."""
309
-
310
- def submit(self, fn, *args):
311
- done = Future()
312
- done.set_result(fn(*args))
313
- return done
314
-
315
-
316
- class _Workers:
317
- """A bounded request-scoped pool whose threads start through process_role."""
318
-
319
- def __init__(self, size):
320
- import queue
321
- import process_role
322
- self.jobs = queue.SimpleQueue()
323
- self.threads = [process_role.thread(target=self._loop, name="usage-activity-read", kind="light")
324
- for _ in range(size)]
325
- if not all(process_role.admitted(job) for job in self.threads):
326
- raise RuntimeError("Usage activity workers were refused.")
327
- for job in self.threads:
328
- job.start()
329
-
330
- def _loop(self):
331
- while (item := self.jobs.get()) is not None:
332
- done, context, fn, args = item
333
- try:
334
- done.set_result(context.run(fn, *args))
335
- except BaseException as exc: # noqa: BLE001
336
- done.set_exception(exc)
337
-
338
- def submit(self, fn, *args):
339
- # A copied context carries the request deadline, its accumulator and any store binding.
340
- done = Future()
341
- self.jobs.put((done, contextvars.copy_context(), fn, args))
342
- return done
343
-
344
- def close(self):
345
- for _ in self.threads:
346
- self.jobs.put(None)
347
- for job in self.threads:
348
- job.join()
349
-
350
-
351
- def _collect_all(session, start, stop, now, user, jobs):
352
- """Read every source and its per-table units concurrently, then merge them in serial order.
353
-
354
- Each call runs in a copy of the caller's context, so the request deadline, the accumulator
355
- and any store binding carry. A unit writes limitations to its own scratch source, and the merge
356
- replays them in table order, stopping at the first failed unit as a serial read would.
357
- """
358
- def run(pool):
359
- units, plans = {}, {}
360
- for index, (name, reader, source, kinds) in enumerate(jobs):
361
- if name in UNITS:
362
- plans[pool.submit(_plan, UNITS[name], session, start, stop, now, source)] = index
363
- else:
364
- whole = partial(reader, session, start, stop, now)
365
- units[index] = [(source, pool.submit(_count, whole, source, start, stop, now, user, kinds))]
366
- # Source order, so the slower cell units queue ahead of the view units.
367
- for planned in list(plans):
368
- index = plans[planned]
369
- source, kinds = jobs[index][2], jobs[index][3]
370
- parts, exc = planned.result()
371
- if exc is not None:
372
- units[index] = exc
373
- continue
374
- units[index] = []
375
- for part in parts:
376
- scratch = {"heldFrom": source["heldFrom"], "limitations": []}
377
- units[index].append((scratch, pool.submit(_count, part, scratch, start, stop, now, user, kinds)))
378
- results = []
379
- for index, (_, _, source, _) in enumerate(jobs):
380
- if isinstance(units[index], Exception):
381
- results.append((None, units[index]))
382
- continue
383
- local, failure = Counter(), None
384
- for scratch, future in units[index]:
385
- counted, failure = future.result()
386
- if scratch is not source:
387
- source["limitations"].extend(scratch["limitations"])
388
- source["heldFrom"] = max(source["heldFrom"], scratch["heldFrom"])
389
- if failure is not None:
390
- break
391
- local.update(counted)
392
- results.append((None, failure) if failure is not None else (local, None))
393
- return results
394
-
395
- if not CONCURRENT_READS or _enlisted():
396
- return run(_Inline())
397
- pool = _Workers(READ_WORKERS)
398
- try:
399
- return run(pool)
400
- finally:
401
- pool.close()
402
 
403
 
404
  def _source_error(exc):
@@ -592,7 +412,6 @@ def read(session, *, from_=None, to=None, user=None, kind=None, now=None):
592
  "view_audit": start, "turn_audit": start}
593
  sources, counts = [], Counter()
594
  totals = {key: 0 for key in selected}
595
- jobs = []
596
  for name, reader in READERS.items():
597
  kinds = [key for key in SOURCE_KINDS[name] if key in selected]
598
  if not kinds:
@@ -602,38 +421,33 @@ def read(session, *, from_=None, to=None, user=None, kind=None, now=None):
602
  if floors[name] > start:
603
  source["limitations"].append({"code": "retention", "cause":
604
  "This source no longer holds the beginning of the requested range."})
605
- jobs.append((name, reader, source, kinds))
606
- for (_, _, source, kinds), (local, exc) in zip(jobs, _collect_all(
607
- session, start, stop, now, user, jobs)):
608
- if exc is not None:
 
 
 
609
  source.update(status="error", error=_source_error(exc))
610
  for key in kinds:
611
  totals[key] = None
612
  else:
613
  source["total"] = sum(local.values())
614
- by_kind = Counter()
615
- for (_, _, key), n in local.items():
616
- by_kind[key] += n
617
- source["byKind"] = {key: by_kind[key] for key in kinds}
618
  source["status"] = "partial" if source["limitations"] else "ok"
619
  counts.update(local)
620
  totals.update(source["byKind"])
621
  sources.append(source)
622
  complete = all(source["status"] == "ok" for source in sources)
623
- # One pass over the cube. A per-day or per-user rescan is quadratic on a 13-month range.
624
- user_kind, day_kind = Counter(), Counter()
625
- for (day_iso, actor, key), n in counts.items():
626
- user_kind[(actor, key)] += n
627
- day_kind[(day_iso, key)] += n
628
- users = sorted({actor for actor, _ in user_kind})
629
  by_user = [{"user": actor, "byKind": {
630
- key: None if totals[key] is None else user_kind[(actor, key)]
631
  for key in selected}} for actor in users]
632
  by_day = []
633
  day = first
634
  while day <= last:
635
- values = {key: None if totals[key] is None else day_kind[(day.isoformat(), key)]
636
- for key in selected}
637
  for source in sources:
638
  if day < _timestamp(source["heldFrom"]).date():
639
  for key in source["kinds"]:
 
2
 
3
  from calendar import monthrange
4
  from collections import Counter
 
 
 
5
  from datetime import date, datetime, time, timedelta, timezone
6
  import base64
7
  import binascii
 
22
  "turn_audit": ("ai_turn",),
23
  }
24
  PAGE = 200
 
 
 
 
 
 
25
  UTC = timezone.utc
26
 
27
 
 
60
  return start, end, now
61
 
62
 
63
+ def _unlimited_prefix(reader):
64
  """Expand readers without a cursor until a short result proves exhaustion."""
65
+ limit = PAGE
66
  while True:
67
  rows = reader(limit)
68
  if len(rows) < limit:
 
115
  with activity_log.read_scope(session.runtime):
116
  while True:
117
  page = activity_log.read(session.runtime, can_read_object=lambda obj: True,
118
+ since=start, until=min(stop, now), cursor=cursor, limit=PAGE)
 
119
  for row in page["records"]:
120
  if _tenant_row(row, session.tenant) and row["kind"] in {"record", "field"}:
121
  yield _timestamp(row["occurred_at"]), row.get("by") or "", row["kind"] + "_edit"
 
146
  return sorted(tables)
147
 
148
 
149
+ def _cells(session, start, stop, now, source):
 
 
 
 
 
 
 
 
 
 
150
  source["retention"] = {"days": history_rows.RETENTION_DAYS}
151
+ for table in _history_tables(session.runtime, now):
152
+ if table == activity_log.DATABASE_KEY:
153
+ continue
154
+ history = user_tables.value_history(table, st=session.runtime,
155
+ retention_days=history_rows.RETENTION_DAYS, extended_entitled=True)
156
+ for row in history["events"]:
157
+ if _tenant_row(row, session.tenant) and row.get("kind") in {"edit", "restore"}:
158
+ for _ in row.get("changes", {}):
159
+ yield _timestamp(row["at"]), row.get("actor") or "", "cell_edit"
160
 
161
 
162
+ def _views(session, start, stop, now, source):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
163
  source["retention"] = {"days": None, "documentRowsPerTable": view_audit.RING_CAP}
164
  from core.view_templates import _WS_KEYS
165
+ keys = set(_WS_KEYS.values()) | {key for key in session.runtime.list_keys("ut_")
166
+ if key.endswith("_table_workspace")}
167
+ for table in sorted(keys):
168
+ home = view_audit.home(session.runtime, table)
169
+ data = None
170
+ if home == view_audit.DOCUMENT:
171
+ # Read outside rows(), whose legacy document fallback swallows store failures.
172
+ data = session.runtime.get(table) or {}
173
+ ring = data.get(view_audit.AUDIT_KEY) or []
174
+ if len(ring) >= view_audit.RING_CAP:
175
+ held = min(_timestamp(row["ts"]) for row in ring).isoformat()
176
+ source["heldFrom"] = max(source["heldFrom"], held)
177
+ source["limitations"].append({"code": "view_ring", "cause":
178
+ "A view document retains only its newest 2000 audit rows."})
179
+ rows = _unlimited_prefix(lambda limit: view_audit.rows(session.runtime,
180
+ table_key=table, since=math.nextafter(start.timestamp(), -math.inf), limit=limit, data=data))
181
+ for row in rows:
182
+ if _tenant_row(row, session.tenant):
183
+ yield _timestamp(row["ts"]), row.get("actor") or "", "view_edit"
184
 
185
 
186
  def _turns(session, start, stop, now, source):
 
219
 
220
  READERS = {"security_events": _security, "activity_events": _activity,
221
  "value_history": _cells, "view_audit": _views, "turn_audit": _turns}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
222
 
223
 
224
  def _source_error(exc):
 
412
  "view_audit": start, "turn_audit": start}
413
  sources, counts = [], Counter()
414
  totals = {key: 0 for key in selected}
 
415
  for name, reader in READERS.items():
416
  kinds = [key for key in SOURCE_KINDS[name] if key in selected]
417
  if not kinds:
 
421
  if floors[name] > start:
422
  source["limitations"].append({"code": "retention", "cause":
423
  "This source no longer holds the beginning of the requested range."})
424
+ local = Counter()
425
+ try:
426
+ for at, actor, event_kind in reader(session, start, stop, now, source):
427
+ if (start <= at < stop and at <= now and event_kind in kinds
428
+ and (user is None or actor == user)):
429
+ local[(at.date().isoformat(), actor, event_kind)] += 1
430
+ except Exception as exc:
431
  source.update(status="error", error=_source_error(exc))
432
  for key in kinds:
433
  totals[key] = None
434
  else:
435
  source["total"] = sum(local.values())
436
+ source["byKind"] = {key: sum(n for (_, _, k), n in local.items() if k == key) for key in kinds}
 
 
 
437
  source["status"] = "partial" if source["limitations"] else "ok"
438
  counts.update(local)
439
  totals.update(source["byKind"])
440
  sources.append(source)
441
  complete = all(source["status"] == "ok" for source in sources)
442
+ users = sorted({actor for _, actor, _ in counts})
 
 
 
 
 
443
  by_user = [{"user": actor, "byKind": {
444
+ key: None if totals[key] is None else sum(n for (_, a, k), n in counts.items() if a == actor and k == key)
445
  for key in selected}} for actor in users]
446
  by_day = []
447
  day = first
448
  while day <= last:
449
+ values = {key: None if totals[key] is None else sum(
450
+ n for (d, _, k), n in counts.items() if d == day.isoformat() and k == key) for key in selected}
451
  for source in sources:
452
  if day < _timestamp(source["heldFrom"]).date():
453
  for key in source["kinds"]:
web/public/AIOS_DEPLOY_RECEIPT CHANGED
@@ -1 +1 @@
1
- e24ae1f591624feeaabc31247172674d4edf31c580859afa74a11832d74760d9
 
1
+ 428d0323b94de1ae0403b46ecabc028cbf42c06b72d796886310199c77689c96