Spaces:
Running
Running
Commit ·
3741d5c
1
Parent(s): b3eac77
Detailed result view: readiness, family overview, grouped findings, episode explorer, JSON download
Browse files- .DS_Store +0 -0
- .gitignore +1 -0
- README.md +27 -0
- __pycache__/app.cpython-314.pyc +0 -0
- __pycache__/cards.cpython-314.pyc +0 -0
- __pycache__/quality_card.cpython-314.pyc +0 -0
- app.py +18 -4
- cards.py +31 -7
- publisher_grader/presentation/__pycache__/__init__.cpython-314.pyc +0 -0
- publisher_grader/presentation/__pycache__/action_model.cpython-314.pyc +0 -0
- publisher_grader/presentation/__pycache__/report_model.cpython-314.pyc +0 -0
- publisher_grader/presentation/__pycache__/video_model.cpython-314.pyc +0 -0
- quality_card.py +163 -0
- tests/__pycache__/test_app.cpython-314.pyc +0 -0
- tests/__pycache__/test_quality_card.cpython-314.pyc +0 -0
- tests/test_quality_card.py +101 -0
- tests/test_quick_grade.py +5 -0
.DS_Store
ADDED
|
Binary file (6.15 kB). View file
|
|
|
.gitignore
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
.venv/
|
README.md
CHANGED
|
@@ -48,3 +48,30 @@ python app.py
|
|
| 48 |
```
|
| 49 |
|
| 50 |
The numeric integration test uses a tiny local LeRobot recording with a declared camera and no media file. No production report is submitted by the tests.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
```
|
| 49 |
|
| 50 |
The numeric integration test uses a tiny local LeRobot recording with a declared camera and no media file. No production report is submitted by the tests.
|
| 51 |
+
|
| 52 |
+
## Robotics Dataset Quality result
|
| 53 |
+
|
| 54 |
+
The selected result expands into Training Readiness, Quality overview, Fix these
|
| 55 |
+
first, Dataset overview, Episode explorer, and Technical details. Featured cards
|
| 56 |
+
remain compact. The Space keeps using the bounded numeric worker and vendored
|
| 57 |
+
converter; no grading policy, sidecar, video scope or publishing behavior changes.
|
| 58 |
+
|
| 59 |
+
- Training Readiness displays the engine's readiness value (including undefined
|
| 60 |
+
values and their recorded reasons). Pass/review/blocked/unknown counts remain
|
| 61 |
+
the engine's decisions. Known blockers stay visible even with unknown episodes.
|
| 62 |
+
- Family rows summarize computed results, unavailable checks and recorded finding
|
| 63 |
+
consequences. They are not 0–100 scores. Inventory completeness is shown for
|
| 64 |
+
structure; it is not a schema quality score. Diversity is not inferred from
|
| 65 |
+
assessment coverage, and absent published details are labeled “not provided.”
|
| 66 |
+
- Issues are grouped by metric, source address, unit and consequence. Affected
|
| 67 |
+
episode counts are deduplicated within each group; warning counts count finding
|
| 68 |
+
records and may overlap blockers. Severity alone never creates a blocker.
|
| 69 |
+
- Evidence and episode links expose recorded values, source coordinates and support.
|
| 70 |
+
Recorded skew never becomes a physical camera/action latency claim. Review
|
| 71 |
+
guidance is explicitly UI guidance, not an engine repair or causal diagnosis.
|
| 72 |
+
- Download assessment JSON exports the current converted card (or stored summary
|
| 73 |
+
payload for a published lookup). This is not the complete raw CLI report:
|
| 74 |
+
metric examples and support intervals can be compacted by the existing converter.
|
| 75 |
+
Missing raw plots/thresholds require the full CLI report. Downloads expire with
|
| 76 |
+
Gradio's cache cleanup. Legacy published summaries retain their full-report link
|
| 77 |
+
and explicitly show when episode/diagnostic details are absent.
|
__pycache__/app.cpython-314.pyc
CHANGED
|
Binary files a/__pycache__/app.cpython-314.pyc and b/__pycache__/app.cpython-314.pyc differ
|
|
|
__pycache__/cards.cpython-314.pyc
CHANGED
|
Binary files a/__pycache__/cards.cpython-314.pyc and b/__pycache__/cards.cpython-314.pyc differ
|
|
|
__pycache__/quality_card.cpython-314.pyc
ADDED
|
Binary file (19.6 kB). View file
|
|
|
app.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
| 1 |
"""Kalanos HF Space: bounded numeric audits and published assessment examples."""
|
|
|
|
| 2 |
import logging
|
|
|
|
| 3 |
import os
|
| 4 |
from concurrent.futures import ThreadPoolExecutor
|
| 5 |
from html import escape
|
|
@@ -90,7 +92,7 @@ def run(ref):
|
|
| 90 |
# Always run fresh. Existing published reports never short-circuit grading.
|
| 91 |
card = quick_grade(ref)
|
| 92 |
payload = card_payload(card, KALANOS_API)
|
| 93 |
-
return _result(
|
| 94 |
except ValueError as exc:
|
| 95 |
return _result('', '', _err(str(exc)), None)
|
| 96 |
except Exception:
|
|
@@ -107,19 +109,29 @@ def lookup(ref):
|
|
| 107 |
return _result('', '', _err(str(exc)), None)
|
| 108 |
payload = published(ref)
|
| 109 |
if payload:
|
| 110 |
-
return _result(
|
| 111 |
if dataset_exists(ref) is False:
|
| 112 |
return _result('', '', _err(NOT_FOUND.format(ref=ref)), None)
|
| 113 |
return _result('', '', _err(f'No published report for {ref} on Kalanos Hub yet. Grade dataset runs a fresh quick audit.'), None)
|
| 114 |
|
| 115 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
# Publishing from the Space is intentionally disabled. The publisher upserts
|
| 117 |
# reports by slug alone (hf/<org>/<name>), so a visitor's quick audit submitted
|
| 118 |
# with a personal CLI key would replace the registry's full report for that
|
| 119 |
# dataset. Re-enable once /api/reports scopes slug conflicts by tenant.
|
| 120 |
# Until then: copy the HF card Markdown, or publish from the CLI.
|
| 121 |
|
| 122 |
-
with gr.Blocks(title='Kalanos · Robotics Dataset Quality') as demo:
|
| 123 |
gr.HTML(header(KALANOS_API))
|
| 124 |
with gr.Column(elem_classes='k-grader'):
|
| 125 |
gr.HTML(grader_intro())
|
|
@@ -129,8 +141,10 @@ with gr.Blocks(title='Kalanos · Robotics Dataset Quality') as demo:
|
|
| 129 |
# Secondary action stays clear; report cards themselves use compact icons.
|
| 130 |
lookup_btn = gr.Button('View published report', scale=1, elem_classes='k-secondary')
|
| 131 |
status = gr.HTML('', elem_classes='k-status')
|
| 132 |
-
out = gr.HTML()
|
| 133 |
report_state = gr.State(None, time_to_live=600)
|
|
|
|
|
|
|
|
|
|
| 134 |
with gr.Accordion('HF card Markdown', open=False, elem_classes='k-accordion', visible=False) as card_acc:
|
| 135 |
card_text = gr.Code(language='markdown', label='Copy into your existing dataset README', interactive=False)
|
| 136 |
featured = gr.HTML(section('Featured dataset reports'))
|
|
|
|
| 1 |
"""Kalanos HF Space: bounded numeric audits and published assessment examples."""
|
| 2 |
+
import json
|
| 3 |
import logging
|
| 4 |
+
import tempfile
|
| 5 |
import os
|
| 6 |
from concurrent.futures import ThreadPoolExecutor
|
| 7 |
from html import escape
|
|
|
|
| 92 |
# Always run fresh. Existing published reports never short-circuit grading.
|
| 93 |
card = quick_grade(ref)
|
| 94 |
payload = card_payload(card, KALANOS_API)
|
| 95 |
+
return _result(render(payload, published=False, detailed=True), payload['markdown'], _ok('Quick audit complete. Video: not assessed.'), card)
|
| 96 |
except ValueError as exc:
|
| 97 |
return _result('', '', _err(str(exc)), None)
|
| 98 |
except Exception:
|
|
|
|
| 109 |
return _result('', '', _err(str(exc)), None)
|
| 110 |
payload = published(ref)
|
| 111 |
if payload:
|
| 112 |
+
return _result(render(payload, detailed=True), payload['markdown'], _ok('Stored assessment shown with its recorded date and scope. Grade dataset runs a fresh audit.'), payload)
|
| 113 |
if dataset_exists(ref) is False:
|
| 114 |
return _result('', '', _err(NOT_FOUND.format(ref=ref)), None)
|
| 115 |
return _result('', '', _err(f'No published report for {ref} on Kalanos Hub yet. Grade dataset runs a fresh quick audit.'), None)
|
| 116 |
|
| 117 |
|
| 118 |
+
def download_assessment(card):
|
| 119 |
+
"""Gradio serves this session's converted assessment and expires cached files."""
|
| 120 |
+
if card is None:
|
| 121 |
+
return gr.update(value=None, visible=False)
|
| 122 |
+
with tempfile.NamedTemporaryFile(mode='w', suffix='.json', prefix='kalanos-assessment-', delete=False) as file:
|
| 123 |
+
json.dump(card, file, indent=2, ensure_ascii=False)
|
| 124 |
+
download.temp_files.add(file.name)
|
| 125 |
+
return gr.update(value=file.name, visible=True)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
# Publishing from the Space is intentionally disabled. The publisher upserts
|
| 129 |
# reports by slug alone (hf/<org>/<name>), so a visitor's quick audit submitted
|
| 130 |
# with a personal CLI key would replace the registry's full report for that
|
| 131 |
# dataset. Re-enable once /api/reports scopes slug conflicts by tenant.
|
| 132 |
# Until then: copy the HF card Markdown, or publish from the CLI.
|
| 133 |
|
| 134 |
+
with gr.Blocks(title='Kalanos · Robotics Dataset Quality', delete_cache=(600, 600)) as demo:
|
| 135 |
gr.HTML(header(KALANOS_API))
|
| 136 |
with gr.Column(elem_classes='k-grader'):
|
| 137 |
gr.HTML(grader_intro())
|
|
|
|
| 141 |
# Secondary action stays clear; report cards themselves use compact icons.
|
| 142 |
lookup_btn = gr.Button('View published report', scale=1, elem_classes='k-secondary')
|
| 143 |
status = gr.HTML('', elem_classes='k-status')
|
|
|
|
| 144 |
report_state = gr.State(None, time_to_live=600)
|
| 145 |
+
download = gr.DownloadButton('Download assessment JSON (compact)', visible=False)
|
| 146 |
+
report_state.change(download_assessment, inputs=report_state, outputs=download)
|
| 147 |
+
out = gr.HTML()
|
| 148 |
with gr.Accordion('HF card Markdown', open=False, elem_classes='k-accordion', visible=False) as card_acc:
|
| 149 |
card_text = gr.Code(language='markdown', label='Copy into your existing dataset README', interactive=False)
|
| 150 |
featured = gr.HTML(section('Featured dataset reports'))
|
cards.py
CHANGED
|
@@ -161,14 +161,16 @@ def card_payload(card, base):
|
|
| 161 |
return dict(dataset=ref, revision=revision, assessed_at=card['graded_at'], report_id=card['report_id'],
|
| 162 |
requirements=d['scope']['requirements_id'], readiness=d['readiness']['score'],
|
| 163 |
counts=d['counts'], details=rows, markdown=markdown, notes=card.get('notes', []),
|
| 164 |
-
dataset_url='https://huggingface.co/datasets/' + quote(ref, safe='/'), report_url=None
|
|
|
|
|
|
|
| 165 |
|
| 166 |
|
| 167 |
def _status(c):
|
| 168 |
-
if not c.get('inventory_complete') or c.get('unknown') or not c.get('total'):
|
| 169 |
-
return 'incomplete', 'Incomplete evidence'
|
| 170 |
if c.get('blocked'):
|
| 171 |
return 'blocked', 'Blocking findings'
|
|
|
|
|
|
|
| 172 |
if c.get('review'):
|
| 173 |
return 'review', 'Review required'
|
| 174 |
return 'pass', 'Pass within scope'
|
|
@@ -205,7 +207,7 @@ def _assessed(ts):
|
|
| 205 |
return str(ts)
|
| 206 |
|
| 207 |
|
| 208 |
-
def render(payload, published=True):
|
| 209 |
c = payload['counts']
|
| 210 |
score = payload['readiness']
|
| 211 |
status, status_text = _status(c)
|
|
@@ -221,11 +223,33 @@ def render(payload, published=True):
|
|
| 221 |
stats = ''.join(f'<div><strong>{v}</strong><span>{label}</span></div>' for v, label in [
|
| 222 |
(c.get('total', '—'), 'Episodes assessed'), (f'{c.get("pass_count", 0)}/{c.get("total", 0)}', 'Eligible episodes'),
|
| 223 |
(c.get('review', 0), 'Requiring review'), (c.get('blocked', 0), 'Blocked')])
|
|
|
|
| 224 |
kind = 'Published report' if published else 'Unpublished quick audit · video not assessed'
|
| 225 |
-
|
| 226 |
f'<div class="k-card-id">{escape(payload["dataset"])}</div>'
|
| 227 |
-
f'<div class="k-score"><span class="k-eyebrow">
|
| 228 |
-
f'
|
| 229 |
f'<p class="k-note">{escape(_note(payload, status))}</p>'
|
| 230 |
f'<div class="k-meta">{escape(payload["requirements"])}<br>Assessed {escape(_assessed(payload["assessed_at"]))}<br>{kind}</div>'
|
| 231 |
f'<div class="k-actions">{actions}</div></article>')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
return dict(dataset=ref, revision=revision, assessed_at=card['graded_at'], report_id=card['report_id'],
|
| 162 |
requirements=d['scope']['requirements_id'], readiness=d['readiness']['score'],
|
| 163 |
counts=d['counts'], details=rows, markdown=markdown, notes=card.get('notes', []),
|
| 164 |
+
dataset_url='https://huggingface.co/datasets/' + quote(ref, safe='/'), report_url=None,
|
| 165 |
+
presentation=card.get('presentation'), dataset_metadata=card.get('dataset'),
|
| 166 |
+
readiness_detail=d['readiness'])
|
| 167 |
|
| 168 |
|
| 169 |
def _status(c):
|
|
|
|
|
|
|
| 170 |
if c.get('blocked'):
|
| 171 |
return 'blocked', 'Blocking findings'
|
| 172 |
+
if not c.get('inventory_complete') or c.get('unknown') or not c.get('total'):
|
| 173 |
+
return 'incomplete', 'Incomplete evidence'
|
| 174 |
if c.get('review'):
|
| 175 |
return 'review', 'Review required'
|
| 176 |
return 'pass', 'Pass within scope'
|
|
|
|
| 207 |
return str(ts)
|
| 208 |
|
| 209 |
|
| 210 |
+
def render(payload, published=True, detailed=False):
|
| 211 |
c = payload['counts']
|
| 212 |
score = payload['readiness']
|
| 213 |
status, status_text = _status(c)
|
|
|
|
| 223 |
stats = ''.join(f'<div><strong>{v}</strong><span>{label}</span></div>' for v, label in [
|
| 224 |
(c.get('total', '—'), 'Episodes assessed'), (f'{c.get("pass_count", 0)}/{c.get("total", 0)}', 'Eligible episodes'),
|
| 225 |
(c.get('review', 0), 'Requiring review'), (c.get('blocked', 0), 'Blocked')])
|
| 226 |
+
stats_html = f'<div class="k-stats">{stats}</div>' if not detailed else ''
|
| 227 |
kind = 'Published report' if published else 'Unpublished quick audit · video not assessed'
|
| 228 |
+
result = (f'<article class="k-card"><span class="k-pill {status}">{status_text}</span><h3>{escape(name)}</h3>'
|
| 229 |
f'<div class="k-card-id">{escape(payload["dataset"])}</div>'
|
| 230 |
+
f'<div class="k-score"><span class="k-eyebrow">Training Readiness</span>{score_html}</div>'
|
| 231 |
+
f'{stats_html}{_bar(c)}'
|
| 232 |
f'<p class="k-note">{escape(_note(payload, status))}</p>'
|
| 233 |
f'<div class="k-meta">{escape(payload["requirements"])}<br>Assessed {escape(_assessed(payload["assessed_at"]))}<br>{kind}</div>'
|
| 234 |
f'<div class="k-actions">{actions}</div></article>')
|
| 235 |
+
|
| 236 |
+
if detailed:
|
| 237 |
+
from quality_card import render_detail
|
| 238 |
+
return render_detail(payload, result)
|
| 239 |
+
return result
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
CSS += """
|
| 243 |
+
.k-detail{background:#fff;border:1px solid var(--k-line);padding:24px;margin-bottom:16px}
|
| 244 |
+
.k-detail-head{display:grid;grid-template-columns:minmax(260px,1fr) 1fr;gap:28px}
|
| 245 |
+
.k-detail-head .k-card{border:0;padding:0}.k-detail-intro{padding:16px;background:var(--k-sage);align-self:start}
|
| 246 |
+
.k-detail section{border-top:1px solid var(--k-line);margin-top:24px;padding-top:16px}
|
| 247 |
+
.k-detail h3{font-size:21px;margin:0 0 12px}.k-detail h4{font-size:16px;margin:10px 0}
|
| 248 |
+
.k-detail p{font-size:14px;line-height:1.6}.k-detail summary{cursor:pointer;padding:10px 0;font-weight:500}
|
| 249 |
+
.k-detail details{border-bottom:1px solid var(--k-line)}.k-detail details:target{outline:2px solid var(--k-link);outline-offset:4px}
|
| 250 |
+
.k-detail table{border-collapse:collapse;width:100%;font-size:13px}.k-detail th,.k-detail td{text-align:left;padding:12px 8px;border-bottom:1px solid var(--k-line)}
|
| 251 |
+
.k-table-scroll{overflow-x:auto}.k-evidence{white-space:pre-wrap;overflow-wrap:anywhere;max-height:360px;overflow:auto;background:#f6f7f3;padding:12px;font-size:12px}
|
| 252 |
+
.k-finding{border-left:3px solid var(--k-line);padding:12px 16px;margin:12px 0;background:#fafbf8}
|
| 253 |
+
.k-finding a{display:inline-block;margin:4px 12px 4px 0}.k-detail :focus-visible{outline:3px solid #dc8437;outline-offset:3px}
|
| 254 |
+
@media(max-width:760px){.k-detail{padding:16px}.k-detail-head{grid-template-columns:1fr;gap:12px}}
|
| 255 |
+
"""
|
publisher_grader/presentation/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (2.66 kB). View file
|
|
|
publisher_grader/presentation/__pycache__/action_model.cpython-314.pyc
ADDED
|
Binary file (28 kB). View file
|
|
|
publisher_grader/presentation/__pycache__/report_model.cpython-314.pyc
ADDED
|
Binary file (26.9 kB). View file
|
|
|
publisher_grader/presentation/__pycache__/video_model.cpython-314.pyc
ADDED
|
Binary file (5.91 kB). View file
|
|
|
quality_card.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Detailed Space view over the existing public-engine presentation model.
|
| 2 |
+
|
| 3 |
+
No policy thresholds, eligibility decisions, family scores or physical causes
|
| 4 |
+
are inferred here. Missing presentation data remains explicitly unavailable.
|
| 5 |
+
"""
|
| 6 |
+
from collections import Counter
|
| 7 |
+
from hashlib import sha256
|
| 8 |
+
from html import escape
|
| 9 |
+
import json
|
| 10 |
+
|
| 11 |
+
from publisher_grader.presentation.action_model import LABELS
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
TIMING_NOTE = ('Recorded timestamp skew or correspondence does not prove physical camera/action '
|
| 15 |
+
'latency. Capture alignment requires verified capture-clock evidence for both streams.')
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def text(value):
|
| 19 |
+
return escape(str(value if value is not None else 'Not recorded'))
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def evidence(value):
|
| 23 |
+
return '<pre class="k-evidence">' + text(json.dumps(value, indent=2, ensure_ascii=False)) + '</pre>'
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def findings(model):
|
| 27 |
+
"""Keep source, unit and consequence separate; count unique episodes per issue."""
|
| 28 |
+
groups = {}
|
| 29 |
+
for episode in model.get('episodes') or []:
|
| 30 |
+
for finding in episode.get('findings') or []:
|
| 31 |
+
key = tuple(finding.get(k) for k in ('metric_id', 'node', 'source_field',
|
| 32 |
+
'source_index', 'stream', 'instance', 'channel', 'subject_level', 'unit', 'consequence'))
|
| 33 |
+
group = groups.setdefault(key, {'finding': finding, 'episodes': set(), 'records': []})
|
| 34 |
+
group['episodes'].add(episode['id'])
|
| 35 |
+
group['records'].append({'episode': episode['id'], **finding})
|
| 36 |
+
ranks = {'block': 0, 'blocked': 0, 'review': 1, 'report_only': 2}
|
| 37 |
+
return sorted(groups.values(), key=lambda g: (ranks.get(g['finding'].get('consequence'), 3),
|
| 38 |
+
-len(g['episodes']), str(g['finding'].get('metric_id'))))
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def family_rows(model):
|
| 42 |
+
"""Diagnostic summaries, never derived readiness or universal diversity scores."""
|
| 43 |
+
all_findings = [f for e in model.get('episodes') or [] for f in e.get('findings') or []]
|
| 44 |
+
rows = []
|
| 45 |
+
for family, label in [('timing', 'Timing'), ('integrity', 'Signal integrity'),
|
| 46 |
+
('motion', 'Motion'), ('coverage', 'Coverage / diversity'),
|
| 47 |
+
('schema', 'Structure / schema')]:
|
| 48 |
+
metrics = [m for m in model.get('metrics') or [] if m['id'].startswith(family + '.')]
|
| 49 |
+
relevant = [f for f in all_findings if f.get('family') == family or
|
| 50 |
+
str(f.get('metric_id', '')).startswith(family + '.')]
|
| 51 |
+
consequences = Counter(f.get('consequence') for f in relevant)
|
| 52 |
+
computed = sum((m.get('availability') or {}).get('computed', 0) for m in metrics)
|
| 53 |
+
gaps = sum(sum((m.get('availability') or {}).get(s, 0) for s in
|
| 54 |
+
('unavailable', 'skipped', 'error')) for m in metrics)
|
| 55 |
+
if consequences['block'] or consequences['blocked']:
|
| 56 |
+
status = 'BLOCKING FINDINGS'
|
| 57 |
+
elif consequences['review']:
|
| 58 |
+
status = 'REVIEW FINDINGS'
|
| 59 |
+
elif relevant:
|
| 60 |
+
status = 'OBSERVATIONS'
|
| 61 |
+
elif computed:
|
| 62 |
+
status = 'COMPUTED'
|
| 63 |
+
else:
|
| 64 |
+
status = 'NOT ASSESSED'
|
| 65 |
+
detail = f'{computed} computed metric results · {gaps} unavailable/skipped/error'
|
| 66 |
+
if relevant:
|
| 67 |
+
detail += f' · {len(relevant)} finding records'
|
| 68 |
+
if family == 'schema' and not metrics and not relevant:
|
| 69 |
+
inventory = model.get('inventory') or {}
|
| 70 |
+
status = ('INVENTORY COMPLETE' if inventory.get('complete') is True else
|
| 71 |
+
'INVENTORY INCOMPLETE' if inventory.get('complete') is False else 'NOT PROVIDED')
|
| 72 |
+
detail = (f"{inventory.get('loaded', 'Unknown')} loaded / {inventory.get('expected', 'unknown')} expected episodes; "
|
| 73 |
+
'inventory accounting is not a schema quality score')
|
| 74 |
+
elif 'metrics' not in model:
|
| 75 |
+
status, detail = 'NOT PROVIDED', 'Diagnostic details are absent from this payload'
|
| 76 |
+
rows.append((label, status, detail))
|
| 77 |
+
return rows
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _issue(group, episode_links):
|
| 81 |
+
f = group['finding']
|
| 82 |
+
consequence = f.get('consequence') or 'unspecified'
|
| 83 |
+
style = {'block': 'blocked', 'blocked': 'blocked', 'review': 'review'}.get(consequence, 'incomplete')
|
| 84 |
+
metric = f.get('metric_id') or 'Unspecified metric'
|
| 85 |
+
name = LABELS.get(metric, metric.rsplit('.', 1)[-1].replace('_', ' ').capitalize())
|
| 86 |
+
source = f.get('source_field') or f.get('stream') or 'Source not resolved'
|
| 87 |
+
if f.get('source_index') is not None:
|
| 88 |
+
source += f"[{f['source_index']}]"
|
| 89 |
+
episodes = ' '.join(f'<a href="#{episode_links[e]}">{text(e.split("::")[-1])}</a>'
|
| 90 |
+
for e in sorted(group['episodes']))
|
| 91 |
+
return (f'<article class="k-finding"><span class="k-pill {style}">{text(consequence.replace("_", " "))}</span>'
|
| 92 |
+
f'<h4>{text(name)}</h4><p>{text(source)} · {len(group["episodes"])} affected episode(s)</p>'
|
| 93 |
+
f'<p>Recorded example: {text(f.get("value"))} {text(f.get("unit") or "")} · {text(metric)}</p>'
|
| 94 |
+
'<details><summary>Evidence — recorded values and support</summary>'
|
| 95 |
+
+ evidence(group['records']) + '</details>'
|
| 96 |
+
f'<details><summary>View {len(group["episodes"])} affected episode(s)</summary><p>{episodes}</p></details>'
|
| 97 |
+
'<details><summary>Review guidance</summary><p>Inspect the recorded source and support intervals. '
|
| 98 |
+
'Validate units, signal roles, clocks and task context before changing the data; rerun after confirmed corrections. '
|
| 99 |
+
'This is UI guidance, not an engine diagnosis or automatic repair.</p></details></article>')
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def render_detail(payload, summary):
|
| 103 |
+
model = payload.get('presentation') or {}
|
| 104 |
+
groups = findings(model)
|
| 105 |
+
prefix = 'quality-' + sha256(str((payload.get('dataset'), payload.get('report_id'))).encode()).hexdigest()[:12]
|
| 106 |
+
episode_links = {e['id']: f'{prefix}-episode-{i}' for i, e in enumerate(model.get('episodes') or [])}
|
| 107 |
+
blockers = [g for g in groups if g['finding'].get('consequence') in ('block', 'blocked')]
|
| 108 |
+
other = [g for g in groups if g not in blockers]
|
| 109 |
+
warnings = sum(f.get('severity') == 'warning' for e in model.get('episodes') or []
|
| 110 |
+
for f in e.get('findings') or [])
|
| 111 |
+
count_note = (f'{len(blockers)} blocking issue group(s) · {warnings} warning finding record(s)'
|
| 112 |
+
if 'episodes' in model else 'Detailed findings are unavailable in this assessment payload.')
|
| 113 |
+
readiness_reasons = (payload.get('readiness_detail') or model.get('readiness') or {}).get('reasons') or []
|
| 114 |
+
actions = f'<a class="k-btn" href="#{prefix}-blockers">View blockers ({len(blockers)})</a>' if blockers else ''
|
| 115 |
+
head = (f'<div class="k-detail-head">{summary}<div class="k-detail-intro"><span class="k-eyebrow">Assessment</span>'
|
| 116 |
+
f'<p>{text(count_note)}</p>'
|
| 117 |
+
+ (f'<p>Readiness: {text("; ".join(readiness_reasons))}</p>' if readiness_reasons else '')
|
| 118 |
+
+ '<p>Training Readiness is the engine’s technical quality result within the recorded scope. '
|
| 119 |
+
'It does not predict training success.</p><p>Diagnostic summaries below do not contribute new scores or change eligibility.</p>'
|
| 120 |
+
f'{actions}</div></div>')
|
| 121 |
+
rows = ''.join(f'<tr><th scope="row">{text(label)}</th><td>{text(state)}</td><td>{text(detail)}</td></tr>'
|
| 122 |
+
for label, state, detail in family_rows(model))
|
| 123 |
+
quality = ('<section><h3>Quality overview</h3><p>Diagnostic families · no synthetic 0–100 scores. '
|
| 124 |
+
'Computed does not mean all checks passed. Coverage / diversity is distinct from assessment coverage.</p>'
|
| 125 |
+
f'<div class="k-table-scroll"><table><thead><tr><th>Family</th><th>Recorded summary</th><th>Evidence coverage</th></tr></thead><tbody>{rows}</tbody></table></div></section>')
|
| 126 |
+
issues = (f'<section><h3>Fix these first</h3><p>{text(TIMING_NOTE)}</p>'
|
| 127 |
+
'<p>Only recorded block/review consequences change eligibility; critical or warning severity alone does not.</p>'
|
| 128 |
+
f'<div id="{prefix}-blockers"><h4>Blocking findings</h4>'
|
| 129 |
+
+ (''.join(_issue(g, episode_links) for g in blockers) if blockers else
|
| 130 |
+
'<p>No blocking issue details recorded in this payload. Refer to the episode counts above.</p>') + '</div>')
|
| 131 |
+
if other:
|
| 132 |
+
issues += '<h4>Review and diagnostic findings</h4>' + ''.join(_issue(g, episode_links) for g in other[:3])
|
| 133 |
+
if len(other) > 3:
|
| 134 |
+
issues += f'<details><summary>Show {len(other)-3} additional issue groups</summary>' + ''.join(_issue(g, episode_links) for g in other[3:]) + '</details>'
|
| 135 |
+
gaps = [a for a in model.get('attention') or [] if a.get('category') in ('gap', 'inventory')]
|
| 136 |
+
if gaps:
|
| 137 |
+
issues += '<details><summary>Assessment gaps (not defect claims)</summary>' + ''.join(
|
| 138 |
+
f'<p><strong>{text(a.get("name"))}</strong> — {text(a.get("observation"))}</p>' for a in gaps) + '</details>'
|
| 139 |
+
issues += '</section>'
|
| 140 |
+
meta = payload.get('dataset_metadata') or {}
|
| 141 |
+
duration = model.get('duration')
|
| 142 |
+
known = model.get('durationKnown', 0)
|
| 143 |
+
dataset = (f'<section><h3>Dataset overview</h3><p>{text(payload["counts"].get("total"))} episodes assessed'
|
| 144 |
+
+ (f' · {duration:g} recorded seconds ({known} episodes with duration)' if duration is not None else '')
|
| 145 |
+
+ f'</p><p>Format: {text(meta.get("format"))} · Robot type: {text(meta.get("embodiment"))}</p>'
|
| 146 |
+
f'<p>Source fields: {text(", ".join(meta.get("sensors") or []) or None)}</p>'
|
| 147 |
+
'<p>Camera hardware count and arm DoF are not inferred from source fields.</p></section>')
|
| 148 |
+
explorer = '<section><h3>Episode explorer</h3>'
|
| 149 |
+
for e in model.get('episodes') or []:
|
| 150 |
+
explorer += (f'<details id="{episode_links[e["id"]]}"><summary>{text(e.get("label") or e["id"])} · '
|
| 151 |
+
f'{text(e.get("status"))} · {len(e.get("findings") or [])} finding(s)</summary>'
|
| 152 |
+
+ evidence({k: e.get(k) for k in ('id', 'status', 'reasons', 'samples', 'duration', 'tasks', 'findings')}) + '</details>')
|
| 153 |
+
if not model.get('episodes'):
|
| 154 |
+
explorer += '<p>Episode details are not included. Open the published full report when available.</p>'
|
| 155 |
+
explorer += '</section>'
|
| 156 |
+
technical = ('<section><details><summary>Technical details</summary><p>Converted CLI assessment: metric summaries include up to three '
|
| 157 |
+
'examples; support intervals may be truncated, with counts retained. Full raw metrics, plots and policy thresholds '
|
| 158 |
+
'require the full CLI report when absent here.</p>'
|
| 159 |
+
+ evidence({'scope': model.get('scope'), 'readiness': payload.get('readiness_detail') or model.get('readiness'), 'assessment_coverage': model.get('coverage'),
|
| 160 |
+
'inventory': model.get('inventory'), 'metrics': model.get('metrics'),
|
| 161 |
+
'notes': (payload.get('notes') or []) + (model.get('notes') or [])})
|
| 162 |
+
+ '</details></section>')
|
| 163 |
+
return f'<div class="k-html k-detail">{head}{quality}{issues}{dataset}{explorer}{technical}</div>'
|
tests/__pycache__/test_app.cpython-314.pyc
CHANGED
|
Binary files a/tests/__pycache__/test_app.cpython-314.pyc and b/tests/__pycache__/test_app.cpython-314.pyc differ
|
|
|
tests/__pycache__/test_quality_card.cpython-314.pyc
ADDED
|
Binary file (9.25 kB). View file
|
|
|
tests/test_quality_card.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import copy
|
| 2 |
+
import json
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
import unittest
|
| 5 |
+
from unittest.mock import patch
|
| 6 |
+
|
| 7 |
+
from cards import render
|
| 8 |
+
from quality_card import family_rows, findings
|
| 9 |
+
from app import download_assessment, download, lookup
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def payload():
|
| 13 |
+
finding = {'metric_id': 'timing.recorded_skew', 'family': 'timing',
|
| 14 |
+
'consequence': 'report_only', 'severity': 'critical', 'value': 180,
|
| 15 |
+
'unit': 'ms', 'source_field': 'action', 'node': 'n0',
|
| 16 |
+
'evidence': {'interpretation': 'recorded timestamps'},
|
| 17 |
+
'support': {'intervals': [{'start': 1, 'end': 3}]}}
|
| 18 |
+
return {'dataset': 'example/data', 'report_id': 'r', 'readiness': None,
|
| 19 |
+
'counts': {'total': 2, 'pass_count': 1, 'review': 0, 'blocked': 0,
|
| 20 |
+
'unknown': 1, 'inventory_complete': False},
|
| 21 |
+
'requirements': 'numeric-core-v1', 'assessed_at': '2026-10-05T00:00:00Z',
|
| 22 |
+
'details': [], 'markdown': 'card', 'dataset_url': 'https://huggingface.co/datasets/example/data',
|
| 23 |
+
'presentation': {'episodes': [
|
| 24 |
+
{'id': 'e0', 'label': 'e0', 'status': 'pass', 'findings': [finding, copy.deepcopy(finding)]},
|
| 25 |
+
{'id': 'e1', 'label': 'e1', 'status': 'unknown', 'findings': [copy.deepcopy(finding)]}],
|
| 26 |
+
'metrics': [{'id': 'timing.recorded_skew', 'availability': {'computed': 3, 'unavailable': 1}}],
|
| 27 |
+
'readiness': {'score': None, 'reasons': ['inventory_incomplete']}}}
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class QualityCardTests(unittest.TestCase):
|
| 31 |
+
def test_report_only_critical_is_not_blocking(self):
|
| 32 |
+
p = payload()
|
| 33 |
+
self.assertEqual(family_rows(p['presentation'])[0][1], 'OBSERVATIONS')
|
| 34 |
+
html = render(p, detailed=True)
|
| 35 |
+
self.assertIn('0 blocking issue group(s)', html)
|
| 36 |
+
self.assertNotIn('View blockers (1)', html)
|
| 37 |
+
self.assertIn('Not established', html)
|
| 38 |
+
self.assertIn('inventory_incomplete', html)
|
| 39 |
+
self.assertIn('does not prove physical camera/action latency', html)
|
| 40 |
+
self.assertNotIn('GOOD', html)
|
| 41 |
+
|
| 42 |
+
def test_grouping_deduplicates_episodes_not_sources_or_consequences(self):
|
| 43 |
+
p = payload()
|
| 44 |
+
self.assertEqual(len(findings(p['presentation'])[0]['episodes']), 2)
|
| 45 |
+
f = copy.deepcopy(p['presentation']['episodes'][0]['findings'][0])
|
| 46 |
+
f['consequence'] = 'block'
|
| 47 |
+
p['presentation']['episodes'][0]['findings'].append(copy.deepcopy(f))
|
| 48 |
+
self.assertEqual(len(findings(p['presentation'])), 2)
|
| 49 |
+
self.assertEqual(family_rows(p['presentation'])[0][1], 'BLOCKING FINDINGS')
|
| 50 |
+
f['source_field'] = 'another_action'
|
| 51 |
+
p['presentation']['episodes'][0]['findings'].append(copy.deepcopy(f))
|
| 52 |
+
self.assertEqual(len(findings(p['presentation'])), 3)
|
| 53 |
+
|
| 54 |
+
def test_missing_published_detail_is_not_pass_or_unassessed(self):
|
| 55 |
+
p = payload()
|
| 56 |
+
del p['presentation']
|
| 57 |
+
html = render(p, detailed=True)
|
| 58 |
+
self.assertIn('NOT PROVIDED', html)
|
| 59 |
+
self.assertIn('Detailed findings are unavailable', html)
|
| 60 |
+
self.assertNotIn('0 blocking issue group(s)', html)
|
| 61 |
+
with patch('app.published', return_value=p):
|
| 62 |
+
self.assertEqual(lookup('example/data')[4], p)
|
| 63 |
+
|
| 64 |
+
def test_escaping_and_episode_links(self):
|
| 65 |
+
from html.parser import HTMLParser
|
| 66 |
+
class Links(HTMLParser):
|
| 67 |
+
def __init__(self):
|
| 68 |
+
super().__init__(); self.ids = set(); self.targets = set()
|
| 69 |
+
def handle_starttag(self, tag, attrs):
|
| 70 |
+
a = dict(attrs)
|
| 71 |
+
if 'id' in a: self.ids.add(a['id'])
|
| 72 |
+
if a.get('href', '').startswith('#'): self.targets.add(a['href'][1:])
|
| 73 |
+
p = payload()
|
| 74 |
+
p['presentation']['episodes'][0]['findings'][0]['source_field'] = '<script>bad()</script>'
|
| 75 |
+
html = render(p, detailed=True)
|
| 76 |
+
self.assertNotIn('<script>', html)
|
| 77 |
+
self.assertIn('<script>', html)
|
| 78 |
+
parsed = Links(); parsed.feed(html)
|
| 79 |
+
self.assertTrue(parsed.targets)
|
| 80 |
+
self.assertTrue(parsed.targets <= parsed.ids)
|
| 81 |
+
|
| 82 |
+
def test_blocked_status_remains_visible_with_unknown_episodes(self):
|
| 83 |
+
p = payload(); p['counts']['blocked'] = 1
|
| 84 |
+
self.assertIn('Blocking findings', render(p, detailed=True))
|
| 85 |
+
self.assertIn('1 unknown', render(p, detailed=True))
|
| 86 |
+
|
| 87 |
+
def test_download_roundtrip_and_reset(self):
|
| 88 |
+
p = payload()
|
| 89 |
+
update = download_assessment(p)
|
| 90 |
+
path = Path(update['value'])
|
| 91 |
+
try:
|
| 92 |
+
self.assertEqual(json.loads(path.read_text()), p)
|
| 93 |
+
self.assertIn(str(path), download.temp_files)
|
| 94 |
+
self.assertEqual(download_assessment(None)['visible'], False)
|
| 95 |
+
self.assertIsNone(download_assessment(None)['value'])
|
| 96 |
+
finally:
|
| 97 |
+
path.unlink(); download.temp_files.discard(str(path))
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
if __name__ == '__main__':
|
| 101 |
+
unittest.main()
|
tests/test_quick_grade.py
CHANGED
|
@@ -61,6 +61,11 @@ class QuickGradeTests(unittest.TestCase):
|
|
| 61 |
payload=card_payload(card,'https://hub.kalanos.ai')
|
| 62 |
self.assertIn('Unpublished quick audit', payload['markdown'])
|
| 63 |
self.assertIn('aria-label="Preview and copy HF dataset card"',render(payload,False))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
Path('/tmp/kalanos-quick-card.json').write_text(json.dumps(card))
|
| 65 |
|
| 66 |
if __name__=='__main__':
|
|
|
|
| 61 |
payload=card_payload(card,'https://hub.kalanos.ai')
|
| 62 |
self.assertIn('Unpublished quick audit', payload['markdown'])
|
| 63 |
self.assertIn('aria-label="Preview and copy HF dataset card"',render(payload,False))
|
| 64 |
+
detailed = render(payload, False, detailed=True)
|
| 65 |
+
self.assertIn('INVENTORY COMPLETE', detailed)
|
| 66 |
+
self.assertIn('Training Readiness', detailed)
|
| 67 |
+
self.assertIn('Episode explorer', detailed)
|
| 68 |
+
self.assertIn('Video: not assessed', detailed)
|
| 69 |
Path('/tmp/kalanos-quick-card.json').write_text(json.dumps(card))
|
| 70 |
|
| 71 |
if __name__=='__main__':
|