Add taxonomy docs and richer telemetry error accounting

README.md

@@ -92,6 +92,7 @@ Smart home safety defaults:
 - Integration runbook: [`docs/operations/integration-policy.md`](docs/operations/integration-policy.md).
 - Release checklist: [`docs/operations/release-checklist.md`](docs/operations/release-checklist.md).
 - Security maintenance: [`docs/operations/security-maintenance.md`](docs/operations/security-maintenance.md).
+- Error taxonomy: [`docs/operations/error-taxonomy.md`](docs/operations/error-taxonomy.md).
 - Todoist integration:
   - `TODOIST_PERMISSION_PROFILE=readonly|control`
   - `readonly` allows `todoist_list_tasks` and denies `todoist_add_task`

TODO.md

@@ -65,14 +65,14 @@ Files: `src/jarvis/tools/services.py`, `tests/test_tools.py`, `docs/operations/h
 
 ## 4) Telemetry and Taxonomy (8 items)
 
-- [ ] `T01` Add taxonomy reference doc with each error code and owning tool families.
-- [ ] `T02` Add test asserting taxonomy doc examples stay in sync with constants.
-- [ ] `T03` Track unknown summary detail count in telemetry snapshot.
-- [ ] `T04` Add tests for unknown summary detail accounting.
-- [ ] `T05` Add explicit counter for per-code service error totals in telemetry snapshot.
-- [ ] `T06` Add tests for per-code aggregation stability.
-- [ ] `T07` Add NaN/inf guard tests for telemetry averages.
-- [ ] `T08` Add startup log line summarizing taxonomy sizes (service/storage subsets).
+- [x] `T01` Add taxonomy reference doc with each error code and owning tool families.
+- [x] `T02` Add test asserting taxonomy doc examples stay in sync with constants.
+- [x] `T03` Track unknown summary detail count in telemetry snapshot.
+- [x] `T04` Add tests for unknown summary detail accounting.
+- [x] `T05` Add explicit counter for per-code service error totals in telemetry snapshot.
+- [x] `T06` Add tests for per-code aggregation stability.
+- [x] `T07` Add NaN/inf guard tests for telemetry averages.
+- [x] `T08` Add startup log line summarizing taxonomy sizes (service/storage subsets).
 
 Files: `src/jarvis/tool_errors.py`, `src/jarvis/__main__.py`, `tests/test_main_lifecycle.py`, `docs/operations/error-taxonomy.md`
 

docs/operations/error-taxonomy.md

@@ -0,0 +1,63 @@
+# Error Taxonomy Reference
+
+Canonical tool error taxonomy shared by service tools and telemetry.
+
+## Ownership by Tool Family
+
+| Error code | Owning tool families |
+|---|---|
+| `api_error` | `pushover_notify` |
+| `auth` | `smart_home`, `smart_home_state`, `todoist_*`, `pushover_notify` |
+| `cancelled` | `smart_home*`, `todoist_*`, `pushover_notify` |
+| `http_error` | `smart_home*`, `todoist_*`, `pushover_notify` |
+| `invalid_data` | `smart_home`, `task_plan_*`, `memory_*` |
+| `invalid_json` | `smart_home*`, `todoist_*`, `pushover_notify` |
+| `invalid_plan` | `task_plan_*` |
+| `invalid_status` | `task_plan_update` |
+| `invalid_steps` | `task_plan_create` |
+| `missing_config` | `smart_home*`, `todoist_*`, `pushover_notify` |
+| `missing_entity` | `smart_home_state` |
+| `missing_fields` | `smart_home`, `todoist_add_task`, `pushover_notify` |
+| `missing_plan` | `task_plan_*` |
+| `missing_query` | `memory_search` |
+| `missing_store` | `memory_*`, `task_plan_*` |
+| `missing_text` | `memory_add` |
+| `network_client_error` | `smart_home*`, `todoist_*`, `pushover_notify` |
+| `not_found` | `smart_home*`, `task_plan_*` |
+| `policy` | cross-cutting tool policy enforcement |
+| `storage_error` | `memory_*`, `task_plan_*` |
+| `summary_unavailable` | `tool_summary*`, `system_status` |
+| `timeout` | `smart_home*`, `todoist_*`, `pushover_notify` |
+| `unexpected` | cross-cutting runtime guard |
+| `unknown_error` | fallback normalization bucket |
+
+## Canonical Code Set (Machine-Checked)
+
+Do not edit this list without updating `src/jarvis/tool_errors.py` and tests.
+
+<!-- SERVICE_ERROR_CODES_START -->
+api_error
+auth
+cancelled
+http_error
+invalid_data
+invalid_json
+invalid_plan
+invalid_status
+invalid_steps
+missing_config
+missing_entity
+missing_fields
+missing_plan
+missing_query
+missing_store
+missing_text
+network_client_error
+not_found
+policy
+storage_error
+summary_unavailable
+timeout
+unexpected
+unknown_error
+<!-- SERVICE_ERROR_CODES_END -->

src/jarvis/__main__.py

@@ -193,8 +193,10 @@ class Jarvis:
             "tts_first_audio_count": 0.0,
             "service_errors": 0.0,
             "storage_errors": 0.0,
+            "unknown_summary_details": 0.0,
             "fallback_responses": 0.0,
         }
+        self._telemetry_error_counts: dict[str, float] = {}
 
     def start(self) -> None:
         """Initialize all subsystems."""
@@ -266,6 +268,7 @@ class Jarvis:
             f"Persona style: {self.config.persona_style}",
             f"Config warnings: {warning_count}",
             f"Tool policy: allow={len(self.config.tool_allowlist)} deny={len(self.config.tool_denylist)}",
+            f"Error taxonomy: total={len(TOOL_SERVICE_ERROR_CODES)} service={len(TELEMETRY_SERVICE_ERROR_DETAILS)} storage={len(TELEMETRY_STORAGE_ERROR_DETAILS)}",
         ]
 
     def _refresh_tool_error_counters(self) -> None:
@@ -275,6 +278,8 @@ class Jarvis:
             return
         service_errors = 0
         storage_errors = 0
+        unknown_summary_details = 0
+        per_code: dict[str, float] = {}
         for item in recent:
             if not isinstance(item, dict):
                 continue
@@ -282,30 +287,53 @@ class Jarvis:
             detail = str(item.get("detail", ""))
             if status != "error":
                 continue
+            if detail in TOOL_SERVICE_ERROR_CODES:
+                per_code[detail] = per_code.get(detail, 0.0) + 1.0
             if detail in TELEMETRY_STORAGE_ERROR_DETAILS:
                 storage_errors += 1
                 continue
             if detail in TELEMETRY_SERVICE_ERROR_DETAILS:
                 service_errors += 1
+                continue
+            unknown_summary_details += 1
         self._telemetry["service_errors"] = float(service_errors)
         self._telemetry["storage_errors"] = float(storage_errors)
+        self._telemetry["unknown_summary_details"] = float(unknown_summary_details)
+        self._telemetry_error_counts = {name: per_code[name] for name in sorted(per_code)}
+
+    def _telemetry_snapshot(self) -> dict[str, float | dict[str, float]]:
+        def metric(key: str) -> float:
+            value = self._telemetry.get(key, 0.0)
+            if not math.isfinite(value):
+                return 0.0
+            return value
 
-    def _telemetry_snapshot(self) -> dict[str, float]:
         def avg(total_key: str, count_key: str) -> float:
-            count = self._telemetry.get(count_key, 0.0)
-            if count <= 0:
+            count = metric(count_key)
+            if count <= 0.0:
                 return 0.0
-            return self._telemetry.get(total_key, 0.0) / count
+            total = metric(total_key)
+            value = total / count
+            if not math.isfinite(value):
+                return 0.0
+            return value
 
+        counts = {
+            name: value
+            for name, value in getattr(self, "_telemetry_error_counts", {}).items()
+            if math.isfinite(value)
+        }
         return {
-            "turns": self._telemetry.get("turns", 0.0),
-            "barge_ins": self._telemetry.get("barge_ins", 0.0),
+            "turns": metric("turns"),
+            "barge_ins": metric("barge_ins"),
             "avg_stt_latency_ms": avg("stt_latency_total_ms", "stt_latency_count"),
             "avg_llm_first_sentence_ms": avg("llm_first_sentence_total_ms", "llm_first_sentence_count"),
             "avg_tts_first_audio_ms": avg("tts_first_audio_total_ms", "tts_first_audio_count"),
-            "service_errors": self._telemetry.get("service_errors", 0.0),
-            "storage_errors": self._telemetry.get("storage_errors", 0.0),
-            "fallback_responses": self._telemetry.get("fallback_responses", 0.0),
+            "service_errors": metric("service_errors"),
+            "storage_errors": metric("storage_errors"),
+            "unknown_summary_details": metric("unknown_summary_details"),
+            "service_error_counts": counts,
+            "fallback_responses": metric("fallback_responses"),
         }
 
     def stop(self) -> None:

tests/test_main_lifecycle.py

@@ -1,6 +1,7 @@
 """Lifecycle robustness tests for jarvis.__main__.Jarvis."""
 
 import pytest
+from pathlib import Path
 from types import SimpleNamespace
 from unittest.mock import MagicMock, AsyncMock
 
@@ -79,6 +80,21 @@ def test_startup_summary_lines_include_core_status():
     assert "Memory: enabled" in joined
     assert "Config warnings: 0" in joined
     assert "Tool policy: allow=1 deny=2" in joined
+    assert "Error taxonomy: total=" in joined
+
+
+def test_error_taxonomy_doc_matches_constants():
+    from jarvis.tool_errors import TOOL_SERVICE_ERROR_CODES
+
+    doc_path = Path(__file__).resolve().parents[1] / "docs" / "operations" / "error-taxonomy.md"
+    text = doc_path.read_text(encoding="utf-8")
+    start = "<!-- SERVICE_ERROR_CODES_START -->"
+    end = "<!-- SERVICE_ERROR_CODES_END -->"
+    assert start in text and end in text
+
+    block = text.split(start, 1)[1].split(end, 1)[0]
+    documented = {line.strip() for line in block.splitlines() if line.strip()}
+    assert documented == TOOL_SERVICE_ERROR_CODES
 
 
 def test_telemetry_snapshot_averages():
@@ -104,6 +120,7 @@ def test_telemetry_snapshot_averages():
     assert snapshot["avg_tts_first_audio_ms"] == 250.0
     assert snapshot["service_errors"] == 3.0
     assert snapshot["storage_errors"] == 1.0
+    assert snapshot["unknown_summary_details"] == 0.0
     assert snapshot["fallback_responses"] == 4.0
 
 
@@ -121,6 +138,7 @@ def test_refresh_tool_error_counters():
 
     assert jarvis._telemetry["service_errors"] == 1.0
     assert jarvis._telemetry["storage_errors"] == 1.0
+    assert jarvis._telemetry["unknown_summary_details"] == 0.0
 
 
 def test_refresh_tool_error_counters_classifies_missing_store_as_storage():
@@ -137,6 +155,7 @@ def test_refresh_tool_error_counters_classifies_missing_store_as_storage():
 
     assert jarvis._telemetry["service_errors"] == 1.0
     assert jarvis._telemetry["storage_errors"] == 2.0
+    assert jarvis._telemetry["unknown_summary_details"] == 0.0
 
 
 def test_refresh_tool_error_counters_includes_network_taxonomy():
@@ -153,6 +172,80 @@ def test_refresh_tool_error_counters_includes_network_taxonomy():
 
     assert jarvis._telemetry["service_errors"] == 3.0
     assert jarvis._telemetry["storage_errors"] == 0.0
+    assert jarvis._telemetry["unknown_summary_details"] == 0.0
+
+
+def test_refresh_tool_error_counters_tracks_unknown_summary_detail_count():
+    jarvis = Jarvis.__new__(Jarvis)
+    jarvis._telemetry = {"service_errors": 0.0, "storage_errors": 0.0, "unknown_summary_details": 0.0}
+    from unittest.mock import patch
+
+    with patch("jarvis.__main__.list_summaries", return_value=[
+        {"status": "error", "detail": "timeout"},
+        {"status": "error", "detail": "not_a_real_code"},
+        {"status": "error", "detail": "another_unknown"},
+    ]):
+        Jarvis._refresh_tool_error_counters(jarvis)
+
+    assert jarvis._telemetry["service_errors"] == 1.0
+    assert jarvis._telemetry["storage_errors"] == 0.0
+    assert jarvis._telemetry["unknown_summary_details"] == 2.0
+
+
+def test_refresh_tool_error_counters_reports_per_code_totals():
+    jarvis = Jarvis.__new__(Jarvis)
+    jarvis._telemetry = {"service_errors": 0.0, "storage_errors": 0.0, "unknown_summary_details": 0.0}
+    from unittest.mock import patch
+
+    with patch("jarvis.__main__.list_summaries", return_value=[
+        {"status": "error", "detail": "timeout"},
+        {"status": "error", "detail": "timeout"},
+        {"status": "error", "detail": "storage_error"},
+        {"status": "error", "detail": "unknown_error"},
+        {"status": "error", "detail": "not_a_real_code"},
+        {"status": "ok", "detail": "timeout"},
+    ]):
+        Jarvis._refresh_tool_error_counters(jarvis)
+
+    assert jarvis._telemetry_error_counts == {
+        "storage_error": 1.0,
+        "timeout": 2.0,
+        "unknown_error": 1.0,
+    }
+
+    snapshot = Jarvis._telemetry_snapshot(jarvis)
+    assert snapshot["service_error_counts"] == jarvis._telemetry_error_counts
+    assert snapshot["unknown_summary_details"] == 1.0
+
+
+def test_telemetry_snapshot_guards_non_finite_values():
+    jarvis = Jarvis.__new__(Jarvis)
+    jarvis._telemetry = {
+        "turns": float("nan"),
+        "barge_ins": float("inf"),
+        "stt_latency_total_ms": float("inf"),
+        "stt_latency_count": 1.0,
+        "llm_first_sentence_total_ms": 100.0,
+        "llm_first_sentence_count": float("nan"),
+        "tts_first_audio_total_ms": 100.0,
+        "tts_first_audio_count": 0.0,
+        "service_errors": float("nan"),
+        "storage_errors": float("inf"),
+        "unknown_summary_details": float("nan"),
+        "fallback_responses": float("inf"),
+    }
+    jarvis._telemetry_error_counts = {"timeout": float("nan"), "auth": 2.0}
+
+    snapshot = Jarvis._telemetry_snapshot(jarvis)
+    assert snapshot["turns"] == 0.0
+    assert snapshot["barge_ins"] == 0.0
+    assert snapshot["avg_stt_latency_ms"] == 0.0
+    assert snapshot["avg_llm_first_sentence_ms"] == 0.0
+    assert snapshot["service_errors"] == 0.0
+    assert snapshot["storage_errors"] == 0.0
+    assert snapshot["unknown_summary_details"] == 0.0
+    assert snapshot["fallback_responses"] == 0.0
+    assert snapshot["service_error_counts"] == {"auth": 2.0}
 
 
 def test_telemetry_error_taxonomy_matches_service_error_codes():