"""Normalized evidence ledger. Environmental observations are never spacecraft telemetry.""" import hashlib import json from datetime import datetime, timezone, timedelta from threading import RLock from typing import Literal, Optional from pydantic import BaseModel, ConfigDict, Field, model_validator from . import store LOCK = RLock() UNITS = {'precipitation_mm': ('mm', 0, 500), 'water_level_m': ('m', -500, 10000), 'temperature_c': ('degC', -80, 100), 'humidity_pct': ('%', 0, 100)} def utc(value=None): if value is None: return datetime.now(timezone.utc) d = value if isinstance(value, datetime) else datetime.fromisoformat(value.replace('Z', '+00:00')) if d.tzinfo is None: raise ValueError('A timezone is required') return d.astimezone(timezone.utc) def digest(value): return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode()).hexdigest() class Observation(BaseModel): model_config = ConfigDict(extra='forbid', allow_inf_nan=False) message_id: str = Field(min_length=1, max_length=160) source_id: str = Field(min_length=1, max_length=100) instrument_id: str = Field(min_length=1, max_length=100) platform: Literal['ground', 'external', 'simulation'] category: Literal['observed', 'external', 'simulation'] product: Literal['observation', 'forecast'] = 'observation' acquired_at: datetime valid_at: datetime longitude: float = Field(ge=-180, le=180) latitude: float = Field(ge=-80, le=80) measurement: Literal['precipitation_mm', 'water_level_m', 'temperature_c', 'humidity_pct'] value: Optional[float] = None unit: str interval_minutes: int = Field(default=60, ge=1, le=1440) quality_flags: list[Literal['sensor_fault', 'suspect', 'missing', 'uncalibrated']] = Field(default_factory=list, max_length=4) metadata: dict = Field(default_factory=dict) @model_validator(mode='after') def check(self): self.acquired_at, self.valid_at = utc(self.acquired_at), utc(self.valid_at) now = utc() if self.acquired_at > now + timedelta(minutes=5): raise ValueError('Acquisition time cannot be in the future') if self.product == 'observation' and self.valid_at > now + timedelta(minutes=5): raise ValueError('An observed measurement cannot have a future valid time') if self.valid_at > now + timedelta(days=8): raise ValueError('Forecast horizon exceeds eight days') unit, low, high = UNITS[self.measurement] if self.unit != unit or self.value is not None and not low <= self.value <= high: raise ValueError(f'{self.measurement} requires {unit} in [{low}, {high}]') if self.platform != {'observed': 'ground', 'external': 'external', 'simulation': 'simulation'}[self.category]: raise ValueError('Platform and evidence category disagree') if self.product == 'forecast' and self.category == 'observed': raise ValueError('A forecast is not an observation') if len(json.dumps(self.metadata)) > 8000: raise ValueError('Observation metadata is limited to 8 KB') return self def quality(record, now=None, max_age_hours=6): now = utc(now) if record.get('value') is None: return 'missing' if set(record.get('quality_flags', [])) & {'sensor_fault', 'suspect', 'missing'}: return 'rejected' stamp = record['acquired_at'] if record['product'] == 'forecast' else record['valid_at'] return 'stale' if (now - utc(stamp)).total_seconds() > max_age_hours * 3600 else 'valid' def ingest(obs: Observation): body = obs.model_dump(mode='json') identity = digest([obs.category, obs.source_id, obs.instrument_id, obs.message_id]) checksum = digest(body) with LOCK: try: old = store.get('observations', identity) except KeyError: old = None if old: if old['checksum'] != checksum: store.event('Conflicting duplicate observation rejected', obs.category, observation_id=identity) raise ValueError('Message identity already exists with different contents') return {**old, 'duplicate': True} record = store.put('observations', {**body, 'ingested_at': store.now(), 'checksum': checksum, 'validation_status': quality(body), 'processing_status': 'pending'}, identity) if record['validation_status'] != 'valid': store.event('Observation stored with ' + record['validation_status'] + ' quality', obs.category, observation_id=identity) return {**record, 'duplicate': False} def weather_adapter(weather): """Open-Meteo hourly precipitation is accumulation over the preceding hour.""" if weather.get('provenance') not in ('EXTERNAL FORECAST', 'SIMULATION FORECAST'): return [] # Reanalysis and unlocated CSV files must never become future evidence. loc = weather['location']; simulated = weather['provenance'] == 'SIMULATION FORECAST' unit = weather.get('units', {}).get('precipitation') if unit != 'mm': raise ValueError('Weather precipitation must declare millimeters') times, values = weather['hourly']['time'], weather['hourly']['precipitation'] if len(times) != len(values) or len(times) > 200 or len(set(times)) != len(times): raise ValueError('Weather timestamps and values must be unique aligned hourly series') parsed = [utc(t if t.endswith('Z') or '+' in t[10:] else t + '+00:00') for t in times] if parsed != sorted(parsed): raise ValueError('Weather series must be chronological') normalized = [] for t, value in zip(parsed, values): normalized.append(Observation(message_id=weather['id'] + ':' + t.isoformat(), source_id='satforge-replay' if simulated else 'open-meteo', instrument_id='synthetic-grid' if simulated else 'forecast-grid', platform='simulation' if simulated else 'external', category='simulation' if simulated else 'external', product='forecast', acquired_at=weather['retrieved_at'], valid_at=t, longitude=loc['requested_lon'], latitude=loc['requested_lat'], measurement='precipitation_mm', value=value, unit=unit, metadata={'weather_id': weather['id'], 'source_url': weather.get('source_url'), 'timestamp_basis': 'provider retrieval; issue time unavailable', 'spatial_support': 'single model grid point, not catchment average'})) # Validate the whole provider response before committing any evidence. return [ingest(o) for o in normalized] def telemetry_ingest(t): from .operations import power identity = digest([t.device_id, t.packet_id or t.measured_at.isoformat()]) checksum = digest(t.model_dump(mode='json')) with LOCK: try: old = store.get('telemetry', identity) except KeyError: old = None if old: if old.get('checksum') != checksum: raise ValueError('Conflicting telemetry packet identity') return {**old, 'duplicate': True} body = power(t) body.update(checksum=checksum, validation_status='stale' if (utc()-utc(t.measured_at)).total_seconds()>900 else 'valid', platform='ESP32 ground prototype; orbital platform unverified', category='observed') result = store.put('telemetry', body, identity) store.event('Telemetry packet validated', 'MEASURED', device_id=t.device_id, telemetry_id=identity) return {**result, 'duplicate': False} def mission_status(now=None): from .operations import schedule, EnergyPolicy now = utc(now) rows = sorted(store.all_records('telemetry'), key=lambda r: r['measured_at'], reverse=True) latest = rows[0] if rows else None age = max(0, (now-utc(latest['measured_at'])).total_seconds()) if latest else None recent = [r for r in rows if latest and r['device_id'] == latest['device_id']][:100] sequences = sorted(set(r['sequence'] for r in recent if r.get('sequence') is not None)) try: policy = EnergyPolicy(**{k:v for k,v in store.get('config','energy-policy').items() if k!='id'}) except KeyError: policy = EnergyPolicy() return {'status': 'NO TELEMETRY RECEIVED' if not latest else 'STALE TELEMETRY' if age>900 else 'RECENT GROUND TELEMETRY', 'last_packet_at': max((r['received_at'] for r in rows), default=None), 'last_measurement_at': latest['measured_at'] if latest else None, 'age_seconds': age, 'samples': rows[:500], 'orbital_connection': 'NOT VERIFIED', 'command_interface': 'NOT IMPLEMENTED', 'sequence_gaps': sum(max(0,b-a-1) for a,b in zip(sequences,sequences[1:])) if len(sequences)>1 else None, 'sequence_caveat': 'Only supplied sequence numbers for one device; reboot/wrap not inferred. Packet loss rate is unavailable.', 'policy': schedule(latest if latest and age<=900 else {}, policy), 'policy_scope': 'Ground prototype planning only; no spacecraft commands sent', 'instruments': {'BME280': 'optional environmental context; not ground rainfall', 'BH1750': 'optional illuminance', 'INA219': 'optional solar voltage/current', 'camera': 'disabled by default; board confirmation required', 'rain_gauge': 'not instrumented', 'water_level': 'not instrumented', 'orbital_radio': 'not configured'}}