Download backend/observations.py from m4az19/saef-api: direct link, hf CLI and curl.
- Browser
- Download file 9.48 kB
-
https://huggingface.co/spaces/m4az19/saef-api/resolve/main/backend/observations.py
- Command line
-
hf download hf://spaces/m4az19/saef-api/backend/observations.py
-
curl -L -o observations.py https://huggingface.co/spaces/m4az19/saef-api/resolve/main/backend/observations.py
9.48 kB
| """Normalized evidence ledger. Environmental observations are never spacecraft telemetry.""" | |
| import hashlib | |
| import json | |
| from datetime import datetime, timezone, timedelta | |
| from threading import RLock | |
| from typing import Literal, Optional | |
| from pydantic import BaseModel, ConfigDict, Field, model_validator | |
| from . import store | |
| LOCK = RLock() | |
| UNITS = {'precipitation_mm': ('mm', 0, 500), 'water_level_m': ('m', -500, 10000), | |
| 'temperature_c': ('degC', -80, 100), 'humidity_pct': ('%', 0, 100)} | |
| def utc(value=None): | |
| if value is None: | |
| return datetime.now(timezone.utc) | |
| d = value if isinstance(value, datetime) else datetime.fromisoformat(value.replace('Z', '+00:00')) | |
| if d.tzinfo is None: | |
| raise ValueError('A timezone is required') | |
| return d.astimezone(timezone.utc) | |
| def digest(value): | |
| return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode()).hexdigest() | |
| class Observation(BaseModel): | |
| model_config = ConfigDict(extra='forbid', allow_inf_nan=False) | |
| message_id: str = Field(min_length=1, max_length=160) | |
| source_id: str = Field(min_length=1, max_length=100) | |
| instrument_id: str = Field(min_length=1, max_length=100) | |
| platform: Literal['ground', 'external', 'simulation'] | |
| category: Literal['observed', 'external', 'simulation'] | |
| product: Literal['observation', 'forecast'] = 'observation' | |
| acquired_at: datetime | |
| valid_at: datetime | |
| longitude: float = Field(ge=-180, le=180) | |
| latitude: float = Field(ge=-80, le=80) | |
| measurement: Literal['precipitation_mm', 'water_level_m', 'temperature_c', 'humidity_pct'] | |
| value: Optional[float] = None | |
| unit: str | |
| interval_minutes: int = Field(default=60, ge=1, le=1440) | |
| quality_flags: list[Literal['sensor_fault', 'suspect', 'missing', 'uncalibrated']] = Field(default_factory=list, max_length=4) | |
| metadata: dict = Field(default_factory=dict) | |
| def check(self): | |
| self.acquired_at, self.valid_at = utc(self.acquired_at), utc(self.valid_at) | |
| now = utc() | |
| if self.acquired_at > now + timedelta(minutes=5): | |
| raise ValueError('Acquisition time cannot be in the future') | |
| if self.product == 'observation' and self.valid_at > now + timedelta(minutes=5): | |
| raise ValueError('An observed measurement cannot have a future valid time') | |
| if self.valid_at > now + timedelta(days=8): | |
| raise ValueError('Forecast horizon exceeds eight days') | |
| unit, low, high = UNITS[self.measurement] | |
| if self.unit != unit or self.value is not None and not low <= self.value <= high: | |
| raise ValueError(f'{self.measurement} requires {unit} in [{low}, {high}]') | |
| if self.platform != {'observed': 'ground', 'external': 'external', 'simulation': 'simulation'}[self.category]: | |
| raise ValueError('Platform and evidence category disagree') | |
| if self.product == 'forecast' and self.category == 'observed': | |
| raise ValueError('A forecast is not an observation') | |
| if len(json.dumps(self.metadata)) > 8000: | |
| raise ValueError('Observation metadata is limited to 8 KB') | |
| return self | |
| def quality(record, now=None, max_age_hours=6): | |
| now = utc(now) | |
| if record.get('value') is None: | |
| return 'missing' | |
| if set(record.get('quality_flags', [])) & {'sensor_fault', 'suspect', 'missing'}: | |
| return 'rejected' | |
| stamp = record['acquired_at'] if record['product'] == 'forecast' else record['valid_at'] | |
| return 'stale' if (now - utc(stamp)).total_seconds() > max_age_hours * 3600 else 'valid' | |
| def ingest(obs: Observation): | |
| body = obs.model_dump(mode='json') | |
| identity = digest([obs.category, obs.source_id, obs.instrument_id, obs.message_id]) | |
| checksum = digest(body) | |
| with LOCK: | |
| try: | |
| old = store.get('observations', identity) | |
| except KeyError: | |
| old = None | |
| if old: | |
| if old['checksum'] != checksum: | |
| store.event('Conflicting duplicate observation rejected', obs.category, observation_id=identity) | |
| raise ValueError('Message identity already exists with different contents') | |
| return {**old, 'duplicate': True} | |
| record = store.put('observations', {**body, 'ingested_at': store.now(), 'checksum': checksum, | |
| 'validation_status': quality(body), 'processing_status': 'pending'}, identity) | |
| if record['validation_status'] != 'valid': | |
| store.event('Observation stored with ' + record['validation_status'] + ' quality', obs.category, observation_id=identity) | |
| return {**record, 'duplicate': False} | |
| def weather_adapter(weather): | |
| """Open-Meteo hourly precipitation is accumulation over the preceding hour.""" | |
| if weather.get('provenance') not in ('EXTERNAL FORECAST', 'SIMULATION FORECAST'): | |
| return [] # Reanalysis and unlocated CSV files must never become future evidence. | |
| loc = weather['location']; simulated = weather['provenance'] == 'SIMULATION FORECAST' | |
| unit = weather.get('units', {}).get('precipitation') | |
| if unit != 'mm': | |
| raise ValueError('Weather precipitation must declare millimeters') | |
| times, values = weather['hourly']['time'], weather['hourly']['precipitation'] | |
| if len(times) != len(values) or len(times) > 200 or len(set(times)) != len(times): | |
| raise ValueError('Weather timestamps and values must be unique aligned hourly series') | |
| parsed = [utc(t if t.endswith('Z') or '+' in t[10:] else t + '+00:00') for t in times] | |
| if parsed != sorted(parsed): | |
| raise ValueError('Weather series must be chronological') | |
| normalized = [] | |
| for t, value in zip(parsed, values): | |
| normalized.append(Observation(message_id=weather['id'] + ':' + t.isoformat(), | |
| source_id='satforge-replay' if simulated else 'open-meteo', instrument_id='synthetic-grid' if simulated else 'forecast-grid', | |
| platform='simulation' if simulated else 'external', category='simulation' if simulated else 'external', product='forecast', | |
| acquired_at=weather['retrieved_at'], valid_at=t, longitude=loc['requested_lon'], latitude=loc['requested_lat'], | |
| measurement='precipitation_mm', value=value, unit=unit, | |
| metadata={'weather_id': weather['id'], 'source_url': weather.get('source_url'), 'timestamp_basis': 'provider retrieval; issue time unavailable', | |
| 'spatial_support': 'single model grid point, not catchment average'})) | |
| # Validate the whole provider response before committing any evidence. | |
| return [ingest(o) for o in normalized] | |
| def telemetry_ingest(t): | |
| from .operations import power | |
| identity = digest([t.device_id, t.packet_id or t.measured_at.isoformat()]) | |
| checksum = digest(t.model_dump(mode='json')) | |
| with LOCK: | |
| try: | |
| old = store.get('telemetry', identity) | |
| except KeyError: | |
| old = None | |
| if old: | |
| if old.get('checksum') != checksum: | |
| raise ValueError('Conflicting telemetry packet identity') | |
| return {**old, 'duplicate': True} | |
| body = power(t) | |
| body.update(checksum=checksum, validation_status='stale' if (utc()-utc(t.measured_at)).total_seconds()>900 else 'valid', | |
| platform='ESP32 ground prototype; orbital platform unverified', category='observed') | |
| result = store.put('telemetry', body, identity) | |
| store.event('Telemetry packet validated', 'MEASURED', device_id=t.device_id, telemetry_id=identity) | |
| return {**result, 'duplicate': False} | |
| def mission_status(now=None): | |
| from .operations import schedule, EnergyPolicy | |
| now = utc(now) | |
| rows = sorted(store.all_records('telemetry'), key=lambda r: r['measured_at'], reverse=True) | |
| latest = rows[0] if rows else None | |
| age = max(0, (now-utc(latest['measured_at'])).total_seconds()) if latest else None | |
| recent = [r for r in rows if latest and r['device_id'] == latest['device_id']][:100] | |
| sequences = sorted(set(r['sequence'] for r in recent if r.get('sequence') is not None)) | |
| try: | |
| policy = EnergyPolicy(**{k:v for k,v in store.get('config','energy-policy').items() if k!='id'}) | |
| except KeyError: | |
| policy = EnergyPolicy() | |
| return {'status': 'NO TELEMETRY RECEIVED' if not latest else 'STALE TELEMETRY' if age>900 else 'RECENT GROUND TELEMETRY', | |
| 'last_packet_at': max((r['received_at'] for r in rows), default=None), 'last_measurement_at': latest['measured_at'] if latest else None, | |
| 'age_seconds': age, 'samples': rows[:500], 'orbital_connection': 'NOT VERIFIED', 'command_interface': 'NOT IMPLEMENTED', | |
| 'sequence_gaps': sum(max(0,b-a-1) for a,b in zip(sequences,sequences[1:])) if len(sequences)>1 else None, | |
| 'sequence_caveat': 'Only supplied sequence numbers for one device; reboot/wrap not inferred. Packet loss rate is unavailable.', | |
| 'policy': schedule(latest if latest and age<=900 else {}, policy), | |
| 'policy_scope': 'Ground prototype planning only; no spacecraft commands sent', | |
| 'instruments': {'BME280': 'optional environmental context; not ground rainfall', 'BH1750': 'optional illuminance', | |
| 'INA219': 'optional solar voltage/current', 'camera': 'disabled by default; board confirmation required', | |
| 'rain_gauge': 'not instrumented', 'water_level': 'not instrumented', 'orbital_radio': 'not configured'}} | |