Raster Viewer: first version
Browse files- README.md +61 -5
- app.js +310 -0
- build_index.py +72 -0
- footprints.js +68 -0
- footprints/DEM.parquet +3 -0
- footprints/S1RTC.parquet +3 -0
- footprints/S2L1C.parquet +3 -0
- footprints/S2L2A.parquet +3 -0
- footprints/datasets.json +52 -0
- index.html +223 -17
- labels.js +48 -0
- mt-icon.svg +22 -0
- parquet.js +39 -0
- places.json +0 -0
- samples.js +29 -0
- tiles.js +80 -0
README.md
CHANGED
|
@@ -1,10 +1,66 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: static
|
| 7 |
pinned: false
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Major TOM Raster Viewer
|
| 3 |
+
emoji: 🛰️
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: green
|
| 6 |
sdk: static
|
| 7 |
pinned: false
|
| 8 |
+
short_description: Major TOM Core samples on a 3D globe, read in the browser
|
| 9 |
---
|
| 10 |
|
| 11 |
+
> **CLAUDE-GENERATED DOCUMENT** — automated LLM-generated content. Verify before relying on it.
|
| 12 |
+
|
| 13 |
+
> **Review status:** draft for review, 2026-11-02. Space not yet created.
|
| 14 |
+
|
| 15 |
+
# Major TOM Raster Viewer
|
| 16 |
+
|
| 17 |
+
A web page that shows where the Major TOM Core datasets have samples, on a 3D globe, and loads the image of the
|
| 18 |
+
sample in a clicked cell. It runs in the browser and reads the Core Parquet files on Hugging Face directly: no
|
| 19 |
+
server. Plain HTML and JavaScript, libraries from jsDelivr at pinned versions.
|
| 20 |
+
|
| 21 |
+
The [Core Viewer](https://huggingface.co/spaces/Major-TOM/MajorTOM-Core-Viewer) (a Python server) stays as it is,
|
| 22 |
+
with its text search and the AlphaEarth and VIIRS layers.
|
| 23 |
+
|
| 24 |
+
## Flow
|
| 25 |
+
|
| 26 |
+
```
|
| 27 |
+
build_index.py (once): each Core dataset's metadata.parquet -> footprints/<name>.parquet, sorted by cell
|
| 28 |
+
panel: dataset
|
| 29 |
+
-> footprints.js: the two position columns of its footprint file -> cells with a sample -> coverage tiles
|
| 30 |
+
-> app.js: MapLibre globe; cursor shows the cell and whether it has a sample
|
| 31 |
+
click a cell -> its footprint rows (page index) -> the image file on Hugging Face (one row group per sample)
|
| 32 |
+
-> samples.js: that row group's thumbnail column -> panel, with acquisition time, cloud cover or orbit, product
|
| 33 |
+
```
|
| 34 |
+
|
| 35 |
+
## Files
|
| 36 |
+
|
| 37 |
+
| File | What it does |
|
| 38 |
+
|---|---|
|
| 39 |
+
| `index.html` | page layout and styles (as the Climate Globe) |
|
| 40 |
+
| `app.js` | globe, dataset list, loading flag, cursor, selected cell |
|
| 41 |
+
| `footprints.js` | a dataset's footprints: coverage mask, a cell's samples |
|
| 42 |
+
| `samples.js` | one sample's thumbnail from its image file |
|
| 43 |
+
| `parquet.js` | range requests for hyparquet, with bytes and time counted |
|
| 44 |
+
| `tiles.js`, `labels.js`, `places.json` | copied from the Climate Globe: cells and tiles, place names |
|
| 45 |
+
| `build_index.py` | Core `metadata.parquet` -> `footprints/<name>.parquet` and `footprints/datasets.json` |
|
| 46 |
+
| `footprints/` | built files, not in git: uploaded to the Space once (S2L2A and S2L1C 20.5 MB each, S1RTC 12.1 MB, DEM 4.5 MB) |
|
| 47 |
+
|
| 48 |
+
## Reads
|
| 49 |
+
|
| 50 |
+
| Read | Bytes | Time |
|
| 51 |
+
|---|---|---|
|
| 52 |
+
| S2L2A footprints (coverage of 2,245,886 samples) | 3.9 MB in 37 requests | 0.7 s |
|
| 53 |
+
| One S2L2A thumbnail, first from its image file (footer 1.5 MB) | 1.1 MB in 4 requests | 1.8 s |
|
| 54 |
+
|
| 55 |
+
Reading each dataset's `metadata.parquet` from Hugging Face instead was measured first: S2L2A's has one row group
|
| 56 |
+
per image file (4,492), so its positions took 462 s in 4,492 requests; S1RTC's and DEM's have two row groups, so
|
| 57 |
+
one sample's row cost 23-27 MB. Hence the footprint files.
|
| 58 |
+
|
| 59 |
+
## Run locally
|
| 60 |
+
|
| 61 |
+
```
|
| 62 |
+
cd apps/raster-viewer && python build_index.py /data/Major-TOM footprints # once
|
| 63 |
+
pytest -m network # serves the folder with byte ranges
|
| 64 |
+
```
|
| 65 |
+
|
| 66 |
+
The page needs a server that answers HTTP range requests (the tests' server does; `python -m http.server` does not).
|
app.js
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// The page: a MapLibre globe of where a Core dataset has samples, and the image of the sample in a clicked cell.
|
| 2 |
+
//
|
| 3 |
+
// dataset (panel) -> footprints/<name>.parquet, two position columns (footprints.js) -> mask on the cell array
|
| 4 |
+
// -> coverage tiles drawn in the browser (tiles.js) -> globe
|
| 5 |
+
// click -> cell -> its samples (footprint rows, page index) -> image file on Hugging Face, one row group,
|
| 6 |
+
// thumbnail column (samples.js) -> panel
|
| 7 |
+
// cursor -> cell under it -> outline + name, position, whether it has a sample
|
| 8 |
+
|
| 9 |
+
import * as maplibregl from 'https://cdn.jsdelivr.net/npm/maplibre-gl@6.14.0/dist/maplibre-gl.mjs'
|
| 10 |
+
import { feature } from 'https://cdn.jsdelivr.net/npm/topojson-client@3.1.0/+esm'
|
| 11 |
+
import { DATASETS, loadCoverage, samplesAt } from './footprints.js'
|
| 12 |
+
import { placeLabels } from './labels.js'
|
| 13 |
+
import { imageUrl, loadThumbnail } from './samples.js'
|
| 14 |
+
import { cellAt, cellIndex, renderTile } from './tiles.js'
|
| 15 |
+
|
| 16 |
+
const $ = id => document.getElementById(id)
|
| 17 |
+
const COLOURS = { S2L2A: [57, 181, 74], S2L1C: [0, 113, 188], S1RTC: [241, 90, 36], DEM: [180, 140, 90] }
|
| 18 |
+
|
| 19 |
+
const state = { name: null, info: null, coverage: null, version: 0, loading: 0 }
|
| 20 |
+
|
| 21 |
+
// ---- map ---------------------------------------------------------------------------------------------------
|
| 22 |
+
|
| 23 |
+
/** Coverage of the shown dataset: its colour where a cell has a sample, transparent elsewhere. */
|
| 24 |
+
const field = {
|
| 25 |
+
pixel(lon, lat, out, o) {
|
| 26 |
+
const mask = state.coverage?.mask
|
| 27 |
+
if (!mask || !mask[cellIndex(lon, lat)]) { out[o + 3] = 0; return }
|
| 28 |
+
const [r, g, b] = COLOURS[state.name] ?? [57, 181, 74]
|
| 29 |
+
out[o] = r; out[o + 1] = g; out[o + 2] = b; out[o + 3] = 130 // coastlines show through
|
| 30 |
+
},
|
| 31 |
+
}
|
| 32 |
+
maplibregl.addProtocol('mt', async params => {
|
| 33 |
+
const [z, x, y] = params.url.split('/').slice(-3).map(Number)
|
| 34 |
+
return { data: await renderTile(field, z, x, y) }
|
| 35 |
+
})
|
| 36 |
+
|
| 37 |
+
const map = new maplibregl.Map({
|
| 38 |
+
container: 'map', center: [12, 22], zoom: innerWidth < 820 ? 1.3 : 2.25, attributionControl: false, maxZoom: 12,
|
| 39 |
+
style: {
|
| 40 |
+
version: 8, projection: { type: 'globe' },
|
| 41 |
+
sky: { 'atmosphere-blend': ['interpolate', ['linear'], ['zoom'], 0, 0.6, 4, 0.3, 7, 0] },
|
| 42 |
+
sources: {
|
| 43 |
+
field: { type: 'raster', tiles: ['mt://0/{z}/{x}/{y}'], tileSize: 256, maxzoom: 9 },
|
| 44 |
+
land: { type: 'geojson', data: { type: 'FeatureCollection', features: [] } },
|
| 45 |
+
coast: { type: 'geojson', data: { type: 'FeatureCollection', features: [] } },
|
| 46 |
+
cell: { type: 'geojson', data: { type: 'FeatureCollection', features: [] } },
|
| 47 |
+
selected: { type: 'geojson', data: { type: 'FeatureCollection', features: [] } },
|
| 48 |
+
},
|
| 49 |
+
layers: [
|
| 50 |
+
{ id: 'ocean', type: 'background', paint: { 'background-color': '#2a2d33' } },
|
| 51 |
+
{ id: 'land', type: 'fill', source: 'land', paint: { 'fill-color': '#3b3f47', 'fill-antialias': false } },
|
| 52 |
+
{ id: 'field', type: 'raster', source: 'field', paint: { 'raster-resampling': 'nearest', 'raster-fade-duration': 0 } },
|
| 53 |
+
{ id: 'coast', type: 'line', source: 'coast', paint: { 'line-color': 'rgba(20,20,24,0.55)', 'line-width': 0.6 } },
|
| 54 |
+
{ id: 'cell', type: 'line', source: 'cell', paint: { 'line-color': '#f15a24', 'line-width': 1.25, 'line-opacity': 0.8 } },
|
| 55 |
+
{ id: 'selected', type: 'line', source: 'selected', paint: { 'line-color': '#f15a24', 'line-width': 2.5 } },
|
| 56 |
+
],
|
| 57 |
+
},
|
| 58 |
+
})
|
| 59 |
+
map.addControl(new maplibregl.NavigationControl({ showCompass: false }), 'bottom-right')
|
| 60 |
+
const places = placeLabels(map, $('places'))
|
| 61 |
+
$('show-places').addEventListener('change', e => places.show(e.target.checked))
|
| 62 |
+
map.addControl(new maplibregl.AttributionControl({ compact: true,
|
| 63 |
+
customAttribution: 'Coastlines and places: Natural Earth · Data: <a href="https://huggingface.co/Major-TOM">Major TOM Core</a>' }))
|
| 64 |
+
|
| 65 |
+
map.on('load', async () => {
|
| 66 |
+
const topo = await (await fetch('https://cdn.jsdelivr.net/npm/world-atlas@2.0.2/land-50m.json')).json()
|
| 67 |
+
map.getSource('coast').setData(coastLines(feature(topo, topo.objects.land)))
|
| 68 |
+
map.getSource('land').setData(continuousRings(feature(topo, topo.objects.land)))
|
| 69 |
+
const datasets = await DATASETS
|
| 70 |
+
buildDatasetList(datasets)
|
| 71 |
+
setDataset(Object.keys(datasets)[0], datasets)
|
| 72 |
+
})
|
| 73 |
+
|
| 74 |
+
/** Land polygons whose rings step across the date line made continuous (as climate-globe/app.js). */
|
| 75 |
+
function continuousRings(land) {
|
| 76 |
+
const unwrap = ring => {
|
| 77 |
+
if (ring.some(([, y]) => y <= -89.99)) return ring
|
| 78 |
+
const out = [ring[0]]
|
| 79 |
+
for (let i = 1, shift = 0; i < ring.length; i++) {
|
| 80 |
+
const dx = ring[i][0] + shift - out[i - 1][0]
|
| 81 |
+
if (dx > 180) shift -= 360; else if (dx < -180) shift += 360
|
| 82 |
+
out.push([ring[i][0] + shift, ring[i][1]])
|
| 83 |
+
}
|
| 84 |
+
return out
|
| 85 |
+
}
|
| 86 |
+
for (const f of land.features) {
|
| 87 |
+
const g = f.geometry
|
| 88 |
+
g.coordinates = g.type === 'Polygon' ? g.coordinates.map(unwrap) : g.coordinates.map(p => p.map(unwrap))
|
| 89 |
+
}
|
| 90 |
+
return land
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
/** Coastlines from the land polygons, without the edges that are not coast (as climate-globe/app.js). */
|
| 94 |
+
function coastLines(land) {
|
| 95 |
+
const artificial = ([x1, y1], [x2, y2]) => (Math.abs(x1) >= 179.99 && Math.abs(x2) >= 179.99) ||
|
| 96 |
+
(y1 <= -89.99 && y2 <= -89.99) || Math.abs(x2 - x1) > 180
|
| 97 |
+
const lines = []
|
| 98 |
+
for (const f of land.features) {
|
| 99 |
+
const polygons = f.geometry.type === 'Polygon' ? [f.geometry.coordinates] : f.geometry.coordinates
|
| 100 |
+
for (const ring of polygons.flat()) {
|
| 101 |
+
let line = [ring[0]]
|
| 102 |
+
for (let i = 1; i < ring.length; i++) {
|
| 103 |
+
if (artificial(ring[i - 1], ring[i])) { if (line.length > 1) lines.push(line); line = [ring[i]] }
|
| 104 |
+
else line.push(ring[i])
|
| 105 |
+
}
|
| 106 |
+
if (line.length > 1) lines.push(line)
|
| 107 |
+
}
|
| 108 |
+
}
|
| 109 |
+
return { type: 'Feature', geometry: { type: 'MultiLineString', coordinates: lines } }
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
// ---- cursor ------------------------------------------------------------------------------------------------
|
| 113 |
+
|
| 114 |
+
let cursor = null
|
| 115 |
+
|
| 116 |
+
/** Whether a screen point lies on the globe (not the space around it). */
|
| 117 |
+
function onGlobe(point, lngLat) {
|
| 118 |
+
if (typeof map.transform?.isPointOnMapSurface === 'function') return map.transform.isPointOnMapSurface(point)
|
| 119 |
+
const back = map.project(lngLat)
|
| 120 |
+
return Math.hypot(back.x - point.x, back.y - point.y) < 2
|
| 121 |
+
}
|
| 122 |
+
map.on('mousemove', e => {
|
| 123 |
+
if (onGlobe(e.point, e.lngLat)) { cursor = e.lngLat; describe() } else clearCursor()
|
| 124 |
+
})
|
| 125 |
+
window.globe = map // for automated checks
|
| 126 |
+
window.pickState = () => pick
|
| 127 |
+
|
| 128 |
+
const outline = ([w, s, e, n]) => ({ type: 'Feature', geometry: { type: 'Polygon', coordinates: [[[w, s], [e, s], [e, n], [w, n], [w, s]]] } })
|
| 129 |
+
|
| 130 |
+
function describe() {
|
| 131 |
+
if (!cursor) return
|
| 132 |
+
const cell = cellAt(cursor.lng, cursor.lat)
|
| 133 |
+
map.getSource('cell').setData(outline(cell.bounds))
|
| 134 |
+
$('cursor-cell').textContent = cell.name
|
| 135 |
+
$('cursor-pos').textContent = `${cursor.lat.toFixed(2)}°, ${cursor.lng.toFixed(2)}°`
|
| 136 |
+
$('cursor-value').textContent = !state.coverage ? '–' : state.coverage.mask[cell.index] ? 'yes, click to load' : 'none'
|
| 137 |
+
}
|
| 138 |
+
function clearCursor() {
|
| 139 |
+
cursor = null
|
| 140 |
+
map.getSource('cell').setData({ type: 'FeatureCollection', features: [] })
|
| 141 |
+
for (const id of ['cursor-cell', 'cursor-pos', 'cursor-value']) $(id).textContent = '–'
|
| 142 |
+
}
|
| 143 |
+
map.getCanvas().addEventListener('mouseleave', clearCursor)
|
| 144 |
+
|
| 145 |
+
// ---- loading indicator (as climate-globe/app.js) -------------------------------------------------------------
|
| 146 |
+
|
| 147 |
+
const bars = [...document.querySelectorAll('#loader polygon')] // 4 columns x 5, each top to bottom
|
| 148 |
+
const still = matchMedia('(prefers-reduced-motion: reduce)').matches
|
| 149 |
+
let loaderTimer = null, animation = null, tick = 0
|
| 150 |
+
|
| 151 |
+
function animateFlag() {
|
| 152 |
+
const n = bars.length, k = tick++ % (2 * n) // first n ticks fill, next n empty
|
| 153 |
+
if (k < n) bars[k].classList.add('on')
|
| 154 |
+
else { const i = k - n, col = Math.floor(i / 5); bars[col * 5 + 4 - (i % 5)].classList.remove('on') }
|
| 155 |
+
}
|
| 156 |
+
function startLoader(text) {
|
| 157 |
+
for (const b of bars) b.classList.remove('on')
|
| 158 |
+
tick = 0
|
| 159 |
+
$('loader-phase').textContent = 'Loading'
|
| 160 |
+
$('loader-text').textContent = text
|
| 161 |
+
$('loader-bytes').textContent = ''
|
| 162 |
+
clearTimeout(loaderTimer); clearInterval(animation)
|
| 163 |
+
loaderTimer = setTimeout(() => { // fast and cached loads: no flash
|
| 164 |
+
$('loader').hidden = false
|
| 165 |
+
if (still) bars.slice(0, 10).forEach(b => b.classList.add('on'))
|
| 166 |
+
else animation = setInterval(animateFlag, 70)
|
| 167 |
+
}, 150)
|
| 168 |
+
}
|
| 169 |
+
function loaderProgress(bytes, expected) {
|
| 170 |
+
$('loader-bytes').textContent = expected ? `${(bytes / 1e6).toFixed(1)} of ${(Math.max(expected, bytes) / 1e6).toFixed(1)} MB`
|
| 171 |
+
: `${(bytes / 1e6).toFixed(1)} MB`
|
| 172 |
+
}
|
| 173 |
+
function stopLoader() {
|
| 174 |
+
clearTimeout(loaderTimer); clearInterval(animation)
|
| 175 |
+
$('loader').hidden = true
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
// ---- dataset -----------------------------------------------------------------------------------------------
|
| 179 |
+
|
| 180 |
+
const kb = bytes => (bytes >= 1e6 ? `${(bytes / 1e6).toFixed(1)} MB` : `${Math.round(bytes / 1e3)} KB`)
|
| 181 |
+
const secs = ms => `${(ms / 1000).toFixed(1)} s`
|
| 182 |
+
|
| 183 |
+
function buildDatasetList(datasets) {
|
| 184 |
+
$('datasets').replaceChildren(...Object.entries(datasets).map(([name, d]) => {
|
| 185 |
+
const label = document.createElement('label')
|
| 186 |
+
label.className = 'option'
|
| 187 |
+
label.innerHTML = `<input type="radio" name="dataset" value="${name}"><span><b>${d.label}</b>` +
|
| 188 |
+
`<small>Core-${name} · ${d.samples.toLocaleString('en')} samples</small></span>`
|
| 189 |
+
label.querySelector('input').addEventListener('change', () => setDataset(name, datasets))
|
| 190 |
+
return label
|
| 191 |
+
}))
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
async function setDataset(name, datasets) {
|
| 195 |
+
const ticket = ++state.loading, info = datasets[name]
|
| 196 |
+
state.name = name; state.info = info; state.coverage = null
|
| 197 |
+
document.querySelector(`input[name=dataset][value="${name}"]`).checked = true
|
| 198 |
+
$('dataset-link').href = `https://huggingface.co/datasets/Major-TOM/Core-${name}`
|
| 199 |
+
$('dataset-link').textContent = `huggingface.co/datasets/Major-TOM/Core-${name}`
|
| 200 |
+
$('status').textContent = ''
|
| 201 |
+
startLoader(`${info.label} · footprints of ${info.samples.toLocaleString('en')} samples`)
|
| 202 |
+
let coverage
|
| 203 |
+
try {
|
| 204 |
+
coverage = await loadCoverage(name, (bytes, expected) => { if (ticket === state.loading) loaderProgress(bytes, expected) })
|
| 205 |
+
} catch (error) {
|
| 206 |
+
if (ticket === state.loading) { stopLoader(); $('status').textContent = `Could not load: ${error.message}` }
|
| 207 |
+
return
|
| 208 |
+
}
|
| 209 |
+
if (ticket !== state.loading) return
|
| 210 |
+
state.coverage = coverage
|
| 211 |
+
$('loader-phase').textContent = 'Drawing'
|
| 212 |
+
const t0 = performance.now()
|
| 213 |
+
map.getSource('field').setTiles([`mt://${++state.version}/{z}/{x}/{y}`])
|
| 214 |
+
describe()
|
| 215 |
+
if (pick.cell) pickCell()
|
| 216 |
+
map.once('idle', () => {
|
| 217 |
+
if (ticket !== state.loading) return
|
| 218 |
+
$('stat-footprints').textContent = `${kb(coverage.stats.bytes)} in ${coverage.stats.requests} requests · ${secs(coverage.stats.read)}`
|
| 219 |
+
$('stat-samples').textContent = coverage.samples.toLocaleString('en')
|
| 220 |
+
$('stat-draw').textContent = secs(performance.now() - t0)
|
| 221 |
+
document.body.dataset.loaded = name // for automated checks
|
| 222 |
+
stopLoader()
|
| 223 |
+
})
|
| 224 |
+
}
|
| 225 |
+
|
| 226 |
+
// ---- selected cell -----------------------------------------------------------------------------------------
|
| 227 |
+
// click -> cell -> its samples (footprint rows) -> buttons when there are several -> thumbnail of the chosen one
|
| 228 |
+
|
| 229 |
+
const pick = { cell: null, samples: [], chosen: 0, thumbnail: null, ticket: 0 }
|
| 230 |
+
|
| 231 |
+
map.on('click', e => {
|
| 232 |
+
if (!onGlobe(e.point, e.lngLat)) return
|
| 233 |
+
pick.cell = { ...cellAt(e.lngLat.lng, e.lngLat.lat), lng: e.lngLat.lng, lat: e.lngLat.lat }
|
| 234 |
+
map.getSource('selected').setData(outline(pick.cell.bounds))
|
| 235 |
+
$('pick').hidden = false
|
| 236 |
+
$('pick-cell').textContent = pick.cell.name
|
| 237 |
+
$('pick-pos').textContent = `${pick.cell.lat.toFixed(2)}°, ${pick.cell.lng.toFixed(2)}°`
|
| 238 |
+
$('pick').scrollIntoView({ block: 'nearest', behavior: 'smooth' })
|
| 239 |
+
pickCell()
|
| 240 |
+
})
|
| 241 |
+
$('pick-clear').addEventListener('click', () => {
|
| 242 |
+
pick.cell = null; pick.ticket++; $('pick').hidden = true
|
| 243 |
+
map.getSource('selected').setData({ type: 'FeatureCollection', features: [] })
|
| 244 |
+
})
|
| 245 |
+
|
| 246 |
+
/** The selected cell in the shown dataset: its samples, then the first one's thumbnail. */
|
| 247 |
+
async function pickCell() {
|
| 248 |
+
const ticket = ++pick.ticket, { row, col } = pick.cell, name = state.name
|
| 249 |
+
pick.samples = []; pick.thumbnail = null
|
| 250 |
+
showThumbnail(null, 'Looking up the cell…')
|
| 251 |
+
$('choices').replaceChildren(); $('pick-meta').replaceChildren(); $('pick-stats').textContent = ''
|
| 252 |
+
let samples
|
| 253 |
+
try {
|
| 254 |
+
samples = await samplesAt(name, row, col)
|
| 255 |
+
} catch (error) {
|
| 256 |
+
if (ticket === pick.ticket) showThumbnail(null, `Could not load: ${error.message}`)
|
| 257 |
+
return
|
| 258 |
+
}
|
| 259 |
+
if (ticket !== pick.ticket) return
|
| 260 |
+
pick.samples = samples
|
| 261 |
+
if (!samples.length) { showThumbnail(null, `No ${state.info.label} sample in this cell`); return }
|
| 262 |
+
if (samples.length > 1) {
|
| 263 |
+
$('choices').replaceChildren(...samples.map((s, i) => {
|
| 264 |
+
const b = document.createElement('button')
|
| 265 |
+
b.textContent = s.timestamp ? when(s.timestamp) : `sample ${i + 1}`
|
| 266 |
+
b.addEventListener('click', () => choose(i))
|
| 267 |
+
return b
|
| 268 |
+
}))
|
| 269 |
+
}
|
| 270 |
+
choose(0)
|
| 271 |
+
}
|
| 272 |
+
|
| 273 |
+
async function choose(i) {
|
| 274 |
+
const ticket = ++pick.ticket, sample = pick.samples[i], info = state.info
|
| 275 |
+
pick.chosen = i
|
| 276 |
+
for (const [k, b] of [...$('choices').children].entries()) b.setAttribute('aria-pressed', String(k === i))
|
| 277 |
+
meta(sample, null, info)
|
| 278 |
+
showThumbnail(null, 'Loading the thumbnail…')
|
| 279 |
+
try {
|
| 280 |
+
const t = await loadThumbnail(info, sample)
|
| 281 |
+
if (ticket !== pick.ticket) { URL.revokeObjectURL(t.src); return }
|
| 282 |
+
if (pick.thumbnail) URL.revokeObjectURL(pick.thumbnail.src)
|
| 283 |
+
pick.thumbnail = t
|
| 284 |
+
showThumbnail(t.src)
|
| 285 |
+
meta(sample, t.productId, info)
|
| 286 |
+
$('pick-stats').textContent = `Thumbnail: ${kb(t.stats.bytes)} in ${t.stats.requests} requests · ${secs(t.stats.read)}`
|
| 287 |
+
} catch (error) {
|
| 288 |
+
if (ticket === pick.ticket) showThumbnail(null, `Could not load: ${error.message}`)
|
| 289 |
+
}
|
| 290 |
+
}
|
| 291 |
+
|
| 292 |
+
function showThumbnail(src, note = '') {
|
| 293 |
+
$('thumb').hidden = !src
|
| 294 |
+
if (src) $('thumb').src = src
|
| 295 |
+
$('thumb-note').textContent = src ? '' : note
|
| 296 |
+
}
|
| 297 |
+
|
| 298 |
+
/** "20200213T023821" -> "2020-02-13 02:38 UTC". */
|
| 299 |
+
const when = t => `${t.slice(0, 4)}-${t.slice(4, 6)}-${t.slice(6, 8)} ${t.slice(9, 11)}:${t.slice(11, 13)} UTC`
|
| 300 |
+
|
| 301 |
+
function meta(sample, productId, info) {
|
| 302 |
+
const rows = []
|
| 303 |
+
if (sample.timestamp) rows.push(['Acquired', when(sample.timestamp)])
|
| 304 |
+
if (sample.cloud_cover !== undefined && sample.cloud_cover !== null) rows.push(['Cloud cover', `${sample.cloud_cover.toFixed(1)}%`])
|
| 305 |
+
if (sample.orbit_state) rows.push(['Orbit', sample.orbit_state])
|
| 306 |
+
if (productId) rows.push(['Product', productId])
|
| 307 |
+
const url = imageUrl(info, sample.part)
|
| 308 |
+
rows.push(['File', `<a href="${url.replace('/resolve/', '/blob/')}" target="_blank" rel="noopener">part_${String(sample.part).padStart(5, '0')}</a>, row ${sample.parquet_row}`])
|
| 309 |
+
$('pick-meta').innerHTML = rows.map(([k, v]) => `<dt>${k}</dt><dd>${v}</dd>`).join('')
|
| 310 |
+
}
|
build_index.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Footprint index of each Core dataset, for the Raster Viewer: where its samples are and where each one is stored.
|
| 2 |
+
|
| 3 |
+
<core>/Core-<name>/metadata.parquet (one row per sample: grid_row_u, grid_col_r, parquet_url, parquet_row, ...)
|
| 4 |
+
-> parquet_url -> part number (every URL is <base>/images/part_NNNNN.parquet; checked)
|
| 5 |
+
-> rows sorted by grid_row_u, grid_col_r
|
| 6 |
+
-> footprints/<name>.parquet: grid_row_u, grid_col_r, part, parquet_row, timestamp, + cloud_cover / orbit_state
|
| 7 |
+
row groups of 65,536 rows with statistics and a page index, no value dictionaries (as the climate files):
|
| 8 |
+
the app reads the two position columns for the coverage map and, for a click, the page holding the cell
|
| 9 |
+
-> footprints/datasets.json: per dataset its label, the URL base of its image files, its samples
|
| 10 |
+
|
| 11 |
+
python build_index.py /data/Major-TOM <out folder> (Slurm, .venv311)
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import json
|
| 16 |
+
import re
|
| 17 |
+
import sys
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
import numpy as np
|
| 21 |
+
import pyarrow as pa
|
| 22 |
+
import pyarrow.compute as pc
|
| 23 |
+
import pyarrow.parquet as pq
|
| 24 |
+
|
| 25 |
+
DATASETS = {"S2L2A": "Sentinel-2 L2A", "S2L1C": "Sentinel-2 L1C", "S1RTC": "Sentinel-1 RTC", "DEM": "COP-DEM"}
|
| 26 |
+
EXTRA = ["timestamp", "cloud_cover", "orbit_state"] # kept where a dataset has them
|
| 27 |
+
ROW_GROUP = 65_536
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def index(metadata: Path, name: str) -> tuple[pa.Table, str]:
|
| 31 |
+
"""The footprint table of one dataset, and the URL base of its image files."""
|
| 32 |
+
have = pq.read_schema(metadata).names
|
| 33 |
+
table = pq.read_table(metadata, columns=["grid_row_u", "grid_col_r", "parquet_url", "parquet_row",
|
| 34 |
+
*[c for c in EXTRA if c in have]])
|
| 35 |
+
base = f"https://huggingface.co/datasets/Major-TOM/Core-{name}/resolve/main/images/"
|
| 36 |
+
urls = table.column("parquet_url").to_pylist()
|
| 37 |
+
part = [re.fullmatch(re.escape(base) + r"part_(\d{5})\.parquet", u) for u in urls]
|
| 38 |
+
if not all(part):
|
| 39 |
+
raise ValueError(f"{metadata}: a parquet_url outside {base}part_NNNNN.parquet")
|
| 40 |
+
out = pa.table({"grid_row_u": pc.cast(table.column("grid_row_u"), pa.int16()),
|
| 41 |
+
"grid_col_r": pc.cast(table.column("grid_col_r"), pa.int16()),
|
| 42 |
+
"part": pa.array(np.array([int(m[1]) for m in part], np.int16)),
|
| 43 |
+
"parquet_row": pc.cast(table.column("parquet_row"), pa.int16()),
|
| 44 |
+
**{c: table.column(c) for c in EXTRA if c in have}})
|
| 45 |
+
if "cloud_cover" in out.column_names:
|
| 46 |
+
out = out.set_column(out.schema.get_field_index("cloud_cover"), "cloud_cover",
|
| 47 |
+
pc.cast(out.column("cloud_cover"), pa.float32()))
|
| 48 |
+
return out.sort_by([("grid_row_u", "ascending"), ("grid_col_r", "ascending")]), base
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def write(table: pa.Table, path: Path) -> None:
|
| 52 |
+
keys = ["grid_row_u", "grid_col_r", "orbit_state"]
|
| 53 |
+
pq.write_table(table, path, row_group_size=ROW_GROUP, compression="zstd", write_statistics=True,
|
| 54 |
+
write_page_index=True, use_dictionary=[c for c in keys if c in table.column_names],
|
| 55 |
+
data_page_size=4096)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def main(core: str, out: str) -> None:
|
| 59 |
+
core, out = Path(core), Path(out)
|
| 60 |
+
out.mkdir(parents=True, exist_ok=True)
|
| 61 |
+
listing = {}
|
| 62 |
+
for name, label in DATASETS.items():
|
| 63 |
+
table, base = index(core / f"Core-{name}" / "metadata.parquet", name)
|
| 64 |
+
write(table, out / f"{name}.parquet")
|
| 65 |
+
listing[name] = {"label": label, "images": base, "samples": table.num_rows,
|
| 66 |
+
"columns": table.column_names}
|
| 67 |
+
print(f"{name}: {table.num_rows:,} samples -> {(out / f'{name}.parquet').stat().st_size / 1e6:.1f} MB", flush=True)
|
| 68 |
+
(out / "datasets.json").write_text(json.dumps(listing, indent=1) + "\n")
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
if __name__ == "__main__":
|
| 72 |
+
main(*sys.argv[1:3])
|
footprints.js
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Where a Core dataset's samples are, and where each one is stored: its footprint file (build_index.py).
|
| 2 |
+
//
|
| 3 |
+
// footprints/<name>.parquet: one row per sample, sorted by grid_row_u, grid_col_r; page index
|
| 4 |
+
// -> loadCoverage(): the two position columns -> index on the (row, column) array of every sample, ascending
|
| 5 |
+
// -> mask (coverage tiles) and, for a click, a binary search for the cell's rows
|
| 6 |
+
// -> samplesAt(): the cell's rows only, through the page index -> part, parquet_row, timestamp, ...
|
| 7 |
+
|
| 8 |
+
import { COLS, ROWS } from './tiles.js'
|
| 9 |
+
import { compressors, countedFile, newStats, parquetMetadataAsync, parquetRead } from './parquet.js'
|
| 10 |
+
|
| 11 |
+
export const DATASETS = fetch('footprints/datasets.json' + location.search).then(r => r.json())
|
| 12 |
+
|
| 13 |
+
const footprintUrl = name => new URL(`footprints/${name}.parquet`, location.href).href.split('?')[0] + location.search
|
| 14 |
+
const loaded = new Map() // name -> {file, metadata, index, mask}
|
| 15 |
+
|
| 16 |
+
/** A dataset's samples: index of each on the (row, column) array (ascending, the file's order) and a mask of the
|
| 17 |
+
* cells with a sample. onProgress(bytes so far, bytes expected). Read once per dataset. */
|
| 18 |
+
export function loadCoverage(name, onProgress = () => {}) {
|
| 19 |
+
if (!loaded.has(name)) loaded.set(name, read(name, onProgress))
|
| 20 |
+
return loaded.get(name)
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
async function read(name, onProgress) {
|
| 24 |
+
const stats = newStats(), file = await countedFile(footprintUrl(name), stats)
|
| 25 |
+
const metadata = await parquetMetadataAsync(file)
|
| 26 |
+
let expected = 0
|
| 27 |
+
for (const g of metadata.row_groups) {
|
| 28 |
+
for (const { meta_data: m } of g.columns) {
|
| 29 |
+
if (['grid_row_u', 'grid_col_r'].includes(m.path_in_schema[0])) expected += Number(m.total_compressed_size)
|
| 30 |
+
}
|
| 31 |
+
}
|
| 32 |
+
const n = Number(metadata.num_rows), index = new Int32Array(n), mask = new Uint8Array(ROWS * COLS)
|
| 33 |
+
const rows = new Int16Array(n), cols = new Int16Array(n)
|
| 34 |
+
await parquetRead({
|
| 35 |
+
file, metadata, compressors, columns: ['grid_row_u', 'grid_col_r'],
|
| 36 |
+
onChunk({ columnName, columnData, rowStart }) {
|
| 37 |
+
if (columnName === 'grid_row_u') rows.set(columnData, rowStart)
|
| 38 |
+
else cols.set(columnData, rowStart)
|
| 39 |
+
onProgress(stats.bytes, expected)
|
| 40 |
+
},
|
| 41 |
+
})
|
| 42 |
+
for (let i = 0; i < n; i++) {
|
| 43 |
+
index[i] = (rows[i] + ROWS / 2) * COLS + cols[i] + COLS / 2
|
| 44 |
+
mask[index[i]] = 1
|
| 45 |
+
}
|
| 46 |
+
stats.read = performance.now() - stats.t0
|
| 47 |
+
return { file, metadata, index, mask, stats, samples: n }
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
/** First position in the ascending `index` not below `k`. */
|
| 51 |
+
function lowerBound(index, k) {
|
| 52 |
+
let lo = 0, hi = index.length
|
| 53 |
+
while (lo < hi) { const mid = (lo + hi) >> 1; if (index[mid] < k) lo = mid + 1; else hi = mid }
|
| 54 |
+
return lo
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
/** The samples of one cell (usually one; a few cells have two): their rows of the footprint file, read through
|
| 58 |
+
* the page index, as objects with that file's columns. */
|
| 59 |
+
export async function samplesAt(name, row, col) {
|
| 60 |
+
const { file, metadata, index } = await loadCoverage(name), k = (row + ROWS / 2) * COLS + col + COLS / 2
|
| 61 |
+
const first = lowerBound(index, k), last = lowerBound(index, k + 1)
|
| 62 |
+
if (first === last) return []
|
| 63 |
+
const columns = metadata.schema.slice(1).map(c => c.name)
|
| 64 |
+
let out = []
|
| 65 |
+
await parquetRead({ file, metadata, compressors, useOffsetIndex: true, rowStart: first, rowEnd: last, rowFormat: 'object',
|
| 66 |
+
onComplete(rows) { out = rows } })
|
| 67 |
+
return out.map(r => Object.fromEntries(columns.map(c => [c, typeof r[c] === 'bigint' ? Number(r[c]) : r[c]])))
|
| 68 |
+
}
|
footprints/DEM.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6b4d67e3b790f7f216374a27a3b9289db9fd63e260e4ab9ffc5e9d15ba057347
|
| 3 |
+
size 4482569
|
footprints/S1RTC.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3089bd10a123b5052a5ee5a8c2956822f182c7401231b2f67cb9c30a84322ac3
|
| 3 |
+
size 12103409
|
footprints/S2L1C.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1cc52c2e3b5ad3cf415b136eaa8cd93761b8fec5a9828d7a00ad88f5d1159752
|
| 3 |
+
size 20484283
|
footprints/S2L2A.parquet
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1cc52c2e3b5ad3cf415b136eaa8cd93761b8fec5a9828d7a00ad88f5d1159752
|
| 3 |
+
size 20484283
|
footprints/datasets.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"S2L2A": {
|
| 3 |
+
"label": "Sentinel-2 L2A",
|
| 4 |
+
"images": "https://huggingface.co/datasets/Major-TOM/Core-S2L2A/resolve/main/images/",
|
| 5 |
+
"samples": 2245886,
|
| 6 |
+
"columns": [
|
| 7 |
+
"grid_row_u",
|
| 8 |
+
"grid_col_r",
|
| 9 |
+
"part",
|
| 10 |
+
"parquet_row",
|
| 11 |
+
"timestamp",
|
| 12 |
+
"cloud_cover"
|
| 13 |
+
]
|
| 14 |
+
},
|
| 15 |
+
"S2L1C": {
|
| 16 |
+
"label": "Sentinel-2 L1C",
|
| 17 |
+
"images": "https://huggingface.co/datasets/Major-TOM/Core-S2L1C/resolve/main/images/",
|
| 18 |
+
"samples": 2245886,
|
| 19 |
+
"columns": [
|
| 20 |
+
"grid_row_u",
|
| 21 |
+
"grid_col_r",
|
| 22 |
+
"part",
|
| 23 |
+
"parquet_row",
|
| 24 |
+
"timestamp",
|
| 25 |
+
"cloud_cover"
|
| 26 |
+
]
|
| 27 |
+
},
|
| 28 |
+
"S1RTC": {
|
| 29 |
+
"label": "Sentinel-1 RTC",
|
| 30 |
+
"images": "https://huggingface.co/datasets/Major-TOM/Core-S1RTC/resolve/main/images/",
|
| 31 |
+
"samples": 1469955,
|
| 32 |
+
"columns": [
|
| 33 |
+
"grid_row_u",
|
| 34 |
+
"grid_col_r",
|
| 35 |
+
"part",
|
| 36 |
+
"parquet_row",
|
| 37 |
+
"timestamp",
|
| 38 |
+
"orbit_state"
|
| 39 |
+
]
|
| 40 |
+
},
|
| 41 |
+
"DEM": {
|
| 42 |
+
"label": "COP-DEM",
|
| 43 |
+
"images": "https://huggingface.co/datasets/Major-TOM/Core-DEM/resolve/main/images/",
|
| 44 |
+
"samples": 1837843,
|
| 45 |
+
"columns": [
|
| 46 |
+
"grid_row_u",
|
| 47 |
+
"grid_col_r",
|
| 48 |
+
"part",
|
| 49 |
+
"parquet_row"
|
| 50 |
+
]
|
| 51 |
+
}
|
| 52 |
+
}
|
index.html
CHANGED
|
@@ -1,19 +1,225 @@
|
|
| 1 |
<!doctype html>
|
| 2 |
-
<html>
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
</html>
|
|
|
|
| 1 |
<!doctype html>
|
| 2 |
+
<html lang="en">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="utf-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
| 6 |
+
<title>Major TOM Raster Viewer</title>
|
| 7 |
+
<link rel="icon" href="mt-icon.svg">
|
| 8 |
+
<link rel="preconnect" href="https://fonts.googleapis.com">
|
| 9 |
+
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=Source+Sans+3:wght@400;600;700&display=swap">
|
| 10 |
+
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/maplibre-gl@6.14.0/dist/maplibre-gl.css">
|
| 11 |
+
<style>
|
| 12 |
+
:root {
|
| 13 |
+
/* Major TOM logo colours */
|
| 14 |
+
--mt-blue: #0071bc; --mt-navy: #2e3192; --mt-green: #39b54a; --mt-orange: #f15a24;
|
| 15 |
+
/* light interface, as Hugging Face in light mode */
|
| 16 |
+
--page: #f9fafb; --surface: #ffffff; --ink: #111827; --ink-2: #4b5563; --ink-3: #6b7280;
|
| 17 |
+
--line: #e5e7eb; --tint: #eff6fc; --globe-space: #f3f4f6;
|
| 18 |
+
--sans: 'Source Sans 3', system-ui, sans-serif; --mono: 'IBM Plex Mono', ui-monospace, monospace;
|
| 19 |
+
}
|
| 20 |
+
* { box-sizing: border-box; }
|
| 21 |
+
html, body { height: 100%; margin: 0; }
|
| 22 |
+
body { background: var(--page); color: var(--ink); font: 15px/1.45 var(--sans); display: flex; flex-direction: column; }
|
| 23 |
+
|
| 24 |
+
header { background: var(--surface); border-bottom: 1px solid var(--line); position: relative; }
|
| 25 |
+
.brand-strip { display: flex; height: 3px; }
|
| 26 |
+
.brand-strip span { flex: 1; }
|
| 27 |
+
.bar { display: flex; align-items: center; gap: 12px; padding: 10px 20px; }
|
| 28 |
+
.bar img { width: 26px; height: 26px; }
|
| 29 |
+
.crumbs { font-family: var(--mono); font-size: 15px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
| 30 |
+
.crumbs .org { color: var(--ink-3); }
|
| 31 |
+
.crumbs b { font-weight: 500; }
|
| 32 |
+
.pill { font-size: 12px; color: var(--ink-2); border: 1px solid var(--line); border-radius: 999px; padding: 1px 9px; white-space: nowrap; }
|
| 33 |
+
#loader { position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%); z-index: 3; display: flex;
|
| 34 |
+
flex-direction: column; align-items: center; gap: 6px; min-width: 220px; padding: 18px 22px 14px;
|
| 35 |
+
background: rgba(255,255,255,.94); border: 1px solid var(--line); border-radius: 12px; box-shadow: 0 4px 16px rgba(17,24,39,.08); }
|
| 36 |
+
#loader[hidden] { display: none; }
|
| 37 |
+
#loader svg { width: 64px; height: 56px; margin-bottom: 4px; }
|
| 38 |
+
#loader polygon { fill: var(--line); transition: fill .12s; }
|
| 39 |
+
#loader polygon.on { fill: var(--on); }
|
| 40 |
+
#loader-phase { font-size: 11px; font-weight: 600; letter-spacing: .06em; text-transform: uppercase; color: var(--ink-3); }
|
| 41 |
+
#loader-text { font-size: 14px; color: var(--ink); text-align: center; }
|
| 42 |
+
#loader-bytes { font-family: var(--mono); font-size: 12px; color: var(--ink-3); min-height: 1em; }
|
| 43 |
+
.bar nav { margin-left: auto; display: flex; gap: 16px; font-size: 14px; }
|
| 44 |
+
.bar nav a { color: var(--ink-2); text-decoration: none; white-space: nowrap; }
|
| 45 |
+
.bar nav a:hover { color: var(--mt-blue); }
|
| 46 |
+
|
| 47 |
+
main { flex: 1; display: flex; min-height: 0; }
|
| 48 |
+
#map-wrap { flex: 1; position: relative; background: var(--globe-space); }
|
| 49 |
+
#map { position: absolute; inset: 0; }
|
| 50 |
+
.places { position: absolute; inset: 0; pointer-events: none; overflow: hidden; z-index: 1; }
|
| 51 |
+
.place { position: absolute; left: 0; top: 0; margin: -7px 0 0 -4px; display: flex; align-items: center; gap: 4px;
|
| 52 |
+
font: 600 12px/14px var(--sans); color: var(--ink); white-space: nowrap;
|
| 53 |
+
text-shadow: 0 0 2px #fff, 0 0 3px #fff, 0 0 5px rgba(255,255,255,.9); }
|
| 54 |
+
.place i { font-style: normal; font-size: 8px; color: var(--ink-2); }
|
| 55 |
+
.place.capital span { font-weight: 700; }
|
| 56 |
+
.place.peak i, .place.low i, .place.pass i { color: #6b4f2a; }
|
| 57 |
+
.place.peak span, .place.low span, .place.pass span, .place.falls span { font-weight: 400; font-style: italic; }
|
| 58 |
+
.toggle { display: flex; gap: 8px; align-items: center; font-size: 14px; cursor: pointer; }
|
| 59 |
+
.toggle input { accent-color: var(--mt-blue); }
|
| 60 |
+
#status:not(:empty) { position: absolute; top: 14px; left: 14px; z-index: 2; background: var(--surface);
|
| 61 |
+
border: 1px solid var(--line); border-radius: 8px; padding: 4px 10px; font-size: 13px; color: var(--ink-2); }
|
| 62 |
+
|
| 63 |
+
aside { width: 344px; flex: none; background: var(--surface); border-left: 1px solid var(--line); overflow-y: auto; }
|
| 64 |
+
section { padding: 14px 18px 16px; border-bottom: 1px solid var(--line); }
|
| 65 |
+
h2 { font-size: 13px; font-weight: 600; color: var(--ink-3); margin: 0 0 10px; }
|
| 66 |
+
|
| 67 |
+
.option { display: flex; gap: 10px; align-items: flex-start; padding: 8px 10px; border: 1px solid var(--line);
|
| 68 |
+
border-radius: 8px; margin-bottom: 6px; cursor: pointer; }
|
| 69 |
+
.option:has(input:checked) { border-color: var(--mt-blue); background: var(--tint); }
|
| 70 |
+
.option input { margin: 4px 0 0; accent-color: var(--mt-blue); }
|
| 71 |
+
.option b { display: block; font-weight: 600; }
|
| 72 |
+
.option small { display: block; color: var(--ink-3); font-family: var(--mono); font-size: 12px; }
|
| 73 |
+
.note { font-size: 13px; color: var(--ink-3); margin: 0; }
|
| 74 |
+
|
| 75 |
+
.pick-head { display: flex; justify-content: space-between; align-items: baseline; }
|
| 76 |
+
.pick-head h2 { margin-bottom: 4px; }
|
| 77 |
+
.link { font: inherit; font-size: 13px; color: var(--mt-blue); background: none; border: 0; padding: 0; cursor: pointer; }
|
| 78 |
+
.pick-name { display: flex; justify-content: space-between; gap: 8px; font-family: var(--mono); font-size: 13px; margin-bottom: 10px; }
|
| 79 |
+
.pick-name b { font-weight: 500; }
|
| 80 |
+
.pick-name span { color: var(--ink-3); }
|
| 81 |
+
#thumb-wrap { position: relative; aspect-ratio: 1; background: var(--globe-space); border-radius: 8px; overflow: hidden;
|
| 82 |
+
margin-bottom: 10px; display: flex; align-items: center; justify-content: center; color: var(--ink-3); font-size: 13px; }
|
| 83 |
+
#thumb { width: 100%; height: 100%; object-fit: contain; display: block; }
|
| 84 |
+
#thumb[hidden] { display: none; }
|
| 85 |
+
#choices { display: flex; flex-wrap: wrap; gap: 6px; margin-bottom: 8px; }
|
| 86 |
+
#choices:empty { display: none; }
|
| 87 |
+
#choices button { font: 12px var(--mono); padding: 3px 8px; border: 1px solid var(--line); border-radius: 999px;
|
| 88 |
+
background: var(--surface); color: var(--ink-2); cursor: pointer; }
|
| 89 |
+
#choices button[aria-pressed=true] { border-color: var(--mt-blue); color: var(--mt-blue); background: var(--tint); }
|
| 90 |
+
.tiny { font: 11px/1.4 var(--mono); color: var(--ink-3); min-height: 1em; margin-top: 8px; }
|
| 91 |
+
dl { display: grid; grid-template-columns: auto 1fr; gap: 3px 14px; margin: 0; font-size: 14px; }
|
| 92 |
+
dt { color: var(--ink-3); }
|
| 93 |
+
dd { margin: 0; font-family: var(--mono); font-size: 13px; text-align: right; overflow-wrap: anywhere; }
|
| 94 |
+
dd a { color: var(--mt-blue); }
|
| 95 |
+
.source { font-size: 12px; color: var(--ink-3); padding: 12px 18px 18px; }
|
| 96 |
+
.source a { color: var(--mt-blue); }
|
| 97 |
+
|
| 98 |
+
.maplibregl-ctrl-attrib { font-family: var(--sans); }
|
| 99 |
+
|
| 100 |
+
@media (max-width: 820px) {
|
| 101 |
+
main { flex-direction: column; overflow-y: auto; }
|
| 102 |
+
#map-wrap { flex: none; height: 62vh; }
|
| 103 |
+
aside { width: 100%; border-left: 0; border-top: 1px solid var(--line); overflow: visible; }
|
| 104 |
+
.bar { padding: 10px 16px; }
|
| 105 |
+
.bar nav, .pill { display: none; }
|
| 106 |
+
}
|
| 107 |
+
</style>
|
| 108 |
+
</head>
|
| 109 |
+
<body>
|
| 110 |
+
<header>
|
| 111 |
+
<div class="brand-strip" aria-hidden="true">
|
| 112 |
+
<span style="background:var(--mt-blue)"></span><span style="background:var(--mt-navy)"></span>
|
| 113 |
+
<span style="background:var(--mt-green)"></span><span style="background:var(--mt-orange)"></span>
|
| 114 |
+
</div>
|
| 115 |
+
<div class="bar">
|
| 116 |
+
<img src="mt-icon.svg" alt="">
|
| 117 |
+
<div class="crumbs"><span class="org">Major-TOM /</span> <b>Raster Viewer</b></div>
|
| 118 |
+
<span class="pill">serverless</span>
|
| 119 |
+
<nav>
|
| 120 |
+
<a href="https://huggingface.co/Major-TOM">Datasets</a>
|
| 121 |
+
<a href="https://huggingface.co/spaces/Major-TOM/MajorTOM-Core-Viewer">Core Viewer</a>
|
| 122 |
+
<a href="https://github.com/ESA-PhiLab/Major-TOM">Code</a>
|
| 123 |
+
</nav>
|
| 124 |
+
</div>
|
| 125 |
+
</header>
|
| 126 |
+
|
| 127 |
+
<main>
|
| 128 |
+
<div id="map-wrap">
|
| 129 |
+
<div id="map"></div>
|
| 130 |
+
<div id="places" class="places" aria-hidden="true"></div>
|
| 131 |
+
<div id="status"></div>
|
| 132 |
+
<div id="loader" role="status" aria-live="polite" hidden>
|
| 133 |
+
<!-- the Major TOM flag (mt-icon.svg, its 20 bars): they light up as the data arrives -->
|
| 134 |
+
<svg viewBox="398 10 100 86" aria-hidden="true">
|
| 135 |
+
<polygon style="--on:var(--mt-navy)" points="420.45 14.5 402.24 23.6 402.24 34.39 420.45 25.29"/>
|
| 136 |
+
<polygon style="--on:var(--mt-navy)" points="420.45 28.89 402.24 37.99 402.24 48.79 420.45 39.69"/>
|
| 137 |
+
<polygon style="--on:var(--mt-navy)" points="420.45 43.28 402.24 52.38 402.24 63.18 420.45 54.08"/>
|
| 138 |
+
<polygon style="--on:var(--mt-navy)" points="420.45 57.67 402.24 66.78 402.24 77.57 420.45 68.47"/>
|
| 139 |
+
<polygon style="--on:var(--mt-navy)" points="420.45 72.07 402.24 81.17 402.24 92.6 420.45 83.04"/>
|
| 140 |
+
<polygon style="--on:var(--mt-blue)" points="424.04 14.26 446.45 22.88 446.45 33.67 424.04 25.05"/>
|
| 141 |
+
<polygon style="--on:var(--mt-blue)" points="424.04 28.65 446.45 37.26 446.45 48.06 424.04 39.45"/>
|
| 142 |
+
<polygon style="--on:var(--mt-blue)" points="424.04 43.04 446.45 51.66 446.45 62.46 424.04 53.84"/>
|
| 143 |
+
<polygon style="--on:var(--mt-blue)" points="424.04 57.43 446.45 66.05 446.45 76.85 424.04 68.23"/>
|
| 144 |
+
<polygon style="--on:var(--mt-blue)" points="424.04 71.83 446.45 80.45 446.45 91.84 424.04 82.79"/>
|
| 145 |
+
<polygon style="--on:var(--mt-green)" points="450.04 22.91 472.45 14.29 472.45 25.08 450.04 33.7"/>
|
| 146 |
+
<polygon style="--on:var(--mt-green)" points="450.04 37.3 472.45 28.68 472.45 39.48 450.04 48.1"/>
|
| 147 |
+
<polygon style="--on:var(--mt-green)" points="450.04 51.69 472.45 43.07 472.45 53.87 450.04 62.49"/>
|
| 148 |
+
<polygon style="--on:var(--mt-green)" points="450.04 66.08 472.45 57.47 472.45 68.27 450.04 76.88"/>
|
| 149 |
+
<polygon style="--on:var(--mt-green)" points="450.04 80.48 472.45 71.86 472.45 82.82 450.04 91.88"/>
|
| 150 |
+
<polygon style="--on:var(--mt-orange)" points="476.04 14.5 494.24 23.6 494.24 34.39 476.04 25.29"/>
|
| 151 |
+
<polygon style="--on:var(--mt-orange)" points="476.04 28.89 494.24 37.99 494.24 48.79 476.04 39.69"/>
|
| 152 |
+
<polygon style="--on:var(--mt-orange)" points="476.04 43.28 494.24 52.38 494.24 63.18 476.04 54.08"/>
|
| 153 |
+
<polygon style="--on:var(--mt-orange)" points="476.04 57.67 494.24 66.78 494.24 77.57 476.04 68.47"/>
|
| 154 |
+
<polygon style="--on:var(--mt-orange)" points="476.04 72.07 494.24 81.17 494.24 92.6 476.04 83.04"/>
|
| 155 |
+
</svg>
|
| 156 |
+
<div id="loader-phase"></div>
|
| 157 |
+
<div id="loader-text"></div>
|
| 158 |
+
<div id="loader-bytes"></div>
|
| 159 |
+
</div>
|
| 160 |
+
</div>
|
| 161 |
+
|
| 162 |
+
<aside>
|
| 163 |
+
<section>
|
| 164 |
+
<h2>Dataset</h2>
|
| 165 |
+
<div id="datasets"></div>
|
| 166 |
+
<p class="note">Runs in your browser: reads the Core Parquet files on Hugging Face directly, no server. Click a
|
| 167 |
+
cell with a sample to load its image.</p>
|
| 168 |
+
</section>
|
| 169 |
+
|
| 170 |
+
<section id="pick" hidden>
|
| 171 |
+
<div class="pick-head"><h2>Selected cell</h2><button id="pick-clear" class="link">Clear</button></div>
|
| 172 |
+
<div class="pick-name"><b id="pick-cell"></b><span id="pick-pos"></span></div>
|
| 173 |
+
<div id="choices"></div>
|
| 174 |
+
<div id="thumb-wrap"><img id="thumb" alt="Thumbnail of the selected sample" hidden><span id="thumb-note"></span></div>
|
| 175 |
+
<dl id="pick-meta"></dl>
|
| 176 |
+
<div class="tiny" id="pick-stats"></div>
|
| 177 |
+
</section>
|
| 178 |
+
|
| 179 |
+
<section>
|
| 180 |
+
<h2>Map</h2>
|
| 181 |
+
<label class="toggle"><input type="checkbox" id="show-places" checked> Place names (cities, peaks) from zoom 5</label>
|
| 182 |
+
</section>
|
| 183 |
+
|
| 184 |
+
<section>
|
| 185 |
+
<h2>Cursor</h2>
|
| 186 |
+
<dl>
|
| 187 |
+
<dt>Cell</dt><dd id="cursor-cell">–</dd>
|
| 188 |
+
<dt>Latitude, longitude</dt><dd id="cursor-pos">–</dd>
|
| 189 |
+
<dt>Sample</dt><dd id="cursor-value">–</dd>
|
| 190 |
+
</dl>
|
| 191 |
+
</section>
|
| 192 |
+
|
| 193 |
+
<section>
|
| 194 |
+
<h2>Last load</h2>
|
| 195 |
+
<dl>
|
| 196 |
+
<dt>Footprints</dt><dd id="stat-footprints">–</dd>
|
| 197 |
+
<dt>Samples</dt><dd id="stat-samples">–</dd>
|
| 198 |
+
<dt>Draw on globe</dt><dd id="stat-draw">–</dd>
|
| 199 |
+
</dl>
|
| 200 |
+
</section>
|
| 201 |
+
|
| 202 |
+
<div class="source">Reads <a id="dataset-link" href="https://huggingface.co/Major-TOM">huggingface.co/Major-TOM</a>
|
| 203 |
+
· footprints from each dataset's metadata.parquet</div>
|
| 204 |
+
</aside>
|
| 205 |
+
</main>
|
| 206 |
+
|
| 207 |
+
<script>
|
| 208 |
+
// A private Space is shown in huggingface.co's frame through a signed link (?__sign=...), and only requests that
|
| 209 |
+
// carry the signature are let through there: modules imported by other modules would not. So every file of the app
|
| 210 |
+
// is asked for with it (an import map, then app.js). With no signature (public Space, local) nothing changes.
|
| 211 |
+
{
|
| 212 |
+
const sign = location.search, files = ['app.js', 'footprints.js', 'samples.js', 'parquet.js', 'tiles.js', 'labels.js']
|
| 213 |
+
const url = file => new URL(file, location.href).href.split('?')[0]
|
| 214 |
+
const map = document.createElement('script')
|
| 215 |
+
map.type = 'importmap'
|
| 216 |
+
map.textContent = JSON.stringify({ imports: Object.fromEntries(files.map(f => [url(f), url(f) + sign])) })
|
| 217 |
+
document.head.append(map)
|
| 218 |
+
const app = document.createElement('script')
|
| 219 |
+
app.type = 'module'
|
| 220 |
+
app.src = 'app.js' + sign
|
| 221 |
+
document.body.append(app)
|
| 222 |
+
}
|
| 223 |
+
</script>
|
| 224 |
+
</body>
|
| 225 |
</html>
|
labels.js
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Place names over the globe from zoom 5: cities and landmarks (places.json, Natural Earth), as HTML labels in the
|
| 2 |
+
// page's fonts. After each move: places in view, in the file's order (largest first), each placed unless it would
|
| 3 |
+
// overlap one already placed; at most MAX labels. Points behind the globe are left out.
|
| 4 |
+
|
| 5 |
+
const MIN_ZOOM = 5, MAX = 80, PAD = 3
|
| 6 |
+
const MARK = { capital: '■', city: '●', peak: '▲', low: '▼', pass: '⌇', falls: '≈', pole: '+' }
|
| 7 |
+
|
| 8 |
+
export function placeLabels(map, layer) {
|
| 9 |
+
let rows = [], on = true, frame = 0
|
| 10 |
+
fetch('places.json' + location.search).then(r => r.json()).then(d => { rows = d.rows; draw() })
|
| 11 |
+
|
| 12 |
+
/** Whether a place is on the visible side of the globe (MapLibre's own test where available). */
|
| 13 |
+
const visible = (lon, lat) => {
|
| 14 |
+
const t = map.transform
|
| 15 |
+
if (typeof t?.isLocationOccluded === 'function') return !t.isLocationOccluded({ lng: lon, lat })
|
| 16 |
+
const c = map.getCenter(), rad = Math.PI / 180 // within 60° of the centre
|
| 17 |
+
const cos = Math.sin(lat * rad) * Math.sin(c.lat * rad) + Math.cos(lat * rad) * Math.cos(c.lat * rad) * Math.cos((lon - c.lng) * rad)
|
| 18 |
+
return cos > 0.5
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
function draw() {
|
| 22 |
+
frame = 0
|
| 23 |
+
layer.replaceChildren()
|
| 24 |
+
if (!on || map.getZoom() < MIN_ZOOM) return
|
| 25 |
+
const { width, height } = map.getCanvas().getBoundingClientRect(), taken = []
|
| 26 |
+
for (const [kind, name, lon, lat, size] of rows) {
|
| 27 |
+
if (taken.length >= MAX) break
|
| 28 |
+
const p = map.project([lon, lat])
|
| 29 |
+
if (p.x < 0 || p.y < 0 || p.x > width || p.y > height || !visible(lon, lat)) continue
|
| 30 |
+
const text = kind === 'peak' || kind === 'low' ? `${name} ${size.toLocaleString('en')} m` : name
|
| 31 |
+
// box estimated from the text (12 px Source Sans: ~6.3 px a character), so only placed labels are made
|
| 32 |
+
const box = { left: p.x - 5, right: p.x + 12 + text.length * 6.3, top: p.y - 9, bottom: p.y + 9 }
|
| 33 |
+
if (taken.some(b => box.left < b.right + PAD && box.right + PAD > b.left && box.top < b.bottom + PAD && box.bottom + PAD > b.top)) continue
|
| 34 |
+
taken.push(box)
|
| 35 |
+
const label = document.createElement('div')
|
| 36 |
+
label.className = `place ${kind}`
|
| 37 |
+
label.innerHTML = `<i>${MARK[kind]}</i><span></span>`
|
| 38 |
+
label.querySelector('span').textContent = text
|
| 39 |
+
label.style.transform = `translate(${Math.round(p.x)}px, ${Math.round(p.y)}px)`
|
| 40 |
+
layer.append(label)
|
| 41 |
+
}
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
const later = () => { if (!frame) frame = requestAnimationFrame(draw) }
|
| 45 |
+
map.on('move', later)
|
| 46 |
+
map.on('resize', later)
|
| 47 |
+
return { show(value) { on = value; later() } }
|
| 48 |
+
}
|
mt-icon.svg
ADDED
|
|
parquet.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Remote Parquet files over HTTP range requests, with the bytes, requests and network time each read costs
|
| 2 |
+
// (as climate-globe/data.js).
|
| 3 |
+
|
| 4 |
+
export { parquetMetadataAsync, parquetRead } from 'https://cdn.jsdelivr.net/npm/hyparquet@1.31.4/+esm'
|
| 5 |
+
export { compressors } from 'https://cdn.jsdelivr.net/npm/hyparquet-compressors@1.1.2/+esm'
|
| 6 |
+
|
| 7 |
+
export const TAIL = 131072 // a file's first request: its last 128 KB, which hold most footers here
|
| 8 |
+
|
| 9 |
+
export const newStats = () => ({ bytes: 0, requests: 0, network: 0, active: 0, t0: performance.now() })
|
| 10 |
+
|
| 11 |
+
/** One range request, counted: bytes, requests, and network time (while any request is open). */
|
| 12 |
+
async function rangeGet(url, range, stats) {
|
| 13 |
+
if (stats.active++ === 0) stats.since = performance.now()
|
| 14 |
+
try {
|
| 15 |
+
const r = await fetch(url, { headers: { Range: `bytes=${range}` } })
|
| 16 |
+
if (!r.ok) throw new Error(`${url}: HTTP ${r.status}`)
|
| 17 |
+
const buffer = await r.arrayBuffer()
|
| 18 |
+
stats.bytes += buffer.byteLength; stats.requests += 1
|
| 19 |
+
return [r, buffer]
|
| 20 |
+
} finally {
|
| 21 |
+
if (--stats.active === 0) stats.network += performance.now() - stats.since
|
| 22 |
+
}
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
/** An AsyncBuffer for hyparquet over HTTP range requests. Opening it reads the file's tail, which gives the file
|
| 26 |
+
* size (Content-Range) and the footer in one request, where a size request and a footer request would take two. */
|
| 27 |
+
export async function countedFile(url, stats, tailSize = TAIL) {
|
| 28 |
+
const [r, tail] = await rangeGet(url, `-${tailSize}`, stats)
|
| 29 |
+
const range = r.headers.get('content-range') // "bytes a-b/size"; absent if whole
|
| 30 |
+
const byteLength = range ? Number(range.split('/')[1]) : tail.byteLength
|
| 31 |
+
const tailStart = byteLength - tail.byteLength
|
| 32 |
+
return {
|
| 33 |
+
byteLength,
|
| 34 |
+
async slice(start, end = byteLength) {
|
| 35 |
+
if (start >= tailStart) return tail.slice(start - tailStart, end - tailStart)
|
| 36 |
+
return (await rangeGet(url, `${start}-${end - 1}`, stats))[1]
|
| 37 |
+
},
|
| 38 |
+
}
|
| 39 |
+
}
|
places.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
samples.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// One Core sample from its image file on Hugging Face: the row group that holds it, thumbnail column only.
|
| 2 |
+
//
|
| 3 |
+
// footprint row (part, parquet_row) -> <images>/part_NNNNN.parquet (one row group per sample)
|
| 4 |
+
// -> footer (kept per file: 0.2-1.5 MB) -> that row group's thumbnail (and product_id where the file has it)
|
| 5 |
+
// -> an object URL for <img>
|
| 6 |
+
|
| 7 |
+
import { compressors, countedFile, newStats, parquetMetadataAsync, parquetRead, TAIL } from './parquet.js'
|
| 8 |
+
|
| 9 |
+
const footers = new Map() // image file URL -> its metadata, after the first read
|
| 10 |
+
|
| 11 |
+
export const imageUrl = (info, part) => `${info.images}part_${String(part).padStart(5, '0')}.parquet`
|
| 12 |
+
|
| 13 |
+
/** The thumbnail of one sample: {src (object URL), productId, url, stats}. */
|
| 14 |
+
export async function loadThumbnail(info, sample) {
|
| 15 |
+
const stats = newStats(), url = imageUrl(info, sample.part)
|
| 16 |
+
const file = await countedFile(url, stats, footers.has(url) ? 8 : TAIL) // 8 bytes: the size only
|
| 17 |
+
if (!footers.has(url)) footers.set(url, await parquetMetadataAsync(file, { initialFetchSize: TAIL }))
|
| 18 |
+
const metadata = footers.get(url), names = metadata.schema.slice(1).map(c => c.name)
|
| 19 |
+
let rowStart = 0
|
| 20 |
+
for (let g = 0; g < sample.parquet_row; g++) rowStart += Number(metadata.row_groups[g].num_rows)
|
| 21 |
+
const columns = ['thumbnail', ...(names.includes('product_id') ? ['product_id'] : [])]
|
| 22 |
+
let row
|
| 23 |
+
await parquetRead({ file, metadata, compressors, columns, rowStart, rowEnd: rowStart + 1, rowFormat: 'object',
|
| 24 |
+
utf8: false, onComplete(rows) { row = rows[0] } }) // utf8: the thumbnail stays bytes
|
| 25 |
+
if (!row?.thumbnail) throw new Error(`${url}: no thumbnail in row ${rowStart}`)
|
| 26 |
+
stats.read = performance.now() - stats.t0
|
| 27 |
+
const text = v => (v instanceof Uint8Array ? new TextDecoder().decode(v) : v ?? null)
|
| 28 |
+
return { src: URL.createObjectURL(new Blob([row.thumbnail])), productId: text(row.product_id), url, stats }
|
| 29 |
+
}
|
tiles.js
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// What the globe draws: map tiles made in the browser from the current field, and the Major TOM cell under a point.
|
| 2 |
+
//
|
| 3 |
+
// MapLibre asks for tile z/x/y (Web Mercator, 256 px)
|
| 4 |
+
// -> each pixel's longitude and latitude
|
| 5 |
+
// -> the Major TOM cell it falls in (row from latitude, then column within that row's own width)
|
| 6 |
+
// -> grid field: the cell's value -> colour scale; image field: the preview pixel at that lon/lat
|
| 7 |
+
// -> ImageBitmap handed back to MapLibre (no encoding: MapLibre uses decoded pixels as they are)
|
| 8 |
+
|
| 9 |
+
export const ROWS = 2004, COLS = 4008 // the (row, column) array: rows -1002..1001, columns -2004..2003
|
| 10 |
+
const R_KM = 6378.137, D_KM = 10, DLAT = 180 / ROWS
|
| 11 |
+
const TILE = 256
|
| 12 |
+
|
| 13 |
+
/** Columns in grid row r (as majortom.cells: from the latitude of the row's lower edge). */
|
| 14 |
+
const nCols = r => Math.max(Math.ceil((2 * Math.PI * R_KM * Math.cos((r * DLAT * Math.PI) / 180)) / D_KM), 1)
|
| 15 |
+
|
| 16 |
+
/** Row and column of the cell under (lon, lat), lon in -180..180. */
|
| 17 |
+
function rowCol(lon, lat) {
|
| 18 |
+
const row = Math.min(Math.max(Math.floor(lat / DLAT), -ROWS / 2), ROWS / 2 - 1)
|
| 19 |
+
const n = nCols(row)
|
| 20 |
+
const col = Math.min(Math.max(Math.floor(lon / (360 / n)), -Math.floor(n / 2)), n - 1 - Math.floor(n / 2))
|
| 21 |
+
return [row, col, n]
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
/** Index of the cell under (lon, lat) in the (row, column) array, or at `level` in its 2^level-coarser block means
|
| 25 |
+
* (a dataset's overview); used for every pixel, so no objects. */
|
| 26 |
+
export function cellIndex(lon, lat, level = 0) {
|
| 27 |
+
const row = Math.min(Math.max(Math.floor(lat / DLAT), -ROWS / 2), ROWS / 2 - 1)
|
| 28 |
+
const n = nCols(row)
|
| 29 |
+
const col = Math.min(Math.max(Math.floor(lon / (360 / n)), -Math.floor(n / 2)), n - 1 - Math.floor(n / 2))
|
| 30 |
+
if (!level) return (row + ROWS / 2) * COLS + col + COLS / 2
|
| 31 |
+
return ((row + ROWS / 2) >> level) * Math.ceil(COLS / 2 ** level) + ((col + COLS / 2) >> level)
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
/** The cell under (lon, lat), for the cursor: row, column, name (as majortom.cells), index, bounds in degrees. */
|
| 35 |
+
export function cellAt(lon, lat) {
|
| 36 |
+
lon = ((lon + 180) % 360 + 360) % 360 - 180
|
| 37 |
+
const [row, col, n] = rowCol(lon, lat), dlon = 360 / n
|
| 38 |
+
return {
|
| 39 |
+
row, col, index: (row + ROWS / 2) * COLS + col + COLS / 2,
|
| 40 |
+
name: `${Math.abs(row)}${row >= 0 ? 'U' : 'D'}_${Math.abs(col)}${col >= 0 ? 'R' : 'L'}`,
|
| 41 |
+
bounds: [col * dlon, row * DLAT, (col + 1) * dlon, (row + 1) * DLAT],
|
| 42 |
+
}
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
/** A field of values, one per cell, on the (row, column) array (or its block means at `level`); NaN where there
|
| 46 |
+
* is no cell or no value. */
|
| 47 |
+
export function gridField(values, scale, level = 0) {
|
| 48 |
+
return {
|
| 49 |
+
values, scale, level,
|
| 50 |
+
valueAt: (lon, lat) => values[cellIndex(((lon + 180) % 360 + 360) % 360 - 180, lat, level)],
|
| 51 |
+
pixel(lon, lat, out, o) {
|
| 52 |
+
const v = values[cellIndex(lon, lat, level)]
|
| 53 |
+
if (Number.isNaN(v)) { out[o + 3] = 0; return }
|
| 54 |
+
const i = Math.round(scale.position(v) * 255) * 3
|
| 55 |
+
out[o] = scale.lut[i]; out[o + 1] = scale.lut[i + 1]; out[o + 2] = scale.lut[i + 2]; out[o + 3] = 255
|
| 56 |
+
},
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
/** A field read from a preview image of the (row, column) array, north up (build_images.py): pixel (x, y) is the
|
| 61 |
+
* cell grid_col_r = x - 2004, grid_row_u = 1001 - y, so a point finds its pixel exactly as live data finds its value. */
|
| 62 |
+
export function imageField(rgba, scale) {
|
| 63 |
+
const colour = (lon, lat, out, o) => {
|
| 64 |
+
const k = cellIndex(lon, lat), i = ((ROWS - 1 - Math.floor(k / COLS)) * COLS + (k % COLS)) * 4
|
| 65 |
+
out[o] = rgba[i]; out[o + 1] = rgba[i + 1]; out[o + 2] = rgba[i + 2]; out[o + 3] = rgba[i + 3]
|
| 66 |
+
}
|
| 67 |
+
return { scale, valueAt: () => undefined, pixel: colour } // colours only: exact values come from the Parquet files
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
/** One 256 px tile of `field` at z/x/y. */
|
| 71 |
+
export async function renderTile(field, z, x, y) {
|
| 72 |
+
const n = 2 ** z, out = new Uint8ClampedArray(TILE * TILE * 4)
|
| 73 |
+
const lons = new Float64Array(TILE)
|
| 74 |
+
for (let i = 0; i < TILE; i++) lons[i] = ((x + (i + 0.5) / TILE) / n) * 360 - 180
|
| 75 |
+
for (let j = 0; j < TILE; j++) {
|
| 76 |
+
const lat = (Math.atan(Math.sinh(Math.PI - (2 * Math.PI * (y + (j + 0.5) / TILE)) / n)) * 180) / Math.PI
|
| 77 |
+
for (let i = 0; i < TILE; i++) field.pixel(lons[i], lat, out, (j * TILE + i) * 4)
|
| 78 |
+
}
|
| 79 |
+
return createImageBitmap(new ImageData(out, TILE, TILE))
|
| 80 |
+
}
|