Add 260927_mvt_dinov3b_512_hexaview_fly38

260927_mvt_dinov3b_512_hexaview_fly38/README.md

@@ -0,0 +1,30 @@
+# `mvt_dinov3b_512_hexaview_fly38` -- the hexaview detector, round 11
+
+A deeperfly 2 detector (`deeperfly-detector-1`, family `mvt`): a multiview transformer on a DINOv3 ViT-B/16 that predicts all 38 points of the `fly38` skeleton in the six views of the hexaview rig -- two cameras, each seeing the arena directly and through two prisms. It windows each view on the fly frame by frame, and a video holds three views of one fly, so it runs with its locator, `260920_fast_efficientvit_b1_352_hexaview_locator`, beside it.
+
+Exported 2026-09-28 from `hexaview/runs/r11ship_all`, trained 2026-09-27: round 10's recipe unchanged, retrained on the labels of 2026-09-27.
+
+| | |
+|---|---|
+| file | `mvt_dinov3b_512_hexaview_fly38.pth`, 383,792,459 bytes |
+| sha256 | `a832147c1bef46006b7e034215467bca1505a0aba32a01e581db02e8fa8db285` |
+| skeleton | `fly38` -- `fly38.toml` beside this file |
+| backbone | `vit_base_patch16_dinov3.lvd1689m` (timm) |
+| input | 512x512 per view: a square window centered on the fly in each frame |
+| trained on | 16 hexaview recordings, 3,018 items (`report.json`); no holdout |
+| recipe | `config.toml` beside this file |
+
+## What it scores
+
+No held-out number of its own: a shipping model trains on every label. On the deeperfly repository's `examples/hexaview/` clip (250 frames of `260819_002`), the pictorial 3D reprojects at 3.15 px median through the recording's calibration. That is agreement across the six views, not an error against labels.
+
+## Using it
+
+```toml
+[pose2d]
+weights = ["/mnt/upramdya/data/TL/deeperfly-models/260927_mvt_dinov3b_512_hexaview_fly38/mvt_dinov3b_512_hexaview_fly38.pth"]
+window = "per_frame"
+locator = "/mnt/upramdya/data/TL/deeperfly-models/260920_fast_efficientvit_b1_352_hexaview_locator/fast_efficientvit_b1_352_hexaview_locator.pth"
+```
+
+Copying this directory to the machine that runs it and naming the local path is the better habit. Verify a copy with `sha256sum -c SHA256SUMS`.

260927_mvt_dinov3b_512_hexaview_fly38/SHA256SUMS

@@ -0,0 +1,4 @@
+5aaa8857861800b1a797dc69a7da51838ebaa03059112f9608592f6e6dc11c3b  ./config.toml
+56f9a33ea5c286bda06e599602e7b7adaefb7ba29a74a2516548b7cbb7776e52  ./fly38.toml
+a832147c1bef46006b7e034215467bca1505a0aba32a01e581db02e8fa8db285  ./mvt_dinov3b_512_hexaview_fly38.pth
+d15e43352ce008d3e5365b6aa126d5849f59d88ea7dfbd94838647129c9995a2  ./report.json

260927_mvt_dinov3b_512_hexaview_fly38/config.toml

@@ -0,0 +1,100 @@
+# r11ship_all: round 11's hexaview ship -- r10ship_all's recipe unchanged, retrained on
+# the labels of 2026-09-27 (504 frames in 17 projects: the 10 of round 10 plus the 7
+# large-arena GE recordings kept after the desync audit) by the code of main 796ae43.
+# The split is still splits/r10_all.toml: its one exclusion, 260804_002 frame 29500,
+# is still labeled. 260730_001's realignment (2026-09-25) split its labeled frames
+# 441250 and 445750 into t-1 (top views only) and t (bottom views only); each is a
+# partially labeled moment, which the fill completes like any other. No holdout.
+#
+# `transductive = true` as in `r6ship_all`: the ship trains on every label, so a
+# teacher that has seen them all leaks nothing it does not already have.
+
+format_version = 3
+
+[data]
+kind = "project"
+projects = ["../recordings/*"]
+split = "../splits/r10_all.toml"
+cache = "../cache"
+
+[data.crops]
+tightness = [0.75, 2.2]  # per-sample crop around the animal, log-uniform
+jitter = 0.1
+eval_tightness = 1.25
+full_p = 0.0            # see above: a whole video frame is not one view's
+
+[data.fill]
+weight = 0.25
+max_residual_px = 10.0
+
+[model]
+input_hw = [512, 512]
+family = "mvt"
+backbone = "vit_base_patch16_dinov3.lvd1689m"
+pretrained = true
+attn_scopes = "llllllllglgl"
+margin = 64
+reach = 2
+drop_path = 0.0
+token_mask = 0.0
+token_pad = 2
+init_from = "/home/tlam/deeperfly-projects/octacam/runs/mvt_dinov3/ship_vitb_distill_s0/model.pth"
+
+[augment]
+degrees = 180.0              # every view, day 1: a walking fly faces anywhere
+scale = [1.0, 1.0]           # the crop sampler supplies scale and shift
+aspect = [0.85, 1.18]
+translate = [0.0, 0.0]
+flip_p = 0.5
+brightness = 0.3
+contrast = 0.3
+gamma = [0.7, 1.4]
+blur_sigma = [0.5, 1.5]
+noise_std = 0.03
+photometric_p = 0.5
+
+[train]
+grad_checkpoint = true
+steps = 5000
+images_per_batch = 24        # four 6-view moments per step
+grad_clip = 1.0
+amp = true
+ema_decay = 0.999
+view_drop_p = 0.3
+view_min = 1
+log_every = 100
+seed = 0
+out = "../runs/r11ship_all"
+
+[train.optimizer]
+lr = 1e-4
+weight_decay = 0.05
+warmup_steps = 500
+
+[train.loss]
+sigma = 2.0
+field_weight = 1.0
+coord_weight = 0.0
+tri_weight = 0.0             # the calibration's units are arbitrary
+
+[train.eval]
+every = 0
+
+[compute]
+workers = 8
+
+[unlabeled]
+# No `teacher`: the teacher's points, confidences and windows come from each
+# project's merged detections (ADR-0083): r10ship_all, or r10ship_all_realigned.
+n_per_project = 250
+transductive = true
+agree_px = 6.0
+min_views = 3
+conf_min = 0.3
+geom_weight = 0.3
+weight = 0.5
+mask_ratios = [0.0, 0.5, 0.9]
+masked_weight = 1.0
+unmasked_weight = 0.5
+crop_tightness = [0.75, 1.6]
+cache = "../cache/unlabeled"

260927_mvt_dinov3b_512_hexaview_fly38/fly38.toml

@@ -0,0 +1,129 @@
+# fly38 -- the 38 tracked points of a tethered Drosophila: six 5-point legs, two
+# antennae, the neck, and a 5-point dorsal-midline abdomen chain (the tergite stripes).
+#
+# A skeleton is points, edges and point_symmetries -- the same decomposition as SLEAP
+# and DeepLabCut -- plus an optional palette (point_colors / edge_colors). The palette
+# is decoration, not identity: it draws the points but is kept out of the digest, so
+# recoloring never reports a different skeleton.
+
+[skeleton]
+name = "fly38"
+
+# The points IN ORDER: channel i of the detector is points[i]. This is the contract
+# with the weights.
+points = [
+    "lf_thorax_coxa", "lf_coxa_trochanter", "lf_femur_tibia", "lf_tibia_tarsus", "lf_pretarsus",
+    "lm_thorax_coxa", "lm_coxa_trochanter", "lm_femur_tibia", "lm_tibia_tarsus", "lm_pretarsus",
+    "lh_thorax_coxa", "lh_coxa_trochanter", "lh_femur_tibia", "lh_tibia_tarsus", "lh_pretarsus",
+    "l_antenna",
+    "rf_thorax_coxa", "rf_coxa_trochanter", "rf_femur_tibia", "rf_tibia_tarsus", "rf_pretarsus",
+    "rm_thorax_coxa", "rm_coxa_trochanter", "rm_femur_tibia", "rm_tibia_tarsus", "rm_pretarsus",
+    "rh_thorax_coxa", "rh_coxa_trochanter", "rh_femur_tibia", "rh_tibia_tarsus", "rh_pretarsus",
+    "r_antenna",
+    "neck",
+    "abdomen0", "abdomen1", "abdomen2", "abdomen3", "abdomen4",
+]
+
+# The edges, as pairs of points. The antennae and neck appear in no edge -- tracked
+# points with no bone.
+edges = [
+    ["lf_thorax_coxa", "lf_coxa_trochanter"], ["lf_coxa_trochanter", "lf_femur_tibia"],
+    ["lf_femur_tibia", "lf_tibia_tarsus"], ["lf_tibia_tarsus", "lf_pretarsus"],
+    ["lm_thorax_coxa", "lm_coxa_trochanter"], ["lm_coxa_trochanter", "lm_femur_tibia"],
+    ["lm_femur_tibia", "lm_tibia_tarsus"], ["lm_tibia_tarsus", "lm_pretarsus"],
+    ["lh_thorax_coxa", "lh_coxa_trochanter"], ["lh_coxa_trochanter", "lh_femur_tibia"],
+    ["lh_femur_tibia", "lh_tibia_tarsus"], ["lh_tibia_tarsus", "lh_pretarsus"],
+    ["rf_thorax_coxa", "rf_coxa_trochanter"], ["rf_coxa_trochanter", "rf_femur_tibia"],
+    ["rf_femur_tibia", "rf_tibia_tarsus"], ["rf_tibia_tarsus", "rf_pretarsus"],
+    ["rm_thorax_coxa", "rm_coxa_trochanter"], ["rm_coxa_trochanter", "rm_femur_tibia"],
+    ["rm_femur_tibia", "rm_tibia_tarsus"], ["rm_tibia_tarsus", "rm_pretarsus"],
+    ["rh_thorax_coxa", "rh_coxa_trochanter"], ["rh_coxa_trochanter", "rh_femur_tibia"],
+    ["rh_femur_tibia", "rh_tibia_tarsus"], ["rh_tibia_tarsus", "rh_pretarsus"],
+    ["abdomen0", "abdomen1"], ["abdomen1", "abdomen2"], ["abdomen2", "abdomen3"], ["abdomen3", "abdomen4"],
+]
+
+# What each point becomes under a left/right flip. Write each pair once; anything absent
+# is on the midline and maps to itself (hence no neck, no abdomen). The loader checks
+# the implied mirror against `edges`: applying it must give the edge set back, so a
+# wrong side or two joints of one leg exchanged is named rather than trained on silently.
+point_symmetries = [
+    ["lf_thorax_coxa", "rf_thorax_coxa"],
+    ["lf_coxa_trochanter", "rf_coxa_trochanter"],
+    ["lf_femur_tibia", "rf_femur_tibia"],
+    ["lf_tibia_tarsus", "rf_tibia_tarsus"],
+    ["lf_pretarsus", "rf_pretarsus"],
+    ["lm_thorax_coxa", "rm_thorax_coxa"],
+    ["lm_coxa_trochanter", "rm_coxa_trochanter"],
+    ["lm_femur_tibia", "rm_femur_tibia"],
+    ["lm_tibia_tarsus", "rm_tibia_tarsus"],
+    ["lm_pretarsus", "rm_pretarsus"],
+    ["lh_thorax_coxa", "rh_thorax_coxa"],
+    ["lh_coxa_trochanter", "rh_coxa_trochanter"],
+    ["lh_femur_tibia", "rh_femur_tibia"],
+    ["lh_tibia_tarsus", "rh_tibia_tarsus"],
+    ["lh_pretarsus", "rh_pretarsus"],
+    ["l_antenna", "r_antenna"],
+]
+
+# The palette. One #rrggbb per point; a key is a point name or a `*` pattern (an exact
+# name beats a pattern). Optional and may be partial -- an unnamed point takes tab10 by
+# index. The scheme is anatomical, so it reads at a glance: left = blue, right = red,
+# front -> hind lightening within a side; the midline neck and abdomen green. Edges take
+# the average of their endpoints, so every leg bone comes out its own limb's color.
+[skeleton.point_colors]
+"lf_*" = "#0f7399"
+"lm_*" = "#1a8daf"
+"lh_*" = "#75becb"
+l_antenna = "#0a4f6b"
+"rf_*" = "#ba1e31"
+"rm_*" = "#c9564f"
+"rh_*" = "#d58579"
+r_antenna = "#8c1525"
+neck = "#15a315"
+"abdomen*" = "#61e47b"
+
+# A canonical neutral posture (NeuroMechFly's rest pose), one [x, y, z] per point
+# in millimeters, with the body center of mass at the origin (x anterior, y the
+# left-right axis, z dorsal). It is NOT part of the skeleton's identity (out of the
+# digest); it draws the skeleton in 3D and SEEDS labeling: projected through a
+# recording's calibration it drops a rough draggable instance into each view when
+# nothing else (a prior labeled frame, another view) is available to seed from.
+[skeleton.default_pose]
+lf_thorax_coxa     = [ 0.42966,  0.17200,  0.25226]
+lf_coxa_trochanter = [ 0.31028,  0.06883, -0.07688]
+lf_femur_tibia     = [ 0.85227,  0.42043,  0.20537]
+lf_tibia_tarsus    = [ 1.05606,  0.65936, -0.20658]
+lf_pretarsus       = [ 1.45133,  1.02885, -0.58856]
+lm_thorax_coxa     = [ 0.03366,  0.12500,  0.00026]
+lm_coxa_trochanter = [ 0.00831,  0.09367, -0.17620]
+lm_femur_tibia     = [ 0.11746,  0.70120,  0.30719]
+lm_tibia_tarsus    = [ 0.23055,  0.96680, -0.29411]
+lm_pretarsus       = [ 0.38484,  1.49119, -0.73358]
+lh_thorax_coxa     = [-0.19334,  0.08700, -0.01474]
+lh_coxa_trochanter = [-0.27402,  0.09655, -0.19640]
+lh_femur_tibia     = [-0.64760,  0.63110,  0.32665]
+lh_tibia_tarsus    = [-0.96496,  0.77982, -0.26073]
+lh_pretarsus       = [-1.40791,  1.03885, -0.86325]
+l_antenna          = [ 1.04836,  0.08600,  0.56596]
+rf_thorax_coxa     = [ 0.42966, -0.17200,  0.25226]
+rf_coxa_trochanter = [ 0.31028, -0.06883, -0.07688]
+rf_femur_tibia     = [ 0.85227, -0.42043,  0.20537]
+rf_tibia_tarsus    = [ 1.05606, -0.65936, -0.20658]
+rf_pretarsus       = [ 1.45133, -1.02885, -0.58856]
+rm_thorax_coxa     = [ 0.03366, -0.12500,  0.00026]
+rm_coxa_trochanter = [ 0.00831, -0.09367, -0.17620]
+rm_femur_tibia     = [ 0.11746, -0.70120,  0.30719]
+rm_tibia_tarsus    = [ 0.23055, -0.96680, -0.29411]
+rm_pretarsus       = [ 0.38484, -1.49119, -0.73358]
+rh_thorax_coxa     = [-0.19334, -0.08700, -0.01474]
+rh_coxa_trochanter = [-0.27402, -0.09655, -0.19640]
+rh_femur_tibia     = [-0.64760, -0.63110,  0.32665]
+rh_tibia_tarsus    = [-0.96496, -0.77982, -0.26073]
+rh_pretarsus       = [-1.40791, -1.03885, -0.86325]
+r_antenna          = [ 1.04836, -0.08600,  0.56596]
+neck               = [ 0.61936,  0.00000,  0.48296]
+abdomen0           = [-0.71698,  0.00000,  0.71444]
+abdomen1           = [-0.91268,  0.00000,  0.66169]
+abdomen2           = [-1.10060,  0.00000,  0.61850]
+abdomen3           = [-1.27163,  0.00000,  0.56551]
+abdomen4           = [-1.48737,  0.00000,  0.46364]

260927_mvt_dinov3b_512_hexaview_fly38/mvt_dinov3b_512_hexaview_fly38.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a832147c1bef46006b7e034215467bca1505a0aba32a01e581db02e8fa8db285
+size 383792459

260927_mvt_dinov3b_512_hexaview_fly38/report.json

@@ -0,0 +1,75 @@
+{
+ "projects": {
+  "260730_001": "/home/tlam/deeperfly-projects/hexaview/recordings/260730_001/deeperfly",
+  "260804_001": "/home/tlam/deeperfly-projects/hexaview/recordings/260804_001/deeperfly",
+  "260804_002": "/home/tlam/deeperfly-projects/hexaview/recordings/260804_002/deeperfly",
+  "260804_003": "/home/tlam/deeperfly-projects/hexaview/recordings/260804_003/deeperfly",
+  "260804_004": "/home/tlam/deeperfly-projects/hexaview/recordings/260804_004/deeperfly",
+  "260818_001": "/home/tlam/deeperfly-projects/hexaview/recordings/260818_001/deeperfly",
+  "260819_001": "/home/tlam/deeperfly-projects/hexaview/recordings/260819_001/deeperfly",
+  "260819_002": "/home/tlam/deeperfly-projects/hexaview/recordings/260819_002/deeperfly",
+  "260819_003": "/home/tlam/deeperfly-projects/hexaview/recordings/260819_003/deeperfly",
+  "260821_002": "/home/tlam/deeperfly-projects/hexaview/recordings/260821_002/deeperfly",
+  "260916_Fly10_001": "/home/tlam/deeperfly-projects/hexaview/recordings/260916_Fly10_001/deeperfly",
+  "260916_Fly1_001": "/home/tlam/deeperfly-projects/hexaview/recordings/260916_Fly1_001/deeperfly",
+  "260916_Fly3_001": "/home/tlam/deeperfly-projects/hexaview/recordings/260916_Fly3_001/deeperfly",
+  "260916_Fly5_001": "/home/tlam/deeperfly-projects/hexaview/recordings/260916_Fly5_001/deeperfly",
+  "260916_Fly6_001": "/home/tlam/deeperfly-projects/hexaview/recordings/260916_Fly6_001/deeperfly",
+  "260916_Fly9_001": "/home/tlam/deeperfly-projects/hexaview/recordings/260916_Fly9_001/deeperfly",
+  "260917_Fly8_001": "/home/tlam/deeperfly-projects/hexaview/recordings/260917_Fly8_001/deeperfly"
+ },
+ "train_projects": [
+  "260730_001",
+  "260804_001",
+  "260804_002",
+  "260804_003",
+  "260804_004",
+  "260818_001",
+  "260819_001",
+  "260819_002",
+  "260819_003",
+  "260821_002",
+  "260916_Fly10_001",
+  "260916_Fly1_001",
+  "260916_Fly3_001",
+  "260916_Fly5_001",
+  "260916_Fly9_001",
+  "260917_Fly8_001"
+ ],
+ "holdout": [],
+ "holdout_frames": {},
+ "n_train_items": 3018,
+ "augment": {
+  "degrees": 180.0,
+  "scale": [
+   1.0,
+   1.0
+  ],
+  "aspect": [
+   0.85,
+   1.18
+  ],
+  "translate": [
+   0.0,
+   0.0
+  ],
+  "flip_p": 0.5,
+  "brightness": 0.3,
+  "contrast": 0.3,
+  "gamma": [
+   0.7,
+   1.4
+  ],
+  "blur_sigma": [
+   0.5,
+   1.5
+  ],
+  "noise_std": 0.03,
+  "photometric_p": 0.5
+ },
+ "final_loss": 7.084183692932129,
+ "holdout_report": null,
+ "select_best": null,
+ "best_step": null,
+ "best_score": null
+}