d1-omni-600M-CoreAI / metadata.json
mlboydaisuke's picture
d1-omni-600M on Core AI: fp16 decision graphs (L64-L4096), vision and audio graphs, Swift host, reference fixture
914184e verified
Raw History Blame Contribute Delete
20.4 kB
{
"schema": "d1-omni-coreai/1",
"model": {
"source": "LiquidAI/d1-omni-600M",
"revision": "414f8d6438174f5b2133a9c21a478fc42625e308",
"license": "LFM Open License v1.0",
"conversion": "coreai-model-zoo conversion/d1_omni (re-authored in plain PyTorch, coreai-torch 0.4.1; the graphs' debug locations stripped with coreai-torch strip_debug_info, conversion/d1_omni/strip_ship.py)"
},
"runtime": "Core AI (macOS 27 / iOS 27)",
"precision": "fp16 (weights and compute; the exceptions are listed in each graph's metadata.json)",
"graphs": {
"decision": {
"buckets": [
64,
128,
256,
512,
1024,
2048,
4096
],
"bucket_rule": "the smallest L that holds P + len(ids) (host.py bucket_for(positions, ALL_BUCKETS))",
"inputs": {
"input_ids": {
"dtype": "int32",
"shape": [
1,
"L"
],
"values": "0 on [0, P), the row's ids on [P, P+n), 0 (<|pad|>) after"
},
"prefix_embeds": {
"dtype": "float32",
"shape": [
1,
"L",
1024
],
"values": "the image / audio prefix on [0, P), 0.0 elsewhere (finite)"
},
"pad_mask": {
"dtype": "float32",
"shape": [
1,
"L"
],
"values": "1.0 on [0, P+n), 0.0 after"
},
"prefix_mask": {
"dtype": "float32",
"shape": [
1,
"L"
],
"values": "1.0 on [0, P), 0.0 after"
},
"keep_right": {
"dtype": "float32",
"shape": [
1,
"L"
],
"values": "0.0 at P-1 when P > 0, 1.0 elsewhere"
},
"qtype_onehot": {
"dtype": "float32",
"shape": [
1,
3
],
"order": [
"choice",
"score",
"noul"
]
}
},
"outputs": {
"scores": {
"dtype": "float32",
"shape": [
1,
"L"
],
"read": "at P + markers (the <|mask|> positions)"
}
},
"max_length": 16384,
"image_text_length": 896,
"audio_text_length": 15360,
"min_text_positions": 64,
"prefix_hidden": 1024,
"folders": {
"macos": {
"decide-fp16-L64": "macos/decide-fp16-L64",
"decide-fp16-L128": "macos/decide-fp16-L128",
"decide-fp16-L256": "macos/decide-fp16-L256",
"decide-fp16-L512": "macos/decide-fp16-L512",
"decide-fp16-L1024": "macos/decide-fp16-L1024",
"decide-fp16-L2048": "macos/decide-fp16-L2048",
"decide-fp16-L4096": "macos/decide-fp16-L4096"
},
"ios": {
"decide-fp16-L64": "ios/decide-fp16-L64",
"decide-fp16-L128": "ios/decide-fp16-L128",
"decide-fp16-L256": "ios/decide-fp16-L256",
"decide-fp16-L512": "ios/decide-fp16-L512",
"decide-fp16-L1024": "ios/decide-fp16-L1024",
"decide-fp16-L2048": "ios/decide-fp16-L2048",
"decide-fp16-L4096": "ios/decide-fp16-L4096"
},
"ios-h19p": {
"decide-fp16-L64": "ios-h19p/decide-fp16-L64",
"decide-fp16-L128": "ios-h19p/decide-fp16-L128",
"decide-fp16-L256": "ios-h19p/decide-fp16-L256",
"decide-fp16-L512": "ios-h19p/decide-fp16-L512",
"decide-fp16-L1024": "ios-h19p/decide-fp16-L1024",
"decide-fp16-L2048": "ios-h19p/decide-fp16-L2048"
}
}
},
"vision": {
"role": "d1-omni image prefix: one crop per call",
"inputs": {
"pixel_values": {
"dtype": "float32",
"shape": [
1,
1024,
768
],
"values": "the crop's 16x16 patches row-major over (ph, pw), each [py][px][c], (x - 127.5) / 127.5; 0.0 past ph*pw"
},
"pos_embed": {
"dtype": "float32",
"shape": [
1,
1024,
768
],
"values": "position_table resized to (ph, pw), bilinear + antialias (host.position_embeddings / position_embeddings_numpy), row-major; 0.0 past ph*pw"
},
"patch_mask": {
"dtype": "float32",
"shape": [
1,
1024
],
"values": "1.0 on the ph*pw patches, 0.0 after"
},
"unshuffle_index": {
"dtype": "int32",
"shape": [
256,
4
],
"values": "token (i, j) = row i * (pw/2) + j: patches 2i*pw+2j, 2i*pw+2j+1, (2i+1)*pw+2j, (2i+1)*pw+2j+1; 0 past (ph/2)(pw/2) (host.unshuffle_index)"
}
},
"outputs": {
"prefix": {
"dtype": "float32",
"shape": [
1,
256,
1024
],
"read": "rows [0, (ph/2)(pw/2)): the crop's prefix embeddings"
}
},
"max_patches": 1024,
"max_tokens": 256,
"patch": 16,
"crops": "vision.preprocess(): an image whose rounded area exceeds 2 x 512^2 is cut into the 512 px tiles of the closest grid (2 to 10 tiles, row-major) plus a thumbnail, else the thumbnail alone; each crop resized by torchvision (bilinear, antialias) from the RGB image (host.layout / image_crops)",
"prefix": "each crop's first (ph/2)(pw/2) output rows, crops in order, images in request order; this is the decision graph's prefix_embeds [0, P)",
"position_table": {
"file": "position_table.f32",
"shape": [
16,
16,
768
],
"dtype": "float32",
"byte_order": "little",
"layout": "row-major (row, column, channel)",
"resize": "per crop to (ph, pw): bilinear, align_corners false, antialias (torch's float32 kernel: width pass then height pass, fused multiply-add per tap; host.position_embeddings_numpy)",
"sha256": "69b3c175e58af01f99eb4aa566e2c238d2e8b24c9a0d505fbcb6d5e3a095bf6f"
},
"precision_detail": {
"weights": "fp16",
"compute": "fp16 (LayerNorm, the attention softmax and the projector's gelu in fp32)"
},
"folders": {
"macos": {
"vision-fp16": "macos/vision-fp16"
},
"ios": {
"vision-fp16": "ios/vision-fp16"
},
"ios-h19p": {
"vision-fp16": "ios-h19p/vision-fp16"
}
}
},
"audio": {
"buckets_s": [
5,
10,
20,
30
],
"per_bucket": {
"5s": {
"bucket": {
"seconds": 5,
"F": 501,
"F2": 251,
"F4": 126,
"T": 63,
"holds": "clips of n samples with 1 + n // 160 <= 501 after waveform()'s cut and pad (host.audio_bucket_for: the smallest of [5, 10, 20, 30] s that holds it)",
"max_prefix_rows": 63
},
"inputs": {
"mel": {
"dtype": "float32",
"shape": [
1,
128,
501
],
"values": "the clip's normalised log-mel (mel_host: waveform() + MelFrontend), columns t >= frames 0.0, zero-padded to F"
},
"mask_f": {
"dtype": "float32",
"shape": [
1,
501
],
"values": "1.0 on t < l0 = frames (n // 160)"
},
"mask_f2": {
"dtype": "float32",
"shape": [
1,
251
],
"values": "1.0 on t < l1 = (l0 - 1) // 2 + 1"
},
"mask_f4": {
"dtype": "float32",
"shape": [
1,
126
],
"values": "1.0 on t < l2 = (l1 - 1) // 2 + 1"
},
"mask_t": {
"dtype": "float32",
"shape": [
1,
63
],
"values": "1.0 on t < P = l3 = (l2 - 1) // 2 + 1"
}
},
"outputs": {
"prefix": {
"dtype": "float32",
"shape": [
1,
63,
1024
],
"read": "rows [0, P): the clip's prefix embeddings"
}
}
},
"10s": {
"bucket": {
"seconds": 10,
"F": 1001,
"F2": 501,
"F4": 251,
"T": 126,
"holds": "clips of n samples with 1 + n // 160 <= 1001 after waveform()'s cut and pad (host.audio_bucket_for: the smallest of [5, 10, 20, 30] s that holds it)",
"max_prefix_rows": 125
},
"inputs": {
"mel": {
"dtype": "float32",
"shape": [
1,
128,
1001
],
"values": "the clip's normalised log-mel (mel_host: waveform() + MelFrontend), columns t >= frames 0.0, zero-padded to F"
},
"mask_f": {
"dtype": "float32",
"shape": [
1,
1001
],
"values": "1.0 on t < l0 = frames (n // 160)"
},
"mask_f2": {
"dtype": "float32",
"shape": [
1,
501
],
"values": "1.0 on t < l1 = (l0 - 1) // 2 + 1"
},
"mask_f4": {
"dtype": "float32",
"shape": [
1,
251
],
"values": "1.0 on t < l2 = (l1 - 1) // 2 + 1"
},
"mask_t": {
"dtype": "float32",
"shape": [
1,
126
],
"values": "1.0 on t < P = l3 = (l2 - 1) // 2 + 1"
}
},
"outputs": {
"prefix": {
"dtype": "float32",
"shape": [
1,
126,
1024
],
"read": "rows [0, P): the clip's prefix embeddings"
}
}
},
"20s": {
"bucket": {
"seconds": 20,
"F": 2001,
"F2": 1001,
"F4": 501,
"T": 251,
"holds": "clips of n samples with 1 + n // 160 <= 2001 after waveform()'s cut and pad (host.audio_bucket_for: the smallest of [5, 10, 20, 30] s that holds it)",
"max_prefix_rows": 250
},
"inputs": {
"mel": {
"dtype": "float32",
"shape": [
1,
128,
2001
],
"values": "the clip's normalised log-mel (mel_host: waveform() + MelFrontend), columns t >= frames 0.0, zero-padded to F"
},
"mask_f": {
"dtype": "float32",
"shape": [
1,
2001
],
"values": "1.0 on t < l0 = frames (n // 160)"
},
"mask_f2": {
"dtype": "float32",
"shape": [
1,
1001
],
"values": "1.0 on t < l1 = (l0 - 1) // 2 + 1"
},
"mask_f4": {
"dtype": "float32",
"shape": [
1,
501
],
"values": "1.0 on t < l2 = (l1 - 1) // 2 + 1"
},
"mask_t": {
"dtype": "float32",
"shape": [
1,
251
],
"values": "1.0 on t < P = l3 = (l2 - 1) // 2 + 1"
}
},
"outputs": {
"prefix": {
"dtype": "float32",
"shape": [
1,
251,
1024
],
"read": "rows [0, P): the clip's prefix embeddings"
}
}
},
"30s": {
"bucket": {
"seconds": 30,
"F": 3001,
"F2": 1501,
"F4": 751,
"T": 376,
"holds": "clips of n samples with 1 + n // 160 <= 3001 after waveform()'s cut and pad (host.audio_bucket_for: the smallest of [5, 10, 20, 30] s that holds it)",
"max_prefix_rows": 375
},
"inputs": {
"mel": {
"dtype": "float32",
"shape": [
1,
128,
3001
],
"values": "the clip's normalised log-mel (mel_host: waveform() + MelFrontend), columns t >= frames 0.0, zero-padded to F"
},
"mask_f": {
"dtype": "float32",
"shape": [
1,
3001
],
"values": "1.0 on t < l0 = frames (n // 160)"
},
"mask_f2": {
"dtype": "float32",
"shape": [
1,
1501
],
"values": "1.0 on t < l1 = (l0 - 1) // 2 + 1"
},
"mask_f4": {
"dtype": "float32",
"shape": [
1,
751
],
"values": "1.0 on t < l2 = (l1 - 1) // 2 + 1"
},
"mask_t": {
"dtype": "float32",
"shape": [
1,
376
],
"values": "1.0 on t < P = l3 = (l2 - 1) // 2 + 1"
}
},
"outputs": {
"prefix": {
"dtype": "float32",
"shape": [
1,
376,
1024
],
"read": "rows [0, P): the clip's prefix embeddings"
}
}
}
},
"role": "d1-omni audio prefix: one clip per call",
"waveform": "16 kHz mono; int16 / 32768 or float32 as is; cut to 480,000 samples (30 s); zero-padded to 8,000 (0.5 s); no resampling (the publisher's waveform())",
"mel": {
"spec": "mel_host.py (module docstring): preemphasis 0.97; center=True constant-padded STFT, n_fft 512, hop 160, Hann(400, periodic=False) centred in 512; power (sqrt(re^2+im^2))^2; Slaney mel 128 (mel_filters_128x257_f32.bin); log(x + 2^-24); per mel row over the frames t < n // 160: (x - mean) / (std(ddof 1) + 1e-5); 0.0 after",
"python": "mel_host.mel_torch (the publisher's code, bit for bit)",
"swift_form": "mel_host.mel_numpy (the same steps in NumPy, float64)"
},
"filterbank": {
"file": "mel_filters_128x257_f32.bin",
"shape": [
128,
257
],
"dtype": "float32",
"byte_order": "little",
"layout": "row-major (mel row, FFT bin)",
"made_by": "the publisher's slaney_filterbank() (librosa's Slaney mel, 0 to 8 kHz)",
"sha256": "bce5ec5f194a5913f6508cee5a85512e7bad2352db8fc28f5c6ff75af8b09137"
},
"prefix": "the graph's first P rows (P = host.audio_prefix_length(n)); this is the decision graph's prefix_embeds [0, P) with max_len = min(15360, 16384 - P) and state None -> {}",
"precision_detail": {
"weights": "fp16 (every Linear / Conv weight and bias, pos_bias_u / v)",
"compute": "fp16 (LayerNorm, the attention softmax, the BatchNorm affine, both GELUs and the output in fp32)",
"constants_fp32": "the relative-position table [1, 2T-1, 512] and the folded BatchNorm scale / shift"
},
"folders": {
"macos": {
"audio-fp16-5s": "macos/audio-fp16-5s",
"audio-fp16-10s": "macos/audio-fp16-10s",
"audio-fp16-20s": "macos/audio-fp16-20s",
"audio-fp16-30s": "macos/audio-fp16-30s"
},
"ios": {
"audio-fp16-5s": "ios/audio-fp16-5s",
"audio-fp16-10s": "ios/audio-fp16-10s",
"audio-fp16-20s": "ios/audio-fp16-20s",
"audio-fp16-30s": "ios/audio-fp16-30s"
},
"ios-h19p": {
"audio-fp16-5s": "ios-h19p/audio-fp16-5s",
"audio-fp16-10s": "ios-h19p/audio-fp16-10s",
"audio-fp16-20s": "ios-h19p/audio-fp16-20s",
"audio-fp16-30s": "ios-h19p/audio-fp16-30s"
}
}
}
},
"token_ids": {
"<|pad|>": 0,
"<|startoftext|>": 1,
"<|im_end|>": 7,
"<|mask|>": 16,
"<|reserved_7|>": 17,
"<|reserved_8|>": 18,
"<|reserved_9|>": 19,
"<|reserved_10|>": 20,
"<|reserved_11|>": 21
},
"delimiters": {
"state": "<|reserved_7|>",
"q": "<|reserved_8|>",
"opt": "<|reserved_9|>",
"opt_end": "<|reserved_10|>",
"decide": "<|reserved_11|>",
"marker": "<|mask|>"
},
"temperatures": {
"choice:11+": 1.372515082359314,
"choice:2": 1.7465145587921143,
"choice:3-5": 1.3998981714248657,
"choice:6-10": 1.1751071214675903,
"noul:2": 1.6663223505020142,
"score:3-5": 1.7301132678985596,
"score:6-10": 1.0,
"choice": 1.0,
"score": 1.0,
"noul": 1.0
},
"temperature_rule": "text rows: marker logits / temperatures[f'{type}:{2|3-5|6-10|11+}' by option count] if present else temperatures[type]; image and audio rows: no temperature",
"noul_order": [
"false",
"true"
],
"noul_reported": [
"yes",
"no"
],
"tokenizer": {
"folder": "tokenizer/",
"encode": "tok(escape(s), add_special_tokens=False): no <|startoftext|> from the post-processor; escape() rewrites every <|name|> in caller text as <¦name¦>"
},
"host": {
"python": "coreai-model-zoo conversion/d1_omni/host.py (request_rows, graph_inputs, probabilities_from_logits, image_crops_inputs, audio_inputs)",
"swift": "coreai-model-zoo apps/D1Omni (library D1Omni, CLI d1omni; a copy in host/Package.swift and host/Sources): text, image and audio requests; the JPEG rule is in host/README.md"
},
"platforms": {
"macos": "the .aimodel; the Mac specializes it at load (the Swift host's JIT and AOT gave the same marker logits bit for bit and the same ms)",
"ios": "the same .aimodel; every phone specializes it on its first load",
"ios-h19p": "iPhone 18 Pro (h19p) GPU AOT of the same bundles except decide-fp16-L4096 (its compile exceeds the phone's per-process memory limit at the first call: a phone runs ios/decide-fp16-L4096); another phone generation refuses them at load and takes ios/"
},
"staged": {
"decide-fp16-L64": {
"bundle": "d1_omni_decide_fp16_L64.aimodel",
"bundle_bytes": 761631060,
"main_hash": "e9e43ae73782c138dc785bc7fb14d484319926e95c19d08506b3c83bec357f74",
"aimodelc_h19p": "d1_omni_decide_fp16_L64.h19p.aimodelc",
"aimodelc_h19p_bytes": 761747067,
"aimodelc_h19p_main_hash": "eff8f9efff9098fbaebd4ec088d08430f9b5b1b040d1d82a2c0edce25fe7b88f"
},
"decide-fp16-L128": {
"bundle": "d1_omni_decide_fp16_L128.aimodel",
"bundle_bytes": 761647561,
"main_hash": "aa6af50cfa06c729f4a618ec0cb6d7a85515ed7f32159e81e5e49991417bf79f",
"aimodelc_h19p": "d1_omni_decide_fp16_L128.h19p.aimodelc",
"aimodelc_h19p_bytes": 761763726,
"aimodelc_h19p_main_hash": "fe544a478d29e743b5e615a18b4d4f98925ba89905dda872efe0e341fe68fd26"
},
"decide-fp16-L256": {
"bundle": "d1_omni_decide_fp16_L256.aimodel",
"bundle_bytes": 761680330,
"main_hash": "eb4b4f9dd21f8454b49d7433b31e18b2a00c39215f5049c7137ff53306ee943b",
"aimodelc_h19p": "d1_omni_decide_fp16_L256.h19p.aimodelc",
"aimodelc_h19p_bytes": 761796331,
"aimodelc_h19p_main_hash": "7ba3b11ff7d253b1f97ecca8c177d18f953b9e844ff9eabda6b28454b71de937"
},
"decide-fp16-L512": {
"bundle": "d1_omni_decide_fp16_L512.aimodel",
"bundle_bytes": 761745868,
"main_hash": "4f73c0b4742272e6e8c57a0368ca14521ae7845b3e5d469e697aa5e4027e48b7",
"aimodelc_h19p": "d1_omni_decide_fp16_L512.h19p.aimodelc",
"aimodelc_h19p_bytes": 761861994,
"aimodelc_h19p_main_hash": "1d54cd013075829da188bc5be1ed7ee1f3f071371aa391f2bc61738c9f37558e"
},
"decide-fp16-L1024": {
"bundle": "d1_omni_decide_fp16_L1024.aimodel",
"bundle_bytes": 761876787,
"main_hash": "92bf9613ca22138709d741196796fcde9302f01b66cfade2e5c9b6237ef35c73",
"aimodelc_h19p": "d1_omni_decide_fp16_L1024.h19p.aimodelc",
"aimodelc_h19p_bytes": 761993099,
"aimodelc_h19p_main_hash": "c21863c40b02ef7c0e3e7d87297016864d209ab306bc16bc551c7841202b1cd7"
},
"decide-fp16-L2048": {
"bundle": "d1_omni_decide_fp16_L2048.aimodel",
"bundle_bytes": 762139098,
"main_hash": "51b6cbe1b3c13267c544d7f596bdc265e7214b68d6386a03dd1f31c45d1a4ee5",
"aimodelc_h19p": "d1_omni_decide_fp16_L2048.h19p.aimodelc",
"aimodelc_h19p_bytes": 762255228,
"aimodelc_h19p_main_hash": "117d089252f05d23fdcdbaebcead8da2fc4bf1f69da18f6f873ee000c487332d"
},
"decide-fp16-L4096": {
"bundle": "d1_omni_decide_fp16_L4096.aimodel",
"bundle_bytes": 762676555,
"main_hash": "4c38676fe90011b05fc79e6c5cca04fe2eb0f9b72c92e71aec64e1d3ffa664a5"
},
"vision-fp16": {
"bundle": "d1_omni_vision_fp16.aimodel",
"bundle_bytes": 188218374,
"main_hash": "d65cf00787bab8bd5ee390cc8820ab600ba44cfe5a72ee43574a0ffd3d3d148c",
"aimodelc_h19p": "d1_omni_vision_fp16.h19p.aimodelc",
"aimodelc_h19p_bytes": 188303983,
"aimodelc_h19p_main_hash": "fc3e1278f91c97cdf9419aa154b5041e7f6fb82525d245c4135aab972d83bef1"
},
"audio-fp16-5s": {
"bundle": "d1_omni_audio_fp16_5s.aimodel",
"bundle_bytes": 224920212,
"main_hash": "b3240c83a181d87adda6fa97fe52661ab1fa6d82b6c62b651d5481cfe5138685",
"aimodelc_h19p": "d1_omni_audio_fp16_5s.h19p.aimodelc",
"aimodelc_h19p_bytes": 225226318,
"aimodelc_h19p_main_hash": "09a5141b87f4abc2be1169bbda2528ad492cfb9681ad9bc8761d237c9ad7839c"
},
"audio-fp16-10s": {
"bundle": "d1_omni_audio_fp16_10s.aimodel",
"bundle_bytes": 225049263,
"main_hash": "0efcce362b71f05bb90eb11eb22b087521e990135f2bfdd2efd72a6b213eaa01",
"aimodelc_h19p": "d1_omni_audio_fp16_10s.h19p.aimodelc",
"aimodelc_h19p_bytes": 225355344,
"aimodelc_h19p_main_hash": "0a66cbd973f93c56bcc086df520399d97a05c0857d8aab73c3e58e59209a12ae"
},
"audio-fp16-20s": {
"bundle": "d1_omni_audio_fp16_20s.aimodel",
"bundle_bytes": 225305277,
"main_hash": "9de9ec76dc8c6abb22688ce7e842cb056cee0ca15ffe6cb3a66ecdfe0dab515e",
"aimodelc_h19p": "d1_omni_audio_fp16_20s.h19p.aimodelc",
"aimodelc_h19p_bytes": 225611159,
"aimodelc_h19p_main_hash": "0039312e4d292a6c00f18480a8dc90c66743762546123ffa842f166e1d382e5c"
},
"audio-fp16-30s": {
"bundle": "d1_omni_audio_fp16_30s.aimodel",
"bundle_bytes": 225561282,
"main_hash": "6bb3911322160cd29ec93e72c5d78e7c9ab07b3a1073552fe2fda053bee61c65",
"aimodelc_h19p": "d1_omni_audio_fp16_30s.h19p.aimodelc",
"aimodelc_h19p_bytes": 225867256,
"aimodelc_h19p_main_hash": "1354d0680107fc4b551fe93c35ab5806dc6c882bbe5c32f3f295f1b6d1e798ab"
}
},
"hash_note": "main_hash = the .aimodel's main.hash, sha256 of its main.mlirb (the name of the compile-cache entry a JIT load creates); aimodelc_h19p_main_hash = the AOT's main.hash"
}