{ "schema": "d1-omni-coreai/1", "model": { "source": "LiquidAI/d1-omni-600M", "revision": "414f8d6438174f5b2133a9c21a478fc42625e308", "license": "LFM Open License v1.0", "conversion": "coreai-model-zoo conversion/d1_omni (re-authored in plain PyTorch, coreai-torch 0.4.1; the graphs' debug locations stripped with coreai-torch strip_debug_info, conversion/d1_omni/strip_ship.py)" }, "runtime": "Core AI (macOS 27 / iOS 27)", "precision": "fp16 (weights and compute; the exceptions are listed in each graph's metadata.json)", "graphs": { "decision": { "buckets": [ 64, 128, 256, 512, 1024, 2048, 4096 ], "bucket_rule": "the smallest L that holds P + len(ids) (host.py bucket_for(positions, ALL_BUCKETS))", "inputs": { "input_ids": { "dtype": "int32", "shape": [ 1, "L" ], "values": "0 on [0, P), the row's ids on [P, P+n), 0 (<|pad|>) after" }, "prefix_embeds": { "dtype": "float32", "shape": [ 1, "L", 1024 ], "values": "the image / audio prefix on [0, P), 0.0 elsewhere (finite)" }, "pad_mask": { "dtype": "float32", "shape": [ 1, "L" ], "values": "1.0 on [0, P+n), 0.0 after" }, "prefix_mask": { "dtype": "float32", "shape": [ 1, "L" ], "values": "1.0 on [0, P), 0.0 after" }, "keep_right": { "dtype": "float32", "shape": [ 1, "L" ], "values": "0.0 at P-1 when P > 0, 1.0 elsewhere" }, "qtype_onehot": { "dtype": "float32", "shape": [ 1, 3 ], "order": [ "choice", "score", "noul" ] } }, "outputs": { "scores": { "dtype": "float32", "shape": [ 1, "L" ], "read": "at P + markers (the <|mask|> positions)" } }, "max_length": 16384, "image_text_length": 896, "audio_text_length": 15360, "min_text_positions": 64, "prefix_hidden": 1024, "folders": { "macos": { "decide-fp16-L64": "macos/decide-fp16-L64", "decide-fp16-L128": "macos/decide-fp16-L128", "decide-fp16-L256": "macos/decide-fp16-L256", "decide-fp16-L512": "macos/decide-fp16-L512", "decide-fp16-L1024": "macos/decide-fp16-L1024", "decide-fp16-L2048": "macos/decide-fp16-L2048", "decide-fp16-L4096": "macos/decide-fp16-L4096" }, "ios": { "decide-fp16-L64": "ios/decide-fp16-L64", "decide-fp16-L128": "ios/decide-fp16-L128", "decide-fp16-L256": "ios/decide-fp16-L256", "decide-fp16-L512": "ios/decide-fp16-L512", "decide-fp16-L1024": "ios/decide-fp16-L1024", "decide-fp16-L2048": "ios/decide-fp16-L2048", "decide-fp16-L4096": "ios/decide-fp16-L4096" }, "ios-h19p": { "decide-fp16-L64": "ios-h19p/decide-fp16-L64", "decide-fp16-L128": "ios-h19p/decide-fp16-L128", "decide-fp16-L256": "ios-h19p/decide-fp16-L256", "decide-fp16-L512": "ios-h19p/decide-fp16-L512", "decide-fp16-L1024": "ios-h19p/decide-fp16-L1024", "decide-fp16-L2048": "ios-h19p/decide-fp16-L2048" } } }, "vision": { "role": "d1-omni image prefix: one crop per call", "inputs": { "pixel_values": { "dtype": "float32", "shape": [ 1, 1024, 768 ], "values": "the crop's 16x16 patches row-major over (ph, pw), each [py][px][c], (x - 127.5) / 127.5; 0.0 past ph*pw" }, "pos_embed": { "dtype": "float32", "shape": [ 1, 1024, 768 ], "values": "position_table resized to (ph, pw), bilinear + antialias (host.position_embeddings / position_embeddings_numpy), row-major; 0.0 past ph*pw" }, "patch_mask": { "dtype": "float32", "shape": [ 1, 1024 ], "values": "1.0 on the ph*pw patches, 0.0 after" }, "unshuffle_index": { "dtype": "int32", "shape": [ 256, 4 ], "values": "token (i, j) = row i * (pw/2) + j: patches 2i*pw+2j, 2i*pw+2j+1, (2i+1)*pw+2j, (2i+1)*pw+2j+1; 0 past (ph/2)(pw/2) (host.unshuffle_index)" } }, "outputs": { "prefix": { "dtype": "float32", "shape": [ 1, 256, 1024 ], "read": "rows [0, (ph/2)(pw/2)): the crop's prefix embeddings" } }, "max_patches": 1024, "max_tokens": 256, "patch": 16, "crops": "vision.preprocess(): an image whose rounded area exceeds 2 x 512^2 is cut into the 512 px tiles of the closest grid (2 to 10 tiles, row-major) plus a thumbnail, else the thumbnail alone; each crop resized by torchvision (bilinear, antialias) from the RGB image (host.layout / image_crops)", "prefix": "each crop's first (ph/2)(pw/2) output rows, crops in order, images in request order; this is the decision graph's prefix_embeds [0, P)", "position_table": { "file": "position_table.f32", "shape": [ 16, 16, 768 ], "dtype": "float32", "byte_order": "little", "layout": "row-major (row, column, channel)", "resize": "per crop to (ph, pw): bilinear, align_corners false, antialias (torch's float32 kernel: width pass then height pass, fused multiply-add per tap; host.position_embeddings_numpy)", "sha256": "69b3c175e58af01f99eb4aa566e2c238d2e8b24c9a0d505fbcb6d5e3a095bf6f" }, "precision_detail": { "weights": "fp16", "compute": "fp16 (LayerNorm, the attention softmax and the projector's gelu in fp32)" }, "folders": { "macos": { "vision-fp16": "macos/vision-fp16" }, "ios": { "vision-fp16": "ios/vision-fp16" }, "ios-h19p": { "vision-fp16": "ios-h19p/vision-fp16" } } }, "audio": { "buckets_s": [ 5, 10, 20, 30 ], "per_bucket": { "5s": { "bucket": { "seconds": 5, "F": 501, "F2": 251, "F4": 126, "T": 63, "holds": "clips of n samples with 1 + n // 160 <= 501 after waveform()'s cut and pad (host.audio_bucket_for: the smallest of [5, 10, 20, 30] s that holds it)", "max_prefix_rows": 63 }, "inputs": { "mel": { "dtype": "float32", "shape": [ 1, 128, 501 ], "values": "the clip's normalised log-mel (mel_host: waveform() + MelFrontend), columns t >= frames 0.0, zero-padded to F" }, "mask_f": { "dtype": "float32", "shape": [ 1, 501 ], "values": "1.0 on t < l0 = frames (n // 160)" }, "mask_f2": { "dtype": "float32", "shape": [ 1, 251 ], "values": "1.0 on t < l1 = (l0 - 1) // 2 + 1" }, "mask_f4": { "dtype": "float32", "shape": [ 1, 126 ], "values": "1.0 on t < l2 = (l1 - 1) // 2 + 1" }, "mask_t": { "dtype": "float32", "shape": [ 1, 63 ], "values": "1.0 on t < P = l3 = (l2 - 1) // 2 + 1" } }, "outputs": { "prefix": { "dtype": "float32", "shape": [ 1, 63, 1024 ], "read": "rows [0, P): the clip's prefix embeddings" } } }, "10s": { "bucket": { "seconds": 10, "F": 1001, "F2": 501, "F4": 251, "T": 126, "holds": "clips of n samples with 1 + n // 160 <= 1001 after waveform()'s cut and pad (host.audio_bucket_for: the smallest of [5, 10, 20, 30] s that holds it)", "max_prefix_rows": 125 }, "inputs": { "mel": { "dtype": "float32", "shape": [ 1, 128, 1001 ], "values": "the clip's normalised log-mel (mel_host: waveform() + MelFrontend), columns t >= frames 0.0, zero-padded to F" }, "mask_f": { "dtype": "float32", "shape": [ 1, 1001 ], "values": "1.0 on t < l0 = frames (n // 160)" }, "mask_f2": { "dtype": "float32", "shape": [ 1, 501 ], "values": "1.0 on t < l1 = (l0 - 1) // 2 + 1" }, "mask_f4": { "dtype": "float32", "shape": [ 1, 251 ], "values": "1.0 on t < l2 = (l1 - 1) // 2 + 1" }, "mask_t": { "dtype": "float32", "shape": [ 1, 126 ], "values": "1.0 on t < P = l3 = (l2 - 1) // 2 + 1" } }, "outputs": { "prefix": { "dtype": "float32", "shape": [ 1, 126, 1024 ], "read": "rows [0, P): the clip's prefix embeddings" } } }, "20s": { "bucket": { "seconds": 20, "F": 2001, "F2": 1001, "F4": 501, "T": 251, "holds": "clips of n samples with 1 + n // 160 <= 2001 after waveform()'s cut and pad (host.audio_bucket_for: the smallest of [5, 10, 20, 30] s that holds it)", "max_prefix_rows": 250 }, "inputs": { "mel": { "dtype": "float32", "shape": [ 1, 128, 2001 ], "values": "the clip's normalised log-mel (mel_host: waveform() + MelFrontend), columns t >= frames 0.0, zero-padded to F" }, "mask_f": { "dtype": "float32", "shape": [ 1, 2001 ], "values": "1.0 on t < l0 = frames (n // 160)" }, "mask_f2": { "dtype": "float32", "shape": [ 1, 1001 ], "values": "1.0 on t < l1 = (l0 - 1) // 2 + 1" }, "mask_f4": { "dtype": "float32", "shape": [ 1, 501 ], "values": "1.0 on t < l2 = (l1 - 1) // 2 + 1" }, "mask_t": { "dtype": "float32", "shape": [ 1, 251 ], "values": "1.0 on t < P = l3 = (l2 - 1) // 2 + 1" } }, "outputs": { "prefix": { "dtype": "float32", "shape": [ 1, 251, 1024 ], "read": "rows [0, P): the clip's prefix embeddings" } } }, "30s": { "bucket": { "seconds": 30, "F": 3001, "F2": 1501, "F4": 751, "T": 376, "holds": "clips of n samples with 1 + n // 160 <= 3001 after waveform()'s cut and pad (host.audio_bucket_for: the smallest of [5, 10, 20, 30] s that holds it)", "max_prefix_rows": 375 }, "inputs": { "mel": { "dtype": "float32", "shape": [ 1, 128, 3001 ], "values": "the clip's normalised log-mel (mel_host: waveform() + MelFrontend), columns t >= frames 0.0, zero-padded to F" }, "mask_f": { "dtype": "float32", "shape": [ 1, 3001 ], "values": "1.0 on t < l0 = frames (n // 160)" }, "mask_f2": { "dtype": "float32", "shape": [ 1, 1501 ], "values": "1.0 on t < l1 = (l0 - 1) // 2 + 1" }, "mask_f4": { "dtype": "float32", "shape": [ 1, 751 ], "values": "1.0 on t < l2 = (l1 - 1) // 2 + 1" }, "mask_t": { "dtype": "float32", "shape": [ 1, 376 ], "values": "1.0 on t < P = l3 = (l2 - 1) // 2 + 1" } }, "outputs": { "prefix": { "dtype": "float32", "shape": [ 1, 376, 1024 ], "read": "rows [0, P): the clip's prefix embeddings" } } } }, "role": "d1-omni audio prefix: one clip per call", "waveform": "16 kHz mono; int16 / 32768 or float32 as is; cut to 480,000 samples (30 s); zero-padded to 8,000 (0.5 s); no resampling (the publisher's waveform())", "mel": { "spec": "mel_host.py (module docstring): preemphasis 0.97; center=True constant-padded STFT, n_fft 512, hop 160, Hann(400, periodic=False) centred in 512; power (sqrt(re^2+im^2))^2; Slaney mel 128 (mel_filters_128x257_f32.bin); log(x + 2^-24); per mel row over the frames t < n // 160: (x - mean) / (std(ddof 1) + 1e-5); 0.0 after", "python": "mel_host.mel_torch (the publisher's code, bit for bit)", "swift_form": "mel_host.mel_numpy (the same steps in NumPy, float64)" }, "filterbank": { "file": "mel_filters_128x257_f32.bin", "shape": [ 128, 257 ], "dtype": "float32", "byte_order": "little", "layout": "row-major (mel row, FFT bin)", "made_by": "the publisher's slaney_filterbank() (librosa's Slaney mel, 0 to 8 kHz)", "sha256": "bce5ec5f194a5913f6508cee5a85512e7bad2352db8fc28f5c6ff75af8b09137" }, "prefix": "the graph's first P rows (P = host.audio_prefix_length(n)); this is the decision graph's prefix_embeds [0, P) with max_len = min(15360, 16384 - P) and state None -> {}", "precision_detail": { "weights": "fp16 (every Linear / Conv weight and bias, pos_bias_u / v)", "compute": "fp16 (LayerNorm, the attention softmax, the BatchNorm affine, both GELUs and the output in fp32)", "constants_fp32": "the relative-position table [1, 2T-1, 512] and the folded BatchNorm scale / shift" }, "folders": { "macos": { "audio-fp16-5s": "macos/audio-fp16-5s", "audio-fp16-10s": "macos/audio-fp16-10s", "audio-fp16-20s": "macos/audio-fp16-20s", "audio-fp16-30s": "macos/audio-fp16-30s" }, "ios": { "audio-fp16-5s": "ios/audio-fp16-5s", "audio-fp16-10s": "ios/audio-fp16-10s", "audio-fp16-20s": "ios/audio-fp16-20s", "audio-fp16-30s": "ios/audio-fp16-30s" }, "ios-h19p": { "audio-fp16-5s": "ios-h19p/audio-fp16-5s", "audio-fp16-10s": "ios-h19p/audio-fp16-10s", "audio-fp16-20s": "ios-h19p/audio-fp16-20s", "audio-fp16-30s": "ios-h19p/audio-fp16-30s" } } } }, "token_ids": { "<|pad|>": 0, "<|startoftext|>": 1, "<|im_end|>": 7, "<|mask|>": 16, "<|reserved_7|>": 17, "<|reserved_8|>": 18, "<|reserved_9|>": 19, "<|reserved_10|>": 20, "<|reserved_11|>": 21 }, "delimiters": { "state": "<|reserved_7|>", "q": "<|reserved_8|>", "opt": "<|reserved_9|>", "opt_end": "<|reserved_10|>", "decide": "<|reserved_11|>", "marker": "<|mask|>" }, "temperatures": { "choice:11+": 1.372515082359314, "choice:2": 1.7465145587921143, "choice:3-5": 1.3998981714248657, "choice:6-10": 1.1751071214675903, "noul:2": 1.6663223505020142, "score:3-5": 1.7301132678985596, "score:6-10": 1.0, "choice": 1.0, "score": 1.0, "noul": 1.0 }, "temperature_rule": "text rows: marker logits / temperatures[f'{type}:{2|3-5|6-10|11+}' by option count] if present else temperatures[type]; image and audio rows: no temperature", "noul_order": [ "false", "true" ], "noul_reported": [ "yes", "no" ], "tokenizer": { "folder": "tokenizer/", "encode": "tok(escape(s), add_special_tokens=False): no <|startoftext|> from the post-processor; escape() rewrites every <|name|> in caller text as <¦name¦>" }, "host": { "python": "coreai-model-zoo conversion/d1_omni/host.py (request_rows, graph_inputs, probabilities_from_logits, image_crops_inputs, audio_inputs)", "swift": "coreai-model-zoo apps/D1Omni (library D1Omni, CLI d1omni; a copy in host/Package.swift and host/Sources): text, image and audio requests; the JPEG rule is in host/README.md" }, "platforms": { "macos": "the .aimodel; the Mac specializes it at load (the Swift host's JIT and AOT gave the same marker logits bit for bit and the same ms)", "ios": "the same .aimodel; every phone specializes it on its first load", "ios-h19p": "iPhone 18 Pro (h19p) GPU AOT of the same bundles except decide-fp16-L4096 (its compile exceeds the phone's per-process memory limit at the first call: a phone runs ios/decide-fp16-L4096); another phone generation refuses them at load and takes ios/" }, "staged": { "decide-fp16-L64": { "bundle": "d1_omni_decide_fp16_L64.aimodel", "bundle_bytes": 761631060, "main_hash": "e9e43ae73782c138dc785bc7fb14d484319926e95c19d08506b3c83bec357f74", "aimodelc_h19p": "d1_omni_decide_fp16_L64.h19p.aimodelc", "aimodelc_h19p_bytes": 761747067, "aimodelc_h19p_main_hash": "eff8f9efff9098fbaebd4ec088d08430f9b5b1b040d1d82a2c0edce25fe7b88f" }, "decide-fp16-L128": { "bundle": "d1_omni_decide_fp16_L128.aimodel", "bundle_bytes": 761647561, "main_hash": "aa6af50cfa06c729f4a618ec0cb6d7a85515ed7f32159e81e5e49991417bf79f", "aimodelc_h19p": "d1_omni_decide_fp16_L128.h19p.aimodelc", "aimodelc_h19p_bytes": 761763726, "aimodelc_h19p_main_hash": "fe544a478d29e743b5e615a18b4d4f98925ba89905dda872efe0e341fe68fd26" }, "decide-fp16-L256": { "bundle": "d1_omni_decide_fp16_L256.aimodel", "bundle_bytes": 761680330, "main_hash": "eb4b4f9dd21f8454b49d7433b31e18b2a00c39215f5049c7137ff53306ee943b", "aimodelc_h19p": "d1_omni_decide_fp16_L256.h19p.aimodelc", "aimodelc_h19p_bytes": 761796331, "aimodelc_h19p_main_hash": "7ba3b11ff7d253b1f97ecca8c177d18f953b9e844ff9eabda6b28454b71de937" }, "decide-fp16-L512": { "bundle": "d1_omni_decide_fp16_L512.aimodel", "bundle_bytes": 761745868, "main_hash": "4f73c0b4742272e6e8c57a0368ca14521ae7845b3e5d469e697aa5e4027e48b7", "aimodelc_h19p": "d1_omni_decide_fp16_L512.h19p.aimodelc", "aimodelc_h19p_bytes": 761861994, "aimodelc_h19p_main_hash": "1d54cd013075829da188bc5be1ed7ee1f3f071371aa391f2bc61738c9f37558e" }, "decide-fp16-L1024": { "bundle": "d1_omni_decide_fp16_L1024.aimodel", "bundle_bytes": 761876787, "main_hash": "92bf9613ca22138709d741196796fcde9302f01b66cfade2e5c9b6237ef35c73", "aimodelc_h19p": "d1_omni_decide_fp16_L1024.h19p.aimodelc", "aimodelc_h19p_bytes": 761993099, "aimodelc_h19p_main_hash": "c21863c40b02ef7c0e3e7d87297016864d209ab306bc16bc551c7841202b1cd7" }, "decide-fp16-L2048": { "bundle": "d1_omni_decide_fp16_L2048.aimodel", "bundle_bytes": 762139098, "main_hash": "51b6cbe1b3c13267c544d7f596bdc265e7214b68d6386a03dd1f31c45d1a4ee5", "aimodelc_h19p": "d1_omni_decide_fp16_L2048.h19p.aimodelc", "aimodelc_h19p_bytes": 762255228, "aimodelc_h19p_main_hash": "117d089252f05d23fdcdbaebcead8da2fc4bf1f69da18f6f873ee000c487332d" }, "decide-fp16-L4096": { "bundle": "d1_omni_decide_fp16_L4096.aimodel", "bundle_bytes": 762676555, "main_hash": "4c38676fe90011b05fc79e6c5cca04fe2eb0f9b72c92e71aec64e1d3ffa664a5" }, "vision-fp16": { "bundle": "d1_omni_vision_fp16.aimodel", "bundle_bytes": 188218374, "main_hash": "d65cf00787bab8bd5ee390cc8820ab600ba44cfe5a72ee43574a0ffd3d3d148c", "aimodelc_h19p": "d1_omni_vision_fp16.h19p.aimodelc", "aimodelc_h19p_bytes": 188303983, "aimodelc_h19p_main_hash": "fc3e1278f91c97cdf9419aa154b5041e7f6fb82525d245c4135aab972d83bef1" }, "audio-fp16-5s": { "bundle": "d1_omni_audio_fp16_5s.aimodel", "bundle_bytes": 224920212, "main_hash": "b3240c83a181d87adda6fa97fe52661ab1fa6d82b6c62b651d5481cfe5138685", "aimodelc_h19p": "d1_omni_audio_fp16_5s.h19p.aimodelc", "aimodelc_h19p_bytes": 225226318, "aimodelc_h19p_main_hash": "09a5141b87f4abc2be1169bbda2528ad492cfb9681ad9bc8761d237c9ad7839c" }, "audio-fp16-10s": { "bundle": "d1_omni_audio_fp16_10s.aimodel", "bundle_bytes": 225049263, "main_hash": "0efcce362b71f05bb90eb11eb22b087521e990135f2bfdd2efd72a6b213eaa01", "aimodelc_h19p": "d1_omni_audio_fp16_10s.h19p.aimodelc", "aimodelc_h19p_bytes": 225355344, "aimodelc_h19p_main_hash": "0a66cbd973f93c56bcc086df520399d97a05c0857d8aab73c3e58e59209a12ae" }, "audio-fp16-20s": { "bundle": "d1_omni_audio_fp16_20s.aimodel", "bundle_bytes": 225305277, "main_hash": "9de9ec76dc8c6abb22688ce7e842cb056cee0ca15ffe6cb3a66ecdfe0dab515e", "aimodelc_h19p": "d1_omni_audio_fp16_20s.h19p.aimodelc", "aimodelc_h19p_bytes": 225611159, "aimodelc_h19p_main_hash": "0039312e4d292a6c00f18480a8dc90c66743762546123ffa842f166e1d382e5c" }, "audio-fp16-30s": { "bundle": "d1_omni_audio_fp16_30s.aimodel", "bundle_bytes": 225561282, "main_hash": "6bb3911322160cd29ec93e72c5d78e7c9ab07b3a1073552fe2fda053bee61c65", "aimodelc_h19p": "d1_omni_audio_fp16_30s.h19p.aimodelc", "aimodelc_h19p_bytes": 225867256, "aimodelc_h19p_main_hash": "1354d0680107fc4b551fe93c35ab5806dc6c882bbe5c32f3f295f1b6d1e798ab" } }, "hash_note": "main_hash = the .aimodel's main.hash, sha256 of its main.mlirb (the name of the compile-cache entry a JIT load creates); aimodelc_h19p_main_hash = the AOT's main.hash" }