Release complete mixed Q3/Q4 Audio8 with portable CPU, MLX and exact GGUF GPU consumers
c49eca9 verified about 1 month ago
28.7 kB
#include "audio8_model.h"
#include "ggml-alloc.h"
#include "gguf.h"

#include <algorithm>
#include <bit>
#include <cstdio>
#include <cstring>
#include <initializer_list>
#include <limits>
#include <map>
#include <set>
#include <stdexcept>
#include <type_traits>
#include <utility>
#include <vector>

namespace audio8 {
namespace {
constexpr std::size_t kTensorCount = 938;
constexpr std::uint64_t kMetadataLimit = 64ULL * 1024 * 1024;
constexpr std::uint64_t kSourcePayloadBytes = 2313973312ULL;
constexpr std::uint64_t kOriginalBytes = 21474880ULL;
constexpr const char * kRevision = "b4413de154ed6bdef0a4011028b1ebd12aca8152";
constexpr const char * kGgmlRevision = "84e76d8a23162eca70490da131945ebec1f09bf4";
constexpr const char * kAlias = "language_model.lm_head.weight";
constexpr const char * kEmbedding = "language_model.model.embed_tokens.weight";
constexpr const char * kAliasesJson =
    "{\"language_model.lm_head.weight\":\"language_model.model.embed_tokens.weight\"}";

struct FileCloser {
    void operator()(FILE * file) const noexcept {
        if (file != nullptr) std::fclose(file);
    }
};

void require(bool ok, const std::string & message) {
    if (!ok) throw std::runtime_error("Audio8 GGUF: " + message);
}

std::uint64_t add(std::uint64_t a, std::uint64_t b) {
    require(a <= std::numeric_limits<std::uint64_t>::max()-b, "size overflow");
    return a+b;
}

std::uint64_t aligned(std::uint64_t n) { return add(n, 31)/32*32; }

void read_exact(FILE * file, void * out, std::size_t bytes) {
    require(bytes == 0 || std::fread(out, 1, bytes, file) == bytes, "truncated file/read error");
}

void seek(FILE * file, std::uint64_t offset) {
    require(offset <= static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max()), "file offset overflow");
#if defined(_WIN32)
    require(_fseeki64(file, static_cast<__int64>(offset), SEEK_SET) == 0, "seek failed");
#else
    static_assert(sizeof(off_t) >= 8, "Audio8 loader requires 64-bit file offsets");
    require(fseeko(file, static_cast<off_t>(offset), SEEK_SET) == 0, "seek failed");
#endif
}

std::uint64_t tell(FILE * file) {
#if defined(_WIN32)
    const auto result = _ftelli64(file);
#else
    const auto result = ftello(file);
#endif
    require(result >= 0, "tell failed");
    return static_cast<std::uint64_t>(result);
}

std::uint64_t file_size(FILE * file) {
#if defined(_WIN32)
    require(_fseeki64(file, 0, SEEK_END) == 0, "end seek failed");
#else
    require(fseeko(file, 0, SEEK_END) == 0, "end seek failed");
#endif
    const auto result = tell(file);
    seek(file, 0);
    return result;
}

template<typename T> T scalar(FILE * file) {
    static_assert(std::is_unsigned_v<T>);
    std::array<std::uint8_t, sizeof(T)> bytes{};
    read_exact(file, bytes.data(), bytes.size());
    T value = 0;
    for (std::size_t i = 0; i < bytes.size(); ++i) value |= static_cast<T>(bytes[i]) << (i*8);
    return value;
}

void skip(FILE * file, std::uint64_t count, std::uint64_t file_bytes) {
    const auto end = add(tell(file), count);
    require(end <= file_bytes && end <= kMetadataLimit, "metadata length exceeds bound");
    seek(file, end);
}

void skip_string(FILE * file, std::uint64_t maximum, std::uint64_t file_bytes) {
    const auto n = scalar<std::uint64_t>(file);
    require(n <= maximum, "oversized metadata string");
    require(add(tell(file),n)<=file_bytes && add(tell(file),n)<=kMetadataLimit,"string extends beyond metadata");
    std::array<std::uint8_t,4096> block{};
    for (std::uint64_t remaining=n; remaining;) {
        const auto count=static_cast<std::size_t>(std::min<std::uint64_t>(remaining,block.size()));
        read_exact(file,block.data(),count);
        require(std::all_of(block.begin(),block.begin()+count,[](std::uint8_t c){return c!=0;}),"NUL in metadata string");
        remaining-=count;
    }
}

// Bounds are checked before calling the general GGUF parser, which may allocate
// metadata based on lengths. This scanner never allocates a declared string.
std::uint64_t preflight(FILE * file, std::uint64_t file_bytes) {
    std::array<char,4> magic{}; read_exact(file, magic.data(), magic.size());
    require(std::memcmp(magic.data(), "GGUF", 4) == 0 && scalar<std::uint32_t>(file) == 3, "magic/version");
    require(scalar<std::uint64_t>(file) == kTensorCount, "expected all 938 source tensors");
    const auto keys = scalar<std::uint64_t>(file);
    require(keys == 64 || keys == 72, "metadata key count for custom formats v1/v2");
    std::set<std::string> seen_keys;
    for (std::uint64_t i = 0; i < keys; ++i) {
        const auto key_size = scalar<std::uint64_t>(file);
        require(key_size > 0 && key_size <= 512, "metadata key length");
        std::string name(key_size,'\0'); read_exact(file,name.data(),name.size());
        require(std::all_of(name.begin(),name.end(),[](unsigned char c){return c>=32 && c<127;})
                && seen_keys.insert(name).second,"invalid or duplicate metadata key");
        switch (scalar<std::uint32_t>(file)) {
            case GGUF_TYPE_UINT32: skip(file, 4, file_bytes); break;
            case GGUF_TYPE_BOOL: require(scalar<std::uint8_t>(file) <= 1, "invalid metadata bool"); break;
            case GGUF_TYPE_STRING: skip_string(file, 32*1024*1024, file_bytes); break;
            case GGUF_TYPE_ARRAY: {
                require(scalar<std::uint32_t>(file) == GGUF_TYPE_UINT8, "unsupported metadata array type");
                const auto size = scalar<std::uint64_t>(file);
                require(size <= 4*1024*1024, "metadata byte array exceeds bound");
                skip(file, size, file_bytes); break;
            }
            default: throw std::runtime_error("Audio8 GGUF: unsupported emitted metadata type");
        }
    }
    std::uint64_t cursor = 0;
    for (std::size_t i = 0; i < kTensorCount; ++i) {
        const auto name_size = scalar<std::uint64_t>(file);
        require(name_size > 0 && name_size < GGML_MAX_NAME, "tensor name length");
        std::string name(name_size,'\0'); read_exact(file,name.data(),name.size());
        require(std::all_of(name.begin(),name.end(),[](unsigned char c){return c>=32 && c<127;}),"invalid tensor name bytes");
        const auto rank = scalar<std::uint32_t>(file);
        require(rank >= 1 && rank <= 4, "tensor rank");
        std::uint64_t elements = 1, first = 0;
        for (std::uint32_t dim = 0; dim < rank; ++dim) {
            const auto n = scalar<std::uint64_t>(file);
            require(n > 0 && n <= 1024*1024 && elements <= (1ULL<<32)/n, "tensor dimension/product bound");
            if (dim == 0) first = n;
            elements *= n;
        }
        const auto type = scalar<std::uint32_t>(file);
        require(type == GGML_TYPE_Q4_0 || type == GGML_TYPE_BF16, "source tensor dtype");
        require(type != GGML_TYPE_Q4_0 || (rank == 2 && first % 64 == 0), "source Q4 group64 row shape");
        const auto bytes = type == GGML_TYPE_Q4_0 ? elements/32*18 : elements*2;
        require(scalar<std::uint64_t>(file) == cursor, "noncanonical or overlapping tensor offsets");
        cursor = aligned(add(cursor, bytes));
        require(cursor <= file_bytes && tell(file) <= kMetadataLimit, "tensor directory bound");
    }
    const auto unpadded=tell(file);
    const auto data_start = aligned(unpadded);
    require(data_start <= kMetadataLimit && add(data_start,cursor) == file_bytes, "file coverage/trailing bytes");
    std::array<std::uint8_t,32> pad{};
    const auto count=static_cast<std::size_t>(data_start-unpadded);
    read_exact(file,pad.data(),count);
    require(std::all_of(pad.begin(),pad.begin()+count,[](std::uint8_t c){return c==0;}),"nonzero metadata padding");
    return data_start;
}

bool valid_hash(std::string_view text) {
    return text.size() == 64 && std::all_of(text.begin(), text.end(), [](char x) {
        return (x >= '0' && x <= '9') || (x >= 'a' && x <= 'f');
    });
}

std::string hex(const std::array<std::uint8_t,32> & bytes) {
    constexpr char digits[] = "0123456789abcdef";
    std::string result; result.reserve(64);
    for (const auto byte : bytes) { result.push_back(digits[byte>>4]); result.push_back(digits[byte&15]); }
    return result;
}

std::unique_ptr<Sha256> hasher(const Sha256Factory & factory) {
    require(static_cast<bool>(factory), "missing SHA256 factory");
    auto result = factory(); require(result != nullptr, "null SHA256 instance"); return result;
}

std::string digest(const Sha256Factory & factory, std::span<const std::uint8_t> bytes) {
    auto h = hasher(factory); h->update(bytes); return hex(h->finish());
}

std::span<const std::uint8_t> as_bytes(std::string_view text) {
    return {reinterpret_cast<const std::uint8_t *>(text.data()), text.size()};
}

int64_t key(const gguf_context * meta, std::string_view name, gguf_type type) {
    const auto index = gguf_find_key(meta, std::string(name).c_str());
    require(index >= 0 && gguf_get_kv_type(meta,index) == type, "missing/wrong metadata type: " + std::string(name));
    return index;
}

std::string_view text(const gguf_context * meta, std::string_view name) {
    const char * value = gguf_get_val_str(meta,key(meta,name,GGUF_TYPE_STRING));
    require(value != nullptr, "null metadata string");
    return value;
}

std::span<const std::uint8_t> array(const gguf_context * meta, std::string_view name) {
    const auto index = key(meta,name,GGUF_TYPE_ARRAY);
    require(gguf_get_arr_type(meta,index) == GGUF_TYPE_UINT8, "auxiliary array dtype");
    return {static_cast<const std::uint8_t *>(gguf_get_arr_data(meta,index)), gguf_get_arr_n(meta,index)};
}

struct Shape {
    ggml_type type;
    std::array<int64_t,4> ne = {1,1,1,1};
    int rank;
};

std::map<std::string,Shape,std::less<>> schema() {
    std::map<std::string,Shape,std::less<>> shapes;
    const auto put = [&](std::string name, ggml_type type, std::initializer_list<int64_t> dims) {
        Shape s{type,{1,1,1,1},static_cast<int>(dims.size())};
        std::copy(dims.begin(),dims.end(),s.ne.begin());
        require(shapes.emplace(std::move(name),s).second,"duplicate internal schema name");
    };
    const auto b = GGML_TYPE_BF16, q = GGML_TYPE_Q4_0;
    put("audio_tower.embedder.conv1.weight",b,{3,128,1280});
    put("audio_tower.embedder.conv1.bias",b,{1280});
    put("audio_tower.embedder.conv2.weight",b,{3,1280,1280});
    put("audio_tower.embedder.conv2.bias",b,{1280});
    put("audio_tower.norm.weight",b,{1280});
    for (int i=0; i<32; ++i) {
        const auto p="audio_tower.layers."+std::to_string(i)+".";
        put(p+"final_layer_norm.weight",b,{1280});
        put(p+"self_attn_layer_norm.weight",b,{1280});
        put(p+"mlp.down_proj.weight",q,{5120,1280});
        put(p+"mlp.down_proj.bias",b,{1280});
        put(p+"mlp.gate_proj.weight",q,{1280,5120});
        put(p+"mlp.up_proj.weight",q,{1280,5120});
        for (const auto * projection : {"q_proj","k_proj","v_proj"})
            put(p+"self_attn."+projection+".weight",q,{1280,2048});
        put(p+"self_attn.q_proj.bias",b,{2048});
        put(p+"self_attn.v_proj.bias",b,{2048});
        put(p+"self_attn.o_proj.weight",q,{2048,1280});
        put(p+"self_attn.o_proj.bias",b,{1280});
    }
    put("frame_len_embedding.weight",b,{2048,3});
    put(kEmbedding,q,{2048,151936});
    put("language_model.model.norm.weight",b,{2048});
    put("multi_modal_projector.linear_1.weight",q,{10240,2048});
    put("multi_modal_projector.linear_2.weight",q,{2048,2048});
    for (int i=0; i<36; ++i) {
        const auto p="language_model.model.layers."+std::to_string(i)+".";
        put(p+"ada_rms_norm.linear1.weight",b,{2048,32});
        put(p+"ada_rms_norm.linear2.weight",b,{32,2048});
        put(p+"input_layernorm.weight",b,{2048});
        put(p+"post_attention_layernorm.weight",b,{2048});
        put(p+"mlp.down_proj.weight",q,{11008,2048});
        put(p+"mlp.gate_proj.weight",q,{2048,11008});
        put(p+"mlp.up_proj.weight",q,{2048,11008});
        put(p+"self_attn.q_proj.weight",q,{2048,2048});
        put(p+"self_attn.q_proj.bias",b,{2048});
        put(p+"self_attn.o_proj.weight",q,{2048,2048});
        for (const auto * projection : {"k_proj","v_proj"}) {
            put(p+"self_attn."+projection+".weight",q,{2048,256});
            put(p+"self_attn."+projection+".bias",b,{256});
        }
    }
    for (int i=0; i<4; ++i) {
        const auto p="semantic_vad_heads."+std::to_string(i)+".";
        put(p+"weight",b,{2048,8}); put(p+"bias",b,{8});
    }
    require(shapes.size()==kTensorCount,"internal schema count");
    return shapes;
}

void verify_metadata(const gguf_context * meta, const Sha256Factory & hash) {
    require(gguf_get_version(meta)==3 && gguf_get_alignment(meta)==32,"GGUF format/alignment");
    require(text(meta,"general.architecture")=="audio8", "architecture must be audio8");
    require(text(meta,"general.license")=="apache-2.0", "model license metadata");
    const auto format=gguf_get_val_u32(meta,key(meta,"audio8.format_version",GGUF_TYPE_UINT32));
    require(format==1 || format==2,"supported Audio8 format version");
    require(gguf_get_n_kv(meta)==(format==1 ? 64 : 72),"format-specific metadata key count");
    for (const auto &[name,value] : std::initializer_list<std::pair<const char *,std::uint32_t>>{
            {"general.quantization_version",2},
            {"audio8.mapping.source_group_size",64},{"audio8.mapping.destination_group_size",32}})
        require(gguf_get_val_u32(meta,key(meta,name,GGUF_TYPE_UINT32))==value,"format/mapping integer metadata");
    require(!gguf_get_val_bool(meta,key(meta,"audio8.backend_verified",GGUF_TYPE_BOOL)) &&
            !gguf_get_val_bool(meta,key(meta,"audio8.mapping.refit",GGUF_TYPE_BOOL)),"unexpected artifact claim/refit");
    require(text(meta,"audio8.mapping")== (format==1 ? "q4g64-zp7-to-q4_0g32-zp8-v1" : "q3fullg64-zp4-and-q4g64-zp7-to-q4_0g32-zp8-v2"), "mapping version");
    if (format==2) {
        require(text(meta,"audio8.source.bundle_manifest_sha256")=="f132d8c4dd0af2eaf49cef779635e83678e3169f002015e9a0a35c9f6f685656", "Q3 native manifest identity");
        require(text(meta,"audio8.source.weights_sha256")=="e38b5a14c8636e1d9a586cd725538d17164554c8ef52f60f7fd293791acaa6c4", "Q3 native weights identity");
        require(text(meta,"audio8.aux.provenance/q3-overlay-manifest.json.sha256")=="0e77d69bf604f6d4815a3397206efa9b34f13cc0edb4999efd8e0cabae4c320e", "Q3 fitted overlay identity");
        require(text(meta,"audio8.aux.provenance/q3-native-manifest.json.sha256")==text(meta,"audio8.source.bundle_manifest_sha256"), "Q3 provenance manifest identity");
    }
    require(text(meta,"audio8.source.model_id")=="Edge0/Audio8-ASR-Infinite" &&
            text(meta,"audio8.source.revision")==kRevision &&
            text(meta,"audio8.ggml_source_revision")==kGgmlRevision,"upstream identity");
    require(text(meta,"audio8.tensor.aliases")==kAliasesJson,"single tied-head alias required");
    const auto manifest_hash=text(meta,"audio8.source.bundle_manifest_sha256");
    require(valid_hash(manifest_hash) && digest(hash,as_bytes(text(meta,"audio8.source.bundle_manifest")))==manifest_hash,
            "embedded source manifest hash");
    require(valid_hash(text(meta,"audio8.source.weights_sha256")),"source weight identity");
    const std::pair<const char *,const char *> assets[] = {
        {"chat_template.jinja","cd8e9439f0570856fd70470bf8889ebd8b5d1107207f67a5efb46e342330527f"},
        {"config.json","744baed356a8c86a9791fc76cecb73219ed4aff795147c6cdc082337a13f019c"},
        {"generation_config.json","d3f96050ed55e02136d3cea2d1c03c0163d8e93631cdfb87bf0e82945ed2db64"},
        {"model.safetensors.index.json","18307b2e6abe5e41a867b27158c65c4097e6ee1b16a87369006dc2d4df1546fe"},
        {"preprocessor_config.json","ee0dac67c684faa527db5d26a75109e168723fdae38b49e0c1bb471a953f6c45"},
        {"tokenizer.json","72f8b7e878ee6606c309881f68201e277f9dae50175a387074a5181d771b9599"},
        {"tokenizer_config.json","ceca11f549d6a8550430c7272e9aa40582bffd633f8998335883a428f4d77fb9"},
    };
    for (const auto &[name,expected] : assets) {
        const std::string k=std::string("audio8.asset.")+name;
        require(text(meta,k+".sha256")==expected && digest(hash,as_bytes(text(meta,k)))==expected,
                "source config/tokenizer asset hash: "+k);
    }
    const std::pair<const char *,const char *> auxiliary[] = {
        {"frontend.window_f32le","c692d19f0b2bd745f0dfc967261c3e3d21a2473e485a677d219e9b4a73e0218e"},
        {"frontend.filters_f32le","c342ed6d06c90b42990bf9e4a3a07ca54f3f1422f7bc3c66b3177a774395df5a"},
        {"tokenizer.a8tok","7fdc1e903297ea1fa438b77503db993c50a67f8b325b4e55754f08b8579c5410"},
        {"source_model_card_utf8","791e07ad5efa25a1ae4dde9f795d3d041bd06487eef9b39b07ddef1ad0a6e5e2"},
    };
    for (const auto &[name,expected] : auxiliary) {
        const std::string k=std::string("audio8.aux.")+name;
        require(text(meta,k+".sha256")==expected && digest(hash,array(meta,k))==expected,
                "frontend/tokenizer/model-card hash: "+k);
    }
    require(array(meta,"audio8.aux.frontend.window_f32le").size()==400*4 &&
            array(meta,"audio8.aux.frontend.filters_f32le").size()==128*201*4,"frontend dimensions");
    const auto tok=array(meta,"audio8.aux.tokenizer.a8tok");
    require(tok.size()==1734837 && std::memcmp(tok.data(),"A8TOK001",8)==0,"native tokenizer format");
    // Preserve and check every embedded provenance sidecar; never execute it.
    std::size_t sidecars=0;
    for (int64_t i=0; i<gguf_get_n_kv(meta); ++i) {
        const std::string_view name=gguf_get_key(meta,i);
        if (name.starts_with("audio8.aux.provenance/") && !name.ends_with(".sha256")) {
            const auto expected=text(meta,std::string(name)+".sha256");
            require(valid_hash(expected) && digest(hash,array(meta,name))==expected,"provenance sidecar hash");
            ++sidecars;
        }
    }
    require(sidecars==(format==1 ? 11U : 14U),"incomplete quantization provenance sidecars");
}

void validate_q4(std::span<const std::uint8_t> bytes) {
    require(bytes.size()%36==0,"Q4 group64 staging alignment");
    for (std::size_t i=0; i<bytes.size(); i+=36) {
        const std::uint16_t scale=bytes[i] | (std::uint16_t(bytes[i+1])<<8);
        // Keep the success path allocation-free while inspecting every packed
        // byte. Constructing std::string errors per code is expensive at 4B weights.
        if ((scale&0x7c00)==0x7c00 || ((scale&0x8000)!=0 && scale!=0x8000))
            require(false,"invalid Q4 scale");
        if (bytes[i]!=bytes[i+18] || bytes[i+1]!=bytes[i+19])
            require(false,"nonduplicated group64 scale");
        for (const auto start : {i+2,i+20})
            for (std::size_t j=0; j<16; ++j)
                if ((bytes[start+j]&15)==0 || (bytes[start+j]>>4)==0)
                    require(false,"unused -8 code in exact Q4 mapping");
    }
}

void validate_bf16(std::span<const std::uint8_t> bytes) {
    require(bytes.size()%2==0,"BF16 staging alignment");
    for (std::size_t i=0; i<bytes.size(); i+=2) {
        const std::uint16_t bits=bytes[i] | (std::uint16_t(bytes[i+1])<<8);
        if ((bits&0x7f80)==0x7f80) require(false,"nonfinite original BF16 weight");
    }
}

} // namespace

struct Model::Impl {
    gguf_context * metadata=nullptr;
    ggml_context * context=nullptr;
    ggml_backend_buffer_t buffer=nullptr;
    ggml_backend_t backend=nullptr; // Borrowed, never freed here.
    std::map<std::string,ggml_tensor *,std::less<>> tensors;
    Audio8Config config;
    OriginalWeightPolicy original_weights=OriginalWeightPolicy::PreserveBf16;
    std::uint64_t source_bytes=0, resident_bytes=0, allocated_bytes=0;
    std::string file_hash, manifest_hash;
    ~Impl() {
        if (buffer) ggml_backend_buffer_free(buffer);
        if (context) ggml_free(context);
        if (metadata) gguf_free(metadata);
    }
};

Model::Model(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {}
Model::~Model() = default;

std::unique_ptr<Model> Model::load(const std::string & path, ggml_backend_t backend,
                                  const LoadOptions & options) {
    static_assert(sizeof(std::size_t)>=8,"Audio8 requires a 64-bit host");
    require(std::endian::native==std::endian::little,"big-endian upload is not implemented");
    require(backend!=nullptr && static_cast<bool>(options.sha256),"backend and SHA256 factory required");
    require(valid_hash(options.expected_gguf_sha256),"external full-file SHA256 pin required");
    require(options.staging_bytes>=4096 && options.staging_bytes<=4*1024*1024,"staging bound");
    require(options.maximum_file_bytes>=kSourcePayloadBytes && options.maximum_file_bytes<=8ULL*1024*1024*1024,
            "maximum file size bound");
    require(options.original_weights==OriginalWeightPolicy::PreserveBf16 ||
            options.original_weights==OriginalWeightPolicy::PromoteBf16ToF32,"unsupported original weight conversion");
    std::unique_ptr<FILE,FileCloser> file(std::fopen(path.c_str(),"rb"));
    require(file!=nullptr,"cannot open input");
    const auto bytes=file_size(file.get());
    require(bytes>=24 && bytes<=options.maximum_file_bytes,"input file size bound");
    const auto data_start=preflight(file.get(),bytes);
    seek(file.get(),0);
    auto result=std::make_unique<Impl>();
    result->backend=backend; result->original_weights=options.original_weights;
    result->metadata=gguf_init_from_file_ptr(file.get(),gguf_init_params{true,nullptr});
    require(result->metadata!=nullptr,"GGUF parse failed");
    require(gguf_get_data_offset(result->metadata)==data_start && gguf_get_n_tensors(result->metadata)==kTensorCount,
            "GGUF parser disagrees with bounded preflight");
    verify_metadata(result->metadata,options.sha256);
    result->manifest_hash=text(result->metadata,"audio8.source.bundle_manifest_sha256");
    const auto expected=schema();
    const auto context_bytes=ggml_tensor_overhead()*(kTensorCount+8);
    result->context=ggml_init(ggml_init_params{context_bytes,nullptr,true});
    require(result->context!=nullptr,"weight tensor metadata allocation failed");
    std::size_t index=0;
    for (const auto &[name,shape] : expected) {
        require(name==gguf_get_tensor_name(result->metadata,index),"missing/extra/reordered tensor: "+name);
        require(gguf_get_tensor_type(result->metadata,index)==shape.type,"source tensor dtype: "+name);
        const auto * dims=gguf_get_tensor_ne(result->metadata,index);
        require(std::equal(shape.ne.begin(),shape.ne.end(),dims),"source tensor shape: "+name);
        const auto target_type=shape.type==GGML_TYPE_BF16 && options.original_weights==OriginalWeightPolicy::PromoteBf16ToF32
            ? GGML_TYPE_F32 : shape.type;
        auto * tensor=ggml_new_tensor(result->context,target_type,shape.rank,shape.ne.data());
        require(tensor!=nullptr,"weight tensor creation failed");
        ggml_set_name(tensor,name.c_str());
        result->tensors.emplace(name,tensor);
        result->source_bytes+=gguf_get_tensor_size(result->metadata,index);
        result->resident_bytes+=ggml_nbytes(tensor);
        ++index;
    }
    require(result->source_bytes==kSourcePayloadBytes,"source weight payload count");
    require(result->resident_bytes==kSourcePayloadBytes+(options.original_weights==OriginalWeightPolicy::PromoteBf16ToF32?kOriginalBytes:0),
            "resident weight payload count");
    result->buffer=ggml_backend_alloc_ctx_tensors(result->context,backend);
    require(result->buffer!=nullptr,"backend weight allocation failed");
    if (options.require_device_buffer)
        require(!ggml_backend_buft_is_host(ggml_backend_buffer_get_type(result->buffer)),"requested device weights landed in host buffer");
    ggml_backend_buffer_set_usage(result->buffer,GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
    result->allocated_bytes=ggml_backend_buffer_get_size(result->buffer);

    // Hash precisely the bytes uploaded, including header, metadata and padding.
    // Never retain/mmap the entire packed file next to backend-resident weights.
    auto whole=hasher(options.sha256);
    std::vector<std::uint8_t> staging(options.staging_bytes);
    std::vector<std::uint32_t> widened;
    if (options.original_weights==OriginalWeightPolicy::PromoteBf16ToF32) widened.resize(options.staging_bytes/2);
    seek(file.get(),0);
    for (std::uint64_t remaining=data_start; remaining;) {
        const auto count=static_cast<std::size_t>(std::min<std::uint64_t>(remaining,staging.size()));
        read_exact(file.get(),staging.data(),count); whole->update({staging.data(),count}); remaining-=count;
    }
    index=0;
    for (const auto &[name,shape] : expected) {
        auto * tensor=result->tensors.at(name);
        const auto source_size=gguf_get_tensor_size(result->metadata,index);
        require(tell(file.get())==data_start+gguf_get_tensor_offset(result->metadata,index),"source stream position");
        const std::size_t unit=shape.type==GGML_TYPE_Q4_0?36:2;
        const std::size_t chunk=staging.size()/unit*unit;
        std::size_t uploaded=0;
        for (std::size_t remaining=source_size; remaining;) {
            const auto count=std::min(remaining,chunk);
            read_exact(file.get(),staging.data(),count);
            const std::span<const std::uint8_t> block(staging.data(),count);
            whole->update(block);
            if (shape.type==GGML_TYPE_Q4_0) validate_q4(block); else validate_bf16(block);
            if (tensor->type==GGML_TYPE_F32 && shape.type==GGML_TYPE_BF16) {
                for (std::size_t i=0; i<count/2; ++i)
                    widened[i]=(std::uint32_t(staging[2*i]) | (std::uint32_t(staging[2*i+1])<<8))<<16;
                ggml_backend_tensor_set(tensor,widened.data(),uploaded,count*2);
                uploaded+=count*2;
            } else {
                ggml_backend_tensor_set(tensor,block.data(),uploaded,count);
                uploaded+=count;
            }
            remaining-=count;
        }
        require(uploaded==ggml_nbytes(tensor),"backend tensor upload coverage");
        const auto padding=static_cast<std::size_t>(aligned(tell(file.get()))-tell(file.get()));
        read_exact(file.get(),staging.data(),padding);
        require(std::all_of(staging.begin(),staging.begin()+padding,[](std::uint8_t x){return x==0;}),"nonzero tensor padding");
        whole->update({staging.data(),padding});
        ++index;
    }
    require(tell(file.get())==bytes && std::fgetc(file.get())==EOF && !std::ferror(file.get()),"trailing data/read error");
    result->file_hash=hex(whole->finish());
    require(result->file_hash==options.expected_gguf_sha256,"full-file SHA256 mismatch; model discarded");
    ggml_backend_synchronize(backend);
    // staging, widened and FILE are destroyed before the model is returned.
    return std::unique_ptr<Model>(new Model(std::move(result)));
}

ggml_tensor * Model::tensor(std::string_view source_name) const {
    if (source_name==kAlias) source_name=kEmbedding;
    const auto found=impl_->tensors.find(source_name);
    require(found!=impl_->tensors.end(),"unknown tensor: "+std::string(source_name));
    return found->second;
}
const Audio8Config & Model::config() const noexcept { return impl_->config; }
ggml_backend_t Model::backend() const noexcept { return impl_->backend; }
OriginalWeightPolicy Model::original_weight_policy() const noexcept { return impl_->original_weights; }
std::uint64_t Model::source_weight_payload_bytes() const noexcept { return impl_->source_bytes; }
std::uint64_t Model::resident_weight_payload_bytes() const noexcept { return impl_->resident_bytes; }
std::uint64_t Model::allocated_weight_buffer_bytes() const noexcept { return impl_->allocated_bytes; }
std::size_t Model::tensor_count() const noexcept { return impl_->tensors.size(); }
const std::string & Model::file_sha256() const noexcept { return impl_->file_hash; }
const std::string & Model::source_manifest_sha256() const noexcept { return impl_->manifest_hash; }
std::string_view Model::metadata_text(std::string_view name) const { return text(impl_->metadata,name); }
std::span<const std::uint8_t> Model::auxiliary(std::string_view name) const {
    return array(impl_->metadata,std::string("audio8.aux.")+std::string(name));
}
std::span<const std::uint8_t> Model::frontend_window_f32le() const { return auxiliary("frontend.window_f32le"); }
std::span<const std::uint8_t> Model::frontend_filters_f32le() const { return auxiliary("frontend.filters_f32le"); }
std::span<const std::uint8_t> Model::tokenizer_bytes() const { return auxiliary("tokenizer.a8tok"); }

} // namespace audio8