File size: 6,310 Bytes
f4f166e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
//! `.aeg` single-artifact bundle (plan E6.1): one file containing the whole
//! resident skull - base shards, LoRA adapter, needle engine asset, tokenizer,
//! config - so distribution matches local-LLM model files.
//!
//! Format (v1): magic b"AEG1" | u32 entry_count | per entry:
//!   u16 name_len | name | u64 data_len | data
//! First entry SHOULD be "_manifest.json" describing components.

use std::io::{Read, Write};
use std::path::Path;

use crate::error::{Error, Result};

const MAGIC: &[u8; 4] = b"AEG1";

pub fn pack(entries: Vec<(String, Vec<u8>)>, out_path: &str) -> Result<()> {
    let mut f = std::fs::File::create(out_path).map_err(|e| Error::Model(format!("io: {e}")))?;
    f.write_all(MAGIC).map_err(|e| Error::Model(format!("io: {e}")))?;
    f.write_all(&(entries.len() as u32).to_le_bytes())
        .map_err(|e| Error::Model(format!("io: {e}")))?;
    for (name, data) in &entries {
        let nb = name.as_bytes();
        if nb.len() > u16::MAX as usize {
            return Err(Error::Model("entry name too long".into()));
        }
        f.write_all(&(nb.len() as u16).to_le_bytes())
            .map_err(|e| Error::Model(format!("io: {e}")))?;
        f.write_all(nb).map_err(|e| Error::Model(format!("io: {e}")))?;
        f.write_all(&(data.len() as u64).to_le_bytes())
            .map_err(|e| Error::Model(format!("io: {e}")))?;
        f.write_all(data).map_err(|e| Error::Model(format!("io: {e}")))?;
    }
    Ok(())
}

/// Pack a directory recursively (relative paths become entry names).
pub fn pack_dir(dir: &str, out_path: &str, manifest_json: Option<&str>) -> Result<usize> {
    let mut entries: Vec<(String, Vec<u8>)> = Vec::new();
    if let Some(m) = manifest_json {
        entries.push(("_manifest.json".into(), m.as_bytes().to_vec()));
    }
    collect_dir(Path::new(dir), "", &mut entries)?;
    let n = entries.len();
    pack(entries, out_path)?;
    Ok(n)
}

fn collect_dir(
    root: &Path,
    rel: &str,
    out: &mut Vec<(String, Vec<u8>)>,
) -> Result<()> {
    for e in std::fs::read_dir(root).map_err(|e| Error::Model(format!("io: {e}")))? {
        let p = e.map_err(|e| Error::Model(format!("io: {e}")))?.path();
        let fname = p
            .file_name()
            .map(|s| s.to_string_lossy().to_string())
            .unwrap_or_default();
        let rname = if rel.is_empty() { fname.clone() } else { format!("{rel}/{fname}") };
        if p.is_dir() {
            collect_dir(&p, &rname, out)?;
        } else {
            let data = std::fs::read(&p).map_err(|e| Error::Model(format!("io: {e}")))?;
            out.push((rname, data));
        }
    }
    Ok(())
}

pub fn list(path: &str) -> Result<Vec<String>> {
    let (_names, names_only) = read_all(path)?;
    Ok(names_only)
}

/// Returns (entries_with_data, names_only).
pub fn read_all(path: &str) -> Result<(Vec<(String, Vec<u8>)>, Vec<String>)> {
    let mut f = std::fs::File::open(path).map_err(|e| Error::Model(format!("io: {e}")))?;
    let mut magic = [0u8; 4];
    f.read_exact(&mut magic).map_err(|e| Error::Model(format!("io: {e}")))?;
    if &magic != MAGIC {
        return Err(Error::Model("not an .aeg bundle".into()));
    }
    let mut cnt_b = [0u8; 4];
    f.read_exact(&mut cnt_b).map_err(|e| Error::Model(format!("io: {e}")))?;
    let count = u32::from_le_bytes(cnt_b) as usize;
    let mut entries = Vec::with_capacity(count);
    let mut names = Vec::with_capacity(count);
    for _ in 0..count {
        let mut nb = [0u8; 2];
        f.read_exact(&mut nb).map_err(|e| Error::Model(format!("io: {e}")))?;
        let nlen = u16::from_le_bytes(nb) as usize;
        let mut nm = vec![0u8; nlen];
        f.read_exact(&mut nm).map_err(|e| Error::Model(format!("io: {e}")))?;
        let mut db = [0u8; 8];
        f.read_exact(&mut db).map_err(|e| Error::Model(format!("io: {e}")))?;
        let dlen = u64::from_le_bytes(db) as usize;
        let mut data = vec![0u8; dlen];
        f.read_exact(&mut data).map_err(|e| Error::Model(format!("io: {e}")))?;
        let name = String::from_utf8(nm).map_err(|_| Error::Model("bad name".into()))?;
        names.push(name.clone());
        entries.push((name, data));
    }
    Ok((entries, names))
}

pub fn unpack(path: &str, out_dir: &str) -> Result<usize> {
    let (entries, _) = read_all(path)?;
    for (name, data) in &entries {
        if name == "_manifest.json" {
            continue; // metadata pseudo-entry, not a materialized file
        }
        let dest = Path::new(out_dir).join(name.replace('/', std::path::MAIN_SEPARATOR_STR));
        if let Some(parent) = dest.parent() {
            std::fs::create_dir_all(parent).map_err(|e| Error::Model(format!("io: {e}")))?;
        }
        std::fs::write(&dest, data).map_err(|e| Error::Model(format!("io: {e}")))?;
    }
    Ok(entries.len())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn roundtrip_binary_and_manifest() {
        let dir = std::env::temp_dir().join("ae_bundle_test");
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(dir.join("sub")).unwrap();
        std::fs::write(dir.join("a.bin"), vec![1u8, 2, 3, 250]).unwrap();
        std::fs::write(dir.join("sub").join("cfg.json"), b"{\"vocab\":8192}").unwrap();

        let bundle = dir.join("skull.aeg");
        let manifest = r#"{"version":1,"components":["base","adapter","needle","tokenizer"]}"#;
        let n = pack_dir(dir.to_str().unwrap(), bundle.to_str().unwrap(), Some(manifest)).unwrap();
        assert_eq!(n, 3); // manifest + a.bin + sub/cfg.json

        let names = list(bundle.to_str().unwrap()).unwrap();
        assert!(names.contains(&"_manifest.json".to_string()));
        assert!(names.contains(&"sub/cfg.json".to_string()));

        let outd = dir.join("out");
        unpack(bundle.to_str().unwrap(), outd.to_str().unwrap()).unwrap();
        assert_eq!(
            std::fs::read(outd.join("a.bin")).unwrap(),
            vec![1u8, 2, 3, 250]
        );
        assert!(outd.join("sub").join("cfg.json").exists());
        assert!(!outd.join("_manifest.json").exists(), "manifest not materialized");

        // corrupt magic must fail loudly
        let bad = dir.join("bad.aeg");
        std::fs::write(&bad, b"NOPE").unwrap();
        assert!(list(bad.to_str().unwrap()).is_err());
    }
}