File size: 6,310 Bytes
f4f166e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | //! `.aeg` single-artifact bundle (plan E6.1): one file containing the whole
//! resident skull - base shards, LoRA adapter, needle engine asset, tokenizer,
//! config - so distribution matches local-LLM model files.
//!
//! Format (v1): magic b"AEG1" | u32 entry_count | per entry:
//! u16 name_len | name | u64 data_len | data
//! First entry SHOULD be "_manifest.json" describing components.
use std::io::{Read, Write};
use std::path::Path;
use crate::error::{Error, Result};
const MAGIC: &[u8; 4] = b"AEG1";
pub fn pack(entries: Vec<(String, Vec<u8>)>, out_path: &str) -> Result<()> {
let mut f = std::fs::File::create(out_path).map_err(|e| Error::Model(format!("io: {e}")))?;
f.write_all(MAGIC).map_err(|e| Error::Model(format!("io: {e}")))?;
f.write_all(&(entries.len() as u32).to_le_bytes())
.map_err(|e| Error::Model(format!("io: {e}")))?;
for (name, data) in &entries {
let nb = name.as_bytes();
if nb.len() > u16::MAX as usize {
return Err(Error::Model("entry name too long".into()));
}
f.write_all(&(nb.len() as u16).to_le_bytes())
.map_err(|e| Error::Model(format!("io: {e}")))?;
f.write_all(nb).map_err(|e| Error::Model(format!("io: {e}")))?;
f.write_all(&(data.len() as u64).to_le_bytes())
.map_err(|e| Error::Model(format!("io: {e}")))?;
f.write_all(data).map_err(|e| Error::Model(format!("io: {e}")))?;
}
Ok(())
}
/// Pack a directory recursively (relative paths become entry names).
pub fn pack_dir(dir: &str, out_path: &str, manifest_json: Option<&str>) -> Result<usize> {
let mut entries: Vec<(String, Vec<u8>)> = Vec::new();
if let Some(m) = manifest_json {
entries.push(("_manifest.json".into(), m.as_bytes().to_vec()));
}
collect_dir(Path::new(dir), "", &mut entries)?;
let n = entries.len();
pack(entries, out_path)?;
Ok(n)
}
fn collect_dir(
root: &Path,
rel: &str,
out: &mut Vec<(String, Vec<u8>)>,
) -> Result<()> {
for e in std::fs::read_dir(root).map_err(|e| Error::Model(format!("io: {e}")))? {
let p = e.map_err(|e| Error::Model(format!("io: {e}")))?.path();
let fname = p
.file_name()
.map(|s| s.to_string_lossy().to_string())
.unwrap_or_default();
let rname = if rel.is_empty() { fname.clone() } else { format!("{rel}/{fname}") };
if p.is_dir() {
collect_dir(&p, &rname, out)?;
} else {
let data = std::fs::read(&p).map_err(|e| Error::Model(format!("io: {e}")))?;
out.push((rname, data));
}
}
Ok(())
}
pub fn list(path: &str) -> Result<Vec<String>> {
let (_names, names_only) = read_all(path)?;
Ok(names_only)
}
/// Returns (entries_with_data, names_only).
pub fn read_all(path: &str) -> Result<(Vec<(String, Vec<u8>)>, Vec<String>)> {
let mut f = std::fs::File::open(path).map_err(|e| Error::Model(format!("io: {e}")))?;
let mut magic = [0u8; 4];
f.read_exact(&mut magic).map_err(|e| Error::Model(format!("io: {e}")))?;
if &magic != MAGIC {
return Err(Error::Model("not an .aeg bundle".into()));
}
let mut cnt_b = [0u8; 4];
f.read_exact(&mut cnt_b).map_err(|e| Error::Model(format!("io: {e}")))?;
let count = u32::from_le_bytes(cnt_b) as usize;
let mut entries = Vec::with_capacity(count);
let mut names = Vec::with_capacity(count);
for _ in 0..count {
let mut nb = [0u8; 2];
f.read_exact(&mut nb).map_err(|e| Error::Model(format!("io: {e}")))?;
let nlen = u16::from_le_bytes(nb) as usize;
let mut nm = vec![0u8; nlen];
f.read_exact(&mut nm).map_err(|e| Error::Model(format!("io: {e}")))?;
let mut db = [0u8; 8];
f.read_exact(&mut db).map_err(|e| Error::Model(format!("io: {e}")))?;
let dlen = u64::from_le_bytes(db) as usize;
let mut data = vec![0u8; dlen];
f.read_exact(&mut data).map_err(|e| Error::Model(format!("io: {e}")))?;
let name = String::from_utf8(nm).map_err(|_| Error::Model("bad name".into()))?;
names.push(name.clone());
entries.push((name, data));
}
Ok((entries, names))
}
pub fn unpack(path: &str, out_dir: &str) -> Result<usize> {
let (entries, _) = read_all(path)?;
for (name, data) in &entries {
if name == "_manifest.json" {
continue; // metadata pseudo-entry, not a materialized file
}
let dest = Path::new(out_dir).join(name.replace('/', std::path::MAIN_SEPARATOR_STR));
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent).map_err(|e| Error::Model(format!("io: {e}")))?;
}
std::fs::write(&dest, data).map_err(|e| Error::Model(format!("io: {e}")))?;
}
Ok(entries.len())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn roundtrip_binary_and_manifest() {
let dir = std::env::temp_dir().join("ae_bundle_test");
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(dir.join("sub")).unwrap();
std::fs::write(dir.join("a.bin"), vec![1u8, 2, 3, 250]).unwrap();
std::fs::write(dir.join("sub").join("cfg.json"), b"{\"vocab\":8192}").unwrap();
let bundle = dir.join("skull.aeg");
let manifest = r#"{"version":1,"components":["base","adapter","needle","tokenizer"]}"#;
let n = pack_dir(dir.to_str().unwrap(), bundle.to_str().unwrap(), Some(manifest)).unwrap();
assert_eq!(n, 3); // manifest + a.bin + sub/cfg.json
let names = list(bundle.to_str().unwrap()).unwrap();
assert!(names.contains(&"_manifest.json".to_string()));
assert!(names.contains(&"sub/cfg.json".to_string()));
let outd = dir.join("out");
unpack(bundle.to_str().unwrap(), outd.to_str().unwrap()).unwrap();
assert_eq!(
std::fs::read(outd.join("a.bin")).unwrap(),
vec![1u8, 2, 3, 250]
);
assert!(outd.join("sub").join("cfg.json").exists());
assert!(!outd.join("_manifest.json").exists(), "manifest not materialized");
// corrupt magic must fail loudly
let bad = dir.join("bad.aeg");
std::fs::write(&bad, b"NOPE").unwrap();
assert!(list(bad.to_str().unwrap()).is_err());
}
}
|