alter_ego / serving /src /bundle.rs
geraldaton20's picture
initial upload: source code (no build artifacts)
f4f166e verified
Raw History Blame Contribute Delete
6.31 kB
//! `.aeg` single-artifact bundle (plan E6.1): one file containing the whole
//! resident skull - base shards, LoRA adapter, needle engine asset, tokenizer,
//! config - so distribution matches local-LLM model files.
//!
//! Format (v1): magic b"AEG1" | u32 entry_count | per entry:
//! u16 name_len | name | u64 data_len | data
//! First entry SHOULD be "_manifest.json" describing components.
use std::io::{Read, Write};
use std::path::Path;
use crate::error::{Error, Result};
const MAGIC: &[u8; 4] = b"AEG1";
pub fn pack(entries: Vec<(String, Vec<u8>)>, out_path: &str) -> Result<()> {
let mut f = std::fs::File::create(out_path).map_err(|e| Error::Model(format!("io: {e}")))?;
f.write_all(MAGIC).map_err(|e| Error::Model(format!("io: {e}")))?;
f.write_all(&(entries.len() as u32).to_le_bytes())
.map_err(|e| Error::Model(format!("io: {e}")))?;
for (name, data) in &entries {
let nb = name.as_bytes();
if nb.len() > u16::MAX as usize {
return Err(Error::Model("entry name too long".into()));
}
f.write_all(&(nb.len() as u16).to_le_bytes())
.map_err(|e| Error::Model(format!("io: {e}")))?;
f.write_all(nb).map_err(|e| Error::Model(format!("io: {e}")))?;
f.write_all(&(data.len() as u64).to_le_bytes())
.map_err(|e| Error::Model(format!("io: {e}")))?;
f.write_all(data).map_err(|e| Error::Model(format!("io: {e}")))?;
}
Ok(())
}
/// Pack a directory recursively (relative paths become entry names).
pub fn pack_dir(dir: &str, out_path: &str, manifest_json: Option<&str>) -> Result<usize> {
let mut entries: Vec<(String, Vec<u8>)> = Vec::new();
if let Some(m) = manifest_json {
entries.push(("_manifest.json".into(), m.as_bytes().to_vec()));
}
collect_dir(Path::new(dir), "", &mut entries)?;
let n = entries.len();
pack(entries, out_path)?;
Ok(n)
}
fn collect_dir(
root: &Path,
rel: &str,
out: &mut Vec<(String, Vec<u8>)>,
) -> Result<()> {
for e in std::fs::read_dir(root).map_err(|e| Error::Model(format!("io: {e}")))? {
let p = e.map_err(|e| Error::Model(format!("io: {e}")))?.path();
let fname = p
.file_name()
.map(|s| s.to_string_lossy().to_string())
.unwrap_or_default();
let rname = if rel.is_empty() { fname.clone() } else { format!("{rel}/{fname}") };
if p.is_dir() {
collect_dir(&p, &rname, out)?;
} else {
let data = std::fs::read(&p).map_err(|e| Error::Model(format!("io: {e}")))?;
out.push((rname, data));
}
}
Ok(())
}
pub fn list(path: &str) -> Result<Vec<String>> {
let (_names, names_only) = read_all(path)?;
Ok(names_only)
}
/// Returns (entries_with_data, names_only).
pub fn read_all(path: &str) -> Result<(Vec<(String, Vec<u8>)>, Vec<String>)> {
let mut f = std::fs::File::open(path).map_err(|e| Error::Model(format!("io: {e}")))?;
let mut magic = [0u8; 4];
f.read_exact(&mut magic).map_err(|e| Error::Model(format!("io: {e}")))?;
if &magic != MAGIC {
return Err(Error::Model("not an .aeg bundle".into()));
}
let mut cnt_b = [0u8; 4];
f.read_exact(&mut cnt_b).map_err(|e| Error::Model(format!("io: {e}")))?;
let count = u32::from_le_bytes(cnt_b) as usize;
let mut entries = Vec::with_capacity(count);
let mut names = Vec::with_capacity(count);
for _ in 0..count {
let mut nb = [0u8; 2];
f.read_exact(&mut nb).map_err(|e| Error::Model(format!("io: {e}")))?;
let nlen = u16::from_le_bytes(nb) as usize;
let mut nm = vec![0u8; nlen];
f.read_exact(&mut nm).map_err(|e| Error::Model(format!("io: {e}")))?;
let mut db = [0u8; 8];
f.read_exact(&mut db).map_err(|e| Error::Model(format!("io: {e}")))?;
let dlen = u64::from_le_bytes(db) as usize;
let mut data = vec![0u8; dlen];
f.read_exact(&mut data).map_err(|e| Error::Model(format!("io: {e}")))?;
let name = String::from_utf8(nm).map_err(|_| Error::Model("bad name".into()))?;
names.push(name.clone());
entries.push((name, data));
}
Ok((entries, names))
}
pub fn unpack(path: &str, out_dir: &str) -> Result<usize> {
let (entries, _) = read_all(path)?;
for (name, data) in &entries {
if name == "_manifest.json" {
continue; // metadata pseudo-entry, not a materialized file
}
let dest = Path::new(out_dir).join(name.replace('/', std::path::MAIN_SEPARATOR_STR));
if let Some(parent) = dest.parent() {
std::fs::create_dir_all(parent).map_err(|e| Error::Model(format!("io: {e}")))?;
}
std::fs::write(&dest, data).map_err(|e| Error::Model(format!("io: {e}")))?;
}
Ok(entries.len())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn roundtrip_binary_and_manifest() {
let dir = std::env::temp_dir().join("ae_bundle_test");
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(dir.join("sub")).unwrap();
std::fs::write(dir.join("a.bin"), vec![1u8, 2, 3, 250]).unwrap();
std::fs::write(dir.join("sub").join("cfg.json"), b"{\"vocab\":8192}").unwrap();
let bundle = dir.join("skull.aeg");
let manifest = r#"{"version":1,"components":["base","adapter","needle","tokenizer"]}"#;
let n = pack_dir(dir.to_str().unwrap(), bundle.to_str().unwrap(), Some(manifest)).unwrap();
assert_eq!(n, 3); // manifest + a.bin + sub/cfg.json
let names = list(bundle.to_str().unwrap()).unwrap();
assert!(names.contains(&"_manifest.json".to_string()));
assert!(names.contains(&"sub/cfg.json".to_string()));
let outd = dir.join("out");
unpack(bundle.to_str().unwrap(), outd.to_str().unwrap()).unwrap();
assert_eq!(
std::fs::read(outd.join("a.bin")).unwrap(),
vec![1u8, 2, 3, 250]
);
assert!(outd.join("sub").join("cfg.json").exists());
assert!(!outd.join("_manifest.json").exists(), "manifest not materialized");
// corrupt magic must fail loudly
let bad = dir.join("bad.aeg");
std::fs::write(&bad, b"NOPE").unwrap();
assert!(list(bad.to_str().unwrap()).is_err());
}
}