//! `.aeg` single-artifact bundle (plan E6.1): one file containing the whole //! resident skull - base shards, LoRA adapter, needle engine asset, tokenizer, //! config - so distribution matches local-LLM model files. //! //! Format (v1): magic b"AEG1" | u32 entry_count | per entry: //! u16 name_len | name | u64 data_len | data //! First entry SHOULD be "_manifest.json" describing components. use std::io::{Read, Write}; use std::path::Path; use crate::error::{Error, Result}; const MAGIC: &[u8; 4] = b"AEG1"; pub fn pack(entries: Vec<(String, Vec)>, out_path: &str) -> Result<()> { let mut f = std::fs::File::create(out_path).map_err(|e| Error::Model(format!("io: {e}")))?; f.write_all(MAGIC).map_err(|e| Error::Model(format!("io: {e}")))?; f.write_all(&(entries.len() as u32).to_le_bytes()) .map_err(|e| Error::Model(format!("io: {e}")))?; for (name, data) in &entries { let nb = name.as_bytes(); if nb.len() > u16::MAX as usize { return Err(Error::Model("entry name too long".into())); } f.write_all(&(nb.len() as u16).to_le_bytes()) .map_err(|e| Error::Model(format!("io: {e}")))?; f.write_all(nb).map_err(|e| Error::Model(format!("io: {e}")))?; f.write_all(&(data.len() as u64).to_le_bytes()) .map_err(|e| Error::Model(format!("io: {e}")))?; f.write_all(data).map_err(|e| Error::Model(format!("io: {e}")))?; } Ok(()) } /// Pack a directory recursively (relative paths become entry names). pub fn pack_dir(dir: &str, out_path: &str, manifest_json: Option<&str>) -> Result { let mut entries: Vec<(String, Vec)> = Vec::new(); if let Some(m) = manifest_json { entries.push(("_manifest.json".into(), m.as_bytes().to_vec())); } collect_dir(Path::new(dir), "", &mut entries)?; let n = entries.len(); pack(entries, out_path)?; Ok(n) } fn collect_dir( root: &Path, rel: &str, out: &mut Vec<(String, Vec)>, ) -> Result<()> { for e in std::fs::read_dir(root).map_err(|e| Error::Model(format!("io: {e}")))? { let p = e.map_err(|e| Error::Model(format!("io: {e}")))?.path(); let fname = p .file_name() .map(|s| s.to_string_lossy().to_string()) .unwrap_or_default(); let rname = if rel.is_empty() { fname.clone() } else { format!("{rel}/{fname}") }; if p.is_dir() { collect_dir(&p, &rname, out)?; } else { let data = std::fs::read(&p).map_err(|e| Error::Model(format!("io: {e}")))?; out.push((rname, data)); } } Ok(()) } pub fn list(path: &str) -> Result> { let (_names, names_only) = read_all(path)?; Ok(names_only) } /// Returns (entries_with_data, names_only). pub fn read_all(path: &str) -> Result<(Vec<(String, Vec)>, Vec)> { let mut f = std::fs::File::open(path).map_err(|e| Error::Model(format!("io: {e}")))?; let mut magic = [0u8; 4]; f.read_exact(&mut magic).map_err(|e| Error::Model(format!("io: {e}")))?; if &magic != MAGIC { return Err(Error::Model("not an .aeg bundle".into())); } let mut cnt_b = [0u8; 4]; f.read_exact(&mut cnt_b).map_err(|e| Error::Model(format!("io: {e}")))?; let count = u32::from_le_bytes(cnt_b) as usize; let mut entries = Vec::with_capacity(count); let mut names = Vec::with_capacity(count); for _ in 0..count { let mut nb = [0u8; 2]; f.read_exact(&mut nb).map_err(|e| Error::Model(format!("io: {e}")))?; let nlen = u16::from_le_bytes(nb) as usize; let mut nm = vec![0u8; nlen]; f.read_exact(&mut nm).map_err(|e| Error::Model(format!("io: {e}")))?; let mut db = [0u8; 8]; f.read_exact(&mut db).map_err(|e| Error::Model(format!("io: {e}")))?; let dlen = u64::from_le_bytes(db) as usize; let mut data = vec![0u8; dlen]; f.read_exact(&mut data).map_err(|e| Error::Model(format!("io: {e}")))?; let name = String::from_utf8(nm).map_err(|_| Error::Model("bad name".into()))?; names.push(name.clone()); entries.push((name, data)); } Ok((entries, names)) } pub fn unpack(path: &str, out_dir: &str) -> Result { let (entries, _) = read_all(path)?; for (name, data) in &entries { if name == "_manifest.json" { continue; // metadata pseudo-entry, not a materialized file } let dest = Path::new(out_dir).join(name.replace('/', std::path::MAIN_SEPARATOR_STR)); if let Some(parent) = dest.parent() { std::fs::create_dir_all(parent).map_err(|e| Error::Model(format!("io: {e}")))?; } std::fs::write(&dest, data).map_err(|e| Error::Model(format!("io: {e}")))?; } Ok(entries.len()) } #[cfg(test)] mod tests { use super::*; #[test] fn roundtrip_binary_and_manifest() { let dir = std::env::temp_dir().join("ae_bundle_test"); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(dir.join("sub")).unwrap(); std::fs::write(dir.join("a.bin"), vec![1u8, 2, 3, 250]).unwrap(); std::fs::write(dir.join("sub").join("cfg.json"), b"{\"vocab\":8192}").unwrap(); let bundle = dir.join("skull.aeg"); let manifest = r#"{"version":1,"components":["base","adapter","needle","tokenizer"]}"#; let n = pack_dir(dir.to_str().unwrap(), bundle.to_str().unwrap(), Some(manifest)).unwrap(); assert_eq!(n, 3); // manifest + a.bin + sub/cfg.json let names = list(bundle.to_str().unwrap()).unwrap(); assert!(names.contains(&"_manifest.json".to_string())); assert!(names.contains(&"sub/cfg.json".to_string())); let outd = dir.join("out"); unpack(bundle.to_str().unwrap(), outd.to_str().unwrap()).unwrap(); assert_eq!( std::fs::read(outd.join("a.bin")).unwrap(), vec![1u8, 2, 3, 250] ); assert!(outd.join("sub").join("cfg.json").exists()); assert!(!outd.join("_manifest.json").exists(), "manifest not materialized"); // corrupt magic must fail loudly let bad = dir.join("bad.aeg"); std::fs::write(&bad, b"NOPE").unwrap(); assert!(list(bad.to_str().unwrap()).is_err()); } }