Download serving/src/bundle.rs from geraldaton20/alter_ego: direct link, hf CLI and curl.
- Browser
- Download file 6.31 kB
-
https://huggingface.co/geraldaton20/alter_ego/resolve/main/serving/src/bundle.rs
- Command line
-
hf download hf://geraldaton20/alter_ego/serving/src/bundle.rs
-
curl -L -o bundle.rs https://huggingface.co/geraldaton20/alter_ego/resolve/main/serving/src/bundle.rs
6.31 kB
| //! `.aeg` single-artifact bundle (plan E6.1): one file containing the whole | |
| //! resident skull - base shards, LoRA adapter, needle engine asset, tokenizer, | |
| //! config - so distribution matches local-LLM model files. | |
| //! | |
| //! Format (v1): magic b"AEG1" | u32 entry_count | per entry: | |
| //! u16 name_len | name | u64 data_len | data | |
| //! First entry SHOULD be "_manifest.json" describing components. | |
| use std::io::{Read, Write}; | |
| use std::path::Path; | |
| use crate::error::{Error, Result}; | |
| const MAGIC: &[u8; 4] = b"AEG1"; | |
| pub fn pack(entries: Vec<(String, Vec<u8>)>, out_path: &str) -> Result<()> { | |
| let mut f = std::fs::File::create(out_path).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| f.write_all(MAGIC).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| f.write_all(&(entries.len() as u32).to_le_bytes()) | |
| .map_err(|e| Error::Model(format!("io: {e}")))?; | |
| for (name, data) in &entries { | |
| let nb = name.as_bytes(); | |
| if nb.len() > u16::MAX as usize { | |
| return Err(Error::Model("entry name too long".into())); | |
| } | |
| f.write_all(&(nb.len() as u16).to_le_bytes()) | |
| .map_err(|e| Error::Model(format!("io: {e}")))?; | |
| f.write_all(nb).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| f.write_all(&(data.len() as u64).to_le_bytes()) | |
| .map_err(|e| Error::Model(format!("io: {e}")))?; | |
| f.write_all(data).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| } | |
| Ok(()) | |
| } | |
| /// Pack a directory recursively (relative paths become entry names). | |
| pub fn pack_dir(dir: &str, out_path: &str, manifest_json: Option<&str>) -> Result<usize> { | |
| let mut entries: Vec<(String, Vec<u8>)> = Vec::new(); | |
| if let Some(m) = manifest_json { | |
| entries.push(("_manifest.json".into(), m.as_bytes().to_vec())); | |
| } | |
| collect_dir(Path::new(dir), "", &mut entries)?; | |
| let n = entries.len(); | |
| pack(entries, out_path)?; | |
| Ok(n) | |
| } | |
| fn collect_dir( | |
| root: &Path, | |
| rel: &str, | |
| out: &mut Vec<(String, Vec<u8>)>, | |
| ) -> Result<()> { | |
| for e in std::fs::read_dir(root).map_err(|e| Error::Model(format!("io: {e}")))? { | |
| let p = e.map_err(|e| Error::Model(format!("io: {e}")))?.path(); | |
| let fname = p | |
| .file_name() | |
| .map(|s| s.to_string_lossy().to_string()) | |
| .unwrap_or_default(); | |
| let rname = if rel.is_empty() { fname.clone() } else { format!("{rel}/{fname}") }; | |
| if p.is_dir() { | |
| collect_dir(&p, &rname, out)?; | |
| } else { | |
| let data = std::fs::read(&p).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| out.push((rname, data)); | |
| } | |
| } | |
| Ok(()) | |
| } | |
| pub fn list(path: &str) -> Result<Vec<String>> { | |
| let (_names, names_only) = read_all(path)?; | |
| Ok(names_only) | |
| } | |
| /// Returns (entries_with_data, names_only). | |
| pub fn read_all(path: &str) -> Result<(Vec<(String, Vec<u8>)>, Vec<String>)> { | |
| let mut f = std::fs::File::open(path).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| let mut magic = [0u8; 4]; | |
| f.read_exact(&mut magic).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| if &magic != MAGIC { | |
| return Err(Error::Model("not an .aeg bundle".into())); | |
| } | |
| let mut cnt_b = [0u8; 4]; | |
| f.read_exact(&mut cnt_b).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| let count = u32::from_le_bytes(cnt_b) as usize; | |
| let mut entries = Vec::with_capacity(count); | |
| let mut names = Vec::with_capacity(count); | |
| for _ in 0..count { | |
| let mut nb = [0u8; 2]; | |
| f.read_exact(&mut nb).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| let nlen = u16::from_le_bytes(nb) as usize; | |
| let mut nm = vec![0u8; nlen]; | |
| f.read_exact(&mut nm).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| let mut db = [0u8; 8]; | |
| f.read_exact(&mut db).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| let dlen = u64::from_le_bytes(db) as usize; | |
| let mut data = vec![0u8; dlen]; | |
| f.read_exact(&mut data).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| let name = String::from_utf8(nm).map_err(|_| Error::Model("bad name".into()))?; | |
| names.push(name.clone()); | |
| entries.push((name, data)); | |
| } | |
| Ok((entries, names)) | |
| } | |
| pub fn unpack(path: &str, out_dir: &str) -> Result<usize> { | |
| let (entries, _) = read_all(path)?; | |
| for (name, data) in &entries { | |
| if name == "_manifest.json" { | |
| continue; // metadata pseudo-entry, not a materialized file | |
| } | |
| let dest = Path::new(out_dir).join(name.replace('/', std::path::MAIN_SEPARATOR_STR)); | |
| if let Some(parent) = dest.parent() { | |
| std::fs::create_dir_all(parent).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| } | |
| std::fs::write(&dest, data).map_err(|e| Error::Model(format!("io: {e}")))?; | |
| } | |
| Ok(entries.len()) | |
| } | |
| mod tests { | |
| use super::*; | |
| fn roundtrip_binary_and_manifest() { | |
| let dir = std::env::temp_dir().join("ae_bundle_test"); | |
| let _ = std::fs::remove_dir_all(&dir); | |
| std::fs::create_dir_all(dir.join("sub")).unwrap(); | |
| std::fs::write(dir.join("a.bin"), vec![1u8, 2, 3, 250]).unwrap(); | |
| std::fs::write(dir.join("sub").join("cfg.json"), b"{\"vocab\":8192}").unwrap(); | |
| let bundle = dir.join("skull.aeg"); | |
| let manifest = r#"{"version":1,"components":["base","adapter","needle","tokenizer"]}"#; | |
| let n = pack_dir(dir.to_str().unwrap(), bundle.to_str().unwrap(), Some(manifest)).unwrap(); | |
| assert_eq!(n, 3); // manifest + a.bin + sub/cfg.json | |
| let names = list(bundle.to_str().unwrap()).unwrap(); | |
| assert!(names.contains(&"_manifest.json".to_string())); | |
| assert!(names.contains(&"sub/cfg.json".to_string())); | |
| let outd = dir.join("out"); | |
| unpack(bundle.to_str().unwrap(), outd.to_str().unwrap()).unwrap(); | |
| assert_eq!( | |
| std::fs::read(outd.join("a.bin")).unwrap(), | |
| vec![1u8, 2, 3, 250] | |
| ); | |
| assert!(outd.join("sub").join("cfg.json").exists()); | |
| assert!(!outd.join("_manifest.json").exists(), "manifest not materialized"); | |
| // corrupt magic must fail loudly | |
| let bad = dir.join("bad.aeg"); | |
| std::fs::write(&bad, b"NOPE").unwrap(); | |
| assert!(list(bad.to_str().unwrap()).is_err()); | |
| } | |
| } | |