Download crates/mb3-txt/src/encoding.rs from nmarshall/nanobrush: direct link, hf CLI and curl.
- Browser
- Download file 2.74 kB
-
https://huggingface.co/nmarshall/nanobrush/resolve/main/crates/mb3-txt/src/encoding.rs
- Command line
-
hf download hf://nmarshall/nanobrush/crates/mb3-txt/src/encoding.rs
-
curl -L -o encoding.rs https://huggingface.co/nmarshall/nanobrush/resolve/main/crates/mb3-txt/src/encoding.rs
2.74 kB
| use super::*; | |
| /// Explicit source decoding. Latin1 is ISO-8859-1, never Windows-1252 or guessed. | |
| pub enum Encoding { | |
| Utf8, | |
| Latin1, | |
| } | |
| pub fn decode_with_encoding( | |
| bytes: &[u8], | |
| encoding: Encoding, | |
| limits: &Limits, | |
| ) -> Result<mb3_dat::World> { | |
| if encoding == Encoding::Utf8 { | |
| return decode(bytes, limits); | |
| } | |
| if bytes.len() > limits.max_bytes { | |
| return Err(err(0, ErrorKind::LimitExceeded("file bytes"))); | |
| } | |
| if bytes.starts_with(b"\xef\xbb\xbf") { | |
| return Err(err(0, ErrorKind::ConflictingBom)); | |
| } | |
| let n = bytes | |
| .len() | |
| .checked_add(bytes.iter().filter(|b| **b >= 128).count()) | |
| .ok_or_else(|| err(0, ErrorKind::CountOverflow))?; | |
| if n > limits.max_bytes { | |
| return Err(err(0, ErrorKind::LimitExceeded("transcoded bytes"))); | |
| } | |
| let owned = limits | |
| .max_owned_bytes | |
| .checked_sub(n) | |
| .ok_or_else(|| err(0, ErrorKind::LimitExceeded("transcode storage")))?; | |
| let mut text = Vec::new(); | |
| text.try_reserve_exact(n) | |
| .map_err(|_| err(0, ErrorKind::Allocation))?; | |
| for b in bytes { | |
| let mut buf = [0; 4]; | |
| text.extend_from_slice(char::from(*b).encode_utf8(&mut buf).as_bytes()); | |
| } | |
| decode( | |
| &text, | |
| &Limits { | |
| max_owned_bytes: owned, | |
| ..*limits | |
| }, | |
| ) | |
| } | |
| mod tests { | |
| use super::*; | |
| fn explicit_encoding_limits_and_bom() { | |
| let input = b"world(computedFeatures false stringProperties[\"\xe9\"] )"; | |
| assert!(decode(input, &Limits::default()).is_err()); | |
| let w = decode_with_encoding(input, Encoding::Latin1, &Limits::default()).unwrap(); | |
| assert_eq!(w.string_properties, ["é"]); | |
| assert_eq!( | |
| decode(&encode(&w, &Limits::default()).unwrap(), &Limits::default()).unwrap(), | |
| w | |
| ); | |
| assert!( | |
| decode_with_encoding( | |
| b"\xef\xbb\xbfworld(computedFeatures false)", | |
| Encoding::Latin1, | |
| &Limits::default() | |
| ) | |
| .is_err() | |
| ); | |
| assert!( | |
| decode_with_encoding( | |
| input, | |
| Encoding::Latin1, | |
| &Limits { | |
| max_bytes: input.len(), | |
| ..Limits::default() | |
| } | |
| ) | |
| .is_err() | |
| ); | |
| assert!( | |
| decode_with_encoding( | |
| input, | |
| Encoding::Latin1, | |
| &Limits { | |
| max_owned_bytes: 1, | |
| ..Limits::default() | |
| } | |
| ) | |
| .is_err() | |
| ); | |
| } | |
| } | |