use super::*; /// Explicit source decoding. Latin1 is ISO-8859-1, never Windows-1252 or guessed. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum Encoding { #[default] Utf8, Latin1, } pub fn decode_with_encoding( bytes: &[u8], encoding: Encoding, limits: &Limits, ) -> Result { if encoding == Encoding::Utf8 { return decode(bytes, limits); } if bytes.len() > limits.max_bytes { return Err(err(0, ErrorKind::LimitExceeded("file bytes"))); } if bytes.starts_with(b"\xef\xbb\xbf") { return Err(err(0, ErrorKind::ConflictingBom)); } let n = bytes .len() .checked_add(bytes.iter().filter(|b| **b >= 128).count()) .ok_or_else(|| err(0, ErrorKind::CountOverflow))?; if n > limits.max_bytes { return Err(err(0, ErrorKind::LimitExceeded("transcoded bytes"))); } let owned = limits .max_owned_bytes .checked_sub(n) .ok_or_else(|| err(0, ErrorKind::LimitExceeded("transcode storage")))?; let mut text = Vec::new(); text.try_reserve_exact(n) .map_err(|_| err(0, ErrorKind::Allocation))?; for b in bytes { let mut buf = [0; 4]; text.extend_from_slice(char::from(*b).encode_utf8(&mut buf).as_bytes()); } decode( &text, &Limits { max_owned_bytes: owned, ..*limits }, ) } #[cfg(test)] mod tests { use super::*; #[test] fn explicit_encoding_limits_and_bom() { let input = b"world(computedFeatures false stringProperties[\"\xe9\"] )"; assert!(decode(input, &Limits::default()).is_err()); let w = decode_with_encoding(input, Encoding::Latin1, &Limits::default()).unwrap(); assert_eq!(w.string_properties, ["é"]); assert_eq!( decode(&encode(&w, &Limits::default()).unwrap(), &Limits::default()).unwrap(), w ); assert!( decode_with_encoding( b"\xef\xbb\xbfworld(computedFeatures false)", Encoding::Latin1, &Limits::default() ) .is_err() ); assert!( decode_with_encoding( input, Encoding::Latin1, &Limits { max_bytes: input.len(), ..Limits::default() } ) .is_err() ); assert!( decode_with_encoding( input, Encoding::Latin1, &Limits { max_owned_bytes: 1, ..Limits::default() } ) .is_err() ); } }