File size: 2,744 Bytes
b3b2f4a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | use super::*;
/// Explicit source decoding. Latin1 is ISO-8859-1, never Windows-1252 or guessed.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Encoding {
#[default]
Utf8,
Latin1,
}
pub fn decode_with_encoding(
bytes: &[u8],
encoding: Encoding,
limits: &Limits,
) -> Result<mb3_dat::World> {
if encoding == Encoding::Utf8 {
return decode(bytes, limits);
}
if bytes.len() > limits.max_bytes {
return Err(err(0, ErrorKind::LimitExceeded("file bytes")));
}
if bytes.starts_with(b"\xef\xbb\xbf") {
return Err(err(0, ErrorKind::ConflictingBom));
}
let n = bytes
.len()
.checked_add(bytes.iter().filter(|b| **b >= 128).count())
.ok_or_else(|| err(0, ErrorKind::CountOverflow))?;
if n > limits.max_bytes {
return Err(err(0, ErrorKind::LimitExceeded("transcoded bytes")));
}
let owned = limits
.max_owned_bytes
.checked_sub(n)
.ok_or_else(|| err(0, ErrorKind::LimitExceeded("transcode storage")))?;
let mut text = Vec::new();
text.try_reserve_exact(n)
.map_err(|_| err(0, ErrorKind::Allocation))?;
for b in bytes {
let mut buf = [0; 4];
text.extend_from_slice(char::from(*b).encode_utf8(&mut buf).as_bytes());
}
decode(
&text,
&Limits {
max_owned_bytes: owned,
..*limits
},
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn explicit_encoding_limits_and_bom() {
let input = b"world(computedFeatures false stringProperties[\"\xe9\"] )";
assert!(decode(input, &Limits::default()).is_err());
let w = decode_with_encoding(input, Encoding::Latin1, &Limits::default()).unwrap();
assert_eq!(w.string_properties, ["é"]);
assert_eq!(
decode(&encode(&w, &Limits::default()).unwrap(), &Limits::default()).unwrap(),
w
);
assert!(
decode_with_encoding(
b"\xef\xbb\xbfworld(computedFeatures false)",
Encoding::Latin1,
&Limits::default()
)
.is_err()
);
assert!(
decode_with_encoding(
input,
Encoding::Latin1,
&Limits {
max_bytes: input.len(),
..Limits::default()
}
)
.is_err()
);
assert!(
decode_with_encoding(
input,
Encoding::Latin1,
&Limits {
max_owned_bytes: 1,
..Limits::default()
}
)
.is_err()
);
}
}
|