nmarshall commited on
Commit
b3b2f4a
·
1 Parent(s): 05e776e

txt: explicit latin1

Browse files
crates/mb3-txt/src/encoding.rs ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use super::*;
2
+ /// Explicit source decoding. Latin1 is ISO-8859-1, never Windows-1252 or guessed.
3
+ #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
4
+ pub enum Encoding {
5
+ #[default]
6
+ Utf8,
7
+ Latin1,
8
+ }
9
+ pub fn decode_with_encoding(
10
+ bytes: &[u8],
11
+ encoding: Encoding,
12
+ limits: &Limits,
13
+ ) -> Result<mb3_dat::World> {
14
+ if encoding == Encoding::Utf8 {
15
+ return decode(bytes, limits);
16
+ }
17
+ if bytes.len() > limits.max_bytes {
18
+ return Err(err(0, ErrorKind::LimitExceeded("file bytes")));
19
+ }
20
+ if bytes.starts_with(b"\xef\xbb\xbf") {
21
+ return Err(err(0, ErrorKind::ConflictingBom));
22
+ }
23
+ let n = bytes
24
+ .len()
25
+ .checked_add(bytes.iter().filter(|b| **b >= 128).count())
26
+ .ok_or_else(|| err(0, ErrorKind::CountOverflow))?;
27
+ if n > limits.max_bytes {
28
+ return Err(err(0, ErrorKind::LimitExceeded("transcoded bytes")));
29
+ }
30
+ let owned = limits
31
+ .max_owned_bytes
32
+ .checked_sub(n)
33
+ .ok_or_else(|| err(0, ErrorKind::LimitExceeded("transcode storage")))?;
34
+ let mut text = Vec::new();
35
+ text.try_reserve_exact(n)
36
+ .map_err(|_| err(0, ErrorKind::Allocation))?;
37
+ for b in bytes {
38
+ let mut buf = [0; 4];
39
+ text.extend_from_slice(char::from(*b).encode_utf8(&mut buf).as_bytes());
40
+ }
41
+ decode(
42
+ &text,
43
+ &Limits {
44
+ max_owned_bytes: owned,
45
+ ..*limits
46
+ },
47
+ )
48
+ }
49
+ #[cfg(test)]
50
+ mod tests {
51
+ use super::*;
52
+ #[test]
53
+ fn explicit_encoding_limits_and_bom() {
54
+ let input = b"world(computedFeatures false stringProperties[\"\xe9\"] )";
55
+ assert!(decode(input, &Limits::default()).is_err());
56
+ let w = decode_with_encoding(input, Encoding::Latin1, &Limits::default()).unwrap();
57
+ assert_eq!(w.string_properties, ["é"]);
58
+ assert_eq!(
59
+ decode(&encode(&w, &Limits::default()).unwrap(), &Limits::default()).unwrap(),
60
+ w
61
+ );
62
+ assert!(
63
+ decode_with_encoding(
64
+ b"\xef\xbb\xbfworld(computedFeatures false)",
65
+ Encoding::Latin1,
66
+ &Limits::default()
67
+ )
68
+ .is_err()
69
+ );
70
+ assert!(
71
+ decode_with_encoding(
72
+ input,
73
+ Encoding::Latin1,
74
+ &Limits {
75
+ max_bytes: input.len(),
76
+ ..Limits::default()
77
+ }
78
+ )
79
+ .is_err()
80
+ );
81
+ assert!(
82
+ decode_with_encoding(
83
+ input,
84
+ Encoding::Latin1,
85
+ &Limits {
86
+ max_owned_bytes: 1,
87
+ ..Limits::default()
88
+ }
89
+ )
90
+ .is_err()
91
+ );
92
+ }
93
+ }
crates/mb3-txt/src/lib.rs CHANGED
@@ -9,6 +9,8 @@
9
  //! assert_eq!(mb3_txt::decode(&bytes, &Limits::default())?, world);
10
  //! # Ok::<(), mb3_txt::Error>(())
11
  //! ```
 
 
12
  mod parser;
13
  mod writer;
14
  use mb3_dat::Limits;
@@ -23,6 +25,7 @@ pub use writer::encode;
23
  pub enum ErrorKind {
24
  Io(io::Error),
25
  InvalidUtf8,
 
26
  Syntax(&'static str),
27
  InvalidNumber,
28
  NonFinite,
@@ -159,7 +162,14 @@ fn append(out: &mut Vec<u8>, bytes: &[u8], limit: usize) -> Result<()> {
159
  Ok(())
160
  }
161
  /// Bounded buffering, then strict decoding; interrupted and short reads work.
162
- pub fn read(mut source: impl Read, limits: &Limits) -> Result<mb3_dat::World> {
 
 
 
 
 
 
 
163
  let mut data = Vec::new();
164
  let mut chunk = [0; 8192];
165
  loop {
@@ -176,7 +186,7 @@ pub fn read(mut source: impl Read, limits: &Limits) -> Result<mb3_dat::World> {
176
  };
177
  append(&mut data, &chunk[..n], limits.max_bytes)?;
178
  }
179
- decode(&data, limits)
180
  }
181
  /// Validate/encode before touching the destination; does not flush or promise
182
  /// filesystem atomicity. Later I/O failure can leave a completed prefix.
 
9
  //! assert_eq!(mb3_txt::decode(&bytes, &Limits::default())?, world);
10
  //! # Ok::<(), mb3_txt::Error>(())
11
  //! ```
12
+ mod encoding;
13
+ pub use encoding::{Encoding, decode_with_encoding};
14
  mod parser;
15
  mod writer;
16
  use mb3_dat::Limits;
 
25
  pub enum ErrorKind {
26
  Io(io::Error),
27
  InvalidUtf8,
28
+ ConflictingBom,
29
  Syntax(&'static str),
30
  InvalidNumber,
31
  NonFinite,
 
162
  Ok(())
163
  }
164
  /// Bounded buffering, then strict decoding; interrupted and short reads work.
165
+ pub fn read(source: impl Read, limits: &Limits) -> Result<mb3_dat::World> {
166
+ read_with_encoding(source, Encoding::Utf8, limits)
167
+ }
168
+ pub fn read_with_encoding(
169
+ mut source: impl Read,
170
+ encoding: Encoding,
171
+ limits: &Limits,
172
+ ) -> Result<mb3_dat::World> {
173
  let mut data = Vec::new();
174
  let mut chunk = [0; 8192];
175
  loop {
 
186
  };
187
  append(&mut data, &chunk[..n], limits.max_bytes)?;
188
  }
189
+ decode_with_encoding(&data, encoding, limits)
190
  }
191
  /// Validate/encode before touching the destination; does not flush or promise
192
  /// filesystem atomicity. Later I/O failure can leave a completed prefix.
crates/nanobrush-cli/src/lib.rs CHANGED
@@ -8,7 +8,7 @@ use std::io::{self, BufRead, BufReader, Write};
8
  use std::path::{Path, PathBuf};
9
  use std::sync::atomic::{AtomicU64, Ordering};
10
 
11
- const HELP: &str = "nanobrush preflight INPUT --to txt|dat-201501280|dat-201512200 [LIMITS]\nnanobrush inspect INPUT [--from auto|txt|dat] [LIMITS]\nnanobrush validate INPUT [--from auto|txt|dat] [LIMITS]\nnanobrush convert INPUT OUTPUT --to txt|dat-201501280|dat-201512200 [--from auto|txt|dat] [--force] [LIMITS]\nLIMITS: --max-bytes N --max-items N --max-string-bytes N --max-owned-bytes N\nUse -- before positional paths beginning with a dash. TXT input is UTF-8; output has a BOM.\nExit: 0 success/help, 1 invalid data or I/O failure, 2 usage error.\n";
12
  #[derive(Debug)]
13
  pub struct Error {
14
  pub usage: bool,
@@ -56,6 +56,7 @@ struct Options {
56
  to: Option<To>,
57
  force: bool,
58
  limits: Limits,
 
59
  }
60
  fn parse(args: impl IntoIterator<Item = OsString>) -> Result<Option<Options>> {
61
  let mut args = args.into_iter();
@@ -81,6 +82,7 @@ fn parse(args: impl IntoIterator<Item = OsString>) -> Result<Option<Options>> {
81
  to: None,
82
  force: false,
83
  limits: Limits::default(),
 
84
  };
85
  let mut positional = false;
86
  let mut seen = std::collections::HashSet::new();
@@ -104,6 +106,7 @@ fn parse(args: impl IntoIterator<Item = OsString>) -> Result<Option<Options>> {
104
  if !matches!(
105
  flag,
106
  "--from"
 
107
  | "--to"
108
  | "--max-bytes"
109
  | "--max-items"
@@ -119,6 +122,13 @@ fn parse(args: impl IntoIterator<Item = OsString>) -> Result<Option<Options>> {
119
  .to_str()
120
  .ok_or_else(|| usage("option value must be UTF-8"))?;
121
  match flag {
 
 
 
 
 
 
 
122
  "--from" => {
123
  options.from = match value {
124
  "auto" => InputFormat::Auto,
@@ -175,7 +185,12 @@ struct Input {
175
  world: World,
176
  version: Option<Version>,
177
  }
178
- fn load(path: &Path, from: InputFormat, limits: &Limits) -> Result<Input> {
 
 
 
 
 
179
  let mut file =
180
  BufReader::new(File::open(path).map_err(|e| fail(format!("{}: {e}", path.display())))?);
181
  let dat = match from {
@@ -184,6 +199,9 @@ fn load(path: &Path, from: InputFormat, limits: &Limits) -> Result<Input> {
184
  InputFormat::Auto => file.fill_buf()?.starts_with(b"Mb3World"),
185
  };
186
  if dat {
 
 
 
187
  let d =
188
  mb3_dat::read(file, limits).map_err(|e| fail(format!("{}: {e}", path.display())))?;
189
  Ok(Input {
@@ -191,8 +209,8 @@ fn load(path: &Path, from: InputFormat, limits: &Limits) -> Result<Input> {
191
  version: Some(d.version),
192
  })
193
  } else {
194
- let world =
195
- mb3_txt::read(file, limits).map_err(|e| fail(format!("{}: {e}", path.display())))?;
196
  Ok(Input {
197
  world,
198
  version: None,
@@ -295,7 +313,7 @@ pub fn run(args: impl IntoIterator<Item = OsString>, stdout: &mut dyn Write) ->
295
  stdout.write_all(HELP.as_bytes())?;
296
  return Ok(());
297
  };
298
- let input = load(&o.paths[0], o.from, &o.limits)?;
299
  match o.command.as_str() {
300
  "preflight" => {
301
  let target = match o.to.ok_or_else(|| usage("missing --to"))? {
 
8
  use std::path::{Path, PathBuf};
9
  use std::sync::atomic::{AtomicU64, Ordering};
10
 
11
+ const HELP: &str = "nanobrush preflight INPUT --to txt|dat-201501280|dat-201512200 [LIMITS]\nnanobrush inspect INPUT [--from auto|txt|dat] [LIMITS]\nnanobrush validate INPUT [--from auto|txt|dat] [LIMITS]\nnanobrush convert INPUT OUTPUT --to txt|dat-201501280|dat-201512200 [--from auto|txt|dat] [--force] [LIMITS]\nLIMITS: --max-bytes N --max-items N --max-string-bytes N --max-owned-bytes N\nTXT option: --text-encoding utf8|latin1 (default utf8).\nUse -- before positional paths beginning with a dash. TXT input is UTF-8; output has a BOM.\nExit: 0 success/help, 1 invalid data or I/O failure, 2 usage error.\n";
12
  #[derive(Debug)]
13
  pub struct Error {
14
  pub usage: bool,
 
56
  to: Option<To>,
57
  force: bool,
58
  limits: Limits,
59
+ encoding: mb3_txt::Encoding,
60
  }
61
  fn parse(args: impl IntoIterator<Item = OsString>) -> Result<Option<Options>> {
62
  let mut args = args.into_iter();
 
82
  to: None,
83
  force: false,
84
  limits: Limits::default(),
85
+ encoding: mb3_txt::Encoding::Utf8,
86
  };
87
  let mut positional = false;
88
  let mut seen = std::collections::HashSet::new();
 
106
  if !matches!(
107
  flag,
108
  "--from"
109
+ | "--text-encoding"
110
  | "--to"
111
  | "--max-bytes"
112
  | "--max-items"
 
122
  .to_str()
123
  .ok_or_else(|| usage("option value must be UTF-8"))?;
124
  match flag {
125
+ "--text-encoding" => {
126
+ options.encoding = match value {
127
+ "utf8" => mb3_txt::Encoding::Utf8,
128
+ "latin1" => mb3_txt::Encoding::Latin1,
129
+ _ => return Err(usage("--text-encoding requires utf8 or latin1")),
130
+ }
131
+ }
132
  "--from" => {
133
  options.from = match value {
134
  "auto" => InputFormat::Auto,
 
185
  world: World,
186
  version: Option<Version>,
187
  }
188
+ fn load(
189
+ path: &Path,
190
+ from: InputFormat,
191
+ encoding: mb3_txt::Encoding,
192
+ limits: &Limits,
193
+ ) -> Result<Input> {
194
  let mut file =
195
  BufReader::new(File::open(path).map_err(|e| fail(format!("{}: {e}", path.display())))?);
196
  let dat = match from {
 
199
  InputFormat::Auto => file.fill_buf()?.starts_with(b"Mb3World"),
200
  };
201
  if dat {
202
+ if encoding != mb3_txt::Encoding::Utf8 {
203
+ return Err(fail("--text-encoding latin1 requires TXT input"));
204
+ }
205
  let d =
206
  mb3_dat::read(file, limits).map_err(|e| fail(format!("{}: {e}", path.display())))?;
207
  Ok(Input {
 
209
  version: Some(d.version),
210
  })
211
  } else {
212
+ let world = mb3_txt::read_with_encoding(file, encoding, limits)
213
+ .map_err(|e| fail(format!("{}: {e}", path.display())))?;
214
  Ok(Input {
215
  world,
216
  version: None,
 
313
  stdout.write_all(HELP.as_bytes())?;
314
  return Ok(());
315
  };
316
+ let input = load(&o.paths[0], o.from, o.encoding, &o.limits)?;
317
  match o.command.as_str() {
318
  "preflight" => {
319
  let target = match o.to.ok_or_else(|| usage("missing --to"))? {
docs/PROGRESS_2026-10-26.md CHANGED
@@ -5,7 +5,7 @@ Scope: extend native headless formats and geometry. Each stage is independently
5
  | Stage | TODO | Scope | Status/evidence |
6
  | --- | --- | --- | --- |
7
  | 1 | P6.23 | Structured preflight reports | Complete: `preflight INPUT --to FORMAT` emits schema-1 JSON with grouped ASCII/downgrade/duplicate-name incompatibilities; reports never mutate documents. Parser/encoder validation remains authoritative. Unit tests and existing CLI unit regressions pass. |
8
- | 2 | P6.24 | Explicit Latin-1 TXT input | Pending |
9
  | 3 | P6.25 | Bounded computed OBJ writer | Pending |
10
  | 4 | P6.26 | CLI OBJ with loss acknowledgement | Pending |
11
  | 5 | P5.15 | Geometry primitives/bounds | Pending |
 
5
  | Stage | TODO | Scope | Status/evidence |
6
  | --- | --- | --- | --- |
7
  | 1 | P6.23 | Structured preflight reports | Complete: `preflight INPUT --to FORMAT` emits schema-1 JSON with grouped ASCII/downgrade/duplicate-name incompatibilities; reports never mutate documents. Parser/encoder validation remains authoritative. Unit tests and existing CLI unit regressions pass. |
8
+ | 2 | P6.24 | Explicit Latin-1 TXT input | Complete: ISO-8859-1 API and `--text-encoding latin1`, no guessing, conflicting UTF-8 BOM rejected; transcoded bytes and scratch ownership are bounded. UTF-8 remains default and output. Encoding and CLI unit suites pass. |
9
  | 3 | P6.25 | Bounded computed OBJ writer | Pending |
10
  | 4 | P6.26 | CLI OBJ with loss acknowledgement | Pending |
11
  | 5 | P5.15 | Geometry primitives/bounds | Pending |
docs/RUST_TXT.md CHANGED
@@ -53,3 +53,5 @@ Native Unicode output follows the recovered grammar/encoding contract; a broad n
53
  - The property-optimization fixture retains `[unused, rare, common]`, while the original CLI emitted `[common, rare]`; Unicode source fixtures retain `é`, while DAT conversion refuses replacement and the historical BOM-less path drifted.
54
 
55
  Twelve unit tests cover lexer errors, comments, exponent syntax, escapes/UTF-8, finite extremes, signed zero, proper-prefix truncations, all resource policies, array consistency, streaming I/O, 3,000 bounded deterministic mutations and rejection of recursive-looking syntax. Five integration tests require hydrated E02 fixtures. Coverage-guided fuzzing, large real TXT files, explicit legacy Latin-1 recoding, native-editor interoperability and cross-platform execution remain open. See [E04](experiments/txt-cli-2026-10-21.md) for reproduction.
 
 
 
53
  - The property-optimization fixture retains `[unused, rare, common]`, while the original CLI emitted `[common, rare]`; Unicode source fixtures retain `é`, while DAT conversion refuses replacement and the historical BOM-less path drifted.
54
 
55
  Twelve unit tests cover lexer errors, comments, exponent syntax, escapes/UTF-8, finite extremes, signed zero, proper-prefix truncations, all resource policies, array consistency, streaming I/O, 3,000 bounded deterministic mutations and rejection of recursive-looking syntax. Five integration tests require hydrated E02 fixtures. Coverage-guided fuzzing, large real TXT files, explicit legacy Latin-1 recoding, native-editor interoperability and cross-platform execution remain open. See [E04](experiments/txt-cli-2026-10-21.md) for reproduction.
56
+
57
+ Explicit legacy input is now available through `Encoding::Latin1`, `decode_with_encoding` and `read_with_encoding`; the CLI uses `--text-encoding latin1`. ISO-8859-1 byte values map directly to Unicode, including C1 controls; this is not CP1252. UTF-8 BOM conflicts fail. Limits apply to raw and expanded UTF-8 bytes, and transcoding storage is subtracted from decoder ownership allowance. Parse error offsets in this mode refer to expanded UTF-8 bytes. Output stays BOM-prefixed UTF-8; no heuristic fallback occurs.
docs/TODO.md CHANGED
@@ -159,6 +159,8 @@ Each task ID is stable. Split tasks into child IDs as evidence grows. Record dep
159
 
160
  - [x] P6.23 Implement grouped, versioned conversion preflight reports for TXT/DAT incompatibilities without mutation ([ten-task ledger](PROGRESS_2026-10-26.md)).
161
 
 
 
162
  ## P7 — Input, cameras, grids, views, and editing workflows
163
 
164
  **Depends on:** P4 dispatch and P5 geometry. **Exit:** complete interaction/state-machine specifications and reproducible replays.
 
159
 
160
  - [x] P6.23 Implement grouped, versioned conversion preflight reports for TXT/DAT incompatibilities without mutation ([ten-task ledger](PROGRESS_2026-10-26.md)).
161
 
162
+ - [x] P6.24 Support explicit bounded ISO-8859-1 TXT input and CLI selection without altering the UTF-8 default or output ([ledger](PROGRESS_2026-10-26.md)).
163
+
164
  ## P7 — Input, cameras, grids, views, and editing workflows
165
 
166
  **Depends on:** P4 dispatch and P5 geometry. **Exit:** complete interaction/state-machine specifications and reproducible replays.