staccs commited on
Commit
1bbdee7
·
verified ·
1 Parent(s): fda8632

ds4-winhip: Windows/HIP 7.2 build tree, launcher, supervisor, bench + notes (part 2)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. crates/ds4-dist/src/lib.rs +121 -0
  2. crates/ds4-dist/src/memory_snapshot.rs +186 -0
  3. crates/ds4-dist/src/native_snapshot.rs +126 -0
  4. crates/ds4-dist/src/options.rs +351 -0
  5. crates/ds4-dist/src/plan.rs +157 -0
  6. crates/ds4-dist/src/prefetch.rs +278 -0
  7. crates/ds4-dist/src/prefetch_local.rs +253 -0
  8. crates/ds4-dist/src/reconnect.rs +217 -0
  9. crates/ds4-dist/src/relay.rs +217 -0
  10. crates/ds4-dist/src/route.rs +162 -0
  11. crates/ds4-dist/src/serve_local.rs +127 -0
  12. crates/ds4-dist/src/snapshot.rs +337 -0
  13. crates/ds4-dist/src/snapshot_gather.rs +186 -0
  14. crates/ds4-dist/src/snapshot_merge.rs +415 -0
  15. crates/ds4-dist/src/snapshot_temp.rs +99 -0
  16. crates/ds4-dist/src/transport.rs +30 -0
  17. crates/ds4-dist/src/work.rs +207 -0
  18. crates/ds4-dist/src/worker.rs +424 -0
  19. crates/ds4-dist/src/worker_snapshot.rs +291 -0
  20. crates/ds4-dist/tests/forward.rs +128 -0
  21. crates/ds4-dist/tests/prefetch.rs +267 -0
  22. crates/ds4-dist/tests/relay.rs +296 -0
  23. crates/ds4-dist/tests/runtime.rs +1257 -0
  24. crates/ds4-dist/tests/snapshot_store.rs +214 -0
  25. crates/ds4-dist/tests/snapshot_transport.rs +646 -0
  26. crates/ds4-dist/tests/wire_oracle.rs +398 -0
  27. crates/ds4-kv/Cargo.toml +10 -0
  28. crates/ds4-kv/src/extensions.rs +245 -0
  29. crates/ds4-kv/src/format.rs +781 -0
  30. crates/ds4-kv/src/host.rs +64 -0
  31. crates/ds4-kv/src/identity.rs +42 -0
  32. crates/ds4-kv/src/lib.rs +28 -0
  33. crates/ds4-kv/src/policy.rs +340 -0
  34. crates/ds4-kv/src/sha1.rs +140 -0
  35. crates/ds4-kv/src/store.rs +2406 -0
  36. crates/ds4-kv/tests/bank_thinking.rs +250 -0
  37. crates/ds4-kv/tests/four_way.rs +375 -0
  38. crates/ds4-perf-gpu/Cargo.toml +25 -0
  39. crates/ds4-perf-gpu/src/calibration.rs +176 -0
  40. crates/ds4-perf-gpu/src/collector.rs +288 -0
  41. crates/ds4-perf-gpu/src/device.rs +51 -0
  42. crates/ds4-perf-gpu/src/lib.rs +11 -0
  43. crates/ds4-perf-gpu/src/main.rs +23 -0
  44. crates/ds4-perf-gpu/tests/live.rs +14 -0
  45. crates/ds4-perf/Cargo.toml +18 -0
  46. crates/ds4-perf/src/artifact.rs +564 -0
  47. crates/ds4-perf/src/bench.rs +84 -0
  48. crates/ds4-perf/src/cli.rs +214 -0
  49. crates/ds4-perf/src/compare.rs +695 -0
  50. crates/ds4-perf/src/csv.rs +155 -0
crates/ds4-dist/src/lib.rs ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Distributed coordinator/worker wire and blocking runtime. Explicit integer
2
+ //! codecs only — do not `#[repr(C)]` records onto the socket.
3
+
4
+ mod activation;
5
+ mod assemble;
6
+ mod codec;
7
+ mod coordinator;
8
+ mod exec;
9
+ mod forward;
10
+ mod hash;
11
+ mod hops;
12
+ mod memory_snapshot;
13
+ mod native_snapshot;
14
+ mod options;
15
+ mod plan;
16
+ mod prefetch;
17
+ mod prefetch_local;
18
+ mod reconnect;
19
+ mod relay;
20
+ mod route;
21
+ mod serve_local;
22
+ mod snapshot;
23
+ mod snapshot_gather;
24
+ mod snapshot_merge;
25
+ mod snapshot_temp;
26
+ mod transport;
27
+ mod work;
28
+ mod worker;
29
+ mod worker_snapshot;
30
+
31
+ pub use activation::{
32
+ bits_or_default, bits_valid, decode_activation, encode_activation, f16_to_f32, f32_to_f16,
33
+ f32_to_f8_e4m3, f8_e4m3_to_f32, values_from_wire_bytes, wire_bytes, wire_bytes_from_f32_bytes,
34
+ BITS_DEFAULT,
35
+ };
36
+ pub use assemble::{
37
+ assemble_worker, print_worker_listen_banner, slice_meta, worker_hello, worker_listen_banner,
38
+ worker_listen_port, worker_model_name, worker_plan, AssembledWorker, SliceMeta,
39
+ WorkerListenBanner, WorkerPlan, UNKNOWN_MODEL_NAME,
40
+ };
41
+ pub use codec::{
42
+ bytes_have_nul, decode_frame_header, decode_hello_payload, decode_snapshot_begin_body,
43
+ decode_snapshot_chunk_body, decode_snapshot_done_body, decode_snapshot_load_begin_body,
44
+ decode_tokens_be, encode_error_frame, encode_frame_header, encode_hello_payload,
45
+ encode_snapshot_begin_body, encode_snapshot_chunk_body, encode_snapshot_done_body,
46
+ encode_tokens_be, get_u32_be, put_u32_be, u64_from_halves, u64_to_halves, CodecError,
47
+ FrameHeader, Hello, ResultHdr, Route, RouteReturn, SnapshotBegin, SnapshotChunk, SnapshotDone,
48
+ SnapshotReq, Telemetry, Work, FRAME_HEADER_BYTES, HELLO_FIXED_BYTES, MAGIC, MAX_MODEL_NAME,
49
+ MSG_ERROR, MSG_HELLO, MSG_RESULT, MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK, MSG_SNAPSHOT_DONE,
50
+ MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK, RESULT_ACK, RESULT_FIXED_BYTES,
51
+ RESULT_HIDDEN_STATE, RESULT_LOGITS, ROUTE_FIXED_BYTES, ROUTE_F_OUTPUT_LOGITS,
52
+ ROUTE_RETURN_FIXED_BYTES, ROUTE_RETURN_UPSTREAM, SNAPSHOT_BEGIN_FIXED_BYTES,
53
+ SNAPSHOT_CHUNK_BYTES, SNAPSHOT_CHUNK_FIXED_BYTES, SNAPSHOT_DONE_FIXED_BYTES,
54
+ SNAPSHOT_REQ_FIXED_BYTES, TELEMETRY_FIXED_BYTES, WORK_FIXED_BYTES, WORK_F_ACK_ONLY,
55
+ WORK_F_INPUT_HC, WORK_F_OUTPUT_LOGITS, WORK_F_RESET_SESSION, WORK_F_VALID_MASK,
56
+ };
57
+ pub use coordinator::{
58
+ accept_data_client, accept_loop, dispatch_eval, format_telemetry_line, listen,
59
+ open_data_listener, recv_hello_only, token_span_hashes, Coordinator, EvalOutcome,
60
+ RegisteredWorker, SharedCoordinator, ERR_DATA_LISTEN_PORT,
61
+ };
62
+ pub use exec::{SliceExec, WorkOutput, WorkRequest};
63
+ pub use forward::{
64
+ forward_window, forward_window_from, opened_forwarder_message, PendingQueue, PendingRequest,
65
+ ERR_CLOSED_WHILE_RESULT, ERR_FORWARD, ERR_FORWARD_HIDDEN, ERR_INVALID_RESULT, ERR_NEXT_CLOSED,
66
+ ERR_OOM_FORWARDER, ERR_OOM_TRACK, ERR_RELAY_THREAD, ERR_RESULT_METADATA, ERR_RESULT_TOO_LARGE,
67
+ ERR_TELEMETRY_TOO_LARGE, FORWARD_WINDOW_DEFAULT, FORWARD_WINDOW_MAX, FORWARD_WINDOW_MIN,
68
+ };
69
+ pub use hash::{token_hash_prefix, token_hash_update, token_hash_update_span, TOKEN_HASH_INIT};
70
+ pub use hops::ForwarderPool;
71
+ pub use native_snapshot::{
72
+ apply_snapshot_load, copy_chunked, create_temp, dispatch_worker_snapshot,
73
+ prepare_snapshot_save, MemorySnapshotStore, SnapshotLoad, SnapshotSave, SnapshotStore,
74
+ TempShard, LOAD_PREFIX, SAVE_PREFIX,
75
+ };
76
+ pub use options::{
77
+ parse_cli, parse_cli_arg, parse_layers, parse_role, prepare_engine_options, resolved_layer_end,
78
+ validate_layers_for_model, validate_options, CliResult, Layers, Options, Role, USAGE,
79
+ };
80
+ pub use plan::{build_route_plan, register_worker, CoordinatorView, RoutePlan, WorkerInfo};
81
+ pub use prefetch::{
82
+ prefetch_depth, prefetch_depth_from, prefetch_disabled, prefetch_disabled_from,
83
+ prefetch_enabled_message, JobQueue, ERR_OOM_QUEUE, ERR_OOM_READ, PREFETCH_DEPTH_DEFAULT,
84
+ PREFETCH_DEPTH_MAX, PREFETCH_DEPTH_MIN,
85
+ };
86
+ pub use prefetch_local::{
87
+ local_prefetch_enabled, local_prefetch_enabled_from, serve_prefetch_local_with, PrefetchJob,
88
+ };
89
+ pub use reconnect::{
90
+ cleared_sessions_message, connect_endpoint, connect_endpoint_once, connect_error,
91
+ connect_retryable, connected_message, disconnected_message, hello_failed_message, peer_name,
92
+ reconnect_local, reconnect_with, retrying_message, sleep_reconnect, LocalReconnect,
93
+ CONNECT_RETRY_ATTEMPTS, CONNECT_RETRY_DELAY, RECONNECT_SLEEP,
94
+ };
95
+ pub use relay::{
96
+ forward_work_blocking, local_work_telemetry, now_sec, prepend_telemetry, usec_since, Forwarder,
97
+ };
98
+ pub use route::{
99
+ decode_route_blob, encode_route_blob, validate_route_blob, ReturnTarget, RouteEntry,
100
+ };
101
+ pub use snapshot::{coordinator_load_snapshot, coordinator_save_snapshot, SnapshotMeta};
102
+ pub use snapshot_gather::{
103
+ gather_kv_shards, scatter_kv_shards, GatherShard, GatheredShards, ERR_EMPTY_SHARD,
104
+ ERR_NO_TIMELINE,
105
+ };
106
+ pub use snapshot_merge::{
107
+ kv_state_bytes, layer_tensor_bytes, merge_session_payload, parse_layer_payload, DistKvLayout,
108
+ ParsedLayerShard, ERR_METADATA, ERR_TENSOR_BYTES, LAYER_MAGIC, SESSION_MAGIC,
109
+ };
110
+ pub use transport::{read_frame, write_frame};
111
+ pub use work::{
112
+ decode_logits_payload, decode_result_body, decode_work_body, encode_logits_payload,
113
+ encode_result_body, encode_result_frame, encode_work_body, encode_work_frame,
114
+ error_result_frame, ok_result_hdr, result_hash, result_request_id, work_with_ids, ResultBody,
115
+ WorkBody,
116
+ };
117
+ pub use worker::{recv_hello, send_hello, Worker};
118
+ pub use worker_snapshot::{
119
+ worker_handle_snapshot_load, worker_handle_snapshot_load_restore, worker_handle_snapshot_save,
120
+ WorkerLoadOffer, WorkerSnapshotIdentity,
121
+ };
crates/ds4-dist/src/memory_snapshot.rs ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! In-memory SnapshotStore with C worker status strings.
2
+
3
+ use std::collections::HashMap;
4
+ use std::io::{self, Read, Seek, SeekFrom, Write};
5
+
6
+ use crate::hash::token_hash_prefix;
7
+ use crate::snapshot_temp::{copy_chunked, create_temp, TempShard, LOAD_PREFIX, SAVE_PREFIX};
8
+
9
+ pub struct SnapshotSave {
10
+ pub session_id: u64,
11
+ pub token_count: u32,
12
+ pub token_hash: u64,
13
+ }
14
+
15
+ pub struct SnapshotLoad<'a> {
16
+ pub session_id: u64,
17
+ pub tokens: &'a [i32],
18
+ pub token_hash: u64,
19
+ pub payload_bytes: u64,
20
+ }
21
+
22
+ pub trait SnapshotStore {
23
+ type SaveReader: Read;
24
+ fn save(&mut self, req: SnapshotSave) -> Result<(Self::SaveReader, u64), String>;
25
+ fn load(&mut self, req: SnapshotLoad<'_>, payload: &mut dyn Read) -> Result<(), String>;
26
+ }
27
+
28
+ struct StoredSession {
29
+ tokens: Vec<i32>,
30
+ shard: TempShard,
31
+ token_hash: u64,
32
+ token_hash_valid: bool,
33
+ }
34
+
35
+ pub struct MemorySnapshotStore {
36
+ sessions: HashMap<u64, StoredSession>,
37
+ fail_restore: bool,
38
+ }
39
+
40
+ impl MemorySnapshotStore {
41
+ pub fn new() -> Self {
42
+ Self {
43
+ sessions: HashMap::new(),
44
+ fail_restore: false,
45
+ }
46
+ }
47
+
48
+ pub fn insert(
49
+ &mut self,
50
+ session_id: u64,
51
+ tokens: Vec<i32>,
52
+ shard: &[u8],
53
+ ) -> Result<(), String> {
54
+ let mut tmp = create_temp(SAVE_PREFIX)
55
+ .map_err(|_| "failed to create worker snapshot temp file".to_string())?;
56
+ copy_chunked(&mut io::Cursor::new(shard), &mut tmp, shard.len() as u64)
57
+ .map_err(|_| "failed to save worker KV shard".to_string())?;
58
+ tmp.flush()
59
+ .map_err(|_| "failed to flush worker KV shard".to_string())?;
60
+ tmp.rewind()
61
+ .map_err(|_| "failed to rewind worker KV shard".to_string())?;
62
+ let token_hash = token_hash_prefix(&tokens);
63
+ self.sessions.insert(
64
+ session_id,
65
+ StoredSession {
66
+ tokens,
67
+ shard: tmp,
68
+ token_hash,
69
+ token_hash_valid: true,
70
+ },
71
+ );
72
+ Ok(())
73
+ }
74
+
75
+ pub fn fail_restore(&mut self, fail: bool) {
76
+ self.fail_restore = fail;
77
+ }
78
+
79
+ pub fn token_hash_valid(&self, session_id: u64) -> bool {
80
+ self.sessions
81
+ .get(&session_id)
82
+ .is_some_and(|s| s.token_hash_valid)
83
+ }
84
+
85
+ pub fn token_hash(&self, session_id: u64) -> Option<u64> {
86
+ self.sessions.get(&session_id).map(|s| s.token_hash)
87
+ }
88
+
89
+ pub fn tokens(&self, session_id: u64) -> Option<&[i32]> {
90
+ self.sessions.get(&session_id).map(|s| s.tokens.as_slice())
91
+ }
92
+
93
+ pub fn read_shard(&mut self, session_id: u64) -> Result<Vec<u8>, String> {
94
+ let session = self
95
+ .sessions
96
+ .get_mut(&session_id)
97
+ .ok_or_else(|| "worker has no distributed session to snapshot".to_string())?;
98
+ session
99
+ .shard
100
+ .rewind()
101
+ .map_err(|_| "failed to rewind worker KV shard".to_string())?;
102
+ let mut out = Vec::new();
103
+ session
104
+ .shard
105
+ .read_to_end(&mut out)
106
+ .map_err(|_| "failed to read worker KV shard".to_string())?;
107
+ session
108
+ .shard
109
+ .rewind()
110
+ .map_err(|_| "failed to rewind worker KV shard".to_string())?;
111
+ Ok(out)
112
+ }
113
+ }
114
+
115
+ impl Default for MemorySnapshotStore {
116
+ fn default() -> Self {
117
+ Self::new()
118
+ }
119
+ }
120
+
121
+ impl SnapshotStore for MemorySnapshotStore {
122
+ type SaveReader = TempShard;
123
+
124
+ fn save(&mut self, req: SnapshotSave) -> Result<(TempShard, u64), String> {
125
+ let session = self
126
+ .sessions
127
+ .get_mut(&req.session_id)
128
+ .ok_or_else(|| "worker has no distributed session to snapshot".to_string())?;
129
+ if session.tokens.len() as u32 != req.token_count {
130
+ return Err("worker snapshot token count mismatch".into());
131
+ }
132
+ if token_hash_prefix(&session.tokens) != req.token_hash {
133
+ return Err("worker snapshot token hash mismatch".into());
134
+ }
135
+ let mut tmp = create_temp(SAVE_PREFIX)
136
+ .map_err(|_| "failed to create worker snapshot temp file".to_string())?;
137
+ session
138
+ .shard
139
+ .rewind()
140
+ .map_err(|_| "failed to rewind worker KV shard".to_string())?;
141
+ let len = session
142
+ .shard
143
+ .seek(SeekFrom::End(0))
144
+ .map_err(|_| "failed to measure worker KV shard".to_string())?;
145
+ session
146
+ .shard
147
+ .rewind()
148
+ .map_err(|_| "failed to rewind worker KV shard".to_string())?;
149
+ copy_chunked(&mut session.shard, &mut tmp, len)
150
+ .map_err(|_| "failed to save worker KV shard".to_string())?;
151
+ tmp.flush()
152
+ .map_err(|_| "failed to flush worker KV shard".to_string())?;
153
+ let payload_bytes = tmp
154
+ .stream_position()
155
+ .map_err(|_| "failed to measure worker KV shard".to_string())?;
156
+ tmp.rewind()
157
+ .map_err(|_| "failed to rewind worker KV shard".to_string())?;
158
+ Ok((tmp, payload_bytes))
159
+ }
160
+
161
+ fn load(&mut self, req: SnapshotLoad<'_>, payload: &mut dyn Read) -> Result<(), String> {
162
+ if self.fail_restore {
163
+ if let Some(session) = self.sessions.get_mut(&req.session_id) {
164
+ session.token_hash_valid = false;
165
+ }
166
+ return Err("failed to restore worker KV shard".into());
167
+ }
168
+ let mut shard = create_temp(LOAD_PREFIX)
169
+ .map_err(|_| "failed to restore worker KV shard".to_string())?;
170
+ copy_chunked(payload, &mut shard, req.payload_bytes)
171
+ .map_err(|_| "failed to restore worker KV shard".to_string())?;
172
+ shard
173
+ .rewind()
174
+ .map_err(|_| "failed to restore worker KV shard".to_string())?;
175
+ self.sessions.insert(
176
+ req.session_id,
177
+ StoredSession {
178
+ tokens: req.tokens.to_vec(),
179
+ shard,
180
+ token_hash: req.token_hash,
181
+ token_hash_valid: true,
182
+ },
183
+ );
184
+ Ok(())
185
+ }
186
+ }
crates/ds4-dist/src/native_snapshot.rs ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Bind snapshot frames to a SnapshotStore with C restore-before-DONE order.
2
+
3
+ use std::fs::File;
4
+ use std::io::{self, Read, Write};
5
+
6
+ use crate::codec::{
7
+ u64_from_halves, SnapshotReq, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ,
8
+ SNAPSHOT_REQ_FIXED_BYTES,
9
+ };
10
+ use crate::worker_snapshot::{
11
+ worker_handle_snapshot_load_restore, worker_handle_snapshot_save, WorkerLoadOffer,
12
+ WorkerSnapshotIdentity,
13
+ };
14
+
15
+ pub use crate::memory_snapshot::{MemorySnapshotStore, SnapshotLoad, SnapshotSave, SnapshotStore};
16
+ pub use crate::snapshot_temp::{copy_chunked, create_temp, TempShard, LOAD_PREFIX, SAVE_PREFIX};
17
+
18
+ pub fn prepare_snapshot_save<Store: SnapshotStore>(
19
+ store: &mut Store,
20
+ identity: WorkerSnapshotIdentity,
21
+ req_body: &[u8],
22
+ ) -> Result<(Store::SaveReader, u64), String> {
23
+ if req_body.len() != SNAPSHOT_REQ_FIXED_BYTES {
24
+ return Err(String::new());
25
+ }
26
+ let req = SnapshotReq::decode(req_body).map_err(|e| e.to_string())?;
27
+ if !identity.accepts(
28
+ req.model_id,
29
+ req.layer_start,
30
+ req.layer_end,
31
+ req.token_count,
32
+ ) {
33
+ return Err(String::new());
34
+ }
35
+ store.save(SnapshotSave {
36
+ session_id: u64_from_halves(req.session_hi, req.session_lo),
37
+ token_count: req.token_count,
38
+ token_hash: u64_from_halves(req.token_hash_hi, req.token_hash_lo),
39
+ })
40
+ }
41
+
42
+ pub fn apply_snapshot_load<S, Store>(
43
+ stream: &mut S,
44
+ identity: WorkerSnapshotIdentity,
45
+ begin_body: &[u8],
46
+ vocab_size: u32,
47
+ store: &mut Store,
48
+ ) -> Result<WorkerLoadOffer, String>
49
+ where
50
+ S: Read + Write,
51
+ Store: SnapshotStore,
52
+ {
53
+ match create_temp(LOAD_PREFIX) {
54
+ Ok(mut tmp) => {
55
+ let path = tmp.path.clone();
56
+ worker_handle_snapshot_load_restore(
57
+ stream,
58
+ identity,
59
+ begin_body,
60
+ vocab_size,
61
+ &mut tmp,
62
+ |offer| {
63
+ let mut replay = File::open(&path)
64
+ .map_err(|_| "failed to rewind worker KV shard restore file".to_string())?;
65
+ store.load(
66
+ SnapshotLoad {
67
+ session_id: offer.session_id,
68
+ tokens: &offer.tokens,
69
+ token_hash: offer.token_hash,
70
+ payload_bytes: offer.payload_bytes,
71
+ },
72
+ &mut replay,
73
+ )
74
+ },
75
+ )
76
+ }
77
+ Err(_) => {
78
+ let mut sink = io::sink();
79
+ worker_handle_snapshot_load_restore(
80
+ stream,
81
+ identity,
82
+ begin_body,
83
+ vocab_size,
84
+ &mut sink,
85
+ |_| Err("failed to create worker snapshot restore temp file".into()),
86
+ )
87
+ }
88
+ }
89
+ }
90
+
91
+ pub fn dispatch_worker_snapshot<Stream, Store>(
92
+ stream: &mut Stream,
93
+ typ: u32,
94
+ body: &[u8],
95
+ identity: WorkerSnapshotIdentity,
96
+ vocab: u32,
97
+ store: &mut Store,
98
+ ) -> Result<Option<(u64, u64)>, String>
99
+ where
100
+ Stream: Read + Write,
101
+ Store: SnapshotStore,
102
+ {
103
+ match typ {
104
+ MSG_SNAPSHOT_SAVE_REQ => {
105
+ match prepare_snapshot_save(store, identity, body) {
106
+ Ok((mut reader, n)) => {
107
+ worker_handle_snapshot_save(stream, identity, body, Ok((&mut reader, n)))?;
108
+ }
109
+ Err(e) => {
110
+ worker_handle_snapshot_save::<_, io::Cursor<Vec<u8>>>(
111
+ stream,
112
+ identity,
113
+ body,
114
+ Err(e),
115
+ )?;
116
+ }
117
+ }
118
+ Ok(None)
119
+ }
120
+ MSG_SNAPSHOT_LOAD_BEGIN => {
121
+ let offer = apply_snapshot_load(stream, identity, body, vocab, store)?;
122
+ Ok(Some((offer.session_id, offer.token_hash)))
123
+ }
124
+ _ => Err(format!("rejected unsupported frame type {typ}")),
125
+ }
126
+ }
crates/ds4-dist/src/options.rs ADDED
@@ -0,0 +1,351 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! CLI + option validation matching `ds4_dist_parse_cli_arg` / `dist_validate_options`.
2
+
3
+ use crate::activation::bits_valid;
4
+
5
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
6
+ pub enum Role {
7
+ None = 0,
8
+ Coordinator = 1,
9
+ Worker = 2,
10
+ }
11
+
12
+ impl Role {
13
+ pub fn name(self) -> &'static str {
14
+ match self {
15
+ Role::None => "none",
16
+ Role::Coordinator => "coordinator",
17
+ Role::Worker => "worker",
18
+ }
19
+ }
20
+ }
21
+
22
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
23
+ pub struct Layers {
24
+ pub start: u32,
25
+ pub end: u32,
26
+ pub has_output: bool,
27
+ pub set: bool,
28
+ }
29
+
30
+ #[derive(Debug, Clone, PartialEq, Eq, Default)]
31
+ pub struct Options {
32
+ pub role: Role,
33
+ pub layers: Layers,
34
+ pub listen_host: Option<String>,
35
+ pub listen_port: i32,
36
+ pub coordinator_host: Option<String>,
37
+ pub coordinator_port: i32,
38
+ pub prefill_chunk: u32,
39
+ pub prefill_window: u32,
40
+ pub activation_bits: u32,
41
+ pub replay_check: bool,
42
+ pub debug: bool,
43
+ }
44
+
45
+ impl Default for Role {
46
+ fn default() -> Self {
47
+ Role::None
48
+ }
49
+ }
50
+
51
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
52
+ pub enum CliResult {
53
+ Error,
54
+ NotMatched,
55
+ Matched,
56
+ }
57
+
58
+ pub fn parse_role(s: &str) -> Option<Role> {
59
+ match s {
60
+ "none" => Some(Role::None),
61
+ "coordinator" => Some(Role::Coordinator),
62
+ "worker" => Some(Role::Worker),
63
+ _ => None,
64
+ }
65
+ }
66
+
67
+ pub fn parse_layers(s: &str) -> Result<Layers, String> {
68
+ let Some(colon) = s.find(':') else {
69
+ return Err("expected A:B or A:output".into());
70
+ };
71
+ if colon == 0 || colon + 1 == s.len() {
72
+ return Err("expected A:B or A:output".into());
73
+ }
74
+ if s[colon + 1..].contains(':') {
75
+ return Err("layer range has too many ':' separators".into());
76
+ }
77
+ let start =
78
+ parse_u32_component(&s[..colon]).ok_or_else(|| format!("invalid start layer in {s}"))?;
79
+ let end_s = &s[colon + 1..];
80
+ if end_s == "output" {
81
+ return Ok(Layers {
82
+ start,
83
+ end: u32::MAX,
84
+ has_output: true,
85
+ set: true,
86
+ });
87
+ }
88
+ let end = parse_u32_component(end_s).ok_or_else(|| format!("invalid end layer in {s}"))?;
89
+ if end < start {
90
+ return Err(format!("layer range end precedes start in {s}"));
91
+ }
92
+ Ok(Layers {
93
+ start,
94
+ end,
95
+ has_output: false,
96
+ set: true,
97
+ })
98
+ }
99
+
100
+ fn parse_u32_component(s: &str) -> Option<u32> {
101
+ if s.is_empty() || s.len() >= 32 {
102
+ return None;
103
+ }
104
+ if s.as_bytes().iter().any(|c| !c.is_ascii_digit()) {
105
+ return None;
106
+ }
107
+ s.parse().ok()
108
+ }
109
+
110
+ fn parse_port(s: &str, arg: &str) -> Result<i32, String> {
111
+ let v: i64 = s
112
+ .parse()
113
+ .map_err(|_| format!("invalid value for {arg}: {s}"))?;
114
+ if v <= 0 || v > 65535 {
115
+ return Err(format!("invalid value for {arg}: {s}"));
116
+ }
117
+ Ok(v as i32)
118
+ }
119
+
120
+ fn parse_positive_u32(s: &str, name: &str) -> Result<u32, String> {
121
+ if s.is_empty() {
122
+ return Err(format!("{name} requires a positive integer"));
123
+ }
124
+ let v: u64 = s
125
+ .parse()
126
+ .map_err(|_| format!("invalid value for {name}: {s}"))?;
127
+ if v == 0 || v > u64::from(u32::MAX) {
128
+ return Err(format!("invalid value for {name}: {s}"));
129
+ }
130
+ Ok(v as u32)
131
+ }
132
+
133
+ pub fn parse_cli_arg(
134
+ arg: &str,
135
+ rest: &mut impl Iterator<Item = String>,
136
+ opt: &mut Options,
137
+ ) -> Result<CliResult, String> {
138
+ match arg {
139
+ "--role" => {
140
+ let role = rest
141
+ .next()
142
+ .ok_or_else(|| "--role requires an argument".to_string())?;
143
+ let parsed = parse_role(&role).ok_or_else(|| {
144
+ format!("invalid distributed role: {role} (valid roles: none, coordinator, worker)")
145
+ })?;
146
+ opt.role = parsed;
147
+ Ok(CliResult::Matched)
148
+ }
149
+ "--layers" => {
150
+ let layers = rest
151
+ .next()
152
+ .ok_or_else(|| "--layers requires an argument".to_string())?;
153
+ match parse_layers(&layers) {
154
+ Ok(l) => {
155
+ opt.layers = l;
156
+ Ok(CliResult::Matched)
157
+ }
158
+ Err(detail) => Err(format!("invalid --layers {layers}: {detail}")),
159
+ }
160
+ }
161
+ "--listen" => {
162
+ if opt.listen_host.is_some() || opt.listen_port != 0 {
163
+ return Err("specify --listen only once".into());
164
+ }
165
+ let host = rest
166
+ .next()
167
+ .ok_or_else(|| "--listen requires an argument".to_string())?;
168
+ let port = rest
169
+ .next()
170
+ .ok_or_else(|| "--listen requires an argument".to_string())?;
171
+ opt.listen_port = parse_port(&port, "--listen")?;
172
+ opt.listen_host = Some(host);
173
+ Ok(CliResult::Matched)
174
+ }
175
+ "--coordinator" => {
176
+ if opt.coordinator_host.is_some() || opt.coordinator_port != 0 {
177
+ return Err("specify --coordinator only once".into());
178
+ }
179
+ let host = rest
180
+ .next()
181
+ .ok_or_else(|| "--coordinator requires an argument".to_string())?;
182
+ let port = rest
183
+ .next()
184
+ .ok_or_else(|| "--coordinator requires an argument".to_string())?;
185
+ opt.coordinator_port = parse_port(&port, "--coordinator")?;
186
+ opt.coordinator_host = Some(host);
187
+ Ok(CliResult::Matched)
188
+ }
189
+ "--dist-prefill-chunk" => {
190
+ let value = rest
191
+ .next()
192
+ .ok_or_else(|| "--dist-prefill-chunk requires an argument".to_string())?;
193
+ opt.prefill_chunk = parse_positive_u32(&value, "--dist-prefill-chunk")?;
194
+ Ok(CliResult::Matched)
195
+ }
196
+ "--dist-prefill-window" => {
197
+ let value = rest
198
+ .next()
199
+ .ok_or_else(|| "--dist-prefill-window requires an argument".to_string())?;
200
+ opt.prefill_window = parse_positive_u32(&value, "--dist-prefill-window")?;
201
+ if opt.prefill_window > 64 {
202
+ return Err("--dist-prefill-window must be <= 64".into());
203
+ }
204
+ Ok(CliResult::Matched)
205
+ }
206
+ "--dist-activation-bits" => {
207
+ let value = rest
208
+ .next()
209
+ .ok_or_else(|| "--dist-activation-bits requires an argument".to_string())?;
210
+ let bits = parse_positive_u32(&value, "--dist-activation-bits")?;
211
+ if !bits_valid(bits) {
212
+ return Err("--dist-activation-bits must be 32, 16, or 8".into());
213
+ }
214
+ opt.activation_bits = bits;
215
+ Ok(CliResult::Matched)
216
+ }
217
+ "--dist-replay-check" => {
218
+ opt.replay_check = true;
219
+ Ok(CliResult::Matched)
220
+ }
221
+ "--debug" => {
222
+ opt.debug = true;
223
+ Ok(CliResult::Matched)
224
+ }
225
+ _ => Ok(CliResult::NotMatched),
226
+ }
227
+ }
228
+
229
+ pub fn parse_cli(args: &[String]) -> Result<(Options, Vec<String>), String> {
230
+ let mut opt = Options::default();
231
+ let mut unmatched = Vec::new();
232
+ let mut rest = args.iter().cloned().collect::<Vec<_>>().into_iter();
233
+ while let Some(arg) = rest.next() {
234
+ match parse_cli_arg(&arg, &mut rest, &mut opt)? {
235
+ CliResult::Matched => {}
236
+ CliResult::NotMatched => unmatched.push(arg),
237
+ CliResult::Error => unreachable!(),
238
+ }
239
+ }
240
+ Ok((opt, unmatched))
241
+ }
242
+
243
+ pub fn validate_options(opt: &Options) -> Result<(), String> {
244
+ if opt.role == Role::None {
245
+ if opt.layers.set
246
+ || opt.listen_host.is_some()
247
+ || opt.listen_port != 0
248
+ || opt.coordinator_host.is_some()
249
+ || opt.coordinator_port != 0
250
+ || opt.prefill_chunk != 0
251
+ || opt.prefill_window != 0
252
+ || opt.activation_bits != 0
253
+ {
254
+ return Err("distributed options require --role coordinator or --role worker".into());
255
+ }
256
+ return Ok(());
257
+ }
258
+ if !opt.layers.set {
259
+ return Err(format!("--role {} requires --layers", opt.role.name()));
260
+ }
261
+ if opt.prefill_window > 64 {
262
+ return Err("--dist-prefill-window must be <= 64".into());
263
+ }
264
+ if opt.activation_bits != 0 && !bits_valid(opt.activation_bits) {
265
+ return Err("--dist-activation-bits must be 32, 16, or 8".into());
266
+ }
267
+ match opt.role {
268
+ Role::Coordinator => {
269
+ if opt.listen_host.is_none() || opt.listen_port <= 0 {
270
+ return Err("--role coordinator requires --listen HOST PORT".into());
271
+ }
272
+ if opt.coordinator_host.is_some() || opt.coordinator_port != 0 {
273
+ return Err("--role coordinator must not use --coordinator".into());
274
+ }
275
+ Ok(())
276
+ }
277
+ Role::Worker => {
278
+ if opt.coordinator_host.is_none() || opt.coordinator_port <= 0 {
279
+ return Err("--role worker requires --coordinator HOST PORT".into());
280
+ }
281
+ if opt.prefill_chunk != 0 {
282
+ return Err("--dist-prefill-chunk requires --role coordinator".into());
283
+ }
284
+ if opt.prefill_window != 0 {
285
+ return Err("--dist-prefill-window requires --role coordinator".into());
286
+ }
287
+ if opt.activation_bits != 0 {
288
+ return Err("--dist-activation-bits requires --role coordinator".into());
289
+ }
290
+ Ok(())
291
+ }
292
+ Role::None => Ok(()),
293
+ }
294
+ }
295
+
296
+ pub fn validate_layers_for_model(opt: &Options, n_layers: u32) -> Result<(), String> {
297
+ if opt.role == Role::None || !opt.layers.set {
298
+ return Ok(());
299
+ }
300
+ if n_layers == 0 {
301
+ return Err("model reports no layers".into());
302
+ }
303
+ let last = n_layers - 1;
304
+ if opt.layers.start > last {
305
+ return Err(format!("layer range starts past final model layer {last}"));
306
+ }
307
+ if !opt.layers.has_output && opt.layers.end > last {
308
+ return Err(format!("layer range ends past final model layer {last}"));
309
+ }
310
+ if opt.role == Role::Coordinator && opt.layers.start != 0 {
311
+ return Err("coordinator layer range must start at layer 0".into());
312
+ }
313
+ Ok(())
314
+ }
315
+
316
+ pub fn resolved_layer_end(layers: &Layers, n_layers: u32) -> u32 {
317
+ if layers.has_output {
318
+ n_layers.saturating_sub(1)
319
+ } else {
320
+ layers.end
321
+ }
322
+ }
323
+
324
+ pub fn prepare_engine_options(opt: &Options) -> Result<(), String> {
325
+ validate_options(opt)?;
326
+ if opt.replay_check && opt.role != Role::Coordinator {
327
+ return Err("--dist-replay-check requires --role coordinator".into());
328
+ }
329
+ Ok(())
330
+ }
331
+
332
+ pub const USAGE: &str = "\
333
+ --role ROLE\n\
334
+ Distributed role: coordinator or worker.\n\
335
+ --layers A:B\n\
336
+ Inclusive distributed layer slice, e.g. 10:20 or 21:output.\n\
337
+ --listen HOST PORT\n\
338
+ Coordinator TCP listen address. Workers may later use it to force their data listener.\n\
339
+ --coordinator HOST PORT\n\
340
+ Coordinator TCP address for --role worker.\n\
341
+ --dist-prefill-chunk N\n\
342
+ Coordinator prefill pipeline chunk size. Default: session cap, normally 4096.\n\
343
+ Non-default values are experimental and can change logits unless validated.\n\
344
+ --dist-prefill-window N\n\
345
+ Coordinator max end-to-end prefill chunks in flight. Default: workers+2, capped at 8.\n\
346
+ --dist-activation-bits N\n\
347
+ Coordinator hidden-state transport width: 32, 16, or 8. Default: 32.\n\
348
+ --dist-replay-check\n\
349
+ Coordinator diagnostic: reset and replay the prompt, then compare logits.\n\
350
+ --debug\n\
351
+ Print coordinator route/debug logs. Workers keep their normal logs without this.\n";
crates/ds4-dist/src/plan.rs ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Worker registry and route search. Matches `dist_coordinator_build_route_plan`.
2
+
3
+ use crate::codec::{ROUTE_F_OUTPUT_LOGITS, ROUTE_RETURN_UPSTREAM};
4
+ use crate::route::{encode_route_blob, ReturnTarget, RouteEntry};
5
+
6
+ #[derive(Debug, Clone, PartialEq, Eq)]
7
+ pub struct WorkerInfo {
8
+ pub peer_host: String,
9
+ pub listen_port: u32,
10
+ pub model_id: u32,
11
+ pub quant_bits: u32,
12
+ pub layer_start: u32,
13
+ pub layer_end: u32,
14
+ pub has_output: bool,
15
+ pub has_hidden: bool,
16
+ }
17
+
18
+ #[derive(Debug, Clone)]
19
+ pub struct CoordinatorView {
20
+ pub n_layers: u32,
21
+ pub local_start: u32,
22
+ pub local_end: u32,
23
+ pub local_has_output: bool,
24
+ pub local_can_output_head: bool,
25
+ }
26
+
27
+ #[derive(Debug, Clone, PartialEq, Eq)]
28
+ pub struct RoutePlan {
29
+ pub entries: Vec<RouteEntry>,
30
+ pub blob: Vec<u8>,
31
+ }
32
+
33
+ fn worker_cmp(a: &WorkerInfo, b: &WorkerInfo) -> std::cmp::Ordering {
34
+ a.layer_start
35
+ .cmp(&b.layer_start)
36
+ .then_with(|| b.has_output.cmp(&a.has_output))
37
+ .then_with(|| b.layer_end.cmp(&a.layer_end))
38
+ }
39
+
40
+ fn candidate_ok(view: &CoordinatorView, w: &WorkerInfo, last: u32) -> bool {
41
+ let needs_hidden = w.layer_end < last || !w.has_output;
42
+ if needs_hidden && !w.has_hidden {
43
+ return false;
44
+ }
45
+ if w.layer_end >= last && !w.has_output && !view.local_can_output_head {
46
+ return false;
47
+ }
48
+ true
49
+ }
50
+
51
+ fn search(
52
+ view: &CoordinatorView,
53
+ workers: &[WorkerInfo],
54
+ next: u32,
55
+ last: u32,
56
+ path: &mut Vec<usize>,
57
+ missing: &mut u32,
58
+ ) -> bool {
59
+ let mut saw_start = false;
60
+ for (i, w) in workers.iter().enumerate() {
61
+ if w.layer_start < next {
62
+ continue;
63
+ }
64
+ if w.layer_start > next {
65
+ break;
66
+ }
67
+ saw_start = true;
68
+ if !candidate_ok(view, w, last) {
69
+ continue;
70
+ }
71
+ path.push(i);
72
+ if w.layer_end >= last {
73
+ return true;
74
+ }
75
+ let mut child_missing = w.layer_end + 1;
76
+ if search(view, workers, child_missing, last, path, &mut child_missing) {
77
+ return true;
78
+ }
79
+ if child_missing > *missing {
80
+ *missing = child_missing;
81
+ }
82
+ path.pop();
83
+ }
84
+ if !saw_start && next > *missing {
85
+ *missing = next;
86
+ }
87
+ false
88
+ }
89
+
90
+ pub fn build_route_plan(
91
+ view: &CoordinatorView,
92
+ workers: &[WorkerInfo],
93
+ ) -> Result<RoutePlan, String> {
94
+ if view.local_start != 0 {
95
+ return Err("coordinator route does not start at layer 0".into());
96
+ }
97
+ let last = view.n_layers.saturating_sub(1);
98
+ if view.local_end == last && (view.local_has_output || view.local_can_output_head) {
99
+ return Ok(RoutePlan {
100
+ entries: Vec::new(),
101
+ blob: Vec::new(),
102
+ });
103
+ }
104
+
105
+ let mut sorted = workers.to_vec();
106
+ sorted.sort_by(worker_cmp);
107
+ let next = view.local_end + 1;
108
+ let mut path = Vec::new();
109
+ let mut missing = next;
110
+ if !search(view, &sorted, next, last, &mut path, &mut missing) {
111
+ return Err(format!(
112
+ "distributed route incomplete: missing layer {missing}"
113
+ ));
114
+ }
115
+
116
+ let mut entries = Vec::new();
117
+ for idx in path {
118
+ let w = &sorted[idx];
119
+ entries.push(RouteEntry {
120
+ host: w.peer_host.clone(),
121
+ port: w.listen_port,
122
+ layer_start: w.layer_start,
123
+ layer_end: w.layer_end,
124
+ flags: if w.has_output {
125
+ ROUTE_F_OUTPUT_LOGITS
126
+ } else {
127
+ 0
128
+ },
129
+ });
130
+ }
131
+ let blob = if entries.is_empty() {
132
+ Vec::new()
133
+ } else {
134
+ encode_route_blob(
135
+ &entries,
136
+ &ReturnTarget {
137
+ kind: ROUTE_RETURN_UPSTREAM,
138
+ host: String::new(),
139
+ port: 0,
140
+ },
141
+ )
142
+ .map_err(|e| e.to_string())?
143
+ };
144
+ Ok(RoutePlan { entries, blob })
145
+ }
146
+
147
+ /// Replace a stale worker with the same host/model/layers/output (C prepends).
148
+ pub fn register_worker(list: &mut Vec<WorkerInfo>, neu: WorkerInfo) {
149
+ list.retain(|old| {
150
+ !(old.peer_host == neu.peer_host
151
+ && old.model_id == neu.model_id
152
+ && old.layer_start == neu.layer_start
153
+ && old.layer_end == neu.layer_end
154
+ && old.has_output == neu.has_output)
155
+ });
156
+ list.insert(0, neu);
157
+ }
crates/ds4-dist/src/prefetch.rs ADDED
@@ -0,0 +1,278 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Worker receive prefetch: `DS4_DIST_WORKER_PREFETCH_DEPTH` and the C job queue.
2
+
3
+ use std::collections::VecDeque;
4
+ use std::io::{self, Write};
5
+ use std::net::TcpStream;
6
+ use std::sync::{Arc, Condvar, Mutex};
7
+ use std::thread;
8
+
9
+ use crate::codec::{MSG_ERROR, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK};
10
+ use crate::exec::SliceExec;
11
+ use crate::native_snapshot::SnapshotStore;
12
+ use crate::transport::{read_frame, write_frame};
13
+ use crate::worker::Worker;
14
+
15
+ pub const PREFETCH_DEPTH_DEFAULT: u32 = 2;
16
+ pub const PREFETCH_DEPTH_MIN: u32 = 1;
17
+ pub const PREFETCH_DEPTH_MAX: u32 = 8;
18
+ pub const ERR_OOM_QUEUE: &str = "out of memory queueing distributed WORK";
19
+ pub const ERR_OOM_READ: &str = "out of memory reading distributed WORK frame";
20
+
21
+ pub fn prefetch_depth_from(env: Option<&str>) -> u32 {
22
+ let Some(raw) = env else {
23
+ return PREFETCH_DEPTH_DEFAULT;
24
+ };
25
+ if raw.is_empty() {
26
+ return PREFETCH_DEPTH_DEFAULT;
27
+ }
28
+ match parse_strtol(raw) {
29
+ Some(v) if v >= i64::from(PREFETCH_DEPTH_MIN) && v <= i64::from(PREFETCH_DEPTH_MAX) => {
30
+ v as u32
31
+ }
32
+ _ => PREFETCH_DEPTH_DEFAULT,
33
+ }
34
+ }
35
+
36
+ pub fn prefetch_depth() -> u32 {
37
+ prefetch_depth_from(
38
+ std::env::var("DS4_DIST_WORKER_PREFETCH_DEPTH")
39
+ .ok()
40
+ .as_deref(),
41
+ )
42
+ }
43
+
44
+ pub fn prefetch_disabled_from(env: Option<&std::ffi::OsStr>) -> bool {
45
+ env.is_some()
46
+ }
47
+
48
+ pub fn prefetch_disabled() -> bool {
49
+ prefetch_disabled_from(std::env::var_os("DS4_DIST_DISABLE_WORKER_PREFETCH").as_deref())
50
+ }
51
+
52
+ pub fn prefetch_enabled_message(depth: u32) -> String {
53
+ format!("ds4: distributed worker: receive prefetch depth {depth} enabled")
54
+ }
55
+
56
+ fn parse_strtol(s: &str) -> Option<i64> {
57
+ let s = s.trim_start_matches(|c: char| c.is_ascii_whitespace());
58
+ if s.is_empty() {
59
+ return None;
60
+ }
61
+ let (neg, digits) = match s.as_bytes()[0] {
62
+ b'+' => (false, &s[1..]),
63
+ b'-' => (true, &s[1..]),
64
+ _ => (false, s),
65
+ };
66
+ if digits.is_empty() || !digits.as_bytes()[0].is_ascii_digit() {
67
+ return None;
68
+ }
69
+ if !digits.bytes().all(|b| b.is_ascii_digit()) {
70
+ return None;
71
+ }
72
+ let v = digits.parse::<i64>().ok()?;
73
+ Some(if neg { -v } else { v })
74
+ }
75
+
76
+ struct QueueInner<T> {
77
+ depth: u32,
78
+ queued: u32,
79
+ jobs: VecDeque<T>,
80
+ closed: bool,
81
+ canceled: bool,
82
+ }
83
+
84
+ pub struct JobQueue<T = Vec<u8>> {
85
+ inner: Mutex<QueueInner<T>>,
86
+ not_empty: Condvar,
87
+ not_full: Condvar,
88
+ }
89
+
90
+ impl<T> JobQueue<T> {
91
+ pub fn new(depth: u32) -> Self {
92
+ let depth = depth.max(1);
93
+ Self {
94
+ inner: Mutex::new(QueueInner {
95
+ depth,
96
+ queued: 0,
97
+ jobs: VecDeque::new(),
98
+ closed: false,
99
+ canceled: false,
100
+ }),
101
+ not_empty: Condvar::new(),
102
+ not_full: Condvar::new(),
103
+ }
104
+ }
105
+
106
+ pub fn depth(&self) -> u32 {
107
+ self.inner.lock().expect("prefetch queue").depth
108
+ }
109
+
110
+ pub fn queued(&self) -> u32 {
111
+ self.inner.lock().expect("prefetch queue").queued
112
+ }
113
+
114
+ pub fn enqueue(&self, job: T) -> bool {
115
+ let mut g = self.inner.lock().expect("prefetch queue");
116
+ while !g.closed && !g.canceled && g.queued >= g.depth {
117
+ g = self.not_full.wait(g).expect("prefetch queue");
118
+ }
119
+ if g.closed || g.canceled {
120
+ return false;
121
+ }
122
+ g.jobs.push_back(job);
123
+ g.queued += 1;
124
+ self.not_empty.notify_one();
125
+ true
126
+ }
127
+
128
+ pub fn wait_until_queued(&self, n: u32) {
129
+ let mut g = self.inner.lock().expect("prefetch queue");
130
+ while g.queued < n && !g.closed && !g.canceled {
131
+ g = self.not_empty.wait(g).expect("prefetch queue");
132
+ }
133
+ }
134
+
135
+ fn take_job(g: &mut QueueInner<T>, not_full: &Condvar) -> Option<T> {
136
+ if g.canceled || g.jobs.is_empty() {
137
+ return None;
138
+ }
139
+ let job = g.jobs.pop_front();
140
+ g.queued = g.queued.saturating_sub(1);
141
+ not_full.notify_one();
142
+ job
143
+ }
144
+
145
+ pub fn try_pop(&self) -> Option<T> {
146
+ let mut g = self.inner.lock().expect("prefetch queue");
147
+ Self::take_job(&mut g, &self.not_full)
148
+ }
149
+
150
+ pub fn pop(&self) -> Option<T> {
151
+ let mut g = self.inner.lock().expect("prefetch queue");
152
+ while g.jobs.is_empty() && !g.closed && !g.canceled {
153
+ g = self.not_empty.wait(g).expect("prefetch queue");
154
+ }
155
+ Self::take_job(&mut g, &self.not_full)
156
+ }
157
+
158
+ pub fn finish(&self) {
159
+ let mut g = self.inner.lock().expect("prefetch queue");
160
+ g.closed = true;
161
+ self.not_empty.notify_all();
162
+ self.not_full.notify_all();
163
+ }
164
+
165
+ pub fn cancel(&self) {
166
+ let mut g = self.inner.lock().expect("prefetch queue");
167
+ g.closed = true;
168
+ g.canceled = true;
169
+ g.jobs.clear();
170
+ g.queued = 0;
171
+ self.not_empty.notify_all();
172
+ self.not_full.notify_all();
173
+ }
174
+ }
175
+
176
+ pub(crate) struct MutexWrite<'a>(pub &'a Mutex<TcpStream>);
177
+
178
+ impl Write for MutexWrite<'_> {
179
+ fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
180
+ self.0.lock().expect("worker write").write(buf)
181
+ }
182
+
183
+ fn flush(&mut self) -> io::Result<()> {
184
+ self.0.lock().expect("worker write").flush()
185
+ }
186
+ }
187
+
188
+ impl<E: SliceExec + Send, S: SnapshotStore + Send> Worker<E, S> {
189
+ pub fn serve_prefetch(&mut self, stream: &mut TcpStream) -> io::Result<()> {
190
+ let depth = prefetch_depth();
191
+ let queue = Arc::new(JobQueue::<Vec<u8>>::new(depth));
192
+ let worker = Mutex::new(&mut *self);
193
+ let writer = Arc::new(Mutex::new(stream.try_clone()?));
194
+ let mut reader = stream.try_clone()?;
195
+ worker
196
+ .lock()
197
+ .expect("worker hops")
198
+ .bind_hops(Arc::clone(&writer));
199
+ eprintln!("{}", prefetch_enabled_message(depth));
200
+
201
+ let rc = thread::scope(|scope| {
202
+ let eval = scope.spawn(|| -> io::Result<()> {
203
+ while let Some(payload) = queue.pop() {
204
+ let reply = {
205
+ let mut w = worker.lock().expect("worker eval");
206
+ w.process_work(&payload, &mut MutexWrite(writer.as_ref()))?
207
+ };
208
+ if let Some(frame) = reply {
209
+ writer.lock().expect("worker write").write_all(&frame)?;
210
+ }
211
+ }
212
+ Ok(())
213
+ });
214
+
215
+ let mut loop_err: Option<io::Error> = None;
216
+ loop {
217
+ let (typ, body) = match read_frame(&mut reader) {
218
+ Ok(v) => v,
219
+ Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => break,
220
+ Err(e) => {
221
+ loop_err = Some(e);
222
+ break;
223
+ }
224
+ };
225
+ if typ == MSG_ERROR {
226
+ loop_err = Some(io::Error::new(
227
+ io::ErrorKind::Other,
228
+ format!("coordinator error: {}", String::from_utf8_lossy(&body)),
229
+ ));
230
+ break;
231
+ }
232
+ if typ == MSG_WORK {
233
+ if !queue.enqueue(body) {
234
+ loop_err = Some(io::Error::new(
235
+ io::ErrorKind::Interrupted,
236
+ "prefetch queue canceled",
237
+ ));
238
+ break;
239
+ }
240
+ continue;
241
+ }
242
+ if typ == MSG_SNAPSHOT_SAVE_REQ || typ == MSG_SNAPSHOT_LOAD_BEGIN {
243
+ let mut w = worker.lock().expect("worker snapshot");
244
+ let mut out = writer.lock().expect("worker snapshot");
245
+ w.apply_snapshot(&mut *out, typ, &body)?;
246
+ continue;
247
+ }
248
+ {
249
+ let mut out = writer.lock().expect("worker write");
250
+ write_frame(
251
+ &mut *out,
252
+ MSG_ERROR,
253
+ b"unsupported distributed worker frame",
254
+ )?;
255
+ }
256
+ loop_err = Some(io::Error::new(
257
+ io::ErrorKind::InvalidData,
258
+ format!("rejected unsupported frame type {typ}"),
259
+ ));
260
+ break;
261
+ }
262
+
263
+ if loop_err.is_some() {
264
+ queue.cancel();
265
+ } else {
266
+ queue.finish();
267
+ }
268
+ let eval_rc = eval.join().expect("prefetch eval");
269
+ match (loop_err, eval_rc) {
270
+ (Some(e), _) => Err(e),
271
+ (None, Err(e)) => Err(e),
272
+ (None, Ok(())) => Ok(()),
273
+ }
274
+ });
275
+ self.shutdown_hops();
276
+ rc
277
+ }
278
+ }
crates/ds4-dist/src/prefetch_local.rs ADDED
@@ -0,0 +1,253 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! !Send receive-prefetch: queue on a helper thread, eval on the session thread.
2
+
3
+ use std::io::{self, ErrorKind, Write};
4
+ use std::net::{Shutdown, TcpStream};
5
+ use std::sync::mpsc::{self, Receiver};
6
+ use std::sync::{Arc, Mutex};
7
+ use std::thread::{self, JoinHandle};
8
+
9
+ use crate::codec::{MSG_ERROR, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK};
10
+ use crate::exec::SliceExec;
11
+ use crate::native_snapshot::SnapshotStore;
12
+ use crate::prefetch::{prefetch_depth, prefetch_enabled_message, JobQueue, MutexWrite};
13
+ use crate::serve_local::{drive_ready, Drive, IDLE};
14
+ use crate::transport::{read_frame, write_frame};
15
+ use crate::worker::Worker;
16
+
17
+ pub enum PrefetchJob {
18
+ Work(Vec<u8>),
19
+ Snapshot { typ: u32, body: Vec<u8> },
20
+ }
21
+
22
+ pub fn local_prefetch_enabled_from(env: Option<&std::ffi::OsStr>) -> bool {
23
+ !crate::prefetch::prefetch_disabled_from(env)
24
+ }
25
+
26
+ pub fn local_prefetch_enabled() -> bool {
27
+ local_prefetch_enabled_from(std::env::var_os("DS4_DIST_DISABLE_WORKER_PREFETCH").as_deref())
28
+ }
29
+
30
+ fn recv_prefetch(
31
+ mut reader: TcpStream,
32
+ queue: &JobQueue<PrefetchJob>,
33
+ writer: &Mutex<TcpStream>,
34
+ mut on_job: impl FnMut(),
35
+ ) -> Option<io::Error> {
36
+ let err = loop {
37
+ match read_frame(&mut reader) {
38
+ Ok((typ, body)) if typ == MSG_WORK => {
39
+ if !queue.enqueue(PrefetchJob::Work(body)) {
40
+ break Some(io::Error::new(
41
+ ErrorKind::Interrupted,
42
+ "prefetch queue canceled",
43
+ ));
44
+ }
45
+ on_job();
46
+ }
47
+ Ok((typ, body)) if typ == MSG_SNAPSHOT_SAVE_REQ || typ == MSG_SNAPSHOT_LOAD_BEGIN => {
48
+ if !queue.enqueue(PrefetchJob::Snapshot { typ, body }) {
49
+ break Some(io::Error::new(
50
+ ErrorKind::Interrupted,
51
+ "prefetch queue canceled",
52
+ ));
53
+ }
54
+ on_job();
55
+ }
56
+ Ok((typ, body)) if typ == MSG_ERROR => {
57
+ break Some(io::Error::new(
58
+ ErrorKind::Other,
59
+ format!("coordinator error: {}", String::from_utf8_lossy(&body)),
60
+ ));
61
+ }
62
+ Ok((typ, _)) => {
63
+ let _ = write_frame(
64
+ &mut *writer.lock().expect("worker write"),
65
+ MSG_ERROR,
66
+ b"unsupported distributed worker frame",
67
+ );
68
+ break Some(io::Error::new(
69
+ ErrorKind::InvalidData,
70
+ format!("rejected unsupported frame type {typ}"),
71
+ ));
72
+ }
73
+ Err(e) if e.kind() == ErrorKind::UnexpectedEof => break None,
74
+ Err(e)
75
+ if matches!(
76
+ e.kind(),
77
+ ErrorKind::TimedOut | ErrorKind::WouldBlock | ErrorKind::Interrupted
78
+ ) =>
79
+ {
80
+ continue;
81
+ }
82
+ Err(e) => break Some(e),
83
+ }
84
+ };
85
+ if err.is_some() {
86
+ queue.cancel();
87
+ } else {
88
+ queue.finish();
89
+ }
90
+ on_job();
91
+ err
92
+ }
93
+
94
+ fn apply_prefetch_job<E, S>(
95
+ worker: &mut Worker<E, S>,
96
+ writer: &Arc<Mutex<TcpStream>>,
97
+ job: PrefetchJob,
98
+ ) -> io::Result<()>
99
+ where
100
+ E: SliceExec,
101
+ S: SnapshotStore,
102
+ {
103
+ match job {
104
+ PrefetchJob::Work(payload) => {
105
+ let reply = worker.process_work(&payload, &mut MutexWrite(writer))?;
106
+ if let Some(frame) = reply {
107
+ writer.lock().expect("worker write").write_all(&frame)?;
108
+ }
109
+ Ok(())
110
+ }
111
+ PrefetchJob::Snapshot { typ, body } => {
112
+ let mut out = writer.lock().expect("worker snapshot");
113
+ worker.apply_snapshot(&mut *out, typ, &body)
114
+ }
115
+ }
116
+ }
117
+
118
+ pub fn serve_prefetch_local_with<E, S>(
119
+ worker: &mut Worker<E, S>,
120
+ stream: &mut TcpStream,
121
+ queue: &JobQueue<PrefetchJob>,
122
+ ) -> io::Result<()>
123
+ where
124
+ E: SliceExec,
125
+ S: SnapshotStore,
126
+ {
127
+ let writer = Arc::new(Mutex::new(stream.try_clone()?));
128
+ let reader = stream.try_clone()?;
129
+ worker.bind_hops(Arc::clone(&writer));
130
+ eprintln!("{}", prefetch_enabled_message(queue.depth()));
131
+ let rc = thread::scope(|scope| {
132
+ let recv = scope.spawn(|| recv_prefetch(reader, queue, &writer, || {}));
133
+ let mut eval_err = None;
134
+ while let Some(job) = queue.pop() {
135
+ if let Err(e) = apply_prefetch_job(worker, &writer, job) {
136
+ eval_err = Some(e);
137
+ queue.cancel();
138
+ break;
139
+ }
140
+ }
141
+ match (recv.join().expect("prefetch recv"), eval_err) {
142
+ (Some(e), _) => Err(e),
143
+ (None, Some(e)) => Err(e),
144
+ (None, None) => Ok(()),
145
+ }
146
+ });
147
+ worker.shutdown_hops();
148
+ rc
149
+ }
150
+
151
+ struct PrefetchConn {
152
+ queue: Arc<JobQueue<PrefetchJob>>,
153
+ writer: Arc<Mutex<TcpStream>>,
154
+ recv: Option<JoinHandle<()>>,
155
+ }
156
+
157
+ fn start_prefetch_recv(
158
+ stream: TcpStream,
159
+ wake: mpsc::Sender<usize>,
160
+ idx: usize,
161
+ ) -> io::Result<PrefetchConn> {
162
+ let _ = stream.set_nodelay(true);
163
+ let writer = Arc::new(Mutex::new(stream.try_clone()?));
164
+ let queue = Arc::new(JobQueue::<PrefetchJob>::new(prefetch_depth()));
165
+ let q = Arc::clone(&queue);
166
+ let w = Arc::clone(&writer);
167
+ eprintln!("{}", prefetch_enabled_message(queue.depth()));
168
+ let recv = thread::Builder::new()
169
+ .name("ds4-dist-prefetch-recv".into())
170
+ .spawn(move || {
171
+ let _ = recv_prefetch(stream, &q, &w, || {
172
+ let _ = wake.send(idx);
173
+ });
174
+ })?;
175
+ Ok(PrefetchConn {
176
+ queue,
177
+ writer,
178
+ recv: Some(recv),
179
+ })
180
+ }
181
+
182
+ fn shutdown_prefetch_conns(conns: &mut [PrefetchConn]) {
183
+ for conn in conns.iter() {
184
+ conn.queue.cancel();
185
+ let _ = conn
186
+ .writer
187
+ .lock()
188
+ .expect("worker write")
189
+ .shutdown(Shutdown::Both);
190
+ }
191
+ for conn in conns.iter_mut() {
192
+ if let Some(handle) = conn.recv.take() {
193
+ let _ = handle.join();
194
+ }
195
+ }
196
+ }
197
+
198
+ fn drain_conn<E, S>(worker: &mut Worker<E, S>, conn: &PrefetchConn) -> io::Result<()>
199
+ where
200
+ E: SliceExec,
201
+ S: SnapshotStore,
202
+ {
203
+ worker.bind_hops(Arc::clone(&conn.writer));
204
+ while let Some(job) = conn.queue.try_pop() {
205
+ apply_prefetch_job(worker, &conn.writer, job)?;
206
+ }
207
+ Ok(())
208
+ }
209
+
210
+ pub(crate) fn serve_coordinator_and_hops_prefetch<E, S, Stop>(
211
+ worker: &mut Worker<E, S>,
212
+ coordinator: &mut TcpStream,
213
+ hops: Option<&Receiver<TcpStream>>,
214
+ should_stop: &mut Stop,
215
+ ) -> io::Result<()>
216
+ where
217
+ E: SliceExec,
218
+ S: SnapshotStore,
219
+ Stop: FnMut() -> bool,
220
+ {
221
+ coordinator.set_read_timeout(Some(IDLE))?;
222
+ let (wake_tx, wake_rx) = mpsc::channel();
223
+ let mut conns: Vec<PrefetchConn> = Vec::new();
224
+ let rc = 'run: loop {
225
+ if should_stop() {
226
+ break Ok(());
227
+ }
228
+ if let Some(rx) = hops {
229
+ while let Ok(hop) = rx.try_recv() {
230
+ let idx = conns.len();
231
+ match start_prefetch_recv(hop, wake_tx.clone(), idx) {
232
+ Ok(conn) => conns.push(conn),
233
+ Err(e) => break 'run Err(e),
234
+ }
235
+ }
236
+ }
237
+ while let Ok(idx) = wake_rx.try_recv() {
238
+ if let Some(conn) = conns.get(idx) {
239
+ if let Err(e) = drain_conn(worker, conn) {
240
+ break 'run Err(e);
241
+ }
242
+ }
243
+ }
244
+ match drive_ready(worker, coordinator) {
245
+ Drive::Continue => {}
246
+ Drive::Closed => break Ok(()),
247
+ Drive::Failed(e) => break Err(e),
248
+ }
249
+ };
250
+ shutdown_prefetch_conns(&mut conns);
251
+ worker.shutdown_hops();
252
+ rc
253
+ }
crates/ds4-dist/src/reconnect.rs ADDED
@@ -0,0 +1,217 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Worker reconnect: connect retries, HELLO, session clear, 1s backoff.
2
+
3
+ use std::io::{self, ErrorKind};
4
+ use std::net::{SocketAddr, TcpListener, TcpStream};
5
+ use std::thread;
6
+ use std::time::Duration;
7
+
8
+ use crate::codec::Hello;
9
+ use crate::exec::SliceExec;
10
+ use crate::native_snapshot::SnapshotStore;
11
+ use crate::prefetch::prefetch_disabled;
12
+ use crate::prefetch_local::{local_prefetch_enabled, serve_coordinator_and_hops_prefetch};
13
+ use crate::serve_local::{serve_coordinator_and_hops, spawn_hop_accept, HopAccept};
14
+ use crate::worker::{send_hello, Worker};
15
+
16
+ pub const RECONNECT_SLEEP: Duration = Duration::from_secs(1);
17
+ pub const CONNECT_RETRY_DELAY: Duration = Duration::from_millis(25);
18
+ pub const CONNECT_RETRY_ATTEMPTS: u32 = 200;
19
+
20
+ pub fn sleep_reconnect() {
21
+ thread::sleep(RECONNECT_SLEEP);
22
+ }
23
+
24
+ pub fn connect_retryable(err: &io::Error) -> bool {
25
+ matches!(
26
+ err.kind(),
27
+ ErrorKind::ConnectionRefused
28
+ | ErrorKind::HostUnreachable
29
+ | ErrorKind::NetworkUnreachable
30
+ | ErrorKind::TimedOut
31
+ | ErrorKind::AddrNotAvailable
32
+ )
33
+ }
34
+
35
+ pub fn connect_error(host: &str, port: u16, err: &io::Error) -> String {
36
+ format!("unable to connect to {host}:{port}: {err}")
37
+ }
38
+
39
+ pub fn retrying_message(err: &str) -> String {
40
+ format!("ds4: distributed worker: {err}; retrying")
41
+ }
42
+
43
+ pub fn connected_message(host: &str, port: &str) -> String {
44
+ format!("ds4: distributed worker: connected to coordinator {host}:{port}")
45
+ }
46
+
47
+ pub fn hello_failed_message(err: &io::Error) -> String {
48
+ format!("ds4: distributed worker: failed to send HELLO: {err}")
49
+ }
50
+
51
+ pub fn cleared_sessions_message(n: u32) -> String {
52
+ format!("ds4: distributed worker: cleared {n} sessions after coordinator disconnect")
53
+ }
54
+
55
+ pub fn disconnected_message(after_error: bool) -> String {
56
+ if after_error {
57
+ "ds4: distributed worker: coordinator disconnected after error; reconnecting".into()
58
+ } else {
59
+ "ds4: distributed worker: coordinator disconnected; reconnecting".into()
60
+ }
61
+ }
62
+
63
+ pub fn connect_endpoint_once(host: &str, port: u16) -> io::Result<TcpStream> {
64
+ let stream = TcpStream::connect((host, port))?;
65
+ let _ = stream.set_nodelay(true);
66
+ Ok(stream)
67
+ }
68
+
69
+ pub fn connect_endpoint(host: &str, port: u16) -> io::Result<TcpStream> {
70
+ let mut last = None;
71
+ for _ in 0..CONNECT_RETRY_ATTEMPTS {
72
+ match connect_endpoint_once(host, port) {
73
+ Ok(s) => return Ok(s),
74
+ Err(e) => {
75
+ let retry = connect_retryable(&e);
76
+ last = Some(e);
77
+ if !retry {
78
+ break;
79
+ }
80
+ thread::sleep(CONNECT_RETRY_DELAY);
81
+ }
82
+ }
83
+ }
84
+ Err(last.unwrap_or_else(|| io::Error::new(ErrorKind::Other, "unable to connect")))
85
+ }
86
+
87
+ pub fn peer_name(stream: &TcpStream) -> (String, String) {
88
+ match stream.peer_addr() {
89
+ Ok(SocketAddr::V4(a)) => (a.ip().to_string(), a.port().to_string()),
90
+ Ok(SocketAddr::V6(a)) => (a.ip().to_string(), a.port().to_string()),
91
+ Err(_) => ("?".into(), "?".into()),
92
+ }
93
+ }
94
+
95
+ pub struct LocalReconnect<'a, Connect, Sleep, Stop> {
96
+ pub connect: Connect,
97
+ pub hello: &'a Hello,
98
+ pub model_name: &'a str,
99
+ pub sleep: Sleep,
100
+ pub should_stop: Stop,
101
+ pub listener: Option<&'a TcpListener>,
102
+ }
103
+
104
+ fn reconnect_loop<E, S, Connect, Sleep, Stop, Serve>(
105
+ worker: &mut Worker<E, S>,
106
+ connect: &mut Connect,
107
+ hello: &Hello,
108
+ model_name: &str,
109
+ sleep: &mut Sleep,
110
+ should_stop: &mut Stop,
111
+ mut serve: Serve,
112
+ ) -> io::Result<()>
113
+ where
114
+ E: SliceExec,
115
+ S: SnapshotStore,
116
+ Connect: FnMut() -> io::Result<TcpStream>,
117
+ Sleep: FnMut(),
118
+ Stop: FnMut() -> bool,
119
+ Serve: FnMut(&mut Worker<E, S>, &mut TcpStream, &mut Stop) -> io::Result<()>,
120
+ {
121
+ while !should_stop() {
122
+ let mut stream = match connect() {
123
+ Ok(s) => s,
124
+ Err(e) => {
125
+ eprintln!("{}", retrying_message(&e.to_string()));
126
+ sleep();
127
+ continue;
128
+ }
129
+ };
130
+ let (peer_host, peer_port) = peer_name(&stream);
131
+ eprintln!("{}", connected_message(&peer_host, &peer_port));
132
+ if let Err(e) = send_hello(&mut stream, hello, model_name) {
133
+ eprintln!("{}", hello_failed_message(&e));
134
+ sleep();
135
+ continue;
136
+ }
137
+ let rc = serve(worker, &mut stream, should_stop);
138
+ let after_error = rc.is_err();
139
+ let dropped = worker.clear_sessions();
140
+ if dropped != 0 {
141
+ eprintln!("{}", cleared_sessions_message(dropped));
142
+ }
143
+ eprintln!("{}", disconnected_message(after_error));
144
+ if should_stop() {
145
+ return rc;
146
+ }
147
+ sleep();
148
+ if after_error {
149
+ let _ = rc;
150
+ }
151
+ }
152
+ Ok(())
153
+ }
154
+
155
+ pub fn reconnect_with<E, S, Connect, Sleep, Stop>(
156
+ worker: &mut Worker<E, S>,
157
+ mut connect: Connect,
158
+ hello: &Hello,
159
+ model_name: &str,
160
+ mut sleep: Sleep,
161
+ mut should_stop: Stop,
162
+ use_prefetch: bool,
163
+ ) -> io::Result<()>
164
+ where
165
+ E: SliceExec + Send,
166
+ S: SnapshotStore + Send,
167
+ Connect: FnMut() -> io::Result<TcpStream>,
168
+ Sleep: FnMut(),
169
+ Stop: FnMut() -> bool,
170
+ {
171
+ reconnect_loop(
172
+ worker,
173
+ &mut connect,
174
+ hello,
175
+ model_name,
176
+ &mut sleep,
177
+ &mut should_stop,
178
+ |w, stream, _stop| {
179
+ if use_prefetch && !prefetch_disabled() {
180
+ w.serve_prefetch(stream)
181
+ } else {
182
+ w.serve(stream)
183
+ }
184
+ },
185
+ )
186
+ }
187
+
188
+ /// Reconnect a `!Send` worker, polling the coordinator and accepted hop streams.
189
+ pub fn reconnect_local<E, S, Connect, Sleep, Stop>(
190
+ worker: &mut Worker<E, S>,
191
+ mut spec: LocalReconnect<'_, Connect, Sleep, Stop>,
192
+ ) -> io::Result<()>
193
+ where
194
+ E: SliceExec,
195
+ S: SnapshotStore,
196
+ Connect: FnMut() -> io::Result<TcpStream>,
197
+ Sleep: FnMut(),
198
+ Stop: FnMut() -> bool,
199
+ {
200
+ let hops = spec.listener.map(spawn_hop_accept).transpose()?;
201
+ let rx = hops.as_ref().and_then(HopAccept::receiver);
202
+ reconnect_loop(
203
+ worker,
204
+ &mut spec.connect,
205
+ spec.hello,
206
+ spec.model_name,
207
+ &mut spec.sleep,
208
+ &mut spec.should_stop,
209
+ |w, stream, stop| {
210
+ if local_prefetch_enabled() {
211
+ serve_coordinator_and_hops_prefetch(w, stream, rx, stop)
212
+ } else {
213
+ serve_coordinator_and_hops(w, stream, rx, stop)
214
+ }
215
+ },
216
+ )
217
+ }
crates/ds4-dist/src/relay.rs ADDED
@@ -0,0 +1,217 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Persistent worker-to-worker forwarder and RESULT relay thread.
2
+
3
+ use std::io::{self, Write};
4
+ use std::net::TcpStream;
5
+ use std::sync::atomic::{AtomicBool, Ordering};
6
+ use std::sync::{Arc, Mutex};
7
+ use std::thread::{self, JoinHandle};
8
+ use std::time::Instant;
9
+
10
+ use crate::codec::{Telemetry, MSG_RESULT};
11
+ use crate::forward::{
12
+ opened_forwarder_message, PendingQueue, PendingRequest, ERR_FORWARD, ERR_INVALID_RESULT,
13
+ ERR_NEXT_CLOSED, ERR_RESULT_METADATA, ERR_RESULT_TOO_LARGE, ERR_TELEMETRY_TOO_LARGE,
14
+ };
15
+ use crate::reconnect::connect_endpoint;
16
+ use crate::route::RouteEntry;
17
+ use crate::transport::read_frame;
18
+ use crate::work::{
19
+ decode_result_body, encode_result_frame, encode_work_frame, result_request_id, WorkBody,
20
+ };
21
+
22
+ pub fn now_sec() -> f64 {
23
+ static START: std::sync::OnceLock<Instant> = std::sync::OnceLock::new();
24
+ START.get_or_init(Instant::now).elapsed().as_secs_f64()
25
+ }
26
+
27
+ pub fn local_work_telemetry(
28
+ work: &crate::codec::Work,
29
+ eval_usec: u32,
30
+ output_bytes: u32,
31
+ ) -> Telemetry {
32
+ Telemetry {
33
+ layer_start: work.layer_start,
34
+ layer_end: work.layer_end,
35
+ route_index: work.route_index,
36
+ pos0: work.pos0,
37
+ n_tokens: work.n_tokens,
38
+ eval_usec,
39
+ downstream_wait_usec: 0,
40
+ forward_send_usec: 0,
41
+ input_bytes: work.token_bytes.saturating_add(work.input_hc_bytes),
42
+ output_bytes,
43
+ }
44
+ }
45
+
46
+ pub fn usec_since(t0: f64, t1: f64) -> u32 {
47
+ if t1 <= t0 {
48
+ return 0;
49
+ }
50
+ let usec = (t1 - t0) * 1_000_000.0;
51
+ if usec >= f64::from(u32::MAX) {
52
+ u32::MAX
53
+ } else {
54
+ (usec + 0.5) as u32
55
+ }
56
+ }
57
+
58
+ pub fn prepend_telemetry(result_payload: &[u8], local: Telemetry) -> Result<Vec<u8>, &'static str> {
59
+ let mut body = decode_result_body(result_payload).map_err(|_| ERR_RESULT_METADATA)?;
60
+ if body.telemetry.len() >= u32::MAX as usize {
61
+ return Err(ERR_TELEMETRY_TOO_LARGE);
62
+ }
63
+ body.telemetry.push(local);
64
+ encode_result_frame(&body).map_err(|_| ERR_RESULT_TOO_LARGE)
65
+ }
66
+
67
+ pub struct Forwarder {
68
+ pub host: String,
69
+ pub port: u32,
70
+ writer: Mutex<TcpStream>,
71
+ queue: Arc<PendingQueue>,
72
+ stop: Arc<AtomicBool>,
73
+ relay: Option<JoinHandle<()>>,
74
+ }
75
+
76
+ impl Forwarder {
77
+ pub fn connect(
78
+ host: &str,
79
+ port: u32,
80
+ window: u32,
81
+ on_result: impl Fn(Vec<u8>) + Send + 'static,
82
+ on_error: impl Fn(u64, &'static str) + Send + 'static,
83
+ ) -> Result<Self, String> {
84
+ let stream = connect_endpoint(host, port as u16).map_err(|e| e.to_string())?;
85
+ let mut reader = stream.try_clone().map_err(|e| e.to_string())?;
86
+ let queue = Arc::new(PendingQueue::new(window));
87
+ let stop = Arc::new(AtomicBool::new(false));
88
+ let relay_q = Arc::clone(&queue);
89
+ let relay_stop = Arc::clone(&stop);
90
+ let relay = thread::Builder::new()
91
+ .name(format!("ds4-dist-relay-{host}:{port}"))
92
+ .spawn(move || relay_main(&mut reader, &relay_q, &relay_stop, on_result, on_error))
93
+ .map_err(|e| e.to_string())?;
94
+ eprintln!("{}", opened_forwarder_message(host, port, window));
95
+ Ok(Self {
96
+ host: host.to_string(),
97
+ port,
98
+ writer: Mutex::new(stream),
99
+ queue,
100
+ stop,
101
+ relay: Some(relay),
102
+ })
103
+ }
104
+
105
+ pub fn send_work(&self, request: PendingRequest, frame: &[u8]) -> Result<(), &'static str> {
106
+ let request_id = request.request_id;
107
+ self.queue.enqueue(request)?;
108
+ let send_t0 = now_sec();
109
+ let write_rc = self.writer.lock().expect("forward send").write_all(frame);
110
+ let send_t1 = now_sec();
111
+ self.queue
112
+ .note_send_done(request_id, usec_since(send_t0, send_t1), send_t1);
113
+ if write_rc.is_err() {
114
+ self.queue.remove(request_id);
115
+ let _ = self
116
+ .writer
117
+ .lock()
118
+ .expect("forward send")
119
+ .shutdown(std::net::Shutdown::Both);
120
+ return Err(ERR_FORWARD);
121
+ }
122
+ Ok(())
123
+ }
124
+
125
+ pub fn shutdown(&mut self) {
126
+ self.stop.store(true, Ordering::SeqCst);
127
+ self.queue.close();
128
+ if let Ok(w) = self.writer.lock() {
129
+ let _ = w.shutdown(std::net::Shutdown::Both);
130
+ }
131
+ if let Some(relay) = self.relay.take() {
132
+ let _ = relay.join();
133
+ }
134
+ self.queue.clear();
135
+ }
136
+ }
137
+
138
+ impl Drop for Forwarder {
139
+ fn drop(&mut self) {
140
+ self.shutdown();
141
+ }
142
+ }
143
+
144
+ fn relay_main(
145
+ reader: &mut TcpStream,
146
+ queue: &PendingQueue,
147
+ stop: &AtomicBool,
148
+ on_result: impl Fn(Vec<u8>),
149
+ on_error: impl Fn(u64, &'static str),
150
+ ) {
151
+ while !stop.load(Ordering::Relaxed) {
152
+ let (typ, body) = match read_frame(reader) {
153
+ Ok(v) => v,
154
+ Err(e) if e.kind() == io::ErrorKind::UnexpectedEof || stop.load(Ordering::Relaxed) => {
155
+ if let Some(pending) = queue.pop() {
156
+ on_error(pending.request_id, ERR_NEXT_CLOSED);
157
+ }
158
+ break;
159
+ }
160
+ Err(_) => {
161
+ if let Some(pending) = queue.pop() {
162
+ on_error(pending.request_id, ERR_NEXT_CLOSED);
163
+ }
164
+ break;
165
+ }
166
+ };
167
+ if typ != MSG_RESULT {
168
+ if let Some(pending) = queue.pop() {
169
+ on_error(pending.request_id, ERR_INVALID_RESULT);
170
+ }
171
+ break;
172
+ }
173
+ let decoded = match decode_result_body(&body) {
174
+ Ok(v) => v,
175
+ Err(_) => {
176
+ if let Some(pending) = queue.pop() {
177
+ on_error(pending.request_id, ERR_RESULT_METADATA);
178
+ }
179
+ break;
180
+ }
181
+ };
182
+ let Some(mut pending) = queue.pop() else {
183
+ break;
184
+ };
185
+ if result_request_id(&decoded.hdr) != pending.request_id {
186
+ on_error(pending.request_id, ERR_RESULT_METADATA);
187
+ break;
188
+ }
189
+ pending.telemetry.downstream_wait_usec = usec_since(pending.downstream_t0, now_sec());
190
+ match prepend_telemetry(&body, pending.telemetry) {
191
+ Ok(frame) => on_result(frame),
192
+ Err(msg) => on_error(pending.request_id, msg),
193
+ }
194
+ }
195
+ queue.close();
196
+ }
197
+
198
+ pub fn forward_work_blocking<W: Write>(
199
+ next: &RouteEntry,
200
+ body: &WorkBody,
201
+ upstream: &mut W,
202
+ mut local: Telemetry,
203
+ ) -> Result<(), String> {
204
+ let mut hop = connect_endpoint(&next.host, next.port as u16).map_err(|_| ERR_FORWARD)?;
205
+ let frame = encode_work_frame(body).map_err(|e| e.to_string())?;
206
+ let send_t0 = now_sec();
207
+ hop.write_all(&frame).map_err(|_| ERR_FORWARD)?;
208
+ let send_t1 = now_sec();
209
+ local.forward_send_usec = usec_since(send_t0, send_t1);
210
+ let (typ, reply) = read_frame(&mut hop).map_err(|_| ERR_FORWARD)?;
211
+ if typ != MSG_RESULT {
212
+ return Err(ERR_INVALID_RESULT.into());
213
+ }
214
+ local.downstream_wait_usec = usec_since(send_t1, now_sec());
215
+ let prepended = prepend_telemetry(&reply, local).map_err(|e| e.to_string())?;
216
+ upstream.write_all(&prepended).map_err(|e| e.to_string())
217
+ }
crates/ds4-dist/src/route.rs ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Route blob: `[Route + host]*` then `RouteReturn + host`. No struct memcpy.
2
+
3
+ use crate::codec::{
4
+ bytes_have_nul, CodecError, Route, RouteReturn, NI_MAXHOST, ROUTE_FIXED_BYTES,
5
+ ROUTE_F_OUTPUT_LOGITS, ROUTE_RETURN_FIXED_BYTES, ROUTE_RETURN_UPSTREAM,
6
+ };
7
+
8
+ #[derive(Debug, Clone, PartialEq, Eq)]
9
+ pub struct RouteEntry {
10
+ pub host: String,
11
+ pub port: u32,
12
+ pub layer_start: u32,
13
+ pub layer_end: u32,
14
+ pub flags: u32,
15
+ }
16
+
17
+ #[derive(Debug, Clone, PartialEq, Eq)]
18
+ pub struct ReturnTarget {
19
+ pub kind: u32,
20
+ pub host: String,
21
+ pub port: u32,
22
+ }
23
+
24
+ pub fn encode_route_blob(
25
+ entries: &[RouteEntry],
26
+ ret: &ReturnTarget,
27
+ ) -> Result<Vec<u8>, CodecError> {
28
+ let mut out = Vec::new();
29
+ for e in entries {
30
+ let host = e.host.as_bytes();
31
+ if host.is_empty() || host.len() >= NI_MAXHOST || bytes_have_nul(host) {
32
+ return Err(CodecError::Invalid("invalid route host length"));
33
+ }
34
+ let rec = Route {
35
+ host_len: host.len() as u32,
36
+ port: e.port,
37
+ layer_start: e.layer_start,
38
+ layer_end: e.layer_end,
39
+ flags: e.flags,
40
+ };
41
+ out.extend_from_slice(&rec.encode());
42
+ out.extend_from_slice(host);
43
+ }
44
+ let rhost = ret.host.as_bytes();
45
+ if rhost.len() >= NI_MAXHOST || bytes_have_nul(rhost) {
46
+ return Err(CodecError::Invalid(
47
+ "invalid route final destination host length",
48
+ ));
49
+ }
50
+ let rec = RouteReturn {
51
+ kind: ret.kind,
52
+ host_len: rhost.len() as u32,
53
+ port: ret.port,
54
+ };
55
+ out.extend_from_slice(&rec.encode());
56
+ out.extend_from_slice(rhost);
57
+ Ok(out)
58
+ }
59
+
60
+ pub fn decode_route_blob(
61
+ blob: &[u8],
62
+ route_count: u32,
63
+ ) -> Result<(Vec<RouteEntry>, ReturnTarget), CodecError> {
64
+ let mut remaining = blob;
65
+ let mut entries = Vec::new();
66
+ for _ in 0..route_count {
67
+ if remaining.len() < ROUTE_FIXED_BYTES {
68
+ return Err(CodecError::Invalid("truncated route entry"));
69
+ }
70
+ let fixed = Route::decode(remaining)?;
71
+ remaining = &remaining[ROUTE_FIXED_BYTES..];
72
+ if fixed.host_len == 0
73
+ || fixed.host_len as usize >= NI_MAXHOST
74
+ || fixed.host_len as usize > remaining.len()
75
+ {
76
+ return Err(CodecError::Invalid("invalid route host length"));
77
+ }
78
+ let host = &remaining[..fixed.host_len as usize];
79
+ if bytes_have_nul(host) {
80
+ return Err(CodecError::Invalid("route host contains NUL bytes"));
81
+ }
82
+ entries.push(RouteEntry {
83
+ host: String::from_utf8_lossy(host).into_owned(),
84
+ port: fixed.port,
85
+ layer_start: fixed.layer_start,
86
+ layer_end: fixed.layer_end,
87
+ flags: fixed.flags,
88
+ });
89
+ remaining = &remaining[fixed.host_len as usize..];
90
+ }
91
+ if remaining.len() < ROUTE_RETURN_FIXED_BYTES {
92
+ return Err(CodecError::Invalid(
93
+ "route payload missing final destination",
94
+ ));
95
+ }
96
+ let ret = RouteReturn::decode(remaining)?;
97
+ remaining = &remaining[ROUTE_RETURN_FIXED_BYTES..];
98
+ if ret.host_len as usize >= NI_MAXHOST || ret.host_len as usize > remaining.len() {
99
+ return Err(CodecError::Invalid(
100
+ "invalid route final destination host length",
101
+ ));
102
+ }
103
+ let host = &remaining[..ret.host_len as usize];
104
+ if bytes_have_nul(host) {
105
+ return Err(CodecError::Invalid(
106
+ "route final destination host contains NUL bytes",
107
+ ));
108
+ }
109
+ remaining = &remaining[ret.host_len as usize..];
110
+ if !remaining.is_empty() {
111
+ return Err(CodecError::Invalid("route payload has trailing bytes"));
112
+ }
113
+ Ok((
114
+ entries,
115
+ ReturnTarget {
116
+ kind: ret.kind,
117
+ host: String::from_utf8_lossy(host).into_owned(),
118
+ port: ret.port,
119
+ },
120
+ ))
121
+ }
122
+
123
+ pub fn validate_route_blob(blob: &[u8], route_count: u32, n_layers: u32) -> Result<(), CodecError> {
124
+ if route_count == 0 {
125
+ return if blob.is_empty() {
126
+ Ok(())
127
+ } else {
128
+ Err(CodecError::Invalid(
129
+ "route payload has entries without a route count",
130
+ ))
131
+ };
132
+ }
133
+ let (entries, ret) = decode_route_blob(blob, route_count)?;
134
+ let mut prev_end = u32::MAX;
135
+ for (i, e) in entries.iter().enumerate() {
136
+ if e.port == 0 || e.port > 65535 {
137
+ return Err(CodecError::Invalid("invalid route port"));
138
+ }
139
+ if e.layer_start >= n_layers || e.layer_end >= n_layers || e.layer_end < e.layer_start {
140
+ return Err(CodecError::Invalid("invalid route layer range"));
141
+ }
142
+ if (e.flags & !ROUTE_F_OUTPUT_LOGITS) != 0 {
143
+ return Err(CodecError::Invalid("invalid route flags"));
144
+ }
145
+ if (e.flags & ROUTE_F_OUTPUT_LOGITS) != 0 && e.layer_end + 1 != n_layers {
146
+ return Err(CodecError::Invalid("route logits require final layer"));
147
+ }
148
+ if i != 0 && e.layer_start != prev_end + 1 {
149
+ return Err(CodecError::Invalid("route layer ranges are not contiguous"));
150
+ }
151
+ prev_end = e.layer_end;
152
+ }
153
+ if ret.kind != ROUTE_RETURN_UPSTREAM {
154
+ return Err(CodecError::Invalid("unsupported route final destination"));
155
+ }
156
+ if !ret.host.is_empty() || ret.port != 0 {
157
+ return Err(CodecError::Invalid(
158
+ "invalid upstream route final destination",
159
+ ));
160
+ }
161
+ Ok(())
162
+ }
crates/ds4-dist/src/serve_local.rs ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Same-thread coordinator + hop mux. Accept thread only accepts.
2
+
3
+ use std::io::{self, ErrorKind};
4
+ use std::net::{SocketAddr, TcpListener, TcpStream};
5
+ use std::sync::mpsc::{self, Receiver};
6
+ use std::sync::{Arc, Mutex};
7
+ use std::thread::{self, JoinHandle};
8
+ use std::time::Duration;
9
+
10
+ use crate::coordinator::accept_data_client;
11
+ use crate::exec::SliceExec;
12
+ use crate::native_snapshot::SnapshotStore;
13
+ use crate::worker::Worker;
14
+
15
+ pub(crate) const IDLE: Duration = Duration::from_millis(25);
16
+
17
+ pub(crate) struct HopAccept {
18
+ rx: Option<Receiver<TcpStream>>,
19
+ wakeup: SocketAddr,
20
+ handle: Option<JoinHandle<()>>,
21
+ }
22
+
23
+ impl HopAccept {
24
+ pub(crate) fn receiver(&self) -> Option<&Receiver<TcpStream>> {
25
+ self.rx.as_ref()
26
+ }
27
+ }
28
+
29
+ impl Drop for HopAccept {
30
+ fn drop(&mut self) {
31
+ self.rx.take();
32
+ let _ = TcpStream::connect(self.wakeup);
33
+ if let Some(handle) = self.handle.take() {
34
+ let _ = handle.join();
35
+ }
36
+ }
37
+ }
38
+
39
+ pub(crate) fn spawn_hop_accept(listener: &TcpListener) -> io::Result<HopAccept> {
40
+ let listener = listener.try_clone()?;
41
+ let wakeup = listener.local_addr()?;
42
+ let (tx, rx) = mpsc::channel();
43
+ let handle = thread::Builder::new()
44
+ .name("ds4-dist-hop-accept".into())
45
+ .spawn(move || loop {
46
+ match accept_data_client(&listener) {
47
+ Ok(stream) => {
48
+ if tx.send(stream).is_err() {
49
+ break;
50
+ }
51
+ }
52
+ Err(_) => break,
53
+ }
54
+ })?;
55
+ Ok(HopAccept {
56
+ rx: Some(rx),
57
+ wakeup,
58
+ handle: Some(handle),
59
+ })
60
+ }
61
+
62
+ pub(crate) enum Drive {
63
+ Continue,
64
+ Closed,
65
+ Failed(io::Error),
66
+ }
67
+
68
+ pub(crate) fn drive_ready<E, S>(worker: &mut Worker<E, S>, stream: &mut TcpStream) -> Drive
69
+ where
70
+ E: SliceExec,
71
+ S: SnapshotStore,
72
+ {
73
+ match worker.serve_once(stream) {
74
+ Ok(()) => Drive::Continue,
75
+ Err(e) => match e.kind() {
76
+ ErrorKind::TimedOut | ErrorKind::WouldBlock | ErrorKind::Interrupted => Drive::Continue,
77
+ ErrorKind::UnexpectedEof | ErrorKind::InvalidData => Drive::Closed,
78
+ _ => Drive::Failed(e),
79
+ },
80
+ }
81
+ }
82
+
83
+ pub(crate) fn serve_coordinator_and_hops<E, S, Stop>(
84
+ worker: &mut Worker<E, S>,
85
+ coordinator: &mut TcpStream,
86
+ hops: Option<&Receiver<TcpStream>>,
87
+ should_stop: &mut Stop,
88
+ ) -> io::Result<()>
89
+ where
90
+ E: SliceExec,
91
+ S: SnapshotStore,
92
+ Stop: FnMut() -> bool,
93
+ {
94
+ coordinator.set_read_timeout(Some(IDLE))?;
95
+ let mut accepted = Vec::new();
96
+ let rc = loop {
97
+ if should_stop() {
98
+ break Ok(());
99
+ }
100
+ if let Some(rx) = hops {
101
+ while let Ok(hop) = rx.try_recv() {
102
+ let _ = hop.set_nodelay(true);
103
+ let _ = hop.set_read_timeout(Some(IDLE));
104
+ accepted.push(hop);
105
+ }
106
+ }
107
+ let mut i = 0;
108
+ while i < accepted.len() {
109
+ if let Ok(clone) = accepted[i].try_clone() {
110
+ worker.bind_hops(Arc::new(Mutex::new(clone)));
111
+ }
112
+ match drive_ready(worker, &mut accepted[i]) {
113
+ Drive::Continue => i += 1,
114
+ Drive::Closed | Drive::Failed(_) => {
115
+ accepted.swap_remove(i);
116
+ }
117
+ }
118
+ }
119
+ match drive_ready(worker, coordinator) {
120
+ Drive::Continue => {}
121
+ Drive::Closed => break Ok(()),
122
+ Drive::Failed(e) => break Err(e),
123
+ }
124
+ };
125
+ worker.shutdown_hops();
126
+ rc
127
+ }
crates/ds4-dist/src/snapshot.rs ADDED
@@ -0,0 +1,337 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Coordinator-side DS4D snapshot save/load transport.
2
+
3
+ use std::io::{self, Read, Write};
4
+
5
+ use crate::codec::{
6
+ decode_frame_header, encode_frame_header, u64_from_halves, u64_to_halves, FrameHeader,
7
+ SnapshotBegin, SnapshotChunk, SnapshotDone, SnapshotReq, FRAME_HEADER_BYTES,
8
+ MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK, MSG_SNAPSHOT_DONE, MSG_SNAPSHOT_LOAD_BEGIN,
9
+ MSG_SNAPSHOT_SAVE_REQ, SNAPSHOT_BEGIN_FIXED_BYTES, SNAPSHOT_CHUNK_BYTES,
10
+ SNAPSHOT_CHUNK_FIXED_BYTES, SNAPSHOT_DONE_FIXED_BYTES,
11
+ };
12
+ use crate::transport::write_frame;
13
+
14
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
15
+ pub struct SnapshotMeta {
16
+ pub model_id: u32,
17
+ pub session_id: u64,
18
+ pub request_id: u64,
19
+ pub token_hash: u64,
20
+ pub layer_start: u32,
21
+ pub layer_end: u32,
22
+ }
23
+
24
+ fn token_count(tokens: &[i32]) -> Result<u32, String> {
25
+ u32::try_from(tokens.len()).map_err(|_| "invalid distributed snapshot token length".to_string())
26
+ }
27
+
28
+ fn c_message(message: &[u8]) -> String {
29
+ let end = message
30
+ .iter()
31
+ .position(|&b| b == 0)
32
+ .unwrap_or(message.len());
33
+ String::from_utf8_lossy(&message[..end]).into_owned()
34
+ }
35
+
36
+ fn read_header<R: Read>(r: &mut R, closed: &'static str) -> Result<FrameHeader, String> {
37
+ let mut buf = [0; FRAME_HEADER_BYTES];
38
+ r.read_exact(&mut buf).map_err(|e| {
39
+ if e.kind() == io::ErrorKind::UnexpectedEof {
40
+ closed.to_string()
41
+ } else {
42
+ format!("failed to read frame header: {e}")
43
+ }
44
+ })?;
45
+ decode_frame_header(&buf).map_err(|e| e.to_string())
46
+ }
47
+
48
+ fn read_or<R: Read>(r: &mut R, buf: &mut [u8], message: &'static str) -> Result<(), String> {
49
+ r.read_exact(buf).map_err(|_| message.to_string())
50
+ }
51
+
52
+ fn discard<R: Read>(r: &mut R, mut bytes: u32) -> io::Result<()> {
53
+ let mut buf = [0; 4096];
54
+ while bytes != 0 {
55
+ let n = bytes.min(buf.len() as u32) as usize;
56
+ r.read_exact(&mut buf[..n])?;
57
+ bytes -= n as u32;
58
+ }
59
+ Ok(())
60
+ }
61
+
62
+ fn read_message<R: Read>(
63
+ r: &mut R,
64
+ bytes: u32,
65
+ read_error: &'static str,
66
+ discard_error: &'static str,
67
+ ) -> Result<Vec<u8>, String> {
68
+ let copied = bytes.min(255) as usize;
69
+ let mut message = vec![0; copied];
70
+ read_or(r, &mut message, read_error)?;
71
+ discard(r, bytes - copied as u32).map_err(|_| discard_error.to_string())?;
72
+ Ok(message)
73
+ }
74
+
75
+ fn read_begin<R: Read>(r: &mut R) -> Result<(SnapshotBegin, Vec<u8>), String> {
76
+ let header = read_header(r, "distributed worker closed snapshot connection")?;
77
+ if header.typ != MSG_SNAPSHOT_BEGIN || header.bytes < SNAPSHOT_BEGIN_FIXED_BYTES as u32 {
78
+ let _ = discard(r, header.bytes);
79
+ return Err("distributed worker returned invalid snapshot frame".into());
80
+ }
81
+
82
+ let mut fixed = [0; SNAPSHOT_BEGIN_FIXED_BYTES];
83
+ read_or(r, &mut fixed, "failed to read distributed snapshot header")?;
84
+ let begin = SnapshotBegin::decode(&fixed)
85
+ .map_err(|_| "invalid distributed snapshot response header".to_string())?;
86
+ let mut body = header.bytes - SNAPSHOT_BEGIN_FIXED_BYTES as u32;
87
+ let expected_token_bytes = u64::from(begin.token_count) * 4;
88
+ if expected_token_bytes > u64::from(u32::MAX)
89
+ || begin.token_bytes != expected_token_bytes as u32
90
+ || begin.token_bytes > body
91
+ || begin.message_bytes > body - begin.token_bytes
92
+ {
93
+ let _ = discard(r, body);
94
+ return Err("invalid distributed snapshot response header".into());
95
+ }
96
+
97
+ discard(r, begin.token_bytes)
98
+ .map_err(|_| "failed to discard distributed snapshot response tokens".to_string())?;
99
+ body -= begin.token_bytes;
100
+ let message = read_message(
101
+ r,
102
+ begin.message_bytes,
103
+ "failed to read distributed snapshot response message",
104
+ "failed to discard distributed snapshot response message",
105
+ )?;
106
+ body -= begin.message_bytes;
107
+ discard(r, body).map_err(|_| {
108
+ "failed to discard trailing distributed snapshot response bytes".to_string()
109
+ })?;
110
+ Ok((begin, message))
111
+ }
112
+
113
+ fn read_done<R: Read>(r: &mut R, request_id: u64) -> Result<(), String> {
114
+ let header = read_header(r, "distributed worker closed before snapshot completion")?;
115
+ if header.typ != MSG_SNAPSHOT_DONE || header.bytes < SNAPSHOT_DONE_FIXED_BYTES as u32 {
116
+ let _ = discard(r, header.bytes);
117
+ return Err("distributed worker returned invalid snapshot completion frame".into());
118
+ }
119
+
120
+ let mut fixed = [0; SNAPSHOT_DONE_FIXED_BYTES];
121
+ read_or(
122
+ r,
123
+ &mut fixed,
124
+ "failed to read distributed snapshot completion",
125
+ )?;
126
+ let done = SnapshotDone::decode(&fixed)
127
+ .map_err(|_| "invalid distributed snapshot completion message".to_string())?;
128
+ let mut body = header.bytes - SNAPSHOT_DONE_FIXED_BYTES as u32;
129
+ if done.message_bytes > body {
130
+ let _ = discard(r, body);
131
+ return Err("invalid distributed snapshot completion message".into());
132
+ }
133
+ let message = read_message(
134
+ r,
135
+ done.message_bytes,
136
+ "failed to read distributed snapshot completion message",
137
+ "failed to discard distributed snapshot completion message",
138
+ )?;
139
+ body -= done.message_bytes;
140
+ discard(r, body).map_err(|_| {
141
+ "failed to discard trailing distributed snapshot completion bytes".to_string()
142
+ })?;
143
+
144
+ if u64_from_halves(done.request_hi, done.request_lo) != request_id {
145
+ return Err("distributed snapshot completion request mismatch".into());
146
+ }
147
+ if done.status != 0 {
148
+ let message = c_message(&message);
149
+ return Err(if message.is_empty() {
150
+ "distributed worker failed snapshot request".into()
151
+ } else {
152
+ message
153
+ });
154
+ }
155
+ Ok(())
156
+ }
157
+
158
+ /// Ask a connected worker for one KV shard and stream it into `payload`.
159
+ pub fn coordinator_save_snapshot<S: Read + Write, W: Write>(
160
+ stream: &mut S,
161
+ meta: SnapshotMeta,
162
+ tokens: &[i32],
163
+ payload: &mut W,
164
+ ) -> Result<u64, String> {
165
+ let token_count = token_count(tokens)?;
166
+ let (session_hi, session_lo) = u64_to_halves(meta.session_id);
167
+ let (request_hi, request_lo) = u64_to_halves(meta.request_id);
168
+ let (token_hash_hi, token_hash_lo) = u64_to_halves(meta.token_hash);
169
+ let request = SnapshotReq {
170
+ model_id: meta.model_id,
171
+ session_hi,
172
+ session_lo,
173
+ request_hi,
174
+ request_lo,
175
+ token_hash_hi,
176
+ token_hash_lo,
177
+ token_count,
178
+ layer_start: meta.layer_start,
179
+ layer_end: meta.layer_end,
180
+ };
181
+ write_frame(stream, MSG_SNAPSHOT_SAVE_REQ, &request.encode())
182
+ .map_err(|_| "failed to request distributed KV shard".to_string())?;
183
+
184
+ let (begin, message) = read_begin(stream)?;
185
+ if begin.status != 0 {
186
+ let message = c_message(&message);
187
+ return Err(if message.is_empty() {
188
+ "distributed worker refused KV snapshot".into()
189
+ } else {
190
+ message
191
+ });
192
+ }
193
+
194
+ let payload_bytes = u64_from_halves(begin.payload_hi, begin.payload_lo);
195
+ if begin.model_id != meta.model_id
196
+ || u64_from_halves(begin.session_hi, begin.session_lo) != meta.session_id
197
+ || u64_from_halves(begin.request_hi, begin.request_lo) != meta.request_id
198
+ || u64_from_halves(begin.token_hash_hi, begin.token_hash_lo) != meta.token_hash
199
+ || (begin.token_count != 0 && begin.token_count != token_count)
200
+ || begin.layer_start != meta.layer_start
201
+ || begin.layer_end != meta.layer_end
202
+ {
203
+ return Err("distributed KV shard metadata mismatch".into());
204
+ }
205
+
206
+ let mut buf = Vec::new();
207
+ if payload_bytes != 0 {
208
+ buf.try_reserve_exact(SNAPSHOT_CHUNK_BYTES)
209
+ .map_err(|_| "out of memory receiving distributed KV shard".to_string())?;
210
+ buf.resize(SNAPSHOT_CHUNK_BYTES, 0);
211
+ }
212
+ let mut received = 0u64;
213
+ while received < payload_bytes {
214
+ let header = read_header(stream, "distributed worker closed while sending KV shard")?;
215
+ if header.typ != MSG_SNAPSHOT_CHUNK || header.bytes < SNAPSHOT_CHUNK_FIXED_BYTES as u32 {
216
+ let _ = discard(stream, header.bytes);
217
+ return Err("expected distributed KV shard chunk".into());
218
+ }
219
+
220
+ let mut fixed = [0; SNAPSHOT_CHUNK_FIXED_BYTES];
221
+ read_or(
222
+ stream,
223
+ &mut fixed,
224
+ "failed to read distributed KV shard chunk header",
225
+ )?;
226
+ let chunk = SnapshotChunk::decode(&fixed)
227
+ .map_err(|_| "invalid distributed KV shard chunk".to_string())?;
228
+ let chunk_bytes = header.bytes - SNAPSHOT_CHUNK_FIXED_BYTES as u32;
229
+ if u64_from_halves(chunk.request_hi, chunk.request_lo) != meta.request_id
230
+ || chunk.chunk_bytes != chunk_bytes
231
+ || chunk_bytes as usize > SNAPSHOT_CHUNK_BYTES
232
+ || u64::from(chunk_bytes) > payload_bytes - received
233
+ {
234
+ let _ = discard(stream, chunk_bytes);
235
+ return Err("invalid distributed KV shard chunk".into());
236
+ }
237
+ let chunk_bytes = chunk_bytes as usize;
238
+ read_or(
239
+ stream,
240
+ &mut buf[..chunk_bytes],
241
+ "failed to read distributed KV shard chunk",
242
+ )?;
243
+ payload
244
+ .write_all(&buf[..chunk_bytes])
245
+ .map_err(|_| "failed to write distributed KV shard".to_string())?;
246
+ received += chunk_bytes as u64;
247
+ }
248
+ read_done(stream, meta.request_id)?;
249
+ Ok(payload_bytes)
250
+ }
251
+
252
+ /// Stream one caller-owned KV shard payload to a connected worker.
253
+ pub fn coordinator_load_snapshot<S: Read + Write, R: Read>(
254
+ stream: &mut S,
255
+ meta: SnapshotMeta,
256
+ tokens: &[i32],
257
+ payload: &mut R,
258
+ payload_bytes: u64,
259
+ ) -> Result<(), String> {
260
+ let token_count = token_count(tokens)?;
261
+ let token_bytes = token_count
262
+ .checked_mul(4)
263
+ .ok_or_else(|| "invalid distributed snapshot token length".to_string())?;
264
+ let (session_hi, session_lo) = u64_to_halves(meta.session_id);
265
+ let (request_hi, request_lo) = u64_to_halves(meta.request_id);
266
+ let (token_hash_hi, token_hash_lo) = u64_to_halves(meta.token_hash);
267
+ let (payload_hi, payload_lo) = u64_to_halves(payload_bytes);
268
+ let begin = SnapshotBegin {
269
+ model_id: meta.model_id,
270
+ session_hi,
271
+ session_lo,
272
+ request_hi,
273
+ request_lo,
274
+ token_hash_hi,
275
+ token_hash_lo,
276
+ token_count,
277
+ layer_start: meta.layer_start,
278
+ layer_end: meta.layer_end,
279
+ payload_hi,
280
+ payload_lo,
281
+ status: 0,
282
+ token_bytes,
283
+ message_bytes: 0,
284
+ };
285
+ let frame_bytes = SNAPSHOT_BEGIN_FIXED_BYTES as u64 + u64::from(token_bytes);
286
+ if frame_bytes > u64::from(u32::MAX) {
287
+ return Err("distributed snapshot begin frame is too large".into());
288
+ }
289
+ let send_begin = || "failed to send distributed KV shard restore request".to_string();
290
+ stream
291
+ .write_all(&encode_frame_header(
292
+ MSG_SNAPSHOT_LOAD_BEGIN,
293
+ frame_bytes as u32,
294
+ ))
295
+ .map_err(|_| send_begin())?;
296
+ stream
297
+ .write_all(&begin.encode())
298
+ .map_err(|_| send_begin())?;
299
+ for &token in tokens {
300
+ stream
301
+ .write_all(&(token as u32).to_be_bytes())
302
+ .map_err(|_| send_begin())?;
303
+ }
304
+
305
+ let mut buf = Vec::new();
306
+ if payload_bytes != 0 {
307
+ buf.try_reserve_exact(SNAPSHOT_CHUNK_BYTES)
308
+ .map_err(|_| "failed to send distributed KV shard restore payload".to_string())?;
309
+ buf.resize(SNAPSHOT_CHUNK_BYTES, 0);
310
+ }
311
+ let mut remaining = payload_bytes;
312
+ while remaining != 0 {
313
+ let n = remaining.min(SNAPSHOT_CHUNK_BYTES as u64) as usize;
314
+ read_or(
315
+ payload,
316
+ &mut buf[..n],
317
+ "failed to send distributed KV shard restore payload",
318
+ )?;
319
+ let (request_hi, request_lo) = u64_to_halves(meta.request_id);
320
+ let chunk = SnapshotChunk {
321
+ request_hi,
322
+ request_lo,
323
+ chunk_bytes: n as u32,
324
+ };
325
+ let frame_bytes = SNAPSHOT_CHUNK_FIXED_BYTES as u32 + n as u32;
326
+ let send_chunk = || "failed to send distributed KV shard restore payload".to_string();
327
+ stream
328
+ .write_all(&encode_frame_header(MSG_SNAPSHOT_CHUNK, frame_bytes))
329
+ .map_err(|_| send_chunk())?;
330
+ stream
331
+ .write_all(&chunk.encode())
332
+ .map_err(|_| send_chunk())?;
333
+ stream.write_all(&buf[..n]).map_err(|_| send_chunk())?;
334
+ remaining -= n as u64;
335
+ }
336
+ read_done(stream, meta.request_id)
337
+ }
crates/ds4-dist/src/snapshot_gather.rs ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Coordinator KV gather order from `ds4_dist_session_save_payload`.
2
+ //!
3
+ //! Host owns shard walk, empty-shard refusal, and token hashing. Layer
4
+ //! GPU bytes stay native; DSV4 header merge lives in `snapshot_merge`.
5
+
6
+ use crate::hash::token_hash_prefix;
7
+
8
+ pub const ERR_NO_TIMELINE: &str = "distributed session has no valid token timeline";
9
+ pub const ERR_EMPTY_SHARD: &str = "distributed KV shard is empty";
10
+
11
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
12
+ pub struct GatherShard {
13
+ pub layer_start: u32,
14
+ pub layer_end: u32,
15
+ pub local: bool,
16
+ }
17
+
18
+ #[derive(Debug, Clone, PartialEq, Eq)]
19
+ pub struct GatheredShards {
20
+ pub token_count: u32,
21
+ pub token_hash: u64,
22
+ pub shards: Vec<(GatherShard, Vec<u8>)>,
23
+ }
24
+
25
+ /// C save walk: shard 0 is the owner layer payload; later shards are
26
+ /// remote. Empty shard bytes fail before any DSV4 merge.
27
+ pub fn gather_kv_shards<F>(
28
+ tokens: &[i32],
29
+ plan: &[GatherShard],
30
+ mut fetch: F,
31
+ ) -> Result<GatheredShards, String>
32
+ where
33
+ F: FnMut(GatherShard) -> Result<Vec<u8>, String>,
34
+ {
35
+ if tokens.len() > u32::MAX as usize {
36
+ return Err(ERR_NO_TIMELINE.into());
37
+ }
38
+ if plan.is_empty() {
39
+ return Err(ERR_EMPTY_SHARD.into());
40
+ }
41
+ let token_count = tokens.len() as u32;
42
+ let token_hash = token_hash_prefix(tokens);
43
+ let mut shards = Vec::with_capacity(plan.len());
44
+ for (i, spec) in plan.iter().copied().enumerate() {
45
+ if i == 0 && !spec.local {
46
+ return Err("distributed KV shard 0 must be the owner layer".into());
47
+ }
48
+ if i != 0 && spec.local {
49
+ return Err("distributed KV remote shard marked local".into());
50
+ }
51
+ let bytes = fetch(spec)?;
52
+ if bytes.is_empty() {
53
+ return Err(ERR_EMPTY_SHARD.into());
54
+ }
55
+ shards.push((spec, bytes));
56
+ }
57
+ Ok(GatheredShards {
58
+ token_count,
59
+ token_hash,
60
+ shards,
61
+ })
62
+ }
63
+
64
+ /// C load walk: shard 0 is the owner layer payload; later shards are
65
+ /// remote. Host owns the order. Layer apply and DSV4 parse stay native.
66
+ pub fn scatter_kv_shards<F>(
67
+ tokens: &[i32],
68
+ plan: &[GatherShard],
69
+ shards: &[Vec<u8>],
70
+ mut apply: F,
71
+ ) -> Result<u64, String>
72
+ where
73
+ F: FnMut(GatherShard, &[u8]) -> Result<(), String>,
74
+ {
75
+ if tokens.len() > u32::MAX as usize {
76
+ return Err(ERR_NO_TIMELINE.into());
77
+ }
78
+ if plan.is_empty() || plan.len() != shards.len() {
79
+ return Err(ERR_EMPTY_SHARD.into());
80
+ }
81
+ for (i, (spec, bytes)) in plan.iter().copied().zip(shards.iter()).enumerate() {
82
+ if i == 0 && !spec.local {
83
+ return Err("distributed KV shard 0 must be the owner layer".into());
84
+ }
85
+ if i != 0 && spec.local {
86
+ return Err("distributed KV remote shard marked local".into());
87
+ }
88
+ if bytes.is_empty() {
89
+ return Err(ERR_EMPTY_SHARD.into());
90
+ }
91
+ apply(spec, bytes)?;
92
+ }
93
+ Ok(token_hash_prefix(tokens))
94
+ }
95
+
96
+ #[cfg(test)]
97
+ mod tests {
98
+ use super::*;
99
+
100
+ fn two_shard_plan() -> [GatherShard; 2] {
101
+ [
102
+ GatherShard {
103
+ layer_start: 0,
104
+ layer_end: 10,
105
+ local: true,
106
+ },
107
+ GatherShard {
108
+ layer_start: 10,
109
+ layer_end: 20,
110
+ local: false,
111
+ },
112
+ ]
113
+ }
114
+
115
+ #[test]
116
+ fn gather_walks_owner_then_remote() {
117
+ let tokens = [1i32, 2, 3];
118
+ let got = gather_kv_shards(&tokens, &two_shard_plan(), |spec| {
119
+ Ok(vec![spec.layer_start as u8, spec.layer_end as u8])
120
+ })
121
+ .unwrap();
122
+ assert_eq!(got.token_count, 3);
123
+ assert_eq!(got.token_hash, token_hash_prefix(&tokens));
124
+ assert_eq!(got.shards.len(), 2);
125
+ assert!(got.shards[0].0.local);
126
+ assert!(!got.shards[1].0.local);
127
+ assert_eq!(got.shards[0].1, vec![0, 10]);
128
+ assert_eq!(got.shards[1].1, vec![10, 20]);
129
+ }
130
+
131
+ #[test]
132
+ fn gather_refuses_empty_shard() {
133
+ let err = gather_kv_shards(&[1], &two_shard_plan(), |spec| {
134
+ if spec.local {
135
+ Ok(vec![1])
136
+ } else {
137
+ Ok(vec![])
138
+ }
139
+ })
140
+ .unwrap_err();
141
+ assert_eq!(err, ERR_EMPTY_SHARD);
142
+ }
143
+
144
+ #[test]
145
+ fn gather_requires_owner_first() {
146
+ let plan = [GatherShard {
147
+ layer_start: 0,
148
+ layer_end: 1,
149
+ local: false,
150
+ }];
151
+ let err = gather_kv_shards(&[1], &plan, |_| Ok(vec![1])).unwrap_err();
152
+ assert!(err.contains("owner layer"));
153
+ }
154
+
155
+ #[test]
156
+ fn gather_accepts_empty_token_timeline() {
157
+ let got = gather_kv_shards(&[], &two_shard_plan(), |_| Ok(vec![1])).unwrap();
158
+ assert_eq!(got.token_count, 0);
159
+ assert_eq!(got.token_hash, token_hash_prefix(&[]));
160
+ }
161
+
162
+ #[test]
163
+ fn scatter_walks_owner_then_remote() {
164
+ let tokens = [7i32];
165
+ let mut seen = Vec::new();
166
+ let hash = scatter_kv_shards(
167
+ &tokens,
168
+ &two_shard_plan(),
169
+ &[vec![1], vec![2]],
170
+ |spec, bytes| {
171
+ seen.push((spec.local, bytes.to_vec()));
172
+ Ok(())
173
+ },
174
+ )
175
+ .unwrap();
176
+ assert_eq!(hash, token_hash_prefix(&tokens));
177
+ assert_eq!(seen, vec![(true, vec![1]), (false, vec![2])]);
178
+ }
179
+
180
+ #[test]
181
+ fn scatter_refuses_empty_shard() {
182
+ let err = scatter_kv_shards(&[1], &two_shard_plan(), &[vec![1], vec![]], |_, _| Ok(()))
183
+ .unwrap_err();
184
+ assert_eq!(err, ERR_EMPTY_SHARD);
185
+ }
186
+ }
crates/ds4-dist/src/snapshot_merge.rs ADDED
@@ -0,0 +1,415 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Coordinator DSV4 assembly from `ds4_dist_session_save_payload`.
2
+ //!
3
+ //! Host owns layer-header parse, layout match, tensor-size arithmetic, and
4
+ //! write order. Layer GPU bytes and logits stay opaque blobs.
5
+
6
+ use crate::snapshot_gather::ERR_EMPTY_SHARD;
7
+
8
+ pub const LAYER_MAGIC: u32 = 0x4C56_5344; /* "DSVL" */
9
+ pub const LAYER_VERSION: u32 = 1;
10
+ pub const LAYER_U32_FIELDS: usize = 14;
11
+ pub const SESSION_MAGIC: u32 = 0x3456_5344; /* "DSV4" */
12
+ pub const SESSION_VERSION: u32 = 3;
13
+ pub const SESSION_U32_FIELDS: usize = 13;
14
+
15
+ pub const ERR_UNSUPPORTED_LAYER: &str = "unsupported distributed KV layer payload";
16
+ pub const ERR_RANGE: &str = "distributed KV shard range mismatch";
17
+ pub const ERR_LAYOUT: &str = "distributed KV shard layout is invalid";
18
+ pub const ERR_LAYOUTS_DIFFER: &str = "distributed KV shards use different layouts";
19
+ pub const ERR_COMP_ROWS: &str = "distributed KV shard has invalid compressed row count";
20
+ pub const ERR_INDEX_ROWS: &str = "distributed KV shard has invalid indexer row count";
21
+ pub const ERR_TENSOR_OVERFLOW: &str = "distributed KV shard tensor size overflow";
22
+ pub const ERR_TENSOR_BYTES: &str = "distributed KV shard tensor byte count mismatch";
23
+ pub const ERR_METADATA: &str = "distributed KV shard metadata mismatch";
24
+
25
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
26
+ pub struct DistKvLayout {
27
+ pub ctx: u32,
28
+ pub prefill_cap: u32,
29
+ pub raw_cap: u32,
30
+ pub raw_window: u32,
31
+ pub comp_cap: u32,
32
+ pub token_count: u32,
33
+ pub n_layers: u32,
34
+ pub head_dim: u32,
35
+ pub indexer_head_dim: u32,
36
+ pub vocab: u32,
37
+ pub raw_live: u32,
38
+ }
39
+
40
+ impl DistKvLayout {
41
+ pub fn matches_core(&self, other: &Self) -> bool {
42
+ self.ctx == other.ctx
43
+ && self.prefill_cap == other.prefill_cap
44
+ && self.raw_cap == other.raw_cap
45
+ && self.raw_window == other.raw_window
46
+ && self.comp_cap == other.comp_cap
47
+ && self.token_count == other.token_count
48
+ && self.n_layers == other.n_layers
49
+ && self.head_dim == other.head_dim
50
+ && self.indexer_head_dim == other.indexer_head_dim
51
+ && self.raw_live == other.raw_live
52
+ }
53
+
54
+ pub fn raw_live_valid(&self) -> bool {
55
+ if self.raw_window == 0 || self.raw_cap == 0 {
56
+ return false;
57
+ }
58
+ let expected = self.token_count.min(self.raw_window);
59
+ self.raw_live == expected && self.raw_live <= self.raw_cap
60
+ }
61
+ }
62
+
63
+ #[derive(Debug, Clone, PartialEq, Eq)]
64
+ pub struct ParsedLayerShard {
65
+ pub layout: DistKvLayout,
66
+ pub layer_start: u32,
67
+ pub layer_end: u32,
68
+ pub n_comp: Vec<u32>,
69
+ pub n_index_comp: Vec<u32>,
70
+ pub tensor: Vec<u8>,
71
+ }
72
+
73
+ fn get_u32(bytes: &[u8]) -> Option<u32> {
74
+ bytes.get(..4)?.try_into().ok().map(u32::from_le_bytes)
75
+ }
76
+
77
+ fn put_u32(out: &mut Vec<u8>, v: u32) {
78
+ out.extend_from_slice(&v.to_le_bytes());
79
+ }
80
+
81
+ /// C `dist_kv_state_bytes`.
82
+ pub fn kv_state_bytes(ratio: u32, head_dim: u32) -> Option<u64> {
83
+ let coff = if ratio == 4 { 2u64 } else { 1u64 };
84
+ coff.checked_mul(u64::from(head_dim))?
85
+ .checked_mul(coff)?
86
+ .checked_mul(u64::from(ratio))?
87
+ .checked_mul(4)
88
+ }
89
+
90
+ /// C `dist_kv_layer_tensor_bytes` once the host compress ratio is known.
91
+ pub fn layer_tensor_bytes(
92
+ layout: &DistKvLayout,
93
+ ratio: u32,
94
+ n_comp: u32,
95
+ n_index_comp: u32,
96
+ ) -> Option<u64> {
97
+ let mut bytes = u64::from(layout.raw_live)
98
+ .checked_mul(u64::from(layout.head_dim))?
99
+ .checked_mul(4)?;
100
+ if ratio == 0 {
101
+ return Some(bytes);
102
+ }
103
+ bytes = bytes.checked_add(
104
+ u64::from(n_comp)
105
+ .checked_mul(u64::from(layout.head_dim))?
106
+ .checked_mul(4)?,
107
+ )?;
108
+ let attn = kv_state_bytes(ratio, layout.head_dim)?;
109
+ bytes = bytes.checked_add(attn)?.checked_add(attn)?;
110
+ if ratio == 4 {
111
+ bytes = bytes.checked_add(
112
+ u64::from(n_index_comp)
113
+ .checked_mul(u64::from(layout.indexer_head_dim))?
114
+ .checked_mul(4)?,
115
+ )?;
116
+ let index = kv_state_bytes(ratio, layout.indexer_head_dim)?;
117
+ bytes = bytes.checked_add(index)?.checked_add(index)?;
118
+ }
119
+ Some(bytes)
120
+ }
121
+
122
+ pub fn parse_layer_payload(
123
+ bytes: &[u8],
124
+ expected_start: u32,
125
+ expected_end: u32,
126
+ compress: &[u32],
127
+ ) -> Result<ParsedLayerShard, String> {
128
+ if bytes.len() < LAYER_U32_FIELDS * 4 {
129
+ return Err(ERR_UNSUPPORTED_LAYER.into());
130
+ }
131
+ let mut h = [0u32; LAYER_U32_FIELDS];
132
+ for (i, slot) in h.iter_mut().enumerate() {
133
+ *slot = get_u32(&bytes[i * 4..]).ok_or(ERR_UNSUPPORTED_LAYER)?;
134
+ }
135
+ if h[0] != LAYER_MAGIC || h[1] != LAYER_VERSION {
136
+ return Err(ERR_UNSUPPORTED_LAYER.into());
137
+ }
138
+ let layout = DistKvLayout {
139
+ ctx: h[2],
140
+ prefill_cap: h[3],
141
+ raw_cap: h[4],
142
+ raw_window: h[5],
143
+ comp_cap: h[6],
144
+ token_count: h[7],
145
+ n_layers: h[8],
146
+ head_dim: h[9],
147
+ indexer_head_dim: h[10],
148
+ vocab: 0,
149
+ raw_live: h[13],
150
+ };
151
+ let layer_start = h[11];
152
+ let layer_end = h[12];
153
+ if layer_start != expected_start
154
+ || layer_end != expected_end
155
+ || layer_start > layer_end
156
+ || layer_end >= layout.n_layers
157
+ {
158
+ return Err(ERR_RANGE.into());
159
+ }
160
+ if !layout.raw_live_valid() {
161
+ return Err(ERR_LAYOUT.into());
162
+ }
163
+ let slice = (layer_end - layer_start + 1) as usize;
164
+ let mut off = LAYER_U32_FIELDS * 4;
165
+ let mut n_comp = vec![0u32; layout.n_layers as usize];
166
+ let mut n_index_comp = vec![0u32; layout.n_layers as usize];
167
+ for i in 0..slice {
168
+ let il = (layer_start as usize) + i;
169
+ let v = get_u32(bytes.get(off..).unwrap_or(&[])).ok_or(ERR_UNSUPPORTED_LAYER)?;
170
+ if v > layout.comp_cap {
171
+ return Err(ERR_COMP_ROWS.into());
172
+ }
173
+ n_comp[il] = v;
174
+ off += 4;
175
+ }
176
+ for i in 0..slice {
177
+ let il = (layer_start as usize) + i;
178
+ let v = get_u32(bytes.get(off..).unwrap_or(&[])).ok_or(ERR_UNSUPPORTED_LAYER)?;
179
+ if v > layout.comp_cap {
180
+ return Err(ERR_INDEX_ROWS.into());
181
+ }
182
+ n_index_comp[il] = v;
183
+ off += 4;
184
+ }
185
+ let mut expected_tensor = 0u64;
186
+ for il in layer_start..=layer_end {
187
+ let ratio = compress.get(il as usize).copied().unwrap_or(0);
188
+ let layer_bytes = layer_tensor_bytes(
189
+ &layout,
190
+ ratio,
191
+ n_comp[il as usize],
192
+ n_index_comp[il as usize],
193
+ )
194
+ .ok_or(ERR_TENSOR_OVERFLOW)?;
195
+ expected_tensor = expected_tensor
196
+ .checked_add(layer_bytes)
197
+ .ok_or(ERR_TENSOR_OVERFLOW)?;
198
+ }
199
+ let tensor = bytes.get(off..).unwrap_or(&[]).to_vec();
200
+ if tensor.len() as u64 != expected_tensor {
201
+ return Err(ERR_TENSOR_BYTES.into());
202
+ }
203
+ Ok(ParsedLayerShard {
204
+ layout,
205
+ layer_start,
206
+ layer_end,
207
+ n_comp,
208
+ n_index_comp,
209
+ tensor,
210
+ })
211
+ }
212
+
213
+ /// C save merge after shards are gathered: header, tokens, logits,
214
+ /// n_comp, n_index_comp, then shard tensor tails in walk order.
215
+ pub fn merge_session_payload(
216
+ tokens: &[i32],
217
+ logits: &[f32],
218
+ n_layers: u32,
219
+ shards: &[ParsedLayerShard],
220
+ compress: &[u32],
221
+ ) -> Result<Vec<u8>, String> {
222
+ if shards.is_empty() {
223
+ return Err(ERR_EMPTY_SHARD.into());
224
+ }
225
+ let token_count = u32::try_from(tokens.len()).map_err(|_| ERR_METADATA.to_string())?;
226
+ let vocab = u32::try_from(logits.len()).map_err(|_| ERR_METADATA.to_string())?;
227
+ let mut layout = shards[0].layout;
228
+ layout.vocab = vocab;
229
+ if layout.token_count != token_count || layout.n_layers != n_layers || !layout.raw_live_valid()
230
+ {
231
+ return Err(ERR_METADATA.into());
232
+ }
233
+ let mut n_comp = vec![0u32; n_layers as usize];
234
+ let mut n_index_comp = vec![0u32; n_layers as usize];
235
+ let mut covered = vec![false; n_layers as usize];
236
+ for shard in shards {
237
+ if !layout.matches_core(&shard.layout) {
238
+ return Err(ERR_LAYOUTS_DIFFER.into());
239
+ }
240
+ let mut shard_bytes = 0u64;
241
+ for il in shard.layer_start..=shard.layer_end {
242
+ let i = il as usize;
243
+ if covered[i] {
244
+ return Err(ERR_RANGE.into());
245
+ }
246
+ covered[i] = true;
247
+ n_comp[i] = shard.n_comp[i];
248
+ n_index_comp[i] = shard.n_index_comp[i];
249
+ let ratio = compress.get(i).copied().unwrap_or(0);
250
+ let layer_bytes = layer_tensor_bytes(&layout, ratio, n_comp[i], n_index_comp[i])
251
+ .ok_or(ERR_TENSOR_OVERFLOW)?;
252
+ shard_bytes = shard_bytes
253
+ .checked_add(layer_bytes)
254
+ .ok_or(ERR_TENSOR_OVERFLOW)?;
255
+ }
256
+ if shard.tensor.len() as u64 != shard_bytes {
257
+ return Err(ERR_TENSOR_BYTES.into());
258
+ }
259
+ }
260
+ if covered.iter().any(|c| !*c) {
261
+ return Err(ERR_METADATA.into());
262
+ }
263
+ let mut out = Vec::new();
264
+ for f in [
265
+ SESSION_MAGIC,
266
+ SESSION_VERSION,
267
+ layout.ctx,
268
+ layout.prefill_cap,
269
+ layout.raw_cap,
270
+ layout.raw_window,
271
+ layout.comp_cap,
272
+ layout.token_count,
273
+ layout.n_layers,
274
+ layout.head_dim,
275
+ layout.indexer_head_dim,
276
+ layout.vocab,
277
+ layout.raw_live,
278
+ ] {
279
+ put_u32(&mut out, f);
280
+ }
281
+ for &tok in tokens {
282
+ put_u32(&mut out, tok as u32);
283
+ }
284
+ for &logit in logits {
285
+ out.extend_from_slice(&logit.to_le_bytes());
286
+ }
287
+ for v in &n_comp {
288
+ put_u32(&mut out, *v);
289
+ }
290
+ for v in &n_index_comp {
291
+ put_u32(&mut out, *v);
292
+ }
293
+ for shard in shards {
294
+ out.extend_from_slice(&shard.tensor);
295
+ }
296
+ Ok(out)
297
+ }
298
+
299
+ #[cfg(test)]
300
+ mod tests {
301
+ use super::*;
302
+
303
+ fn layout() -> DistKvLayout {
304
+ DistKvLayout {
305
+ ctx: 16,
306
+ prefill_cap: 8,
307
+ raw_cap: 16,
308
+ raw_window: 4,
309
+ comp_cap: 8,
310
+ token_count: 2,
311
+ n_layers: 2,
312
+ head_dim: 4,
313
+ indexer_head_dim: 2,
314
+ vocab: 0,
315
+ raw_live: 2,
316
+ }
317
+ }
318
+
319
+ fn encode_layer(
320
+ start: u32,
321
+ end: u32,
322
+ n_comp: &[u32],
323
+ n_index: &[u32],
324
+ tensor: &[u8],
325
+ ) -> Vec<u8> {
326
+ let l = layout();
327
+ let mut out = Vec::new();
328
+ for f in [
329
+ LAYER_MAGIC,
330
+ LAYER_VERSION,
331
+ l.ctx,
332
+ l.prefill_cap,
333
+ l.raw_cap,
334
+ l.raw_window,
335
+ l.comp_cap,
336
+ l.token_count,
337
+ l.n_layers,
338
+ l.head_dim,
339
+ l.indexer_head_dim,
340
+ start,
341
+ end,
342
+ l.raw_live,
343
+ ] {
344
+ put_u32(&mut out, f);
345
+ }
346
+ for il in start..=end {
347
+ put_u32(&mut out, n_comp[il as usize]);
348
+ }
349
+ for il in start..=end {
350
+ put_u32(&mut out, n_index[il as usize]);
351
+ }
352
+ out.extend_from_slice(tensor);
353
+ out
354
+ }
355
+
356
+ #[test]
357
+ fn layer_tensor_bytes_ratio_zero_is_raw_only() {
358
+ let l = layout();
359
+ assert_eq!(layer_tensor_bytes(&l, 0, 3, 9), Some(2 * 4 * 4));
360
+ }
361
+
362
+ #[test]
363
+ fn parse_and_merge_two_uncompressed_shards() {
364
+ let compress = [0u32, 0];
365
+ let t0 = vec![1u8; 32];
366
+ let t1 = vec![2u8; 32];
367
+ let s0 = parse_layer_payload(&encode_layer(0, 0, &[0, 0], &[0, 0], &t0), 0, 0, &compress)
368
+ .unwrap();
369
+ let s1 = parse_layer_payload(&encode_layer(1, 1, &[0, 0], &[0, 0], &t1), 1, 1, &compress)
370
+ .unwrap();
371
+ let tokens = [7i32, 8];
372
+ let logits = [0.5f32, -1.0];
373
+ let merged = merge_session_payload(&tokens, &logits, 2, &[s0, s1], &compress).unwrap();
374
+ assert_eq!(&merged[..4], &SESSION_MAGIC.to_le_bytes());
375
+ assert_eq!(&merged[4..8], &SESSION_VERSION.to_le_bytes());
376
+ let header = SESSION_U32_FIELDS * 4;
377
+ assert_eq!(&merged[header..header + 4], &7u32.to_le_bytes());
378
+ assert_eq!(&merged[header + 4..header + 8], &8u32.to_le_bytes());
379
+ let after_tokens = header + 8;
380
+ assert_eq!(
381
+ &merged[after_tokens..after_tokens + 4],
382
+ &0.5f32.to_le_bytes()
383
+ );
384
+ let after_logits = after_tokens + 8;
385
+ let after_counts = after_logits + 16;
386
+ assert_eq!(&merged[after_counts..after_counts + 32], &t0[..]);
387
+ assert_eq!(&merged[after_counts + 32..after_counts + 64], &t1[..]);
388
+ }
389
+
390
+ #[test]
391
+ fn parse_refuses_tensor_byte_mismatch() {
392
+ let compress = [0u32, 0];
393
+ let err = parse_layer_payload(
394
+ &encode_layer(0, 0, &[0, 0], &[0, 0], &[1, 2, 3]),
395
+ 0,
396
+ 0,
397
+ &compress,
398
+ )
399
+ .unwrap_err();
400
+ assert_eq!(err, ERR_TENSOR_BYTES);
401
+ }
402
+
403
+ #[test]
404
+ fn merge_refuses_layout_mismatch() {
405
+ let compress = [0u32, 0];
406
+ let t = vec![1u8; 32];
407
+ let s0 = parse_layer_payload(&encode_layer(0, 0, &[0, 0], &[0, 0], &t), 0, 0, &compress)
408
+ .unwrap();
409
+ let mut other = encode_layer(1, 1, &[0, 0], &[0, 0], &t);
410
+ other[2 * 4..2 * 4 + 4].copy_from_slice(&32u32.to_le_bytes());
411
+ let s1 = parse_layer_payload(&other, 1, 1, &compress).unwrap();
412
+ let err = merge_session_payload(&[7, 8], &[0.0, 0.0], 2, &[s0, s1], &compress).unwrap_err();
413
+ assert_eq!(err, ERR_LAYOUTS_DIFFER);
414
+ }
415
+ }
crates/ds4-dist/src/snapshot_temp.rs ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Bounded temp files for worker KV shard save/load.
2
+
3
+ use std::fs::{File, OpenOptions};
4
+ use std::io::{self, Read, Seek, SeekFrom, Write};
5
+ use std::path::PathBuf;
6
+
7
+ use crate::codec::SNAPSHOT_CHUNK_BYTES;
8
+
9
+ pub const SAVE_PREFIX: &str = "ds4-dist-save";
10
+ pub const LOAD_PREFIX: &str = "ds4-dist-load";
11
+
12
+ pub struct TempShard {
13
+ pub(crate) file: File,
14
+ pub(crate) path: PathBuf,
15
+ }
16
+
17
+ impl TempShard {
18
+ pub fn path(&self) -> &std::path::Path {
19
+ &self.path
20
+ }
21
+ }
22
+
23
+ impl Drop for TempShard {
24
+ fn drop(&mut self) {
25
+ let _ = std::fs::remove_file(&self.path);
26
+ }
27
+ }
28
+
29
+ impl Read for TempShard {
30
+ fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
31
+ self.file.read(buf)
32
+ }
33
+ }
34
+
35
+ impl Write for TempShard {
36
+ fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
37
+ self.file.write(buf)
38
+ }
39
+
40
+ fn flush(&mut self) -> io::Result<()> {
41
+ self.file.flush()
42
+ }
43
+ }
44
+
45
+ impl Seek for TempShard {
46
+ fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
47
+ self.file.seek(pos)
48
+ }
49
+ }
50
+
51
+ fn temp_root() -> PathBuf {
52
+ #[cfg(unix)]
53
+ {
54
+ PathBuf::from("/tmp")
55
+ }
56
+ #[cfg(not(unix))]
57
+ {
58
+ std::env::temp_dir()
59
+ }
60
+ }
61
+
62
+ pub fn create_temp(prefix: &str) -> io::Result<TempShard> {
63
+ let root = temp_root();
64
+ for n in 0u32..1024 {
65
+ let path = root.join(format!("{prefix}.{}.{:04x}", std::process::id(), n));
66
+ match OpenOptions::new()
67
+ .read(true)
68
+ .write(true)
69
+ .create_new(true)
70
+ .open(&path)
71
+ {
72
+ Ok(file) => return Ok(TempShard { file, path }),
73
+ Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue,
74
+ Err(e) => return Err(e),
75
+ }
76
+ }
77
+ Err(io::Error::new(
78
+ io::ErrorKind::AlreadyExists,
79
+ "temp name space exhausted",
80
+ ))
81
+ }
82
+
83
+ pub fn copy_chunked(
84
+ src: &mut dyn Read,
85
+ dest: &mut dyn Write,
86
+ mut remaining: u64,
87
+ ) -> io::Result<()> {
88
+ if remaining == 0 {
89
+ return Ok(());
90
+ }
91
+ let mut buf = vec![0u8; SNAPSHOT_CHUNK_BYTES];
92
+ while remaining != 0 {
93
+ let n = remaining.min(SNAPSHOT_CHUNK_BYTES as u64) as usize;
94
+ src.read_exact(&mut buf[..n])?;
95
+ dest.write_all(&buf[..n])?;
96
+ remaining -= n as u64;
97
+ }
98
+ Ok(())
99
+ }
crates/ds4-dist/src/transport.rs ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Blocking frame I/O. Same layout as `dist_write_frame_header` / `dist_read_frame_header`.
2
+
3
+ use std::io::{self, Read, Write};
4
+
5
+ use crate::codec::{decode_frame_header, encode_frame_header, CodecError, FRAME_HEADER_BYTES};
6
+
7
+ pub fn write_frame<W: Write>(w: &mut W, typ: u32, payload: &[u8]) -> io::Result<()> {
8
+ let bytes = u32::try_from(payload.len())
9
+ .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "distributed frame too large"))?;
10
+ w.write_all(&encode_frame_header(typ, bytes))?;
11
+ w.write_all(payload)?;
12
+ Ok(())
13
+ }
14
+
15
+ pub fn read_frame<R: Read>(r: &mut R) -> io::Result<(u32, Vec<u8>)> {
16
+ let mut hdr = [0u8; FRAME_HEADER_BYTES];
17
+ r.read_exact(&mut hdr)?;
18
+ let h = decode_frame_header(&hdr).map_err(|e| match e {
19
+ CodecError::BadMagic(m) => io::Error::new(
20
+ io::ErrorKind::InvalidData,
21
+ format!("bad frame magic 0x{m:08x}"),
22
+ ),
23
+ other => io::Error::new(io::ErrorKind::InvalidData, other.to_string()),
24
+ })?;
25
+ let mut payload = vec![0u8; h.bytes as usize];
26
+ if h.bytes != 0 {
27
+ r.read_exact(&mut payload)?;
28
+ }
29
+ Ok((h.typ, payload))
30
+ }
crates/ds4-dist/src/work.rs ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! WORK / RESULT frame bodies. Tokens are BE u32; activations use the packers.
2
+
3
+ use crate::activation::{bits_or_default, decode_activation, encode_activation};
4
+ use crate::codec::{
5
+ decode_tokens_be, encode_frame_header, encode_tokens_be, u64_from_halves, u64_to_halves,
6
+ CodecError, ResultHdr, Telemetry, Work, MSG_RESULT, MSG_WORK, RESULT_FIXED_BYTES,
7
+ TELEMETRY_FIXED_BYTES, WORK_FIXED_BYTES,
8
+ };
9
+
10
+ #[derive(Debug, Clone)]
11
+ pub struct WorkBody {
12
+ pub work: Work,
13
+ pub tokens: Vec<i32>,
14
+ pub input_hc: Vec<f32>,
15
+ pub route_blob: Vec<u8>,
16
+ }
17
+
18
+ pub fn encode_work_body(body: &WorkBody) -> Result<Vec<u8>, CodecError> {
19
+ if body.tokens.is_empty() {
20
+ return Err(CodecError::Invalid("WORK frame has no tokens"));
21
+ }
22
+ let mut work = body.work;
23
+ work.n_tokens = body.tokens.len() as u32;
24
+ work.token_bytes = work.n_tokens * 4;
25
+ let hc_bits = bits_or_default(work.input_hc_bits);
26
+ let hc_wire = if body.input_hc.is_empty() {
27
+ Vec::new()
28
+ } else {
29
+ encode_activation(&body.input_hc, hc_bits)
30
+ .ok_or(CodecError::Invalid("invalid distributed activation width"))?
31
+ };
32
+ work.input_hc_bytes = hc_wire.len() as u32;
33
+ work.input_hc_bits = hc_bits;
34
+ work.route_bytes = body.route_blob.len() as u32;
35
+ let mut out = work.encode().to_vec();
36
+ out.extend_from_slice(&encode_tokens_be(&body.tokens));
37
+ out.extend_from_slice(&hc_wire);
38
+ out.extend_from_slice(&body.route_blob);
39
+ Ok(out)
40
+ }
41
+
42
+ pub fn encode_work_frame(body: &WorkBody) -> Result<Vec<u8>, CodecError> {
43
+ let payload = encode_work_body(body)?;
44
+ let mut frame = encode_frame_header(MSG_WORK, payload.len() as u32).to_vec();
45
+ frame.extend_from_slice(&payload);
46
+ Ok(frame)
47
+ }
48
+
49
+ pub fn decode_work_body(buf: &[u8]) -> Result<WorkBody, CodecError> {
50
+ if buf.len() < WORK_FIXED_BYTES {
51
+ return Err(CodecError::Invalid("truncated distributed WORK frame"));
52
+ }
53
+ let work = Work::decode(buf)?;
54
+ let rest = &buf[WORK_FIXED_BYTES..];
55
+ let expected =
56
+ work.token_bytes as usize + work.input_hc_bytes as usize + work.route_bytes as usize;
57
+ if work.token_bytes as usize != work.n_tokens as usize * 4 || rest.len() != expected {
58
+ return Err(CodecError::Invalid(
59
+ "invalid distributed WORK payload sizes",
60
+ ));
61
+ }
62
+ let (tok, rest) = rest.split_at(work.token_bytes as usize);
63
+ let tokens = decode_tokens_be(tok)?;
64
+ let (hc, rest) = rest.split_at(work.input_hc_bytes as usize);
65
+ let input_hc = if hc.is_empty() {
66
+ Vec::new()
67
+ } else {
68
+ decode_activation(hc, work.input_hc_bits)
69
+ .ok_or(CodecError::Invalid("invalid distributed activation width"))?
70
+ };
71
+ Ok(WorkBody {
72
+ work,
73
+ tokens,
74
+ input_hc,
75
+ route_blob: rest.to_vec(),
76
+ })
77
+ }
78
+
79
+ #[derive(Debug, Clone)]
80
+ pub struct ResultBody {
81
+ pub hdr: ResultHdr,
82
+ pub telemetry: Vec<Telemetry>,
83
+ pub payload: Vec<u8>,
84
+ }
85
+
86
+ pub fn encode_result_body(body: &ResultBody) -> Result<Vec<u8>, CodecError> {
87
+ let mut hdr = body.hdr;
88
+ hdr.telemetry_count = body.telemetry.len() as u32;
89
+ hdr.telemetry_bytes = hdr.telemetry_count * TELEMETRY_FIXED_BYTES as u32;
90
+ hdr.payload_bytes = body.payload.len() as u32;
91
+ let mut out = hdr.encode().to_vec();
92
+ for t in &body.telemetry {
93
+ out.extend_from_slice(&t.encode());
94
+ }
95
+ out.extend_from_slice(&body.payload);
96
+ Ok(out)
97
+ }
98
+
99
+ pub fn encode_result_frame(body: &ResultBody) -> Result<Vec<u8>, CodecError> {
100
+ let payload = encode_result_body(body)?;
101
+ let mut frame = encode_frame_header(MSG_RESULT, payload.len() as u32).to_vec();
102
+ frame.extend_from_slice(&payload);
103
+ Ok(frame)
104
+ }
105
+
106
+ pub fn decode_result_body(buf: &[u8]) -> Result<ResultBody, CodecError> {
107
+ if buf.len() < RESULT_FIXED_BYTES {
108
+ return Err(CodecError::Invalid(
109
+ "distributed worker returned invalid frame",
110
+ ));
111
+ }
112
+ let hdr = ResultHdr::decode(buf)?;
113
+ let rest = &buf[RESULT_FIXED_BYTES..];
114
+ if hdr.telemetry_bytes as usize % TELEMETRY_FIXED_BYTES != 0
115
+ || hdr.telemetry_count as usize != hdr.telemetry_bytes as usize / TELEMETRY_FIXED_BYTES
116
+ || hdr.telemetry_bytes as usize > rest.len()
117
+ || hdr.payload_bytes as usize != rest.len() - hdr.telemetry_bytes as usize
118
+ {
119
+ return Err(CodecError::Invalid(
120
+ "distributed result telemetry metadata mismatch",
121
+ ));
122
+ }
123
+ let (tel, rest) = rest.split_at(hdr.telemetry_bytes as usize);
124
+ let mut telemetry = Vec::new();
125
+ for chunk in tel.chunks_exact(TELEMETRY_FIXED_BYTES) {
126
+ telemetry.push(Telemetry::decode(chunk)?);
127
+ }
128
+ if rest.len() != hdr.payload_bytes as usize {
129
+ return Err(CodecError::Truncated);
130
+ }
131
+ Ok(ResultBody {
132
+ hdr,
133
+ telemetry,
134
+ payload: rest.to_vec(),
135
+ })
136
+ }
137
+
138
+ pub fn result_request_id(hdr: &ResultHdr) -> u64 {
139
+ u64_from_halves(hdr.request_hi, hdr.request_lo)
140
+ }
141
+
142
+ pub fn result_hash(hdr: &ResultHdr) -> u64 {
143
+ u64_from_halves(hdr.result_hash_hi, hdr.result_hash_lo)
144
+ }
145
+
146
+ pub fn encode_logits_payload(logits: &[f32]) -> Vec<u8> {
147
+ encode_activation(logits, 32).unwrap_or_default()
148
+ }
149
+
150
+ pub fn decode_logits_payload(buf: &[u8]) -> Result<Vec<f32>, CodecError> {
151
+ decode_activation(buf, 32).ok_or(CodecError::Invalid("invalid logits payload"))
152
+ }
153
+
154
+ pub fn work_with_ids(mut work: Work, session: u64, request: u64, prefix: u64, result: u64) -> Work {
155
+ let (shi, slo) = u64_to_halves(session);
156
+ let (rhi, rlo) = u64_to_halves(request);
157
+ let (phi, plo) = u64_to_halves(prefix);
158
+ let (ehi, elo) = u64_to_halves(result);
159
+ work.session_hi = shi;
160
+ work.session_lo = slo;
161
+ work.request_hi = rhi;
162
+ work.request_lo = rlo;
163
+ work.prefix_hash_hi = phi;
164
+ work.prefix_hash_lo = plo;
165
+ work.result_hash_hi = ehi;
166
+ work.result_hash_lo = elo;
167
+ work
168
+ }
169
+
170
+ pub fn ok_result_hdr(request: u64, hash: u64, kind: u32, bits: u32) -> ResultHdr {
171
+ let (rhi, rlo) = u64_to_halves(request);
172
+ let (hhi, hlo) = u64_to_halves(hash);
173
+ ResultHdr {
174
+ request_hi: rhi,
175
+ request_lo: rlo,
176
+ result_hash_hi: hhi,
177
+ result_hash_lo: hlo,
178
+ status: 0,
179
+ result_kind: kind,
180
+ telemetry_count: 0,
181
+ telemetry_bytes: 0,
182
+ payload_bytes: 0,
183
+ payload_bits: bits,
184
+ }
185
+ }
186
+
187
+ pub fn error_result_frame(request: u64, msg: &str) -> Vec<u8> {
188
+ let (rhi, rlo) = u64_to_halves(request);
189
+ let payload = msg.as_bytes().to_vec();
190
+ let body = ResultBody {
191
+ hdr: ResultHdr {
192
+ request_hi: rhi,
193
+ request_lo: rlo,
194
+ result_hash_hi: 0,
195
+ result_hash_lo: 0,
196
+ status: 1,
197
+ result_kind: 0,
198
+ telemetry_count: 0,
199
+ telemetry_bytes: 0,
200
+ payload_bytes: payload.len() as u32,
201
+ payload_bits: 0,
202
+ },
203
+ telemetry: Vec::new(),
204
+ payload,
205
+ };
206
+ encode_result_frame(&body).unwrap_or_else(|_| crate::codec::encode_error_frame(msg))
207
+ }
crates/ds4-dist/src/worker.rs ADDED
@@ -0,0 +1,424 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Worker control loop: HELLO, WORK parse/validate, slice eval, RESULT.
2
+
3
+ use std::collections::HashMap;
4
+ use std::io::{Read, Write};
5
+ use std::net::TcpStream;
6
+ use std::sync::{Arc, Mutex};
7
+
8
+ use crate::activation::{bits_or_default, wire_bytes};
9
+ use crate::codec::{
10
+ decode_hello_payload, encode_hello_payload, u64_from_halves, CodecError, Hello, Work,
11
+ MSG_ERROR, MSG_HELLO, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK, RESULT_ACK,
12
+ RESULT_HIDDEN_STATE, RESULT_LOGITS, ROUTE_F_OUTPUT_LOGITS, ROUTE_RETURN_UPSTREAM,
13
+ WORK_FIXED_BYTES, WORK_F_ACK_ONLY, WORK_F_INPUT_HC, WORK_F_OUTPUT_LOGITS, WORK_F_RESET_SESSION,
14
+ WORK_F_VALID_MASK,
15
+ };
16
+ use crate::exec::{SliceExec, WorkRequest};
17
+ use crate::forward::ERR_FORWARD_HIDDEN;
18
+ use crate::hash::{token_hash_update_span, TOKEN_HASH_INIT};
19
+ use crate::hops::ForwarderPool;
20
+ use crate::native_snapshot::{dispatch_worker_snapshot, MemorySnapshotStore, SnapshotStore};
21
+ use crate::relay::{forward_work_blocking, local_work_telemetry, now_sec, usec_since};
22
+ use crate::route::{decode_route_blob, validate_route_blob};
23
+ use crate::transport::{read_frame, write_frame};
24
+ use crate::work::{
25
+ decode_work_body, encode_logits_payload, encode_result_frame, error_result_frame,
26
+ ok_result_hdr, ResultBody, WorkBody,
27
+ };
28
+ use crate::worker_snapshot::WorkerSnapshotIdentity;
29
+
30
+ pub fn send_hello<W: Write>(w: &mut W, hello: &Hello, model_name: &str) -> std::io::Result<()> {
31
+ let payload = encode_hello_payload(hello, model_name)
32
+ .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e.to_string()))?;
33
+ write_frame(w, MSG_HELLO, &payload)
34
+ }
35
+
36
+ pub fn recv_hello<R: Read>(r: &mut R) -> std::io::Result<(Hello, String)> {
37
+ let (typ, body) = read_frame(r)?;
38
+ if typ != MSG_HELLO {
39
+ return Err(std::io::Error::new(
40
+ std::io::ErrorKind::InvalidData,
41
+ format!("expected HELLO frame, got type {typ}"),
42
+ ));
43
+ }
44
+ decode_hello_payload(&body)
45
+ .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))
46
+ }
47
+
48
+ pub struct Worker<E: SliceExec, S: SnapshotStore = MemorySnapshotStore> {
49
+ pub exec: E,
50
+ sessions: HashMap<u64, u64>,
51
+ store: S,
52
+ hops: ForwarderPool,
53
+ }
54
+
55
+ impl<E: SliceExec> Worker<E> {
56
+ pub fn new(exec: E) -> Self {
57
+ Self::with_store(exec, MemorySnapshotStore::new())
58
+ }
59
+ }
60
+
61
+ impl<E: SliceExec, S: SnapshotStore> Worker<E, S> {
62
+ pub fn with_store(exec: E, store: S) -> Self {
63
+ Self {
64
+ exec,
65
+ sessions: HashMap::new(),
66
+ store,
67
+ hops: ForwarderPool::new(),
68
+ }
69
+ }
70
+
71
+ pub fn store(&self) -> &S {
72
+ &self.store
73
+ }
74
+
75
+ pub fn store_mut(&mut self) -> &mut S {
76
+ &mut self.store
77
+ }
78
+
79
+ pub fn session_count(&self) -> usize {
80
+ self.sessions.len()
81
+ }
82
+
83
+ pub fn clear_sessions(&mut self) -> u32 {
84
+ let n = u32::try_from(self.sessions.len()).unwrap_or(u32::MAX);
85
+ self.sessions.clear();
86
+ n
87
+ }
88
+
89
+ pub(crate) fn apply_snapshot<Stream: Read + Write>(
90
+ &mut self,
91
+ stream: &mut Stream,
92
+ typ: u32,
93
+ body: &[u8],
94
+ ) -> std::io::Result<()> {
95
+ match dispatch_worker_snapshot(
96
+ stream,
97
+ typ,
98
+ body,
99
+ self.snapshot_identity(),
100
+ self.exec.vocab(),
101
+ &mut self.store,
102
+ ) {
103
+ Ok(Some((session_id, token_hash))) => {
104
+ self.sessions.insert(session_id, token_hash);
105
+ }
106
+ Ok(None) => {}
107
+ Err(_) => {}
108
+ }
109
+ Ok(())
110
+ }
111
+
112
+ fn snapshot_identity(&self) -> WorkerSnapshotIdentity {
113
+ WorkerSnapshotIdentity {
114
+ model_id: self.exec.model_id(),
115
+ layer_start: self.exec.layer_start(),
116
+ layer_end: self.exec.layer_end(),
117
+ ctx_size: self.exec.ctx_size(),
118
+ }
119
+ }
120
+
121
+ pub fn hello(&self, listen_port: u32, ctx_size: u32, model_name: &str) -> (Hello, String) {
122
+ (
123
+ Hello {
124
+ model_id: self.exec.model_id(),
125
+ quant_bits: 0,
126
+ layer_start: self.exec.layer_start(),
127
+ layer_end: self.exec.layer_end(),
128
+ has_output: if self.exec.has_output() { 1 } else { 0 },
129
+ has_hidden: 1,
130
+ ctx_size,
131
+ n_layers: self.exec.n_layers(),
132
+ listen_port,
133
+ model_name_len: 0,
134
+ },
135
+ model_name.to_string(),
136
+ )
137
+ }
138
+
139
+ pub(crate) fn bind_hops(&mut self, upstream: Arc<Mutex<TcpStream>>) {
140
+ self.hops.bind(upstream);
141
+ }
142
+
143
+ pub(crate) fn shutdown_hops(&mut self) {
144
+ self.hops.shutdown();
145
+ }
146
+
147
+ pub fn serve(&mut self, stream: &mut TcpStream) -> std::io::Result<()> {
148
+ if let Ok(clone) = stream.try_clone() {
149
+ self.bind_hops(Arc::new(Mutex::new(clone)));
150
+ }
151
+ let rc = self.drive(stream);
152
+ self.shutdown_hops();
153
+ rc
154
+ }
155
+
156
+ pub fn serve_once(&mut self, stream: &mut TcpStream) -> std::io::Result<()> {
157
+ self.drive_one(stream)
158
+ }
159
+
160
+ fn drive<Stream: Read + Write>(&mut self, stream: &mut Stream) -> std::io::Result<()> {
161
+ loop {
162
+ match self.drive_one(stream) {
163
+ Ok(()) => {}
164
+ Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(()),
165
+ Err(e) => return Err(e),
166
+ }
167
+ }
168
+ }
169
+
170
+ fn drive_one<Stream: Read + Write>(&mut self, stream: &mut Stream) -> std::io::Result<()> {
171
+ let (typ, body) = read_frame(stream)?;
172
+ if typ == MSG_ERROR {
173
+ return Err(std::io::Error::new(
174
+ std::io::ErrorKind::Other,
175
+ format!("coordinator error: {}", String::from_utf8_lossy(&body)),
176
+ ));
177
+ }
178
+ match typ {
179
+ MSG_WORK => {
180
+ let reply = self.process_work(&body, stream)?;
181
+ if let Some(frame) = reply {
182
+ stream.write_all(&frame)?;
183
+ }
184
+ Ok(())
185
+ }
186
+ MSG_SNAPSHOT_SAVE_REQ | MSG_SNAPSHOT_LOAD_BEGIN => {
187
+ self.apply_snapshot(stream, typ, &body)
188
+ }
189
+ _ => {
190
+ write_frame(stream, MSG_ERROR, b"unsupported distributed worker frame")?;
191
+ Err(std::io::Error::new(
192
+ std::io::ErrorKind::InvalidData,
193
+ format!("rejected unsupported frame type {typ}"),
194
+ ))
195
+ }
196
+ }
197
+ }
198
+
199
+ pub(crate) fn process_work<W: Write>(
200
+ &mut self,
201
+ payload: &[u8],
202
+ stream: &mut W,
203
+ ) -> std::io::Result<Option<Vec<u8>>> {
204
+ if payload.len() < WORK_FIXED_BYTES {
205
+ return Ok(Some(error_result_frame(
206
+ 0,
207
+ "truncated distributed WORK frame",
208
+ )));
209
+ }
210
+ let request_id = match Work::decode(payload) {
211
+ Ok(w) => u64_from_halves(w.request_hi, w.request_lo),
212
+ Err(_) => 0,
213
+ };
214
+ let body = match decode_work_body(payload) {
215
+ Ok(b) => b,
216
+ Err(e) => {
217
+ let msg = match e {
218
+ CodecError::Invalid(s) => s,
219
+ _ => "truncated distributed WORK frame",
220
+ };
221
+ return Ok(Some(error_result_frame(request_id, msg)));
222
+ }
223
+ };
224
+ match self.handle_decoded(body, stream) {
225
+ Ok(frame) => Ok(frame),
226
+ Err(msg) => Ok(Some(error_result_frame(request_id, &msg))),
227
+ }
228
+ }
229
+
230
+ fn handle_decoded<W: Write>(
231
+ &mut self,
232
+ body: WorkBody,
233
+ stream: &mut W,
234
+ ) -> Result<Option<Vec<u8>>, String> {
235
+ let work = body.work;
236
+ let request_id = u64_from_halves(work.request_hi, work.request_lo);
237
+ let session_id = u64_from_halves(work.session_hi, work.session_lo);
238
+ let prefix_hash = u64_from_halves(work.prefix_hash_hi, work.prefix_hash_lo);
239
+ let result_hash = u64_from_halves(work.result_hash_hi, work.result_hash_lo);
240
+
241
+ let fail = |m: String| -> Result<Option<Vec<u8>>, String> {
242
+ Ok(Some(error_result_frame(request_id, &m)))
243
+ };
244
+
245
+ if work.route_count == 0 {
246
+ return fail("WORK frame is missing distributed route".into());
247
+ }
248
+ if work.route_index >= work.route_count {
249
+ return fail("invalid distributed WORK route metadata".into());
250
+ }
251
+ if work.model_id != self.exec.model_id() {
252
+ return fail(format!(
253
+ "model id mismatch: work={} worker={}",
254
+ work.model_id,
255
+ self.exec.model_id()
256
+ ));
257
+ }
258
+ if work.layer_start != self.exec.layer_start() || work.layer_end != self.exec.layer_end() {
259
+ return fail(format!(
260
+ "worker is assigned layers {}:{} but request asked for {}:{}",
261
+ self.exec.layer_start(),
262
+ self.exec.layer_end(),
263
+ work.layer_start,
264
+ work.layer_end
265
+ ));
266
+ }
267
+ if (work.flags & !WORK_F_VALID_MASK) != 0 {
268
+ return fail("invalid distributed WORK flags".into());
269
+ }
270
+ if body.tokens.is_empty() {
271
+ return fail("WORK frame has no tokens".into());
272
+ }
273
+ let ctx = self.exec.ctx_size();
274
+ if work.pos0 > ctx || work.n_tokens > ctx.saturating_sub(work.pos0) {
275
+ return fail("WORK token span exceeds worker context".into());
276
+ }
277
+ let output_logits = (work.flags & WORK_F_OUTPUT_LOGITS) != 0;
278
+ let input_hc_present = (work.flags & WORK_F_INPUT_HC) != 0;
279
+ let ack_only = (work.flags & WORK_F_ACK_ONLY) != 0;
280
+ if input_hc_present && work.layer_start == 0 {
281
+ return fail("layer 0 WORK must not provide input hidden-state".into());
282
+ }
283
+ if !input_hc_present && work.layer_start != 0 {
284
+ return fail("nonzero layer WORK requires input hidden-state".into());
285
+ }
286
+ if output_logits && !self.exec.has_output() {
287
+ return fail("worker was not assigned the output head".into());
288
+ }
289
+ if output_logits && work.layer_end + 1 != self.exec.n_layers() {
290
+ return fail("WORK logits require final transformer layer".into());
291
+ }
292
+ for &t in &body.tokens {
293
+ if t < 0 || (t as u32) >= self.exec.vocab() {
294
+ return fail("WORK token id is outside the model vocabulary".into());
295
+ }
296
+ }
297
+ if token_hash_update_span(prefix_hash, &body.tokens) != result_hash {
298
+ return fail("WORK token prefix hash metadata mismatch".into());
299
+ }
300
+
301
+ let hc_values = self.exec.hidden_values();
302
+ let expected_hc = (work.n_tokens as u64).saturating_mul(hc_values) as usize;
303
+ if input_hc_present && body.input_hc.len() != expected_hc {
304
+ return fail("input hidden-state size does not match token span".into());
305
+ }
306
+ if !input_hc_present && !body.input_hc.is_empty() {
307
+ return fail("WORK frame has hidden bytes without input flag".into());
308
+ }
309
+
310
+ if let Err(e) =
311
+ validate_route_blob(&body.route_blob, work.route_count, self.exec.n_layers())
312
+ {
313
+ return fail(e.to_string());
314
+ }
315
+ let (entries, ret) =
316
+ decode_route_blob(&body.route_blob, work.route_count).map_err(|e| e.to_string())?;
317
+ let current = entries
318
+ .get(work.route_index as usize)
319
+ .ok_or_else(|| "invalid route entry index".to_string())?;
320
+ if current.layer_start != work.layer_start || current.layer_end != work.layer_end {
321
+ return fail("WORK layer range does not match route entry".into());
322
+ }
323
+ let route_out = (current.flags & ROUTE_F_OUTPUT_LOGITS) != 0;
324
+ if route_out != output_logits {
325
+ return fail("WORK logits flag does not match route entry".into());
326
+ }
327
+ let has_next = (work.route_index + 1) < work.route_count;
328
+ if has_next && output_logits {
329
+ return fail("non-final route entry requested logits".into());
330
+ }
331
+ if !has_next && ret.kind != ROUTE_RETURN_UPSTREAM {
332
+ return fail("unsupported final result destination".into());
333
+ }
334
+
335
+ let reset = (work.flags & WORK_F_RESET_SESSION) != 0;
336
+ if reset {
337
+ self.sessions.insert(session_id, TOKEN_HASH_INIT);
338
+ }
339
+ let sess_hash = *self.sessions.entry(session_id).or_insert(TOKEN_HASH_INIT);
340
+ if sess_hash != prefix_hash {
341
+ return fail("worker KV prefix hash mismatch".into());
342
+ }
343
+
344
+ let final_ack_only = ack_only && !has_next;
345
+ let local_output = output_logits && !has_next && !final_ack_only;
346
+ let produce_hidden = !local_output && !final_ack_only;
347
+ let eval_t0 = now_sec();
348
+ let out = match self.exec.eval(&WorkRequest {
349
+ session_id,
350
+ request_id,
351
+ tokens: body.tokens.clone(),
352
+ pos0: work.pos0,
353
+ layer_start: work.layer_start,
354
+ layer_end: work.layer_end,
355
+ reset,
356
+ produce_hidden,
357
+ produce_logits: local_output,
358
+ input_hc: body.input_hc.clone(),
359
+ }) {
360
+ Ok(v) => v,
361
+ Err(e) => return fail(e),
362
+ };
363
+ let eval_usec = usec_since(eval_t0, now_sec());
364
+ self.sessions.insert(session_id, result_hash);
365
+
366
+ if has_next {
367
+ let next = entries[(work.route_index + 1) as usize].clone();
368
+ let mut fwd = body.clone();
369
+ fwd.work.layer_start = next.layer_start;
370
+ fwd.work.layer_end = next.layer_end;
371
+ fwd.work.route_index = work.route_index + 1;
372
+ fwd.work.flags |= WORK_F_INPUT_HC;
373
+ if (next.flags & ROUTE_F_OUTPUT_LOGITS) != 0 {
374
+ fwd.work.flags |= WORK_F_OUTPUT_LOGITS;
375
+ } else {
376
+ fwd.work.flags &= !WORK_F_OUTPUT_LOGITS;
377
+ }
378
+ let hidden = out.hidden.unwrap_or_default();
379
+ let output_bytes =
380
+ match wire_bytes(bits_or_default(work.input_hc_bits), hidden.len() as u64) {
381
+ Some(n) => n,
382
+ None => return fail(ERR_FORWARD_HIDDEN.into()),
383
+ };
384
+ let telemetry = local_work_telemetry(&work, eval_usec, output_bytes);
385
+ fwd.input_hc = hidden;
386
+ let hop = if self.hops.is_bound() {
387
+ self.hops.forward(&next, &fwd, telemetry)
388
+ } else {
389
+ forward_work_blocking(&next, &fwd, stream, telemetry)
390
+ };
391
+ if let Err(msg) = hop {
392
+ return fail(msg);
393
+ }
394
+ return Ok(None);
395
+ }
396
+
397
+ let (kind, payload, bits) = if final_ack_only {
398
+ (RESULT_ACK, Vec::new(), 0u32)
399
+ } else if local_output {
400
+ let logits = out
401
+ .logits
402
+ .ok_or_else(|| "slice exec returned no logits".to_string())?;
403
+ (RESULT_LOGITS, encode_logits_payload(&logits), 32)
404
+ } else {
405
+ let hidden = out
406
+ .hidden
407
+ .ok_or_else(|| "slice exec returned no hidden-state".to_string())?;
408
+ let bits = bits_or_default(work.input_hc_bits);
409
+ let wire = crate::activation::encode_activation(&hidden, bits)
410
+ .ok_or_else(|| "invalid output hidden-state size".to_string())?;
411
+ (RESULT_HIDDEN_STATE, wire, bits)
412
+ };
413
+ let mut hdr = ok_result_hdr(request_id, result_hash, kind, bits);
414
+ hdr.payload_bytes = payload.len() as u32;
415
+ let telemetry = local_work_telemetry(&work, eval_usec, payload.len() as u32);
416
+ let frame = encode_result_frame(&ResultBody {
417
+ hdr,
418
+ telemetry: vec![telemetry],
419
+ payload,
420
+ })
421
+ .map_err(|e| e.to_string())?;
422
+ Ok(Some(frame))
423
+ }
424
+ }
crates/ds4-dist/src/worker_snapshot.rs ADDED
@@ -0,0 +1,291 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Worker-side DS4D snapshot save/load frame handling.
2
+
3
+ use std::io::{Read, Write};
4
+
5
+ use crate::codec::{
6
+ decode_snapshot_chunk_body, decode_snapshot_load_begin_body, encode_snapshot_begin_body,
7
+ encode_snapshot_chunk_body, encode_snapshot_done_body, u64_from_halves, u64_to_halves,
8
+ SnapshotBegin, SnapshotReq, MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK, MSG_SNAPSHOT_DONE,
9
+ SNAPSHOT_BEGIN_FIXED_BYTES, SNAPSHOT_CHUNK_BYTES, SNAPSHOT_REQ_FIXED_BYTES,
10
+ };
11
+ use crate::hash::token_hash_prefix;
12
+ use crate::transport::{read_frame, write_frame};
13
+
14
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
15
+ pub struct WorkerSnapshotIdentity {
16
+ pub model_id: u32,
17
+ pub layer_start: u32,
18
+ pub layer_end: u32,
19
+ pub ctx_size: u32,
20
+ }
21
+
22
+ fn write_begin<S: Write>(
23
+ stream: &mut S,
24
+ begin: &SnapshotBegin,
25
+ tokens: &[i32],
26
+ message: &[u8],
27
+ ) -> Result<(), String> {
28
+ let body = encode_snapshot_begin_body(begin, tokens, message).map_err(|e| e.to_string())?;
29
+ write_frame(stream, MSG_SNAPSHOT_BEGIN, &body).map_err(|e| e.to_string())
30
+ }
31
+
32
+ fn write_error_begin<S: Write>(
33
+ stream: &mut S,
34
+ identity: WorkerSnapshotIdentity,
35
+ request_id: u64,
36
+ session_id: u64,
37
+ message: &str,
38
+ ) -> Result<(), String> {
39
+ let (session_hi, session_lo) = u64_to_halves(session_id);
40
+ let (request_hi, request_lo) = u64_to_halves(request_id);
41
+ let begin = SnapshotBegin {
42
+ model_id: identity.model_id,
43
+ session_hi,
44
+ session_lo,
45
+ request_hi,
46
+ request_lo,
47
+ token_hash_hi: 0,
48
+ token_hash_lo: 0,
49
+ token_count: 0,
50
+ layer_start: identity.layer_start,
51
+ layer_end: identity.layer_end,
52
+ payload_hi: 0,
53
+ payload_lo: 0,
54
+ status: 1,
55
+ token_bytes: 0,
56
+ message_bytes: message.len() as u32,
57
+ };
58
+ write_begin(stream, &begin, &[], message.as_bytes())
59
+ }
60
+
61
+ fn write_done<S: Write>(
62
+ stream: &mut S,
63
+ request_id: u64,
64
+ status: u32,
65
+ message: &str,
66
+ ) -> Result<(), String> {
67
+ let body = encode_snapshot_done_body(request_id, status, message.as_bytes())
68
+ .map_err(|e| e.to_string())?;
69
+ write_frame(stream, MSG_SNAPSHOT_DONE, &body).map_err(|e| e.to_string())
70
+ }
71
+
72
+ fn write_chunks<S: Write, R: Read>(
73
+ stream: &mut S,
74
+ request_id: u64,
75
+ payload: &mut R,
76
+ mut remaining: u64,
77
+ ) -> Result<(), String> {
78
+ let mut buf = Vec::new();
79
+ if remaining != 0 {
80
+ buf.try_reserve_exact(SNAPSHOT_CHUNK_BYTES)
81
+ .map_err(|_| "out of memory sending worker KV shard".to_string())?;
82
+ buf.resize(SNAPSHOT_CHUNK_BYTES, 0);
83
+ }
84
+ while remaining != 0 {
85
+ let n = remaining.min(SNAPSHOT_CHUNK_BYTES as u64) as usize;
86
+ payload
87
+ .read_exact(&mut buf[..n])
88
+ .map_err(|_| "failed to read worker KV shard".to_string())?;
89
+ let body = encode_snapshot_chunk_body(request_id, &buf[..n]).map_err(|e| e.to_string())?;
90
+ write_frame(stream, MSG_SNAPSHOT_CHUNK, &body).map_err(|e| e.to_string())?;
91
+ remaining -= n as u64;
92
+ }
93
+ Ok(())
94
+ }
95
+
96
+ impl WorkerSnapshotIdentity {
97
+ pub fn accepts(self, model_id: u32, start: u32, end: u32, tokens: u32) -> bool {
98
+ model_id == self.model_id
99
+ && start == self.layer_start
100
+ && end == self.layer_end
101
+ && tokens <= self.ctx_size
102
+ }
103
+ }
104
+
105
+ /// Answer one SAVE_REQ body with BEGIN+CHUNK+DONE, or a C error BEGIN.
106
+ pub fn worker_handle_snapshot_save<S, R>(
107
+ stream: &mut S,
108
+ identity: WorkerSnapshotIdentity,
109
+ req_body: &[u8],
110
+ payload: Result<(&mut R, u64), String>,
111
+ ) -> Result<(), String>
112
+ where
113
+ S: Write,
114
+ R: Read,
115
+ {
116
+ if req_body.len() != SNAPSHOT_REQ_FIXED_BYTES {
117
+ return write_error_begin(
118
+ stream,
119
+ identity,
120
+ 0,
121
+ 0,
122
+ "invalid distributed snapshot save request",
123
+ );
124
+ }
125
+ let req = SnapshotReq::decode(req_body).map_err(|e| e.to_string())?;
126
+ let request_id = u64_from_halves(req.request_hi, req.request_lo);
127
+ let session_id = u64_from_halves(req.session_hi, req.session_lo);
128
+ if !identity.accepts(
129
+ req.model_id,
130
+ req.layer_start,
131
+ req.layer_end,
132
+ req.token_count,
133
+ ) {
134
+ return write_error_begin(
135
+ stream,
136
+ identity,
137
+ request_id,
138
+ session_id,
139
+ "snapshot save request does not match worker state",
140
+ );
141
+ }
142
+ let (payload, payload_bytes) = match payload {
143
+ Ok(ok) => ok,
144
+ Err(message) => {
145
+ return write_error_begin(stream, identity, request_id, session_id, &message);
146
+ }
147
+ };
148
+ let (payload_hi, payload_lo) = u64_to_halves(payload_bytes);
149
+ let begin = SnapshotBegin {
150
+ model_id: identity.model_id,
151
+ session_hi: req.session_hi,
152
+ session_lo: req.session_lo,
153
+ request_hi: req.request_hi,
154
+ request_lo: req.request_lo,
155
+ token_hash_hi: req.token_hash_hi,
156
+ token_hash_lo: req.token_hash_lo,
157
+ token_count: 0,
158
+ layer_start: identity.layer_start,
159
+ layer_end: identity.layer_end,
160
+ payload_hi,
161
+ payload_lo,
162
+ status: 0,
163
+ token_bytes: 0,
164
+ message_bytes: 0,
165
+ };
166
+ write_begin(stream, &begin, &[], b"")?;
167
+ write_chunks(stream, request_id, payload, payload_bytes)?;
168
+ write_done(stream, request_id, 0, "")
169
+ }
170
+
171
+ /// Consume one LOAD_BEGIN body plus following CHUNKs, then send DONE.
172
+ pub fn worker_handle_snapshot_load<S, W>(
173
+ stream: &mut S,
174
+ identity: WorkerSnapshotIdentity,
175
+ begin_body: &[u8],
176
+ vocab_size: u32,
177
+ payload: &mut W,
178
+ ) -> Result<WorkerLoadOffer, String>
179
+ where
180
+ S: Read + Write,
181
+ W: Write,
182
+ {
183
+ worker_handle_snapshot_load_restore(stream, identity, begin_body, vocab_size, payload, |_| {
184
+ Ok(())
185
+ })
186
+ }
187
+
188
+ /// Same as [`worker_handle_snapshot_load`], but runs `restore` before DONE.
189
+ pub fn worker_handle_snapshot_load_restore<S, W, F>(
190
+ stream: &mut S,
191
+ identity: WorkerSnapshotIdentity,
192
+ begin_body: &[u8],
193
+ vocab_size: u32,
194
+ payload: &mut W,
195
+ restore: F,
196
+ ) -> Result<WorkerLoadOffer, String>
197
+ where
198
+ S: Read + Write,
199
+ W: Write,
200
+ F: FnOnce(&WorkerLoadOffer) -> Result<(), String>,
201
+ {
202
+ if begin_body.len() < SNAPSHOT_BEGIN_FIXED_BYTES {
203
+ return Err("invalid distributed snapshot load header".into());
204
+ }
205
+
206
+ let Ok((begin, tokens, _)) = decode_snapshot_load_begin_body(begin_body) else {
207
+ write_done(stream, 0, 1, "invalid distributed snapshot load header")?;
208
+ return Err("invalid distributed snapshot load header".into());
209
+ };
210
+ let request_id = u64_from_halves(begin.request_hi, begin.request_lo);
211
+ let session_id = u64_from_halves(begin.session_hi, begin.session_lo);
212
+ let token_hash = u64_from_halves(begin.token_hash_hi, begin.token_hash_lo);
213
+ let payload_bytes = u64_from_halves(begin.payload_hi, begin.payload_lo);
214
+ let mut err = if tokens
215
+ .iter()
216
+ .any(|token| *token < 0 || (*token as u32) >= vocab_size)
217
+ {
218
+ "snapshot token id is outside the model vocabulary"
219
+ } else if token_hash_prefix(&tokens) != token_hash {
220
+ "snapshot load token hash mismatch"
221
+ } else if !identity.accepts(
222
+ begin.model_id,
223
+ begin.layer_start,
224
+ begin.layer_end,
225
+ begin.token_count,
226
+ ) {
227
+ "snapshot load request does not match worker state"
228
+ } else {
229
+ ""
230
+ }
231
+ .to_string();
232
+
233
+ let mut received = 0u64;
234
+ while err.is_empty() && received < payload_bytes {
235
+ let (typ, body) = match read_frame(stream) {
236
+ Ok(frame) => frame,
237
+ Err(_) => {
238
+ let _ = write_done(stream, request_id, 1, "expected distributed snapshot chunk");
239
+ return Err("expected distributed snapshot chunk".into());
240
+ }
241
+ };
242
+ if typ != MSG_SNAPSHOT_CHUNK {
243
+ err = "expected distributed snapshot chunk".into();
244
+ break;
245
+ }
246
+ match decode_snapshot_chunk_body(&body, request_id, payload_bytes - received) {
247
+ Ok((_, chunk)) => {
248
+ if payload.write_all(chunk).is_err() {
249
+ err = "failed to write worker KV shard temp file".into();
250
+ break;
251
+ }
252
+ received += chunk.len() as u64;
253
+ }
254
+ Err(_) => {
255
+ err = "invalid distributed snapshot chunk".into();
256
+ break;
257
+ }
258
+ }
259
+ }
260
+
261
+ if err.is_empty() && payload.flush().is_err() {
262
+ err = "failed to flush worker KV shard restore file".into();
263
+ }
264
+ let offer = WorkerLoadOffer {
265
+ session_id,
266
+ request_id,
267
+ token_hash,
268
+ tokens,
269
+ payload_bytes,
270
+ };
271
+ if err.is_empty() {
272
+ if let Err(message) = restore(&offer) {
273
+ err = message;
274
+ }
275
+ }
276
+ let status = u32::from(!err.is_empty());
277
+ write_done(stream, request_id, status, &err)?;
278
+ if !err.is_empty() {
279
+ return Err(err);
280
+ }
281
+ Ok(offer)
282
+ }
283
+
284
+ #[derive(Debug, Clone, PartialEq, Eq)]
285
+ pub struct WorkerLoadOffer {
286
+ pub session_id: u64,
287
+ pub request_id: u64,
288
+ pub token_hash: u64,
289
+ pub tokens: Vec<i32>,
290
+ pub payload_bytes: u64,
291
+ }
crates/ds4-dist/tests/forward.rs ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Forward-window env parse and pending-request queue.
2
+
3
+ use std::sync::mpsc;
4
+ use std::thread;
5
+ use std::time::Duration;
6
+
7
+ use ds4_dist::{
8
+ forward_window_from, opened_forwarder_message, PendingQueue, PendingRequest, Telemetry,
9
+ ERR_CLOSED_WHILE_RESULT, ERR_FORWARD, ERR_FORWARD_HIDDEN, ERR_INVALID_RESULT, ERR_NEXT_CLOSED,
10
+ ERR_OOM_FORWARDER, ERR_OOM_TRACK, ERR_RELAY_THREAD, ERR_RESULT_METADATA, ERR_RESULT_TOO_LARGE,
11
+ ERR_TELEMETRY_TOO_LARGE, FORWARD_WINDOW_DEFAULT, FORWARD_WINDOW_MAX, FORWARD_WINDOW_MIN,
12
+ };
13
+
14
+ fn pending(id: u64) -> PendingRequest {
15
+ PendingRequest {
16
+ request_id: id,
17
+ telemetry: Telemetry {
18
+ layer_start: 0,
19
+ layer_end: 1,
20
+ route_index: 0,
21
+ pos0: 0,
22
+ n_tokens: 1,
23
+ eval_usec: 0,
24
+ downstream_wait_usec: 0,
25
+ forward_send_usec: 0,
26
+ input_bytes: 0,
27
+ output_bytes: 0,
28
+ },
29
+ downstream_t0: 0.0,
30
+ }
31
+ }
32
+
33
+ #[test]
34
+ fn forward_window_matches_c() {
35
+ assert_eq!(forward_window_from(None), FORWARD_WINDOW_DEFAULT);
36
+ assert_eq!(forward_window_from(Some("")), FORWARD_WINDOW_DEFAULT);
37
+ assert_eq!(forward_window_from(Some("4")), 4);
38
+ assert_eq!(forward_window_from(Some("1")), FORWARD_WINDOW_MIN);
39
+ assert_eq!(forward_window_from(Some("64")), FORWARD_WINDOW_MAX);
40
+ assert_eq!(forward_window_from(Some("0")), FORWARD_WINDOW_DEFAULT);
41
+ assert_eq!(forward_window_from(Some("65")), FORWARD_WINDOW_DEFAULT);
42
+ assert_eq!(forward_window_from(Some("-1")), FORWARD_WINDOW_DEFAULT);
43
+ assert_eq!(forward_window_from(Some("8 ")), FORWARD_WINDOW_DEFAULT);
44
+ assert_eq!(forward_window_from(Some(" 8")), 8);
45
+ assert_eq!(forward_window_from(Some("+16")), 16);
46
+ }
47
+
48
+ #[test]
49
+ fn forwarder_strings_match_c() {
50
+ assert_eq!(
51
+ opened_forwarder_message("10.0.0.2", 7100, 4),
52
+ "ds4: distributed worker: opened pipelined worker-to-worker connection to 10.0.0.2:7100 (window 4)"
53
+ );
54
+ assert_eq!(ERR_NEXT_CLOSED, "next worker closed connection");
55
+ assert_eq!(
56
+ ERR_INVALID_RESULT,
57
+ "next worker did not return valid RESULT"
58
+ );
59
+ assert_eq!(
60
+ ERR_CLOSED_WHILE_RESULT,
61
+ "next worker closed while returning RESULT"
62
+ );
63
+ assert_eq!(ERR_RESULT_METADATA, "next worker RESULT metadata mismatch");
64
+ assert_eq!(
65
+ ERR_TELEMETRY_TOO_LARGE,
66
+ "distributed telemetry chain is too large"
67
+ );
68
+ assert_eq!(
69
+ ERR_RESULT_TOO_LARGE,
70
+ "distributed RESULT frame is too large"
71
+ );
72
+ assert_eq!(
73
+ ERR_OOM_FORWARDER,
74
+ "out of memory creating worker-to-worker forwarder"
75
+ );
76
+ assert_eq!(
77
+ ERR_RELAY_THREAD,
78
+ "failed to start worker-to-worker relay thread"
79
+ );
80
+ assert_eq!(ERR_OOM_TRACK, "out of memory tracking forwarded request");
81
+ assert_eq!(ERR_FORWARD, "failed to forward distributed work");
82
+ assert_eq!(ERR_FORWARD_HIDDEN, "invalid forwarded hidden-state size");
83
+ }
84
+
85
+ #[test]
86
+ fn pending_queue_blocks_at_window_and_pop_unblocks() {
87
+ let q = PendingQueue::new(2);
88
+ assert_eq!(q.depth(), 2);
89
+ assert!(q.enqueue(pending(1)).is_ok());
90
+ assert!(q.enqueue(pending(2)).is_ok());
91
+ let (tx, rx) = mpsc::channel();
92
+ thread::scope(|s| {
93
+ s.spawn(|| {
94
+ assert!(q.enqueue(pending(3)).is_ok());
95
+ tx.send(()).unwrap();
96
+ });
97
+ thread::sleep(Duration::from_millis(30));
98
+ assert!(rx.try_recv().is_err());
99
+ assert_eq!(q.pop().unwrap().request_id, 1);
100
+ rx.recv_timeout(Duration::from_secs(2)).unwrap();
101
+ });
102
+ assert_eq!(q.pop().unwrap().request_id, 2);
103
+ assert_eq!(q.pop().unwrap().request_id, 3);
104
+ assert!(q.pop().is_none());
105
+ }
106
+
107
+ #[test]
108
+ fn pending_queue_remove_and_note_send_done() {
109
+ let q = PendingQueue::new(4);
110
+ assert!(q.enqueue(pending(9)).is_ok());
111
+ assert!(q.enqueue(pending(10)).is_ok());
112
+ q.note_send_done(10, 42, 1.5);
113
+ assert!(q.remove(9));
114
+ let got = q.pop().unwrap();
115
+ assert_eq!(got.request_id, 10);
116
+ assert_eq!(got.telemetry.forward_send_usec, 42);
117
+ assert_eq!(got.downstream_t0, 1.5);
118
+ assert!(!q.remove(9));
119
+ }
120
+
121
+ #[test]
122
+ fn pending_queue_close_rejects_enqueue() {
123
+ let q = PendingQueue::new(2);
124
+ q.close();
125
+ assert_eq!(q.enqueue(pending(1)), Err(ERR_NEXT_CLOSED));
126
+ q.clear();
127
+ assert!(q.pop().is_none());
128
+ }
crates/ds4-dist/tests/prefetch.rs ADDED
@@ -0,0 +1,267 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Prefetch depth, job queue, reconnect strings, and pipelined serve.
2
+
3
+ use std::io;
4
+ use std::net::{TcpListener, TcpStream};
5
+ use std::sync::atomic::{AtomicU32, Ordering};
6
+ use std::sync::mpsc;
7
+ use std::thread;
8
+ use std::time::Duration;
9
+
10
+ use ds4_dist::{
11
+ cleared_sessions_message, connect_retryable, connected_message, disconnected_message,
12
+ hello_failed_message, prefetch_depth_from, prefetch_disabled_from, prefetch_enabled_message,
13
+ reconnect_with, retrying_message, send_hello, JobQueue, SliceExec, WorkOutput, WorkRequest,
14
+ Worker, ERR_OOM_QUEUE, ERR_OOM_READ, PREFETCH_DEPTH_DEFAULT, PREFETCH_DEPTH_MAX,
15
+ PREFETCH_DEPTH_MIN,
16
+ };
17
+
18
+ #[derive(Clone)]
19
+ struct MockExec {
20
+ model_id: u32,
21
+ n_layers: u32,
22
+ vocab: u32,
23
+ ctx_size: u32,
24
+ hidden_values: u64,
25
+ has_output: bool,
26
+ layer_start: u32,
27
+ layer_end: u32,
28
+ hidden: Vec<f32>,
29
+ logits: Vec<f32>,
30
+ }
31
+
32
+ impl SliceExec for MockExec {
33
+ fn model_id(&self) -> u32 {
34
+ self.model_id
35
+ }
36
+ fn n_layers(&self) -> u32 {
37
+ self.n_layers
38
+ }
39
+ fn vocab(&self) -> u32 {
40
+ self.vocab
41
+ }
42
+ fn ctx_size(&self) -> u32 {
43
+ self.ctx_size
44
+ }
45
+ fn hidden_values(&self) -> u64 {
46
+ self.hidden_values
47
+ }
48
+ fn has_output(&self) -> bool {
49
+ self.has_output
50
+ }
51
+ fn layer_start(&self) -> u32 {
52
+ self.layer_start
53
+ }
54
+ fn layer_end(&self) -> u32 {
55
+ self.layer_end
56
+ }
57
+ fn eval(&mut self, req: &WorkRequest) -> Result<WorkOutput, String> {
58
+ Ok(WorkOutput {
59
+ hidden: if req.produce_hidden {
60
+ Some(self.hidden.clone())
61
+ } else {
62
+ None
63
+ },
64
+ logits: if req.produce_logits {
65
+ Some(self.logits.clone())
66
+ } else {
67
+ None
68
+ },
69
+ })
70
+ }
71
+ }
72
+
73
+ fn mock() -> MockExec {
74
+ MockExec {
75
+ model_id: 3,
76
+ n_layers: 4,
77
+ vocab: 8,
78
+ ctx_size: 128,
79
+ hidden_values: 2,
80
+ has_output: true,
81
+ layer_start: 2,
82
+ layer_end: 3,
83
+ hidden: vec![1.0, 2.0],
84
+ logits: vec![0.1, 0.2, 0.7, 0.0],
85
+ }
86
+ }
87
+
88
+ fn tune(s: &TcpStream) {
89
+ let _ = s.set_nodelay(true);
90
+ let _ = s.set_read_timeout(Some(Duration::from_secs(5)));
91
+ let _ = s.set_write_timeout(Some(Duration::from_secs(5)));
92
+ }
93
+
94
+ #[test]
95
+ fn prefetch_depth_matches_c() {
96
+ assert_eq!(prefetch_depth_from(None), PREFETCH_DEPTH_DEFAULT);
97
+ assert_eq!(prefetch_depth_from(Some("")), PREFETCH_DEPTH_DEFAULT);
98
+ assert_eq!(prefetch_depth_from(Some("2")), 2);
99
+ assert_eq!(prefetch_depth_from(Some("1")), PREFETCH_DEPTH_MIN);
100
+ assert_eq!(prefetch_depth_from(Some("8")), PREFETCH_DEPTH_MAX);
101
+ assert_eq!(prefetch_depth_from(Some("0")), PREFETCH_DEPTH_DEFAULT);
102
+ assert_eq!(prefetch_depth_from(Some("9")), PREFETCH_DEPTH_DEFAULT);
103
+ assert_eq!(prefetch_depth_from(Some("-1")), PREFETCH_DEPTH_DEFAULT);
104
+ assert_eq!(prefetch_depth_from(Some("3 ")), PREFETCH_DEPTH_DEFAULT);
105
+ assert_eq!(prefetch_depth_from(Some(" 4")), 4);
106
+ assert_eq!(prefetch_depth_from(Some("+5")), 5);
107
+ assert_eq!(prefetch_depth_from(Some("2foo")), PREFETCH_DEPTH_DEFAULT);
108
+ }
109
+
110
+ #[test]
111
+ fn prefetch_disable_is_presence() {
112
+ assert!(!prefetch_disabled_from(None));
113
+ assert!(prefetch_disabled_from(Some(std::ffi::OsStr::new(""))));
114
+ assert!(prefetch_disabled_from(Some(std::ffi::OsStr::new("1"))));
115
+ }
116
+
117
+ #[test]
118
+ fn prefetch_and_oom_strings_match_c() {
119
+ assert_eq!(
120
+ prefetch_enabled_message(2),
121
+ "ds4: distributed worker: receive prefetch depth 2 enabled"
122
+ );
123
+ assert_eq!(ERR_OOM_QUEUE, "out of memory queueing distributed WORK");
124
+ assert_eq!(ERR_OOM_READ, "out of memory reading distributed WORK frame");
125
+ }
126
+
127
+ #[test]
128
+ fn job_queue_blocks_at_depth_and_finish_drains() {
129
+ let q = JobQueue::new(2);
130
+ assert_eq!(q.depth(), 2);
131
+ assert!(q.enqueue(vec![1]));
132
+ assert!(q.enqueue(vec![2]));
133
+ let (tx, rx) = mpsc::channel();
134
+ thread::scope(|s| {
135
+ s.spawn(|| {
136
+ assert!(q.enqueue(vec![3]));
137
+ tx.send(()).unwrap();
138
+ });
139
+ thread::sleep(Duration::from_millis(30));
140
+ assert!(rx.try_recv().is_err());
141
+ assert_eq!(q.pop(), Some(vec![1]));
142
+ rx.recv_timeout(Duration::from_secs(2)).unwrap();
143
+ });
144
+ assert_eq!(q.pop(), Some(vec![2]));
145
+ q.finish();
146
+ assert_eq!(q.pop(), Some(vec![3]));
147
+ assert_eq!(q.pop(), None);
148
+ }
149
+
150
+ #[test]
151
+ fn job_queue_cancel_drops_queued() {
152
+ let q = JobQueue::new(2);
153
+ assert!(q.enqueue(vec![9]));
154
+ q.cancel();
155
+ assert!(!q.enqueue(vec![8]));
156
+ assert_eq!(q.pop(), None);
157
+ }
158
+
159
+ #[test]
160
+ fn reconnect_messages_match_c() {
161
+ assert_eq!(
162
+ retrying_message("unable to connect to 127.0.0.1:9: Connection refused"),
163
+ "ds4: distributed worker: unable to connect to 127.0.0.1:9: Connection refused; retrying"
164
+ );
165
+ assert_eq!(
166
+ connected_message("127.0.0.1", "7000"),
167
+ "ds4: distributed worker: connected to coordinator 127.0.0.1:7000"
168
+ );
169
+ let err = io::Error::from_raw_os_error(32);
170
+ assert!(
171
+ hello_failed_message(&err).starts_with("ds4: distributed worker: failed to send HELLO:")
172
+ );
173
+ assert_eq!(
174
+ cleared_sessions_message(3),
175
+ "ds4: distributed worker: cleared 3 sessions after coordinator disconnect"
176
+ );
177
+ assert_eq!(
178
+ disconnected_message(false),
179
+ "ds4: distributed worker: coordinator disconnected; reconnecting"
180
+ );
181
+ assert_eq!(
182
+ disconnected_message(true),
183
+ "ds4: distributed worker: coordinator disconnected after error; reconnecting"
184
+ );
185
+ }
186
+
187
+ #[test]
188
+ fn connect_retryable_matches_c_errno_set() {
189
+ assert!(connect_retryable(&io::Error::from_raw_os_error(111))); // ECONNREFUSED
190
+ assert!(connect_retryable(&io::Error::from_raw_os_error(113))); // EHOSTUNREACH
191
+ assert!(connect_retryable(&io::Error::from_raw_os_error(101))); // ENETUNREACH
192
+ assert!(connect_retryable(&io::Error::from_raw_os_error(110))); // ETIMEDOUT
193
+ assert!(connect_retryable(&io::Error::from_raw_os_error(99))); // EADDRNOTAVAIL
194
+ assert!(!connect_retryable(&io::Error::from_raw_os_error(13))); // EACCES
195
+ }
196
+
197
+ #[test]
198
+ fn clear_sessions_returns_count() {
199
+ let mut worker = Worker::new(mock());
200
+ assert_eq!(worker.session_count(), 0);
201
+ assert_eq!(worker.clear_sessions(), 0);
202
+ }
203
+
204
+ #[test]
205
+ fn reconnect_retries_then_serves_and_clears() {
206
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
207
+ let addr = listener.local_addr().unwrap();
208
+ let attempts = AtomicU32::new(0);
209
+ let (ready_tx, ready_rx) = mpsc::channel();
210
+ let worker_thread = thread::spawn(move || {
211
+ let mut worker = Worker::new(mock());
212
+ let (hello, name) = worker.hello(7000, 128, "mock");
213
+ let mut sleeps = 0u32;
214
+ reconnect_with(
215
+ &mut worker,
216
+ || {
217
+ let n = attempts.fetch_add(1, Ordering::SeqCst);
218
+ if n == 0 {
219
+ Err(io::Error::new(io::ErrorKind::ConnectionRefused, "refused"))
220
+ } else {
221
+ let s = TcpStream::connect(addr)?;
222
+ tune(&s);
223
+ Ok(s)
224
+ }
225
+ },
226
+ &hello,
227
+ &name,
228
+ || {
229
+ sleeps += 1;
230
+ },
231
+ || attempts.load(Ordering::SeqCst) > 1 && ready_rx.try_recv().is_ok(),
232
+ false,
233
+ )
234
+ .unwrap();
235
+ assert_eq!(worker.session_count(), 0);
236
+ assert!(sleeps >= 1);
237
+ });
238
+
239
+ let (mut stream, _) = listener.accept().unwrap();
240
+ tune(&stream);
241
+ let _ = ds4_dist::recv_hello(&mut stream).unwrap();
242
+ ready_tx.send(()).unwrap();
243
+ drop(stream);
244
+ worker_thread.join().unwrap();
245
+ }
246
+
247
+ #[test]
248
+ fn serve_prefetch_hello_then_eof() {
249
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
250
+ let addr = listener.local_addr().unwrap();
251
+ let worker_thread = thread::spawn(move || {
252
+ let mut stream = TcpStream::connect(addr).unwrap();
253
+ tune(&stream);
254
+ let mut worker = Worker::new(mock());
255
+ let (hello, name) = worker.hello(7000, 128, "mock");
256
+ send_hello(&mut stream, &hello, &name).unwrap();
257
+ worker.serve_prefetch(&mut stream).unwrap();
258
+ assert_eq!(worker.clear_sessions(), 0);
259
+ });
260
+ let (mut stream, _) = listener.accept().unwrap();
261
+ tune(&stream);
262
+ let (hello, name) = ds4_dist::recv_hello(&mut stream).unwrap();
263
+ assert_eq!(hello.model_id, 3);
264
+ assert_eq!(name, "mock");
265
+ drop(stream);
266
+ worker_thread.join().unwrap();
267
+ }
crates/ds4-dist/tests/relay.rs ADDED
@@ -0,0 +1,296 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Persistent forwarder relay and telemetry prepend.
2
+
3
+ use std::io::Write;
4
+ use std::net::{TcpListener, TcpStream};
5
+ use std::sync::{mpsc, Arc, Mutex};
6
+ use std::thread;
7
+ use std::time::Duration;
8
+
9
+ use ds4_dist::{
10
+ decode_result_body, encode_result_frame, forward_work_blocking, local_work_telemetry,
11
+ ok_result_hdr, prepend_telemetry, read_frame, result_request_id, usec_since, Forwarder,
12
+ ForwarderPool, PendingRequest, ResultBody, RouteEntry, Telemetry, Work, WorkBody,
13
+ ERR_NEXT_CLOSED, FRAME_HEADER_BYTES, MSG_RESULT, RESULT_ACK,
14
+ };
15
+
16
+ fn pending(id: u64) -> PendingRequest {
17
+ PendingRequest {
18
+ request_id: id,
19
+ telemetry: Telemetry {
20
+ layer_start: 2,
21
+ layer_end: 3,
22
+ route_index: 1,
23
+ pos0: 0,
24
+ n_tokens: 1,
25
+ eval_usec: 11,
26
+ downstream_wait_usec: 0,
27
+ forward_send_usec: 0,
28
+ input_bytes: 8,
29
+ output_bytes: 4,
30
+ },
31
+ downstream_t0: 0.0,
32
+ }
33
+ }
34
+
35
+ fn result_frame(request_id: u64, payload: &[u8]) -> Vec<u8> {
36
+ let mut hdr = ok_result_hdr(request_id, 0, RESULT_ACK, 0);
37
+ hdr.payload_bytes = payload.len() as u32;
38
+ encode_result_frame(&ResultBody {
39
+ hdr,
40
+ telemetry: Vec::new(),
41
+ payload: payload.to_vec(),
42
+ })
43
+ .unwrap()
44
+ }
45
+
46
+ fn tune(s: &std::net::TcpStream) {
47
+ let _ = s.set_nodelay(true);
48
+ let _ = s.set_read_timeout(Some(Duration::from_secs(2)));
49
+ let _ = s.set_write_timeout(Some(Duration::from_secs(2)));
50
+ }
51
+
52
+ #[test]
53
+ fn local_work_telemetry_matches_c_fields() {
54
+ let work = Work {
55
+ layer_start: 2,
56
+ layer_end: 3,
57
+ route_index: 1,
58
+ pos0: 4,
59
+ n_tokens: 2,
60
+ token_bytes: 8,
61
+ input_hc_bytes: 16,
62
+ ..Work::default()
63
+ };
64
+ let tel = local_work_telemetry(&work, 12, 32);
65
+ assert_eq!(tel.layer_start, 2);
66
+ assert_eq!(tel.layer_end, 3);
67
+ assert_eq!(tel.route_index, 1);
68
+ assert_eq!(tel.pos0, 4);
69
+ assert_eq!(tel.n_tokens, 2);
70
+ assert_eq!(tel.eval_usec, 12);
71
+ assert_eq!(tel.downstream_wait_usec, 0);
72
+ assert_eq!(tel.forward_send_usec, 0);
73
+ assert_eq!(tel.input_bytes, 24);
74
+ assert_eq!(tel.output_bytes, 32);
75
+ }
76
+
77
+ #[test]
78
+ fn usec_since_matches_c_rounding() {
79
+ assert_eq!(usec_since(1.0, 1.0), 0);
80
+ assert_eq!(usec_since(1.0, 0.5), 0);
81
+ assert_eq!(usec_since(0.0, 0.000_001), 1);
82
+ assert_eq!(usec_since(0.0, 1.5), 1_500_000);
83
+ }
84
+
85
+ #[test]
86
+ fn prepend_telemetry_appends_local_record() {
87
+ let frame = result_frame(9, b"ok");
88
+ let body = &frame[FRAME_HEADER_BYTES..];
89
+ let local = pending(9).telemetry;
90
+ let out = prepend_telemetry(body, local).unwrap();
91
+ let (typ, payload) = {
92
+ let mut cur = std::io::Cursor::new(out);
93
+ read_frame(&mut cur).unwrap()
94
+ };
95
+ assert_eq!(typ, MSG_RESULT);
96
+ let decoded = decode_result_body(&payload).unwrap();
97
+ assert_eq!(result_request_id(&decoded.hdr), 9);
98
+ assert_eq!(decoded.payload, b"ok");
99
+ assert_eq!(decoded.telemetry.len(), 1);
100
+ assert_eq!(decoded.telemetry[0], local);
101
+ }
102
+
103
+ #[test]
104
+ fn forwarder_relays_result_and_prepends_telemetry() {
105
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
106
+ let addr = listener.local_addr().unwrap();
107
+ let (tx, rx) = mpsc::channel();
108
+ let err_tx = tx.clone();
109
+ let mut forwarder = Forwarder::connect(
110
+ "127.0.0.1",
111
+ addr.port() as u32,
112
+ 4,
113
+ move |frame| {
114
+ let _ = tx.send(Ok(frame));
115
+ },
116
+ move |id, msg| {
117
+ let _ = err_tx.send(Err((id, msg)));
118
+ },
119
+ )
120
+ .unwrap();
121
+
122
+ let next = thread::spawn(move || {
123
+ let (mut stream, _) = listener.accept().unwrap();
124
+ tune(&stream);
125
+ let (_typ, body) = read_frame(&mut stream).unwrap();
126
+ assert_eq!(body, b"WORK");
127
+ stream.write_all(&result_frame(7, b"logits")).unwrap();
128
+ });
129
+
130
+ forwarder
131
+ .send_work(pending(7), &{
132
+ let mut frame = ds4_dist::encode_frame_header(ds4_dist::MSG_WORK, 4).to_vec();
133
+ frame.extend_from_slice(b"WORK");
134
+ frame
135
+ })
136
+ .unwrap();
137
+ let frame = rx
138
+ .recv_timeout(Duration::from_secs(2))
139
+ .unwrap()
140
+ .expect("relayed RESULT");
141
+ let mut cur = std::io::Cursor::new(frame);
142
+ let (typ, payload) = read_frame(&mut cur).unwrap();
143
+ assert_eq!(typ, MSG_RESULT);
144
+ let decoded = decode_result_body(&payload).unwrap();
145
+ assert_eq!(decoded.payload, b"logits");
146
+ assert_eq!(decoded.telemetry.len(), 1);
147
+ assert_eq!(decoded.telemetry[0].layer_start, 2);
148
+ assert_eq!(decoded.telemetry[0].eval_usec, 11);
149
+ forwarder.shutdown();
150
+ next.join().unwrap();
151
+ }
152
+
153
+ fn work_frame(payload: &[u8]) -> Vec<u8> {
154
+ let mut frame =
155
+ ds4_dist::encode_frame_header(ds4_dist::MSG_WORK, payload.len() as u32).to_vec();
156
+ frame.extend_from_slice(payload);
157
+ frame
158
+ }
159
+
160
+ #[test]
161
+ fn forwarder_reports_closed_connection() {
162
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
163
+ let addr = listener.local_addr().unwrap();
164
+ let (tx, rx) = mpsc::channel();
165
+ let mut forwarder = Forwarder::connect(
166
+ "127.0.0.1",
167
+ addr.port() as u32,
168
+ 2,
169
+ move |_| {},
170
+ move |id, msg| {
171
+ let _ = tx.send((id, msg));
172
+ },
173
+ )
174
+ .unwrap();
175
+ let (stream, _) = listener.accept().unwrap();
176
+ tune(&stream);
177
+ forwarder.send_work(pending(3), &work_frame(&[1])).unwrap();
178
+ drop(stream);
179
+ let (id, msg) = rx.recv_timeout(Duration::from_secs(2)).unwrap();
180
+ assert_eq!(id, 3);
181
+ assert_eq!(msg, ERR_NEXT_CLOSED);
182
+ forwarder.shutdown();
183
+ }
184
+
185
+ #[test]
186
+ fn blocking_forward_prepends_local_telemetry() {
187
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
188
+ let addr = listener.local_addr().unwrap();
189
+ let next = RouteEntry {
190
+ host: "127.0.0.1".into(),
191
+ port: addr.port() as u32,
192
+ layer_start: 2,
193
+ layer_end: 3,
194
+ flags: 0,
195
+ };
196
+ let work = Work {
197
+ layer_start: 1,
198
+ layer_end: 1,
199
+ route_index: 0,
200
+ pos0: 4,
201
+ n_tokens: 1,
202
+ token_bytes: 4,
203
+ request_lo: 7,
204
+ ..Work::default()
205
+ };
206
+ let body = WorkBody {
207
+ work,
208
+ tokens: vec![1],
209
+ input_hc: vec![1.0, 2.0],
210
+ route_blob: Vec::new(),
211
+ };
212
+ let local = local_work_telemetry(&work, 11, 8);
213
+ let next_thread = thread::spawn(move || {
214
+ let (mut stream, _) = listener.accept().unwrap();
215
+ tune(&stream);
216
+ let (_typ, _body) = read_frame(&mut stream).unwrap();
217
+ stream.write_all(&result_frame(7, b"logits")).unwrap();
218
+ });
219
+ let mut upstream = Vec::new();
220
+ forward_work_blocking(&next, &body, &mut upstream, local).unwrap();
221
+ let mut cur = std::io::Cursor::new(upstream);
222
+ let (typ, payload) = read_frame(&mut cur).unwrap();
223
+ assert_eq!(typ, MSG_RESULT);
224
+ let decoded = decode_result_body(&payload).unwrap();
225
+ assert_eq!(decoded.payload, b"logits");
226
+ assert_eq!(decoded.telemetry.len(), 1);
227
+ assert_eq!(decoded.telemetry[0].eval_usec, 11);
228
+ assert_eq!(decoded.telemetry[0].layer_start, 1);
229
+ assert_eq!(decoded.telemetry[0].input_bytes, 4);
230
+ assert_eq!(decoded.telemetry[0].output_bytes, 8);
231
+ next_thread.join().unwrap();
232
+ }
233
+
234
+ #[test]
235
+ fn pool_reuses_forwarder_and_prepends_telemetry() {
236
+ let next_l = TcpListener::bind("127.0.0.1:0").unwrap();
237
+ let up_l = TcpListener::bind("127.0.0.1:0").unwrap();
238
+ let next_addr = next_l.local_addr().unwrap();
239
+ let up_addr = up_l.local_addr().unwrap();
240
+ let upstream = TcpStream::connect(up_addr).unwrap();
241
+ tune(&upstream);
242
+ let (mut up_peer, _) = up_l.accept().unwrap();
243
+ tune(&up_peer);
244
+ let mut pool = ForwarderPool::new();
245
+ pool.bind(Arc::new(Mutex::new(upstream)));
246
+ let next = RouteEntry {
247
+ host: "127.0.0.1".into(),
248
+ port: next_addr.port() as u32,
249
+ layer_start: 2,
250
+ layer_end: 3,
251
+ flags: 0,
252
+ };
253
+ let work = Work {
254
+ layer_start: 1,
255
+ layer_end: 1,
256
+ n_tokens: 1,
257
+ token_bytes: 4,
258
+ request_lo: 7,
259
+ ..Work::default()
260
+ };
261
+ let body = WorkBody {
262
+ work,
263
+ tokens: vec![1],
264
+ input_hc: vec![1.0, 2.0],
265
+ route_blob: Vec::new(),
266
+ };
267
+ let next_thread = thread::spawn(move || {
268
+ let (mut stream, _) = next_l.accept().unwrap();
269
+ tune(&stream);
270
+ let (_typ, _body) = read_frame(&mut stream).unwrap();
271
+ stream.write_all(&result_frame(7, b"logits")).unwrap();
272
+ let (_typ, _body) = read_frame(&mut stream).unwrap();
273
+ stream.write_all(&result_frame(8, b"again")).unwrap();
274
+ });
275
+ pool.forward(&next, &body, local_work_telemetry(&work, 11, 8))
276
+ .unwrap();
277
+ assert_eq!(pool.len(), 1);
278
+ let mut body2 = body;
279
+ body2.work.request_lo = 8;
280
+ pool.forward(&next, &body2, local_work_telemetry(&work, 12, 8))
281
+ .unwrap();
282
+ assert_eq!(pool.len(), 1);
283
+ let (typ, payload) = read_frame(&mut up_peer).unwrap();
284
+ assert_eq!(typ, MSG_RESULT);
285
+ let decoded = decode_result_body(&payload).unwrap();
286
+ assert_eq!(decoded.payload, b"logits");
287
+ assert_eq!(decoded.telemetry.len(), 1);
288
+ assert_eq!(decoded.telemetry[0].eval_usec, 11);
289
+ let (typ, payload) = read_frame(&mut up_peer).unwrap();
290
+ assert_eq!(typ, MSG_RESULT);
291
+ let decoded = decode_result_body(&payload).unwrap();
292
+ assert_eq!(decoded.payload, b"again");
293
+ assert_eq!(decoded.telemetry[0].eval_usec, 12);
294
+ pool.shutdown();
295
+ next_thread.join().unwrap();
296
+ }
crates/ds4-dist/tests/runtime.rs ADDED
@@ -0,0 +1,1257 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Coordinator/worker runtime: CLI strings, route search, HELLO/WORK/RESULT.
2
+
3
+ use ds4_dist::{
4
+ build_route_plan, decode_logits_payload, decode_result_body, dispatch_eval, encode_route_blob,
5
+ encode_work_frame, format_telemetry_line, parse_cli, parse_layers, parse_role,
6
+ prefetch_disabled_from, prepare_engine_options, read_frame, reconnect_local, register_worker,
7
+ resolved_layer_end, send_hello, serve_prefetch_local_with, token_hash_update_span,
8
+ token_span_hashes, validate_layers_for_model, validate_options, work_with_ids, write_frame,
9
+ Coordinator, CoordinatorView, EvalOutcome, JobQueue, LocalReconnect, PrefetchJob, ReturnTarget,
10
+ RouteEntry, SliceExec, Telemetry, Work, WorkBody, WorkOutput, WorkRequest, Worker, WorkerInfo,
11
+ ERR_NEXT_CLOSED, MSG_ERROR, MSG_RESULT, RESULT_LOGITS, ROUTE_F_OUTPUT_LOGITS,
12
+ ROUTE_RETURN_UPSTREAM, TOKEN_HASH_INIT, WORK_F_INPUT_HC, WORK_F_OUTPUT_LOGITS,
13
+ WORK_F_RESET_SESSION,
14
+ };
15
+ use std::io::{Read, Write};
16
+ use std::marker::PhantomData;
17
+ use std::net::{SocketAddr, TcpListener, TcpStream};
18
+ use std::sync::{mpsc, Arc, Condvar, Mutex};
19
+ use std::thread;
20
+ use std::time::Duration;
21
+
22
+ fn oracle() -> std::path::PathBuf {
23
+ if let Ok(p) = std::env::var("DS4_DIST_C_ORACLE") {
24
+ return std::path::PathBuf::from(p);
25
+ }
26
+ std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/dist_c_oracle")
27
+ }
28
+
29
+ fn require_oracle() -> std::path::PathBuf {
30
+ let p = oracle();
31
+ assert!(
32
+ p.exists(),
33
+ "build the C oracle first: make tests/parity/dist_c_oracle (missing {})",
34
+ p.display()
35
+ );
36
+ p
37
+ }
38
+
39
+ fn c_out(args: &[&str]) -> String {
40
+ let out = std::process::Command::new(require_oracle())
41
+ .args(args)
42
+ .output()
43
+ .expect("run dist_c_oracle");
44
+ assert!(
45
+ out.status.success(),
46
+ "oracle failed: {}",
47
+ String::from_utf8_lossy(&out.stderr)
48
+ );
49
+ String::from_utf8_lossy(&out.stdout).into_owned()
50
+ }
51
+
52
+ fn rust_cli(args: &[&str]) -> Result<(), String> {
53
+ let owned: Vec<String> = args.iter().map(|s| (*s).to_string()).collect();
54
+ let (opt, unmatched) = parse_cli(&owned)?;
55
+ if let Some(u) = unmatched.first() {
56
+ return Err(format!("unmatched {u}"));
57
+ }
58
+ validate_options(&opt)
59
+ }
60
+
61
+ fn rust_cli_layers(n_layers: u32, args: &[&str]) -> Result<(), String> {
62
+ let owned: Vec<String> = args.iter().map(|s| (*s).to_string()).collect();
63
+ let (opt, unmatched) = parse_cli(&owned)?;
64
+ if let Some(u) = unmatched.first() {
65
+ return Err(format!("unmatched {u}"));
66
+ }
67
+ validate_options(&opt)?;
68
+ validate_layers_for_model(&opt, n_layers)
69
+ }
70
+
71
+ fn assert_cli(args: &[&str]) {
72
+ let mut cargs = vec!["cli"];
73
+ cargs.extend_from_slice(args);
74
+ let c = c_out(&cargs);
75
+ let r = rust_cli(args);
76
+ if c == "OK" {
77
+ r.expect("rust rejected a CLI C accepts");
78
+ } else {
79
+ let msg = c.strip_prefix("ERROR:").unwrap_or(&c);
80
+ let err = r.expect_err("rust accepted a CLI C rejects");
81
+ assert_eq!(err, msg, "args={args:?}");
82
+ }
83
+ }
84
+
85
+ #[derive(Clone)]
86
+ struct MockExec {
87
+ model_id: u32,
88
+ n_layers: u32,
89
+ vocab: u32,
90
+ ctx_size: u32,
91
+ hidden_values: u64,
92
+ has_output: bool,
93
+ layer_start: u32,
94
+ layer_end: u32,
95
+ hidden: Vec<f32>,
96
+ logits: Vec<f32>,
97
+ }
98
+
99
+ impl SliceExec for MockExec {
100
+ fn model_id(&self) -> u32 {
101
+ self.model_id
102
+ }
103
+ fn n_layers(&self) -> u32 {
104
+ self.n_layers
105
+ }
106
+ fn vocab(&self) -> u32 {
107
+ self.vocab
108
+ }
109
+ fn ctx_size(&self) -> u32 {
110
+ self.ctx_size
111
+ }
112
+ fn hidden_values(&self) -> u64 {
113
+ self.hidden_values
114
+ }
115
+ fn has_output(&self) -> bool {
116
+ self.has_output
117
+ }
118
+ fn layer_start(&self) -> u32 {
119
+ self.layer_start
120
+ }
121
+ fn layer_end(&self) -> u32 {
122
+ self.layer_end
123
+ }
124
+ fn eval(&mut self, req: &WorkRequest) -> Result<WorkOutput, String> {
125
+ Ok(WorkOutput {
126
+ hidden: if req.produce_hidden {
127
+ Some(self.hidden.clone())
128
+ } else {
129
+ None
130
+ },
131
+ logits: if req.produce_logits {
132
+ Some(self.logits.clone())
133
+ } else {
134
+ None
135
+ },
136
+ })
137
+ }
138
+ }
139
+
140
+ fn tune(s: &TcpStream) {
141
+ let _ = s.set_nodelay(true);
142
+ let _ = s.set_read_timeout(Some(Duration::from_secs(5)));
143
+ let _ = s.set_write_timeout(Some(Duration::from_secs(5)));
144
+ }
145
+
146
+ #[test]
147
+ fn telemetry_line_matches_c_format() {
148
+ let tel = Telemetry {
149
+ layer_start: 2,
150
+ layer_end: 3,
151
+ route_index: 1,
152
+ pos0: 4,
153
+ n_tokens: 2,
154
+ eval_usec: 1500,
155
+ downstream_wait_usec: 250,
156
+ forward_send_usec: 100,
157
+ input_bytes: 2 * 1024 * 1024,
158
+ output_bytes: 512 * 1024,
159
+ };
160
+ assert_eq!(
161
+ format_telemetry_line(9, 0, &tel),
162
+ "ds4: distributed telemetry: request=9 hop=0 layers=2:3 route=1 pos=4 tokens=2 eval=1.500ms downstream_wait=0.250ms forward_send=0.100ms input=2.00MiB output=0.50MiB\n"
163
+ );
164
+ }
165
+
166
+ #[test]
167
+ fn layers_and_role_match_c() {
168
+ assert_eq!(c_out(&["layers", "0:1"]), "start=0 end=1 has_output=0");
169
+ assert_eq!(
170
+ c_out(&["layers", "21:output"]),
171
+ "start=21 end=4294967295 has_output=1"
172
+ );
173
+ let l = parse_layers("21:output").unwrap();
174
+ assert_eq!(l.start, 21);
175
+ assert!(l.has_output);
176
+ assert_eq!(resolved_layer_end(&l, 53), 52);
177
+ assert_eq!(c_out(&["layers", "foo"]), "ERROR:expected A:B or A:output");
178
+ assert_eq!(parse_layers("foo").unwrap_err(), "expected A:B or A:output");
179
+ assert_eq!(
180
+ c_out(&["layers", "x:1"]),
181
+ "ERROR:invalid start layer in x:1"
182
+ );
183
+ assert_eq!(
184
+ parse_layers("x:1").unwrap_err(),
185
+ "invalid start layer in x:1"
186
+ );
187
+ assert_eq!(
188
+ c_out(&["layers", "3:1"]),
189
+ "ERROR:layer range end precedes start in 3:1"
190
+ );
191
+ assert_eq!(c_out(&["role", "coordinator"]), "coordinator");
192
+ assert_eq!(parse_role("coordinator"), Some(ds4_dist::Role::Coordinator));
193
+ assert!(c_out(&["role", "boss"]).starts_with("ERROR:invalid distributed role: boss"));
194
+ assert!(parse_role("boss").is_none());
195
+ }
196
+
197
+ #[test]
198
+ fn cli_validate_matches_c() {
199
+ assert_cli(&[]);
200
+ assert_cli(&["--listen", "127.0.0.1", "7000"]);
201
+ assert_cli(&["--role", "coordinator"]);
202
+ assert_cli(&[
203
+ "--role",
204
+ "coordinator",
205
+ "--layers",
206
+ "0:1",
207
+ "--listen",
208
+ "127.0.0.1",
209
+ "7000",
210
+ ]);
211
+ assert_cli(&[
212
+ "--role",
213
+ "coordinator",
214
+ "--layers",
215
+ "0:1",
216
+ "--listen",
217
+ "127.0.0.1",
218
+ "7000",
219
+ "--coordinator",
220
+ "10.0.0.1",
221
+ "9",
222
+ ]);
223
+ assert_cli(&["--role", "worker", "--layers", "2:output"]);
224
+ assert_cli(&[
225
+ "--role",
226
+ "worker",
227
+ "--layers",
228
+ "2:output",
229
+ "--coordinator",
230
+ "127.0.0.1",
231
+ "7000",
232
+ ]);
233
+ assert_cli(&[
234
+ "--role",
235
+ "worker",
236
+ "--layers",
237
+ "2:output",
238
+ "--coordinator",
239
+ "127.0.0.1",
240
+ "7000",
241
+ "--dist-prefill-chunk",
242
+ "128",
243
+ ]);
244
+ assert_cli(&["--role", "nope"]);
245
+ assert_cli(&["--layers", "0:1:2"]);
246
+ assert_cli(&["--dist-prefill-window", "99"]);
247
+ assert_cli(&[
248
+ "--role",
249
+ "coordinator",
250
+ "--layers",
251
+ "0:1",
252
+ "--listen",
253
+ "127.0.0.1",
254
+ "0",
255
+ ]);
256
+ }
257
+
258
+ #[test]
259
+ fn layers_for_model_match_c() {
260
+ let args = [
261
+ "--role",
262
+ "coordinator",
263
+ "--layers",
264
+ "2:3",
265
+ "--listen",
266
+ "127.0.0.1",
267
+ "7000",
268
+ ];
269
+ let c = c_out(&[
270
+ "layers-for-model",
271
+ "4",
272
+ "--role",
273
+ "coordinator",
274
+ "--layers",
275
+ "2:3",
276
+ "--listen",
277
+ "127.0.0.1",
278
+ "7000",
279
+ ]);
280
+ let r = rust_cli_layers(4, &args);
281
+ assert_eq!(c, "ERROR:coordinator layer range must start at layer 0");
282
+ assert_eq!(
283
+ r.unwrap_err(),
284
+ "coordinator layer range must start at layer 0"
285
+ );
286
+
287
+ let past = rust_cli_layers(
288
+ 4,
289
+ &[
290
+ "--role",
291
+ "worker",
292
+ "--layers",
293
+ "9:9",
294
+ "--coordinator",
295
+ "127.0.0.1",
296
+ "1",
297
+ ],
298
+ );
299
+ assert_eq!(
300
+ past.unwrap_err(),
301
+ "layer range starts past final model layer 3"
302
+ );
303
+ }
304
+
305
+ #[test]
306
+ fn replay_check_requires_coordinator() {
307
+ let owned = vec![
308
+ "--role".into(),
309
+ "worker".into(),
310
+ "--layers".into(),
311
+ "2:output".into(),
312
+ "--coordinator".into(),
313
+ "127.0.0.1".into(),
314
+ "7000".into(),
315
+ "--dist-replay-check".into(),
316
+ ];
317
+ let (opt, _) = parse_cli(&owned).unwrap();
318
+ validate_options(&opt).unwrap();
319
+ assert_eq!(
320
+ prepare_engine_options(&opt).unwrap_err(),
321
+ "--dist-replay-check requires --role coordinator"
322
+ );
323
+ }
324
+
325
+ fn worker(start: u32, end: u32, output: bool) -> WorkerInfo {
326
+ WorkerInfo {
327
+ peer_host: format!("10.0.0.{end}"),
328
+ listen_port: 7000 + end,
329
+ model_id: 3,
330
+ quant_bits: 2,
331
+ layer_start: start,
332
+ layer_end: end,
333
+ has_output: output,
334
+ has_hidden: true,
335
+ }
336
+ }
337
+
338
+ #[test]
339
+ fn route_plan_local_complete_is_empty() {
340
+ let view = CoordinatorView {
341
+ n_layers: 4,
342
+ local_start: 0,
343
+ local_end: 3,
344
+ local_has_output: true,
345
+ local_can_output_head: false,
346
+ };
347
+ let plan = build_route_plan(&view, &[]).unwrap();
348
+ assert!(plan.entries.is_empty());
349
+ assert!(plan.blob.is_empty());
350
+ }
351
+
352
+ #[test]
353
+ fn route_plan_requires_layer_zero() {
354
+ let view = CoordinatorView {
355
+ n_layers: 4,
356
+ local_start: 1,
357
+ local_end: 1,
358
+ local_has_output: false,
359
+ local_can_output_head: false,
360
+ };
361
+ assert_eq!(
362
+ build_route_plan(&view, &[]).unwrap_err(),
363
+ "coordinator route does not start at layer 0"
364
+ );
365
+ }
366
+
367
+ #[test]
368
+ fn route_plan_missing_layer_message() {
369
+ let view = CoordinatorView {
370
+ n_layers: 4,
371
+ local_start: 0,
372
+ local_end: 1,
373
+ local_has_output: false,
374
+ local_can_output_head: false,
375
+ };
376
+ assert_eq!(
377
+ build_route_plan(&view, &[]).unwrap_err(),
378
+ "distributed route incomplete: missing layer 2"
379
+ );
380
+ }
381
+
382
+ #[test]
383
+ fn route_plan_prefers_output_then_longer_end() {
384
+ let view = CoordinatorView {
385
+ n_layers: 6,
386
+ local_start: 0,
387
+ local_end: 1,
388
+ local_has_output: false,
389
+ local_can_output_head: false,
390
+ };
391
+ let workers = vec![worker(2, 3, false), worker(2, 5, true), worker(2, 4, false)];
392
+ let plan = build_route_plan(&view, &workers).unwrap();
393
+ assert_eq!(plan.entries.len(), 1);
394
+ assert_eq!(plan.entries[0].layer_end, 5);
395
+ assert_ne!(plan.entries[0].flags, 0);
396
+ }
397
+
398
+ #[test]
399
+ fn register_worker_replaces_stale_and_prepends() {
400
+ let mut list = vec![worker(2, 5, true)];
401
+ list[0].peer_host = "10.0.0.9".into();
402
+ let neu = WorkerInfo {
403
+ peer_host: "10.0.0.9".into(),
404
+ listen_port: 7101,
405
+ model_id: 3,
406
+ quant_bits: 4,
407
+ layer_start: 2,
408
+ layer_end: 5,
409
+ has_output: true,
410
+ has_hidden: true,
411
+ };
412
+ register_worker(&mut list, neu.clone());
413
+ assert_eq!(list.len(), 1);
414
+ assert_eq!(list[0].listen_port, 7101);
415
+ let other = worker(2, 3, false);
416
+ register_worker(&mut list, other);
417
+ assert_eq!(list.len(), 2);
418
+ assert_eq!(list[0].layer_end, 3);
419
+ }
420
+
421
+ #[test]
422
+ fn single_hop_hello_work_logits() {
423
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
424
+ let addr = listener.local_addr().unwrap();
425
+ let logits = vec![0.1f32, 0.2, 0.7, 0.0];
426
+ let hidden = vec![1.0f32, 2.0, 3.0, 4.0];
427
+ let exec = MockExec {
428
+ model_id: 3,
429
+ n_layers: 4,
430
+ vocab: 8,
431
+ ctx_size: 128,
432
+ hidden_values: 2,
433
+ has_output: true,
434
+ layer_start: 2,
435
+ layer_end: 3,
436
+ hidden: hidden.clone(),
437
+ logits: logits.clone(),
438
+ };
439
+ let worker_thread = thread::spawn(move || {
440
+ let mut stream = TcpStream::connect(addr).unwrap();
441
+ tune(&stream);
442
+ let mut worker = Worker::new(exec);
443
+ let (hello, name) = worker.hello(7000, 128, "mock");
444
+ send_hello(&mut stream, &hello, &name).unwrap();
445
+ worker.serve(&mut stream).unwrap();
446
+ });
447
+
448
+ let (mut stream, _) = listener.accept().unwrap();
449
+ tune(&stream);
450
+ let view = CoordinatorView {
451
+ n_layers: 4,
452
+ local_start: 0,
453
+ local_end: 1,
454
+ local_has_output: false,
455
+ local_can_output_head: false,
456
+ };
457
+ let mut coord = Coordinator::new(view, 3, 32);
458
+ coord.accept_hello(&mut stream, "127.0.0.1").unwrap();
459
+ let tokens = [1i32, 2];
460
+ let out = dispatch_eval(
461
+ &coord,
462
+ &mut stream,
463
+ &tokens,
464
+ 0,
465
+ 9,
466
+ 11,
467
+ true,
468
+ TOKEN_HASH_INIT,
469
+ &hidden,
470
+ )
471
+ .unwrap();
472
+ match out {
473
+ EvalOutcome::Logits(v) => assert_eq!(v, logits),
474
+ other => panic!("expected logits, got {other:?}"),
475
+ }
476
+ drop(stream);
477
+ worker_thread.join().unwrap();
478
+ }
479
+
480
+ #[test]
481
+ fn two_hop_blocking_forward() {
482
+ let data = TcpListener::bind("127.0.0.1:0").unwrap();
483
+ let data_port = data.local_addr().unwrap().port() as u32;
484
+ let coord_l = TcpListener::bind("127.0.0.1:0").unwrap();
485
+ let coord_addr = coord_l.local_addr().unwrap();
486
+ let (ready_tx, ready_rx) = mpsc::channel::<()>();
487
+
488
+ let tail = MockExec {
489
+ model_id: 3,
490
+ n_layers: 4,
491
+ vocab: 8,
492
+ ctx_size: 128,
493
+ hidden_values: 2,
494
+ has_output: true,
495
+ layer_start: 2,
496
+ layer_end: 3,
497
+ hidden: vec![9.0, 8.0],
498
+ logits: vec![0.0, 1.0, 0.0, 0.0],
499
+ };
500
+ let mid = MockExec {
501
+ model_id: 3,
502
+ n_layers: 4,
503
+ vocab: 8,
504
+ ctx_size: 128,
505
+ hidden_values: 2,
506
+ has_output: false,
507
+ layer_start: 1,
508
+ layer_end: 1,
509
+ hidden: vec![0.5, 1.5],
510
+ logits: vec![],
511
+ };
512
+
513
+ let tail_thread = thread::spawn(move || {
514
+ let mut hello_s = TcpStream::connect(coord_addr).unwrap();
515
+ tune(&hello_s);
516
+ let mut worker = Worker::new(tail);
517
+ let (hello, name) = worker.hello(data_port, 128, "tail");
518
+ send_hello(&mut hello_s, &hello, &name).unwrap();
519
+ ready_tx.send(()).unwrap();
520
+ let (mut work_s, _) = data.accept().unwrap();
521
+ tune(&work_s);
522
+ worker.serve(&mut work_s).unwrap();
523
+ });
524
+ ready_rx.recv_timeout(Duration::from_secs(5)).unwrap();
525
+
526
+ let mid_thread = thread::spawn(move || {
527
+ let mut stream = TcpStream::connect(coord_addr).unwrap();
528
+ tune(&stream);
529
+ let mut worker = Worker::new(mid);
530
+ let (hello, name) = worker.hello(7100, 128, "mid");
531
+ send_hello(&mut stream, &hello, &name).unwrap();
532
+ worker.serve(&mut stream).unwrap();
533
+ });
534
+
535
+ let mut first = None;
536
+ let view = CoordinatorView {
537
+ n_layers: 4,
538
+ local_start: 0,
539
+ local_end: 0,
540
+ local_has_output: false,
541
+ local_can_output_head: false,
542
+ };
543
+ let mut coord = Coordinator::new(view, 3, 32);
544
+ for _ in 0..2 {
545
+ let (mut s, _) = coord_l.accept().unwrap();
546
+ tune(&s);
547
+ let (hello, _) = coord.accept_hello(&mut s, "127.0.0.1").unwrap();
548
+ if hello.layer_start == 1 {
549
+ first = Some(s);
550
+ }
551
+ }
552
+ let mut stream = first.expect("mid worker stream");
553
+ let tokens = [3i32];
554
+ let hc = vec![0.25f32, 0.75];
555
+ let out = dispatch_eval(
556
+ &coord,
557
+ &mut stream,
558
+ &tokens,
559
+ 0,
560
+ 1,
561
+ 2,
562
+ true,
563
+ TOKEN_HASH_INIT,
564
+ &hc,
565
+ )
566
+ .unwrap();
567
+ match out {
568
+ EvalOutcome::Logits(v) => assert_eq!(v, vec![0.0, 1.0, 0.0, 0.0]),
569
+ other => panic!("expected logits, got {other:?}"),
570
+ }
571
+ drop(stream);
572
+ mid_thread.join().unwrap();
573
+ tail_thread.join().unwrap();
574
+ }
575
+
576
+ #[test]
577
+ fn worker_rejects_model_mismatch_with_c_string() {
578
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
579
+ let addr = listener.local_addr().unwrap();
580
+ let exec = MockExec {
581
+ model_id: 3,
582
+ n_layers: 4,
583
+ vocab: 8,
584
+ ctx_size: 128,
585
+ hidden_values: 2,
586
+ has_output: true,
587
+ layer_start: 2,
588
+ layer_end: 3,
589
+ hidden: vec![0.0; 4],
590
+ logits: vec![1.0; 4],
591
+ };
592
+ let worker_thread = thread::spawn(move || {
593
+ let mut stream = TcpStream::connect(addr).unwrap();
594
+ tune(&stream);
595
+ let mut worker = Worker::new(exec);
596
+ let (hello, name) = worker.hello(7000, 128, "mock");
597
+ send_hello(&mut stream, &hello, &name).unwrap();
598
+ worker.serve(&mut stream).unwrap();
599
+ });
600
+ let (mut stream, _) = listener.accept().unwrap();
601
+ tune(&stream);
602
+ let view = CoordinatorView {
603
+ n_layers: 4,
604
+ local_start: 0,
605
+ local_end: 1,
606
+ local_has_output: false,
607
+ local_can_output_head: false,
608
+ };
609
+ let mut coord = Coordinator::new(view, 99, 32);
610
+ coord.accept_hello(&mut stream, "127.0.0.1").unwrap();
611
+ let err = dispatch_eval(
612
+ &coord,
613
+ &mut stream,
614
+ &[1],
615
+ 0,
616
+ 1,
617
+ 1,
618
+ true,
619
+ TOKEN_HASH_INIT,
620
+ &[0.0, 0.0],
621
+ )
622
+ .unwrap_err();
623
+ assert_eq!(err, "model id mismatch: work=99 worker=3");
624
+ drop(stream);
625
+ worker_thread.join().unwrap();
626
+ }
627
+
628
+ #[test]
629
+ fn prefix_hash_continues_committed_timeline() {
630
+ let committed = [7i32, 8];
631
+ let span = [9i32];
632
+ let (prefix, result) = token_span_hashes(&committed, &span);
633
+ assert_eq!(prefix, ds4_dist::token_hash_prefix(&committed));
634
+ assert_eq!(result, ds4_dist::token_hash_update_span(prefix, &span));
635
+ assert_ne!(prefix, TOKEN_HASH_INIT);
636
+ }
637
+
638
+ #[test]
639
+ fn usage_text_mentions_roles() {
640
+ assert!(ds4_dist::USAGE.contains("--role ROLE"));
641
+ assert!(ds4_dist::USAGE.contains("21:output"));
642
+ }
643
+
644
+ #[test]
645
+ fn open_data_listener_assigns_ephemeral_port() {
646
+ let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
647
+ assert_ne!(port, 0);
648
+ assert_eq!(listener.local_addr().unwrap().port(), port);
649
+ let _stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap();
650
+ }
651
+
652
+ #[test]
653
+ fn open_data_listener_accept_data_client_returns_client() {
654
+ let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
655
+ let connector = thread::spawn(move || TcpStream::connect(("127.0.0.1", port)).unwrap());
656
+ let client = ds4_dist::accept_data_client(&listener).unwrap();
657
+ let _peer = connector.join().unwrap();
658
+ assert!(client.nodelay().unwrap());
659
+ }
660
+
661
+ #[test]
662
+ fn open_data_listener_accept_data_client_errors_after_close() {
663
+ use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd};
664
+
665
+ let (listener, _port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
666
+ let raw = listener.into_raw_fd();
667
+ // SAFETY: `raw` is exclusively owned; OwnedFd closes it.
668
+ drop(unsafe { OwnedFd::from_raw_fd(raw) });
669
+ // SAFETY: `raw` is a closed fd. Reconstruct only so accept observes the
670
+ // error, then forget so Drop does not close twice.
671
+ let listener = unsafe { TcpListener::from_raw_fd(raw) };
672
+ let err = ds4_dist::accept_data_client(&listener);
673
+ std::mem::forget(listener);
674
+ assert!(err.is_err());
675
+ }
676
+
677
+ fn hop_logits_exec() -> (MockExec, Vec<f32>) {
678
+ let logits = vec![0.1f32, 0.2, 0.7, 0.0];
679
+ (
680
+ MockExec {
681
+ model_id: 3,
682
+ n_layers: 4,
683
+ vocab: 8,
684
+ ctx_size: 128,
685
+ hidden_values: 2,
686
+ has_output: true,
687
+ layer_start: 2,
688
+ layer_end: 3,
689
+ hidden: vec![1.0, 2.0, 3.0, 4.0],
690
+ logits: logits.clone(),
691
+ },
692
+ logits,
693
+ )
694
+ }
695
+
696
+ fn hop_work_frame(port: u16) -> Vec<u8> {
697
+ hop_work_frame_with_id(port, 11)
698
+ }
699
+
700
+ fn hop_work_frame_with_id(port: u16, request_id: u64) -> Vec<u8> {
701
+ let tokens = [1i32, 2];
702
+ let prefix = TOKEN_HASH_INIT;
703
+ let result = token_hash_update_span(prefix, &tokens);
704
+ let route_blob = encode_route_blob(
705
+ &[RouteEntry {
706
+ host: "127.0.0.1".into(),
707
+ port: u32::from(port),
708
+ layer_start: 2,
709
+ layer_end: 3,
710
+ flags: ROUTE_F_OUTPUT_LOGITS,
711
+ }],
712
+ &ReturnTarget {
713
+ kind: ROUTE_RETURN_UPSTREAM,
714
+ host: String::new(),
715
+ port: 0,
716
+ },
717
+ )
718
+ .unwrap();
719
+ let work = work_with_ids(
720
+ Work {
721
+ model_id: 3,
722
+ pos0: 0,
723
+ n_tokens: tokens.len() as u32,
724
+ layer_start: 2,
725
+ layer_end: 3,
726
+ flags: WORK_F_INPUT_HC | WORK_F_OUTPUT_LOGITS | WORK_F_RESET_SESSION,
727
+ input_hc_bits: 32,
728
+ route_count: 1,
729
+ route_index: 0,
730
+ ..Work::default()
731
+ },
732
+ 9,
733
+ request_id,
734
+ prefix,
735
+ result,
736
+ );
737
+ encode_work_frame(&WorkBody {
738
+ work,
739
+ tokens: tokens.to_vec(),
740
+ input_hc: vec![1.0, 2.0, 3.0, 4.0],
741
+ route_blob,
742
+ })
743
+ .unwrap()
744
+ }
745
+
746
+ #[test]
747
+ fn accepted_hop_serve_once_returns_result_for_work() {
748
+ // Given: a data listener and a MockExec worker that owns the final hop
749
+ let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
750
+ let frame = hop_work_frame(port);
751
+ let client = thread::spawn(move || {
752
+ let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap();
753
+ tune(&stream);
754
+ stream.write_all(&frame).unwrap();
755
+ read_frame(&mut stream).unwrap()
756
+ });
757
+
758
+ // When: accept the hop and serve one WORK
759
+ let mut hop = ds4_dist::accept_data_client(&listener).unwrap();
760
+ tune(&hop);
761
+ let (exec, logits) = hop_logits_exec();
762
+ let mut worker = Worker::new(exec);
763
+ worker.serve_once(&mut hop).unwrap();
764
+
765
+ // Then: the client reads a RESULT with the mock logits
766
+ let (typ, reply) = client.join().unwrap();
767
+ assert_eq!(typ, MSG_RESULT);
768
+ let result = decode_result_body(&reply).unwrap();
769
+ assert_eq!(result.hdr.status, 0);
770
+ assert_eq!(result.hdr.result_kind, RESULT_LOGITS);
771
+ assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits);
772
+ }
773
+
774
+ #[test]
775
+ fn accepted_hop_serve_once_unknown_frame_yields_error() {
776
+ // Given: a data listener and a MockExec worker
777
+ let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
778
+ let client = thread::spawn(move || {
779
+ let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap();
780
+ tune(&stream);
781
+ write_frame(&mut stream, 99, b"not-a-work-frame").unwrap();
782
+ read_frame(&mut stream).unwrap()
783
+ });
784
+
785
+ // When: accept the hop and serve one unknown frame
786
+ let mut hop = ds4_dist::accept_data_client(&listener).unwrap();
787
+ tune(&hop);
788
+ let (exec, _) = hop_logits_exec();
789
+ let mut worker = Worker::new(exec);
790
+ let err = worker.serve_once(&mut hop).unwrap_err();
791
+
792
+ // Then: the worker rejects the type and the client reads an ERROR frame
793
+ assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
794
+ let (typ, body) = client.join().unwrap();
795
+ assert_eq!(typ, MSG_ERROR);
796
+ assert_eq!(body, b"unsupported distributed worker frame");
797
+ }
798
+
799
+ #[test]
800
+ fn reconnect_local_keeps_coordinator_open_while_hop_work_completes() {
801
+ // Given: a coordinator TCP and a worker data listener
802
+ let coord_listener = TcpListener::bind("127.0.0.1:0").unwrap();
803
+ let coord_addr = coord_listener.local_addr().unwrap();
804
+ let (data_listener, data_port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
805
+ let frame = hop_work_frame(data_port);
806
+ let (stop_tx, stop_rx) = mpsc::channel();
807
+ let (exec, logits) = hop_logits_exec();
808
+
809
+ let worker_thread = thread::spawn(move || {
810
+ let mut worker = Worker::new(exec);
811
+ let (hello, name) = worker.hello(u32::from(data_port), 128, "mock");
812
+ let mut stopped = false;
813
+ reconnect_local(
814
+ &mut worker,
815
+ LocalReconnect {
816
+ connect: || {
817
+ let stream = TcpStream::connect(coord_addr)?;
818
+ tune(&stream);
819
+ Ok(stream)
820
+ },
821
+ hello: &hello,
822
+ model_name: &name,
823
+ sleep: || {},
824
+ should_stop: || {
825
+ stopped |= stop_rx.try_recv().is_ok();
826
+ stopped
827
+ },
828
+ listener: Some(&data_listener),
829
+ },
830
+ )
831
+ });
832
+
833
+ // When: HELLO stays on the coordinator while a hop WORK is served
834
+ let (mut coord, _) = coord_listener.accept().unwrap();
835
+ tune(&coord);
836
+ let _ = ds4_dist::recv_hello(&mut coord).unwrap();
837
+
838
+ let mut hop = TcpStream::connect(("127.0.0.1", data_port)).unwrap();
839
+ tune(&hop);
840
+ hop.write_all(&frame).unwrap();
841
+ let (typ, reply) = read_frame(&mut hop).unwrap();
842
+
843
+ // Then: hop RESULT arrives and the coordinator TCP is still open
844
+ assert_eq!(typ, MSG_RESULT);
845
+ let result = decode_result_body(&reply).unwrap();
846
+ assert_eq!(result.hdr.status, 0);
847
+ assert_eq!(result.hdr.result_kind, RESULT_LOGITS);
848
+ assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits);
849
+ assert!(coord.peer_addr().is_ok());
850
+ coord
851
+ .set_read_timeout(Some(Duration::from_millis(50)))
852
+ .unwrap();
853
+ let mut probe = [0u8; 1];
854
+ let idle = coord.read(&mut probe);
855
+ assert!(
856
+ matches!(&idle, Err(e) if e.kind() == std::io::ErrorKind::TimedOut
857
+ || e.kind() == std::io::ErrorKind::WouldBlock),
858
+ "coordinator must stay open (got {idle:?})"
859
+ );
860
+
861
+ stop_tx.send(()).unwrap();
862
+ drop(coord);
863
+ worker_thread.join().unwrap().unwrap();
864
+ }
865
+
866
+ fn middle_exec() -> MockExec {
867
+ MockExec {
868
+ model_id: 3,
869
+ n_layers: 4,
870
+ vocab: 8,
871
+ ctx_size: 128,
872
+ hidden_values: 2,
873
+ has_output: false,
874
+ layer_start: 0,
875
+ layer_end: 1,
876
+ hidden: vec![1.0, 2.0, 3.0, 4.0],
877
+ logits: Vec::new(),
878
+ }
879
+ }
880
+
881
+ fn two_hop_work_frame(middle_port: u16, next_port: u16) -> Vec<u8> {
882
+ let tokens = [1i32, 2];
883
+ let prefix = TOKEN_HASH_INIT;
884
+ let result = token_hash_update_span(prefix, &tokens);
885
+ let route_blob = encode_route_blob(
886
+ &[
887
+ RouteEntry {
888
+ host: "127.0.0.1".into(),
889
+ port: u32::from(middle_port),
890
+ layer_start: 0,
891
+ layer_end: 1,
892
+ flags: 0,
893
+ },
894
+ RouteEntry {
895
+ host: "127.0.0.1".into(),
896
+ port: u32::from(next_port),
897
+ layer_start: 2,
898
+ layer_end: 3,
899
+ flags: ROUTE_F_OUTPUT_LOGITS,
900
+ },
901
+ ],
902
+ &ReturnTarget {
903
+ kind: ROUTE_RETURN_UPSTREAM,
904
+ host: String::new(),
905
+ port: 0,
906
+ },
907
+ )
908
+ .unwrap();
909
+ let work = work_with_ids(
910
+ Work {
911
+ model_id: 3,
912
+ pos0: 0,
913
+ n_tokens: tokens.len() as u32,
914
+ layer_start: 0,
915
+ layer_end: 1,
916
+ flags: WORK_F_RESET_SESSION,
917
+ input_hc_bits: 32,
918
+ route_count: 2,
919
+ route_index: 0,
920
+ ..Work::default()
921
+ },
922
+ 9,
923
+ 11,
924
+ prefix,
925
+ result,
926
+ );
927
+ encode_work_frame(&WorkBody {
928
+ work,
929
+ tokens: tokens.to_vec(),
930
+ input_hc: Vec::new(),
931
+ route_blob,
932
+ })
933
+ .unwrap()
934
+ }
935
+
936
+ fn spawn_reconnect_worker(
937
+ exec: MockExec,
938
+ coord_addr: SocketAddr,
939
+ data_listener: TcpListener,
940
+ data_port: u16,
941
+ stop_rx: mpsc::Receiver<()>,
942
+ ) -> thread::JoinHandle<std::io::Result<()>> {
943
+ thread::spawn(move || {
944
+ let mut worker = Worker::new(exec);
945
+ let (hello, name) = worker.hello(u32::from(data_port), 128, "mock");
946
+ let mut stopped = false;
947
+ reconnect_local(
948
+ &mut worker,
949
+ LocalReconnect {
950
+ connect: || {
951
+ let stream = TcpStream::connect(coord_addr)?;
952
+ tune(&stream);
953
+ Ok(stream)
954
+ },
955
+ hello: &hello,
956
+ model_name: &name,
957
+ sleep: || {},
958
+ should_stop: || {
959
+ stopped |= stop_rx.try_recv().is_ok();
960
+ stopped
961
+ },
962
+ listener: Some(&data_listener),
963
+ },
964
+ )
965
+ })
966
+ }
967
+
968
+ fn accept_hello(listener: &TcpListener) -> TcpStream {
969
+ let (stream, _) = listener.accept().unwrap();
970
+ tune(&stream);
971
+ let mut stream = stream;
972
+ let _ = ds4_dist::recv_hello(&mut stream).unwrap();
973
+ stream
974
+ }
975
+
976
+ fn assert_c_shaped_telemetry(request_id: u64, hop: u32, tel: &Telemetry) {
977
+ let line = format_telemetry_line(request_id, hop, tel);
978
+ assert!(
979
+ line.starts_with("ds4: distributed telemetry: "),
980
+ "C telemetry prefix missing: {line:?}"
981
+ );
982
+ assert!(line.contains(&format!("request={request_id}")));
983
+ assert!(line.contains(&format!("hop={hop}")));
984
+ assert!(line.contains(&format!("layers={}:{}", tel.layer_start, tel.layer_end)));
985
+ assert!(line.contains("eval="));
986
+ assert!(line.contains("downstream_wait="));
987
+ assert!(line.contains("forward_send="));
988
+ assert!(line.contains("input="));
989
+ assert!(line.contains("output="));
990
+ assert!(line.contains("MiB"));
991
+ assert!(line.ends_with('\n'));
992
+ }
993
+
994
+ #[test]
995
+ fn reconnect_local_two_hop_result_includes_c_shaped_telemetry() {
996
+ // Given: production reconnect_local mux on a middle worker and a final hop
997
+ let coord_listener = TcpListener::bind("127.0.0.1:0").unwrap();
998
+ let coord_addr = coord_listener.local_addr().unwrap();
999
+ let (middle_listener, middle_port) =
1000
+ ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
1001
+ let (final_listener, final_port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
1002
+ let (stop_mid_tx, stop_mid_rx) = mpsc::channel();
1003
+ let (stop_fin_tx, stop_fin_rx) = mpsc::channel();
1004
+ let (final_exec, logits) = hop_logits_exec();
1005
+
1006
+ let final_thread = spawn_reconnect_worker(
1007
+ final_exec,
1008
+ coord_addr,
1009
+ final_listener,
1010
+ final_port,
1011
+ stop_fin_rx,
1012
+ );
1013
+ let middle_thread = spawn_reconnect_worker(
1014
+ middle_exec(),
1015
+ coord_addr,
1016
+ middle_listener,
1017
+ middle_port,
1018
+ stop_mid_rx,
1019
+ );
1020
+ let coord_final = accept_hello(&coord_listener);
1021
+ let coord_middle = accept_hello(&coord_listener);
1022
+
1023
+ // When: a two-hop WORK is sent on the middle worker data port
1024
+ let frame = two_hop_work_frame(middle_port, final_port);
1025
+ let mut hop = TcpStream::connect(("127.0.0.1", middle_port)).unwrap();
1026
+ tune(&hop);
1027
+ hop.write_all(&frame).unwrap();
1028
+ let (typ, reply) = read_frame(&mut hop).unwrap();
1029
+
1030
+ // Then: hop RESULT carries both hops and format_telemetry_line is C-shaped
1031
+ assert_eq!(typ, MSG_RESULT);
1032
+ let result = decode_result_body(&reply).unwrap();
1033
+ assert_eq!(result.hdr.status, 0);
1034
+ assert_eq!(result.hdr.result_kind, RESULT_LOGITS);
1035
+ assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits);
1036
+ assert_eq!(result.telemetry.len(), 2);
1037
+ assert_eq!(result.telemetry[0].layer_start, 2);
1038
+ assert_eq!(result.telemetry[0].layer_end, 3);
1039
+ assert_eq!(result.telemetry[0].route_index, 1);
1040
+ assert_eq!(result.telemetry[1].layer_start, 0);
1041
+ assert_eq!(result.telemetry[1].layer_end, 1);
1042
+ assert_eq!(result.telemetry[1].route_index, 0);
1043
+ assert_c_shaped_telemetry(11, 0, &result.telemetry[0]);
1044
+ assert_c_shaped_telemetry(11, 1, &result.telemetry[1]);
1045
+
1046
+ stop_mid_tx.send(()).unwrap();
1047
+ stop_fin_tx.send(()).unwrap();
1048
+ drop(hop);
1049
+ drop(coord_final);
1050
+ drop(coord_middle);
1051
+ middle_thread.join().unwrap().unwrap();
1052
+ final_thread.join().unwrap().unwrap();
1053
+ }
1054
+
1055
+ #[test]
1056
+ fn reconnect_local_missing_next_hop_keeps_c_forward_error() {
1057
+ // Given: a middle worker whose next hop accepts then closes
1058
+ let coord_listener = TcpListener::bind("127.0.0.1:0").unwrap();
1059
+ let coord_addr = coord_listener.local_addr().unwrap();
1060
+ let (middle_listener, middle_port) =
1061
+ ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
1062
+ let next_listener = TcpListener::bind("127.0.0.1:0").unwrap();
1063
+ let next_port = next_listener.local_addr().unwrap().port();
1064
+ let (stop_tx, stop_rx) = mpsc::channel();
1065
+ let middle_thread = spawn_reconnect_worker(
1066
+ middle_exec(),
1067
+ coord_addr,
1068
+ middle_listener,
1069
+ middle_port,
1070
+ stop_rx,
1071
+ );
1072
+ let coord = accept_hello(&coord_listener);
1073
+ let next_thread = thread::spawn(move || {
1074
+ let (stream, _) = next_listener.accept().unwrap();
1075
+ drop(stream);
1076
+ });
1077
+
1078
+ // When: two-hop WORK is forwarded to the closed next hop
1079
+ let frame = two_hop_work_frame(middle_port, next_port);
1080
+ let mut hop = TcpStream::connect(("127.0.0.1", middle_port)).unwrap();
1081
+ tune(&hop);
1082
+ hop.write_all(&frame).unwrap();
1083
+ let (typ, reply) = read_frame(&mut hop).unwrap();
1084
+
1085
+ // Then: RESULT uses the existing C forward-error string
1086
+ assert_eq!(typ, MSG_RESULT);
1087
+ let result = decode_result_body(&reply).unwrap();
1088
+ assert_ne!(result.hdr.status, 0);
1089
+ let msg = String::from_utf8_lossy(&result.payload);
1090
+ assert_eq!(msg, ERR_NEXT_CLOSED);
1091
+ assert!(!msg.contains("telemetry"));
1092
+ next_thread.join().unwrap();
1093
+
1094
+ stop_tx.send(()).unwrap();
1095
+ drop(hop);
1096
+ drop(coord);
1097
+ middle_thread.join().unwrap().unwrap();
1098
+ }
1099
+
1100
+ struct EvalGate {
1101
+ started: Mutex<u32>,
1102
+ started_cv: Condvar,
1103
+ release: Mutex<bool>,
1104
+ release_cv: Condvar,
1105
+ }
1106
+
1107
+ impl EvalGate {
1108
+ fn new() -> Arc<Self> {
1109
+ Arc::new(Self {
1110
+ started: Mutex::new(0),
1111
+ started_cv: Condvar::new(),
1112
+ release: Mutex::new(false),
1113
+ release_cv: Condvar::new(),
1114
+ })
1115
+ }
1116
+
1117
+ fn wait_started(&self) {
1118
+ let mut n = self.started.lock().expect("eval gate");
1119
+ while *n == 0 {
1120
+ n = self.started_cv.wait(n).expect("eval gate");
1121
+ }
1122
+ }
1123
+
1124
+ fn release(&self) {
1125
+ *self.release.lock().expect("eval gate") = true;
1126
+ self.release_cv.notify_all();
1127
+ }
1128
+ }
1129
+
1130
+ struct NotSendExec {
1131
+ inner: MockExec,
1132
+ gate: Arc<EvalGate>,
1133
+ _not_send: PhantomData<*const ()>,
1134
+ }
1135
+
1136
+ impl SliceExec for NotSendExec {
1137
+ fn model_id(&self) -> u32 {
1138
+ self.inner.model_id()
1139
+ }
1140
+ fn n_layers(&self) -> u32 {
1141
+ self.inner.n_layers()
1142
+ }
1143
+ fn vocab(&self) -> u32 {
1144
+ self.inner.vocab()
1145
+ }
1146
+ fn ctx_size(&self) -> u32 {
1147
+ self.inner.ctx_size()
1148
+ }
1149
+ fn hidden_values(&self) -> u64 {
1150
+ self.inner.hidden_values()
1151
+ }
1152
+ fn has_output(&self) -> bool {
1153
+ self.inner.has_output()
1154
+ }
1155
+ fn layer_start(&self) -> u32 {
1156
+ self.inner.layer_start()
1157
+ }
1158
+ fn layer_end(&self) -> u32 {
1159
+ self.inner.layer_end()
1160
+ }
1161
+ fn eval(&mut self, req: &WorkRequest) -> Result<WorkOutput, String> {
1162
+ {
1163
+ let mut n = self.gate.started.lock().expect("eval gate");
1164
+ *n += 1;
1165
+ self.gate.started_cv.notify_all();
1166
+ }
1167
+ let mut released = self.gate.release.lock().expect("eval gate");
1168
+ while !*released {
1169
+ released = self.gate.release_cv.wait(released).expect("eval gate");
1170
+ }
1171
+ self.inner.eval(req)
1172
+ }
1173
+ }
1174
+
1175
+ fn assert_local_prefetch_accepts_not_send<E, S>(
1176
+ worker: &mut Worker<E, S>,
1177
+ stream: &mut TcpStream,
1178
+ queue: &JobQueue<PrefetchJob>,
1179
+ ) -> std::io::Result<()>
1180
+ where
1181
+ E: SliceExec,
1182
+ S: ds4_dist::SnapshotStore,
1183
+ {
1184
+ serve_prefetch_local_with(worker, stream, queue)
1185
+ }
1186
+
1187
+ #[test]
1188
+ fn prefetch_env_unset_enables_local_prefetch() {
1189
+ // Given: C default is prefetch on unless DS4_DIST_DISABLE_WORKER_PREFETCH is present
1190
+ // When: the disable env is unset
1191
+ // Then: local reconnect must take the prefetch path
1192
+ assert!(!prefetch_disabled_from(None));
1193
+ assert!(ds4_dist::local_prefetch_enabled_from(None));
1194
+ }
1195
+
1196
+ #[test]
1197
+ fn prefetch_env_set_disables_local_prefetch() {
1198
+ // Given: C turns prefetch off when the disable env exists (any value)
1199
+ // When: the disable env is present
1200
+ // Then: local reconnect must not use prefetch
1201
+ assert!(prefetch_disabled_from(Some(std::ffi::OsStr::new("1"))));
1202
+ assert!(!ds4_dist::local_prefetch_enabled_from(Some(
1203
+ std::ffi::OsStr::new("1")
1204
+ )));
1205
+ }
1206
+
1207
+ #[test]
1208
+ fn serve_prefetch_local_depth_2_queues_second_work_during_not_send_eval() {
1209
+ // Given: a !Send exec and a depth-2 local prefetch queue
1210
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1211
+ let port = listener.local_addr().unwrap().port();
1212
+ let (inner, logits) = hop_logits_exec();
1213
+ let gate = EvalGate::new();
1214
+ let queue = Arc::new(JobQueue::<PrefetchJob>::new(2));
1215
+ let frame1 = hop_work_frame_with_id(port, 11);
1216
+ let frame2 = hop_work_frame_with_id(port, 12);
1217
+ let client_gate = Arc::clone(&gate);
1218
+ let client_queue = Arc::clone(&queue);
1219
+
1220
+ let client = thread::spawn(move || {
1221
+ let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap();
1222
+ tune(&stream);
1223
+ stream.write_all(&frame1).unwrap();
1224
+ client_gate.wait_started();
1225
+ stream.write_all(&frame2).unwrap();
1226
+ client_queue.wait_until_queued(1);
1227
+ let queued_during_eval = client_queue.queued();
1228
+ client_gate.release();
1229
+ let first = read_frame(&mut stream).unwrap();
1230
+ let second = read_frame(&mut stream).unwrap();
1231
+ (queued_during_eval, first, second)
1232
+ });
1233
+
1234
+ // When: session-thread prefetch serves both WORKs on a !Send worker
1235
+ let mut stream = listener.accept().unwrap().0;
1236
+ tune(&stream);
1237
+ let mut worker = Worker::new(NotSendExec {
1238
+ inner,
1239
+ gate,
1240
+ _not_send: PhantomData,
1241
+ });
1242
+ assert_local_prefetch_accepts_not_send(&mut worker, &mut stream, &queue).unwrap();
1243
+
1244
+ // Then: WORK2 was queued while eval1 was still in progress
1245
+ let (queued_during_eval, first, second) = client.join().unwrap();
1246
+ assert!(
1247
+ queued_during_eval >= 1,
1248
+ "depth 2 must queue WORK2 during eval1, queued={queued_during_eval}"
1249
+ );
1250
+ for (typ, reply) in [first, second] {
1251
+ assert_eq!(typ, MSG_RESULT);
1252
+ let result = decode_result_body(&reply).unwrap();
1253
+ assert_eq!(result.hdr.status, 0);
1254
+ assert_eq!(result.hdr.result_kind, RESULT_LOGITS);
1255
+ assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits);
1256
+ }
1257
+ }
crates/ds4-dist/tests/snapshot_store.rs ADDED
@@ -0,0 +1,214 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::io::Cursor;
2
+ use std::net::{TcpListener, TcpStream};
3
+ use std::thread;
4
+ use std::time::Duration;
5
+
6
+ use ds4_dist::{
7
+ coordinator_load_snapshot, coordinator_save_snapshot, token_hash_prefix, MemorySnapshotStore,
8
+ SliceExec, SnapshotLoad, SnapshotMeta, SnapshotSave, SnapshotStore, WorkOutput, WorkRequest,
9
+ Worker,
10
+ };
11
+
12
+ #[derive(Clone)]
13
+ struct MockExec {
14
+ model_id: u32,
15
+ n_layers: u32,
16
+ vocab: u32,
17
+ ctx_size: u32,
18
+ layer_start: u32,
19
+ layer_end: u32,
20
+ }
21
+
22
+ impl SliceExec for MockExec {
23
+ fn model_id(&self) -> u32 {
24
+ self.model_id
25
+ }
26
+ fn n_layers(&self) -> u32 {
27
+ self.n_layers
28
+ }
29
+ fn vocab(&self) -> u32 {
30
+ self.vocab
31
+ }
32
+ fn ctx_size(&self) -> u32 {
33
+ self.ctx_size
34
+ }
35
+ fn hidden_values(&self) -> u64 {
36
+ 2
37
+ }
38
+ fn has_output(&self) -> bool {
39
+ true
40
+ }
41
+ fn layer_start(&self) -> u32 {
42
+ self.layer_start
43
+ }
44
+ fn layer_end(&self) -> u32 {
45
+ self.layer_end
46
+ }
47
+ fn eval(&mut self, _req: &WorkRequest) -> Result<WorkOutput, String> {
48
+ Ok(WorkOutput {
49
+ hidden: None,
50
+ logits: Some(vec![0.0; 4]),
51
+ })
52
+ }
53
+ }
54
+
55
+ fn exec() -> MockExec {
56
+ MockExec {
57
+ model_id: 3,
58
+ n_layers: 16,
59
+ vocab: 32,
60
+ ctx_size: 128,
61
+ layer_start: 8,
62
+ layer_end: 12,
63
+ }
64
+ }
65
+
66
+ fn meta(tokens: &[i32]) -> SnapshotMeta {
67
+ SnapshotMeta {
68
+ model_id: 3,
69
+ session_id: 0x11_2233,
70
+ request_id: 0x44_5566,
71
+ token_hash: token_hash_prefix(tokens),
72
+ layer_start: 8,
73
+ layer_end: 12,
74
+ }
75
+ }
76
+
77
+ fn tune(s: &TcpStream) {
78
+ let _ = s.set_nodelay(true);
79
+ let _ = s.set_read_timeout(Some(Duration::from_secs(5)));
80
+ let _ = s.set_write_timeout(Some(Duration::from_secs(5)));
81
+ }
82
+
83
+ #[test]
84
+ fn store_save_reports_missing_session_with_c_text() {
85
+ let mut store = MemorySnapshotStore::new();
86
+ let err = store
87
+ .save(SnapshotSave {
88
+ session_id: 7,
89
+ token_count: 1,
90
+ token_hash: 1,
91
+ })
92
+ .err()
93
+ .unwrap();
94
+ assert_eq!(err, "worker has no distributed session to snapshot");
95
+ }
96
+
97
+ #[test]
98
+ fn store_save_reports_token_count_mismatch_with_c_text() {
99
+ let mut store = MemorySnapshotStore::new();
100
+ store.insert(7, vec![1, 2], b"shard").unwrap();
101
+ let err = store
102
+ .save(SnapshotSave {
103
+ session_id: 7,
104
+ token_count: 1,
105
+ token_hash: token_hash_prefix(&[1, 2]),
106
+ })
107
+ .err()
108
+ .unwrap();
109
+ assert_eq!(err, "worker snapshot token count mismatch");
110
+ }
111
+
112
+ #[test]
113
+ fn store_save_reports_token_hash_mismatch_with_c_text() {
114
+ let mut store = MemorySnapshotStore::new();
115
+ store.insert(7, vec![1, 2], b"shard").unwrap();
116
+ let err = store
117
+ .save(SnapshotSave {
118
+ session_id: 7,
119
+ token_count: 2,
120
+ token_hash: token_hash_prefix(&[1, 2]) ^ 1,
121
+ })
122
+ .err()
123
+ .unwrap();
124
+ assert_eq!(err, "worker snapshot token hash mismatch");
125
+ }
126
+
127
+ #[test]
128
+ fn store_load_sets_hash_valid_and_clears_it_on_restore_failure() {
129
+ let mut store = MemorySnapshotStore::new();
130
+ store.insert(7, vec![1], b"old").unwrap();
131
+ assert!(store.token_hash_valid(7));
132
+
133
+ let tokens = [3, 4];
134
+ let hash = token_hash_prefix(&tokens);
135
+ store
136
+ .load(
137
+ SnapshotLoad {
138
+ session_id: 7,
139
+ tokens: &tokens,
140
+ token_hash: hash,
141
+ payload_bytes: 4,
142
+ },
143
+ &mut Cursor::new(b"next"),
144
+ )
145
+ .unwrap();
146
+ assert!(store.token_hash_valid(7));
147
+ assert_eq!(store.token_hash(7), Some(hash));
148
+ assert_eq!(store.tokens(7), Some(tokens.as_slice()));
149
+ assert_eq!(store.read_shard(7).unwrap(), b"next");
150
+
151
+ store.fail_restore(true);
152
+ let err = store
153
+ .load(
154
+ SnapshotLoad {
155
+ session_id: 7,
156
+ tokens: &tokens,
157
+ token_hash: hash,
158
+ payload_bytes: 3,
159
+ },
160
+ &mut Cursor::new(b"bad"),
161
+ )
162
+ .unwrap_err();
163
+ assert_eq!(err, "failed to restore worker KV shard");
164
+ assert!(!store.token_hash_valid(7));
165
+ }
166
+
167
+ #[test]
168
+ fn worker_serve_save_load_round_trips_memory_shard() {
169
+ let tokens = [7i32, 8];
170
+ let meta = meta(&tokens);
171
+ let first = b"kv-shard-one".to_vec();
172
+ let second = b"kv-shard-two".to_vec();
173
+ let mut store = MemorySnapshotStore::new();
174
+ store
175
+ .insert(meta.session_id, tokens.to_vec(), &first)
176
+ .unwrap();
177
+
178
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
179
+ let addr = listener.local_addr().unwrap();
180
+ let worker_thread = thread::spawn(move || {
181
+ let mut stream = TcpStream::connect(addr).unwrap();
182
+ tune(&stream);
183
+ let mut worker = Worker::with_store(exec(), store);
184
+ worker.serve(&mut stream).unwrap();
185
+ });
186
+
187
+ let (mut stream, _) = listener.accept().unwrap();
188
+ tune(&stream);
189
+ let mut saved = Vec::new();
190
+ assert_eq!(
191
+ coordinator_save_snapshot(&mut stream, meta, &tokens, &mut saved).unwrap(),
192
+ first.len() as u64
193
+ );
194
+ assert_eq!(saved, first);
195
+
196
+ coordinator_load_snapshot(
197
+ &mut stream,
198
+ meta,
199
+ &tokens,
200
+ &mut Cursor::new(&second),
201
+ second.len() as u64,
202
+ )
203
+ .unwrap();
204
+
205
+ let mut saved = Vec::new();
206
+ assert_eq!(
207
+ coordinator_save_snapshot(&mut stream, meta, &tokens, &mut saved).unwrap(),
208
+ second.len() as u64
209
+ );
210
+ assert_eq!(saved, second);
211
+
212
+ drop(stream);
213
+ worker_thread.join().unwrap();
214
+ }
crates/ds4-dist/tests/snapshot_transport.rs ADDED
@@ -0,0 +1,646 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::io::{self, Cursor, Read, Write};
2
+
3
+ use ds4_dist::{
4
+ coordinator_load_snapshot, coordinator_save_snapshot, decode_snapshot_chunk_body,
5
+ decode_snapshot_load_begin_body, encode_frame_header, encode_snapshot_begin_body,
6
+ encode_snapshot_chunk_body, encode_snapshot_done_body, read_frame, token_hash_prefix,
7
+ worker_handle_snapshot_load, worker_handle_snapshot_save, write_frame, SnapshotBegin,
8
+ SnapshotMeta, SnapshotReq, WorkerSnapshotIdentity, MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK,
9
+ MSG_SNAPSHOT_DONE, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, SNAPSHOT_CHUNK_BYTES,
10
+ SNAPSHOT_REQ_FIXED_BYTES,
11
+ };
12
+
13
+ #[derive(Default)]
14
+ struct ScriptedStream {
15
+ read: Cursor<Vec<u8>>,
16
+ written: Vec<u8>,
17
+ }
18
+
19
+ impl ScriptedStream {
20
+ fn new(read: Vec<u8>) -> Self {
21
+ Self {
22
+ read: Cursor::new(read),
23
+ written: Vec::new(),
24
+ }
25
+ }
26
+ }
27
+
28
+ impl Read for ScriptedStream {
29
+ fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
30
+ self.read.read(buf)
31
+ }
32
+ }
33
+
34
+ impl Write for ScriptedStream {
35
+ fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
36
+ self.written.extend_from_slice(buf);
37
+ Ok(buf.len())
38
+ }
39
+
40
+ fn flush(&mut self) -> io::Result<()> {
41
+ Ok(())
42
+ }
43
+ }
44
+
45
+ struct RejectLargeReads {
46
+ read: Cursor<Vec<u8>>,
47
+ written: Vec<u8>,
48
+ largest_read: usize,
49
+ }
50
+
51
+ impl RejectLargeReads {
52
+ fn new(read: Vec<u8>) -> Self {
53
+ Self {
54
+ read: Cursor::new(read),
55
+ written: Vec::new(),
56
+ largest_read: 0,
57
+ }
58
+ }
59
+ }
60
+
61
+ impl Read for RejectLargeReads {
62
+ fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
63
+ self.largest_read = self.largest_read.max(buf.len());
64
+ assert!(buf.len() <= SNAPSHOT_CHUNK_BYTES);
65
+ self.read.read(buf)
66
+ }
67
+ }
68
+
69
+ impl Write for RejectLargeReads {
70
+ fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
71
+ self.written.extend_from_slice(buf);
72
+ Ok(buf.len())
73
+ }
74
+
75
+ fn flush(&mut self) -> io::Result<()> {
76
+ Ok(())
77
+ }
78
+ }
79
+
80
+ fn meta() -> SnapshotMeta {
81
+ SnapshotMeta {
82
+ model_id: 3,
83
+ session_id: 0x11_2233,
84
+ request_id: 0x44_5566,
85
+ token_hash: 0x77_8899,
86
+ layer_start: 8,
87
+ layer_end: 12,
88
+ }
89
+ }
90
+
91
+ fn begin(meta: SnapshotMeta, token_count: u32, payload_bytes: u64) -> SnapshotBegin {
92
+ let (session_hi, session_lo) = ds4_dist::u64_to_halves(meta.session_id);
93
+ let (request_hi, request_lo) = ds4_dist::u64_to_halves(meta.request_id);
94
+ let (token_hash_hi, token_hash_lo) = ds4_dist::u64_to_halves(meta.token_hash);
95
+ let (payload_hi, payload_lo) = ds4_dist::u64_to_halves(payload_bytes);
96
+ SnapshotBegin {
97
+ model_id: meta.model_id,
98
+ session_hi,
99
+ session_lo,
100
+ request_hi,
101
+ request_lo,
102
+ token_hash_hi,
103
+ token_hash_lo,
104
+ token_count,
105
+ layer_start: meta.layer_start,
106
+ layer_end: meta.layer_end,
107
+ payload_hi,
108
+ payload_lo,
109
+ status: 0,
110
+ token_bytes: token_count * 4,
111
+ message_bytes: 0,
112
+ }
113
+ }
114
+
115
+ fn push_frame(out: &mut Vec<u8>, typ: u32, body: &[u8]) {
116
+ write_frame(out, typ, body).unwrap();
117
+ }
118
+
119
+ #[test]
120
+ fn coordinator_save_follows_c_frame_order_and_lengths() {
121
+ let meta = meta();
122
+ let tokens = [7, 8];
123
+ let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3];
124
+ payload[SNAPSHOT_CHUNK_BYTES] = 1;
125
+ payload[SNAPSHOT_CHUNK_BYTES + 1] = 2;
126
+ payload[SNAPSHOT_CHUNK_BYTES + 2] = 3;
127
+ let mut scripted = Vec::new();
128
+
129
+ let mut begin_body = encode_snapshot_begin_body(
130
+ &begin(meta, tokens.len() as u32, payload.len() as u64),
131
+ &tokens,
132
+ b"",
133
+ )
134
+ .unwrap();
135
+ begin_body.extend_from_slice(b"ignored trailing begin bytes");
136
+ push_frame(&mut scripted, MSG_SNAPSHOT_BEGIN, &begin_body);
137
+ push_frame(
138
+ &mut scripted,
139
+ MSG_SNAPSHOT_CHUNK,
140
+ &encode_snapshot_chunk_body(meta.request_id, &payload[..SNAPSHOT_CHUNK_BYTES]).unwrap(),
141
+ );
142
+ push_frame(
143
+ &mut scripted,
144
+ MSG_SNAPSHOT_CHUNK,
145
+ &encode_snapshot_chunk_body(meta.request_id, &payload[SNAPSHOT_CHUNK_BYTES..]).unwrap(),
146
+ );
147
+ let mut done = encode_snapshot_done_body(meta.request_id, 0, b"").unwrap();
148
+ done.extend_from_slice(b"ignored trailing done bytes");
149
+ push_frame(&mut scripted, MSG_SNAPSHOT_DONE, &done);
150
+
151
+ let mut stream = ScriptedStream::new(scripted);
152
+ let mut saved = Vec::new();
153
+ assert_eq!(
154
+ coordinator_save_snapshot(&mut stream, meta, &tokens, &mut saved).unwrap(),
155
+ payload.len() as u64
156
+ );
157
+ assert_eq!(saved, payload);
158
+
159
+ let mut written = Cursor::new(stream.written);
160
+ let (typ, body) = read_frame(&mut written).unwrap();
161
+ assert_eq!(typ, MSG_SNAPSHOT_SAVE_REQ);
162
+ let request = SnapshotReq::decode(&body).unwrap();
163
+ assert_eq!(request.model_id, meta.model_id);
164
+ assert_eq!(
165
+ ds4_dist::u64_from_halves(request.session_hi, request.session_lo),
166
+ meta.session_id
167
+ );
168
+ assert_eq!(
169
+ ds4_dist::u64_from_halves(request.request_hi, request.request_lo),
170
+ meta.request_id
171
+ );
172
+ assert_eq!(request.token_count, tokens.len() as u32);
173
+ assert!(read_frame(&mut written).is_err());
174
+ }
175
+
176
+ #[test]
177
+ fn coordinator_save_rejects_status_metadata_and_frame_sequence_errors() {
178
+ let meta = meta();
179
+ let tokens = [7, 8];
180
+
181
+ let mut refused = begin(meta, 0, 0);
182
+ refused.status = 1;
183
+ refused.message_bytes = 12;
184
+ let body = encode_snapshot_begin_body(&refused, &[], b"refused\0tail").unwrap();
185
+ let mut frames = Vec::new();
186
+ push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
187
+ let mut saved = Vec::new();
188
+ assert_eq!(
189
+ coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
190
+ .unwrap_err(),
191
+ "refused"
192
+ );
193
+
194
+ let message = vec![b'x'; 300];
195
+ refused.message_bytes = message.len() as u32;
196
+ let body = encode_snapshot_begin_body(&refused, &[], &message).unwrap();
197
+ let mut frames = Vec::new();
198
+ push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
199
+ let mut saved = Vec::new();
200
+ assert_eq!(
201
+ coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
202
+ .unwrap_err(),
203
+ "x".repeat(255)
204
+ );
205
+
206
+ let mut wrong = begin(meta, tokens.len() as u32, 0);
207
+ wrong.layer_end += 1;
208
+ let body = encode_snapshot_begin_body(&wrong, &tokens, b"").unwrap();
209
+ let mut frames = Vec::new();
210
+ push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
211
+ let mut saved = Vec::new();
212
+ assert_eq!(
213
+ coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
214
+ .unwrap_err(),
215
+ "distributed KV shard metadata mismatch"
216
+ );
217
+
218
+ let body = encode_snapshot_begin_body(&begin(meta, 0, 4), &[], b"").unwrap();
219
+ let mut frames = Vec::new();
220
+ push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
221
+ push_frame(
222
+ &mut frames,
223
+ MSG_SNAPSHOT_CHUNK,
224
+ &encode_snapshot_chunk_body(meta.request_id, b"ab").unwrap(),
225
+ );
226
+ push_frame(
227
+ &mut frames,
228
+ MSG_SNAPSHOT_DONE,
229
+ &encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(),
230
+ );
231
+ let mut saved = Vec::new();
232
+ assert_eq!(
233
+ coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
234
+ .unwrap_err(),
235
+ "expected distributed KV shard chunk"
236
+ );
237
+
238
+ let body = encode_snapshot_begin_body(&begin(meta, 0, 2), &[], b"").unwrap();
239
+ let mut frames = Vec::new();
240
+ push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
241
+ push_frame(
242
+ &mut frames,
243
+ MSG_SNAPSHOT_CHUNK,
244
+ &encode_snapshot_chunk_body(meta.request_id, b"abc").unwrap(),
245
+ );
246
+ let mut saved = Vec::new();
247
+ assert_eq!(
248
+ coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
249
+ .unwrap_err(),
250
+ "invalid distributed KV shard chunk"
251
+ );
252
+
253
+ let body = encode_snapshot_begin_body(&begin(meta, 0, 1), &[], b"").unwrap();
254
+ let mut frames = Vec::new();
255
+ push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
256
+ push_frame(
257
+ &mut frames,
258
+ MSG_SNAPSHOT_CHUNK,
259
+ &encode_snapshot_chunk_body(meta.request_id + 1, b"a").unwrap(),
260
+ );
261
+ let mut saved = Vec::new();
262
+ assert_eq!(
263
+ coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
264
+ .unwrap_err(),
265
+ "invalid distributed KV shard chunk"
266
+ );
267
+
268
+ let body = encode_snapshot_begin_body(&begin(meta, 0, 1), &[], b"").unwrap();
269
+ let mut frames = Vec::new();
270
+ push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
271
+ for _ in 0..2 {
272
+ push_frame(
273
+ &mut frames,
274
+ MSG_SNAPSHOT_CHUNK,
275
+ &encode_snapshot_chunk_body(meta.request_id, b"a").unwrap(),
276
+ );
277
+ }
278
+ let mut saved = Vec::new();
279
+ assert_eq!(
280
+ coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
281
+ .unwrap_err(),
282
+ "distributed worker returned invalid snapshot completion frame"
283
+ );
284
+
285
+ let body = encode_snapshot_begin_body(&begin(meta, 0, 0), &[], b"").unwrap();
286
+ let mut frames = Vec::new();
287
+ push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
288
+ push_frame(
289
+ &mut frames,
290
+ MSG_SNAPSHOT_DONE,
291
+ &encode_snapshot_done_body(meta.request_id + 1, 0, b"").unwrap(),
292
+ );
293
+ let mut saved = Vec::new();
294
+ assert_eq!(
295
+ coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
296
+ .unwrap_err(),
297
+ "distributed snapshot completion request mismatch"
298
+ );
299
+ }
300
+
301
+ #[test]
302
+ fn coordinator_save_bounds_peer_lengths_and_reports_partial_stages() {
303
+ let meta = meta();
304
+
305
+ let mut stream =
306
+ RejectLargeReads::new(encode_frame_header(MSG_SNAPSHOT_BEGIN, u32::MAX).to_vec());
307
+ let mut saved = Vec::new();
308
+ assert_eq!(
309
+ coordinator_save_snapshot(&mut stream, meta, &[], &mut saved).unwrap_err(),
310
+ "failed to read distributed snapshot header"
311
+ );
312
+ assert!(stream.largest_read <= 60);
313
+
314
+ let mut partial_done = Vec::new();
315
+ push_frame(
316
+ &mut partial_done,
317
+ MSG_SNAPSHOT_BEGIN,
318
+ &encode_snapshot_begin_body(&begin(meta, 0, 0), &[], b"").unwrap(),
319
+ );
320
+ partial_done.extend_from_slice(&encode_frame_header(MSG_SNAPSHOT_DONE, 16));
321
+ partial_done.extend_from_slice(&[0; 8]);
322
+ let mut saved = Vec::new();
323
+ assert_eq!(
324
+ coordinator_save_snapshot(
325
+ &mut ScriptedStream::new(partial_done),
326
+ meta,
327
+ &[],
328
+ &mut saved,
329
+ )
330
+ .unwrap_err(),
331
+ "failed to read distributed snapshot completion"
332
+ );
333
+ }
334
+
335
+ #[test]
336
+ fn coordinator_load_chunks_payload_and_checks_completion() {
337
+ let meta = meta();
338
+ let tokens = vec![7, 8];
339
+ let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3];
340
+ payload[SNAPSHOT_CHUNK_BYTES] = 1;
341
+ payload[SNAPSHOT_CHUNK_BYTES + 1] = 2;
342
+ payload[SNAPSHOT_CHUNK_BYTES + 2] = 3;
343
+
344
+ let mut reply = Vec::new();
345
+ let mut done = encode_snapshot_done_body(meta.request_id, 0, b"").unwrap();
346
+ done.extend_from_slice(b"trailing");
347
+ push_frame(&mut reply, MSG_SNAPSHOT_DONE, &done);
348
+ let mut stream = ScriptedStream::new(reply);
349
+ let payload_len = payload.len() as u64;
350
+ coordinator_load_snapshot(
351
+ &mut stream,
352
+ meta,
353
+ &tokens,
354
+ &mut Cursor::new(&payload),
355
+ payload_len,
356
+ )
357
+ .unwrap();
358
+
359
+ let mut written = Cursor::new(stream.written);
360
+ let (typ, body) = read_frame(&mut written).unwrap();
361
+ assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN);
362
+ let (header, got_tokens, message) = decode_snapshot_load_begin_body(&body).unwrap();
363
+ assert_eq!(got_tokens, tokens);
364
+ assert!(message.is_empty());
365
+ assert_eq!(
366
+ ds4_dist::u64_from_halves(header.payload_hi, header.payload_lo),
367
+ payload.len() as u64
368
+ );
369
+
370
+ let (typ, body) = read_frame(&mut written).unwrap();
371
+ assert_eq!(typ, MSG_SNAPSHOT_CHUNK);
372
+ let (_, chunk) =
373
+ decode_snapshot_chunk_body(&body, meta.request_id, payload.len() as u64).unwrap();
374
+ assert_eq!(chunk.len(), SNAPSHOT_CHUNK_BYTES);
375
+ let (typ, body) = read_frame(&mut written).unwrap();
376
+ assert_eq!(typ, MSG_SNAPSHOT_CHUNK);
377
+ let (_, chunk) = decode_snapshot_chunk_body(&body, meta.request_id, 3).unwrap();
378
+ assert_eq!(chunk, [1, 2, 3]);
379
+ assert!(read_frame(&mut written).is_err());
380
+
381
+ let mut failed = Vec::new();
382
+ push_frame(
383
+ &mut failed,
384
+ MSG_SNAPSHOT_DONE,
385
+ &encode_snapshot_done_body(meta.request_id, 1, b"restore failed").unwrap(),
386
+ );
387
+ assert_eq!(
388
+ coordinator_load_snapshot(
389
+ &mut ScriptedStream::new(failed),
390
+ meta,
391
+ &tokens,
392
+ &mut Cursor::new(b"payload"),
393
+ 7,
394
+ )
395
+ .unwrap_err(),
396
+ "restore failed"
397
+ );
398
+ }
399
+
400
+ #[test]
401
+ fn coordinator_zero_payload_does_not_require_or_send_chunks() {
402
+ let meta = meta();
403
+
404
+ let mut save_reply = Vec::new();
405
+ push_frame(
406
+ &mut save_reply,
407
+ MSG_SNAPSHOT_BEGIN,
408
+ &encode_snapshot_begin_body(&begin(meta, 0, 0), &[], b"").unwrap(),
409
+ );
410
+ push_frame(
411
+ &mut save_reply,
412
+ MSG_SNAPSHOT_DONE,
413
+ &encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(),
414
+ );
415
+ let mut saved = Vec::new();
416
+ assert_eq!(
417
+ coordinator_save_snapshot(&mut ScriptedStream::new(save_reply), meta, &[], &mut saved,)
418
+ .unwrap(),
419
+ 0
420
+ );
421
+ assert!(saved.is_empty());
422
+
423
+ let mut reply = Vec::new();
424
+ push_frame(
425
+ &mut reply,
426
+ MSG_SNAPSHOT_DONE,
427
+ &encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(),
428
+ );
429
+ let mut stream = ScriptedStream::new(reply);
430
+ let mut source = Cursor::new(b"not part of the declared payload");
431
+ coordinator_load_snapshot(&mut stream, meta, &[], &mut source, 0).unwrap();
432
+ assert_eq!(source.position(), 0);
433
+
434
+ let mut written = Cursor::new(stream.written);
435
+ let (typ, _) = read_frame(&mut written).unwrap();
436
+ assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN);
437
+ assert!(read_frame(&mut written).is_err());
438
+ }
439
+
440
+ fn identity(meta: SnapshotMeta) -> WorkerSnapshotIdentity {
441
+ WorkerSnapshotIdentity {
442
+ model_id: meta.model_id,
443
+ layer_start: meta.layer_start,
444
+ layer_end: meta.layer_end,
445
+ ctx_size: 128,
446
+ }
447
+ }
448
+
449
+ fn hashed_meta(tokens: &[i32]) -> SnapshotMeta {
450
+ let mut meta = meta();
451
+ meta.token_hash = token_hash_prefix(tokens);
452
+ meta
453
+ }
454
+
455
+ #[cfg(unix)]
456
+ fn duplex_save(meta: SnapshotMeta, tokens: &[i32], payload: Vec<u8>) -> (Vec<u8>, u64) {
457
+ let (mut coord, mut worker) = std::os::unix::net::UnixStream::pair().unwrap();
458
+ let identity = identity(meta);
459
+ let sent = payload.clone();
460
+ let handle = std::thread::spawn(move || {
461
+ let (typ, body) = read_frame(&mut worker).unwrap();
462
+ assert_eq!(typ, MSG_SNAPSHOT_SAVE_REQ);
463
+ let mut src = Cursor::new(sent);
464
+ let len = src.get_ref().len() as u64;
465
+ worker_handle_snapshot_save(&mut worker, identity, &body, Ok((&mut src, len))).unwrap();
466
+ });
467
+ let mut saved = Vec::new();
468
+ let n = coordinator_save_snapshot(&mut coord, meta, tokens, &mut saved).unwrap();
469
+ handle.join().unwrap();
470
+ (saved, n)
471
+ }
472
+
473
+ #[cfg(unix)]
474
+ fn duplex_load(meta: SnapshotMeta, tokens: &[i32], payload: Vec<u8>) -> Vec<u8> {
475
+ let (mut coord, mut worker) = std::os::unix::net::UnixStream::pair().unwrap();
476
+ let identity = identity(meta);
477
+ let handle = std::thread::spawn(move || {
478
+ let (typ, body) = read_frame(&mut worker).unwrap();
479
+ assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN);
480
+ let mut restored = Vec::new();
481
+ let offer =
482
+ worker_handle_snapshot_load(&mut worker, identity, &body, 32, &mut restored).unwrap();
483
+ (restored, offer)
484
+ });
485
+ coordinator_load_snapshot(
486
+ &mut coord,
487
+ meta,
488
+ tokens,
489
+ &mut Cursor::new(&payload),
490
+ payload.len() as u64,
491
+ )
492
+ .unwrap();
493
+ let (restored, offer) = handle.join().unwrap();
494
+ assert_eq!(offer.session_id, meta.session_id);
495
+ assert_eq!(offer.request_id, meta.request_id);
496
+ assert_eq!(offer.token_hash, meta.token_hash);
497
+ assert_eq!(offer.tokens, tokens);
498
+ restored
499
+ }
500
+
501
+ #[cfg(unix)]
502
+ #[test]
503
+ fn worker_save_answers_coordinator_with_c_frame_order() {
504
+ let tokens = [7, 8];
505
+ let meta = hashed_meta(&tokens);
506
+ let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3];
507
+ payload[SNAPSHOT_CHUNK_BYTES] = 1;
508
+ payload[SNAPSHOT_CHUNK_BYTES + 1] = 2;
509
+ payload[SNAPSHOT_CHUNK_BYTES + 2] = 3;
510
+ let (saved, n) = duplex_save(meta, &tokens, payload.clone());
511
+ assert_eq!(n, payload.len() as u64);
512
+ assert_eq!(saved, payload);
513
+ }
514
+
515
+ #[cfg(unix)]
516
+ #[test]
517
+ fn worker_load_accepts_coordinator_chunks_and_reports_tokens() {
518
+ let tokens = [7, 8];
519
+ let meta = hashed_meta(&tokens);
520
+ let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3];
521
+ payload[SNAPSHOT_CHUNK_BYTES] = 9;
522
+ let restored = duplex_load(meta, &tokens, payload.clone());
523
+ assert_eq!(restored, payload);
524
+ }
525
+
526
+ #[cfg(unix)]
527
+ #[test]
528
+ fn worker_zero_payload_round_trips_without_chunks() {
529
+ let meta = hashed_meta(&[]);
530
+ let (saved, n) = duplex_save(meta, &[], Vec::new());
531
+ assert_eq!(n, 0);
532
+ assert!(saved.is_empty());
533
+ assert!(duplex_load(meta, &[], Vec::new()).is_empty());
534
+ }
535
+
536
+ #[test]
537
+ fn worker_save_rejects_bad_size_and_identity_with_c_text() {
538
+ let meta = meta();
539
+ let identity = identity(meta);
540
+ let mut stream = ScriptedStream::new(Vec::new());
541
+ worker_handle_snapshot_save::<_, Cursor<Vec<u8>>>(
542
+ &mut stream,
543
+ identity,
544
+ &[0; SNAPSHOT_REQ_FIXED_BYTES - 1],
545
+ Err("unused".into()),
546
+ )
547
+ .unwrap();
548
+ let mut written = Cursor::new(stream.written);
549
+ let (typ, body) = read_frame(&mut written).unwrap();
550
+ assert_eq!(typ, MSG_SNAPSHOT_BEGIN);
551
+ let (begin, _, message) = ds4_dist::decode_snapshot_begin_body(&body).unwrap();
552
+ assert_eq!(begin.status, 1);
553
+ assert_eq!(message, b"invalid distributed snapshot save request");
554
+
555
+ let mut request = ScriptedStream::new(Vec::new());
556
+ let mut unused = Vec::new();
557
+ coordinator_save_snapshot(&mut request, meta, &[7], &mut unused).unwrap_err();
558
+ let mut written = Cursor::new(request.written);
559
+ let (typ, body) = read_frame(&mut written).unwrap();
560
+ assert_eq!(typ, MSG_SNAPSHOT_SAVE_REQ);
561
+ let mut wrong = identity;
562
+ wrong.layer_end += 1;
563
+ let mut reply = ScriptedStream::new(Vec::new());
564
+ worker_handle_snapshot_save::<_, Cursor<Vec<u8>>>(
565
+ &mut reply,
566
+ wrong,
567
+ &body,
568
+ Err("unused".into()),
569
+ )
570
+ .unwrap();
571
+ let mut written = Cursor::new(reply.written);
572
+ let (typ, body) = read_frame(&mut written).unwrap();
573
+ assert_eq!(typ, MSG_SNAPSHOT_BEGIN);
574
+ let (begin, _, message) = ds4_dist::decode_snapshot_begin_body(&body).unwrap();
575
+ assert_eq!(begin.status, 1);
576
+ assert_eq!(
577
+ std::str::from_utf8(&message).unwrap(),
578
+ "snapshot save request does not match worker state"
579
+ );
580
+ }
581
+
582
+ #[test]
583
+ fn worker_save_forwards_session_error_text() {
584
+ let meta = meta();
585
+ let mut request = ScriptedStream::new(Vec::new());
586
+ let mut unused = Vec::new();
587
+ coordinator_save_snapshot(&mut request, meta, &[7], &mut unused).unwrap_err();
588
+ let mut written = Cursor::new(request.written);
589
+ let (_, body) = read_frame(&mut written).unwrap();
590
+ let mut reply = ScriptedStream::new(Vec::new());
591
+ worker_handle_snapshot_save::<_, Cursor<Vec<u8>>>(
592
+ &mut reply,
593
+ identity(meta),
594
+ &body,
595
+ Err("worker has no distributed session to snapshot".into()),
596
+ )
597
+ .unwrap();
598
+ let mut written = Cursor::new(reply.written);
599
+ let (_, body) = read_frame(&mut written).unwrap();
600
+ let (begin, _, message) = ds4_dist::decode_snapshot_begin_body(&body).unwrap();
601
+ assert_eq!(begin.status, 1);
602
+ assert_eq!(
603
+ std::str::from_utf8(&message).unwrap(),
604
+ "worker has no distributed session to snapshot"
605
+ );
606
+ }
607
+
608
+ #[test]
609
+ fn worker_load_rejects_header_hash_and_identity_with_c_text() {
610
+ let identity = identity(meta());
611
+ let mut stream = ScriptedStream::new(Vec::new());
612
+ assert_eq!(
613
+ worker_handle_snapshot_load(&mut stream, identity, &[0; 8], 32, &mut Vec::new())
614
+ .unwrap_err(),
615
+ "invalid distributed snapshot load header"
616
+ );
617
+ assert!(stream.written.is_empty());
618
+
619
+ let tokens = [7, 8];
620
+ let mut meta = hashed_meta(&tokens);
621
+ meta.token_hash ^= 1;
622
+ let mut request = ScriptedStream::new({
623
+ let mut done = Vec::new();
624
+ push_frame(
625
+ &mut done,
626
+ MSG_SNAPSHOT_DONE,
627
+ &encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(),
628
+ );
629
+ done
630
+ });
631
+ coordinator_load_snapshot(&mut request, meta, &tokens, &mut Cursor::new(b""), 0).unwrap();
632
+ let mut written = Cursor::new(request.written);
633
+ let (typ, body) = read_frame(&mut written).unwrap();
634
+ assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN);
635
+ let mut reply = ScriptedStream::new(Vec::new());
636
+ assert_eq!(
637
+ worker_handle_snapshot_load(&mut reply, identity, &body, 32, &mut Vec::new()).unwrap_err(),
638
+ "snapshot load token hash mismatch"
639
+ );
640
+ let mut written = Cursor::new(reply.written);
641
+ let (typ, body) = read_frame(&mut written).unwrap();
642
+ assert_eq!(typ, MSG_SNAPSHOT_DONE);
643
+ let (done, message) = ds4_dist::decode_snapshot_done_body(&body, meta.request_id).unwrap();
644
+ assert_eq!(done.status, 1);
645
+ assert_eq!(message, b"snapshot load token hash mismatch");
646
+ }
crates/ds4-dist/tests/wire_oracle.rs ADDED
@@ -0,0 +1,398 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! C↔Rust DS4D wire bytes. No `#[repr(C)]` on the Rust side.
2
+
3
+ use ds4_dist::{
4
+ decode_frame_header, decode_hello_payload, decode_snapshot_begin_body,
5
+ decode_snapshot_chunk_body, decode_snapshot_done_body, decode_snapshot_load_begin_body,
6
+ encode_activation, encode_frame_header, encode_hello_payload, encode_route_blob,
7
+ encode_snapshot_begin_body, encode_snapshot_chunk_body, encode_snapshot_done_body,
8
+ encode_tokens_be, f32_to_f16, f32_to_f8_e4m3, read_frame, token_hash_prefix,
9
+ validate_route_blob, write_frame, Hello, ResultHdr, ReturnTarget, RouteEntry, SnapshotBegin,
10
+ SnapshotChunk, SnapshotDone, SnapshotReq, Work, FRAME_HEADER_BYTES, HELLO_FIXED_BYTES, MAGIC,
11
+ MSG_HELLO, MSG_WORK, RESULT_FIXED_BYTES, ROUTE_RETURN_UPSTREAM, SNAPSHOT_CHUNK_BYTES,
12
+ WORK_FIXED_BYTES,
13
+ };
14
+ use std::path::PathBuf;
15
+ use std::process::Command;
16
+
17
+ fn oracle() -> PathBuf {
18
+ if let Ok(p) = std::env::var("DS4_DIST_C_ORACLE") {
19
+ return PathBuf::from(p);
20
+ }
21
+ PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/dist_c_oracle")
22
+ }
23
+
24
+ fn require_oracle() -> PathBuf {
25
+ let p = oracle();
26
+ assert!(
27
+ p.exists(),
28
+ "build the C oracle first: make tests/parity/dist_c_oracle (missing {})",
29
+ p.display()
30
+ );
31
+ p
32
+ }
33
+
34
+ fn c_out(args: &[&str]) -> String {
35
+ let out = Command::new(require_oracle())
36
+ .args(args)
37
+ .output()
38
+ .expect("run dist_c_oracle");
39
+ assert!(
40
+ out.status.success(),
41
+ "oracle failed: {}",
42
+ String::from_utf8_lossy(&out.stderr)
43
+ );
44
+ String::from_utf8_lossy(&out.stdout).into_owned()
45
+ }
46
+
47
+ fn hex(bytes: &[u8]) -> String {
48
+ bytes.iter().map(|b| format!("{b:02x}")).collect()
49
+ }
50
+
51
+ #[test]
52
+ fn struct_sizes_match_c() {
53
+ let line = c_out(&["sizes"]);
54
+ assert!(line.contains("frame 12"), "{line}");
55
+ assert!(line.contains("hello 40"), "{line}");
56
+ assert!(line.contains("work 80"), "{line}");
57
+ assert!(line.contains(&format!("work {WORK_FIXED_BYTES}")), "{line}");
58
+ assert_eq!(FRAME_HEADER_BYTES, 12);
59
+ assert_eq!(HELLO_FIXED_BYTES, 40);
60
+ assert_eq!(RESULT_FIXED_BYTES, 40);
61
+ }
62
+
63
+ #[test]
64
+ fn frame_header_matches_c() {
65
+ let rust = encode_frame_header(MSG_WORK, 80);
66
+ assert_eq!(hex(&rust), c_out(&["frame", "3", "80"]));
67
+ let h = decode_frame_header(&rust).unwrap();
68
+ assert_eq!(h.typ, MSG_WORK);
69
+ assert_eq!(h.bytes, 80);
70
+ assert_eq!(u32::from_be_bytes(rust[0..4].try_into().unwrap()), MAGIC);
71
+ }
72
+
73
+ #[test]
74
+ fn hello_matches_c() {
75
+ let rec = Hello {
76
+ model_id: 3,
77
+ quant_bits: 2,
78
+ layer_start: 2,
79
+ layer_end: 51,
80
+ has_output: 1,
81
+ has_hidden: 1,
82
+ ctx_size: 8192,
83
+ n_layers: 53,
84
+ listen_port: 7000,
85
+ model_name_len: 0,
86
+ };
87
+ let rust = encode_hello_payload(&rec, "motif-3").unwrap();
88
+ assert_eq!(
89
+ hex(&rust),
90
+ c_out(&["hello", "3", "2", "2", "51", "1", "1", "8192", "53", "7000", "motif-3"])
91
+ );
92
+ let (back, name) = decode_hello_payload(&rust).unwrap();
93
+ assert_eq!(name, "motif-3");
94
+ assert_eq!(back.model_id, 3);
95
+ assert_eq!(back.listen_port, 7000);
96
+ }
97
+
98
+ #[test]
99
+ fn work_and_tokens_match_c() {
100
+ let w = Work {
101
+ model_id: 3,
102
+ session_hi: 0,
103
+ session_lo: 9,
104
+ request_hi: 0,
105
+ request_lo: 11,
106
+ prefix_hash_hi: 1,
107
+ prefix_hash_lo: 2,
108
+ result_hash_hi: 3,
109
+ result_hash_lo: 4,
110
+ pos0: 16,
111
+ n_tokens: 3,
112
+ layer_start: 2,
113
+ layer_end: 10,
114
+ flags: 1,
115
+ token_bytes: 12,
116
+ input_hc_bytes: 0,
117
+ input_hc_bits: 32,
118
+ route_count: 1,
119
+ route_index: 0,
120
+ route_bytes: 0,
121
+ };
122
+ assert_eq!(
123
+ hex(&w.encode()),
124
+ c_out(&[
125
+ "work", "3", "0", "9", "0", "11", "1", "2", "3", "4", "16", "3", "2", "10", "1", "12",
126
+ "0", "32", "1", "0", "0"
127
+ ])
128
+ );
129
+ let tokens = [7i32, -1, 256];
130
+ assert_eq!(
131
+ hex(&encode_tokens_be(&tokens)),
132
+ c_out(&["tokens", "7", "-1", "256"])
133
+ );
134
+ }
135
+
136
+ #[test]
137
+ fn token_hash_matches_c() {
138
+ assert_eq!(
139
+ format!("{:016x}", token_hash_prefix(&[1, 2, 3, 99])),
140
+ c_out(&["token-hash", "1", "2", "3", "99"])
141
+ );
142
+ }
143
+
144
+ #[test]
145
+ fn route_blob_round_trip() {
146
+ let entries = [RouteEntry {
147
+ host: "10.0.0.2".into(),
148
+ port: 7100,
149
+ layer_start: 2,
150
+ layer_end: 20,
151
+ flags: 0,
152
+ }];
153
+ let ret = ReturnTarget {
154
+ kind: ROUTE_RETURN_UPSTREAM,
155
+ host: String::new(),
156
+ port: 0,
157
+ };
158
+ let blob = encode_route_blob(&entries, &ret).unwrap();
159
+ assert_eq!(
160
+ hex(&blob[..20 + 8]),
161
+ c_out(&["route", "10.0.0.2", "7100", "2", "20", "0"])
162
+ );
163
+ validate_route_blob(&blob, 1, 53).unwrap();
164
+ }
165
+
166
+ #[test]
167
+ fn result_matches_c() {
168
+ let r = ResultHdr {
169
+ request_hi: 0,
170
+ request_lo: 11,
171
+ result_hash_hi: 3,
172
+ result_hash_lo: 4,
173
+ status: 0,
174
+ result_kind: 2,
175
+ telemetry_count: 1,
176
+ telemetry_bytes: 40,
177
+ payload_bytes: 16,
178
+ payload_bits: 32,
179
+ };
180
+ assert_eq!(
181
+ hex(&r.encode()),
182
+ c_out(&["result", "0", "11", "3", "4", "0", "2", "1", "40", "16", "32"])
183
+ );
184
+ }
185
+
186
+ #[test]
187
+ fn f16_f8_match_c() {
188
+ let samples = [
189
+ 0x00000000u32,
190
+ 0x3f800000,
191
+ 0xbf800000,
192
+ 0x3f000000,
193
+ 0x7f800000,
194
+ 0x7fc00000,
195
+ 0x00000001,
196
+ 0x3fc00000,
197
+ ];
198
+ for bits in samples {
199
+ let hex_bits = format!("{bits:08x}");
200
+ let f = f32::from_bits(bits);
201
+ assert_eq!(
202
+ hex(&f32_to_f16(f).to_le_bytes()),
203
+ c_out(&["f16", &hex_bits]),
204
+ "f16 {hex_bits}"
205
+ );
206
+ assert_eq!(
207
+ hex(&[f32_to_f8_e4m3(f)]),
208
+ c_out(&["f8", &hex_bits]),
209
+ "f8 {hex_bits}"
210
+ );
211
+ }
212
+ }
213
+
214
+ #[test]
215
+ fn blocking_hello_round_trip() {
216
+ use std::net::{TcpListener, TcpStream};
217
+ use std::thread;
218
+
219
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
220
+ let addr = listener.local_addr().unwrap();
221
+ let rec = Hello {
222
+ model_id: 1,
223
+ quant_bits: 4,
224
+ layer_start: 0,
225
+ layer_end: 11,
226
+ has_output: 0,
227
+ has_hidden: 1,
228
+ ctx_size: 2048,
229
+ n_layers: 48,
230
+ listen_port: 9,
231
+ model_name_len: 0,
232
+ };
233
+ let payload = encode_hello_payload(&rec, "solar").unwrap();
234
+ let reply = payload.clone();
235
+ let server = thread::spawn(move || {
236
+ let (mut s, _) = listener.accept().unwrap();
237
+ let (typ, body) = read_frame(&mut s).unwrap();
238
+ assert_eq!(typ, MSG_HELLO);
239
+ let (h, name) = decode_hello_payload(&body).unwrap();
240
+ assert_eq!(name, "solar");
241
+ assert_eq!(h.n_layers, 48);
242
+ write_frame(&mut s, MSG_HELLO, &reply).unwrap();
243
+ });
244
+ let mut client = TcpStream::connect(addr).unwrap();
245
+ write_frame(&mut client, MSG_HELLO, &payload).unwrap();
246
+ let (typ, body) = read_frame(&mut client).unwrap();
247
+ assert_eq!(typ, MSG_HELLO);
248
+ let (_, name) = decode_hello_payload(&body).unwrap();
249
+ assert_eq!(name, "solar");
250
+ server.join().unwrap();
251
+ }
252
+
253
+ #[test]
254
+ fn activation_f32_is_le_host_layout() {
255
+ let src = [1.0f32, -2.5];
256
+ let wire = encode_activation(&src, 32).unwrap();
257
+ let mut expect = Vec::new();
258
+ expect.extend_from_slice(&1.0f32.to_bits().to_le_bytes());
259
+ expect.extend_from_slice(&(-2.5f32).to_bits().to_le_bytes());
260
+ assert_eq!(wire, expect);
261
+ }
262
+
263
+ #[test]
264
+ fn snapshot_records_match_c() {
265
+ let req = SnapshotReq {
266
+ model_id: 3,
267
+ session_hi: 1,
268
+ session_lo: 2,
269
+ request_hi: 3,
270
+ request_lo: 4,
271
+ token_hash_hi: 5,
272
+ token_hash_lo: 6,
273
+ token_count: 7,
274
+ layer_start: 8,
275
+ layer_end: 9,
276
+ };
277
+ assert_eq!(
278
+ hex(&req.encode()),
279
+ c_out(&[
280
+ "snapshot-req",
281
+ "3",
282
+ "1",
283
+ "2",
284
+ "3",
285
+ "4",
286
+ "5",
287
+ "6",
288
+ "7",
289
+ "8",
290
+ "9"
291
+ ])
292
+ );
293
+
294
+ let begin = SnapshotBegin {
295
+ model_id: 3,
296
+ session_hi: 1,
297
+ session_lo: 2,
298
+ request_hi: 3,
299
+ request_lo: 4,
300
+ token_hash_hi: 5,
301
+ token_hash_lo: 6,
302
+ token_count: 2,
303
+ layer_start: 8,
304
+ layer_end: 9,
305
+ payload_hi: 10,
306
+ payload_lo: 11,
307
+ status: 0,
308
+ token_bytes: 8,
309
+ message_bytes: 0,
310
+ };
311
+ assert_eq!(
312
+ hex(&begin.encode()),
313
+ c_out(&[
314
+ "snapshot-begin",
315
+ "3",
316
+ "1",
317
+ "2",
318
+ "3",
319
+ "4",
320
+ "5",
321
+ "6",
322
+ "2",
323
+ "8",
324
+ "9",
325
+ "10",
326
+ "11",
327
+ "0",
328
+ "8",
329
+ "0"
330
+ ])
331
+ );
332
+
333
+ let chunk = SnapshotChunk {
334
+ request_hi: 3,
335
+ request_lo: 4,
336
+ chunk_bytes: 5,
337
+ };
338
+ assert_eq!(
339
+ hex(&chunk.encode()),
340
+ c_out(&["snapshot-chunk", "3", "4", "5"])
341
+ );
342
+
343
+ let done = SnapshotDone {
344
+ request_hi: 3,
345
+ request_lo: 4,
346
+ status: 1,
347
+ message_bytes: 5,
348
+ };
349
+ assert_eq!(
350
+ hex(&done.encode()),
351
+ c_out(&["snapshot-done", "3", "4", "1", "5"])
352
+ );
353
+ }
354
+
355
+ #[test]
356
+ fn snapshot_bodies_enforce_c_lengths_and_ids() {
357
+ let begin = SnapshotBegin {
358
+ model_id: 3,
359
+ session_hi: 0,
360
+ session_lo: 2,
361
+ request_hi: 0,
362
+ request_lo: 4,
363
+ token_hash_hi: 0,
364
+ token_hash_lo: 6,
365
+ token_count: 2,
366
+ layer_start: 8,
367
+ layer_end: 9,
368
+ payload_hi: 0,
369
+ payload_lo: 5,
370
+ status: 0,
371
+ token_bytes: 8,
372
+ message_bytes: 0,
373
+ };
374
+ let body = encode_snapshot_begin_body(&begin, &[7, 8], b"").unwrap();
375
+ let (back, tokens, message) = decode_snapshot_load_begin_body(&body).unwrap();
376
+ assert_eq!(back, begin);
377
+ assert_eq!(tokens, [7, 8]);
378
+ assert!(message.is_empty());
379
+
380
+ let mut bad = begin;
381
+ bad.token_bytes = 4;
382
+ assert!(encode_snapshot_begin_body(&bad, &[7, 8], b"").is_err());
383
+ assert!(decode_snapshot_begin_body(&body[..body.len() - 1]).is_err());
384
+
385
+ let payload = b"kv-shard";
386
+ let chunk = encode_snapshot_chunk_body(4, payload).unwrap();
387
+ let (_, decoded) = decode_snapshot_chunk_body(&chunk, 4, payload.len() as u64).unwrap();
388
+ assert_eq!(decoded, payload);
389
+ assert!(decode_snapshot_chunk_body(&chunk, 5, payload.len() as u64).is_err());
390
+ assert!(decode_snapshot_chunk_body(&chunk, 4, (payload.len() - 1) as u64).is_err());
391
+ assert!(encode_snapshot_chunk_body(4, &vec![0; SNAPSHOT_CHUNK_BYTES + 1]).is_err());
392
+
393
+ let done = encode_snapshot_done_body(4, 1, b"failed").unwrap();
394
+ let (header, message) = decode_snapshot_done_body(&done, 4).unwrap();
395
+ assert_eq!(header.status, 1);
396
+ assert_eq!(message, b"failed");
397
+ assert!(decode_snapshot_done_body(&done, 5).is_err());
398
+ }
crates/ds4-kv/Cargo.toml ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ [package]
2
+ name = "ds4-kv"
3
+ version.workspace = true
4
+ edition.workspace = true
5
+ license.workspace = true
6
+ publish.workspace = true
7
+ description = "KVC checkpoint store (Phase 4)"
8
+
9
+ [lib]
10
+ path = "src/lib.rs"
crates/ds4-kv/src/extensions.rs ADDED
@@ -0,0 +1,245 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Bank thinking bit + KTM extension records.
2
+ //!
3
+ //! Thinking is `EXT_THINKING_VISIBLE` on the KVC header (no extra magic).
4
+ //! The only appended trailer magic is C's `KTM\x01`. A missing or invalid
5
+ //! section is ignored the way `kv_tool_map_load_from_pos` returns 0.
6
+
7
+ use crate::format::{
8
+ Record, EXT_BANK_REPLAY_V1, EXT_IMAGE_PIXELS_V2, EXT_RESPONSES_VISIBLE, EXT_SESSION_TITLE,
9
+ EXT_THINKING_VISIBLE, EXT_TOOL_MAP,
10
+ };
11
+
12
+ pub const KTM_MAGIC: [u8; 3] = [b'K', b'T', b'M'];
13
+ pub const KTM_VERSION: u8 = 1;
14
+ const KTM_HEADER: [u8; 4] = [b'K', b'T', b'M', KTM_VERSION];
15
+ const DEFAULT_MAX_IDS: usize = 100_000;
16
+ const MAX_ID_BYTES: usize = 256;
17
+ const MAX_DSML_BYTES: usize = 512 * 1024 * 1024;
18
+
19
+ #[derive(Clone, Debug, PartialEq, Eq)]
20
+ pub struct ExtensionRecord {
21
+ pub id: Vec<u8>,
22
+ pub body: Vec<u8>,
23
+ }
24
+
25
+ #[derive(Clone, Debug, Default, PartialEq, Eq)]
26
+ pub struct BankThinkingExtensions {
27
+ pub thinking_visible: bool,
28
+ pub records: Vec<ExtensionRecord>,
29
+ }
30
+
31
+ impl BankThinkingExtensions {
32
+ pub fn ext_flags(&self) -> u8 {
33
+ self.wire(0).0
34
+ }
35
+
36
+ pub fn encode_trailer(&self) -> Vec<u8> {
37
+ self.wire(0).1
38
+ }
39
+
40
+ fn wire(&self, keep: u8) -> (u8, Vec<u8>) {
41
+ let trailer = encode_ktm(&self.records);
42
+ (
43
+ bank_persist_ext_flags(keep, self.thinking_visible, !trailer.is_empty()),
44
+ trailer,
45
+ )
46
+ }
47
+
48
+ pub fn restore(ext_flags: u8, trailer: &[u8]) -> Self {
49
+ Self {
50
+ thinking_visible: ext_flags & EXT_THINKING_VISIBLE != 0,
51
+ records: decode_ktm(trailer),
52
+ }
53
+ }
54
+ }
55
+
56
+ impl Record {
57
+ pub fn persist_bank_extensions(&mut self, ext: &BankThinkingExtensions) {
58
+ let (flags, trailer) = ext.wire(self.header.ext_flags);
59
+ self.header.ext_flags = flags;
60
+ self.trailer = trailer;
61
+ self.header.text_bytes = self.text.len() as u32;
62
+ self.header.payload_bytes = self.payload.len() as u64;
63
+ }
64
+ }
65
+
66
+ /// Bank persist always sets `EXT_BANK_REPLAY_V1`. Responses/title bits on
67
+ /// the incoming header survive; thinking and KTM are applied from `ext`.
68
+ pub fn bank_persist_ext_flags(keep: u8, thinking_visible: bool, has_trailer: bool) -> u8 {
69
+ let mut flags = EXT_BANK_REPLAY_V1
70
+ | (keep & (EXT_RESPONSES_VISIBLE | EXT_SESSION_TITLE | EXT_IMAGE_PIXELS_V2));
71
+ if thinking_visible {
72
+ flags |= EXT_THINKING_VISIBLE;
73
+ }
74
+ if has_trailer {
75
+ flags |= EXT_TOOL_MAP;
76
+ }
77
+ flags
78
+ }
79
+
80
+ fn wire_ok(id: &[u8], body: &[u8]) -> bool {
81
+ !id.is_empty()
82
+ && !body.is_empty()
83
+ && id.len() <= MAX_ID_BYTES
84
+ && body.len() <= MAX_DSML_BYTES
85
+ && !id.contains(&0)
86
+ && !body.contains(&0)
87
+ }
88
+
89
+ pub fn encode_ktm(records: &[ExtensionRecord]) -> Vec<u8> {
90
+ let kept: Vec<&ExtensionRecord> = records
91
+ .iter()
92
+ .filter(|record| wire_ok(&record.id, &record.body))
93
+ .collect();
94
+ if kept.is_empty() {
95
+ return Vec::new();
96
+ }
97
+ let Ok(count) = u32::try_from(kept.len()) else {
98
+ return Vec::new();
99
+ };
100
+ let mut out = Vec::from(KTM_HEADER);
101
+ out.extend_from_slice(&count.to_le_bytes());
102
+ for record in kept {
103
+ let id_len = record.id.len() as u32;
104
+ let body_len = record.body.len() as u32;
105
+ out.extend_from_slice(&id_len.to_le_bytes());
106
+ out.extend_from_slice(&body_len.to_le_bytes());
107
+ out.extend_from_slice(&record.id);
108
+ out.extend_from_slice(&record.body);
109
+ }
110
+ out
111
+ }
112
+
113
+ pub fn decode_ktm(bytes: &[u8]) -> Vec<ExtensionRecord> {
114
+ decode_ktm_bounded(bytes, DEFAULT_MAX_IDS)
115
+ }
116
+
117
+ fn decode_ktm_bounded(bytes: &[u8], max_ids: usize) -> Vec<ExtensionRecord> {
118
+ if bytes.len() < 8 || bytes[..4] != KTM_HEADER {
119
+ return Vec::new();
120
+ }
121
+ let count = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]);
122
+ let max_ids = if max_ids == 0 {
123
+ DEFAULT_MAX_IDS
124
+ } else {
125
+ max_ids
126
+ };
127
+ let max_count = (max_ids as u64).saturating_mul(4);
128
+ if u64::from(count) > max_count {
129
+ return Vec::new();
130
+ }
131
+ let mut records = Vec::new();
132
+ let mut pos = 8usize;
133
+ for _ in 0..count {
134
+ let Some(lens_end) = pos.checked_add(8).filter(|&end| end <= bytes.len()) else {
135
+ return records;
136
+ };
137
+ let id_len =
138
+ u32::from_le_bytes([bytes[pos], bytes[pos + 1], bytes[pos + 2], bytes[pos + 3]])
139
+ as usize;
140
+ let body_len = u32::from_le_bytes([
141
+ bytes[pos + 4],
142
+ bytes[pos + 5],
143
+ bytes[pos + 6],
144
+ bytes[pos + 7],
145
+ ]) as usize;
146
+ if id_len == 0 || id_len > MAX_ID_BYTES || body_len == 0 || body_len > MAX_DSML_BYTES {
147
+ return records;
148
+ }
149
+ pos = lens_end;
150
+ let Some(id_end) = pos.checked_add(id_len).filter(|&end| end <= bytes.len()) else {
151
+ return records;
152
+ };
153
+ let Some(body_end) = id_end
154
+ .checked_add(body_len)
155
+ .filter(|&end| end <= bytes.len())
156
+ else {
157
+ return records;
158
+ };
159
+ let id = until_nul(&bytes[pos..id_end]);
160
+ let body = until_nul(&bytes[id_end..body_end]);
161
+ if !id.is_empty() && !body.is_empty() {
162
+ records.push(ExtensionRecord {
163
+ id: id.to_vec(),
164
+ body: body.to_vec(),
165
+ });
166
+ }
167
+ pos = body_end;
168
+ }
169
+ records
170
+ }
171
+
172
+ fn until_nul(bytes: &[u8]) -> &[u8] {
173
+ match bytes.iter().position(|&b| b == 0) {
174
+ Some(i) => &bytes[..i],
175
+ None => bytes,
176
+ }
177
+ }
178
+
179
+ #[cfg(test)]
180
+ mod tests {
181
+ use super::*;
182
+
183
+ #[test]
184
+ fn empty_records_write_no_ktm_header() {
185
+ let ext = BankThinkingExtensions {
186
+ thinking_visible: true,
187
+ records: Vec::new(),
188
+ };
189
+ assert_eq!(ext.ext_flags(), EXT_BANK_REPLAY_V1 | EXT_THINKING_VISIBLE);
190
+ assert!(ext.encode_trailer().is_empty());
191
+ }
192
+
193
+ #[test]
194
+ fn invalid_records_do_not_set_tool_map() {
195
+ let ext = BankThinkingExtensions {
196
+ thinking_visible: false,
197
+ records: vec![ExtensionRecord {
198
+ id: Vec::new(),
199
+ body: b"x".to_vec(),
200
+ }],
201
+ };
202
+ assert_eq!(ext.ext_flags(), EXT_BANK_REPLAY_V1);
203
+ assert!(ext.encode_trailer().is_empty());
204
+ }
205
+
206
+ #[test]
207
+ fn persist_bank_extensions_keeps_identity_responses_and_title() {
208
+ // Given: a bank record that already carries Responses/title bits
209
+ let mut rec = Record {
210
+ header: crate::format::Header {
211
+ quant_bits: 2,
212
+ reason: crate::format::Reason::BankEvict,
213
+ ext_flags: EXT_RESPONSES_VISIBLE | EXT_SESSION_TITLE | EXT_IMAGE_PIXELS_V2,
214
+ model_id: 1,
215
+ tokens: 8,
216
+ hits: 0,
217
+ ctx_size: 4096,
218
+ created_at: 1,
219
+ last_used: 1,
220
+ payload_bytes: 1,
221
+ text_bytes: 4,
222
+ },
223
+ text: b"keep".to_vec(),
224
+ payload: b"P".to_vec(),
225
+ trailer: Vec::new(),
226
+ };
227
+ let ext = BankThinkingExtensions {
228
+ thinking_visible: true,
229
+ records: Vec::new(),
230
+ };
231
+
232
+ // When: thinking/KTM extensions are persisted onto that record
233
+ rec.persist_bank_extensions(&ext);
234
+
235
+ // Then: BANK_REPLAY_V1 + thinking land, and the two prior bits survive
236
+ assert_eq!(
237
+ rec.header.ext_flags,
238
+ EXT_BANK_REPLAY_V1
239
+ | EXT_THINKING_VISIBLE
240
+ | EXT_RESPONSES_VISIBLE
241
+ | EXT_SESSION_TITLE
242
+ | EXT_IMAGE_PIXELS_V2
243
+ );
244
+ }
245
+ }
crates/ds4-kv/src/format.rs ADDED
@@ -0,0 +1,781 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! KVC file envelope: 48-byte header + text length + text + payload.
2
+ //!
3
+ //! Matches `ds4_kvstore_fill_header` / `ds4_kvstore_read_header` and the
4
+ //! store fwrite order in `ds4_kvstore.c`. Do not invent a new format.
5
+
6
+ use crate::sha1::sha1_hex;
7
+ use std::fs;
8
+ use std::io::{self, Read, Seek, SeekFrom, Write};
9
+ use std::path::{Path, PathBuf};
10
+ use std::sync::atomic::{AtomicU64, Ordering};
11
+
12
+ pub const FIXED_HEADER: usize = 48;
13
+ pub const MAGIC: [u8; 3] = [b'K', b'V', b'C'];
14
+ pub const VERSION: u8 = 1;
15
+ pub const PAYLOAD_ABI: u8 = 2;
16
+
17
+ static TEMP_SEQ: AtomicU64 = AtomicU64::new(0);
18
+
19
+ pub const EXT_TOOL_MAP: u8 = 1 << 0;
20
+ pub const EXT_RESPONSES_VISIBLE: u8 = 1 << 1;
21
+ pub const EXT_THINKING_VISIBLE: u8 = 1 << 2;
22
+ pub const EXT_SESSION_TITLE: u8 = 1 << 3;
23
+ pub const EXT_BANK_REPLAY_V1: u8 = 1 << 4;
24
+ pub const EXT_IMAGE_PIXELS_V2: u8 = 1 << 5;
25
+
26
+ #[derive(Clone, Copy, Debug, PartialEq, Eq)]
27
+ #[repr(u8)]
28
+ pub enum Reason {
29
+ Unknown = 0,
30
+ Cold = 1,
31
+ Continued = 2,
32
+ Evict = 3,
33
+ Shutdown = 4,
34
+ AgentSystem = 5,
35
+ AgentSession = 6,
36
+ BankEvict = 7,
37
+ BankShutdown = 8,
38
+ BankCheckpoint = 9,
39
+ }
40
+
41
+ impl Reason {
42
+ pub fn from_u8(v: u8) -> Self {
43
+ match v {
44
+ 1 => Self::Cold,
45
+ 2 => Self::Continued,
46
+ 3 => Self::Evict,
47
+ 4 => Self::Shutdown,
48
+ 5 => Self::AgentSystem,
49
+ 6 => Self::AgentSession,
50
+ 7 => Self::BankEvict,
51
+ 8 => Self::BankShutdown,
52
+ 9 => Self::BankCheckpoint,
53
+ _ => Self::Unknown,
54
+ }
55
+ }
56
+
57
+ pub fn from_name(name: &str) -> Self {
58
+ match name {
59
+ "cold" => Self::Cold,
60
+ "continued" => Self::Continued,
61
+ "evict" => Self::Evict,
62
+ "shutdown" => Self::Shutdown,
63
+ "agent-system" => Self::AgentSystem,
64
+ "agent-session" => Self::AgentSession,
65
+ "bank-evict" => Self::BankEvict,
66
+ "bank-shutdown" => Self::BankShutdown,
67
+ "bank-checkpoint" => Self::BankCheckpoint,
68
+ _ => Self::Unknown,
69
+ }
70
+ }
71
+
72
+ pub fn family(self) -> u8 {
73
+ match self as u8 {
74
+ 1..=4 => 1,
75
+ 5..=6 => 2,
76
+ 7..=9 => 3,
77
+ _ => 0,
78
+ }
79
+ }
80
+ }
81
+
82
+ #[derive(Clone, Debug, PartialEq, Eq)]
83
+ pub struct Header {
84
+ pub quant_bits: u8,
85
+ pub reason: Reason,
86
+ pub ext_flags: u8,
87
+ pub model_id: u8,
88
+ pub tokens: u32,
89
+ pub hits: u32,
90
+ pub ctx_size: u32,
91
+ pub created_at: u64,
92
+ pub last_used: u64,
93
+ pub payload_bytes: u64,
94
+ pub text_bytes: u32,
95
+ }
96
+
97
+ #[derive(Clone, Debug, PartialEq, Eq)]
98
+ pub struct Record {
99
+ pub header: Header,
100
+ pub text: Vec<u8>,
101
+ pub payload: Vec<u8>,
102
+ pub trailer: Vec<u8>,
103
+ }
104
+
105
+ #[derive(Clone, Debug, PartialEq, Eq)]
106
+ pub(crate) struct Metadata {
107
+ pub(crate) header: Header,
108
+ pub(crate) payload_offset: u64,
109
+ pub(crate) trailer_bytes: u64,
110
+ pub(crate) file_size: u64,
111
+ }
112
+
113
+ #[derive(Clone, Debug, PartialEq, Eq)]
114
+ pub struct Envelope {
115
+ pub header: Header,
116
+ pub text: Vec<u8>,
117
+ pub payload_offset: u64,
118
+ pub trailer_bytes: u64,
119
+ pub file_size: u64,
120
+ }
121
+
122
+ #[derive(Debug)]
123
+ pub enum FormatError {
124
+ Io(io::Error),
125
+ BadMagic,
126
+ BadVersion,
127
+ BadPayloadAbi,
128
+ BadQuant,
129
+ ZeroTokens,
130
+ Truncated,
131
+ TextLenMismatch,
132
+ TrailerTooLarge,
133
+ }
134
+
135
+ impl From<io::Error> for FormatError {
136
+ fn from(e: io::Error) -> Self {
137
+ Self::Io(e)
138
+ }
139
+ }
140
+
141
+ impl std::fmt::Display for FormatError {
142
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
143
+ match self {
144
+ Self::Io(e) => write!(f, "{e}"),
145
+ Self::BadMagic => write!(f, "not a KVC file"),
146
+ Self::BadVersion => write!(f, "unsupported KVC version"),
147
+ Self::BadPayloadAbi => write!(f, "unsupported KVC payload ABI"),
148
+ Self::BadQuant => write!(f, "quant_bits must be 2 or 4"),
149
+ Self::ZeroTokens => write!(f, "token count is zero"),
150
+ Self::Truncated => write!(f, "truncated KVC file"),
151
+ Self::TextLenMismatch => write!(f, "text length does not match header"),
152
+ Self::TrailerTooLarge => write!(f, "KVC trailer exceeds read bound"),
153
+ }
154
+ }
155
+ }
156
+
157
+ impl std::error::Error for FormatError {}
158
+
159
+ pub fn le_put32(p: &mut [u8], v: u32) {
160
+ p[0] = v as u8;
161
+ p[1] = (v >> 8) as u8;
162
+ p[2] = (v >> 16) as u8;
163
+ p[3] = (v >> 24) as u8;
164
+ }
165
+
166
+ pub fn le_get32(p: &[u8]) -> u32 {
167
+ u32::from(p[0]) | (u32::from(p[1]) << 8) | (u32::from(p[2]) << 16) | (u32::from(p[3]) << 24)
168
+ }
169
+
170
+ pub fn le_put64(p: &mut [u8], v: u64) {
171
+ for i in 0..8 {
172
+ p[i] = (v >> (8 * i)) as u8;
173
+ }
174
+ }
175
+
176
+ pub fn le_get64(p: &[u8]) -> u64 {
177
+ let mut v = 0u64;
178
+ for i in (0..8).rev() {
179
+ v = (v << 8) | u64::from(p[i]);
180
+ }
181
+ v
182
+ }
183
+
184
+ pub fn fill_header(
185
+ model_id: u8,
186
+ quant_bits: u8,
187
+ reason: Reason,
188
+ ext_flags: u8,
189
+ tokens: u32,
190
+ hits: u32,
191
+ ctx_size: u32,
192
+ created_at: u64,
193
+ last_used: u64,
194
+ payload_bytes: u64,
195
+ ) -> [u8; FIXED_HEADER] {
196
+ let mut h = [0u8; FIXED_HEADER];
197
+ h[0] = MAGIC[0];
198
+ h[1] = MAGIC[1];
199
+ h[2] = MAGIC[2];
200
+ h[3] = VERSION;
201
+ h[4] = quant_bits;
202
+ h[5] = reason as u8;
203
+ h[6] = ext_flags;
204
+ h[7] = model_id;
205
+ le_put32(&mut h[8..12], tokens);
206
+ le_put32(&mut h[12..16], hits);
207
+ le_put32(&mut h[16..20], ctx_size);
208
+ h[20] = PAYLOAD_ABI;
209
+ le_put64(&mut h[24..32], created_at);
210
+ le_put64(&mut h[32..40], last_used);
211
+ le_put64(&mut h[40..48], payload_bytes);
212
+ h
213
+ }
214
+
215
+ pub fn parse_header(h: &[u8; FIXED_HEADER], text_bytes: u32) -> Result<Header, FormatError> {
216
+ if h[0] != MAGIC[0] || h[1] != MAGIC[1] || h[2] != MAGIC[2] {
217
+ return Err(FormatError::BadMagic);
218
+ }
219
+ if h[3] != VERSION {
220
+ return Err(FormatError::BadVersion);
221
+ }
222
+ if h[20] != PAYLOAD_ABI {
223
+ return Err(FormatError::BadPayloadAbi);
224
+ }
225
+ let tokens = le_get32(&h[8..12]);
226
+ let quant_bits = h[4];
227
+ if tokens == 0 {
228
+ return Err(FormatError::ZeroTokens);
229
+ }
230
+ if quant_bits != 2 && quant_bits != 4 {
231
+ return Err(FormatError::BadQuant);
232
+ }
233
+ Ok(Header {
234
+ quant_bits,
235
+ reason: Reason::from_u8(h[5]),
236
+ ext_flags: h[6],
237
+ model_id: h[7],
238
+ tokens,
239
+ hits: le_get32(&h[12..16]),
240
+ ctx_size: le_get32(&h[16..20]),
241
+ created_at: le_get64(&h[24..32]),
242
+ last_used: le_get64(&h[32..40]),
243
+ payload_bytes: le_get64(&h[40..48]),
244
+ text_bytes,
245
+ })
246
+ }
247
+
248
+ pub fn encode_file(record: &Record) -> Vec<u8> {
249
+ let mut h = fill_header(
250
+ record.header.model_id,
251
+ record.header.quant_bits,
252
+ record.header.reason,
253
+ record.header.ext_flags,
254
+ record.header.tokens,
255
+ record.header.hits,
256
+ record.header.ctx_size,
257
+ record.header.created_at,
258
+ record.header.last_used,
259
+ record.payload.len() as u64,
260
+ );
261
+ // payload_bytes in the header follows the payload vector, matching C.
262
+ le_put64(&mut h[40..48], record.payload.len() as u64);
263
+ let mut out = Vec::with_capacity(
264
+ FIXED_HEADER + 4 + record.text.len() + record.payload.len() + record.trailer.len(),
265
+ );
266
+ out.extend_from_slice(&h);
267
+ let mut tb = [0u8; 4];
268
+ le_put32(&mut tb, record.text.len() as u32);
269
+ out.extend_from_slice(&tb);
270
+ out.extend_from_slice(&record.text);
271
+ out.extend_from_slice(&record.payload);
272
+ out.extend_from_slice(&record.trailer);
273
+ out
274
+ }
275
+
276
+ pub fn decode_file(bytes: &[u8]) -> Result<Record, FormatError> {
277
+ if bytes.len() < FIXED_HEADER + 4 {
278
+ return Err(FormatError::Truncated);
279
+ }
280
+ let mut hdr = [0u8; FIXED_HEADER];
281
+ hdr.copy_from_slice(&bytes[..FIXED_HEADER]);
282
+ let text_bytes = le_get32(&bytes[FIXED_HEADER..FIXED_HEADER + 4]);
283
+ let header = parse_header(&hdr, text_bytes)?;
284
+ let text_start = FIXED_HEADER + 4;
285
+ let text_end = text_start
286
+ .checked_add(text_bytes as usize)
287
+ .ok_or(FormatError::Truncated)?;
288
+ let payload_end = text_end
289
+ .checked_add(header.payload_bytes as usize)
290
+ .ok_or(FormatError::Truncated)?;
291
+ if bytes.len() < payload_end {
292
+ return Err(FormatError::Truncated);
293
+ }
294
+ Ok(Record {
295
+ header,
296
+ text: bytes[text_start..text_end].to_vec(),
297
+ payload: bytes[text_end..payload_end].to_vec(),
298
+ trailer: bytes[payload_end..].to_vec(),
299
+ })
300
+ }
301
+
302
+ pub fn write_path(path: &Path, record: &Record) -> Result<(), FormatError> {
303
+ let bytes = encode_file(record);
304
+ if let Some(parent) = path.parent() {
305
+ fs::create_dir_all(parent)?;
306
+ }
307
+ let tmp = path.with_extension("kv.tmp");
308
+ {
309
+ let mut f = fs::File::create(&tmp)?;
310
+ f.write_all(&bytes)?;
311
+ f.flush()?;
312
+ }
313
+ fs::rename(&tmp, path)?;
314
+ Ok(())
315
+ }
316
+
317
+ #[cfg(test)]
318
+ pub(crate) fn write_stream(
319
+ path: &Path,
320
+ header: &Header,
321
+ text: &[u8],
322
+ payload: &mut impl Read,
323
+ payload_bytes: u64,
324
+ trailer: &[u8],
325
+ ) -> Result<(), FormatError> {
326
+ let tmp = stage_stream(path, header, text, payload, payload_bytes, trailer)?;
327
+ let result = fs::rename(&tmp, path).map_err(FormatError::Io);
328
+ if result.is_err() {
329
+ let _ = fs::remove_file(&tmp);
330
+ }
331
+ result
332
+ }
333
+
334
+ pub(crate) fn stage_stream(
335
+ path: &Path,
336
+ header: &Header,
337
+ text: &[u8],
338
+ payload: &mut impl Read,
339
+ payload_bytes: u64,
340
+ trailer: &[u8],
341
+ ) -> Result<PathBuf, FormatError> {
342
+ if text.len() > u32::MAX as usize {
343
+ return Err(FormatError::TextLenMismatch);
344
+ }
345
+ if let Some(parent) = path.parent() {
346
+ fs::create_dir_all(parent)?;
347
+ }
348
+ let (tmp, mut f) = loop {
349
+ let seq = TEMP_SEQ.fetch_add(1, Ordering::Relaxed);
350
+ let tmp = path.with_extension(format!("kv.tmp.{}.{}", std::process::id(), seq));
351
+ match fs::OpenOptions::new()
352
+ .write(true)
353
+ .create_new(true)
354
+ .open(&tmp)
355
+ {
356
+ Ok(f) => break (tmp, f),
357
+ Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue,
358
+ Err(e) => return Err(FormatError::Io(e)),
359
+ }
360
+ };
361
+ let result: Result<(), FormatError> = (|| {
362
+ let h = fill_header(
363
+ header.model_id,
364
+ header.quant_bits,
365
+ header.reason,
366
+ header.ext_flags,
367
+ header.tokens,
368
+ header.hits,
369
+ header.ctx_size,
370
+ header.created_at,
371
+ header.last_used,
372
+ payload_bytes,
373
+ );
374
+ f.write_all(&h)?;
375
+ let mut raw_text_bytes = [0u8; 4];
376
+ le_put32(&mut raw_text_bytes, text.len() as u32);
377
+ f.write_all(&raw_text_bytes)?;
378
+ f.write_all(text)?;
379
+ let copied = io::copy(&mut payload.take(payload_bytes), &mut f)?;
380
+ if copied != payload_bytes {
381
+ return Err(FormatError::Truncated);
382
+ }
383
+ f.write_all(trailer)?;
384
+ f.flush()?;
385
+ drop(f);
386
+ Ok(())
387
+ })();
388
+ if let Err(error) = result {
389
+ let _ = fs::remove_file(&tmp);
390
+ return Err(error);
391
+ }
392
+ Ok(tmp)
393
+ }
394
+
395
+ pub fn read_path(path: &Path) -> Result<Record, FormatError> {
396
+ let mut f = fs::File::open(path)?;
397
+ let mut bytes = Vec::new();
398
+ f.read_to_end(&mut bytes)?;
399
+ decode_file(&bytes)
400
+ }
401
+
402
+ pub fn read_envelope(path: &Path) -> Result<Envelope, FormatError> {
403
+ let (metadata, text) = read_text_prefix(path, usize::MAX)?;
404
+ if text.len() != metadata.header.text_bytes as usize {
405
+ return Err(FormatError::Truncated);
406
+ }
407
+ Ok(Envelope {
408
+ header: metadata.header,
409
+ text,
410
+ payload_offset: metadata.payload_offset,
411
+ trailer_bytes: metadata.trailer_bytes,
412
+ file_size: metadata.file_size,
413
+ })
414
+ }
415
+
416
+ pub fn read_trailer(path: &Path, max_bytes: u64) -> Result<(Header, Vec<u8>), FormatError> {
417
+ let mut file = fs::File::open(path)?;
418
+ let metadata = read_metadata_file(&mut file)?;
419
+ if metadata.trailer_bytes > max_bytes {
420
+ return Err(FormatError::TrailerTooLarge);
421
+ }
422
+ let trailer_start = metadata
423
+ .payload_offset
424
+ .checked_add(metadata.header.payload_bytes)
425
+ .ok_or(FormatError::Truncated)?;
426
+ let trailer_bytes =
427
+ usize::try_from(metadata.trailer_bytes).map_err(|_| FormatError::TrailerTooLarge)?;
428
+ file.seek(SeekFrom::Start(trailer_start))?;
429
+ let mut trailer = vec![0; trailer_bytes];
430
+ read_exact(&mut file, &mut trailer)?;
431
+ Ok((metadata.header, trailer))
432
+ }
433
+
434
+ pub(crate) fn read_metadata(path: &Path) -> Result<Metadata, FormatError> {
435
+ let mut f = fs::File::open(path)?;
436
+ read_metadata_file(&mut f)
437
+ }
438
+
439
+ /// The header, and as much of the text as the file still holds, without
440
+ /// asking it for the payload behind them. A truncation can take the
441
+ /// payload and leave what says which conversation the record was keyed by.
442
+ pub(crate) fn read_header_text(
443
+ path: &Path,
444
+ max_bytes: usize,
445
+ ) -> Result<(Header, Vec<u8>), FormatError> {
446
+ let mut f = fs::File::open(path)?;
447
+ let mut raw_header = [0u8; FIXED_HEADER];
448
+ read_exact(&mut f, &mut raw_header)?;
449
+ let mut raw_text_bytes = [0u8; 4];
450
+ read_exact(&mut f, &mut raw_text_bytes)?;
451
+ let header = parse_header(&raw_header, le_get32(&raw_text_bytes))?;
452
+ let want = (header.text_bytes as usize).min(max_bytes);
453
+ let mut text = Vec::new();
454
+ f.take(want as u64).read_to_end(&mut text)?;
455
+ Ok((header, text))
456
+ }
457
+
458
+ pub(crate) fn read_text_prefix(
459
+ path: &Path,
460
+ max_bytes: usize,
461
+ ) -> Result<(Metadata, Vec<u8>), FormatError> {
462
+ let mut f = fs::File::open(path)?;
463
+ let metadata = read_metadata_file(&mut f)?;
464
+ let want = (metadata.header.text_bytes as usize).min(max_bytes);
465
+ let mut text = Vec::with_capacity(want);
466
+ f.take(want as u64).read_to_end(&mut text)?;
467
+ Ok((metadata, text))
468
+ }
469
+
470
+ fn read_metadata_file(f: &mut fs::File) -> Result<Metadata, FormatError> {
471
+ let file_size = f.metadata()?.len();
472
+ if file_size < (FIXED_HEADER + 4) as u64 {
473
+ return Err(FormatError::Truncated);
474
+ }
475
+
476
+ let mut raw_header = [0u8; FIXED_HEADER];
477
+ read_exact(f, &mut raw_header)?;
478
+ let mut raw_text_bytes = [0u8; 4];
479
+ read_exact(f, &mut raw_text_bytes)?;
480
+ let text_bytes = le_get32(&raw_text_bytes);
481
+ let header = parse_header(&raw_header, text_bytes)?;
482
+ let payload_offset = (FIXED_HEADER + 4) as u64 + u64::from(text_bytes);
483
+ let payload_end = payload_offset
484
+ .checked_add(header.payload_bytes)
485
+ .ok_or(FormatError::Truncated)?;
486
+ if file_size < payload_end {
487
+ return Err(FormatError::Truncated);
488
+ }
489
+
490
+ Ok(Metadata {
491
+ header,
492
+ payload_offset,
493
+ trailer_bytes: file_size - payload_end,
494
+ file_size,
495
+ })
496
+ }
497
+
498
+ fn read_exact(f: &mut fs::File, buf: &mut [u8]) -> Result<(), FormatError> {
499
+ match f.read_exact(buf) {
500
+ Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => Err(FormatError::Truncated),
501
+ result => result.map_err(FormatError::Io),
502
+ }
503
+ }
504
+
505
+ pub fn text_sha_hex(text: &[u8]) -> String {
506
+ sha1_hex(text)
507
+ }
508
+
509
+ pub fn sha_hex_name(name: &str) -> Option<String> {
510
+ if name.len() != 43 || !name.ends_with(".kv") {
511
+ return None;
512
+ }
513
+ let hex = &name[..40];
514
+ if !hex.bytes().all(|b| b.is_ascii_hexdigit()) {
515
+ return None;
516
+ }
517
+ Some(hex.to_ascii_lowercase())
518
+ }
519
+
520
+ pub fn path_for_sha(dir: &Path, sha: &str) -> PathBuf {
521
+ dir.join(format!("{sha}.kv"))
522
+ }
523
+
524
+ pub fn key_kind(ext_flags: u8) -> &'static str {
525
+ if ext_flags & EXT_RESPONSES_VISIBLE != 0 {
526
+ "responses-visible"
527
+ } else if ext_flags & EXT_THINKING_VISIBLE != 0 {
528
+ "thinking-visible"
529
+ } else {
530
+ "token-text"
531
+ }
532
+ }
533
+
534
+ pub fn is_bank_replay_v1(reason: Reason, ext_flags: u8) -> bool {
535
+ reason.family() == 3 && ext_flags & EXT_BANK_REPLAY_V1 != 0
536
+ }
537
+
538
+ pub fn is_automatic_exact_replay(reason: Reason, ext_flags: u8) -> bool {
539
+ match reason.family() {
540
+ 1 => ext_flags & EXT_BANK_REPLAY_V1 == 0,
541
+ _ => is_bank_replay_v1(reason, ext_flags),
542
+ }
543
+ }
544
+
545
+ #[cfg(test)]
546
+ mod tests {
547
+ use super::*;
548
+
549
+ struct ErrorAfterPrefix {
550
+ offset: usize,
551
+ }
552
+
553
+ impl Read for ErrorAfterPrefix {
554
+ fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
555
+ const PREFIX: &[u8] = b"DSV4";
556
+ if self.offset == PREFIX.len() {
557
+ return Err(io::Error::other("injected payload read failure"));
558
+ }
559
+ let n = buf.len().min(PREFIX.len() - self.offset);
560
+ buf[..n].copy_from_slice(&PREFIX[self.offset..self.offset + n]);
561
+ self.offset += n;
562
+ Ok(n)
563
+ }
564
+ }
565
+
566
+ fn sample() -> Record {
567
+ Record {
568
+ header: Header {
569
+ quant_bits: 2,
570
+ reason: Reason::Cold,
571
+ ext_flags: 0,
572
+ model_id: 0,
573
+ tokens: 512,
574
+ hits: 3,
575
+ ctx_size: 2048,
576
+ created_at: 1_700_000_000,
577
+ last_used: 1_700_000_100,
578
+ payload_bytes: 4,
579
+ text_bytes: 5,
580
+ },
581
+ text: b"hello".to_vec(),
582
+ payload: b"ABCD".to_vec(),
583
+ trailer: Vec::new(),
584
+ }
585
+ }
586
+
587
+ #[test]
588
+ fn header_round_trip() {
589
+ let rec = sample();
590
+ let bytes = encode_file(&rec);
591
+ let got = decode_file(&bytes).unwrap();
592
+ assert_eq!(got.text, rec.text);
593
+ assert_eq!(got.payload, rec.payload);
594
+ assert_eq!(got.header.tokens, 512);
595
+ assert_eq!(got.header.quant_bits, 2);
596
+ assert_eq!(got.header.reason, Reason::Cold);
597
+ assert_eq!(got.header.ctx_size, 2048);
598
+ assert_eq!(&bytes[..3], b"KVC");
599
+ assert_eq!(bytes[3], 1);
600
+ assert_eq!(bytes[20], 2);
601
+ }
602
+
603
+ #[test]
604
+ fn envelope_has_offsets_without_payload() {
605
+ let dir = std::env::temp_dir().join(format!("ds4-kv-envelope-{}", std::process::id()));
606
+ let path = dir.join("sample.kv");
607
+ let _ = fs::remove_dir_all(&dir);
608
+ let mut rec = sample();
609
+ rec.trailer = b"end".to_vec();
610
+ write_path(&path, &rec).unwrap();
611
+
612
+ let got = read_envelope(&path).unwrap();
613
+ assert_eq!(got.header, rec.header);
614
+ assert_eq!(got.text, rec.text);
615
+ assert_eq!(
616
+ got.payload_offset,
617
+ (FIXED_HEADER + 4 + rec.text.len()) as u64
618
+ );
619
+ assert_eq!(got.trailer_bytes, rec.trailer.len() as u64);
620
+ assert_eq!(got.file_size, fs::metadata(&path).unwrap().len());
621
+
622
+ let _ = fs::remove_dir_all(&dir);
623
+ }
624
+
625
+ #[test]
626
+ fn trailer_read_seeks_past_sparse_payload_and_honors_bound() {
627
+ use std::io::{Seek, SeekFrom};
628
+
629
+ const PAYLOAD_BYTES: u64 = 4 * 1024 * 1024 * 1024;
630
+ let dir =
631
+ std::env::temp_dir().join(format!("ds4-kv-trailer-sparse-{}", std::process::id()));
632
+ let path = dir.join("sample.kv");
633
+ let _ = fs::remove_dir_all(&dir);
634
+ fs::create_dir_all(&dir).unwrap();
635
+
636
+ let rec = sample();
637
+ let mut file = fs::File::create(&path).unwrap();
638
+ file.write_all(&fill_header(
639
+ rec.header.model_id,
640
+ rec.header.quant_bits,
641
+ rec.header.reason,
642
+ EXT_TOOL_MAP,
643
+ rec.header.tokens,
644
+ rec.header.hits,
645
+ rec.header.ctx_size,
646
+ rec.header.created_at,
647
+ rec.header.last_used,
648
+ PAYLOAD_BYTES,
649
+ ))
650
+ .unwrap();
651
+ let mut text_bytes = [0u8; 4];
652
+ le_put32(&mut text_bytes, rec.text.len() as u32);
653
+ file.write_all(&text_bytes).unwrap();
654
+ file.write_all(&rec.text).unwrap();
655
+ file.seek(SeekFrom::Current(PAYLOAD_BYTES as i64)).unwrap();
656
+ file.write_all(b"\0\xffTR").unwrap();
657
+ drop(file);
658
+
659
+ let (header, trailer) = read_trailer(&path, 4).unwrap();
660
+ assert_eq!(header.payload_bytes, PAYLOAD_BYTES);
661
+ assert_eq!(header.ext_flags, EXT_TOOL_MAP);
662
+ assert_eq!(trailer, b"\0\xffTR");
663
+ assert!(matches!(
664
+ read_trailer(&path, 3),
665
+ Err(FormatError::TrailerTooLarge)
666
+ ));
667
+
668
+ let _ = fs::remove_dir_all(&dir);
669
+ }
670
+
671
+ #[test]
672
+ fn rejects_wrong_quant() {
673
+ let mut rec = sample();
674
+ rec.header.quant_bits = 8;
675
+ let bytes = encode_file(&rec);
676
+ assert!(matches!(decode_file(&bytes), Err(FormatError::BadQuant)));
677
+ }
678
+
679
+ #[test]
680
+ fn filename_is_text_sha() {
681
+ assert_eq!(sha_hex_name("not-a-kv"), None);
682
+ let sha = text_sha_hex(b"hello");
683
+ assert_eq!(sha.len(), 40);
684
+ assert_eq!(
685
+ sha_hex_name(&format!("{sha}.kv")).as_deref(),
686
+ Some(sha.as_str())
687
+ );
688
+ }
689
+
690
+ #[test]
691
+ fn stream_failure_preserves_destination_and_removes_temp() {
692
+ let dir =
693
+ std::env::temp_dir().join(format!("ds4-kv-stream-failure-{}", std::process::id()));
694
+ let path = dir.join("entry.kv");
695
+ let _ = fs::remove_dir_all(&dir);
696
+ fs::create_dir_all(&dir).unwrap();
697
+ fs::write(&path, b"original KVC bytes").unwrap();
698
+
699
+ let rec = sample();
700
+ let mut payload = ErrorAfterPrefix { offset: 0 };
701
+ let result = write_stream(&path, &rec.header, &rec.text, &mut payload, 8, &rec.trailer);
702
+
703
+ assert!(result.is_err());
704
+ assert_eq!(fs::read(&path).unwrap(), b"original KVC bytes");
705
+ let mut names: Vec<_> = fs::read_dir(&dir)
706
+ .unwrap()
707
+ .map(|entry| entry.unwrap().file_name())
708
+ .collect();
709
+ names.sort();
710
+ assert_eq!(names, [std::ffi::OsString::from("entry.kv")]);
711
+
712
+ let mut short_payload = &b"DSV4"[..];
713
+ let result = write_stream(
714
+ &path,
715
+ &rec.header,
716
+ &rec.text,
717
+ &mut short_payload,
718
+ 8,
719
+ &rec.trailer,
720
+ );
721
+ assert!(result.is_err());
722
+ assert_eq!(fs::read(&path).unwrap(), b"original KVC bytes");
723
+ let names: Vec<_> = fs::read_dir(&dir)
724
+ .unwrap()
725
+ .map(|entry| entry.unwrap().file_name())
726
+ .collect();
727
+ assert_eq!(names, [std::ffi::OsString::from("entry.kv")]);
728
+
729
+ let _ = fs::remove_dir_all(&dir);
730
+ }
731
+
732
+ #[test]
733
+ fn stream_layout_matches_buffered_encoder() {
734
+ let dir = std::env::temp_dir().join(format!("ds4-kv-stream-layout-{}", std::process::id()));
735
+ let path = dir.join("entry.kv");
736
+ let _ = fs::remove_dir_all(&dir);
737
+
738
+ let mut rec = sample();
739
+ rec.trailer = b"KVT1 trailer".to_vec();
740
+ let mut payload = rec.payload.as_slice();
741
+ write_stream(
742
+ &path,
743
+ &rec.header,
744
+ &rec.text,
745
+ &mut payload,
746
+ rec.payload.len() as u64,
747
+ &rec.trailer,
748
+ )
749
+ .unwrap();
750
+ assert_eq!(fs::read(&path).unwrap(), encode_file(&rec));
751
+
752
+ let _ = fs::remove_dir_all(&dir);
753
+ }
754
+
755
+ #[test]
756
+ fn staged_stream_leaves_destination_until_commit() {
757
+ let dir = std::env::temp_dir().join(format!("ds4-kv-stream-stage-{}", std::process::id()));
758
+ let path = dir.join("entry.kv");
759
+ let _ = fs::remove_dir_all(&dir);
760
+ fs::create_dir_all(&dir).unwrap();
761
+ fs::write(&path, b"original KVC bytes").unwrap();
762
+
763
+ let rec = sample();
764
+ let mut payload = rec.payload.as_slice();
765
+ let staged = stage_stream(
766
+ &path,
767
+ &rec.header,
768
+ &rec.text,
769
+ &mut payload,
770
+ rec.payload.len() as u64,
771
+ &rec.trailer,
772
+ )
773
+ .unwrap();
774
+ assert_eq!(fs::read(&path).unwrap(), b"original KVC bytes");
775
+ assert_eq!(fs::read(&staged).unwrap(), encode_file(&rec));
776
+
777
+ fs::rename(staged, &path).unwrap();
778
+ assert_eq!(fs::read(&path).unwrap(), encode_file(&rec));
779
+ let _ = fs::remove_dir_all(&dir);
780
+ }
781
+ }
crates/ds4-kv/src/host.rs ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Host-ledger token counts for KV policy. Callers pass these fields;
2
+ //! policy never reads a native session or CUDA scratch.
3
+
4
+ use crate::policy::{bank_checkpoint_due, continued_store_target, Options};
5
+
6
+ #[derive(Clone, Copy, Debug, PartialEq, Eq)]
7
+ pub struct HostKvView {
8
+ pub live_tokens: i32,
9
+ pub stored_tokens: i32,
10
+ }
11
+
12
+ pub fn continued_store_target_from_host(opt: &Options, host: HostKvView) -> i32 {
13
+ continued_store_target(opt, host.stored_tokens, host.live_tokens)
14
+ }
15
+
16
+ pub fn bank_checkpoint_due_from_host(opt: &Options, host: HostKvView) -> bool {
17
+ bank_checkpoint_due(opt, host.live_tokens, host.stored_tokens)
18
+ }
19
+
20
+ #[cfg(test)]
21
+ mod tests {
22
+ use super::*;
23
+
24
+ #[test]
25
+ fn continued_target_uses_host_live_tokens_not_native_pos() {
26
+ let opt = Options::default();
27
+ let due = HostKvView {
28
+ live_tokens: 10240,
29
+ stored_tokens: 0,
30
+ };
31
+ let raw_interval = HostKvView {
32
+ live_tokens: 10000,
33
+ stored_tokens: 0,
34
+ };
35
+ assert_eq!(continued_store_target_from_host(&opt, due), 10240);
36
+ assert_eq!(continued_store_target_from_host(&opt, raw_interval), 0);
37
+ }
38
+
39
+ #[test]
40
+ fn bank_checkpoint_due_uses_host_committed_frontier() {
41
+ let opt = Options::default();
42
+ assert!(bank_checkpoint_due_from_host(
43
+ &opt,
44
+ HostKvView {
45
+ live_tokens: 10240,
46
+ stored_tokens: 0,
47
+ }
48
+ ));
49
+ assert!(!bank_checkpoint_due_from_host(
50
+ &opt,
51
+ HostKvView {
52
+ live_tokens: 10000,
53
+ stored_tokens: 0,
54
+ }
55
+ ));
56
+ assert!(!bank_checkpoint_due_from_host(
57
+ &opt,
58
+ HostKvView {
59
+ live_tokens: 9999,
60
+ stored_tokens: 0,
61
+ }
62
+ ));
63
+ }
64
+ }
crates/ds4-kv/src/identity.rs ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Fixed identity footer after opaque payload and optional KTM records.
2
+ //! Reading identity never scans payload bytes or allocates from file lengths.
3
+
4
+ use std::fs::File;
5
+ use std::io::{Read, Seek, SeekFrom};
6
+ use std::path::Path;
7
+
8
+ pub(crate) const FLAG: u8 = 1 << 6;
9
+ pub(crate) const BYTES: usize = 40;
10
+ const MAGIC: &[u8; 4] = b"KVI\x01";
11
+ const END: &[u8; 4] = b"KVI!";
12
+
13
+ pub(crate) fn append(trailer: &mut Vec<u8>, identity: &[u8; 32]) {
14
+ trailer.extend_from_slice(MAGIC);
15
+ trailer.extend_from_slice(identity);
16
+ trailer.extend_from_slice(END);
17
+ }
18
+
19
+ pub(crate) fn read(path: &Path, file_size: u64, trailer_bytes: u64, flags: u8) -> Option<[u8; 32]> {
20
+ if flags & FLAG == 0 || trailer_bytes < BYTES as u64 || file_size < BYTES as u64 {
21
+ return None;
22
+ }
23
+ let mut file = File::open(path).ok()?;
24
+ if file.metadata().ok()?.len() != file_size {
25
+ return None;
26
+ }
27
+ file.seek(SeekFrom::Start(file_size - BYTES as u64)).ok()?;
28
+ let mut footer = [0; BYTES];
29
+ file.read_exact(&mut footer).ok()?;
30
+ decode(flags, &footer)
31
+ }
32
+
33
+ pub(crate) fn decode(flags: u8, trailer: &[u8]) -> Option<[u8; 32]> {
34
+ if flags & FLAG == 0 || trailer.len() < BYTES {
35
+ return None;
36
+ }
37
+ let footer = &trailer[trailer.len() - BYTES..];
38
+ if &footer[..4] != MAGIC || &footer[36..] != END {
39
+ return None;
40
+ }
41
+ footer[4..36].try_into().ok()
42
+ }
crates/ds4-kv/src/lib.rs ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! KVC disk store. File format, SHA key, eviction, and prefix lookup
2
+ //! match `ds4_kvstore.c` at `v0.6.3-dfm`. Payload bytes stay opaque.
3
+
4
+ mod extensions;
5
+ mod format;
6
+ mod host;
7
+ mod identity;
8
+ mod policy;
9
+ mod sha1;
10
+ mod store;
11
+
12
+ pub use extensions::{
13
+ bank_persist_ext_flags, decode_ktm, encode_ktm, BankThinkingExtensions, ExtensionRecord,
14
+ KTM_MAGIC, KTM_VERSION,
15
+ };
16
+ pub use format::{
17
+ decode_file, encode_file, fill_header, key_kind, parse_header, path_for_sha, read_envelope,
18
+ read_path, read_trailer, sha_hex_name, text_sha_hex, write_path, Envelope, FormatError, Header,
19
+ Reason, Record, EXT_BANK_REPLAY_V1, EXT_IMAGE_PIXELS_V2, EXT_RESPONSES_VISIBLE,
20
+ EXT_SESSION_TITLE, EXT_THINKING_VISIBLE, EXT_TOOL_MAP, FIXED_HEADER, PAYLOAD_ABI, VERSION,
21
+ };
22
+ pub use host::{bank_checkpoint_due_from_host, continued_store_target_from_host, HostKvView};
23
+ pub use policy::{
24
+ bank_checkpoint_due, chat_anchor_pos, continued_store_target, eviction_score, file_size_fits,
25
+ store_len, EvictionContext, Options, ScoreEntry,
26
+ };
27
+ pub use sha1::sha1_hex;
28
+ pub use store::{Entry, PayloadGuard, PayloadTemp, PrefixAnswer, Store};
crates/ds4-kv/src/policy.rs ADDED
@@ -0,0 +1,340 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Store-length, continued pacing, budget, and eviction scoring.
2
+ //! Port of the corresponding helpers in `ds4_kvstore.c`.
3
+
4
+ use crate::format::{text_sha_hex, Reason};
5
+
6
+ pub const DEFAULT_MB: u64 = 4096;
7
+ pub const HIT_HALF_LIFE_SECONDS: u64 = 6 * 60 * 60;
8
+ pub const DEFAULT_MIN_TOKENS: i32 = 512;
9
+ pub const DEFAULT_COLD_MAX_TOKENS: i32 = 30000;
10
+ pub const DEFAULT_BOUNDARY_TRIM_TOKENS: i32 = 32;
11
+ pub const DEFAULT_BOUNDARY_ALIGN_TOKENS: i32 = 2048;
12
+ pub const DEFAULT_CONTINUED_INTERVAL_TOKENS: i32 = 10000;
13
+ const MIN_EFFECTIVE_HITS: f64 = 0.01;
14
+ const CONTINUED_PREFIX_MIN_FACTOR: f64 = 0.05;
15
+ const CONTINUED_PREFIX_HIT_FACTOR: f64 = 0.45;
16
+ const ANCHOR_REASON_SCORE_FACTOR: f64 = 2.0;
17
+
18
+ #[derive(Clone, Copy, Debug, PartialEq, Eq)]
19
+ pub struct Options {
20
+ pub min_tokens: i32,
21
+ pub cold_max_tokens: i32,
22
+ pub continued_interval_tokens: i32,
23
+ pub boundary_trim_tokens: i32,
24
+ pub boundary_align_tokens: i32,
25
+ }
26
+
27
+ impl Default for Options {
28
+ fn default() -> Self {
29
+ Self {
30
+ min_tokens: DEFAULT_MIN_TOKENS,
31
+ cold_max_tokens: DEFAULT_COLD_MAX_TOKENS,
32
+ continued_interval_tokens: DEFAULT_CONTINUED_INTERVAL_TOKENS,
33
+ boundary_trim_tokens: DEFAULT_BOUNDARY_TRIM_TOKENS,
34
+ boundary_align_tokens: DEFAULT_BOUNDARY_ALIGN_TOKENS,
35
+ }
36
+ }
37
+ }
38
+
39
+ #[derive(Clone, Debug)]
40
+ pub struct EvictionContext<'a> {
41
+ pub text: &'a [u8],
42
+ pub model_id: u8,
43
+ pub quant_bits: u8,
44
+ pub ctx_size: u32,
45
+ pub reject_different_quant: bool,
46
+ }
47
+
48
+ #[derive(Clone, Debug)]
49
+ pub struct ScoreEntry<'a> {
50
+ pub sha: &'a str,
51
+ pub quant_bits: u8,
52
+ pub model_id: u8,
53
+ pub reason: Reason,
54
+ pub tokens: u32,
55
+ pub hits: u32,
56
+ pub ctx_size: u32,
57
+ pub created_at: u64,
58
+ pub last_used: u64,
59
+ pub text_bytes: u64,
60
+ pub file_size: u64,
61
+ }
62
+
63
+ pub fn store_len(opt: &Options, tokens: i32) -> i32 {
64
+ let trim = opt.boundary_trim_tokens;
65
+ let align = opt.boundary_align_tokens;
66
+ if tokens > opt.min_tokens + trim {
67
+ let mut stable = tokens - trim;
68
+ if align > 0 {
69
+ stable -= stable % align;
70
+ }
71
+ if stable >= opt.min_tokens {
72
+ return stable;
73
+ }
74
+ }
75
+ tokens
76
+ }
77
+
78
+ pub fn continued_step(opt: &Options) -> i32 {
79
+ if opt.continued_interval_tokens <= 0 {
80
+ return 0;
81
+ }
82
+ let mut step = opt.continued_interval_tokens;
83
+ let align = opt.boundary_align_tokens;
84
+ if align > 0 {
85
+ let rounded =
86
+ ((i64::from(step) + i64::from(align) - 1) / i64::from(align)) * i64::from(align);
87
+ step = i32::try_from(rounded).unwrap_or(0);
88
+ }
89
+ step
90
+ }
91
+
92
+ pub fn continued_store_target(opt: &Options, last_store_tokens: i32, live_tokens: i32) -> i32 {
93
+ let step = continued_step(opt);
94
+ if step <= 0 {
95
+ return 0;
96
+ }
97
+ if live_tokens < opt.min_tokens {
98
+ return 0;
99
+ }
100
+ if live_tokens % step != 0 {
101
+ return 0;
102
+ }
103
+ if live_tokens <= last_store_tokens {
104
+ return 0;
105
+ }
106
+ live_tokens
107
+ }
108
+
109
+ pub fn bank_checkpoint_due(opt: &Options, committed: i32, stored_tokens: i32) -> bool {
110
+ let step = continued_step(opt);
111
+ if step <= 0 || committed < opt.min_tokens || committed < stored_tokens + step {
112
+ return false;
113
+ }
114
+ true
115
+ }
116
+
117
+ pub fn file_size_bytes(text_bytes: u64, payload_bytes: u64, trailer_bytes: u64) -> Option<u64> {
118
+ const FIXED: u64 = 48 + 4;
119
+ FIXED
120
+ .checked_add(text_bytes)?
121
+ .checked_add(payload_bytes)?
122
+ .checked_add(trailer_bytes)
123
+ }
124
+
125
+ pub fn budget_required(file_bytes: u64) -> Option<u64> {
126
+ let mut slack = file_bytes / 100;
127
+ if file_bytes % 100 != 0 {
128
+ slack += 1;
129
+ }
130
+ file_bytes.checked_add(slack)
131
+ }
132
+
133
+ pub fn file_size_fits(
134
+ budget_bytes: u64,
135
+ text_bytes: u64,
136
+ payload_bytes: u64,
137
+ trailer_bytes: u64,
138
+ ) -> bool {
139
+ let Some(file_bytes) = file_size_bytes(text_bytes, payload_bytes, trailer_bytes) else {
140
+ return false;
141
+ };
142
+ if budget_bytes == 0 {
143
+ return true;
144
+ }
145
+ match budget_required(file_bytes) {
146
+ Some(required) => required <= budget_bytes,
147
+ None => false,
148
+ }
149
+ }
150
+
151
+ fn reason_is_anchor(reason: Reason) -> bool {
152
+ matches!(
153
+ reason,
154
+ Reason::Cold
155
+ | Reason::Evict
156
+ | Reason::Shutdown
157
+ | Reason::BankEvict
158
+ | Reason::BankShutdown
159
+ | Reason::BankCheckpoint
160
+ )
161
+ }
162
+
163
+ fn incoming_supersedes_continued(e: &ScoreEntry<'_>, incoming: &EvictionContext<'_>) -> bool {
164
+ if e.reason != Reason::Continued && e.reason != Reason::BankCheckpoint {
165
+ return false;
166
+ }
167
+ if e.text_bytes == 0 || e.text_bytes >= incoming.text.len() as u64 {
168
+ return false;
169
+ }
170
+ if e.model_id != incoming.model_id {
171
+ return false;
172
+ }
173
+ if incoming.reject_different_quant && e.quant_bits != incoming.quant_bits {
174
+ return false;
175
+ }
176
+ if incoming.ctx_size > e.ctx_size {
177
+ return false;
178
+ }
179
+ let prefix = &incoming.text[..e.text_bytes as usize];
180
+ text_sha_hex(prefix) == e.sha
181
+ }
182
+
183
+ pub fn eviction_score(e: &ScoreEntry<'_>, now: u64, incoming: Option<&EvictionContext<'_>>) -> f64 {
184
+ if e.file_size == 0 {
185
+ return 0.0;
186
+ }
187
+ let mut effective_hits = e.hits as f64;
188
+ let used_at = if e.last_used != 0 {
189
+ e.last_used
190
+ } else {
191
+ e.created_at
192
+ };
193
+ if used_at == 0 {
194
+ effective_hits = 0.0;
195
+ } else if now > used_at {
196
+ let elapsed = (now - used_at) as f64;
197
+ effective_hits *= (-elapsed / HIT_HALF_LIFE_SECONDS as f64).exp2();
198
+ if effective_hits < MIN_EFFECTIVE_HITS {
199
+ effective_hits = 0.0;
200
+ }
201
+ }
202
+ let mut score = (effective_hits + 1.0) * (e.tokens as f64) / (e.file_size as f64);
203
+ if reason_is_anchor(e.reason) {
204
+ score *= ANCHOR_REASON_SCORE_FACTOR;
205
+ }
206
+ if let Some(incoming) = incoming {
207
+ if incoming_supersedes_continued(e, incoming) {
208
+ let h = if effective_hits > 0.0 {
209
+ effective_hits / (effective_hits + 1.0)
210
+ } else {
211
+ 0.0
212
+ };
213
+ score *= CONTINUED_PREFIX_MIN_FACTOR + CONTINUED_PREFIX_HIT_FACTOR * h;
214
+ }
215
+ }
216
+ score
217
+ }
218
+
219
+ pub fn chat_anchor_pos(
220
+ opt: &Options,
221
+ prompt: &[i32],
222
+ user_token_id: i32,
223
+ assistant_token_id: i32,
224
+ ) -> i32 {
225
+ if user_token_id < 0 || assistant_token_id < 0 {
226
+ return -1;
227
+ }
228
+ let mut last_user = -1;
229
+ for (i, &token) in prompt.iter().enumerate() {
230
+ if token == assistant_token_id {
231
+ break;
232
+ }
233
+ if token == user_token_id {
234
+ last_user = i as i32;
235
+ }
236
+ }
237
+ if last_user >= opt.min_tokens {
238
+ last_user
239
+ } else {
240
+ -1
241
+ }
242
+ }
243
+
244
+ #[cfg(test)]
245
+ mod tests {
246
+ use super::*;
247
+
248
+ #[test]
249
+ fn store_len_trims_to_align() {
250
+ let opt = Options::default();
251
+ assert_eq!(store_len(&opt, 4096 + 40), 4096);
252
+ assert_eq!(store_len(&opt, 100), 100);
253
+ }
254
+
255
+ #[test]
256
+ fn continued_target_needs_aligned_interval() {
257
+ let opt = Options::default();
258
+ assert_eq!(continued_store_target(&opt, 0, 8192), 0);
259
+ assert_eq!(continued_store_target(&opt, 0, 10240), 10240);
260
+ assert_eq!(continued_store_target(&opt, 10240, 10240), 0);
261
+ }
262
+
263
+ #[test]
264
+ fn continued_step_maps_default_10000_interval_to_10240() {
265
+ let opt = Options::default();
266
+ assert_eq!(DEFAULT_CONTINUED_INTERVAL_TOKENS, 10000);
267
+ assert_eq!(
268
+ opt.continued_interval_tokens,
269
+ DEFAULT_CONTINUED_INTERVAL_TOKENS
270
+ );
271
+ assert_eq!(continued_step(&opt), 10240);
272
+ }
273
+
274
+ #[test]
275
+ fn bank_checkpoint_due_when_committed_at_10240_aligned_frontier() {
276
+ let opt = Options::default();
277
+ assert!(bank_checkpoint_due(&opt, 10240, 0));
278
+ }
279
+
280
+ #[test]
281
+ fn bank_checkpoint_not_due_when_committed_is_raw_10000_interval() {
282
+ let opt = Options::default();
283
+ assert!(!bank_checkpoint_due(&opt, 10000, 0));
284
+ }
285
+
286
+ #[test]
287
+ fn bank_checkpoint_not_due_when_committed_is_9999() {
288
+ let opt = Options::default();
289
+ assert!(!bank_checkpoint_due(&opt, 9999, 0));
290
+ }
291
+
292
+ #[test]
293
+ fn continued_step_disables_unrepresentable_aligned_interval() {
294
+ let opt = Options {
295
+ min_tokens: 1,
296
+ continued_interval_tokens: i32::MAX,
297
+ boundary_align_tokens: 2048,
298
+ ..Options::default()
299
+ };
300
+ assert_eq!(continued_step(&opt), 0);
301
+ assert_eq!(continued_store_target(&opt, 0, 2048), 0);
302
+ }
303
+
304
+ #[test]
305
+ fn budget_adds_one_percent() {
306
+ assert_eq!(budget_required(100), Some(101));
307
+ assert_eq!(budget_required(101), Some(103));
308
+ }
309
+
310
+ #[test]
311
+ fn equal_length_incoming_does_not_discount_continued() {
312
+ let text = b"same prefix";
313
+ let sha = text_sha_hex(text);
314
+ let entry = ScoreEntry {
315
+ sha: &sha,
316
+ quant_bits: 2,
317
+ model_id: 0,
318
+ reason: Reason::Continued,
319
+ tokens: 512,
320
+ hits: 0,
321
+ ctx_size: 2048,
322
+ created_at: 1000,
323
+ last_used: 1000,
324
+ text_bytes: text.len() as u64,
325
+ file_size: 4096,
326
+ };
327
+ let incoming = EvictionContext {
328
+ text,
329
+ model_id: 0,
330
+ quant_bits: 2,
331
+ ctx_size: 2048,
332
+ reject_different_quant: true,
333
+ };
334
+
335
+ assert_eq!(
336
+ eviction_score(&entry, 1000, Some(&incoming)),
337
+ eviction_score(&entry, 1000, None)
338
+ );
339
+ }
340
+ }
crates/ds4-kv/src/sha1.rs ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Byte-identical port of the SHA-1 in `ds4_kvstore.c`.
2
+
3
+ pub fn sha1(bytes: &[u8]) -> [u8; 20] {
4
+ let mut c = Ctx::new();
5
+ c.update(bytes);
6
+ c.finish()
7
+ }
8
+
9
+ pub fn sha1_hex(bytes: &[u8]) -> String {
10
+ hex20(&sha1(bytes))
11
+ }
12
+
13
+ struct Ctx {
14
+ h: [u32; 5],
15
+ bytes: u64,
16
+ block: [u8; 64],
17
+ used: usize,
18
+ }
19
+
20
+ impl Ctx {
21
+ fn new() -> Self {
22
+ Self {
23
+ h: [
24
+ 0x6745_2301,
25
+ 0xefcd_ab89,
26
+ 0x98ba_dcfe,
27
+ 0x1032_5476,
28
+ 0xc3d2_e1f0,
29
+ ],
30
+ bytes: 0,
31
+ block: [0; 64],
32
+ used: 0,
33
+ }
34
+ }
35
+
36
+ fn update(&mut self, mut input: &[u8]) {
37
+ self.bytes += input.len() as u64;
38
+ while !input.is_empty() {
39
+ let n = (64 - self.used).min(input.len());
40
+ self.block[self.used..self.used + n].copy_from_slice(&input[..n]);
41
+ self.used += n;
42
+ input = &input[n..];
43
+ if self.used == 64 {
44
+ self.transform();
45
+ self.used = 0;
46
+ }
47
+ }
48
+ }
49
+
50
+ fn finish(&mut self) -> [u8; 20] {
51
+ let bits = self.bytes * 8;
52
+ self.update(&[0x80]);
53
+ let zero = [0u8; 1];
54
+ while self.used != 56 {
55
+ self.update(&zero);
56
+ }
57
+ let mut len = [0u8; 8];
58
+ for i in 0..8 {
59
+ len[7 - i] = (bits >> (8 * i)) as u8;
60
+ }
61
+ self.update(&len);
62
+ let mut out = [0u8; 20];
63
+ for i in 0..5 {
64
+ out[i * 4] = (self.h[i] >> 24) as u8;
65
+ out[i * 4 + 1] = (self.h[i] >> 16) as u8;
66
+ out[i * 4 + 2] = (self.h[i] >> 8) as u8;
67
+ out[i * 4 + 3] = self.h[i] as u8;
68
+ }
69
+ out
70
+ }
71
+
72
+ fn transform(&mut self) {
73
+ let mut w = [0u32; 80];
74
+ for i in 0..16 {
75
+ w[i] = (u32::from(self.block[i * 4]) << 24)
76
+ | (u32::from(self.block[i * 4 + 1]) << 16)
77
+ | (u32::from(self.block[i * 4 + 2]) << 8)
78
+ | u32::from(self.block[i * 4 + 3]);
79
+ }
80
+ for i in 16..80 {
81
+ w[i] = rol32(w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16], 1);
82
+ }
83
+
84
+ let mut a = self.h[0];
85
+ let mut b = self.h[1];
86
+ let mut cc = self.h[2];
87
+ let mut d = self.h[3];
88
+ let mut e = self.h[4];
89
+ for i in 0..80 {
90
+ let (f, k) = if i < 20 {
91
+ ((b & cc) | ((!b) & d), 0x5a82_7999)
92
+ } else if i < 40 {
93
+ (b ^ cc ^ d, 0x6ed9_eba1)
94
+ } else if i < 60 {
95
+ ((b & cc) | (b & d) | (cc & d), 0x8f1b_bcdc)
96
+ } else {
97
+ (b ^ cc ^ d, 0xca62_c1d6)
98
+ };
99
+ let tmp = rol32(a, 5)
100
+ .wrapping_add(f)
101
+ .wrapping_add(e)
102
+ .wrapping_add(k)
103
+ .wrapping_add(w[i]);
104
+ e = d;
105
+ d = cc;
106
+ cc = rol32(b, 30);
107
+ b = a;
108
+ a = tmp;
109
+ }
110
+ self.h[0] = self.h[0].wrapping_add(a);
111
+ self.h[1] = self.h[1].wrapping_add(b);
112
+ self.h[2] = self.h[2].wrapping_add(cc);
113
+ self.h[3] = self.h[3].wrapping_add(d);
114
+ self.h[4] = self.h[4].wrapping_add(e);
115
+ }
116
+ }
117
+
118
+ fn rol32(v: u32, n: i32) -> u32 {
119
+ v.rotate_left(n as u32)
120
+ }
121
+
122
+ fn hex20(digest: &[u8; 20]) -> String {
123
+ const HEX: &[u8; 16] = b"0123456789abcdef";
124
+ let mut out = String::with_capacity(40);
125
+ for &b in digest {
126
+ out.push(HEX[(b >> 4) as usize] as char);
127
+ out.push(HEX[(b & 15) as usize] as char);
128
+ }
129
+ out
130
+ }
131
+
132
+ #[cfg(test)]
133
+ mod tests {
134
+ use super::*;
135
+
136
+ #[test]
137
+ fn sha1_abc_matches_fips() {
138
+ assert_eq!(sha1_hex(b"abc"), "a9993e364706816aba3e25717850c26c9cd0d89d");
139
+ }
140
+ }
crates/ds4-kv/src/store.rs ADDED
@@ -0,0 +1,2406 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Directory-backed KVC catalog: refresh, prefix lookup, LCP, evict.
2
+
3
+ use crate::format::{
4
+ fill_header, is_automatic_exact_replay, is_bank_replay_v1, path_for_sha, read_envelope,
5
+ read_header_text, read_metadata, read_path, read_text_prefix, sha_hex_name, stage_stream,
6
+ text_sha_hex, write_path, Envelope, FormatError, Header, Reason, Record, EXT_IMAGE_PIXELS_V2,
7
+ EXT_TOOL_MAP,
8
+ };
9
+ use crate::policy::{
10
+ eviction_score, file_size_bytes, file_size_fits, EvictionContext, Options, ScoreEntry,
11
+ DEFAULT_MB,
12
+ };
13
+ use std::fs;
14
+ use std::io::{self, Read as _, Seek as _, SeekFrom, Write as _};
15
+ #[cfg(unix)]
16
+ use std::os::fd::AsRawFd as _;
17
+ #[cfg(unix)]
18
+ use std::os::unix::fs::{DirBuilderExt as _, OpenOptionsExt as _};
19
+ use std::path::{Path, PathBuf};
20
+ use std::sync::atomic::{AtomicU64, Ordering};
21
+ use std::time::{SystemTime, UNIX_EPOCH};
22
+
23
+ static PAYLOAD_TEMP_SEQ: AtomicU64 = AtomicU64::new(0);
24
+
25
+ /// Whether a record must leave something to prefill. The bank lane admits
26
+ /// only records it can extend; the serial lane may replay an exact one.
27
+ #[derive(Clone, Copy, PartialEq, Eq)]
28
+ enum Suffix {
29
+ Optional,
30
+ Required,
31
+ }
32
+
33
+ #[derive(Clone, Copy, PartialEq, Eq)]
34
+ enum FileAccess {
35
+ ReadOnly,
36
+ ReadWrite,
37
+ }
38
+
39
+ /// What a search would do with the best record it can see for a key. The
40
+ /// store answers what it holds; the lane turns that into a miss reason.
41
+ #[derive(Clone, Copy, Debug, PartialEq, Eq)]
42
+ pub enum PrefixAnswer {
43
+ /// Nothing is keyed by this prompt.
44
+ None,
45
+ /// A record the search takes. Only the caller's own budget can refuse
46
+ /// it now, and the reuse contract has no reason for that.
47
+ Usable,
48
+ /// A record ruled out by model, quantization or context.
49
+ Mismatch,
50
+ /// A record shallower than the store's current minimum — written
51
+ /// before the minimum was raised, and skipped by every search since.
52
+ Shallow,
53
+ }
54
+
55
+ #[derive(Clone, Debug)]
56
+ pub struct Entry {
57
+ pub sha: String,
58
+ pub path: PathBuf,
59
+ pub header: Header,
60
+ pub file_size: u64,
61
+ identity: Option<[u8; 32]>,
62
+ }
63
+
64
+ /// A file the catalog cannot use: its payload no longer holds what its
65
+ /// header describes. The name still says which text it was keyed by.
66
+ #[derive(Clone, Debug)]
67
+ struct Damaged {
68
+ sha: String,
69
+ path: PathBuf,
70
+ reason: Reason,
71
+ ext_flags: u8,
72
+ text_bytes: u32,
73
+ }
74
+
75
+ #[derive(Debug)]
76
+ pub struct Store {
77
+ pub dir: PathBuf,
78
+ pub budget_bytes: u64,
79
+ pub reject_different_quant: bool,
80
+ pub opt: Options,
81
+ pub continued_last_store_tokens: i32,
82
+ entries: Vec<Entry>,
83
+ damaged: Vec<Damaged>,
84
+ identity: Option<[u8; 32]>,
85
+ }
86
+
87
+ #[derive(Debug)]
88
+ pub struct PayloadTemp {
89
+ path: PathBuf,
90
+ }
91
+
92
+ /// Holds the validated inode while a native loader reopens its descriptor path.
93
+ /// An atomic replacement at the catalog path cannot swap the loaded artifact.
94
+ #[derive(Debug)]
95
+ pub struct PayloadGuard {
96
+ _file: fs::File,
97
+ path: PathBuf,
98
+ }
99
+
100
+ impl PayloadGuard {
101
+ pub fn path(&self) -> &Path {
102
+ &self.path
103
+ }
104
+ }
105
+
106
+ impl PayloadTemp {
107
+ pub fn path(&self) -> &Path {
108
+ &self.path
109
+ }
110
+ }
111
+
112
+ impl Drop for PayloadTemp {
113
+ fn drop(&mut self) {
114
+ let _ = fs::remove_file(&self.path);
115
+ }
116
+ }
117
+
118
+ impl Store {
119
+ pub fn open(
120
+ dir: impl AsRef<Path>,
121
+ budget_mb: u64,
122
+ reject_different_quant: bool,
123
+ opt: Options,
124
+ ) -> io::Result<Self> {
125
+ let dir = dir.as_ref().to_path_buf();
126
+ #[cfg(unix)]
127
+ {
128
+ let mut builder = fs::DirBuilder::new();
129
+ builder.recursive(true).mode(0o700).create(&dir)?;
130
+ }
131
+ #[cfg(not(unix))]
132
+ fs::create_dir_all(&dir)?;
133
+ let budget_mb = if budget_mb == 0 {
134
+ DEFAULT_MB
135
+ } else {
136
+ budget_mb
137
+ };
138
+ let mut store = Self {
139
+ dir,
140
+ budget_bytes: budget_mb * 1024 * 1024,
141
+ reject_different_quant,
142
+ opt,
143
+ continued_last_store_tokens: 0,
144
+ entries: Vec::new(),
145
+ damaged: Vec::new(),
146
+ identity: None,
147
+ };
148
+ store.evict(0, None);
149
+ Ok(store)
150
+ }
151
+
152
+ /// Require this artifact/runtime identity for all subsequent reads and
153
+ /// writes. Legacy entries remain in the shared budget, but cannot be reused.
154
+ pub fn bind_identity(&mut self, identity: [u8; 32]) {
155
+ self.identity = Some(identity);
156
+ self.continued_last_store_tokens = 0;
157
+ self.refresh();
158
+ }
159
+
160
+ fn matches_identity(&self, flags: u8, identity: Option<[u8; 32]>) -> bool {
161
+ match self.identity {
162
+ Some(expected) => flags & crate::identity::FLAG != 0 && identity == Some(expected),
163
+ None => flags & crate::identity::FLAG == 0,
164
+ }
165
+ }
166
+
167
+ fn envelope_identity(&self, path: &Path, e: &Envelope) -> bool {
168
+ self.matches_identity(
169
+ e.header.ext_flags,
170
+ crate::identity::read(path, e.file_size, e.trailer_bytes, e.header.ext_flags),
171
+ )
172
+ }
173
+
174
+ fn check_identity(&self, path: &Path, e: &Envelope) -> io::Result<()> {
175
+ if self.envelope_identity(path, e) {
176
+ return Ok(());
177
+ }
178
+ Err(io::Error::new(
179
+ io::ErrorKind::InvalidData,
180
+ "KVC artifact/runtime identity mismatch",
181
+ ))
182
+ }
183
+
184
+ fn scoped_trailer(&self, header: &mut Header, trailer: &[u8]) -> Vec<u8> {
185
+ let mut out = trailer.to_vec();
186
+ if let Some(identity) = self.identity {
187
+ header.ext_flags |= crate::identity::FLAG;
188
+ crate::identity::append(&mut out, &identity);
189
+ }
190
+ out
191
+ }
192
+
193
+ fn visible_envelope(&self, mut envelope: Envelope) -> Envelope {
194
+ // Callers validate protocol extensions independently. Identity has
195
+ // already been checked; it is neither a tool map nor payload content.
196
+ if self.identity.is_some() {
197
+ envelope.header.ext_flags &= !crate::identity::FLAG;
198
+ envelope.trailer_bytes -= crate::identity::BYTES as u64;
199
+ }
200
+ envelope
201
+ }
202
+
203
+ pub fn payload_temp(&self) -> io::Result<PayloadTemp> {
204
+ loop {
205
+ let seq = PAYLOAD_TEMP_SEQ.fetch_add(1, Ordering::Relaxed);
206
+ let path = self
207
+ .dir
208
+ .join(format!(".payload.{}.{}", std::process::id(), seq));
209
+ let mut options = fs::OpenOptions::new();
210
+ options.write(true).create_new(true);
211
+ #[cfg(unix)]
212
+ options.mode(0o600);
213
+ match options.open(&path) {
214
+ Ok(_) => return Ok(PayloadTemp { path }),
215
+ Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
216
+ Err(error) => return Err(error),
217
+ }
218
+ }
219
+ }
220
+
221
+ pub fn refresh(&mut self) {
222
+ self.entries.clear();
223
+ self.damaged.clear();
224
+ let Ok(rd) = fs::read_dir(&self.dir) else {
225
+ return;
226
+ };
227
+ for ent in rd.flatten() {
228
+ let name = ent.file_name();
229
+ let Some(name) = name.to_str() else { continue };
230
+ let Some(sha) = sha_hex_name(name) else {
231
+ continue;
232
+ };
233
+ let path = ent.path();
234
+ let Ok(metadata) = read_metadata(&path) else {
235
+ // No search can use it, but it can still be the reason a
236
+ // prompt keyed by its text finds nothing.
237
+ if let Ok((header, _)) = read_header_text(&path, 0) {
238
+ self.damaged.push(Damaged {
239
+ sha,
240
+ path,
241
+ reason: header.reason,
242
+ ext_flags: header.ext_flags,
243
+ text_bytes: header.text_bytes,
244
+ });
245
+ }
246
+ continue;
247
+ };
248
+ let identity = crate::identity::read(
249
+ &path,
250
+ metadata.file_size,
251
+ metadata.trailer_bytes,
252
+ metadata.header.ext_flags,
253
+ );
254
+ self.entries.push(Entry {
255
+ sha,
256
+ path,
257
+ header: metadata.header,
258
+ file_size: metadata.file_size,
259
+ identity,
260
+ });
261
+ }
262
+ }
263
+
264
+ pub fn entries(&self) -> &[Entry] {
265
+ &self.entries
266
+ }
267
+
268
+ pub fn write(&mut self, mut record: Record) -> io::Result<PathBuf> {
269
+ record.trailer = self.scoped_trailer(&mut record.header, &record.trailer);
270
+ if !file_size_fits(
271
+ self.budget_bytes,
272
+ record.text.len() as u64,
273
+ record.payload.len() as u64,
274
+ record.trailer.len() as u64,
275
+ ) {
276
+ return Err(io::Error::new(
277
+ io::ErrorKind::OutOfMemory,
278
+ "KVC file exceeds budget",
279
+ ));
280
+ }
281
+ record.header.text_bytes = record.text.len() as u32;
282
+ record.header.payload_bytes = record.payload.len() as u64;
283
+ let sha = text_sha_hex(&record.text);
284
+ let path = path_for_sha(&self.dir, &sha);
285
+ let incoming = EvictionContext {
286
+ text: &record.text,
287
+ model_id: record.header.model_id,
288
+ quant_bits: record.header.quant_bits,
289
+ ctx_size: record.header.ctx_size,
290
+ reject_different_quant: self.reject_different_quant,
291
+ };
292
+ let extra = 48
293
+ + 4
294
+ + record.text.len() as u64
295
+ + record.payload.len() as u64
296
+ + record.trailer.len() as u64;
297
+ self.evict(extra, Some(&incoming));
298
+ write_path(&path, &record).map_err(|e| io::Error::other(e))?;
299
+ self.refresh();
300
+ Ok(path)
301
+ }
302
+
303
+ /// Return an already-compatible KVC entry before the caller stages its
304
+ /// opaque native payload. A nonempty trailer follows the writer fast path.
305
+ pub fn reuse_compatible(
306
+ &mut self,
307
+ mut header: Header,
308
+ text: &[u8],
309
+ trailer: &[u8],
310
+ ) -> io::Result<Option<PathBuf>> {
311
+ if !trailer.is_empty() {
312
+ header.ext_flags |= EXT_TOOL_MAP;
313
+ }
314
+ let sha = text_sha_hex(text);
315
+ let path = path_for_sha(&self.dir, &sha);
316
+ let Ok(existing) = read_envelope(&path) else {
317
+ return Ok(None);
318
+ };
319
+ let compatible = self.envelope_identity(&path, &existing)
320
+ && existing.header.model_id == header.model_id
321
+ && (!self.reject_different_quant || existing.header.quant_bits == header.quant_bits)
322
+ && existing.header.ctx_size <= header.ctx_size
323
+ && is_automatic_exact_replay(existing.header.reason, existing.header.ext_flags)
324
+ && is_automatic_exact_replay(header.reason, header.ext_flags)
325
+ && existing.text == text
326
+ && text_sha_hex(&existing.text) == sha;
327
+ if !compatible {
328
+ return Ok(None);
329
+ }
330
+ let visible = self.visible_envelope(existing.clone());
331
+ if trailer.is_empty() {
332
+ // Revalidate header and footer through one descriptor on the
333
+ // no-write fast path too; catalog paths may be atomically replaced.
334
+ self.payload_file(&path, &visible, FileAccess::ReadOnly)?;
335
+ } else {
336
+ let trailer = self.scoped_trailer(&mut header, trailer);
337
+ let mut file = self.payload_file(&path, &visible, FileAccess::ReadWrite)?;
338
+ rewrite_compatible_trailer(&mut file, &existing, header.ext_flags, &trailer)?;
339
+ }
340
+ Ok(Some(path))
341
+ }
342
+
343
+ /// Store an opaque native payload without buffering it in Rust memory.
344
+ /// A nonempty trailer is the frozen server tool-map suffix.
345
+ pub fn write_payload_file(
346
+ &mut self,
347
+ mut header: Header,
348
+ text: &[u8],
349
+ payload_path: &Path,
350
+ trailer: &[u8],
351
+ ) -> io::Result<PathBuf> {
352
+ if text.len() > u32::MAX as usize {
353
+ return Err(io::Error::new(
354
+ io::ErrorKind::InvalidInput,
355
+ "KVC text exceeds u32 length",
356
+ ));
357
+ }
358
+ if !trailer.is_empty() {
359
+ header.ext_flags |= EXT_TOOL_MAP;
360
+ }
361
+ if let Some(path) = self.reuse_compatible(header.clone(), text, trailer)? {
362
+ return Ok(path);
363
+ }
364
+ let trailer = self.scoped_trailer(&mut header, trailer);
365
+ let sha = text_sha_hex(text);
366
+ let path = path_for_sha(&self.dir, &sha);
367
+ if payload_path == path {
368
+ return Err(io::Error::new(
369
+ io::ErrorKind::InvalidInput,
370
+ "KVC payload source aliases destination",
371
+ ));
372
+ }
373
+ let mut payload = fs::File::open(payload_path)?;
374
+ let payload_bytes = payload.metadata()?.len();
375
+ let extra = file_size_bytes(text.len() as u64, payload_bytes, trailer.len() as u64)
376
+ .ok_or_else(|| {
377
+ io::Error::new(io::ErrorKind::InvalidInput, "KVC file size overflows")
378
+ })?;
379
+ if !file_size_fits(
380
+ self.budget_bytes,
381
+ text.len() as u64,
382
+ payload_bytes,
383
+ trailer.len() as u64,
384
+ ) {
385
+ return Err(io::Error::new(
386
+ io::ErrorKind::OutOfMemory,
387
+ "KVC file exceeds budget",
388
+ ));
389
+ }
390
+ header.text_bytes = text.len() as u32;
391
+ header.payload_bytes = payload_bytes;
392
+ let incoming = EvictionContext {
393
+ text,
394
+ model_id: header.model_id,
395
+ quant_bits: header.quant_bits,
396
+ ctx_size: header.ctx_size,
397
+ reject_different_quant: self.reject_different_quant,
398
+ };
399
+ let staged = stage_stream(&path, &header, text, &mut payload, payload_bytes, &trailer)
400
+ .map_err(format_io_error)?;
401
+ self.evict_excluding(extra, Some(&incoming), Some(&path));
402
+ if let Err(error) = fs::rename(&staged, &path) {
403
+ let _ = fs::remove_file(staged);
404
+ return Err(error);
405
+ }
406
+ self.refresh();
407
+ Ok(path)
408
+ }
409
+
410
+ pub fn read(&self, path: &Path) -> io::Result<Record> {
411
+ let envelope = read_envelope(path).map_err(format_io_error)?;
412
+ self.check_identity(path, &envelope)?;
413
+ let mut record = read_path(path).map_err(io::Error::other)?;
414
+ self.check_trailer(record.header.ext_flags, &record.trailer)?;
415
+ if self.identity.is_some() {
416
+ record
417
+ .trailer
418
+ .truncate(record.trailer.len().saturating_sub(crate::identity::BYTES));
419
+ record.header.ext_flags &= !crate::identity::FLAG;
420
+ }
421
+ Ok(record)
422
+ }
423
+
424
+ pub fn open_payload(&self, path: &Path, expected: &Envelope) -> io::Result<PayloadGuard> {
425
+ let file = self.payload_file(path, expected, FileAccess::ReadOnly)?;
426
+ #[cfg(target_os = "linux")]
427
+ let held_path = PathBuf::from(format!("/proc/self/fd/{}", file.as_raw_fd()));
428
+ #[cfg(all(unix, not(target_os = "linux")))]
429
+ let held_path = PathBuf::from(format!("/dev/fd/{}", file.as_raw_fd()));
430
+ #[cfg(not(unix))]
431
+ let held_path = {
432
+ return Err(io::Error::new(
433
+ io::ErrorKind::Unsupported,
434
+ "held KVC payloads need Unix descriptors",
435
+ ));
436
+ };
437
+ Ok(PayloadGuard {
438
+ _file: file,
439
+ path: held_path,
440
+ })
441
+ }
442
+
443
+ fn payload_file(
444
+ &self,
445
+ path: &Path,
446
+ expected: &Envelope,
447
+ access: FileAccess,
448
+ ) -> io::Result<fs::File> {
449
+ let mut file = fs::OpenOptions::new()
450
+ .read(true)
451
+ .write(access == FileAccess::ReadWrite)
452
+ .open(path)?;
453
+ let mut raw = [0; crate::format::FIXED_HEADER];
454
+ file.read_exact(&mut raw)?;
455
+ let mut text_len = [0; 4];
456
+ file.read_exact(&mut text_len)?;
457
+ let header = crate::format::parse_header(&raw, u32::from_le_bytes(text_len))
458
+ .map_err(format_io_error)?;
459
+ let payload_offset =
460
+ (crate::format::FIXED_HEADER + 4) as u64 + u64::from(header.text_bytes);
461
+ let file_size = file.metadata()?.len();
462
+ let payload_end = payload_offset
463
+ .checked_add(header.payload_bytes)
464
+ .filter(|end| *end <= file_size)
465
+ .ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "truncated KVC payload"))?;
466
+ let trailer_bytes = file_size - payload_end;
467
+ let mut footer = [0; crate::identity::BYTES];
468
+ let identity = if header.ext_flags & crate::identity::FLAG != 0
469
+ && trailer_bytes >= footer.len() as u64
470
+ {
471
+ file.seek(SeekFrom::Start(file_size - footer.len() as u64))?;
472
+ file.read_exact(&mut footer)?;
473
+ crate::identity::decode(header.ext_flags, &footer)
474
+ } else {
475
+ None
476
+ };
477
+ let wanted = &expected.header;
478
+ if !self.matches_identity(header.ext_flags, identity)
479
+ || payload_offset != expected.payload_offset
480
+ || header.payload_bytes != wanted.payload_bytes
481
+ || header.model_id != wanted.model_id
482
+ || header.quant_bits != wanted.quant_bits
483
+ || header.ctx_size != wanted.ctx_size
484
+ || header.tokens != wanted.tokens
485
+ || header.reason != wanted.reason
486
+ || header.ext_flags & !crate::identity::FLAG != wanted.ext_flags
487
+ {
488
+ return Err(io::Error::new(
489
+ io::ErrorKind::InvalidData,
490
+ "KVC payload identity changed",
491
+ ));
492
+ }
493
+ // Match the selected text on this same descriptor, in bounded chunks.
494
+ file.seek(SeekFrom::Start((crate::format::FIXED_HEADER + 4) as u64))?;
495
+ let mut buffer = [0; 4096];
496
+ for bytes in expected.text.chunks(buffer.len()) {
497
+ file.read_exact(&mut buffer[..bytes.len()])?;
498
+ if &buffer[..bytes.len()] != bytes {
499
+ return Err(io::Error::new(
500
+ io::ErrorKind::InvalidData,
501
+ "KVC payload text changed",
502
+ ));
503
+ }
504
+ }
505
+ Ok(file)
506
+ }
507
+
508
+ /// Read tool metadata with the same required identity as payload candidates.
509
+ /// The identity footer is never exposed as a tool-map record.
510
+ pub fn read_trailer(&self, path: &Path, max_bytes: u64) -> io::Result<(Header, Vec<u8>)> {
511
+ let (mut header, mut trailer) =
512
+ crate::format::read_trailer(path, max_bytes).map_err(format_io_error)?;
513
+ self.check_trailer(header.ext_flags, &trailer)?;
514
+ if self.identity.is_some() {
515
+ trailer.truncate(trailer.len() - crate::identity::BYTES);
516
+ header.ext_flags &= !crate::identity::FLAG;
517
+ }
518
+ Ok((header, trailer))
519
+ }
520
+
521
+ fn check_trailer(&self, flags: u8, trailer: &[u8]) -> io::Result<()> {
522
+ if self.matches_identity(flags, crate::identity::decode(flags, trailer)) {
523
+ return Ok(());
524
+ }
525
+ Err(io::Error::new(
526
+ io::ErrorKind::InvalidData,
527
+ "KVC artifact/runtime identity mismatch",
528
+ ))
529
+ }
530
+
531
+ pub fn find_text_prefix(
532
+ &mut self,
533
+ prompt: &[u8],
534
+ model_id: u8,
535
+ quant_bits: u8,
536
+ ctx_size: u32,
537
+ ) -> Option<usize> {
538
+ self.find_prefix(prompt, model_id, quant_bits, ctx_size, false)
539
+ }
540
+
541
+ /// Reopen the selected file and repeat the C loader's header, text, and
542
+ /// filename-hash checks before a native payload restore consumes it.
543
+ pub fn text_prefix_candidate(
544
+ &mut self,
545
+ prompt: &[u8],
546
+ model_id: u8,
547
+ quant_bits: u8,
548
+ ctx_size: u32,
549
+ ) -> io::Result<Option<(PathBuf, Envelope)>> {
550
+ self.prefix_candidate(prompt, model_id, quant_bits, ctx_size, false)
551
+ }
552
+
553
+ pub fn bank_text_prefix_candidate(
554
+ &mut self,
555
+ prompt: &[u8],
556
+ model_id: u8,
557
+ quant_bits: u8,
558
+ ctx_size: u32,
559
+ ) -> io::Result<Option<(PathBuf, Envelope)>> {
560
+ self.prefix_candidate(prompt, model_id, quant_bits, ctx_size, true)
561
+ }
562
+
563
+ pub fn bank_text_prefix_candidate_identity(
564
+ &mut self,
565
+ prompt: &[u8],
566
+ model_id: u8,
567
+ quant_bits: u8,
568
+ ctx_size: u32,
569
+ identity_flags: u8,
570
+ ) -> io::Result<Option<(PathBuf, Envelope)>> {
571
+ self.prefix_candidate_identity(
572
+ prompt,
573
+ model_id,
574
+ quant_bits,
575
+ ctx_size,
576
+ true,
577
+ EXT_IMAGE_PIXELS_V2,
578
+ identity_flags,
579
+ )
580
+ }
581
+
582
+ pub fn bank_text_lcp_candidate(
583
+ &mut self,
584
+ prompt: &[u8],
585
+ model_id: u8,
586
+ quant_bits: u8,
587
+ ctx_size: u32,
588
+ min_lcp: usize,
589
+ ) -> io::Result<Option<(PathBuf, Envelope, usize)>> {
590
+ self.bank_text_lcp_candidate_with_identity(
591
+ prompt, model_id, quant_bits, ctx_size, min_lcp, 0, 0,
592
+ )
593
+ }
594
+
595
+ pub fn bank_text_lcp_candidate_identity(
596
+ &mut self,
597
+ prompt: &[u8],
598
+ model_id: u8,
599
+ quant_bits: u8,
600
+ ctx_size: u32,
601
+ min_lcp: usize,
602
+ identity_flags: u8,
603
+ ) -> io::Result<Option<(PathBuf, Envelope, usize)>> {
604
+ self.bank_text_lcp_candidate_with_identity(
605
+ prompt,
606
+ model_id,
607
+ quant_bits,
608
+ ctx_size,
609
+ min_lcp,
610
+ EXT_IMAGE_PIXELS_V2,
611
+ identity_flags,
612
+ )
613
+ }
614
+
615
+ fn bank_text_lcp_candidate_with_identity(
616
+ &mut self,
617
+ prompt: &[u8],
618
+ model_id: u8,
619
+ quant_bits: u8,
620
+ ctx_size: u32,
621
+ min_lcp: usize,
622
+ identity_mask: u8,
623
+ identity_flags: u8,
624
+ ) -> io::Result<Option<(PathBuf, Envelope, usize)>> {
625
+ let Some((index, _)) = self.find_text_lcp_with_identity(
626
+ prompt,
627
+ model_id,
628
+ quant_bits,
629
+ ctx_size,
630
+ min_lcp,
631
+ identity_mask,
632
+ identity_flags,
633
+ ) else {
634
+ return Ok(None);
635
+ };
636
+ let entry = self.entries[index].clone();
637
+ let envelope = read_envelope(&entry.path).map_err(format_io_error)?;
638
+ let header = &envelope.header;
639
+ let unchanged = header.model_id == entry.header.model_id
640
+ && header.quant_bits == entry.header.quant_bits
641
+ && header.ctx_size == entry.header.ctx_size
642
+ && header.tokens == entry.header.tokens
643
+ && header.text_bytes == entry.header.text_bytes;
644
+ let lcp = prompt
645
+ .iter()
646
+ .zip(&envelope.text)
647
+ .take_while(|(left, right)| left == right)
648
+ .count();
649
+ // Corruption past the shared prefix still leaves the scan's answer
650
+ // standing, so check the record against its own name here too.
651
+ if !unchanged || text_sha_hex(&envelope.text) != entry.sha {
652
+ return Err(io::Error::new(
653
+ io::ErrorKind::InvalidData,
654
+ "KVC record does not match its name",
655
+ ));
656
+ }
657
+
658
+ if !self.envelope_identity(&entry.path, &envelope)
659
+ || !is_automatic_exact_replay(header.reason, header.ext_flags)
660
+ || header.model_id != model_id
661
+ || (self.reject_different_quant && header.quant_bits != quant_bits)
662
+ || header.ctx_size > ctx_size
663
+ || header.ext_flags & identity_mask != identity_flags & identity_mask
664
+ || lcp < min_lcp
665
+ || 8 * (lcp as u64) < u64::from(header.text_bytes)
666
+ {
667
+ return Ok(None);
668
+ }
669
+ Ok(Some((entry.path, self.visible_envelope(envelope), lcp)))
670
+ }
671
+
672
+ fn prefix_candidate(
673
+ &mut self,
674
+ prompt: &[u8],
675
+ model_id: u8,
676
+ quant_bits: u8,
677
+ ctx_size: u32,
678
+ require_suffix: bool,
679
+ ) -> io::Result<Option<(PathBuf, Envelope)>> {
680
+ self.prefix_candidate_identity(prompt, model_id, quant_bits, ctx_size, require_suffix, 0, 0)
681
+ }
682
+
683
+ fn prefix_candidate_identity(
684
+ &mut self,
685
+ prompt: &[u8],
686
+ model_id: u8,
687
+ quant_bits: u8,
688
+ ctx_size: u32,
689
+ require_suffix: bool,
690
+ identity_mask: u8,
691
+ identity_flags: u8,
692
+ ) -> io::Result<Option<(PathBuf, Envelope)>> {
693
+ let index = if require_suffix {
694
+ self.find_prefix_with_identity(
695
+ prompt,
696
+ model_id,
697
+ quant_bits,
698
+ ctx_size,
699
+ true,
700
+ identity_mask,
701
+ identity_flags,
702
+ )
703
+ } else {
704
+ self.find_prefix_with_identity(
705
+ prompt,
706
+ model_id,
707
+ quant_bits,
708
+ ctx_size,
709
+ false,
710
+ identity_mask,
711
+ identity_flags,
712
+ )
713
+ };
714
+ let Some(index) = index else {
715
+ return Ok(None);
716
+ };
717
+ let entry = self.entries[index].clone();
718
+ let envelope = read_envelope(&entry.path).map_err(format_io_error)?;
719
+ let header = &envelope.header;
720
+ let unchanged = header.model_id == entry.header.model_id
721
+ && header.quant_bits == entry.header.quant_bits
722
+ && header.ctx_size == entry.header.ctx_size
723
+ && header.tokens == entry.header.tokens
724
+ && header.text_bytes == entry.header.text_bytes;
725
+ // The record no longer describes itself: its text does not hash to
726
+ // its own name, it is not this prompt's prefix after all, or the
727
+ // header moved under the read. The candidate was selected and then
728
+ // refused by its own contents, which is not an absence.
729
+ if !unchanged
730
+ || text_sha_hex(&envelope.text) != entry.sha
731
+ || !prompt.starts_with(&envelope.text)
732
+ {
733
+ return Err(io::Error::new(
734
+ io::ErrorKind::InvalidData,
735
+ "KVC record does not match its name",
736
+ ));
737
+ }
738
+
739
+ // Sound, but not usable in this shape: nothing left to prefill, or
740
+ // an identity this search does not take.
741
+ let bank_without_logits = is_bank_replay_v1(header.reason, header.ext_flags)
742
+ && envelope.text.len() == prompt.len();
743
+ let missing_suffix = require_suffix && envelope.text.len() >= prompt.len();
744
+ if !self.envelope_identity(&entry.path, &envelope)
745
+ || !is_automatic_exact_replay(header.reason, header.ext_flags)
746
+ || header.model_id != model_id
747
+ || header.ext_flags & identity_mask != identity_flags & identity_mask
748
+ || bank_without_logits
749
+ || missing_suffix
750
+ {
751
+ return Ok(None);
752
+ }
753
+ Ok(Some((entry.path, self.visible_envelope(envelope))))
754
+ }
755
+
756
+ pub fn touch_hit(&mut self, path: &Path) -> io::Result<()> {
757
+ let envelope = read_envelope(path).map_err(format_io_error)?;
758
+ self.check_identity(path, &envelope)?;
759
+ let mut file = self.payload_file(
760
+ path,
761
+ &self.visible_envelope(envelope.clone()),
762
+ FileAccess::ReadWrite,
763
+ )?;
764
+ let header = envelope.header;
765
+ let raw = fill_header(
766
+ header.model_id,
767
+ header.quant_bits,
768
+ header.reason,
769
+ header.ext_flags,
770
+ header.tokens,
771
+ header.hits.wrapping_add(1),
772
+ header.ctx_size,
773
+ header.created_at,
774
+ now_secs(),
775
+ header.payload_bytes,
776
+ );
777
+ file.seek(SeekFrom::Start(0))?;
778
+ file.write_all(&raw)?;
779
+ file.flush()?;
780
+ self.refresh();
781
+ Ok(())
782
+ }
783
+
784
+ pub fn discard(&mut self, path: &Path) -> io::Result<()> {
785
+ self.discard_inner(path, true)
786
+ }
787
+
788
+ pub fn discard_bank(&mut self, path: &Path) -> io::Result<()> {
789
+ self.discard_inner(path, false)
790
+ }
791
+
792
+ fn discard_inner(&mut self, path: &Path, reset_continued: bool) -> io::Result<()> {
793
+ if !self.entries.iter().any(|entry| entry.path == path) {
794
+ return Err(io::Error::new(
795
+ io::ErrorKind::InvalidInput,
796
+ "KVC discard path is not a catalog entry",
797
+ ));
798
+ }
799
+ if reset_continued {
800
+ self.continued_last_store_tokens = 0;
801
+ }
802
+ match fs::remove_file(path) {
803
+ Ok(()) => {}
804
+ Err(error) if error.kind() == io::ErrorKind::NotFound => {}
805
+ Err(error) => return Err(error),
806
+ }
807
+ self.refresh();
808
+ Ok(())
809
+ }
810
+
811
+ pub fn find_bank_text_prefix(
812
+ &mut self,
813
+ prompt: &[u8],
814
+ model_id: u8,
815
+ quant_bits: u8,
816
+ ctx_size: u32,
817
+ ) -> Option<usize> {
818
+ self.find_prefix(prompt, model_id, quant_bits, ctx_size, true)
819
+ }
820
+
821
+ fn find_prefix(
822
+ &mut self,
823
+ prompt: &[u8],
824
+ model_id: u8,
825
+ quant_bits: u8,
826
+ ctx_size: u32,
827
+ require_suffix: bool,
828
+ ) -> Option<usize> {
829
+ self.find_prefix_with_identity(prompt, model_id, quant_bits, ctx_size, require_suffix, 0, 0)
830
+ }
831
+
832
+ fn find_prefix_with_identity(
833
+ &mut self,
834
+ prompt: &[u8],
835
+ model_id: u8,
836
+ quant_bits: u8,
837
+ ctx_size: u32,
838
+ require_suffix: bool,
839
+ identity_mask: u8,
840
+ identity_flags: u8,
841
+ ) -> Option<usize> {
842
+ self.refresh();
843
+ let prompt_bytes = prompt.len();
844
+ let mut best: Option<usize> = None;
845
+ for (i, e) in self.entries.iter().enumerate() {
846
+ if !self.matches_identity(e.header.ext_flags, e.identity)
847
+ || !is_automatic_exact_replay(e.header.reason, e.header.ext_flags)
848
+ {
849
+ continue;
850
+ }
851
+ if e.header.ext_flags & identity_mask != identity_flags & identity_mask {
852
+ continue;
853
+ }
854
+ if e.header.text_bytes as usize > prompt_bytes {
855
+ continue;
856
+ }
857
+ if (require_suffix || is_bank_replay_v1(e.header.reason, e.header.ext_flags))
858
+ && e.header.text_bytes as usize == prompt_bytes
859
+ {
860
+ continue;
861
+ }
862
+ if (e.header.tokens as i32) < self.opt.min_tokens {
863
+ continue;
864
+ }
865
+ if e.header.model_id != model_id {
866
+ continue;
867
+ }
868
+ if ctx_size < e.header.ctx_size {
869
+ continue;
870
+ }
871
+ if self.reject_different_quant && e.header.quant_bits != quant_bits {
872
+ continue;
873
+ }
874
+ if let Some(b) = best {
875
+ let be = &self.entries[b];
876
+ if (e.header.text_bytes as usize) < be.header.text_bytes as usize {
877
+ continue;
878
+ }
879
+ if e.header.text_bytes == be.header.text_bytes
880
+ && e.header.tokens <= be.header.tokens
881
+ {
882
+ continue;
883
+ }
884
+ }
885
+ let sha = text_sha_hex(&prompt[..e.header.text_bytes as usize]);
886
+ if sha == e.sha {
887
+ best = Some(i);
888
+ }
889
+ }
890
+ best
891
+ }
892
+
893
+ /// What the serial search would do with this prompt: it keys on the
894
+ /// rendered text and may replay a record of exactly that length.
895
+ pub fn prefix_answer(
896
+ &mut self,
897
+ prompt: &[u8],
898
+ model_id: u8,
899
+ quant_bits: u8,
900
+ ctx_size: u32,
901
+ ) -> PrefixAnswer {
902
+ self.prefix_answer_masked(
903
+ prompt,
904
+ model_id,
905
+ quant_bits,
906
+ ctx_size,
907
+ Suffix::Optional,
908
+ 0,
909
+ 0,
910
+ )
911
+ }
912
+
913
+ /// What the bank search would do: it keys an image request by its
914
+ /// media-marked cache text and admits only records it can extend, so
915
+ /// asking about the rendered prompt would see nothing at all.
916
+ pub fn bank_prefix_answer(
917
+ &mut self,
918
+ prompt: &[u8],
919
+ model_id: u8,
920
+ quant_bits: u8,
921
+ ctx_size: u32,
922
+ identity_flags: u8,
923
+ ) -> PrefixAnswer {
924
+ self.prefix_answer_masked(
925
+ prompt,
926
+ model_id,
927
+ quant_bits,
928
+ ctx_size,
929
+ Suffix::Required,
930
+ EXT_IMAGE_PIXELS_V2,
931
+ identity_flags,
932
+ )
933
+ }
934
+
935
+ fn prefix_answer_masked(
936
+ &mut self,
937
+ prompt: &[u8],
938
+ model_id: u8,
939
+ quant_bits: u8,
940
+ ctx_size: u32,
941
+ suffix: Suffix,
942
+ identity_mask: u8,
943
+ identity_flags: u8,
944
+ ) -> PrefixAnswer {
945
+ self.refresh();
946
+ let reject_quant = self.reject_different_quant;
947
+ let min_tokens = self.opt.min_tokens;
948
+ let mut answer = PrefixAnswer::None;
949
+
950
+ for e in &self.entries {
951
+ if !is_automatic_exact_replay(e.header.reason, e.header.ext_flags)
952
+ || e.header.text_bytes as usize > prompt.len()
953
+ {
954
+ continue;
955
+ }
956
+ let exact_length = e.header.text_bytes as usize == prompt.len();
957
+ if suffix == Suffix::Required && exact_length {
958
+ // Nothing left to prefill from it in this shape.
959
+ continue;
960
+ }
961
+ if text_sha_hex(&prompt[..e.header.text_bytes as usize]) != e.sha {
962
+ continue;
963
+ }
964
+
965
+ // A bank snapshot carries no logits, so an exact replay cannot
966
+ // use one: the record is there and its layout refuses. The media
967
+ // flag is identity in the same way — a record keyed by this text
968
+ // without it is a mismatch, not one about another conversation.
969
+ let compatible = !(exact_length
970
+ && is_bank_replay_v1(e.header.reason, e.header.ext_flags))
971
+ && self.matches_identity(e.header.ext_flags, e.identity)
972
+ && e.header.model_id == model_id
973
+ && ctx_size >= e.header.ctx_size
974
+ && (!reject_quant || e.header.quant_bits == quant_bits)
975
+ && e.header.ext_flags & identity_mask == identity_flags & identity_mask;
976
+ if compatible && (e.header.tokens as i32) >= min_tokens {
977
+ return PrefixAnswer::Usable;
978
+ }
979
+ let refused = if compatible {
980
+ PrefixAnswer::Shallow
981
+ } else {
982
+ PrefixAnswer::Mismatch
983
+ };
984
+ answer = weaker_answer(answer, refused);
985
+ }
986
+
987
+ // A record whose payload was truncated away is not in the catalog,
988
+ // but its name still says it was keyed by this text. It is why the
989
+ // prompt finds nothing, and that is a mismatch.
990
+ if self.damaged_prefix(prompt, suffix) {
991
+ answer = weaker_answer(answer, PrefixAnswer::Mismatch);
992
+ }
993
+
994
+ answer
995
+ }
996
+
997
+ /// A file the catalog dropped that still shares this prompt's required
998
+ /// prefix. Its payload is gone; the text in front of it need not be.
999
+ fn damaged_lcp(&self, prompt: &[u8], min_lcp: usize) -> bool {
1000
+ self.damaged.iter().any(|d| {
1001
+ if !is_automatic_exact_replay(d.reason, d.ext_flags)
1002
+ || (d.text_bytes as usize) < min_lcp
1003
+ || u64::from(d.text_bytes) > 8 * prompt.len() as u64
1004
+ {
1005
+ return false;
1006
+ }
1007
+ let want = (d.text_bytes as usize).min(prompt.len());
1008
+ let Ok((_, text)) = read_header_text(&d.path, want) else {
1009
+ return false;
1010
+ };
1011
+ let lcp = text
1012
+ .iter()
1013
+ .zip(prompt)
1014
+ .take_while(|(stored, asked)| stored == asked)
1015
+ .count();
1016
+ lcp >= min_lcp && 8 * (lcp as u64) >= u64::from(d.text_bytes)
1017
+ })
1018
+ }
1019
+
1020
+ /// A file the catalog dropped, keyed by a prefix of this prompt.
1021
+ fn damaged_prefix(&self, prompt: &[u8], suffix: Suffix) -> bool {
1022
+ self.damaged.iter().any(|d| {
1023
+ let text_bytes = d.text_bytes as usize;
1024
+ if !is_automatic_exact_replay(d.reason, d.ext_flags)
1025
+ || text_bytes > prompt.len()
1026
+ || (suffix == Suffix::Required && text_bytes == prompt.len())
1027
+ {
1028
+ return false;
1029
+ }
1030
+ text_sha_hex(&prompt[..text_bytes]) == d.sha
1031
+ })
1032
+ }
1033
+
1034
+ /// The LCP form: an edited prompt diverges from the record it shares a
1035
+ /// prefix with, so the exact question cannot see it at all. `min_lcp`
1036
+ /// is the partial minimum the bank search admits from.
1037
+ pub fn bank_lcp_answer(
1038
+ &mut self,
1039
+ prompt: &[u8],
1040
+ model_id: u8,
1041
+ quant_bits: u8,
1042
+ ctx_size: u32,
1043
+ min_lcp: usize,
1044
+ identity_flags: u8,
1045
+ ) -> PrefixAnswer {
1046
+ if min_lcp == 0 || prompt.len() < min_lcp {
1047
+ return PrefixAnswer::None;
1048
+ }
1049
+ self.refresh();
1050
+ let reject_quant = self.reject_different_quant;
1051
+ let min_tokens = self.opt.min_tokens;
1052
+ let mut answer = PrefixAnswer::None;
1053
+
1054
+ for e in &self.entries {
1055
+ if !is_automatic_exact_replay(e.header.reason, e.header.ext_flags)
1056
+ || (e.header.text_bytes as usize) < min_lcp
1057
+ || u64::from(e.header.text_bytes) > 8 * prompt.len() as u64
1058
+ {
1059
+ continue;
1060
+ }
1061
+ let want = (e.header.text_bytes as usize).min(prompt.len());
1062
+ let Ok((metadata, text)) = read_text_prefix(&e.path, want) else {
1063
+ continue;
1064
+ };
1065
+ if metadata.header.text_bytes != e.header.text_bytes {
1066
+ continue;
1067
+ }
1068
+ let lcp = text
1069
+ .iter()
1070
+ .zip(prompt)
1071
+ .take_while(|(stored, asked)| stored == asked)
1072
+ .count();
1073
+ if lcp < min_lcp || 8 * (lcp as u64) < u64::from(e.header.text_bytes) {
1074
+ continue;
1075
+ }
1076
+
1077
+ let compatible = self.matches_identity(e.header.ext_flags, e.identity)
1078
+ && e.header.model_id == model_id
1079
+ && ctx_size >= e.header.ctx_size
1080
+ && (!reject_quant || e.header.quant_bits == quant_bits)
1081
+ && e.header.ext_flags & EXT_IMAGE_PIXELS_V2 == identity_flags & EXT_IMAGE_PIXELS_V2;
1082
+ if compatible && (e.header.tokens as i32) >= min_tokens {
1083
+ return PrefixAnswer::Usable;
1084
+ }
1085
+ let refused = if compatible {
1086
+ PrefixAnswer::Shallow
1087
+ } else {
1088
+ PrefixAnswer::Mismatch
1089
+ };
1090
+ answer = weaker_answer(answer, refused);
1091
+ }
1092
+
1093
+ // An edited prompt diverges from the record, so the damaged list has
1094
+ // to be read through the same shared prefix, not by name. A record
1095
+ // no search would replay from is not a refusal either.
1096
+ if self.damaged_lcp(prompt, min_lcp) {
1097
+ answer = weaker_answer(answer, PrefixAnswer::Mismatch);
1098
+ }
1099
+
1100
+ answer
1101
+ }
1102
+
1103
+ pub fn find_text_lcp(
1104
+ &mut self,
1105
+ prompt: &[u8],
1106
+ model_id: u8,
1107
+ quant_bits: u8,
1108
+ ctx_size: u32,
1109
+ min_lcp: usize,
1110
+ ) -> Option<(usize, usize)> {
1111
+ self.find_text_lcp_with_identity(prompt, model_id, quant_bits, ctx_size, min_lcp, 0, 0)
1112
+ }
1113
+
1114
+ fn find_text_lcp_with_identity(
1115
+ &mut self,
1116
+ prompt: &[u8],
1117
+ model_id: u8,
1118
+ quant_bits: u8,
1119
+ ctx_size: u32,
1120
+ min_lcp: usize,
1121
+ identity_mask: u8,
1122
+ identity_flags: u8,
1123
+ ) -> Option<(usize, usize)> {
1124
+ if min_lcp == 0 || prompt.len() < min_lcp {
1125
+ return None;
1126
+ }
1127
+ self.refresh();
1128
+ let mut best: Option<(usize, usize, u64)> = None;
1129
+ for (i, e) in self.entries.iter().enumerate() {
1130
+ if !self.matches_identity(e.header.ext_flags, e.identity)
1131
+ || !is_automatic_exact_replay(e.header.reason, e.header.ext_flags)
1132
+ {
1133
+ continue;
1134
+ }
1135
+ if e.header.ext_flags & identity_mask != identity_flags & identity_mask {
1136
+ continue;
1137
+ }
1138
+ if (e.header.tokens as i32) < self.opt.min_tokens {
1139
+ continue;
1140
+ }
1141
+ if e.header.model_id != model_id || ctx_size < e.header.ctx_size {
1142
+ continue;
1143
+ }
1144
+ if self.reject_different_quant && e.header.quant_bits != quant_bits {
1145
+ continue;
1146
+ }
1147
+ if (e.header.text_bytes as usize) < min_lcp {
1148
+ continue;
1149
+ }
1150
+ if u64::from(e.header.text_bytes) > 8 * prompt.len() as u64 {
1151
+ continue;
1152
+ }
1153
+ let want = (e.header.text_bytes as usize).min(prompt.len());
1154
+ let Ok((metadata, text)) = read_text_prefix(&e.path, want) else {
1155
+ continue;
1156
+ };
1157
+ if metadata.header.text_bytes != e.header.text_bytes {
1158
+ continue;
1159
+ }
1160
+ let mut lcp = 0;
1161
+ while lcp < text.len() && text[lcp] == prompt[lcp] {
1162
+ lcp += 1;
1163
+ }
1164
+ if lcp < min_lcp {
1165
+ continue;
1166
+ }
1167
+ if 8 * (lcp as u64) < u64::from(e.header.text_bytes) {
1168
+ continue;
1169
+ }
1170
+ if let Some((_, best_lcp, best_text)) = best {
1171
+ if lcp < best_lcp {
1172
+ continue;
1173
+ }
1174
+ if lcp == best_lcp && u64::from(e.header.text_bytes) >= best_text {
1175
+ continue;
1176
+ }
1177
+ }
1178
+ best = Some((i, lcp, u64::from(e.header.text_bytes)));
1179
+ }
1180
+ best.map(|(i, lcp, _)| (i, lcp))
1181
+ }
1182
+
1183
+ pub fn evict(&mut self, extra_bytes: u64, incoming: Option<&EvictionContext<'_>>) {
1184
+ self.evict_excluding(extra_bytes, incoming, None);
1185
+ }
1186
+
1187
+ fn evict_excluding(
1188
+ &mut self,
1189
+ extra_bytes: u64,
1190
+ incoming: Option<&EvictionContext<'_>>,
1191
+ protected: Option<&Path>,
1192
+ ) {
1193
+ if self.budget_bytes == 0 || extra_bytes > self.budget_bytes {
1194
+ return;
1195
+ }
1196
+ self.refresh();
1197
+ let now = now_secs();
1198
+ let target = self.budget_bytes - extra_bytes;
1199
+ loop {
1200
+ let total: u64 = self
1201
+ .entries
1202
+ .iter()
1203
+ .filter(|e| protected.is_none_or(|path| path != e.path))
1204
+ .map(|e| e.file_size)
1205
+ .sum();
1206
+ if total <= target || self.entries.is_empty() {
1207
+ break;
1208
+ }
1209
+ let mut victim: Option<usize> = None;
1210
+ let mut victim_score = 0.0;
1211
+ for i in 0..self.entries.len() {
1212
+ if protected.is_some_and(|path| path == self.entries[i].path) {
1213
+ continue;
1214
+ }
1215
+ let score = self.score_at(i, now, incoming);
1216
+ if victim.is_none()
1217
+ || score < victim_score
1218
+ || (score == victim_score
1219
+ && self.entries[i].header.last_used
1220
+ < self.entries[victim.unwrap()].header.last_used)
1221
+ {
1222
+ victim = Some(i);
1223
+ victim_score = score;
1224
+ }
1225
+ }
1226
+ let Some(victim) = victim else { break };
1227
+ let path = self.entries[victim].path.clone();
1228
+ let unlinked = fs::remove_file(path).is_ok();
1229
+ self.entries.remove(victim);
1230
+ if !unlinked {
1231
+ break;
1232
+ }
1233
+ }
1234
+ }
1235
+
1236
+ fn score_at(&self, i: usize, now: u64, incoming: Option<&EvictionContext<'_>>) -> f64 {
1237
+ let e = &self.entries[i];
1238
+ eviction_score(
1239
+ &ScoreEntry {
1240
+ sha: &e.sha,
1241
+ quant_bits: e.header.quant_bits,
1242
+ model_id: e.header.model_id,
1243
+ reason: e.header.reason,
1244
+ tokens: e.header.tokens,
1245
+ hits: e.header.hits,
1246
+ ctx_size: e.header.ctx_size,
1247
+ created_at: e.header.created_at,
1248
+ last_used: e.header.last_used,
1249
+ text_bytes: u64::from(e.header.text_bytes),
1250
+ file_size: e.file_size,
1251
+ },
1252
+ now,
1253
+ incoming,
1254
+ )
1255
+ }
1256
+ }
1257
+
1258
+ fn rewrite_compatible_trailer(
1259
+ file: &mut fs::File,
1260
+ existing: &Envelope,
1261
+ incoming_ext_flags: u8,
1262
+ trailer: &[u8],
1263
+ ) -> io::Result<()> {
1264
+ // The C fast path is O(trailer); copying a multi-GiB payload here would
1265
+ // turn a compatible save into a full checkpoint rewrite.
1266
+ let payload_end = existing
1267
+ .payload_offset
1268
+ .checked_add(existing.header.payload_bytes)
1269
+ .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "KVC payload end overflows"))?;
1270
+ file.seek(SeekFrom::Start(payload_end))?;
1271
+ file.set_len(payload_end)?;
1272
+ file.write_all(trailer)?;
1273
+ file.flush()?;
1274
+
1275
+ let mut header = existing.header.clone();
1276
+ header.ext_flags |= incoming_ext_flags & EXT_TOOL_MAP;
1277
+ header.last_used = now_secs();
1278
+ let raw = fill_header(
1279
+ header.model_id,
1280
+ header.quant_bits,
1281
+ header.reason,
1282
+ header.ext_flags,
1283
+ header.tokens,
1284
+ header.hits,
1285
+ header.ctx_size,
1286
+ header.created_at,
1287
+ header.last_used,
1288
+ header.payload_bytes,
1289
+ );
1290
+ file.seek(SeekFrom::Start(0))?;
1291
+ file.write_all(&raw)?;
1292
+ file.flush()
1293
+ }
1294
+
1295
+ /// Fold in a record the search would not take. A mismatch outranks one
1296
+ /// that is only too shallow, the way the lanes report the two, and both
1297
+ /// outrank having seen nothing.
1298
+ fn weaker_answer(seen: PrefixAnswer, refused: PrefixAnswer) -> PrefixAnswer {
1299
+ if refused == PrefixAnswer::Mismatch || seen == PrefixAnswer::None {
1300
+ return refused;
1301
+ }
1302
+ seen
1303
+ }
1304
+
1305
+ fn format_io_error(error: FormatError) -> io::Error {
1306
+ match error {
1307
+ FormatError::Io(error) => error,
1308
+ FormatError::Truncated => {
1309
+ io::Error::new(io::ErrorKind::UnexpectedEof, "truncated KVC payload")
1310
+ }
1311
+ other => io::Error::new(io::ErrorKind::InvalidData, other),
1312
+ }
1313
+ }
1314
+
1315
+ fn now_secs() -> u64 {
1316
+ SystemTime::now()
1317
+ .duration_since(UNIX_EPOCH)
1318
+ .map(|d| d.as_secs())
1319
+ .unwrap_or(0)
1320
+ }
1321
+
1322
+ #[cfg(test)]
1323
+ mod tests {
1324
+ use super::*;
1325
+ use crate::format::{
1326
+ fill_header, le_put32, read_metadata, read_text_prefix, Header, Reason, FIXED_HEADER,
1327
+ };
1328
+ #[cfg(unix)]
1329
+ use std::os::unix::fs::PermissionsExt as _;
1330
+
1331
+ fn rec(text: &[u8], tokens: u32) -> Record {
1332
+ Record {
1333
+ header: Header {
1334
+ quant_bits: 2,
1335
+ reason: Reason::Cold,
1336
+ ext_flags: 0,
1337
+ model_id: 0,
1338
+ tokens,
1339
+ hits: 0,
1340
+ ctx_size: 2048,
1341
+ created_at: 1,
1342
+ last_used: 1,
1343
+ payload_bytes: 1,
1344
+ text_bytes: text.len() as u32,
1345
+ },
1346
+ text: text.to_vec(),
1347
+ payload: vec![0xAA],
1348
+ trailer: Vec::new(),
1349
+ }
1350
+ }
1351
+
1352
+ #[test]
1353
+ fn identity_required_for_reuse() {
1354
+ let dir = std::env::temp_dir().join(format!("ds4-kv-identity-{}", std::process::id()));
1355
+ let _ = fs::remove_dir_all(&dir);
1356
+ let options = Options {
1357
+ min_tokens: 1,
1358
+ ..Options::default()
1359
+ };
1360
+ let mut store = Store::open(&dir, 16, false, options).unwrap();
1361
+ let row = rec(b"same prompt", 8);
1362
+ let path = store.write(row.clone()).unwrap();
1363
+ store.bind_identity([1; 32]);
1364
+ assert!(store
1365
+ .text_prefix_candidate(b"same prompt plus", 0, 2, 2048)
1366
+ .unwrap()
1367
+ .is_none());
1368
+ assert_eq!(
1369
+ store.prefix_answer(b"same prompt plus", 0, 2, 2048),
1370
+ PrefixAnswer::Mismatch
1371
+ );
1372
+ assert!(store.read(&path).is_err());
1373
+ assert!(store
1374
+ .reuse_compatible(row.header.clone(), &row.text, &[])
1375
+ .unwrap()
1376
+ .is_none());
1377
+ store.write(row.clone()).unwrap();
1378
+ assert!(store
1379
+ .text_prefix_candidate(b"same prompt plus", 0, 2, 2048)
1380
+ .unwrap()
1381
+ .is_some());
1382
+ assert!(store
1383
+ .bank_text_lcp_candidate(b"same prompt edit", 0, 2, 2048, 4)
1384
+ .unwrap()
1385
+ .is_some());
1386
+ assert_eq!(store.read(&path).unwrap(), row);
1387
+ let mut other = Store::open(&dir, 16, false, options).unwrap();
1388
+ other.bind_identity([2; 32]);
1389
+ assert!(other
1390
+ .text_prefix_candidate(b"same prompt plus", 0, 2, 2048)
1391
+ .unwrap()
1392
+ .is_none());
1393
+ assert!(other
1394
+ .bank_text_lcp_candidate(b"same prompt edit", 0, 2, 2048, 4)
1395
+ .unwrap()
1396
+ .is_none());
1397
+ assert!(other
1398
+ .reuse_compatible(row.header.clone(), &row.text, &[])
1399
+ .unwrap()
1400
+ .is_none());
1401
+ assert!(other.read(&path).is_err());
1402
+ assert!(other.touch_hit(&path).is_err());
1403
+ let (_, envelope) = store
1404
+ .text_prefix_candidate(b"same prompt plus", 0, 2, 2048)
1405
+ .unwrap()
1406
+ .unwrap();
1407
+ let held = store.open_payload(&path, &envelope).unwrap();
1408
+ let mut replacement = row.clone();
1409
+ replacement.payload = vec![0xBB];
1410
+ other.write(replacement).unwrap();
1411
+ assert_eq!(read_path(held.path()).unwrap().payload, row.payload);
1412
+ assert!(store.open_payload(&path, &envelope).is_err());
1413
+ assert!(store.read(&path).is_err());
1414
+ assert_eq!(other.entries().len(), 1);
1415
+ fs::remove_dir_all(dir).unwrap();
1416
+ }
1417
+
1418
+ #[test]
1419
+ fn identity_keeps_tool_trailer() {
1420
+ let dir = std::env::temp_dir().join(format!("ds4-kv-id-tools-{}", std::process::id()));
1421
+ let _ = fs::remove_dir_all(&dir);
1422
+ let mut store = Store::open(
1423
+ &dir,
1424
+ 16,
1425
+ false,
1426
+ Options {
1427
+ min_tokens: 1,
1428
+ ..Options::default()
1429
+ },
1430
+ )
1431
+ .unwrap();
1432
+ store.bind_identity([3; 32]);
1433
+ let mut row = rec(b"tool prompt", 8);
1434
+ let tools = vec![crate::ExtensionRecord {
1435
+ id: b"tool-1".to_vec(),
1436
+ body: b"call()".to_vec(),
1437
+ }];
1438
+ row.header.ext_flags = EXT_TOOL_MAP;
1439
+ row.trailer = crate::encode_ktm(&tools);
1440
+ let path = store.write(row.clone()).unwrap();
1441
+ assert_eq!(store.read(&path).unwrap(), row);
1442
+ let raw = read_path(&path).unwrap();
1443
+ assert_eq!(crate::decode_ktm(&raw.trailer), tools);
1444
+ assert!(store
1445
+ .reuse_compatible(row.header.clone(), &row.text, &row.trailer)
1446
+ .unwrap()
1447
+ .is_some());
1448
+ let loaded = store.read(&path).unwrap();
1449
+ assert_eq!(loaded.trailer, row.trailer);
1450
+ assert_eq!(loaded.payload, row.payload);
1451
+ assert_eq!(store.read_trailer(&path, 4096).unwrap().1, row.trailer);
1452
+ store.bind_identity([4; 32]);
1453
+ assert!(store.read_trailer(&path, 4096).is_err());
1454
+ store.bind_identity([3; 32]);
1455
+ let bytes = fs::read(&path).unwrap();
1456
+ fs::write(&path, &bytes[..bytes.len() - 1]).unwrap();
1457
+ assert!(store
1458
+ .text_prefix_candidate(b"tool prompt suffix", 0, 2, 2048)
1459
+ .unwrap()
1460
+ .is_none());
1461
+ assert!(store.read(&path).is_err());
1462
+ fs::remove_dir_all(dir).unwrap();
1463
+ }
1464
+
1465
+ #[test]
1466
+ fn identities_share_one_budget() {
1467
+ let dir = std::env::temp_dir().join(format!("ds4-kv-id-budget-{}", std::process::id()));
1468
+ let _ = fs::remove_dir_all(&dir);
1469
+ let mut store = Store::open(
1470
+ &dir,
1471
+ 16,
1472
+ false,
1473
+ Options {
1474
+ min_tokens: 1,
1475
+ ..Options::default()
1476
+ },
1477
+ )
1478
+ .unwrap();
1479
+ store.bind_identity([1; 32]);
1480
+ let first = store.write(rec(b"old artifact", 8)).unwrap();
1481
+ let bytes = fs::metadata(&first).unwrap().len();
1482
+ store.budget_bytes = bytes + 16; // Includes the policy's one-percent reserve.
1483
+ store.bind_identity([2; 32]);
1484
+ let second = store.write(rec(b"new artifact", 8)).unwrap();
1485
+ assert!(!first.exists() && second.exists());
1486
+ assert_eq!(store.entries().len(), 1);
1487
+ assert_eq!(fs::metadata(second).unwrap().len(), bytes);
1488
+ fs::remove_dir_all(dir).unwrap();
1489
+ }
1490
+
1491
+ #[cfg(unix)]
1492
+ #[test]
1493
+ fn new_nested_store_directories_are_private() {
1494
+ let base = std::env::temp_dir().join(format!("ds4-kv-private-dir-{}", std::process::id()));
1495
+ let dir = base.join("nested/store");
1496
+ let _ = fs::remove_dir_all(&base);
1497
+
1498
+ let _store = Store::open(&dir, 16, false, Options::default()).unwrap();
1499
+ for path in [&base, &base.join("nested"), &dir] {
1500
+ assert_eq!(fs::metadata(path).unwrap().permissions().mode() & 0o077, 0);
1501
+ }
1502
+
1503
+ let _ = fs::remove_dir_all(&base);
1504
+ }
1505
+
1506
+ #[cfg(unix)]
1507
+ #[test]
1508
+ fn existing_store_directory_mode_is_unchanged() {
1509
+ let dir = std::env::temp_dir().join(format!("ds4-kv-existing-dir-{}", std::process::id()));
1510
+ let _ = fs::remove_dir_all(&dir);
1511
+ fs::create_dir_all(&dir).unwrap();
1512
+ fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).unwrap();
1513
+
1514
+ let _store = Store::open(&dir, 16, false, Options::default()).unwrap();
1515
+ assert_eq!(
1516
+ fs::metadata(&dir).unwrap().permissions().mode() & 0o777,
1517
+ 0o755
1518
+ );
1519
+
1520
+ let _ = fs::remove_dir_all(&dir);
1521
+ }
1522
+
1523
+ #[cfg(unix)]
1524
+ #[test]
1525
+ fn payload_temp_is_private_and_drop_removes_it() {
1526
+ let dir = std::env::temp_dir().join(format!("ds4-kv-payload-temp-{}", std::process::id()));
1527
+ let _ = fs::remove_dir_all(&dir);
1528
+ let store = Store::open(&dir, 16, false, Options::default()).unwrap();
1529
+
1530
+ let temp = store.payload_temp().unwrap();
1531
+ let path = temp.path().to_path_buf();
1532
+ assert!(path.exists());
1533
+ assert_eq!(
1534
+ fs::metadata(&path).unwrap().permissions().mode() & 0o777,
1535
+ 0o600
1536
+ );
1537
+
1538
+ drop(temp);
1539
+ assert!(!path.exists());
1540
+ let _ = fs::remove_dir_all(&dir);
1541
+ }
1542
+
1543
+ #[test]
1544
+ fn payload_temps_are_distinct_and_drop_independently() {
1545
+ let dir = std::env::temp_dir().join(format!("ds4-kv-payload-pair-{}", std::process::id()));
1546
+ let _ = fs::remove_dir_all(&dir);
1547
+ let store = Store::open(&dir, 16, false, Options::default()).unwrap();
1548
+
1549
+ let first = store.payload_temp().unwrap();
1550
+ let second = store.payload_temp().unwrap();
1551
+ let first_path = first.path().to_path_buf();
1552
+ let second_path = second.path().to_path_buf();
1553
+ assert_ne!(first_path, second_path);
1554
+ assert!(first_path.exists());
1555
+ assert!(second_path.exists());
1556
+
1557
+ drop(first);
1558
+ assert!(!first_path.exists());
1559
+ assert!(second_path.exists());
1560
+ drop(second);
1561
+ assert!(!second_path.exists());
1562
+ let _ = fs::remove_dir_all(&dir);
1563
+ }
1564
+
1565
+ #[test]
1566
+ fn lcp_refuses_a_moved_record() {
1567
+ let dir = std::env::temp_dir().join(format!("ds4-kv-lcp-candidate-{}", std::process::id()));
1568
+ let _ = fs::remove_dir_all(&dir);
1569
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1570
+ let mut record = rec(b"shared opening/stored tail", 512);
1571
+ record.header.reason = Reason::BankShutdown;
1572
+ record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
1573
+ let path = store.write(record).unwrap();
1574
+ let edited = b"shared opening/edited tail";
1575
+
1576
+ let (got, _, lcp) = store
1577
+ .bank_text_lcp_candidate(edited, 0, 2, 8192, 8)
1578
+ .unwrap()
1579
+ .unwrap();
1580
+ assert_eq!(got, path);
1581
+ assert_eq!(lcp, 15);
1582
+
1583
+ // Corrupt the text past the shared prefix: the scan still picks the
1584
+ // record, and only the full-text check can see that it moved.
1585
+ let mut file = fs::OpenOptions::new().write(true).open(&path).unwrap();
1586
+ file.seek(SeekFrom::Start((FIXED_HEADER + 4 + 20) as u64))
1587
+ .unwrap();
1588
+ file.write_all(b"X").unwrap();
1589
+ file.flush().unwrap();
1590
+ assert_eq!(
1591
+ store
1592
+ .bank_text_lcp_candidate(edited, 0, 2, 8192, 8)
1593
+ .unwrap_err()
1594
+ .kind(),
1595
+ io::ErrorKind::InvalidData
1596
+ );
1597
+
1598
+ let _ = fs::remove_dir_all(&dir);
1599
+ }
1600
+
1601
+ #[test]
1602
+ fn a_cut_record_keeps_its_key() {
1603
+ let dir = std::env::temp_dir().join(format!("ds4-kv-truncated-{}", std::process::id()));
1604
+ let _ = fs::remove_dir_all(&dir);
1605
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1606
+ let path = store.write(rec(b"shared prefix", 512)).unwrap();
1607
+
1608
+ // Cut the payload away: the header and text stay, so the catalog
1609
+ // drops the record while its name still says what it was keyed by.
1610
+ fs::OpenOptions::new()
1611
+ .write(true)
1612
+ .open(&path)
1613
+ .unwrap()
1614
+ .set_len(FIXED_HEADER as u64 + 4 + 13)
1615
+ .unwrap();
1616
+
1617
+ assert_eq!(
1618
+ store.prefix_answer(b"shared prefix and suffix", 0, 2, 2048),
1619
+ PrefixAnswer::Mismatch
1620
+ );
1621
+ assert_eq!(
1622
+ store.prefix_answer(b"another conversation", 0, 2, 2048),
1623
+ PrefixAnswer::None
1624
+ );
1625
+
1626
+ // A record no search replays from is not a refusal: it was never a
1627
+ // candidate, damaged or not.
1628
+ let mut session = rec(b"agent opening", 512);
1629
+ session.header.reason = Reason::AgentSession;
1630
+ let session_path = store.write(session).unwrap();
1631
+ fs::OpenOptions::new()
1632
+ .write(true)
1633
+ .open(&session_path)
1634
+ .unwrap()
1635
+ .set_len(FIXED_HEADER as u64 + 4 + 13)
1636
+ .unwrap();
1637
+ assert_eq!(
1638
+ store.prefix_answer(b"agent opening and more", 0, 2, 2048),
1639
+ PrefixAnswer::None
1640
+ );
1641
+ assert_eq!(
1642
+ store.bank_lcp_answer(b"agent openinG edited", 0, 2, 2048, 8, 0),
1643
+ PrefixAnswer::None
1644
+ );
1645
+
1646
+ // An edited prompt diverges from the record, so only the shared
1647
+ // prefix can find it — and the text in front of the payload is
1648
+ // still there to be read.
1649
+ assert_eq!(
1650
+ store.bank_lcp_answer(b"shared prefiX edited", 0, 2, 2048, 8, 0),
1651
+ PrefixAnswer::Mismatch
1652
+ );
1653
+ assert_eq!(
1654
+ store.bank_lcp_answer(b"another opening turn", 0, 2, 2048, 8, 0),
1655
+ PrefixAnswer::None
1656
+ );
1657
+
1658
+ let _ = fs::remove_dir_all(&dir);
1659
+ }
1660
+
1661
+ #[test]
1662
+ fn an_exact_bank_answers_layout() {
1663
+ let dir = std::env::temp_dir().join(format!("ds4-kv-exact-bank-{}", std::process::id()));
1664
+ let _ = fs::remove_dir_all(&dir);
1665
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1666
+ let mut record = rec(b"exact conversation", 512);
1667
+ record.header.reason = Reason::BankShutdown;
1668
+ record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
1669
+ store.write(record).unwrap();
1670
+
1671
+ // A bank snapshot has no logits to replay from, so an exact serial
1672
+ // prompt is refused by the record's layout, not by its absence.
1673
+ assert_eq!(
1674
+ store.prefix_answer(b"exact conversation", 0, 2, 2048),
1675
+ PrefixAnswer::Mismatch
1676
+ );
1677
+ // With a suffix to prefill, the same record is what the search takes.
1678
+ assert_eq!(
1679
+ store.prefix_answer(b"exact conversation plus a turn", 0, 2, 2048),
1680
+ PrefixAnswer::Usable
1681
+ );
1682
+
1683
+ let _ = fs::remove_dir_all(&dir);
1684
+ }
1685
+
1686
+ #[test]
1687
+ fn a_thin_record_answers_shallow() {
1688
+ let dir = std::env::temp_dir().join(format!("ds4-kv-shallow-{}", std::process::id()));
1689
+ let _ = fs::remove_dir_all(&dir);
1690
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1691
+ // Written while the minimum was lower; every search skips it now.
1692
+ store.write(rec(b"shared opening", 8)).unwrap();
1693
+
1694
+ assert_eq!(
1695
+ store.prefix_answer(b"shared opening and more", 0, 2, 2048),
1696
+ PrefixAnswer::Shallow
1697
+ );
1698
+ assert_eq!(
1699
+ store.prefix_answer(b"another conversation", 0, 2, 2048),
1700
+ PrefixAnswer::None
1701
+ );
1702
+
1703
+ let _ = fs::remove_dir_all(&dir);
1704
+ }
1705
+
1706
+ #[test]
1707
+ fn an_image_key_owns_its_answer() {
1708
+ let dir = std::env::temp_dir().join(format!("ds4-kv-image-key-{}", std::process::id()));
1709
+ let _ = fs::remove_dir_all(&dir);
1710
+ let mut store = Store::open(&dir, 16, true, Options::default()).unwrap();
1711
+ let mut record = rec(b"chat\xffDS4IMG2 turn", 512);
1712
+ record.header.reason = Reason::BankCheckpoint;
1713
+ record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1 | EXT_IMAGE_PIXELS_V2;
1714
+ store.write(record).unwrap();
1715
+ let key = b"chat\xffDS4IMG2 turn and one more";
1716
+
1717
+ // The identity the record was stored with: the search takes it.
1718
+ assert_eq!(
1719
+ store.bank_prefix_answer(key, 0, 2, 2048, EXT_IMAGE_PIXELS_V2),
1720
+ PrefixAnswer::Usable
1721
+ );
1722
+ // A different quantization rules it out, so the miss is a mismatch.
1723
+ assert_eq!(
1724
+ store.bank_prefix_answer(key, 0, 4, 2048, EXT_IMAGE_PIXELS_V2),
1725
+ PrefixAnswer::Mismatch
1726
+ );
1727
+ // The rendered prompt is not the key it was stored under, so the
1728
+ // text-keyed search sees nothing. Asked under this key without the
1729
+ // media flag, the record is there and its layout rules it out.
1730
+ assert_eq!(
1731
+ store.prefix_answer(b"chat turn and one more", 0, 4, 2048),
1732
+ PrefixAnswer::None
1733
+ );
1734
+ assert_eq!(
1735
+ store.bank_prefix_answer(key, 0, 2, 2048, 0),
1736
+ PrefixAnswer::Mismatch
1737
+ );
1738
+
1739
+ let _ = fs::remove_dir_all(&dir);
1740
+ }
1741
+
1742
+ #[test]
1743
+ fn an_edit_mismatches_by_lcp() {
1744
+ let dir = std::env::temp_dir().join(format!("ds4-kv-lcp-identity-{}", std::process::id()));
1745
+ let _ = fs::remove_dir_all(&dir);
1746
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1747
+ store
1748
+ .write(rec(b"shared opening turn/ORIGINAL", 512))
1749
+ .unwrap();
1750
+ let edited = b"shared opening turn/EDITED CONTINUATION";
1751
+
1752
+ // The stored text diverges from the prompt, so the exact-prefix
1753
+ // question cannot see the record at all.
1754
+ assert_eq!(
1755
+ store.bank_prefix_answer(edited, 1, 2, 2048, 0),
1756
+ PrefixAnswer::None
1757
+ );
1758
+ // Through the LCP the partial search would have taken it, and only
1759
+ // the model it was written for rules it out.
1760
+ assert_eq!(
1761
+ store.bank_lcp_answer(edited, 1, 2, 2048, 8, 0),
1762
+ PrefixAnswer::Mismatch
1763
+ );
1764
+ // The identity it was stored with is not a mismatch.
1765
+ assert_eq!(
1766
+ store.bank_lcp_answer(edited, 0, 2, 2048, 8, 0),
1767
+ PrefixAnswer::Usable
1768
+ );
1769
+ // Neither is a prompt that shares nothing with it.
1770
+ assert_eq!(
1771
+ store.bank_lcp_answer(b"a different opening", 1, 2, 2048, 8, 0),
1772
+ PrefixAnswer::None
1773
+ );
1774
+
1775
+ let _ = fs::remove_dir_all(&dir);
1776
+ }
1777
+
1778
+ #[test]
1779
+ fn discard_removes_existing_candidate_and_refreshes_index() {
1780
+ let dir = std::env::temp_dir().join(format!("ds4-kv-discard-{}", std::process::id()));
1781
+ let _ = fs::remove_dir_all(&dir);
1782
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1783
+ let discarded = store.write(rec(b"discarded", 512)).unwrap();
1784
+ let retained = store.write(rec(b"retained", 1024)).unwrap();
1785
+ store.continued_last_store_tokens = 512;
1786
+ assert_eq!(store.entries().len(), 2);
1787
+
1788
+ store.discard(&discarded).unwrap();
1789
+
1790
+ assert!(!discarded.exists());
1791
+ assert!(retained.exists());
1792
+ assert_eq!(store.entries().len(), 1);
1793
+ assert_eq!(store.entries()[0].path, retained);
1794
+ assert_eq!(store.continued_last_store_tokens, 0);
1795
+ let _ = fs::remove_dir_all(&dir);
1796
+ }
1797
+
1798
+ #[test]
1799
+ fn bank_discard_preserves_the_serial_checkpoint_marker() {
1800
+ let dir = std::env::temp_dir().join(format!("ds4-kv-bank-discard-{}", std::process::id()));
1801
+ let _ = fs::remove_dir_all(&dir);
1802
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1803
+ let discarded = store.write(rec(b"bank-corrupt", 512)).unwrap();
1804
+ store.continued_last_store_tokens = 777;
1805
+
1806
+ store.discard_bank(&discarded).unwrap();
1807
+
1808
+ assert!(!discarded.exists());
1809
+ assert!(store.entries().is_empty());
1810
+ assert_eq!(store.continued_last_store_tokens, 777);
1811
+ let _ = fs::remove_dir_all(&dir);
1812
+ }
1813
+
1814
+ #[test]
1815
+ fn discard_missing_candidate_is_ok_and_refreshes_index() {
1816
+ let dir =
1817
+ std::env::temp_dir().join(format!("ds4-kv-discard-missing-{}", std::process::id()));
1818
+ let _ = fs::remove_dir_all(&dir);
1819
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1820
+ let missing = store.write(rec(b"missing", 512)).unwrap();
1821
+ store.continued_last_store_tokens = 512;
1822
+ fs::remove_file(&missing).unwrap();
1823
+ assert_eq!(store.entries().len(), 1);
1824
+
1825
+ store.discard(&missing).unwrap();
1826
+
1827
+ assert!(store.entries().is_empty());
1828
+ assert_eq!(store.continued_last_store_tokens, 0);
1829
+ let _ = fs::remove_dir_all(&dir);
1830
+ }
1831
+
1832
+ #[test]
1833
+ fn discard_rejects_paths_outside_the_catalog() {
1834
+ let dir = std::env::temp_dir().join(format!("ds4-kv-discard-scope-{}", std::process::id()));
1835
+ let _ = fs::remove_dir_all(&dir);
1836
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1837
+ let outside = dir.with_extension("outside");
1838
+ fs::write(&outside, b"keep").unwrap();
1839
+
1840
+ let error = store.discard(&outside).unwrap_err();
1841
+
1842
+ assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
1843
+ assert_eq!(fs::read(&outside).unwrap(), b"keep");
1844
+ fs::remove_file(outside).unwrap();
1845
+ let _ = fs::remove_dir_all(&dir);
1846
+ }
1847
+
1848
+ #[test]
1849
+ fn prefix_prefers_longest_matching_text() {
1850
+ let dir = std::env::temp_dir().join(format!("ds4-kv-prefix-{}", std::process::id()));
1851
+ let _ = fs::remove_dir_all(&dir);
1852
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1853
+ store.write(rec(b"hello", 512)).unwrap();
1854
+ store.write(rec(b"hello world", 512)).unwrap();
1855
+ let idx = store.find_text_prefix(b"hello world!", 0, 2, 2048).unwrap();
1856
+ assert_eq!(store.entries()[idx].header.text_bytes, 11);
1857
+ let _ = fs::remove_dir_all(&dir);
1858
+ }
1859
+
1860
+ #[test]
1861
+ fn bank_lookup_requires_suffix() {
1862
+ let dir = std::env::temp_dir().join(format!("ds4-kv-bank-{}", std::process::id()));
1863
+ let _ = fs::remove_dir_all(&dir);
1864
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1865
+ let mut r = rec(b"hello", 512);
1866
+ r.header.reason = Reason::BankEvict;
1867
+ r.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
1868
+ store.write(r).unwrap();
1869
+ assert!(store.find_bank_text_prefix(b"hello", 0, 2, 2048).is_none());
1870
+ assert!(store.find_bank_text_prefix(b"hello!", 0, 2, 2048).is_some());
1871
+ let _ = fs::remove_dir_all(&dir);
1872
+ }
1873
+
1874
+ #[test]
1875
+ fn bank_identity_lookup_separates_image_and_text_records() {
1876
+ let dir = std::env::temp_dir().join(format!("ds4-kv-bank-identity-{}", std::process::id()));
1877
+ let _ = fs::remove_dir_all(&dir);
1878
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1879
+ let mut plain = rec(b"plain identity", 512);
1880
+ plain.header.reason = Reason::BankCheckpoint;
1881
+ plain.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
1882
+ store.write(plain).unwrap();
1883
+ let mut image = rec(b"image identity", 512);
1884
+ image.header.reason = Reason::BankCheckpoint;
1885
+ image.header.ext_flags =
1886
+ crate::format::EXT_BANK_REPLAY_V1 | crate::format::EXT_IMAGE_PIXELS_V2;
1887
+ store.write(image).unwrap();
1888
+
1889
+ assert!(store
1890
+ .bank_text_prefix_candidate_identity(
1891
+ b"plain identity tail",
1892
+ 0,
1893
+ 2,
1894
+ 2048,
1895
+ EXT_IMAGE_PIXELS_V2,
1896
+ )
1897
+ .unwrap()
1898
+ .is_none());
1899
+ assert!(store
1900
+ .bank_text_prefix_candidate_identity(b"image identity tail", 0, 2, 2048, 0)
1901
+ .unwrap()
1902
+ .is_none());
1903
+ assert!(store
1904
+ .bank_text_prefix_candidate_identity(
1905
+ b"image identity tail",
1906
+ 0,
1907
+ 2,
1908
+ 2048,
1909
+ EXT_IMAGE_PIXELS_V2,
1910
+ )
1911
+ .unwrap()
1912
+ .is_some());
1913
+ assert!(
1914
+ store
1915
+ .bank_text_lcp_candidate_identity(
1916
+ b"image identitX",
1917
+ 0,
1918
+ 2,
1919
+ 2048,
1920
+ 8,
1921
+ EXT_IMAGE_PIXELS_V2,
1922
+ )
1923
+ .unwrap()
1924
+ .is_some()
1925
+ );
1926
+ let _ = fs::remove_dir_all(&dir);
1927
+ }
1928
+
1929
+ #[test]
1930
+ fn lcp_candidate_reopens_and_revalidates_the_selected_record() {
1931
+ let dir = std::env::temp_dir().join(format!("ds4-kv-lcp-{}", std::process::id()));
1932
+ let _ = fs::remove_dir_all(&dir);
1933
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
1934
+ let mut record = rec(b"shared alpha", 512);
1935
+ record.header.reason = Reason::BankCheckpoint;
1936
+ record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
1937
+ let path = store.write(record).unwrap();
1938
+
1939
+ let (candidate, envelope, lcp) = store
1940
+ .bank_text_lcp_candidate(b"shared beta", 0, 2, 2048, 6)
1941
+ .unwrap()
1942
+ .unwrap();
1943
+ assert_eq!(candidate, path);
1944
+ assert_eq!(envelope.text, b"shared alpha");
1945
+ assert_eq!(lcp, b"shared ".len());
1946
+
1947
+ let mut file = fs::OpenOptions::new()
1948
+ .read(true)
1949
+ .write(true)
1950
+ .open(&path)
1951
+ .unwrap();
1952
+ file.seek(SeekFrom::Start((FIXED_HEADER + 4) as u64))
1953
+ .unwrap();
1954
+ file.write_all(b"X").unwrap();
1955
+ file.flush().unwrap();
1956
+ assert!(store
1957
+ .bank_text_lcp_candidate(b"shared beta", 0, 2, 2048, 6)
1958
+ .unwrap()
1959
+ .is_none());
1960
+ let _ = fs::remove_dir_all(&dir);
1961
+ }
1962
+
1963
+ #[test]
1964
+ fn indexes_sparse_payload_without_reading_it() {
1965
+ const PAYLOAD_BYTES: u64 = 4 * 1024 * 1024 * 1024;
1966
+ const TRAILER_BYTES: u64 = 17;
1967
+
1968
+ let dir = std::env::temp_dir().join(format!("ds4-kv-sparse-{}", std::process::id()));
1969
+ let _ = fs::remove_dir_all(&dir);
1970
+ fs::create_dir_all(&dir).unwrap();
1971
+ let text = [b'x'; 64];
1972
+ let prompt = [b'x'; 8];
1973
+ let path = path_for_sha(&dir, &text_sha_hex(&text));
1974
+ let header = fill_header(0, 2, Reason::Cold, 0, 512, 0, 2048, 1, 1, PAYLOAD_BYTES);
1975
+ let mut text_len = [0u8; 4];
1976
+ le_put32(&mut text_len, text.len() as u32);
1977
+ let mut file = fs::File::create(&path).unwrap();
1978
+ file.write_all(&header).unwrap();
1979
+ file.write_all(&text_len).unwrap();
1980
+ file.write_all(&text).unwrap();
1981
+ let payload_offset = (FIXED_HEADER + 4 + text.len()) as u64;
1982
+ file.set_len(payload_offset + PAYLOAD_BYTES + TRAILER_BYTES)
1983
+ .unwrap();
1984
+ drop(file);
1985
+
1986
+ let metadata = read_metadata(&path).unwrap();
1987
+ assert_eq!(metadata.header.text_bytes, text.len() as u32);
1988
+ let (_, prefix) = read_text_prefix(&path, prompt.len()).unwrap();
1989
+ assert_eq!(prefix, prompt);
1990
+
1991
+ let mut store = Store::open(&dir, 8192, false, Options::default()).unwrap();
1992
+ assert_eq!(store.entries().len(), 1);
1993
+ assert_eq!(
1994
+ store.entries()[0].file_size,
1995
+ payload_offset + PAYLOAD_BYTES + TRAILER_BYTES
1996
+ );
1997
+ let (_, lcp) = store.find_text_lcp(&prompt, 0, 2, 2048, 5).unwrap();
1998
+ assert_eq!(lcp, prompt.len());
1999
+
2000
+ let _ = fs::remove_dir_all(&dir);
2001
+ }
2002
+
2003
+ #[cfg(unix)]
2004
+ #[test]
2005
+ fn eviction_stops_after_unlink_failure() {
2006
+ let dir = std::env::temp_dir().join(format!("ds4-kv-evict-unlink-{}", std::process::id()));
2007
+ let _ = fs::remove_dir_all(&dir);
2008
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2009
+ store.write(rec(b"first", 512)).unwrap();
2010
+ store.write(rec(b"second", 1024)).unwrap();
2011
+
2012
+ let original_permissions = fs::metadata(&dir).unwrap().permissions();
2013
+ fs::set_permissions(&dir, fs::Permissions::from_mode(0o555)).unwrap();
2014
+ store.budget_bytes = 1;
2015
+ store.evict(0, None);
2016
+ let disk_files = fs::read_dir(&dir).unwrap().count();
2017
+ let remaining_tokens: Vec<_> = store.entries().iter().map(|e| e.header.tokens).collect();
2018
+ fs::set_permissions(&dir, original_permissions).unwrap();
2019
+ let _ = fs::remove_dir_all(&dir);
2020
+
2021
+ assert_eq!(disk_files, 2);
2022
+ assert_eq!(remaining_tokens, vec![1024]);
2023
+ }
2024
+
2025
+ #[test]
2026
+ fn compatible_empty_trailer_reuses_existing_bytes() {
2027
+ let dir = std::env::temp_dir().join(format!("ds4-kv-reuse-empty-{}", std::process::id()));
2028
+ let _ = fs::remove_dir_all(&dir);
2029
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2030
+ let mut existing = rec(b"same prompt", 512);
2031
+ existing.header.hits = 7;
2032
+ existing.trailer = b"old trailer".to_vec();
2033
+ let path = store.write(existing).unwrap();
2034
+ let sibling = store.write(rec(b"sibling", 1024)).unwrap();
2035
+ let before = fs::read(&path).unwrap();
2036
+
2037
+ let mut incoming = rec(b"same prompt", 2048).header;
2038
+ incoming.quant_bits = 4;
2039
+ incoming.reason = Reason::Continued;
2040
+ incoming.ext_flags = crate::format::EXT_RESPONSES_VISIBLE;
2041
+ incoming.ctx_size = 4096;
2042
+ incoming.created_at = 99;
2043
+ incoming.last_used = 99;
2044
+ store.budget_bytes = 1;
2045
+
2046
+ let got = store
2047
+ .write_payload_file(incoming, b"same prompt", &dir.join("missing-payload"), b"")
2048
+ .unwrap();
2049
+ assert_eq!(got, path);
2050
+ assert_eq!(fs::read(&path).unwrap(), before);
2051
+ assert!(sibling.exists());
2052
+ let _ = fs::remove_dir_all(&dir);
2053
+ }
2054
+
2055
+ #[test]
2056
+ fn reuse_compatible_empty_trailer_returns_existing_without_staging() {
2057
+ let dir =
2058
+ std::env::temp_dir().join(format!("ds4-kv-preflight-empty-{}", std::process::id()));
2059
+ let _ = fs::remove_dir_all(&dir);
2060
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2061
+ let mut existing = rec(b"same prompt", 512);
2062
+ existing.header.hits = 7;
2063
+ existing.trailer = b"old trailer".to_vec();
2064
+ let path = store.write(existing).unwrap();
2065
+ let sibling = store.write(rec(b"sibling", 1024)).unwrap();
2066
+ let before = fs::read(&path).unwrap();
2067
+ let sibling_before = fs::read(&sibling).unwrap();
2068
+
2069
+ let mut incoming = rec(b"same prompt", 2048).header;
2070
+ incoming.quant_bits = 4;
2071
+ incoming.reason = Reason::Continued;
2072
+ incoming.ext_flags = crate::format::EXT_RESPONSES_VISIBLE;
2073
+ incoming.ctx_size = 4096;
2074
+ incoming.created_at = 99;
2075
+ incoming.last_used = 99;
2076
+ store.budget_bytes = 1;
2077
+
2078
+ let got = store
2079
+ .reuse_compatible(incoming, b"same prompt", b"")
2080
+ .unwrap();
2081
+
2082
+ assert_eq!(got, Some(path.clone()));
2083
+ assert_eq!(fs::read(&path).unwrap(), before);
2084
+ assert_eq!(fs::read(&sibling).unwrap(), sibling_before);
2085
+ assert_eq!(fs::read_dir(&dir).unwrap().count(), 2);
2086
+ let _ = fs::remove_dir_all(&dir);
2087
+ }
2088
+
2089
+ #[test]
2090
+ fn reuse_compatible_incompatible_entry_returns_none_without_mutation() {
2091
+ let dir = std::env::temp_dir().join(format!(
2092
+ "ds4-kv-preflight-incompatible-{}",
2093
+ std::process::id()
2094
+ ));
2095
+ let _ = fs::remove_dir_all(&dir);
2096
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2097
+ let path = store.write(rec(b"same prompt", 512)).unwrap();
2098
+ let sibling = store.write(rec(b"sibling", 1024)).unwrap();
2099
+ let before = fs::read(&path).unwrap();
2100
+ let sibling_before = fs::read(&sibling).unwrap();
2101
+ let mut incoming = rec(b"same prompt", 512).header;
2102
+ incoming.model_id = 1;
2103
+ store.budget_bytes = 1;
2104
+
2105
+ let got = store
2106
+ .reuse_compatible(incoming, b"same prompt", b"")
2107
+ .unwrap();
2108
+
2109
+ assert_eq!(got, None);
2110
+ assert_eq!(fs::read(&path).unwrap(), before);
2111
+ assert_eq!(fs::read(&sibling).unwrap(), sibling_before);
2112
+ assert_eq!(store.entries().len(), 2);
2113
+ let _ = fs::remove_dir_all(&dir);
2114
+ }
2115
+
2116
+ #[test]
2117
+ fn reuse_compatible_nonempty_trailer_matches_writer_fast_path() {
2118
+ let dir =
2119
+ std::env::temp_dir().join(format!("ds4-kv-preflight-trailer-{}", std::process::id()));
2120
+ let _ = fs::remove_dir_all(&dir);
2121
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2122
+ let mut existing = rec(b"same prompt", 512);
2123
+ existing.header.hits = 7;
2124
+ existing.header.created_at = 42;
2125
+ existing.header.last_used = 43;
2126
+ existing.header.ext_flags = crate::format::EXT_RESPONSES_VISIBLE;
2127
+ existing.payload = b"opaque payload".to_vec();
2128
+ existing.trailer = b"old trailer".to_vec();
2129
+ let path = store.write(existing).unwrap();
2130
+ let sibling = store.write(rec(b"sibling", 1024)).unwrap();
2131
+ let before = read_path(&path).unwrap();
2132
+
2133
+ let mut incoming = rec(b"same prompt", 2048).header;
2134
+ incoming.reason = Reason::Continued;
2135
+ incoming.ext_flags = crate::format::EXT_THINKING_VISIBLE;
2136
+ incoming.ctx_size = 4096;
2137
+ incoming.created_at = 99;
2138
+ incoming.last_used = 99;
2139
+ store.budget_bytes = 1;
2140
+ let started = now_secs();
2141
+
2142
+ let got = store
2143
+ .reuse_compatible(incoming, b"same prompt", b"new tool map")
2144
+ .unwrap();
2145
+ let after = read_path(&path).unwrap();
2146
+
2147
+ assert_eq!(got, Some(path));
2148
+ assert_eq!(after.text, before.text);
2149
+ assert_eq!(after.payload, before.payload);
2150
+ assert_eq!(after.trailer, b"new tool map");
2151
+ assert_eq!(after.header.model_id, before.header.model_id);
2152
+ assert_eq!(after.header.quant_bits, before.header.quant_bits);
2153
+ assert_eq!(after.header.reason, before.header.reason);
2154
+ assert_eq!(after.header.tokens, before.header.tokens);
2155
+ assert_eq!(after.header.hits, before.header.hits);
2156
+ assert_eq!(after.header.ctx_size, before.header.ctx_size);
2157
+ assert_eq!(after.header.created_at, before.header.created_at);
2158
+ assert_eq!(after.header.payload_bytes, before.header.payload_bytes);
2159
+ assert_eq!(after.header.text_bytes, before.header.text_bytes);
2160
+ assert_eq!(
2161
+ after.header.ext_flags,
2162
+ before.header.ext_flags | crate::format::EXT_TOOL_MAP
2163
+ );
2164
+ assert!(after.header.last_used >= started);
2165
+ assert!(sibling.exists());
2166
+ let _ = fs::remove_dir_all(&dir);
2167
+ }
2168
+
2169
+ #[test]
2170
+ fn compatible_trailer_rewrite_preserves_payload_and_header() {
2171
+ let dir = std::env::temp_dir().join(format!("ds4-kv-reuse-trailer-{}", std::process::id()));
2172
+ let _ = fs::remove_dir_all(&dir);
2173
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2174
+ let mut existing = rec(b"same prompt", 512);
2175
+ existing.header.hits = 7;
2176
+ existing.header.created_at = 42;
2177
+ existing.header.last_used = 43;
2178
+ existing.header.ext_flags = crate::format::EXT_RESPONSES_VISIBLE;
2179
+ existing.payload = b"opaque payload".to_vec();
2180
+ existing.trailer = b"old trailer".to_vec();
2181
+ let path = store.write(existing).unwrap();
2182
+ let sibling = store.write(rec(b"sibling", 1024)).unwrap();
2183
+ let before = read_path(&path).unwrap();
2184
+
2185
+ let mut incoming = rec(b"same prompt", 2048).header;
2186
+ incoming.reason = Reason::Continued;
2187
+ incoming.ext_flags = crate::format::EXT_THINKING_VISIBLE;
2188
+ incoming.ctx_size = 4096;
2189
+ incoming.created_at = 99;
2190
+ incoming.last_used = 99;
2191
+ store.budget_bytes = 1;
2192
+ let started = now_secs();
2193
+
2194
+ store
2195
+ .write_payload_file(
2196
+ incoming,
2197
+ b"same prompt",
2198
+ &dir.join("missing-payload"),
2199
+ b"new tool map",
2200
+ )
2201
+ .unwrap();
2202
+ let got = read_path(&path).unwrap();
2203
+ assert_eq!(got.text, before.text);
2204
+ assert_eq!(got.payload, before.payload);
2205
+ assert_eq!(got.trailer, b"new tool map");
2206
+ assert_eq!(got.header.model_id, before.header.model_id);
2207
+ assert_eq!(got.header.quant_bits, before.header.quant_bits);
2208
+ assert_eq!(got.header.reason, before.header.reason);
2209
+ assert_eq!(got.header.tokens, before.header.tokens);
2210
+ assert_eq!(got.header.hits, before.header.hits);
2211
+ assert_eq!(got.header.ctx_size, before.header.ctx_size);
2212
+ assert_eq!(got.header.created_at, before.header.created_at);
2213
+ assert_eq!(got.header.payload_bytes, before.header.payload_bytes);
2214
+ assert_eq!(got.header.text_bytes, before.header.text_bytes);
2215
+ assert_eq!(
2216
+ got.header.ext_flags,
2217
+ before.header.ext_flags | crate::format::EXT_TOOL_MAP
2218
+ );
2219
+ assert!(got.header.last_used >= started);
2220
+ assert!(sibling.exists());
2221
+ let _ = fs::remove_dir_all(&dir);
2222
+ }
2223
+
2224
+ #[test]
2225
+ fn payload_destination_alias_is_rejected_without_data_loss() {
2226
+ let dir = std::env::temp_dir().join(format!("ds4-kv-alias-{}", std::process::id()));
2227
+ let _ = fs::remove_dir_all(&dir);
2228
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2229
+ let path = store.write(rec(b"same prompt", 512)).unwrap();
2230
+ let before = fs::read(&path).unwrap();
2231
+ let mut incompatible = rec(b"same prompt", 512).header;
2232
+ incompatible.model_id = 1;
2233
+
2234
+ let error = store
2235
+ .write_payload_file(incompatible, b"same prompt", &path, b"")
2236
+ .unwrap_err();
2237
+ assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
2238
+ assert_eq!(fs::read(&path).unwrap(), before);
2239
+ let _ = fs::remove_dir_all(&dir);
2240
+ }
2241
+
2242
+ #[test]
2243
+ fn stream_truncation_maps_to_unexpected_eof() {
2244
+ assert_eq!(
2245
+ format_io_error(FormatError::Truncated).kind(),
2246
+ io::ErrorKind::UnexpectedEof
2247
+ );
2248
+ }
2249
+
2250
+ #[test]
2251
+ fn failed_incompatible_stage_preserves_existing_entries() {
2252
+ let dir = std::env::temp_dir().join(format!("ds4-kv-stage-fail-{}", std::process::id()));
2253
+ let _ = fs::remove_dir_all(&dir);
2254
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2255
+ let target = store.write(rec(b"same prompt", 8192)).unwrap();
2256
+ let sibling = store.write(rec(b"sibling", 512)).unwrap();
2257
+ let target_before = fs::read(&target).unwrap();
2258
+ let sibling_before = fs::read(&sibling).unwrap();
2259
+ let mut incoming = rec(b"same prompt", 1024).header;
2260
+ incoming.model_id = 1;
2261
+
2262
+ let error = store
2263
+ .write_payload_file(incoming, b"same prompt", &dir.join("missing-payload"), b"")
2264
+ .unwrap_err();
2265
+ assert_eq!(error.kind(), io::ErrorKind::NotFound);
2266
+ assert_eq!(fs::read(&target).unwrap(), target_before);
2267
+ assert_eq!(fs::read(&sibling).unwrap(), sibling_before);
2268
+ let _ = fs::remove_dir_all(&dir);
2269
+ }
2270
+
2271
+ #[test]
2272
+ fn incompatible_replacement_credits_target_and_keeps_sibling() {
2273
+ let dir = std::env::temp_dir().join(format!("ds4-kv-replace-{}", std::process::id()));
2274
+ let _ = fs::remove_dir_all(&dir);
2275
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2276
+ let text = b"same prompt";
2277
+ let target = store.write(rec(text, 8192)).unwrap();
2278
+ let sibling = store.write(rec(b"sibling", 512)).unwrap();
2279
+ let payload = b"new payload";
2280
+ let payload_path = dir.join("incoming.payload");
2281
+ fs::write(&payload_path, payload).unwrap();
2282
+ let incoming_bytes = file_size_bytes(text.len() as u64, payload.len() as u64, 0).unwrap();
2283
+ store.budget_bytes = fs::metadata(&sibling).unwrap().len() + incoming_bytes;
2284
+ let mut incoming = rec(text, 1024).header;
2285
+ incoming.model_id = 1;
2286
+
2287
+ let got = store
2288
+ .write_payload_file(incoming, text, &payload_path, b"")
2289
+ .unwrap();
2290
+ let replaced = read_path(&target).unwrap();
2291
+ assert_eq!(got, target);
2292
+ assert_eq!(replaced.header.model_id, 1);
2293
+ assert_eq!(replaced.payload, payload);
2294
+ assert!(sibling.exists());
2295
+ let _ = fs::remove_dir_all(&dir);
2296
+ }
2297
+
2298
+ #[test]
2299
+ fn candidate_reopens_and_validates_text() {
2300
+ let dir = std::env::temp_dir().join(format!("ds4-kv-candidate-{}", std::process::id()));
2301
+ let _ = fs::remove_dir_all(&dir);
2302
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2303
+ let path = store.write(rec(b"shared prefix", 512)).unwrap();
2304
+
2305
+ let (got_path, envelope) = store
2306
+ .text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192)
2307
+ .unwrap()
2308
+ .unwrap();
2309
+ assert_eq!(got_path, path);
2310
+ assert_eq!(envelope.text, b"shared prefix");
2311
+
2312
+ let mut file = fs::OpenOptions::new().write(true).open(&path).unwrap();
2313
+ file.seek(SeekFrom::Start((crate::format::FIXED_HEADER + 4) as u64))
2314
+ .unwrap();
2315
+ file.write_all(b"X").unwrap();
2316
+ file.flush().unwrap();
2317
+ // The text no longer hashes to its own name: the record is refused
2318
+ // by its contents, which is not the same as not being there.
2319
+ assert_eq!(
2320
+ store
2321
+ .text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192)
2322
+ .unwrap_err()
2323
+ .kind(),
2324
+ io::ErrorKind::InvalidData
2325
+ );
2326
+ let _ = fs::remove_dir_all(&dir);
2327
+ }
2328
+
2329
+ #[test]
2330
+ fn bank_candidate_requires_a_valid_strict_suffix() {
2331
+ let dir =
2332
+ std::env::temp_dir().join(format!("ds4-kv-bank-candidate-{}", std::process::id()));
2333
+ let _ = fs::remove_dir_all(&dir);
2334
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2335
+ let mut record = rec(b"shared prefix", 512);
2336
+ record.header.reason = Reason::BankShutdown;
2337
+ record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
2338
+ let path = store.write(record).unwrap();
2339
+ let mut serial = rec(b"serial exact", 512);
2340
+ serial.header.reason = Reason::Shutdown;
2341
+ let serial_path = store.write(serial).unwrap();
2342
+
2343
+ assert!(store
2344
+ .bank_text_prefix_candidate(b"shared prefix", 0, 2, 8192)
2345
+ .unwrap()
2346
+ .is_none());
2347
+ assert!(store
2348
+ .bank_text_prefix_candidate(b"serial exact", 0, 2, 8192)
2349
+ .unwrap()
2350
+ .is_none());
2351
+ assert_eq!(
2352
+ store
2353
+ .bank_text_prefix_candidate(b"serial exact suffix", 0, 2, 8192)
2354
+ .unwrap()
2355
+ .unwrap()
2356
+ .0,
2357
+ serial_path
2358
+ );
2359
+ let (got_path, envelope) = store
2360
+ .bank_text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192)
2361
+ .unwrap()
2362
+ .unwrap();
2363
+ assert_eq!(got_path, path);
2364
+ assert_eq!(envelope.text, b"shared prefix");
2365
+
2366
+ let mut file = fs::OpenOptions::new().write(true).open(&path).unwrap();
2367
+ file.seek(SeekFrom::Start((crate::format::FIXED_HEADER + 4) as u64))
2368
+ .unwrap();
2369
+ file.write_all(b"X").unwrap();
2370
+ file.flush().unwrap();
2371
+ assert_eq!(
2372
+ store
2373
+ .bank_text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192)
2374
+ .unwrap_err()
2375
+ .kind(),
2376
+ io::ErrorKind::InvalidData
2377
+ );
2378
+ let _ = fs::remove_dir_all(&dir);
2379
+ }
2380
+
2381
+ #[test]
2382
+ fn touch_hit_preserves_payload_and_trailer() {
2383
+ let dir = std::env::temp_dir().join(format!("ds4-kv-touch-{}", std::process::id()));
2384
+ let _ = fs::remove_dir_all(&dir);
2385
+ let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
2386
+ let mut record = rec(b"shared prefix", 512);
2387
+ record.header.hits = 7;
2388
+ record.header.created_at = 42;
2389
+ record.header.last_used = 43;
2390
+ record.payload = b"opaque payload".to_vec();
2391
+ record.trailer = b"trailer".to_vec();
2392
+ let path = store.write(record).unwrap();
2393
+ let before = read_path(&path).unwrap();
2394
+ let started = now_secs();
2395
+
2396
+ store.touch_hit(&path).unwrap();
2397
+ let after = read_path(&path).unwrap();
2398
+ assert_eq!(after.header.hits, before.header.hits + 1);
2399
+ assert_eq!(after.header.created_at, before.header.created_at);
2400
+ assert!(after.header.last_used >= started);
2401
+ assert_eq!(after.text, before.text);
2402
+ assert_eq!(after.payload, before.payload);
2403
+ assert_eq!(after.trailer, before.trailer);
2404
+ let _ = fs::remove_dir_all(&dir);
2405
+ }
2406
+ }
crates/ds4-kv/tests/bank_thinking.rs ADDED
@@ -0,0 +1,250 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Bank thinking bit + one KTM extension record.
2
+ //!
3
+ //! Given / When / Then against the C KVC envelope: thinking is
4
+ //! `EXT_THINKING_VISIBLE`, the only post-DSV4 magic trailer is `KTM\x01`.
5
+
6
+ use ds4_kv::{
7
+ decode_file, encode_file, read_path, write_path, BankThinkingExtensions, ExtensionRecord,
8
+ Header, Reason, Record, EXT_BANK_REPLAY_V1, EXT_THINKING_VISIBLE, EXT_TOOL_MAP,
9
+ };
10
+ use std::fs;
11
+ use std::path::{Path, PathBuf};
12
+ use std::process::Command;
13
+
14
+ fn sample_bank() -> Record {
15
+ Record {
16
+ header: Header {
17
+ quant_bits: 2,
18
+ reason: Reason::BankCheckpoint,
19
+ ext_flags: EXT_BANK_REPLAY_V1,
20
+ model_id: 3,
21
+ tokens: 64,
22
+ hits: 1,
23
+ ctx_size: 4096,
24
+ created_at: 1_700_000_000,
25
+ last_used: 1_700_000_050,
26
+ payload_bytes: 6,
27
+ text_bytes: 11,
28
+ },
29
+ text: b"hello world".to_vec(),
30
+ payload: b"PAYLOD".to_vec(),
31
+ trailer: Vec::new(),
32
+ }
33
+ }
34
+
35
+ fn tmp(name: &str) -> PathBuf {
36
+ let dir =
37
+ std::env::temp_dir().join(format!("ds4-kv-bank-think-{}-{}", std::process::id(), name));
38
+ let _ = fs::remove_dir_all(&dir);
39
+ fs::create_dir_all(&dir).unwrap();
40
+ dir.join("bank.kv")
41
+ }
42
+
43
+ fn oracle() -> PathBuf {
44
+ if let Ok(p) = std::env::var("DS4_KV_C_ORACLE") {
45
+ return PathBuf::from(p);
46
+ }
47
+ PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/kv_c_oracle")
48
+ }
49
+
50
+ fn hex(bytes: &[u8]) -> String {
51
+ bytes.iter().map(|b| format!("{b:02x}")).collect()
52
+ }
53
+
54
+ fn unhex(s: &str) -> Vec<u8> {
55
+ (0..s.len())
56
+ .step_by(2)
57
+ .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
58
+ .collect()
59
+ }
60
+
61
+ fn c_write(path: &Path, rec: &Record) {
62
+ let status = Command::new(oracle())
63
+ .args([
64
+ "write",
65
+ "--path",
66
+ path.to_str().unwrap(),
67
+ "--text-hex",
68
+ &hex(&rec.text),
69
+ "--payload-hex",
70
+ &hex(&rec.payload),
71
+ "--model-id",
72
+ &rec.header.model_id.to_string(),
73
+ "--quant",
74
+ &rec.header.quant_bits.to_string(),
75
+ "--reason",
76
+ &(rec.header.reason as u8).to_string(),
77
+ "--ext",
78
+ &rec.header.ext_flags.to_string(),
79
+ "--tokens",
80
+ &rec.header.tokens.to_string(),
81
+ "--hits",
82
+ &rec.header.hits.to_string(),
83
+ "--ctx",
84
+ &rec.header.ctx_size.to_string(),
85
+ "--created",
86
+ &rec.header.created_at.to_string(),
87
+ "--used",
88
+ &rec.header.last_used.to_string(),
89
+ "--trailer-hex",
90
+ &hex(&rec.trailer),
91
+ ])
92
+ .status()
93
+ .unwrap();
94
+ assert!(status.success(), "C write failed");
95
+ }
96
+
97
+ fn c_read(path: &Path) -> Record {
98
+ let out = Command::new(oracle())
99
+ .args(["read", path.to_str().unwrap()])
100
+ .output()
101
+ .unwrap();
102
+ assert!(
103
+ out.status.success(),
104
+ "C read failed: {}",
105
+ String::from_utf8_lossy(&out.stderr)
106
+ );
107
+ let text = String::from_utf8(out.stdout).unwrap();
108
+ let mut rec = sample_bank();
109
+ for line in text.lines() {
110
+ let Some((k, v)) = line.split_once('=') else {
111
+ continue;
112
+ };
113
+ match k {
114
+ "model_id" => rec.header.model_id = v.parse().unwrap(),
115
+ "quant_bits" => rec.header.quant_bits = v.parse().unwrap(),
116
+ "reason" => rec.header.reason = Reason::from_u8(v.parse().unwrap()),
117
+ "ext_flags" => rec.header.ext_flags = v.parse().unwrap(),
118
+ "tokens" => rec.header.tokens = v.parse().unwrap(),
119
+ "hits" => rec.header.hits = v.parse().unwrap(),
120
+ "ctx_size" => rec.header.ctx_size = v.parse().unwrap(),
121
+ "created_at" => rec.header.created_at = v.parse().unwrap(),
122
+ "last_used" => rec.header.last_used = v.parse().unwrap(),
123
+ "payload_bytes" => rec.header.payload_bytes = v.parse().unwrap(),
124
+ "text_bytes" => rec.header.text_bytes = v.parse().unwrap(),
125
+ "text_hex" => rec.text = unhex(v),
126
+ "payload_hex" => rec.payload = unhex(v),
127
+ "trailer_hex" => rec.trailer = unhex(v),
128
+ _ => {}
129
+ }
130
+ }
131
+ rec
132
+ }
133
+
134
+ fn one_extension() -> BankThinkingExtensions {
135
+ BankThinkingExtensions {
136
+ thinking_visible: true,
137
+ records: vec![ExtensionRecord {
138
+ id: b"call_abc".to_vec(),
139
+ body: b"<dsml>function_calls>x".to_vec(),
140
+ }],
141
+ }
142
+ }
143
+
144
+ #[test]
145
+ fn bank_save_load_thinking_bit_and_one_extension_record() {
146
+ // Given: a bank checkpoint with thinking-visible text and one KTM record
147
+ let ext = one_extension();
148
+ let mut rec = sample_bank();
149
+ rec.persist_bank_extensions(&ext);
150
+
151
+ // When: the envelope is written and read back
152
+ let path = tmp("roundtrip");
153
+ write_path(&path, &rec).unwrap();
154
+ let got = read_path(&path).unwrap();
155
+ let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer);
156
+
157
+ // Then: thinking bit + BANK_REPLAY_V1 + TOOL_MAP and the one record survive
158
+ assert_eq!(
159
+ got.header.ext_flags,
160
+ EXT_BANK_REPLAY_V1 | EXT_THINKING_VISIBLE | EXT_TOOL_MAP
161
+ );
162
+ assert!(restored.thinking_visible);
163
+ assert_eq!(restored.records, ext.records);
164
+ assert_eq!(&got.trailer[..4], b"KTM\x01");
165
+ assert_eq!(encode_file(&rec)[..3], *b"KVC");
166
+
167
+ let _ = fs::remove_dir_all(path.parent().unwrap());
168
+ }
169
+
170
+ #[test]
171
+ fn missing_extension_section_is_ignored_like_c() {
172
+ // Given: a bank file with replay-v1 only (no thinking bit, no trailer)
173
+ let rec = sample_bank();
174
+ let bytes = encode_file(&rec);
175
+ let got = decode_file(&bytes).unwrap();
176
+
177
+ // When: restore runs on a missing section
178
+ let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer);
179
+
180
+ // Then: C-like ignore — thinking off, zero records, file still valid
181
+ assert_eq!(got.header.ext_flags, EXT_BANK_REPLAY_V1);
182
+ assert!(!restored.thinking_visible);
183
+ assert!(restored.records.is_empty());
184
+ }
185
+
186
+ #[test]
187
+ fn invalid_ktm_magic_is_ignored_like_c() {
188
+ // Given: a trailer that is not KTM\x01
189
+ let restored = BankThinkingExtensions::restore(EXT_TOOL_MAP, b"XXX\x01\x01\0\0\0");
190
+
191
+ // When/Then: C kv_tool_map_load_from_pos returns 0
192
+ assert!(!restored.thinking_visible);
193
+ assert!(restored.records.is_empty());
194
+ }
195
+
196
+ #[test]
197
+ fn rust_bank_thinking_extension_is_c_readable() {
198
+ let oracle = oracle();
199
+ assert!(
200
+ oracle.exists(),
201
+ "build the C oracle first: make tests/parity/kv_c_oracle (missing {})",
202
+ oracle.display()
203
+ );
204
+
205
+ let ext = one_extension();
206
+ let mut rec = sample_bank();
207
+ rec.persist_bank_extensions(&ext);
208
+ rec.header.text_bytes = rec.text.len() as u32;
209
+ rec.header.payload_bytes = rec.payload.len() as u64;
210
+
211
+ let path = tmp("rust-c");
212
+ write_path(&path, &rec).unwrap();
213
+ let got = c_read(&path);
214
+ let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer);
215
+
216
+ assert_eq!(got.header.ext_flags, rec.header.ext_flags);
217
+ assert_eq!(got.trailer, rec.trailer);
218
+ assert!(restored.thinking_visible);
219
+ assert_eq!(restored.records, ext.records);
220
+
221
+ let _ = fs::remove_dir_all(path.parent().unwrap());
222
+ }
223
+
224
+ #[test]
225
+ fn c_bank_thinking_extension_loads_in_rust() {
226
+ let oracle = oracle();
227
+ assert!(
228
+ oracle.exists(),
229
+ "build the C oracle first: make tests/parity/kv_c_oracle (missing {})",
230
+ oracle.display()
231
+ );
232
+
233
+ let ext = one_extension();
234
+ let mut rec = sample_bank();
235
+ rec.persist_bank_extensions(&ext);
236
+ rec.header.text_bytes = rec.text.len() as u32;
237
+ rec.header.payload_bytes = rec.payload.len() as u64;
238
+
239
+ let path = tmp("c-rust");
240
+ c_write(&path, &rec);
241
+ let got = decode_file(&fs::read(&path).unwrap()).unwrap();
242
+ let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer);
243
+
244
+ assert_eq!(got.header.ext_flags, rec.header.ext_flags);
245
+ assert_eq!(got.trailer, rec.trailer);
246
+ assert!(restored.thinking_visible);
247
+ assert_eq!(restored.records, ext.records);
248
+
249
+ let _ = fs::remove_dir_all(path.parent().unwrap());
250
+ }
crates/ds4-kv/tests/four_way.rs ADDED
@@ -0,0 +1,375 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! Four-way KVC matrix: C save/load × Rust save/load.
2
+
3
+ use ds4_kv::{
4
+ chat_anchor_pos, continued_store_target, decode_file, encode_file, eviction_score, read_path,
5
+ read_trailer, store_len, write_path, Header, Options, Reason, Record, ScoreEntry, Store,
6
+ EXT_TOOL_MAP,
7
+ };
8
+ use std::fs;
9
+ use std::path::{Path, PathBuf};
10
+ use std::process::Command;
11
+
12
+ fn oracle() -> PathBuf {
13
+ if let Ok(p) = std::env::var("DS4_KV_C_ORACLE") {
14
+ return PathBuf::from(p);
15
+ }
16
+ PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/kv_c_oracle")
17
+ }
18
+
19
+ fn require_oracle() -> PathBuf {
20
+ let p = oracle();
21
+ assert!(
22
+ p.exists(),
23
+ "build the C oracle first: make tests/parity/kv_c_oracle (missing {})",
24
+ p.display()
25
+ );
26
+ p
27
+ }
28
+
29
+ fn hex(bytes: &[u8]) -> String {
30
+ bytes.iter().map(|b| format!("{b:02x}")).collect()
31
+ }
32
+
33
+ fn unhex(s: &str) -> Vec<u8> {
34
+ (0..s.len())
35
+ .step_by(2)
36
+ .map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
37
+ .collect()
38
+ }
39
+
40
+ fn sample() -> Record {
41
+ Record {
42
+ header: Header {
43
+ quant_bits: 2,
44
+ reason: Reason::Cold,
45
+ ext_flags: 0,
46
+ model_id: 3,
47
+ tokens: 512,
48
+ hits: 7,
49
+ ctx_size: 4096,
50
+ created_at: 1_700_000_000,
51
+ last_used: 1_700_000_050,
52
+ payload_bytes: 6,
53
+ text_bytes: 11,
54
+ },
55
+ text: b"hello world".to_vec(),
56
+ payload: b"PAYLOD".to_vec(),
57
+ trailer: Vec::new(),
58
+ }
59
+ }
60
+
61
+ fn c_write(path: &Path, rec: &Record) {
62
+ let status = Command::new(require_oracle())
63
+ .args([
64
+ "write",
65
+ "--path",
66
+ path.to_str().unwrap(),
67
+ "--text-hex",
68
+ &hex(&rec.text),
69
+ "--payload-hex",
70
+ &hex(&rec.payload),
71
+ "--model-id",
72
+ &rec.header.model_id.to_string(),
73
+ "--quant",
74
+ &rec.header.quant_bits.to_string(),
75
+ "--reason",
76
+ &(rec.header.reason as u8).to_string(),
77
+ "--ext",
78
+ &rec.header.ext_flags.to_string(),
79
+ "--tokens",
80
+ &rec.header.tokens.to_string(),
81
+ "--hits",
82
+ &rec.header.hits.to_string(),
83
+ "--ctx",
84
+ &rec.header.ctx_size.to_string(),
85
+ "--created",
86
+ &rec.header.created_at.to_string(),
87
+ "--used",
88
+ &rec.header.last_used.to_string(),
89
+ "--trailer-hex",
90
+ &hex(&rec.trailer),
91
+ ])
92
+ .status()
93
+ .unwrap();
94
+ assert!(status.success(), "C write failed");
95
+ }
96
+
97
+ fn c_read(path: &Path) -> Record {
98
+ let out = Command::new(require_oracle())
99
+ .args(["read", path.to_str().unwrap()])
100
+ .output()
101
+ .unwrap();
102
+ assert!(
103
+ out.status.success(),
104
+ "C read failed: {}",
105
+ String::from_utf8_lossy(&out.stderr)
106
+ );
107
+ let text = String::from_utf8(out.stdout).unwrap();
108
+ let mut rec = sample();
109
+ for line in text.lines() {
110
+ let Some((k, v)) = line.split_once('=') else {
111
+ continue;
112
+ };
113
+ match k {
114
+ "model_id" => rec.header.model_id = v.parse().unwrap(),
115
+ "quant_bits" => rec.header.quant_bits = v.parse().unwrap(),
116
+ "reason" => rec.header.reason = Reason::from_u8(v.parse().unwrap()),
117
+ "ext_flags" => rec.header.ext_flags = v.parse().unwrap(),
118
+ "tokens" => rec.header.tokens = v.parse().unwrap(),
119
+ "hits" => rec.header.hits = v.parse().unwrap(),
120
+ "ctx_size" => rec.header.ctx_size = v.parse().unwrap(),
121
+ "created_at" => rec.header.created_at = v.parse().unwrap(),
122
+ "last_used" => rec.header.last_used = v.parse().unwrap(),
123
+ "payload_bytes" => rec.header.payload_bytes = v.parse().unwrap(),
124
+ "text_bytes" => rec.header.text_bytes = v.parse().unwrap(),
125
+ "text_hex" => rec.text = unhex(v),
126
+ "payload_hex" => rec.payload = unhex(v),
127
+ "trailer_hex" => rec.trailer = unhex(v),
128
+ _ => {}
129
+ }
130
+ }
131
+ rec
132
+ }
133
+
134
+ fn tmp(name: &str) -> PathBuf {
135
+ let dir = std::env::temp_dir().join(format!("ds4-kv-4way-{}", std::process::id()));
136
+ fs::create_dir_all(&dir).unwrap();
137
+ dir.join(name)
138
+ }
139
+
140
+ fn same_envelope(a: &Record, b: &Record) {
141
+ assert_eq!(a.header.model_id, b.header.model_id);
142
+ assert_eq!(a.header.quant_bits, b.header.quant_bits);
143
+ assert_eq!(a.header.reason, b.header.reason);
144
+ assert_eq!(a.header.ext_flags, b.header.ext_flags);
145
+ assert_eq!(a.header.tokens, b.header.tokens);
146
+ assert_eq!(a.header.hits, b.header.hits);
147
+ assert_eq!(a.header.ctx_size, b.header.ctx_size);
148
+ assert_eq!(a.header.created_at, b.header.created_at);
149
+ assert_eq!(a.header.last_used, b.header.last_used);
150
+ assert_eq!(a.header.payload_bytes, b.header.payload_bytes);
151
+ assert_eq!(a.header.text_bytes, b.header.text_bytes);
152
+ assert_eq!(a.text, b.text);
153
+ assert_eq!(a.payload, b.payload);
154
+ assert_eq!(a.trailer, b.trailer);
155
+ }
156
+
157
+ #[test]
158
+ fn rust_save_rust_load() {
159
+ let rec = sample();
160
+ let path = tmp("rr.kv");
161
+ write_path(&path, &rec).unwrap();
162
+ let got = read_path(&path).unwrap();
163
+ same_envelope(&rec, &got);
164
+ assert_eq!(encode_file(&rec), fs::read(&path).unwrap());
165
+ }
166
+
167
+ #[test]
168
+ fn c_save_c_load() {
169
+ let rec = sample();
170
+ let path = tmp("cc.kv");
171
+ c_write(&path, &rec);
172
+ let got = c_read(&path);
173
+ same_envelope(&rec, &got);
174
+ }
175
+
176
+ #[test]
177
+ fn rust_save_c_load() {
178
+ let rec = sample();
179
+ let path = tmp("rc.kv");
180
+ write_path(&path, &rec).unwrap();
181
+ let got = c_read(&path);
182
+ same_envelope(&rec, &got);
183
+ }
184
+
185
+ #[test]
186
+ fn c_save_rust_load() {
187
+ let rec = sample();
188
+ let path = tmp("cr.kv");
189
+ c_write(&path, &rec);
190
+ let got = decode_file(&fs::read(&path).unwrap()).unwrap();
191
+ same_envelope(&rec, &got);
192
+ }
193
+
194
+ #[test]
195
+ fn c_save_rust_trailer_only_load() {
196
+ let mut rec = sample();
197
+ rec.header.ext_flags = EXT_TOOL_MAP;
198
+ rec.trailer = b"KTM\x01\x01\0\0\0\xff".to_vec();
199
+ let path = tmp("cr-trailer.kv");
200
+ c_write(&path, &rec);
201
+
202
+ let (header, trailer) = read_trailer(&path, rec.trailer.len() as u64).unwrap();
203
+ assert_eq!(header.ext_flags, EXT_TOOL_MAP);
204
+ assert_eq!(trailer, rec.trailer);
205
+ }
206
+
207
+ #[test]
208
+ fn rust_stream_payload_with_trailer_loads_in_c() {
209
+ let base = tmp("stream-rust-c");
210
+ let store_dir = base.join("store");
211
+ let _ = fs::remove_dir_all(&base);
212
+ fs::create_dir_all(&base).unwrap();
213
+
214
+ let mut rec = sample();
215
+ rec.payload = b"DSV4 opaque native payload\0\xff".to_vec();
216
+ rec.trailer = b"KVT1 tool-map trailer".to_vec();
217
+ rec.header.ext_flags = ds4_kv::EXT_TOOL_MAP;
218
+ rec.header.text_bytes = rec.text.len() as u32;
219
+ rec.header.payload_bytes = rec.payload.len() as u64;
220
+ let payload_path = ds4_kv::path_for_sha(&store_dir, &ds4_kv::text_sha_hex(&rec.text))
221
+ .with_extension(format!("kv.tmp.{}", std::process::id()));
222
+ fs::create_dir_all(&store_dir).unwrap();
223
+ fs::write(&payload_path, &rec.payload).unwrap();
224
+
225
+ let mut store = Store::open(&store_dir, 16, false, Options::default()).unwrap();
226
+ let path = store
227
+ .write_payload_file(rec.header.clone(), &rec.text, &payload_path, &rec.trailer)
228
+ .unwrap();
229
+ assert_eq!(fs::read(&payload_path).unwrap(), rec.payload);
230
+ let got = c_read(&path);
231
+ same_envelope(&rec, &got);
232
+
233
+ let _ = fs::remove_dir_all(&base);
234
+ }
235
+
236
+ #[test]
237
+ fn sha1_matches_c() {
238
+ let text = b"hello world";
239
+ let out = Command::new(require_oracle())
240
+ .args(["sha1", "--text-hex", &hex(text)])
241
+ .output()
242
+ .unwrap();
243
+ assert!(out.status.success());
244
+ let c = String::from_utf8(out.stdout).unwrap();
245
+ assert_eq!(c.trim(), ds4_kv::text_sha_hex(text));
246
+ }
247
+
248
+ #[test]
249
+ fn store_len_matches_c() {
250
+ for tokens in [100, 512, 2048, 4096 + 40, 20000] {
251
+ let out = Command::new(require_oracle())
252
+ .args(["store-len", "--tokens", &tokens.to_string()])
253
+ .output()
254
+ .unwrap();
255
+ assert!(out.status.success());
256
+ let c: i32 = String::from_utf8(out.stdout)
257
+ .unwrap()
258
+ .trim()
259
+ .parse()
260
+ .unwrap();
261
+ assert_eq!(store_len(&Options::default(), tokens), c, "tokens={tokens}");
262
+ }
263
+ }
264
+
265
+ #[test]
266
+ fn continued_target_matches_c() {
267
+ for (name, min, interval, align, last, live, expected) in [
268
+ ("default", 512, 10000, 2048, 0, 10240, 10240),
269
+ ("disabled", 512, 0, 2048, 0, 10240, 0),
270
+ ("custom-align", 512, 10000, 4096, 0, 12288, 12288),
271
+ ("below-min", 20000, 10000, 2048, 0, 10240, 0),
272
+ ("prior-marker", 512, 10000, 2048, 10240, 10240, 0),
273
+ ("nonmultiple", 512, 10000, 2048, 0, 10239, 0),
274
+ ("serial-align1", 512, 6800, 1, 0, 6800, 6800),
275
+ ] {
276
+ let out = Command::new(require_oracle())
277
+ .args([
278
+ "continued-target",
279
+ "--min",
280
+ &min.to_string(),
281
+ "--interval",
282
+ &interval.to_string(),
283
+ "--align",
284
+ &align.to_string(),
285
+ "--last",
286
+ &last.to_string(),
287
+ "--live",
288
+ &live.to_string(),
289
+ ])
290
+ .output()
291
+ .unwrap();
292
+ assert!(out.status.success(), "C continued-target failed for {name}");
293
+ let c: i32 = String::from_utf8(out.stdout)
294
+ .unwrap()
295
+ .trim()
296
+ .parse()
297
+ .unwrap();
298
+ assert_eq!(c, expected, "C case={name}");
299
+ let opt = Options {
300
+ min_tokens: min,
301
+ continued_interval_tokens: interval,
302
+ boundary_align_tokens: align,
303
+ ..Options::default()
304
+ };
305
+ assert_eq!(continued_store_target(&opt, last, live), c, "case={name}");
306
+ }
307
+ }
308
+
309
+ #[test]
310
+ fn chat_anchor_matches_c() {
311
+ let mut tokens = vec![7; 600];
312
+ tokens[512] = 99;
313
+ tokens[530] = 99;
314
+ tokens[550] = 100;
315
+ tokens[560] = 99;
316
+ let mut args = vec!["chat-anchor".to_string(), "99".into(), "100".into()];
317
+ args.extend(tokens.iter().map(i32::to_string));
318
+ let out = Command::new(require_oracle()).args(&args).output().unwrap();
319
+ assert!(out.status.success());
320
+ let c: i32 = String::from_utf8(out.stdout)
321
+ .unwrap()
322
+ .trim()
323
+ .parse()
324
+ .unwrap();
325
+ assert_eq!(chat_anchor_pos(&Options::default(), &tokens, 99, 100), c);
326
+ assert_eq!(c, 530);
327
+ }
328
+
329
+ #[test]
330
+ fn eviction_score_matches_c() {
331
+ let out = Command::new(require_oracle())
332
+ .args([
333
+ "score",
334
+ "--hits",
335
+ "4",
336
+ "--tokens",
337
+ "512",
338
+ "--file-size",
339
+ "4096",
340
+ "--reason",
341
+ "1",
342
+ "--created",
343
+ "1000",
344
+ "--used",
345
+ "1000",
346
+ "--now",
347
+ "1000",
348
+ ])
349
+ .output()
350
+ .unwrap();
351
+ assert!(out.status.success());
352
+ let c: f64 = String::from_utf8(out.stdout)
353
+ .unwrap()
354
+ .trim()
355
+ .parse()
356
+ .unwrap();
357
+ let rust = eviction_score(
358
+ &ScoreEntry {
359
+ sha: "x",
360
+ quant_bits: 2,
361
+ model_id: 0,
362
+ reason: Reason::Cold,
363
+ tokens: 512,
364
+ hits: 4,
365
+ ctx_size: 2048,
366
+ created_at: 1000,
367
+ last_used: 1000,
368
+ text_bytes: 11,
369
+ file_size: 4096,
370
+ },
371
+ 1000,
372
+ None,
373
+ );
374
+ assert!((rust - c).abs() <= 1e-12, "rust={rust} c={c}");
375
+ }
crates/ds4-perf-gpu/Cargo.toml ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [package]
2
+ name = "ds4-perf-gpu"
3
+ version.workspace = true
4
+ edition.workspace = true
5
+ license.workspace = true
6
+ publish.workspace = true
7
+ description = "Isolated CUDA calibration and CUPTI collection for ds4-perf"
8
+
9
+ [lib]
10
+ crate-type = ["rlib", "cdylib"]
11
+
12
+ [[bin]]
13
+ name = "ds4-perf-gpu"
14
+ path = "src/main.rs"
15
+ required-features = ["cuda"]
16
+
17
+ [features]
18
+ default = []
19
+ cuda = ["dep:cudarc", "dep:libc"]
20
+
21
+ [dependencies]
22
+ ds4-perf = { path = "../ds4-perf" }
23
+ serde_json = "1"
24
+ cudarc = { version = "0.19.9", optional = true, default-features = false, features = ["std", "driver", "nvrtc", "cupti", "dynamic-loading", "cuda-13030"] }
25
+ libc = { version = "0.2", optional = true }
crates/ds4-perf-gpu/src/calibration.rs ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use cudarc::{
2
+ driver::{CudaContext, LaunchConfig, PushKernelArg},
3
+ nvrtc::{compile_ptx_with_opts, CompileOptions},
4
+ };
5
+ use ds4_perf::machine::Calibration;
6
+ use std::time::Instant;
7
+
8
+ const WARMUPS: u32 = 2;
9
+ const REPEATS: u32 = 7;
10
+ const FMA_ITERATIONS: i32 = 4096;
11
+ const LAUNCHES: u32 = 256;
12
+ const SOURCE: &str = r#"
13
+ extern "C" __global__ void copy_words(float *out, const float *input, int n) {
14
+ int i = blockIdx.x * blockDim.x + threadIdx.x;
15
+ if (i < n) out[i] = input[i];
16
+ }
17
+ extern "C" __global__ void fma_envelope(float *out, int iterations) {
18
+ int i = blockIdx.x * blockDim.x + threadIdx.x;
19
+ float a=0.1f,b=0.2f,c=0.3f,d=0.4f,e=0.5f,f=0.6f,g=0.7f,h=0.8f;
20
+ for (int j=0; j<iterations; ++j) {
21
+ a=fmaf(a,0.9999f,0.0001f); b=fmaf(b,0.9999f,0.0001f);
22
+ c=fmaf(c,0.9999f,0.0001f); d=fmaf(d,0.9999f,0.0001f);
23
+ e=fmaf(e,0.9999f,0.0001f); f=fmaf(f,0.9999f,0.0001f);
24
+ g=fmaf(g,0.9999f,0.0001f); h=fmaf(h,0.9999f,0.0001f);
25
+ }
26
+ out[i]=a+b+c+d+e+f+g+h;
27
+ }
28
+ extern "C" __global__ void empty_launch() {}
29
+ "#;
30
+
31
+ pub fn run(ordinal: usize) -> Result<Calibration, String> {
32
+ measure(ordinal).map_err(|e| e.to_string())
33
+ }
34
+
35
+ fn measure(ordinal: usize) -> Result<Calibration, Box<dyn std::error::Error>> {
36
+ let gpu = crate::device::inspect(ordinal)?;
37
+ let ctx = CudaContext::new(ordinal)?;
38
+ let stream = ctx.default_stream();
39
+ // Larger than L2; bounded to 512 MiB total so this is not a memory stress test.
40
+ let bytes = (64 * 1024 * 1024usize).max(gpu.l2_bytes.max(0) as usize * 2);
41
+ if bytes > 256 * 1024 * 1024 || gpu.free_memory_bytes < (bytes * 4) as u64 {
42
+ return Err("insufficient bounded calibration headroom".into());
43
+ }
44
+ let n = (bytes / 4) as i32;
45
+ let source = stream.clone_htod(&vec![1.0f32; n as usize])?;
46
+ let mut target = stream.alloc_zeros::<f32>(n as usize)?;
47
+ let workers = gpu.multiprocessors as u32 * 256 * 8;
48
+ let mut compute = stream.alloc_zeros::<f32>(workers as usize)?;
49
+ let ptx = compile_ptx_with_opts(
50
+ SOURCE,
51
+ CompileOptions {
52
+ options: vec![format!(
53
+ "--gpu-architecture=compute_{}{}",
54
+ gpu.compute_major, gpu.compute_minor
55
+ )],
56
+ ..Default::default()
57
+ },
58
+ )?;
59
+ let ptx_sha256 = ds4_perf::artifact::hash_bytes(ptx.to_src().as_bytes());
60
+ let module = ctx.load_module(ptx)?;
61
+ let copy = module.load_function("copy_words")?;
62
+ let fma = module.load_function("fma_envelope")?;
63
+ let empty = module.load_function("empty_launch")?;
64
+ let mut copy_gb_s = Vec::new();
65
+ let mut fp32_gflop_s = Vec::new();
66
+ let mut launch_us = Vec::new();
67
+ let mut copy_times = Vec::new();
68
+ let mut compute_times = Vec::new();
69
+ let mut launch_times = Vec::new();
70
+ for repetition in 0..WARMUPS + REPEATS {
71
+ let start =
72
+ stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?;
73
+ // All pointers are cudarc-owned slices on this stream; n bounds the kernel.
74
+ unsafe {
75
+ stream
76
+ .launch_builder(&copy)
77
+ .arg(&mut target)
78
+ .arg(&source)
79
+ .arg(&n)
80
+ .launch(LaunchConfig::for_num_elems(n as u32))?;
81
+ }
82
+ let end =
83
+ stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?;
84
+ end.synchronize()?;
85
+ let copy_ms = start.elapsed_ms(&end)? as f64;
86
+
87
+ let start =
88
+ stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?;
89
+ // Exactly workers threads write an allocation of workers f32 elements.
90
+ unsafe {
91
+ stream
92
+ .launch_builder(&fma)
93
+ .arg(&mut compute)
94
+ .arg(&FMA_ITERATIONS)
95
+ .launch(LaunchConfig {
96
+ grid_dim: (workers / 256, 1, 1),
97
+ block_dim: (256, 1, 1),
98
+ shared_mem_bytes: 0,
99
+ })?;
100
+ }
101
+ let end =
102
+ stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?;
103
+ end.synchronize()?;
104
+ let compute_ms = start.elapsed_ms(&end)? as f64;
105
+
106
+ let host_start = Instant::now();
107
+ for _ in 0..LAUNCHES {
108
+ // This kernel has no arguments or memory accesses.
109
+ unsafe {
110
+ stream.launch_builder(&empty).launch(LaunchConfig {
111
+ grid_dim: (1, 1, 1),
112
+ block_dim: (1, 1, 1),
113
+ shared_mem_bytes: 0,
114
+ })?;
115
+ }
116
+ }
117
+ stream.synchronize()?;
118
+ let launch_ms = host_start.elapsed().as_secs_f64() * 1000.0;
119
+ let latency = launch_ms * 1000.0 / LAUNCHES as f64;
120
+ if repetition >= WARMUPS {
121
+ copy_gb_s.push(2.0 * bytes as f64 / (copy_ms * 1e6));
122
+ fp32_gflop_s.push(workers as f64 * FMA_ITERATIONS as f64 * 16.0 / (compute_ms * 1e6));
123
+ launch_us.push(latency);
124
+ copy_times.push(copy_ms);
125
+ compute_times.push(compute_ms);
126
+ launch_times.push(launch_ms);
127
+ }
128
+ }
129
+ if stream.clone_dtoh(&target)?.iter().any(|v| *v != 1.0) {
130
+ return Err("copy calibration validation failed".into());
131
+ }
132
+ let compute_result = stream.clone_dtoh(&compute)?;
133
+ if !valid_compute(&compute_result) {
134
+ return Err("FMA calibration validation failed".into());
135
+ }
136
+ if [&copy_gb_s, &fp32_gflop_s, &launch_us]
137
+ .iter()
138
+ .any(|values| values.iter().any(|v| !v.is_finite() || *v <= 0.0))
139
+ {
140
+ return Err("invalid calibration timing".into());
141
+ }
142
+ Ok(Calibration { ptx_sha256, method:"cuda-event-copy-fp32-fma-v1".into(),source_sha256:ds4_perf::artifact::hash_bytes(SOURCE.as_bytes()),
143
+ copy_ms:copy_times,compute_ms:compute_times,launch_batch_ms:launch_times,
144
+ compute_threads:workers,compute_block_threads:256,fma_iterations:FMA_ITERATIONS as u32,launch_count:LAUNCHES,
145
+ gpu, transfer_bytes: bytes as u64, warmups: WARMUPS, repeats: REPEATS, copy_gb_s, fp32_gflop_s, launch_us,
146
+ notes: vec!["Copy counts read plus write bytes, uses CUDA event time, and exceeds L2 capacity.".into(),
147
+ "FP32 measures eight independent SIMT FMA chains; it is not Tensor Core peak throughput.".into(),
148
+ "Launch latency is host wall time for 256 empty kernels plus completion synchronization, per kernel.".into(),
149
+ "Observed rates depend on current clocks, temperature and other processes; this is a measured envelope, not a hardware guarantee.".into()] })
150
+ }
151
+
152
+ fn valid_compute(values: &[f32]) -> bool {
153
+ let mut expected = [0.1f32, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8];
154
+ for _ in 0..FMA_ITERATIONS {
155
+ for value in &mut expected {
156
+ *value = value.mul_add(0.9999, 0.0001);
157
+ }
158
+ }
159
+ let expected: f32 = expected.iter().sum();
160
+ !values.is_empty()
161
+ && values
162
+ .iter()
163
+ .all(|v| v.is_finite() && (*v - expected).abs() <= 0.00002)
164
+ }
165
+
166
+ #[cfg(test)]
167
+ mod tests {
168
+ #[test]
169
+ fn rejects_short_fma_work() {
170
+ let mut one_step = [0.1f32, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8];
171
+ for value in &mut one_step {
172
+ *value = value.mul_add(0.9999, 0.0001);
173
+ }
174
+ assert!(!super::valid_compute(&[one_step.iter().sum()]));
175
+ }
176
+ }
crates/ds4-perf-gpu/src/collector.rs ADDED
@@ -0,0 +1,288 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // CUPTI injection follows NVIDIA's cupti_trace_injection sample. NVTX injection
2
+ // points directly to libcupti; all application annotations remain NVIDIA nvtx.
3
+ use cudarc::{
4
+ cupti::{
5
+ result::activity,
6
+ sys::{self, CUpti_ActivityKind::*},
7
+ },
8
+ driver,
9
+ };
10
+ use ds4_perf::trace::Event;
11
+ use std::{
12
+ alloc::{alloc, dealloc, Layout},
13
+ ffi::CStr,
14
+ fs::OpenOptions,
15
+ io::{BufWriter, Write},
16
+ sync::{
17
+ atomic::{AtomicU64, Ordering},
18
+ Mutex, OnceLock,
19
+ },
20
+ };
21
+
22
+ const BUFFER_BYTES: usize = 8 * 1024 * 1024;
23
+ const BUFFER_ALIGN: usize = 8;
24
+ static OUTPUT: OnceLock<Mutex<BufWriter<std::fs::File>>> = OnceLock::new();
25
+ static INITIALIZED: OnceLock<Result<(), String>> = OnceLock::new();
26
+ static ERRORS: AtomicU64 = AtomicU64::new(0);
27
+ static DROPPED: AtomicU64 = AtomicU64::new(0);
28
+
29
+ fn emit(event: Event) {
30
+ let Some(output) = OUTPUT.get() else {
31
+ return;
32
+ };
33
+ let Ok(mut output) = output.lock() else {
34
+ ERRORS.fetch_add(1, Ordering::Relaxed);
35
+ return;
36
+ };
37
+ if serde_json::to_writer(&mut *output, &event).is_err() || output.write_all(b"\n").is_err() {
38
+ ERRORS.fetch_add(1, Ordering::Relaxed);
39
+ }
40
+ }
41
+
42
+ fn error(message: String) {
43
+ ERRORS.fetch_add(1, Ordering::Relaxed);
44
+ emit(Event::Error { message });
45
+ }
46
+
47
+ // These callbacks are called by CUPTI with its documented output pointers.
48
+ extern "C" fn request_buffer(buffer: *mut *mut u8, size: *mut usize, records: *mut usize) {
49
+ let layout = Layout::from_size_align(BUFFER_BYTES, BUFFER_ALIGN).expect("constant layout");
50
+ unsafe {
51
+ *buffer = alloc(layout);
52
+ *size = if (*buffer).is_null() { 0 } else { BUFFER_BYTES };
53
+ *records = 0;
54
+ if (*buffer).is_null() {
55
+ ERRORS.fetch_add(1, Ordering::Relaxed);
56
+ }
57
+ }
58
+ }
59
+
60
+ extern "C" fn complete_buffer(
61
+ context: driver::sys::CUcontext,
62
+ stream: u32,
63
+ buffer: *mut u8,
64
+ _size: usize,
65
+ valid: usize,
66
+ ) {
67
+ // No panic may cross CUPTI's C boundary. Record failures for the parent.
68
+ let result = std::panic::catch_unwind(|| unsafe { consume(buffer, valid) });
69
+ match result {
70
+ Ok(Ok(())) => {}
71
+ Ok(Err(message)) => error(message),
72
+ Err(_) => error("activity callback panicked".into()),
73
+ }
74
+ let mut dropped = 0;
75
+ // CUPTI owns context lifetime throughout this callback.
76
+ match unsafe { activity::get_num_dropped_records(context, stream, &mut dropped) } {
77
+ Ok(()) => {
78
+ DROPPED.fetch_add(dropped as u64, Ordering::Relaxed);
79
+ }
80
+ Err(e) => error(e.to_string()),
81
+ }
82
+ if !buffer.is_null() {
83
+ // This is exactly the allocation made by request_buffer.
84
+ unsafe {
85
+ dealloc(
86
+ buffer,
87
+ Layout::from_size_align(BUFFER_BYTES, BUFFER_ALIGN).expect("constant layout"),
88
+ );
89
+ }
90
+ }
91
+ }
92
+
93
+ unsafe fn string(pointer: *const std::ffi::c_char) -> String {
94
+ if pointer.is_null() {
95
+ return String::new();
96
+ }
97
+ // CUPTI guarantees activity strings remain valid until this callback returns.
98
+ unsafe { CStr::from_ptr(pointer) }
99
+ .to_string_lossy()
100
+ .into_owned()
101
+ }
102
+
103
+ unsafe fn consume(buffer: *mut u8, valid: usize) -> Result<(), String> {
104
+ if valid == 0 {
105
+ return Ok(());
106
+ }
107
+ if buffer.is_null() || valid > BUFFER_BYTES {
108
+ return Err("invalid CUPTI buffer length".into());
109
+ }
110
+ let mut record = std::ptr::null_mut();
111
+ loop {
112
+ match unsafe { activity::get_next_record(buffer, valid, &mut record) } {
113
+ Ok(()) => {}
114
+ Err(cudarc::cupti::result::CuptiError(
115
+ sys::CUptiResult::CUPTI_ERROR_MAX_LIMIT_REACHED,
116
+ )) => return Ok(()),
117
+ Err(error) => return Err(error.to_string()),
118
+ }
119
+ if record.is_null() {
120
+ return Err("null activity record".into());
121
+ }
122
+ // Runtime API version is checked before enable. Cast only the documented
123
+ // activity kind to its CUDA 13.3 record; copy values before buffer release.
124
+ match unsafe { (*record).kind } {
125
+ CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL => {
126
+ let k = unsafe { &*record.cast::<sys::CUpti_ActivityKernel12>() };
127
+ if [k.gridX, k.gridY, k.gridZ, k.blockX, k.blockY, k.blockZ]
128
+ .iter()
129
+ .any(|v| *v < 0)
130
+ {
131
+ return Err("negative launch dimension".into());
132
+ }
133
+ emit(Event::Kernel {
134
+ name: unsafe { string(k.name) },
135
+ start: k.start,
136
+ end: k.end,
137
+ device: k.deviceId,
138
+ context: k.contextId,
139
+ stream: k.streamId,
140
+ correlation: k.correlationId,
141
+ grid: [k.gridX as u32, k.gridY as u32, k.gridZ as u32],
142
+ block: [k.blockX as u32, k.blockY as u32, k.blockZ as u32],
143
+ registers: k.registersPerThread as u32,
144
+ shared_bytes: k.staticSharedMemory.max(0) as u64
145
+ + k.dynamicSharedMemory.max(0) as u64,
146
+ graph_id: k.graphId,
147
+ });
148
+ }
149
+ CUPTI_ACTIVITY_KIND_MARKER => {
150
+ let m = unsafe { &*record.cast::<sys::CUpti_ActivityMarker2>() };
151
+ if !matches!(
152
+ m.objectKind,
153
+ sys::CUpti_ActivityObjectKind::CUPTI_ACTIVITY_OBJECT_THREAD
154
+ | sys::CUpti_ActivityObjectKind::CUPTI_ACTIVITY_OBJECT_PROCESS
155
+ ) {
156
+ continue;
157
+ }
158
+ let id = unsafe { m.objectId.pt };
159
+ emit(Event::Marker {
160
+ name: unsafe { string(m.name) },
161
+ domain: unsafe { string(m.domain) },
162
+ timestamp: m.timestamp,
163
+ id: m.id,
164
+ flags: m.flags as u32,
165
+ pid: id.processId,
166
+ tid: id.threadId,
167
+ });
168
+ }
169
+ CUPTI_ACTIVITY_KIND_MEMCPY => {
170
+ let m = unsafe { &*record.cast::<sys::CUpti_ActivityMemcpy6>() };
171
+ emit(Event::Memop {
172
+ operation: "memcpy".into(),
173
+ start: m.start,
174
+ end: m.end,
175
+ bytes: m.bytes,
176
+ device: m.deviceId,
177
+ context: m.contextId,
178
+ stream: m.streamId,
179
+ });
180
+ }
181
+ CUPTI_ACTIVITY_KIND_MEMSET => {
182
+ let m = unsafe { &*record.cast::<sys::CUpti_ActivityMemset4>() };
183
+ emit(Event::Memop {
184
+ operation: "memset".into(),
185
+ start: m.start,
186
+ end: m.end,
187
+ bytes: m.bytes,
188
+ device: m.deviceId,
189
+ context: m.contextId,
190
+ stream: m.streamId,
191
+ });
192
+ }
193
+ _ => {}
194
+ }
195
+ }
196
+ }
197
+
198
+ extern "C" fn finish() {
199
+ let _ = std::panic::catch_unwind(|| {
200
+ if let Err(e) =
201
+ activity::flush_all(sys::CUpti_ActivityFlag::CUPTI_ACTIVITY_FLAG_FLUSH_FORCED as u32)
202
+ {
203
+ error(e.to_string());
204
+ }
205
+ emit(Event::End {
206
+ dropped: DROPPED.load(Ordering::Relaxed),
207
+ errors: ERRORS.load(Ordering::Relaxed),
208
+ });
209
+ if let Some(output) = OUTPUT.get() {
210
+ if let Ok(mut output) = output.lock() {
211
+ let _ = output.flush();
212
+ let _ = output.get_ref().sync_all();
213
+ }
214
+ }
215
+ });
216
+ }
217
+
218
+ fn initialize() -> Result<(), String> {
219
+ let path =
220
+ std::env::var_os("DS4_PERF_CUPTI_OUTPUT").ok_or("DS4_PERF_CUPTI_OUTPUT is required")?;
221
+ let file = OpenOptions::new()
222
+ .create_new(true)
223
+ .write(true)
224
+ .open(path)
225
+ .map_err(|e| e.to_string())?;
226
+ OUTPUT
227
+ .set(Mutex::new(BufWriter::new(file)))
228
+ .map_err(|_| "collector already initialized")?;
229
+ let mut version = 0;
230
+ // cudarc supplies the binding; the output pointer is a live u32 local.
231
+ unsafe { sys::cuptiGetVersion(&mut version) }
232
+ .result()
233
+ .map_err(|e| e.to_string())?;
234
+ emit(Event::Start {
235
+ schema_version: 1,
236
+ api_version: version,
237
+ pid: std::process::id(),
238
+ });
239
+ if version != sys::CUPTI_API_VERSION {
240
+ return Err(format!(
241
+ "CUPTI record ABI mismatch: runtime {version}, bindings {}",
242
+ sys::CUPTI_API_VERSION
243
+ ));
244
+ }
245
+ activity::register_callbacks(Some(request_buffer), Some(complete_buffer))
246
+ .map_err(|e| e.to_string())?;
247
+ for kind in [
248
+ CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL,
249
+ CUPTI_ACTIVITY_KIND_MEMCPY,
250
+ CUPTI_ACTIVITY_KIND_MEMSET,
251
+ CUPTI_ACTIVITY_KIND_MARKER,
252
+ ] {
253
+ activity::enable(kind).map_err(|e| e.to_string())?;
254
+ }
255
+ // Linux atexit is the documented CUPTI injection sample flush point.
256
+ if unsafe { libc::atexit(finish) } != 0 {
257
+ return Err("cannot register CUPTI exit flush".into());
258
+ }
259
+ Ok(())
260
+ }
261
+
262
+ #[unsafe(no_mangle)]
263
+ pub extern "C" fn InitializeInjection() -> i32 {
264
+ let result = std::panic::catch_unwind(|| INITIALIZED.get_or_init(initialize));
265
+ match result {
266
+ Ok(Ok(())) => 1,
267
+ Ok(Err(message)) => {
268
+ error(message.clone());
269
+ flush_output();
270
+ 0
271
+ }
272
+ Err(_) => {
273
+ error("CUPTI initialization panicked".into());
274
+ flush_output();
275
+ 0
276
+ }
277
+ }
278
+ }
279
+
280
+ fn flush_output() {
281
+ if let Some(output) = OUTPUT.get() {
282
+ if let Ok(mut output) = output.lock() {
283
+ if output.flush().is_err() || output.get_ref().sync_all().is_err() {
284
+ eprintln!("ds4-perf CUPTI: could not persist collector output");
285
+ }
286
+ }
287
+ }
288
+ }
crates/ds4-perf-gpu/src/device.rs ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use cudarc::driver::{sys::CUdevice_attribute::*, CudaContext};
2
+ use ds4_perf::machine::Gpu;
3
+
4
+ pub fn inspect(ordinal: usize) -> Result<Gpu, String> {
5
+ let ctx = CudaContext::new(ordinal).map_err(|e| e.to_string())?;
6
+ let attr = |key| ctx.attribute(key).map_err(|e| e.to_string());
7
+ let (free, total) = ctx.mem_get_info().map_err(|e| e.to_string())?;
8
+ let uuid = ctx.uuid().map_err(|e| e.to_string())?;
9
+ let mut driver_version = 0;
10
+ // CUDA writes one integer to this live local, through cudarc's bindings.
11
+ unsafe { cudarc::driver::sys::cuDriverGetVersion(&mut driver_version) }
12
+ .result()
13
+ .map_err(|e| e.to_string())?;
14
+ let mut unavailable = std::collections::BTreeMap::new();
15
+ let mut optional = |name: &str, key| match attr(key) {
16
+ Ok(value) if value > 0 => Some(value),
17
+ other => {
18
+ unavailable.insert(name.into(), format!("{other:?}"));
19
+ None
20
+ }
21
+ };
22
+ let memory_bus_bits = optional(
23
+ "memory_bus_bits",
24
+ CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH,
25
+ );
26
+ let memory_clock_khz = optional("memory_clock_khz", CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE);
27
+ let clock_khz = optional("clock_khz", CU_DEVICE_ATTRIBUTE_CLOCK_RATE);
28
+ Ok(Gpu {
29
+ ordinal,
30
+ name: ctx.name().map_err(|e| e.to_string())?,
31
+ uuid: uuid.bytes.iter().map(|b| format!("{:02x}", *b)).collect(),
32
+ compute_major: attr(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR)?,
33
+ compute_minor: attr(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR)?,
34
+ driver_version,
35
+ total_memory_bytes: total as u64,
36
+ free_memory_bytes: free as u64,
37
+ multiprocessors: attr(CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT)?,
38
+ warp_size: attr(CU_DEVICE_ATTRIBUTE_WARP_SIZE)?,
39
+ max_threads_per_block: attr(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK)?,
40
+ max_threads_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR)?,
41
+ max_blocks_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_BLOCKS_PER_MULTIPROCESSOR)?,
42
+ registers_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_MULTIPROCESSOR)?,
43
+ shared_bytes_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR)?,
44
+ shared_bytes_per_block: attr(CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN)?,
45
+ l2_bytes: attr(CU_DEVICE_ATTRIBUTE_L2_CACHE_SIZE)?,
46
+ memory_bus_bits,
47
+ memory_clock_khz,
48
+ clock_khz,
49
+ unavailable,
50
+ })
51
+ }
crates/ds4-perf-gpu/src/lib.rs ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // This helper owns profiler CUDA calls; it is never an inference dependency.
2
+ #![cfg_attr(not(feature = "cuda"), forbid(unsafe_code))]
3
+
4
+ #[cfg(feature = "cuda")]
5
+ pub mod device;
6
+
7
+ #[cfg(feature = "cuda")]
8
+ pub mod calibration;
9
+
10
+ #[cfg(feature = "cuda")]
11
+ mod collector;
crates/ds4-perf-gpu/src/main.rs ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ fn main() {
2
+ let args: Vec<_> = std::env::args().skip(1).collect();
3
+ let result = match args.as_slice() {
4
+ [command, ordinal] if command == "inspect" => ordinal
5
+ .parse::<usize>()
6
+ .map_err(|e| e.to_string())
7
+ .and_then(ds4_perf_gpu::device::inspect)
8
+ .and_then(|gpu| serde_json::to_string(&gpu).map_err(|e| e.to_string())),
9
+ [command, ordinal] if command == "calibrate" => ordinal
10
+ .parse::<usize>()
11
+ .map_err(|e| e.to_string())
12
+ .and_then(ds4_perf_gpu::calibration::run)
13
+ .and_then(|calibration| serde_json::to_string(&calibration).map_err(|e| e.to_string())),
14
+ _ => Err("usage: ds4-perf-gpu inspect|calibrate DEVICE".into()),
15
+ };
16
+ match result {
17
+ Ok(text) => println!("{text}"),
18
+ Err(error) => {
19
+ eprintln!("ds4-perf-gpu: {error}");
20
+ std::process::exit(1);
21
+ }
22
+ }
23
+ }
crates/ds4-perf-gpu/tests/live.rs ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #![cfg(feature = "cuda")]
2
+
3
+ #[test]
4
+ #[ignore = "requires an idle CUDA device and NVRTC"]
5
+ fn measured_envelope() {
6
+ let result = ds4_perf_gpu::calibration::run(0).unwrap();
7
+ for samples in [&result.copy_gb_s, &result.fp32_gflop_s, &result.launch_us] {
8
+ assert_eq!(samples.len(), result.repeats as usize);
9
+ assert!(samples
10
+ .iter()
11
+ .all(|value| value.is_finite() && *value > 0.0));
12
+ }
13
+ assert!(result.transfer_bytes > result.gpu.l2_bytes as u64);
14
+ }
crates/ds4-perf/Cargo.toml ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [package]
2
+ name = "ds4-perf"
3
+ version.workspace = true
4
+ edition.workspace = true
5
+ license.workspace = true
6
+ publish.workspace = true
7
+ description = "Model-free performance orchestration and Nsight diagnostics for ds4"
8
+
9
+ [dependencies]
10
+ clap = { version = "4.6", features = ["derive"] }
11
+ serde = { version = "1", features = ["derive"] }
12
+ # Evidence summaries must retain the exact f64 samples used for decisions.
13
+ serde_json = { version = "1", features = ["float_roundtrip"] }
14
+ sha2 = { version = "0.11", default-features = false }
15
+ ctrlc = { version = "3.5", features = ["termination"] }
16
+
17
+ [target.'cfg(unix)'.dependencies]
18
+ nix = { version = "0.31", default-features = false, features = ["signal", "process"] }
crates/ds4-perf/src/artifact.rs ADDED
@@ -0,0 +1,564 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use serde::{de::DeserializeOwned, Deserialize, Serialize};
2
+ use sha2::{Digest, Sha256};
3
+ use std::{
4
+ fs,
5
+ io::{Read, Write},
6
+ path::Path,
7
+ time::SystemTime,
8
+ };
9
+
10
+ /// Each payload evolves independently. Unknown versions fail closed.
11
+ pub trait Payload {
12
+ const KIND: &'static str;
13
+ const SCHEMA: u32 = 1;
14
+ fn validate(&self) -> Result<(), String>;
15
+ }
16
+
17
+ #[derive(Clone, Debug, Serialize, Deserialize)]
18
+ #[serde(deny_unknown_fields)]
19
+ pub struct Reference {
20
+ pub path: String,
21
+ pub sha256: String,
22
+ }
23
+
24
+ #[derive(Debug, Serialize, Deserialize)]
25
+ #[serde(deny_unknown_fields)]
26
+ pub struct Artifact<T> {
27
+ pub schema_version: u32,
28
+ pub kind: String,
29
+ pub producer_version: String,
30
+ pub created_unix: u64,
31
+ pub complete: bool,
32
+ pub inputs: Vec<Reference>,
33
+ pub warnings: Vec<String>,
34
+ pub data: Option<T>,
35
+ }
36
+
37
+ impl<T: Payload> Artifact<T> {
38
+ pub fn new(data: T) -> Self {
39
+ Self {
40
+ schema_version: T::SCHEMA,
41
+ kind: T::KIND.into(),
42
+ producer_version: env!("CARGO_PKG_VERSION").into(),
43
+ created_unix: SystemTime::now()
44
+ .duration_since(SystemTime::UNIX_EPOCH)
45
+ .unwrap_or_default()
46
+ .as_secs(),
47
+ complete: true,
48
+ inputs: Vec::new(),
49
+ warnings: Vec::new(),
50
+ data: Some(data),
51
+ }
52
+ }
53
+ pub fn failed(message: String) -> Self {
54
+ Self {
55
+ schema_version: T::SCHEMA,
56
+ kind: T::KIND.into(),
57
+ producer_version: env!("CARGO_PKG_VERSION").into(),
58
+ created_unix: SystemTime::now()
59
+ .duration_since(SystemTime::UNIX_EPOCH)
60
+ .unwrap_or_default()
61
+ .as_secs(),
62
+ complete: false,
63
+ inputs: Vec::new(),
64
+ warnings: vec![message],
65
+ data: None,
66
+ }
67
+ }
68
+ pub fn require(&self) -> Result<&T, String> {
69
+ if !self.complete {
70
+ return Err(format!("incomplete {} evidence", T::KIND));
71
+ }
72
+ self.data
73
+ .as_ref()
74
+ .ok_or_else(|| format!("missing {} payload", T::KIND))
75
+ }
76
+ }
77
+
78
+ fn hex(bytes: impl IntoIterator<Item = u8>) -> String {
79
+ bytes
80
+ .into_iter()
81
+ .map(|byte| format!("{byte:02x}"))
82
+ .collect()
83
+ }
84
+
85
+ pub fn hash_bytes(bytes: &[u8]) -> String {
86
+ hex(Sha256::digest(bytes))
87
+ }
88
+
89
+ pub fn hash(path: &Path) -> Result<String, String> {
90
+ let mut file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
91
+ hash_file(&mut file)
92
+ }
93
+
94
+ fn hash_file(file: &mut fs::File) -> Result<String, String> {
95
+ let mut state = Sha256::new();
96
+ let mut buffer = [0u8; 64 * 1024];
97
+ loop {
98
+ let count = file.read(&mut buffer).map_err(|e| e.to_string())?;
99
+ if count == 0 {
100
+ break;
101
+ }
102
+ state.update(&buffer[..count]);
103
+ }
104
+ Ok(hex(state.finalize()))
105
+ }
106
+
107
+ #[derive(Clone, PartialEq, Eq)]
108
+ struct FileStamp {
109
+ size: u64,
110
+ modified: SystemTime,
111
+ #[cfg(unix)]
112
+ identity: [u64; 6],
113
+ }
114
+
115
+ impl FileStamp {
116
+ fn read(metadata: &fs::Metadata) -> Result<Self, String> {
117
+ #[cfg(unix)]
118
+ use std::os::unix::fs::MetadataExt;
119
+ if !metadata.is_file() {
120
+ return Err("evidence input must be a regular file".into());
121
+ }
122
+ Ok(Self {
123
+ size: metadata.len(),
124
+ modified: metadata.modified().map_err(|e| e.to_string())?,
125
+ #[cfg(unix)]
126
+ identity: [
127
+ metadata.dev(),
128
+ metadata.ino(),
129
+ metadata.mtime() as u64,
130
+ metadata.mtime_nsec() as u64,
131
+ metadata.ctime() as u64,
132
+ metadata.ctime_nsec() as u64,
133
+ ],
134
+ })
135
+ }
136
+ }
137
+
138
+ /// Share full-byte verification only within one operation. Reuse never trusts
139
+ /// an expected hash alone: replacement or metadata drift invalidates the read.
140
+ #[derive(Default)]
141
+ pub struct Verification {
142
+ files: std::collections::BTreeMap<std::path::PathBuf, (FileStamp, String)>,
143
+ aliases: std::collections::BTreeMap<std::path::PathBuf, std::path::PathBuf>,
144
+ #[cfg(test)]
145
+ full_reads: usize,
146
+ }
147
+
148
+ impl Verification {
149
+ pub fn verify(&mut self, path: &Path, expected: &str) -> Result<(), String> {
150
+ let alias = std::env::current_dir()
151
+ .map_err(|e| e.to_string())?
152
+ .join(path);
153
+ let canonical = path.canonicalize().map_err(|e| e.to_string())?;
154
+ if self
155
+ .aliases
156
+ .get(&alias)
157
+ .is_some_and(|previous| previous != &canonical)
158
+ {
159
+ return Err(format!("evidence path changed: {}", path.display()));
160
+ }
161
+ let mut file = fs::File::open(&canonical).map_err(|e| e.to_string())?;
162
+ let before = FileStamp::read(&file.metadata().map_err(|e| e.to_string())?)?;
163
+ let digest = match self.files.get(&canonical) {
164
+ Some((stamp, digest)) => {
165
+ if *stamp != before {
166
+ return Err(format!(
167
+ "evidence changed during verification: {}",
168
+ path.display()
169
+ ));
170
+ }
171
+ digest.clone()
172
+ }
173
+ None => {
174
+ #[cfg(test)]
175
+ {
176
+ self.full_reads += 1;
177
+ }
178
+ hash_file(&mut file)?
179
+ }
180
+ };
181
+ if digest != expected {
182
+ return Err(format!("evidence changed: {}", path.display()));
183
+ }
184
+ let after = FileStamp::read(&file.metadata().map_err(|e| e.to_string())?)?;
185
+ let current = FileStamp::read(&fs::metadata(&canonical).map_err(|e| e.to_string())?)?;
186
+ if before != after
187
+ || before != current
188
+ || path.canonicalize().ok().as_ref() != Some(&canonical)
189
+ {
190
+ return Err(format!(
191
+ "evidence changed during verification: {}",
192
+ path.display()
193
+ ));
194
+ }
195
+ self.aliases.insert(alias, canonical.clone());
196
+ self.files.insert(canonical, (before, digest));
197
+ Ok(())
198
+ }
199
+
200
+ pub fn finish(&self) -> Result<(), String> {
201
+ for (alias, target) in &self.aliases {
202
+ if alias.canonicalize().ok().as_ref() != Some(target) {
203
+ return Err(format!("evidence path changed: {}", alias.display()));
204
+ }
205
+ }
206
+ for (path, (before, _)) in &self.files {
207
+ let current = FileStamp::read(&fs::metadata(path).map_err(|e| e.to_string())?)?;
208
+ if *before != current {
209
+ return Err(format!(
210
+ "evidence changed during verification: {}",
211
+ path.display()
212
+ ));
213
+ }
214
+ }
215
+ Ok(())
216
+ }
217
+ }
218
+
219
+ pub fn reference(path: &Path, directory: &Path) -> Result<Reference, String> {
220
+ let canonical = path.canonicalize().map_err(|e| e.to_string())?;
221
+ let base = directory.canonicalize().map_err(|e| e.to_string())?;
222
+ let relative = canonical.strip_prefix(base).unwrap_or(&canonical);
223
+ Ok(Reference {
224
+ path: relative.to_string_lossy().into(),
225
+ sha256: hash(&canonical)?,
226
+ })
227
+ }
228
+
229
+ pub fn save<T: Serialize + Payload>(path: &Path, value: &Artifact<T>) -> Result<(), String> {
230
+ if value.kind != T::KIND || value.schema_version != T::SCHEMA {
231
+ return Err("artifact type/schema mismatch".into());
232
+ }
233
+ if let Some(data) = &value.data {
234
+ data.validate()?;
235
+ }
236
+ if value.complete && value.data.is_none() {
237
+ return Err("complete artifact has no data".into());
238
+ }
239
+ let mut bytes = serde_json::to_vec_pretty(value).map_err(|e| e.to_string())?;
240
+ bytes.push(b'\n');
241
+ let nonce = SystemTime::now()
242
+ .duration_since(SystemTime::UNIX_EPOCH)
243
+ .unwrap_or_default()
244
+ .as_nanos();
245
+ let temporary = path.with_extension(format!("tmp-{}-{nonce}", std::process::id()));
246
+ let result = (|| {
247
+ let mut file = fs::OpenOptions::new()
248
+ .create_new(true)
249
+ .write(true)
250
+ .open(&temporary)?;
251
+ file.write_all(&bytes)?;
252
+ file.sync_all()?;
253
+ // Atomic publication with no overwrite, even if another writer wins.
254
+ fs::hard_link(&temporary, path)
255
+ })();
256
+ let _ = fs::remove_file(temporary);
257
+ result.map_err(|e| format!("{}: {e}", path.display()))
258
+ }
259
+
260
+ pub fn load<T: DeserializeOwned + Payload>(path: &Path) -> Result<Artifact<T>, String> {
261
+ let mut checked = Verification::default();
262
+ let value = load_verified(path, &mut checked)?;
263
+ checked.finish()?;
264
+ Ok(value)
265
+ }
266
+
267
+ pub fn load_verified<T: DeserializeOwned + Payload>(
268
+ path: &Path,
269
+ checked: &mut Verification,
270
+ ) -> Result<Artifact<T>, String> {
271
+ let file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
272
+ const MAX_JSON: u64 = 64 * 1024 * 1024;
273
+ if file.metadata().map_err(|e| e.to_string())?.len() > MAX_JSON {
274
+ return Err("artifact exceeds 64 MiB".into());
275
+ }
276
+ let mut bytes = Vec::new();
277
+ file.take(MAX_JSON + 1)
278
+ .read_to_end(&mut bytes)
279
+ .map_err(|e| e.to_string())?;
280
+ if bytes.len() as u64 > MAX_JSON {
281
+ return Err("artifact exceeds 64 MiB".into());
282
+ }
283
+ checked.verify(path, &hash_bytes(&bytes))?;
284
+ let value: Artifact<T> =
285
+ serde_json::from_slice(&bytes).map_err(|e| format!("{}: {e}", path.display()))?;
286
+ if value.schema_version != T::SCHEMA || value.kind != T::KIND {
287
+ return Err(format!(
288
+ "{}: expected {} schema {}",
289
+ path.display(),
290
+ T::KIND,
291
+ T::SCHEMA
292
+ ));
293
+ }
294
+ if let Some(data) = &value.data {
295
+ data.validate()?;
296
+ }
297
+ if value.complete {
298
+ value.require()?;
299
+ }
300
+ verify_refs(
301
+ path,
302
+ &value.inputs,
303
+ checked,
304
+ &mut std::collections::BTreeSet::new(),
305
+ 0,
306
+ )?;
307
+ Ok(value)
308
+ }
309
+
310
+ fn verify_refs(
311
+ path: &Path,
312
+ refs: &[Reference],
313
+ checked: &mut Verification,
314
+ traversed: &mut std::collections::BTreeSet<std::path::PathBuf>,
315
+ depth: usize,
316
+ ) -> Result<(), String> {
317
+ if depth > 32 {
318
+ return Err("evidence reference depth exceeds 32".into());
319
+ }
320
+ for input in refs {
321
+ let alias = path.parent().unwrap_or(Path::new(".")).join(&input.path);
322
+ let target = alias.canonicalize().map_err(|e| e.to_string())?;
323
+ checked.verify(&alias, &input.sha256)?;
324
+ if !traversed.insert(target.clone()) {
325
+ continue;
326
+ }
327
+ if target.extension().is_none_or(|e| e != "json") {
328
+ continue;
329
+ }
330
+ let file = fs::File::open(&target).map_err(|e| e.to_string())?;
331
+ if file.metadata().map_err(|e| e.to_string())?.len() > 64 * 1024 * 1024 {
332
+ return Err("JSON evidence exceeds 64 MiB".into());
333
+ }
334
+ let value: serde_json::Value = serde_json::from_reader(file).map_err(|e| e.to_string())?;
335
+ if value.get("kind").is_none() || value.get("schema_version").is_none() {
336
+ continue;
337
+ }
338
+ let refs: Vec<Reference> = serde_json::from_value(
339
+ value
340
+ .get("inputs")
341
+ .ok_or("nested artifact inputs missing")?
342
+ .clone(),
343
+ )
344
+ .map_err(|e| e.to_string())?;
345
+ verify_refs(&target, &refs, checked, traversed, depth + 1)?;
346
+ }
347
+ Ok(())
348
+ }
349
+
350
+ pub fn directory(path: &Path) -> Result<(), String> {
351
+ if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) {
352
+ fs::create_dir_all(parent).map_err(|e| e.to_string())?;
353
+ }
354
+ fs::create_dir(path).map_err(|e| {
355
+ format!(
356
+ "cannot create new evidence directory {}: {e}",
357
+ path.display()
358
+ )
359
+ })
360
+ }
361
+
362
+ /// Import an inspected machine/calibration bundle with its relative raw inputs.
363
+ /// External references require re-inspection into a self-contained directory.
364
+ pub fn bundle<T: DeserializeOwned + Payload>(
365
+ path: &Path,
366
+ out: &Path,
367
+ ) -> Result<std::path::PathBuf, String> {
368
+ load::<T>(path)?;
369
+ let source = path.canonicalize().map_err(|e| e.to_string())?;
370
+ let root = source.parent().ok_or("bundle has no parent")?;
371
+ directory(out)?;
372
+ let mut visited = std::collections::BTreeSet::new();
373
+ let mut pending = vec![source.clone()];
374
+ let mut total = 0u64;
375
+ while let Some(path) = pending.pop() {
376
+ let path = path.canonicalize().map_err(|e| e.to_string())?;
377
+ if !visited.insert(path.clone()) {
378
+ continue;
379
+ }
380
+ let relative = path
381
+ .strip_prefix(root)
382
+ .map_err(|_| "bundle has external raw inputs; create a self-contained inspection")?;
383
+ total = total
384
+ .checked_add(fs::metadata(&path).map_err(|e| e.to_string())?.len())
385
+ .ok_or("bundle size overflow")?;
386
+ if total > 256 * 1024 * 1024 {
387
+ return Err("machine/calibration bundle exceeds 256 MiB".into());
388
+ }
389
+ let destination = out.join(relative);
390
+ if let Some(parent) = destination.parent() {
391
+ fs::create_dir_all(parent).map_err(|e| e.to_string())?;
392
+ }
393
+ let mut source_file = fs::File::open(&path).map_err(|e| e.to_string())?;
394
+ let mut copy = fs::OpenOptions::new()
395
+ .create_new(true)
396
+ .write(true)
397
+ .open(&destination)
398
+ .map_err(|e| e.to_string())?;
399
+ std::io::copy(&mut source_file, &mut copy).map_err(|e| e.to_string())?;
400
+ if path.extension().is_none_or(|e| e != "json") {
401
+ continue;
402
+ }
403
+ let value: serde_json::Value =
404
+ serde_json::from_reader(fs::File::open(&path).map_err(|e| e.to_string())?)
405
+ .map_err(|e| e.to_string())?;
406
+ if value.get("kind").is_none() || value.get("schema_version").is_none() {
407
+ continue;
408
+ }
409
+ let inputs: Vec<Reference> =
410
+ serde_json::from_value(value.get("inputs").ok_or("bundle inputs missing")?.clone())
411
+ .map_err(|e| e.to_string())?;
412
+ pending.extend(
413
+ inputs
414
+ .into_iter()
415
+ .map(|r| path.parent().unwrap().join(r.path)),
416
+ );
417
+ }
418
+ let imported = out.join(source.file_name().unwrap());
419
+ load::<T>(&imported)?;
420
+ Ok(imported)
421
+ }
422
+
423
+ #[cfg(test)]
424
+ mod tests {
425
+ use super::*;
426
+ #[derive(Debug, Serialize, Deserialize)]
427
+ struct Example {
428
+ value: u32,
429
+ }
430
+ impl Payload for Example {
431
+ const KIND: &'static str = "example";
432
+ fn validate(&self) -> Result<(), String> {
433
+ Ok(())
434
+ }
435
+ }
436
+ #[test]
437
+ fn verification_rejects_drift_and_conflicting_digest() {
438
+ let root = std::env::temp_dir().join(format!("ds4-verified-{}", std::process::id()));
439
+ directory(&root).unwrap();
440
+ let path = root.join("input");
441
+ fs::write(&path, b"first").unwrap();
442
+ let expected = hash(&path).unwrap();
443
+ let mut checked = Verification::default();
444
+ checked.verify(&path, &expected).unwrap();
445
+ checked.verify(&path, &expected).unwrap();
446
+ assert_eq!(checked.full_reads, 1);
447
+ assert!(checked.verify(&path, &hash_bytes(b"other")).is_err());
448
+ fs::write(&path, b"other").unwrap();
449
+ assert!(checked.verify(&path, &expected).is_err());
450
+ assert!(checked.finish().is_err());
451
+
452
+ let mut replaced = Verification::default();
453
+ replaced.verify(&path, &hash_bytes(b"other")).unwrap();
454
+ let replacement = root.join("replacement");
455
+ fs::write(&replacement, b"other").unwrap();
456
+ fs::rename(replacement, &path).unwrap();
457
+ assert!(
458
+ replaced.finish().is_err(),
459
+ "same bytes on another inode must fail"
460
+ );
461
+ fs::remove_dir_all(root).unwrap();
462
+ }
463
+
464
+ #[test]
465
+ fn operation_reuses_shared_inputs_across_artifacts() {
466
+ let root = std::env::temp_dir().join(format!("ds4-shared-{}", std::process::id()));
467
+ directory(&root).unwrap();
468
+ let input = root.join("weights");
469
+ fs::write(&input, b"same input").unwrap();
470
+ for index in 0..2 {
471
+ let mut evidence = Artifact::new(Example { value: index });
472
+ evidence.inputs.push(reference(&input, &root).unwrap());
473
+ save(&root.join(format!("{index}.json")), &evidence).unwrap();
474
+ }
475
+ let mut checked = Verification::default();
476
+ for index in 0..2 {
477
+ load_verified::<Example>(&root.join(format!("{index}.json")), &mut checked).unwrap();
478
+ }
479
+ assert_eq!(checked.full_reads, 3, "two envelopes and one shared input");
480
+ checked.finish().unwrap();
481
+ let envelope = root.join("0.json");
482
+ let bytes = fs::read(&envelope).unwrap();
483
+ fs::write(&envelope, [bytes.as_slice(), b"\n"].concat()).unwrap();
484
+ assert!(
485
+ checked.finish().is_err(),
486
+ "root envelope must remain pinned"
487
+ );
488
+ fs::write(&input, b"new input!").unwrap();
489
+ assert!(load_verified::<Example>(&root.join("1.json"), &mut checked).is_err());
490
+ assert!(checked.finish().is_err());
491
+ fs::remove_dir_all(root).unwrap();
492
+ }
493
+
494
+ #[cfg(unix)]
495
+ #[test]
496
+ fn verification_rejects_retargeted_alias() {
497
+ let root = std::env::temp_dir().join(format!("ds4-alias-{}", std::process::id()));
498
+ directory(&root).unwrap();
499
+ fs::write(root.join("a"), b"same").unwrap();
500
+ fs::write(root.join("b"), b"same").unwrap();
501
+ let alias = root.join("alias");
502
+ std::os::unix::fs::symlink(root.join("a"), &alias).unwrap();
503
+ let mut checked = Verification::default();
504
+ checked.verify(&alias, &hash_bytes(b"same")).unwrap();
505
+ fs::remove_file(&alias).unwrap();
506
+ std::os::unix::fs::symlink(root.join("b"), &alias).unwrap();
507
+ assert!(checked.finish().is_err());
508
+ assert!(checked.verify(&alias, &hash_bytes(b"same")).is_err());
509
+ fs::remove_dir_all(root).unwrap();
510
+ }
511
+ #[test]
512
+ fn transitive_hashes() {
513
+ let root = std::env::temp_dir().join(format!("ds4-tree-{}", std::process::id()));
514
+ directory(&root).unwrap();
515
+ fs::write(root.join("raw"), "original").unwrap();
516
+ let mut first = Artifact::new(Example { value: 1 });
517
+ first
518
+ .inputs
519
+ .push(reference(&root.join("raw"), &root).unwrap());
520
+ save(&root.join("first.json"), &first).unwrap();
521
+ let mut second = Artifact::new(Example { value: 2 });
522
+ second
523
+ .inputs
524
+ .push(reference(&root.join("first.json"), &root).unwrap());
525
+ save(&root.join("second.json"), &second).unwrap();
526
+ fs::write(root.join("raw"), "altered").unwrap();
527
+ let rejected = load::<Example>(&root.join("second.json")).is_err();
528
+ fs::remove_dir_all(root).unwrap();
529
+ assert!(rejected, "nested evidence tampering must fail");
530
+ }
531
+
532
+ #[test]
533
+ fn moved_evidence_and_tampering() {
534
+ let root = std::env::temp_dir().join(format!("ds4-evidence-{}", std::process::id()));
535
+ let source = root.join("source");
536
+ directory(&source).unwrap();
537
+ fs::write(source.join("raw"), "original").unwrap();
538
+ let mut artifact = Artifact::new(Example { value: 7 });
539
+ artifact
540
+ .inputs
541
+ .push(reference(&source.join("raw"), &source).unwrap());
542
+ save(&source.join("example.json"), &artifact).unwrap();
543
+ assert!(save(
544
+ &source.join("example.json"),
545
+ &Artifact::new(Example { value: 99 })
546
+ )
547
+ .is_err());
548
+ let moved = root.join("moved");
549
+ fs::rename(&source, &moved).unwrap();
550
+ assert_eq!(
551
+ load::<Example>(&moved.join("example.json"))
552
+ .unwrap()
553
+ .require()
554
+ .unwrap()
555
+ .value,
556
+ 7
557
+ );
558
+ fs::write(moved.join("raw"), "changed").unwrap();
559
+ assert!(load::<Example>(&moved.join("example.json"))
560
+ .unwrap_err()
561
+ .contains("evidence changed"));
562
+ fs::remove_dir_all(root).unwrap();
563
+ }
564
+ }
crates/ds4-perf/src/bench.rs ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use crate::csv;
2
+ use std::io::BufRead;
3
+
4
+ pub const HEADER: &str =
5
+ "ctx_tokens,prefill_tokens,prefill_tps,gen_tokens,gen_tps,first_token_sec,kvcache_bytes";
6
+
7
+ #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
8
+ #[serde(deny_unknown_fields)]
9
+ pub struct Row {
10
+ pub ctx: u64,
11
+ pub prefill_tokens: u64,
12
+ pub prefill_tps: f64,
13
+ pub gen_tokens: u64,
14
+ pub gen_tps: f64,
15
+ pub first_token_sec: f64,
16
+ pub kvcache_bytes: u64,
17
+ }
18
+
19
+ impl Row {
20
+ pub fn csv(&self) -> String {
21
+ format!(
22
+ "{},{},{},{},{},{},{}\n",
23
+ self.ctx,
24
+ self.prefill_tokens,
25
+ self.prefill_tps,
26
+ self.gen_tokens,
27
+ self.gen_tps,
28
+ self.first_token_sec,
29
+ self.kvcache_bytes
30
+ )
31
+ }
32
+ }
33
+
34
+ pub fn parse(reader: impl BufRead) -> Result<Vec<Row>, String> {
35
+ let mut header = Vec::new();
36
+ let mut rows = Vec::new();
37
+ csv::records(reader, |row| {
38
+ if csv::column(&row, &["prefill_tps"]).is_some() {
39
+ header = row;
40
+ return Ok(());
41
+ }
42
+ if header.is_empty() || row.iter().all(|v| v.trim().is_empty()) {
43
+ return Ok(());
44
+ }
45
+ let field = |name| {
46
+ csv::value(&row, &header, &[name])
47
+ .ok_or_else(|| format!("missing ds4-bench column: {name}"))
48
+ };
49
+ let integer = |name| {
50
+ field(name)?
51
+ .parse::<u64>()
52
+ .map_err(|_| format!("invalid {name}"))
53
+ };
54
+ rows.push(Row {
55
+ ctx: integer("ctx_tokens")?,
56
+ prefill_tokens: integer("prefill_tokens")?,
57
+ prefill_tps: csv::number(field("prefill_tps")?)?,
58
+ gen_tokens: integer("gen_tokens")?,
59
+ gen_tps: csv::number(field("gen_tps")?)?,
60
+ first_token_sec: csv::number(field("first_token_sec")?)?,
61
+ kvcache_bytes: integer("kvcache_bytes")?,
62
+ });
63
+ Ok(())
64
+ })?;
65
+ Ok(rows)
66
+ }
67
+
68
+ #[cfg(test)]
69
+ mod tests {
70
+ use super::*;
71
+
72
+ #[test]
73
+ fn bench_named_columns() {
74
+ let input = include_str!("../tests/fixtures/bench.csv");
75
+ let rows = parse(input.as_bytes()).unwrap();
76
+ assert_eq!(rows.len(), 1);
77
+ assert_eq!(rows[0].prefill_tps, 1502.8);
78
+ assert_eq!(rows[0].gen_tps, 24.3);
79
+ assert_eq!(rows[0].first_token_sec, 0.0512);
80
+ assert_eq!(rows[0].kvcache_bytes, 4294967296);
81
+ assert_eq!(rows[0].ctx, 8192);
82
+ assert!(parse(b"opaque output\n".as_slice()).unwrap().is_empty());
83
+ }
84
+ }
crates/ds4-perf/src/cli.rs ADDED
@@ -0,0 +1,214 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use clap::{Args, Parser, Subcommand, ValueEnum};
2
+ use std::{ffi::OsString, path::PathBuf};
3
+
4
+ #[derive(Parser, Debug)]
5
+ #[command(
6
+ version,
7
+ about = "Reproducible CUDA performance evidence and experiments"
8
+ )]
9
+ pub struct Cli {
10
+ #[command(subcommand)]
11
+ pub command: Command,
12
+ }
13
+
14
+ #[derive(Subcommand, Debug)]
15
+ pub enum Command {
16
+ /// Inspect machine capabilities; optionally measure the device envelope.
17
+ Inspect(Inspect),
18
+ /// Compatibility entry point for the v0.1.0 capability report.
19
+ #[command(hide = true)]
20
+ Doctor {
21
+ #[arg(long, default_value = "./ds4-bench")]
22
+ bench: String,
23
+ },
24
+ /// Profile a fresh benchmark and identify execution-path bottlenecks.
25
+ Scout(Scout),
26
+ /// Compare matched unprofiled experiments and correctness evidence.
27
+ Compare(Compare),
28
+ /// Choose experiments from evidence; --auto executes a bounded loop.
29
+ Optimize(Optimize),
30
+ /// Measure ordered HTTP conversation workloads and check their contracts.
31
+ Serving(Serving),
32
+ /// Propose one-control experiments from the server's capability report.
33
+ ServingControls(ServingControls),
34
+ /// Select a profile only from complete, repeated serving contracts.
35
+ ServingProfile(ServingProfile),
36
+ /// Check or execute a pinned serving profile; additional options are forbidden.
37
+ ApplyProfile(ApplyProfile),
38
+ }
39
+
40
+ #[derive(Args, Debug)]
41
+ pub struct ServingControls {
42
+ #[arg(long)]
43
+ pub plan: PathBuf,
44
+ #[arg(long)]
45
+ pub out: PathBuf,
46
+ }
47
+
48
+ #[derive(Args, Debug)]
49
+ pub struct ServingProfile {
50
+ #[arg(long)]
51
+ pub plan: PathBuf,
52
+ #[arg(long)]
53
+ pub out: PathBuf,
54
+ }
55
+
56
+ #[derive(Args, Debug)]
57
+ pub struct ApplyProfile {
58
+ #[arg(long)]
59
+ pub profile: PathBuf,
60
+ #[arg(long)]
61
+ pub out: PathBuf,
62
+ #[arg(long)]
63
+ pub execute: bool,
64
+ }
65
+
66
+ #[derive(Args, Debug)]
67
+ pub struct Serving {
68
+ #[command(flatten)]
69
+ pub budget: Budget,
70
+ /// Local server origin, e.g. http://127.0.0.1:8002.
71
+ #[arg(long)]
72
+ pub url: String,
73
+ #[arg(long)]
74
+ pub workload: PathBuf,
75
+ #[arg(long)]
76
+ pub out: PathBuf,
77
+ /// Replay the ordered workload; KV is deliberately preserved between repeats.
78
+ #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..=100))]
79
+ pub repeats: u32,
80
+ /// Pin the local server executable, start time and command line.
81
+ #[arg(long)]
82
+ pub server_pid: Option<u32>,
83
+ /// Physical NVIDIA device ordinal used for clock/temperature snapshots.
84
+ #[arg(long, default_value_t = 0)]
85
+ pub device: usize,
86
+ }
87
+
88
+ #[derive(Args, Debug)]
89
+ pub struct Inspect {
90
+ #[arg(long)]
91
+ pub out: PathBuf,
92
+ #[arg(long)]
93
+ pub calibrate: bool,
94
+ #[arg(long, default_value_t = 0)]
95
+ pub device: usize,
96
+ #[arg(long)]
97
+ pub gpu_helper: Option<PathBuf>,
98
+ #[arg(long)]
99
+ pub bench: Option<String>,
100
+ }
101
+
102
+ #[derive(Clone, Copy, Debug, ValueEnum)]
103
+ pub enum Collector {
104
+ Nsys,
105
+ Cupti,
106
+ }
107
+
108
+ #[derive(Args, Clone, Debug)]
109
+ pub struct Scout {
110
+ #[arg(skip)]
111
+ pub exact_environment: bool,
112
+ #[command(flatten)]
113
+ pub budget: Budget,
114
+ #[arg(long)]
115
+ pub out: PathBuf,
116
+ #[arg(long, value_enum, default_value = "nsys")]
117
+ pub collector: Collector,
118
+ #[arg(long)]
119
+ pub fit: bool,
120
+ /// Request ds4-bench token and full-vocabulary frontier proof files.
121
+ #[arg(long)]
122
+ pub proof: bool,
123
+ /// CUPTI injection library (default: beside ds4-perf-gpu).
124
+ #[arg(long, requires = "collector")]
125
+ pub cupti_library: Option<PathBuf>,
126
+ /// NVIDIA libcupti.so used for NVTX injection and activity collection.
127
+ #[arg(long)]
128
+ pub cupti_sdk: Option<PathBuf>,
129
+ #[arg(long)]
130
+ pub ncu: bool,
131
+ #[arg(long)]
132
+ pub machine: Option<PathBuf>,
133
+ #[arg(long, requires = "fit")]
134
+ pub calibration: Option<PathBuf>,
135
+ #[arg(long)]
136
+ pub gpu_helper: Option<PathBuf>,
137
+ #[arg(long, default_value_t = 0)]
138
+ pub device: usize,
139
+ #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..=100))]
140
+ pub repeats: u32,
141
+ /// Explicit workload/cache contract used when comparing runs.
142
+ #[arg(long)]
143
+ pub workload: Option<PathBuf>,
144
+ #[arg(long, default_value = "inherited", value_parser = ["inherited", "warmup-then-fresh"])]
145
+ pub cache_policy: String,
146
+ /// Allowlisted runtime control, passed directly to the child process.
147
+ #[arg(long = "env")]
148
+ pub environment: Vec<String>,
149
+ #[arg(last = true, required = true)]
150
+ pub command: Vec<OsString>,
151
+ }
152
+
153
+ #[derive(Args, Debug)]
154
+ pub struct Compare {
155
+ #[arg(long)]
156
+ pub baseline: PathBuf,
157
+ #[arg(long)]
158
+ pub candidate: PathBuf,
159
+ #[arg(long)]
160
+ pub out: PathBuf,
161
+ /// Exit unsuccessfully on regression or insufficient comparison evidence.
162
+ #[arg(long)]
163
+ pub regression: bool,
164
+ #[arg(long, default_value_t = 3.0)]
165
+ pub max_slowdown_percent: f64,
166
+ #[arg(long, default_value_t = 0.0001)]
167
+ pub logit_atol: f64,
168
+ #[arg(long, default_value_t = 0.0001)]
169
+ pub logit_rtol: f64,
170
+ /// Relaxed contract for summation-order changes: when positive, each
171
+ /// frontier must stay within this relative RMS of the baseline at 1024
172
+ /// tokens (the bound grows with log2 of the context) and keep its argmax;
173
+ /// greedy sequences may diverge. Zero keeps the exact contract.
174
+ #[arg(long, default_value_t = 0.0)]
175
+ pub logit_rel_rms: f64,
176
+ }
177
+
178
+ #[derive(Args, Debug)]
179
+ pub struct Optimize {
180
+ #[command(flatten)]
181
+ pub budget: Budget,
182
+ #[arg(long)]
183
+ pub scout: PathBuf,
184
+ #[arg(long)]
185
+ pub out: PathBuf,
186
+ /// Run evidence-selected experiments and compare each candidate.
187
+ #[arg(long = "auto")]
188
+ pub automatic: bool,
189
+ #[arg(long)]
190
+ pub plan: Option<PathBuf>,
191
+ #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(u32).range(1..=32))]
192
+ pub rounds: u32,
193
+ #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(u32).range(3..=100))]
194
+ pub repeats: u32,
195
+ }
196
+
197
+ #[derive(Args, Clone, Debug, serde::Serialize, serde::Deserialize)]
198
+ #[serde(deny_unknown_fields)]
199
+ pub struct Budget {
200
+ /// Deadline for each benchmark/profiler process, including descendants.
201
+ #[arg(long,default_value_t=1800,value_parser=clap::value_parser!(u64).range(1..=86400))]
202
+ pub timeout_seconds: u64,
203
+ /// Total raw output budget per scout; auto also bounds the campaign.
204
+ #[arg(long,default_value_t=2048,value_parser=clap::value_parser!(u64).range(1..=65536))]
205
+ pub max_output_mib: u64,
206
+ }
207
+ impl Budget {
208
+ pub fn limits(&self) -> crate::process::Limits {
209
+ crate::process::Limits {
210
+ timeout: std::time::Duration::from_secs(self.timeout_seconds),
211
+ bytes: self.max_output_mib.saturating_mul(1024 * 1024),
212
+ }
213
+ }
214
+ }
crates/ds4-perf/src/compare.rs ADDED
@@ -0,0 +1,695 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use crate::{
2
+ cli,
3
+ experiment::{self, Scout},
4
+ fit,
5
+ };
6
+ use ds4_perf::artifact::{self, Artifact, Payload, Reference};
7
+ use serde::{Deserialize, Serialize};
8
+ use std::{collections::BTreeMap, fs, path::Path};
9
+
10
+ #[derive(Debug, Serialize, Deserialize)]
11
+ #[serde(deny_unknown_fields)]
12
+ pub struct Delta {
13
+ pub metric: String,
14
+ pub ctx: u64,
15
+ pub unit: String,
16
+ pub baseline: Vec<f64>,
17
+ pub candidate: Vec<f64>,
18
+ pub baseline_median: f64,
19
+ pub candidate_median: f64,
20
+ pub median_percent: f64,
21
+ pub lower_percent: f64,
22
+ pub upper_percent: f64,
23
+ }
24
+
25
+ #[derive(Debug, Default, Serialize, Deserialize)]
26
+ #[serde(deny_unknown_fields)]
27
+ pub struct LogitError {
28
+ pub max_abs: f64,
29
+ pub max_scaled: f64,
30
+ pub bad: u64,
31
+ pub checked: u64,
32
+ /// Largest relative RMS of a candidate frontier against its reference
33
+ /// (sqrt(sum diff^2 / sum ref^2)); the relaxed contract's statistic.
34
+ #[serde(default)]
35
+ pub rel_rms: f64,
36
+ /// Largest `rel_rms / rel_rms_bound(ctx)`; above 1 the relaxed contract
37
+ /// is violated.
38
+ #[serde(default)]
39
+ pub rel_rms_scaled: f64,
40
+ }
41
+
42
+ /// Reference context of the relaxed contract: `--logit-rel-rms` is the bound
43
+ /// at this width and grows with log2(ctx) from it.
44
+ const REL_RMS_REFERENCE_CTX: f64 = 1024.0;
45
+
46
+ /// Context-dependent relative RMS bound. Reordering noise accumulates with
47
+ /// the attended length but the model's amplification saturates, and a
48
+ /// one-ulp perturbation of Inkling MQ85GB measured 0.063 at 16 tokens and
49
+ /// 0.105 at 515 (log-linear), so the bound grows logarithmically: at 2K it
50
+ /// is 1.1x the reference, at 8K 1.3x, at 64K 1.6x, never linear in ctx.
51
+ fn rel_rms_bound(reference: f64, ctx: u64) -> f64 {
52
+ reference * (ctx.max(2) as f64).log2() / REL_RMS_REFERENCE_CTX.log2()
53
+ }
54
+
55
+ #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
56
+ #[serde(rename_all = "snake_case")]
57
+ pub enum Verdict {
58
+ Pass,
59
+ Improved,
60
+ Regressed,
61
+ Inconclusive,
62
+ Incorrect,
63
+ Incomparable,
64
+ }
65
+
66
+ #[derive(Debug, Serialize, Deserialize)]
67
+ #[serde(deny_unknown_fields)]
68
+ pub struct Comparison {
69
+ pub verdict: Verdict,
70
+ pub metrics: Vec<Delta>,
71
+ pub correctness: LogitError,
72
+ /// Number of proof comparisons with a greedy sequence or argmax mismatch.
73
+ pub token_mismatches: u64,
74
+ /// Number of proof comparisons whose frontier argmax differs (a subset of
75
+ /// `token_mismatches`).
76
+ #[serde(default)]
77
+ pub argmax_mismatches: u64,
78
+ pub max_slowdown_percent: f64,
79
+ pub logit_atol: f64,
80
+ pub logit_rtol: f64,
81
+ /// Relaxed contract when positive: a frontier is correct when its relative
82
+ /// RMS stays within `rel_rms_bound(ctx)` (this value at 1024 tokens,
83
+ /// growing with log2(ctx)) and its argmax matches; greedy sequences may
84
+ /// diverge. Zero keeps the per-logit atol/rtol and sequence contract.
85
+ #[serde(default)]
86
+ pub logit_rel_rms: f64,
87
+ pub changes: BTreeMap<String, String>,
88
+ pub reasons: Vec<String>,
89
+ pub method: String,
90
+ }
91
+
92
+ impl Payload for Comparison {
93
+ const KIND: &'static str = "compare";
94
+ fn validate(&self) -> Result<(), String> {
95
+ if self.method != "sample-extrema-envelope-v1" {
96
+ return Err("unsupported comparison method".into());
97
+ }
98
+ if [
99
+ self.max_slowdown_percent,
100
+ self.logit_atol,
101
+ self.logit_rtol,
102
+ self.logit_rel_rms,
103
+ self.correctness.max_abs,
104
+ self.correctness.max_scaled,
105
+ self.correctness.rel_rms,
106
+ self.correctness.rel_rms_scaled,
107
+ ]
108
+ .into_iter()
109
+ .any(|v| !v.is_finite() || v < 0.0)
110
+ {
111
+ return Err("invalid comparison policy or result".into());
112
+ }
113
+ if self.verdict == Verdict::Incomparable {
114
+ if self.reasons.is_empty() || !self.metrics.is_empty() {
115
+ return Err("incomparable result lacks a reason".into());
116
+ }
117
+ return Ok(());
118
+ }
119
+ if self.metrics.is_empty()
120
+ || self.correctness.checked == 0
121
+ || self.correctness.bad > self.correctness.checked
122
+ || self.argmax_mismatches > self.token_mismatches
123
+ {
124
+ return Err("conclusive comparison lacks timing/correctness evidence".into());
125
+ }
126
+ let expected = verdict(
127
+ &self.metrics,
128
+ &self.correctness,
129
+ self.token_mismatches,
130
+ self.argmax_mismatches,
131
+ self.max_slowdown_percent,
132
+ self.logit_rel_rms,
133
+ );
134
+ if self.verdict != expected {
135
+ return Err("comparison verdict disagrees with evidence".into());
136
+ }
137
+ for d in &self.metrics {
138
+ if [
139
+ d.baseline_median,
140
+ d.candidate_median,
141
+ d.median_percent,
142
+ d.lower_percent,
143
+ d.upper_percent,
144
+ ]
145
+ .into_iter()
146
+ .any(|v| !v.is_finite())
147
+ || d.baseline
148
+ .iter()
149
+ .chain(&d.candidate)
150
+ .any(|v| !v.is_finite() || *v <= 0.0)
151
+ {
152
+ return Err("invalid comparison samples".into());
153
+ }
154
+ let computed = delta(&d.metric, d.ctx, &d.baseline, &d.candidate)?;
155
+ if d.baseline.len() < 3
156
+ || d.baseline.len() != d.candidate.len()
157
+ || d.unit != computed.unit
158
+ || [
159
+ d.baseline_median,
160
+ d.candidate_median,
161
+ d.median_percent,
162
+ d.lower_percent,
163
+ d.upper_percent,
164
+ ] != [
165
+ computed.baseline_median,
166
+ computed.candidate_median,
167
+ computed.median_percent,
168
+ computed.lower_percent,
169
+ computed.upper_percent,
170
+ ]
171
+ {
172
+ return Err("comparison summary disagrees with raw samples".into());
173
+ }
174
+ }
175
+ Ok(())
176
+ }
177
+ }
178
+
179
+ fn delta(metric: &str, ctx: u64, a: &[f64], b: &[f64]) -> Result<Delta, String> {
180
+ if a.is_empty() || b.is_empty() || a.iter().chain(b).any(|v| !v.is_finite() || *v <= 0.0) {
181
+ return Err("nonpositive or missing timing samples".into());
182
+ }
183
+ let baseline_median = fit::median(a);
184
+ let candidate_median = fit::median(b);
185
+ let extrema = |v: &[f64]| {
186
+ (
187
+ v.iter().copied().fold(f64::INFINITY, f64::min),
188
+ v.iter().copied().fold(0.0, f64::max),
189
+ )
190
+ };
191
+ let (amin, amax) = extrema(a);
192
+ let (bmin, bmax) = extrema(b);
193
+ Ok(Delta {
194
+ metric: metric.into(),
195
+ ctx,
196
+ unit: if metric == "first_token" {
197
+ "seconds"
198
+ } else {
199
+ "seconds/token"
200
+ }
201
+ .into(),
202
+ baseline: a.into(),
203
+ candidate: b.into(),
204
+ baseline_median,
205
+ candidate_median,
206
+ median_percent: 100.0 * (candidate_median / baseline_median - 1.0),
207
+ lower_percent: 100.0 * (bmin / amax - 1.0),
208
+ upper_percent: 100.0 * (bmax / amin - 1.0),
209
+ })
210
+ }
211
+
212
+ fn logit_error(a: &[f64], b: &[f64], atol: f64, rtol: f64) -> Result<LogitError, String> {
213
+ if a.is_empty() || a.len() != b.len() || a.iter().chain(b).any(|v| !v.is_finite()) {
214
+ return Err("invalid full-vocabulary logits".into());
215
+ }
216
+ let mut error = LogitError::default();
217
+ let (mut diff2, mut ref2) = (0.0f64, 0.0f64);
218
+ for (a, b) in a.iter().zip(b) {
219
+ let difference = (a - b).abs();
220
+ let tolerance = atol + rtol * a.abs();
221
+ error.max_abs = error.max_abs.max(difference);
222
+ // A zero tolerance is valid for an exact parity contract.
223
+ error.max_scaled = error
224
+ .max_scaled
225
+ .max((difference / tolerance.max(f64::MIN_POSITIVE)).min(f64::MAX));
226
+ error.bad += u64::from(difference > tolerance);
227
+ error.checked += 1;
228
+ diff2 += difference * difference;
229
+ ref2 += a * a;
230
+ }
231
+ error.rel_rms = (diff2 / ref2.max(f64::MIN_POSITIVE)).sqrt().min(f64::MAX);
232
+ Ok(error)
233
+ }
234
+
235
+ #[derive(Deserialize)]
236
+ #[serde(deny_unknown_fields)]
237
+ struct Frontier {
238
+ source: String,
239
+ model: String,
240
+ backend: String,
241
+ quality: bool,
242
+ quant_bits: u64,
243
+ prompt_tokens: u64,
244
+ frontier_tokens: u64,
245
+ prefill_tokens: u64,
246
+ ctx: u64,
247
+ vocab: usize,
248
+ argmax_id: usize,
249
+ argmax_logit: f64,
250
+ logits: Vec<f64>,
251
+ }
252
+
253
+ fn proof_file<T: serde::de::DeserializeOwned>(
254
+ root: &Path,
255
+ reference: &Reference,
256
+ ) -> Result<T, String> {
257
+ let path = root.join(&reference.path);
258
+ if artifact::hash(&path)? != reference.sha256 {
259
+ return Err("correctness evidence changed".into());
260
+ }
261
+ let file = fs::File::open(&path).map_err(|e| e.to_string())?;
262
+ if file.metadata().map_err(|e| e.to_string())?.len() > 64 * 1024 * 1024 {
263
+ return Err("proof file exceeds 64 MiB".into());
264
+ }
265
+ serde_json::from_reader(file).map_err(|e| format!("{}: {e}", path.display()))
266
+ }
267
+
268
+ type FrontierProofs = BTreeMap<u64, Vec<(Frontier, Vec<i32>)>>;
269
+
270
+ fn proofs(root: &Path, scout: &Scout) -> Result<FrontierProofs, String> {
271
+ let mut proofs: BTreeMap<_, Vec<_>> = BTreeMap::new();
272
+ for sample in &scout.samples {
273
+ for row in &sample.rows {
274
+ let name = format!("frontier_{:06}.logits.json", row.ctx);
275
+ let frontier: Frontier = proof_file(
276
+ root,
277
+ sample
278
+ .proof_files
279
+ .get(&name)
280
+ .ok_or("missing full-vocabulary frontier; use scout --proof")?,
281
+ )?;
282
+ let name = format!("tokens-{}.json", row.ctx);
283
+ let tokens: Vec<i32> = proof_file(
284
+ root,
285
+ sample
286
+ .proof_files
287
+ .get(&name)
288
+ .ok_or("missing greedy-token evidence; use scout --proof")?,
289
+ )?;
290
+ if frontier.source != "ds4-bench"
291
+ || frontier.backend != "cuda"
292
+ || frontier.model.is_empty()
293
+ || frontier.vocab == 0
294
+ || frontier.logits.len() != frontier.vocab
295
+ || frontier.argmax_id >= frontier.vocab
296
+ || frontier.frontier_tokens != row.ctx
297
+ || frontier.prompt_tokens != row.ctx
298
+ || frontier.prefill_tokens != row.prefill_tokens
299
+ || frontier.ctx <= row.ctx
300
+ || tokens.len() as u64 != row.gen_tokens
301
+ || tokens
302
+ .iter()
303
+ .any(|t| *t < 0 || *t as usize >= frontier.vocab)
304
+ || frontier.logits.iter().any(|v| !v.is_finite())
305
+ || !frontier.argmax_logit.is_finite()
306
+ || frontier.logits[frontier.argmax_id] != frontier.argmax_logit
307
+ || frontier.logits.iter().any(|v| *v > frontier.argmax_logit)
308
+ {
309
+ return Err("frontier/token proof disagrees with benchmark row".into());
310
+ }
311
+ if scout.workload.is_some()
312
+ && scout.model_argument.as_deref() != Some(Path::new(&frontier.model))
313
+ {
314
+ return Err(
315
+ "proof model differs from the captured and hashed workload argument".into(),
316
+ );
317
+ }
318
+ proofs.entry(row.ctx).or_default().push((frontier, tokens));
319
+ }
320
+ }
321
+ Ok(proofs)
322
+ }
323
+
324
+ pub fn validate_proofs(root: &Path, scout: &Scout) -> Result<(), String> {
325
+ if scout.samples.len() != scout.requested_repeats as usize
326
+ || scout.samples.iter().any(|s| s.rows.is_empty())
327
+ {
328
+ return Err("missing samples for requested proof".into());
329
+ }
330
+ proofs(root, scout).map(|_| ())
331
+ }
332
+
333
+ fn identity(a: &Scout, b: &Scout) -> Result<BTreeMap<String, String>, String> {
334
+ if !a.process_scope.verified
335
+ || !b.process_scope.verified
336
+ || a.process_scope.expected_owners != b.process_scope.expected_owners
337
+ {
338
+ return Err("GPU process snapshots do not match the intended owners for both runs".into());
339
+ }
340
+ if !a.unreviewed_environment.is_empty() || !b.unreviewed_environment.is_empty() {
341
+ return Err("unreviewed DS4/CUDA/LD environment controls prevent matched comparison; see scout.json".into());
342
+ }
343
+ let (Some(ga), Some(gb)) = (&a.gpu, &b.gpu) else {
344
+ return Err("comparison requires verified CUDA device identity".into());
345
+ };
346
+ if !ga.same_device(gb) {
347
+ return Err("device or driver differs".into());
348
+ }
349
+ if a.command[1..] != b.command[1..] || a.cwd != b.cwd {
350
+ return Err("benchmark arguments or working directory differ".into());
351
+ }
352
+ if a.cache_policy != "warmup-then-fresh" || a.cache_policy != b.cache_policy {
353
+ return Err("comparison requires matching explicit --cache-policy".into());
354
+ }
355
+ let (Some(wa), Some(wb)) = (&a.workload, &b.workload) else {
356
+ return Err(
357
+ "comparison requires --workload with hashed model/prompt and cache contract".into(),
358
+ );
359
+ };
360
+ let hashes = |w: &experiment::Workload| {
361
+ w.files
362
+ .iter()
363
+ .map(|(k, v)| (k.clone(), v.sha256.clone()))
364
+ .collect::<BTreeMap<_, _>>()
365
+ };
366
+ if wa.name != wb.name
367
+ || wa.family != wb.family
368
+ || hashes(wa) != hashes(wb)
369
+ || wa.shape != wb.shape
370
+ || wa.cache_state != wb.cache_state
371
+ {
372
+ return Err("workload model/prompt/shape/cache identity differs".into());
373
+ }
374
+ let mut changes = BTreeMap::new();
375
+ if a.executable_sha256 != b.executable_sha256 {
376
+ changes.insert(
377
+ "executable_sha256".into(),
378
+ format!("{} -> {}", a.executable_sha256, b.executable_sha256),
379
+ );
380
+ }
381
+ let keys: std::collections::BTreeSet<_> =
382
+ a.environment.keys().chain(b.environment.keys()).collect();
383
+ for key in keys {
384
+ if a.environment.get(key) == b.environment.get(key) {
385
+ continue;
386
+ }
387
+ if !crate::knobs::tunable(key) {
388
+ return Err(format!("non-tuning runtime/memory policy changed: {key}"));
389
+ }
390
+ for value in [a.environment.get(key), b.environment.get(key)]
391
+ .into_iter()
392
+ .flatten()
393
+ {
394
+ crate::knobs::validate(key, value, &wa.family)?;
395
+ }
396
+ changes.insert(
397
+ key.clone(),
398
+ format!(
399
+ "{:?} -> {:?}",
400
+ a.environment.get(key),
401
+ b.environment.get(key)
402
+ ),
403
+ );
404
+ }
405
+ Ok(changes)
406
+ }
407
+
408
+ fn analyze(
409
+ a: &Scout,
410
+ ar: &Path,
411
+ b: &Scout,
412
+ br: &Path,
413
+ args: &cli::Compare,
414
+ ) -> Result<Comparison, String> {
415
+ let changes = identity(a, b)?;
416
+ if a.requested_repeats != b.requested_repeats
417
+ || a.samples.len() != a.requested_repeats as usize
418
+ || b.samples.len() != b.requested_repeats as usize
419
+ {
420
+ return Err("comparison requires equal complete requested sample counts".into());
421
+ }
422
+ if a.samples.len() < 3 || b.samples.len() < 3 {
423
+ return Err("comparison requires at least three fresh unprofiled samples per side".into());
424
+ }
425
+ type TimingPair = (Vec<f64>, Vec<f64>);
426
+ let mut timing: BTreeMap<(String, u64), TimingPair> = BTreeMap::new();
427
+ let shape = |s: &experiment::Sample| {
428
+ s.rows
429
+ .iter()
430
+ .map(|r| (r.ctx, r.prefill_tokens, r.gen_tokens, r.kvcache_bytes))
431
+ .collect::<Vec<_>>()
432
+ };
433
+ let expected = shape(&a.samples[0]);
434
+ if expected.is_empty()
435
+ || expected
436
+ .iter()
437
+ .map(|r| r.0)
438
+ .collect::<std::collections::BTreeSet<_>>()
439
+ .len()
440
+ != expected.len()
441
+ {
442
+ return Err("empty or duplicate benchmark frontiers".into());
443
+ }
444
+ for (candidate, scout) in [(false, a), (true, b)] {
445
+ for sample in &scout.samples {
446
+ if shape(sample) != expected {
447
+ return Err("token counts/frontiers/KV allocation differ across samples".into());
448
+ }
449
+ for row in &sample.rows {
450
+ for (name, value) in [
451
+ ("prefill", 1.0 / row.prefill_tps),
452
+ ("decode", 1.0 / row.gen_tps),
453
+ ("first_token", row.first_token_sec),
454
+ ] {
455
+ let entry = timing.entry((name.into(), row.ctx)).or_default();
456
+ if candidate {
457
+ entry.1.push(value);
458
+ } else {
459
+ entry.0.push(value);
460
+ }
461
+ }
462
+ }
463
+ }
464
+ }
465
+ let baseline = proofs(ar, a)?;
466
+ let candidate = proofs(br, b)?;
467
+ let mut correctness = LogitError::default();
468
+ let mut token_mismatches = 0;
469
+ let mut argmax_mismatches = 0;
470
+ for (ctx, base) in &baseline {
471
+ let (reference, tokens) = &base[0];
472
+ let candidates = candidate.get(ctx).ok_or("missing candidate frontier")?;
473
+ for (frontier, ids) in base.iter().chain(candidates) {
474
+ if frontier.source != reference.source
475
+ || frontier.backend != reference.backend
476
+ || frontier.model != reference.model
477
+ || frontier.quality != reference.quality
478
+ || frontier.quant_bits != reference.quant_bits
479
+ || frontier.ctx != reference.ctx
480
+ {
481
+ return Err("quality/quantization/context proof metadata differs".into());
482
+ }
483
+ token_mismatches +=
484
+ u64::from(ids != tokens || frontier.argmax_id != reference.argmax_id);
485
+ argmax_mismatches += u64::from(frontier.argmax_id != reference.argmax_id);
486
+ let error = logit_error(
487
+ &reference.logits,
488
+ &frontier.logits,
489
+ args.logit_atol,
490
+ args.logit_rtol,
491
+ )?;
492
+ correctness.max_abs = correctness.max_abs.max(error.max_abs);
493
+ correctness.max_scaled = correctness.max_scaled.max(error.max_scaled);
494
+ correctness.bad += error.bad;
495
+ correctness.checked += error.checked;
496
+ correctness.rel_rms = correctness.rel_rms.max(error.rel_rms);
497
+ if args.logit_rel_rms > 0.0 {
498
+ let scaled = error.rel_rms / rel_rms_bound(args.logit_rel_rms, *ctx);
499
+ correctness.rel_rms_scaled = correctness.rel_rms_scaled.max(scaled.min(f64::MAX));
500
+ }
501
+ }
502
+ }
503
+ let metrics = timing
504
+ .into_iter()
505
+ .map(|((name, ctx), (a, b))| delta(&name, ctx, &a, &b))
506
+ .collect::<Result<Vec<_>, _>>()?;
507
+ let verdict = verdict(
508
+ &metrics,
509
+ &correctness,
510
+ token_mismatches,
511
+ argmax_mismatches,
512
+ args.max_slowdown_percent,
513
+ args.logit_rel_rms,
514
+ );
515
+ Ok(Comparison {
516
+ verdict,
517
+ metrics,
518
+ correctness,
519
+ token_mismatches,
520
+ argmax_mismatches,
521
+ max_slowdown_percent: args.max_slowdown_percent,
522
+ logit_atol: args.logit_atol,
523
+ logit_rtol: args.logit_rtol,
524
+ logit_rel_rms: args.logit_rel_rms,
525
+ changes,
526
+ reasons: Vec::new(),
527
+ method: "sample-extrema-envelope-v1".into(),
528
+ })
529
+ }
530
+
531
+ fn verdict(
532
+ metrics: &[Delta],
533
+ correctness: &LogitError,
534
+ mismatches: u64,
535
+ argmax_mismatches: u64,
536
+ tolerance: f64,
537
+ logit_rel_rms: f64,
538
+ ) -> Verdict {
539
+ // The relaxed contract accepts summation-order changes that a chaotic
540
+ // model amplifies into different greedy continuations: it bounds the
541
+ // frontier's relative RMS per context and keeps its argmax instead of
542
+ // every logit and the whole sequence.
543
+ let incorrect = if logit_rel_rms > 0.0 {
544
+ correctness.rel_rms_scaled > 1.0 || argmax_mismatches > 0
545
+ } else {
546
+ correctness.bad > 0 || mismatches > 0
547
+ };
548
+ if incorrect {
549
+ return Verdict::Incorrect;
550
+ }
551
+ if metrics.iter().any(|d| d.lower_percent > tolerance) {
552
+ return Verdict::Regressed;
553
+ }
554
+ if metrics.iter().any(|d| d.upper_percent > tolerance) {
555
+ return Verdict::Inconclusive;
556
+ }
557
+ if metrics
558
+ .iter()
559
+ .any(|d| d.metric != "first_token" && d.upper_percent < -1.0)
560
+ {
561
+ return Verdict::Improved;
562
+ }
563
+ Verdict::Pass
564
+ }
565
+
566
+ pub fn run(args: &cli::Compare) -> Result<(), String> {
567
+ if [
568
+ args.max_slowdown_percent,
569
+ args.logit_atol,
570
+ args.logit_rtol,
571
+ args.logit_rel_rms,
572
+ ]
573
+ .into_iter()
574
+ .any(|v| !v.is_finite() || v < 0.0)
575
+ {
576
+ return Err("comparison tolerances must be finite and nonnegative".into());
577
+ }
578
+ artifact::directory(&args.out)?;
579
+ let result = (|| {
580
+ let a = experiment::load(&args.baseline)?;
581
+ let b = experiment::load(&args.candidate)?;
582
+ analyze(
583
+ a.require()?,
584
+ args.baseline.parent().unwrap_or(Path::new(".")),
585
+ b.require()?,
586
+ args.candidate.parent().unwrap_or(Path::new(".")),
587
+ args,
588
+ )
589
+ })();
590
+ let data = match result {
591
+ Ok(data) => data,
592
+ Err(error) => Comparison {
593
+ verdict: Verdict::Incomparable,
594
+ metrics: Vec::new(),
595
+ correctness: LogitError::default(),
596
+ token_mismatches: 0,
597
+ argmax_mismatches: 0,
598
+ max_slowdown_percent: args.max_slowdown_percent,
599
+ logit_atol: args.logit_atol,
600
+ logit_rtol: args.logit_rtol,
601
+ logit_rel_rms: args.logit_rel_rms,
602
+ changes: BTreeMap::new(),
603
+ reasons: vec![error],
604
+ method: "sample-extrema-envelope-v1".into(),
605
+ },
606
+ };
607
+ let passed = matches!(data.verdict, Verdict::Pass | Verdict::Improved);
608
+ let mut artifact = Artifact::new(data);
609
+ artifact.complete = artifact
610
+ .data
611
+ .as_ref()
612
+ .is_some_and(|d| d.verdict != Verdict::Incomparable);
613
+ for path in [&args.baseline, &args.candidate] {
614
+ if path.is_file() {
615
+ artifact.inputs.push(artifact::reference(path, &args.out)?);
616
+ }
617
+ }
618
+ artifact::save(&args.out.join("compare.json"), &artifact)?;
619
+ println!(
620
+ "compare: {:?}; {}",
621
+ artifact.data.as_ref().unwrap().verdict,
622
+ args.out.join("compare.json").display()
623
+ );
624
+ if args.regression && !passed {
625
+ return Err("regression gate did not pass; see compare.json".into());
626
+ }
627
+ Ok(())
628
+ }
629
+
630
+ #[cfg(test)]
631
+ mod tests {
632
+ use super::*;
633
+ #[test]
634
+ fn preserves_live_roundtrip() {
635
+ // These measured Qwen ratios exposed a lossy JSON parse on reload.
636
+ let recorded: Comparison =
637
+ serde_json::from_str(include_str!("../tests/fixtures/compare-live.json")).unwrap();
638
+ recorded.validate().unwrap();
639
+ let encoded = serde_json::to_vec(&recorded).unwrap();
640
+ let mut loaded: Comparison = serde_json::from_slice(&encoded).unwrap();
641
+ loaded.validate().unwrap();
642
+ loaded.metrics[0].median_percent += 0.01;
643
+ assert!(loaded.validate().is_err());
644
+ }
645
+
646
+ #[test]
647
+ fn separates_regression_noise() {
648
+ let stable = delta("prefill", 2048, &[1.0, 1.01, 0.99], &[1.1, 1.11, 1.09]).unwrap();
649
+ assert!(stable.lower_percent > 3.0);
650
+ let noise = delta("prefill", 2048, &[0.9, 1.0, 1.1], &[0.9, 1.03, 1.2]).unwrap();
651
+ assert!(noise.lower_percent < 3.0 && noise.upper_percent > 3.0);
652
+ assert!(delta("prefill", 2048, &[1.0], &[0.0]).is_err());
653
+ assert!(delta("prefill", 2048, &[1.0], &[f64::NAN]).is_err());
654
+ }
655
+ #[test]
656
+ fn checks_every_logit() {
657
+ let a = vec![0.0, 1.0, -2.0, 0.001];
658
+ assert!(logit_error(&a, &a, 1e-4, 1e-4).unwrap().bad == 0);
659
+ let mut b = a.clone();
660
+ b[3] = 0.01;
661
+ assert_eq!(logit_error(&a, &b, 1e-4, 1e-4).unwrap().bad, 1);
662
+ assert!(logit_error(&a, &[f64::NAN], 1e-4, 1e-4).is_err());
663
+ }
664
+ #[test]
665
+ fn relaxed_contract_bounds_rms_and_argmax() {
666
+ let a = vec![0.0, 3.0, -4.0];
667
+ let b = vec![0.3, 3.0, -4.0];
668
+ let mut error = logit_error(&a, &b, 1e-4, 1e-4).unwrap();
669
+ assert_eq!(error.bad, 1);
670
+ assert!((error.rel_rms - 0.06).abs() < 1e-12);
671
+ // The bound is the reference at 1024 tokens and grows with log2(ctx).
672
+ assert!((rel_rms_bound(0.1, 1024) - 0.1).abs() < 1e-12);
673
+ assert!((rel_rms_bound(0.1, 8192) - 0.13).abs() < 1e-12);
674
+ assert!(rel_rms_bound(0.1, 65536) < 0.17);
675
+ let stable = delta("prefill", 2048, &[1.0, 1.01, 0.99], &[0.9, 0.91, 0.89]).unwrap();
676
+ let metrics = [stable];
677
+ // Strict: any bad logit or sequence mismatch is incorrect.
678
+ assert_eq!(
679
+ verdict(&metrics, &error, 1, 0, 3.0, 0.0),
680
+ Verdict::Incorrect
681
+ );
682
+ // Relaxed: the sequence may diverge while the scaled RMS and argmax hold.
683
+ error.rel_rms_scaled = error.rel_rms / rel_rms_bound(0.1, 2048);
684
+ assert_eq!(verdict(&metrics, &error, 1, 0, 3.0, 0.1), Verdict::Improved);
685
+ assert_eq!(
686
+ verdict(&metrics, &error, 1, 1, 3.0, 0.1),
687
+ Verdict::Incorrect
688
+ );
689
+ error.rel_rms_scaled = error.rel_rms / rel_rms_bound(0.05, 2048);
690
+ assert_eq!(
691
+ verdict(&metrics, &error, 1, 0, 3.0, 0.05),
692
+ Verdict::Incorrect
693
+ );
694
+ }
695
+ }
crates/ds4-perf/src/csv.rs ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::io::{self, BufRead};
2
+
3
+ // Read records incrementally: GPU traces can contain millions of launches.
4
+ // Quotes, escaped quotes and embedded newlines are legal in Nsight names.
5
+ pub fn records(
6
+ reader: impl BufRead,
7
+ mut visit: impl FnMut(Vec<String>) -> Result<(), String>,
8
+ ) -> Result<(), String> {
9
+ let mut field = Vec::new();
10
+ let mut row = Vec::new();
11
+ let mut quoted = false;
12
+ let mut closed = false;
13
+ for byte in reader.bytes() {
14
+ let byte = byte.map_err(|e| e.to_string())?;
15
+ if quoted {
16
+ if byte == b'"' {
17
+ quoted = false;
18
+ closed = true;
19
+ } else {
20
+ field.push(byte);
21
+ }
22
+ continue;
23
+ }
24
+ match byte {
25
+ b'"' if closed => {
26
+ field.push(b'"');
27
+ quoted = true;
28
+ closed = false;
29
+ }
30
+ b'"' if field.is_empty() => quoted = true,
31
+ b',' | b'\n' => {
32
+ row.push(take_field(&mut field)?);
33
+ closed = false;
34
+ if byte == b'\n' {
35
+ visit(std::mem::take(&mut row))?;
36
+ }
37
+ }
38
+ b'\r' => {}
39
+ b' ' | b'\t' if closed => {}
40
+ _ if closed => return Err("characters after closing CSV quote".into()),
41
+ _ => field.push(byte),
42
+ }
43
+ }
44
+ if quoted {
45
+ return Err("unterminated CSV quote".into());
46
+ }
47
+ if !field.is_empty() || !row.is_empty() || closed {
48
+ row.push(take_field(&mut field)?);
49
+ visit(row)?;
50
+ }
51
+ Ok(())
52
+ }
53
+
54
+ fn take_field(bytes: &mut Vec<u8>) -> Result<String, String> {
55
+ String::from_utf8(std::mem::take(bytes)).map_err(|e| format!("CSV is not UTF-8: {e}"))
56
+ }
57
+
58
+ pub fn field(value: &str) -> String {
59
+ if value.contains([',', '"', '\r', '\n']) {
60
+ format!("\"{}\"", value.replace('"', "\"\""))
61
+ } else {
62
+ value.into()
63
+ }
64
+ }
65
+
66
+ pub fn key(value: &str) -> String {
67
+ value
68
+ .trim()
69
+ .trim_start_matches('\u{feff}')
70
+ .to_ascii_lowercase()
71
+ }
72
+
73
+ pub fn column(header: &[String], names: &[&str]) -> Option<usize> {
74
+ header.iter().position(|s| names.contains(&key(s).as_str()))
75
+ }
76
+
77
+ pub fn value<'a>(row: &'a [String], header: &[String], names: &[&str]) -> Option<&'a str> {
78
+ row.get(column(header, names)?)
79
+ .map(|s| s.trim())
80
+ .filter(|s| !s.is_empty())
81
+ }
82
+
83
+ pub fn number(value: &str) -> Result<f64, String> {
84
+ value
85
+ .trim()
86
+ .parse::<f64>()
87
+ .ok()
88
+ .filter(|v| v.is_finite() && *v >= 0.0)
89
+ .ok_or_else(|| format!("invalid nonnegative number: {value}"))
90
+ }
91
+
92
+ pub fn time(row: &[String], header: &[String], name: &str) -> Option<f64> {
93
+ for (unit, factor) in [
94
+ ("ns", 1.0),
95
+ ("us", 1_000.0),
96
+ ("ms", 1_000_000.0),
97
+ ("s", 1_000_000_000.0),
98
+ ] {
99
+ let col = format!("{} ({unit})", name.to_ascii_lowercase());
100
+ if let Some(value) = value(row, header, &[&col]) {
101
+ return number(value)
102
+ .ok()
103
+ .map(|v| v * factor)
104
+ .filter(|v| v.is_finite());
105
+ }
106
+ }
107
+ None
108
+ }
109
+
110
+ pub fn open(path: &std::path::Path) -> io::Result<io::BufReader<std::fs::File>> {
111
+ std::fs::File::open(path).map(io::BufReader::new)
112
+ }
113
+
114
+ pub fn table(
115
+ reader: impl BufRead,
116
+ marker: &str,
117
+ mut visit: impl FnMut(&[String], &[String]) -> Result<(), String>,
118
+ ) -> Result<(), String> {
119
+ let mut header = Vec::new();
120
+ records(reader, |row| {
121
+ if column(&row, &[marker]).is_some() {
122
+ header = row;
123
+ return Ok(());
124
+ }
125
+ if header.is_empty() || row.iter().all(|s| s.trim().is_empty()) {
126
+ return Ok(());
127
+ }
128
+ if row.len() != header.len() {
129
+ return Err("unexpected CSV record width after header".into());
130
+ }
131
+ visit(&header, &row)
132
+ })
133
+ }
134
+
135
+ #[cfg(test)]
136
+ mod tests {
137
+ use super::*;
138
+
139
+ #[test]
140
+ fn quoted_templates_and_newlines() {
141
+ let input = "Name,Count\r\n\"foo<int, 128, bar<float>>\",2\r\n\"line\n\"\"two\"\"\",3";
142
+ let mut rows = Vec::new();
143
+ records(input.as_bytes(), |row| {
144
+ rows.push(row);
145
+ Ok(())
146
+ })
147
+ .unwrap();
148
+ assert_eq!(rows[1], ["foo<int, 128, bar<float>>", "2"]);
149
+ assert_eq!(rows[2], ["line\n\"two\"", "3"]);
150
+ assert!(records(b"\"bad".as_slice(), |_| Ok(())).is_err());
151
+ for invalid in ["NaN", "inf", "-1"] {
152
+ assert!(number(invalid).is_err());
153
+ }
154
+ }
155
+ }