ds4-winhip: Windows/HIP 7.2 build tree, launcher, supervisor, bench + notes (part 2)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- crates/ds4-dist/src/lib.rs +121 -0
- crates/ds4-dist/src/memory_snapshot.rs +186 -0
- crates/ds4-dist/src/native_snapshot.rs +126 -0
- crates/ds4-dist/src/options.rs +351 -0
- crates/ds4-dist/src/plan.rs +157 -0
- crates/ds4-dist/src/prefetch.rs +278 -0
- crates/ds4-dist/src/prefetch_local.rs +253 -0
- crates/ds4-dist/src/reconnect.rs +217 -0
- crates/ds4-dist/src/relay.rs +217 -0
- crates/ds4-dist/src/route.rs +162 -0
- crates/ds4-dist/src/serve_local.rs +127 -0
- crates/ds4-dist/src/snapshot.rs +337 -0
- crates/ds4-dist/src/snapshot_gather.rs +186 -0
- crates/ds4-dist/src/snapshot_merge.rs +415 -0
- crates/ds4-dist/src/snapshot_temp.rs +99 -0
- crates/ds4-dist/src/transport.rs +30 -0
- crates/ds4-dist/src/work.rs +207 -0
- crates/ds4-dist/src/worker.rs +424 -0
- crates/ds4-dist/src/worker_snapshot.rs +291 -0
- crates/ds4-dist/tests/forward.rs +128 -0
- crates/ds4-dist/tests/prefetch.rs +267 -0
- crates/ds4-dist/tests/relay.rs +296 -0
- crates/ds4-dist/tests/runtime.rs +1257 -0
- crates/ds4-dist/tests/snapshot_store.rs +214 -0
- crates/ds4-dist/tests/snapshot_transport.rs +646 -0
- crates/ds4-dist/tests/wire_oracle.rs +398 -0
- crates/ds4-kv/Cargo.toml +10 -0
- crates/ds4-kv/src/extensions.rs +245 -0
- crates/ds4-kv/src/format.rs +781 -0
- crates/ds4-kv/src/host.rs +64 -0
- crates/ds4-kv/src/identity.rs +42 -0
- crates/ds4-kv/src/lib.rs +28 -0
- crates/ds4-kv/src/policy.rs +340 -0
- crates/ds4-kv/src/sha1.rs +140 -0
- crates/ds4-kv/src/store.rs +2406 -0
- crates/ds4-kv/tests/bank_thinking.rs +250 -0
- crates/ds4-kv/tests/four_way.rs +375 -0
- crates/ds4-perf-gpu/Cargo.toml +25 -0
- crates/ds4-perf-gpu/src/calibration.rs +176 -0
- crates/ds4-perf-gpu/src/collector.rs +288 -0
- crates/ds4-perf-gpu/src/device.rs +51 -0
- crates/ds4-perf-gpu/src/lib.rs +11 -0
- crates/ds4-perf-gpu/src/main.rs +23 -0
- crates/ds4-perf-gpu/tests/live.rs +14 -0
- crates/ds4-perf/Cargo.toml +18 -0
- crates/ds4-perf/src/artifact.rs +564 -0
- crates/ds4-perf/src/bench.rs +84 -0
- crates/ds4-perf/src/cli.rs +214 -0
- crates/ds4-perf/src/compare.rs +695 -0
- crates/ds4-perf/src/csv.rs +155 -0
crates/ds4-dist/src/lib.rs
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Distributed coordinator/worker wire and blocking runtime. Explicit integer
|
| 2 |
+
//! codecs only — do not `#[repr(C)]` records onto the socket.
|
| 3 |
+
|
| 4 |
+
mod activation;
|
| 5 |
+
mod assemble;
|
| 6 |
+
mod codec;
|
| 7 |
+
mod coordinator;
|
| 8 |
+
mod exec;
|
| 9 |
+
mod forward;
|
| 10 |
+
mod hash;
|
| 11 |
+
mod hops;
|
| 12 |
+
mod memory_snapshot;
|
| 13 |
+
mod native_snapshot;
|
| 14 |
+
mod options;
|
| 15 |
+
mod plan;
|
| 16 |
+
mod prefetch;
|
| 17 |
+
mod prefetch_local;
|
| 18 |
+
mod reconnect;
|
| 19 |
+
mod relay;
|
| 20 |
+
mod route;
|
| 21 |
+
mod serve_local;
|
| 22 |
+
mod snapshot;
|
| 23 |
+
mod snapshot_gather;
|
| 24 |
+
mod snapshot_merge;
|
| 25 |
+
mod snapshot_temp;
|
| 26 |
+
mod transport;
|
| 27 |
+
mod work;
|
| 28 |
+
mod worker;
|
| 29 |
+
mod worker_snapshot;
|
| 30 |
+
|
| 31 |
+
pub use activation::{
|
| 32 |
+
bits_or_default, bits_valid, decode_activation, encode_activation, f16_to_f32, f32_to_f16,
|
| 33 |
+
f32_to_f8_e4m3, f8_e4m3_to_f32, values_from_wire_bytes, wire_bytes, wire_bytes_from_f32_bytes,
|
| 34 |
+
BITS_DEFAULT,
|
| 35 |
+
};
|
| 36 |
+
pub use assemble::{
|
| 37 |
+
assemble_worker, print_worker_listen_banner, slice_meta, worker_hello, worker_listen_banner,
|
| 38 |
+
worker_listen_port, worker_model_name, worker_plan, AssembledWorker, SliceMeta,
|
| 39 |
+
WorkerListenBanner, WorkerPlan, UNKNOWN_MODEL_NAME,
|
| 40 |
+
};
|
| 41 |
+
pub use codec::{
|
| 42 |
+
bytes_have_nul, decode_frame_header, decode_hello_payload, decode_snapshot_begin_body,
|
| 43 |
+
decode_snapshot_chunk_body, decode_snapshot_done_body, decode_snapshot_load_begin_body,
|
| 44 |
+
decode_tokens_be, encode_error_frame, encode_frame_header, encode_hello_payload,
|
| 45 |
+
encode_snapshot_begin_body, encode_snapshot_chunk_body, encode_snapshot_done_body,
|
| 46 |
+
encode_tokens_be, get_u32_be, put_u32_be, u64_from_halves, u64_to_halves, CodecError,
|
| 47 |
+
FrameHeader, Hello, ResultHdr, Route, RouteReturn, SnapshotBegin, SnapshotChunk, SnapshotDone,
|
| 48 |
+
SnapshotReq, Telemetry, Work, FRAME_HEADER_BYTES, HELLO_FIXED_BYTES, MAGIC, MAX_MODEL_NAME,
|
| 49 |
+
MSG_ERROR, MSG_HELLO, MSG_RESULT, MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK, MSG_SNAPSHOT_DONE,
|
| 50 |
+
MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK, RESULT_ACK, RESULT_FIXED_BYTES,
|
| 51 |
+
RESULT_HIDDEN_STATE, RESULT_LOGITS, ROUTE_FIXED_BYTES, ROUTE_F_OUTPUT_LOGITS,
|
| 52 |
+
ROUTE_RETURN_FIXED_BYTES, ROUTE_RETURN_UPSTREAM, SNAPSHOT_BEGIN_FIXED_BYTES,
|
| 53 |
+
SNAPSHOT_CHUNK_BYTES, SNAPSHOT_CHUNK_FIXED_BYTES, SNAPSHOT_DONE_FIXED_BYTES,
|
| 54 |
+
SNAPSHOT_REQ_FIXED_BYTES, TELEMETRY_FIXED_BYTES, WORK_FIXED_BYTES, WORK_F_ACK_ONLY,
|
| 55 |
+
WORK_F_INPUT_HC, WORK_F_OUTPUT_LOGITS, WORK_F_RESET_SESSION, WORK_F_VALID_MASK,
|
| 56 |
+
};
|
| 57 |
+
pub use coordinator::{
|
| 58 |
+
accept_data_client, accept_loop, dispatch_eval, format_telemetry_line, listen,
|
| 59 |
+
open_data_listener, recv_hello_only, token_span_hashes, Coordinator, EvalOutcome,
|
| 60 |
+
RegisteredWorker, SharedCoordinator, ERR_DATA_LISTEN_PORT,
|
| 61 |
+
};
|
| 62 |
+
pub use exec::{SliceExec, WorkOutput, WorkRequest};
|
| 63 |
+
pub use forward::{
|
| 64 |
+
forward_window, forward_window_from, opened_forwarder_message, PendingQueue, PendingRequest,
|
| 65 |
+
ERR_CLOSED_WHILE_RESULT, ERR_FORWARD, ERR_FORWARD_HIDDEN, ERR_INVALID_RESULT, ERR_NEXT_CLOSED,
|
| 66 |
+
ERR_OOM_FORWARDER, ERR_OOM_TRACK, ERR_RELAY_THREAD, ERR_RESULT_METADATA, ERR_RESULT_TOO_LARGE,
|
| 67 |
+
ERR_TELEMETRY_TOO_LARGE, FORWARD_WINDOW_DEFAULT, FORWARD_WINDOW_MAX, FORWARD_WINDOW_MIN,
|
| 68 |
+
};
|
| 69 |
+
pub use hash::{token_hash_prefix, token_hash_update, token_hash_update_span, TOKEN_HASH_INIT};
|
| 70 |
+
pub use hops::ForwarderPool;
|
| 71 |
+
pub use native_snapshot::{
|
| 72 |
+
apply_snapshot_load, copy_chunked, create_temp, dispatch_worker_snapshot,
|
| 73 |
+
prepare_snapshot_save, MemorySnapshotStore, SnapshotLoad, SnapshotSave, SnapshotStore,
|
| 74 |
+
TempShard, LOAD_PREFIX, SAVE_PREFIX,
|
| 75 |
+
};
|
| 76 |
+
pub use options::{
|
| 77 |
+
parse_cli, parse_cli_arg, parse_layers, parse_role, prepare_engine_options, resolved_layer_end,
|
| 78 |
+
validate_layers_for_model, validate_options, CliResult, Layers, Options, Role, USAGE,
|
| 79 |
+
};
|
| 80 |
+
pub use plan::{build_route_plan, register_worker, CoordinatorView, RoutePlan, WorkerInfo};
|
| 81 |
+
pub use prefetch::{
|
| 82 |
+
prefetch_depth, prefetch_depth_from, prefetch_disabled, prefetch_disabled_from,
|
| 83 |
+
prefetch_enabled_message, JobQueue, ERR_OOM_QUEUE, ERR_OOM_READ, PREFETCH_DEPTH_DEFAULT,
|
| 84 |
+
PREFETCH_DEPTH_MAX, PREFETCH_DEPTH_MIN,
|
| 85 |
+
};
|
| 86 |
+
pub use prefetch_local::{
|
| 87 |
+
local_prefetch_enabled, local_prefetch_enabled_from, serve_prefetch_local_with, PrefetchJob,
|
| 88 |
+
};
|
| 89 |
+
pub use reconnect::{
|
| 90 |
+
cleared_sessions_message, connect_endpoint, connect_endpoint_once, connect_error,
|
| 91 |
+
connect_retryable, connected_message, disconnected_message, hello_failed_message, peer_name,
|
| 92 |
+
reconnect_local, reconnect_with, retrying_message, sleep_reconnect, LocalReconnect,
|
| 93 |
+
CONNECT_RETRY_ATTEMPTS, CONNECT_RETRY_DELAY, RECONNECT_SLEEP,
|
| 94 |
+
};
|
| 95 |
+
pub use relay::{
|
| 96 |
+
forward_work_blocking, local_work_telemetry, now_sec, prepend_telemetry, usec_since, Forwarder,
|
| 97 |
+
};
|
| 98 |
+
pub use route::{
|
| 99 |
+
decode_route_blob, encode_route_blob, validate_route_blob, ReturnTarget, RouteEntry,
|
| 100 |
+
};
|
| 101 |
+
pub use snapshot::{coordinator_load_snapshot, coordinator_save_snapshot, SnapshotMeta};
|
| 102 |
+
pub use snapshot_gather::{
|
| 103 |
+
gather_kv_shards, scatter_kv_shards, GatherShard, GatheredShards, ERR_EMPTY_SHARD,
|
| 104 |
+
ERR_NO_TIMELINE,
|
| 105 |
+
};
|
| 106 |
+
pub use snapshot_merge::{
|
| 107 |
+
kv_state_bytes, layer_tensor_bytes, merge_session_payload, parse_layer_payload, DistKvLayout,
|
| 108 |
+
ParsedLayerShard, ERR_METADATA, ERR_TENSOR_BYTES, LAYER_MAGIC, SESSION_MAGIC,
|
| 109 |
+
};
|
| 110 |
+
pub use transport::{read_frame, write_frame};
|
| 111 |
+
pub use work::{
|
| 112 |
+
decode_logits_payload, decode_result_body, decode_work_body, encode_logits_payload,
|
| 113 |
+
encode_result_body, encode_result_frame, encode_work_body, encode_work_frame,
|
| 114 |
+
error_result_frame, ok_result_hdr, result_hash, result_request_id, work_with_ids, ResultBody,
|
| 115 |
+
WorkBody,
|
| 116 |
+
};
|
| 117 |
+
pub use worker::{recv_hello, send_hello, Worker};
|
| 118 |
+
pub use worker_snapshot::{
|
| 119 |
+
worker_handle_snapshot_load, worker_handle_snapshot_load_restore, worker_handle_snapshot_save,
|
| 120 |
+
WorkerLoadOffer, WorkerSnapshotIdentity,
|
| 121 |
+
};
|
crates/ds4-dist/src/memory_snapshot.rs
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! In-memory SnapshotStore with C worker status strings.
|
| 2 |
+
|
| 3 |
+
use std::collections::HashMap;
|
| 4 |
+
use std::io::{self, Read, Seek, SeekFrom, Write};
|
| 5 |
+
|
| 6 |
+
use crate::hash::token_hash_prefix;
|
| 7 |
+
use crate::snapshot_temp::{copy_chunked, create_temp, TempShard, LOAD_PREFIX, SAVE_PREFIX};
|
| 8 |
+
|
| 9 |
+
pub struct SnapshotSave {
|
| 10 |
+
pub session_id: u64,
|
| 11 |
+
pub token_count: u32,
|
| 12 |
+
pub token_hash: u64,
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
pub struct SnapshotLoad<'a> {
|
| 16 |
+
pub session_id: u64,
|
| 17 |
+
pub tokens: &'a [i32],
|
| 18 |
+
pub token_hash: u64,
|
| 19 |
+
pub payload_bytes: u64,
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
pub trait SnapshotStore {
|
| 23 |
+
type SaveReader: Read;
|
| 24 |
+
fn save(&mut self, req: SnapshotSave) -> Result<(Self::SaveReader, u64), String>;
|
| 25 |
+
fn load(&mut self, req: SnapshotLoad<'_>, payload: &mut dyn Read) -> Result<(), String>;
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
struct StoredSession {
|
| 29 |
+
tokens: Vec<i32>,
|
| 30 |
+
shard: TempShard,
|
| 31 |
+
token_hash: u64,
|
| 32 |
+
token_hash_valid: bool,
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
pub struct MemorySnapshotStore {
|
| 36 |
+
sessions: HashMap<u64, StoredSession>,
|
| 37 |
+
fail_restore: bool,
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
impl MemorySnapshotStore {
|
| 41 |
+
pub fn new() -> Self {
|
| 42 |
+
Self {
|
| 43 |
+
sessions: HashMap::new(),
|
| 44 |
+
fail_restore: false,
|
| 45 |
+
}
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
pub fn insert(
|
| 49 |
+
&mut self,
|
| 50 |
+
session_id: u64,
|
| 51 |
+
tokens: Vec<i32>,
|
| 52 |
+
shard: &[u8],
|
| 53 |
+
) -> Result<(), String> {
|
| 54 |
+
let mut tmp = create_temp(SAVE_PREFIX)
|
| 55 |
+
.map_err(|_| "failed to create worker snapshot temp file".to_string())?;
|
| 56 |
+
copy_chunked(&mut io::Cursor::new(shard), &mut tmp, shard.len() as u64)
|
| 57 |
+
.map_err(|_| "failed to save worker KV shard".to_string())?;
|
| 58 |
+
tmp.flush()
|
| 59 |
+
.map_err(|_| "failed to flush worker KV shard".to_string())?;
|
| 60 |
+
tmp.rewind()
|
| 61 |
+
.map_err(|_| "failed to rewind worker KV shard".to_string())?;
|
| 62 |
+
let token_hash = token_hash_prefix(&tokens);
|
| 63 |
+
self.sessions.insert(
|
| 64 |
+
session_id,
|
| 65 |
+
StoredSession {
|
| 66 |
+
tokens,
|
| 67 |
+
shard: tmp,
|
| 68 |
+
token_hash,
|
| 69 |
+
token_hash_valid: true,
|
| 70 |
+
},
|
| 71 |
+
);
|
| 72 |
+
Ok(())
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
pub fn fail_restore(&mut self, fail: bool) {
|
| 76 |
+
self.fail_restore = fail;
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
pub fn token_hash_valid(&self, session_id: u64) -> bool {
|
| 80 |
+
self.sessions
|
| 81 |
+
.get(&session_id)
|
| 82 |
+
.is_some_and(|s| s.token_hash_valid)
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
pub fn token_hash(&self, session_id: u64) -> Option<u64> {
|
| 86 |
+
self.sessions.get(&session_id).map(|s| s.token_hash)
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
pub fn tokens(&self, session_id: u64) -> Option<&[i32]> {
|
| 90 |
+
self.sessions.get(&session_id).map(|s| s.tokens.as_slice())
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
pub fn read_shard(&mut self, session_id: u64) -> Result<Vec<u8>, String> {
|
| 94 |
+
let session = self
|
| 95 |
+
.sessions
|
| 96 |
+
.get_mut(&session_id)
|
| 97 |
+
.ok_or_else(|| "worker has no distributed session to snapshot".to_string())?;
|
| 98 |
+
session
|
| 99 |
+
.shard
|
| 100 |
+
.rewind()
|
| 101 |
+
.map_err(|_| "failed to rewind worker KV shard".to_string())?;
|
| 102 |
+
let mut out = Vec::new();
|
| 103 |
+
session
|
| 104 |
+
.shard
|
| 105 |
+
.read_to_end(&mut out)
|
| 106 |
+
.map_err(|_| "failed to read worker KV shard".to_string())?;
|
| 107 |
+
session
|
| 108 |
+
.shard
|
| 109 |
+
.rewind()
|
| 110 |
+
.map_err(|_| "failed to rewind worker KV shard".to_string())?;
|
| 111 |
+
Ok(out)
|
| 112 |
+
}
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
impl Default for MemorySnapshotStore {
|
| 116 |
+
fn default() -> Self {
|
| 117 |
+
Self::new()
|
| 118 |
+
}
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
impl SnapshotStore for MemorySnapshotStore {
|
| 122 |
+
type SaveReader = TempShard;
|
| 123 |
+
|
| 124 |
+
fn save(&mut self, req: SnapshotSave) -> Result<(TempShard, u64), String> {
|
| 125 |
+
let session = self
|
| 126 |
+
.sessions
|
| 127 |
+
.get_mut(&req.session_id)
|
| 128 |
+
.ok_or_else(|| "worker has no distributed session to snapshot".to_string())?;
|
| 129 |
+
if session.tokens.len() as u32 != req.token_count {
|
| 130 |
+
return Err("worker snapshot token count mismatch".into());
|
| 131 |
+
}
|
| 132 |
+
if token_hash_prefix(&session.tokens) != req.token_hash {
|
| 133 |
+
return Err("worker snapshot token hash mismatch".into());
|
| 134 |
+
}
|
| 135 |
+
let mut tmp = create_temp(SAVE_PREFIX)
|
| 136 |
+
.map_err(|_| "failed to create worker snapshot temp file".to_string())?;
|
| 137 |
+
session
|
| 138 |
+
.shard
|
| 139 |
+
.rewind()
|
| 140 |
+
.map_err(|_| "failed to rewind worker KV shard".to_string())?;
|
| 141 |
+
let len = session
|
| 142 |
+
.shard
|
| 143 |
+
.seek(SeekFrom::End(0))
|
| 144 |
+
.map_err(|_| "failed to measure worker KV shard".to_string())?;
|
| 145 |
+
session
|
| 146 |
+
.shard
|
| 147 |
+
.rewind()
|
| 148 |
+
.map_err(|_| "failed to rewind worker KV shard".to_string())?;
|
| 149 |
+
copy_chunked(&mut session.shard, &mut tmp, len)
|
| 150 |
+
.map_err(|_| "failed to save worker KV shard".to_string())?;
|
| 151 |
+
tmp.flush()
|
| 152 |
+
.map_err(|_| "failed to flush worker KV shard".to_string())?;
|
| 153 |
+
let payload_bytes = tmp
|
| 154 |
+
.stream_position()
|
| 155 |
+
.map_err(|_| "failed to measure worker KV shard".to_string())?;
|
| 156 |
+
tmp.rewind()
|
| 157 |
+
.map_err(|_| "failed to rewind worker KV shard".to_string())?;
|
| 158 |
+
Ok((tmp, payload_bytes))
|
| 159 |
+
}
|
| 160 |
+
|
| 161 |
+
fn load(&mut self, req: SnapshotLoad<'_>, payload: &mut dyn Read) -> Result<(), String> {
|
| 162 |
+
if self.fail_restore {
|
| 163 |
+
if let Some(session) = self.sessions.get_mut(&req.session_id) {
|
| 164 |
+
session.token_hash_valid = false;
|
| 165 |
+
}
|
| 166 |
+
return Err("failed to restore worker KV shard".into());
|
| 167 |
+
}
|
| 168 |
+
let mut shard = create_temp(LOAD_PREFIX)
|
| 169 |
+
.map_err(|_| "failed to restore worker KV shard".to_string())?;
|
| 170 |
+
copy_chunked(payload, &mut shard, req.payload_bytes)
|
| 171 |
+
.map_err(|_| "failed to restore worker KV shard".to_string())?;
|
| 172 |
+
shard
|
| 173 |
+
.rewind()
|
| 174 |
+
.map_err(|_| "failed to restore worker KV shard".to_string())?;
|
| 175 |
+
self.sessions.insert(
|
| 176 |
+
req.session_id,
|
| 177 |
+
StoredSession {
|
| 178 |
+
tokens: req.tokens.to_vec(),
|
| 179 |
+
shard,
|
| 180 |
+
token_hash: req.token_hash,
|
| 181 |
+
token_hash_valid: true,
|
| 182 |
+
},
|
| 183 |
+
);
|
| 184 |
+
Ok(())
|
| 185 |
+
}
|
| 186 |
+
}
|
crates/ds4-dist/src/native_snapshot.rs
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Bind snapshot frames to a SnapshotStore with C restore-before-DONE order.
|
| 2 |
+
|
| 3 |
+
use std::fs::File;
|
| 4 |
+
use std::io::{self, Read, Write};
|
| 5 |
+
|
| 6 |
+
use crate::codec::{
|
| 7 |
+
u64_from_halves, SnapshotReq, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ,
|
| 8 |
+
SNAPSHOT_REQ_FIXED_BYTES,
|
| 9 |
+
};
|
| 10 |
+
use crate::worker_snapshot::{
|
| 11 |
+
worker_handle_snapshot_load_restore, worker_handle_snapshot_save, WorkerLoadOffer,
|
| 12 |
+
WorkerSnapshotIdentity,
|
| 13 |
+
};
|
| 14 |
+
|
| 15 |
+
pub use crate::memory_snapshot::{MemorySnapshotStore, SnapshotLoad, SnapshotSave, SnapshotStore};
|
| 16 |
+
pub use crate::snapshot_temp::{copy_chunked, create_temp, TempShard, LOAD_PREFIX, SAVE_PREFIX};
|
| 17 |
+
|
| 18 |
+
pub fn prepare_snapshot_save<Store: SnapshotStore>(
|
| 19 |
+
store: &mut Store,
|
| 20 |
+
identity: WorkerSnapshotIdentity,
|
| 21 |
+
req_body: &[u8],
|
| 22 |
+
) -> Result<(Store::SaveReader, u64), String> {
|
| 23 |
+
if req_body.len() != SNAPSHOT_REQ_FIXED_BYTES {
|
| 24 |
+
return Err(String::new());
|
| 25 |
+
}
|
| 26 |
+
let req = SnapshotReq::decode(req_body).map_err(|e| e.to_string())?;
|
| 27 |
+
if !identity.accepts(
|
| 28 |
+
req.model_id,
|
| 29 |
+
req.layer_start,
|
| 30 |
+
req.layer_end,
|
| 31 |
+
req.token_count,
|
| 32 |
+
) {
|
| 33 |
+
return Err(String::new());
|
| 34 |
+
}
|
| 35 |
+
store.save(SnapshotSave {
|
| 36 |
+
session_id: u64_from_halves(req.session_hi, req.session_lo),
|
| 37 |
+
token_count: req.token_count,
|
| 38 |
+
token_hash: u64_from_halves(req.token_hash_hi, req.token_hash_lo),
|
| 39 |
+
})
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
pub fn apply_snapshot_load<S, Store>(
|
| 43 |
+
stream: &mut S,
|
| 44 |
+
identity: WorkerSnapshotIdentity,
|
| 45 |
+
begin_body: &[u8],
|
| 46 |
+
vocab_size: u32,
|
| 47 |
+
store: &mut Store,
|
| 48 |
+
) -> Result<WorkerLoadOffer, String>
|
| 49 |
+
where
|
| 50 |
+
S: Read + Write,
|
| 51 |
+
Store: SnapshotStore,
|
| 52 |
+
{
|
| 53 |
+
match create_temp(LOAD_PREFIX) {
|
| 54 |
+
Ok(mut tmp) => {
|
| 55 |
+
let path = tmp.path.clone();
|
| 56 |
+
worker_handle_snapshot_load_restore(
|
| 57 |
+
stream,
|
| 58 |
+
identity,
|
| 59 |
+
begin_body,
|
| 60 |
+
vocab_size,
|
| 61 |
+
&mut tmp,
|
| 62 |
+
|offer| {
|
| 63 |
+
let mut replay = File::open(&path)
|
| 64 |
+
.map_err(|_| "failed to rewind worker KV shard restore file".to_string())?;
|
| 65 |
+
store.load(
|
| 66 |
+
SnapshotLoad {
|
| 67 |
+
session_id: offer.session_id,
|
| 68 |
+
tokens: &offer.tokens,
|
| 69 |
+
token_hash: offer.token_hash,
|
| 70 |
+
payload_bytes: offer.payload_bytes,
|
| 71 |
+
},
|
| 72 |
+
&mut replay,
|
| 73 |
+
)
|
| 74 |
+
},
|
| 75 |
+
)
|
| 76 |
+
}
|
| 77 |
+
Err(_) => {
|
| 78 |
+
let mut sink = io::sink();
|
| 79 |
+
worker_handle_snapshot_load_restore(
|
| 80 |
+
stream,
|
| 81 |
+
identity,
|
| 82 |
+
begin_body,
|
| 83 |
+
vocab_size,
|
| 84 |
+
&mut sink,
|
| 85 |
+
|_| Err("failed to create worker snapshot restore temp file".into()),
|
| 86 |
+
)
|
| 87 |
+
}
|
| 88 |
+
}
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
pub fn dispatch_worker_snapshot<Stream, Store>(
|
| 92 |
+
stream: &mut Stream,
|
| 93 |
+
typ: u32,
|
| 94 |
+
body: &[u8],
|
| 95 |
+
identity: WorkerSnapshotIdentity,
|
| 96 |
+
vocab: u32,
|
| 97 |
+
store: &mut Store,
|
| 98 |
+
) -> Result<Option<(u64, u64)>, String>
|
| 99 |
+
where
|
| 100 |
+
Stream: Read + Write,
|
| 101 |
+
Store: SnapshotStore,
|
| 102 |
+
{
|
| 103 |
+
match typ {
|
| 104 |
+
MSG_SNAPSHOT_SAVE_REQ => {
|
| 105 |
+
match prepare_snapshot_save(store, identity, body) {
|
| 106 |
+
Ok((mut reader, n)) => {
|
| 107 |
+
worker_handle_snapshot_save(stream, identity, body, Ok((&mut reader, n)))?;
|
| 108 |
+
}
|
| 109 |
+
Err(e) => {
|
| 110 |
+
worker_handle_snapshot_save::<_, io::Cursor<Vec<u8>>>(
|
| 111 |
+
stream,
|
| 112 |
+
identity,
|
| 113 |
+
body,
|
| 114 |
+
Err(e),
|
| 115 |
+
)?;
|
| 116 |
+
}
|
| 117 |
+
}
|
| 118 |
+
Ok(None)
|
| 119 |
+
}
|
| 120 |
+
MSG_SNAPSHOT_LOAD_BEGIN => {
|
| 121 |
+
let offer = apply_snapshot_load(stream, identity, body, vocab, store)?;
|
| 122 |
+
Ok(Some((offer.session_id, offer.token_hash)))
|
| 123 |
+
}
|
| 124 |
+
_ => Err(format!("rejected unsupported frame type {typ}")),
|
| 125 |
+
}
|
| 126 |
+
}
|
crates/ds4-dist/src/options.rs
ADDED
|
@@ -0,0 +1,351 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! CLI + option validation matching `ds4_dist_parse_cli_arg` / `dist_validate_options`.
|
| 2 |
+
|
| 3 |
+
use crate::activation::bits_valid;
|
| 4 |
+
|
| 5 |
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
| 6 |
+
pub enum Role {
|
| 7 |
+
None = 0,
|
| 8 |
+
Coordinator = 1,
|
| 9 |
+
Worker = 2,
|
| 10 |
+
}
|
| 11 |
+
|
| 12 |
+
impl Role {
|
| 13 |
+
pub fn name(self) -> &'static str {
|
| 14 |
+
match self {
|
| 15 |
+
Role::None => "none",
|
| 16 |
+
Role::Coordinator => "coordinator",
|
| 17 |
+
Role::Worker => "worker",
|
| 18 |
+
}
|
| 19 |
+
}
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
| 23 |
+
pub struct Layers {
|
| 24 |
+
pub start: u32,
|
| 25 |
+
pub end: u32,
|
| 26 |
+
pub has_output: bool,
|
| 27 |
+
pub set: bool,
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
| 31 |
+
pub struct Options {
|
| 32 |
+
pub role: Role,
|
| 33 |
+
pub layers: Layers,
|
| 34 |
+
pub listen_host: Option<String>,
|
| 35 |
+
pub listen_port: i32,
|
| 36 |
+
pub coordinator_host: Option<String>,
|
| 37 |
+
pub coordinator_port: i32,
|
| 38 |
+
pub prefill_chunk: u32,
|
| 39 |
+
pub prefill_window: u32,
|
| 40 |
+
pub activation_bits: u32,
|
| 41 |
+
pub replay_check: bool,
|
| 42 |
+
pub debug: bool,
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
impl Default for Role {
|
| 46 |
+
fn default() -> Self {
|
| 47 |
+
Role::None
|
| 48 |
+
}
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
| 52 |
+
pub enum CliResult {
|
| 53 |
+
Error,
|
| 54 |
+
NotMatched,
|
| 55 |
+
Matched,
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
pub fn parse_role(s: &str) -> Option<Role> {
|
| 59 |
+
match s {
|
| 60 |
+
"none" => Some(Role::None),
|
| 61 |
+
"coordinator" => Some(Role::Coordinator),
|
| 62 |
+
"worker" => Some(Role::Worker),
|
| 63 |
+
_ => None,
|
| 64 |
+
}
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
pub fn parse_layers(s: &str) -> Result<Layers, String> {
|
| 68 |
+
let Some(colon) = s.find(':') else {
|
| 69 |
+
return Err("expected A:B or A:output".into());
|
| 70 |
+
};
|
| 71 |
+
if colon == 0 || colon + 1 == s.len() {
|
| 72 |
+
return Err("expected A:B or A:output".into());
|
| 73 |
+
}
|
| 74 |
+
if s[colon + 1..].contains(':') {
|
| 75 |
+
return Err("layer range has too many ':' separators".into());
|
| 76 |
+
}
|
| 77 |
+
let start =
|
| 78 |
+
parse_u32_component(&s[..colon]).ok_or_else(|| format!("invalid start layer in {s}"))?;
|
| 79 |
+
let end_s = &s[colon + 1..];
|
| 80 |
+
if end_s == "output" {
|
| 81 |
+
return Ok(Layers {
|
| 82 |
+
start,
|
| 83 |
+
end: u32::MAX,
|
| 84 |
+
has_output: true,
|
| 85 |
+
set: true,
|
| 86 |
+
});
|
| 87 |
+
}
|
| 88 |
+
let end = parse_u32_component(end_s).ok_or_else(|| format!("invalid end layer in {s}"))?;
|
| 89 |
+
if end < start {
|
| 90 |
+
return Err(format!("layer range end precedes start in {s}"));
|
| 91 |
+
}
|
| 92 |
+
Ok(Layers {
|
| 93 |
+
start,
|
| 94 |
+
end,
|
| 95 |
+
has_output: false,
|
| 96 |
+
set: true,
|
| 97 |
+
})
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
fn parse_u32_component(s: &str) -> Option<u32> {
|
| 101 |
+
if s.is_empty() || s.len() >= 32 {
|
| 102 |
+
return None;
|
| 103 |
+
}
|
| 104 |
+
if s.as_bytes().iter().any(|c| !c.is_ascii_digit()) {
|
| 105 |
+
return None;
|
| 106 |
+
}
|
| 107 |
+
s.parse().ok()
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
fn parse_port(s: &str, arg: &str) -> Result<i32, String> {
|
| 111 |
+
let v: i64 = s
|
| 112 |
+
.parse()
|
| 113 |
+
.map_err(|_| format!("invalid value for {arg}: {s}"))?;
|
| 114 |
+
if v <= 0 || v > 65535 {
|
| 115 |
+
return Err(format!("invalid value for {arg}: {s}"));
|
| 116 |
+
}
|
| 117 |
+
Ok(v as i32)
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
fn parse_positive_u32(s: &str, name: &str) -> Result<u32, String> {
|
| 121 |
+
if s.is_empty() {
|
| 122 |
+
return Err(format!("{name} requires a positive integer"));
|
| 123 |
+
}
|
| 124 |
+
let v: u64 = s
|
| 125 |
+
.parse()
|
| 126 |
+
.map_err(|_| format!("invalid value for {name}: {s}"))?;
|
| 127 |
+
if v == 0 || v > u64::from(u32::MAX) {
|
| 128 |
+
return Err(format!("invalid value for {name}: {s}"));
|
| 129 |
+
}
|
| 130 |
+
Ok(v as u32)
|
| 131 |
+
}
|
| 132 |
+
|
| 133 |
+
pub fn parse_cli_arg(
|
| 134 |
+
arg: &str,
|
| 135 |
+
rest: &mut impl Iterator<Item = String>,
|
| 136 |
+
opt: &mut Options,
|
| 137 |
+
) -> Result<CliResult, String> {
|
| 138 |
+
match arg {
|
| 139 |
+
"--role" => {
|
| 140 |
+
let role = rest
|
| 141 |
+
.next()
|
| 142 |
+
.ok_or_else(|| "--role requires an argument".to_string())?;
|
| 143 |
+
let parsed = parse_role(&role).ok_or_else(|| {
|
| 144 |
+
format!("invalid distributed role: {role} (valid roles: none, coordinator, worker)")
|
| 145 |
+
})?;
|
| 146 |
+
opt.role = parsed;
|
| 147 |
+
Ok(CliResult::Matched)
|
| 148 |
+
}
|
| 149 |
+
"--layers" => {
|
| 150 |
+
let layers = rest
|
| 151 |
+
.next()
|
| 152 |
+
.ok_or_else(|| "--layers requires an argument".to_string())?;
|
| 153 |
+
match parse_layers(&layers) {
|
| 154 |
+
Ok(l) => {
|
| 155 |
+
opt.layers = l;
|
| 156 |
+
Ok(CliResult::Matched)
|
| 157 |
+
}
|
| 158 |
+
Err(detail) => Err(format!("invalid --layers {layers}: {detail}")),
|
| 159 |
+
}
|
| 160 |
+
}
|
| 161 |
+
"--listen" => {
|
| 162 |
+
if opt.listen_host.is_some() || opt.listen_port != 0 {
|
| 163 |
+
return Err("specify --listen only once".into());
|
| 164 |
+
}
|
| 165 |
+
let host = rest
|
| 166 |
+
.next()
|
| 167 |
+
.ok_or_else(|| "--listen requires an argument".to_string())?;
|
| 168 |
+
let port = rest
|
| 169 |
+
.next()
|
| 170 |
+
.ok_or_else(|| "--listen requires an argument".to_string())?;
|
| 171 |
+
opt.listen_port = parse_port(&port, "--listen")?;
|
| 172 |
+
opt.listen_host = Some(host);
|
| 173 |
+
Ok(CliResult::Matched)
|
| 174 |
+
}
|
| 175 |
+
"--coordinator" => {
|
| 176 |
+
if opt.coordinator_host.is_some() || opt.coordinator_port != 0 {
|
| 177 |
+
return Err("specify --coordinator only once".into());
|
| 178 |
+
}
|
| 179 |
+
let host = rest
|
| 180 |
+
.next()
|
| 181 |
+
.ok_or_else(|| "--coordinator requires an argument".to_string())?;
|
| 182 |
+
let port = rest
|
| 183 |
+
.next()
|
| 184 |
+
.ok_or_else(|| "--coordinator requires an argument".to_string())?;
|
| 185 |
+
opt.coordinator_port = parse_port(&port, "--coordinator")?;
|
| 186 |
+
opt.coordinator_host = Some(host);
|
| 187 |
+
Ok(CliResult::Matched)
|
| 188 |
+
}
|
| 189 |
+
"--dist-prefill-chunk" => {
|
| 190 |
+
let value = rest
|
| 191 |
+
.next()
|
| 192 |
+
.ok_or_else(|| "--dist-prefill-chunk requires an argument".to_string())?;
|
| 193 |
+
opt.prefill_chunk = parse_positive_u32(&value, "--dist-prefill-chunk")?;
|
| 194 |
+
Ok(CliResult::Matched)
|
| 195 |
+
}
|
| 196 |
+
"--dist-prefill-window" => {
|
| 197 |
+
let value = rest
|
| 198 |
+
.next()
|
| 199 |
+
.ok_or_else(|| "--dist-prefill-window requires an argument".to_string())?;
|
| 200 |
+
opt.prefill_window = parse_positive_u32(&value, "--dist-prefill-window")?;
|
| 201 |
+
if opt.prefill_window > 64 {
|
| 202 |
+
return Err("--dist-prefill-window must be <= 64".into());
|
| 203 |
+
}
|
| 204 |
+
Ok(CliResult::Matched)
|
| 205 |
+
}
|
| 206 |
+
"--dist-activation-bits" => {
|
| 207 |
+
let value = rest
|
| 208 |
+
.next()
|
| 209 |
+
.ok_or_else(|| "--dist-activation-bits requires an argument".to_string())?;
|
| 210 |
+
let bits = parse_positive_u32(&value, "--dist-activation-bits")?;
|
| 211 |
+
if !bits_valid(bits) {
|
| 212 |
+
return Err("--dist-activation-bits must be 32, 16, or 8".into());
|
| 213 |
+
}
|
| 214 |
+
opt.activation_bits = bits;
|
| 215 |
+
Ok(CliResult::Matched)
|
| 216 |
+
}
|
| 217 |
+
"--dist-replay-check" => {
|
| 218 |
+
opt.replay_check = true;
|
| 219 |
+
Ok(CliResult::Matched)
|
| 220 |
+
}
|
| 221 |
+
"--debug" => {
|
| 222 |
+
opt.debug = true;
|
| 223 |
+
Ok(CliResult::Matched)
|
| 224 |
+
}
|
| 225 |
+
_ => Ok(CliResult::NotMatched),
|
| 226 |
+
}
|
| 227 |
+
}
|
| 228 |
+
|
| 229 |
+
pub fn parse_cli(args: &[String]) -> Result<(Options, Vec<String>), String> {
|
| 230 |
+
let mut opt = Options::default();
|
| 231 |
+
let mut unmatched = Vec::new();
|
| 232 |
+
let mut rest = args.iter().cloned().collect::<Vec<_>>().into_iter();
|
| 233 |
+
while let Some(arg) = rest.next() {
|
| 234 |
+
match parse_cli_arg(&arg, &mut rest, &mut opt)? {
|
| 235 |
+
CliResult::Matched => {}
|
| 236 |
+
CliResult::NotMatched => unmatched.push(arg),
|
| 237 |
+
CliResult::Error => unreachable!(),
|
| 238 |
+
}
|
| 239 |
+
}
|
| 240 |
+
Ok((opt, unmatched))
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
pub fn validate_options(opt: &Options) -> Result<(), String> {
|
| 244 |
+
if opt.role == Role::None {
|
| 245 |
+
if opt.layers.set
|
| 246 |
+
|| opt.listen_host.is_some()
|
| 247 |
+
|| opt.listen_port != 0
|
| 248 |
+
|| opt.coordinator_host.is_some()
|
| 249 |
+
|| opt.coordinator_port != 0
|
| 250 |
+
|| opt.prefill_chunk != 0
|
| 251 |
+
|| opt.prefill_window != 0
|
| 252 |
+
|| opt.activation_bits != 0
|
| 253 |
+
{
|
| 254 |
+
return Err("distributed options require --role coordinator or --role worker".into());
|
| 255 |
+
}
|
| 256 |
+
return Ok(());
|
| 257 |
+
}
|
| 258 |
+
if !opt.layers.set {
|
| 259 |
+
return Err(format!("--role {} requires --layers", opt.role.name()));
|
| 260 |
+
}
|
| 261 |
+
if opt.prefill_window > 64 {
|
| 262 |
+
return Err("--dist-prefill-window must be <= 64".into());
|
| 263 |
+
}
|
| 264 |
+
if opt.activation_bits != 0 && !bits_valid(opt.activation_bits) {
|
| 265 |
+
return Err("--dist-activation-bits must be 32, 16, or 8".into());
|
| 266 |
+
}
|
| 267 |
+
match opt.role {
|
| 268 |
+
Role::Coordinator => {
|
| 269 |
+
if opt.listen_host.is_none() || opt.listen_port <= 0 {
|
| 270 |
+
return Err("--role coordinator requires --listen HOST PORT".into());
|
| 271 |
+
}
|
| 272 |
+
if opt.coordinator_host.is_some() || opt.coordinator_port != 0 {
|
| 273 |
+
return Err("--role coordinator must not use --coordinator".into());
|
| 274 |
+
}
|
| 275 |
+
Ok(())
|
| 276 |
+
}
|
| 277 |
+
Role::Worker => {
|
| 278 |
+
if opt.coordinator_host.is_none() || opt.coordinator_port <= 0 {
|
| 279 |
+
return Err("--role worker requires --coordinator HOST PORT".into());
|
| 280 |
+
}
|
| 281 |
+
if opt.prefill_chunk != 0 {
|
| 282 |
+
return Err("--dist-prefill-chunk requires --role coordinator".into());
|
| 283 |
+
}
|
| 284 |
+
if opt.prefill_window != 0 {
|
| 285 |
+
return Err("--dist-prefill-window requires --role coordinator".into());
|
| 286 |
+
}
|
| 287 |
+
if opt.activation_bits != 0 {
|
| 288 |
+
return Err("--dist-activation-bits requires --role coordinator".into());
|
| 289 |
+
}
|
| 290 |
+
Ok(())
|
| 291 |
+
}
|
| 292 |
+
Role::None => Ok(()),
|
| 293 |
+
}
|
| 294 |
+
}
|
| 295 |
+
|
| 296 |
+
pub fn validate_layers_for_model(opt: &Options, n_layers: u32) -> Result<(), String> {
|
| 297 |
+
if opt.role == Role::None || !opt.layers.set {
|
| 298 |
+
return Ok(());
|
| 299 |
+
}
|
| 300 |
+
if n_layers == 0 {
|
| 301 |
+
return Err("model reports no layers".into());
|
| 302 |
+
}
|
| 303 |
+
let last = n_layers - 1;
|
| 304 |
+
if opt.layers.start > last {
|
| 305 |
+
return Err(format!("layer range starts past final model layer {last}"));
|
| 306 |
+
}
|
| 307 |
+
if !opt.layers.has_output && opt.layers.end > last {
|
| 308 |
+
return Err(format!("layer range ends past final model layer {last}"));
|
| 309 |
+
}
|
| 310 |
+
if opt.role == Role::Coordinator && opt.layers.start != 0 {
|
| 311 |
+
return Err("coordinator layer range must start at layer 0".into());
|
| 312 |
+
}
|
| 313 |
+
Ok(())
|
| 314 |
+
}
|
| 315 |
+
|
| 316 |
+
pub fn resolved_layer_end(layers: &Layers, n_layers: u32) -> u32 {
|
| 317 |
+
if layers.has_output {
|
| 318 |
+
n_layers.saturating_sub(1)
|
| 319 |
+
} else {
|
| 320 |
+
layers.end
|
| 321 |
+
}
|
| 322 |
+
}
|
| 323 |
+
|
| 324 |
+
pub fn prepare_engine_options(opt: &Options) -> Result<(), String> {
|
| 325 |
+
validate_options(opt)?;
|
| 326 |
+
if opt.replay_check && opt.role != Role::Coordinator {
|
| 327 |
+
return Err("--dist-replay-check requires --role coordinator".into());
|
| 328 |
+
}
|
| 329 |
+
Ok(())
|
| 330 |
+
}
|
| 331 |
+
|
| 332 |
+
pub const USAGE: &str = "\
|
| 333 |
+
--role ROLE\n\
|
| 334 |
+
Distributed role: coordinator or worker.\n\
|
| 335 |
+
--layers A:B\n\
|
| 336 |
+
Inclusive distributed layer slice, e.g. 10:20 or 21:output.\n\
|
| 337 |
+
--listen HOST PORT\n\
|
| 338 |
+
Coordinator TCP listen address. Workers may later use it to force their data listener.\n\
|
| 339 |
+
--coordinator HOST PORT\n\
|
| 340 |
+
Coordinator TCP address for --role worker.\n\
|
| 341 |
+
--dist-prefill-chunk N\n\
|
| 342 |
+
Coordinator prefill pipeline chunk size. Default: session cap, normally 4096.\n\
|
| 343 |
+
Non-default values are experimental and can change logits unless validated.\n\
|
| 344 |
+
--dist-prefill-window N\n\
|
| 345 |
+
Coordinator max end-to-end prefill chunks in flight. Default: workers+2, capped at 8.\n\
|
| 346 |
+
--dist-activation-bits N\n\
|
| 347 |
+
Coordinator hidden-state transport width: 32, 16, or 8. Default: 32.\n\
|
| 348 |
+
--dist-replay-check\n\
|
| 349 |
+
Coordinator diagnostic: reset and replay the prompt, then compare logits.\n\
|
| 350 |
+
--debug\n\
|
| 351 |
+
Print coordinator route/debug logs. Workers keep their normal logs without this.\n";
|
crates/ds4-dist/src/plan.rs
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Worker registry and route search. Matches `dist_coordinator_build_route_plan`.
|
| 2 |
+
|
| 3 |
+
use crate::codec::{ROUTE_F_OUTPUT_LOGITS, ROUTE_RETURN_UPSTREAM};
|
| 4 |
+
use crate::route::{encode_route_blob, ReturnTarget, RouteEntry};
|
| 5 |
+
|
| 6 |
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
| 7 |
+
pub struct WorkerInfo {
|
| 8 |
+
pub peer_host: String,
|
| 9 |
+
pub listen_port: u32,
|
| 10 |
+
pub model_id: u32,
|
| 11 |
+
pub quant_bits: u32,
|
| 12 |
+
pub layer_start: u32,
|
| 13 |
+
pub layer_end: u32,
|
| 14 |
+
pub has_output: bool,
|
| 15 |
+
pub has_hidden: bool,
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
#[derive(Debug, Clone)]
|
| 19 |
+
pub struct CoordinatorView {
|
| 20 |
+
pub n_layers: u32,
|
| 21 |
+
pub local_start: u32,
|
| 22 |
+
pub local_end: u32,
|
| 23 |
+
pub local_has_output: bool,
|
| 24 |
+
pub local_can_output_head: bool,
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
| 28 |
+
pub struct RoutePlan {
|
| 29 |
+
pub entries: Vec<RouteEntry>,
|
| 30 |
+
pub blob: Vec<u8>,
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
fn worker_cmp(a: &WorkerInfo, b: &WorkerInfo) -> std::cmp::Ordering {
|
| 34 |
+
a.layer_start
|
| 35 |
+
.cmp(&b.layer_start)
|
| 36 |
+
.then_with(|| b.has_output.cmp(&a.has_output))
|
| 37 |
+
.then_with(|| b.layer_end.cmp(&a.layer_end))
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
fn candidate_ok(view: &CoordinatorView, w: &WorkerInfo, last: u32) -> bool {
|
| 41 |
+
let needs_hidden = w.layer_end < last || !w.has_output;
|
| 42 |
+
if needs_hidden && !w.has_hidden {
|
| 43 |
+
return false;
|
| 44 |
+
}
|
| 45 |
+
if w.layer_end >= last && !w.has_output && !view.local_can_output_head {
|
| 46 |
+
return false;
|
| 47 |
+
}
|
| 48 |
+
true
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
fn search(
|
| 52 |
+
view: &CoordinatorView,
|
| 53 |
+
workers: &[WorkerInfo],
|
| 54 |
+
next: u32,
|
| 55 |
+
last: u32,
|
| 56 |
+
path: &mut Vec<usize>,
|
| 57 |
+
missing: &mut u32,
|
| 58 |
+
) -> bool {
|
| 59 |
+
let mut saw_start = false;
|
| 60 |
+
for (i, w) in workers.iter().enumerate() {
|
| 61 |
+
if w.layer_start < next {
|
| 62 |
+
continue;
|
| 63 |
+
}
|
| 64 |
+
if w.layer_start > next {
|
| 65 |
+
break;
|
| 66 |
+
}
|
| 67 |
+
saw_start = true;
|
| 68 |
+
if !candidate_ok(view, w, last) {
|
| 69 |
+
continue;
|
| 70 |
+
}
|
| 71 |
+
path.push(i);
|
| 72 |
+
if w.layer_end >= last {
|
| 73 |
+
return true;
|
| 74 |
+
}
|
| 75 |
+
let mut child_missing = w.layer_end + 1;
|
| 76 |
+
if search(view, workers, child_missing, last, path, &mut child_missing) {
|
| 77 |
+
return true;
|
| 78 |
+
}
|
| 79 |
+
if child_missing > *missing {
|
| 80 |
+
*missing = child_missing;
|
| 81 |
+
}
|
| 82 |
+
path.pop();
|
| 83 |
+
}
|
| 84 |
+
if !saw_start && next > *missing {
|
| 85 |
+
*missing = next;
|
| 86 |
+
}
|
| 87 |
+
false
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
pub fn build_route_plan(
|
| 91 |
+
view: &CoordinatorView,
|
| 92 |
+
workers: &[WorkerInfo],
|
| 93 |
+
) -> Result<RoutePlan, String> {
|
| 94 |
+
if view.local_start != 0 {
|
| 95 |
+
return Err("coordinator route does not start at layer 0".into());
|
| 96 |
+
}
|
| 97 |
+
let last = view.n_layers.saturating_sub(1);
|
| 98 |
+
if view.local_end == last && (view.local_has_output || view.local_can_output_head) {
|
| 99 |
+
return Ok(RoutePlan {
|
| 100 |
+
entries: Vec::new(),
|
| 101 |
+
blob: Vec::new(),
|
| 102 |
+
});
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
let mut sorted = workers.to_vec();
|
| 106 |
+
sorted.sort_by(worker_cmp);
|
| 107 |
+
let next = view.local_end + 1;
|
| 108 |
+
let mut path = Vec::new();
|
| 109 |
+
let mut missing = next;
|
| 110 |
+
if !search(view, &sorted, next, last, &mut path, &mut missing) {
|
| 111 |
+
return Err(format!(
|
| 112 |
+
"distributed route incomplete: missing layer {missing}"
|
| 113 |
+
));
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
let mut entries = Vec::new();
|
| 117 |
+
for idx in path {
|
| 118 |
+
let w = &sorted[idx];
|
| 119 |
+
entries.push(RouteEntry {
|
| 120 |
+
host: w.peer_host.clone(),
|
| 121 |
+
port: w.listen_port,
|
| 122 |
+
layer_start: w.layer_start,
|
| 123 |
+
layer_end: w.layer_end,
|
| 124 |
+
flags: if w.has_output {
|
| 125 |
+
ROUTE_F_OUTPUT_LOGITS
|
| 126 |
+
} else {
|
| 127 |
+
0
|
| 128 |
+
},
|
| 129 |
+
});
|
| 130 |
+
}
|
| 131 |
+
let blob = if entries.is_empty() {
|
| 132 |
+
Vec::new()
|
| 133 |
+
} else {
|
| 134 |
+
encode_route_blob(
|
| 135 |
+
&entries,
|
| 136 |
+
&ReturnTarget {
|
| 137 |
+
kind: ROUTE_RETURN_UPSTREAM,
|
| 138 |
+
host: String::new(),
|
| 139 |
+
port: 0,
|
| 140 |
+
},
|
| 141 |
+
)
|
| 142 |
+
.map_err(|e| e.to_string())?
|
| 143 |
+
};
|
| 144 |
+
Ok(RoutePlan { entries, blob })
|
| 145 |
+
}
|
| 146 |
+
|
| 147 |
+
/// Replace a stale worker with the same host/model/layers/output (C prepends).
|
| 148 |
+
pub fn register_worker(list: &mut Vec<WorkerInfo>, neu: WorkerInfo) {
|
| 149 |
+
list.retain(|old| {
|
| 150 |
+
!(old.peer_host == neu.peer_host
|
| 151 |
+
&& old.model_id == neu.model_id
|
| 152 |
+
&& old.layer_start == neu.layer_start
|
| 153 |
+
&& old.layer_end == neu.layer_end
|
| 154 |
+
&& old.has_output == neu.has_output)
|
| 155 |
+
});
|
| 156 |
+
list.insert(0, neu);
|
| 157 |
+
}
|
crates/ds4-dist/src/prefetch.rs
ADDED
|
@@ -0,0 +1,278 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Worker receive prefetch: `DS4_DIST_WORKER_PREFETCH_DEPTH` and the C job queue.
|
| 2 |
+
|
| 3 |
+
use std::collections::VecDeque;
|
| 4 |
+
use std::io::{self, Write};
|
| 5 |
+
use std::net::TcpStream;
|
| 6 |
+
use std::sync::{Arc, Condvar, Mutex};
|
| 7 |
+
use std::thread;
|
| 8 |
+
|
| 9 |
+
use crate::codec::{MSG_ERROR, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK};
|
| 10 |
+
use crate::exec::SliceExec;
|
| 11 |
+
use crate::native_snapshot::SnapshotStore;
|
| 12 |
+
use crate::transport::{read_frame, write_frame};
|
| 13 |
+
use crate::worker::Worker;
|
| 14 |
+
|
| 15 |
+
pub const PREFETCH_DEPTH_DEFAULT: u32 = 2;
|
| 16 |
+
pub const PREFETCH_DEPTH_MIN: u32 = 1;
|
| 17 |
+
pub const PREFETCH_DEPTH_MAX: u32 = 8;
|
| 18 |
+
pub const ERR_OOM_QUEUE: &str = "out of memory queueing distributed WORK";
|
| 19 |
+
pub const ERR_OOM_READ: &str = "out of memory reading distributed WORK frame";
|
| 20 |
+
|
| 21 |
+
pub fn prefetch_depth_from(env: Option<&str>) -> u32 {
|
| 22 |
+
let Some(raw) = env else {
|
| 23 |
+
return PREFETCH_DEPTH_DEFAULT;
|
| 24 |
+
};
|
| 25 |
+
if raw.is_empty() {
|
| 26 |
+
return PREFETCH_DEPTH_DEFAULT;
|
| 27 |
+
}
|
| 28 |
+
match parse_strtol(raw) {
|
| 29 |
+
Some(v) if v >= i64::from(PREFETCH_DEPTH_MIN) && v <= i64::from(PREFETCH_DEPTH_MAX) => {
|
| 30 |
+
v as u32
|
| 31 |
+
}
|
| 32 |
+
_ => PREFETCH_DEPTH_DEFAULT,
|
| 33 |
+
}
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
pub fn prefetch_depth() -> u32 {
|
| 37 |
+
prefetch_depth_from(
|
| 38 |
+
std::env::var("DS4_DIST_WORKER_PREFETCH_DEPTH")
|
| 39 |
+
.ok()
|
| 40 |
+
.as_deref(),
|
| 41 |
+
)
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
pub fn prefetch_disabled_from(env: Option<&std::ffi::OsStr>) -> bool {
|
| 45 |
+
env.is_some()
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
pub fn prefetch_disabled() -> bool {
|
| 49 |
+
prefetch_disabled_from(std::env::var_os("DS4_DIST_DISABLE_WORKER_PREFETCH").as_deref())
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
pub fn prefetch_enabled_message(depth: u32) -> String {
|
| 53 |
+
format!("ds4: distributed worker: receive prefetch depth {depth} enabled")
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
fn parse_strtol(s: &str) -> Option<i64> {
|
| 57 |
+
let s = s.trim_start_matches(|c: char| c.is_ascii_whitespace());
|
| 58 |
+
if s.is_empty() {
|
| 59 |
+
return None;
|
| 60 |
+
}
|
| 61 |
+
let (neg, digits) = match s.as_bytes()[0] {
|
| 62 |
+
b'+' => (false, &s[1..]),
|
| 63 |
+
b'-' => (true, &s[1..]),
|
| 64 |
+
_ => (false, s),
|
| 65 |
+
};
|
| 66 |
+
if digits.is_empty() || !digits.as_bytes()[0].is_ascii_digit() {
|
| 67 |
+
return None;
|
| 68 |
+
}
|
| 69 |
+
if !digits.bytes().all(|b| b.is_ascii_digit()) {
|
| 70 |
+
return None;
|
| 71 |
+
}
|
| 72 |
+
let v = digits.parse::<i64>().ok()?;
|
| 73 |
+
Some(if neg { -v } else { v })
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
struct QueueInner<T> {
|
| 77 |
+
depth: u32,
|
| 78 |
+
queued: u32,
|
| 79 |
+
jobs: VecDeque<T>,
|
| 80 |
+
closed: bool,
|
| 81 |
+
canceled: bool,
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
pub struct JobQueue<T = Vec<u8>> {
|
| 85 |
+
inner: Mutex<QueueInner<T>>,
|
| 86 |
+
not_empty: Condvar,
|
| 87 |
+
not_full: Condvar,
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
impl<T> JobQueue<T> {
|
| 91 |
+
pub fn new(depth: u32) -> Self {
|
| 92 |
+
let depth = depth.max(1);
|
| 93 |
+
Self {
|
| 94 |
+
inner: Mutex::new(QueueInner {
|
| 95 |
+
depth,
|
| 96 |
+
queued: 0,
|
| 97 |
+
jobs: VecDeque::new(),
|
| 98 |
+
closed: false,
|
| 99 |
+
canceled: false,
|
| 100 |
+
}),
|
| 101 |
+
not_empty: Condvar::new(),
|
| 102 |
+
not_full: Condvar::new(),
|
| 103 |
+
}
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
pub fn depth(&self) -> u32 {
|
| 107 |
+
self.inner.lock().expect("prefetch queue").depth
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
pub fn queued(&self) -> u32 {
|
| 111 |
+
self.inner.lock().expect("prefetch queue").queued
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
pub fn enqueue(&self, job: T) -> bool {
|
| 115 |
+
let mut g = self.inner.lock().expect("prefetch queue");
|
| 116 |
+
while !g.closed && !g.canceled && g.queued >= g.depth {
|
| 117 |
+
g = self.not_full.wait(g).expect("prefetch queue");
|
| 118 |
+
}
|
| 119 |
+
if g.closed || g.canceled {
|
| 120 |
+
return false;
|
| 121 |
+
}
|
| 122 |
+
g.jobs.push_back(job);
|
| 123 |
+
g.queued += 1;
|
| 124 |
+
self.not_empty.notify_one();
|
| 125 |
+
true
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
pub fn wait_until_queued(&self, n: u32) {
|
| 129 |
+
let mut g = self.inner.lock().expect("prefetch queue");
|
| 130 |
+
while g.queued < n && !g.closed && !g.canceled {
|
| 131 |
+
g = self.not_empty.wait(g).expect("prefetch queue");
|
| 132 |
+
}
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
fn take_job(g: &mut QueueInner<T>, not_full: &Condvar) -> Option<T> {
|
| 136 |
+
if g.canceled || g.jobs.is_empty() {
|
| 137 |
+
return None;
|
| 138 |
+
}
|
| 139 |
+
let job = g.jobs.pop_front();
|
| 140 |
+
g.queued = g.queued.saturating_sub(1);
|
| 141 |
+
not_full.notify_one();
|
| 142 |
+
job
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
pub fn try_pop(&self) -> Option<T> {
|
| 146 |
+
let mut g = self.inner.lock().expect("prefetch queue");
|
| 147 |
+
Self::take_job(&mut g, &self.not_full)
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
pub fn pop(&self) -> Option<T> {
|
| 151 |
+
let mut g = self.inner.lock().expect("prefetch queue");
|
| 152 |
+
while g.jobs.is_empty() && !g.closed && !g.canceled {
|
| 153 |
+
g = self.not_empty.wait(g).expect("prefetch queue");
|
| 154 |
+
}
|
| 155 |
+
Self::take_job(&mut g, &self.not_full)
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
pub fn finish(&self) {
|
| 159 |
+
let mut g = self.inner.lock().expect("prefetch queue");
|
| 160 |
+
g.closed = true;
|
| 161 |
+
self.not_empty.notify_all();
|
| 162 |
+
self.not_full.notify_all();
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
pub fn cancel(&self) {
|
| 166 |
+
let mut g = self.inner.lock().expect("prefetch queue");
|
| 167 |
+
g.closed = true;
|
| 168 |
+
g.canceled = true;
|
| 169 |
+
g.jobs.clear();
|
| 170 |
+
g.queued = 0;
|
| 171 |
+
self.not_empty.notify_all();
|
| 172 |
+
self.not_full.notify_all();
|
| 173 |
+
}
|
| 174 |
+
}
|
| 175 |
+
|
| 176 |
+
pub(crate) struct MutexWrite<'a>(pub &'a Mutex<TcpStream>);
|
| 177 |
+
|
| 178 |
+
impl Write for MutexWrite<'_> {
|
| 179 |
+
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
| 180 |
+
self.0.lock().expect("worker write").write(buf)
|
| 181 |
+
}
|
| 182 |
+
|
| 183 |
+
fn flush(&mut self) -> io::Result<()> {
|
| 184 |
+
self.0.lock().expect("worker write").flush()
|
| 185 |
+
}
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
impl<E: SliceExec + Send, S: SnapshotStore + Send> Worker<E, S> {
|
| 189 |
+
pub fn serve_prefetch(&mut self, stream: &mut TcpStream) -> io::Result<()> {
|
| 190 |
+
let depth = prefetch_depth();
|
| 191 |
+
let queue = Arc::new(JobQueue::<Vec<u8>>::new(depth));
|
| 192 |
+
let worker = Mutex::new(&mut *self);
|
| 193 |
+
let writer = Arc::new(Mutex::new(stream.try_clone()?));
|
| 194 |
+
let mut reader = stream.try_clone()?;
|
| 195 |
+
worker
|
| 196 |
+
.lock()
|
| 197 |
+
.expect("worker hops")
|
| 198 |
+
.bind_hops(Arc::clone(&writer));
|
| 199 |
+
eprintln!("{}", prefetch_enabled_message(depth));
|
| 200 |
+
|
| 201 |
+
let rc = thread::scope(|scope| {
|
| 202 |
+
let eval = scope.spawn(|| -> io::Result<()> {
|
| 203 |
+
while let Some(payload) = queue.pop() {
|
| 204 |
+
let reply = {
|
| 205 |
+
let mut w = worker.lock().expect("worker eval");
|
| 206 |
+
w.process_work(&payload, &mut MutexWrite(writer.as_ref()))?
|
| 207 |
+
};
|
| 208 |
+
if let Some(frame) = reply {
|
| 209 |
+
writer.lock().expect("worker write").write_all(&frame)?;
|
| 210 |
+
}
|
| 211 |
+
}
|
| 212 |
+
Ok(())
|
| 213 |
+
});
|
| 214 |
+
|
| 215 |
+
let mut loop_err: Option<io::Error> = None;
|
| 216 |
+
loop {
|
| 217 |
+
let (typ, body) = match read_frame(&mut reader) {
|
| 218 |
+
Ok(v) => v,
|
| 219 |
+
Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => break,
|
| 220 |
+
Err(e) => {
|
| 221 |
+
loop_err = Some(e);
|
| 222 |
+
break;
|
| 223 |
+
}
|
| 224 |
+
};
|
| 225 |
+
if typ == MSG_ERROR {
|
| 226 |
+
loop_err = Some(io::Error::new(
|
| 227 |
+
io::ErrorKind::Other,
|
| 228 |
+
format!("coordinator error: {}", String::from_utf8_lossy(&body)),
|
| 229 |
+
));
|
| 230 |
+
break;
|
| 231 |
+
}
|
| 232 |
+
if typ == MSG_WORK {
|
| 233 |
+
if !queue.enqueue(body) {
|
| 234 |
+
loop_err = Some(io::Error::new(
|
| 235 |
+
io::ErrorKind::Interrupted,
|
| 236 |
+
"prefetch queue canceled",
|
| 237 |
+
));
|
| 238 |
+
break;
|
| 239 |
+
}
|
| 240 |
+
continue;
|
| 241 |
+
}
|
| 242 |
+
if typ == MSG_SNAPSHOT_SAVE_REQ || typ == MSG_SNAPSHOT_LOAD_BEGIN {
|
| 243 |
+
let mut w = worker.lock().expect("worker snapshot");
|
| 244 |
+
let mut out = writer.lock().expect("worker snapshot");
|
| 245 |
+
w.apply_snapshot(&mut *out, typ, &body)?;
|
| 246 |
+
continue;
|
| 247 |
+
}
|
| 248 |
+
{
|
| 249 |
+
let mut out = writer.lock().expect("worker write");
|
| 250 |
+
write_frame(
|
| 251 |
+
&mut *out,
|
| 252 |
+
MSG_ERROR,
|
| 253 |
+
b"unsupported distributed worker frame",
|
| 254 |
+
)?;
|
| 255 |
+
}
|
| 256 |
+
loop_err = Some(io::Error::new(
|
| 257 |
+
io::ErrorKind::InvalidData,
|
| 258 |
+
format!("rejected unsupported frame type {typ}"),
|
| 259 |
+
));
|
| 260 |
+
break;
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
if loop_err.is_some() {
|
| 264 |
+
queue.cancel();
|
| 265 |
+
} else {
|
| 266 |
+
queue.finish();
|
| 267 |
+
}
|
| 268 |
+
let eval_rc = eval.join().expect("prefetch eval");
|
| 269 |
+
match (loop_err, eval_rc) {
|
| 270 |
+
(Some(e), _) => Err(e),
|
| 271 |
+
(None, Err(e)) => Err(e),
|
| 272 |
+
(None, Ok(())) => Ok(()),
|
| 273 |
+
}
|
| 274 |
+
});
|
| 275 |
+
self.shutdown_hops();
|
| 276 |
+
rc
|
| 277 |
+
}
|
| 278 |
+
}
|
crates/ds4-dist/src/prefetch_local.rs
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! !Send receive-prefetch: queue on a helper thread, eval on the session thread.
|
| 2 |
+
|
| 3 |
+
use std::io::{self, ErrorKind, Write};
|
| 4 |
+
use std::net::{Shutdown, TcpStream};
|
| 5 |
+
use std::sync::mpsc::{self, Receiver};
|
| 6 |
+
use std::sync::{Arc, Mutex};
|
| 7 |
+
use std::thread::{self, JoinHandle};
|
| 8 |
+
|
| 9 |
+
use crate::codec::{MSG_ERROR, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK};
|
| 10 |
+
use crate::exec::SliceExec;
|
| 11 |
+
use crate::native_snapshot::SnapshotStore;
|
| 12 |
+
use crate::prefetch::{prefetch_depth, prefetch_enabled_message, JobQueue, MutexWrite};
|
| 13 |
+
use crate::serve_local::{drive_ready, Drive, IDLE};
|
| 14 |
+
use crate::transport::{read_frame, write_frame};
|
| 15 |
+
use crate::worker::Worker;
|
| 16 |
+
|
| 17 |
+
pub enum PrefetchJob {
|
| 18 |
+
Work(Vec<u8>),
|
| 19 |
+
Snapshot { typ: u32, body: Vec<u8> },
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
pub fn local_prefetch_enabled_from(env: Option<&std::ffi::OsStr>) -> bool {
|
| 23 |
+
!crate::prefetch::prefetch_disabled_from(env)
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
pub fn local_prefetch_enabled() -> bool {
|
| 27 |
+
local_prefetch_enabled_from(std::env::var_os("DS4_DIST_DISABLE_WORKER_PREFETCH").as_deref())
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
fn recv_prefetch(
|
| 31 |
+
mut reader: TcpStream,
|
| 32 |
+
queue: &JobQueue<PrefetchJob>,
|
| 33 |
+
writer: &Mutex<TcpStream>,
|
| 34 |
+
mut on_job: impl FnMut(),
|
| 35 |
+
) -> Option<io::Error> {
|
| 36 |
+
let err = loop {
|
| 37 |
+
match read_frame(&mut reader) {
|
| 38 |
+
Ok((typ, body)) if typ == MSG_WORK => {
|
| 39 |
+
if !queue.enqueue(PrefetchJob::Work(body)) {
|
| 40 |
+
break Some(io::Error::new(
|
| 41 |
+
ErrorKind::Interrupted,
|
| 42 |
+
"prefetch queue canceled",
|
| 43 |
+
));
|
| 44 |
+
}
|
| 45 |
+
on_job();
|
| 46 |
+
}
|
| 47 |
+
Ok((typ, body)) if typ == MSG_SNAPSHOT_SAVE_REQ || typ == MSG_SNAPSHOT_LOAD_BEGIN => {
|
| 48 |
+
if !queue.enqueue(PrefetchJob::Snapshot { typ, body }) {
|
| 49 |
+
break Some(io::Error::new(
|
| 50 |
+
ErrorKind::Interrupted,
|
| 51 |
+
"prefetch queue canceled",
|
| 52 |
+
));
|
| 53 |
+
}
|
| 54 |
+
on_job();
|
| 55 |
+
}
|
| 56 |
+
Ok((typ, body)) if typ == MSG_ERROR => {
|
| 57 |
+
break Some(io::Error::new(
|
| 58 |
+
ErrorKind::Other,
|
| 59 |
+
format!("coordinator error: {}", String::from_utf8_lossy(&body)),
|
| 60 |
+
));
|
| 61 |
+
}
|
| 62 |
+
Ok((typ, _)) => {
|
| 63 |
+
let _ = write_frame(
|
| 64 |
+
&mut *writer.lock().expect("worker write"),
|
| 65 |
+
MSG_ERROR,
|
| 66 |
+
b"unsupported distributed worker frame",
|
| 67 |
+
);
|
| 68 |
+
break Some(io::Error::new(
|
| 69 |
+
ErrorKind::InvalidData,
|
| 70 |
+
format!("rejected unsupported frame type {typ}"),
|
| 71 |
+
));
|
| 72 |
+
}
|
| 73 |
+
Err(e) if e.kind() == ErrorKind::UnexpectedEof => break None,
|
| 74 |
+
Err(e)
|
| 75 |
+
if matches!(
|
| 76 |
+
e.kind(),
|
| 77 |
+
ErrorKind::TimedOut | ErrorKind::WouldBlock | ErrorKind::Interrupted
|
| 78 |
+
) =>
|
| 79 |
+
{
|
| 80 |
+
continue;
|
| 81 |
+
}
|
| 82 |
+
Err(e) => break Some(e),
|
| 83 |
+
}
|
| 84 |
+
};
|
| 85 |
+
if err.is_some() {
|
| 86 |
+
queue.cancel();
|
| 87 |
+
} else {
|
| 88 |
+
queue.finish();
|
| 89 |
+
}
|
| 90 |
+
on_job();
|
| 91 |
+
err
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
fn apply_prefetch_job<E, S>(
|
| 95 |
+
worker: &mut Worker<E, S>,
|
| 96 |
+
writer: &Arc<Mutex<TcpStream>>,
|
| 97 |
+
job: PrefetchJob,
|
| 98 |
+
) -> io::Result<()>
|
| 99 |
+
where
|
| 100 |
+
E: SliceExec,
|
| 101 |
+
S: SnapshotStore,
|
| 102 |
+
{
|
| 103 |
+
match job {
|
| 104 |
+
PrefetchJob::Work(payload) => {
|
| 105 |
+
let reply = worker.process_work(&payload, &mut MutexWrite(writer))?;
|
| 106 |
+
if let Some(frame) = reply {
|
| 107 |
+
writer.lock().expect("worker write").write_all(&frame)?;
|
| 108 |
+
}
|
| 109 |
+
Ok(())
|
| 110 |
+
}
|
| 111 |
+
PrefetchJob::Snapshot { typ, body } => {
|
| 112 |
+
let mut out = writer.lock().expect("worker snapshot");
|
| 113 |
+
worker.apply_snapshot(&mut *out, typ, &body)
|
| 114 |
+
}
|
| 115 |
+
}
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
pub fn serve_prefetch_local_with<E, S>(
|
| 119 |
+
worker: &mut Worker<E, S>,
|
| 120 |
+
stream: &mut TcpStream,
|
| 121 |
+
queue: &JobQueue<PrefetchJob>,
|
| 122 |
+
) -> io::Result<()>
|
| 123 |
+
where
|
| 124 |
+
E: SliceExec,
|
| 125 |
+
S: SnapshotStore,
|
| 126 |
+
{
|
| 127 |
+
let writer = Arc::new(Mutex::new(stream.try_clone()?));
|
| 128 |
+
let reader = stream.try_clone()?;
|
| 129 |
+
worker.bind_hops(Arc::clone(&writer));
|
| 130 |
+
eprintln!("{}", prefetch_enabled_message(queue.depth()));
|
| 131 |
+
let rc = thread::scope(|scope| {
|
| 132 |
+
let recv = scope.spawn(|| recv_prefetch(reader, queue, &writer, || {}));
|
| 133 |
+
let mut eval_err = None;
|
| 134 |
+
while let Some(job) = queue.pop() {
|
| 135 |
+
if let Err(e) = apply_prefetch_job(worker, &writer, job) {
|
| 136 |
+
eval_err = Some(e);
|
| 137 |
+
queue.cancel();
|
| 138 |
+
break;
|
| 139 |
+
}
|
| 140 |
+
}
|
| 141 |
+
match (recv.join().expect("prefetch recv"), eval_err) {
|
| 142 |
+
(Some(e), _) => Err(e),
|
| 143 |
+
(None, Some(e)) => Err(e),
|
| 144 |
+
(None, None) => Ok(()),
|
| 145 |
+
}
|
| 146 |
+
});
|
| 147 |
+
worker.shutdown_hops();
|
| 148 |
+
rc
|
| 149 |
+
}
|
| 150 |
+
|
| 151 |
+
struct PrefetchConn {
|
| 152 |
+
queue: Arc<JobQueue<PrefetchJob>>,
|
| 153 |
+
writer: Arc<Mutex<TcpStream>>,
|
| 154 |
+
recv: Option<JoinHandle<()>>,
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
fn start_prefetch_recv(
|
| 158 |
+
stream: TcpStream,
|
| 159 |
+
wake: mpsc::Sender<usize>,
|
| 160 |
+
idx: usize,
|
| 161 |
+
) -> io::Result<PrefetchConn> {
|
| 162 |
+
let _ = stream.set_nodelay(true);
|
| 163 |
+
let writer = Arc::new(Mutex::new(stream.try_clone()?));
|
| 164 |
+
let queue = Arc::new(JobQueue::<PrefetchJob>::new(prefetch_depth()));
|
| 165 |
+
let q = Arc::clone(&queue);
|
| 166 |
+
let w = Arc::clone(&writer);
|
| 167 |
+
eprintln!("{}", prefetch_enabled_message(queue.depth()));
|
| 168 |
+
let recv = thread::Builder::new()
|
| 169 |
+
.name("ds4-dist-prefetch-recv".into())
|
| 170 |
+
.spawn(move || {
|
| 171 |
+
let _ = recv_prefetch(stream, &q, &w, || {
|
| 172 |
+
let _ = wake.send(idx);
|
| 173 |
+
});
|
| 174 |
+
})?;
|
| 175 |
+
Ok(PrefetchConn {
|
| 176 |
+
queue,
|
| 177 |
+
writer,
|
| 178 |
+
recv: Some(recv),
|
| 179 |
+
})
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
fn shutdown_prefetch_conns(conns: &mut [PrefetchConn]) {
|
| 183 |
+
for conn in conns.iter() {
|
| 184 |
+
conn.queue.cancel();
|
| 185 |
+
let _ = conn
|
| 186 |
+
.writer
|
| 187 |
+
.lock()
|
| 188 |
+
.expect("worker write")
|
| 189 |
+
.shutdown(Shutdown::Both);
|
| 190 |
+
}
|
| 191 |
+
for conn in conns.iter_mut() {
|
| 192 |
+
if let Some(handle) = conn.recv.take() {
|
| 193 |
+
let _ = handle.join();
|
| 194 |
+
}
|
| 195 |
+
}
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
fn drain_conn<E, S>(worker: &mut Worker<E, S>, conn: &PrefetchConn) -> io::Result<()>
|
| 199 |
+
where
|
| 200 |
+
E: SliceExec,
|
| 201 |
+
S: SnapshotStore,
|
| 202 |
+
{
|
| 203 |
+
worker.bind_hops(Arc::clone(&conn.writer));
|
| 204 |
+
while let Some(job) = conn.queue.try_pop() {
|
| 205 |
+
apply_prefetch_job(worker, &conn.writer, job)?;
|
| 206 |
+
}
|
| 207 |
+
Ok(())
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
pub(crate) fn serve_coordinator_and_hops_prefetch<E, S, Stop>(
|
| 211 |
+
worker: &mut Worker<E, S>,
|
| 212 |
+
coordinator: &mut TcpStream,
|
| 213 |
+
hops: Option<&Receiver<TcpStream>>,
|
| 214 |
+
should_stop: &mut Stop,
|
| 215 |
+
) -> io::Result<()>
|
| 216 |
+
where
|
| 217 |
+
E: SliceExec,
|
| 218 |
+
S: SnapshotStore,
|
| 219 |
+
Stop: FnMut() -> bool,
|
| 220 |
+
{
|
| 221 |
+
coordinator.set_read_timeout(Some(IDLE))?;
|
| 222 |
+
let (wake_tx, wake_rx) = mpsc::channel();
|
| 223 |
+
let mut conns: Vec<PrefetchConn> = Vec::new();
|
| 224 |
+
let rc = 'run: loop {
|
| 225 |
+
if should_stop() {
|
| 226 |
+
break Ok(());
|
| 227 |
+
}
|
| 228 |
+
if let Some(rx) = hops {
|
| 229 |
+
while let Ok(hop) = rx.try_recv() {
|
| 230 |
+
let idx = conns.len();
|
| 231 |
+
match start_prefetch_recv(hop, wake_tx.clone(), idx) {
|
| 232 |
+
Ok(conn) => conns.push(conn),
|
| 233 |
+
Err(e) => break 'run Err(e),
|
| 234 |
+
}
|
| 235 |
+
}
|
| 236 |
+
}
|
| 237 |
+
while let Ok(idx) = wake_rx.try_recv() {
|
| 238 |
+
if let Some(conn) = conns.get(idx) {
|
| 239 |
+
if let Err(e) = drain_conn(worker, conn) {
|
| 240 |
+
break 'run Err(e);
|
| 241 |
+
}
|
| 242 |
+
}
|
| 243 |
+
}
|
| 244 |
+
match drive_ready(worker, coordinator) {
|
| 245 |
+
Drive::Continue => {}
|
| 246 |
+
Drive::Closed => break Ok(()),
|
| 247 |
+
Drive::Failed(e) => break Err(e),
|
| 248 |
+
}
|
| 249 |
+
};
|
| 250 |
+
shutdown_prefetch_conns(&mut conns);
|
| 251 |
+
worker.shutdown_hops();
|
| 252 |
+
rc
|
| 253 |
+
}
|
crates/ds4-dist/src/reconnect.rs
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Worker reconnect: connect retries, HELLO, session clear, 1s backoff.
|
| 2 |
+
|
| 3 |
+
use std::io::{self, ErrorKind};
|
| 4 |
+
use std::net::{SocketAddr, TcpListener, TcpStream};
|
| 5 |
+
use std::thread;
|
| 6 |
+
use std::time::Duration;
|
| 7 |
+
|
| 8 |
+
use crate::codec::Hello;
|
| 9 |
+
use crate::exec::SliceExec;
|
| 10 |
+
use crate::native_snapshot::SnapshotStore;
|
| 11 |
+
use crate::prefetch::prefetch_disabled;
|
| 12 |
+
use crate::prefetch_local::{local_prefetch_enabled, serve_coordinator_and_hops_prefetch};
|
| 13 |
+
use crate::serve_local::{serve_coordinator_and_hops, spawn_hop_accept, HopAccept};
|
| 14 |
+
use crate::worker::{send_hello, Worker};
|
| 15 |
+
|
| 16 |
+
pub const RECONNECT_SLEEP: Duration = Duration::from_secs(1);
|
| 17 |
+
pub const CONNECT_RETRY_DELAY: Duration = Duration::from_millis(25);
|
| 18 |
+
pub const CONNECT_RETRY_ATTEMPTS: u32 = 200;
|
| 19 |
+
|
| 20 |
+
pub fn sleep_reconnect() {
|
| 21 |
+
thread::sleep(RECONNECT_SLEEP);
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
pub fn connect_retryable(err: &io::Error) -> bool {
|
| 25 |
+
matches!(
|
| 26 |
+
err.kind(),
|
| 27 |
+
ErrorKind::ConnectionRefused
|
| 28 |
+
| ErrorKind::HostUnreachable
|
| 29 |
+
| ErrorKind::NetworkUnreachable
|
| 30 |
+
| ErrorKind::TimedOut
|
| 31 |
+
| ErrorKind::AddrNotAvailable
|
| 32 |
+
)
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
pub fn connect_error(host: &str, port: u16, err: &io::Error) -> String {
|
| 36 |
+
format!("unable to connect to {host}:{port}: {err}")
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
pub fn retrying_message(err: &str) -> String {
|
| 40 |
+
format!("ds4: distributed worker: {err}; retrying")
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
pub fn connected_message(host: &str, port: &str) -> String {
|
| 44 |
+
format!("ds4: distributed worker: connected to coordinator {host}:{port}")
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
pub fn hello_failed_message(err: &io::Error) -> String {
|
| 48 |
+
format!("ds4: distributed worker: failed to send HELLO: {err}")
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
pub fn cleared_sessions_message(n: u32) -> String {
|
| 52 |
+
format!("ds4: distributed worker: cleared {n} sessions after coordinator disconnect")
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
pub fn disconnected_message(after_error: bool) -> String {
|
| 56 |
+
if after_error {
|
| 57 |
+
"ds4: distributed worker: coordinator disconnected after error; reconnecting".into()
|
| 58 |
+
} else {
|
| 59 |
+
"ds4: distributed worker: coordinator disconnected; reconnecting".into()
|
| 60 |
+
}
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
pub fn connect_endpoint_once(host: &str, port: u16) -> io::Result<TcpStream> {
|
| 64 |
+
let stream = TcpStream::connect((host, port))?;
|
| 65 |
+
let _ = stream.set_nodelay(true);
|
| 66 |
+
Ok(stream)
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
pub fn connect_endpoint(host: &str, port: u16) -> io::Result<TcpStream> {
|
| 70 |
+
let mut last = None;
|
| 71 |
+
for _ in 0..CONNECT_RETRY_ATTEMPTS {
|
| 72 |
+
match connect_endpoint_once(host, port) {
|
| 73 |
+
Ok(s) => return Ok(s),
|
| 74 |
+
Err(e) => {
|
| 75 |
+
let retry = connect_retryable(&e);
|
| 76 |
+
last = Some(e);
|
| 77 |
+
if !retry {
|
| 78 |
+
break;
|
| 79 |
+
}
|
| 80 |
+
thread::sleep(CONNECT_RETRY_DELAY);
|
| 81 |
+
}
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
Err(last.unwrap_or_else(|| io::Error::new(ErrorKind::Other, "unable to connect")))
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
pub fn peer_name(stream: &TcpStream) -> (String, String) {
|
| 88 |
+
match stream.peer_addr() {
|
| 89 |
+
Ok(SocketAddr::V4(a)) => (a.ip().to_string(), a.port().to_string()),
|
| 90 |
+
Ok(SocketAddr::V6(a)) => (a.ip().to_string(), a.port().to_string()),
|
| 91 |
+
Err(_) => ("?".into(), "?".into()),
|
| 92 |
+
}
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
pub struct LocalReconnect<'a, Connect, Sleep, Stop> {
|
| 96 |
+
pub connect: Connect,
|
| 97 |
+
pub hello: &'a Hello,
|
| 98 |
+
pub model_name: &'a str,
|
| 99 |
+
pub sleep: Sleep,
|
| 100 |
+
pub should_stop: Stop,
|
| 101 |
+
pub listener: Option<&'a TcpListener>,
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
fn reconnect_loop<E, S, Connect, Sleep, Stop, Serve>(
|
| 105 |
+
worker: &mut Worker<E, S>,
|
| 106 |
+
connect: &mut Connect,
|
| 107 |
+
hello: &Hello,
|
| 108 |
+
model_name: &str,
|
| 109 |
+
sleep: &mut Sleep,
|
| 110 |
+
should_stop: &mut Stop,
|
| 111 |
+
mut serve: Serve,
|
| 112 |
+
) -> io::Result<()>
|
| 113 |
+
where
|
| 114 |
+
E: SliceExec,
|
| 115 |
+
S: SnapshotStore,
|
| 116 |
+
Connect: FnMut() -> io::Result<TcpStream>,
|
| 117 |
+
Sleep: FnMut(),
|
| 118 |
+
Stop: FnMut() -> bool,
|
| 119 |
+
Serve: FnMut(&mut Worker<E, S>, &mut TcpStream, &mut Stop) -> io::Result<()>,
|
| 120 |
+
{
|
| 121 |
+
while !should_stop() {
|
| 122 |
+
let mut stream = match connect() {
|
| 123 |
+
Ok(s) => s,
|
| 124 |
+
Err(e) => {
|
| 125 |
+
eprintln!("{}", retrying_message(&e.to_string()));
|
| 126 |
+
sleep();
|
| 127 |
+
continue;
|
| 128 |
+
}
|
| 129 |
+
};
|
| 130 |
+
let (peer_host, peer_port) = peer_name(&stream);
|
| 131 |
+
eprintln!("{}", connected_message(&peer_host, &peer_port));
|
| 132 |
+
if let Err(e) = send_hello(&mut stream, hello, model_name) {
|
| 133 |
+
eprintln!("{}", hello_failed_message(&e));
|
| 134 |
+
sleep();
|
| 135 |
+
continue;
|
| 136 |
+
}
|
| 137 |
+
let rc = serve(worker, &mut stream, should_stop);
|
| 138 |
+
let after_error = rc.is_err();
|
| 139 |
+
let dropped = worker.clear_sessions();
|
| 140 |
+
if dropped != 0 {
|
| 141 |
+
eprintln!("{}", cleared_sessions_message(dropped));
|
| 142 |
+
}
|
| 143 |
+
eprintln!("{}", disconnected_message(after_error));
|
| 144 |
+
if should_stop() {
|
| 145 |
+
return rc;
|
| 146 |
+
}
|
| 147 |
+
sleep();
|
| 148 |
+
if after_error {
|
| 149 |
+
let _ = rc;
|
| 150 |
+
}
|
| 151 |
+
}
|
| 152 |
+
Ok(())
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
pub fn reconnect_with<E, S, Connect, Sleep, Stop>(
|
| 156 |
+
worker: &mut Worker<E, S>,
|
| 157 |
+
mut connect: Connect,
|
| 158 |
+
hello: &Hello,
|
| 159 |
+
model_name: &str,
|
| 160 |
+
mut sleep: Sleep,
|
| 161 |
+
mut should_stop: Stop,
|
| 162 |
+
use_prefetch: bool,
|
| 163 |
+
) -> io::Result<()>
|
| 164 |
+
where
|
| 165 |
+
E: SliceExec + Send,
|
| 166 |
+
S: SnapshotStore + Send,
|
| 167 |
+
Connect: FnMut() -> io::Result<TcpStream>,
|
| 168 |
+
Sleep: FnMut(),
|
| 169 |
+
Stop: FnMut() -> bool,
|
| 170 |
+
{
|
| 171 |
+
reconnect_loop(
|
| 172 |
+
worker,
|
| 173 |
+
&mut connect,
|
| 174 |
+
hello,
|
| 175 |
+
model_name,
|
| 176 |
+
&mut sleep,
|
| 177 |
+
&mut should_stop,
|
| 178 |
+
|w, stream, _stop| {
|
| 179 |
+
if use_prefetch && !prefetch_disabled() {
|
| 180 |
+
w.serve_prefetch(stream)
|
| 181 |
+
} else {
|
| 182 |
+
w.serve(stream)
|
| 183 |
+
}
|
| 184 |
+
},
|
| 185 |
+
)
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
/// Reconnect a `!Send` worker, polling the coordinator and accepted hop streams.
|
| 189 |
+
pub fn reconnect_local<E, S, Connect, Sleep, Stop>(
|
| 190 |
+
worker: &mut Worker<E, S>,
|
| 191 |
+
mut spec: LocalReconnect<'_, Connect, Sleep, Stop>,
|
| 192 |
+
) -> io::Result<()>
|
| 193 |
+
where
|
| 194 |
+
E: SliceExec,
|
| 195 |
+
S: SnapshotStore,
|
| 196 |
+
Connect: FnMut() -> io::Result<TcpStream>,
|
| 197 |
+
Sleep: FnMut(),
|
| 198 |
+
Stop: FnMut() -> bool,
|
| 199 |
+
{
|
| 200 |
+
let hops = spec.listener.map(spawn_hop_accept).transpose()?;
|
| 201 |
+
let rx = hops.as_ref().and_then(HopAccept::receiver);
|
| 202 |
+
reconnect_loop(
|
| 203 |
+
worker,
|
| 204 |
+
&mut spec.connect,
|
| 205 |
+
spec.hello,
|
| 206 |
+
spec.model_name,
|
| 207 |
+
&mut spec.sleep,
|
| 208 |
+
&mut spec.should_stop,
|
| 209 |
+
|w, stream, stop| {
|
| 210 |
+
if local_prefetch_enabled() {
|
| 211 |
+
serve_coordinator_and_hops_prefetch(w, stream, rx, stop)
|
| 212 |
+
} else {
|
| 213 |
+
serve_coordinator_and_hops(w, stream, rx, stop)
|
| 214 |
+
}
|
| 215 |
+
},
|
| 216 |
+
)
|
| 217 |
+
}
|
crates/ds4-dist/src/relay.rs
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Persistent worker-to-worker forwarder and RESULT relay thread.
|
| 2 |
+
|
| 3 |
+
use std::io::{self, Write};
|
| 4 |
+
use std::net::TcpStream;
|
| 5 |
+
use std::sync::atomic::{AtomicBool, Ordering};
|
| 6 |
+
use std::sync::{Arc, Mutex};
|
| 7 |
+
use std::thread::{self, JoinHandle};
|
| 8 |
+
use std::time::Instant;
|
| 9 |
+
|
| 10 |
+
use crate::codec::{Telemetry, MSG_RESULT};
|
| 11 |
+
use crate::forward::{
|
| 12 |
+
opened_forwarder_message, PendingQueue, PendingRequest, ERR_FORWARD, ERR_INVALID_RESULT,
|
| 13 |
+
ERR_NEXT_CLOSED, ERR_RESULT_METADATA, ERR_RESULT_TOO_LARGE, ERR_TELEMETRY_TOO_LARGE,
|
| 14 |
+
};
|
| 15 |
+
use crate::reconnect::connect_endpoint;
|
| 16 |
+
use crate::route::RouteEntry;
|
| 17 |
+
use crate::transport::read_frame;
|
| 18 |
+
use crate::work::{
|
| 19 |
+
decode_result_body, encode_result_frame, encode_work_frame, result_request_id, WorkBody,
|
| 20 |
+
};
|
| 21 |
+
|
| 22 |
+
pub fn now_sec() -> f64 {
|
| 23 |
+
static START: std::sync::OnceLock<Instant> = std::sync::OnceLock::new();
|
| 24 |
+
START.get_or_init(Instant::now).elapsed().as_secs_f64()
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
pub fn local_work_telemetry(
|
| 28 |
+
work: &crate::codec::Work,
|
| 29 |
+
eval_usec: u32,
|
| 30 |
+
output_bytes: u32,
|
| 31 |
+
) -> Telemetry {
|
| 32 |
+
Telemetry {
|
| 33 |
+
layer_start: work.layer_start,
|
| 34 |
+
layer_end: work.layer_end,
|
| 35 |
+
route_index: work.route_index,
|
| 36 |
+
pos0: work.pos0,
|
| 37 |
+
n_tokens: work.n_tokens,
|
| 38 |
+
eval_usec,
|
| 39 |
+
downstream_wait_usec: 0,
|
| 40 |
+
forward_send_usec: 0,
|
| 41 |
+
input_bytes: work.token_bytes.saturating_add(work.input_hc_bytes),
|
| 42 |
+
output_bytes,
|
| 43 |
+
}
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
pub fn usec_since(t0: f64, t1: f64) -> u32 {
|
| 47 |
+
if t1 <= t0 {
|
| 48 |
+
return 0;
|
| 49 |
+
}
|
| 50 |
+
let usec = (t1 - t0) * 1_000_000.0;
|
| 51 |
+
if usec >= f64::from(u32::MAX) {
|
| 52 |
+
u32::MAX
|
| 53 |
+
} else {
|
| 54 |
+
(usec + 0.5) as u32
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
pub fn prepend_telemetry(result_payload: &[u8], local: Telemetry) -> Result<Vec<u8>, &'static str> {
|
| 59 |
+
let mut body = decode_result_body(result_payload).map_err(|_| ERR_RESULT_METADATA)?;
|
| 60 |
+
if body.telemetry.len() >= u32::MAX as usize {
|
| 61 |
+
return Err(ERR_TELEMETRY_TOO_LARGE);
|
| 62 |
+
}
|
| 63 |
+
body.telemetry.push(local);
|
| 64 |
+
encode_result_frame(&body).map_err(|_| ERR_RESULT_TOO_LARGE)
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
pub struct Forwarder {
|
| 68 |
+
pub host: String,
|
| 69 |
+
pub port: u32,
|
| 70 |
+
writer: Mutex<TcpStream>,
|
| 71 |
+
queue: Arc<PendingQueue>,
|
| 72 |
+
stop: Arc<AtomicBool>,
|
| 73 |
+
relay: Option<JoinHandle<()>>,
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
impl Forwarder {
|
| 77 |
+
pub fn connect(
|
| 78 |
+
host: &str,
|
| 79 |
+
port: u32,
|
| 80 |
+
window: u32,
|
| 81 |
+
on_result: impl Fn(Vec<u8>) + Send + 'static,
|
| 82 |
+
on_error: impl Fn(u64, &'static str) + Send + 'static,
|
| 83 |
+
) -> Result<Self, String> {
|
| 84 |
+
let stream = connect_endpoint(host, port as u16).map_err(|e| e.to_string())?;
|
| 85 |
+
let mut reader = stream.try_clone().map_err(|e| e.to_string())?;
|
| 86 |
+
let queue = Arc::new(PendingQueue::new(window));
|
| 87 |
+
let stop = Arc::new(AtomicBool::new(false));
|
| 88 |
+
let relay_q = Arc::clone(&queue);
|
| 89 |
+
let relay_stop = Arc::clone(&stop);
|
| 90 |
+
let relay = thread::Builder::new()
|
| 91 |
+
.name(format!("ds4-dist-relay-{host}:{port}"))
|
| 92 |
+
.spawn(move || relay_main(&mut reader, &relay_q, &relay_stop, on_result, on_error))
|
| 93 |
+
.map_err(|e| e.to_string())?;
|
| 94 |
+
eprintln!("{}", opened_forwarder_message(host, port, window));
|
| 95 |
+
Ok(Self {
|
| 96 |
+
host: host.to_string(),
|
| 97 |
+
port,
|
| 98 |
+
writer: Mutex::new(stream),
|
| 99 |
+
queue,
|
| 100 |
+
stop,
|
| 101 |
+
relay: Some(relay),
|
| 102 |
+
})
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
pub fn send_work(&self, request: PendingRequest, frame: &[u8]) -> Result<(), &'static str> {
|
| 106 |
+
let request_id = request.request_id;
|
| 107 |
+
self.queue.enqueue(request)?;
|
| 108 |
+
let send_t0 = now_sec();
|
| 109 |
+
let write_rc = self.writer.lock().expect("forward send").write_all(frame);
|
| 110 |
+
let send_t1 = now_sec();
|
| 111 |
+
self.queue
|
| 112 |
+
.note_send_done(request_id, usec_since(send_t0, send_t1), send_t1);
|
| 113 |
+
if write_rc.is_err() {
|
| 114 |
+
self.queue.remove(request_id);
|
| 115 |
+
let _ = self
|
| 116 |
+
.writer
|
| 117 |
+
.lock()
|
| 118 |
+
.expect("forward send")
|
| 119 |
+
.shutdown(std::net::Shutdown::Both);
|
| 120 |
+
return Err(ERR_FORWARD);
|
| 121 |
+
}
|
| 122 |
+
Ok(())
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
pub fn shutdown(&mut self) {
|
| 126 |
+
self.stop.store(true, Ordering::SeqCst);
|
| 127 |
+
self.queue.close();
|
| 128 |
+
if let Ok(w) = self.writer.lock() {
|
| 129 |
+
let _ = w.shutdown(std::net::Shutdown::Both);
|
| 130 |
+
}
|
| 131 |
+
if let Some(relay) = self.relay.take() {
|
| 132 |
+
let _ = relay.join();
|
| 133 |
+
}
|
| 134 |
+
self.queue.clear();
|
| 135 |
+
}
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
impl Drop for Forwarder {
|
| 139 |
+
fn drop(&mut self) {
|
| 140 |
+
self.shutdown();
|
| 141 |
+
}
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
fn relay_main(
|
| 145 |
+
reader: &mut TcpStream,
|
| 146 |
+
queue: &PendingQueue,
|
| 147 |
+
stop: &AtomicBool,
|
| 148 |
+
on_result: impl Fn(Vec<u8>),
|
| 149 |
+
on_error: impl Fn(u64, &'static str),
|
| 150 |
+
) {
|
| 151 |
+
while !stop.load(Ordering::Relaxed) {
|
| 152 |
+
let (typ, body) = match read_frame(reader) {
|
| 153 |
+
Ok(v) => v,
|
| 154 |
+
Err(e) if e.kind() == io::ErrorKind::UnexpectedEof || stop.load(Ordering::Relaxed) => {
|
| 155 |
+
if let Some(pending) = queue.pop() {
|
| 156 |
+
on_error(pending.request_id, ERR_NEXT_CLOSED);
|
| 157 |
+
}
|
| 158 |
+
break;
|
| 159 |
+
}
|
| 160 |
+
Err(_) => {
|
| 161 |
+
if let Some(pending) = queue.pop() {
|
| 162 |
+
on_error(pending.request_id, ERR_NEXT_CLOSED);
|
| 163 |
+
}
|
| 164 |
+
break;
|
| 165 |
+
}
|
| 166 |
+
};
|
| 167 |
+
if typ != MSG_RESULT {
|
| 168 |
+
if let Some(pending) = queue.pop() {
|
| 169 |
+
on_error(pending.request_id, ERR_INVALID_RESULT);
|
| 170 |
+
}
|
| 171 |
+
break;
|
| 172 |
+
}
|
| 173 |
+
let decoded = match decode_result_body(&body) {
|
| 174 |
+
Ok(v) => v,
|
| 175 |
+
Err(_) => {
|
| 176 |
+
if let Some(pending) = queue.pop() {
|
| 177 |
+
on_error(pending.request_id, ERR_RESULT_METADATA);
|
| 178 |
+
}
|
| 179 |
+
break;
|
| 180 |
+
}
|
| 181 |
+
};
|
| 182 |
+
let Some(mut pending) = queue.pop() else {
|
| 183 |
+
break;
|
| 184 |
+
};
|
| 185 |
+
if result_request_id(&decoded.hdr) != pending.request_id {
|
| 186 |
+
on_error(pending.request_id, ERR_RESULT_METADATA);
|
| 187 |
+
break;
|
| 188 |
+
}
|
| 189 |
+
pending.telemetry.downstream_wait_usec = usec_since(pending.downstream_t0, now_sec());
|
| 190 |
+
match prepend_telemetry(&body, pending.telemetry) {
|
| 191 |
+
Ok(frame) => on_result(frame),
|
| 192 |
+
Err(msg) => on_error(pending.request_id, msg),
|
| 193 |
+
}
|
| 194 |
+
}
|
| 195 |
+
queue.close();
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
pub fn forward_work_blocking<W: Write>(
|
| 199 |
+
next: &RouteEntry,
|
| 200 |
+
body: &WorkBody,
|
| 201 |
+
upstream: &mut W,
|
| 202 |
+
mut local: Telemetry,
|
| 203 |
+
) -> Result<(), String> {
|
| 204 |
+
let mut hop = connect_endpoint(&next.host, next.port as u16).map_err(|_| ERR_FORWARD)?;
|
| 205 |
+
let frame = encode_work_frame(body).map_err(|e| e.to_string())?;
|
| 206 |
+
let send_t0 = now_sec();
|
| 207 |
+
hop.write_all(&frame).map_err(|_| ERR_FORWARD)?;
|
| 208 |
+
let send_t1 = now_sec();
|
| 209 |
+
local.forward_send_usec = usec_since(send_t0, send_t1);
|
| 210 |
+
let (typ, reply) = read_frame(&mut hop).map_err(|_| ERR_FORWARD)?;
|
| 211 |
+
if typ != MSG_RESULT {
|
| 212 |
+
return Err(ERR_INVALID_RESULT.into());
|
| 213 |
+
}
|
| 214 |
+
local.downstream_wait_usec = usec_since(send_t1, now_sec());
|
| 215 |
+
let prepended = prepend_telemetry(&reply, local).map_err(|e| e.to_string())?;
|
| 216 |
+
upstream.write_all(&prepended).map_err(|e| e.to_string())
|
| 217 |
+
}
|
crates/ds4-dist/src/route.rs
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Route blob: `[Route + host]*` then `RouteReturn + host`. No struct memcpy.
|
| 2 |
+
|
| 3 |
+
use crate::codec::{
|
| 4 |
+
bytes_have_nul, CodecError, Route, RouteReturn, NI_MAXHOST, ROUTE_FIXED_BYTES,
|
| 5 |
+
ROUTE_F_OUTPUT_LOGITS, ROUTE_RETURN_FIXED_BYTES, ROUTE_RETURN_UPSTREAM,
|
| 6 |
+
};
|
| 7 |
+
|
| 8 |
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
| 9 |
+
pub struct RouteEntry {
|
| 10 |
+
pub host: String,
|
| 11 |
+
pub port: u32,
|
| 12 |
+
pub layer_start: u32,
|
| 13 |
+
pub layer_end: u32,
|
| 14 |
+
pub flags: u32,
|
| 15 |
+
}
|
| 16 |
+
|
| 17 |
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
| 18 |
+
pub struct ReturnTarget {
|
| 19 |
+
pub kind: u32,
|
| 20 |
+
pub host: String,
|
| 21 |
+
pub port: u32,
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
pub fn encode_route_blob(
|
| 25 |
+
entries: &[RouteEntry],
|
| 26 |
+
ret: &ReturnTarget,
|
| 27 |
+
) -> Result<Vec<u8>, CodecError> {
|
| 28 |
+
let mut out = Vec::new();
|
| 29 |
+
for e in entries {
|
| 30 |
+
let host = e.host.as_bytes();
|
| 31 |
+
if host.is_empty() || host.len() >= NI_MAXHOST || bytes_have_nul(host) {
|
| 32 |
+
return Err(CodecError::Invalid("invalid route host length"));
|
| 33 |
+
}
|
| 34 |
+
let rec = Route {
|
| 35 |
+
host_len: host.len() as u32,
|
| 36 |
+
port: e.port,
|
| 37 |
+
layer_start: e.layer_start,
|
| 38 |
+
layer_end: e.layer_end,
|
| 39 |
+
flags: e.flags,
|
| 40 |
+
};
|
| 41 |
+
out.extend_from_slice(&rec.encode());
|
| 42 |
+
out.extend_from_slice(host);
|
| 43 |
+
}
|
| 44 |
+
let rhost = ret.host.as_bytes();
|
| 45 |
+
if rhost.len() >= NI_MAXHOST || bytes_have_nul(rhost) {
|
| 46 |
+
return Err(CodecError::Invalid(
|
| 47 |
+
"invalid route final destination host length",
|
| 48 |
+
));
|
| 49 |
+
}
|
| 50 |
+
let rec = RouteReturn {
|
| 51 |
+
kind: ret.kind,
|
| 52 |
+
host_len: rhost.len() as u32,
|
| 53 |
+
port: ret.port,
|
| 54 |
+
};
|
| 55 |
+
out.extend_from_slice(&rec.encode());
|
| 56 |
+
out.extend_from_slice(rhost);
|
| 57 |
+
Ok(out)
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
pub fn decode_route_blob(
|
| 61 |
+
blob: &[u8],
|
| 62 |
+
route_count: u32,
|
| 63 |
+
) -> Result<(Vec<RouteEntry>, ReturnTarget), CodecError> {
|
| 64 |
+
let mut remaining = blob;
|
| 65 |
+
let mut entries = Vec::new();
|
| 66 |
+
for _ in 0..route_count {
|
| 67 |
+
if remaining.len() < ROUTE_FIXED_BYTES {
|
| 68 |
+
return Err(CodecError::Invalid("truncated route entry"));
|
| 69 |
+
}
|
| 70 |
+
let fixed = Route::decode(remaining)?;
|
| 71 |
+
remaining = &remaining[ROUTE_FIXED_BYTES..];
|
| 72 |
+
if fixed.host_len == 0
|
| 73 |
+
|| fixed.host_len as usize >= NI_MAXHOST
|
| 74 |
+
|| fixed.host_len as usize > remaining.len()
|
| 75 |
+
{
|
| 76 |
+
return Err(CodecError::Invalid("invalid route host length"));
|
| 77 |
+
}
|
| 78 |
+
let host = &remaining[..fixed.host_len as usize];
|
| 79 |
+
if bytes_have_nul(host) {
|
| 80 |
+
return Err(CodecError::Invalid("route host contains NUL bytes"));
|
| 81 |
+
}
|
| 82 |
+
entries.push(RouteEntry {
|
| 83 |
+
host: String::from_utf8_lossy(host).into_owned(),
|
| 84 |
+
port: fixed.port,
|
| 85 |
+
layer_start: fixed.layer_start,
|
| 86 |
+
layer_end: fixed.layer_end,
|
| 87 |
+
flags: fixed.flags,
|
| 88 |
+
});
|
| 89 |
+
remaining = &remaining[fixed.host_len as usize..];
|
| 90 |
+
}
|
| 91 |
+
if remaining.len() < ROUTE_RETURN_FIXED_BYTES {
|
| 92 |
+
return Err(CodecError::Invalid(
|
| 93 |
+
"route payload missing final destination",
|
| 94 |
+
));
|
| 95 |
+
}
|
| 96 |
+
let ret = RouteReturn::decode(remaining)?;
|
| 97 |
+
remaining = &remaining[ROUTE_RETURN_FIXED_BYTES..];
|
| 98 |
+
if ret.host_len as usize >= NI_MAXHOST || ret.host_len as usize > remaining.len() {
|
| 99 |
+
return Err(CodecError::Invalid(
|
| 100 |
+
"invalid route final destination host length",
|
| 101 |
+
));
|
| 102 |
+
}
|
| 103 |
+
let host = &remaining[..ret.host_len as usize];
|
| 104 |
+
if bytes_have_nul(host) {
|
| 105 |
+
return Err(CodecError::Invalid(
|
| 106 |
+
"route final destination host contains NUL bytes",
|
| 107 |
+
));
|
| 108 |
+
}
|
| 109 |
+
remaining = &remaining[ret.host_len as usize..];
|
| 110 |
+
if !remaining.is_empty() {
|
| 111 |
+
return Err(CodecError::Invalid("route payload has trailing bytes"));
|
| 112 |
+
}
|
| 113 |
+
Ok((
|
| 114 |
+
entries,
|
| 115 |
+
ReturnTarget {
|
| 116 |
+
kind: ret.kind,
|
| 117 |
+
host: String::from_utf8_lossy(host).into_owned(),
|
| 118 |
+
port: ret.port,
|
| 119 |
+
},
|
| 120 |
+
))
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
pub fn validate_route_blob(blob: &[u8], route_count: u32, n_layers: u32) -> Result<(), CodecError> {
|
| 124 |
+
if route_count == 0 {
|
| 125 |
+
return if blob.is_empty() {
|
| 126 |
+
Ok(())
|
| 127 |
+
} else {
|
| 128 |
+
Err(CodecError::Invalid(
|
| 129 |
+
"route payload has entries without a route count",
|
| 130 |
+
))
|
| 131 |
+
};
|
| 132 |
+
}
|
| 133 |
+
let (entries, ret) = decode_route_blob(blob, route_count)?;
|
| 134 |
+
let mut prev_end = u32::MAX;
|
| 135 |
+
for (i, e) in entries.iter().enumerate() {
|
| 136 |
+
if e.port == 0 || e.port > 65535 {
|
| 137 |
+
return Err(CodecError::Invalid("invalid route port"));
|
| 138 |
+
}
|
| 139 |
+
if e.layer_start >= n_layers || e.layer_end >= n_layers || e.layer_end < e.layer_start {
|
| 140 |
+
return Err(CodecError::Invalid("invalid route layer range"));
|
| 141 |
+
}
|
| 142 |
+
if (e.flags & !ROUTE_F_OUTPUT_LOGITS) != 0 {
|
| 143 |
+
return Err(CodecError::Invalid("invalid route flags"));
|
| 144 |
+
}
|
| 145 |
+
if (e.flags & ROUTE_F_OUTPUT_LOGITS) != 0 && e.layer_end + 1 != n_layers {
|
| 146 |
+
return Err(CodecError::Invalid("route logits require final layer"));
|
| 147 |
+
}
|
| 148 |
+
if i != 0 && e.layer_start != prev_end + 1 {
|
| 149 |
+
return Err(CodecError::Invalid("route layer ranges are not contiguous"));
|
| 150 |
+
}
|
| 151 |
+
prev_end = e.layer_end;
|
| 152 |
+
}
|
| 153 |
+
if ret.kind != ROUTE_RETURN_UPSTREAM {
|
| 154 |
+
return Err(CodecError::Invalid("unsupported route final destination"));
|
| 155 |
+
}
|
| 156 |
+
if !ret.host.is_empty() || ret.port != 0 {
|
| 157 |
+
return Err(CodecError::Invalid(
|
| 158 |
+
"invalid upstream route final destination",
|
| 159 |
+
));
|
| 160 |
+
}
|
| 161 |
+
Ok(())
|
| 162 |
+
}
|
crates/ds4-dist/src/serve_local.rs
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Same-thread coordinator + hop mux. Accept thread only accepts.
|
| 2 |
+
|
| 3 |
+
use std::io::{self, ErrorKind};
|
| 4 |
+
use std::net::{SocketAddr, TcpListener, TcpStream};
|
| 5 |
+
use std::sync::mpsc::{self, Receiver};
|
| 6 |
+
use std::sync::{Arc, Mutex};
|
| 7 |
+
use std::thread::{self, JoinHandle};
|
| 8 |
+
use std::time::Duration;
|
| 9 |
+
|
| 10 |
+
use crate::coordinator::accept_data_client;
|
| 11 |
+
use crate::exec::SliceExec;
|
| 12 |
+
use crate::native_snapshot::SnapshotStore;
|
| 13 |
+
use crate::worker::Worker;
|
| 14 |
+
|
| 15 |
+
pub(crate) const IDLE: Duration = Duration::from_millis(25);
|
| 16 |
+
|
| 17 |
+
pub(crate) struct HopAccept {
|
| 18 |
+
rx: Option<Receiver<TcpStream>>,
|
| 19 |
+
wakeup: SocketAddr,
|
| 20 |
+
handle: Option<JoinHandle<()>>,
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
impl HopAccept {
|
| 24 |
+
pub(crate) fn receiver(&self) -> Option<&Receiver<TcpStream>> {
|
| 25 |
+
self.rx.as_ref()
|
| 26 |
+
}
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
impl Drop for HopAccept {
|
| 30 |
+
fn drop(&mut self) {
|
| 31 |
+
self.rx.take();
|
| 32 |
+
let _ = TcpStream::connect(self.wakeup);
|
| 33 |
+
if let Some(handle) = self.handle.take() {
|
| 34 |
+
let _ = handle.join();
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
pub(crate) fn spawn_hop_accept(listener: &TcpListener) -> io::Result<HopAccept> {
|
| 40 |
+
let listener = listener.try_clone()?;
|
| 41 |
+
let wakeup = listener.local_addr()?;
|
| 42 |
+
let (tx, rx) = mpsc::channel();
|
| 43 |
+
let handle = thread::Builder::new()
|
| 44 |
+
.name("ds4-dist-hop-accept".into())
|
| 45 |
+
.spawn(move || loop {
|
| 46 |
+
match accept_data_client(&listener) {
|
| 47 |
+
Ok(stream) => {
|
| 48 |
+
if tx.send(stream).is_err() {
|
| 49 |
+
break;
|
| 50 |
+
}
|
| 51 |
+
}
|
| 52 |
+
Err(_) => break,
|
| 53 |
+
}
|
| 54 |
+
})?;
|
| 55 |
+
Ok(HopAccept {
|
| 56 |
+
rx: Some(rx),
|
| 57 |
+
wakeup,
|
| 58 |
+
handle: Some(handle),
|
| 59 |
+
})
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
pub(crate) enum Drive {
|
| 63 |
+
Continue,
|
| 64 |
+
Closed,
|
| 65 |
+
Failed(io::Error),
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
pub(crate) fn drive_ready<E, S>(worker: &mut Worker<E, S>, stream: &mut TcpStream) -> Drive
|
| 69 |
+
where
|
| 70 |
+
E: SliceExec,
|
| 71 |
+
S: SnapshotStore,
|
| 72 |
+
{
|
| 73 |
+
match worker.serve_once(stream) {
|
| 74 |
+
Ok(()) => Drive::Continue,
|
| 75 |
+
Err(e) => match e.kind() {
|
| 76 |
+
ErrorKind::TimedOut | ErrorKind::WouldBlock | ErrorKind::Interrupted => Drive::Continue,
|
| 77 |
+
ErrorKind::UnexpectedEof | ErrorKind::InvalidData => Drive::Closed,
|
| 78 |
+
_ => Drive::Failed(e),
|
| 79 |
+
},
|
| 80 |
+
}
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
pub(crate) fn serve_coordinator_and_hops<E, S, Stop>(
|
| 84 |
+
worker: &mut Worker<E, S>,
|
| 85 |
+
coordinator: &mut TcpStream,
|
| 86 |
+
hops: Option<&Receiver<TcpStream>>,
|
| 87 |
+
should_stop: &mut Stop,
|
| 88 |
+
) -> io::Result<()>
|
| 89 |
+
where
|
| 90 |
+
E: SliceExec,
|
| 91 |
+
S: SnapshotStore,
|
| 92 |
+
Stop: FnMut() -> bool,
|
| 93 |
+
{
|
| 94 |
+
coordinator.set_read_timeout(Some(IDLE))?;
|
| 95 |
+
let mut accepted = Vec::new();
|
| 96 |
+
let rc = loop {
|
| 97 |
+
if should_stop() {
|
| 98 |
+
break Ok(());
|
| 99 |
+
}
|
| 100 |
+
if let Some(rx) = hops {
|
| 101 |
+
while let Ok(hop) = rx.try_recv() {
|
| 102 |
+
let _ = hop.set_nodelay(true);
|
| 103 |
+
let _ = hop.set_read_timeout(Some(IDLE));
|
| 104 |
+
accepted.push(hop);
|
| 105 |
+
}
|
| 106 |
+
}
|
| 107 |
+
let mut i = 0;
|
| 108 |
+
while i < accepted.len() {
|
| 109 |
+
if let Ok(clone) = accepted[i].try_clone() {
|
| 110 |
+
worker.bind_hops(Arc::new(Mutex::new(clone)));
|
| 111 |
+
}
|
| 112 |
+
match drive_ready(worker, &mut accepted[i]) {
|
| 113 |
+
Drive::Continue => i += 1,
|
| 114 |
+
Drive::Closed | Drive::Failed(_) => {
|
| 115 |
+
accepted.swap_remove(i);
|
| 116 |
+
}
|
| 117 |
+
}
|
| 118 |
+
}
|
| 119 |
+
match drive_ready(worker, coordinator) {
|
| 120 |
+
Drive::Continue => {}
|
| 121 |
+
Drive::Closed => break Ok(()),
|
| 122 |
+
Drive::Failed(e) => break Err(e),
|
| 123 |
+
}
|
| 124 |
+
};
|
| 125 |
+
worker.shutdown_hops();
|
| 126 |
+
rc
|
| 127 |
+
}
|
crates/ds4-dist/src/snapshot.rs
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Coordinator-side DS4D snapshot save/load transport.
|
| 2 |
+
|
| 3 |
+
use std::io::{self, Read, Write};
|
| 4 |
+
|
| 5 |
+
use crate::codec::{
|
| 6 |
+
decode_frame_header, encode_frame_header, u64_from_halves, u64_to_halves, FrameHeader,
|
| 7 |
+
SnapshotBegin, SnapshotChunk, SnapshotDone, SnapshotReq, FRAME_HEADER_BYTES,
|
| 8 |
+
MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK, MSG_SNAPSHOT_DONE, MSG_SNAPSHOT_LOAD_BEGIN,
|
| 9 |
+
MSG_SNAPSHOT_SAVE_REQ, SNAPSHOT_BEGIN_FIXED_BYTES, SNAPSHOT_CHUNK_BYTES,
|
| 10 |
+
SNAPSHOT_CHUNK_FIXED_BYTES, SNAPSHOT_DONE_FIXED_BYTES,
|
| 11 |
+
};
|
| 12 |
+
use crate::transport::write_frame;
|
| 13 |
+
|
| 14 |
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
| 15 |
+
pub struct SnapshotMeta {
|
| 16 |
+
pub model_id: u32,
|
| 17 |
+
pub session_id: u64,
|
| 18 |
+
pub request_id: u64,
|
| 19 |
+
pub token_hash: u64,
|
| 20 |
+
pub layer_start: u32,
|
| 21 |
+
pub layer_end: u32,
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
fn token_count(tokens: &[i32]) -> Result<u32, String> {
|
| 25 |
+
u32::try_from(tokens.len()).map_err(|_| "invalid distributed snapshot token length".to_string())
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
fn c_message(message: &[u8]) -> String {
|
| 29 |
+
let end = message
|
| 30 |
+
.iter()
|
| 31 |
+
.position(|&b| b == 0)
|
| 32 |
+
.unwrap_or(message.len());
|
| 33 |
+
String::from_utf8_lossy(&message[..end]).into_owned()
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
fn read_header<R: Read>(r: &mut R, closed: &'static str) -> Result<FrameHeader, String> {
|
| 37 |
+
let mut buf = [0; FRAME_HEADER_BYTES];
|
| 38 |
+
r.read_exact(&mut buf).map_err(|e| {
|
| 39 |
+
if e.kind() == io::ErrorKind::UnexpectedEof {
|
| 40 |
+
closed.to_string()
|
| 41 |
+
} else {
|
| 42 |
+
format!("failed to read frame header: {e}")
|
| 43 |
+
}
|
| 44 |
+
})?;
|
| 45 |
+
decode_frame_header(&buf).map_err(|e| e.to_string())
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
fn read_or<R: Read>(r: &mut R, buf: &mut [u8], message: &'static str) -> Result<(), String> {
|
| 49 |
+
r.read_exact(buf).map_err(|_| message.to_string())
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
fn discard<R: Read>(r: &mut R, mut bytes: u32) -> io::Result<()> {
|
| 53 |
+
let mut buf = [0; 4096];
|
| 54 |
+
while bytes != 0 {
|
| 55 |
+
let n = bytes.min(buf.len() as u32) as usize;
|
| 56 |
+
r.read_exact(&mut buf[..n])?;
|
| 57 |
+
bytes -= n as u32;
|
| 58 |
+
}
|
| 59 |
+
Ok(())
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
fn read_message<R: Read>(
|
| 63 |
+
r: &mut R,
|
| 64 |
+
bytes: u32,
|
| 65 |
+
read_error: &'static str,
|
| 66 |
+
discard_error: &'static str,
|
| 67 |
+
) -> Result<Vec<u8>, String> {
|
| 68 |
+
let copied = bytes.min(255) as usize;
|
| 69 |
+
let mut message = vec![0; copied];
|
| 70 |
+
read_or(r, &mut message, read_error)?;
|
| 71 |
+
discard(r, bytes - copied as u32).map_err(|_| discard_error.to_string())?;
|
| 72 |
+
Ok(message)
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
fn read_begin<R: Read>(r: &mut R) -> Result<(SnapshotBegin, Vec<u8>), String> {
|
| 76 |
+
let header = read_header(r, "distributed worker closed snapshot connection")?;
|
| 77 |
+
if header.typ != MSG_SNAPSHOT_BEGIN || header.bytes < SNAPSHOT_BEGIN_FIXED_BYTES as u32 {
|
| 78 |
+
let _ = discard(r, header.bytes);
|
| 79 |
+
return Err("distributed worker returned invalid snapshot frame".into());
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
let mut fixed = [0; SNAPSHOT_BEGIN_FIXED_BYTES];
|
| 83 |
+
read_or(r, &mut fixed, "failed to read distributed snapshot header")?;
|
| 84 |
+
let begin = SnapshotBegin::decode(&fixed)
|
| 85 |
+
.map_err(|_| "invalid distributed snapshot response header".to_string())?;
|
| 86 |
+
let mut body = header.bytes - SNAPSHOT_BEGIN_FIXED_BYTES as u32;
|
| 87 |
+
let expected_token_bytes = u64::from(begin.token_count) * 4;
|
| 88 |
+
if expected_token_bytes > u64::from(u32::MAX)
|
| 89 |
+
|| begin.token_bytes != expected_token_bytes as u32
|
| 90 |
+
|| begin.token_bytes > body
|
| 91 |
+
|| begin.message_bytes > body - begin.token_bytes
|
| 92 |
+
{
|
| 93 |
+
let _ = discard(r, body);
|
| 94 |
+
return Err("invalid distributed snapshot response header".into());
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
discard(r, begin.token_bytes)
|
| 98 |
+
.map_err(|_| "failed to discard distributed snapshot response tokens".to_string())?;
|
| 99 |
+
body -= begin.token_bytes;
|
| 100 |
+
let message = read_message(
|
| 101 |
+
r,
|
| 102 |
+
begin.message_bytes,
|
| 103 |
+
"failed to read distributed snapshot response message",
|
| 104 |
+
"failed to discard distributed snapshot response message",
|
| 105 |
+
)?;
|
| 106 |
+
body -= begin.message_bytes;
|
| 107 |
+
discard(r, body).map_err(|_| {
|
| 108 |
+
"failed to discard trailing distributed snapshot response bytes".to_string()
|
| 109 |
+
})?;
|
| 110 |
+
Ok((begin, message))
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
fn read_done<R: Read>(r: &mut R, request_id: u64) -> Result<(), String> {
|
| 114 |
+
let header = read_header(r, "distributed worker closed before snapshot completion")?;
|
| 115 |
+
if header.typ != MSG_SNAPSHOT_DONE || header.bytes < SNAPSHOT_DONE_FIXED_BYTES as u32 {
|
| 116 |
+
let _ = discard(r, header.bytes);
|
| 117 |
+
return Err("distributed worker returned invalid snapshot completion frame".into());
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
let mut fixed = [0; SNAPSHOT_DONE_FIXED_BYTES];
|
| 121 |
+
read_or(
|
| 122 |
+
r,
|
| 123 |
+
&mut fixed,
|
| 124 |
+
"failed to read distributed snapshot completion",
|
| 125 |
+
)?;
|
| 126 |
+
let done = SnapshotDone::decode(&fixed)
|
| 127 |
+
.map_err(|_| "invalid distributed snapshot completion message".to_string())?;
|
| 128 |
+
let mut body = header.bytes - SNAPSHOT_DONE_FIXED_BYTES as u32;
|
| 129 |
+
if done.message_bytes > body {
|
| 130 |
+
let _ = discard(r, body);
|
| 131 |
+
return Err("invalid distributed snapshot completion message".into());
|
| 132 |
+
}
|
| 133 |
+
let message = read_message(
|
| 134 |
+
r,
|
| 135 |
+
done.message_bytes,
|
| 136 |
+
"failed to read distributed snapshot completion message",
|
| 137 |
+
"failed to discard distributed snapshot completion message",
|
| 138 |
+
)?;
|
| 139 |
+
body -= done.message_bytes;
|
| 140 |
+
discard(r, body).map_err(|_| {
|
| 141 |
+
"failed to discard trailing distributed snapshot completion bytes".to_string()
|
| 142 |
+
})?;
|
| 143 |
+
|
| 144 |
+
if u64_from_halves(done.request_hi, done.request_lo) != request_id {
|
| 145 |
+
return Err("distributed snapshot completion request mismatch".into());
|
| 146 |
+
}
|
| 147 |
+
if done.status != 0 {
|
| 148 |
+
let message = c_message(&message);
|
| 149 |
+
return Err(if message.is_empty() {
|
| 150 |
+
"distributed worker failed snapshot request".into()
|
| 151 |
+
} else {
|
| 152 |
+
message
|
| 153 |
+
});
|
| 154 |
+
}
|
| 155 |
+
Ok(())
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
/// Ask a connected worker for one KV shard and stream it into `payload`.
|
| 159 |
+
pub fn coordinator_save_snapshot<S: Read + Write, W: Write>(
|
| 160 |
+
stream: &mut S,
|
| 161 |
+
meta: SnapshotMeta,
|
| 162 |
+
tokens: &[i32],
|
| 163 |
+
payload: &mut W,
|
| 164 |
+
) -> Result<u64, String> {
|
| 165 |
+
let token_count = token_count(tokens)?;
|
| 166 |
+
let (session_hi, session_lo) = u64_to_halves(meta.session_id);
|
| 167 |
+
let (request_hi, request_lo) = u64_to_halves(meta.request_id);
|
| 168 |
+
let (token_hash_hi, token_hash_lo) = u64_to_halves(meta.token_hash);
|
| 169 |
+
let request = SnapshotReq {
|
| 170 |
+
model_id: meta.model_id,
|
| 171 |
+
session_hi,
|
| 172 |
+
session_lo,
|
| 173 |
+
request_hi,
|
| 174 |
+
request_lo,
|
| 175 |
+
token_hash_hi,
|
| 176 |
+
token_hash_lo,
|
| 177 |
+
token_count,
|
| 178 |
+
layer_start: meta.layer_start,
|
| 179 |
+
layer_end: meta.layer_end,
|
| 180 |
+
};
|
| 181 |
+
write_frame(stream, MSG_SNAPSHOT_SAVE_REQ, &request.encode())
|
| 182 |
+
.map_err(|_| "failed to request distributed KV shard".to_string())?;
|
| 183 |
+
|
| 184 |
+
let (begin, message) = read_begin(stream)?;
|
| 185 |
+
if begin.status != 0 {
|
| 186 |
+
let message = c_message(&message);
|
| 187 |
+
return Err(if message.is_empty() {
|
| 188 |
+
"distributed worker refused KV snapshot".into()
|
| 189 |
+
} else {
|
| 190 |
+
message
|
| 191 |
+
});
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
let payload_bytes = u64_from_halves(begin.payload_hi, begin.payload_lo);
|
| 195 |
+
if begin.model_id != meta.model_id
|
| 196 |
+
|| u64_from_halves(begin.session_hi, begin.session_lo) != meta.session_id
|
| 197 |
+
|| u64_from_halves(begin.request_hi, begin.request_lo) != meta.request_id
|
| 198 |
+
|| u64_from_halves(begin.token_hash_hi, begin.token_hash_lo) != meta.token_hash
|
| 199 |
+
|| (begin.token_count != 0 && begin.token_count != token_count)
|
| 200 |
+
|| begin.layer_start != meta.layer_start
|
| 201 |
+
|| begin.layer_end != meta.layer_end
|
| 202 |
+
{
|
| 203 |
+
return Err("distributed KV shard metadata mismatch".into());
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
let mut buf = Vec::new();
|
| 207 |
+
if payload_bytes != 0 {
|
| 208 |
+
buf.try_reserve_exact(SNAPSHOT_CHUNK_BYTES)
|
| 209 |
+
.map_err(|_| "out of memory receiving distributed KV shard".to_string())?;
|
| 210 |
+
buf.resize(SNAPSHOT_CHUNK_BYTES, 0);
|
| 211 |
+
}
|
| 212 |
+
let mut received = 0u64;
|
| 213 |
+
while received < payload_bytes {
|
| 214 |
+
let header = read_header(stream, "distributed worker closed while sending KV shard")?;
|
| 215 |
+
if header.typ != MSG_SNAPSHOT_CHUNK || header.bytes < SNAPSHOT_CHUNK_FIXED_BYTES as u32 {
|
| 216 |
+
let _ = discard(stream, header.bytes);
|
| 217 |
+
return Err("expected distributed KV shard chunk".into());
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
let mut fixed = [0; SNAPSHOT_CHUNK_FIXED_BYTES];
|
| 221 |
+
read_or(
|
| 222 |
+
stream,
|
| 223 |
+
&mut fixed,
|
| 224 |
+
"failed to read distributed KV shard chunk header",
|
| 225 |
+
)?;
|
| 226 |
+
let chunk = SnapshotChunk::decode(&fixed)
|
| 227 |
+
.map_err(|_| "invalid distributed KV shard chunk".to_string())?;
|
| 228 |
+
let chunk_bytes = header.bytes - SNAPSHOT_CHUNK_FIXED_BYTES as u32;
|
| 229 |
+
if u64_from_halves(chunk.request_hi, chunk.request_lo) != meta.request_id
|
| 230 |
+
|| chunk.chunk_bytes != chunk_bytes
|
| 231 |
+
|| chunk_bytes as usize > SNAPSHOT_CHUNK_BYTES
|
| 232 |
+
|| u64::from(chunk_bytes) > payload_bytes - received
|
| 233 |
+
{
|
| 234 |
+
let _ = discard(stream, chunk_bytes);
|
| 235 |
+
return Err("invalid distributed KV shard chunk".into());
|
| 236 |
+
}
|
| 237 |
+
let chunk_bytes = chunk_bytes as usize;
|
| 238 |
+
read_or(
|
| 239 |
+
stream,
|
| 240 |
+
&mut buf[..chunk_bytes],
|
| 241 |
+
"failed to read distributed KV shard chunk",
|
| 242 |
+
)?;
|
| 243 |
+
payload
|
| 244 |
+
.write_all(&buf[..chunk_bytes])
|
| 245 |
+
.map_err(|_| "failed to write distributed KV shard".to_string())?;
|
| 246 |
+
received += chunk_bytes as u64;
|
| 247 |
+
}
|
| 248 |
+
read_done(stream, meta.request_id)?;
|
| 249 |
+
Ok(payload_bytes)
|
| 250 |
+
}
|
| 251 |
+
|
| 252 |
+
/// Stream one caller-owned KV shard payload to a connected worker.
|
| 253 |
+
pub fn coordinator_load_snapshot<S: Read + Write, R: Read>(
|
| 254 |
+
stream: &mut S,
|
| 255 |
+
meta: SnapshotMeta,
|
| 256 |
+
tokens: &[i32],
|
| 257 |
+
payload: &mut R,
|
| 258 |
+
payload_bytes: u64,
|
| 259 |
+
) -> Result<(), String> {
|
| 260 |
+
let token_count = token_count(tokens)?;
|
| 261 |
+
let token_bytes = token_count
|
| 262 |
+
.checked_mul(4)
|
| 263 |
+
.ok_or_else(|| "invalid distributed snapshot token length".to_string())?;
|
| 264 |
+
let (session_hi, session_lo) = u64_to_halves(meta.session_id);
|
| 265 |
+
let (request_hi, request_lo) = u64_to_halves(meta.request_id);
|
| 266 |
+
let (token_hash_hi, token_hash_lo) = u64_to_halves(meta.token_hash);
|
| 267 |
+
let (payload_hi, payload_lo) = u64_to_halves(payload_bytes);
|
| 268 |
+
let begin = SnapshotBegin {
|
| 269 |
+
model_id: meta.model_id,
|
| 270 |
+
session_hi,
|
| 271 |
+
session_lo,
|
| 272 |
+
request_hi,
|
| 273 |
+
request_lo,
|
| 274 |
+
token_hash_hi,
|
| 275 |
+
token_hash_lo,
|
| 276 |
+
token_count,
|
| 277 |
+
layer_start: meta.layer_start,
|
| 278 |
+
layer_end: meta.layer_end,
|
| 279 |
+
payload_hi,
|
| 280 |
+
payload_lo,
|
| 281 |
+
status: 0,
|
| 282 |
+
token_bytes,
|
| 283 |
+
message_bytes: 0,
|
| 284 |
+
};
|
| 285 |
+
let frame_bytes = SNAPSHOT_BEGIN_FIXED_BYTES as u64 + u64::from(token_bytes);
|
| 286 |
+
if frame_bytes > u64::from(u32::MAX) {
|
| 287 |
+
return Err("distributed snapshot begin frame is too large".into());
|
| 288 |
+
}
|
| 289 |
+
let send_begin = || "failed to send distributed KV shard restore request".to_string();
|
| 290 |
+
stream
|
| 291 |
+
.write_all(&encode_frame_header(
|
| 292 |
+
MSG_SNAPSHOT_LOAD_BEGIN,
|
| 293 |
+
frame_bytes as u32,
|
| 294 |
+
))
|
| 295 |
+
.map_err(|_| send_begin())?;
|
| 296 |
+
stream
|
| 297 |
+
.write_all(&begin.encode())
|
| 298 |
+
.map_err(|_| send_begin())?;
|
| 299 |
+
for &token in tokens {
|
| 300 |
+
stream
|
| 301 |
+
.write_all(&(token as u32).to_be_bytes())
|
| 302 |
+
.map_err(|_| send_begin())?;
|
| 303 |
+
}
|
| 304 |
+
|
| 305 |
+
let mut buf = Vec::new();
|
| 306 |
+
if payload_bytes != 0 {
|
| 307 |
+
buf.try_reserve_exact(SNAPSHOT_CHUNK_BYTES)
|
| 308 |
+
.map_err(|_| "failed to send distributed KV shard restore payload".to_string())?;
|
| 309 |
+
buf.resize(SNAPSHOT_CHUNK_BYTES, 0);
|
| 310 |
+
}
|
| 311 |
+
let mut remaining = payload_bytes;
|
| 312 |
+
while remaining != 0 {
|
| 313 |
+
let n = remaining.min(SNAPSHOT_CHUNK_BYTES as u64) as usize;
|
| 314 |
+
read_or(
|
| 315 |
+
payload,
|
| 316 |
+
&mut buf[..n],
|
| 317 |
+
"failed to send distributed KV shard restore payload",
|
| 318 |
+
)?;
|
| 319 |
+
let (request_hi, request_lo) = u64_to_halves(meta.request_id);
|
| 320 |
+
let chunk = SnapshotChunk {
|
| 321 |
+
request_hi,
|
| 322 |
+
request_lo,
|
| 323 |
+
chunk_bytes: n as u32,
|
| 324 |
+
};
|
| 325 |
+
let frame_bytes = SNAPSHOT_CHUNK_FIXED_BYTES as u32 + n as u32;
|
| 326 |
+
let send_chunk = || "failed to send distributed KV shard restore payload".to_string();
|
| 327 |
+
stream
|
| 328 |
+
.write_all(&encode_frame_header(MSG_SNAPSHOT_CHUNK, frame_bytes))
|
| 329 |
+
.map_err(|_| send_chunk())?;
|
| 330 |
+
stream
|
| 331 |
+
.write_all(&chunk.encode())
|
| 332 |
+
.map_err(|_| send_chunk())?;
|
| 333 |
+
stream.write_all(&buf[..n]).map_err(|_| send_chunk())?;
|
| 334 |
+
remaining -= n as u64;
|
| 335 |
+
}
|
| 336 |
+
read_done(stream, meta.request_id)
|
| 337 |
+
}
|
crates/ds4-dist/src/snapshot_gather.rs
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Coordinator KV gather order from `ds4_dist_session_save_payload`.
|
| 2 |
+
//!
|
| 3 |
+
//! Host owns shard walk, empty-shard refusal, and token hashing. Layer
|
| 4 |
+
//! GPU bytes stay native; DSV4 header merge lives in `snapshot_merge`.
|
| 5 |
+
|
| 6 |
+
use crate::hash::token_hash_prefix;
|
| 7 |
+
|
| 8 |
+
pub const ERR_NO_TIMELINE: &str = "distributed session has no valid token timeline";
|
| 9 |
+
pub const ERR_EMPTY_SHARD: &str = "distributed KV shard is empty";
|
| 10 |
+
|
| 11 |
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
| 12 |
+
pub struct GatherShard {
|
| 13 |
+
pub layer_start: u32,
|
| 14 |
+
pub layer_end: u32,
|
| 15 |
+
pub local: bool,
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
| 19 |
+
pub struct GatheredShards {
|
| 20 |
+
pub token_count: u32,
|
| 21 |
+
pub token_hash: u64,
|
| 22 |
+
pub shards: Vec<(GatherShard, Vec<u8>)>,
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
/// C save walk: shard 0 is the owner layer payload; later shards are
|
| 26 |
+
/// remote. Empty shard bytes fail before any DSV4 merge.
|
| 27 |
+
pub fn gather_kv_shards<F>(
|
| 28 |
+
tokens: &[i32],
|
| 29 |
+
plan: &[GatherShard],
|
| 30 |
+
mut fetch: F,
|
| 31 |
+
) -> Result<GatheredShards, String>
|
| 32 |
+
where
|
| 33 |
+
F: FnMut(GatherShard) -> Result<Vec<u8>, String>,
|
| 34 |
+
{
|
| 35 |
+
if tokens.len() > u32::MAX as usize {
|
| 36 |
+
return Err(ERR_NO_TIMELINE.into());
|
| 37 |
+
}
|
| 38 |
+
if plan.is_empty() {
|
| 39 |
+
return Err(ERR_EMPTY_SHARD.into());
|
| 40 |
+
}
|
| 41 |
+
let token_count = tokens.len() as u32;
|
| 42 |
+
let token_hash = token_hash_prefix(tokens);
|
| 43 |
+
let mut shards = Vec::with_capacity(plan.len());
|
| 44 |
+
for (i, spec) in plan.iter().copied().enumerate() {
|
| 45 |
+
if i == 0 && !spec.local {
|
| 46 |
+
return Err("distributed KV shard 0 must be the owner layer".into());
|
| 47 |
+
}
|
| 48 |
+
if i != 0 && spec.local {
|
| 49 |
+
return Err("distributed KV remote shard marked local".into());
|
| 50 |
+
}
|
| 51 |
+
let bytes = fetch(spec)?;
|
| 52 |
+
if bytes.is_empty() {
|
| 53 |
+
return Err(ERR_EMPTY_SHARD.into());
|
| 54 |
+
}
|
| 55 |
+
shards.push((spec, bytes));
|
| 56 |
+
}
|
| 57 |
+
Ok(GatheredShards {
|
| 58 |
+
token_count,
|
| 59 |
+
token_hash,
|
| 60 |
+
shards,
|
| 61 |
+
})
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
/// C load walk: shard 0 is the owner layer payload; later shards are
|
| 65 |
+
/// remote. Host owns the order. Layer apply and DSV4 parse stay native.
|
| 66 |
+
pub fn scatter_kv_shards<F>(
|
| 67 |
+
tokens: &[i32],
|
| 68 |
+
plan: &[GatherShard],
|
| 69 |
+
shards: &[Vec<u8>],
|
| 70 |
+
mut apply: F,
|
| 71 |
+
) -> Result<u64, String>
|
| 72 |
+
where
|
| 73 |
+
F: FnMut(GatherShard, &[u8]) -> Result<(), String>,
|
| 74 |
+
{
|
| 75 |
+
if tokens.len() > u32::MAX as usize {
|
| 76 |
+
return Err(ERR_NO_TIMELINE.into());
|
| 77 |
+
}
|
| 78 |
+
if plan.is_empty() || plan.len() != shards.len() {
|
| 79 |
+
return Err(ERR_EMPTY_SHARD.into());
|
| 80 |
+
}
|
| 81 |
+
for (i, (spec, bytes)) in plan.iter().copied().zip(shards.iter()).enumerate() {
|
| 82 |
+
if i == 0 && !spec.local {
|
| 83 |
+
return Err("distributed KV shard 0 must be the owner layer".into());
|
| 84 |
+
}
|
| 85 |
+
if i != 0 && spec.local {
|
| 86 |
+
return Err("distributed KV remote shard marked local".into());
|
| 87 |
+
}
|
| 88 |
+
if bytes.is_empty() {
|
| 89 |
+
return Err(ERR_EMPTY_SHARD.into());
|
| 90 |
+
}
|
| 91 |
+
apply(spec, bytes)?;
|
| 92 |
+
}
|
| 93 |
+
Ok(token_hash_prefix(tokens))
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
#[cfg(test)]
|
| 97 |
+
mod tests {
|
| 98 |
+
use super::*;
|
| 99 |
+
|
| 100 |
+
fn two_shard_plan() -> [GatherShard; 2] {
|
| 101 |
+
[
|
| 102 |
+
GatherShard {
|
| 103 |
+
layer_start: 0,
|
| 104 |
+
layer_end: 10,
|
| 105 |
+
local: true,
|
| 106 |
+
},
|
| 107 |
+
GatherShard {
|
| 108 |
+
layer_start: 10,
|
| 109 |
+
layer_end: 20,
|
| 110 |
+
local: false,
|
| 111 |
+
},
|
| 112 |
+
]
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
#[test]
|
| 116 |
+
fn gather_walks_owner_then_remote() {
|
| 117 |
+
let tokens = [1i32, 2, 3];
|
| 118 |
+
let got = gather_kv_shards(&tokens, &two_shard_plan(), |spec| {
|
| 119 |
+
Ok(vec![spec.layer_start as u8, spec.layer_end as u8])
|
| 120 |
+
})
|
| 121 |
+
.unwrap();
|
| 122 |
+
assert_eq!(got.token_count, 3);
|
| 123 |
+
assert_eq!(got.token_hash, token_hash_prefix(&tokens));
|
| 124 |
+
assert_eq!(got.shards.len(), 2);
|
| 125 |
+
assert!(got.shards[0].0.local);
|
| 126 |
+
assert!(!got.shards[1].0.local);
|
| 127 |
+
assert_eq!(got.shards[0].1, vec![0, 10]);
|
| 128 |
+
assert_eq!(got.shards[1].1, vec![10, 20]);
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
#[test]
|
| 132 |
+
fn gather_refuses_empty_shard() {
|
| 133 |
+
let err = gather_kv_shards(&[1], &two_shard_plan(), |spec| {
|
| 134 |
+
if spec.local {
|
| 135 |
+
Ok(vec![1])
|
| 136 |
+
} else {
|
| 137 |
+
Ok(vec![])
|
| 138 |
+
}
|
| 139 |
+
})
|
| 140 |
+
.unwrap_err();
|
| 141 |
+
assert_eq!(err, ERR_EMPTY_SHARD);
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
#[test]
|
| 145 |
+
fn gather_requires_owner_first() {
|
| 146 |
+
let plan = [GatherShard {
|
| 147 |
+
layer_start: 0,
|
| 148 |
+
layer_end: 1,
|
| 149 |
+
local: false,
|
| 150 |
+
}];
|
| 151 |
+
let err = gather_kv_shards(&[1], &plan, |_| Ok(vec![1])).unwrap_err();
|
| 152 |
+
assert!(err.contains("owner layer"));
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
#[test]
|
| 156 |
+
fn gather_accepts_empty_token_timeline() {
|
| 157 |
+
let got = gather_kv_shards(&[], &two_shard_plan(), |_| Ok(vec![1])).unwrap();
|
| 158 |
+
assert_eq!(got.token_count, 0);
|
| 159 |
+
assert_eq!(got.token_hash, token_hash_prefix(&[]));
|
| 160 |
+
}
|
| 161 |
+
|
| 162 |
+
#[test]
|
| 163 |
+
fn scatter_walks_owner_then_remote() {
|
| 164 |
+
let tokens = [7i32];
|
| 165 |
+
let mut seen = Vec::new();
|
| 166 |
+
let hash = scatter_kv_shards(
|
| 167 |
+
&tokens,
|
| 168 |
+
&two_shard_plan(),
|
| 169 |
+
&[vec![1], vec![2]],
|
| 170 |
+
|spec, bytes| {
|
| 171 |
+
seen.push((spec.local, bytes.to_vec()));
|
| 172 |
+
Ok(())
|
| 173 |
+
},
|
| 174 |
+
)
|
| 175 |
+
.unwrap();
|
| 176 |
+
assert_eq!(hash, token_hash_prefix(&tokens));
|
| 177 |
+
assert_eq!(seen, vec![(true, vec![1]), (false, vec![2])]);
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
#[test]
|
| 181 |
+
fn scatter_refuses_empty_shard() {
|
| 182 |
+
let err = scatter_kv_shards(&[1], &two_shard_plan(), &[vec![1], vec![]], |_, _| Ok(()))
|
| 183 |
+
.unwrap_err();
|
| 184 |
+
assert_eq!(err, ERR_EMPTY_SHARD);
|
| 185 |
+
}
|
| 186 |
+
}
|
crates/ds4-dist/src/snapshot_merge.rs
ADDED
|
@@ -0,0 +1,415 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Coordinator DSV4 assembly from `ds4_dist_session_save_payload`.
|
| 2 |
+
//!
|
| 3 |
+
//! Host owns layer-header parse, layout match, tensor-size arithmetic, and
|
| 4 |
+
//! write order. Layer GPU bytes and logits stay opaque blobs.
|
| 5 |
+
|
| 6 |
+
use crate::snapshot_gather::ERR_EMPTY_SHARD;
|
| 7 |
+
|
| 8 |
+
pub const LAYER_MAGIC: u32 = 0x4C56_5344; /* "DSVL" */
|
| 9 |
+
pub const LAYER_VERSION: u32 = 1;
|
| 10 |
+
pub const LAYER_U32_FIELDS: usize = 14;
|
| 11 |
+
pub const SESSION_MAGIC: u32 = 0x3456_5344; /* "DSV4" */
|
| 12 |
+
pub const SESSION_VERSION: u32 = 3;
|
| 13 |
+
pub const SESSION_U32_FIELDS: usize = 13;
|
| 14 |
+
|
| 15 |
+
pub const ERR_UNSUPPORTED_LAYER: &str = "unsupported distributed KV layer payload";
|
| 16 |
+
pub const ERR_RANGE: &str = "distributed KV shard range mismatch";
|
| 17 |
+
pub const ERR_LAYOUT: &str = "distributed KV shard layout is invalid";
|
| 18 |
+
pub const ERR_LAYOUTS_DIFFER: &str = "distributed KV shards use different layouts";
|
| 19 |
+
pub const ERR_COMP_ROWS: &str = "distributed KV shard has invalid compressed row count";
|
| 20 |
+
pub const ERR_INDEX_ROWS: &str = "distributed KV shard has invalid indexer row count";
|
| 21 |
+
pub const ERR_TENSOR_OVERFLOW: &str = "distributed KV shard tensor size overflow";
|
| 22 |
+
pub const ERR_TENSOR_BYTES: &str = "distributed KV shard tensor byte count mismatch";
|
| 23 |
+
pub const ERR_METADATA: &str = "distributed KV shard metadata mismatch";
|
| 24 |
+
|
| 25 |
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
| 26 |
+
pub struct DistKvLayout {
|
| 27 |
+
pub ctx: u32,
|
| 28 |
+
pub prefill_cap: u32,
|
| 29 |
+
pub raw_cap: u32,
|
| 30 |
+
pub raw_window: u32,
|
| 31 |
+
pub comp_cap: u32,
|
| 32 |
+
pub token_count: u32,
|
| 33 |
+
pub n_layers: u32,
|
| 34 |
+
pub head_dim: u32,
|
| 35 |
+
pub indexer_head_dim: u32,
|
| 36 |
+
pub vocab: u32,
|
| 37 |
+
pub raw_live: u32,
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
impl DistKvLayout {
|
| 41 |
+
pub fn matches_core(&self, other: &Self) -> bool {
|
| 42 |
+
self.ctx == other.ctx
|
| 43 |
+
&& self.prefill_cap == other.prefill_cap
|
| 44 |
+
&& self.raw_cap == other.raw_cap
|
| 45 |
+
&& self.raw_window == other.raw_window
|
| 46 |
+
&& self.comp_cap == other.comp_cap
|
| 47 |
+
&& self.token_count == other.token_count
|
| 48 |
+
&& self.n_layers == other.n_layers
|
| 49 |
+
&& self.head_dim == other.head_dim
|
| 50 |
+
&& self.indexer_head_dim == other.indexer_head_dim
|
| 51 |
+
&& self.raw_live == other.raw_live
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
pub fn raw_live_valid(&self) -> bool {
|
| 55 |
+
if self.raw_window == 0 || self.raw_cap == 0 {
|
| 56 |
+
return false;
|
| 57 |
+
}
|
| 58 |
+
let expected = self.token_count.min(self.raw_window);
|
| 59 |
+
self.raw_live == expected && self.raw_live <= self.raw_cap
|
| 60 |
+
}
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
| 64 |
+
pub struct ParsedLayerShard {
|
| 65 |
+
pub layout: DistKvLayout,
|
| 66 |
+
pub layer_start: u32,
|
| 67 |
+
pub layer_end: u32,
|
| 68 |
+
pub n_comp: Vec<u32>,
|
| 69 |
+
pub n_index_comp: Vec<u32>,
|
| 70 |
+
pub tensor: Vec<u8>,
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
fn get_u32(bytes: &[u8]) -> Option<u32> {
|
| 74 |
+
bytes.get(..4)?.try_into().ok().map(u32::from_le_bytes)
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
fn put_u32(out: &mut Vec<u8>, v: u32) {
|
| 78 |
+
out.extend_from_slice(&v.to_le_bytes());
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
/// C `dist_kv_state_bytes`.
|
| 82 |
+
pub fn kv_state_bytes(ratio: u32, head_dim: u32) -> Option<u64> {
|
| 83 |
+
let coff = if ratio == 4 { 2u64 } else { 1u64 };
|
| 84 |
+
coff.checked_mul(u64::from(head_dim))?
|
| 85 |
+
.checked_mul(coff)?
|
| 86 |
+
.checked_mul(u64::from(ratio))?
|
| 87 |
+
.checked_mul(4)
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
/// C `dist_kv_layer_tensor_bytes` once the host compress ratio is known.
|
| 91 |
+
pub fn layer_tensor_bytes(
|
| 92 |
+
layout: &DistKvLayout,
|
| 93 |
+
ratio: u32,
|
| 94 |
+
n_comp: u32,
|
| 95 |
+
n_index_comp: u32,
|
| 96 |
+
) -> Option<u64> {
|
| 97 |
+
let mut bytes = u64::from(layout.raw_live)
|
| 98 |
+
.checked_mul(u64::from(layout.head_dim))?
|
| 99 |
+
.checked_mul(4)?;
|
| 100 |
+
if ratio == 0 {
|
| 101 |
+
return Some(bytes);
|
| 102 |
+
}
|
| 103 |
+
bytes = bytes.checked_add(
|
| 104 |
+
u64::from(n_comp)
|
| 105 |
+
.checked_mul(u64::from(layout.head_dim))?
|
| 106 |
+
.checked_mul(4)?,
|
| 107 |
+
)?;
|
| 108 |
+
let attn = kv_state_bytes(ratio, layout.head_dim)?;
|
| 109 |
+
bytes = bytes.checked_add(attn)?.checked_add(attn)?;
|
| 110 |
+
if ratio == 4 {
|
| 111 |
+
bytes = bytes.checked_add(
|
| 112 |
+
u64::from(n_index_comp)
|
| 113 |
+
.checked_mul(u64::from(layout.indexer_head_dim))?
|
| 114 |
+
.checked_mul(4)?,
|
| 115 |
+
)?;
|
| 116 |
+
let index = kv_state_bytes(ratio, layout.indexer_head_dim)?;
|
| 117 |
+
bytes = bytes.checked_add(index)?.checked_add(index)?;
|
| 118 |
+
}
|
| 119 |
+
Some(bytes)
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
pub fn parse_layer_payload(
|
| 123 |
+
bytes: &[u8],
|
| 124 |
+
expected_start: u32,
|
| 125 |
+
expected_end: u32,
|
| 126 |
+
compress: &[u32],
|
| 127 |
+
) -> Result<ParsedLayerShard, String> {
|
| 128 |
+
if bytes.len() < LAYER_U32_FIELDS * 4 {
|
| 129 |
+
return Err(ERR_UNSUPPORTED_LAYER.into());
|
| 130 |
+
}
|
| 131 |
+
let mut h = [0u32; LAYER_U32_FIELDS];
|
| 132 |
+
for (i, slot) in h.iter_mut().enumerate() {
|
| 133 |
+
*slot = get_u32(&bytes[i * 4..]).ok_or(ERR_UNSUPPORTED_LAYER)?;
|
| 134 |
+
}
|
| 135 |
+
if h[0] != LAYER_MAGIC || h[1] != LAYER_VERSION {
|
| 136 |
+
return Err(ERR_UNSUPPORTED_LAYER.into());
|
| 137 |
+
}
|
| 138 |
+
let layout = DistKvLayout {
|
| 139 |
+
ctx: h[2],
|
| 140 |
+
prefill_cap: h[3],
|
| 141 |
+
raw_cap: h[4],
|
| 142 |
+
raw_window: h[5],
|
| 143 |
+
comp_cap: h[6],
|
| 144 |
+
token_count: h[7],
|
| 145 |
+
n_layers: h[8],
|
| 146 |
+
head_dim: h[9],
|
| 147 |
+
indexer_head_dim: h[10],
|
| 148 |
+
vocab: 0,
|
| 149 |
+
raw_live: h[13],
|
| 150 |
+
};
|
| 151 |
+
let layer_start = h[11];
|
| 152 |
+
let layer_end = h[12];
|
| 153 |
+
if layer_start != expected_start
|
| 154 |
+
|| layer_end != expected_end
|
| 155 |
+
|| layer_start > layer_end
|
| 156 |
+
|| layer_end >= layout.n_layers
|
| 157 |
+
{
|
| 158 |
+
return Err(ERR_RANGE.into());
|
| 159 |
+
}
|
| 160 |
+
if !layout.raw_live_valid() {
|
| 161 |
+
return Err(ERR_LAYOUT.into());
|
| 162 |
+
}
|
| 163 |
+
let slice = (layer_end - layer_start + 1) as usize;
|
| 164 |
+
let mut off = LAYER_U32_FIELDS * 4;
|
| 165 |
+
let mut n_comp = vec![0u32; layout.n_layers as usize];
|
| 166 |
+
let mut n_index_comp = vec![0u32; layout.n_layers as usize];
|
| 167 |
+
for i in 0..slice {
|
| 168 |
+
let il = (layer_start as usize) + i;
|
| 169 |
+
let v = get_u32(bytes.get(off..).unwrap_or(&[])).ok_or(ERR_UNSUPPORTED_LAYER)?;
|
| 170 |
+
if v > layout.comp_cap {
|
| 171 |
+
return Err(ERR_COMP_ROWS.into());
|
| 172 |
+
}
|
| 173 |
+
n_comp[il] = v;
|
| 174 |
+
off += 4;
|
| 175 |
+
}
|
| 176 |
+
for i in 0..slice {
|
| 177 |
+
let il = (layer_start as usize) + i;
|
| 178 |
+
let v = get_u32(bytes.get(off..).unwrap_or(&[])).ok_or(ERR_UNSUPPORTED_LAYER)?;
|
| 179 |
+
if v > layout.comp_cap {
|
| 180 |
+
return Err(ERR_INDEX_ROWS.into());
|
| 181 |
+
}
|
| 182 |
+
n_index_comp[il] = v;
|
| 183 |
+
off += 4;
|
| 184 |
+
}
|
| 185 |
+
let mut expected_tensor = 0u64;
|
| 186 |
+
for il in layer_start..=layer_end {
|
| 187 |
+
let ratio = compress.get(il as usize).copied().unwrap_or(0);
|
| 188 |
+
let layer_bytes = layer_tensor_bytes(
|
| 189 |
+
&layout,
|
| 190 |
+
ratio,
|
| 191 |
+
n_comp[il as usize],
|
| 192 |
+
n_index_comp[il as usize],
|
| 193 |
+
)
|
| 194 |
+
.ok_or(ERR_TENSOR_OVERFLOW)?;
|
| 195 |
+
expected_tensor = expected_tensor
|
| 196 |
+
.checked_add(layer_bytes)
|
| 197 |
+
.ok_or(ERR_TENSOR_OVERFLOW)?;
|
| 198 |
+
}
|
| 199 |
+
let tensor = bytes.get(off..).unwrap_or(&[]).to_vec();
|
| 200 |
+
if tensor.len() as u64 != expected_tensor {
|
| 201 |
+
return Err(ERR_TENSOR_BYTES.into());
|
| 202 |
+
}
|
| 203 |
+
Ok(ParsedLayerShard {
|
| 204 |
+
layout,
|
| 205 |
+
layer_start,
|
| 206 |
+
layer_end,
|
| 207 |
+
n_comp,
|
| 208 |
+
n_index_comp,
|
| 209 |
+
tensor,
|
| 210 |
+
})
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
+
/// C save merge after shards are gathered: header, tokens, logits,
|
| 214 |
+
/// n_comp, n_index_comp, then shard tensor tails in walk order.
|
| 215 |
+
pub fn merge_session_payload(
|
| 216 |
+
tokens: &[i32],
|
| 217 |
+
logits: &[f32],
|
| 218 |
+
n_layers: u32,
|
| 219 |
+
shards: &[ParsedLayerShard],
|
| 220 |
+
compress: &[u32],
|
| 221 |
+
) -> Result<Vec<u8>, String> {
|
| 222 |
+
if shards.is_empty() {
|
| 223 |
+
return Err(ERR_EMPTY_SHARD.into());
|
| 224 |
+
}
|
| 225 |
+
let token_count = u32::try_from(tokens.len()).map_err(|_| ERR_METADATA.to_string())?;
|
| 226 |
+
let vocab = u32::try_from(logits.len()).map_err(|_| ERR_METADATA.to_string())?;
|
| 227 |
+
let mut layout = shards[0].layout;
|
| 228 |
+
layout.vocab = vocab;
|
| 229 |
+
if layout.token_count != token_count || layout.n_layers != n_layers || !layout.raw_live_valid()
|
| 230 |
+
{
|
| 231 |
+
return Err(ERR_METADATA.into());
|
| 232 |
+
}
|
| 233 |
+
let mut n_comp = vec![0u32; n_layers as usize];
|
| 234 |
+
let mut n_index_comp = vec![0u32; n_layers as usize];
|
| 235 |
+
let mut covered = vec![false; n_layers as usize];
|
| 236 |
+
for shard in shards {
|
| 237 |
+
if !layout.matches_core(&shard.layout) {
|
| 238 |
+
return Err(ERR_LAYOUTS_DIFFER.into());
|
| 239 |
+
}
|
| 240 |
+
let mut shard_bytes = 0u64;
|
| 241 |
+
for il in shard.layer_start..=shard.layer_end {
|
| 242 |
+
let i = il as usize;
|
| 243 |
+
if covered[i] {
|
| 244 |
+
return Err(ERR_RANGE.into());
|
| 245 |
+
}
|
| 246 |
+
covered[i] = true;
|
| 247 |
+
n_comp[i] = shard.n_comp[i];
|
| 248 |
+
n_index_comp[i] = shard.n_index_comp[i];
|
| 249 |
+
let ratio = compress.get(i).copied().unwrap_or(0);
|
| 250 |
+
let layer_bytes = layer_tensor_bytes(&layout, ratio, n_comp[i], n_index_comp[i])
|
| 251 |
+
.ok_or(ERR_TENSOR_OVERFLOW)?;
|
| 252 |
+
shard_bytes = shard_bytes
|
| 253 |
+
.checked_add(layer_bytes)
|
| 254 |
+
.ok_or(ERR_TENSOR_OVERFLOW)?;
|
| 255 |
+
}
|
| 256 |
+
if shard.tensor.len() as u64 != shard_bytes {
|
| 257 |
+
return Err(ERR_TENSOR_BYTES.into());
|
| 258 |
+
}
|
| 259 |
+
}
|
| 260 |
+
if covered.iter().any(|c| !*c) {
|
| 261 |
+
return Err(ERR_METADATA.into());
|
| 262 |
+
}
|
| 263 |
+
let mut out = Vec::new();
|
| 264 |
+
for f in [
|
| 265 |
+
SESSION_MAGIC,
|
| 266 |
+
SESSION_VERSION,
|
| 267 |
+
layout.ctx,
|
| 268 |
+
layout.prefill_cap,
|
| 269 |
+
layout.raw_cap,
|
| 270 |
+
layout.raw_window,
|
| 271 |
+
layout.comp_cap,
|
| 272 |
+
layout.token_count,
|
| 273 |
+
layout.n_layers,
|
| 274 |
+
layout.head_dim,
|
| 275 |
+
layout.indexer_head_dim,
|
| 276 |
+
layout.vocab,
|
| 277 |
+
layout.raw_live,
|
| 278 |
+
] {
|
| 279 |
+
put_u32(&mut out, f);
|
| 280 |
+
}
|
| 281 |
+
for &tok in tokens {
|
| 282 |
+
put_u32(&mut out, tok as u32);
|
| 283 |
+
}
|
| 284 |
+
for &logit in logits {
|
| 285 |
+
out.extend_from_slice(&logit.to_le_bytes());
|
| 286 |
+
}
|
| 287 |
+
for v in &n_comp {
|
| 288 |
+
put_u32(&mut out, *v);
|
| 289 |
+
}
|
| 290 |
+
for v in &n_index_comp {
|
| 291 |
+
put_u32(&mut out, *v);
|
| 292 |
+
}
|
| 293 |
+
for shard in shards {
|
| 294 |
+
out.extend_from_slice(&shard.tensor);
|
| 295 |
+
}
|
| 296 |
+
Ok(out)
|
| 297 |
+
}
|
| 298 |
+
|
| 299 |
+
#[cfg(test)]
|
| 300 |
+
mod tests {
|
| 301 |
+
use super::*;
|
| 302 |
+
|
| 303 |
+
fn layout() -> DistKvLayout {
|
| 304 |
+
DistKvLayout {
|
| 305 |
+
ctx: 16,
|
| 306 |
+
prefill_cap: 8,
|
| 307 |
+
raw_cap: 16,
|
| 308 |
+
raw_window: 4,
|
| 309 |
+
comp_cap: 8,
|
| 310 |
+
token_count: 2,
|
| 311 |
+
n_layers: 2,
|
| 312 |
+
head_dim: 4,
|
| 313 |
+
indexer_head_dim: 2,
|
| 314 |
+
vocab: 0,
|
| 315 |
+
raw_live: 2,
|
| 316 |
+
}
|
| 317 |
+
}
|
| 318 |
+
|
| 319 |
+
fn encode_layer(
|
| 320 |
+
start: u32,
|
| 321 |
+
end: u32,
|
| 322 |
+
n_comp: &[u32],
|
| 323 |
+
n_index: &[u32],
|
| 324 |
+
tensor: &[u8],
|
| 325 |
+
) -> Vec<u8> {
|
| 326 |
+
let l = layout();
|
| 327 |
+
let mut out = Vec::new();
|
| 328 |
+
for f in [
|
| 329 |
+
LAYER_MAGIC,
|
| 330 |
+
LAYER_VERSION,
|
| 331 |
+
l.ctx,
|
| 332 |
+
l.prefill_cap,
|
| 333 |
+
l.raw_cap,
|
| 334 |
+
l.raw_window,
|
| 335 |
+
l.comp_cap,
|
| 336 |
+
l.token_count,
|
| 337 |
+
l.n_layers,
|
| 338 |
+
l.head_dim,
|
| 339 |
+
l.indexer_head_dim,
|
| 340 |
+
start,
|
| 341 |
+
end,
|
| 342 |
+
l.raw_live,
|
| 343 |
+
] {
|
| 344 |
+
put_u32(&mut out, f);
|
| 345 |
+
}
|
| 346 |
+
for il in start..=end {
|
| 347 |
+
put_u32(&mut out, n_comp[il as usize]);
|
| 348 |
+
}
|
| 349 |
+
for il in start..=end {
|
| 350 |
+
put_u32(&mut out, n_index[il as usize]);
|
| 351 |
+
}
|
| 352 |
+
out.extend_from_slice(tensor);
|
| 353 |
+
out
|
| 354 |
+
}
|
| 355 |
+
|
| 356 |
+
#[test]
|
| 357 |
+
fn layer_tensor_bytes_ratio_zero_is_raw_only() {
|
| 358 |
+
let l = layout();
|
| 359 |
+
assert_eq!(layer_tensor_bytes(&l, 0, 3, 9), Some(2 * 4 * 4));
|
| 360 |
+
}
|
| 361 |
+
|
| 362 |
+
#[test]
|
| 363 |
+
fn parse_and_merge_two_uncompressed_shards() {
|
| 364 |
+
let compress = [0u32, 0];
|
| 365 |
+
let t0 = vec![1u8; 32];
|
| 366 |
+
let t1 = vec![2u8; 32];
|
| 367 |
+
let s0 = parse_layer_payload(&encode_layer(0, 0, &[0, 0], &[0, 0], &t0), 0, 0, &compress)
|
| 368 |
+
.unwrap();
|
| 369 |
+
let s1 = parse_layer_payload(&encode_layer(1, 1, &[0, 0], &[0, 0], &t1), 1, 1, &compress)
|
| 370 |
+
.unwrap();
|
| 371 |
+
let tokens = [7i32, 8];
|
| 372 |
+
let logits = [0.5f32, -1.0];
|
| 373 |
+
let merged = merge_session_payload(&tokens, &logits, 2, &[s0, s1], &compress).unwrap();
|
| 374 |
+
assert_eq!(&merged[..4], &SESSION_MAGIC.to_le_bytes());
|
| 375 |
+
assert_eq!(&merged[4..8], &SESSION_VERSION.to_le_bytes());
|
| 376 |
+
let header = SESSION_U32_FIELDS * 4;
|
| 377 |
+
assert_eq!(&merged[header..header + 4], &7u32.to_le_bytes());
|
| 378 |
+
assert_eq!(&merged[header + 4..header + 8], &8u32.to_le_bytes());
|
| 379 |
+
let after_tokens = header + 8;
|
| 380 |
+
assert_eq!(
|
| 381 |
+
&merged[after_tokens..after_tokens + 4],
|
| 382 |
+
&0.5f32.to_le_bytes()
|
| 383 |
+
);
|
| 384 |
+
let after_logits = after_tokens + 8;
|
| 385 |
+
let after_counts = after_logits + 16;
|
| 386 |
+
assert_eq!(&merged[after_counts..after_counts + 32], &t0[..]);
|
| 387 |
+
assert_eq!(&merged[after_counts + 32..after_counts + 64], &t1[..]);
|
| 388 |
+
}
|
| 389 |
+
|
| 390 |
+
#[test]
|
| 391 |
+
fn parse_refuses_tensor_byte_mismatch() {
|
| 392 |
+
let compress = [0u32, 0];
|
| 393 |
+
let err = parse_layer_payload(
|
| 394 |
+
&encode_layer(0, 0, &[0, 0], &[0, 0], &[1, 2, 3]),
|
| 395 |
+
0,
|
| 396 |
+
0,
|
| 397 |
+
&compress,
|
| 398 |
+
)
|
| 399 |
+
.unwrap_err();
|
| 400 |
+
assert_eq!(err, ERR_TENSOR_BYTES);
|
| 401 |
+
}
|
| 402 |
+
|
| 403 |
+
#[test]
|
| 404 |
+
fn merge_refuses_layout_mismatch() {
|
| 405 |
+
let compress = [0u32, 0];
|
| 406 |
+
let t = vec![1u8; 32];
|
| 407 |
+
let s0 = parse_layer_payload(&encode_layer(0, 0, &[0, 0], &[0, 0], &t), 0, 0, &compress)
|
| 408 |
+
.unwrap();
|
| 409 |
+
let mut other = encode_layer(1, 1, &[0, 0], &[0, 0], &t);
|
| 410 |
+
other[2 * 4..2 * 4 + 4].copy_from_slice(&32u32.to_le_bytes());
|
| 411 |
+
let s1 = parse_layer_payload(&other, 1, 1, &compress).unwrap();
|
| 412 |
+
let err = merge_session_payload(&[7, 8], &[0.0, 0.0], 2, &[s0, s1], &compress).unwrap_err();
|
| 413 |
+
assert_eq!(err, ERR_LAYOUTS_DIFFER);
|
| 414 |
+
}
|
| 415 |
+
}
|
crates/ds4-dist/src/snapshot_temp.rs
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Bounded temp files for worker KV shard save/load.
|
| 2 |
+
|
| 3 |
+
use std::fs::{File, OpenOptions};
|
| 4 |
+
use std::io::{self, Read, Seek, SeekFrom, Write};
|
| 5 |
+
use std::path::PathBuf;
|
| 6 |
+
|
| 7 |
+
use crate::codec::SNAPSHOT_CHUNK_BYTES;
|
| 8 |
+
|
| 9 |
+
pub const SAVE_PREFIX: &str = "ds4-dist-save";
|
| 10 |
+
pub const LOAD_PREFIX: &str = "ds4-dist-load";
|
| 11 |
+
|
| 12 |
+
pub struct TempShard {
|
| 13 |
+
pub(crate) file: File,
|
| 14 |
+
pub(crate) path: PathBuf,
|
| 15 |
+
}
|
| 16 |
+
|
| 17 |
+
impl TempShard {
|
| 18 |
+
pub fn path(&self) -> &std::path::Path {
|
| 19 |
+
&self.path
|
| 20 |
+
}
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
impl Drop for TempShard {
|
| 24 |
+
fn drop(&mut self) {
|
| 25 |
+
let _ = std::fs::remove_file(&self.path);
|
| 26 |
+
}
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
impl Read for TempShard {
|
| 30 |
+
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
| 31 |
+
self.file.read(buf)
|
| 32 |
+
}
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
impl Write for TempShard {
|
| 36 |
+
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
| 37 |
+
self.file.write(buf)
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
fn flush(&mut self) -> io::Result<()> {
|
| 41 |
+
self.file.flush()
|
| 42 |
+
}
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
impl Seek for TempShard {
|
| 46 |
+
fn seek(&mut self, pos: SeekFrom) -> io::Result<u64> {
|
| 47 |
+
self.file.seek(pos)
|
| 48 |
+
}
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
fn temp_root() -> PathBuf {
|
| 52 |
+
#[cfg(unix)]
|
| 53 |
+
{
|
| 54 |
+
PathBuf::from("/tmp")
|
| 55 |
+
}
|
| 56 |
+
#[cfg(not(unix))]
|
| 57 |
+
{
|
| 58 |
+
std::env::temp_dir()
|
| 59 |
+
}
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
pub fn create_temp(prefix: &str) -> io::Result<TempShard> {
|
| 63 |
+
let root = temp_root();
|
| 64 |
+
for n in 0u32..1024 {
|
| 65 |
+
let path = root.join(format!("{prefix}.{}.{:04x}", std::process::id(), n));
|
| 66 |
+
match OpenOptions::new()
|
| 67 |
+
.read(true)
|
| 68 |
+
.write(true)
|
| 69 |
+
.create_new(true)
|
| 70 |
+
.open(&path)
|
| 71 |
+
{
|
| 72 |
+
Ok(file) => return Ok(TempShard { file, path }),
|
| 73 |
+
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue,
|
| 74 |
+
Err(e) => return Err(e),
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
Err(io::Error::new(
|
| 78 |
+
io::ErrorKind::AlreadyExists,
|
| 79 |
+
"temp name space exhausted",
|
| 80 |
+
))
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
pub fn copy_chunked(
|
| 84 |
+
src: &mut dyn Read,
|
| 85 |
+
dest: &mut dyn Write,
|
| 86 |
+
mut remaining: u64,
|
| 87 |
+
) -> io::Result<()> {
|
| 88 |
+
if remaining == 0 {
|
| 89 |
+
return Ok(());
|
| 90 |
+
}
|
| 91 |
+
let mut buf = vec![0u8; SNAPSHOT_CHUNK_BYTES];
|
| 92 |
+
while remaining != 0 {
|
| 93 |
+
let n = remaining.min(SNAPSHOT_CHUNK_BYTES as u64) as usize;
|
| 94 |
+
src.read_exact(&mut buf[..n])?;
|
| 95 |
+
dest.write_all(&buf[..n])?;
|
| 96 |
+
remaining -= n as u64;
|
| 97 |
+
}
|
| 98 |
+
Ok(())
|
| 99 |
+
}
|
crates/ds4-dist/src/transport.rs
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Blocking frame I/O. Same layout as `dist_write_frame_header` / `dist_read_frame_header`.
|
| 2 |
+
|
| 3 |
+
use std::io::{self, Read, Write};
|
| 4 |
+
|
| 5 |
+
use crate::codec::{decode_frame_header, encode_frame_header, CodecError, FRAME_HEADER_BYTES};
|
| 6 |
+
|
| 7 |
+
pub fn write_frame<W: Write>(w: &mut W, typ: u32, payload: &[u8]) -> io::Result<()> {
|
| 8 |
+
let bytes = u32::try_from(payload.len())
|
| 9 |
+
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "distributed frame too large"))?;
|
| 10 |
+
w.write_all(&encode_frame_header(typ, bytes))?;
|
| 11 |
+
w.write_all(payload)?;
|
| 12 |
+
Ok(())
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
pub fn read_frame<R: Read>(r: &mut R) -> io::Result<(u32, Vec<u8>)> {
|
| 16 |
+
let mut hdr = [0u8; FRAME_HEADER_BYTES];
|
| 17 |
+
r.read_exact(&mut hdr)?;
|
| 18 |
+
let h = decode_frame_header(&hdr).map_err(|e| match e {
|
| 19 |
+
CodecError::BadMagic(m) => io::Error::new(
|
| 20 |
+
io::ErrorKind::InvalidData,
|
| 21 |
+
format!("bad frame magic 0x{m:08x}"),
|
| 22 |
+
),
|
| 23 |
+
other => io::Error::new(io::ErrorKind::InvalidData, other.to_string()),
|
| 24 |
+
})?;
|
| 25 |
+
let mut payload = vec![0u8; h.bytes as usize];
|
| 26 |
+
if h.bytes != 0 {
|
| 27 |
+
r.read_exact(&mut payload)?;
|
| 28 |
+
}
|
| 29 |
+
Ok((h.typ, payload))
|
| 30 |
+
}
|
crates/ds4-dist/src/work.rs
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! WORK / RESULT frame bodies. Tokens are BE u32; activations use the packers.
|
| 2 |
+
|
| 3 |
+
use crate::activation::{bits_or_default, decode_activation, encode_activation};
|
| 4 |
+
use crate::codec::{
|
| 5 |
+
decode_tokens_be, encode_frame_header, encode_tokens_be, u64_from_halves, u64_to_halves,
|
| 6 |
+
CodecError, ResultHdr, Telemetry, Work, MSG_RESULT, MSG_WORK, RESULT_FIXED_BYTES,
|
| 7 |
+
TELEMETRY_FIXED_BYTES, WORK_FIXED_BYTES,
|
| 8 |
+
};
|
| 9 |
+
|
| 10 |
+
#[derive(Debug, Clone)]
|
| 11 |
+
pub struct WorkBody {
|
| 12 |
+
pub work: Work,
|
| 13 |
+
pub tokens: Vec<i32>,
|
| 14 |
+
pub input_hc: Vec<f32>,
|
| 15 |
+
pub route_blob: Vec<u8>,
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
pub fn encode_work_body(body: &WorkBody) -> Result<Vec<u8>, CodecError> {
|
| 19 |
+
if body.tokens.is_empty() {
|
| 20 |
+
return Err(CodecError::Invalid("WORK frame has no tokens"));
|
| 21 |
+
}
|
| 22 |
+
let mut work = body.work;
|
| 23 |
+
work.n_tokens = body.tokens.len() as u32;
|
| 24 |
+
work.token_bytes = work.n_tokens * 4;
|
| 25 |
+
let hc_bits = bits_or_default(work.input_hc_bits);
|
| 26 |
+
let hc_wire = if body.input_hc.is_empty() {
|
| 27 |
+
Vec::new()
|
| 28 |
+
} else {
|
| 29 |
+
encode_activation(&body.input_hc, hc_bits)
|
| 30 |
+
.ok_or(CodecError::Invalid("invalid distributed activation width"))?
|
| 31 |
+
};
|
| 32 |
+
work.input_hc_bytes = hc_wire.len() as u32;
|
| 33 |
+
work.input_hc_bits = hc_bits;
|
| 34 |
+
work.route_bytes = body.route_blob.len() as u32;
|
| 35 |
+
let mut out = work.encode().to_vec();
|
| 36 |
+
out.extend_from_slice(&encode_tokens_be(&body.tokens));
|
| 37 |
+
out.extend_from_slice(&hc_wire);
|
| 38 |
+
out.extend_from_slice(&body.route_blob);
|
| 39 |
+
Ok(out)
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
pub fn encode_work_frame(body: &WorkBody) -> Result<Vec<u8>, CodecError> {
|
| 43 |
+
let payload = encode_work_body(body)?;
|
| 44 |
+
let mut frame = encode_frame_header(MSG_WORK, payload.len() as u32).to_vec();
|
| 45 |
+
frame.extend_from_slice(&payload);
|
| 46 |
+
Ok(frame)
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
pub fn decode_work_body(buf: &[u8]) -> Result<WorkBody, CodecError> {
|
| 50 |
+
if buf.len() < WORK_FIXED_BYTES {
|
| 51 |
+
return Err(CodecError::Invalid("truncated distributed WORK frame"));
|
| 52 |
+
}
|
| 53 |
+
let work = Work::decode(buf)?;
|
| 54 |
+
let rest = &buf[WORK_FIXED_BYTES..];
|
| 55 |
+
let expected =
|
| 56 |
+
work.token_bytes as usize + work.input_hc_bytes as usize + work.route_bytes as usize;
|
| 57 |
+
if work.token_bytes as usize != work.n_tokens as usize * 4 || rest.len() != expected {
|
| 58 |
+
return Err(CodecError::Invalid(
|
| 59 |
+
"invalid distributed WORK payload sizes",
|
| 60 |
+
));
|
| 61 |
+
}
|
| 62 |
+
let (tok, rest) = rest.split_at(work.token_bytes as usize);
|
| 63 |
+
let tokens = decode_tokens_be(tok)?;
|
| 64 |
+
let (hc, rest) = rest.split_at(work.input_hc_bytes as usize);
|
| 65 |
+
let input_hc = if hc.is_empty() {
|
| 66 |
+
Vec::new()
|
| 67 |
+
} else {
|
| 68 |
+
decode_activation(hc, work.input_hc_bits)
|
| 69 |
+
.ok_or(CodecError::Invalid("invalid distributed activation width"))?
|
| 70 |
+
};
|
| 71 |
+
Ok(WorkBody {
|
| 72 |
+
work,
|
| 73 |
+
tokens,
|
| 74 |
+
input_hc,
|
| 75 |
+
route_blob: rest.to_vec(),
|
| 76 |
+
})
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
#[derive(Debug, Clone)]
|
| 80 |
+
pub struct ResultBody {
|
| 81 |
+
pub hdr: ResultHdr,
|
| 82 |
+
pub telemetry: Vec<Telemetry>,
|
| 83 |
+
pub payload: Vec<u8>,
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
pub fn encode_result_body(body: &ResultBody) -> Result<Vec<u8>, CodecError> {
|
| 87 |
+
let mut hdr = body.hdr;
|
| 88 |
+
hdr.telemetry_count = body.telemetry.len() as u32;
|
| 89 |
+
hdr.telemetry_bytes = hdr.telemetry_count * TELEMETRY_FIXED_BYTES as u32;
|
| 90 |
+
hdr.payload_bytes = body.payload.len() as u32;
|
| 91 |
+
let mut out = hdr.encode().to_vec();
|
| 92 |
+
for t in &body.telemetry {
|
| 93 |
+
out.extend_from_slice(&t.encode());
|
| 94 |
+
}
|
| 95 |
+
out.extend_from_slice(&body.payload);
|
| 96 |
+
Ok(out)
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
pub fn encode_result_frame(body: &ResultBody) -> Result<Vec<u8>, CodecError> {
|
| 100 |
+
let payload = encode_result_body(body)?;
|
| 101 |
+
let mut frame = encode_frame_header(MSG_RESULT, payload.len() as u32).to_vec();
|
| 102 |
+
frame.extend_from_slice(&payload);
|
| 103 |
+
Ok(frame)
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
pub fn decode_result_body(buf: &[u8]) -> Result<ResultBody, CodecError> {
|
| 107 |
+
if buf.len() < RESULT_FIXED_BYTES {
|
| 108 |
+
return Err(CodecError::Invalid(
|
| 109 |
+
"distributed worker returned invalid frame",
|
| 110 |
+
));
|
| 111 |
+
}
|
| 112 |
+
let hdr = ResultHdr::decode(buf)?;
|
| 113 |
+
let rest = &buf[RESULT_FIXED_BYTES..];
|
| 114 |
+
if hdr.telemetry_bytes as usize % TELEMETRY_FIXED_BYTES != 0
|
| 115 |
+
|| hdr.telemetry_count as usize != hdr.telemetry_bytes as usize / TELEMETRY_FIXED_BYTES
|
| 116 |
+
|| hdr.telemetry_bytes as usize > rest.len()
|
| 117 |
+
|| hdr.payload_bytes as usize != rest.len() - hdr.telemetry_bytes as usize
|
| 118 |
+
{
|
| 119 |
+
return Err(CodecError::Invalid(
|
| 120 |
+
"distributed result telemetry metadata mismatch",
|
| 121 |
+
));
|
| 122 |
+
}
|
| 123 |
+
let (tel, rest) = rest.split_at(hdr.telemetry_bytes as usize);
|
| 124 |
+
let mut telemetry = Vec::new();
|
| 125 |
+
for chunk in tel.chunks_exact(TELEMETRY_FIXED_BYTES) {
|
| 126 |
+
telemetry.push(Telemetry::decode(chunk)?);
|
| 127 |
+
}
|
| 128 |
+
if rest.len() != hdr.payload_bytes as usize {
|
| 129 |
+
return Err(CodecError::Truncated);
|
| 130 |
+
}
|
| 131 |
+
Ok(ResultBody {
|
| 132 |
+
hdr,
|
| 133 |
+
telemetry,
|
| 134 |
+
payload: rest.to_vec(),
|
| 135 |
+
})
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
pub fn result_request_id(hdr: &ResultHdr) -> u64 {
|
| 139 |
+
u64_from_halves(hdr.request_hi, hdr.request_lo)
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
pub fn result_hash(hdr: &ResultHdr) -> u64 {
|
| 143 |
+
u64_from_halves(hdr.result_hash_hi, hdr.result_hash_lo)
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
pub fn encode_logits_payload(logits: &[f32]) -> Vec<u8> {
|
| 147 |
+
encode_activation(logits, 32).unwrap_or_default()
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
pub fn decode_logits_payload(buf: &[u8]) -> Result<Vec<f32>, CodecError> {
|
| 151 |
+
decode_activation(buf, 32).ok_or(CodecError::Invalid("invalid logits payload"))
|
| 152 |
+
}
|
| 153 |
+
|
| 154 |
+
pub fn work_with_ids(mut work: Work, session: u64, request: u64, prefix: u64, result: u64) -> Work {
|
| 155 |
+
let (shi, slo) = u64_to_halves(session);
|
| 156 |
+
let (rhi, rlo) = u64_to_halves(request);
|
| 157 |
+
let (phi, plo) = u64_to_halves(prefix);
|
| 158 |
+
let (ehi, elo) = u64_to_halves(result);
|
| 159 |
+
work.session_hi = shi;
|
| 160 |
+
work.session_lo = slo;
|
| 161 |
+
work.request_hi = rhi;
|
| 162 |
+
work.request_lo = rlo;
|
| 163 |
+
work.prefix_hash_hi = phi;
|
| 164 |
+
work.prefix_hash_lo = plo;
|
| 165 |
+
work.result_hash_hi = ehi;
|
| 166 |
+
work.result_hash_lo = elo;
|
| 167 |
+
work
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
pub fn ok_result_hdr(request: u64, hash: u64, kind: u32, bits: u32) -> ResultHdr {
|
| 171 |
+
let (rhi, rlo) = u64_to_halves(request);
|
| 172 |
+
let (hhi, hlo) = u64_to_halves(hash);
|
| 173 |
+
ResultHdr {
|
| 174 |
+
request_hi: rhi,
|
| 175 |
+
request_lo: rlo,
|
| 176 |
+
result_hash_hi: hhi,
|
| 177 |
+
result_hash_lo: hlo,
|
| 178 |
+
status: 0,
|
| 179 |
+
result_kind: kind,
|
| 180 |
+
telemetry_count: 0,
|
| 181 |
+
telemetry_bytes: 0,
|
| 182 |
+
payload_bytes: 0,
|
| 183 |
+
payload_bits: bits,
|
| 184 |
+
}
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
pub fn error_result_frame(request: u64, msg: &str) -> Vec<u8> {
|
| 188 |
+
let (rhi, rlo) = u64_to_halves(request);
|
| 189 |
+
let payload = msg.as_bytes().to_vec();
|
| 190 |
+
let body = ResultBody {
|
| 191 |
+
hdr: ResultHdr {
|
| 192 |
+
request_hi: rhi,
|
| 193 |
+
request_lo: rlo,
|
| 194 |
+
result_hash_hi: 0,
|
| 195 |
+
result_hash_lo: 0,
|
| 196 |
+
status: 1,
|
| 197 |
+
result_kind: 0,
|
| 198 |
+
telemetry_count: 0,
|
| 199 |
+
telemetry_bytes: 0,
|
| 200 |
+
payload_bytes: payload.len() as u32,
|
| 201 |
+
payload_bits: 0,
|
| 202 |
+
},
|
| 203 |
+
telemetry: Vec::new(),
|
| 204 |
+
payload,
|
| 205 |
+
};
|
| 206 |
+
encode_result_frame(&body).unwrap_or_else(|_| crate::codec::encode_error_frame(msg))
|
| 207 |
+
}
|
crates/ds4-dist/src/worker.rs
ADDED
|
@@ -0,0 +1,424 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Worker control loop: HELLO, WORK parse/validate, slice eval, RESULT.
|
| 2 |
+
|
| 3 |
+
use std::collections::HashMap;
|
| 4 |
+
use std::io::{Read, Write};
|
| 5 |
+
use std::net::TcpStream;
|
| 6 |
+
use std::sync::{Arc, Mutex};
|
| 7 |
+
|
| 8 |
+
use crate::activation::{bits_or_default, wire_bytes};
|
| 9 |
+
use crate::codec::{
|
| 10 |
+
decode_hello_payload, encode_hello_payload, u64_from_halves, CodecError, Hello, Work,
|
| 11 |
+
MSG_ERROR, MSG_HELLO, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, MSG_WORK, RESULT_ACK,
|
| 12 |
+
RESULT_HIDDEN_STATE, RESULT_LOGITS, ROUTE_F_OUTPUT_LOGITS, ROUTE_RETURN_UPSTREAM,
|
| 13 |
+
WORK_FIXED_BYTES, WORK_F_ACK_ONLY, WORK_F_INPUT_HC, WORK_F_OUTPUT_LOGITS, WORK_F_RESET_SESSION,
|
| 14 |
+
WORK_F_VALID_MASK,
|
| 15 |
+
};
|
| 16 |
+
use crate::exec::{SliceExec, WorkRequest};
|
| 17 |
+
use crate::forward::ERR_FORWARD_HIDDEN;
|
| 18 |
+
use crate::hash::{token_hash_update_span, TOKEN_HASH_INIT};
|
| 19 |
+
use crate::hops::ForwarderPool;
|
| 20 |
+
use crate::native_snapshot::{dispatch_worker_snapshot, MemorySnapshotStore, SnapshotStore};
|
| 21 |
+
use crate::relay::{forward_work_blocking, local_work_telemetry, now_sec, usec_since};
|
| 22 |
+
use crate::route::{decode_route_blob, validate_route_blob};
|
| 23 |
+
use crate::transport::{read_frame, write_frame};
|
| 24 |
+
use crate::work::{
|
| 25 |
+
decode_work_body, encode_logits_payload, encode_result_frame, error_result_frame,
|
| 26 |
+
ok_result_hdr, ResultBody, WorkBody,
|
| 27 |
+
};
|
| 28 |
+
use crate::worker_snapshot::WorkerSnapshotIdentity;
|
| 29 |
+
|
| 30 |
+
pub fn send_hello<W: Write>(w: &mut W, hello: &Hello, model_name: &str) -> std::io::Result<()> {
|
| 31 |
+
let payload = encode_hello_payload(hello, model_name)
|
| 32 |
+
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e.to_string()))?;
|
| 33 |
+
write_frame(w, MSG_HELLO, &payload)
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
pub fn recv_hello<R: Read>(r: &mut R) -> std::io::Result<(Hello, String)> {
|
| 37 |
+
let (typ, body) = read_frame(r)?;
|
| 38 |
+
if typ != MSG_HELLO {
|
| 39 |
+
return Err(std::io::Error::new(
|
| 40 |
+
std::io::ErrorKind::InvalidData,
|
| 41 |
+
format!("expected HELLO frame, got type {typ}"),
|
| 42 |
+
));
|
| 43 |
+
}
|
| 44 |
+
decode_hello_payload(&body)
|
| 45 |
+
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
pub struct Worker<E: SliceExec, S: SnapshotStore = MemorySnapshotStore> {
|
| 49 |
+
pub exec: E,
|
| 50 |
+
sessions: HashMap<u64, u64>,
|
| 51 |
+
store: S,
|
| 52 |
+
hops: ForwarderPool,
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
impl<E: SliceExec> Worker<E> {
|
| 56 |
+
pub fn new(exec: E) -> Self {
|
| 57 |
+
Self::with_store(exec, MemorySnapshotStore::new())
|
| 58 |
+
}
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
impl<E: SliceExec, S: SnapshotStore> Worker<E, S> {
|
| 62 |
+
pub fn with_store(exec: E, store: S) -> Self {
|
| 63 |
+
Self {
|
| 64 |
+
exec,
|
| 65 |
+
sessions: HashMap::new(),
|
| 66 |
+
store,
|
| 67 |
+
hops: ForwarderPool::new(),
|
| 68 |
+
}
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
pub fn store(&self) -> &S {
|
| 72 |
+
&self.store
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
pub fn store_mut(&mut self) -> &mut S {
|
| 76 |
+
&mut self.store
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
pub fn session_count(&self) -> usize {
|
| 80 |
+
self.sessions.len()
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
pub fn clear_sessions(&mut self) -> u32 {
|
| 84 |
+
let n = u32::try_from(self.sessions.len()).unwrap_or(u32::MAX);
|
| 85 |
+
self.sessions.clear();
|
| 86 |
+
n
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
pub(crate) fn apply_snapshot<Stream: Read + Write>(
|
| 90 |
+
&mut self,
|
| 91 |
+
stream: &mut Stream,
|
| 92 |
+
typ: u32,
|
| 93 |
+
body: &[u8],
|
| 94 |
+
) -> std::io::Result<()> {
|
| 95 |
+
match dispatch_worker_snapshot(
|
| 96 |
+
stream,
|
| 97 |
+
typ,
|
| 98 |
+
body,
|
| 99 |
+
self.snapshot_identity(),
|
| 100 |
+
self.exec.vocab(),
|
| 101 |
+
&mut self.store,
|
| 102 |
+
) {
|
| 103 |
+
Ok(Some((session_id, token_hash))) => {
|
| 104 |
+
self.sessions.insert(session_id, token_hash);
|
| 105 |
+
}
|
| 106 |
+
Ok(None) => {}
|
| 107 |
+
Err(_) => {}
|
| 108 |
+
}
|
| 109 |
+
Ok(())
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
fn snapshot_identity(&self) -> WorkerSnapshotIdentity {
|
| 113 |
+
WorkerSnapshotIdentity {
|
| 114 |
+
model_id: self.exec.model_id(),
|
| 115 |
+
layer_start: self.exec.layer_start(),
|
| 116 |
+
layer_end: self.exec.layer_end(),
|
| 117 |
+
ctx_size: self.exec.ctx_size(),
|
| 118 |
+
}
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
pub fn hello(&self, listen_port: u32, ctx_size: u32, model_name: &str) -> (Hello, String) {
|
| 122 |
+
(
|
| 123 |
+
Hello {
|
| 124 |
+
model_id: self.exec.model_id(),
|
| 125 |
+
quant_bits: 0,
|
| 126 |
+
layer_start: self.exec.layer_start(),
|
| 127 |
+
layer_end: self.exec.layer_end(),
|
| 128 |
+
has_output: if self.exec.has_output() { 1 } else { 0 },
|
| 129 |
+
has_hidden: 1,
|
| 130 |
+
ctx_size,
|
| 131 |
+
n_layers: self.exec.n_layers(),
|
| 132 |
+
listen_port,
|
| 133 |
+
model_name_len: 0,
|
| 134 |
+
},
|
| 135 |
+
model_name.to_string(),
|
| 136 |
+
)
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
pub(crate) fn bind_hops(&mut self, upstream: Arc<Mutex<TcpStream>>) {
|
| 140 |
+
self.hops.bind(upstream);
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
pub(crate) fn shutdown_hops(&mut self) {
|
| 144 |
+
self.hops.shutdown();
|
| 145 |
+
}
|
| 146 |
+
|
| 147 |
+
pub fn serve(&mut self, stream: &mut TcpStream) -> std::io::Result<()> {
|
| 148 |
+
if let Ok(clone) = stream.try_clone() {
|
| 149 |
+
self.bind_hops(Arc::new(Mutex::new(clone)));
|
| 150 |
+
}
|
| 151 |
+
let rc = self.drive(stream);
|
| 152 |
+
self.shutdown_hops();
|
| 153 |
+
rc
|
| 154 |
+
}
|
| 155 |
+
|
| 156 |
+
pub fn serve_once(&mut self, stream: &mut TcpStream) -> std::io::Result<()> {
|
| 157 |
+
self.drive_one(stream)
|
| 158 |
+
}
|
| 159 |
+
|
| 160 |
+
fn drive<Stream: Read + Write>(&mut self, stream: &mut Stream) -> std::io::Result<()> {
|
| 161 |
+
loop {
|
| 162 |
+
match self.drive_one(stream) {
|
| 163 |
+
Ok(()) => {}
|
| 164 |
+
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(()),
|
| 165 |
+
Err(e) => return Err(e),
|
| 166 |
+
}
|
| 167 |
+
}
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
fn drive_one<Stream: Read + Write>(&mut self, stream: &mut Stream) -> std::io::Result<()> {
|
| 171 |
+
let (typ, body) = read_frame(stream)?;
|
| 172 |
+
if typ == MSG_ERROR {
|
| 173 |
+
return Err(std::io::Error::new(
|
| 174 |
+
std::io::ErrorKind::Other,
|
| 175 |
+
format!("coordinator error: {}", String::from_utf8_lossy(&body)),
|
| 176 |
+
));
|
| 177 |
+
}
|
| 178 |
+
match typ {
|
| 179 |
+
MSG_WORK => {
|
| 180 |
+
let reply = self.process_work(&body, stream)?;
|
| 181 |
+
if let Some(frame) = reply {
|
| 182 |
+
stream.write_all(&frame)?;
|
| 183 |
+
}
|
| 184 |
+
Ok(())
|
| 185 |
+
}
|
| 186 |
+
MSG_SNAPSHOT_SAVE_REQ | MSG_SNAPSHOT_LOAD_BEGIN => {
|
| 187 |
+
self.apply_snapshot(stream, typ, &body)
|
| 188 |
+
}
|
| 189 |
+
_ => {
|
| 190 |
+
write_frame(stream, MSG_ERROR, b"unsupported distributed worker frame")?;
|
| 191 |
+
Err(std::io::Error::new(
|
| 192 |
+
std::io::ErrorKind::InvalidData,
|
| 193 |
+
format!("rejected unsupported frame type {typ}"),
|
| 194 |
+
))
|
| 195 |
+
}
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
pub(crate) fn process_work<W: Write>(
|
| 200 |
+
&mut self,
|
| 201 |
+
payload: &[u8],
|
| 202 |
+
stream: &mut W,
|
| 203 |
+
) -> std::io::Result<Option<Vec<u8>>> {
|
| 204 |
+
if payload.len() < WORK_FIXED_BYTES {
|
| 205 |
+
return Ok(Some(error_result_frame(
|
| 206 |
+
0,
|
| 207 |
+
"truncated distributed WORK frame",
|
| 208 |
+
)));
|
| 209 |
+
}
|
| 210 |
+
let request_id = match Work::decode(payload) {
|
| 211 |
+
Ok(w) => u64_from_halves(w.request_hi, w.request_lo),
|
| 212 |
+
Err(_) => 0,
|
| 213 |
+
};
|
| 214 |
+
let body = match decode_work_body(payload) {
|
| 215 |
+
Ok(b) => b,
|
| 216 |
+
Err(e) => {
|
| 217 |
+
let msg = match e {
|
| 218 |
+
CodecError::Invalid(s) => s,
|
| 219 |
+
_ => "truncated distributed WORK frame",
|
| 220 |
+
};
|
| 221 |
+
return Ok(Some(error_result_frame(request_id, msg)));
|
| 222 |
+
}
|
| 223 |
+
};
|
| 224 |
+
match self.handle_decoded(body, stream) {
|
| 225 |
+
Ok(frame) => Ok(frame),
|
| 226 |
+
Err(msg) => Ok(Some(error_result_frame(request_id, &msg))),
|
| 227 |
+
}
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
fn handle_decoded<W: Write>(
|
| 231 |
+
&mut self,
|
| 232 |
+
body: WorkBody,
|
| 233 |
+
stream: &mut W,
|
| 234 |
+
) -> Result<Option<Vec<u8>>, String> {
|
| 235 |
+
let work = body.work;
|
| 236 |
+
let request_id = u64_from_halves(work.request_hi, work.request_lo);
|
| 237 |
+
let session_id = u64_from_halves(work.session_hi, work.session_lo);
|
| 238 |
+
let prefix_hash = u64_from_halves(work.prefix_hash_hi, work.prefix_hash_lo);
|
| 239 |
+
let result_hash = u64_from_halves(work.result_hash_hi, work.result_hash_lo);
|
| 240 |
+
|
| 241 |
+
let fail = |m: String| -> Result<Option<Vec<u8>>, String> {
|
| 242 |
+
Ok(Some(error_result_frame(request_id, &m)))
|
| 243 |
+
};
|
| 244 |
+
|
| 245 |
+
if work.route_count == 0 {
|
| 246 |
+
return fail("WORK frame is missing distributed route".into());
|
| 247 |
+
}
|
| 248 |
+
if work.route_index >= work.route_count {
|
| 249 |
+
return fail("invalid distributed WORK route metadata".into());
|
| 250 |
+
}
|
| 251 |
+
if work.model_id != self.exec.model_id() {
|
| 252 |
+
return fail(format!(
|
| 253 |
+
"model id mismatch: work={} worker={}",
|
| 254 |
+
work.model_id,
|
| 255 |
+
self.exec.model_id()
|
| 256 |
+
));
|
| 257 |
+
}
|
| 258 |
+
if work.layer_start != self.exec.layer_start() || work.layer_end != self.exec.layer_end() {
|
| 259 |
+
return fail(format!(
|
| 260 |
+
"worker is assigned layers {}:{} but request asked for {}:{}",
|
| 261 |
+
self.exec.layer_start(),
|
| 262 |
+
self.exec.layer_end(),
|
| 263 |
+
work.layer_start,
|
| 264 |
+
work.layer_end
|
| 265 |
+
));
|
| 266 |
+
}
|
| 267 |
+
if (work.flags & !WORK_F_VALID_MASK) != 0 {
|
| 268 |
+
return fail("invalid distributed WORK flags".into());
|
| 269 |
+
}
|
| 270 |
+
if body.tokens.is_empty() {
|
| 271 |
+
return fail("WORK frame has no tokens".into());
|
| 272 |
+
}
|
| 273 |
+
let ctx = self.exec.ctx_size();
|
| 274 |
+
if work.pos0 > ctx || work.n_tokens > ctx.saturating_sub(work.pos0) {
|
| 275 |
+
return fail("WORK token span exceeds worker context".into());
|
| 276 |
+
}
|
| 277 |
+
let output_logits = (work.flags & WORK_F_OUTPUT_LOGITS) != 0;
|
| 278 |
+
let input_hc_present = (work.flags & WORK_F_INPUT_HC) != 0;
|
| 279 |
+
let ack_only = (work.flags & WORK_F_ACK_ONLY) != 0;
|
| 280 |
+
if input_hc_present && work.layer_start == 0 {
|
| 281 |
+
return fail("layer 0 WORK must not provide input hidden-state".into());
|
| 282 |
+
}
|
| 283 |
+
if !input_hc_present && work.layer_start != 0 {
|
| 284 |
+
return fail("nonzero layer WORK requires input hidden-state".into());
|
| 285 |
+
}
|
| 286 |
+
if output_logits && !self.exec.has_output() {
|
| 287 |
+
return fail("worker was not assigned the output head".into());
|
| 288 |
+
}
|
| 289 |
+
if output_logits && work.layer_end + 1 != self.exec.n_layers() {
|
| 290 |
+
return fail("WORK logits require final transformer layer".into());
|
| 291 |
+
}
|
| 292 |
+
for &t in &body.tokens {
|
| 293 |
+
if t < 0 || (t as u32) >= self.exec.vocab() {
|
| 294 |
+
return fail("WORK token id is outside the model vocabulary".into());
|
| 295 |
+
}
|
| 296 |
+
}
|
| 297 |
+
if token_hash_update_span(prefix_hash, &body.tokens) != result_hash {
|
| 298 |
+
return fail("WORK token prefix hash metadata mismatch".into());
|
| 299 |
+
}
|
| 300 |
+
|
| 301 |
+
let hc_values = self.exec.hidden_values();
|
| 302 |
+
let expected_hc = (work.n_tokens as u64).saturating_mul(hc_values) as usize;
|
| 303 |
+
if input_hc_present && body.input_hc.len() != expected_hc {
|
| 304 |
+
return fail("input hidden-state size does not match token span".into());
|
| 305 |
+
}
|
| 306 |
+
if !input_hc_present && !body.input_hc.is_empty() {
|
| 307 |
+
return fail("WORK frame has hidden bytes without input flag".into());
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
if let Err(e) =
|
| 311 |
+
validate_route_blob(&body.route_blob, work.route_count, self.exec.n_layers())
|
| 312 |
+
{
|
| 313 |
+
return fail(e.to_string());
|
| 314 |
+
}
|
| 315 |
+
let (entries, ret) =
|
| 316 |
+
decode_route_blob(&body.route_blob, work.route_count).map_err(|e| e.to_string())?;
|
| 317 |
+
let current = entries
|
| 318 |
+
.get(work.route_index as usize)
|
| 319 |
+
.ok_or_else(|| "invalid route entry index".to_string())?;
|
| 320 |
+
if current.layer_start != work.layer_start || current.layer_end != work.layer_end {
|
| 321 |
+
return fail("WORK layer range does not match route entry".into());
|
| 322 |
+
}
|
| 323 |
+
let route_out = (current.flags & ROUTE_F_OUTPUT_LOGITS) != 0;
|
| 324 |
+
if route_out != output_logits {
|
| 325 |
+
return fail("WORK logits flag does not match route entry".into());
|
| 326 |
+
}
|
| 327 |
+
let has_next = (work.route_index + 1) < work.route_count;
|
| 328 |
+
if has_next && output_logits {
|
| 329 |
+
return fail("non-final route entry requested logits".into());
|
| 330 |
+
}
|
| 331 |
+
if !has_next && ret.kind != ROUTE_RETURN_UPSTREAM {
|
| 332 |
+
return fail("unsupported final result destination".into());
|
| 333 |
+
}
|
| 334 |
+
|
| 335 |
+
let reset = (work.flags & WORK_F_RESET_SESSION) != 0;
|
| 336 |
+
if reset {
|
| 337 |
+
self.sessions.insert(session_id, TOKEN_HASH_INIT);
|
| 338 |
+
}
|
| 339 |
+
let sess_hash = *self.sessions.entry(session_id).or_insert(TOKEN_HASH_INIT);
|
| 340 |
+
if sess_hash != prefix_hash {
|
| 341 |
+
return fail("worker KV prefix hash mismatch".into());
|
| 342 |
+
}
|
| 343 |
+
|
| 344 |
+
let final_ack_only = ack_only && !has_next;
|
| 345 |
+
let local_output = output_logits && !has_next && !final_ack_only;
|
| 346 |
+
let produce_hidden = !local_output && !final_ack_only;
|
| 347 |
+
let eval_t0 = now_sec();
|
| 348 |
+
let out = match self.exec.eval(&WorkRequest {
|
| 349 |
+
session_id,
|
| 350 |
+
request_id,
|
| 351 |
+
tokens: body.tokens.clone(),
|
| 352 |
+
pos0: work.pos0,
|
| 353 |
+
layer_start: work.layer_start,
|
| 354 |
+
layer_end: work.layer_end,
|
| 355 |
+
reset,
|
| 356 |
+
produce_hidden,
|
| 357 |
+
produce_logits: local_output,
|
| 358 |
+
input_hc: body.input_hc.clone(),
|
| 359 |
+
}) {
|
| 360 |
+
Ok(v) => v,
|
| 361 |
+
Err(e) => return fail(e),
|
| 362 |
+
};
|
| 363 |
+
let eval_usec = usec_since(eval_t0, now_sec());
|
| 364 |
+
self.sessions.insert(session_id, result_hash);
|
| 365 |
+
|
| 366 |
+
if has_next {
|
| 367 |
+
let next = entries[(work.route_index + 1) as usize].clone();
|
| 368 |
+
let mut fwd = body.clone();
|
| 369 |
+
fwd.work.layer_start = next.layer_start;
|
| 370 |
+
fwd.work.layer_end = next.layer_end;
|
| 371 |
+
fwd.work.route_index = work.route_index + 1;
|
| 372 |
+
fwd.work.flags |= WORK_F_INPUT_HC;
|
| 373 |
+
if (next.flags & ROUTE_F_OUTPUT_LOGITS) != 0 {
|
| 374 |
+
fwd.work.flags |= WORK_F_OUTPUT_LOGITS;
|
| 375 |
+
} else {
|
| 376 |
+
fwd.work.flags &= !WORK_F_OUTPUT_LOGITS;
|
| 377 |
+
}
|
| 378 |
+
let hidden = out.hidden.unwrap_or_default();
|
| 379 |
+
let output_bytes =
|
| 380 |
+
match wire_bytes(bits_or_default(work.input_hc_bits), hidden.len() as u64) {
|
| 381 |
+
Some(n) => n,
|
| 382 |
+
None => return fail(ERR_FORWARD_HIDDEN.into()),
|
| 383 |
+
};
|
| 384 |
+
let telemetry = local_work_telemetry(&work, eval_usec, output_bytes);
|
| 385 |
+
fwd.input_hc = hidden;
|
| 386 |
+
let hop = if self.hops.is_bound() {
|
| 387 |
+
self.hops.forward(&next, &fwd, telemetry)
|
| 388 |
+
} else {
|
| 389 |
+
forward_work_blocking(&next, &fwd, stream, telemetry)
|
| 390 |
+
};
|
| 391 |
+
if let Err(msg) = hop {
|
| 392 |
+
return fail(msg);
|
| 393 |
+
}
|
| 394 |
+
return Ok(None);
|
| 395 |
+
}
|
| 396 |
+
|
| 397 |
+
let (kind, payload, bits) = if final_ack_only {
|
| 398 |
+
(RESULT_ACK, Vec::new(), 0u32)
|
| 399 |
+
} else if local_output {
|
| 400 |
+
let logits = out
|
| 401 |
+
.logits
|
| 402 |
+
.ok_or_else(|| "slice exec returned no logits".to_string())?;
|
| 403 |
+
(RESULT_LOGITS, encode_logits_payload(&logits), 32)
|
| 404 |
+
} else {
|
| 405 |
+
let hidden = out
|
| 406 |
+
.hidden
|
| 407 |
+
.ok_or_else(|| "slice exec returned no hidden-state".to_string())?;
|
| 408 |
+
let bits = bits_or_default(work.input_hc_bits);
|
| 409 |
+
let wire = crate::activation::encode_activation(&hidden, bits)
|
| 410 |
+
.ok_or_else(|| "invalid output hidden-state size".to_string())?;
|
| 411 |
+
(RESULT_HIDDEN_STATE, wire, bits)
|
| 412 |
+
};
|
| 413 |
+
let mut hdr = ok_result_hdr(request_id, result_hash, kind, bits);
|
| 414 |
+
hdr.payload_bytes = payload.len() as u32;
|
| 415 |
+
let telemetry = local_work_telemetry(&work, eval_usec, payload.len() as u32);
|
| 416 |
+
let frame = encode_result_frame(&ResultBody {
|
| 417 |
+
hdr,
|
| 418 |
+
telemetry: vec![telemetry],
|
| 419 |
+
payload,
|
| 420 |
+
})
|
| 421 |
+
.map_err(|e| e.to_string())?;
|
| 422 |
+
Ok(Some(frame))
|
| 423 |
+
}
|
| 424 |
+
}
|
crates/ds4-dist/src/worker_snapshot.rs
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Worker-side DS4D snapshot save/load frame handling.
|
| 2 |
+
|
| 3 |
+
use std::io::{Read, Write};
|
| 4 |
+
|
| 5 |
+
use crate::codec::{
|
| 6 |
+
decode_snapshot_chunk_body, decode_snapshot_load_begin_body, encode_snapshot_begin_body,
|
| 7 |
+
encode_snapshot_chunk_body, encode_snapshot_done_body, u64_from_halves, u64_to_halves,
|
| 8 |
+
SnapshotBegin, SnapshotReq, MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK, MSG_SNAPSHOT_DONE,
|
| 9 |
+
SNAPSHOT_BEGIN_FIXED_BYTES, SNAPSHOT_CHUNK_BYTES, SNAPSHOT_REQ_FIXED_BYTES,
|
| 10 |
+
};
|
| 11 |
+
use crate::hash::token_hash_prefix;
|
| 12 |
+
use crate::transport::{read_frame, write_frame};
|
| 13 |
+
|
| 14 |
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
| 15 |
+
pub struct WorkerSnapshotIdentity {
|
| 16 |
+
pub model_id: u32,
|
| 17 |
+
pub layer_start: u32,
|
| 18 |
+
pub layer_end: u32,
|
| 19 |
+
pub ctx_size: u32,
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
fn write_begin<S: Write>(
|
| 23 |
+
stream: &mut S,
|
| 24 |
+
begin: &SnapshotBegin,
|
| 25 |
+
tokens: &[i32],
|
| 26 |
+
message: &[u8],
|
| 27 |
+
) -> Result<(), String> {
|
| 28 |
+
let body = encode_snapshot_begin_body(begin, tokens, message).map_err(|e| e.to_string())?;
|
| 29 |
+
write_frame(stream, MSG_SNAPSHOT_BEGIN, &body).map_err(|e| e.to_string())
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
fn write_error_begin<S: Write>(
|
| 33 |
+
stream: &mut S,
|
| 34 |
+
identity: WorkerSnapshotIdentity,
|
| 35 |
+
request_id: u64,
|
| 36 |
+
session_id: u64,
|
| 37 |
+
message: &str,
|
| 38 |
+
) -> Result<(), String> {
|
| 39 |
+
let (session_hi, session_lo) = u64_to_halves(session_id);
|
| 40 |
+
let (request_hi, request_lo) = u64_to_halves(request_id);
|
| 41 |
+
let begin = SnapshotBegin {
|
| 42 |
+
model_id: identity.model_id,
|
| 43 |
+
session_hi,
|
| 44 |
+
session_lo,
|
| 45 |
+
request_hi,
|
| 46 |
+
request_lo,
|
| 47 |
+
token_hash_hi: 0,
|
| 48 |
+
token_hash_lo: 0,
|
| 49 |
+
token_count: 0,
|
| 50 |
+
layer_start: identity.layer_start,
|
| 51 |
+
layer_end: identity.layer_end,
|
| 52 |
+
payload_hi: 0,
|
| 53 |
+
payload_lo: 0,
|
| 54 |
+
status: 1,
|
| 55 |
+
token_bytes: 0,
|
| 56 |
+
message_bytes: message.len() as u32,
|
| 57 |
+
};
|
| 58 |
+
write_begin(stream, &begin, &[], message.as_bytes())
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
fn write_done<S: Write>(
|
| 62 |
+
stream: &mut S,
|
| 63 |
+
request_id: u64,
|
| 64 |
+
status: u32,
|
| 65 |
+
message: &str,
|
| 66 |
+
) -> Result<(), String> {
|
| 67 |
+
let body = encode_snapshot_done_body(request_id, status, message.as_bytes())
|
| 68 |
+
.map_err(|e| e.to_string())?;
|
| 69 |
+
write_frame(stream, MSG_SNAPSHOT_DONE, &body).map_err(|e| e.to_string())
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
fn write_chunks<S: Write, R: Read>(
|
| 73 |
+
stream: &mut S,
|
| 74 |
+
request_id: u64,
|
| 75 |
+
payload: &mut R,
|
| 76 |
+
mut remaining: u64,
|
| 77 |
+
) -> Result<(), String> {
|
| 78 |
+
let mut buf = Vec::new();
|
| 79 |
+
if remaining != 0 {
|
| 80 |
+
buf.try_reserve_exact(SNAPSHOT_CHUNK_BYTES)
|
| 81 |
+
.map_err(|_| "out of memory sending worker KV shard".to_string())?;
|
| 82 |
+
buf.resize(SNAPSHOT_CHUNK_BYTES, 0);
|
| 83 |
+
}
|
| 84 |
+
while remaining != 0 {
|
| 85 |
+
let n = remaining.min(SNAPSHOT_CHUNK_BYTES as u64) as usize;
|
| 86 |
+
payload
|
| 87 |
+
.read_exact(&mut buf[..n])
|
| 88 |
+
.map_err(|_| "failed to read worker KV shard".to_string())?;
|
| 89 |
+
let body = encode_snapshot_chunk_body(request_id, &buf[..n]).map_err(|e| e.to_string())?;
|
| 90 |
+
write_frame(stream, MSG_SNAPSHOT_CHUNK, &body).map_err(|e| e.to_string())?;
|
| 91 |
+
remaining -= n as u64;
|
| 92 |
+
}
|
| 93 |
+
Ok(())
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
impl WorkerSnapshotIdentity {
|
| 97 |
+
pub fn accepts(self, model_id: u32, start: u32, end: u32, tokens: u32) -> bool {
|
| 98 |
+
model_id == self.model_id
|
| 99 |
+
&& start == self.layer_start
|
| 100 |
+
&& end == self.layer_end
|
| 101 |
+
&& tokens <= self.ctx_size
|
| 102 |
+
}
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
/// Answer one SAVE_REQ body with BEGIN+CHUNK+DONE, or a C error BEGIN.
|
| 106 |
+
pub fn worker_handle_snapshot_save<S, R>(
|
| 107 |
+
stream: &mut S,
|
| 108 |
+
identity: WorkerSnapshotIdentity,
|
| 109 |
+
req_body: &[u8],
|
| 110 |
+
payload: Result<(&mut R, u64), String>,
|
| 111 |
+
) -> Result<(), String>
|
| 112 |
+
where
|
| 113 |
+
S: Write,
|
| 114 |
+
R: Read,
|
| 115 |
+
{
|
| 116 |
+
if req_body.len() != SNAPSHOT_REQ_FIXED_BYTES {
|
| 117 |
+
return write_error_begin(
|
| 118 |
+
stream,
|
| 119 |
+
identity,
|
| 120 |
+
0,
|
| 121 |
+
0,
|
| 122 |
+
"invalid distributed snapshot save request",
|
| 123 |
+
);
|
| 124 |
+
}
|
| 125 |
+
let req = SnapshotReq::decode(req_body).map_err(|e| e.to_string())?;
|
| 126 |
+
let request_id = u64_from_halves(req.request_hi, req.request_lo);
|
| 127 |
+
let session_id = u64_from_halves(req.session_hi, req.session_lo);
|
| 128 |
+
if !identity.accepts(
|
| 129 |
+
req.model_id,
|
| 130 |
+
req.layer_start,
|
| 131 |
+
req.layer_end,
|
| 132 |
+
req.token_count,
|
| 133 |
+
) {
|
| 134 |
+
return write_error_begin(
|
| 135 |
+
stream,
|
| 136 |
+
identity,
|
| 137 |
+
request_id,
|
| 138 |
+
session_id,
|
| 139 |
+
"snapshot save request does not match worker state",
|
| 140 |
+
);
|
| 141 |
+
}
|
| 142 |
+
let (payload, payload_bytes) = match payload {
|
| 143 |
+
Ok(ok) => ok,
|
| 144 |
+
Err(message) => {
|
| 145 |
+
return write_error_begin(stream, identity, request_id, session_id, &message);
|
| 146 |
+
}
|
| 147 |
+
};
|
| 148 |
+
let (payload_hi, payload_lo) = u64_to_halves(payload_bytes);
|
| 149 |
+
let begin = SnapshotBegin {
|
| 150 |
+
model_id: identity.model_id,
|
| 151 |
+
session_hi: req.session_hi,
|
| 152 |
+
session_lo: req.session_lo,
|
| 153 |
+
request_hi: req.request_hi,
|
| 154 |
+
request_lo: req.request_lo,
|
| 155 |
+
token_hash_hi: req.token_hash_hi,
|
| 156 |
+
token_hash_lo: req.token_hash_lo,
|
| 157 |
+
token_count: 0,
|
| 158 |
+
layer_start: identity.layer_start,
|
| 159 |
+
layer_end: identity.layer_end,
|
| 160 |
+
payload_hi,
|
| 161 |
+
payload_lo,
|
| 162 |
+
status: 0,
|
| 163 |
+
token_bytes: 0,
|
| 164 |
+
message_bytes: 0,
|
| 165 |
+
};
|
| 166 |
+
write_begin(stream, &begin, &[], b"")?;
|
| 167 |
+
write_chunks(stream, request_id, payload, payload_bytes)?;
|
| 168 |
+
write_done(stream, request_id, 0, "")
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
/// Consume one LOAD_BEGIN body plus following CHUNKs, then send DONE.
|
| 172 |
+
pub fn worker_handle_snapshot_load<S, W>(
|
| 173 |
+
stream: &mut S,
|
| 174 |
+
identity: WorkerSnapshotIdentity,
|
| 175 |
+
begin_body: &[u8],
|
| 176 |
+
vocab_size: u32,
|
| 177 |
+
payload: &mut W,
|
| 178 |
+
) -> Result<WorkerLoadOffer, String>
|
| 179 |
+
where
|
| 180 |
+
S: Read + Write,
|
| 181 |
+
W: Write,
|
| 182 |
+
{
|
| 183 |
+
worker_handle_snapshot_load_restore(stream, identity, begin_body, vocab_size, payload, |_| {
|
| 184 |
+
Ok(())
|
| 185 |
+
})
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
/// Same as [`worker_handle_snapshot_load`], but runs `restore` before DONE.
|
| 189 |
+
pub fn worker_handle_snapshot_load_restore<S, W, F>(
|
| 190 |
+
stream: &mut S,
|
| 191 |
+
identity: WorkerSnapshotIdentity,
|
| 192 |
+
begin_body: &[u8],
|
| 193 |
+
vocab_size: u32,
|
| 194 |
+
payload: &mut W,
|
| 195 |
+
restore: F,
|
| 196 |
+
) -> Result<WorkerLoadOffer, String>
|
| 197 |
+
where
|
| 198 |
+
S: Read + Write,
|
| 199 |
+
W: Write,
|
| 200 |
+
F: FnOnce(&WorkerLoadOffer) -> Result<(), String>,
|
| 201 |
+
{
|
| 202 |
+
if begin_body.len() < SNAPSHOT_BEGIN_FIXED_BYTES {
|
| 203 |
+
return Err("invalid distributed snapshot load header".into());
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
let Ok((begin, tokens, _)) = decode_snapshot_load_begin_body(begin_body) else {
|
| 207 |
+
write_done(stream, 0, 1, "invalid distributed snapshot load header")?;
|
| 208 |
+
return Err("invalid distributed snapshot load header".into());
|
| 209 |
+
};
|
| 210 |
+
let request_id = u64_from_halves(begin.request_hi, begin.request_lo);
|
| 211 |
+
let session_id = u64_from_halves(begin.session_hi, begin.session_lo);
|
| 212 |
+
let token_hash = u64_from_halves(begin.token_hash_hi, begin.token_hash_lo);
|
| 213 |
+
let payload_bytes = u64_from_halves(begin.payload_hi, begin.payload_lo);
|
| 214 |
+
let mut err = if tokens
|
| 215 |
+
.iter()
|
| 216 |
+
.any(|token| *token < 0 || (*token as u32) >= vocab_size)
|
| 217 |
+
{
|
| 218 |
+
"snapshot token id is outside the model vocabulary"
|
| 219 |
+
} else if token_hash_prefix(&tokens) != token_hash {
|
| 220 |
+
"snapshot load token hash mismatch"
|
| 221 |
+
} else if !identity.accepts(
|
| 222 |
+
begin.model_id,
|
| 223 |
+
begin.layer_start,
|
| 224 |
+
begin.layer_end,
|
| 225 |
+
begin.token_count,
|
| 226 |
+
) {
|
| 227 |
+
"snapshot load request does not match worker state"
|
| 228 |
+
} else {
|
| 229 |
+
""
|
| 230 |
+
}
|
| 231 |
+
.to_string();
|
| 232 |
+
|
| 233 |
+
let mut received = 0u64;
|
| 234 |
+
while err.is_empty() && received < payload_bytes {
|
| 235 |
+
let (typ, body) = match read_frame(stream) {
|
| 236 |
+
Ok(frame) => frame,
|
| 237 |
+
Err(_) => {
|
| 238 |
+
let _ = write_done(stream, request_id, 1, "expected distributed snapshot chunk");
|
| 239 |
+
return Err("expected distributed snapshot chunk".into());
|
| 240 |
+
}
|
| 241 |
+
};
|
| 242 |
+
if typ != MSG_SNAPSHOT_CHUNK {
|
| 243 |
+
err = "expected distributed snapshot chunk".into();
|
| 244 |
+
break;
|
| 245 |
+
}
|
| 246 |
+
match decode_snapshot_chunk_body(&body, request_id, payload_bytes - received) {
|
| 247 |
+
Ok((_, chunk)) => {
|
| 248 |
+
if payload.write_all(chunk).is_err() {
|
| 249 |
+
err = "failed to write worker KV shard temp file".into();
|
| 250 |
+
break;
|
| 251 |
+
}
|
| 252 |
+
received += chunk.len() as u64;
|
| 253 |
+
}
|
| 254 |
+
Err(_) => {
|
| 255 |
+
err = "invalid distributed snapshot chunk".into();
|
| 256 |
+
break;
|
| 257 |
+
}
|
| 258 |
+
}
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
if err.is_empty() && payload.flush().is_err() {
|
| 262 |
+
err = "failed to flush worker KV shard restore file".into();
|
| 263 |
+
}
|
| 264 |
+
let offer = WorkerLoadOffer {
|
| 265 |
+
session_id,
|
| 266 |
+
request_id,
|
| 267 |
+
token_hash,
|
| 268 |
+
tokens,
|
| 269 |
+
payload_bytes,
|
| 270 |
+
};
|
| 271 |
+
if err.is_empty() {
|
| 272 |
+
if let Err(message) = restore(&offer) {
|
| 273 |
+
err = message;
|
| 274 |
+
}
|
| 275 |
+
}
|
| 276 |
+
let status = u32::from(!err.is_empty());
|
| 277 |
+
write_done(stream, request_id, status, &err)?;
|
| 278 |
+
if !err.is_empty() {
|
| 279 |
+
return Err(err);
|
| 280 |
+
}
|
| 281 |
+
Ok(offer)
|
| 282 |
+
}
|
| 283 |
+
|
| 284 |
+
#[derive(Debug, Clone, PartialEq, Eq)]
|
| 285 |
+
pub struct WorkerLoadOffer {
|
| 286 |
+
pub session_id: u64,
|
| 287 |
+
pub request_id: u64,
|
| 288 |
+
pub token_hash: u64,
|
| 289 |
+
pub tokens: Vec<i32>,
|
| 290 |
+
pub payload_bytes: u64,
|
| 291 |
+
}
|
crates/ds4-dist/tests/forward.rs
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Forward-window env parse and pending-request queue.
|
| 2 |
+
|
| 3 |
+
use std::sync::mpsc;
|
| 4 |
+
use std::thread;
|
| 5 |
+
use std::time::Duration;
|
| 6 |
+
|
| 7 |
+
use ds4_dist::{
|
| 8 |
+
forward_window_from, opened_forwarder_message, PendingQueue, PendingRequest, Telemetry,
|
| 9 |
+
ERR_CLOSED_WHILE_RESULT, ERR_FORWARD, ERR_FORWARD_HIDDEN, ERR_INVALID_RESULT, ERR_NEXT_CLOSED,
|
| 10 |
+
ERR_OOM_FORWARDER, ERR_OOM_TRACK, ERR_RELAY_THREAD, ERR_RESULT_METADATA, ERR_RESULT_TOO_LARGE,
|
| 11 |
+
ERR_TELEMETRY_TOO_LARGE, FORWARD_WINDOW_DEFAULT, FORWARD_WINDOW_MAX, FORWARD_WINDOW_MIN,
|
| 12 |
+
};
|
| 13 |
+
|
| 14 |
+
fn pending(id: u64) -> PendingRequest {
|
| 15 |
+
PendingRequest {
|
| 16 |
+
request_id: id,
|
| 17 |
+
telemetry: Telemetry {
|
| 18 |
+
layer_start: 0,
|
| 19 |
+
layer_end: 1,
|
| 20 |
+
route_index: 0,
|
| 21 |
+
pos0: 0,
|
| 22 |
+
n_tokens: 1,
|
| 23 |
+
eval_usec: 0,
|
| 24 |
+
downstream_wait_usec: 0,
|
| 25 |
+
forward_send_usec: 0,
|
| 26 |
+
input_bytes: 0,
|
| 27 |
+
output_bytes: 0,
|
| 28 |
+
},
|
| 29 |
+
downstream_t0: 0.0,
|
| 30 |
+
}
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
#[test]
|
| 34 |
+
fn forward_window_matches_c() {
|
| 35 |
+
assert_eq!(forward_window_from(None), FORWARD_WINDOW_DEFAULT);
|
| 36 |
+
assert_eq!(forward_window_from(Some("")), FORWARD_WINDOW_DEFAULT);
|
| 37 |
+
assert_eq!(forward_window_from(Some("4")), 4);
|
| 38 |
+
assert_eq!(forward_window_from(Some("1")), FORWARD_WINDOW_MIN);
|
| 39 |
+
assert_eq!(forward_window_from(Some("64")), FORWARD_WINDOW_MAX);
|
| 40 |
+
assert_eq!(forward_window_from(Some("0")), FORWARD_WINDOW_DEFAULT);
|
| 41 |
+
assert_eq!(forward_window_from(Some("65")), FORWARD_WINDOW_DEFAULT);
|
| 42 |
+
assert_eq!(forward_window_from(Some("-1")), FORWARD_WINDOW_DEFAULT);
|
| 43 |
+
assert_eq!(forward_window_from(Some("8 ")), FORWARD_WINDOW_DEFAULT);
|
| 44 |
+
assert_eq!(forward_window_from(Some(" 8")), 8);
|
| 45 |
+
assert_eq!(forward_window_from(Some("+16")), 16);
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
#[test]
|
| 49 |
+
fn forwarder_strings_match_c() {
|
| 50 |
+
assert_eq!(
|
| 51 |
+
opened_forwarder_message("10.0.0.2", 7100, 4),
|
| 52 |
+
"ds4: distributed worker: opened pipelined worker-to-worker connection to 10.0.0.2:7100 (window 4)"
|
| 53 |
+
);
|
| 54 |
+
assert_eq!(ERR_NEXT_CLOSED, "next worker closed connection");
|
| 55 |
+
assert_eq!(
|
| 56 |
+
ERR_INVALID_RESULT,
|
| 57 |
+
"next worker did not return valid RESULT"
|
| 58 |
+
);
|
| 59 |
+
assert_eq!(
|
| 60 |
+
ERR_CLOSED_WHILE_RESULT,
|
| 61 |
+
"next worker closed while returning RESULT"
|
| 62 |
+
);
|
| 63 |
+
assert_eq!(ERR_RESULT_METADATA, "next worker RESULT metadata mismatch");
|
| 64 |
+
assert_eq!(
|
| 65 |
+
ERR_TELEMETRY_TOO_LARGE,
|
| 66 |
+
"distributed telemetry chain is too large"
|
| 67 |
+
);
|
| 68 |
+
assert_eq!(
|
| 69 |
+
ERR_RESULT_TOO_LARGE,
|
| 70 |
+
"distributed RESULT frame is too large"
|
| 71 |
+
);
|
| 72 |
+
assert_eq!(
|
| 73 |
+
ERR_OOM_FORWARDER,
|
| 74 |
+
"out of memory creating worker-to-worker forwarder"
|
| 75 |
+
);
|
| 76 |
+
assert_eq!(
|
| 77 |
+
ERR_RELAY_THREAD,
|
| 78 |
+
"failed to start worker-to-worker relay thread"
|
| 79 |
+
);
|
| 80 |
+
assert_eq!(ERR_OOM_TRACK, "out of memory tracking forwarded request");
|
| 81 |
+
assert_eq!(ERR_FORWARD, "failed to forward distributed work");
|
| 82 |
+
assert_eq!(ERR_FORWARD_HIDDEN, "invalid forwarded hidden-state size");
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
#[test]
|
| 86 |
+
fn pending_queue_blocks_at_window_and_pop_unblocks() {
|
| 87 |
+
let q = PendingQueue::new(2);
|
| 88 |
+
assert_eq!(q.depth(), 2);
|
| 89 |
+
assert!(q.enqueue(pending(1)).is_ok());
|
| 90 |
+
assert!(q.enqueue(pending(2)).is_ok());
|
| 91 |
+
let (tx, rx) = mpsc::channel();
|
| 92 |
+
thread::scope(|s| {
|
| 93 |
+
s.spawn(|| {
|
| 94 |
+
assert!(q.enqueue(pending(3)).is_ok());
|
| 95 |
+
tx.send(()).unwrap();
|
| 96 |
+
});
|
| 97 |
+
thread::sleep(Duration::from_millis(30));
|
| 98 |
+
assert!(rx.try_recv().is_err());
|
| 99 |
+
assert_eq!(q.pop().unwrap().request_id, 1);
|
| 100 |
+
rx.recv_timeout(Duration::from_secs(2)).unwrap();
|
| 101 |
+
});
|
| 102 |
+
assert_eq!(q.pop().unwrap().request_id, 2);
|
| 103 |
+
assert_eq!(q.pop().unwrap().request_id, 3);
|
| 104 |
+
assert!(q.pop().is_none());
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
#[test]
|
| 108 |
+
fn pending_queue_remove_and_note_send_done() {
|
| 109 |
+
let q = PendingQueue::new(4);
|
| 110 |
+
assert!(q.enqueue(pending(9)).is_ok());
|
| 111 |
+
assert!(q.enqueue(pending(10)).is_ok());
|
| 112 |
+
q.note_send_done(10, 42, 1.5);
|
| 113 |
+
assert!(q.remove(9));
|
| 114 |
+
let got = q.pop().unwrap();
|
| 115 |
+
assert_eq!(got.request_id, 10);
|
| 116 |
+
assert_eq!(got.telemetry.forward_send_usec, 42);
|
| 117 |
+
assert_eq!(got.downstream_t0, 1.5);
|
| 118 |
+
assert!(!q.remove(9));
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
#[test]
|
| 122 |
+
fn pending_queue_close_rejects_enqueue() {
|
| 123 |
+
let q = PendingQueue::new(2);
|
| 124 |
+
q.close();
|
| 125 |
+
assert_eq!(q.enqueue(pending(1)), Err(ERR_NEXT_CLOSED));
|
| 126 |
+
q.clear();
|
| 127 |
+
assert!(q.pop().is_none());
|
| 128 |
+
}
|
crates/ds4-dist/tests/prefetch.rs
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Prefetch depth, job queue, reconnect strings, and pipelined serve.
|
| 2 |
+
|
| 3 |
+
use std::io;
|
| 4 |
+
use std::net::{TcpListener, TcpStream};
|
| 5 |
+
use std::sync::atomic::{AtomicU32, Ordering};
|
| 6 |
+
use std::sync::mpsc;
|
| 7 |
+
use std::thread;
|
| 8 |
+
use std::time::Duration;
|
| 9 |
+
|
| 10 |
+
use ds4_dist::{
|
| 11 |
+
cleared_sessions_message, connect_retryable, connected_message, disconnected_message,
|
| 12 |
+
hello_failed_message, prefetch_depth_from, prefetch_disabled_from, prefetch_enabled_message,
|
| 13 |
+
reconnect_with, retrying_message, send_hello, JobQueue, SliceExec, WorkOutput, WorkRequest,
|
| 14 |
+
Worker, ERR_OOM_QUEUE, ERR_OOM_READ, PREFETCH_DEPTH_DEFAULT, PREFETCH_DEPTH_MAX,
|
| 15 |
+
PREFETCH_DEPTH_MIN,
|
| 16 |
+
};
|
| 17 |
+
|
| 18 |
+
#[derive(Clone)]
|
| 19 |
+
struct MockExec {
|
| 20 |
+
model_id: u32,
|
| 21 |
+
n_layers: u32,
|
| 22 |
+
vocab: u32,
|
| 23 |
+
ctx_size: u32,
|
| 24 |
+
hidden_values: u64,
|
| 25 |
+
has_output: bool,
|
| 26 |
+
layer_start: u32,
|
| 27 |
+
layer_end: u32,
|
| 28 |
+
hidden: Vec<f32>,
|
| 29 |
+
logits: Vec<f32>,
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
impl SliceExec for MockExec {
|
| 33 |
+
fn model_id(&self) -> u32 {
|
| 34 |
+
self.model_id
|
| 35 |
+
}
|
| 36 |
+
fn n_layers(&self) -> u32 {
|
| 37 |
+
self.n_layers
|
| 38 |
+
}
|
| 39 |
+
fn vocab(&self) -> u32 {
|
| 40 |
+
self.vocab
|
| 41 |
+
}
|
| 42 |
+
fn ctx_size(&self) -> u32 {
|
| 43 |
+
self.ctx_size
|
| 44 |
+
}
|
| 45 |
+
fn hidden_values(&self) -> u64 {
|
| 46 |
+
self.hidden_values
|
| 47 |
+
}
|
| 48 |
+
fn has_output(&self) -> bool {
|
| 49 |
+
self.has_output
|
| 50 |
+
}
|
| 51 |
+
fn layer_start(&self) -> u32 {
|
| 52 |
+
self.layer_start
|
| 53 |
+
}
|
| 54 |
+
fn layer_end(&self) -> u32 {
|
| 55 |
+
self.layer_end
|
| 56 |
+
}
|
| 57 |
+
fn eval(&mut self, req: &WorkRequest) -> Result<WorkOutput, String> {
|
| 58 |
+
Ok(WorkOutput {
|
| 59 |
+
hidden: if req.produce_hidden {
|
| 60 |
+
Some(self.hidden.clone())
|
| 61 |
+
} else {
|
| 62 |
+
None
|
| 63 |
+
},
|
| 64 |
+
logits: if req.produce_logits {
|
| 65 |
+
Some(self.logits.clone())
|
| 66 |
+
} else {
|
| 67 |
+
None
|
| 68 |
+
},
|
| 69 |
+
})
|
| 70 |
+
}
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
fn mock() -> MockExec {
|
| 74 |
+
MockExec {
|
| 75 |
+
model_id: 3,
|
| 76 |
+
n_layers: 4,
|
| 77 |
+
vocab: 8,
|
| 78 |
+
ctx_size: 128,
|
| 79 |
+
hidden_values: 2,
|
| 80 |
+
has_output: true,
|
| 81 |
+
layer_start: 2,
|
| 82 |
+
layer_end: 3,
|
| 83 |
+
hidden: vec![1.0, 2.0],
|
| 84 |
+
logits: vec![0.1, 0.2, 0.7, 0.0],
|
| 85 |
+
}
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
fn tune(s: &TcpStream) {
|
| 89 |
+
let _ = s.set_nodelay(true);
|
| 90 |
+
let _ = s.set_read_timeout(Some(Duration::from_secs(5)));
|
| 91 |
+
let _ = s.set_write_timeout(Some(Duration::from_secs(5)));
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
#[test]
|
| 95 |
+
fn prefetch_depth_matches_c() {
|
| 96 |
+
assert_eq!(prefetch_depth_from(None), PREFETCH_DEPTH_DEFAULT);
|
| 97 |
+
assert_eq!(prefetch_depth_from(Some("")), PREFETCH_DEPTH_DEFAULT);
|
| 98 |
+
assert_eq!(prefetch_depth_from(Some("2")), 2);
|
| 99 |
+
assert_eq!(prefetch_depth_from(Some("1")), PREFETCH_DEPTH_MIN);
|
| 100 |
+
assert_eq!(prefetch_depth_from(Some("8")), PREFETCH_DEPTH_MAX);
|
| 101 |
+
assert_eq!(prefetch_depth_from(Some("0")), PREFETCH_DEPTH_DEFAULT);
|
| 102 |
+
assert_eq!(prefetch_depth_from(Some("9")), PREFETCH_DEPTH_DEFAULT);
|
| 103 |
+
assert_eq!(prefetch_depth_from(Some("-1")), PREFETCH_DEPTH_DEFAULT);
|
| 104 |
+
assert_eq!(prefetch_depth_from(Some("3 ")), PREFETCH_DEPTH_DEFAULT);
|
| 105 |
+
assert_eq!(prefetch_depth_from(Some(" 4")), 4);
|
| 106 |
+
assert_eq!(prefetch_depth_from(Some("+5")), 5);
|
| 107 |
+
assert_eq!(prefetch_depth_from(Some("2foo")), PREFETCH_DEPTH_DEFAULT);
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
#[test]
|
| 111 |
+
fn prefetch_disable_is_presence() {
|
| 112 |
+
assert!(!prefetch_disabled_from(None));
|
| 113 |
+
assert!(prefetch_disabled_from(Some(std::ffi::OsStr::new(""))));
|
| 114 |
+
assert!(prefetch_disabled_from(Some(std::ffi::OsStr::new("1"))));
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
#[test]
|
| 118 |
+
fn prefetch_and_oom_strings_match_c() {
|
| 119 |
+
assert_eq!(
|
| 120 |
+
prefetch_enabled_message(2),
|
| 121 |
+
"ds4: distributed worker: receive prefetch depth 2 enabled"
|
| 122 |
+
);
|
| 123 |
+
assert_eq!(ERR_OOM_QUEUE, "out of memory queueing distributed WORK");
|
| 124 |
+
assert_eq!(ERR_OOM_READ, "out of memory reading distributed WORK frame");
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
#[test]
|
| 128 |
+
fn job_queue_blocks_at_depth_and_finish_drains() {
|
| 129 |
+
let q = JobQueue::new(2);
|
| 130 |
+
assert_eq!(q.depth(), 2);
|
| 131 |
+
assert!(q.enqueue(vec![1]));
|
| 132 |
+
assert!(q.enqueue(vec![2]));
|
| 133 |
+
let (tx, rx) = mpsc::channel();
|
| 134 |
+
thread::scope(|s| {
|
| 135 |
+
s.spawn(|| {
|
| 136 |
+
assert!(q.enqueue(vec![3]));
|
| 137 |
+
tx.send(()).unwrap();
|
| 138 |
+
});
|
| 139 |
+
thread::sleep(Duration::from_millis(30));
|
| 140 |
+
assert!(rx.try_recv().is_err());
|
| 141 |
+
assert_eq!(q.pop(), Some(vec![1]));
|
| 142 |
+
rx.recv_timeout(Duration::from_secs(2)).unwrap();
|
| 143 |
+
});
|
| 144 |
+
assert_eq!(q.pop(), Some(vec![2]));
|
| 145 |
+
q.finish();
|
| 146 |
+
assert_eq!(q.pop(), Some(vec![3]));
|
| 147 |
+
assert_eq!(q.pop(), None);
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
#[test]
|
| 151 |
+
fn job_queue_cancel_drops_queued() {
|
| 152 |
+
let q = JobQueue::new(2);
|
| 153 |
+
assert!(q.enqueue(vec![9]));
|
| 154 |
+
q.cancel();
|
| 155 |
+
assert!(!q.enqueue(vec![8]));
|
| 156 |
+
assert_eq!(q.pop(), None);
|
| 157 |
+
}
|
| 158 |
+
|
| 159 |
+
#[test]
|
| 160 |
+
fn reconnect_messages_match_c() {
|
| 161 |
+
assert_eq!(
|
| 162 |
+
retrying_message("unable to connect to 127.0.0.1:9: Connection refused"),
|
| 163 |
+
"ds4: distributed worker: unable to connect to 127.0.0.1:9: Connection refused; retrying"
|
| 164 |
+
);
|
| 165 |
+
assert_eq!(
|
| 166 |
+
connected_message("127.0.0.1", "7000"),
|
| 167 |
+
"ds4: distributed worker: connected to coordinator 127.0.0.1:7000"
|
| 168 |
+
);
|
| 169 |
+
let err = io::Error::from_raw_os_error(32);
|
| 170 |
+
assert!(
|
| 171 |
+
hello_failed_message(&err).starts_with("ds4: distributed worker: failed to send HELLO:")
|
| 172 |
+
);
|
| 173 |
+
assert_eq!(
|
| 174 |
+
cleared_sessions_message(3),
|
| 175 |
+
"ds4: distributed worker: cleared 3 sessions after coordinator disconnect"
|
| 176 |
+
);
|
| 177 |
+
assert_eq!(
|
| 178 |
+
disconnected_message(false),
|
| 179 |
+
"ds4: distributed worker: coordinator disconnected; reconnecting"
|
| 180 |
+
);
|
| 181 |
+
assert_eq!(
|
| 182 |
+
disconnected_message(true),
|
| 183 |
+
"ds4: distributed worker: coordinator disconnected after error; reconnecting"
|
| 184 |
+
);
|
| 185 |
+
}
|
| 186 |
+
|
| 187 |
+
#[test]
|
| 188 |
+
fn connect_retryable_matches_c_errno_set() {
|
| 189 |
+
assert!(connect_retryable(&io::Error::from_raw_os_error(111))); // ECONNREFUSED
|
| 190 |
+
assert!(connect_retryable(&io::Error::from_raw_os_error(113))); // EHOSTUNREACH
|
| 191 |
+
assert!(connect_retryable(&io::Error::from_raw_os_error(101))); // ENETUNREACH
|
| 192 |
+
assert!(connect_retryable(&io::Error::from_raw_os_error(110))); // ETIMEDOUT
|
| 193 |
+
assert!(connect_retryable(&io::Error::from_raw_os_error(99))); // EADDRNOTAVAIL
|
| 194 |
+
assert!(!connect_retryable(&io::Error::from_raw_os_error(13))); // EACCES
|
| 195 |
+
}
|
| 196 |
+
|
| 197 |
+
#[test]
|
| 198 |
+
fn clear_sessions_returns_count() {
|
| 199 |
+
let mut worker = Worker::new(mock());
|
| 200 |
+
assert_eq!(worker.session_count(), 0);
|
| 201 |
+
assert_eq!(worker.clear_sessions(), 0);
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
#[test]
|
| 205 |
+
fn reconnect_retries_then_serves_and_clears() {
|
| 206 |
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 207 |
+
let addr = listener.local_addr().unwrap();
|
| 208 |
+
let attempts = AtomicU32::new(0);
|
| 209 |
+
let (ready_tx, ready_rx) = mpsc::channel();
|
| 210 |
+
let worker_thread = thread::spawn(move || {
|
| 211 |
+
let mut worker = Worker::new(mock());
|
| 212 |
+
let (hello, name) = worker.hello(7000, 128, "mock");
|
| 213 |
+
let mut sleeps = 0u32;
|
| 214 |
+
reconnect_with(
|
| 215 |
+
&mut worker,
|
| 216 |
+
|| {
|
| 217 |
+
let n = attempts.fetch_add(1, Ordering::SeqCst);
|
| 218 |
+
if n == 0 {
|
| 219 |
+
Err(io::Error::new(io::ErrorKind::ConnectionRefused, "refused"))
|
| 220 |
+
} else {
|
| 221 |
+
let s = TcpStream::connect(addr)?;
|
| 222 |
+
tune(&s);
|
| 223 |
+
Ok(s)
|
| 224 |
+
}
|
| 225 |
+
},
|
| 226 |
+
&hello,
|
| 227 |
+
&name,
|
| 228 |
+
|| {
|
| 229 |
+
sleeps += 1;
|
| 230 |
+
},
|
| 231 |
+
|| attempts.load(Ordering::SeqCst) > 1 && ready_rx.try_recv().is_ok(),
|
| 232 |
+
false,
|
| 233 |
+
)
|
| 234 |
+
.unwrap();
|
| 235 |
+
assert_eq!(worker.session_count(), 0);
|
| 236 |
+
assert!(sleeps >= 1);
|
| 237 |
+
});
|
| 238 |
+
|
| 239 |
+
let (mut stream, _) = listener.accept().unwrap();
|
| 240 |
+
tune(&stream);
|
| 241 |
+
let _ = ds4_dist::recv_hello(&mut stream).unwrap();
|
| 242 |
+
ready_tx.send(()).unwrap();
|
| 243 |
+
drop(stream);
|
| 244 |
+
worker_thread.join().unwrap();
|
| 245 |
+
}
|
| 246 |
+
|
| 247 |
+
#[test]
|
| 248 |
+
fn serve_prefetch_hello_then_eof() {
|
| 249 |
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 250 |
+
let addr = listener.local_addr().unwrap();
|
| 251 |
+
let worker_thread = thread::spawn(move || {
|
| 252 |
+
let mut stream = TcpStream::connect(addr).unwrap();
|
| 253 |
+
tune(&stream);
|
| 254 |
+
let mut worker = Worker::new(mock());
|
| 255 |
+
let (hello, name) = worker.hello(7000, 128, "mock");
|
| 256 |
+
send_hello(&mut stream, &hello, &name).unwrap();
|
| 257 |
+
worker.serve_prefetch(&mut stream).unwrap();
|
| 258 |
+
assert_eq!(worker.clear_sessions(), 0);
|
| 259 |
+
});
|
| 260 |
+
let (mut stream, _) = listener.accept().unwrap();
|
| 261 |
+
tune(&stream);
|
| 262 |
+
let (hello, name) = ds4_dist::recv_hello(&mut stream).unwrap();
|
| 263 |
+
assert_eq!(hello.model_id, 3);
|
| 264 |
+
assert_eq!(name, "mock");
|
| 265 |
+
drop(stream);
|
| 266 |
+
worker_thread.join().unwrap();
|
| 267 |
+
}
|
crates/ds4-dist/tests/relay.rs
ADDED
|
@@ -0,0 +1,296 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Persistent forwarder relay and telemetry prepend.
|
| 2 |
+
|
| 3 |
+
use std::io::Write;
|
| 4 |
+
use std::net::{TcpListener, TcpStream};
|
| 5 |
+
use std::sync::{mpsc, Arc, Mutex};
|
| 6 |
+
use std::thread;
|
| 7 |
+
use std::time::Duration;
|
| 8 |
+
|
| 9 |
+
use ds4_dist::{
|
| 10 |
+
decode_result_body, encode_result_frame, forward_work_blocking, local_work_telemetry,
|
| 11 |
+
ok_result_hdr, prepend_telemetry, read_frame, result_request_id, usec_since, Forwarder,
|
| 12 |
+
ForwarderPool, PendingRequest, ResultBody, RouteEntry, Telemetry, Work, WorkBody,
|
| 13 |
+
ERR_NEXT_CLOSED, FRAME_HEADER_BYTES, MSG_RESULT, RESULT_ACK,
|
| 14 |
+
};
|
| 15 |
+
|
| 16 |
+
fn pending(id: u64) -> PendingRequest {
|
| 17 |
+
PendingRequest {
|
| 18 |
+
request_id: id,
|
| 19 |
+
telemetry: Telemetry {
|
| 20 |
+
layer_start: 2,
|
| 21 |
+
layer_end: 3,
|
| 22 |
+
route_index: 1,
|
| 23 |
+
pos0: 0,
|
| 24 |
+
n_tokens: 1,
|
| 25 |
+
eval_usec: 11,
|
| 26 |
+
downstream_wait_usec: 0,
|
| 27 |
+
forward_send_usec: 0,
|
| 28 |
+
input_bytes: 8,
|
| 29 |
+
output_bytes: 4,
|
| 30 |
+
},
|
| 31 |
+
downstream_t0: 0.0,
|
| 32 |
+
}
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
fn result_frame(request_id: u64, payload: &[u8]) -> Vec<u8> {
|
| 36 |
+
let mut hdr = ok_result_hdr(request_id, 0, RESULT_ACK, 0);
|
| 37 |
+
hdr.payload_bytes = payload.len() as u32;
|
| 38 |
+
encode_result_frame(&ResultBody {
|
| 39 |
+
hdr,
|
| 40 |
+
telemetry: Vec::new(),
|
| 41 |
+
payload: payload.to_vec(),
|
| 42 |
+
})
|
| 43 |
+
.unwrap()
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
fn tune(s: &std::net::TcpStream) {
|
| 47 |
+
let _ = s.set_nodelay(true);
|
| 48 |
+
let _ = s.set_read_timeout(Some(Duration::from_secs(2)));
|
| 49 |
+
let _ = s.set_write_timeout(Some(Duration::from_secs(2)));
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
#[test]
|
| 53 |
+
fn local_work_telemetry_matches_c_fields() {
|
| 54 |
+
let work = Work {
|
| 55 |
+
layer_start: 2,
|
| 56 |
+
layer_end: 3,
|
| 57 |
+
route_index: 1,
|
| 58 |
+
pos0: 4,
|
| 59 |
+
n_tokens: 2,
|
| 60 |
+
token_bytes: 8,
|
| 61 |
+
input_hc_bytes: 16,
|
| 62 |
+
..Work::default()
|
| 63 |
+
};
|
| 64 |
+
let tel = local_work_telemetry(&work, 12, 32);
|
| 65 |
+
assert_eq!(tel.layer_start, 2);
|
| 66 |
+
assert_eq!(tel.layer_end, 3);
|
| 67 |
+
assert_eq!(tel.route_index, 1);
|
| 68 |
+
assert_eq!(tel.pos0, 4);
|
| 69 |
+
assert_eq!(tel.n_tokens, 2);
|
| 70 |
+
assert_eq!(tel.eval_usec, 12);
|
| 71 |
+
assert_eq!(tel.downstream_wait_usec, 0);
|
| 72 |
+
assert_eq!(tel.forward_send_usec, 0);
|
| 73 |
+
assert_eq!(tel.input_bytes, 24);
|
| 74 |
+
assert_eq!(tel.output_bytes, 32);
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
#[test]
|
| 78 |
+
fn usec_since_matches_c_rounding() {
|
| 79 |
+
assert_eq!(usec_since(1.0, 1.0), 0);
|
| 80 |
+
assert_eq!(usec_since(1.0, 0.5), 0);
|
| 81 |
+
assert_eq!(usec_since(0.0, 0.000_001), 1);
|
| 82 |
+
assert_eq!(usec_since(0.0, 1.5), 1_500_000);
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
#[test]
|
| 86 |
+
fn prepend_telemetry_appends_local_record() {
|
| 87 |
+
let frame = result_frame(9, b"ok");
|
| 88 |
+
let body = &frame[FRAME_HEADER_BYTES..];
|
| 89 |
+
let local = pending(9).telemetry;
|
| 90 |
+
let out = prepend_telemetry(body, local).unwrap();
|
| 91 |
+
let (typ, payload) = {
|
| 92 |
+
let mut cur = std::io::Cursor::new(out);
|
| 93 |
+
read_frame(&mut cur).unwrap()
|
| 94 |
+
};
|
| 95 |
+
assert_eq!(typ, MSG_RESULT);
|
| 96 |
+
let decoded = decode_result_body(&payload).unwrap();
|
| 97 |
+
assert_eq!(result_request_id(&decoded.hdr), 9);
|
| 98 |
+
assert_eq!(decoded.payload, b"ok");
|
| 99 |
+
assert_eq!(decoded.telemetry.len(), 1);
|
| 100 |
+
assert_eq!(decoded.telemetry[0], local);
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
#[test]
|
| 104 |
+
fn forwarder_relays_result_and_prepends_telemetry() {
|
| 105 |
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 106 |
+
let addr = listener.local_addr().unwrap();
|
| 107 |
+
let (tx, rx) = mpsc::channel();
|
| 108 |
+
let err_tx = tx.clone();
|
| 109 |
+
let mut forwarder = Forwarder::connect(
|
| 110 |
+
"127.0.0.1",
|
| 111 |
+
addr.port() as u32,
|
| 112 |
+
4,
|
| 113 |
+
move |frame| {
|
| 114 |
+
let _ = tx.send(Ok(frame));
|
| 115 |
+
},
|
| 116 |
+
move |id, msg| {
|
| 117 |
+
let _ = err_tx.send(Err((id, msg)));
|
| 118 |
+
},
|
| 119 |
+
)
|
| 120 |
+
.unwrap();
|
| 121 |
+
|
| 122 |
+
let next = thread::spawn(move || {
|
| 123 |
+
let (mut stream, _) = listener.accept().unwrap();
|
| 124 |
+
tune(&stream);
|
| 125 |
+
let (_typ, body) = read_frame(&mut stream).unwrap();
|
| 126 |
+
assert_eq!(body, b"WORK");
|
| 127 |
+
stream.write_all(&result_frame(7, b"logits")).unwrap();
|
| 128 |
+
});
|
| 129 |
+
|
| 130 |
+
forwarder
|
| 131 |
+
.send_work(pending(7), &{
|
| 132 |
+
let mut frame = ds4_dist::encode_frame_header(ds4_dist::MSG_WORK, 4).to_vec();
|
| 133 |
+
frame.extend_from_slice(b"WORK");
|
| 134 |
+
frame
|
| 135 |
+
})
|
| 136 |
+
.unwrap();
|
| 137 |
+
let frame = rx
|
| 138 |
+
.recv_timeout(Duration::from_secs(2))
|
| 139 |
+
.unwrap()
|
| 140 |
+
.expect("relayed RESULT");
|
| 141 |
+
let mut cur = std::io::Cursor::new(frame);
|
| 142 |
+
let (typ, payload) = read_frame(&mut cur).unwrap();
|
| 143 |
+
assert_eq!(typ, MSG_RESULT);
|
| 144 |
+
let decoded = decode_result_body(&payload).unwrap();
|
| 145 |
+
assert_eq!(decoded.payload, b"logits");
|
| 146 |
+
assert_eq!(decoded.telemetry.len(), 1);
|
| 147 |
+
assert_eq!(decoded.telemetry[0].layer_start, 2);
|
| 148 |
+
assert_eq!(decoded.telemetry[0].eval_usec, 11);
|
| 149 |
+
forwarder.shutdown();
|
| 150 |
+
next.join().unwrap();
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
fn work_frame(payload: &[u8]) -> Vec<u8> {
|
| 154 |
+
let mut frame =
|
| 155 |
+
ds4_dist::encode_frame_header(ds4_dist::MSG_WORK, payload.len() as u32).to_vec();
|
| 156 |
+
frame.extend_from_slice(payload);
|
| 157 |
+
frame
|
| 158 |
+
}
|
| 159 |
+
|
| 160 |
+
#[test]
|
| 161 |
+
fn forwarder_reports_closed_connection() {
|
| 162 |
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 163 |
+
let addr = listener.local_addr().unwrap();
|
| 164 |
+
let (tx, rx) = mpsc::channel();
|
| 165 |
+
let mut forwarder = Forwarder::connect(
|
| 166 |
+
"127.0.0.1",
|
| 167 |
+
addr.port() as u32,
|
| 168 |
+
2,
|
| 169 |
+
move |_| {},
|
| 170 |
+
move |id, msg| {
|
| 171 |
+
let _ = tx.send((id, msg));
|
| 172 |
+
},
|
| 173 |
+
)
|
| 174 |
+
.unwrap();
|
| 175 |
+
let (stream, _) = listener.accept().unwrap();
|
| 176 |
+
tune(&stream);
|
| 177 |
+
forwarder.send_work(pending(3), &work_frame(&[1])).unwrap();
|
| 178 |
+
drop(stream);
|
| 179 |
+
let (id, msg) = rx.recv_timeout(Duration::from_secs(2)).unwrap();
|
| 180 |
+
assert_eq!(id, 3);
|
| 181 |
+
assert_eq!(msg, ERR_NEXT_CLOSED);
|
| 182 |
+
forwarder.shutdown();
|
| 183 |
+
}
|
| 184 |
+
|
| 185 |
+
#[test]
|
| 186 |
+
fn blocking_forward_prepends_local_telemetry() {
|
| 187 |
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 188 |
+
let addr = listener.local_addr().unwrap();
|
| 189 |
+
let next = RouteEntry {
|
| 190 |
+
host: "127.0.0.1".into(),
|
| 191 |
+
port: addr.port() as u32,
|
| 192 |
+
layer_start: 2,
|
| 193 |
+
layer_end: 3,
|
| 194 |
+
flags: 0,
|
| 195 |
+
};
|
| 196 |
+
let work = Work {
|
| 197 |
+
layer_start: 1,
|
| 198 |
+
layer_end: 1,
|
| 199 |
+
route_index: 0,
|
| 200 |
+
pos0: 4,
|
| 201 |
+
n_tokens: 1,
|
| 202 |
+
token_bytes: 4,
|
| 203 |
+
request_lo: 7,
|
| 204 |
+
..Work::default()
|
| 205 |
+
};
|
| 206 |
+
let body = WorkBody {
|
| 207 |
+
work,
|
| 208 |
+
tokens: vec![1],
|
| 209 |
+
input_hc: vec![1.0, 2.0],
|
| 210 |
+
route_blob: Vec::new(),
|
| 211 |
+
};
|
| 212 |
+
let local = local_work_telemetry(&work, 11, 8);
|
| 213 |
+
let next_thread = thread::spawn(move || {
|
| 214 |
+
let (mut stream, _) = listener.accept().unwrap();
|
| 215 |
+
tune(&stream);
|
| 216 |
+
let (_typ, _body) = read_frame(&mut stream).unwrap();
|
| 217 |
+
stream.write_all(&result_frame(7, b"logits")).unwrap();
|
| 218 |
+
});
|
| 219 |
+
let mut upstream = Vec::new();
|
| 220 |
+
forward_work_blocking(&next, &body, &mut upstream, local).unwrap();
|
| 221 |
+
let mut cur = std::io::Cursor::new(upstream);
|
| 222 |
+
let (typ, payload) = read_frame(&mut cur).unwrap();
|
| 223 |
+
assert_eq!(typ, MSG_RESULT);
|
| 224 |
+
let decoded = decode_result_body(&payload).unwrap();
|
| 225 |
+
assert_eq!(decoded.payload, b"logits");
|
| 226 |
+
assert_eq!(decoded.telemetry.len(), 1);
|
| 227 |
+
assert_eq!(decoded.telemetry[0].eval_usec, 11);
|
| 228 |
+
assert_eq!(decoded.telemetry[0].layer_start, 1);
|
| 229 |
+
assert_eq!(decoded.telemetry[0].input_bytes, 4);
|
| 230 |
+
assert_eq!(decoded.telemetry[0].output_bytes, 8);
|
| 231 |
+
next_thread.join().unwrap();
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
#[test]
|
| 235 |
+
fn pool_reuses_forwarder_and_prepends_telemetry() {
|
| 236 |
+
let next_l = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 237 |
+
let up_l = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 238 |
+
let next_addr = next_l.local_addr().unwrap();
|
| 239 |
+
let up_addr = up_l.local_addr().unwrap();
|
| 240 |
+
let upstream = TcpStream::connect(up_addr).unwrap();
|
| 241 |
+
tune(&upstream);
|
| 242 |
+
let (mut up_peer, _) = up_l.accept().unwrap();
|
| 243 |
+
tune(&up_peer);
|
| 244 |
+
let mut pool = ForwarderPool::new();
|
| 245 |
+
pool.bind(Arc::new(Mutex::new(upstream)));
|
| 246 |
+
let next = RouteEntry {
|
| 247 |
+
host: "127.0.0.1".into(),
|
| 248 |
+
port: next_addr.port() as u32,
|
| 249 |
+
layer_start: 2,
|
| 250 |
+
layer_end: 3,
|
| 251 |
+
flags: 0,
|
| 252 |
+
};
|
| 253 |
+
let work = Work {
|
| 254 |
+
layer_start: 1,
|
| 255 |
+
layer_end: 1,
|
| 256 |
+
n_tokens: 1,
|
| 257 |
+
token_bytes: 4,
|
| 258 |
+
request_lo: 7,
|
| 259 |
+
..Work::default()
|
| 260 |
+
};
|
| 261 |
+
let body = WorkBody {
|
| 262 |
+
work,
|
| 263 |
+
tokens: vec![1],
|
| 264 |
+
input_hc: vec![1.0, 2.0],
|
| 265 |
+
route_blob: Vec::new(),
|
| 266 |
+
};
|
| 267 |
+
let next_thread = thread::spawn(move || {
|
| 268 |
+
let (mut stream, _) = next_l.accept().unwrap();
|
| 269 |
+
tune(&stream);
|
| 270 |
+
let (_typ, _body) = read_frame(&mut stream).unwrap();
|
| 271 |
+
stream.write_all(&result_frame(7, b"logits")).unwrap();
|
| 272 |
+
let (_typ, _body) = read_frame(&mut stream).unwrap();
|
| 273 |
+
stream.write_all(&result_frame(8, b"again")).unwrap();
|
| 274 |
+
});
|
| 275 |
+
pool.forward(&next, &body, local_work_telemetry(&work, 11, 8))
|
| 276 |
+
.unwrap();
|
| 277 |
+
assert_eq!(pool.len(), 1);
|
| 278 |
+
let mut body2 = body;
|
| 279 |
+
body2.work.request_lo = 8;
|
| 280 |
+
pool.forward(&next, &body2, local_work_telemetry(&work, 12, 8))
|
| 281 |
+
.unwrap();
|
| 282 |
+
assert_eq!(pool.len(), 1);
|
| 283 |
+
let (typ, payload) = read_frame(&mut up_peer).unwrap();
|
| 284 |
+
assert_eq!(typ, MSG_RESULT);
|
| 285 |
+
let decoded = decode_result_body(&payload).unwrap();
|
| 286 |
+
assert_eq!(decoded.payload, b"logits");
|
| 287 |
+
assert_eq!(decoded.telemetry.len(), 1);
|
| 288 |
+
assert_eq!(decoded.telemetry[0].eval_usec, 11);
|
| 289 |
+
let (typ, payload) = read_frame(&mut up_peer).unwrap();
|
| 290 |
+
assert_eq!(typ, MSG_RESULT);
|
| 291 |
+
let decoded = decode_result_body(&payload).unwrap();
|
| 292 |
+
assert_eq!(decoded.payload, b"again");
|
| 293 |
+
assert_eq!(decoded.telemetry[0].eval_usec, 12);
|
| 294 |
+
pool.shutdown();
|
| 295 |
+
next_thread.join().unwrap();
|
| 296 |
+
}
|
crates/ds4-dist/tests/runtime.rs
ADDED
|
@@ -0,0 +1,1257 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Coordinator/worker runtime: CLI strings, route search, HELLO/WORK/RESULT.
|
| 2 |
+
|
| 3 |
+
use ds4_dist::{
|
| 4 |
+
build_route_plan, decode_logits_payload, decode_result_body, dispatch_eval, encode_route_blob,
|
| 5 |
+
encode_work_frame, format_telemetry_line, parse_cli, parse_layers, parse_role,
|
| 6 |
+
prefetch_disabled_from, prepare_engine_options, read_frame, reconnect_local, register_worker,
|
| 7 |
+
resolved_layer_end, send_hello, serve_prefetch_local_with, token_hash_update_span,
|
| 8 |
+
token_span_hashes, validate_layers_for_model, validate_options, work_with_ids, write_frame,
|
| 9 |
+
Coordinator, CoordinatorView, EvalOutcome, JobQueue, LocalReconnect, PrefetchJob, ReturnTarget,
|
| 10 |
+
RouteEntry, SliceExec, Telemetry, Work, WorkBody, WorkOutput, WorkRequest, Worker, WorkerInfo,
|
| 11 |
+
ERR_NEXT_CLOSED, MSG_ERROR, MSG_RESULT, RESULT_LOGITS, ROUTE_F_OUTPUT_LOGITS,
|
| 12 |
+
ROUTE_RETURN_UPSTREAM, TOKEN_HASH_INIT, WORK_F_INPUT_HC, WORK_F_OUTPUT_LOGITS,
|
| 13 |
+
WORK_F_RESET_SESSION,
|
| 14 |
+
};
|
| 15 |
+
use std::io::{Read, Write};
|
| 16 |
+
use std::marker::PhantomData;
|
| 17 |
+
use std::net::{SocketAddr, TcpListener, TcpStream};
|
| 18 |
+
use std::sync::{mpsc, Arc, Condvar, Mutex};
|
| 19 |
+
use std::thread;
|
| 20 |
+
use std::time::Duration;
|
| 21 |
+
|
| 22 |
+
fn oracle() -> std::path::PathBuf {
|
| 23 |
+
if let Ok(p) = std::env::var("DS4_DIST_C_ORACLE") {
|
| 24 |
+
return std::path::PathBuf::from(p);
|
| 25 |
+
}
|
| 26 |
+
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/dist_c_oracle")
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
fn require_oracle() -> std::path::PathBuf {
|
| 30 |
+
let p = oracle();
|
| 31 |
+
assert!(
|
| 32 |
+
p.exists(),
|
| 33 |
+
"build the C oracle first: make tests/parity/dist_c_oracle (missing {})",
|
| 34 |
+
p.display()
|
| 35 |
+
);
|
| 36 |
+
p
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
fn c_out(args: &[&str]) -> String {
|
| 40 |
+
let out = std::process::Command::new(require_oracle())
|
| 41 |
+
.args(args)
|
| 42 |
+
.output()
|
| 43 |
+
.expect("run dist_c_oracle");
|
| 44 |
+
assert!(
|
| 45 |
+
out.status.success(),
|
| 46 |
+
"oracle failed: {}",
|
| 47 |
+
String::from_utf8_lossy(&out.stderr)
|
| 48 |
+
);
|
| 49 |
+
String::from_utf8_lossy(&out.stdout).into_owned()
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
fn rust_cli(args: &[&str]) -> Result<(), String> {
|
| 53 |
+
let owned: Vec<String> = args.iter().map(|s| (*s).to_string()).collect();
|
| 54 |
+
let (opt, unmatched) = parse_cli(&owned)?;
|
| 55 |
+
if let Some(u) = unmatched.first() {
|
| 56 |
+
return Err(format!("unmatched {u}"));
|
| 57 |
+
}
|
| 58 |
+
validate_options(&opt)
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
fn rust_cli_layers(n_layers: u32, args: &[&str]) -> Result<(), String> {
|
| 62 |
+
let owned: Vec<String> = args.iter().map(|s| (*s).to_string()).collect();
|
| 63 |
+
let (opt, unmatched) = parse_cli(&owned)?;
|
| 64 |
+
if let Some(u) = unmatched.first() {
|
| 65 |
+
return Err(format!("unmatched {u}"));
|
| 66 |
+
}
|
| 67 |
+
validate_options(&opt)?;
|
| 68 |
+
validate_layers_for_model(&opt, n_layers)
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
fn assert_cli(args: &[&str]) {
|
| 72 |
+
let mut cargs = vec!["cli"];
|
| 73 |
+
cargs.extend_from_slice(args);
|
| 74 |
+
let c = c_out(&cargs);
|
| 75 |
+
let r = rust_cli(args);
|
| 76 |
+
if c == "OK" {
|
| 77 |
+
r.expect("rust rejected a CLI C accepts");
|
| 78 |
+
} else {
|
| 79 |
+
let msg = c.strip_prefix("ERROR:").unwrap_or(&c);
|
| 80 |
+
let err = r.expect_err("rust accepted a CLI C rejects");
|
| 81 |
+
assert_eq!(err, msg, "args={args:?}");
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
#[derive(Clone)]
|
| 86 |
+
struct MockExec {
|
| 87 |
+
model_id: u32,
|
| 88 |
+
n_layers: u32,
|
| 89 |
+
vocab: u32,
|
| 90 |
+
ctx_size: u32,
|
| 91 |
+
hidden_values: u64,
|
| 92 |
+
has_output: bool,
|
| 93 |
+
layer_start: u32,
|
| 94 |
+
layer_end: u32,
|
| 95 |
+
hidden: Vec<f32>,
|
| 96 |
+
logits: Vec<f32>,
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
impl SliceExec for MockExec {
|
| 100 |
+
fn model_id(&self) -> u32 {
|
| 101 |
+
self.model_id
|
| 102 |
+
}
|
| 103 |
+
fn n_layers(&self) -> u32 {
|
| 104 |
+
self.n_layers
|
| 105 |
+
}
|
| 106 |
+
fn vocab(&self) -> u32 {
|
| 107 |
+
self.vocab
|
| 108 |
+
}
|
| 109 |
+
fn ctx_size(&self) -> u32 {
|
| 110 |
+
self.ctx_size
|
| 111 |
+
}
|
| 112 |
+
fn hidden_values(&self) -> u64 {
|
| 113 |
+
self.hidden_values
|
| 114 |
+
}
|
| 115 |
+
fn has_output(&self) -> bool {
|
| 116 |
+
self.has_output
|
| 117 |
+
}
|
| 118 |
+
fn layer_start(&self) -> u32 {
|
| 119 |
+
self.layer_start
|
| 120 |
+
}
|
| 121 |
+
fn layer_end(&self) -> u32 {
|
| 122 |
+
self.layer_end
|
| 123 |
+
}
|
| 124 |
+
fn eval(&mut self, req: &WorkRequest) -> Result<WorkOutput, String> {
|
| 125 |
+
Ok(WorkOutput {
|
| 126 |
+
hidden: if req.produce_hidden {
|
| 127 |
+
Some(self.hidden.clone())
|
| 128 |
+
} else {
|
| 129 |
+
None
|
| 130 |
+
},
|
| 131 |
+
logits: if req.produce_logits {
|
| 132 |
+
Some(self.logits.clone())
|
| 133 |
+
} else {
|
| 134 |
+
None
|
| 135 |
+
},
|
| 136 |
+
})
|
| 137 |
+
}
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
fn tune(s: &TcpStream) {
|
| 141 |
+
let _ = s.set_nodelay(true);
|
| 142 |
+
let _ = s.set_read_timeout(Some(Duration::from_secs(5)));
|
| 143 |
+
let _ = s.set_write_timeout(Some(Duration::from_secs(5)));
|
| 144 |
+
}
|
| 145 |
+
|
| 146 |
+
#[test]
|
| 147 |
+
fn telemetry_line_matches_c_format() {
|
| 148 |
+
let tel = Telemetry {
|
| 149 |
+
layer_start: 2,
|
| 150 |
+
layer_end: 3,
|
| 151 |
+
route_index: 1,
|
| 152 |
+
pos0: 4,
|
| 153 |
+
n_tokens: 2,
|
| 154 |
+
eval_usec: 1500,
|
| 155 |
+
downstream_wait_usec: 250,
|
| 156 |
+
forward_send_usec: 100,
|
| 157 |
+
input_bytes: 2 * 1024 * 1024,
|
| 158 |
+
output_bytes: 512 * 1024,
|
| 159 |
+
};
|
| 160 |
+
assert_eq!(
|
| 161 |
+
format_telemetry_line(9, 0, &tel),
|
| 162 |
+
"ds4: distributed telemetry: request=9 hop=0 layers=2:3 route=1 pos=4 tokens=2 eval=1.500ms downstream_wait=0.250ms forward_send=0.100ms input=2.00MiB output=0.50MiB\n"
|
| 163 |
+
);
|
| 164 |
+
}
|
| 165 |
+
|
| 166 |
+
#[test]
|
| 167 |
+
fn layers_and_role_match_c() {
|
| 168 |
+
assert_eq!(c_out(&["layers", "0:1"]), "start=0 end=1 has_output=0");
|
| 169 |
+
assert_eq!(
|
| 170 |
+
c_out(&["layers", "21:output"]),
|
| 171 |
+
"start=21 end=4294967295 has_output=1"
|
| 172 |
+
);
|
| 173 |
+
let l = parse_layers("21:output").unwrap();
|
| 174 |
+
assert_eq!(l.start, 21);
|
| 175 |
+
assert!(l.has_output);
|
| 176 |
+
assert_eq!(resolved_layer_end(&l, 53), 52);
|
| 177 |
+
assert_eq!(c_out(&["layers", "foo"]), "ERROR:expected A:B or A:output");
|
| 178 |
+
assert_eq!(parse_layers("foo").unwrap_err(), "expected A:B or A:output");
|
| 179 |
+
assert_eq!(
|
| 180 |
+
c_out(&["layers", "x:1"]),
|
| 181 |
+
"ERROR:invalid start layer in x:1"
|
| 182 |
+
);
|
| 183 |
+
assert_eq!(
|
| 184 |
+
parse_layers("x:1").unwrap_err(),
|
| 185 |
+
"invalid start layer in x:1"
|
| 186 |
+
);
|
| 187 |
+
assert_eq!(
|
| 188 |
+
c_out(&["layers", "3:1"]),
|
| 189 |
+
"ERROR:layer range end precedes start in 3:1"
|
| 190 |
+
);
|
| 191 |
+
assert_eq!(c_out(&["role", "coordinator"]), "coordinator");
|
| 192 |
+
assert_eq!(parse_role("coordinator"), Some(ds4_dist::Role::Coordinator));
|
| 193 |
+
assert!(c_out(&["role", "boss"]).starts_with("ERROR:invalid distributed role: boss"));
|
| 194 |
+
assert!(parse_role("boss").is_none());
|
| 195 |
+
}
|
| 196 |
+
|
| 197 |
+
#[test]
|
| 198 |
+
fn cli_validate_matches_c() {
|
| 199 |
+
assert_cli(&[]);
|
| 200 |
+
assert_cli(&["--listen", "127.0.0.1", "7000"]);
|
| 201 |
+
assert_cli(&["--role", "coordinator"]);
|
| 202 |
+
assert_cli(&[
|
| 203 |
+
"--role",
|
| 204 |
+
"coordinator",
|
| 205 |
+
"--layers",
|
| 206 |
+
"0:1",
|
| 207 |
+
"--listen",
|
| 208 |
+
"127.0.0.1",
|
| 209 |
+
"7000",
|
| 210 |
+
]);
|
| 211 |
+
assert_cli(&[
|
| 212 |
+
"--role",
|
| 213 |
+
"coordinator",
|
| 214 |
+
"--layers",
|
| 215 |
+
"0:1",
|
| 216 |
+
"--listen",
|
| 217 |
+
"127.0.0.1",
|
| 218 |
+
"7000",
|
| 219 |
+
"--coordinator",
|
| 220 |
+
"10.0.0.1",
|
| 221 |
+
"9",
|
| 222 |
+
]);
|
| 223 |
+
assert_cli(&["--role", "worker", "--layers", "2:output"]);
|
| 224 |
+
assert_cli(&[
|
| 225 |
+
"--role",
|
| 226 |
+
"worker",
|
| 227 |
+
"--layers",
|
| 228 |
+
"2:output",
|
| 229 |
+
"--coordinator",
|
| 230 |
+
"127.0.0.1",
|
| 231 |
+
"7000",
|
| 232 |
+
]);
|
| 233 |
+
assert_cli(&[
|
| 234 |
+
"--role",
|
| 235 |
+
"worker",
|
| 236 |
+
"--layers",
|
| 237 |
+
"2:output",
|
| 238 |
+
"--coordinator",
|
| 239 |
+
"127.0.0.1",
|
| 240 |
+
"7000",
|
| 241 |
+
"--dist-prefill-chunk",
|
| 242 |
+
"128",
|
| 243 |
+
]);
|
| 244 |
+
assert_cli(&["--role", "nope"]);
|
| 245 |
+
assert_cli(&["--layers", "0:1:2"]);
|
| 246 |
+
assert_cli(&["--dist-prefill-window", "99"]);
|
| 247 |
+
assert_cli(&[
|
| 248 |
+
"--role",
|
| 249 |
+
"coordinator",
|
| 250 |
+
"--layers",
|
| 251 |
+
"0:1",
|
| 252 |
+
"--listen",
|
| 253 |
+
"127.0.0.1",
|
| 254 |
+
"0",
|
| 255 |
+
]);
|
| 256 |
+
}
|
| 257 |
+
|
| 258 |
+
#[test]
|
| 259 |
+
fn layers_for_model_match_c() {
|
| 260 |
+
let args = [
|
| 261 |
+
"--role",
|
| 262 |
+
"coordinator",
|
| 263 |
+
"--layers",
|
| 264 |
+
"2:3",
|
| 265 |
+
"--listen",
|
| 266 |
+
"127.0.0.1",
|
| 267 |
+
"7000",
|
| 268 |
+
];
|
| 269 |
+
let c = c_out(&[
|
| 270 |
+
"layers-for-model",
|
| 271 |
+
"4",
|
| 272 |
+
"--role",
|
| 273 |
+
"coordinator",
|
| 274 |
+
"--layers",
|
| 275 |
+
"2:3",
|
| 276 |
+
"--listen",
|
| 277 |
+
"127.0.0.1",
|
| 278 |
+
"7000",
|
| 279 |
+
]);
|
| 280 |
+
let r = rust_cli_layers(4, &args);
|
| 281 |
+
assert_eq!(c, "ERROR:coordinator layer range must start at layer 0");
|
| 282 |
+
assert_eq!(
|
| 283 |
+
r.unwrap_err(),
|
| 284 |
+
"coordinator layer range must start at layer 0"
|
| 285 |
+
);
|
| 286 |
+
|
| 287 |
+
let past = rust_cli_layers(
|
| 288 |
+
4,
|
| 289 |
+
&[
|
| 290 |
+
"--role",
|
| 291 |
+
"worker",
|
| 292 |
+
"--layers",
|
| 293 |
+
"9:9",
|
| 294 |
+
"--coordinator",
|
| 295 |
+
"127.0.0.1",
|
| 296 |
+
"1",
|
| 297 |
+
],
|
| 298 |
+
);
|
| 299 |
+
assert_eq!(
|
| 300 |
+
past.unwrap_err(),
|
| 301 |
+
"layer range starts past final model layer 3"
|
| 302 |
+
);
|
| 303 |
+
}
|
| 304 |
+
|
| 305 |
+
#[test]
|
| 306 |
+
fn replay_check_requires_coordinator() {
|
| 307 |
+
let owned = vec![
|
| 308 |
+
"--role".into(),
|
| 309 |
+
"worker".into(),
|
| 310 |
+
"--layers".into(),
|
| 311 |
+
"2:output".into(),
|
| 312 |
+
"--coordinator".into(),
|
| 313 |
+
"127.0.0.1".into(),
|
| 314 |
+
"7000".into(),
|
| 315 |
+
"--dist-replay-check".into(),
|
| 316 |
+
];
|
| 317 |
+
let (opt, _) = parse_cli(&owned).unwrap();
|
| 318 |
+
validate_options(&opt).unwrap();
|
| 319 |
+
assert_eq!(
|
| 320 |
+
prepare_engine_options(&opt).unwrap_err(),
|
| 321 |
+
"--dist-replay-check requires --role coordinator"
|
| 322 |
+
);
|
| 323 |
+
}
|
| 324 |
+
|
| 325 |
+
fn worker(start: u32, end: u32, output: bool) -> WorkerInfo {
|
| 326 |
+
WorkerInfo {
|
| 327 |
+
peer_host: format!("10.0.0.{end}"),
|
| 328 |
+
listen_port: 7000 + end,
|
| 329 |
+
model_id: 3,
|
| 330 |
+
quant_bits: 2,
|
| 331 |
+
layer_start: start,
|
| 332 |
+
layer_end: end,
|
| 333 |
+
has_output: output,
|
| 334 |
+
has_hidden: true,
|
| 335 |
+
}
|
| 336 |
+
}
|
| 337 |
+
|
| 338 |
+
#[test]
|
| 339 |
+
fn route_plan_local_complete_is_empty() {
|
| 340 |
+
let view = CoordinatorView {
|
| 341 |
+
n_layers: 4,
|
| 342 |
+
local_start: 0,
|
| 343 |
+
local_end: 3,
|
| 344 |
+
local_has_output: true,
|
| 345 |
+
local_can_output_head: false,
|
| 346 |
+
};
|
| 347 |
+
let plan = build_route_plan(&view, &[]).unwrap();
|
| 348 |
+
assert!(plan.entries.is_empty());
|
| 349 |
+
assert!(plan.blob.is_empty());
|
| 350 |
+
}
|
| 351 |
+
|
| 352 |
+
#[test]
|
| 353 |
+
fn route_plan_requires_layer_zero() {
|
| 354 |
+
let view = CoordinatorView {
|
| 355 |
+
n_layers: 4,
|
| 356 |
+
local_start: 1,
|
| 357 |
+
local_end: 1,
|
| 358 |
+
local_has_output: false,
|
| 359 |
+
local_can_output_head: false,
|
| 360 |
+
};
|
| 361 |
+
assert_eq!(
|
| 362 |
+
build_route_plan(&view, &[]).unwrap_err(),
|
| 363 |
+
"coordinator route does not start at layer 0"
|
| 364 |
+
);
|
| 365 |
+
}
|
| 366 |
+
|
| 367 |
+
#[test]
|
| 368 |
+
fn route_plan_missing_layer_message() {
|
| 369 |
+
let view = CoordinatorView {
|
| 370 |
+
n_layers: 4,
|
| 371 |
+
local_start: 0,
|
| 372 |
+
local_end: 1,
|
| 373 |
+
local_has_output: false,
|
| 374 |
+
local_can_output_head: false,
|
| 375 |
+
};
|
| 376 |
+
assert_eq!(
|
| 377 |
+
build_route_plan(&view, &[]).unwrap_err(),
|
| 378 |
+
"distributed route incomplete: missing layer 2"
|
| 379 |
+
);
|
| 380 |
+
}
|
| 381 |
+
|
| 382 |
+
#[test]
|
| 383 |
+
fn route_plan_prefers_output_then_longer_end() {
|
| 384 |
+
let view = CoordinatorView {
|
| 385 |
+
n_layers: 6,
|
| 386 |
+
local_start: 0,
|
| 387 |
+
local_end: 1,
|
| 388 |
+
local_has_output: false,
|
| 389 |
+
local_can_output_head: false,
|
| 390 |
+
};
|
| 391 |
+
let workers = vec![worker(2, 3, false), worker(2, 5, true), worker(2, 4, false)];
|
| 392 |
+
let plan = build_route_plan(&view, &workers).unwrap();
|
| 393 |
+
assert_eq!(plan.entries.len(), 1);
|
| 394 |
+
assert_eq!(plan.entries[0].layer_end, 5);
|
| 395 |
+
assert_ne!(plan.entries[0].flags, 0);
|
| 396 |
+
}
|
| 397 |
+
|
| 398 |
+
#[test]
|
| 399 |
+
fn register_worker_replaces_stale_and_prepends() {
|
| 400 |
+
let mut list = vec![worker(2, 5, true)];
|
| 401 |
+
list[0].peer_host = "10.0.0.9".into();
|
| 402 |
+
let neu = WorkerInfo {
|
| 403 |
+
peer_host: "10.0.0.9".into(),
|
| 404 |
+
listen_port: 7101,
|
| 405 |
+
model_id: 3,
|
| 406 |
+
quant_bits: 4,
|
| 407 |
+
layer_start: 2,
|
| 408 |
+
layer_end: 5,
|
| 409 |
+
has_output: true,
|
| 410 |
+
has_hidden: true,
|
| 411 |
+
};
|
| 412 |
+
register_worker(&mut list, neu.clone());
|
| 413 |
+
assert_eq!(list.len(), 1);
|
| 414 |
+
assert_eq!(list[0].listen_port, 7101);
|
| 415 |
+
let other = worker(2, 3, false);
|
| 416 |
+
register_worker(&mut list, other);
|
| 417 |
+
assert_eq!(list.len(), 2);
|
| 418 |
+
assert_eq!(list[0].layer_end, 3);
|
| 419 |
+
}
|
| 420 |
+
|
| 421 |
+
#[test]
|
| 422 |
+
fn single_hop_hello_work_logits() {
|
| 423 |
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 424 |
+
let addr = listener.local_addr().unwrap();
|
| 425 |
+
let logits = vec![0.1f32, 0.2, 0.7, 0.0];
|
| 426 |
+
let hidden = vec![1.0f32, 2.0, 3.0, 4.0];
|
| 427 |
+
let exec = MockExec {
|
| 428 |
+
model_id: 3,
|
| 429 |
+
n_layers: 4,
|
| 430 |
+
vocab: 8,
|
| 431 |
+
ctx_size: 128,
|
| 432 |
+
hidden_values: 2,
|
| 433 |
+
has_output: true,
|
| 434 |
+
layer_start: 2,
|
| 435 |
+
layer_end: 3,
|
| 436 |
+
hidden: hidden.clone(),
|
| 437 |
+
logits: logits.clone(),
|
| 438 |
+
};
|
| 439 |
+
let worker_thread = thread::spawn(move || {
|
| 440 |
+
let mut stream = TcpStream::connect(addr).unwrap();
|
| 441 |
+
tune(&stream);
|
| 442 |
+
let mut worker = Worker::new(exec);
|
| 443 |
+
let (hello, name) = worker.hello(7000, 128, "mock");
|
| 444 |
+
send_hello(&mut stream, &hello, &name).unwrap();
|
| 445 |
+
worker.serve(&mut stream).unwrap();
|
| 446 |
+
});
|
| 447 |
+
|
| 448 |
+
let (mut stream, _) = listener.accept().unwrap();
|
| 449 |
+
tune(&stream);
|
| 450 |
+
let view = CoordinatorView {
|
| 451 |
+
n_layers: 4,
|
| 452 |
+
local_start: 0,
|
| 453 |
+
local_end: 1,
|
| 454 |
+
local_has_output: false,
|
| 455 |
+
local_can_output_head: false,
|
| 456 |
+
};
|
| 457 |
+
let mut coord = Coordinator::new(view, 3, 32);
|
| 458 |
+
coord.accept_hello(&mut stream, "127.0.0.1").unwrap();
|
| 459 |
+
let tokens = [1i32, 2];
|
| 460 |
+
let out = dispatch_eval(
|
| 461 |
+
&coord,
|
| 462 |
+
&mut stream,
|
| 463 |
+
&tokens,
|
| 464 |
+
0,
|
| 465 |
+
9,
|
| 466 |
+
11,
|
| 467 |
+
true,
|
| 468 |
+
TOKEN_HASH_INIT,
|
| 469 |
+
&hidden,
|
| 470 |
+
)
|
| 471 |
+
.unwrap();
|
| 472 |
+
match out {
|
| 473 |
+
EvalOutcome::Logits(v) => assert_eq!(v, logits),
|
| 474 |
+
other => panic!("expected logits, got {other:?}"),
|
| 475 |
+
}
|
| 476 |
+
drop(stream);
|
| 477 |
+
worker_thread.join().unwrap();
|
| 478 |
+
}
|
| 479 |
+
|
| 480 |
+
#[test]
|
| 481 |
+
fn two_hop_blocking_forward() {
|
| 482 |
+
let data = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 483 |
+
let data_port = data.local_addr().unwrap().port() as u32;
|
| 484 |
+
let coord_l = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 485 |
+
let coord_addr = coord_l.local_addr().unwrap();
|
| 486 |
+
let (ready_tx, ready_rx) = mpsc::channel::<()>();
|
| 487 |
+
|
| 488 |
+
let tail = MockExec {
|
| 489 |
+
model_id: 3,
|
| 490 |
+
n_layers: 4,
|
| 491 |
+
vocab: 8,
|
| 492 |
+
ctx_size: 128,
|
| 493 |
+
hidden_values: 2,
|
| 494 |
+
has_output: true,
|
| 495 |
+
layer_start: 2,
|
| 496 |
+
layer_end: 3,
|
| 497 |
+
hidden: vec![9.0, 8.0],
|
| 498 |
+
logits: vec![0.0, 1.0, 0.0, 0.0],
|
| 499 |
+
};
|
| 500 |
+
let mid = MockExec {
|
| 501 |
+
model_id: 3,
|
| 502 |
+
n_layers: 4,
|
| 503 |
+
vocab: 8,
|
| 504 |
+
ctx_size: 128,
|
| 505 |
+
hidden_values: 2,
|
| 506 |
+
has_output: false,
|
| 507 |
+
layer_start: 1,
|
| 508 |
+
layer_end: 1,
|
| 509 |
+
hidden: vec![0.5, 1.5],
|
| 510 |
+
logits: vec![],
|
| 511 |
+
};
|
| 512 |
+
|
| 513 |
+
let tail_thread = thread::spawn(move || {
|
| 514 |
+
let mut hello_s = TcpStream::connect(coord_addr).unwrap();
|
| 515 |
+
tune(&hello_s);
|
| 516 |
+
let mut worker = Worker::new(tail);
|
| 517 |
+
let (hello, name) = worker.hello(data_port, 128, "tail");
|
| 518 |
+
send_hello(&mut hello_s, &hello, &name).unwrap();
|
| 519 |
+
ready_tx.send(()).unwrap();
|
| 520 |
+
let (mut work_s, _) = data.accept().unwrap();
|
| 521 |
+
tune(&work_s);
|
| 522 |
+
worker.serve(&mut work_s).unwrap();
|
| 523 |
+
});
|
| 524 |
+
ready_rx.recv_timeout(Duration::from_secs(5)).unwrap();
|
| 525 |
+
|
| 526 |
+
let mid_thread = thread::spawn(move || {
|
| 527 |
+
let mut stream = TcpStream::connect(coord_addr).unwrap();
|
| 528 |
+
tune(&stream);
|
| 529 |
+
let mut worker = Worker::new(mid);
|
| 530 |
+
let (hello, name) = worker.hello(7100, 128, "mid");
|
| 531 |
+
send_hello(&mut stream, &hello, &name).unwrap();
|
| 532 |
+
worker.serve(&mut stream).unwrap();
|
| 533 |
+
});
|
| 534 |
+
|
| 535 |
+
let mut first = None;
|
| 536 |
+
let view = CoordinatorView {
|
| 537 |
+
n_layers: 4,
|
| 538 |
+
local_start: 0,
|
| 539 |
+
local_end: 0,
|
| 540 |
+
local_has_output: false,
|
| 541 |
+
local_can_output_head: false,
|
| 542 |
+
};
|
| 543 |
+
let mut coord = Coordinator::new(view, 3, 32);
|
| 544 |
+
for _ in 0..2 {
|
| 545 |
+
let (mut s, _) = coord_l.accept().unwrap();
|
| 546 |
+
tune(&s);
|
| 547 |
+
let (hello, _) = coord.accept_hello(&mut s, "127.0.0.1").unwrap();
|
| 548 |
+
if hello.layer_start == 1 {
|
| 549 |
+
first = Some(s);
|
| 550 |
+
}
|
| 551 |
+
}
|
| 552 |
+
let mut stream = first.expect("mid worker stream");
|
| 553 |
+
let tokens = [3i32];
|
| 554 |
+
let hc = vec![0.25f32, 0.75];
|
| 555 |
+
let out = dispatch_eval(
|
| 556 |
+
&coord,
|
| 557 |
+
&mut stream,
|
| 558 |
+
&tokens,
|
| 559 |
+
0,
|
| 560 |
+
1,
|
| 561 |
+
2,
|
| 562 |
+
true,
|
| 563 |
+
TOKEN_HASH_INIT,
|
| 564 |
+
&hc,
|
| 565 |
+
)
|
| 566 |
+
.unwrap();
|
| 567 |
+
match out {
|
| 568 |
+
EvalOutcome::Logits(v) => assert_eq!(v, vec![0.0, 1.0, 0.0, 0.0]),
|
| 569 |
+
other => panic!("expected logits, got {other:?}"),
|
| 570 |
+
}
|
| 571 |
+
drop(stream);
|
| 572 |
+
mid_thread.join().unwrap();
|
| 573 |
+
tail_thread.join().unwrap();
|
| 574 |
+
}
|
| 575 |
+
|
| 576 |
+
#[test]
|
| 577 |
+
fn worker_rejects_model_mismatch_with_c_string() {
|
| 578 |
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 579 |
+
let addr = listener.local_addr().unwrap();
|
| 580 |
+
let exec = MockExec {
|
| 581 |
+
model_id: 3,
|
| 582 |
+
n_layers: 4,
|
| 583 |
+
vocab: 8,
|
| 584 |
+
ctx_size: 128,
|
| 585 |
+
hidden_values: 2,
|
| 586 |
+
has_output: true,
|
| 587 |
+
layer_start: 2,
|
| 588 |
+
layer_end: 3,
|
| 589 |
+
hidden: vec![0.0; 4],
|
| 590 |
+
logits: vec![1.0; 4],
|
| 591 |
+
};
|
| 592 |
+
let worker_thread = thread::spawn(move || {
|
| 593 |
+
let mut stream = TcpStream::connect(addr).unwrap();
|
| 594 |
+
tune(&stream);
|
| 595 |
+
let mut worker = Worker::new(exec);
|
| 596 |
+
let (hello, name) = worker.hello(7000, 128, "mock");
|
| 597 |
+
send_hello(&mut stream, &hello, &name).unwrap();
|
| 598 |
+
worker.serve(&mut stream).unwrap();
|
| 599 |
+
});
|
| 600 |
+
let (mut stream, _) = listener.accept().unwrap();
|
| 601 |
+
tune(&stream);
|
| 602 |
+
let view = CoordinatorView {
|
| 603 |
+
n_layers: 4,
|
| 604 |
+
local_start: 0,
|
| 605 |
+
local_end: 1,
|
| 606 |
+
local_has_output: false,
|
| 607 |
+
local_can_output_head: false,
|
| 608 |
+
};
|
| 609 |
+
let mut coord = Coordinator::new(view, 99, 32);
|
| 610 |
+
coord.accept_hello(&mut stream, "127.0.0.1").unwrap();
|
| 611 |
+
let err = dispatch_eval(
|
| 612 |
+
&coord,
|
| 613 |
+
&mut stream,
|
| 614 |
+
&[1],
|
| 615 |
+
0,
|
| 616 |
+
1,
|
| 617 |
+
1,
|
| 618 |
+
true,
|
| 619 |
+
TOKEN_HASH_INIT,
|
| 620 |
+
&[0.0, 0.0],
|
| 621 |
+
)
|
| 622 |
+
.unwrap_err();
|
| 623 |
+
assert_eq!(err, "model id mismatch: work=99 worker=3");
|
| 624 |
+
drop(stream);
|
| 625 |
+
worker_thread.join().unwrap();
|
| 626 |
+
}
|
| 627 |
+
|
| 628 |
+
#[test]
|
| 629 |
+
fn prefix_hash_continues_committed_timeline() {
|
| 630 |
+
let committed = [7i32, 8];
|
| 631 |
+
let span = [9i32];
|
| 632 |
+
let (prefix, result) = token_span_hashes(&committed, &span);
|
| 633 |
+
assert_eq!(prefix, ds4_dist::token_hash_prefix(&committed));
|
| 634 |
+
assert_eq!(result, ds4_dist::token_hash_update_span(prefix, &span));
|
| 635 |
+
assert_ne!(prefix, TOKEN_HASH_INIT);
|
| 636 |
+
}
|
| 637 |
+
|
| 638 |
+
#[test]
|
| 639 |
+
fn usage_text_mentions_roles() {
|
| 640 |
+
assert!(ds4_dist::USAGE.contains("--role ROLE"));
|
| 641 |
+
assert!(ds4_dist::USAGE.contains("21:output"));
|
| 642 |
+
}
|
| 643 |
+
|
| 644 |
+
#[test]
|
| 645 |
+
fn open_data_listener_assigns_ephemeral_port() {
|
| 646 |
+
let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
|
| 647 |
+
assert_ne!(port, 0);
|
| 648 |
+
assert_eq!(listener.local_addr().unwrap().port(), port);
|
| 649 |
+
let _stream = std::net::TcpStream::connect(("127.0.0.1", port)).unwrap();
|
| 650 |
+
}
|
| 651 |
+
|
| 652 |
+
#[test]
|
| 653 |
+
fn open_data_listener_accept_data_client_returns_client() {
|
| 654 |
+
let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
|
| 655 |
+
let connector = thread::spawn(move || TcpStream::connect(("127.0.0.1", port)).unwrap());
|
| 656 |
+
let client = ds4_dist::accept_data_client(&listener).unwrap();
|
| 657 |
+
let _peer = connector.join().unwrap();
|
| 658 |
+
assert!(client.nodelay().unwrap());
|
| 659 |
+
}
|
| 660 |
+
|
| 661 |
+
#[test]
|
| 662 |
+
fn open_data_listener_accept_data_client_errors_after_close() {
|
| 663 |
+
use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd};
|
| 664 |
+
|
| 665 |
+
let (listener, _port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
|
| 666 |
+
let raw = listener.into_raw_fd();
|
| 667 |
+
// SAFETY: `raw` is exclusively owned; OwnedFd closes it.
|
| 668 |
+
drop(unsafe { OwnedFd::from_raw_fd(raw) });
|
| 669 |
+
// SAFETY: `raw` is a closed fd. Reconstruct only so accept observes the
|
| 670 |
+
// error, then forget so Drop does not close twice.
|
| 671 |
+
let listener = unsafe { TcpListener::from_raw_fd(raw) };
|
| 672 |
+
let err = ds4_dist::accept_data_client(&listener);
|
| 673 |
+
std::mem::forget(listener);
|
| 674 |
+
assert!(err.is_err());
|
| 675 |
+
}
|
| 676 |
+
|
| 677 |
+
fn hop_logits_exec() -> (MockExec, Vec<f32>) {
|
| 678 |
+
let logits = vec![0.1f32, 0.2, 0.7, 0.0];
|
| 679 |
+
(
|
| 680 |
+
MockExec {
|
| 681 |
+
model_id: 3,
|
| 682 |
+
n_layers: 4,
|
| 683 |
+
vocab: 8,
|
| 684 |
+
ctx_size: 128,
|
| 685 |
+
hidden_values: 2,
|
| 686 |
+
has_output: true,
|
| 687 |
+
layer_start: 2,
|
| 688 |
+
layer_end: 3,
|
| 689 |
+
hidden: vec![1.0, 2.0, 3.0, 4.0],
|
| 690 |
+
logits: logits.clone(),
|
| 691 |
+
},
|
| 692 |
+
logits,
|
| 693 |
+
)
|
| 694 |
+
}
|
| 695 |
+
|
| 696 |
+
fn hop_work_frame(port: u16) -> Vec<u8> {
|
| 697 |
+
hop_work_frame_with_id(port, 11)
|
| 698 |
+
}
|
| 699 |
+
|
| 700 |
+
fn hop_work_frame_with_id(port: u16, request_id: u64) -> Vec<u8> {
|
| 701 |
+
let tokens = [1i32, 2];
|
| 702 |
+
let prefix = TOKEN_HASH_INIT;
|
| 703 |
+
let result = token_hash_update_span(prefix, &tokens);
|
| 704 |
+
let route_blob = encode_route_blob(
|
| 705 |
+
&[RouteEntry {
|
| 706 |
+
host: "127.0.0.1".into(),
|
| 707 |
+
port: u32::from(port),
|
| 708 |
+
layer_start: 2,
|
| 709 |
+
layer_end: 3,
|
| 710 |
+
flags: ROUTE_F_OUTPUT_LOGITS,
|
| 711 |
+
}],
|
| 712 |
+
&ReturnTarget {
|
| 713 |
+
kind: ROUTE_RETURN_UPSTREAM,
|
| 714 |
+
host: String::new(),
|
| 715 |
+
port: 0,
|
| 716 |
+
},
|
| 717 |
+
)
|
| 718 |
+
.unwrap();
|
| 719 |
+
let work = work_with_ids(
|
| 720 |
+
Work {
|
| 721 |
+
model_id: 3,
|
| 722 |
+
pos0: 0,
|
| 723 |
+
n_tokens: tokens.len() as u32,
|
| 724 |
+
layer_start: 2,
|
| 725 |
+
layer_end: 3,
|
| 726 |
+
flags: WORK_F_INPUT_HC | WORK_F_OUTPUT_LOGITS | WORK_F_RESET_SESSION,
|
| 727 |
+
input_hc_bits: 32,
|
| 728 |
+
route_count: 1,
|
| 729 |
+
route_index: 0,
|
| 730 |
+
..Work::default()
|
| 731 |
+
},
|
| 732 |
+
9,
|
| 733 |
+
request_id,
|
| 734 |
+
prefix,
|
| 735 |
+
result,
|
| 736 |
+
);
|
| 737 |
+
encode_work_frame(&WorkBody {
|
| 738 |
+
work,
|
| 739 |
+
tokens: tokens.to_vec(),
|
| 740 |
+
input_hc: vec![1.0, 2.0, 3.0, 4.0],
|
| 741 |
+
route_blob,
|
| 742 |
+
})
|
| 743 |
+
.unwrap()
|
| 744 |
+
}
|
| 745 |
+
|
| 746 |
+
#[test]
|
| 747 |
+
fn accepted_hop_serve_once_returns_result_for_work() {
|
| 748 |
+
// Given: a data listener and a MockExec worker that owns the final hop
|
| 749 |
+
let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
|
| 750 |
+
let frame = hop_work_frame(port);
|
| 751 |
+
let client = thread::spawn(move || {
|
| 752 |
+
let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap();
|
| 753 |
+
tune(&stream);
|
| 754 |
+
stream.write_all(&frame).unwrap();
|
| 755 |
+
read_frame(&mut stream).unwrap()
|
| 756 |
+
});
|
| 757 |
+
|
| 758 |
+
// When: accept the hop and serve one WORK
|
| 759 |
+
let mut hop = ds4_dist::accept_data_client(&listener).unwrap();
|
| 760 |
+
tune(&hop);
|
| 761 |
+
let (exec, logits) = hop_logits_exec();
|
| 762 |
+
let mut worker = Worker::new(exec);
|
| 763 |
+
worker.serve_once(&mut hop).unwrap();
|
| 764 |
+
|
| 765 |
+
// Then: the client reads a RESULT with the mock logits
|
| 766 |
+
let (typ, reply) = client.join().unwrap();
|
| 767 |
+
assert_eq!(typ, MSG_RESULT);
|
| 768 |
+
let result = decode_result_body(&reply).unwrap();
|
| 769 |
+
assert_eq!(result.hdr.status, 0);
|
| 770 |
+
assert_eq!(result.hdr.result_kind, RESULT_LOGITS);
|
| 771 |
+
assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits);
|
| 772 |
+
}
|
| 773 |
+
|
| 774 |
+
#[test]
|
| 775 |
+
fn accepted_hop_serve_once_unknown_frame_yields_error() {
|
| 776 |
+
// Given: a data listener and a MockExec worker
|
| 777 |
+
let (listener, port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
|
| 778 |
+
let client = thread::spawn(move || {
|
| 779 |
+
let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap();
|
| 780 |
+
tune(&stream);
|
| 781 |
+
write_frame(&mut stream, 99, b"not-a-work-frame").unwrap();
|
| 782 |
+
read_frame(&mut stream).unwrap()
|
| 783 |
+
});
|
| 784 |
+
|
| 785 |
+
// When: accept the hop and serve one unknown frame
|
| 786 |
+
let mut hop = ds4_dist::accept_data_client(&listener).unwrap();
|
| 787 |
+
tune(&hop);
|
| 788 |
+
let (exec, _) = hop_logits_exec();
|
| 789 |
+
let mut worker = Worker::new(exec);
|
| 790 |
+
let err = worker.serve_once(&mut hop).unwrap_err();
|
| 791 |
+
|
| 792 |
+
// Then: the worker rejects the type and the client reads an ERROR frame
|
| 793 |
+
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
|
| 794 |
+
let (typ, body) = client.join().unwrap();
|
| 795 |
+
assert_eq!(typ, MSG_ERROR);
|
| 796 |
+
assert_eq!(body, b"unsupported distributed worker frame");
|
| 797 |
+
}
|
| 798 |
+
|
| 799 |
+
#[test]
|
| 800 |
+
fn reconnect_local_keeps_coordinator_open_while_hop_work_completes() {
|
| 801 |
+
// Given: a coordinator TCP and a worker data listener
|
| 802 |
+
let coord_listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 803 |
+
let coord_addr = coord_listener.local_addr().unwrap();
|
| 804 |
+
let (data_listener, data_port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
|
| 805 |
+
let frame = hop_work_frame(data_port);
|
| 806 |
+
let (stop_tx, stop_rx) = mpsc::channel();
|
| 807 |
+
let (exec, logits) = hop_logits_exec();
|
| 808 |
+
|
| 809 |
+
let worker_thread = thread::spawn(move || {
|
| 810 |
+
let mut worker = Worker::new(exec);
|
| 811 |
+
let (hello, name) = worker.hello(u32::from(data_port), 128, "mock");
|
| 812 |
+
let mut stopped = false;
|
| 813 |
+
reconnect_local(
|
| 814 |
+
&mut worker,
|
| 815 |
+
LocalReconnect {
|
| 816 |
+
connect: || {
|
| 817 |
+
let stream = TcpStream::connect(coord_addr)?;
|
| 818 |
+
tune(&stream);
|
| 819 |
+
Ok(stream)
|
| 820 |
+
},
|
| 821 |
+
hello: &hello,
|
| 822 |
+
model_name: &name,
|
| 823 |
+
sleep: || {},
|
| 824 |
+
should_stop: || {
|
| 825 |
+
stopped |= stop_rx.try_recv().is_ok();
|
| 826 |
+
stopped
|
| 827 |
+
},
|
| 828 |
+
listener: Some(&data_listener),
|
| 829 |
+
},
|
| 830 |
+
)
|
| 831 |
+
});
|
| 832 |
+
|
| 833 |
+
// When: HELLO stays on the coordinator while a hop WORK is served
|
| 834 |
+
let (mut coord, _) = coord_listener.accept().unwrap();
|
| 835 |
+
tune(&coord);
|
| 836 |
+
let _ = ds4_dist::recv_hello(&mut coord).unwrap();
|
| 837 |
+
|
| 838 |
+
let mut hop = TcpStream::connect(("127.0.0.1", data_port)).unwrap();
|
| 839 |
+
tune(&hop);
|
| 840 |
+
hop.write_all(&frame).unwrap();
|
| 841 |
+
let (typ, reply) = read_frame(&mut hop).unwrap();
|
| 842 |
+
|
| 843 |
+
// Then: hop RESULT arrives and the coordinator TCP is still open
|
| 844 |
+
assert_eq!(typ, MSG_RESULT);
|
| 845 |
+
let result = decode_result_body(&reply).unwrap();
|
| 846 |
+
assert_eq!(result.hdr.status, 0);
|
| 847 |
+
assert_eq!(result.hdr.result_kind, RESULT_LOGITS);
|
| 848 |
+
assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits);
|
| 849 |
+
assert!(coord.peer_addr().is_ok());
|
| 850 |
+
coord
|
| 851 |
+
.set_read_timeout(Some(Duration::from_millis(50)))
|
| 852 |
+
.unwrap();
|
| 853 |
+
let mut probe = [0u8; 1];
|
| 854 |
+
let idle = coord.read(&mut probe);
|
| 855 |
+
assert!(
|
| 856 |
+
matches!(&idle, Err(e) if e.kind() == std::io::ErrorKind::TimedOut
|
| 857 |
+
|| e.kind() == std::io::ErrorKind::WouldBlock),
|
| 858 |
+
"coordinator must stay open (got {idle:?})"
|
| 859 |
+
);
|
| 860 |
+
|
| 861 |
+
stop_tx.send(()).unwrap();
|
| 862 |
+
drop(coord);
|
| 863 |
+
worker_thread.join().unwrap().unwrap();
|
| 864 |
+
}
|
| 865 |
+
|
| 866 |
+
fn middle_exec() -> MockExec {
|
| 867 |
+
MockExec {
|
| 868 |
+
model_id: 3,
|
| 869 |
+
n_layers: 4,
|
| 870 |
+
vocab: 8,
|
| 871 |
+
ctx_size: 128,
|
| 872 |
+
hidden_values: 2,
|
| 873 |
+
has_output: false,
|
| 874 |
+
layer_start: 0,
|
| 875 |
+
layer_end: 1,
|
| 876 |
+
hidden: vec![1.0, 2.0, 3.0, 4.0],
|
| 877 |
+
logits: Vec::new(),
|
| 878 |
+
}
|
| 879 |
+
}
|
| 880 |
+
|
| 881 |
+
fn two_hop_work_frame(middle_port: u16, next_port: u16) -> Vec<u8> {
|
| 882 |
+
let tokens = [1i32, 2];
|
| 883 |
+
let prefix = TOKEN_HASH_INIT;
|
| 884 |
+
let result = token_hash_update_span(prefix, &tokens);
|
| 885 |
+
let route_blob = encode_route_blob(
|
| 886 |
+
&[
|
| 887 |
+
RouteEntry {
|
| 888 |
+
host: "127.0.0.1".into(),
|
| 889 |
+
port: u32::from(middle_port),
|
| 890 |
+
layer_start: 0,
|
| 891 |
+
layer_end: 1,
|
| 892 |
+
flags: 0,
|
| 893 |
+
},
|
| 894 |
+
RouteEntry {
|
| 895 |
+
host: "127.0.0.1".into(),
|
| 896 |
+
port: u32::from(next_port),
|
| 897 |
+
layer_start: 2,
|
| 898 |
+
layer_end: 3,
|
| 899 |
+
flags: ROUTE_F_OUTPUT_LOGITS,
|
| 900 |
+
},
|
| 901 |
+
],
|
| 902 |
+
&ReturnTarget {
|
| 903 |
+
kind: ROUTE_RETURN_UPSTREAM,
|
| 904 |
+
host: String::new(),
|
| 905 |
+
port: 0,
|
| 906 |
+
},
|
| 907 |
+
)
|
| 908 |
+
.unwrap();
|
| 909 |
+
let work = work_with_ids(
|
| 910 |
+
Work {
|
| 911 |
+
model_id: 3,
|
| 912 |
+
pos0: 0,
|
| 913 |
+
n_tokens: tokens.len() as u32,
|
| 914 |
+
layer_start: 0,
|
| 915 |
+
layer_end: 1,
|
| 916 |
+
flags: WORK_F_RESET_SESSION,
|
| 917 |
+
input_hc_bits: 32,
|
| 918 |
+
route_count: 2,
|
| 919 |
+
route_index: 0,
|
| 920 |
+
..Work::default()
|
| 921 |
+
},
|
| 922 |
+
9,
|
| 923 |
+
11,
|
| 924 |
+
prefix,
|
| 925 |
+
result,
|
| 926 |
+
);
|
| 927 |
+
encode_work_frame(&WorkBody {
|
| 928 |
+
work,
|
| 929 |
+
tokens: tokens.to_vec(),
|
| 930 |
+
input_hc: Vec::new(),
|
| 931 |
+
route_blob,
|
| 932 |
+
})
|
| 933 |
+
.unwrap()
|
| 934 |
+
}
|
| 935 |
+
|
| 936 |
+
fn spawn_reconnect_worker(
|
| 937 |
+
exec: MockExec,
|
| 938 |
+
coord_addr: SocketAddr,
|
| 939 |
+
data_listener: TcpListener,
|
| 940 |
+
data_port: u16,
|
| 941 |
+
stop_rx: mpsc::Receiver<()>,
|
| 942 |
+
) -> thread::JoinHandle<std::io::Result<()>> {
|
| 943 |
+
thread::spawn(move || {
|
| 944 |
+
let mut worker = Worker::new(exec);
|
| 945 |
+
let (hello, name) = worker.hello(u32::from(data_port), 128, "mock");
|
| 946 |
+
let mut stopped = false;
|
| 947 |
+
reconnect_local(
|
| 948 |
+
&mut worker,
|
| 949 |
+
LocalReconnect {
|
| 950 |
+
connect: || {
|
| 951 |
+
let stream = TcpStream::connect(coord_addr)?;
|
| 952 |
+
tune(&stream);
|
| 953 |
+
Ok(stream)
|
| 954 |
+
},
|
| 955 |
+
hello: &hello,
|
| 956 |
+
model_name: &name,
|
| 957 |
+
sleep: || {},
|
| 958 |
+
should_stop: || {
|
| 959 |
+
stopped |= stop_rx.try_recv().is_ok();
|
| 960 |
+
stopped
|
| 961 |
+
},
|
| 962 |
+
listener: Some(&data_listener),
|
| 963 |
+
},
|
| 964 |
+
)
|
| 965 |
+
})
|
| 966 |
+
}
|
| 967 |
+
|
| 968 |
+
fn accept_hello(listener: &TcpListener) -> TcpStream {
|
| 969 |
+
let (stream, _) = listener.accept().unwrap();
|
| 970 |
+
tune(&stream);
|
| 971 |
+
let mut stream = stream;
|
| 972 |
+
let _ = ds4_dist::recv_hello(&mut stream).unwrap();
|
| 973 |
+
stream
|
| 974 |
+
}
|
| 975 |
+
|
| 976 |
+
fn assert_c_shaped_telemetry(request_id: u64, hop: u32, tel: &Telemetry) {
|
| 977 |
+
let line = format_telemetry_line(request_id, hop, tel);
|
| 978 |
+
assert!(
|
| 979 |
+
line.starts_with("ds4: distributed telemetry: "),
|
| 980 |
+
"C telemetry prefix missing: {line:?}"
|
| 981 |
+
);
|
| 982 |
+
assert!(line.contains(&format!("request={request_id}")));
|
| 983 |
+
assert!(line.contains(&format!("hop={hop}")));
|
| 984 |
+
assert!(line.contains(&format!("layers={}:{}", tel.layer_start, tel.layer_end)));
|
| 985 |
+
assert!(line.contains("eval="));
|
| 986 |
+
assert!(line.contains("downstream_wait="));
|
| 987 |
+
assert!(line.contains("forward_send="));
|
| 988 |
+
assert!(line.contains("input="));
|
| 989 |
+
assert!(line.contains("output="));
|
| 990 |
+
assert!(line.contains("MiB"));
|
| 991 |
+
assert!(line.ends_with('\n'));
|
| 992 |
+
}
|
| 993 |
+
|
| 994 |
+
#[test]
|
| 995 |
+
fn reconnect_local_two_hop_result_includes_c_shaped_telemetry() {
|
| 996 |
+
// Given: production reconnect_local mux on a middle worker and a final hop
|
| 997 |
+
let coord_listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 998 |
+
let coord_addr = coord_listener.local_addr().unwrap();
|
| 999 |
+
let (middle_listener, middle_port) =
|
| 1000 |
+
ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
|
| 1001 |
+
let (final_listener, final_port) = ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
|
| 1002 |
+
let (stop_mid_tx, stop_mid_rx) = mpsc::channel();
|
| 1003 |
+
let (stop_fin_tx, stop_fin_rx) = mpsc::channel();
|
| 1004 |
+
let (final_exec, logits) = hop_logits_exec();
|
| 1005 |
+
|
| 1006 |
+
let final_thread = spawn_reconnect_worker(
|
| 1007 |
+
final_exec,
|
| 1008 |
+
coord_addr,
|
| 1009 |
+
final_listener,
|
| 1010 |
+
final_port,
|
| 1011 |
+
stop_fin_rx,
|
| 1012 |
+
);
|
| 1013 |
+
let middle_thread = spawn_reconnect_worker(
|
| 1014 |
+
middle_exec(),
|
| 1015 |
+
coord_addr,
|
| 1016 |
+
middle_listener,
|
| 1017 |
+
middle_port,
|
| 1018 |
+
stop_mid_rx,
|
| 1019 |
+
);
|
| 1020 |
+
let coord_final = accept_hello(&coord_listener);
|
| 1021 |
+
let coord_middle = accept_hello(&coord_listener);
|
| 1022 |
+
|
| 1023 |
+
// When: a two-hop WORK is sent on the middle worker data port
|
| 1024 |
+
let frame = two_hop_work_frame(middle_port, final_port);
|
| 1025 |
+
let mut hop = TcpStream::connect(("127.0.0.1", middle_port)).unwrap();
|
| 1026 |
+
tune(&hop);
|
| 1027 |
+
hop.write_all(&frame).unwrap();
|
| 1028 |
+
let (typ, reply) = read_frame(&mut hop).unwrap();
|
| 1029 |
+
|
| 1030 |
+
// Then: hop RESULT carries both hops and format_telemetry_line is C-shaped
|
| 1031 |
+
assert_eq!(typ, MSG_RESULT);
|
| 1032 |
+
let result = decode_result_body(&reply).unwrap();
|
| 1033 |
+
assert_eq!(result.hdr.status, 0);
|
| 1034 |
+
assert_eq!(result.hdr.result_kind, RESULT_LOGITS);
|
| 1035 |
+
assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits);
|
| 1036 |
+
assert_eq!(result.telemetry.len(), 2);
|
| 1037 |
+
assert_eq!(result.telemetry[0].layer_start, 2);
|
| 1038 |
+
assert_eq!(result.telemetry[0].layer_end, 3);
|
| 1039 |
+
assert_eq!(result.telemetry[0].route_index, 1);
|
| 1040 |
+
assert_eq!(result.telemetry[1].layer_start, 0);
|
| 1041 |
+
assert_eq!(result.telemetry[1].layer_end, 1);
|
| 1042 |
+
assert_eq!(result.telemetry[1].route_index, 0);
|
| 1043 |
+
assert_c_shaped_telemetry(11, 0, &result.telemetry[0]);
|
| 1044 |
+
assert_c_shaped_telemetry(11, 1, &result.telemetry[1]);
|
| 1045 |
+
|
| 1046 |
+
stop_mid_tx.send(()).unwrap();
|
| 1047 |
+
stop_fin_tx.send(()).unwrap();
|
| 1048 |
+
drop(hop);
|
| 1049 |
+
drop(coord_final);
|
| 1050 |
+
drop(coord_middle);
|
| 1051 |
+
middle_thread.join().unwrap().unwrap();
|
| 1052 |
+
final_thread.join().unwrap().unwrap();
|
| 1053 |
+
}
|
| 1054 |
+
|
| 1055 |
+
#[test]
|
| 1056 |
+
fn reconnect_local_missing_next_hop_keeps_c_forward_error() {
|
| 1057 |
+
// Given: a middle worker whose next hop accepts then closes
|
| 1058 |
+
let coord_listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 1059 |
+
let coord_addr = coord_listener.local_addr().unwrap();
|
| 1060 |
+
let (middle_listener, middle_port) =
|
| 1061 |
+
ds4_dist::open_data_listener(Some("127.0.0.1"), 0).unwrap();
|
| 1062 |
+
let next_listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 1063 |
+
let next_port = next_listener.local_addr().unwrap().port();
|
| 1064 |
+
let (stop_tx, stop_rx) = mpsc::channel();
|
| 1065 |
+
let middle_thread = spawn_reconnect_worker(
|
| 1066 |
+
middle_exec(),
|
| 1067 |
+
coord_addr,
|
| 1068 |
+
middle_listener,
|
| 1069 |
+
middle_port,
|
| 1070 |
+
stop_rx,
|
| 1071 |
+
);
|
| 1072 |
+
let coord = accept_hello(&coord_listener);
|
| 1073 |
+
let next_thread = thread::spawn(move || {
|
| 1074 |
+
let (stream, _) = next_listener.accept().unwrap();
|
| 1075 |
+
drop(stream);
|
| 1076 |
+
});
|
| 1077 |
+
|
| 1078 |
+
// When: two-hop WORK is forwarded to the closed next hop
|
| 1079 |
+
let frame = two_hop_work_frame(middle_port, next_port);
|
| 1080 |
+
let mut hop = TcpStream::connect(("127.0.0.1", middle_port)).unwrap();
|
| 1081 |
+
tune(&hop);
|
| 1082 |
+
hop.write_all(&frame).unwrap();
|
| 1083 |
+
let (typ, reply) = read_frame(&mut hop).unwrap();
|
| 1084 |
+
|
| 1085 |
+
// Then: RESULT uses the existing C forward-error string
|
| 1086 |
+
assert_eq!(typ, MSG_RESULT);
|
| 1087 |
+
let result = decode_result_body(&reply).unwrap();
|
| 1088 |
+
assert_ne!(result.hdr.status, 0);
|
| 1089 |
+
let msg = String::from_utf8_lossy(&result.payload);
|
| 1090 |
+
assert_eq!(msg, ERR_NEXT_CLOSED);
|
| 1091 |
+
assert!(!msg.contains("telemetry"));
|
| 1092 |
+
next_thread.join().unwrap();
|
| 1093 |
+
|
| 1094 |
+
stop_tx.send(()).unwrap();
|
| 1095 |
+
drop(hop);
|
| 1096 |
+
drop(coord);
|
| 1097 |
+
middle_thread.join().unwrap().unwrap();
|
| 1098 |
+
}
|
| 1099 |
+
|
| 1100 |
+
struct EvalGate {
|
| 1101 |
+
started: Mutex<u32>,
|
| 1102 |
+
started_cv: Condvar,
|
| 1103 |
+
release: Mutex<bool>,
|
| 1104 |
+
release_cv: Condvar,
|
| 1105 |
+
}
|
| 1106 |
+
|
| 1107 |
+
impl EvalGate {
|
| 1108 |
+
fn new() -> Arc<Self> {
|
| 1109 |
+
Arc::new(Self {
|
| 1110 |
+
started: Mutex::new(0),
|
| 1111 |
+
started_cv: Condvar::new(),
|
| 1112 |
+
release: Mutex::new(false),
|
| 1113 |
+
release_cv: Condvar::new(),
|
| 1114 |
+
})
|
| 1115 |
+
}
|
| 1116 |
+
|
| 1117 |
+
fn wait_started(&self) {
|
| 1118 |
+
let mut n = self.started.lock().expect("eval gate");
|
| 1119 |
+
while *n == 0 {
|
| 1120 |
+
n = self.started_cv.wait(n).expect("eval gate");
|
| 1121 |
+
}
|
| 1122 |
+
}
|
| 1123 |
+
|
| 1124 |
+
fn release(&self) {
|
| 1125 |
+
*self.release.lock().expect("eval gate") = true;
|
| 1126 |
+
self.release_cv.notify_all();
|
| 1127 |
+
}
|
| 1128 |
+
}
|
| 1129 |
+
|
| 1130 |
+
struct NotSendExec {
|
| 1131 |
+
inner: MockExec,
|
| 1132 |
+
gate: Arc<EvalGate>,
|
| 1133 |
+
_not_send: PhantomData<*const ()>,
|
| 1134 |
+
}
|
| 1135 |
+
|
| 1136 |
+
impl SliceExec for NotSendExec {
|
| 1137 |
+
fn model_id(&self) -> u32 {
|
| 1138 |
+
self.inner.model_id()
|
| 1139 |
+
}
|
| 1140 |
+
fn n_layers(&self) -> u32 {
|
| 1141 |
+
self.inner.n_layers()
|
| 1142 |
+
}
|
| 1143 |
+
fn vocab(&self) -> u32 {
|
| 1144 |
+
self.inner.vocab()
|
| 1145 |
+
}
|
| 1146 |
+
fn ctx_size(&self) -> u32 {
|
| 1147 |
+
self.inner.ctx_size()
|
| 1148 |
+
}
|
| 1149 |
+
fn hidden_values(&self) -> u64 {
|
| 1150 |
+
self.inner.hidden_values()
|
| 1151 |
+
}
|
| 1152 |
+
fn has_output(&self) -> bool {
|
| 1153 |
+
self.inner.has_output()
|
| 1154 |
+
}
|
| 1155 |
+
fn layer_start(&self) -> u32 {
|
| 1156 |
+
self.inner.layer_start()
|
| 1157 |
+
}
|
| 1158 |
+
fn layer_end(&self) -> u32 {
|
| 1159 |
+
self.inner.layer_end()
|
| 1160 |
+
}
|
| 1161 |
+
fn eval(&mut self, req: &WorkRequest) -> Result<WorkOutput, String> {
|
| 1162 |
+
{
|
| 1163 |
+
let mut n = self.gate.started.lock().expect("eval gate");
|
| 1164 |
+
*n += 1;
|
| 1165 |
+
self.gate.started_cv.notify_all();
|
| 1166 |
+
}
|
| 1167 |
+
let mut released = self.gate.release.lock().expect("eval gate");
|
| 1168 |
+
while !*released {
|
| 1169 |
+
released = self.gate.release_cv.wait(released).expect("eval gate");
|
| 1170 |
+
}
|
| 1171 |
+
self.inner.eval(req)
|
| 1172 |
+
}
|
| 1173 |
+
}
|
| 1174 |
+
|
| 1175 |
+
fn assert_local_prefetch_accepts_not_send<E, S>(
|
| 1176 |
+
worker: &mut Worker<E, S>,
|
| 1177 |
+
stream: &mut TcpStream,
|
| 1178 |
+
queue: &JobQueue<PrefetchJob>,
|
| 1179 |
+
) -> std::io::Result<()>
|
| 1180 |
+
where
|
| 1181 |
+
E: SliceExec,
|
| 1182 |
+
S: ds4_dist::SnapshotStore,
|
| 1183 |
+
{
|
| 1184 |
+
serve_prefetch_local_with(worker, stream, queue)
|
| 1185 |
+
}
|
| 1186 |
+
|
| 1187 |
+
#[test]
|
| 1188 |
+
fn prefetch_env_unset_enables_local_prefetch() {
|
| 1189 |
+
// Given: C default is prefetch on unless DS4_DIST_DISABLE_WORKER_PREFETCH is present
|
| 1190 |
+
// When: the disable env is unset
|
| 1191 |
+
// Then: local reconnect must take the prefetch path
|
| 1192 |
+
assert!(!prefetch_disabled_from(None));
|
| 1193 |
+
assert!(ds4_dist::local_prefetch_enabled_from(None));
|
| 1194 |
+
}
|
| 1195 |
+
|
| 1196 |
+
#[test]
|
| 1197 |
+
fn prefetch_env_set_disables_local_prefetch() {
|
| 1198 |
+
// Given: C turns prefetch off when the disable env exists (any value)
|
| 1199 |
+
// When: the disable env is present
|
| 1200 |
+
// Then: local reconnect must not use prefetch
|
| 1201 |
+
assert!(prefetch_disabled_from(Some(std::ffi::OsStr::new("1"))));
|
| 1202 |
+
assert!(!ds4_dist::local_prefetch_enabled_from(Some(
|
| 1203 |
+
std::ffi::OsStr::new("1")
|
| 1204 |
+
)));
|
| 1205 |
+
}
|
| 1206 |
+
|
| 1207 |
+
#[test]
|
| 1208 |
+
fn serve_prefetch_local_depth_2_queues_second_work_during_not_send_eval() {
|
| 1209 |
+
// Given: a !Send exec and a depth-2 local prefetch queue
|
| 1210 |
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 1211 |
+
let port = listener.local_addr().unwrap().port();
|
| 1212 |
+
let (inner, logits) = hop_logits_exec();
|
| 1213 |
+
let gate = EvalGate::new();
|
| 1214 |
+
let queue = Arc::new(JobQueue::<PrefetchJob>::new(2));
|
| 1215 |
+
let frame1 = hop_work_frame_with_id(port, 11);
|
| 1216 |
+
let frame2 = hop_work_frame_with_id(port, 12);
|
| 1217 |
+
let client_gate = Arc::clone(&gate);
|
| 1218 |
+
let client_queue = Arc::clone(&queue);
|
| 1219 |
+
|
| 1220 |
+
let client = thread::spawn(move || {
|
| 1221 |
+
let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap();
|
| 1222 |
+
tune(&stream);
|
| 1223 |
+
stream.write_all(&frame1).unwrap();
|
| 1224 |
+
client_gate.wait_started();
|
| 1225 |
+
stream.write_all(&frame2).unwrap();
|
| 1226 |
+
client_queue.wait_until_queued(1);
|
| 1227 |
+
let queued_during_eval = client_queue.queued();
|
| 1228 |
+
client_gate.release();
|
| 1229 |
+
let first = read_frame(&mut stream).unwrap();
|
| 1230 |
+
let second = read_frame(&mut stream).unwrap();
|
| 1231 |
+
(queued_during_eval, first, second)
|
| 1232 |
+
});
|
| 1233 |
+
|
| 1234 |
+
// When: session-thread prefetch serves both WORKs on a !Send worker
|
| 1235 |
+
let mut stream = listener.accept().unwrap().0;
|
| 1236 |
+
tune(&stream);
|
| 1237 |
+
let mut worker = Worker::new(NotSendExec {
|
| 1238 |
+
inner,
|
| 1239 |
+
gate,
|
| 1240 |
+
_not_send: PhantomData,
|
| 1241 |
+
});
|
| 1242 |
+
assert_local_prefetch_accepts_not_send(&mut worker, &mut stream, &queue).unwrap();
|
| 1243 |
+
|
| 1244 |
+
// Then: WORK2 was queued while eval1 was still in progress
|
| 1245 |
+
let (queued_during_eval, first, second) = client.join().unwrap();
|
| 1246 |
+
assert!(
|
| 1247 |
+
queued_during_eval >= 1,
|
| 1248 |
+
"depth 2 must queue WORK2 during eval1, queued={queued_during_eval}"
|
| 1249 |
+
);
|
| 1250 |
+
for (typ, reply) in [first, second] {
|
| 1251 |
+
assert_eq!(typ, MSG_RESULT);
|
| 1252 |
+
let result = decode_result_body(&reply).unwrap();
|
| 1253 |
+
assert_eq!(result.hdr.status, 0);
|
| 1254 |
+
assert_eq!(result.hdr.result_kind, RESULT_LOGITS);
|
| 1255 |
+
assert_eq!(decode_logits_payload(&result.payload).unwrap(), logits);
|
| 1256 |
+
}
|
| 1257 |
+
}
|
crates/ds4-dist/tests/snapshot_store.rs
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
use std::io::Cursor;
|
| 2 |
+
use std::net::{TcpListener, TcpStream};
|
| 3 |
+
use std::thread;
|
| 4 |
+
use std::time::Duration;
|
| 5 |
+
|
| 6 |
+
use ds4_dist::{
|
| 7 |
+
coordinator_load_snapshot, coordinator_save_snapshot, token_hash_prefix, MemorySnapshotStore,
|
| 8 |
+
SliceExec, SnapshotLoad, SnapshotMeta, SnapshotSave, SnapshotStore, WorkOutput, WorkRequest,
|
| 9 |
+
Worker,
|
| 10 |
+
};
|
| 11 |
+
|
| 12 |
+
#[derive(Clone)]
|
| 13 |
+
struct MockExec {
|
| 14 |
+
model_id: u32,
|
| 15 |
+
n_layers: u32,
|
| 16 |
+
vocab: u32,
|
| 17 |
+
ctx_size: u32,
|
| 18 |
+
layer_start: u32,
|
| 19 |
+
layer_end: u32,
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
impl SliceExec for MockExec {
|
| 23 |
+
fn model_id(&self) -> u32 {
|
| 24 |
+
self.model_id
|
| 25 |
+
}
|
| 26 |
+
fn n_layers(&self) -> u32 {
|
| 27 |
+
self.n_layers
|
| 28 |
+
}
|
| 29 |
+
fn vocab(&self) -> u32 {
|
| 30 |
+
self.vocab
|
| 31 |
+
}
|
| 32 |
+
fn ctx_size(&self) -> u32 {
|
| 33 |
+
self.ctx_size
|
| 34 |
+
}
|
| 35 |
+
fn hidden_values(&self) -> u64 {
|
| 36 |
+
2
|
| 37 |
+
}
|
| 38 |
+
fn has_output(&self) -> bool {
|
| 39 |
+
true
|
| 40 |
+
}
|
| 41 |
+
fn layer_start(&self) -> u32 {
|
| 42 |
+
self.layer_start
|
| 43 |
+
}
|
| 44 |
+
fn layer_end(&self) -> u32 {
|
| 45 |
+
self.layer_end
|
| 46 |
+
}
|
| 47 |
+
fn eval(&mut self, _req: &WorkRequest) -> Result<WorkOutput, String> {
|
| 48 |
+
Ok(WorkOutput {
|
| 49 |
+
hidden: None,
|
| 50 |
+
logits: Some(vec![0.0; 4]),
|
| 51 |
+
})
|
| 52 |
+
}
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
fn exec() -> MockExec {
|
| 56 |
+
MockExec {
|
| 57 |
+
model_id: 3,
|
| 58 |
+
n_layers: 16,
|
| 59 |
+
vocab: 32,
|
| 60 |
+
ctx_size: 128,
|
| 61 |
+
layer_start: 8,
|
| 62 |
+
layer_end: 12,
|
| 63 |
+
}
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
fn meta(tokens: &[i32]) -> SnapshotMeta {
|
| 67 |
+
SnapshotMeta {
|
| 68 |
+
model_id: 3,
|
| 69 |
+
session_id: 0x11_2233,
|
| 70 |
+
request_id: 0x44_5566,
|
| 71 |
+
token_hash: token_hash_prefix(tokens),
|
| 72 |
+
layer_start: 8,
|
| 73 |
+
layer_end: 12,
|
| 74 |
+
}
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
fn tune(s: &TcpStream) {
|
| 78 |
+
let _ = s.set_nodelay(true);
|
| 79 |
+
let _ = s.set_read_timeout(Some(Duration::from_secs(5)));
|
| 80 |
+
let _ = s.set_write_timeout(Some(Duration::from_secs(5)));
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
#[test]
|
| 84 |
+
fn store_save_reports_missing_session_with_c_text() {
|
| 85 |
+
let mut store = MemorySnapshotStore::new();
|
| 86 |
+
let err = store
|
| 87 |
+
.save(SnapshotSave {
|
| 88 |
+
session_id: 7,
|
| 89 |
+
token_count: 1,
|
| 90 |
+
token_hash: 1,
|
| 91 |
+
})
|
| 92 |
+
.err()
|
| 93 |
+
.unwrap();
|
| 94 |
+
assert_eq!(err, "worker has no distributed session to snapshot");
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
#[test]
|
| 98 |
+
fn store_save_reports_token_count_mismatch_with_c_text() {
|
| 99 |
+
let mut store = MemorySnapshotStore::new();
|
| 100 |
+
store.insert(7, vec![1, 2], b"shard").unwrap();
|
| 101 |
+
let err = store
|
| 102 |
+
.save(SnapshotSave {
|
| 103 |
+
session_id: 7,
|
| 104 |
+
token_count: 1,
|
| 105 |
+
token_hash: token_hash_prefix(&[1, 2]),
|
| 106 |
+
})
|
| 107 |
+
.err()
|
| 108 |
+
.unwrap();
|
| 109 |
+
assert_eq!(err, "worker snapshot token count mismatch");
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
#[test]
|
| 113 |
+
fn store_save_reports_token_hash_mismatch_with_c_text() {
|
| 114 |
+
let mut store = MemorySnapshotStore::new();
|
| 115 |
+
store.insert(7, vec![1, 2], b"shard").unwrap();
|
| 116 |
+
let err = store
|
| 117 |
+
.save(SnapshotSave {
|
| 118 |
+
session_id: 7,
|
| 119 |
+
token_count: 2,
|
| 120 |
+
token_hash: token_hash_prefix(&[1, 2]) ^ 1,
|
| 121 |
+
})
|
| 122 |
+
.err()
|
| 123 |
+
.unwrap();
|
| 124 |
+
assert_eq!(err, "worker snapshot token hash mismatch");
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
#[test]
|
| 128 |
+
fn store_load_sets_hash_valid_and_clears_it_on_restore_failure() {
|
| 129 |
+
let mut store = MemorySnapshotStore::new();
|
| 130 |
+
store.insert(7, vec![1], b"old").unwrap();
|
| 131 |
+
assert!(store.token_hash_valid(7));
|
| 132 |
+
|
| 133 |
+
let tokens = [3, 4];
|
| 134 |
+
let hash = token_hash_prefix(&tokens);
|
| 135 |
+
store
|
| 136 |
+
.load(
|
| 137 |
+
SnapshotLoad {
|
| 138 |
+
session_id: 7,
|
| 139 |
+
tokens: &tokens,
|
| 140 |
+
token_hash: hash,
|
| 141 |
+
payload_bytes: 4,
|
| 142 |
+
},
|
| 143 |
+
&mut Cursor::new(b"next"),
|
| 144 |
+
)
|
| 145 |
+
.unwrap();
|
| 146 |
+
assert!(store.token_hash_valid(7));
|
| 147 |
+
assert_eq!(store.token_hash(7), Some(hash));
|
| 148 |
+
assert_eq!(store.tokens(7), Some(tokens.as_slice()));
|
| 149 |
+
assert_eq!(store.read_shard(7).unwrap(), b"next");
|
| 150 |
+
|
| 151 |
+
store.fail_restore(true);
|
| 152 |
+
let err = store
|
| 153 |
+
.load(
|
| 154 |
+
SnapshotLoad {
|
| 155 |
+
session_id: 7,
|
| 156 |
+
tokens: &tokens,
|
| 157 |
+
token_hash: hash,
|
| 158 |
+
payload_bytes: 3,
|
| 159 |
+
},
|
| 160 |
+
&mut Cursor::new(b"bad"),
|
| 161 |
+
)
|
| 162 |
+
.unwrap_err();
|
| 163 |
+
assert_eq!(err, "failed to restore worker KV shard");
|
| 164 |
+
assert!(!store.token_hash_valid(7));
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
#[test]
|
| 168 |
+
fn worker_serve_save_load_round_trips_memory_shard() {
|
| 169 |
+
let tokens = [7i32, 8];
|
| 170 |
+
let meta = meta(&tokens);
|
| 171 |
+
let first = b"kv-shard-one".to_vec();
|
| 172 |
+
let second = b"kv-shard-two".to_vec();
|
| 173 |
+
let mut store = MemorySnapshotStore::new();
|
| 174 |
+
store
|
| 175 |
+
.insert(meta.session_id, tokens.to_vec(), &first)
|
| 176 |
+
.unwrap();
|
| 177 |
+
|
| 178 |
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 179 |
+
let addr = listener.local_addr().unwrap();
|
| 180 |
+
let worker_thread = thread::spawn(move || {
|
| 181 |
+
let mut stream = TcpStream::connect(addr).unwrap();
|
| 182 |
+
tune(&stream);
|
| 183 |
+
let mut worker = Worker::with_store(exec(), store);
|
| 184 |
+
worker.serve(&mut stream).unwrap();
|
| 185 |
+
});
|
| 186 |
+
|
| 187 |
+
let (mut stream, _) = listener.accept().unwrap();
|
| 188 |
+
tune(&stream);
|
| 189 |
+
let mut saved = Vec::new();
|
| 190 |
+
assert_eq!(
|
| 191 |
+
coordinator_save_snapshot(&mut stream, meta, &tokens, &mut saved).unwrap(),
|
| 192 |
+
first.len() as u64
|
| 193 |
+
);
|
| 194 |
+
assert_eq!(saved, first);
|
| 195 |
+
|
| 196 |
+
coordinator_load_snapshot(
|
| 197 |
+
&mut stream,
|
| 198 |
+
meta,
|
| 199 |
+
&tokens,
|
| 200 |
+
&mut Cursor::new(&second),
|
| 201 |
+
second.len() as u64,
|
| 202 |
+
)
|
| 203 |
+
.unwrap();
|
| 204 |
+
|
| 205 |
+
let mut saved = Vec::new();
|
| 206 |
+
assert_eq!(
|
| 207 |
+
coordinator_save_snapshot(&mut stream, meta, &tokens, &mut saved).unwrap(),
|
| 208 |
+
second.len() as u64
|
| 209 |
+
);
|
| 210 |
+
assert_eq!(saved, second);
|
| 211 |
+
|
| 212 |
+
drop(stream);
|
| 213 |
+
worker_thread.join().unwrap();
|
| 214 |
+
}
|
crates/ds4-dist/tests/snapshot_transport.rs
ADDED
|
@@ -0,0 +1,646 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
use std::io::{self, Cursor, Read, Write};
|
| 2 |
+
|
| 3 |
+
use ds4_dist::{
|
| 4 |
+
coordinator_load_snapshot, coordinator_save_snapshot, decode_snapshot_chunk_body,
|
| 5 |
+
decode_snapshot_load_begin_body, encode_frame_header, encode_snapshot_begin_body,
|
| 6 |
+
encode_snapshot_chunk_body, encode_snapshot_done_body, read_frame, token_hash_prefix,
|
| 7 |
+
worker_handle_snapshot_load, worker_handle_snapshot_save, write_frame, SnapshotBegin,
|
| 8 |
+
SnapshotMeta, SnapshotReq, WorkerSnapshotIdentity, MSG_SNAPSHOT_BEGIN, MSG_SNAPSHOT_CHUNK,
|
| 9 |
+
MSG_SNAPSHOT_DONE, MSG_SNAPSHOT_LOAD_BEGIN, MSG_SNAPSHOT_SAVE_REQ, SNAPSHOT_CHUNK_BYTES,
|
| 10 |
+
SNAPSHOT_REQ_FIXED_BYTES,
|
| 11 |
+
};
|
| 12 |
+
|
| 13 |
+
#[derive(Default)]
|
| 14 |
+
struct ScriptedStream {
|
| 15 |
+
read: Cursor<Vec<u8>>,
|
| 16 |
+
written: Vec<u8>,
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
impl ScriptedStream {
|
| 20 |
+
fn new(read: Vec<u8>) -> Self {
|
| 21 |
+
Self {
|
| 22 |
+
read: Cursor::new(read),
|
| 23 |
+
written: Vec::new(),
|
| 24 |
+
}
|
| 25 |
+
}
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
impl Read for ScriptedStream {
|
| 29 |
+
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
| 30 |
+
self.read.read(buf)
|
| 31 |
+
}
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
impl Write for ScriptedStream {
|
| 35 |
+
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
| 36 |
+
self.written.extend_from_slice(buf);
|
| 37 |
+
Ok(buf.len())
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
fn flush(&mut self) -> io::Result<()> {
|
| 41 |
+
Ok(())
|
| 42 |
+
}
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
struct RejectLargeReads {
|
| 46 |
+
read: Cursor<Vec<u8>>,
|
| 47 |
+
written: Vec<u8>,
|
| 48 |
+
largest_read: usize,
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
impl RejectLargeReads {
|
| 52 |
+
fn new(read: Vec<u8>) -> Self {
|
| 53 |
+
Self {
|
| 54 |
+
read: Cursor::new(read),
|
| 55 |
+
written: Vec::new(),
|
| 56 |
+
largest_read: 0,
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
impl Read for RejectLargeReads {
|
| 62 |
+
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
| 63 |
+
self.largest_read = self.largest_read.max(buf.len());
|
| 64 |
+
assert!(buf.len() <= SNAPSHOT_CHUNK_BYTES);
|
| 65 |
+
self.read.read(buf)
|
| 66 |
+
}
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
impl Write for RejectLargeReads {
|
| 70 |
+
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
| 71 |
+
self.written.extend_from_slice(buf);
|
| 72 |
+
Ok(buf.len())
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
fn flush(&mut self) -> io::Result<()> {
|
| 76 |
+
Ok(())
|
| 77 |
+
}
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
fn meta() -> SnapshotMeta {
|
| 81 |
+
SnapshotMeta {
|
| 82 |
+
model_id: 3,
|
| 83 |
+
session_id: 0x11_2233,
|
| 84 |
+
request_id: 0x44_5566,
|
| 85 |
+
token_hash: 0x77_8899,
|
| 86 |
+
layer_start: 8,
|
| 87 |
+
layer_end: 12,
|
| 88 |
+
}
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
fn begin(meta: SnapshotMeta, token_count: u32, payload_bytes: u64) -> SnapshotBegin {
|
| 92 |
+
let (session_hi, session_lo) = ds4_dist::u64_to_halves(meta.session_id);
|
| 93 |
+
let (request_hi, request_lo) = ds4_dist::u64_to_halves(meta.request_id);
|
| 94 |
+
let (token_hash_hi, token_hash_lo) = ds4_dist::u64_to_halves(meta.token_hash);
|
| 95 |
+
let (payload_hi, payload_lo) = ds4_dist::u64_to_halves(payload_bytes);
|
| 96 |
+
SnapshotBegin {
|
| 97 |
+
model_id: meta.model_id,
|
| 98 |
+
session_hi,
|
| 99 |
+
session_lo,
|
| 100 |
+
request_hi,
|
| 101 |
+
request_lo,
|
| 102 |
+
token_hash_hi,
|
| 103 |
+
token_hash_lo,
|
| 104 |
+
token_count,
|
| 105 |
+
layer_start: meta.layer_start,
|
| 106 |
+
layer_end: meta.layer_end,
|
| 107 |
+
payload_hi,
|
| 108 |
+
payload_lo,
|
| 109 |
+
status: 0,
|
| 110 |
+
token_bytes: token_count * 4,
|
| 111 |
+
message_bytes: 0,
|
| 112 |
+
}
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
fn push_frame(out: &mut Vec<u8>, typ: u32, body: &[u8]) {
|
| 116 |
+
write_frame(out, typ, body).unwrap();
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
#[test]
|
| 120 |
+
fn coordinator_save_follows_c_frame_order_and_lengths() {
|
| 121 |
+
let meta = meta();
|
| 122 |
+
let tokens = [7, 8];
|
| 123 |
+
let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3];
|
| 124 |
+
payload[SNAPSHOT_CHUNK_BYTES] = 1;
|
| 125 |
+
payload[SNAPSHOT_CHUNK_BYTES + 1] = 2;
|
| 126 |
+
payload[SNAPSHOT_CHUNK_BYTES + 2] = 3;
|
| 127 |
+
let mut scripted = Vec::new();
|
| 128 |
+
|
| 129 |
+
let mut begin_body = encode_snapshot_begin_body(
|
| 130 |
+
&begin(meta, tokens.len() as u32, payload.len() as u64),
|
| 131 |
+
&tokens,
|
| 132 |
+
b"",
|
| 133 |
+
)
|
| 134 |
+
.unwrap();
|
| 135 |
+
begin_body.extend_from_slice(b"ignored trailing begin bytes");
|
| 136 |
+
push_frame(&mut scripted, MSG_SNAPSHOT_BEGIN, &begin_body);
|
| 137 |
+
push_frame(
|
| 138 |
+
&mut scripted,
|
| 139 |
+
MSG_SNAPSHOT_CHUNK,
|
| 140 |
+
&encode_snapshot_chunk_body(meta.request_id, &payload[..SNAPSHOT_CHUNK_BYTES]).unwrap(),
|
| 141 |
+
);
|
| 142 |
+
push_frame(
|
| 143 |
+
&mut scripted,
|
| 144 |
+
MSG_SNAPSHOT_CHUNK,
|
| 145 |
+
&encode_snapshot_chunk_body(meta.request_id, &payload[SNAPSHOT_CHUNK_BYTES..]).unwrap(),
|
| 146 |
+
);
|
| 147 |
+
let mut done = encode_snapshot_done_body(meta.request_id, 0, b"").unwrap();
|
| 148 |
+
done.extend_from_slice(b"ignored trailing done bytes");
|
| 149 |
+
push_frame(&mut scripted, MSG_SNAPSHOT_DONE, &done);
|
| 150 |
+
|
| 151 |
+
let mut stream = ScriptedStream::new(scripted);
|
| 152 |
+
let mut saved = Vec::new();
|
| 153 |
+
assert_eq!(
|
| 154 |
+
coordinator_save_snapshot(&mut stream, meta, &tokens, &mut saved).unwrap(),
|
| 155 |
+
payload.len() as u64
|
| 156 |
+
);
|
| 157 |
+
assert_eq!(saved, payload);
|
| 158 |
+
|
| 159 |
+
let mut written = Cursor::new(stream.written);
|
| 160 |
+
let (typ, body) = read_frame(&mut written).unwrap();
|
| 161 |
+
assert_eq!(typ, MSG_SNAPSHOT_SAVE_REQ);
|
| 162 |
+
let request = SnapshotReq::decode(&body).unwrap();
|
| 163 |
+
assert_eq!(request.model_id, meta.model_id);
|
| 164 |
+
assert_eq!(
|
| 165 |
+
ds4_dist::u64_from_halves(request.session_hi, request.session_lo),
|
| 166 |
+
meta.session_id
|
| 167 |
+
);
|
| 168 |
+
assert_eq!(
|
| 169 |
+
ds4_dist::u64_from_halves(request.request_hi, request.request_lo),
|
| 170 |
+
meta.request_id
|
| 171 |
+
);
|
| 172 |
+
assert_eq!(request.token_count, tokens.len() as u32);
|
| 173 |
+
assert!(read_frame(&mut written).is_err());
|
| 174 |
+
}
|
| 175 |
+
|
| 176 |
+
#[test]
|
| 177 |
+
fn coordinator_save_rejects_status_metadata_and_frame_sequence_errors() {
|
| 178 |
+
let meta = meta();
|
| 179 |
+
let tokens = [7, 8];
|
| 180 |
+
|
| 181 |
+
let mut refused = begin(meta, 0, 0);
|
| 182 |
+
refused.status = 1;
|
| 183 |
+
refused.message_bytes = 12;
|
| 184 |
+
let body = encode_snapshot_begin_body(&refused, &[], b"refused\0tail").unwrap();
|
| 185 |
+
let mut frames = Vec::new();
|
| 186 |
+
push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
|
| 187 |
+
let mut saved = Vec::new();
|
| 188 |
+
assert_eq!(
|
| 189 |
+
coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
|
| 190 |
+
.unwrap_err(),
|
| 191 |
+
"refused"
|
| 192 |
+
);
|
| 193 |
+
|
| 194 |
+
let message = vec![b'x'; 300];
|
| 195 |
+
refused.message_bytes = message.len() as u32;
|
| 196 |
+
let body = encode_snapshot_begin_body(&refused, &[], &message).unwrap();
|
| 197 |
+
let mut frames = Vec::new();
|
| 198 |
+
push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
|
| 199 |
+
let mut saved = Vec::new();
|
| 200 |
+
assert_eq!(
|
| 201 |
+
coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
|
| 202 |
+
.unwrap_err(),
|
| 203 |
+
"x".repeat(255)
|
| 204 |
+
);
|
| 205 |
+
|
| 206 |
+
let mut wrong = begin(meta, tokens.len() as u32, 0);
|
| 207 |
+
wrong.layer_end += 1;
|
| 208 |
+
let body = encode_snapshot_begin_body(&wrong, &tokens, b"").unwrap();
|
| 209 |
+
let mut frames = Vec::new();
|
| 210 |
+
push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
|
| 211 |
+
let mut saved = Vec::new();
|
| 212 |
+
assert_eq!(
|
| 213 |
+
coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
|
| 214 |
+
.unwrap_err(),
|
| 215 |
+
"distributed KV shard metadata mismatch"
|
| 216 |
+
);
|
| 217 |
+
|
| 218 |
+
let body = encode_snapshot_begin_body(&begin(meta, 0, 4), &[], b"").unwrap();
|
| 219 |
+
let mut frames = Vec::new();
|
| 220 |
+
push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
|
| 221 |
+
push_frame(
|
| 222 |
+
&mut frames,
|
| 223 |
+
MSG_SNAPSHOT_CHUNK,
|
| 224 |
+
&encode_snapshot_chunk_body(meta.request_id, b"ab").unwrap(),
|
| 225 |
+
);
|
| 226 |
+
push_frame(
|
| 227 |
+
&mut frames,
|
| 228 |
+
MSG_SNAPSHOT_DONE,
|
| 229 |
+
&encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(),
|
| 230 |
+
);
|
| 231 |
+
let mut saved = Vec::new();
|
| 232 |
+
assert_eq!(
|
| 233 |
+
coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
|
| 234 |
+
.unwrap_err(),
|
| 235 |
+
"expected distributed KV shard chunk"
|
| 236 |
+
);
|
| 237 |
+
|
| 238 |
+
let body = encode_snapshot_begin_body(&begin(meta, 0, 2), &[], b"").unwrap();
|
| 239 |
+
let mut frames = Vec::new();
|
| 240 |
+
push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
|
| 241 |
+
push_frame(
|
| 242 |
+
&mut frames,
|
| 243 |
+
MSG_SNAPSHOT_CHUNK,
|
| 244 |
+
&encode_snapshot_chunk_body(meta.request_id, b"abc").unwrap(),
|
| 245 |
+
);
|
| 246 |
+
let mut saved = Vec::new();
|
| 247 |
+
assert_eq!(
|
| 248 |
+
coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
|
| 249 |
+
.unwrap_err(),
|
| 250 |
+
"invalid distributed KV shard chunk"
|
| 251 |
+
);
|
| 252 |
+
|
| 253 |
+
let body = encode_snapshot_begin_body(&begin(meta, 0, 1), &[], b"").unwrap();
|
| 254 |
+
let mut frames = Vec::new();
|
| 255 |
+
push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
|
| 256 |
+
push_frame(
|
| 257 |
+
&mut frames,
|
| 258 |
+
MSG_SNAPSHOT_CHUNK,
|
| 259 |
+
&encode_snapshot_chunk_body(meta.request_id + 1, b"a").unwrap(),
|
| 260 |
+
);
|
| 261 |
+
let mut saved = Vec::new();
|
| 262 |
+
assert_eq!(
|
| 263 |
+
coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
|
| 264 |
+
.unwrap_err(),
|
| 265 |
+
"invalid distributed KV shard chunk"
|
| 266 |
+
);
|
| 267 |
+
|
| 268 |
+
let body = encode_snapshot_begin_body(&begin(meta, 0, 1), &[], b"").unwrap();
|
| 269 |
+
let mut frames = Vec::new();
|
| 270 |
+
push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
|
| 271 |
+
for _ in 0..2 {
|
| 272 |
+
push_frame(
|
| 273 |
+
&mut frames,
|
| 274 |
+
MSG_SNAPSHOT_CHUNK,
|
| 275 |
+
&encode_snapshot_chunk_body(meta.request_id, b"a").unwrap(),
|
| 276 |
+
);
|
| 277 |
+
}
|
| 278 |
+
let mut saved = Vec::new();
|
| 279 |
+
assert_eq!(
|
| 280 |
+
coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
|
| 281 |
+
.unwrap_err(),
|
| 282 |
+
"distributed worker returned invalid snapshot completion frame"
|
| 283 |
+
);
|
| 284 |
+
|
| 285 |
+
let body = encode_snapshot_begin_body(&begin(meta, 0, 0), &[], b"").unwrap();
|
| 286 |
+
let mut frames = Vec::new();
|
| 287 |
+
push_frame(&mut frames, MSG_SNAPSHOT_BEGIN, &body);
|
| 288 |
+
push_frame(
|
| 289 |
+
&mut frames,
|
| 290 |
+
MSG_SNAPSHOT_DONE,
|
| 291 |
+
&encode_snapshot_done_body(meta.request_id + 1, 0, b"").unwrap(),
|
| 292 |
+
);
|
| 293 |
+
let mut saved = Vec::new();
|
| 294 |
+
assert_eq!(
|
| 295 |
+
coordinator_save_snapshot(&mut ScriptedStream::new(frames), meta, &tokens, &mut saved,)
|
| 296 |
+
.unwrap_err(),
|
| 297 |
+
"distributed snapshot completion request mismatch"
|
| 298 |
+
);
|
| 299 |
+
}
|
| 300 |
+
|
| 301 |
+
#[test]
|
| 302 |
+
fn coordinator_save_bounds_peer_lengths_and_reports_partial_stages() {
|
| 303 |
+
let meta = meta();
|
| 304 |
+
|
| 305 |
+
let mut stream =
|
| 306 |
+
RejectLargeReads::new(encode_frame_header(MSG_SNAPSHOT_BEGIN, u32::MAX).to_vec());
|
| 307 |
+
let mut saved = Vec::new();
|
| 308 |
+
assert_eq!(
|
| 309 |
+
coordinator_save_snapshot(&mut stream, meta, &[], &mut saved).unwrap_err(),
|
| 310 |
+
"failed to read distributed snapshot header"
|
| 311 |
+
);
|
| 312 |
+
assert!(stream.largest_read <= 60);
|
| 313 |
+
|
| 314 |
+
let mut partial_done = Vec::new();
|
| 315 |
+
push_frame(
|
| 316 |
+
&mut partial_done,
|
| 317 |
+
MSG_SNAPSHOT_BEGIN,
|
| 318 |
+
&encode_snapshot_begin_body(&begin(meta, 0, 0), &[], b"").unwrap(),
|
| 319 |
+
);
|
| 320 |
+
partial_done.extend_from_slice(&encode_frame_header(MSG_SNAPSHOT_DONE, 16));
|
| 321 |
+
partial_done.extend_from_slice(&[0; 8]);
|
| 322 |
+
let mut saved = Vec::new();
|
| 323 |
+
assert_eq!(
|
| 324 |
+
coordinator_save_snapshot(
|
| 325 |
+
&mut ScriptedStream::new(partial_done),
|
| 326 |
+
meta,
|
| 327 |
+
&[],
|
| 328 |
+
&mut saved,
|
| 329 |
+
)
|
| 330 |
+
.unwrap_err(),
|
| 331 |
+
"failed to read distributed snapshot completion"
|
| 332 |
+
);
|
| 333 |
+
}
|
| 334 |
+
|
| 335 |
+
#[test]
|
| 336 |
+
fn coordinator_load_chunks_payload_and_checks_completion() {
|
| 337 |
+
let meta = meta();
|
| 338 |
+
let tokens = vec![7, 8];
|
| 339 |
+
let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3];
|
| 340 |
+
payload[SNAPSHOT_CHUNK_BYTES] = 1;
|
| 341 |
+
payload[SNAPSHOT_CHUNK_BYTES + 1] = 2;
|
| 342 |
+
payload[SNAPSHOT_CHUNK_BYTES + 2] = 3;
|
| 343 |
+
|
| 344 |
+
let mut reply = Vec::new();
|
| 345 |
+
let mut done = encode_snapshot_done_body(meta.request_id, 0, b"").unwrap();
|
| 346 |
+
done.extend_from_slice(b"trailing");
|
| 347 |
+
push_frame(&mut reply, MSG_SNAPSHOT_DONE, &done);
|
| 348 |
+
let mut stream = ScriptedStream::new(reply);
|
| 349 |
+
let payload_len = payload.len() as u64;
|
| 350 |
+
coordinator_load_snapshot(
|
| 351 |
+
&mut stream,
|
| 352 |
+
meta,
|
| 353 |
+
&tokens,
|
| 354 |
+
&mut Cursor::new(&payload),
|
| 355 |
+
payload_len,
|
| 356 |
+
)
|
| 357 |
+
.unwrap();
|
| 358 |
+
|
| 359 |
+
let mut written = Cursor::new(stream.written);
|
| 360 |
+
let (typ, body) = read_frame(&mut written).unwrap();
|
| 361 |
+
assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN);
|
| 362 |
+
let (header, got_tokens, message) = decode_snapshot_load_begin_body(&body).unwrap();
|
| 363 |
+
assert_eq!(got_tokens, tokens);
|
| 364 |
+
assert!(message.is_empty());
|
| 365 |
+
assert_eq!(
|
| 366 |
+
ds4_dist::u64_from_halves(header.payload_hi, header.payload_lo),
|
| 367 |
+
payload.len() as u64
|
| 368 |
+
);
|
| 369 |
+
|
| 370 |
+
let (typ, body) = read_frame(&mut written).unwrap();
|
| 371 |
+
assert_eq!(typ, MSG_SNAPSHOT_CHUNK);
|
| 372 |
+
let (_, chunk) =
|
| 373 |
+
decode_snapshot_chunk_body(&body, meta.request_id, payload.len() as u64).unwrap();
|
| 374 |
+
assert_eq!(chunk.len(), SNAPSHOT_CHUNK_BYTES);
|
| 375 |
+
let (typ, body) = read_frame(&mut written).unwrap();
|
| 376 |
+
assert_eq!(typ, MSG_SNAPSHOT_CHUNK);
|
| 377 |
+
let (_, chunk) = decode_snapshot_chunk_body(&body, meta.request_id, 3).unwrap();
|
| 378 |
+
assert_eq!(chunk, [1, 2, 3]);
|
| 379 |
+
assert!(read_frame(&mut written).is_err());
|
| 380 |
+
|
| 381 |
+
let mut failed = Vec::new();
|
| 382 |
+
push_frame(
|
| 383 |
+
&mut failed,
|
| 384 |
+
MSG_SNAPSHOT_DONE,
|
| 385 |
+
&encode_snapshot_done_body(meta.request_id, 1, b"restore failed").unwrap(),
|
| 386 |
+
);
|
| 387 |
+
assert_eq!(
|
| 388 |
+
coordinator_load_snapshot(
|
| 389 |
+
&mut ScriptedStream::new(failed),
|
| 390 |
+
meta,
|
| 391 |
+
&tokens,
|
| 392 |
+
&mut Cursor::new(b"payload"),
|
| 393 |
+
7,
|
| 394 |
+
)
|
| 395 |
+
.unwrap_err(),
|
| 396 |
+
"restore failed"
|
| 397 |
+
);
|
| 398 |
+
}
|
| 399 |
+
|
| 400 |
+
#[test]
|
| 401 |
+
fn coordinator_zero_payload_does_not_require_or_send_chunks() {
|
| 402 |
+
let meta = meta();
|
| 403 |
+
|
| 404 |
+
let mut save_reply = Vec::new();
|
| 405 |
+
push_frame(
|
| 406 |
+
&mut save_reply,
|
| 407 |
+
MSG_SNAPSHOT_BEGIN,
|
| 408 |
+
&encode_snapshot_begin_body(&begin(meta, 0, 0), &[], b"").unwrap(),
|
| 409 |
+
);
|
| 410 |
+
push_frame(
|
| 411 |
+
&mut save_reply,
|
| 412 |
+
MSG_SNAPSHOT_DONE,
|
| 413 |
+
&encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(),
|
| 414 |
+
);
|
| 415 |
+
let mut saved = Vec::new();
|
| 416 |
+
assert_eq!(
|
| 417 |
+
coordinator_save_snapshot(&mut ScriptedStream::new(save_reply), meta, &[], &mut saved,)
|
| 418 |
+
.unwrap(),
|
| 419 |
+
0
|
| 420 |
+
);
|
| 421 |
+
assert!(saved.is_empty());
|
| 422 |
+
|
| 423 |
+
let mut reply = Vec::new();
|
| 424 |
+
push_frame(
|
| 425 |
+
&mut reply,
|
| 426 |
+
MSG_SNAPSHOT_DONE,
|
| 427 |
+
&encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(),
|
| 428 |
+
);
|
| 429 |
+
let mut stream = ScriptedStream::new(reply);
|
| 430 |
+
let mut source = Cursor::new(b"not part of the declared payload");
|
| 431 |
+
coordinator_load_snapshot(&mut stream, meta, &[], &mut source, 0).unwrap();
|
| 432 |
+
assert_eq!(source.position(), 0);
|
| 433 |
+
|
| 434 |
+
let mut written = Cursor::new(stream.written);
|
| 435 |
+
let (typ, _) = read_frame(&mut written).unwrap();
|
| 436 |
+
assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN);
|
| 437 |
+
assert!(read_frame(&mut written).is_err());
|
| 438 |
+
}
|
| 439 |
+
|
| 440 |
+
fn identity(meta: SnapshotMeta) -> WorkerSnapshotIdentity {
|
| 441 |
+
WorkerSnapshotIdentity {
|
| 442 |
+
model_id: meta.model_id,
|
| 443 |
+
layer_start: meta.layer_start,
|
| 444 |
+
layer_end: meta.layer_end,
|
| 445 |
+
ctx_size: 128,
|
| 446 |
+
}
|
| 447 |
+
}
|
| 448 |
+
|
| 449 |
+
fn hashed_meta(tokens: &[i32]) -> SnapshotMeta {
|
| 450 |
+
let mut meta = meta();
|
| 451 |
+
meta.token_hash = token_hash_prefix(tokens);
|
| 452 |
+
meta
|
| 453 |
+
}
|
| 454 |
+
|
| 455 |
+
#[cfg(unix)]
|
| 456 |
+
fn duplex_save(meta: SnapshotMeta, tokens: &[i32], payload: Vec<u8>) -> (Vec<u8>, u64) {
|
| 457 |
+
let (mut coord, mut worker) = std::os::unix::net::UnixStream::pair().unwrap();
|
| 458 |
+
let identity = identity(meta);
|
| 459 |
+
let sent = payload.clone();
|
| 460 |
+
let handle = std::thread::spawn(move || {
|
| 461 |
+
let (typ, body) = read_frame(&mut worker).unwrap();
|
| 462 |
+
assert_eq!(typ, MSG_SNAPSHOT_SAVE_REQ);
|
| 463 |
+
let mut src = Cursor::new(sent);
|
| 464 |
+
let len = src.get_ref().len() as u64;
|
| 465 |
+
worker_handle_snapshot_save(&mut worker, identity, &body, Ok((&mut src, len))).unwrap();
|
| 466 |
+
});
|
| 467 |
+
let mut saved = Vec::new();
|
| 468 |
+
let n = coordinator_save_snapshot(&mut coord, meta, tokens, &mut saved).unwrap();
|
| 469 |
+
handle.join().unwrap();
|
| 470 |
+
(saved, n)
|
| 471 |
+
}
|
| 472 |
+
|
| 473 |
+
#[cfg(unix)]
|
| 474 |
+
fn duplex_load(meta: SnapshotMeta, tokens: &[i32], payload: Vec<u8>) -> Vec<u8> {
|
| 475 |
+
let (mut coord, mut worker) = std::os::unix::net::UnixStream::pair().unwrap();
|
| 476 |
+
let identity = identity(meta);
|
| 477 |
+
let handle = std::thread::spawn(move || {
|
| 478 |
+
let (typ, body) = read_frame(&mut worker).unwrap();
|
| 479 |
+
assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN);
|
| 480 |
+
let mut restored = Vec::new();
|
| 481 |
+
let offer =
|
| 482 |
+
worker_handle_snapshot_load(&mut worker, identity, &body, 32, &mut restored).unwrap();
|
| 483 |
+
(restored, offer)
|
| 484 |
+
});
|
| 485 |
+
coordinator_load_snapshot(
|
| 486 |
+
&mut coord,
|
| 487 |
+
meta,
|
| 488 |
+
tokens,
|
| 489 |
+
&mut Cursor::new(&payload),
|
| 490 |
+
payload.len() as u64,
|
| 491 |
+
)
|
| 492 |
+
.unwrap();
|
| 493 |
+
let (restored, offer) = handle.join().unwrap();
|
| 494 |
+
assert_eq!(offer.session_id, meta.session_id);
|
| 495 |
+
assert_eq!(offer.request_id, meta.request_id);
|
| 496 |
+
assert_eq!(offer.token_hash, meta.token_hash);
|
| 497 |
+
assert_eq!(offer.tokens, tokens);
|
| 498 |
+
restored
|
| 499 |
+
}
|
| 500 |
+
|
| 501 |
+
#[cfg(unix)]
|
| 502 |
+
#[test]
|
| 503 |
+
fn worker_save_answers_coordinator_with_c_frame_order() {
|
| 504 |
+
let tokens = [7, 8];
|
| 505 |
+
let meta = hashed_meta(&tokens);
|
| 506 |
+
let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3];
|
| 507 |
+
payload[SNAPSHOT_CHUNK_BYTES] = 1;
|
| 508 |
+
payload[SNAPSHOT_CHUNK_BYTES + 1] = 2;
|
| 509 |
+
payload[SNAPSHOT_CHUNK_BYTES + 2] = 3;
|
| 510 |
+
let (saved, n) = duplex_save(meta, &tokens, payload.clone());
|
| 511 |
+
assert_eq!(n, payload.len() as u64);
|
| 512 |
+
assert_eq!(saved, payload);
|
| 513 |
+
}
|
| 514 |
+
|
| 515 |
+
#[cfg(unix)]
|
| 516 |
+
#[test]
|
| 517 |
+
fn worker_load_accepts_coordinator_chunks_and_reports_tokens() {
|
| 518 |
+
let tokens = [7, 8];
|
| 519 |
+
let meta = hashed_meta(&tokens);
|
| 520 |
+
let mut payload = vec![0x5a; SNAPSHOT_CHUNK_BYTES + 3];
|
| 521 |
+
payload[SNAPSHOT_CHUNK_BYTES] = 9;
|
| 522 |
+
let restored = duplex_load(meta, &tokens, payload.clone());
|
| 523 |
+
assert_eq!(restored, payload);
|
| 524 |
+
}
|
| 525 |
+
|
| 526 |
+
#[cfg(unix)]
|
| 527 |
+
#[test]
|
| 528 |
+
fn worker_zero_payload_round_trips_without_chunks() {
|
| 529 |
+
let meta = hashed_meta(&[]);
|
| 530 |
+
let (saved, n) = duplex_save(meta, &[], Vec::new());
|
| 531 |
+
assert_eq!(n, 0);
|
| 532 |
+
assert!(saved.is_empty());
|
| 533 |
+
assert!(duplex_load(meta, &[], Vec::new()).is_empty());
|
| 534 |
+
}
|
| 535 |
+
|
| 536 |
+
#[test]
|
| 537 |
+
fn worker_save_rejects_bad_size_and_identity_with_c_text() {
|
| 538 |
+
let meta = meta();
|
| 539 |
+
let identity = identity(meta);
|
| 540 |
+
let mut stream = ScriptedStream::new(Vec::new());
|
| 541 |
+
worker_handle_snapshot_save::<_, Cursor<Vec<u8>>>(
|
| 542 |
+
&mut stream,
|
| 543 |
+
identity,
|
| 544 |
+
&[0; SNAPSHOT_REQ_FIXED_BYTES - 1],
|
| 545 |
+
Err("unused".into()),
|
| 546 |
+
)
|
| 547 |
+
.unwrap();
|
| 548 |
+
let mut written = Cursor::new(stream.written);
|
| 549 |
+
let (typ, body) = read_frame(&mut written).unwrap();
|
| 550 |
+
assert_eq!(typ, MSG_SNAPSHOT_BEGIN);
|
| 551 |
+
let (begin, _, message) = ds4_dist::decode_snapshot_begin_body(&body).unwrap();
|
| 552 |
+
assert_eq!(begin.status, 1);
|
| 553 |
+
assert_eq!(message, b"invalid distributed snapshot save request");
|
| 554 |
+
|
| 555 |
+
let mut request = ScriptedStream::new(Vec::new());
|
| 556 |
+
let mut unused = Vec::new();
|
| 557 |
+
coordinator_save_snapshot(&mut request, meta, &[7], &mut unused).unwrap_err();
|
| 558 |
+
let mut written = Cursor::new(request.written);
|
| 559 |
+
let (typ, body) = read_frame(&mut written).unwrap();
|
| 560 |
+
assert_eq!(typ, MSG_SNAPSHOT_SAVE_REQ);
|
| 561 |
+
let mut wrong = identity;
|
| 562 |
+
wrong.layer_end += 1;
|
| 563 |
+
let mut reply = ScriptedStream::new(Vec::new());
|
| 564 |
+
worker_handle_snapshot_save::<_, Cursor<Vec<u8>>>(
|
| 565 |
+
&mut reply,
|
| 566 |
+
wrong,
|
| 567 |
+
&body,
|
| 568 |
+
Err("unused".into()),
|
| 569 |
+
)
|
| 570 |
+
.unwrap();
|
| 571 |
+
let mut written = Cursor::new(reply.written);
|
| 572 |
+
let (typ, body) = read_frame(&mut written).unwrap();
|
| 573 |
+
assert_eq!(typ, MSG_SNAPSHOT_BEGIN);
|
| 574 |
+
let (begin, _, message) = ds4_dist::decode_snapshot_begin_body(&body).unwrap();
|
| 575 |
+
assert_eq!(begin.status, 1);
|
| 576 |
+
assert_eq!(
|
| 577 |
+
std::str::from_utf8(&message).unwrap(),
|
| 578 |
+
"snapshot save request does not match worker state"
|
| 579 |
+
);
|
| 580 |
+
}
|
| 581 |
+
|
| 582 |
+
#[test]
|
| 583 |
+
fn worker_save_forwards_session_error_text() {
|
| 584 |
+
let meta = meta();
|
| 585 |
+
let mut request = ScriptedStream::new(Vec::new());
|
| 586 |
+
let mut unused = Vec::new();
|
| 587 |
+
coordinator_save_snapshot(&mut request, meta, &[7], &mut unused).unwrap_err();
|
| 588 |
+
let mut written = Cursor::new(request.written);
|
| 589 |
+
let (_, body) = read_frame(&mut written).unwrap();
|
| 590 |
+
let mut reply = ScriptedStream::new(Vec::new());
|
| 591 |
+
worker_handle_snapshot_save::<_, Cursor<Vec<u8>>>(
|
| 592 |
+
&mut reply,
|
| 593 |
+
identity(meta),
|
| 594 |
+
&body,
|
| 595 |
+
Err("worker has no distributed session to snapshot".into()),
|
| 596 |
+
)
|
| 597 |
+
.unwrap();
|
| 598 |
+
let mut written = Cursor::new(reply.written);
|
| 599 |
+
let (_, body) = read_frame(&mut written).unwrap();
|
| 600 |
+
let (begin, _, message) = ds4_dist::decode_snapshot_begin_body(&body).unwrap();
|
| 601 |
+
assert_eq!(begin.status, 1);
|
| 602 |
+
assert_eq!(
|
| 603 |
+
std::str::from_utf8(&message).unwrap(),
|
| 604 |
+
"worker has no distributed session to snapshot"
|
| 605 |
+
);
|
| 606 |
+
}
|
| 607 |
+
|
| 608 |
+
#[test]
|
| 609 |
+
fn worker_load_rejects_header_hash_and_identity_with_c_text() {
|
| 610 |
+
let identity = identity(meta());
|
| 611 |
+
let mut stream = ScriptedStream::new(Vec::new());
|
| 612 |
+
assert_eq!(
|
| 613 |
+
worker_handle_snapshot_load(&mut stream, identity, &[0; 8], 32, &mut Vec::new())
|
| 614 |
+
.unwrap_err(),
|
| 615 |
+
"invalid distributed snapshot load header"
|
| 616 |
+
);
|
| 617 |
+
assert!(stream.written.is_empty());
|
| 618 |
+
|
| 619 |
+
let tokens = [7, 8];
|
| 620 |
+
let mut meta = hashed_meta(&tokens);
|
| 621 |
+
meta.token_hash ^= 1;
|
| 622 |
+
let mut request = ScriptedStream::new({
|
| 623 |
+
let mut done = Vec::new();
|
| 624 |
+
push_frame(
|
| 625 |
+
&mut done,
|
| 626 |
+
MSG_SNAPSHOT_DONE,
|
| 627 |
+
&encode_snapshot_done_body(meta.request_id, 0, b"").unwrap(),
|
| 628 |
+
);
|
| 629 |
+
done
|
| 630 |
+
});
|
| 631 |
+
coordinator_load_snapshot(&mut request, meta, &tokens, &mut Cursor::new(b""), 0).unwrap();
|
| 632 |
+
let mut written = Cursor::new(request.written);
|
| 633 |
+
let (typ, body) = read_frame(&mut written).unwrap();
|
| 634 |
+
assert_eq!(typ, MSG_SNAPSHOT_LOAD_BEGIN);
|
| 635 |
+
let mut reply = ScriptedStream::new(Vec::new());
|
| 636 |
+
assert_eq!(
|
| 637 |
+
worker_handle_snapshot_load(&mut reply, identity, &body, 32, &mut Vec::new()).unwrap_err(),
|
| 638 |
+
"snapshot load token hash mismatch"
|
| 639 |
+
);
|
| 640 |
+
let mut written = Cursor::new(reply.written);
|
| 641 |
+
let (typ, body) = read_frame(&mut written).unwrap();
|
| 642 |
+
assert_eq!(typ, MSG_SNAPSHOT_DONE);
|
| 643 |
+
let (done, message) = ds4_dist::decode_snapshot_done_body(&body, meta.request_id).unwrap();
|
| 644 |
+
assert_eq!(done.status, 1);
|
| 645 |
+
assert_eq!(message, b"snapshot load token hash mismatch");
|
| 646 |
+
}
|
crates/ds4-dist/tests/wire_oracle.rs
ADDED
|
@@ -0,0 +1,398 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! C↔Rust DS4D wire bytes. No `#[repr(C)]` on the Rust side.
|
| 2 |
+
|
| 3 |
+
use ds4_dist::{
|
| 4 |
+
decode_frame_header, decode_hello_payload, decode_snapshot_begin_body,
|
| 5 |
+
decode_snapshot_chunk_body, decode_snapshot_done_body, decode_snapshot_load_begin_body,
|
| 6 |
+
encode_activation, encode_frame_header, encode_hello_payload, encode_route_blob,
|
| 7 |
+
encode_snapshot_begin_body, encode_snapshot_chunk_body, encode_snapshot_done_body,
|
| 8 |
+
encode_tokens_be, f32_to_f16, f32_to_f8_e4m3, read_frame, token_hash_prefix,
|
| 9 |
+
validate_route_blob, write_frame, Hello, ResultHdr, ReturnTarget, RouteEntry, SnapshotBegin,
|
| 10 |
+
SnapshotChunk, SnapshotDone, SnapshotReq, Work, FRAME_HEADER_BYTES, HELLO_FIXED_BYTES, MAGIC,
|
| 11 |
+
MSG_HELLO, MSG_WORK, RESULT_FIXED_BYTES, ROUTE_RETURN_UPSTREAM, SNAPSHOT_CHUNK_BYTES,
|
| 12 |
+
WORK_FIXED_BYTES,
|
| 13 |
+
};
|
| 14 |
+
use std::path::PathBuf;
|
| 15 |
+
use std::process::Command;
|
| 16 |
+
|
| 17 |
+
fn oracle() -> PathBuf {
|
| 18 |
+
if let Ok(p) = std::env::var("DS4_DIST_C_ORACLE") {
|
| 19 |
+
return PathBuf::from(p);
|
| 20 |
+
}
|
| 21 |
+
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/dist_c_oracle")
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
fn require_oracle() -> PathBuf {
|
| 25 |
+
let p = oracle();
|
| 26 |
+
assert!(
|
| 27 |
+
p.exists(),
|
| 28 |
+
"build the C oracle first: make tests/parity/dist_c_oracle (missing {})",
|
| 29 |
+
p.display()
|
| 30 |
+
);
|
| 31 |
+
p
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
fn c_out(args: &[&str]) -> String {
|
| 35 |
+
let out = Command::new(require_oracle())
|
| 36 |
+
.args(args)
|
| 37 |
+
.output()
|
| 38 |
+
.expect("run dist_c_oracle");
|
| 39 |
+
assert!(
|
| 40 |
+
out.status.success(),
|
| 41 |
+
"oracle failed: {}",
|
| 42 |
+
String::from_utf8_lossy(&out.stderr)
|
| 43 |
+
);
|
| 44 |
+
String::from_utf8_lossy(&out.stdout).into_owned()
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
fn hex(bytes: &[u8]) -> String {
|
| 48 |
+
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
#[test]
|
| 52 |
+
fn struct_sizes_match_c() {
|
| 53 |
+
let line = c_out(&["sizes"]);
|
| 54 |
+
assert!(line.contains("frame 12"), "{line}");
|
| 55 |
+
assert!(line.contains("hello 40"), "{line}");
|
| 56 |
+
assert!(line.contains("work 80"), "{line}");
|
| 57 |
+
assert!(line.contains(&format!("work {WORK_FIXED_BYTES}")), "{line}");
|
| 58 |
+
assert_eq!(FRAME_HEADER_BYTES, 12);
|
| 59 |
+
assert_eq!(HELLO_FIXED_BYTES, 40);
|
| 60 |
+
assert_eq!(RESULT_FIXED_BYTES, 40);
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
#[test]
|
| 64 |
+
fn frame_header_matches_c() {
|
| 65 |
+
let rust = encode_frame_header(MSG_WORK, 80);
|
| 66 |
+
assert_eq!(hex(&rust), c_out(&["frame", "3", "80"]));
|
| 67 |
+
let h = decode_frame_header(&rust).unwrap();
|
| 68 |
+
assert_eq!(h.typ, MSG_WORK);
|
| 69 |
+
assert_eq!(h.bytes, 80);
|
| 70 |
+
assert_eq!(u32::from_be_bytes(rust[0..4].try_into().unwrap()), MAGIC);
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
#[test]
|
| 74 |
+
fn hello_matches_c() {
|
| 75 |
+
let rec = Hello {
|
| 76 |
+
model_id: 3,
|
| 77 |
+
quant_bits: 2,
|
| 78 |
+
layer_start: 2,
|
| 79 |
+
layer_end: 51,
|
| 80 |
+
has_output: 1,
|
| 81 |
+
has_hidden: 1,
|
| 82 |
+
ctx_size: 8192,
|
| 83 |
+
n_layers: 53,
|
| 84 |
+
listen_port: 7000,
|
| 85 |
+
model_name_len: 0,
|
| 86 |
+
};
|
| 87 |
+
let rust = encode_hello_payload(&rec, "motif-3").unwrap();
|
| 88 |
+
assert_eq!(
|
| 89 |
+
hex(&rust),
|
| 90 |
+
c_out(&["hello", "3", "2", "2", "51", "1", "1", "8192", "53", "7000", "motif-3"])
|
| 91 |
+
);
|
| 92 |
+
let (back, name) = decode_hello_payload(&rust).unwrap();
|
| 93 |
+
assert_eq!(name, "motif-3");
|
| 94 |
+
assert_eq!(back.model_id, 3);
|
| 95 |
+
assert_eq!(back.listen_port, 7000);
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
#[test]
|
| 99 |
+
fn work_and_tokens_match_c() {
|
| 100 |
+
let w = Work {
|
| 101 |
+
model_id: 3,
|
| 102 |
+
session_hi: 0,
|
| 103 |
+
session_lo: 9,
|
| 104 |
+
request_hi: 0,
|
| 105 |
+
request_lo: 11,
|
| 106 |
+
prefix_hash_hi: 1,
|
| 107 |
+
prefix_hash_lo: 2,
|
| 108 |
+
result_hash_hi: 3,
|
| 109 |
+
result_hash_lo: 4,
|
| 110 |
+
pos0: 16,
|
| 111 |
+
n_tokens: 3,
|
| 112 |
+
layer_start: 2,
|
| 113 |
+
layer_end: 10,
|
| 114 |
+
flags: 1,
|
| 115 |
+
token_bytes: 12,
|
| 116 |
+
input_hc_bytes: 0,
|
| 117 |
+
input_hc_bits: 32,
|
| 118 |
+
route_count: 1,
|
| 119 |
+
route_index: 0,
|
| 120 |
+
route_bytes: 0,
|
| 121 |
+
};
|
| 122 |
+
assert_eq!(
|
| 123 |
+
hex(&w.encode()),
|
| 124 |
+
c_out(&[
|
| 125 |
+
"work", "3", "0", "9", "0", "11", "1", "2", "3", "4", "16", "3", "2", "10", "1", "12",
|
| 126 |
+
"0", "32", "1", "0", "0"
|
| 127 |
+
])
|
| 128 |
+
);
|
| 129 |
+
let tokens = [7i32, -1, 256];
|
| 130 |
+
assert_eq!(
|
| 131 |
+
hex(&encode_tokens_be(&tokens)),
|
| 132 |
+
c_out(&["tokens", "7", "-1", "256"])
|
| 133 |
+
);
|
| 134 |
+
}
|
| 135 |
+
|
| 136 |
+
#[test]
|
| 137 |
+
fn token_hash_matches_c() {
|
| 138 |
+
assert_eq!(
|
| 139 |
+
format!("{:016x}", token_hash_prefix(&[1, 2, 3, 99])),
|
| 140 |
+
c_out(&["token-hash", "1", "2", "3", "99"])
|
| 141 |
+
);
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
#[test]
|
| 145 |
+
fn route_blob_round_trip() {
|
| 146 |
+
let entries = [RouteEntry {
|
| 147 |
+
host: "10.0.0.2".into(),
|
| 148 |
+
port: 7100,
|
| 149 |
+
layer_start: 2,
|
| 150 |
+
layer_end: 20,
|
| 151 |
+
flags: 0,
|
| 152 |
+
}];
|
| 153 |
+
let ret = ReturnTarget {
|
| 154 |
+
kind: ROUTE_RETURN_UPSTREAM,
|
| 155 |
+
host: String::new(),
|
| 156 |
+
port: 0,
|
| 157 |
+
};
|
| 158 |
+
let blob = encode_route_blob(&entries, &ret).unwrap();
|
| 159 |
+
assert_eq!(
|
| 160 |
+
hex(&blob[..20 + 8]),
|
| 161 |
+
c_out(&["route", "10.0.0.2", "7100", "2", "20", "0"])
|
| 162 |
+
);
|
| 163 |
+
validate_route_blob(&blob, 1, 53).unwrap();
|
| 164 |
+
}
|
| 165 |
+
|
| 166 |
+
#[test]
|
| 167 |
+
fn result_matches_c() {
|
| 168 |
+
let r = ResultHdr {
|
| 169 |
+
request_hi: 0,
|
| 170 |
+
request_lo: 11,
|
| 171 |
+
result_hash_hi: 3,
|
| 172 |
+
result_hash_lo: 4,
|
| 173 |
+
status: 0,
|
| 174 |
+
result_kind: 2,
|
| 175 |
+
telemetry_count: 1,
|
| 176 |
+
telemetry_bytes: 40,
|
| 177 |
+
payload_bytes: 16,
|
| 178 |
+
payload_bits: 32,
|
| 179 |
+
};
|
| 180 |
+
assert_eq!(
|
| 181 |
+
hex(&r.encode()),
|
| 182 |
+
c_out(&["result", "0", "11", "3", "4", "0", "2", "1", "40", "16", "32"])
|
| 183 |
+
);
|
| 184 |
+
}
|
| 185 |
+
|
| 186 |
+
#[test]
|
| 187 |
+
fn f16_f8_match_c() {
|
| 188 |
+
let samples = [
|
| 189 |
+
0x00000000u32,
|
| 190 |
+
0x3f800000,
|
| 191 |
+
0xbf800000,
|
| 192 |
+
0x3f000000,
|
| 193 |
+
0x7f800000,
|
| 194 |
+
0x7fc00000,
|
| 195 |
+
0x00000001,
|
| 196 |
+
0x3fc00000,
|
| 197 |
+
];
|
| 198 |
+
for bits in samples {
|
| 199 |
+
let hex_bits = format!("{bits:08x}");
|
| 200 |
+
let f = f32::from_bits(bits);
|
| 201 |
+
assert_eq!(
|
| 202 |
+
hex(&f32_to_f16(f).to_le_bytes()),
|
| 203 |
+
c_out(&["f16", &hex_bits]),
|
| 204 |
+
"f16 {hex_bits}"
|
| 205 |
+
);
|
| 206 |
+
assert_eq!(
|
| 207 |
+
hex(&[f32_to_f8_e4m3(f)]),
|
| 208 |
+
c_out(&["f8", &hex_bits]),
|
| 209 |
+
"f8 {hex_bits}"
|
| 210 |
+
);
|
| 211 |
+
}
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
#[test]
|
| 215 |
+
fn blocking_hello_round_trip() {
|
| 216 |
+
use std::net::{TcpListener, TcpStream};
|
| 217 |
+
use std::thread;
|
| 218 |
+
|
| 219 |
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
| 220 |
+
let addr = listener.local_addr().unwrap();
|
| 221 |
+
let rec = Hello {
|
| 222 |
+
model_id: 1,
|
| 223 |
+
quant_bits: 4,
|
| 224 |
+
layer_start: 0,
|
| 225 |
+
layer_end: 11,
|
| 226 |
+
has_output: 0,
|
| 227 |
+
has_hidden: 1,
|
| 228 |
+
ctx_size: 2048,
|
| 229 |
+
n_layers: 48,
|
| 230 |
+
listen_port: 9,
|
| 231 |
+
model_name_len: 0,
|
| 232 |
+
};
|
| 233 |
+
let payload = encode_hello_payload(&rec, "solar").unwrap();
|
| 234 |
+
let reply = payload.clone();
|
| 235 |
+
let server = thread::spawn(move || {
|
| 236 |
+
let (mut s, _) = listener.accept().unwrap();
|
| 237 |
+
let (typ, body) = read_frame(&mut s).unwrap();
|
| 238 |
+
assert_eq!(typ, MSG_HELLO);
|
| 239 |
+
let (h, name) = decode_hello_payload(&body).unwrap();
|
| 240 |
+
assert_eq!(name, "solar");
|
| 241 |
+
assert_eq!(h.n_layers, 48);
|
| 242 |
+
write_frame(&mut s, MSG_HELLO, &reply).unwrap();
|
| 243 |
+
});
|
| 244 |
+
let mut client = TcpStream::connect(addr).unwrap();
|
| 245 |
+
write_frame(&mut client, MSG_HELLO, &payload).unwrap();
|
| 246 |
+
let (typ, body) = read_frame(&mut client).unwrap();
|
| 247 |
+
assert_eq!(typ, MSG_HELLO);
|
| 248 |
+
let (_, name) = decode_hello_payload(&body).unwrap();
|
| 249 |
+
assert_eq!(name, "solar");
|
| 250 |
+
server.join().unwrap();
|
| 251 |
+
}
|
| 252 |
+
|
| 253 |
+
#[test]
|
| 254 |
+
fn activation_f32_is_le_host_layout() {
|
| 255 |
+
let src = [1.0f32, -2.5];
|
| 256 |
+
let wire = encode_activation(&src, 32).unwrap();
|
| 257 |
+
let mut expect = Vec::new();
|
| 258 |
+
expect.extend_from_slice(&1.0f32.to_bits().to_le_bytes());
|
| 259 |
+
expect.extend_from_slice(&(-2.5f32).to_bits().to_le_bytes());
|
| 260 |
+
assert_eq!(wire, expect);
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
#[test]
|
| 264 |
+
fn snapshot_records_match_c() {
|
| 265 |
+
let req = SnapshotReq {
|
| 266 |
+
model_id: 3,
|
| 267 |
+
session_hi: 1,
|
| 268 |
+
session_lo: 2,
|
| 269 |
+
request_hi: 3,
|
| 270 |
+
request_lo: 4,
|
| 271 |
+
token_hash_hi: 5,
|
| 272 |
+
token_hash_lo: 6,
|
| 273 |
+
token_count: 7,
|
| 274 |
+
layer_start: 8,
|
| 275 |
+
layer_end: 9,
|
| 276 |
+
};
|
| 277 |
+
assert_eq!(
|
| 278 |
+
hex(&req.encode()),
|
| 279 |
+
c_out(&[
|
| 280 |
+
"snapshot-req",
|
| 281 |
+
"3",
|
| 282 |
+
"1",
|
| 283 |
+
"2",
|
| 284 |
+
"3",
|
| 285 |
+
"4",
|
| 286 |
+
"5",
|
| 287 |
+
"6",
|
| 288 |
+
"7",
|
| 289 |
+
"8",
|
| 290 |
+
"9"
|
| 291 |
+
])
|
| 292 |
+
);
|
| 293 |
+
|
| 294 |
+
let begin = SnapshotBegin {
|
| 295 |
+
model_id: 3,
|
| 296 |
+
session_hi: 1,
|
| 297 |
+
session_lo: 2,
|
| 298 |
+
request_hi: 3,
|
| 299 |
+
request_lo: 4,
|
| 300 |
+
token_hash_hi: 5,
|
| 301 |
+
token_hash_lo: 6,
|
| 302 |
+
token_count: 2,
|
| 303 |
+
layer_start: 8,
|
| 304 |
+
layer_end: 9,
|
| 305 |
+
payload_hi: 10,
|
| 306 |
+
payload_lo: 11,
|
| 307 |
+
status: 0,
|
| 308 |
+
token_bytes: 8,
|
| 309 |
+
message_bytes: 0,
|
| 310 |
+
};
|
| 311 |
+
assert_eq!(
|
| 312 |
+
hex(&begin.encode()),
|
| 313 |
+
c_out(&[
|
| 314 |
+
"snapshot-begin",
|
| 315 |
+
"3",
|
| 316 |
+
"1",
|
| 317 |
+
"2",
|
| 318 |
+
"3",
|
| 319 |
+
"4",
|
| 320 |
+
"5",
|
| 321 |
+
"6",
|
| 322 |
+
"2",
|
| 323 |
+
"8",
|
| 324 |
+
"9",
|
| 325 |
+
"10",
|
| 326 |
+
"11",
|
| 327 |
+
"0",
|
| 328 |
+
"8",
|
| 329 |
+
"0"
|
| 330 |
+
])
|
| 331 |
+
);
|
| 332 |
+
|
| 333 |
+
let chunk = SnapshotChunk {
|
| 334 |
+
request_hi: 3,
|
| 335 |
+
request_lo: 4,
|
| 336 |
+
chunk_bytes: 5,
|
| 337 |
+
};
|
| 338 |
+
assert_eq!(
|
| 339 |
+
hex(&chunk.encode()),
|
| 340 |
+
c_out(&["snapshot-chunk", "3", "4", "5"])
|
| 341 |
+
);
|
| 342 |
+
|
| 343 |
+
let done = SnapshotDone {
|
| 344 |
+
request_hi: 3,
|
| 345 |
+
request_lo: 4,
|
| 346 |
+
status: 1,
|
| 347 |
+
message_bytes: 5,
|
| 348 |
+
};
|
| 349 |
+
assert_eq!(
|
| 350 |
+
hex(&done.encode()),
|
| 351 |
+
c_out(&["snapshot-done", "3", "4", "1", "5"])
|
| 352 |
+
);
|
| 353 |
+
}
|
| 354 |
+
|
| 355 |
+
#[test]
|
| 356 |
+
fn snapshot_bodies_enforce_c_lengths_and_ids() {
|
| 357 |
+
let begin = SnapshotBegin {
|
| 358 |
+
model_id: 3,
|
| 359 |
+
session_hi: 0,
|
| 360 |
+
session_lo: 2,
|
| 361 |
+
request_hi: 0,
|
| 362 |
+
request_lo: 4,
|
| 363 |
+
token_hash_hi: 0,
|
| 364 |
+
token_hash_lo: 6,
|
| 365 |
+
token_count: 2,
|
| 366 |
+
layer_start: 8,
|
| 367 |
+
layer_end: 9,
|
| 368 |
+
payload_hi: 0,
|
| 369 |
+
payload_lo: 5,
|
| 370 |
+
status: 0,
|
| 371 |
+
token_bytes: 8,
|
| 372 |
+
message_bytes: 0,
|
| 373 |
+
};
|
| 374 |
+
let body = encode_snapshot_begin_body(&begin, &[7, 8], b"").unwrap();
|
| 375 |
+
let (back, tokens, message) = decode_snapshot_load_begin_body(&body).unwrap();
|
| 376 |
+
assert_eq!(back, begin);
|
| 377 |
+
assert_eq!(tokens, [7, 8]);
|
| 378 |
+
assert!(message.is_empty());
|
| 379 |
+
|
| 380 |
+
let mut bad = begin;
|
| 381 |
+
bad.token_bytes = 4;
|
| 382 |
+
assert!(encode_snapshot_begin_body(&bad, &[7, 8], b"").is_err());
|
| 383 |
+
assert!(decode_snapshot_begin_body(&body[..body.len() - 1]).is_err());
|
| 384 |
+
|
| 385 |
+
let payload = b"kv-shard";
|
| 386 |
+
let chunk = encode_snapshot_chunk_body(4, payload).unwrap();
|
| 387 |
+
let (_, decoded) = decode_snapshot_chunk_body(&chunk, 4, payload.len() as u64).unwrap();
|
| 388 |
+
assert_eq!(decoded, payload);
|
| 389 |
+
assert!(decode_snapshot_chunk_body(&chunk, 5, payload.len() as u64).is_err());
|
| 390 |
+
assert!(decode_snapshot_chunk_body(&chunk, 4, (payload.len() - 1) as u64).is_err());
|
| 391 |
+
assert!(encode_snapshot_chunk_body(4, &vec![0; SNAPSHOT_CHUNK_BYTES + 1]).is_err());
|
| 392 |
+
|
| 393 |
+
let done = encode_snapshot_done_body(4, 1, b"failed").unwrap();
|
| 394 |
+
let (header, message) = decode_snapshot_done_body(&done, 4).unwrap();
|
| 395 |
+
assert_eq!(header.status, 1);
|
| 396 |
+
assert_eq!(message, b"failed");
|
| 397 |
+
assert!(decode_snapshot_done_body(&done, 5).is_err());
|
| 398 |
+
}
|
crates/ds4-kv/Cargo.toml
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[package]
|
| 2 |
+
name = "ds4-kv"
|
| 3 |
+
version.workspace = true
|
| 4 |
+
edition.workspace = true
|
| 5 |
+
license.workspace = true
|
| 6 |
+
publish.workspace = true
|
| 7 |
+
description = "KVC checkpoint store (Phase 4)"
|
| 8 |
+
|
| 9 |
+
[lib]
|
| 10 |
+
path = "src/lib.rs"
|
crates/ds4-kv/src/extensions.rs
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Bank thinking bit + KTM extension records.
|
| 2 |
+
//!
|
| 3 |
+
//! Thinking is `EXT_THINKING_VISIBLE` on the KVC header (no extra magic).
|
| 4 |
+
//! The only appended trailer magic is C's `KTM\x01`. A missing or invalid
|
| 5 |
+
//! section is ignored the way `kv_tool_map_load_from_pos` returns 0.
|
| 6 |
+
|
| 7 |
+
use crate::format::{
|
| 8 |
+
Record, EXT_BANK_REPLAY_V1, EXT_IMAGE_PIXELS_V2, EXT_RESPONSES_VISIBLE, EXT_SESSION_TITLE,
|
| 9 |
+
EXT_THINKING_VISIBLE, EXT_TOOL_MAP,
|
| 10 |
+
};
|
| 11 |
+
|
| 12 |
+
pub const KTM_MAGIC: [u8; 3] = [b'K', b'T', b'M'];
|
| 13 |
+
pub const KTM_VERSION: u8 = 1;
|
| 14 |
+
const KTM_HEADER: [u8; 4] = [b'K', b'T', b'M', KTM_VERSION];
|
| 15 |
+
const DEFAULT_MAX_IDS: usize = 100_000;
|
| 16 |
+
const MAX_ID_BYTES: usize = 256;
|
| 17 |
+
const MAX_DSML_BYTES: usize = 512 * 1024 * 1024;
|
| 18 |
+
|
| 19 |
+
#[derive(Clone, Debug, PartialEq, Eq)]
|
| 20 |
+
pub struct ExtensionRecord {
|
| 21 |
+
pub id: Vec<u8>,
|
| 22 |
+
pub body: Vec<u8>,
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
| 26 |
+
pub struct BankThinkingExtensions {
|
| 27 |
+
pub thinking_visible: bool,
|
| 28 |
+
pub records: Vec<ExtensionRecord>,
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
impl BankThinkingExtensions {
|
| 32 |
+
pub fn ext_flags(&self) -> u8 {
|
| 33 |
+
self.wire(0).0
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
pub fn encode_trailer(&self) -> Vec<u8> {
|
| 37 |
+
self.wire(0).1
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
fn wire(&self, keep: u8) -> (u8, Vec<u8>) {
|
| 41 |
+
let trailer = encode_ktm(&self.records);
|
| 42 |
+
(
|
| 43 |
+
bank_persist_ext_flags(keep, self.thinking_visible, !trailer.is_empty()),
|
| 44 |
+
trailer,
|
| 45 |
+
)
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
pub fn restore(ext_flags: u8, trailer: &[u8]) -> Self {
|
| 49 |
+
Self {
|
| 50 |
+
thinking_visible: ext_flags & EXT_THINKING_VISIBLE != 0,
|
| 51 |
+
records: decode_ktm(trailer),
|
| 52 |
+
}
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
impl Record {
|
| 57 |
+
pub fn persist_bank_extensions(&mut self, ext: &BankThinkingExtensions) {
|
| 58 |
+
let (flags, trailer) = ext.wire(self.header.ext_flags);
|
| 59 |
+
self.header.ext_flags = flags;
|
| 60 |
+
self.trailer = trailer;
|
| 61 |
+
self.header.text_bytes = self.text.len() as u32;
|
| 62 |
+
self.header.payload_bytes = self.payload.len() as u64;
|
| 63 |
+
}
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
/// Bank persist always sets `EXT_BANK_REPLAY_V1`. Responses/title bits on
|
| 67 |
+
/// the incoming header survive; thinking and KTM are applied from `ext`.
|
| 68 |
+
pub fn bank_persist_ext_flags(keep: u8, thinking_visible: bool, has_trailer: bool) -> u8 {
|
| 69 |
+
let mut flags = EXT_BANK_REPLAY_V1
|
| 70 |
+
| (keep & (EXT_RESPONSES_VISIBLE | EXT_SESSION_TITLE | EXT_IMAGE_PIXELS_V2));
|
| 71 |
+
if thinking_visible {
|
| 72 |
+
flags |= EXT_THINKING_VISIBLE;
|
| 73 |
+
}
|
| 74 |
+
if has_trailer {
|
| 75 |
+
flags |= EXT_TOOL_MAP;
|
| 76 |
+
}
|
| 77 |
+
flags
|
| 78 |
+
}
|
| 79 |
+
|
| 80 |
+
fn wire_ok(id: &[u8], body: &[u8]) -> bool {
|
| 81 |
+
!id.is_empty()
|
| 82 |
+
&& !body.is_empty()
|
| 83 |
+
&& id.len() <= MAX_ID_BYTES
|
| 84 |
+
&& body.len() <= MAX_DSML_BYTES
|
| 85 |
+
&& !id.contains(&0)
|
| 86 |
+
&& !body.contains(&0)
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
pub fn encode_ktm(records: &[ExtensionRecord]) -> Vec<u8> {
|
| 90 |
+
let kept: Vec<&ExtensionRecord> = records
|
| 91 |
+
.iter()
|
| 92 |
+
.filter(|record| wire_ok(&record.id, &record.body))
|
| 93 |
+
.collect();
|
| 94 |
+
if kept.is_empty() {
|
| 95 |
+
return Vec::new();
|
| 96 |
+
}
|
| 97 |
+
let Ok(count) = u32::try_from(kept.len()) else {
|
| 98 |
+
return Vec::new();
|
| 99 |
+
};
|
| 100 |
+
let mut out = Vec::from(KTM_HEADER);
|
| 101 |
+
out.extend_from_slice(&count.to_le_bytes());
|
| 102 |
+
for record in kept {
|
| 103 |
+
let id_len = record.id.len() as u32;
|
| 104 |
+
let body_len = record.body.len() as u32;
|
| 105 |
+
out.extend_from_slice(&id_len.to_le_bytes());
|
| 106 |
+
out.extend_from_slice(&body_len.to_le_bytes());
|
| 107 |
+
out.extend_from_slice(&record.id);
|
| 108 |
+
out.extend_from_slice(&record.body);
|
| 109 |
+
}
|
| 110 |
+
out
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
pub fn decode_ktm(bytes: &[u8]) -> Vec<ExtensionRecord> {
|
| 114 |
+
decode_ktm_bounded(bytes, DEFAULT_MAX_IDS)
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
fn decode_ktm_bounded(bytes: &[u8], max_ids: usize) -> Vec<ExtensionRecord> {
|
| 118 |
+
if bytes.len() < 8 || bytes[..4] != KTM_HEADER {
|
| 119 |
+
return Vec::new();
|
| 120 |
+
}
|
| 121 |
+
let count = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]);
|
| 122 |
+
let max_ids = if max_ids == 0 {
|
| 123 |
+
DEFAULT_MAX_IDS
|
| 124 |
+
} else {
|
| 125 |
+
max_ids
|
| 126 |
+
};
|
| 127 |
+
let max_count = (max_ids as u64).saturating_mul(4);
|
| 128 |
+
if u64::from(count) > max_count {
|
| 129 |
+
return Vec::new();
|
| 130 |
+
}
|
| 131 |
+
let mut records = Vec::new();
|
| 132 |
+
let mut pos = 8usize;
|
| 133 |
+
for _ in 0..count {
|
| 134 |
+
let Some(lens_end) = pos.checked_add(8).filter(|&end| end <= bytes.len()) else {
|
| 135 |
+
return records;
|
| 136 |
+
};
|
| 137 |
+
let id_len =
|
| 138 |
+
u32::from_le_bytes([bytes[pos], bytes[pos + 1], bytes[pos + 2], bytes[pos + 3]])
|
| 139 |
+
as usize;
|
| 140 |
+
let body_len = u32::from_le_bytes([
|
| 141 |
+
bytes[pos + 4],
|
| 142 |
+
bytes[pos + 5],
|
| 143 |
+
bytes[pos + 6],
|
| 144 |
+
bytes[pos + 7],
|
| 145 |
+
]) as usize;
|
| 146 |
+
if id_len == 0 || id_len > MAX_ID_BYTES || body_len == 0 || body_len > MAX_DSML_BYTES {
|
| 147 |
+
return records;
|
| 148 |
+
}
|
| 149 |
+
pos = lens_end;
|
| 150 |
+
let Some(id_end) = pos.checked_add(id_len).filter(|&end| end <= bytes.len()) else {
|
| 151 |
+
return records;
|
| 152 |
+
};
|
| 153 |
+
let Some(body_end) = id_end
|
| 154 |
+
.checked_add(body_len)
|
| 155 |
+
.filter(|&end| end <= bytes.len())
|
| 156 |
+
else {
|
| 157 |
+
return records;
|
| 158 |
+
};
|
| 159 |
+
let id = until_nul(&bytes[pos..id_end]);
|
| 160 |
+
let body = until_nul(&bytes[id_end..body_end]);
|
| 161 |
+
if !id.is_empty() && !body.is_empty() {
|
| 162 |
+
records.push(ExtensionRecord {
|
| 163 |
+
id: id.to_vec(),
|
| 164 |
+
body: body.to_vec(),
|
| 165 |
+
});
|
| 166 |
+
}
|
| 167 |
+
pos = body_end;
|
| 168 |
+
}
|
| 169 |
+
records
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
fn until_nul(bytes: &[u8]) -> &[u8] {
|
| 173 |
+
match bytes.iter().position(|&b| b == 0) {
|
| 174 |
+
Some(i) => &bytes[..i],
|
| 175 |
+
None => bytes,
|
| 176 |
+
}
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
#[cfg(test)]
|
| 180 |
+
mod tests {
|
| 181 |
+
use super::*;
|
| 182 |
+
|
| 183 |
+
#[test]
|
| 184 |
+
fn empty_records_write_no_ktm_header() {
|
| 185 |
+
let ext = BankThinkingExtensions {
|
| 186 |
+
thinking_visible: true,
|
| 187 |
+
records: Vec::new(),
|
| 188 |
+
};
|
| 189 |
+
assert_eq!(ext.ext_flags(), EXT_BANK_REPLAY_V1 | EXT_THINKING_VISIBLE);
|
| 190 |
+
assert!(ext.encode_trailer().is_empty());
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
#[test]
|
| 194 |
+
fn invalid_records_do_not_set_tool_map() {
|
| 195 |
+
let ext = BankThinkingExtensions {
|
| 196 |
+
thinking_visible: false,
|
| 197 |
+
records: vec![ExtensionRecord {
|
| 198 |
+
id: Vec::new(),
|
| 199 |
+
body: b"x".to_vec(),
|
| 200 |
+
}],
|
| 201 |
+
};
|
| 202 |
+
assert_eq!(ext.ext_flags(), EXT_BANK_REPLAY_V1);
|
| 203 |
+
assert!(ext.encode_trailer().is_empty());
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
#[test]
|
| 207 |
+
fn persist_bank_extensions_keeps_identity_responses_and_title() {
|
| 208 |
+
// Given: a bank record that already carries Responses/title bits
|
| 209 |
+
let mut rec = Record {
|
| 210 |
+
header: crate::format::Header {
|
| 211 |
+
quant_bits: 2,
|
| 212 |
+
reason: crate::format::Reason::BankEvict,
|
| 213 |
+
ext_flags: EXT_RESPONSES_VISIBLE | EXT_SESSION_TITLE | EXT_IMAGE_PIXELS_V2,
|
| 214 |
+
model_id: 1,
|
| 215 |
+
tokens: 8,
|
| 216 |
+
hits: 0,
|
| 217 |
+
ctx_size: 4096,
|
| 218 |
+
created_at: 1,
|
| 219 |
+
last_used: 1,
|
| 220 |
+
payload_bytes: 1,
|
| 221 |
+
text_bytes: 4,
|
| 222 |
+
},
|
| 223 |
+
text: b"keep".to_vec(),
|
| 224 |
+
payload: b"P".to_vec(),
|
| 225 |
+
trailer: Vec::new(),
|
| 226 |
+
};
|
| 227 |
+
let ext = BankThinkingExtensions {
|
| 228 |
+
thinking_visible: true,
|
| 229 |
+
records: Vec::new(),
|
| 230 |
+
};
|
| 231 |
+
|
| 232 |
+
// When: thinking/KTM extensions are persisted onto that record
|
| 233 |
+
rec.persist_bank_extensions(&ext);
|
| 234 |
+
|
| 235 |
+
// Then: BANK_REPLAY_V1 + thinking land, and the two prior bits survive
|
| 236 |
+
assert_eq!(
|
| 237 |
+
rec.header.ext_flags,
|
| 238 |
+
EXT_BANK_REPLAY_V1
|
| 239 |
+
| EXT_THINKING_VISIBLE
|
| 240 |
+
| EXT_RESPONSES_VISIBLE
|
| 241 |
+
| EXT_SESSION_TITLE
|
| 242 |
+
| EXT_IMAGE_PIXELS_V2
|
| 243 |
+
);
|
| 244 |
+
}
|
| 245 |
+
}
|
crates/ds4-kv/src/format.rs
ADDED
|
@@ -0,0 +1,781 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! KVC file envelope: 48-byte header + text length + text + payload.
|
| 2 |
+
//!
|
| 3 |
+
//! Matches `ds4_kvstore_fill_header` / `ds4_kvstore_read_header` and the
|
| 4 |
+
//! store fwrite order in `ds4_kvstore.c`. Do not invent a new format.
|
| 5 |
+
|
| 6 |
+
use crate::sha1::sha1_hex;
|
| 7 |
+
use std::fs;
|
| 8 |
+
use std::io::{self, Read, Seek, SeekFrom, Write};
|
| 9 |
+
use std::path::{Path, PathBuf};
|
| 10 |
+
use std::sync::atomic::{AtomicU64, Ordering};
|
| 11 |
+
|
| 12 |
+
pub const FIXED_HEADER: usize = 48;
|
| 13 |
+
pub const MAGIC: [u8; 3] = [b'K', b'V', b'C'];
|
| 14 |
+
pub const VERSION: u8 = 1;
|
| 15 |
+
pub const PAYLOAD_ABI: u8 = 2;
|
| 16 |
+
|
| 17 |
+
static TEMP_SEQ: AtomicU64 = AtomicU64::new(0);
|
| 18 |
+
|
| 19 |
+
pub const EXT_TOOL_MAP: u8 = 1 << 0;
|
| 20 |
+
pub const EXT_RESPONSES_VISIBLE: u8 = 1 << 1;
|
| 21 |
+
pub const EXT_THINKING_VISIBLE: u8 = 1 << 2;
|
| 22 |
+
pub const EXT_SESSION_TITLE: u8 = 1 << 3;
|
| 23 |
+
pub const EXT_BANK_REPLAY_V1: u8 = 1 << 4;
|
| 24 |
+
pub const EXT_IMAGE_PIXELS_V2: u8 = 1 << 5;
|
| 25 |
+
|
| 26 |
+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
| 27 |
+
#[repr(u8)]
|
| 28 |
+
pub enum Reason {
|
| 29 |
+
Unknown = 0,
|
| 30 |
+
Cold = 1,
|
| 31 |
+
Continued = 2,
|
| 32 |
+
Evict = 3,
|
| 33 |
+
Shutdown = 4,
|
| 34 |
+
AgentSystem = 5,
|
| 35 |
+
AgentSession = 6,
|
| 36 |
+
BankEvict = 7,
|
| 37 |
+
BankShutdown = 8,
|
| 38 |
+
BankCheckpoint = 9,
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
impl Reason {
|
| 42 |
+
pub fn from_u8(v: u8) -> Self {
|
| 43 |
+
match v {
|
| 44 |
+
1 => Self::Cold,
|
| 45 |
+
2 => Self::Continued,
|
| 46 |
+
3 => Self::Evict,
|
| 47 |
+
4 => Self::Shutdown,
|
| 48 |
+
5 => Self::AgentSystem,
|
| 49 |
+
6 => Self::AgentSession,
|
| 50 |
+
7 => Self::BankEvict,
|
| 51 |
+
8 => Self::BankShutdown,
|
| 52 |
+
9 => Self::BankCheckpoint,
|
| 53 |
+
_ => Self::Unknown,
|
| 54 |
+
}
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
pub fn from_name(name: &str) -> Self {
|
| 58 |
+
match name {
|
| 59 |
+
"cold" => Self::Cold,
|
| 60 |
+
"continued" => Self::Continued,
|
| 61 |
+
"evict" => Self::Evict,
|
| 62 |
+
"shutdown" => Self::Shutdown,
|
| 63 |
+
"agent-system" => Self::AgentSystem,
|
| 64 |
+
"agent-session" => Self::AgentSession,
|
| 65 |
+
"bank-evict" => Self::BankEvict,
|
| 66 |
+
"bank-shutdown" => Self::BankShutdown,
|
| 67 |
+
"bank-checkpoint" => Self::BankCheckpoint,
|
| 68 |
+
_ => Self::Unknown,
|
| 69 |
+
}
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
pub fn family(self) -> u8 {
|
| 73 |
+
match self as u8 {
|
| 74 |
+
1..=4 => 1,
|
| 75 |
+
5..=6 => 2,
|
| 76 |
+
7..=9 => 3,
|
| 77 |
+
_ => 0,
|
| 78 |
+
}
|
| 79 |
+
}
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
#[derive(Clone, Debug, PartialEq, Eq)]
|
| 83 |
+
pub struct Header {
|
| 84 |
+
pub quant_bits: u8,
|
| 85 |
+
pub reason: Reason,
|
| 86 |
+
pub ext_flags: u8,
|
| 87 |
+
pub model_id: u8,
|
| 88 |
+
pub tokens: u32,
|
| 89 |
+
pub hits: u32,
|
| 90 |
+
pub ctx_size: u32,
|
| 91 |
+
pub created_at: u64,
|
| 92 |
+
pub last_used: u64,
|
| 93 |
+
pub payload_bytes: u64,
|
| 94 |
+
pub text_bytes: u32,
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
#[derive(Clone, Debug, PartialEq, Eq)]
|
| 98 |
+
pub struct Record {
|
| 99 |
+
pub header: Header,
|
| 100 |
+
pub text: Vec<u8>,
|
| 101 |
+
pub payload: Vec<u8>,
|
| 102 |
+
pub trailer: Vec<u8>,
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
#[derive(Clone, Debug, PartialEq, Eq)]
|
| 106 |
+
pub(crate) struct Metadata {
|
| 107 |
+
pub(crate) header: Header,
|
| 108 |
+
pub(crate) payload_offset: u64,
|
| 109 |
+
pub(crate) trailer_bytes: u64,
|
| 110 |
+
pub(crate) file_size: u64,
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
#[derive(Clone, Debug, PartialEq, Eq)]
|
| 114 |
+
pub struct Envelope {
|
| 115 |
+
pub header: Header,
|
| 116 |
+
pub text: Vec<u8>,
|
| 117 |
+
pub payload_offset: u64,
|
| 118 |
+
pub trailer_bytes: u64,
|
| 119 |
+
pub file_size: u64,
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
#[derive(Debug)]
|
| 123 |
+
pub enum FormatError {
|
| 124 |
+
Io(io::Error),
|
| 125 |
+
BadMagic,
|
| 126 |
+
BadVersion,
|
| 127 |
+
BadPayloadAbi,
|
| 128 |
+
BadQuant,
|
| 129 |
+
ZeroTokens,
|
| 130 |
+
Truncated,
|
| 131 |
+
TextLenMismatch,
|
| 132 |
+
TrailerTooLarge,
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
impl From<io::Error> for FormatError {
|
| 136 |
+
fn from(e: io::Error) -> Self {
|
| 137 |
+
Self::Io(e)
|
| 138 |
+
}
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
+
impl std::fmt::Display for FormatError {
|
| 142 |
+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
| 143 |
+
match self {
|
| 144 |
+
Self::Io(e) => write!(f, "{e}"),
|
| 145 |
+
Self::BadMagic => write!(f, "not a KVC file"),
|
| 146 |
+
Self::BadVersion => write!(f, "unsupported KVC version"),
|
| 147 |
+
Self::BadPayloadAbi => write!(f, "unsupported KVC payload ABI"),
|
| 148 |
+
Self::BadQuant => write!(f, "quant_bits must be 2 or 4"),
|
| 149 |
+
Self::ZeroTokens => write!(f, "token count is zero"),
|
| 150 |
+
Self::Truncated => write!(f, "truncated KVC file"),
|
| 151 |
+
Self::TextLenMismatch => write!(f, "text length does not match header"),
|
| 152 |
+
Self::TrailerTooLarge => write!(f, "KVC trailer exceeds read bound"),
|
| 153 |
+
}
|
| 154 |
+
}
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
impl std::error::Error for FormatError {}
|
| 158 |
+
|
| 159 |
+
pub fn le_put32(p: &mut [u8], v: u32) {
|
| 160 |
+
p[0] = v as u8;
|
| 161 |
+
p[1] = (v >> 8) as u8;
|
| 162 |
+
p[2] = (v >> 16) as u8;
|
| 163 |
+
p[3] = (v >> 24) as u8;
|
| 164 |
+
}
|
| 165 |
+
|
| 166 |
+
pub fn le_get32(p: &[u8]) -> u32 {
|
| 167 |
+
u32::from(p[0]) | (u32::from(p[1]) << 8) | (u32::from(p[2]) << 16) | (u32::from(p[3]) << 24)
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
pub fn le_put64(p: &mut [u8], v: u64) {
|
| 171 |
+
for i in 0..8 {
|
| 172 |
+
p[i] = (v >> (8 * i)) as u8;
|
| 173 |
+
}
|
| 174 |
+
}
|
| 175 |
+
|
| 176 |
+
pub fn le_get64(p: &[u8]) -> u64 {
|
| 177 |
+
let mut v = 0u64;
|
| 178 |
+
for i in (0..8).rev() {
|
| 179 |
+
v = (v << 8) | u64::from(p[i]);
|
| 180 |
+
}
|
| 181 |
+
v
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
pub fn fill_header(
|
| 185 |
+
model_id: u8,
|
| 186 |
+
quant_bits: u8,
|
| 187 |
+
reason: Reason,
|
| 188 |
+
ext_flags: u8,
|
| 189 |
+
tokens: u32,
|
| 190 |
+
hits: u32,
|
| 191 |
+
ctx_size: u32,
|
| 192 |
+
created_at: u64,
|
| 193 |
+
last_used: u64,
|
| 194 |
+
payload_bytes: u64,
|
| 195 |
+
) -> [u8; FIXED_HEADER] {
|
| 196 |
+
let mut h = [0u8; FIXED_HEADER];
|
| 197 |
+
h[0] = MAGIC[0];
|
| 198 |
+
h[1] = MAGIC[1];
|
| 199 |
+
h[2] = MAGIC[2];
|
| 200 |
+
h[3] = VERSION;
|
| 201 |
+
h[4] = quant_bits;
|
| 202 |
+
h[5] = reason as u8;
|
| 203 |
+
h[6] = ext_flags;
|
| 204 |
+
h[7] = model_id;
|
| 205 |
+
le_put32(&mut h[8..12], tokens);
|
| 206 |
+
le_put32(&mut h[12..16], hits);
|
| 207 |
+
le_put32(&mut h[16..20], ctx_size);
|
| 208 |
+
h[20] = PAYLOAD_ABI;
|
| 209 |
+
le_put64(&mut h[24..32], created_at);
|
| 210 |
+
le_put64(&mut h[32..40], last_used);
|
| 211 |
+
le_put64(&mut h[40..48], payload_bytes);
|
| 212 |
+
h
|
| 213 |
+
}
|
| 214 |
+
|
| 215 |
+
pub fn parse_header(h: &[u8; FIXED_HEADER], text_bytes: u32) -> Result<Header, FormatError> {
|
| 216 |
+
if h[0] != MAGIC[0] || h[1] != MAGIC[1] || h[2] != MAGIC[2] {
|
| 217 |
+
return Err(FormatError::BadMagic);
|
| 218 |
+
}
|
| 219 |
+
if h[3] != VERSION {
|
| 220 |
+
return Err(FormatError::BadVersion);
|
| 221 |
+
}
|
| 222 |
+
if h[20] != PAYLOAD_ABI {
|
| 223 |
+
return Err(FormatError::BadPayloadAbi);
|
| 224 |
+
}
|
| 225 |
+
let tokens = le_get32(&h[8..12]);
|
| 226 |
+
let quant_bits = h[4];
|
| 227 |
+
if tokens == 0 {
|
| 228 |
+
return Err(FormatError::ZeroTokens);
|
| 229 |
+
}
|
| 230 |
+
if quant_bits != 2 && quant_bits != 4 {
|
| 231 |
+
return Err(FormatError::BadQuant);
|
| 232 |
+
}
|
| 233 |
+
Ok(Header {
|
| 234 |
+
quant_bits,
|
| 235 |
+
reason: Reason::from_u8(h[5]),
|
| 236 |
+
ext_flags: h[6],
|
| 237 |
+
model_id: h[7],
|
| 238 |
+
tokens,
|
| 239 |
+
hits: le_get32(&h[12..16]),
|
| 240 |
+
ctx_size: le_get32(&h[16..20]),
|
| 241 |
+
created_at: le_get64(&h[24..32]),
|
| 242 |
+
last_used: le_get64(&h[32..40]),
|
| 243 |
+
payload_bytes: le_get64(&h[40..48]),
|
| 244 |
+
text_bytes,
|
| 245 |
+
})
|
| 246 |
+
}
|
| 247 |
+
|
| 248 |
+
pub fn encode_file(record: &Record) -> Vec<u8> {
|
| 249 |
+
let mut h = fill_header(
|
| 250 |
+
record.header.model_id,
|
| 251 |
+
record.header.quant_bits,
|
| 252 |
+
record.header.reason,
|
| 253 |
+
record.header.ext_flags,
|
| 254 |
+
record.header.tokens,
|
| 255 |
+
record.header.hits,
|
| 256 |
+
record.header.ctx_size,
|
| 257 |
+
record.header.created_at,
|
| 258 |
+
record.header.last_used,
|
| 259 |
+
record.payload.len() as u64,
|
| 260 |
+
);
|
| 261 |
+
// payload_bytes in the header follows the payload vector, matching C.
|
| 262 |
+
le_put64(&mut h[40..48], record.payload.len() as u64);
|
| 263 |
+
let mut out = Vec::with_capacity(
|
| 264 |
+
FIXED_HEADER + 4 + record.text.len() + record.payload.len() + record.trailer.len(),
|
| 265 |
+
);
|
| 266 |
+
out.extend_from_slice(&h);
|
| 267 |
+
let mut tb = [0u8; 4];
|
| 268 |
+
le_put32(&mut tb, record.text.len() as u32);
|
| 269 |
+
out.extend_from_slice(&tb);
|
| 270 |
+
out.extend_from_slice(&record.text);
|
| 271 |
+
out.extend_from_slice(&record.payload);
|
| 272 |
+
out.extend_from_slice(&record.trailer);
|
| 273 |
+
out
|
| 274 |
+
}
|
| 275 |
+
|
| 276 |
+
pub fn decode_file(bytes: &[u8]) -> Result<Record, FormatError> {
|
| 277 |
+
if bytes.len() < FIXED_HEADER + 4 {
|
| 278 |
+
return Err(FormatError::Truncated);
|
| 279 |
+
}
|
| 280 |
+
let mut hdr = [0u8; FIXED_HEADER];
|
| 281 |
+
hdr.copy_from_slice(&bytes[..FIXED_HEADER]);
|
| 282 |
+
let text_bytes = le_get32(&bytes[FIXED_HEADER..FIXED_HEADER + 4]);
|
| 283 |
+
let header = parse_header(&hdr, text_bytes)?;
|
| 284 |
+
let text_start = FIXED_HEADER + 4;
|
| 285 |
+
let text_end = text_start
|
| 286 |
+
.checked_add(text_bytes as usize)
|
| 287 |
+
.ok_or(FormatError::Truncated)?;
|
| 288 |
+
let payload_end = text_end
|
| 289 |
+
.checked_add(header.payload_bytes as usize)
|
| 290 |
+
.ok_or(FormatError::Truncated)?;
|
| 291 |
+
if bytes.len() < payload_end {
|
| 292 |
+
return Err(FormatError::Truncated);
|
| 293 |
+
}
|
| 294 |
+
Ok(Record {
|
| 295 |
+
header,
|
| 296 |
+
text: bytes[text_start..text_end].to_vec(),
|
| 297 |
+
payload: bytes[text_end..payload_end].to_vec(),
|
| 298 |
+
trailer: bytes[payload_end..].to_vec(),
|
| 299 |
+
})
|
| 300 |
+
}
|
| 301 |
+
|
| 302 |
+
pub fn write_path(path: &Path, record: &Record) -> Result<(), FormatError> {
|
| 303 |
+
let bytes = encode_file(record);
|
| 304 |
+
if let Some(parent) = path.parent() {
|
| 305 |
+
fs::create_dir_all(parent)?;
|
| 306 |
+
}
|
| 307 |
+
let tmp = path.with_extension("kv.tmp");
|
| 308 |
+
{
|
| 309 |
+
let mut f = fs::File::create(&tmp)?;
|
| 310 |
+
f.write_all(&bytes)?;
|
| 311 |
+
f.flush()?;
|
| 312 |
+
}
|
| 313 |
+
fs::rename(&tmp, path)?;
|
| 314 |
+
Ok(())
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
#[cfg(test)]
|
| 318 |
+
pub(crate) fn write_stream(
|
| 319 |
+
path: &Path,
|
| 320 |
+
header: &Header,
|
| 321 |
+
text: &[u8],
|
| 322 |
+
payload: &mut impl Read,
|
| 323 |
+
payload_bytes: u64,
|
| 324 |
+
trailer: &[u8],
|
| 325 |
+
) -> Result<(), FormatError> {
|
| 326 |
+
let tmp = stage_stream(path, header, text, payload, payload_bytes, trailer)?;
|
| 327 |
+
let result = fs::rename(&tmp, path).map_err(FormatError::Io);
|
| 328 |
+
if result.is_err() {
|
| 329 |
+
let _ = fs::remove_file(&tmp);
|
| 330 |
+
}
|
| 331 |
+
result
|
| 332 |
+
}
|
| 333 |
+
|
| 334 |
+
pub(crate) fn stage_stream(
|
| 335 |
+
path: &Path,
|
| 336 |
+
header: &Header,
|
| 337 |
+
text: &[u8],
|
| 338 |
+
payload: &mut impl Read,
|
| 339 |
+
payload_bytes: u64,
|
| 340 |
+
trailer: &[u8],
|
| 341 |
+
) -> Result<PathBuf, FormatError> {
|
| 342 |
+
if text.len() > u32::MAX as usize {
|
| 343 |
+
return Err(FormatError::TextLenMismatch);
|
| 344 |
+
}
|
| 345 |
+
if let Some(parent) = path.parent() {
|
| 346 |
+
fs::create_dir_all(parent)?;
|
| 347 |
+
}
|
| 348 |
+
let (tmp, mut f) = loop {
|
| 349 |
+
let seq = TEMP_SEQ.fetch_add(1, Ordering::Relaxed);
|
| 350 |
+
let tmp = path.with_extension(format!("kv.tmp.{}.{}", std::process::id(), seq));
|
| 351 |
+
match fs::OpenOptions::new()
|
| 352 |
+
.write(true)
|
| 353 |
+
.create_new(true)
|
| 354 |
+
.open(&tmp)
|
| 355 |
+
{
|
| 356 |
+
Ok(f) => break (tmp, f),
|
| 357 |
+
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue,
|
| 358 |
+
Err(e) => return Err(FormatError::Io(e)),
|
| 359 |
+
}
|
| 360 |
+
};
|
| 361 |
+
let result: Result<(), FormatError> = (|| {
|
| 362 |
+
let h = fill_header(
|
| 363 |
+
header.model_id,
|
| 364 |
+
header.quant_bits,
|
| 365 |
+
header.reason,
|
| 366 |
+
header.ext_flags,
|
| 367 |
+
header.tokens,
|
| 368 |
+
header.hits,
|
| 369 |
+
header.ctx_size,
|
| 370 |
+
header.created_at,
|
| 371 |
+
header.last_used,
|
| 372 |
+
payload_bytes,
|
| 373 |
+
);
|
| 374 |
+
f.write_all(&h)?;
|
| 375 |
+
let mut raw_text_bytes = [0u8; 4];
|
| 376 |
+
le_put32(&mut raw_text_bytes, text.len() as u32);
|
| 377 |
+
f.write_all(&raw_text_bytes)?;
|
| 378 |
+
f.write_all(text)?;
|
| 379 |
+
let copied = io::copy(&mut payload.take(payload_bytes), &mut f)?;
|
| 380 |
+
if copied != payload_bytes {
|
| 381 |
+
return Err(FormatError::Truncated);
|
| 382 |
+
}
|
| 383 |
+
f.write_all(trailer)?;
|
| 384 |
+
f.flush()?;
|
| 385 |
+
drop(f);
|
| 386 |
+
Ok(())
|
| 387 |
+
})();
|
| 388 |
+
if let Err(error) = result {
|
| 389 |
+
let _ = fs::remove_file(&tmp);
|
| 390 |
+
return Err(error);
|
| 391 |
+
}
|
| 392 |
+
Ok(tmp)
|
| 393 |
+
}
|
| 394 |
+
|
| 395 |
+
pub fn read_path(path: &Path) -> Result<Record, FormatError> {
|
| 396 |
+
let mut f = fs::File::open(path)?;
|
| 397 |
+
let mut bytes = Vec::new();
|
| 398 |
+
f.read_to_end(&mut bytes)?;
|
| 399 |
+
decode_file(&bytes)
|
| 400 |
+
}
|
| 401 |
+
|
| 402 |
+
pub fn read_envelope(path: &Path) -> Result<Envelope, FormatError> {
|
| 403 |
+
let (metadata, text) = read_text_prefix(path, usize::MAX)?;
|
| 404 |
+
if text.len() != metadata.header.text_bytes as usize {
|
| 405 |
+
return Err(FormatError::Truncated);
|
| 406 |
+
}
|
| 407 |
+
Ok(Envelope {
|
| 408 |
+
header: metadata.header,
|
| 409 |
+
text,
|
| 410 |
+
payload_offset: metadata.payload_offset,
|
| 411 |
+
trailer_bytes: metadata.trailer_bytes,
|
| 412 |
+
file_size: metadata.file_size,
|
| 413 |
+
})
|
| 414 |
+
}
|
| 415 |
+
|
| 416 |
+
pub fn read_trailer(path: &Path, max_bytes: u64) -> Result<(Header, Vec<u8>), FormatError> {
|
| 417 |
+
let mut file = fs::File::open(path)?;
|
| 418 |
+
let metadata = read_metadata_file(&mut file)?;
|
| 419 |
+
if metadata.trailer_bytes > max_bytes {
|
| 420 |
+
return Err(FormatError::TrailerTooLarge);
|
| 421 |
+
}
|
| 422 |
+
let trailer_start = metadata
|
| 423 |
+
.payload_offset
|
| 424 |
+
.checked_add(metadata.header.payload_bytes)
|
| 425 |
+
.ok_or(FormatError::Truncated)?;
|
| 426 |
+
let trailer_bytes =
|
| 427 |
+
usize::try_from(metadata.trailer_bytes).map_err(|_| FormatError::TrailerTooLarge)?;
|
| 428 |
+
file.seek(SeekFrom::Start(trailer_start))?;
|
| 429 |
+
let mut trailer = vec![0; trailer_bytes];
|
| 430 |
+
read_exact(&mut file, &mut trailer)?;
|
| 431 |
+
Ok((metadata.header, trailer))
|
| 432 |
+
}
|
| 433 |
+
|
| 434 |
+
pub(crate) fn read_metadata(path: &Path) -> Result<Metadata, FormatError> {
|
| 435 |
+
let mut f = fs::File::open(path)?;
|
| 436 |
+
read_metadata_file(&mut f)
|
| 437 |
+
}
|
| 438 |
+
|
| 439 |
+
/// The header, and as much of the text as the file still holds, without
|
| 440 |
+
/// asking it for the payload behind them. A truncation can take the
|
| 441 |
+
/// payload and leave what says which conversation the record was keyed by.
|
| 442 |
+
pub(crate) fn read_header_text(
|
| 443 |
+
path: &Path,
|
| 444 |
+
max_bytes: usize,
|
| 445 |
+
) -> Result<(Header, Vec<u8>), FormatError> {
|
| 446 |
+
let mut f = fs::File::open(path)?;
|
| 447 |
+
let mut raw_header = [0u8; FIXED_HEADER];
|
| 448 |
+
read_exact(&mut f, &mut raw_header)?;
|
| 449 |
+
let mut raw_text_bytes = [0u8; 4];
|
| 450 |
+
read_exact(&mut f, &mut raw_text_bytes)?;
|
| 451 |
+
let header = parse_header(&raw_header, le_get32(&raw_text_bytes))?;
|
| 452 |
+
let want = (header.text_bytes as usize).min(max_bytes);
|
| 453 |
+
let mut text = Vec::new();
|
| 454 |
+
f.take(want as u64).read_to_end(&mut text)?;
|
| 455 |
+
Ok((header, text))
|
| 456 |
+
}
|
| 457 |
+
|
| 458 |
+
pub(crate) fn read_text_prefix(
|
| 459 |
+
path: &Path,
|
| 460 |
+
max_bytes: usize,
|
| 461 |
+
) -> Result<(Metadata, Vec<u8>), FormatError> {
|
| 462 |
+
let mut f = fs::File::open(path)?;
|
| 463 |
+
let metadata = read_metadata_file(&mut f)?;
|
| 464 |
+
let want = (metadata.header.text_bytes as usize).min(max_bytes);
|
| 465 |
+
let mut text = Vec::with_capacity(want);
|
| 466 |
+
f.take(want as u64).read_to_end(&mut text)?;
|
| 467 |
+
Ok((metadata, text))
|
| 468 |
+
}
|
| 469 |
+
|
| 470 |
+
fn read_metadata_file(f: &mut fs::File) -> Result<Metadata, FormatError> {
|
| 471 |
+
let file_size = f.metadata()?.len();
|
| 472 |
+
if file_size < (FIXED_HEADER + 4) as u64 {
|
| 473 |
+
return Err(FormatError::Truncated);
|
| 474 |
+
}
|
| 475 |
+
|
| 476 |
+
let mut raw_header = [0u8; FIXED_HEADER];
|
| 477 |
+
read_exact(f, &mut raw_header)?;
|
| 478 |
+
let mut raw_text_bytes = [0u8; 4];
|
| 479 |
+
read_exact(f, &mut raw_text_bytes)?;
|
| 480 |
+
let text_bytes = le_get32(&raw_text_bytes);
|
| 481 |
+
let header = parse_header(&raw_header, text_bytes)?;
|
| 482 |
+
let payload_offset = (FIXED_HEADER + 4) as u64 + u64::from(text_bytes);
|
| 483 |
+
let payload_end = payload_offset
|
| 484 |
+
.checked_add(header.payload_bytes)
|
| 485 |
+
.ok_or(FormatError::Truncated)?;
|
| 486 |
+
if file_size < payload_end {
|
| 487 |
+
return Err(FormatError::Truncated);
|
| 488 |
+
}
|
| 489 |
+
|
| 490 |
+
Ok(Metadata {
|
| 491 |
+
header,
|
| 492 |
+
payload_offset,
|
| 493 |
+
trailer_bytes: file_size - payload_end,
|
| 494 |
+
file_size,
|
| 495 |
+
})
|
| 496 |
+
}
|
| 497 |
+
|
| 498 |
+
fn read_exact(f: &mut fs::File, buf: &mut [u8]) -> Result<(), FormatError> {
|
| 499 |
+
match f.read_exact(buf) {
|
| 500 |
+
Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => Err(FormatError::Truncated),
|
| 501 |
+
result => result.map_err(FormatError::Io),
|
| 502 |
+
}
|
| 503 |
+
}
|
| 504 |
+
|
| 505 |
+
pub fn text_sha_hex(text: &[u8]) -> String {
|
| 506 |
+
sha1_hex(text)
|
| 507 |
+
}
|
| 508 |
+
|
| 509 |
+
pub fn sha_hex_name(name: &str) -> Option<String> {
|
| 510 |
+
if name.len() != 43 || !name.ends_with(".kv") {
|
| 511 |
+
return None;
|
| 512 |
+
}
|
| 513 |
+
let hex = &name[..40];
|
| 514 |
+
if !hex.bytes().all(|b| b.is_ascii_hexdigit()) {
|
| 515 |
+
return None;
|
| 516 |
+
}
|
| 517 |
+
Some(hex.to_ascii_lowercase())
|
| 518 |
+
}
|
| 519 |
+
|
| 520 |
+
pub fn path_for_sha(dir: &Path, sha: &str) -> PathBuf {
|
| 521 |
+
dir.join(format!("{sha}.kv"))
|
| 522 |
+
}
|
| 523 |
+
|
| 524 |
+
pub fn key_kind(ext_flags: u8) -> &'static str {
|
| 525 |
+
if ext_flags & EXT_RESPONSES_VISIBLE != 0 {
|
| 526 |
+
"responses-visible"
|
| 527 |
+
} else if ext_flags & EXT_THINKING_VISIBLE != 0 {
|
| 528 |
+
"thinking-visible"
|
| 529 |
+
} else {
|
| 530 |
+
"token-text"
|
| 531 |
+
}
|
| 532 |
+
}
|
| 533 |
+
|
| 534 |
+
pub fn is_bank_replay_v1(reason: Reason, ext_flags: u8) -> bool {
|
| 535 |
+
reason.family() == 3 && ext_flags & EXT_BANK_REPLAY_V1 != 0
|
| 536 |
+
}
|
| 537 |
+
|
| 538 |
+
pub fn is_automatic_exact_replay(reason: Reason, ext_flags: u8) -> bool {
|
| 539 |
+
match reason.family() {
|
| 540 |
+
1 => ext_flags & EXT_BANK_REPLAY_V1 == 0,
|
| 541 |
+
_ => is_bank_replay_v1(reason, ext_flags),
|
| 542 |
+
}
|
| 543 |
+
}
|
| 544 |
+
|
| 545 |
+
#[cfg(test)]
|
| 546 |
+
mod tests {
|
| 547 |
+
use super::*;
|
| 548 |
+
|
| 549 |
+
struct ErrorAfterPrefix {
|
| 550 |
+
offset: usize,
|
| 551 |
+
}
|
| 552 |
+
|
| 553 |
+
impl Read for ErrorAfterPrefix {
|
| 554 |
+
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
| 555 |
+
const PREFIX: &[u8] = b"DSV4";
|
| 556 |
+
if self.offset == PREFIX.len() {
|
| 557 |
+
return Err(io::Error::other("injected payload read failure"));
|
| 558 |
+
}
|
| 559 |
+
let n = buf.len().min(PREFIX.len() - self.offset);
|
| 560 |
+
buf[..n].copy_from_slice(&PREFIX[self.offset..self.offset + n]);
|
| 561 |
+
self.offset += n;
|
| 562 |
+
Ok(n)
|
| 563 |
+
}
|
| 564 |
+
}
|
| 565 |
+
|
| 566 |
+
fn sample() -> Record {
|
| 567 |
+
Record {
|
| 568 |
+
header: Header {
|
| 569 |
+
quant_bits: 2,
|
| 570 |
+
reason: Reason::Cold,
|
| 571 |
+
ext_flags: 0,
|
| 572 |
+
model_id: 0,
|
| 573 |
+
tokens: 512,
|
| 574 |
+
hits: 3,
|
| 575 |
+
ctx_size: 2048,
|
| 576 |
+
created_at: 1_700_000_000,
|
| 577 |
+
last_used: 1_700_000_100,
|
| 578 |
+
payload_bytes: 4,
|
| 579 |
+
text_bytes: 5,
|
| 580 |
+
},
|
| 581 |
+
text: b"hello".to_vec(),
|
| 582 |
+
payload: b"ABCD".to_vec(),
|
| 583 |
+
trailer: Vec::new(),
|
| 584 |
+
}
|
| 585 |
+
}
|
| 586 |
+
|
| 587 |
+
#[test]
|
| 588 |
+
fn header_round_trip() {
|
| 589 |
+
let rec = sample();
|
| 590 |
+
let bytes = encode_file(&rec);
|
| 591 |
+
let got = decode_file(&bytes).unwrap();
|
| 592 |
+
assert_eq!(got.text, rec.text);
|
| 593 |
+
assert_eq!(got.payload, rec.payload);
|
| 594 |
+
assert_eq!(got.header.tokens, 512);
|
| 595 |
+
assert_eq!(got.header.quant_bits, 2);
|
| 596 |
+
assert_eq!(got.header.reason, Reason::Cold);
|
| 597 |
+
assert_eq!(got.header.ctx_size, 2048);
|
| 598 |
+
assert_eq!(&bytes[..3], b"KVC");
|
| 599 |
+
assert_eq!(bytes[3], 1);
|
| 600 |
+
assert_eq!(bytes[20], 2);
|
| 601 |
+
}
|
| 602 |
+
|
| 603 |
+
#[test]
|
| 604 |
+
fn envelope_has_offsets_without_payload() {
|
| 605 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-envelope-{}", std::process::id()));
|
| 606 |
+
let path = dir.join("sample.kv");
|
| 607 |
+
let _ = fs::remove_dir_all(&dir);
|
| 608 |
+
let mut rec = sample();
|
| 609 |
+
rec.trailer = b"end".to_vec();
|
| 610 |
+
write_path(&path, &rec).unwrap();
|
| 611 |
+
|
| 612 |
+
let got = read_envelope(&path).unwrap();
|
| 613 |
+
assert_eq!(got.header, rec.header);
|
| 614 |
+
assert_eq!(got.text, rec.text);
|
| 615 |
+
assert_eq!(
|
| 616 |
+
got.payload_offset,
|
| 617 |
+
(FIXED_HEADER + 4 + rec.text.len()) as u64
|
| 618 |
+
);
|
| 619 |
+
assert_eq!(got.trailer_bytes, rec.trailer.len() as u64);
|
| 620 |
+
assert_eq!(got.file_size, fs::metadata(&path).unwrap().len());
|
| 621 |
+
|
| 622 |
+
let _ = fs::remove_dir_all(&dir);
|
| 623 |
+
}
|
| 624 |
+
|
| 625 |
+
#[test]
|
| 626 |
+
fn trailer_read_seeks_past_sparse_payload_and_honors_bound() {
|
| 627 |
+
use std::io::{Seek, SeekFrom};
|
| 628 |
+
|
| 629 |
+
const PAYLOAD_BYTES: u64 = 4 * 1024 * 1024 * 1024;
|
| 630 |
+
let dir =
|
| 631 |
+
std::env::temp_dir().join(format!("ds4-kv-trailer-sparse-{}", std::process::id()));
|
| 632 |
+
let path = dir.join("sample.kv");
|
| 633 |
+
let _ = fs::remove_dir_all(&dir);
|
| 634 |
+
fs::create_dir_all(&dir).unwrap();
|
| 635 |
+
|
| 636 |
+
let rec = sample();
|
| 637 |
+
let mut file = fs::File::create(&path).unwrap();
|
| 638 |
+
file.write_all(&fill_header(
|
| 639 |
+
rec.header.model_id,
|
| 640 |
+
rec.header.quant_bits,
|
| 641 |
+
rec.header.reason,
|
| 642 |
+
EXT_TOOL_MAP,
|
| 643 |
+
rec.header.tokens,
|
| 644 |
+
rec.header.hits,
|
| 645 |
+
rec.header.ctx_size,
|
| 646 |
+
rec.header.created_at,
|
| 647 |
+
rec.header.last_used,
|
| 648 |
+
PAYLOAD_BYTES,
|
| 649 |
+
))
|
| 650 |
+
.unwrap();
|
| 651 |
+
let mut text_bytes = [0u8; 4];
|
| 652 |
+
le_put32(&mut text_bytes, rec.text.len() as u32);
|
| 653 |
+
file.write_all(&text_bytes).unwrap();
|
| 654 |
+
file.write_all(&rec.text).unwrap();
|
| 655 |
+
file.seek(SeekFrom::Current(PAYLOAD_BYTES as i64)).unwrap();
|
| 656 |
+
file.write_all(b"\0\xffTR").unwrap();
|
| 657 |
+
drop(file);
|
| 658 |
+
|
| 659 |
+
let (header, trailer) = read_trailer(&path, 4).unwrap();
|
| 660 |
+
assert_eq!(header.payload_bytes, PAYLOAD_BYTES);
|
| 661 |
+
assert_eq!(header.ext_flags, EXT_TOOL_MAP);
|
| 662 |
+
assert_eq!(trailer, b"\0\xffTR");
|
| 663 |
+
assert!(matches!(
|
| 664 |
+
read_trailer(&path, 3),
|
| 665 |
+
Err(FormatError::TrailerTooLarge)
|
| 666 |
+
));
|
| 667 |
+
|
| 668 |
+
let _ = fs::remove_dir_all(&dir);
|
| 669 |
+
}
|
| 670 |
+
|
| 671 |
+
#[test]
|
| 672 |
+
fn rejects_wrong_quant() {
|
| 673 |
+
let mut rec = sample();
|
| 674 |
+
rec.header.quant_bits = 8;
|
| 675 |
+
let bytes = encode_file(&rec);
|
| 676 |
+
assert!(matches!(decode_file(&bytes), Err(FormatError::BadQuant)));
|
| 677 |
+
}
|
| 678 |
+
|
| 679 |
+
#[test]
|
| 680 |
+
fn filename_is_text_sha() {
|
| 681 |
+
assert_eq!(sha_hex_name("not-a-kv"), None);
|
| 682 |
+
let sha = text_sha_hex(b"hello");
|
| 683 |
+
assert_eq!(sha.len(), 40);
|
| 684 |
+
assert_eq!(
|
| 685 |
+
sha_hex_name(&format!("{sha}.kv")).as_deref(),
|
| 686 |
+
Some(sha.as_str())
|
| 687 |
+
);
|
| 688 |
+
}
|
| 689 |
+
|
| 690 |
+
#[test]
|
| 691 |
+
fn stream_failure_preserves_destination_and_removes_temp() {
|
| 692 |
+
let dir =
|
| 693 |
+
std::env::temp_dir().join(format!("ds4-kv-stream-failure-{}", std::process::id()));
|
| 694 |
+
let path = dir.join("entry.kv");
|
| 695 |
+
let _ = fs::remove_dir_all(&dir);
|
| 696 |
+
fs::create_dir_all(&dir).unwrap();
|
| 697 |
+
fs::write(&path, b"original KVC bytes").unwrap();
|
| 698 |
+
|
| 699 |
+
let rec = sample();
|
| 700 |
+
let mut payload = ErrorAfterPrefix { offset: 0 };
|
| 701 |
+
let result = write_stream(&path, &rec.header, &rec.text, &mut payload, 8, &rec.trailer);
|
| 702 |
+
|
| 703 |
+
assert!(result.is_err());
|
| 704 |
+
assert_eq!(fs::read(&path).unwrap(), b"original KVC bytes");
|
| 705 |
+
let mut names: Vec<_> = fs::read_dir(&dir)
|
| 706 |
+
.unwrap()
|
| 707 |
+
.map(|entry| entry.unwrap().file_name())
|
| 708 |
+
.collect();
|
| 709 |
+
names.sort();
|
| 710 |
+
assert_eq!(names, [std::ffi::OsString::from("entry.kv")]);
|
| 711 |
+
|
| 712 |
+
let mut short_payload = &b"DSV4"[..];
|
| 713 |
+
let result = write_stream(
|
| 714 |
+
&path,
|
| 715 |
+
&rec.header,
|
| 716 |
+
&rec.text,
|
| 717 |
+
&mut short_payload,
|
| 718 |
+
8,
|
| 719 |
+
&rec.trailer,
|
| 720 |
+
);
|
| 721 |
+
assert!(result.is_err());
|
| 722 |
+
assert_eq!(fs::read(&path).unwrap(), b"original KVC bytes");
|
| 723 |
+
let names: Vec<_> = fs::read_dir(&dir)
|
| 724 |
+
.unwrap()
|
| 725 |
+
.map(|entry| entry.unwrap().file_name())
|
| 726 |
+
.collect();
|
| 727 |
+
assert_eq!(names, [std::ffi::OsString::from("entry.kv")]);
|
| 728 |
+
|
| 729 |
+
let _ = fs::remove_dir_all(&dir);
|
| 730 |
+
}
|
| 731 |
+
|
| 732 |
+
#[test]
|
| 733 |
+
fn stream_layout_matches_buffered_encoder() {
|
| 734 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-stream-layout-{}", std::process::id()));
|
| 735 |
+
let path = dir.join("entry.kv");
|
| 736 |
+
let _ = fs::remove_dir_all(&dir);
|
| 737 |
+
|
| 738 |
+
let mut rec = sample();
|
| 739 |
+
rec.trailer = b"KVT1 trailer".to_vec();
|
| 740 |
+
let mut payload = rec.payload.as_slice();
|
| 741 |
+
write_stream(
|
| 742 |
+
&path,
|
| 743 |
+
&rec.header,
|
| 744 |
+
&rec.text,
|
| 745 |
+
&mut payload,
|
| 746 |
+
rec.payload.len() as u64,
|
| 747 |
+
&rec.trailer,
|
| 748 |
+
)
|
| 749 |
+
.unwrap();
|
| 750 |
+
assert_eq!(fs::read(&path).unwrap(), encode_file(&rec));
|
| 751 |
+
|
| 752 |
+
let _ = fs::remove_dir_all(&dir);
|
| 753 |
+
}
|
| 754 |
+
|
| 755 |
+
#[test]
|
| 756 |
+
fn staged_stream_leaves_destination_until_commit() {
|
| 757 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-stream-stage-{}", std::process::id()));
|
| 758 |
+
let path = dir.join("entry.kv");
|
| 759 |
+
let _ = fs::remove_dir_all(&dir);
|
| 760 |
+
fs::create_dir_all(&dir).unwrap();
|
| 761 |
+
fs::write(&path, b"original KVC bytes").unwrap();
|
| 762 |
+
|
| 763 |
+
let rec = sample();
|
| 764 |
+
let mut payload = rec.payload.as_slice();
|
| 765 |
+
let staged = stage_stream(
|
| 766 |
+
&path,
|
| 767 |
+
&rec.header,
|
| 768 |
+
&rec.text,
|
| 769 |
+
&mut payload,
|
| 770 |
+
rec.payload.len() as u64,
|
| 771 |
+
&rec.trailer,
|
| 772 |
+
)
|
| 773 |
+
.unwrap();
|
| 774 |
+
assert_eq!(fs::read(&path).unwrap(), b"original KVC bytes");
|
| 775 |
+
assert_eq!(fs::read(&staged).unwrap(), encode_file(&rec));
|
| 776 |
+
|
| 777 |
+
fs::rename(staged, &path).unwrap();
|
| 778 |
+
assert_eq!(fs::read(&path).unwrap(), encode_file(&rec));
|
| 779 |
+
let _ = fs::remove_dir_all(&dir);
|
| 780 |
+
}
|
| 781 |
+
}
|
crates/ds4-kv/src/host.rs
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Host-ledger token counts for KV policy. Callers pass these fields;
|
| 2 |
+
//! policy never reads a native session or CUDA scratch.
|
| 3 |
+
|
| 4 |
+
use crate::policy::{bank_checkpoint_due, continued_store_target, Options};
|
| 5 |
+
|
| 6 |
+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
| 7 |
+
pub struct HostKvView {
|
| 8 |
+
pub live_tokens: i32,
|
| 9 |
+
pub stored_tokens: i32,
|
| 10 |
+
}
|
| 11 |
+
|
| 12 |
+
pub fn continued_store_target_from_host(opt: &Options, host: HostKvView) -> i32 {
|
| 13 |
+
continued_store_target(opt, host.stored_tokens, host.live_tokens)
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
pub fn bank_checkpoint_due_from_host(opt: &Options, host: HostKvView) -> bool {
|
| 17 |
+
bank_checkpoint_due(opt, host.live_tokens, host.stored_tokens)
|
| 18 |
+
}
|
| 19 |
+
|
| 20 |
+
#[cfg(test)]
|
| 21 |
+
mod tests {
|
| 22 |
+
use super::*;
|
| 23 |
+
|
| 24 |
+
#[test]
|
| 25 |
+
fn continued_target_uses_host_live_tokens_not_native_pos() {
|
| 26 |
+
let opt = Options::default();
|
| 27 |
+
let due = HostKvView {
|
| 28 |
+
live_tokens: 10240,
|
| 29 |
+
stored_tokens: 0,
|
| 30 |
+
};
|
| 31 |
+
let raw_interval = HostKvView {
|
| 32 |
+
live_tokens: 10000,
|
| 33 |
+
stored_tokens: 0,
|
| 34 |
+
};
|
| 35 |
+
assert_eq!(continued_store_target_from_host(&opt, due), 10240);
|
| 36 |
+
assert_eq!(continued_store_target_from_host(&opt, raw_interval), 0);
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
#[test]
|
| 40 |
+
fn bank_checkpoint_due_uses_host_committed_frontier() {
|
| 41 |
+
let opt = Options::default();
|
| 42 |
+
assert!(bank_checkpoint_due_from_host(
|
| 43 |
+
&opt,
|
| 44 |
+
HostKvView {
|
| 45 |
+
live_tokens: 10240,
|
| 46 |
+
stored_tokens: 0,
|
| 47 |
+
}
|
| 48 |
+
));
|
| 49 |
+
assert!(!bank_checkpoint_due_from_host(
|
| 50 |
+
&opt,
|
| 51 |
+
HostKvView {
|
| 52 |
+
live_tokens: 10000,
|
| 53 |
+
stored_tokens: 0,
|
| 54 |
+
}
|
| 55 |
+
));
|
| 56 |
+
assert!(!bank_checkpoint_due_from_host(
|
| 57 |
+
&opt,
|
| 58 |
+
HostKvView {
|
| 59 |
+
live_tokens: 9999,
|
| 60 |
+
stored_tokens: 0,
|
| 61 |
+
}
|
| 62 |
+
));
|
| 63 |
+
}
|
| 64 |
+
}
|
crates/ds4-kv/src/identity.rs
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Fixed identity footer after opaque payload and optional KTM records.
|
| 2 |
+
//! Reading identity never scans payload bytes or allocates from file lengths.
|
| 3 |
+
|
| 4 |
+
use std::fs::File;
|
| 5 |
+
use std::io::{Read, Seek, SeekFrom};
|
| 6 |
+
use std::path::Path;
|
| 7 |
+
|
| 8 |
+
pub(crate) const FLAG: u8 = 1 << 6;
|
| 9 |
+
pub(crate) const BYTES: usize = 40;
|
| 10 |
+
const MAGIC: &[u8; 4] = b"KVI\x01";
|
| 11 |
+
const END: &[u8; 4] = b"KVI!";
|
| 12 |
+
|
| 13 |
+
pub(crate) fn append(trailer: &mut Vec<u8>, identity: &[u8; 32]) {
|
| 14 |
+
trailer.extend_from_slice(MAGIC);
|
| 15 |
+
trailer.extend_from_slice(identity);
|
| 16 |
+
trailer.extend_from_slice(END);
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
pub(crate) fn read(path: &Path, file_size: u64, trailer_bytes: u64, flags: u8) -> Option<[u8; 32]> {
|
| 20 |
+
if flags & FLAG == 0 || trailer_bytes < BYTES as u64 || file_size < BYTES as u64 {
|
| 21 |
+
return None;
|
| 22 |
+
}
|
| 23 |
+
let mut file = File::open(path).ok()?;
|
| 24 |
+
if file.metadata().ok()?.len() != file_size {
|
| 25 |
+
return None;
|
| 26 |
+
}
|
| 27 |
+
file.seek(SeekFrom::Start(file_size - BYTES as u64)).ok()?;
|
| 28 |
+
let mut footer = [0; BYTES];
|
| 29 |
+
file.read_exact(&mut footer).ok()?;
|
| 30 |
+
decode(flags, &footer)
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
pub(crate) fn decode(flags: u8, trailer: &[u8]) -> Option<[u8; 32]> {
|
| 34 |
+
if flags & FLAG == 0 || trailer.len() < BYTES {
|
| 35 |
+
return None;
|
| 36 |
+
}
|
| 37 |
+
let footer = &trailer[trailer.len() - BYTES..];
|
| 38 |
+
if &footer[..4] != MAGIC || &footer[36..] != END {
|
| 39 |
+
return None;
|
| 40 |
+
}
|
| 41 |
+
footer[4..36].try_into().ok()
|
| 42 |
+
}
|
crates/ds4-kv/src/lib.rs
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! KVC disk store. File format, SHA key, eviction, and prefix lookup
|
| 2 |
+
//! match `ds4_kvstore.c` at `v0.6.3-dfm`. Payload bytes stay opaque.
|
| 3 |
+
|
| 4 |
+
mod extensions;
|
| 5 |
+
mod format;
|
| 6 |
+
mod host;
|
| 7 |
+
mod identity;
|
| 8 |
+
mod policy;
|
| 9 |
+
mod sha1;
|
| 10 |
+
mod store;
|
| 11 |
+
|
| 12 |
+
pub use extensions::{
|
| 13 |
+
bank_persist_ext_flags, decode_ktm, encode_ktm, BankThinkingExtensions, ExtensionRecord,
|
| 14 |
+
KTM_MAGIC, KTM_VERSION,
|
| 15 |
+
};
|
| 16 |
+
pub use format::{
|
| 17 |
+
decode_file, encode_file, fill_header, key_kind, parse_header, path_for_sha, read_envelope,
|
| 18 |
+
read_path, read_trailer, sha_hex_name, text_sha_hex, write_path, Envelope, FormatError, Header,
|
| 19 |
+
Reason, Record, EXT_BANK_REPLAY_V1, EXT_IMAGE_PIXELS_V2, EXT_RESPONSES_VISIBLE,
|
| 20 |
+
EXT_SESSION_TITLE, EXT_THINKING_VISIBLE, EXT_TOOL_MAP, FIXED_HEADER, PAYLOAD_ABI, VERSION,
|
| 21 |
+
};
|
| 22 |
+
pub use host::{bank_checkpoint_due_from_host, continued_store_target_from_host, HostKvView};
|
| 23 |
+
pub use policy::{
|
| 24 |
+
bank_checkpoint_due, chat_anchor_pos, continued_store_target, eviction_score, file_size_fits,
|
| 25 |
+
store_len, EvictionContext, Options, ScoreEntry,
|
| 26 |
+
};
|
| 27 |
+
pub use sha1::sha1_hex;
|
| 28 |
+
pub use store::{Entry, PayloadGuard, PayloadTemp, PrefixAnswer, Store};
|
crates/ds4-kv/src/policy.rs
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Store-length, continued pacing, budget, and eviction scoring.
|
| 2 |
+
//! Port of the corresponding helpers in `ds4_kvstore.c`.
|
| 3 |
+
|
| 4 |
+
use crate::format::{text_sha_hex, Reason};
|
| 5 |
+
|
| 6 |
+
pub const DEFAULT_MB: u64 = 4096;
|
| 7 |
+
pub const HIT_HALF_LIFE_SECONDS: u64 = 6 * 60 * 60;
|
| 8 |
+
pub const DEFAULT_MIN_TOKENS: i32 = 512;
|
| 9 |
+
pub const DEFAULT_COLD_MAX_TOKENS: i32 = 30000;
|
| 10 |
+
pub const DEFAULT_BOUNDARY_TRIM_TOKENS: i32 = 32;
|
| 11 |
+
pub const DEFAULT_BOUNDARY_ALIGN_TOKENS: i32 = 2048;
|
| 12 |
+
pub const DEFAULT_CONTINUED_INTERVAL_TOKENS: i32 = 10000;
|
| 13 |
+
const MIN_EFFECTIVE_HITS: f64 = 0.01;
|
| 14 |
+
const CONTINUED_PREFIX_MIN_FACTOR: f64 = 0.05;
|
| 15 |
+
const CONTINUED_PREFIX_HIT_FACTOR: f64 = 0.45;
|
| 16 |
+
const ANCHOR_REASON_SCORE_FACTOR: f64 = 2.0;
|
| 17 |
+
|
| 18 |
+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
| 19 |
+
pub struct Options {
|
| 20 |
+
pub min_tokens: i32,
|
| 21 |
+
pub cold_max_tokens: i32,
|
| 22 |
+
pub continued_interval_tokens: i32,
|
| 23 |
+
pub boundary_trim_tokens: i32,
|
| 24 |
+
pub boundary_align_tokens: i32,
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
impl Default for Options {
|
| 28 |
+
fn default() -> Self {
|
| 29 |
+
Self {
|
| 30 |
+
min_tokens: DEFAULT_MIN_TOKENS,
|
| 31 |
+
cold_max_tokens: DEFAULT_COLD_MAX_TOKENS,
|
| 32 |
+
continued_interval_tokens: DEFAULT_CONTINUED_INTERVAL_TOKENS,
|
| 33 |
+
boundary_trim_tokens: DEFAULT_BOUNDARY_TRIM_TOKENS,
|
| 34 |
+
boundary_align_tokens: DEFAULT_BOUNDARY_ALIGN_TOKENS,
|
| 35 |
+
}
|
| 36 |
+
}
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
#[derive(Clone, Debug)]
|
| 40 |
+
pub struct EvictionContext<'a> {
|
| 41 |
+
pub text: &'a [u8],
|
| 42 |
+
pub model_id: u8,
|
| 43 |
+
pub quant_bits: u8,
|
| 44 |
+
pub ctx_size: u32,
|
| 45 |
+
pub reject_different_quant: bool,
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
#[derive(Clone, Debug)]
|
| 49 |
+
pub struct ScoreEntry<'a> {
|
| 50 |
+
pub sha: &'a str,
|
| 51 |
+
pub quant_bits: u8,
|
| 52 |
+
pub model_id: u8,
|
| 53 |
+
pub reason: Reason,
|
| 54 |
+
pub tokens: u32,
|
| 55 |
+
pub hits: u32,
|
| 56 |
+
pub ctx_size: u32,
|
| 57 |
+
pub created_at: u64,
|
| 58 |
+
pub last_used: u64,
|
| 59 |
+
pub text_bytes: u64,
|
| 60 |
+
pub file_size: u64,
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
pub fn store_len(opt: &Options, tokens: i32) -> i32 {
|
| 64 |
+
let trim = opt.boundary_trim_tokens;
|
| 65 |
+
let align = opt.boundary_align_tokens;
|
| 66 |
+
if tokens > opt.min_tokens + trim {
|
| 67 |
+
let mut stable = tokens - trim;
|
| 68 |
+
if align > 0 {
|
| 69 |
+
stable -= stable % align;
|
| 70 |
+
}
|
| 71 |
+
if stable >= opt.min_tokens {
|
| 72 |
+
return stable;
|
| 73 |
+
}
|
| 74 |
+
}
|
| 75 |
+
tokens
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
pub fn continued_step(opt: &Options) -> i32 {
|
| 79 |
+
if opt.continued_interval_tokens <= 0 {
|
| 80 |
+
return 0;
|
| 81 |
+
}
|
| 82 |
+
let mut step = opt.continued_interval_tokens;
|
| 83 |
+
let align = opt.boundary_align_tokens;
|
| 84 |
+
if align > 0 {
|
| 85 |
+
let rounded =
|
| 86 |
+
((i64::from(step) + i64::from(align) - 1) / i64::from(align)) * i64::from(align);
|
| 87 |
+
step = i32::try_from(rounded).unwrap_or(0);
|
| 88 |
+
}
|
| 89 |
+
step
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
pub fn continued_store_target(opt: &Options, last_store_tokens: i32, live_tokens: i32) -> i32 {
|
| 93 |
+
let step = continued_step(opt);
|
| 94 |
+
if step <= 0 {
|
| 95 |
+
return 0;
|
| 96 |
+
}
|
| 97 |
+
if live_tokens < opt.min_tokens {
|
| 98 |
+
return 0;
|
| 99 |
+
}
|
| 100 |
+
if live_tokens % step != 0 {
|
| 101 |
+
return 0;
|
| 102 |
+
}
|
| 103 |
+
if live_tokens <= last_store_tokens {
|
| 104 |
+
return 0;
|
| 105 |
+
}
|
| 106 |
+
live_tokens
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
pub fn bank_checkpoint_due(opt: &Options, committed: i32, stored_tokens: i32) -> bool {
|
| 110 |
+
let step = continued_step(opt);
|
| 111 |
+
if step <= 0 || committed < opt.min_tokens || committed < stored_tokens + step {
|
| 112 |
+
return false;
|
| 113 |
+
}
|
| 114 |
+
true
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
pub fn file_size_bytes(text_bytes: u64, payload_bytes: u64, trailer_bytes: u64) -> Option<u64> {
|
| 118 |
+
const FIXED: u64 = 48 + 4;
|
| 119 |
+
FIXED
|
| 120 |
+
.checked_add(text_bytes)?
|
| 121 |
+
.checked_add(payload_bytes)?
|
| 122 |
+
.checked_add(trailer_bytes)
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
pub fn budget_required(file_bytes: u64) -> Option<u64> {
|
| 126 |
+
let mut slack = file_bytes / 100;
|
| 127 |
+
if file_bytes % 100 != 0 {
|
| 128 |
+
slack += 1;
|
| 129 |
+
}
|
| 130 |
+
file_bytes.checked_add(slack)
|
| 131 |
+
}
|
| 132 |
+
|
| 133 |
+
pub fn file_size_fits(
|
| 134 |
+
budget_bytes: u64,
|
| 135 |
+
text_bytes: u64,
|
| 136 |
+
payload_bytes: u64,
|
| 137 |
+
trailer_bytes: u64,
|
| 138 |
+
) -> bool {
|
| 139 |
+
let Some(file_bytes) = file_size_bytes(text_bytes, payload_bytes, trailer_bytes) else {
|
| 140 |
+
return false;
|
| 141 |
+
};
|
| 142 |
+
if budget_bytes == 0 {
|
| 143 |
+
return true;
|
| 144 |
+
}
|
| 145 |
+
match budget_required(file_bytes) {
|
| 146 |
+
Some(required) => required <= budget_bytes,
|
| 147 |
+
None => false,
|
| 148 |
+
}
|
| 149 |
+
}
|
| 150 |
+
|
| 151 |
+
fn reason_is_anchor(reason: Reason) -> bool {
|
| 152 |
+
matches!(
|
| 153 |
+
reason,
|
| 154 |
+
Reason::Cold
|
| 155 |
+
| Reason::Evict
|
| 156 |
+
| Reason::Shutdown
|
| 157 |
+
| Reason::BankEvict
|
| 158 |
+
| Reason::BankShutdown
|
| 159 |
+
| Reason::BankCheckpoint
|
| 160 |
+
)
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
fn incoming_supersedes_continued(e: &ScoreEntry<'_>, incoming: &EvictionContext<'_>) -> bool {
|
| 164 |
+
if e.reason != Reason::Continued && e.reason != Reason::BankCheckpoint {
|
| 165 |
+
return false;
|
| 166 |
+
}
|
| 167 |
+
if e.text_bytes == 0 || e.text_bytes >= incoming.text.len() as u64 {
|
| 168 |
+
return false;
|
| 169 |
+
}
|
| 170 |
+
if e.model_id != incoming.model_id {
|
| 171 |
+
return false;
|
| 172 |
+
}
|
| 173 |
+
if incoming.reject_different_quant && e.quant_bits != incoming.quant_bits {
|
| 174 |
+
return false;
|
| 175 |
+
}
|
| 176 |
+
if incoming.ctx_size > e.ctx_size {
|
| 177 |
+
return false;
|
| 178 |
+
}
|
| 179 |
+
let prefix = &incoming.text[..e.text_bytes as usize];
|
| 180 |
+
text_sha_hex(prefix) == e.sha
|
| 181 |
+
}
|
| 182 |
+
|
| 183 |
+
pub fn eviction_score(e: &ScoreEntry<'_>, now: u64, incoming: Option<&EvictionContext<'_>>) -> f64 {
|
| 184 |
+
if e.file_size == 0 {
|
| 185 |
+
return 0.0;
|
| 186 |
+
}
|
| 187 |
+
let mut effective_hits = e.hits as f64;
|
| 188 |
+
let used_at = if e.last_used != 0 {
|
| 189 |
+
e.last_used
|
| 190 |
+
} else {
|
| 191 |
+
e.created_at
|
| 192 |
+
};
|
| 193 |
+
if used_at == 0 {
|
| 194 |
+
effective_hits = 0.0;
|
| 195 |
+
} else if now > used_at {
|
| 196 |
+
let elapsed = (now - used_at) as f64;
|
| 197 |
+
effective_hits *= (-elapsed / HIT_HALF_LIFE_SECONDS as f64).exp2();
|
| 198 |
+
if effective_hits < MIN_EFFECTIVE_HITS {
|
| 199 |
+
effective_hits = 0.0;
|
| 200 |
+
}
|
| 201 |
+
}
|
| 202 |
+
let mut score = (effective_hits + 1.0) * (e.tokens as f64) / (e.file_size as f64);
|
| 203 |
+
if reason_is_anchor(e.reason) {
|
| 204 |
+
score *= ANCHOR_REASON_SCORE_FACTOR;
|
| 205 |
+
}
|
| 206 |
+
if let Some(incoming) = incoming {
|
| 207 |
+
if incoming_supersedes_continued(e, incoming) {
|
| 208 |
+
let h = if effective_hits > 0.0 {
|
| 209 |
+
effective_hits / (effective_hits + 1.0)
|
| 210 |
+
} else {
|
| 211 |
+
0.0
|
| 212 |
+
};
|
| 213 |
+
score *= CONTINUED_PREFIX_MIN_FACTOR + CONTINUED_PREFIX_HIT_FACTOR * h;
|
| 214 |
+
}
|
| 215 |
+
}
|
| 216 |
+
score
|
| 217 |
+
}
|
| 218 |
+
|
| 219 |
+
pub fn chat_anchor_pos(
|
| 220 |
+
opt: &Options,
|
| 221 |
+
prompt: &[i32],
|
| 222 |
+
user_token_id: i32,
|
| 223 |
+
assistant_token_id: i32,
|
| 224 |
+
) -> i32 {
|
| 225 |
+
if user_token_id < 0 || assistant_token_id < 0 {
|
| 226 |
+
return -1;
|
| 227 |
+
}
|
| 228 |
+
let mut last_user = -1;
|
| 229 |
+
for (i, &token) in prompt.iter().enumerate() {
|
| 230 |
+
if token == assistant_token_id {
|
| 231 |
+
break;
|
| 232 |
+
}
|
| 233 |
+
if token == user_token_id {
|
| 234 |
+
last_user = i as i32;
|
| 235 |
+
}
|
| 236 |
+
}
|
| 237 |
+
if last_user >= opt.min_tokens {
|
| 238 |
+
last_user
|
| 239 |
+
} else {
|
| 240 |
+
-1
|
| 241 |
+
}
|
| 242 |
+
}
|
| 243 |
+
|
| 244 |
+
#[cfg(test)]
|
| 245 |
+
mod tests {
|
| 246 |
+
use super::*;
|
| 247 |
+
|
| 248 |
+
#[test]
|
| 249 |
+
fn store_len_trims_to_align() {
|
| 250 |
+
let opt = Options::default();
|
| 251 |
+
assert_eq!(store_len(&opt, 4096 + 40), 4096);
|
| 252 |
+
assert_eq!(store_len(&opt, 100), 100);
|
| 253 |
+
}
|
| 254 |
+
|
| 255 |
+
#[test]
|
| 256 |
+
fn continued_target_needs_aligned_interval() {
|
| 257 |
+
let opt = Options::default();
|
| 258 |
+
assert_eq!(continued_store_target(&opt, 0, 8192), 0);
|
| 259 |
+
assert_eq!(continued_store_target(&opt, 0, 10240), 10240);
|
| 260 |
+
assert_eq!(continued_store_target(&opt, 10240, 10240), 0);
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
#[test]
|
| 264 |
+
fn continued_step_maps_default_10000_interval_to_10240() {
|
| 265 |
+
let opt = Options::default();
|
| 266 |
+
assert_eq!(DEFAULT_CONTINUED_INTERVAL_TOKENS, 10000);
|
| 267 |
+
assert_eq!(
|
| 268 |
+
opt.continued_interval_tokens,
|
| 269 |
+
DEFAULT_CONTINUED_INTERVAL_TOKENS
|
| 270 |
+
);
|
| 271 |
+
assert_eq!(continued_step(&opt), 10240);
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
#[test]
|
| 275 |
+
fn bank_checkpoint_due_when_committed_at_10240_aligned_frontier() {
|
| 276 |
+
let opt = Options::default();
|
| 277 |
+
assert!(bank_checkpoint_due(&opt, 10240, 0));
|
| 278 |
+
}
|
| 279 |
+
|
| 280 |
+
#[test]
|
| 281 |
+
fn bank_checkpoint_not_due_when_committed_is_raw_10000_interval() {
|
| 282 |
+
let opt = Options::default();
|
| 283 |
+
assert!(!bank_checkpoint_due(&opt, 10000, 0));
|
| 284 |
+
}
|
| 285 |
+
|
| 286 |
+
#[test]
|
| 287 |
+
fn bank_checkpoint_not_due_when_committed_is_9999() {
|
| 288 |
+
let opt = Options::default();
|
| 289 |
+
assert!(!bank_checkpoint_due(&opt, 9999, 0));
|
| 290 |
+
}
|
| 291 |
+
|
| 292 |
+
#[test]
|
| 293 |
+
fn continued_step_disables_unrepresentable_aligned_interval() {
|
| 294 |
+
let opt = Options {
|
| 295 |
+
min_tokens: 1,
|
| 296 |
+
continued_interval_tokens: i32::MAX,
|
| 297 |
+
boundary_align_tokens: 2048,
|
| 298 |
+
..Options::default()
|
| 299 |
+
};
|
| 300 |
+
assert_eq!(continued_step(&opt), 0);
|
| 301 |
+
assert_eq!(continued_store_target(&opt, 0, 2048), 0);
|
| 302 |
+
}
|
| 303 |
+
|
| 304 |
+
#[test]
|
| 305 |
+
fn budget_adds_one_percent() {
|
| 306 |
+
assert_eq!(budget_required(100), Some(101));
|
| 307 |
+
assert_eq!(budget_required(101), Some(103));
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
#[test]
|
| 311 |
+
fn equal_length_incoming_does_not_discount_continued() {
|
| 312 |
+
let text = b"same prefix";
|
| 313 |
+
let sha = text_sha_hex(text);
|
| 314 |
+
let entry = ScoreEntry {
|
| 315 |
+
sha: &sha,
|
| 316 |
+
quant_bits: 2,
|
| 317 |
+
model_id: 0,
|
| 318 |
+
reason: Reason::Continued,
|
| 319 |
+
tokens: 512,
|
| 320 |
+
hits: 0,
|
| 321 |
+
ctx_size: 2048,
|
| 322 |
+
created_at: 1000,
|
| 323 |
+
last_used: 1000,
|
| 324 |
+
text_bytes: text.len() as u64,
|
| 325 |
+
file_size: 4096,
|
| 326 |
+
};
|
| 327 |
+
let incoming = EvictionContext {
|
| 328 |
+
text,
|
| 329 |
+
model_id: 0,
|
| 330 |
+
quant_bits: 2,
|
| 331 |
+
ctx_size: 2048,
|
| 332 |
+
reject_different_quant: true,
|
| 333 |
+
};
|
| 334 |
+
|
| 335 |
+
assert_eq!(
|
| 336 |
+
eviction_score(&entry, 1000, Some(&incoming)),
|
| 337 |
+
eviction_score(&entry, 1000, None)
|
| 338 |
+
);
|
| 339 |
+
}
|
| 340 |
+
}
|
crates/ds4-kv/src/sha1.rs
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Byte-identical port of the SHA-1 in `ds4_kvstore.c`.
|
| 2 |
+
|
| 3 |
+
pub fn sha1(bytes: &[u8]) -> [u8; 20] {
|
| 4 |
+
let mut c = Ctx::new();
|
| 5 |
+
c.update(bytes);
|
| 6 |
+
c.finish()
|
| 7 |
+
}
|
| 8 |
+
|
| 9 |
+
pub fn sha1_hex(bytes: &[u8]) -> String {
|
| 10 |
+
hex20(&sha1(bytes))
|
| 11 |
+
}
|
| 12 |
+
|
| 13 |
+
struct Ctx {
|
| 14 |
+
h: [u32; 5],
|
| 15 |
+
bytes: u64,
|
| 16 |
+
block: [u8; 64],
|
| 17 |
+
used: usize,
|
| 18 |
+
}
|
| 19 |
+
|
| 20 |
+
impl Ctx {
|
| 21 |
+
fn new() -> Self {
|
| 22 |
+
Self {
|
| 23 |
+
h: [
|
| 24 |
+
0x6745_2301,
|
| 25 |
+
0xefcd_ab89,
|
| 26 |
+
0x98ba_dcfe,
|
| 27 |
+
0x1032_5476,
|
| 28 |
+
0xc3d2_e1f0,
|
| 29 |
+
],
|
| 30 |
+
bytes: 0,
|
| 31 |
+
block: [0; 64],
|
| 32 |
+
used: 0,
|
| 33 |
+
}
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
fn update(&mut self, mut input: &[u8]) {
|
| 37 |
+
self.bytes += input.len() as u64;
|
| 38 |
+
while !input.is_empty() {
|
| 39 |
+
let n = (64 - self.used).min(input.len());
|
| 40 |
+
self.block[self.used..self.used + n].copy_from_slice(&input[..n]);
|
| 41 |
+
self.used += n;
|
| 42 |
+
input = &input[n..];
|
| 43 |
+
if self.used == 64 {
|
| 44 |
+
self.transform();
|
| 45 |
+
self.used = 0;
|
| 46 |
+
}
|
| 47 |
+
}
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
fn finish(&mut self) -> [u8; 20] {
|
| 51 |
+
let bits = self.bytes * 8;
|
| 52 |
+
self.update(&[0x80]);
|
| 53 |
+
let zero = [0u8; 1];
|
| 54 |
+
while self.used != 56 {
|
| 55 |
+
self.update(&zero);
|
| 56 |
+
}
|
| 57 |
+
let mut len = [0u8; 8];
|
| 58 |
+
for i in 0..8 {
|
| 59 |
+
len[7 - i] = (bits >> (8 * i)) as u8;
|
| 60 |
+
}
|
| 61 |
+
self.update(&len);
|
| 62 |
+
let mut out = [0u8; 20];
|
| 63 |
+
for i in 0..5 {
|
| 64 |
+
out[i * 4] = (self.h[i] >> 24) as u8;
|
| 65 |
+
out[i * 4 + 1] = (self.h[i] >> 16) as u8;
|
| 66 |
+
out[i * 4 + 2] = (self.h[i] >> 8) as u8;
|
| 67 |
+
out[i * 4 + 3] = self.h[i] as u8;
|
| 68 |
+
}
|
| 69 |
+
out
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
fn transform(&mut self) {
|
| 73 |
+
let mut w = [0u32; 80];
|
| 74 |
+
for i in 0..16 {
|
| 75 |
+
w[i] = (u32::from(self.block[i * 4]) << 24)
|
| 76 |
+
| (u32::from(self.block[i * 4 + 1]) << 16)
|
| 77 |
+
| (u32::from(self.block[i * 4 + 2]) << 8)
|
| 78 |
+
| u32::from(self.block[i * 4 + 3]);
|
| 79 |
+
}
|
| 80 |
+
for i in 16..80 {
|
| 81 |
+
w[i] = rol32(w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16], 1);
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
let mut a = self.h[0];
|
| 85 |
+
let mut b = self.h[1];
|
| 86 |
+
let mut cc = self.h[2];
|
| 87 |
+
let mut d = self.h[3];
|
| 88 |
+
let mut e = self.h[4];
|
| 89 |
+
for i in 0..80 {
|
| 90 |
+
let (f, k) = if i < 20 {
|
| 91 |
+
((b & cc) | ((!b) & d), 0x5a82_7999)
|
| 92 |
+
} else if i < 40 {
|
| 93 |
+
(b ^ cc ^ d, 0x6ed9_eba1)
|
| 94 |
+
} else if i < 60 {
|
| 95 |
+
((b & cc) | (b & d) | (cc & d), 0x8f1b_bcdc)
|
| 96 |
+
} else {
|
| 97 |
+
(b ^ cc ^ d, 0xca62_c1d6)
|
| 98 |
+
};
|
| 99 |
+
let tmp = rol32(a, 5)
|
| 100 |
+
.wrapping_add(f)
|
| 101 |
+
.wrapping_add(e)
|
| 102 |
+
.wrapping_add(k)
|
| 103 |
+
.wrapping_add(w[i]);
|
| 104 |
+
e = d;
|
| 105 |
+
d = cc;
|
| 106 |
+
cc = rol32(b, 30);
|
| 107 |
+
b = a;
|
| 108 |
+
a = tmp;
|
| 109 |
+
}
|
| 110 |
+
self.h[0] = self.h[0].wrapping_add(a);
|
| 111 |
+
self.h[1] = self.h[1].wrapping_add(b);
|
| 112 |
+
self.h[2] = self.h[2].wrapping_add(cc);
|
| 113 |
+
self.h[3] = self.h[3].wrapping_add(d);
|
| 114 |
+
self.h[4] = self.h[4].wrapping_add(e);
|
| 115 |
+
}
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
fn rol32(v: u32, n: i32) -> u32 {
|
| 119 |
+
v.rotate_left(n as u32)
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
fn hex20(digest: &[u8; 20]) -> String {
|
| 123 |
+
const HEX: &[u8; 16] = b"0123456789abcdef";
|
| 124 |
+
let mut out = String::with_capacity(40);
|
| 125 |
+
for &b in digest {
|
| 126 |
+
out.push(HEX[(b >> 4) as usize] as char);
|
| 127 |
+
out.push(HEX[(b & 15) as usize] as char);
|
| 128 |
+
}
|
| 129 |
+
out
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
#[cfg(test)]
|
| 133 |
+
mod tests {
|
| 134 |
+
use super::*;
|
| 135 |
+
|
| 136 |
+
#[test]
|
| 137 |
+
fn sha1_abc_matches_fips() {
|
| 138 |
+
assert_eq!(sha1_hex(b"abc"), "a9993e364706816aba3e25717850c26c9cd0d89d");
|
| 139 |
+
}
|
| 140 |
+
}
|
crates/ds4-kv/src/store.rs
ADDED
|
@@ -0,0 +1,2406 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Directory-backed KVC catalog: refresh, prefix lookup, LCP, evict.
|
| 2 |
+
|
| 3 |
+
use crate::format::{
|
| 4 |
+
fill_header, is_automatic_exact_replay, is_bank_replay_v1, path_for_sha, read_envelope,
|
| 5 |
+
read_header_text, read_metadata, read_path, read_text_prefix, sha_hex_name, stage_stream,
|
| 6 |
+
text_sha_hex, write_path, Envelope, FormatError, Header, Reason, Record, EXT_IMAGE_PIXELS_V2,
|
| 7 |
+
EXT_TOOL_MAP,
|
| 8 |
+
};
|
| 9 |
+
use crate::policy::{
|
| 10 |
+
eviction_score, file_size_bytes, file_size_fits, EvictionContext, Options, ScoreEntry,
|
| 11 |
+
DEFAULT_MB,
|
| 12 |
+
};
|
| 13 |
+
use std::fs;
|
| 14 |
+
use std::io::{self, Read as _, Seek as _, SeekFrom, Write as _};
|
| 15 |
+
#[cfg(unix)]
|
| 16 |
+
use std::os::fd::AsRawFd as _;
|
| 17 |
+
#[cfg(unix)]
|
| 18 |
+
use std::os::unix::fs::{DirBuilderExt as _, OpenOptionsExt as _};
|
| 19 |
+
use std::path::{Path, PathBuf};
|
| 20 |
+
use std::sync::atomic::{AtomicU64, Ordering};
|
| 21 |
+
use std::time::{SystemTime, UNIX_EPOCH};
|
| 22 |
+
|
| 23 |
+
static PAYLOAD_TEMP_SEQ: AtomicU64 = AtomicU64::new(0);
|
| 24 |
+
|
| 25 |
+
/// Whether a record must leave something to prefill. The bank lane admits
|
| 26 |
+
/// only records it can extend; the serial lane may replay an exact one.
|
| 27 |
+
#[derive(Clone, Copy, PartialEq, Eq)]
|
| 28 |
+
enum Suffix {
|
| 29 |
+
Optional,
|
| 30 |
+
Required,
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
#[derive(Clone, Copy, PartialEq, Eq)]
|
| 34 |
+
enum FileAccess {
|
| 35 |
+
ReadOnly,
|
| 36 |
+
ReadWrite,
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
/// What a search would do with the best record it can see for a key. The
|
| 40 |
+
/// store answers what it holds; the lane turns that into a miss reason.
|
| 41 |
+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
| 42 |
+
pub enum PrefixAnswer {
|
| 43 |
+
/// Nothing is keyed by this prompt.
|
| 44 |
+
None,
|
| 45 |
+
/// A record the search takes. Only the caller's own budget can refuse
|
| 46 |
+
/// it now, and the reuse contract has no reason for that.
|
| 47 |
+
Usable,
|
| 48 |
+
/// A record ruled out by model, quantization or context.
|
| 49 |
+
Mismatch,
|
| 50 |
+
/// A record shallower than the store's current minimum — written
|
| 51 |
+
/// before the minimum was raised, and skipped by every search since.
|
| 52 |
+
Shallow,
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
#[derive(Clone, Debug)]
|
| 56 |
+
pub struct Entry {
|
| 57 |
+
pub sha: String,
|
| 58 |
+
pub path: PathBuf,
|
| 59 |
+
pub header: Header,
|
| 60 |
+
pub file_size: u64,
|
| 61 |
+
identity: Option<[u8; 32]>,
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
/// A file the catalog cannot use: its payload no longer holds what its
|
| 65 |
+
/// header describes. The name still says which text it was keyed by.
|
| 66 |
+
#[derive(Clone, Debug)]
|
| 67 |
+
struct Damaged {
|
| 68 |
+
sha: String,
|
| 69 |
+
path: PathBuf,
|
| 70 |
+
reason: Reason,
|
| 71 |
+
ext_flags: u8,
|
| 72 |
+
text_bytes: u32,
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
#[derive(Debug)]
|
| 76 |
+
pub struct Store {
|
| 77 |
+
pub dir: PathBuf,
|
| 78 |
+
pub budget_bytes: u64,
|
| 79 |
+
pub reject_different_quant: bool,
|
| 80 |
+
pub opt: Options,
|
| 81 |
+
pub continued_last_store_tokens: i32,
|
| 82 |
+
entries: Vec<Entry>,
|
| 83 |
+
damaged: Vec<Damaged>,
|
| 84 |
+
identity: Option<[u8; 32]>,
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
#[derive(Debug)]
|
| 88 |
+
pub struct PayloadTemp {
|
| 89 |
+
path: PathBuf,
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
/// Holds the validated inode while a native loader reopens its descriptor path.
|
| 93 |
+
/// An atomic replacement at the catalog path cannot swap the loaded artifact.
|
| 94 |
+
#[derive(Debug)]
|
| 95 |
+
pub struct PayloadGuard {
|
| 96 |
+
_file: fs::File,
|
| 97 |
+
path: PathBuf,
|
| 98 |
+
}
|
| 99 |
+
|
| 100 |
+
impl PayloadGuard {
|
| 101 |
+
pub fn path(&self) -> &Path {
|
| 102 |
+
&self.path
|
| 103 |
+
}
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
impl PayloadTemp {
|
| 107 |
+
pub fn path(&self) -> &Path {
|
| 108 |
+
&self.path
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
impl Drop for PayloadTemp {
|
| 113 |
+
fn drop(&mut self) {
|
| 114 |
+
let _ = fs::remove_file(&self.path);
|
| 115 |
+
}
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
impl Store {
|
| 119 |
+
pub fn open(
|
| 120 |
+
dir: impl AsRef<Path>,
|
| 121 |
+
budget_mb: u64,
|
| 122 |
+
reject_different_quant: bool,
|
| 123 |
+
opt: Options,
|
| 124 |
+
) -> io::Result<Self> {
|
| 125 |
+
let dir = dir.as_ref().to_path_buf();
|
| 126 |
+
#[cfg(unix)]
|
| 127 |
+
{
|
| 128 |
+
let mut builder = fs::DirBuilder::new();
|
| 129 |
+
builder.recursive(true).mode(0o700).create(&dir)?;
|
| 130 |
+
}
|
| 131 |
+
#[cfg(not(unix))]
|
| 132 |
+
fs::create_dir_all(&dir)?;
|
| 133 |
+
let budget_mb = if budget_mb == 0 {
|
| 134 |
+
DEFAULT_MB
|
| 135 |
+
} else {
|
| 136 |
+
budget_mb
|
| 137 |
+
};
|
| 138 |
+
let mut store = Self {
|
| 139 |
+
dir,
|
| 140 |
+
budget_bytes: budget_mb * 1024 * 1024,
|
| 141 |
+
reject_different_quant,
|
| 142 |
+
opt,
|
| 143 |
+
continued_last_store_tokens: 0,
|
| 144 |
+
entries: Vec::new(),
|
| 145 |
+
damaged: Vec::new(),
|
| 146 |
+
identity: None,
|
| 147 |
+
};
|
| 148 |
+
store.evict(0, None);
|
| 149 |
+
Ok(store)
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
+
/// Require this artifact/runtime identity for all subsequent reads and
|
| 153 |
+
/// writes. Legacy entries remain in the shared budget, but cannot be reused.
|
| 154 |
+
pub fn bind_identity(&mut self, identity: [u8; 32]) {
|
| 155 |
+
self.identity = Some(identity);
|
| 156 |
+
self.continued_last_store_tokens = 0;
|
| 157 |
+
self.refresh();
|
| 158 |
+
}
|
| 159 |
+
|
| 160 |
+
fn matches_identity(&self, flags: u8, identity: Option<[u8; 32]>) -> bool {
|
| 161 |
+
match self.identity {
|
| 162 |
+
Some(expected) => flags & crate::identity::FLAG != 0 && identity == Some(expected),
|
| 163 |
+
None => flags & crate::identity::FLAG == 0,
|
| 164 |
+
}
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
fn envelope_identity(&self, path: &Path, e: &Envelope) -> bool {
|
| 168 |
+
self.matches_identity(
|
| 169 |
+
e.header.ext_flags,
|
| 170 |
+
crate::identity::read(path, e.file_size, e.trailer_bytes, e.header.ext_flags),
|
| 171 |
+
)
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
fn check_identity(&self, path: &Path, e: &Envelope) -> io::Result<()> {
|
| 175 |
+
if self.envelope_identity(path, e) {
|
| 176 |
+
return Ok(());
|
| 177 |
+
}
|
| 178 |
+
Err(io::Error::new(
|
| 179 |
+
io::ErrorKind::InvalidData,
|
| 180 |
+
"KVC artifact/runtime identity mismatch",
|
| 181 |
+
))
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
fn scoped_trailer(&self, header: &mut Header, trailer: &[u8]) -> Vec<u8> {
|
| 185 |
+
let mut out = trailer.to_vec();
|
| 186 |
+
if let Some(identity) = self.identity {
|
| 187 |
+
header.ext_flags |= crate::identity::FLAG;
|
| 188 |
+
crate::identity::append(&mut out, &identity);
|
| 189 |
+
}
|
| 190 |
+
out
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
fn visible_envelope(&self, mut envelope: Envelope) -> Envelope {
|
| 194 |
+
// Callers validate protocol extensions independently. Identity has
|
| 195 |
+
// already been checked; it is neither a tool map nor payload content.
|
| 196 |
+
if self.identity.is_some() {
|
| 197 |
+
envelope.header.ext_flags &= !crate::identity::FLAG;
|
| 198 |
+
envelope.trailer_bytes -= crate::identity::BYTES as u64;
|
| 199 |
+
}
|
| 200 |
+
envelope
|
| 201 |
+
}
|
| 202 |
+
|
| 203 |
+
pub fn payload_temp(&self) -> io::Result<PayloadTemp> {
|
| 204 |
+
loop {
|
| 205 |
+
let seq = PAYLOAD_TEMP_SEQ.fetch_add(1, Ordering::Relaxed);
|
| 206 |
+
let path = self
|
| 207 |
+
.dir
|
| 208 |
+
.join(format!(".payload.{}.{}", std::process::id(), seq));
|
| 209 |
+
let mut options = fs::OpenOptions::new();
|
| 210 |
+
options.write(true).create_new(true);
|
| 211 |
+
#[cfg(unix)]
|
| 212 |
+
options.mode(0o600);
|
| 213 |
+
match options.open(&path) {
|
| 214 |
+
Ok(_) => return Ok(PayloadTemp { path }),
|
| 215 |
+
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
|
| 216 |
+
Err(error) => return Err(error),
|
| 217 |
+
}
|
| 218 |
+
}
|
| 219 |
+
}
|
| 220 |
+
|
| 221 |
+
pub fn refresh(&mut self) {
|
| 222 |
+
self.entries.clear();
|
| 223 |
+
self.damaged.clear();
|
| 224 |
+
let Ok(rd) = fs::read_dir(&self.dir) else {
|
| 225 |
+
return;
|
| 226 |
+
};
|
| 227 |
+
for ent in rd.flatten() {
|
| 228 |
+
let name = ent.file_name();
|
| 229 |
+
let Some(name) = name.to_str() else { continue };
|
| 230 |
+
let Some(sha) = sha_hex_name(name) else {
|
| 231 |
+
continue;
|
| 232 |
+
};
|
| 233 |
+
let path = ent.path();
|
| 234 |
+
let Ok(metadata) = read_metadata(&path) else {
|
| 235 |
+
// No search can use it, but it can still be the reason a
|
| 236 |
+
// prompt keyed by its text finds nothing.
|
| 237 |
+
if let Ok((header, _)) = read_header_text(&path, 0) {
|
| 238 |
+
self.damaged.push(Damaged {
|
| 239 |
+
sha,
|
| 240 |
+
path,
|
| 241 |
+
reason: header.reason,
|
| 242 |
+
ext_flags: header.ext_flags,
|
| 243 |
+
text_bytes: header.text_bytes,
|
| 244 |
+
});
|
| 245 |
+
}
|
| 246 |
+
continue;
|
| 247 |
+
};
|
| 248 |
+
let identity = crate::identity::read(
|
| 249 |
+
&path,
|
| 250 |
+
metadata.file_size,
|
| 251 |
+
metadata.trailer_bytes,
|
| 252 |
+
metadata.header.ext_flags,
|
| 253 |
+
);
|
| 254 |
+
self.entries.push(Entry {
|
| 255 |
+
sha,
|
| 256 |
+
path,
|
| 257 |
+
header: metadata.header,
|
| 258 |
+
file_size: metadata.file_size,
|
| 259 |
+
identity,
|
| 260 |
+
});
|
| 261 |
+
}
|
| 262 |
+
}
|
| 263 |
+
|
| 264 |
+
pub fn entries(&self) -> &[Entry] {
|
| 265 |
+
&self.entries
|
| 266 |
+
}
|
| 267 |
+
|
| 268 |
+
pub fn write(&mut self, mut record: Record) -> io::Result<PathBuf> {
|
| 269 |
+
record.trailer = self.scoped_trailer(&mut record.header, &record.trailer);
|
| 270 |
+
if !file_size_fits(
|
| 271 |
+
self.budget_bytes,
|
| 272 |
+
record.text.len() as u64,
|
| 273 |
+
record.payload.len() as u64,
|
| 274 |
+
record.trailer.len() as u64,
|
| 275 |
+
) {
|
| 276 |
+
return Err(io::Error::new(
|
| 277 |
+
io::ErrorKind::OutOfMemory,
|
| 278 |
+
"KVC file exceeds budget",
|
| 279 |
+
));
|
| 280 |
+
}
|
| 281 |
+
record.header.text_bytes = record.text.len() as u32;
|
| 282 |
+
record.header.payload_bytes = record.payload.len() as u64;
|
| 283 |
+
let sha = text_sha_hex(&record.text);
|
| 284 |
+
let path = path_for_sha(&self.dir, &sha);
|
| 285 |
+
let incoming = EvictionContext {
|
| 286 |
+
text: &record.text,
|
| 287 |
+
model_id: record.header.model_id,
|
| 288 |
+
quant_bits: record.header.quant_bits,
|
| 289 |
+
ctx_size: record.header.ctx_size,
|
| 290 |
+
reject_different_quant: self.reject_different_quant,
|
| 291 |
+
};
|
| 292 |
+
let extra = 48
|
| 293 |
+
+ 4
|
| 294 |
+
+ record.text.len() as u64
|
| 295 |
+
+ record.payload.len() as u64
|
| 296 |
+
+ record.trailer.len() as u64;
|
| 297 |
+
self.evict(extra, Some(&incoming));
|
| 298 |
+
write_path(&path, &record).map_err(|e| io::Error::other(e))?;
|
| 299 |
+
self.refresh();
|
| 300 |
+
Ok(path)
|
| 301 |
+
}
|
| 302 |
+
|
| 303 |
+
/// Return an already-compatible KVC entry before the caller stages its
|
| 304 |
+
/// opaque native payload. A nonempty trailer follows the writer fast path.
|
| 305 |
+
pub fn reuse_compatible(
|
| 306 |
+
&mut self,
|
| 307 |
+
mut header: Header,
|
| 308 |
+
text: &[u8],
|
| 309 |
+
trailer: &[u8],
|
| 310 |
+
) -> io::Result<Option<PathBuf>> {
|
| 311 |
+
if !trailer.is_empty() {
|
| 312 |
+
header.ext_flags |= EXT_TOOL_MAP;
|
| 313 |
+
}
|
| 314 |
+
let sha = text_sha_hex(text);
|
| 315 |
+
let path = path_for_sha(&self.dir, &sha);
|
| 316 |
+
let Ok(existing) = read_envelope(&path) else {
|
| 317 |
+
return Ok(None);
|
| 318 |
+
};
|
| 319 |
+
let compatible = self.envelope_identity(&path, &existing)
|
| 320 |
+
&& existing.header.model_id == header.model_id
|
| 321 |
+
&& (!self.reject_different_quant || existing.header.quant_bits == header.quant_bits)
|
| 322 |
+
&& existing.header.ctx_size <= header.ctx_size
|
| 323 |
+
&& is_automatic_exact_replay(existing.header.reason, existing.header.ext_flags)
|
| 324 |
+
&& is_automatic_exact_replay(header.reason, header.ext_flags)
|
| 325 |
+
&& existing.text == text
|
| 326 |
+
&& text_sha_hex(&existing.text) == sha;
|
| 327 |
+
if !compatible {
|
| 328 |
+
return Ok(None);
|
| 329 |
+
}
|
| 330 |
+
let visible = self.visible_envelope(existing.clone());
|
| 331 |
+
if trailer.is_empty() {
|
| 332 |
+
// Revalidate header and footer through one descriptor on the
|
| 333 |
+
// no-write fast path too; catalog paths may be atomically replaced.
|
| 334 |
+
self.payload_file(&path, &visible, FileAccess::ReadOnly)?;
|
| 335 |
+
} else {
|
| 336 |
+
let trailer = self.scoped_trailer(&mut header, trailer);
|
| 337 |
+
let mut file = self.payload_file(&path, &visible, FileAccess::ReadWrite)?;
|
| 338 |
+
rewrite_compatible_trailer(&mut file, &existing, header.ext_flags, &trailer)?;
|
| 339 |
+
}
|
| 340 |
+
Ok(Some(path))
|
| 341 |
+
}
|
| 342 |
+
|
| 343 |
+
/// Store an opaque native payload without buffering it in Rust memory.
|
| 344 |
+
/// A nonempty trailer is the frozen server tool-map suffix.
|
| 345 |
+
pub fn write_payload_file(
|
| 346 |
+
&mut self,
|
| 347 |
+
mut header: Header,
|
| 348 |
+
text: &[u8],
|
| 349 |
+
payload_path: &Path,
|
| 350 |
+
trailer: &[u8],
|
| 351 |
+
) -> io::Result<PathBuf> {
|
| 352 |
+
if text.len() > u32::MAX as usize {
|
| 353 |
+
return Err(io::Error::new(
|
| 354 |
+
io::ErrorKind::InvalidInput,
|
| 355 |
+
"KVC text exceeds u32 length",
|
| 356 |
+
));
|
| 357 |
+
}
|
| 358 |
+
if !trailer.is_empty() {
|
| 359 |
+
header.ext_flags |= EXT_TOOL_MAP;
|
| 360 |
+
}
|
| 361 |
+
if let Some(path) = self.reuse_compatible(header.clone(), text, trailer)? {
|
| 362 |
+
return Ok(path);
|
| 363 |
+
}
|
| 364 |
+
let trailer = self.scoped_trailer(&mut header, trailer);
|
| 365 |
+
let sha = text_sha_hex(text);
|
| 366 |
+
let path = path_for_sha(&self.dir, &sha);
|
| 367 |
+
if payload_path == path {
|
| 368 |
+
return Err(io::Error::new(
|
| 369 |
+
io::ErrorKind::InvalidInput,
|
| 370 |
+
"KVC payload source aliases destination",
|
| 371 |
+
));
|
| 372 |
+
}
|
| 373 |
+
let mut payload = fs::File::open(payload_path)?;
|
| 374 |
+
let payload_bytes = payload.metadata()?.len();
|
| 375 |
+
let extra = file_size_bytes(text.len() as u64, payload_bytes, trailer.len() as u64)
|
| 376 |
+
.ok_or_else(|| {
|
| 377 |
+
io::Error::new(io::ErrorKind::InvalidInput, "KVC file size overflows")
|
| 378 |
+
})?;
|
| 379 |
+
if !file_size_fits(
|
| 380 |
+
self.budget_bytes,
|
| 381 |
+
text.len() as u64,
|
| 382 |
+
payload_bytes,
|
| 383 |
+
trailer.len() as u64,
|
| 384 |
+
) {
|
| 385 |
+
return Err(io::Error::new(
|
| 386 |
+
io::ErrorKind::OutOfMemory,
|
| 387 |
+
"KVC file exceeds budget",
|
| 388 |
+
));
|
| 389 |
+
}
|
| 390 |
+
header.text_bytes = text.len() as u32;
|
| 391 |
+
header.payload_bytes = payload_bytes;
|
| 392 |
+
let incoming = EvictionContext {
|
| 393 |
+
text,
|
| 394 |
+
model_id: header.model_id,
|
| 395 |
+
quant_bits: header.quant_bits,
|
| 396 |
+
ctx_size: header.ctx_size,
|
| 397 |
+
reject_different_quant: self.reject_different_quant,
|
| 398 |
+
};
|
| 399 |
+
let staged = stage_stream(&path, &header, text, &mut payload, payload_bytes, &trailer)
|
| 400 |
+
.map_err(format_io_error)?;
|
| 401 |
+
self.evict_excluding(extra, Some(&incoming), Some(&path));
|
| 402 |
+
if let Err(error) = fs::rename(&staged, &path) {
|
| 403 |
+
let _ = fs::remove_file(staged);
|
| 404 |
+
return Err(error);
|
| 405 |
+
}
|
| 406 |
+
self.refresh();
|
| 407 |
+
Ok(path)
|
| 408 |
+
}
|
| 409 |
+
|
| 410 |
+
pub fn read(&self, path: &Path) -> io::Result<Record> {
|
| 411 |
+
let envelope = read_envelope(path).map_err(format_io_error)?;
|
| 412 |
+
self.check_identity(path, &envelope)?;
|
| 413 |
+
let mut record = read_path(path).map_err(io::Error::other)?;
|
| 414 |
+
self.check_trailer(record.header.ext_flags, &record.trailer)?;
|
| 415 |
+
if self.identity.is_some() {
|
| 416 |
+
record
|
| 417 |
+
.trailer
|
| 418 |
+
.truncate(record.trailer.len().saturating_sub(crate::identity::BYTES));
|
| 419 |
+
record.header.ext_flags &= !crate::identity::FLAG;
|
| 420 |
+
}
|
| 421 |
+
Ok(record)
|
| 422 |
+
}
|
| 423 |
+
|
| 424 |
+
pub fn open_payload(&self, path: &Path, expected: &Envelope) -> io::Result<PayloadGuard> {
|
| 425 |
+
let file = self.payload_file(path, expected, FileAccess::ReadOnly)?;
|
| 426 |
+
#[cfg(target_os = "linux")]
|
| 427 |
+
let held_path = PathBuf::from(format!("/proc/self/fd/{}", file.as_raw_fd()));
|
| 428 |
+
#[cfg(all(unix, not(target_os = "linux")))]
|
| 429 |
+
let held_path = PathBuf::from(format!("/dev/fd/{}", file.as_raw_fd()));
|
| 430 |
+
#[cfg(not(unix))]
|
| 431 |
+
let held_path = {
|
| 432 |
+
return Err(io::Error::new(
|
| 433 |
+
io::ErrorKind::Unsupported,
|
| 434 |
+
"held KVC payloads need Unix descriptors",
|
| 435 |
+
));
|
| 436 |
+
};
|
| 437 |
+
Ok(PayloadGuard {
|
| 438 |
+
_file: file,
|
| 439 |
+
path: held_path,
|
| 440 |
+
})
|
| 441 |
+
}
|
| 442 |
+
|
| 443 |
+
fn payload_file(
|
| 444 |
+
&self,
|
| 445 |
+
path: &Path,
|
| 446 |
+
expected: &Envelope,
|
| 447 |
+
access: FileAccess,
|
| 448 |
+
) -> io::Result<fs::File> {
|
| 449 |
+
let mut file = fs::OpenOptions::new()
|
| 450 |
+
.read(true)
|
| 451 |
+
.write(access == FileAccess::ReadWrite)
|
| 452 |
+
.open(path)?;
|
| 453 |
+
let mut raw = [0; crate::format::FIXED_HEADER];
|
| 454 |
+
file.read_exact(&mut raw)?;
|
| 455 |
+
let mut text_len = [0; 4];
|
| 456 |
+
file.read_exact(&mut text_len)?;
|
| 457 |
+
let header = crate::format::parse_header(&raw, u32::from_le_bytes(text_len))
|
| 458 |
+
.map_err(format_io_error)?;
|
| 459 |
+
let payload_offset =
|
| 460 |
+
(crate::format::FIXED_HEADER + 4) as u64 + u64::from(header.text_bytes);
|
| 461 |
+
let file_size = file.metadata()?.len();
|
| 462 |
+
let payload_end = payload_offset
|
| 463 |
+
.checked_add(header.payload_bytes)
|
| 464 |
+
.filter(|end| *end <= file_size)
|
| 465 |
+
.ok_or_else(|| io::Error::new(io::ErrorKind::UnexpectedEof, "truncated KVC payload"))?;
|
| 466 |
+
let trailer_bytes = file_size - payload_end;
|
| 467 |
+
let mut footer = [0; crate::identity::BYTES];
|
| 468 |
+
let identity = if header.ext_flags & crate::identity::FLAG != 0
|
| 469 |
+
&& trailer_bytes >= footer.len() as u64
|
| 470 |
+
{
|
| 471 |
+
file.seek(SeekFrom::Start(file_size - footer.len() as u64))?;
|
| 472 |
+
file.read_exact(&mut footer)?;
|
| 473 |
+
crate::identity::decode(header.ext_flags, &footer)
|
| 474 |
+
} else {
|
| 475 |
+
None
|
| 476 |
+
};
|
| 477 |
+
let wanted = &expected.header;
|
| 478 |
+
if !self.matches_identity(header.ext_flags, identity)
|
| 479 |
+
|| payload_offset != expected.payload_offset
|
| 480 |
+
|| header.payload_bytes != wanted.payload_bytes
|
| 481 |
+
|| header.model_id != wanted.model_id
|
| 482 |
+
|| header.quant_bits != wanted.quant_bits
|
| 483 |
+
|| header.ctx_size != wanted.ctx_size
|
| 484 |
+
|| header.tokens != wanted.tokens
|
| 485 |
+
|| header.reason != wanted.reason
|
| 486 |
+
|| header.ext_flags & !crate::identity::FLAG != wanted.ext_flags
|
| 487 |
+
{
|
| 488 |
+
return Err(io::Error::new(
|
| 489 |
+
io::ErrorKind::InvalidData,
|
| 490 |
+
"KVC payload identity changed",
|
| 491 |
+
));
|
| 492 |
+
}
|
| 493 |
+
// Match the selected text on this same descriptor, in bounded chunks.
|
| 494 |
+
file.seek(SeekFrom::Start((crate::format::FIXED_HEADER + 4) as u64))?;
|
| 495 |
+
let mut buffer = [0; 4096];
|
| 496 |
+
for bytes in expected.text.chunks(buffer.len()) {
|
| 497 |
+
file.read_exact(&mut buffer[..bytes.len()])?;
|
| 498 |
+
if &buffer[..bytes.len()] != bytes {
|
| 499 |
+
return Err(io::Error::new(
|
| 500 |
+
io::ErrorKind::InvalidData,
|
| 501 |
+
"KVC payload text changed",
|
| 502 |
+
));
|
| 503 |
+
}
|
| 504 |
+
}
|
| 505 |
+
Ok(file)
|
| 506 |
+
}
|
| 507 |
+
|
| 508 |
+
/// Read tool metadata with the same required identity as payload candidates.
|
| 509 |
+
/// The identity footer is never exposed as a tool-map record.
|
| 510 |
+
pub fn read_trailer(&self, path: &Path, max_bytes: u64) -> io::Result<(Header, Vec<u8>)> {
|
| 511 |
+
let (mut header, mut trailer) =
|
| 512 |
+
crate::format::read_trailer(path, max_bytes).map_err(format_io_error)?;
|
| 513 |
+
self.check_trailer(header.ext_flags, &trailer)?;
|
| 514 |
+
if self.identity.is_some() {
|
| 515 |
+
trailer.truncate(trailer.len() - crate::identity::BYTES);
|
| 516 |
+
header.ext_flags &= !crate::identity::FLAG;
|
| 517 |
+
}
|
| 518 |
+
Ok((header, trailer))
|
| 519 |
+
}
|
| 520 |
+
|
| 521 |
+
fn check_trailer(&self, flags: u8, trailer: &[u8]) -> io::Result<()> {
|
| 522 |
+
if self.matches_identity(flags, crate::identity::decode(flags, trailer)) {
|
| 523 |
+
return Ok(());
|
| 524 |
+
}
|
| 525 |
+
Err(io::Error::new(
|
| 526 |
+
io::ErrorKind::InvalidData,
|
| 527 |
+
"KVC artifact/runtime identity mismatch",
|
| 528 |
+
))
|
| 529 |
+
}
|
| 530 |
+
|
| 531 |
+
pub fn find_text_prefix(
|
| 532 |
+
&mut self,
|
| 533 |
+
prompt: &[u8],
|
| 534 |
+
model_id: u8,
|
| 535 |
+
quant_bits: u8,
|
| 536 |
+
ctx_size: u32,
|
| 537 |
+
) -> Option<usize> {
|
| 538 |
+
self.find_prefix(prompt, model_id, quant_bits, ctx_size, false)
|
| 539 |
+
}
|
| 540 |
+
|
| 541 |
+
/// Reopen the selected file and repeat the C loader's header, text, and
|
| 542 |
+
/// filename-hash checks before a native payload restore consumes it.
|
| 543 |
+
pub fn text_prefix_candidate(
|
| 544 |
+
&mut self,
|
| 545 |
+
prompt: &[u8],
|
| 546 |
+
model_id: u8,
|
| 547 |
+
quant_bits: u8,
|
| 548 |
+
ctx_size: u32,
|
| 549 |
+
) -> io::Result<Option<(PathBuf, Envelope)>> {
|
| 550 |
+
self.prefix_candidate(prompt, model_id, quant_bits, ctx_size, false)
|
| 551 |
+
}
|
| 552 |
+
|
| 553 |
+
pub fn bank_text_prefix_candidate(
|
| 554 |
+
&mut self,
|
| 555 |
+
prompt: &[u8],
|
| 556 |
+
model_id: u8,
|
| 557 |
+
quant_bits: u8,
|
| 558 |
+
ctx_size: u32,
|
| 559 |
+
) -> io::Result<Option<(PathBuf, Envelope)>> {
|
| 560 |
+
self.prefix_candidate(prompt, model_id, quant_bits, ctx_size, true)
|
| 561 |
+
}
|
| 562 |
+
|
| 563 |
+
pub fn bank_text_prefix_candidate_identity(
|
| 564 |
+
&mut self,
|
| 565 |
+
prompt: &[u8],
|
| 566 |
+
model_id: u8,
|
| 567 |
+
quant_bits: u8,
|
| 568 |
+
ctx_size: u32,
|
| 569 |
+
identity_flags: u8,
|
| 570 |
+
) -> io::Result<Option<(PathBuf, Envelope)>> {
|
| 571 |
+
self.prefix_candidate_identity(
|
| 572 |
+
prompt,
|
| 573 |
+
model_id,
|
| 574 |
+
quant_bits,
|
| 575 |
+
ctx_size,
|
| 576 |
+
true,
|
| 577 |
+
EXT_IMAGE_PIXELS_V2,
|
| 578 |
+
identity_flags,
|
| 579 |
+
)
|
| 580 |
+
}
|
| 581 |
+
|
| 582 |
+
pub fn bank_text_lcp_candidate(
|
| 583 |
+
&mut self,
|
| 584 |
+
prompt: &[u8],
|
| 585 |
+
model_id: u8,
|
| 586 |
+
quant_bits: u8,
|
| 587 |
+
ctx_size: u32,
|
| 588 |
+
min_lcp: usize,
|
| 589 |
+
) -> io::Result<Option<(PathBuf, Envelope, usize)>> {
|
| 590 |
+
self.bank_text_lcp_candidate_with_identity(
|
| 591 |
+
prompt, model_id, quant_bits, ctx_size, min_lcp, 0, 0,
|
| 592 |
+
)
|
| 593 |
+
}
|
| 594 |
+
|
| 595 |
+
pub fn bank_text_lcp_candidate_identity(
|
| 596 |
+
&mut self,
|
| 597 |
+
prompt: &[u8],
|
| 598 |
+
model_id: u8,
|
| 599 |
+
quant_bits: u8,
|
| 600 |
+
ctx_size: u32,
|
| 601 |
+
min_lcp: usize,
|
| 602 |
+
identity_flags: u8,
|
| 603 |
+
) -> io::Result<Option<(PathBuf, Envelope, usize)>> {
|
| 604 |
+
self.bank_text_lcp_candidate_with_identity(
|
| 605 |
+
prompt,
|
| 606 |
+
model_id,
|
| 607 |
+
quant_bits,
|
| 608 |
+
ctx_size,
|
| 609 |
+
min_lcp,
|
| 610 |
+
EXT_IMAGE_PIXELS_V2,
|
| 611 |
+
identity_flags,
|
| 612 |
+
)
|
| 613 |
+
}
|
| 614 |
+
|
| 615 |
+
fn bank_text_lcp_candidate_with_identity(
|
| 616 |
+
&mut self,
|
| 617 |
+
prompt: &[u8],
|
| 618 |
+
model_id: u8,
|
| 619 |
+
quant_bits: u8,
|
| 620 |
+
ctx_size: u32,
|
| 621 |
+
min_lcp: usize,
|
| 622 |
+
identity_mask: u8,
|
| 623 |
+
identity_flags: u8,
|
| 624 |
+
) -> io::Result<Option<(PathBuf, Envelope, usize)>> {
|
| 625 |
+
let Some((index, _)) = self.find_text_lcp_with_identity(
|
| 626 |
+
prompt,
|
| 627 |
+
model_id,
|
| 628 |
+
quant_bits,
|
| 629 |
+
ctx_size,
|
| 630 |
+
min_lcp,
|
| 631 |
+
identity_mask,
|
| 632 |
+
identity_flags,
|
| 633 |
+
) else {
|
| 634 |
+
return Ok(None);
|
| 635 |
+
};
|
| 636 |
+
let entry = self.entries[index].clone();
|
| 637 |
+
let envelope = read_envelope(&entry.path).map_err(format_io_error)?;
|
| 638 |
+
let header = &envelope.header;
|
| 639 |
+
let unchanged = header.model_id == entry.header.model_id
|
| 640 |
+
&& header.quant_bits == entry.header.quant_bits
|
| 641 |
+
&& header.ctx_size == entry.header.ctx_size
|
| 642 |
+
&& header.tokens == entry.header.tokens
|
| 643 |
+
&& header.text_bytes == entry.header.text_bytes;
|
| 644 |
+
let lcp = prompt
|
| 645 |
+
.iter()
|
| 646 |
+
.zip(&envelope.text)
|
| 647 |
+
.take_while(|(left, right)| left == right)
|
| 648 |
+
.count();
|
| 649 |
+
// Corruption past the shared prefix still leaves the scan's answer
|
| 650 |
+
// standing, so check the record against its own name here too.
|
| 651 |
+
if !unchanged || text_sha_hex(&envelope.text) != entry.sha {
|
| 652 |
+
return Err(io::Error::new(
|
| 653 |
+
io::ErrorKind::InvalidData,
|
| 654 |
+
"KVC record does not match its name",
|
| 655 |
+
));
|
| 656 |
+
}
|
| 657 |
+
|
| 658 |
+
if !self.envelope_identity(&entry.path, &envelope)
|
| 659 |
+
|| !is_automatic_exact_replay(header.reason, header.ext_flags)
|
| 660 |
+
|| header.model_id != model_id
|
| 661 |
+
|| (self.reject_different_quant && header.quant_bits != quant_bits)
|
| 662 |
+
|| header.ctx_size > ctx_size
|
| 663 |
+
|| header.ext_flags & identity_mask != identity_flags & identity_mask
|
| 664 |
+
|| lcp < min_lcp
|
| 665 |
+
|| 8 * (lcp as u64) < u64::from(header.text_bytes)
|
| 666 |
+
{
|
| 667 |
+
return Ok(None);
|
| 668 |
+
}
|
| 669 |
+
Ok(Some((entry.path, self.visible_envelope(envelope), lcp)))
|
| 670 |
+
}
|
| 671 |
+
|
| 672 |
+
fn prefix_candidate(
|
| 673 |
+
&mut self,
|
| 674 |
+
prompt: &[u8],
|
| 675 |
+
model_id: u8,
|
| 676 |
+
quant_bits: u8,
|
| 677 |
+
ctx_size: u32,
|
| 678 |
+
require_suffix: bool,
|
| 679 |
+
) -> io::Result<Option<(PathBuf, Envelope)>> {
|
| 680 |
+
self.prefix_candidate_identity(prompt, model_id, quant_bits, ctx_size, require_suffix, 0, 0)
|
| 681 |
+
}
|
| 682 |
+
|
| 683 |
+
fn prefix_candidate_identity(
|
| 684 |
+
&mut self,
|
| 685 |
+
prompt: &[u8],
|
| 686 |
+
model_id: u8,
|
| 687 |
+
quant_bits: u8,
|
| 688 |
+
ctx_size: u32,
|
| 689 |
+
require_suffix: bool,
|
| 690 |
+
identity_mask: u8,
|
| 691 |
+
identity_flags: u8,
|
| 692 |
+
) -> io::Result<Option<(PathBuf, Envelope)>> {
|
| 693 |
+
let index = if require_suffix {
|
| 694 |
+
self.find_prefix_with_identity(
|
| 695 |
+
prompt,
|
| 696 |
+
model_id,
|
| 697 |
+
quant_bits,
|
| 698 |
+
ctx_size,
|
| 699 |
+
true,
|
| 700 |
+
identity_mask,
|
| 701 |
+
identity_flags,
|
| 702 |
+
)
|
| 703 |
+
} else {
|
| 704 |
+
self.find_prefix_with_identity(
|
| 705 |
+
prompt,
|
| 706 |
+
model_id,
|
| 707 |
+
quant_bits,
|
| 708 |
+
ctx_size,
|
| 709 |
+
false,
|
| 710 |
+
identity_mask,
|
| 711 |
+
identity_flags,
|
| 712 |
+
)
|
| 713 |
+
};
|
| 714 |
+
let Some(index) = index else {
|
| 715 |
+
return Ok(None);
|
| 716 |
+
};
|
| 717 |
+
let entry = self.entries[index].clone();
|
| 718 |
+
let envelope = read_envelope(&entry.path).map_err(format_io_error)?;
|
| 719 |
+
let header = &envelope.header;
|
| 720 |
+
let unchanged = header.model_id == entry.header.model_id
|
| 721 |
+
&& header.quant_bits == entry.header.quant_bits
|
| 722 |
+
&& header.ctx_size == entry.header.ctx_size
|
| 723 |
+
&& header.tokens == entry.header.tokens
|
| 724 |
+
&& header.text_bytes == entry.header.text_bytes;
|
| 725 |
+
// The record no longer describes itself: its text does not hash to
|
| 726 |
+
// its own name, it is not this prompt's prefix after all, or the
|
| 727 |
+
// header moved under the read. The candidate was selected and then
|
| 728 |
+
// refused by its own contents, which is not an absence.
|
| 729 |
+
if !unchanged
|
| 730 |
+
|| text_sha_hex(&envelope.text) != entry.sha
|
| 731 |
+
|| !prompt.starts_with(&envelope.text)
|
| 732 |
+
{
|
| 733 |
+
return Err(io::Error::new(
|
| 734 |
+
io::ErrorKind::InvalidData,
|
| 735 |
+
"KVC record does not match its name",
|
| 736 |
+
));
|
| 737 |
+
}
|
| 738 |
+
|
| 739 |
+
// Sound, but not usable in this shape: nothing left to prefill, or
|
| 740 |
+
// an identity this search does not take.
|
| 741 |
+
let bank_without_logits = is_bank_replay_v1(header.reason, header.ext_flags)
|
| 742 |
+
&& envelope.text.len() == prompt.len();
|
| 743 |
+
let missing_suffix = require_suffix && envelope.text.len() >= prompt.len();
|
| 744 |
+
if !self.envelope_identity(&entry.path, &envelope)
|
| 745 |
+
|| !is_automatic_exact_replay(header.reason, header.ext_flags)
|
| 746 |
+
|| header.model_id != model_id
|
| 747 |
+
|| header.ext_flags & identity_mask != identity_flags & identity_mask
|
| 748 |
+
|| bank_without_logits
|
| 749 |
+
|| missing_suffix
|
| 750 |
+
{
|
| 751 |
+
return Ok(None);
|
| 752 |
+
}
|
| 753 |
+
Ok(Some((entry.path, self.visible_envelope(envelope))))
|
| 754 |
+
}
|
| 755 |
+
|
| 756 |
+
pub fn touch_hit(&mut self, path: &Path) -> io::Result<()> {
|
| 757 |
+
let envelope = read_envelope(path).map_err(format_io_error)?;
|
| 758 |
+
self.check_identity(path, &envelope)?;
|
| 759 |
+
let mut file = self.payload_file(
|
| 760 |
+
path,
|
| 761 |
+
&self.visible_envelope(envelope.clone()),
|
| 762 |
+
FileAccess::ReadWrite,
|
| 763 |
+
)?;
|
| 764 |
+
let header = envelope.header;
|
| 765 |
+
let raw = fill_header(
|
| 766 |
+
header.model_id,
|
| 767 |
+
header.quant_bits,
|
| 768 |
+
header.reason,
|
| 769 |
+
header.ext_flags,
|
| 770 |
+
header.tokens,
|
| 771 |
+
header.hits.wrapping_add(1),
|
| 772 |
+
header.ctx_size,
|
| 773 |
+
header.created_at,
|
| 774 |
+
now_secs(),
|
| 775 |
+
header.payload_bytes,
|
| 776 |
+
);
|
| 777 |
+
file.seek(SeekFrom::Start(0))?;
|
| 778 |
+
file.write_all(&raw)?;
|
| 779 |
+
file.flush()?;
|
| 780 |
+
self.refresh();
|
| 781 |
+
Ok(())
|
| 782 |
+
}
|
| 783 |
+
|
| 784 |
+
pub fn discard(&mut self, path: &Path) -> io::Result<()> {
|
| 785 |
+
self.discard_inner(path, true)
|
| 786 |
+
}
|
| 787 |
+
|
| 788 |
+
pub fn discard_bank(&mut self, path: &Path) -> io::Result<()> {
|
| 789 |
+
self.discard_inner(path, false)
|
| 790 |
+
}
|
| 791 |
+
|
| 792 |
+
fn discard_inner(&mut self, path: &Path, reset_continued: bool) -> io::Result<()> {
|
| 793 |
+
if !self.entries.iter().any(|entry| entry.path == path) {
|
| 794 |
+
return Err(io::Error::new(
|
| 795 |
+
io::ErrorKind::InvalidInput,
|
| 796 |
+
"KVC discard path is not a catalog entry",
|
| 797 |
+
));
|
| 798 |
+
}
|
| 799 |
+
if reset_continued {
|
| 800 |
+
self.continued_last_store_tokens = 0;
|
| 801 |
+
}
|
| 802 |
+
match fs::remove_file(path) {
|
| 803 |
+
Ok(()) => {}
|
| 804 |
+
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
| 805 |
+
Err(error) => return Err(error),
|
| 806 |
+
}
|
| 807 |
+
self.refresh();
|
| 808 |
+
Ok(())
|
| 809 |
+
}
|
| 810 |
+
|
| 811 |
+
pub fn find_bank_text_prefix(
|
| 812 |
+
&mut self,
|
| 813 |
+
prompt: &[u8],
|
| 814 |
+
model_id: u8,
|
| 815 |
+
quant_bits: u8,
|
| 816 |
+
ctx_size: u32,
|
| 817 |
+
) -> Option<usize> {
|
| 818 |
+
self.find_prefix(prompt, model_id, quant_bits, ctx_size, true)
|
| 819 |
+
}
|
| 820 |
+
|
| 821 |
+
fn find_prefix(
|
| 822 |
+
&mut self,
|
| 823 |
+
prompt: &[u8],
|
| 824 |
+
model_id: u8,
|
| 825 |
+
quant_bits: u8,
|
| 826 |
+
ctx_size: u32,
|
| 827 |
+
require_suffix: bool,
|
| 828 |
+
) -> Option<usize> {
|
| 829 |
+
self.find_prefix_with_identity(prompt, model_id, quant_bits, ctx_size, require_suffix, 0, 0)
|
| 830 |
+
}
|
| 831 |
+
|
| 832 |
+
fn find_prefix_with_identity(
|
| 833 |
+
&mut self,
|
| 834 |
+
prompt: &[u8],
|
| 835 |
+
model_id: u8,
|
| 836 |
+
quant_bits: u8,
|
| 837 |
+
ctx_size: u32,
|
| 838 |
+
require_suffix: bool,
|
| 839 |
+
identity_mask: u8,
|
| 840 |
+
identity_flags: u8,
|
| 841 |
+
) -> Option<usize> {
|
| 842 |
+
self.refresh();
|
| 843 |
+
let prompt_bytes = prompt.len();
|
| 844 |
+
let mut best: Option<usize> = None;
|
| 845 |
+
for (i, e) in self.entries.iter().enumerate() {
|
| 846 |
+
if !self.matches_identity(e.header.ext_flags, e.identity)
|
| 847 |
+
|| !is_automatic_exact_replay(e.header.reason, e.header.ext_flags)
|
| 848 |
+
{
|
| 849 |
+
continue;
|
| 850 |
+
}
|
| 851 |
+
if e.header.ext_flags & identity_mask != identity_flags & identity_mask {
|
| 852 |
+
continue;
|
| 853 |
+
}
|
| 854 |
+
if e.header.text_bytes as usize > prompt_bytes {
|
| 855 |
+
continue;
|
| 856 |
+
}
|
| 857 |
+
if (require_suffix || is_bank_replay_v1(e.header.reason, e.header.ext_flags))
|
| 858 |
+
&& e.header.text_bytes as usize == prompt_bytes
|
| 859 |
+
{
|
| 860 |
+
continue;
|
| 861 |
+
}
|
| 862 |
+
if (e.header.tokens as i32) < self.opt.min_tokens {
|
| 863 |
+
continue;
|
| 864 |
+
}
|
| 865 |
+
if e.header.model_id != model_id {
|
| 866 |
+
continue;
|
| 867 |
+
}
|
| 868 |
+
if ctx_size < e.header.ctx_size {
|
| 869 |
+
continue;
|
| 870 |
+
}
|
| 871 |
+
if self.reject_different_quant && e.header.quant_bits != quant_bits {
|
| 872 |
+
continue;
|
| 873 |
+
}
|
| 874 |
+
if let Some(b) = best {
|
| 875 |
+
let be = &self.entries[b];
|
| 876 |
+
if (e.header.text_bytes as usize) < be.header.text_bytes as usize {
|
| 877 |
+
continue;
|
| 878 |
+
}
|
| 879 |
+
if e.header.text_bytes == be.header.text_bytes
|
| 880 |
+
&& e.header.tokens <= be.header.tokens
|
| 881 |
+
{
|
| 882 |
+
continue;
|
| 883 |
+
}
|
| 884 |
+
}
|
| 885 |
+
let sha = text_sha_hex(&prompt[..e.header.text_bytes as usize]);
|
| 886 |
+
if sha == e.sha {
|
| 887 |
+
best = Some(i);
|
| 888 |
+
}
|
| 889 |
+
}
|
| 890 |
+
best
|
| 891 |
+
}
|
| 892 |
+
|
| 893 |
+
/// What the serial search would do with this prompt: it keys on the
|
| 894 |
+
/// rendered text and may replay a record of exactly that length.
|
| 895 |
+
pub fn prefix_answer(
|
| 896 |
+
&mut self,
|
| 897 |
+
prompt: &[u8],
|
| 898 |
+
model_id: u8,
|
| 899 |
+
quant_bits: u8,
|
| 900 |
+
ctx_size: u32,
|
| 901 |
+
) -> PrefixAnswer {
|
| 902 |
+
self.prefix_answer_masked(
|
| 903 |
+
prompt,
|
| 904 |
+
model_id,
|
| 905 |
+
quant_bits,
|
| 906 |
+
ctx_size,
|
| 907 |
+
Suffix::Optional,
|
| 908 |
+
0,
|
| 909 |
+
0,
|
| 910 |
+
)
|
| 911 |
+
}
|
| 912 |
+
|
| 913 |
+
/// What the bank search would do: it keys an image request by its
|
| 914 |
+
/// media-marked cache text and admits only records it can extend, so
|
| 915 |
+
/// asking about the rendered prompt would see nothing at all.
|
| 916 |
+
pub fn bank_prefix_answer(
|
| 917 |
+
&mut self,
|
| 918 |
+
prompt: &[u8],
|
| 919 |
+
model_id: u8,
|
| 920 |
+
quant_bits: u8,
|
| 921 |
+
ctx_size: u32,
|
| 922 |
+
identity_flags: u8,
|
| 923 |
+
) -> PrefixAnswer {
|
| 924 |
+
self.prefix_answer_masked(
|
| 925 |
+
prompt,
|
| 926 |
+
model_id,
|
| 927 |
+
quant_bits,
|
| 928 |
+
ctx_size,
|
| 929 |
+
Suffix::Required,
|
| 930 |
+
EXT_IMAGE_PIXELS_V2,
|
| 931 |
+
identity_flags,
|
| 932 |
+
)
|
| 933 |
+
}
|
| 934 |
+
|
| 935 |
+
fn prefix_answer_masked(
|
| 936 |
+
&mut self,
|
| 937 |
+
prompt: &[u8],
|
| 938 |
+
model_id: u8,
|
| 939 |
+
quant_bits: u8,
|
| 940 |
+
ctx_size: u32,
|
| 941 |
+
suffix: Suffix,
|
| 942 |
+
identity_mask: u8,
|
| 943 |
+
identity_flags: u8,
|
| 944 |
+
) -> PrefixAnswer {
|
| 945 |
+
self.refresh();
|
| 946 |
+
let reject_quant = self.reject_different_quant;
|
| 947 |
+
let min_tokens = self.opt.min_tokens;
|
| 948 |
+
let mut answer = PrefixAnswer::None;
|
| 949 |
+
|
| 950 |
+
for e in &self.entries {
|
| 951 |
+
if !is_automatic_exact_replay(e.header.reason, e.header.ext_flags)
|
| 952 |
+
|| e.header.text_bytes as usize > prompt.len()
|
| 953 |
+
{
|
| 954 |
+
continue;
|
| 955 |
+
}
|
| 956 |
+
let exact_length = e.header.text_bytes as usize == prompt.len();
|
| 957 |
+
if suffix == Suffix::Required && exact_length {
|
| 958 |
+
// Nothing left to prefill from it in this shape.
|
| 959 |
+
continue;
|
| 960 |
+
}
|
| 961 |
+
if text_sha_hex(&prompt[..e.header.text_bytes as usize]) != e.sha {
|
| 962 |
+
continue;
|
| 963 |
+
}
|
| 964 |
+
|
| 965 |
+
// A bank snapshot carries no logits, so an exact replay cannot
|
| 966 |
+
// use one: the record is there and its layout refuses. The media
|
| 967 |
+
// flag is identity in the same way — a record keyed by this text
|
| 968 |
+
// without it is a mismatch, not one about another conversation.
|
| 969 |
+
let compatible = !(exact_length
|
| 970 |
+
&& is_bank_replay_v1(e.header.reason, e.header.ext_flags))
|
| 971 |
+
&& self.matches_identity(e.header.ext_flags, e.identity)
|
| 972 |
+
&& e.header.model_id == model_id
|
| 973 |
+
&& ctx_size >= e.header.ctx_size
|
| 974 |
+
&& (!reject_quant || e.header.quant_bits == quant_bits)
|
| 975 |
+
&& e.header.ext_flags & identity_mask == identity_flags & identity_mask;
|
| 976 |
+
if compatible && (e.header.tokens as i32) >= min_tokens {
|
| 977 |
+
return PrefixAnswer::Usable;
|
| 978 |
+
}
|
| 979 |
+
let refused = if compatible {
|
| 980 |
+
PrefixAnswer::Shallow
|
| 981 |
+
} else {
|
| 982 |
+
PrefixAnswer::Mismatch
|
| 983 |
+
};
|
| 984 |
+
answer = weaker_answer(answer, refused);
|
| 985 |
+
}
|
| 986 |
+
|
| 987 |
+
// A record whose payload was truncated away is not in the catalog,
|
| 988 |
+
// but its name still says it was keyed by this text. It is why the
|
| 989 |
+
// prompt finds nothing, and that is a mismatch.
|
| 990 |
+
if self.damaged_prefix(prompt, suffix) {
|
| 991 |
+
answer = weaker_answer(answer, PrefixAnswer::Mismatch);
|
| 992 |
+
}
|
| 993 |
+
|
| 994 |
+
answer
|
| 995 |
+
}
|
| 996 |
+
|
| 997 |
+
/// A file the catalog dropped that still shares this prompt's required
|
| 998 |
+
/// prefix. Its payload is gone; the text in front of it need not be.
|
| 999 |
+
fn damaged_lcp(&self, prompt: &[u8], min_lcp: usize) -> bool {
|
| 1000 |
+
self.damaged.iter().any(|d| {
|
| 1001 |
+
if !is_automatic_exact_replay(d.reason, d.ext_flags)
|
| 1002 |
+
|| (d.text_bytes as usize) < min_lcp
|
| 1003 |
+
|| u64::from(d.text_bytes) > 8 * prompt.len() as u64
|
| 1004 |
+
{
|
| 1005 |
+
return false;
|
| 1006 |
+
}
|
| 1007 |
+
let want = (d.text_bytes as usize).min(prompt.len());
|
| 1008 |
+
let Ok((_, text)) = read_header_text(&d.path, want) else {
|
| 1009 |
+
return false;
|
| 1010 |
+
};
|
| 1011 |
+
let lcp = text
|
| 1012 |
+
.iter()
|
| 1013 |
+
.zip(prompt)
|
| 1014 |
+
.take_while(|(stored, asked)| stored == asked)
|
| 1015 |
+
.count();
|
| 1016 |
+
lcp >= min_lcp && 8 * (lcp as u64) >= u64::from(d.text_bytes)
|
| 1017 |
+
})
|
| 1018 |
+
}
|
| 1019 |
+
|
| 1020 |
+
/// A file the catalog dropped, keyed by a prefix of this prompt.
|
| 1021 |
+
fn damaged_prefix(&self, prompt: &[u8], suffix: Suffix) -> bool {
|
| 1022 |
+
self.damaged.iter().any(|d| {
|
| 1023 |
+
let text_bytes = d.text_bytes as usize;
|
| 1024 |
+
if !is_automatic_exact_replay(d.reason, d.ext_flags)
|
| 1025 |
+
|| text_bytes > prompt.len()
|
| 1026 |
+
|| (suffix == Suffix::Required && text_bytes == prompt.len())
|
| 1027 |
+
{
|
| 1028 |
+
return false;
|
| 1029 |
+
}
|
| 1030 |
+
text_sha_hex(&prompt[..text_bytes]) == d.sha
|
| 1031 |
+
})
|
| 1032 |
+
}
|
| 1033 |
+
|
| 1034 |
+
/// The LCP form: an edited prompt diverges from the record it shares a
|
| 1035 |
+
/// prefix with, so the exact question cannot see it at all. `min_lcp`
|
| 1036 |
+
/// is the partial minimum the bank search admits from.
|
| 1037 |
+
pub fn bank_lcp_answer(
|
| 1038 |
+
&mut self,
|
| 1039 |
+
prompt: &[u8],
|
| 1040 |
+
model_id: u8,
|
| 1041 |
+
quant_bits: u8,
|
| 1042 |
+
ctx_size: u32,
|
| 1043 |
+
min_lcp: usize,
|
| 1044 |
+
identity_flags: u8,
|
| 1045 |
+
) -> PrefixAnswer {
|
| 1046 |
+
if min_lcp == 0 || prompt.len() < min_lcp {
|
| 1047 |
+
return PrefixAnswer::None;
|
| 1048 |
+
}
|
| 1049 |
+
self.refresh();
|
| 1050 |
+
let reject_quant = self.reject_different_quant;
|
| 1051 |
+
let min_tokens = self.opt.min_tokens;
|
| 1052 |
+
let mut answer = PrefixAnswer::None;
|
| 1053 |
+
|
| 1054 |
+
for e in &self.entries {
|
| 1055 |
+
if !is_automatic_exact_replay(e.header.reason, e.header.ext_flags)
|
| 1056 |
+
|| (e.header.text_bytes as usize) < min_lcp
|
| 1057 |
+
|| u64::from(e.header.text_bytes) > 8 * prompt.len() as u64
|
| 1058 |
+
{
|
| 1059 |
+
continue;
|
| 1060 |
+
}
|
| 1061 |
+
let want = (e.header.text_bytes as usize).min(prompt.len());
|
| 1062 |
+
let Ok((metadata, text)) = read_text_prefix(&e.path, want) else {
|
| 1063 |
+
continue;
|
| 1064 |
+
};
|
| 1065 |
+
if metadata.header.text_bytes != e.header.text_bytes {
|
| 1066 |
+
continue;
|
| 1067 |
+
}
|
| 1068 |
+
let lcp = text
|
| 1069 |
+
.iter()
|
| 1070 |
+
.zip(prompt)
|
| 1071 |
+
.take_while(|(stored, asked)| stored == asked)
|
| 1072 |
+
.count();
|
| 1073 |
+
if lcp < min_lcp || 8 * (lcp as u64) < u64::from(e.header.text_bytes) {
|
| 1074 |
+
continue;
|
| 1075 |
+
}
|
| 1076 |
+
|
| 1077 |
+
let compatible = self.matches_identity(e.header.ext_flags, e.identity)
|
| 1078 |
+
&& e.header.model_id == model_id
|
| 1079 |
+
&& ctx_size >= e.header.ctx_size
|
| 1080 |
+
&& (!reject_quant || e.header.quant_bits == quant_bits)
|
| 1081 |
+
&& e.header.ext_flags & EXT_IMAGE_PIXELS_V2 == identity_flags & EXT_IMAGE_PIXELS_V2;
|
| 1082 |
+
if compatible && (e.header.tokens as i32) >= min_tokens {
|
| 1083 |
+
return PrefixAnswer::Usable;
|
| 1084 |
+
}
|
| 1085 |
+
let refused = if compatible {
|
| 1086 |
+
PrefixAnswer::Shallow
|
| 1087 |
+
} else {
|
| 1088 |
+
PrefixAnswer::Mismatch
|
| 1089 |
+
};
|
| 1090 |
+
answer = weaker_answer(answer, refused);
|
| 1091 |
+
}
|
| 1092 |
+
|
| 1093 |
+
// An edited prompt diverges from the record, so the damaged list has
|
| 1094 |
+
// to be read through the same shared prefix, not by name. A record
|
| 1095 |
+
// no search would replay from is not a refusal either.
|
| 1096 |
+
if self.damaged_lcp(prompt, min_lcp) {
|
| 1097 |
+
answer = weaker_answer(answer, PrefixAnswer::Mismatch);
|
| 1098 |
+
}
|
| 1099 |
+
|
| 1100 |
+
answer
|
| 1101 |
+
}
|
| 1102 |
+
|
| 1103 |
+
pub fn find_text_lcp(
|
| 1104 |
+
&mut self,
|
| 1105 |
+
prompt: &[u8],
|
| 1106 |
+
model_id: u8,
|
| 1107 |
+
quant_bits: u8,
|
| 1108 |
+
ctx_size: u32,
|
| 1109 |
+
min_lcp: usize,
|
| 1110 |
+
) -> Option<(usize, usize)> {
|
| 1111 |
+
self.find_text_lcp_with_identity(prompt, model_id, quant_bits, ctx_size, min_lcp, 0, 0)
|
| 1112 |
+
}
|
| 1113 |
+
|
| 1114 |
+
fn find_text_lcp_with_identity(
|
| 1115 |
+
&mut self,
|
| 1116 |
+
prompt: &[u8],
|
| 1117 |
+
model_id: u8,
|
| 1118 |
+
quant_bits: u8,
|
| 1119 |
+
ctx_size: u32,
|
| 1120 |
+
min_lcp: usize,
|
| 1121 |
+
identity_mask: u8,
|
| 1122 |
+
identity_flags: u8,
|
| 1123 |
+
) -> Option<(usize, usize)> {
|
| 1124 |
+
if min_lcp == 0 || prompt.len() < min_lcp {
|
| 1125 |
+
return None;
|
| 1126 |
+
}
|
| 1127 |
+
self.refresh();
|
| 1128 |
+
let mut best: Option<(usize, usize, u64)> = None;
|
| 1129 |
+
for (i, e) in self.entries.iter().enumerate() {
|
| 1130 |
+
if !self.matches_identity(e.header.ext_flags, e.identity)
|
| 1131 |
+
|| !is_automatic_exact_replay(e.header.reason, e.header.ext_flags)
|
| 1132 |
+
{
|
| 1133 |
+
continue;
|
| 1134 |
+
}
|
| 1135 |
+
if e.header.ext_flags & identity_mask != identity_flags & identity_mask {
|
| 1136 |
+
continue;
|
| 1137 |
+
}
|
| 1138 |
+
if (e.header.tokens as i32) < self.opt.min_tokens {
|
| 1139 |
+
continue;
|
| 1140 |
+
}
|
| 1141 |
+
if e.header.model_id != model_id || ctx_size < e.header.ctx_size {
|
| 1142 |
+
continue;
|
| 1143 |
+
}
|
| 1144 |
+
if self.reject_different_quant && e.header.quant_bits != quant_bits {
|
| 1145 |
+
continue;
|
| 1146 |
+
}
|
| 1147 |
+
if (e.header.text_bytes as usize) < min_lcp {
|
| 1148 |
+
continue;
|
| 1149 |
+
}
|
| 1150 |
+
if u64::from(e.header.text_bytes) > 8 * prompt.len() as u64 {
|
| 1151 |
+
continue;
|
| 1152 |
+
}
|
| 1153 |
+
let want = (e.header.text_bytes as usize).min(prompt.len());
|
| 1154 |
+
let Ok((metadata, text)) = read_text_prefix(&e.path, want) else {
|
| 1155 |
+
continue;
|
| 1156 |
+
};
|
| 1157 |
+
if metadata.header.text_bytes != e.header.text_bytes {
|
| 1158 |
+
continue;
|
| 1159 |
+
}
|
| 1160 |
+
let mut lcp = 0;
|
| 1161 |
+
while lcp < text.len() && text[lcp] == prompt[lcp] {
|
| 1162 |
+
lcp += 1;
|
| 1163 |
+
}
|
| 1164 |
+
if lcp < min_lcp {
|
| 1165 |
+
continue;
|
| 1166 |
+
}
|
| 1167 |
+
if 8 * (lcp as u64) < u64::from(e.header.text_bytes) {
|
| 1168 |
+
continue;
|
| 1169 |
+
}
|
| 1170 |
+
if let Some((_, best_lcp, best_text)) = best {
|
| 1171 |
+
if lcp < best_lcp {
|
| 1172 |
+
continue;
|
| 1173 |
+
}
|
| 1174 |
+
if lcp == best_lcp && u64::from(e.header.text_bytes) >= best_text {
|
| 1175 |
+
continue;
|
| 1176 |
+
}
|
| 1177 |
+
}
|
| 1178 |
+
best = Some((i, lcp, u64::from(e.header.text_bytes)));
|
| 1179 |
+
}
|
| 1180 |
+
best.map(|(i, lcp, _)| (i, lcp))
|
| 1181 |
+
}
|
| 1182 |
+
|
| 1183 |
+
pub fn evict(&mut self, extra_bytes: u64, incoming: Option<&EvictionContext<'_>>) {
|
| 1184 |
+
self.evict_excluding(extra_bytes, incoming, None);
|
| 1185 |
+
}
|
| 1186 |
+
|
| 1187 |
+
fn evict_excluding(
|
| 1188 |
+
&mut self,
|
| 1189 |
+
extra_bytes: u64,
|
| 1190 |
+
incoming: Option<&EvictionContext<'_>>,
|
| 1191 |
+
protected: Option<&Path>,
|
| 1192 |
+
) {
|
| 1193 |
+
if self.budget_bytes == 0 || extra_bytes > self.budget_bytes {
|
| 1194 |
+
return;
|
| 1195 |
+
}
|
| 1196 |
+
self.refresh();
|
| 1197 |
+
let now = now_secs();
|
| 1198 |
+
let target = self.budget_bytes - extra_bytes;
|
| 1199 |
+
loop {
|
| 1200 |
+
let total: u64 = self
|
| 1201 |
+
.entries
|
| 1202 |
+
.iter()
|
| 1203 |
+
.filter(|e| protected.is_none_or(|path| path != e.path))
|
| 1204 |
+
.map(|e| e.file_size)
|
| 1205 |
+
.sum();
|
| 1206 |
+
if total <= target || self.entries.is_empty() {
|
| 1207 |
+
break;
|
| 1208 |
+
}
|
| 1209 |
+
let mut victim: Option<usize> = None;
|
| 1210 |
+
let mut victim_score = 0.0;
|
| 1211 |
+
for i in 0..self.entries.len() {
|
| 1212 |
+
if protected.is_some_and(|path| path == self.entries[i].path) {
|
| 1213 |
+
continue;
|
| 1214 |
+
}
|
| 1215 |
+
let score = self.score_at(i, now, incoming);
|
| 1216 |
+
if victim.is_none()
|
| 1217 |
+
|| score < victim_score
|
| 1218 |
+
|| (score == victim_score
|
| 1219 |
+
&& self.entries[i].header.last_used
|
| 1220 |
+
< self.entries[victim.unwrap()].header.last_used)
|
| 1221 |
+
{
|
| 1222 |
+
victim = Some(i);
|
| 1223 |
+
victim_score = score;
|
| 1224 |
+
}
|
| 1225 |
+
}
|
| 1226 |
+
let Some(victim) = victim else { break };
|
| 1227 |
+
let path = self.entries[victim].path.clone();
|
| 1228 |
+
let unlinked = fs::remove_file(path).is_ok();
|
| 1229 |
+
self.entries.remove(victim);
|
| 1230 |
+
if !unlinked {
|
| 1231 |
+
break;
|
| 1232 |
+
}
|
| 1233 |
+
}
|
| 1234 |
+
}
|
| 1235 |
+
|
| 1236 |
+
fn score_at(&self, i: usize, now: u64, incoming: Option<&EvictionContext<'_>>) -> f64 {
|
| 1237 |
+
let e = &self.entries[i];
|
| 1238 |
+
eviction_score(
|
| 1239 |
+
&ScoreEntry {
|
| 1240 |
+
sha: &e.sha,
|
| 1241 |
+
quant_bits: e.header.quant_bits,
|
| 1242 |
+
model_id: e.header.model_id,
|
| 1243 |
+
reason: e.header.reason,
|
| 1244 |
+
tokens: e.header.tokens,
|
| 1245 |
+
hits: e.header.hits,
|
| 1246 |
+
ctx_size: e.header.ctx_size,
|
| 1247 |
+
created_at: e.header.created_at,
|
| 1248 |
+
last_used: e.header.last_used,
|
| 1249 |
+
text_bytes: u64::from(e.header.text_bytes),
|
| 1250 |
+
file_size: e.file_size,
|
| 1251 |
+
},
|
| 1252 |
+
now,
|
| 1253 |
+
incoming,
|
| 1254 |
+
)
|
| 1255 |
+
}
|
| 1256 |
+
}
|
| 1257 |
+
|
| 1258 |
+
fn rewrite_compatible_trailer(
|
| 1259 |
+
file: &mut fs::File,
|
| 1260 |
+
existing: &Envelope,
|
| 1261 |
+
incoming_ext_flags: u8,
|
| 1262 |
+
trailer: &[u8],
|
| 1263 |
+
) -> io::Result<()> {
|
| 1264 |
+
// The C fast path is O(trailer); copying a multi-GiB payload here would
|
| 1265 |
+
// turn a compatible save into a full checkpoint rewrite.
|
| 1266 |
+
let payload_end = existing
|
| 1267 |
+
.payload_offset
|
| 1268 |
+
.checked_add(existing.header.payload_bytes)
|
| 1269 |
+
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "KVC payload end overflows"))?;
|
| 1270 |
+
file.seek(SeekFrom::Start(payload_end))?;
|
| 1271 |
+
file.set_len(payload_end)?;
|
| 1272 |
+
file.write_all(trailer)?;
|
| 1273 |
+
file.flush()?;
|
| 1274 |
+
|
| 1275 |
+
let mut header = existing.header.clone();
|
| 1276 |
+
header.ext_flags |= incoming_ext_flags & EXT_TOOL_MAP;
|
| 1277 |
+
header.last_used = now_secs();
|
| 1278 |
+
let raw = fill_header(
|
| 1279 |
+
header.model_id,
|
| 1280 |
+
header.quant_bits,
|
| 1281 |
+
header.reason,
|
| 1282 |
+
header.ext_flags,
|
| 1283 |
+
header.tokens,
|
| 1284 |
+
header.hits,
|
| 1285 |
+
header.ctx_size,
|
| 1286 |
+
header.created_at,
|
| 1287 |
+
header.last_used,
|
| 1288 |
+
header.payload_bytes,
|
| 1289 |
+
);
|
| 1290 |
+
file.seek(SeekFrom::Start(0))?;
|
| 1291 |
+
file.write_all(&raw)?;
|
| 1292 |
+
file.flush()
|
| 1293 |
+
}
|
| 1294 |
+
|
| 1295 |
+
/// Fold in a record the search would not take. A mismatch outranks one
|
| 1296 |
+
/// that is only too shallow, the way the lanes report the two, and both
|
| 1297 |
+
/// outrank having seen nothing.
|
| 1298 |
+
fn weaker_answer(seen: PrefixAnswer, refused: PrefixAnswer) -> PrefixAnswer {
|
| 1299 |
+
if refused == PrefixAnswer::Mismatch || seen == PrefixAnswer::None {
|
| 1300 |
+
return refused;
|
| 1301 |
+
}
|
| 1302 |
+
seen
|
| 1303 |
+
}
|
| 1304 |
+
|
| 1305 |
+
fn format_io_error(error: FormatError) -> io::Error {
|
| 1306 |
+
match error {
|
| 1307 |
+
FormatError::Io(error) => error,
|
| 1308 |
+
FormatError::Truncated => {
|
| 1309 |
+
io::Error::new(io::ErrorKind::UnexpectedEof, "truncated KVC payload")
|
| 1310 |
+
}
|
| 1311 |
+
other => io::Error::new(io::ErrorKind::InvalidData, other),
|
| 1312 |
+
}
|
| 1313 |
+
}
|
| 1314 |
+
|
| 1315 |
+
fn now_secs() -> u64 {
|
| 1316 |
+
SystemTime::now()
|
| 1317 |
+
.duration_since(UNIX_EPOCH)
|
| 1318 |
+
.map(|d| d.as_secs())
|
| 1319 |
+
.unwrap_or(0)
|
| 1320 |
+
}
|
| 1321 |
+
|
| 1322 |
+
#[cfg(test)]
|
| 1323 |
+
mod tests {
|
| 1324 |
+
use super::*;
|
| 1325 |
+
use crate::format::{
|
| 1326 |
+
fill_header, le_put32, read_metadata, read_text_prefix, Header, Reason, FIXED_HEADER,
|
| 1327 |
+
};
|
| 1328 |
+
#[cfg(unix)]
|
| 1329 |
+
use std::os::unix::fs::PermissionsExt as _;
|
| 1330 |
+
|
| 1331 |
+
fn rec(text: &[u8], tokens: u32) -> Record {
|
| 1332 |
+
Record {
|
| 1333 |
+
header: Header {
|
| 1334 |
+
quant_bits: 2,
|
| 1335 |
+
reason: Reason::Cold,
|
| 1336 |
+
ext_flags: 0,
|
| 1337 |
+
model_id: 0,
|
| 1338 |
+
tokens,
|
| 1339 |
+
hits: 0,
|
| 1340 |
+
ctx_size: 2048,
|
| 1341 |
+
created_at: 1,
|
| 1342 |
+
last_used: 1,
|
| 1343 |
+
payload_bytes: 1,
|
| 1344 |
+
text_bytes: text.len() as u32,
|
| 1345 |
+
},
|
| 1346 |
+
text: text.to_vec(),
|
| 1347 |
+
payload: vec![0xAA],
|
| 1348 |
+
trailer: Vec::new(),
|
| 1349 |
+
}
|
| 1350 |
+
}
|
| 1351 |
+
|
| 1352 |
+
#[test]
|
| 1353 |
+
fn identity_required_for_reuse() {
|
| 1354 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-identity-{}", std::process::id()));
|
| 1355 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1356 |
+
let options = Options {
|
| 1357 |
+
min_tokens: 1,
|
| 1358 |
+
..Options::default()
|
| 1359 |
+
};
|
| 1360 |
+
let mut store = Store::open(&dir, 16, false, options).unwrap();
|
| 1361 |
+
let row = rec(b"same prompt", 8);
|
| 1362 |
+
let path = store.write(row.clone()).unwrap();
|
| 1363 |
+
store.bind_identity([1; 32]);
|
| 1364 |
+
assert!(store
|
| 1365 |
+
.text_prefix_candidate(b"same prompt plus", 0, 2, 2048)
|
| 1366 |
+
.unwrap()
|
| 1367 |
+
.is_none());
|
| 1368 |
+
assert_eq!(
|
| 1369 |
+
store.prefix_answer(b"same prompt plus", 0, 2, 2048),
|
| 1370 |
+
PrefixAnswer::Mismatch
|
| 1371 |
+
);
|
| 1372 |
+
assert!(store.read(&path).is_err());
|
| 1373 |
+
assert!(store
|
| 1374 |
+
.reuse_compatible(row.header.clone(), &row.text, &[])
|
| 1375 |
+
.unwrap()
|
| 1376 |
+
.is_none());
|
| 1377 |
+
store.write(row.clone()).unwrap();
|
| 1378 |
+
assert!(store
|
| 1379 |
+
.text_prefix_candidate(b"same prompt plus", 0, 2, 2048)
|
| 1380 |
+
.unwrap()
|
| 1381 |
+
.is_some());
|
| 1382 |
+
assert!(store
|
| 1383 |
+
.bank_text_lcp_candidate(b"same prompt edit", 0, 2, 2048, 4)
|
| 1384 |
+
.unwrap()
|
| 1385 |
+
.is_some());
|
| 1386 |
+
assert_eq!(store.read(&path).unwrap(), row);
|
| 1387 |
+
let mut other = Store::open(&dir, 16, false, options).unwrap();
|
| 1388 |
+
other.bind_identity([2; 32]);
|
| 1389 |
+
assert!(other
|
| 1390 |
+
.text_prefix_candidate(b"same prompt plus", 0, 2, 2048)
|
| 1391 |
+
.unwrap()
|
| 1392 |
+
.is_none());
|
| 1393 |
+
assert!(other
|
| 1394 |
+
.bank_text_lcp_candidate(b"same prompt edit", 0, 2, 2048, 4)
|
| 1395 |
+
.unwrap()
|
| 1396 |
+
.is_none());
|
| 1397 |
+
assert!(other
|
| 1398 |
+
.reuse_compatible(row.header.clone(), &row.text, &[])
|
| 1399 |
+
.unwrap()
|
| 1400 |
+
.is_none());
|
| 1401 |
+
assert!(other.read(&path).is_err());
|
| 1402 |
+
assert!(other.touch_hit(&path).is_err());
|
| 1403 |
+
let (_, envelope) = store
|
| 1404 |
+
.text_prefix_candidate(b"same prompt plus", 0, 2, 2048)
|
| 1405 |
+
.unwrap()
|
| 1406 |
+
.unwrap();
|
| 1407 |
+
let held = store.open_payload(&path, &envelope).unwrap();
|
| 1408 |
+
let mut replacement = row.clone();
|
| 1409 |
+
replacement.payload = vec![0xBB];
|
| 1410 |
+
other.write(replacement).unwrap();
|
| 1411 |
+
assert_eq!(read_path(held.path()).unwrap().payload, row.payload);
|
| 1412 |
+
assert!(store.open_payload(&path, &envelope).is_err());
|
| 1413 |
+
assert!(store.read(&path).is_err());
|
| 1414 |
+
assert_eq!(other.entries().len(), 1);
|
| 1415 |
+
fs::remove_dir_all(dir).unwrap();
|
| 1416 |
+
}
|
| 1417 |
+
|
| 1418 |
+
#[test]
|
| 1419 |
+
fn identity_keeps_tool_trailer() {
|
| 1420 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-id-tools-{}", std::process::id()));
|
| 1421 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1422 |
+
let mut store = Store::open(
|
| 1423 |
+
&dir,
|
| 1424 |
+
16,
|
| 1425 |
+
false,
|
| 1426 |
+
Options {
|
| 1427 |
+
min_tokens: 1,
|
| 1428 |
+
..Options::default()
|
| 1429 |
+
},
|
| 1430 |
+
)
|
| 1431 |
+
.unwrap();
|
| 1432 |
+
store.bind_identity([3; 32]);
|
| 1433 |
+
let mut row = rec(b"tool prompt", 8);
|
| 1434 |
+
let tools = vec![crate::ExtensionRecord {
|
| 1435 |
+
id: b"tool-1".to_vec(),
|
| 1436 |
+
body: b"call()".to_vec(),
|
| 1437 |
+
}];
|
| 1438 |
+
row.header.ext_flags = EXT_TOOL_MAP;
|
| 1439 |
+
row.trailer = crate::encode_ktm(&tools);
|
| 1440 |
+
let path = store.write(row.clone()).unwrap();
|
| 1441 |
+
assert_eq!(store.read(&path).unwrap(), row);
|
| 1442 |
+
let raw = read_path(&path).unwrap();
|
| 1443 |
+
assert_eq!(crate::decode_ktm(&raw.trailer), tools);
|
| 1444 |
+
assert!(store
|
| 1445 |
+
.reuse_compatible(row.header.clone(), &row.text, &row.trailer)
|
| 1446 |
+
.unwrap()
|
| 1447 |
+
.is_some());
|
| 1448 |
+
let loaded = store.read(&path).unwrap();
|
| 1449 |
+
assert_eq!(loaded.trailer, row.trailer);
|
| 1450 |
+
assert_eq!(loaded.payload, row.payload);
|
| 1451 |
+
assert_eq!(store.read_trailer(&path, 4096).unwrap().1, row.trailer);
|
| 1452 |
+
store.bind_identity([4; 32]);
|
| 1453 |
+
assert!(store.read_trailer(&path, 4096).is_err());
|
| 1454 |
+
store.bind_identity([3; 32]);
|
| 1455 |
+
let bytes = fs::read(&path).unwrap();
|
| 1456 |
+
fs::write(&path, &bytes[..bytes.len() - 1]).unwrap();
|
| 1457 |
+
assert!(store
|
| 1458 |
+
.text_prefix_candidate(b"tool prompt suffix", 0, 2, 2048)
|
| 1459 |
+
.unwrap()
|
| 1460 |
+
.is_none());
|
| 1461 |
+
assert!(store.read(&path).is_err());
|
| 1462 |
+
fs::remove_dir_all(dir).unwrap();
|
| 1463 |
+
}
|
| 1464 |
+
|
| 1465 |
+
#[test]
|
| 1466 |
+
fn identities_share_one_budget() {
|
| 1467 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-id-budget-{}", std::process::id()));
|
| 1468 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1469 |
+
let mut store = Store::open(
|
| 1470 |
+
&dir,
|
| 1471 |
+
16,
|
| 1472 |
+
false,
|
| 1473 |
+
Options {
|
| 1474 |
+
min_tokens: 1,
|
| 1475 |
+
..Options::default()
|
| 1476 |
+
},
|
| 1477 |
+
)
|
| 1478 |
+
.unwrap();
|
| 1479 |
+
store.bind_identity([1; 32]);
|
| 1480 |
+
let first = store.write(rec(b"old artifact", 8)).unwrap();
|
| 1481 |
+
let bytes = fs::metadata(&first).unwrap().len();
|
| 1482 |
+
store.budget_bytes = bytes + 16; // Includes the policy's one-percent reserve.
|
| 1483 |
+
store.bind_identity([2; 32]);
|
| 1484 |
+
let second = store.write(rec(b"new artifact", 8)).unwrap();
|
| 1485 |
+
assert!(!first.exists() && second.exists());
|
| 1486 |
+
assert_eq!(store.entries().len(), 1);
|
| 1487 |
+
assert_eq!(fs::metadata(second).unwrap().len(), bytes);
|
| 1488 |
+
fs::remove_dir_all(dir).unwrap();
|
| 1489 |
+
}
|
| 1490 |
+
|
| 1491 |
+
#[cfg(unix)]
|
| 1492 |
+
#[test]
|
| 1493 |
+
fn new_nested_store_directories_are_private() {
|
| 1494 |
+
let base = std::env::temp_dir().join(format!("ds4-kv-private-dir-{}", std::process::id()));
|
| 1495 |
+
let dir = base.join("nested/store");
|
| 1496 |
+
let _ = fs::remove_dir_all(&base);
|
| 1497 |
+
|
| 1498 |
+
let _store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1499 |
+
for path in [&base, &base.join("nested"), &dir] {
|
| 1500 |
+
assert_eq!(fs::metadata(path).unwrap().permissions().mode() & 0o077, 0);
|
| 1501 |
+
}
|
| 1502 |
+
|
| 1503 |
+
let _ = fs::remove_dir_all(&base);
|
| 1504 |
+
}
|
| 1505 |
+
|
| 1506 |
+
#[cfg(unix)]
|
| 1507 |
+
#[test]
|
| 1508 |
+
fn existing_store_directory_mode_is_unchanged() {
|
| 1509 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-existing-dir-{}", std::process::id()));
|
| 1510 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1511 |
+
fs::create_dir_all(&dir).unwrap();
|
| 1512 |
+
fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).unwrap();
|
| 1513 |
+
|
| 1514 |
+
let _store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1515 |
+
assert_eq!(
|
| 1516 |
+
fs::metadata(&dir).unwrap().permissions().mode() & 0o777,
|
| 1517 |
+
0o755
|
| 1518 |
+
);
|
| 1519 |
+
|
| 1520 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1521 |
+
}
|
| 1522 |
+
|
| 1523 |
+
#[cfg(unix)]
|
| 1524 |
+
#[test]
|
| 1525 |
+
fn payload_temp_is_private_and_drop_removes_it() {
|
| 1526 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-payload-temp-{}", std::process::id()));
|
| 1527 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1528 |
+
let store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1529 |
+
|
| 1530 |
+
let temp = store.payload_temp().unwrap();
|
| 1531 |
+
let path = temp.path().to_path_buf();
|
| 1532 |
+
assert!(path.exists());
|
| 1533 |
+
assert_eq!(
|
| 1534 |
+
fs::metadata(&path).unwrap().permissions().mode() & 0o777,
|
| 1535 |
+
0o600
|
| 1536 |
+
);
|
| 1537 |
+
|
| 1538 |
+
drop(temp);
|
| 1539 |
+
assert!(!path.exists());
|
| 1540 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1541 |
+
}
|
| 1542 |
+
|
| 1543 |
+
#[test]
|
| 1544 |
+
fn payload_temps_are_distinct_and_drop_independently() {
|
| 1545 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-payload-pair-{}", std::process::id()));
|
| 1546 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1547 |
+
let store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1548 |
+
|
| 1549 |
+
let first = store.payload_temp().unwrap();
|
| 1550 |
+
let second = store.payload_temp().unwrap();
|
| 1551 |
+
let first_path = first.path().to_path_buf();
|
| 1552 |
+
let second_path = second.path().to_path_buf();
|
| 1553 |
+
assert_ne!(first_path, second_path);
|
| 1554 |
+
assert!(first_path.exists());
|
| 1555 |
+
assert!(second_path.exists());
|
| 1556 |
+
|
| 1557 |
+
drop(first);
|
| 1558 |
+
assert!(!first_path.exists());
|
| 1559 |
+
assert!(second_path.exists());
|
| 1560 |
+
drop(second);
|
| 1561 |
+
assert!(!second_path.exists());
|
| 1562 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1563 |
+
}
|
| 1564 |
+
|
| 1565 |
+
#[test]
|
| 1566 |
+
fn lcp_refuses_a_moved_record() {
|
| 1567 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-lcp-candidate-{}", std::process::id()));
|
| 1568 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1569 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1570 |
+
let mut record = rec(b"shared opening/stored tail", 512);
|
| 1571 |
+
record.header.reason = Reason::BankShutdown;
|
| 1572 |
+
record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
|
| 1573 |
+
let path = store.write(record).unwrap();
|
| 1574 |
+
let edited = b"shared opening/edited tail";
|
| 1575 |
+
|
| 1576 |
+
let (got, _, lcp) = store
|
| 1577 |
+
.bank_text_lcp_candidate(edited, 0, 2, 8192, 8)
|
| 1578 |
+
.unwrap()
|
| 1579 |
+
.unwrap();
|
| 1580 |
+
assert_eq!(got, path);
|
| 1581 |
+
assert_eq!(lcp, 15);
|
| 1582 |
+
|
| 1583 |
+
// Corrupt the text past the shared prefix: the scan still picks the
|
| 1584 |
+
// record, and only the full-text check can see that it moved.
|
| 1585 |
+
let mut file = fs::OpenOptions::new().write(true).open(&path).unwrap();
|
| 1586 |
+
file.seek(SeekFrom::Start((FIXED_HEADER + 4 + 20) as u64))
|
| 1587 |
+
.unwrap();
|
| 1588 |
+
file.write_all(b"X").unwrap();
|
| 1589 |
+
file.flush().unwrap();
|
| 1590 |
+
assert_eq!(
|
| 1591 |
+
store
|
| 1592 |
+
.bank_text_lcp_candidate(edited, 0, 2, 8192, 8)
|
| 1593 |
+
.unwrap_err()
|
| 1594 |
+
.kind(),
|
| 1595 |
+
io::ErrorKind::InvalidData
|
| 1596 |
+
);
|
| 1597 |
+
|
| 1598 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1599 |
+
}
|
| 1600 |
+
|
| 1601 |
+
#[test]
|
| 1602 |
+
fn a_cut_record_keeps_its_key() {
|
| 1603 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-truncated-{}", std::process::id()));
|
| 1604 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1605 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1606 |
+
let path = store.write(rec(b"shared prefix", 512)).unwrap();
|
| 1607 |
+
|
| 1608 |
+
// Cut the payload away: the header and text stay, so the catalog
|
| 1609 |
+
// drops the record while its name still says what it was keyed by.
|
| 1610 |
+
fs::OpenOptions::new()
|
| 1611 |
+
.write(true)
|
| 1612 |
+
.open(&path)
|
| 1613 |
+
.unwrap()
|
| 1614 |
+
.set_len(FIXED_HEADER as u64 + 4 + 13)
|
| 1615 |
+
.unwrap();
|
| 1616 |
+
|
| 1617 |
+
assert_eq!(
|
| 1618 |
+
store.prefix_answer(b"shared prefix and suffix", 0, 2, 2048),
|
| 1619 |
+
PrefixAnswer::Mismatch
|
| 1620 |
+
);
|
| 1621 |
+
assert_eq!(
|
| 1622 |
+
store.prefix_answer(b"another conversation", 0, 2, 2048),
|
| 1623 |
+
PrefixAnswer::None
|
| 1624 |
+
);
|
| 1625 |
+
|
| 1626 |
+
// A record no search replays from is not a refusal: it was never a
|
| 1627 |
+
// candidate, damaged or not.
|
| 1628 |
+
let mut session = rec(b"agent opening", 512);
|
| 1629 |
+
session.header.reason = Reason::AgentSession;
|
| 1630 |
+
let session_path = store.write(session).unwrap();
|
| 1631 |
+
fs::OpenOptions::new()
|
| 1632 |
+
.write(true)
|
| 1633 |
+
.open(&session_path)
|
| 1634 |
+
.unwrap()
|
| 1635 |
+
.set_len(FIXED_HEADER as u64 + 4 + 13)
|
| 1636 |
+
.unwrap();
|
| 1637 |
+
assert_eq!(
|
| 1638 |
+
store.prefix_answer(b"agent opening and more", 0, 2, 2048),
|
| 1639 |
+
PrefixAnswer::None
|
| 1640 |
+
);
|
| 1641 |
+
assert_eq!(
|
| 1642 |
+
store.bank_lcp_answer(b"agent openinG edited", 0, 2, 2048, 8, 0),
|
| 1643 |
+
PrefixAnswer::None
|
| 1644 |
+
);
|
| 1645 |
+
|
| 1646 |
+
// An edited prompt diverges from the record, so only the shared
|
| 1647 |
+
// prefix can find it — and the text in front of the payload is
|
| 1648 |
+
// still there to be read.
|
| 1649 |
+
assert_eq!(
|
| 1650 |
+
store.bank_lcp_answer(b"shared prefiX edited", 0, 2, 2048, 8, 0),
|
| 1651 |
+
PrefixAnswer::Mismatch
|
| 1652 |
+
);
|
| 1653 |
+
assert_eq!(
|
| 1654 |
+
store.bank_lcp_answer(b"another opening turn", 0, 2, 2048, 8, 0),
|
| 1655 |
+
PrefixAnswer::None
|
| 1656 |
+
);
|
| 1657 |
+
|
| 1658 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1659 |
+
}
|
| 1660 |
+
|
| 1661 |
+
#[test]
|
| 1662 |
+
fn an_exact_bank_answers_layout() {
|
| 1663 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-exact-bank-{}", std::process::id()));
|
| 1664 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1665 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1666 |
+
let mut record = rec(b"exact conversation", 512);
|
| 1667 |
+
record.header.reason = Reason::BankShutdown;
|
| 1668 |
+
record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
|
| 1669 |
+
store.write(record).unwrap();
|
| 1670 |
+
|
| 1671 |
+
// A bank snapshot has no logits to replay from, so an exact serial
|
| 1672 |
+
// prompt is refused by the record's layout, not by its absence.
|
| 1673 |
+
assert_eq!(
|
| 1674 |
+
store.prefix_answer(b"exact conversation", 0, 2, 2048),
|
| 1675 |
+
PrefixAnswer::Mismatch
|
| 1676 |
+
);
|
| 1677 |
+
// With a suffix to prefill, the same record is what the search takes.
|
| 1678 |
+
assert_eq!(
|
| 1679 |
+
store.prefix_answer(b"exact conversation plus a turn", 0, 2, 2048),
|
| 1680 |
+
PrefixAnswer::Usable
|
| 1681 |
+
);
|
| 1682 |
+
|
| 1683 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1684 |
+
}
|
| 1685 |
+
|
| 1686 |
+
#[test]
|
| 1687 |
+
fn a_thin_record_answers_shallow() {
|
| 1688 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-shallow-{}", std::process::id()));
|
| 1689 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1690 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1691 |
+
// Written while the minimum was lower; every search skips it now.
|
| 1692 |
+
store.write(rec(b"shared opening", 8)).unwrap();
|
| 1693 |
+
|
| 1694 |
+
assert_eq!(
|
| 1695 |
+
store.prefix_answer(b"shared opening and more", 0, 2, 2048),
|
| 1696 |
+
PrefixAnswer::Shallow
|
| 1697 |
+
);
|
| 1698 |
+
assert_eq!(
|
| 1699 |
+
store.prefix_answer(b"another conversation", 0, 2, 2048),
|
| 1700 |
+
PrefixAnswer::None
|
| 1701 |
+
);
|
| 1702 |
+
|
| 1703 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1704 |
+
}
|
| 1705 |
+
|
| 1706 |
+
#[test]
|
| 1707 |
+
fn an_image_key_owns_its_answer() {
|
| 1708 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-image-key-{}", std::process::id()));
|
| 1709 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1710 |
+
let mut store = Store::open(&dir, 16, true, Options::default()).unwrap();
|
| 1711 |
+
let mut record = rec(b"chat\xffDS4IMG2 turn", 512);
|
| 1712 |
+
record.header.reason = Reason::BankCheckpoint;
|
| 1713 |
+
record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1 | EXT_IMAGE_PIXELS_V2;
|
| 1714 |
+
store.write(record).unwrap();
|
| 1715 |
+
let key = b"chat\xffDS4IMG2 turn and one more";
|
| 1716 |
+
|
| 1717 |
+
// The identity the record was stored with: the search takes it.
|
| 1718 |
+
assert_eq!(
|
| 1719 |
+
store.bank_prefix_answer(key, 0, 2, 2048, EXT_IMAGE_PIXELS_V2),
|
| 1720 |
+
PrefixAnswer::Usable
|
| 1721 |
+
);
|
| 1722 |
+
// A different quantization rules it out, so the miss is a mismatch.
|
| 1723 |
+
assert_eq!(
|
| 1724 |
+
store.bank_prefix_answer(key, 0, 4, 2048, EXT_IMAGE_PIXELS_V2),
|
| 1725 |
+
PrefixAnswer::Mismatch
|
| 1726 |
+
);
|
| 1727 |
+
// The rendered prompt is not the key it was stored under, so the
|
| 1728 |
+
// text-keyed search sees nothing. Asked under this key without the
|
| 1729 |
+
// media flag, the record is there and its layout rules it out.
|
| 1730 |
+
assert_eq!(
|
| 1731 |
+
store.prefix_answer(b"chat turn and one more", 0, 4, 2048),
|
| 1732 |
+
PrefixAnswer::None
|
| 1733 |
+
);
|
| 1734 |
+
assert_eq!(
|
| 1735 |
+
store.bank_prefix_answer(key, 0, 2, 2048, 0),
|
| 1736 |
+
PrefixAnswer::Mismatch
|
| 1737 |
+
);
|
| 1738 |
+
|
| 1739 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1740 |
+
}
|
| 1741 |
+
|
| 1742 |
+
#[test]
|
| 1743 |
+
fn an_edit_mismatches_by_lcp() {
|
| 1744 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-lcp-identity-{}", std::process::id()));
|
| 1745 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1746 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1747 |
+
store
|
| 1748 |
+
.write(rec(b"shared opening turn/ORIGINAL", 512))
|
| 1749 |
+
.unwrap();
|
| 1750 |
+
let edited = b"shared opening turn/EDITED CONTINUATION";
|
| 1751 |
+
|
| 1752 |
+
// The stored text diverges from the prompt, so the exact-prefix
|
| 1753 |
+
// question cannot see the record at all.
|
| 1754 |
+
assert_eq!(
|
| 1755 |
+
store.bank_prefix_answer(edited, 1, 2, 2048, 0),
|
| 1756 |
+
PrefixAnswer::None
|
| 1757 |
+
);
|
| 1758 |
+
// Through the LCP the partial search would have taken it, and only
|
| 1759 |
+
// the model it was written for rules it out.
|
| 1760 |
+
assert_eq!(
|
| 1761 |
+
store.bank_lcp_answer(edited, 1, 2, 2048, 8, 0),
|
| 1762 |
+
PrefixAnswer::Mismatch
|
| 1763 |
+
);
|
| 1764 |
+
// The identity it was stored with is not a mismatch.
|
| 1765 |
+
assert_eq!(
|
| 1766 |
+
store.bank_lcp_answer(edited, 0, 2, 2048, 8, 0),
|
| 1767 |
+
PrefixAnswer::Usable
|
| 1768 |
+
);
|
| 1769 |
+
// Neither is a prompt that shares nothing with it.
|
| 1770 |
+
assert_eq!(
|
| 1771 |
+
store.bank_lcp_answer(b"a different opening", 1, 2, 2048, 8, 0),
|
| 1772 |
+
PrefixAnswer::None
|
| 1773 |
+
);
|
| 1774 |
+
|
| 1775 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1776 |
+
}
|
| 1777 |
+
|
| 1778 |
+
#[test]
|
| 1779 |
+
fn discard_removes_existing_candidate_and_refreshes_index() {
|
| 1780 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-discard-{}", std::process::id()));
|
| 1781 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1782 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1783 |
+
let discarded = store.write(rec(b"discarded", 512)).unwrap();
|
| 1784 |
+
let retained = store.write(rec(b"retained", 1024)).unwrap();
|
| 1785 |
+
store.continued_last_store_tokens = 512;
|
| 1786 |
+
assert_eq!(store.entries().len(), 2);
|
| 1787 |
+
|
| 1788 |
+
store.discard(&discarded).unwrap();
|
| 1789 |
+
|
| 1790 |
+
assert!(!discarded.exists());
|
| 1791 |
+
assert!(retained.exists());
|
| 1792 |
+
assert_eq!(store.entries().len(), 1);
|
| 1793 |
+
assert_eq!(store.entries()[0].path, retained);
|
| 1794 |
+
assert_eq!(store.continued_last_store_tokens, 0);
|
| 1795 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1796 |
+
}
|
| 1797 |
+
|
| 1798 |
+
#[test]
|
| 1799 |
+
fn bank_discard_preserves_the_serial_checkpoint_marker() {
|
| 1800 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-bank-discard-{}", std::process::id()));
|
| 1801 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1802 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1803 |
+
let discarded = store.write(rec(b"bank-corrupt", 512)).unwrap();
|
| 1804 |
+
store.continued_last_store_tokens = 777;
|
| 1805 |
+
|
| 1806 |
+
store.discard_bank(&discarded).unwrap();
|
| 1807 |
+
|
| 1808 |
+
assert!(!discarded.exists());
|
| 1809 |
+
assert!(store.entries().is_empty());
|
| 1810 |
+
assert_eq!(store.continued_last_store_tokens, 777);
|
| 1811 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1812 |
+
}
|
| 1813 |
+
|
| 1814 |
+
#[test]
|
| 1815 |
+
fn discard_missing_candidate_is_ok_and_refreshes_index() {
|
| 1816 |
+
let dir =
|
| 1817 |
+
std::env::temp_dir().join(format!("ds4-kv-discard-missing-{}", std::process::id()));
|
| 1818 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1819 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1820 |
+
let missing = store.write(rec(b"missing", 512)).unwrap();
|
| 1821 |
+
store.continued_last_store_tokens = 512;
|
| 1822 |
+
fs::remove_file(&missing).unwrap();
|
| 1823 |
+
assert_eq!(store.entries().len(), 1);
|
| 1824 |
+
|
| 1825 |
+
store.discard(&missing).unwrap();
|
| 1826 |
+
|
| 1827 |
+
assert!(store.entries().is_empty());
|
| 1828 |
+
assert_eq!(store.continued_last_store_tokens, 0);
|
| 1829 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1830 |
+
}
|
| 1831 |
+
|
| 1832 |
+
#[test]
|
| 1833 |
+
fn discard_rejects_paths_outside_the_catalog() {
|
| 1834 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-discard-scope-{}", std::process::id()));
|
| 1835 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1836 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1837 |
+
let outside = dir.with_extension("outside");
|
| 1838 |
+
fs::write(&outside, b"keep").unwrap();
|
| 1839 |
+
|
| 1840 |
+
let error = store.discard(&outside).unwrap_err();
|
| 1841 |
+
|
| 1842 |
+
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
|
| 1843 |
+
assert_eq!(fs::read(&outside).unwrap(), b"keep");
|
| 1844 |
+
fs::remove_file(outside).unwrap();
|
| 1845 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1846 |
+
}
|
| 1847 |
+
|
| 1848 |
+
#[test]
|
| 1849 |
+
fn prefix_prefers_longest_matching_text() {
|
| 1850 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-prefix-{}", std::process::id()));
|
| 1851 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1852 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1853 |
+
store.write(rec(b"hello", 512)).unwrap();
|
| 1854 |
+
store.write(rec(b"hello world", 512)).unwrap();
|
| 1855 |
+
let idx = store.find_text_prefix(b"hello world!", 0, 2, 2048).unwrap();
|
| 1856 |
+
assert_eq!(store.entries()[idx].header.text_bytes, 11);
|
| 1857 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1858 |
+
}
|
| 1859 |
+
|
| 1860 |
+
#[test]
|
| 1861 |
+
fn bank_lookup_requires_suffix() {
|
| 1862 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-bank-{}", std::process::id()));
|
| 1863 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1864 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1865 |
+
let mut r = rec(b"hello", 512);
|
| 1866 |
+
r.header.reason = Reason::BankEvict;
|
| 1867 |
+
r.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
|
| 1868 |
+
store.write(r).unwrap();
|
| 1869 |
+
assert!(store.find_bank_text_prefix(b"hello", 0, 2, 2048).is_none());
|
| 1870 |
+
assert!(store.find_bank_text_prefix(b"hello!", 0, 2, 2048).is_some());
|
| 1871 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1872 |
+
}
|
| 1873 |
+
|
| 1874 |
+
#[test]
|
| 1875 |
+
fn bank_identity_lookup_separates_image_and_text_records() {
|
| 1876 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-bank-identity-{}", std::process::id()));
|
| 1877 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1878 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1879 |
+
let mut plain = rec(b"plain identity", 512);
|
| 1880 |
+
plain.header.reason = Reason::BankCheckpoint;
|
| 1881 |
+
plain.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
|
| 1882 |
+
store.write(plain).unwrap();
|
| 1883 |
+
let mut image = rec(b"image identity", 512);
|
| 1884 |
+
image.header.reason = Reason::BankCheckpoint;
|
| 1885 |
+
image.header.ext_flags =
|
| 1886 |
+
crate::format::EXT_BANK_REPLAY_V1 | crate::format::EXT_IMAGE_PIXELS_V2;
|
| 1887 |
+
store.write(image).unwrap();
|
| 1888 |
+
|
| 1889 |
+
assert!(store
|
| 1890 |
+
.bank_text_prefix_candidate_identity(
|
| 1891 |
+
b"plain identity tail",
|
| 1892 |
+
0,
|
| 1893 |
+
2,
|
| 1894 |
+
2048,
|
| 1895 |
+
EXT_IMAGE_PIXELS_V2,
|
| 1896 |
+
)
|
| 1897 |
+
.unwrap()
|
| 1898 |
+
.is_none());
|
| 1899 |
+
assert!(store
|
| 1900 |
+
.bank_text_prefix_candidate_identity(b"image identity tail", 0, 2, 2048, 0)
|
| 1901 |
+
.unwrap()
|
| 1902 |
+
.is_none());
|
| 1903 |
+
assert!(store
|
| 1904 |
+
.bank_text_prefix_candidate_identity(
|
| 1905 |
+
b"image identity tail",
|
| 1906 |
+
0,
|
| 1907 |
+
2,
|
| 1908 |
+
2048,
|
| 1909 |
+
EXT_IMAGE_PIXELS_V2,
|
| 1910 |
+
)
|
| 1911 |
+
.unwrap()
|
| 1912 |
+
.is_some());
|
| 1913 |
+
assert!(
|
| 1914 |
+
store
|
| 1915 |
+
.bank_text_lcp_candidate_identity(
|
| 1916 |
+
b"image identitX",
|
| 1917 |
+
0,
|
| 1918 |
+
2,
|
| 1919 |
+
2048,
|
| 1920 |
+
8,
|
| 1921 |
+
EXT_IMAGE_PIXELS_V2,
|
| 1922 |
+
)
|
| 1923 |
+
.unwrap()
|
| 1924 |
+
.is_some()
|
| 1925 |
+
);
|
| 1926 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1927 |
+
}
|
| 1928 |
+
|
| 1929 |
+
#[test]
|
| 1930 |
+
fn lcp_candidate_reopens_and_revalidates_the_selected_record() {
|
| 1931 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-lcp-{}", std::process::id()));
|
| 1932 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1933 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 1934 |
+
let mut record = rec(b"shared alpha", 512);
|
| 1935 |
+
record.header.reason = Reason::BankCheckpoint;
|
| 1936 |
+
record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
|
| 1937 |
+
let path = store.write(record).unwrap();
|
| 1938 |
+
|
| 1939 |
+
let (candidate, envelope, lcp) = store
|
| 1940 |
+
.bank_text_lcp_candidate(b"shared beta", 0, 2, 2048, 6)
|
| 1941 |
+
.unwrap()
|
| 1942 |
+
.unwrap();
|
| 1943 |
+
assert_eq!(candidate, path);
|
| 1944 |
+
assert_eq!(envelope.text, b"shared alpha");
|
| 1945 |
+
assert_eq!(lcp, b"shared ".len());
|
| 1946 |
+
|
| 1947 |
+
let mut file = fs::OpenOptions::new()
|
| 1948 |
+
.read(true)
|
| 1949 |
+
.write(true)
|
| 1950 |
+
.open(&path)
|
| 1951 |
+
.unwrap();
|
| 1952 |
+
file.seek(SeekFrom::Start((FIXED_HEADER + 4) as u64))
|
| 1953 |
+
.unwrap();
|
| 1954 |
+
file.write_all(b"X").unwrap();
|
| 1955 |
+
file.flush().unwrap();
|
| 1956 |
+
assert!(store
|
| 1957 |
+
.bank_text_lcp_candidate(b"shared beta", 0, 2, 2048, 6)
|
| 1958 |
+
.unwrap()
|
| 1959 |
+
.is_none());
|
| 1960 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1961 |
+
}
|
| 1962 |
+
|
| 1963 |
+
#[test]
|
| 1964 |
+
fn indexes_sparse_payload_without_reading_it() {
|
| 1965 |
+
const PAYLOAD_BYTES: u64 = 4 * 1024 * 1024 * 1024;
|
| 1966 |
+
const TRAILER_BYTES: u64 = 17;
|
| 1967 |
+
|
| 1968 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-sparse-{}", std::process::id()));
|
| 1969 |
+
let _ = fs::remove_dir_all(&dir);
|
| 1970 |
+
fs::create_dir_all(&dir).unwrap();
|
| 1971 |
+
let text = [b'x'; 64];
|
| 1972 |
+
let prompt = [b'x'; 8];
|
| 1973 |
+
let path = path_for_sha(&dir, &text_sha_hex(&text));
|
| 1974 |
+
let header = fill_header(0, 2, Reason::Cold, 0, 512, 0, 2048, 1, 1, PAYLOAD_BYTES);
|
| 1975 |
+
let mut text_len = [0u8; 4];
|
| 1976 |
+
le_put32(&mut text_len, text.len() as u32);
|
| 1977 |
+
let mut file = fs::File::create(&path).unwrap();
|
| 1978 |
+
file.write_all(&header).unwrap();
|
| 1979 |
+
file.write_all(&text_len).unwrap();
|
| 1980 |
+
file.write_all(&text).unwrap();
|
| 1981 |
+
let payload_offset = (FIXED_HEADER + 4 + text.len()) as u64;
|
| 1982 |
+
file.set_len(payload_offset + PAYLOAD_BYTES + TRAILER_BYTES)
|
| 1983 |
+
.unwrap();
|
| 1984 |
+
drop(file);
|
| 1985 |
+
|
| 1986 |
+
let metadata = read_metadata(&path).unwrap();
|
| 1987 |
+
assert_eq!(metadata.header.text_bytes, text.len() as u32);
|
| 1988 |
+
let (_, prefix) = read_text_prefix(&path, prompt.len()).unwrap();
|
| 1989 |
+
assert_eq!(prefix, prompt);
|
| 1990 |
+
|
| 1991 |
+
let mut store = Store::open(&dir, 8192, false, Options::default()).unwrap();
|
| 1992 |
+
assert_eq!(store.entries().len(), 1);
|
| 1993 |
+
assert_eq!(
|
| 1994 |
+
store.entries()[0].file_size,
|
| 1995 |
+
payload_offset + PAYLOAD_BYTES + TRAILER_BYTES
|
| 1996 |
+
);
|
| 1997 |
+
let (_, lcp) = store.find_text_lcp(&prompt, 0, 2, 2048, 5).unwrap();
|
| 1998 |
+
assert_eq!(lcp, prompt.len());
|
| 1999 |
+
|
| 2000 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2001 |
+
}
|
| 2002 |
+
|
| 2003 |
+
#[cfg(unix)]
|
| 2004 |
+
#[test]
|
| 2005 |
+
fn eviction_stops_after_unlink_failure() {
|
| 2006 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-evict-unlink-{}", std::process::id()));
|
| 2007 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2008 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2009 |
+
store.write(rec(b"first", 512)).unwrap();
|
| 2010 |
+
store.write(rec(b"second", 1024)).unwrap();
|
| 2011 |
+
|
| 2012 |
+
let original_permissions = fs::metadata(&dir).unwrap().permissions();
|
| 2013 |
+
fs::set_permissions(&dir, fs::Permissions::from_mode(0o555)).unwrap();
|
| 2014 |
+
store.budget_bytes = 1;
|
| 2015 |
+
store.evict(0, None);
|
| 2016 |
+
let disk_files = fs::read_dir(&dir).unwrap().count();
|
| 2017 |
+
let remaining_tokens: Vec<_> = store.entries().iter().map(|e| e.header.tokens).collect();
|
| 2018 |
+
fs::set_permissions(&dir, original_permissions).unwrap();
|
| 2019 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2020 |
+
|
| 2021 |
+
assert_eq!(disk_files, 2);
|
| 2022 |
+
assert_eq!(remaining_tokens, vec![1024]);
|
| 2023 |
+
}
|
| 2024 |
+
|
| 2025 |
+
#[test]
|
| 2026 |
+
fn compatible_empty_trailer_reuses_existing_bytes() {
|
| 2027 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-reuse-empty-{}", std::process::id()));
|
| 2028 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2029 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2030 |
+
let mut existing = rec(b"same prompt", 512);
|
| 2031 |
+
existing.header.hits = 7;
|
| 2032 |
+
existing.trailer = b"old trailer".to_vec();
|
| 2033 |
+
let path = store.write(existing).unwrap();
|
| 2034 |
+
let sibling = store.write(rec(b"sibling", 1024)).unwrap();
|
| 2035 |
+
let before = fs::read(&path).unwrap();
|
| 2036 |
+
|
| 2037 |
+
let mut incoming = rec(b"same prompt", 2048).header;
|
| 2038 |
+
incoming.quant_bits = 4;
|
| 2039 |
+
incoming.reason = Reason::Continued;
|
| 2040 |
+
incoming.ext_flags = crate::format::EXT_RESPONSES_VISIBLE;
|
| 2041 |
+
incoming.ctx_size = 4096;
|
| 2042 |
+
incoming.created_at = 99;
|
| 2043 |
+
incoming.last_used = 99;
|
| 2044 |
+
store.budget_bytes = 1;
|
| 2045 |
+
|
| 2046 |
+
let got = store
|
| 2047 |
+
.write_payload_file(incoming, b"same prompt", &dir.join("missing-payload"), b"")
|
| 2048 |
+
.unwrap();
|
| 2049 |
+
assert_eq!(got, path);
|
| 2050 |
+
assert_eq!(fs::read(&path).unwrap(), before);
|
| 2051 |
+
assert!(sibling.exists());
|
| 2052 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2053 |
+
}
|
| 2054 |
+
|
| 2055 |
+
#[test]
|
| 2056 |
+
fn reuse_compatible_empty_trailer_returns_existing_without_staging() {
|
| 2057 |
+
let dir =
|
| 2058 |
+
std::env::temp_dir().join(format!("ds4-kv-preflight-empty-{}", std::process::id()));
|
| 2059 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2060 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2061 |
+
let mut existing = rec(b"same prompt", 512);
|
| 2062 |
+
existing.header.hits = 7;
|
| 2063 |
+
existing.trailer = b"old trailer".to_vec();
|
| 2064 |
+
let path = store.write(existing).unwrap();
|
| 2065 |
+
let sibling = store.write(rec(b"sibling", 1024)).unwrap();
|
| 2066 |
+
let before = fs::read(&path).unwrap();
|
| 2067 |
+
let sibling_before = fs::read(&sibling).unwrap();
|
| 2068 |
+
|
| 2069 |
+
let mut incoming = rec(b"same prompt", 2048).header;
|
| 2070 |
+
incoming.quant_bits = 4;
|
| 2071 |
+
incoming.reason = Reason::Continued;
|
| 2072 |
+
incoming.ext_flags = crate::format::EXT_RESPONSES_VISIBLE;
|
| 2073 |
+
incoming.ctx_size = 4096;
|
| 2074 |
+
incoming.created_at = 99;
|
| 2075 |
+
incoming.last_used = 99;
|
| 2076 |
+
store.budget_bytes = 1;
|
| 2077 |
+
|
| 2078 |
+
let got = store
|
| 2079 |
+
.reuse_compatible(incoming, b"same prompt", b"")
|
| 2080 |
+
.unwrap();
|
| 2081 |
+
|
| 2082 |
+
assert_eq!(got, Some(path.clone()));
|
| 2083 |
+
assert_eq!(fs::read(&path).unwrap(), before);
|
| 2084 |
+
assert_eq!(fs::read(&sibling).unwrap(), sibling_before);
|
| 2085 |
+
assert_eq!(fs::read_dir(&dir).unwrap().count(), 2);
|
| 2086 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2087 |
+
}
|
| 2088 |
+
|
| 2089 |
+
#[test]
|
| 2090 |
+
fn reuse_compatible_incompatible_entry_returns_none_without_mutation() {
|
| 2091 |
+
let dir = std::env::temp_dir().join(format!(
|
| 2092 |
+
"ds4-kv-preflight-incompatible-{}",
|
| 2093 |
+
std::process::id()
|
| 2094 |
+
));
|
| 2095 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2096 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2097 |
+
let path = store.write(rec(b"same prompt", 512)).unwrap();
|
| 2098 |
+
let sibling = store.write(rec(b"sibling", 1024)).unwrap();
|
| 2099 |
+
let before = fs::read(&path).unwrap();
|
| 2100 |
+
let sibling_before = fs::read(&sibling).unwrap();
|
| 2101 |
+
let mut incoming = rec(b"same prompt", 512).header;
|
| 2102 |
+
incoming.model_id = 1;
|
| 2103 |
+
store.budget_bytes = 1;
|
| 2104 |
+
|
| 2105 |
+
let got = store
|
| 2106 |
+
.reuse_compatible(incoming, b"same prompt", b"")
|
| 2107 |
+
.unwrap();
|
| 2108 |
+
|
| 2109 |
+
assert_eq!(got, None);
|
| 2110 |
+
assert_eq!(fs::read(&path).unwrap(), before);
|
| 2111 |
+
assert_eq!(fs::read(&sibling).unwrap(), sibling_before);
|
| 2112 |
+
assert_eq!(store.entries().len(), 2);
|
| 2113 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2114 |
+
}
|
| 2115 |
+
|
| 2116 |
+
#[test]
|
| 2117 |
+
fn reuse_compatible_nonempty_trailer_matches_writer_fast_path() {
|
| 2118 |
+
let dir =
|
| 2119 |
+
std::env::temp_dir().join(format!("ds4-kv-preflight-trailer-{}", std::process::id()));
|
| 2120 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2121 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2122 |
+
let mut existing = rec(b"same prompt", 512);
|
| 2123 |
+
existing.header.hits = 7;
|
| 2124 |
+
existing.header.created_at = 42;
|
| 2125 |
+
existing.header.last_used = 43;
|
| 2126 |
+
existing.header.ext_flags = crate::format::EXT_RESPONSES_VISIBLE;
|
| 2127 |
+
existing.payload = b"opaque payload".to_vec();
|
| 2128 |
+
existing.trailer = b"old trailer".to_vec();
|
| 2129 |
+
let path = store.write(existing).unwrap();
|
| 2130 |
+
let sibling = store.write(rec(b"sibling", 1024)).unwrap();
|
| 2131 |
+
let before = read_path(&path).unwrap();
|
| 2132 |
+
|
| 2133 |
+
let mut incoming = rec(b"same prompt", 2048).header;
|
| 2134 |
+
incoming.reason = Reason::Continued;
|
| 2135 |
+
incoming.ext_flags = crate::format::EXT_THINKING_VISIBLE;
|
| 2136 |
+
incoming.ctx_size = 4096;
|
| 2137 |
+
incoming.created_at = 99;
|
| 2138 |
+
incoming.last_used = 99;
|
| 2139 |
+
store.budget_bytes = 1;
|
| 2140 |
+
let started = now_secs();
|
| 2141 |
+
|
| 2142 |
+
let got = store
|
| 2143 |
+
.reuse_compatible(incoming, b"same prompt", b"new tool map")
|
| 2144 |
+
.unwrap();
|
| 2145 |
+
let after = read_path(&path).unwrap();
|
| 2146 |
+
|
| 2147 |
+
assert_eq!(got, Some(path));
|
| 2148 |
+
assert_eq!(after.text, before.text);
|
| 2149 |
+
assert_eq!(after.payload, before.payload);
|
| 2150 |
+
assert_eq!(after.trailer, b"new tool map");
|
| 2151 |
+
assert_eq!(after.header.model_id, before.header.model_id);
|
| 2152 |
+
assert_eq!(after.header.quant_bits, before.header.quant_bits);
|
| 2153 |
+
assert_eq!(after.header.reason, before.header.reason);
|
| 2154 |
+
assert_eq!(after.header.tokens, before.header.tokens);
|
| 2155 |
+
assert_eq!(after.header.hits, before.header.hits);
|
| 2156 |
+
assert_eq!(after.header.ctx_size, before.header.ctx_size);
|
| 2157 |
+
assert_eq!(after.header.created_at, before.header.created_at);
|
| 2158 |
+
assert_eq!(after.header.payload_bytes, before.header.payload_bytes);
|
| 2159 |
+
assert_eq!(after.header.text_bytes, before.header.text_bytes);
|
| 2160 |
+
assert_eq!(
|
| 2161 |
+
after.header.ext_flags,
|
| 2162 |
+
before.header.ext_flags | crate::format::EXT_TOOL_MAP
|
| 2163 |
+
);
|
| 2164 |
+
assert!(after.header.last_used >= started);
|
| 2165 |
+
assert!(sibling.exists());
|
| 2166 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2167 |
+
}
|
| 2168 |
+
|
| 2169 |
+
#[test]
|
| 2170 |
+
fn compatible_trailer_rewrite_preserves_payload_and_header() {
|
| 2171 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-reuse-trailer-{}", std::process::id()));
|
| 2172 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2173 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2174 |
+
let mut existing = rec(b"same prompt", 512);
|
| 2175 |
+
existing.header.hits = 7;
|
| 2176 |
+
existing.header.created_at = 42;
|
| 2177 |
+
existing.header.last_used = 43;
|
| 2178 |
+
existing.header.ext_flags = crate::format::EXT_RESPONSES_VISIBLE;
|
| 2179 |
+
existing.payload = b"opaque payload".to_vec();
|
| 2180 |
+
existing.trailer = b"old trailer".to_vec();
|
| 2181 |
+
let path = store.write(existing).unwrap();
|
| 2182 |
+
let sibling = store.write(rec(b"sibling", 1024)).unwrap();
|
| 2183 |
+
let before = read_path(&path).unwrap();
|
| 2184 |
+
|
| 2185 |
+
let mut incoming = rec(b"same prompt", 2048).header;
|
| 2186 |
+
incoming.reason = Reason::Continued;
|
| 2187 |
+
incoming.ext_flags = crate::format::EXT_THINKING_VISIBLE;
|
| 2188 |
+
incoming.ctx_size = 4096;
|
| 2189 |
+
incoming.created_at = 99;
|
| 2190 |
+
incoming.last_used = 99;
|
| 2191 |
+
store.budget_bytes = 1;
|
| 2192 |
+
let started = now_secs();
|
| 2193 |
+
|
| 2194 |
+
store
|
| 2195 |
+
.write_payload_file(
|
| 2196 |
+
incoming,
|
| 2197 |
+
b"same prompt",
|
| 2198 |
+
&dir.join("missing-payload"),
|
| 2199 |
+
b"new tool map",
|
| 2200 |
+
)
|
| 2201 |
+
.unwrap();
|
| 2202 |
+
let got = read_path(&path).unwrap();
|
| 2203 |
+
assert_eq!(got.text, before.text);
|
| 2204 |
+
assert_eq!(got.payload, before.payload);
|
| 2205 |
+
assert_eq!(got.trailer, b"new tool map");
|
| 2206 |
+
assert_eq!(got.header.model_id, before.header.model_id);
|
| 2207 |
+
assert_eq!(got.header.quant_bits, before.header.quant_bits);
|
| 2208 |
+
assert_eq!(got.header.reason, before.header.reason);
|
| 2209 |
+
assert_eq!(got.header.tokens, before.header.tokens);
|
| 2210 |
+
assert_eq!(got.header.hits, before.header.hits);
|
| 2211 |
+
assert_eq!(got.header.ctx_size, before.header.ctx_size);
|
| 2212 |
+
assert_eq!(got.header.created_at, before.header.created_at);
|
| 2213 |
+
assert_eq!(got.header.payload_bytes, before.header.payload_bytes);
|
| 2214 |
+
assert_eq!(got.header.text_bytes, before.header.text_bytes);
|
| 2215 |
+
assert_eq!(
|
| 2216 |
+
got.header.ext_flags,
|
| 2217 |
+
before.header.ext_flags | crate::format::EXT_TOOL_MAP
|
| 2218 |
+
);
|
| 2219 |
+
assert!(got.header.last_used >= started);
|
| 2220 |
+
assert!(sibling.exists());
|
| 2221 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2222 |
+
}
|
| 2223 |
+
|
| 2224 |
+
#[test]
|
| 2225 |
+
fn payload_destination_alias_is_rejected_without_data_loss() {
|
| 2226 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-alias-{}", std::process::id()));
|
| 2227 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2228 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2229 |
+
let path = store.write(rec(b"same prompt", 512)).unwrap();
|
| 2230 |
+
let before = fs::read(&path).unwrap();
|
| 2231 |
+
let mut incompatible = rec(b"same prompt", 512).header;
|
| 2232 |
+
incompatible.model_id = 1;
|
| 2233 |
+
|
| 2234 |
+
let error = store
|
| 2235 |
+
.write_payload_file(incompatible, b"same prompt", &path, b"")
|
| 2236 |
+
.unwrap_err();
|
| 2237 |
+
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
|
| 2238 |
+
assert_eq!(fs::read(&path).unwrap(), before);
|
| 2239 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2240 |
+
}
|
| 2241 |
+
|
| 2242 |
+
#[test]
|
| 2243 |
+
fn stream_truncation_maps_to_unexpected_eof() {
|
| 2244 |
+
assert_eq!(
|
| 2245 |
+
format_io_error(FormatError::Truncated).kind(),
|
| 2246 |
+
io::ErrorKind::UnexpectedEof
|
| 2247 |
+
);
|
| 2248 |
+
}
|
| 2249 |
+
|
| 2250 |
+
#[test]
|
| 2251 |
+
fn failed_incompatible_stage_preserves_existing_entries() {
|
| 2252 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-stage-fail-{}", std::process::id()));
|
| 2253 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2254 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2255 |
+
let target = store.write(rec(b"same prompt", 8192)).unwrap();
|
| 2256 |
+
let sibling = store.write(rec(b"sibling", 512)).unwrap();
|
| 2257 |
+
let target_before = fs::read(&target).unwrap();
|
| 2258 |
+
let sibling_before = fs::read(&sibling).unwrap();
|
| 2259 |
+
let mut incoming = rec(b"same prompt", 1024).header;
|
| 2260 |
+
incoming.model_id = 1;
|
| 2261 |
+
|
| 2262 |
+
let error = store
|
| 2263 |
+
.write_payload_file(incoming, b"same prompt", &dir.join("missing-payload"), b"")
|
| 2264 |
+
.unwrap_err();
|
| 2265 |
+
assert_eq!(error.kind(), io::ErrorKind::NotFound);
|
| 2266 |
+
assert_eq!(fs::read(&target).unwrap(), target_before);
|
| 2267 |
+
assert_eq!(fs::read(&sibling).unwrap(), sibling_before);
|
| 2268 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2269 |
+
}
|
| 2270 |
+
|
| 2271 |
+
#[test]
|
| 2272 |
+
fn incompatible_replacement_credits_target_and_keeps_sibling() {
|
| 2273 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-replace-{}", std::process::id()));
|
| 2274 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2275 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2276 |
+
let text = b"same prompt";
|
| 2277 |
+
let target = store.write(rec(text, 8192)).unwrap();
|
| 2278 |
+
let sibling = store.write(rec(b"sibling", 512)).unwrap();
|
| 2279 |
+
let payload = b"new payload";
|
| 2280 |
+
let payload_path = dir.join("incoming.payload");
|
| 2281 |
+
fs::write(&payload_path, payload).unwrap();
|
| 2282 |
+
let incoming_bytes = file_size_bytes(text.len() as u64, payload.len() as u64, 0).unwrap();
|
| 2283 |
+
store.budget_bytes = fs::metadata(&sibling).unwrap().len() + incoming_bytes;
|
| 2284 |
+
let mut incoming = rec(text, 1024).header;
|
| 2285 |
+
incoming.model_id = 1;
|
| 2286 |
+
|
| 2287 |
+
let got = store
|
| 2288 |
+
.write_payload_file(incoming, text, &payload_path, b"")
|
| 2289 |
+
.unwrap();
|
| 2290 |
+
let replaced = read_path(&target).unwrap();
|
| 2291 |
+
assert_eq!(got, target);
|
| 2292 |
+
assert_eq!(replaced.header.model_id, 1);
|
| 2293 |
+
assert_eq!(replaced.payload, payload);
|
| 2294 |
+
assert!(sibling.exists());
|
| 2295 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2296 |
+
}
|
| 2297 |
+
|
| 2298 |
+
#[test]
|
| 2299 |
+
fn candidate_reopens_and_validates_text() {
|
| 2300 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-candidate-{}", std::process::id()));
|
| 2301 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2302 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2303 |
+
let path = store.write(rec(b"shared prefix", 512)).unwrap();
|
| 2304 |
+
|
| 2305 |
+
let (got_path, envelope) = store
|
| 2306 |
+
.text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192)
|
| 2307 |
+
.unwrap()
|
| 2308 |
+
.unwrap();
|
| 2309 |
+
assert_eq!(got_path, path);
|
| 2310 |
+
assert_eq!(envelope.text, b"shared prefix");
|
| 2311 |
+
|
| 2312 |
+
let mut file = fs::OpenOptions::new().write(true).open(&path).unwrap();
|
| 2313 |
+
file.seek(SeekFrom::Start((crate::format::FIXED_HEADER + 4) as u64))
|
| 2314 |
+
.unwrap();
|
| 2315 |
+
file.write_all(b"X").unwrap();
|
| 2316 |
+
file.flush().unwrap();
|
| 2317 |
+
// The text no longer hashes to its own name: the record is refused
|
| 2318 |
+
// by its contents, which is not the same as not being there.
|
| 2319 |
+
assert_eq!(
|
| 2320 |
+
store
|
| 2321 |
+
.text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192)
|
| 2322 |
+
.unwrap_err()
|
| 2323 |
+
.kind(),
|
| 2324 |
+
io::ErrorKind::InvalidData
|
| 2325 |
+
);
|
| 2326 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2327 |
+
}
|
| 2328 |
+
|
| 2329 |
+
#[test]
|
| 2330 |
+
fn bank_candidate_requires_a_valid_strict_suffix() {
|
| 2331 |
+
let dir =
|
| 2332 |
+
std::env::temp_dir().join(format!("ds4-kv-bank-candidate-{}", std::process::id()));
|
| 2333 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2334 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2335 |
+
let mut record = rec(b"shared prefix", 512);
|
| 2336 |
+
record.header.reason = Reason::BankShutdown;
|
| 2337 |
+
record.header.ext_flags = crate::format::EXT_BANK_REPLAY_V1;
|
| 2338 |
+
let path = store.write(record).unwrap();
|
| 2339 |
+
let mut serial = rec(b"serial exact", 512);
|
| 2340 |
+
serial.header.reason = Reason::Shutdown;
|
| 2341 |
+
let serial_path = store.write(serial).unwrap();
|
| 2342 |
+
|
| 2343 |
+
assert!(store
|
| 2344 |
+
.bank_text_prefix_candidate(b"shared prefix", 0, 2, 8192)
|
| 2345 |
+
.unwrap()
|
| 2346 |
+
.is_none());
|
| 2347 |
+
assert!(store
|
| 2348 |
+
.bank_text_prefix_candidate(b"serial exact", 0, 2, 8192)
|
| 2349 |
+
.unwrap()
|
| 2350 |
+
.is_none());
|
| 2351 |
+
assert_eq!(
|
| 2352 |
+
store
|
| 2353 |
+
.bank_text_prefix_candidate(b"serial exact suffix", 0, 2, 8192)
|
| 2354 |
+
.unwrap()
|
| 2355 |
+
.unwrap()
|
| 2356 |
+
.0,
|
| 2357 |
+
serial_path
|
| 2358 |
+
);
|
| 2359 |
+
let (got_path, envelope) = store
|
| 2360 |
+
.bank_text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192)
|
| 2361 |
+
.unwrap()
|
| 2362 |
+
.unwrap();
|
| 2363 |
+
assert_eq!(got_path, path);
|
| 2364 |
+
assert_eq!(envelope.text, b"shared prefix");
|
| 2365 |
+
|
| 2366 |
+
let mut file = fs::OpenOptions::new().write(true).open(&path).unwrap();
|
| 2367 |
+
file.seek(SeekFrom::Start((crate::format::FIXED_HEADER + 4) as u64))
|
| 2368 |
+
.unwrap();
|
| 2369 |
+
file.write_all(b"X").unwrap();
|
| 2370 |
+
file.flush().unwrap();
|
| 2371 |
+
assert_eq!(
|
| 2372 |
+
store
|
| 2373 |
+
.bank_text_prefix_candidate(b"shared prefix and suffix", 0, 2, 8192)
|
| 2374 |
+
.unwrap_err()
|
| 2375 |
+
.kind(),
|
| 2376 |
+
io::ErrorKind::InvalidData
|
| 2377 |
+
);
|
| 2378 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2379 |
+
}
|
| 2380 |
+
|
| 2381 |
+
#[test]
|
| 2382 |
+
fn touch_hit_preserves_payload_and_trailer() {
|
| 2383 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-touch-{}", std::process::id()));
|
| 2384 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2385 |
+
let mut store = Store::open(&dir, 16, false, Options::default()).unwrap();
|
| 2386 |
+
let mut record = rec(b"shared prefix", 512);
|
| 2387 |
+
record.header.hits = 7;
|
| 2388 |
+
record.header.created_at = 42;
|
| 2389 |
+
record.header.last_used = 43;
|
| 2390 |
+
record.payload = b"opaque payload".to_vec();
|
| 2391 |
+
record.trailer = b"trailer".to_vec();
|
| 2392 |
+
let path = store.write(record).unwrap();
|
| 2393 |
+
let before = read_path(&path).unwrap();
|
| 2394 |
+
let started = now_secs();
|
| 2395 |
+
|
| 2396 |
+
store.touch_hit(&path).unwrap();
|
| 2397 |
+
let after = read_path(&path).unwrap();
|
| 2398 |
+
assert_eq!(after.header.hits, before.header.hits + 1);
|
| 2399 |
+
assert_eq!(after.header.created_at, before.header.created_at);
|
| 2400 |
+
assert!(after.header.last_used >= started);
|
| 2401 |
+
assert_eq!(after.text, before.text);
|
| 2402 |
+
assert_eq!(after.payload, before.payload);
|
| 2403 |
+
assert_eq!(after.trailer, before.trailer);
|
| 2404 |
+
let _ = fs::remove_dir_all(&dir);
|
| 2405 |
+
}
|
| 2406 |
+
}
|
crates/ds4-kv/tests/bank_thinking.rs
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Bank thinking bit + one KTM extension record.
|
| 2 |
+
//!
|
| 3 |
+
//! Given / When / Then against the C KVC envelope: thinking is
|
| 4 |
+
//! `EXT_THINKING_VISIBLE`, the only post-DSV4 magic trailer is `KTM\x01`.
|
| 5 |
+
|
| 6 |
+
use ds4_kv::{
|
| 7 |
+
decode_file, encode_file, read_path, write_path, BankThinkingExtensions, ExtensionRecord,
|
| 8 |
+
Header, Reason, Record, EXT_BANK_REPLAY_V1, EXT_THINKING_VISIBLE, EXT_TOOL_MAP,
|
| 9 |
+
};
|
| 10 |
+
use std::fs;
|
| 11 |
+
use std::path::{Path, PathBuf};
|
| 12 |
+
use std::process::Command;
|
| 13 |
+
|
| 14 |
+
fn sample_bank() -> Record {
|
| 15 |
+
Record {
|
| 16 |
+
header: Header {
|
| 17 |
+
quant_bits: 2,
|
| 18 |
+
reason: Reason::BankCheckpoint,
|
| 19 |
+
ext_flags: EXT_BANK_REPLAY_V1,
|
| 20 |
+
model_id: 3,
|
| 21 |
+
tokens: 64,
|
| 22 |
+
hits: 1,
|
| 23 |
+
ctx_size: 4096,
|
| 24 |
+
created_at: 1_700_000_000,
|
| 25 |
+
last_used: 1_700_000_050,
|
| 26 |
+
payload_bytes: 6,
|
| 27 |
+
text_bytes: 11,
|
| 28 |
+
},
|
| 29 |
+
text: b"hello world".to_vec(),
|
| 30 |
+
payload: b"PAYLOD".to_vec(),
|
| 31 |
+
trailer: Vec::new(),
|
| 32 |
+
}
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
fn tmp(name: &str) -> PathBuf {
|
| 36 |
+
let dir =
|
| 37 |
+
std::env::temp_dir().join(format!("ds4-kv-bank-think-{}-{}", std::process::id(), name));
|
| 38 |
+
let _ = fs::remove_dir_all(&dir);
|
| 39 |
+
fs::create_dir_all(&dir).unwrap();
|
| 40 |
+
dir.join("bank.kv")
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
fn oracle() -> PathBuf {
|
| 44 |
+
if let Ok(p) = std::env::var("DS4_KV_C_ORACLE") {
|
| 45 |
+
return PathBuf::from(p);
|
| 46 |
+
}
|
| 47 |
+
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/kv_c_oracle")
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
fn hex(bytes: &[u8]) -> String {
|
| 51 |
+
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
fn unhex(s: &str) -> Vec<u8> {
|
| 55 |
+
(0..s.len())
|
| 56 |
+
.step_by(2)
|
| 57 |
+
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
|
| 58 |
+
.collect()
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
fn c_write(path: &Path, rec: &Record) {
|
| 62 |
+
let status = Command::new(oracle())
|
| 63 |
+
.args([
|
| 64 |
+
"write",
|
| 65 |
+
"--path",
|
| 66 |
+
path.to_str().unwrap(),
|
| 67 |
+
"--text-hex",
|
| 68 |
+
&hex(&rec.text),
|
| 69 |
+
"--payload-hex",
|
| 70 |
+
&hex(&rec.payload),
|
| 71 |
+
"--model-id",
|
| 72 |
+
&rec.header.model_id.to_string(),
|
| 73 |
+
"--quant",
|
| 74 |
+
&rec.header.quant_bits.to_string(),
|
| 75 |
+
"--reason",
|
| 76 |
+
&(rec.header.reason as u8).to_string(),
|
| 77 |
+
"--ext",
|
| 78 |
+
&rec.header.ext_flags.to_string(),
|
| 79 |
+
"--tokens",
|
| 80 |
+
&rec.header.tokens.to_string(),
|
| 81 |
+
"--hits",
|
| 82 |
+
&rec.header.hits.to_string(),
|
| 83 |
+
"--ctx",
|
| 84 |
+
&rec.header.ctx_size.to_string(),
|
| 85 |
+
"--created",
|
| 86 |
+
&rec.header.created_at.to_string(),
|
| 87 |
+
"--used",
|
| 88 |
+
&rec.header.last_used.to_string(),
|
| 89 |
+
"--trailer-hex",
|
| 90 |
+
&hex(&rec.trailer),
|
| 91 |
+
])
|
| 92 |
+
.status()
|
| 93 |
+
.unwrap();
|
| 94 |
+
assert!(status.success(), "C write failed");
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
fn c_read(path: &Path) -> Record {
|
| 98 |
+
let out = Command::new(oracle())
|
| 99 |
+
.args(["read", path.to_str().unwrap()])
|
| 100 |
+
.output()
|
| 101 |
+
.unwrap();
|
| 102 |
+
assert!(
|
| 103 |
+
out.status.success(),
|
| 104 |
+
"C read failed: {}",
|
| 105 |
+
String::from_utf8_lossy(&out.stderr)
|
| 106 |
+
);
|
| 107 |
+
let text = String::from_utf8(out.stdout).unwrap();
|
| 108 |
+
let mut rec = sample_bank();
|
| 109 |
+
for line in text.lines() {
|
| 110 |
+
let Some((k, v)) = line.split_once('=') else {
|
| 111 |
+
continue;
|
| 112 |
+
};
|
| 113 |
+
match k {
|
| 114 |
+
"model_id" => rec.header.model_id = v.parse().unwrap(),
|
| 115 |
+
"quant_bits" => rec.header.quant_bits = v.parse().unwrap(),
|
| 116 |
+
"reason" => rec.header.reason = Reason::from_u8(v.parse().unwrap()),
|
| 117 |
+
"ext_flags" => rec.header.ext_flags = v.parse().unwrap(),
|
| 118 |
+
"tokens" => rec.header.tokens = v.parse().unwrap(),
|
| 119 |
+
"hits" => rec.header.hits = v.parse().unwrap(),
|
| 120 |
+
"ctx_size" => rec.header.ctx_size = v.parse().unwrap(),
|
| 121 |
+
"created_at" => rec.header.created_at = v.parse().unwrap(),
|
| 122 |
+
"last_used" => rec.header.last_used = v.parse().unwrap(),
|
| 123 |
+
"payload_bytes" => rec.header.payload_bytes = v.parse().unwrap(),
|
| 124 |
+
"text_bytes" => rec.header.text_bytes = v.parse().unwrap(),
|
| 125 |
+
"text_hex" => rec.text = unhex(v),
|
| 126 |
+
"payload_hex" => rec.payload = unhex(v),
|
| 127 |
+
"trailer_hex" => rec.trailer = unhex(v),
|
| 128 |
+
_ => {}
|
| 129 |
+
}
|
| 130 |
+
}
|
| 131 |
+
rec
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
fn one_extension() -> BankThinkingExtensions {
|
| 135 |
+
BankThinkingExtensions {
|
| 136 |
+
thinking_visible: true,
|
| 137 |
+
records: vec![ExtensionRecord {
|
| 138 |
+
id: b"call_abc".to_vec(),
|
| 139 |
+
body: b"<dsml>function_calls>x".to_vec(),
|
| 140 |
+
}],
|
| 141 |
+
}
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
#[test]
|
| 145 |
+
fn bank_save_load_thinking_bit_and_one_extension_record() {
|
| 146 |
+
// Given: a bank checkpoint with thinking-visible text and one KTM record
|
| 147 |
+
let ext = one_extension();
|
| 148 |
+
let mut rec = sample_bank();
|
| 149 |
+
rec.persist_bank_extensions(&ext);
|
| 150 |
+
|
| 151 |
+
// When: the envelope is written and read back
|
| 152 |
+
let path = tmp("roundtrip");
|
| 153 |
+
write_path(&path, &rec).unwrap();
|
| 154 |
+
let got = read_path(&path).unwrap();
|
| 155 |
+
let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer);
|
| 156 |
+
|
| 157 |
+
// Then: thinking bit + BANK_REPLAY_V1 + TOOL_MAP and the one record survive
|
| 158 |
+
assert_eq!(
|
| 159 |
+
got.header.ext_flags,
|
| 160 |
+
EXT_BANK_REPLAY_V1 | EXT_THINKING_VISIBLE | EXT_TOOL_MAP
|
| 161 |
+
);
|
| 162 |
+
assert!(restored.thinking_visible);
|
| 163 |
+
assert_eq!(restored.records, ext.records);
|
| 164 |
+
assert_eq!(&got.trailer[..4], b"KTM\x01");
|
| 165 |
+
assert_eq!(encode_file(&rec)[..3], *b"KVC");
|
| 166 |
+
|
| 167 |
+
let _ = fs::remove_dir_all(path.parent().unwrap());
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
#[test]
|
| 171 |
+
fn missing_extension_section_is_ignored_like_c() {
|
| 172 |
+
// Given: a bank file with replay-v1 only (no thinking bit, no trailer)
|
| 173 |
+
let rec = sample_bank();
|
| 174 |
+
let bytes = encode_file(&rec);
|
| 175 |
+
let got = decode_file(&bytes).unwrap();
|
| 176 |
+
|
| 177 |
+
// When: restore runs on a missing section
|
| 178 |
+
let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer);
|
| 179 |
+
|
| 180 |
+
// Then: C-like ignore — thinking off, zero records, file still valid
|
| 181 |
+
assert_eq!(got.header.ext_flags, EXT_BANK_REPLAY_V1);
|
| 182 |
+
assert!(!restored.thinking_visible);
|
| 183 |
+
assert!(restored.records.is_empty());
|
| 184 |
+
}
|
| 185 |
+
|
| 186 |
+
#[test]
|
| 187 |
+
fn invalid_ktm_magic_is_ignored_like_c() {
|
| 188 |
+
// Given: a trailer that is not KTM\x01
|
| 189 |
+
let restored = BankThinkingExtensions::restore(EXT_TOOL_MAP, b"XXX\x01\x01\0\0\0");
|
| 190 |
+
|
| 191 |
+
// When/Then: C kv_tool_map_load_from_pos returns 0
|
| 192 |
+
assert!(!restored.thinking_visible);
|
| 193 |
+
assert!(restored.records.is_empty());
|
| 194 |
+
}
|
| 195 |
+
|
| 196 |
+
#[test]
|
| 197 |
+
fn rust_bank_thinking_extension_is_c_readable() {
|
| 198 |
+
let oracle = oracle();
|
| 199 |
+
assert!(
|
| 200 |
+
oracle.exists(),
|
| 201 |
+
"build the C oracle first: make tests/parity/kv_c_oracle (missing {})",
|
| 202 |
+
oracle.display()
|
| 203 |
+
);
|
| 204 |
+
|
| 205 |
+
let ext = one_extension();
|
| 206 |
+
let mut rec = sample_bank();
|
| 207 |
+
rec.persist_bank_extensions(&ext);
|
| 208 |
+
rec.header.text_bytes = rec.text.len() as u32;
|
| 209 |
+
rec.header.payload_bytes = rec.payload.len() as u64;
|
| 210 |
+
|
| 211 |
+
let path = tmp("rust-c");
|
| 212 |
+
write_path(&path, &rec).unwrap();
|
| 213 |
+
let got = c_read(&path);
|
| 214 |
+
let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer);
|
| 215 |
+
|
| 216 |
+
assert_eq!(got.header.ext_flags, rec.header.ext_flags);
|
| 217 |
+
assert_eq!(got.trailer, rec.trailer);
|
| 218 |
+
assert!(restored.thinking_visible);
|
| 219 |
+
assert_eq!(restored.records, ext.records);
|
| 220 |
+
|
| 221 |
+
let _ = fs::remove_dir_all(path.parent().unwrap());
|
| 222 |
+
}
|
| 223 |
+
|
| 224 |
+
#[test]
|
| 225 |
+
fn c_bank_thinking_extension_loads_in_rust() {
|
| 226 |
+
let oracle = oracle();
|
| 227 |
+
assert!(
|
| 228 |
+
oracle.exists(),
|
| 229 |
+
"build the C oracle first: make tests/parity/kv_c_oracle (missing {})",
|
| 230 |
+
oracle.display()
|
| 231 |
+
);
|
| 232 |
+
|
| 233 |
+
let ext = one_extension();
|
| 234 |
+
let mut rec = sample_bank();
|
| 235 |
+
rec.persist_bank_extensions(&ext);
|
| 236 |
+
rec.header.text_bytes = rec.text.len() as u32;
|
| 237 |
+
rec.header.payload_bytes = rec.payload.len() as u64;
|
| 238 |
+
|
| 239 |
+
let path = tmp("c-rust");
|
| 240 |
+
c_write(&path, &rec);
|
| 241 |
+
let got = decode_file(&fs::read(&path).unwrap()).unwrap();
|
| 242 |
+
let restored = BankThinkingExtensions::restore(got.header.ext_flags, &got.trailer);
|
| 243 |
+
|
| 244 |
+
assert_eq!(got.header.ext_flags, rec.header.ext_flags);
|
| 245 |
+
assert_eq!(got.trailer, rec.trailer);
|
| 246 |
+
assert!(restored.thinking_visible);
|
| 247 |
+
assert_eq!(restored.records, ext.records);
|
| 248 |
+
|
| 249 |
+
let _ = fs::remove_dir_all(path.parent().unwrap());
|
| 250 |
+
}
|
crates/ds4-kv/tests/four_way.rs
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//! Four-way KVC matrix: C save/load × Rust save/load.
|
| 2 |
+
|
| 3 |
+
use ds4_kv::{
|
| 4 |
+
chat_anchor_pos, continued_store_target, decode_file, encode_file, eviction_score, read_path,
|
| 5 |
+
read_trailer, store_len, write_path, Header, Options, Reason, Record, ScoreEntry, Store,
|
| 6 |
+
EXT_TOOL_MAP,
|
| 7 |
+
};
|
| 8 |
+
use std::fs;
|
| 9 |
+
use std::path::{Path, PathBuf};
|
| 10 |
+
use std::process::Command;
|
| 11 |
+
|
| 12 |
+
fn oracle() -> PathBuf {
|
| 13 |
+
if let Ok(p) = std::env::var("DS4_KV_C_ORACLE") {
|
| 14 |
+
return PathBuf::from(p);
|
| 15 |
+
}
|
| 16 |
+
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/parity/kv_c_oracle")
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
fn require_oracle() -> PathBuf {
|
| 20 |
+
let p = oracle();
|
| 21 |
+
assert!(
|
| 22 |
+
p.exists(),
|
| 23 |
+
"build the C oracle first: make tests/parity/kv_c_oracle (missing {})",
|
| 24 |
+
p.display()
|
| 25 |
+
);
|
| 26 |
+
p
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
fn hex(bytes: &[u8]) -> String {
|
| 30 |
+
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
fn unhex(s: &str) -> Vec<u8> {
|
| 34 |
+
(0..s.len())
|
| 35 |
+
.step_by(2)
|
| 36 |
+
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
|
| 37 |
+
.collect()
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
fn sample() -> Record {
|
| 41 |
+
Record {
|
| 42 |
+
header: Header {
|
| 43 |
+
quant_bits: 2,
|
| 44 |
+
reason: Reason::Cold,
|
| 45 |
+
ext_flags: 0,
|
| 46 |
+
model_id: 3,
|
| 47 |
+
tokens: 512,
|
| 48 |
+
hits: 7,
|
| 49 |
+
ctx_size: 4096,
|
| 50 |
+
created_at: 1_700_000_000,
|
| 51 |
+
last_used: 1_700_000_050,
|
| 52 |
+
payload_bytes: 6,
|
| 53 |
+
text_bytes: 11,
|
| 54 |
+
},
|
| 55 |
+
text: b"hello world".to_vec(),
|
| 56 |
+
payload: b"PAYLOD".to_vec(),
|
| 57 |
+
trailer: Vec::new(),
|
| 58 |
+
}
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
fn c_write(path: &Path, rec: &Record) {
|
| 62 |
+
let status = Command::new(require_oracle())
|
| 63 |
+
.args([
|
| 64 |
+
"write",
|
| 65 |
+
"--path",
|
| 66 |
+
path.to_str().unwrap(),
|
| 67 |
+
"--text-hex",
|
| 68 |
+
&hex(&rec.text),
|
| 69 |
+
"--payload-hex",
|
| 70 |
+
&hex(&rec.payload),
|
| 71 |
+
"--model-id",
|
| 72 |
+
&rec.header.model_id.to_string(),
|
| 73 |
+
"--quant",
|
| 74 |
+
&rec.header.quant_bits.to_string(),
|
| 75 |
+
"--reason",
|
| 76 |
+
&(rec.header.reason as u8).to_string(),
|
| 77 |
+
"--ext",
|
| 78 |
+
&rec.header.ext_flags.to_string(),
|
| 79 |
+
"--tokens",
|
| 80 |
+
&rec.header.tokens.to_string(),
|
| 81 |
+
"--hits",
|
| 82 |
+
&rec.header.hits.to_string(),
|
| 83 |
+
"--ctx",
|
| 84 |
+
&rec.header.ctx_size.to_string(),
|
| 85 |
+
"--created",
|
| 86 |
+
&rec.header.created_at.to_string(),
|
| 87 |
+
"--used",
|
| 88 |
+
&rec.header.last_used.to_string(),
|
| 89 |
+
"--trailer-hex",
|
| 90 |
+
&hex(&rec.trailer),
|
| 91 |
+
])
|
| 92 |
+
.status()
|
| 93 |
+
.unwrap();
|
| 94 |
+
assert!(status.success(), "C write failed");
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
fn c_read(path: &Path) -> Record {
|
| 98 |
+
let out = Command::new(require_oracle())
|
| 99 |
+
.args(["read", path.to_str().unwrap()])
|
| 100 |
+
.output()
|
| 101 |
+
.unwrap();
|
| 102 |
+
assert!(
|
| 103 |
+
out.status.success(),
|
| 104 |
+
"C read failed: {}",
|
| 105 |
+
String::from_utf8_lossy(&out.stderr)
|
| 106 |
+
);
|
| 107 |
+
let text = String::from_utf8(out.stdout).unwrap();
|
| 108 |
+
let mut rec = sample();
|
| 109 |
+
for line in text.lines() {
|
| 110 |
+
let Some((k, v)) = line.split_once('=') else {
|
| 111 |
+
continue;
|
| 112 |
+
};
|
| 113 |
+
match k {
|
| 114 |
+
"model_id" => rec.header.model_id = v.parse().unwrap(),
|
| 115 |
+
"quant_bits" => rec.header.quant_bits = v.parse().unwrap(),
|
| 116 |
+
"reason" => rec.header.reason = Reason::from_u8(v.parse().unwrap()),
|
| 117 |
+
"ext_flags" => rec.header.ext_flags = v.parse().unwrap(),
|
| 118 |
+
"tokens" => rec.header.tokens = v.parse().unwrap(),
|
| 119 |
+
"hits" => rec.header.hits = v.parse().unwrap(),
|
| 120 |
+
"ctx_size" => rec.header.ctx_size = v.parse().unwrap(),
|
| 121 |
+
"created_at" => rec.header.created_at = v.parse().unwrap(),
|
| 122 |
+
"last_used" => rec.header.last_used = v.parse().unwrap(),
|
| 123 |
+
"payload_bytes" => rec.header.payload_bytes = v.parse().unwrap(),
|
| 124 |
+
"text_bytes" => rec.header.text_bytes = v.parse().unwrap(),
|
| 125 |
+
"text_hex" => rec.text = unhex(v),
|
| 126 |
+
"payload_hex" => rec.payload = unhex(v),
|
| 127 |
+
"trailer_hex" => rec.trailer = unhex(v),
|
| 128 |
+
_ => {}
|
| 129 |
+
}
|
| 130 |
+
}
|
| 131 |
+
rec
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
fn tmp(name: &str) -> PathBuf {
|
| 135 |
+
let dir = std::env::temp_dir().join(format!("ds4-kv-4way-{}", std::process::id()));
|
| 136 |
+
fs::create_dir_all(&dir).unwrap();
|
| 137 |
+
dir.join(name)
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
fn same_envelope(a: &Record, b: &Record) {
|
| 141 |
+
assert_eq!(a.header.model_id, b.header.model_id);
|
| 142 |
+
assert_eq!(a.header.quant_bits, b.header.quant_bits);
|
| 143 |
+
assert_eq!(a.header.reason, b.header.reason);
|
| 144 |
+
assert_eq!(a.header.ext_flags, b.header.ext_flags);
|
| 145 |
+
assert_eq!(a.header.tokens, b.header.tokens);
|
| 146 |
+
assert_eq!(a.header.hits, b.header.hits);
|
| 147 |
+
assert_eq!(a.header.ctx_size, b.header.ctx_size);
|
| 148 |
+
assert_eq!(a.header.created_at, b.header.created_at);
|
| 149 |
+
assert_eq!(a.header.last_used, b.header.last_used);
|
| 150 |
+
assert_eq!(a.header.payload_bytes, b.header.payload_bytes);
|
| 151 |
+
assert_eq!(a.header.text_bytes, b.header.text_bytes);
|
| 152 |
+
assert_eq!(a.text, b.text);
|
| 153 |
+
assert_eq!(a.payload, b.payload);
|
| 154 |
+
assert_eq!(a.trailer, b.trailer);
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
#[test]
|
| 158 |
+
fn rust_save_rust_load() {
|
| 159 |
+
let rec = sample();
|
| 160 |
+
let path = tmp("rr.kv");
|
| 161 |
+
write_path(&path, &rec).unwrap();
|
| 162 |
+
let got = read_path(&path).unwrap();
|
| 163 |
+
same_envelope(&rec, &got);
|
| 164 |
+
assert_eq!(encode_file(&rec), fs::read(&path).unwrap());
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
#[test]
|
| 168 |
+
fn c_save_c_load() {
|
| 169 |
+
let rec = sample();
|
| 170 |
+
let path = tmp("cc.kv");
|
| 171 |
+
c_write(&path, &rec);
|
| 172 |
+
let got = c_read(&path);
|
| 173 |
+
same_envelope(&rec, &got);
|
| 174 |
+
}
|
| 175 |
+
|
| 176 |
+
#[test]
|
| 177 |
+
fn rust_save_c_load() {
|
| 178 |
+
let rec = sample();
|
| 179 |
+
let path = tmp("rc.kv");
|
| 180 |
+
write_path(&path, &rec).unwrap();
|
| 181 |
+
let got = c_read(&path);
|
| 182 |
+
same_envelope(&rec, &got);
|
| 183 |
+
}
|
| 184 |
+
|
| 185 |
+
#[test]
|
| 186 |
+
fn c_save_rust_load() {
|
| 187 |
+
let rec = sample();
|
| 188 |
+
let path = tmp("cr.kv");
|
| 189 |
+
c_write(&path, &rec);
|
| 190 |
+
let got = decode_file(&fs::read(&path).unwrap()).unwrap();
|
| 191 |
+
same_envelope(&rec, &got);
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
#[test]
|
| 195 |
+
fn c_save_rust_trailer_only_load() {
|
| 196 |
+
let mut rec = sample();
|
| 197 |
+
rec.header.ext_flags = EXT_TOOL_MAP;
|
| 198 |
+
rec.trailer = b"KTM\x01\x01\0\0\0\xff".to_vec();
|
| 199 |
+
let path = tmp("cr-trailer.kv");
|
| 200 |
+
c_write(&path, &rec);
|
| 201 |
+
|
| 202 |
+
let (header, trailer) = read_trailer(&path, rec.trailer.len() as u64).unwrap();
|
| 203 |
+
assert_eq!(header.ext_flags, EXT_TOOL_MAP);
|
| 204 |
+
assert_eq!(trailer, rec.trailer);
|
| 205 |
+
}
|
| 206 |
+
|
| 207 |
+
#[test]
|
| 208 |
+
fn rust_stream_payload_with_trailer_loads_in_c() {
|
| 209 |
+
let base = tmp("stream-rust-c");
|
| 210 |
+
let store_dir = base.join("store");
|
| 211 |
+
let _ = fs::remove_dir_all(&base);
|
| 212 |
+
fs::create_dir_all(&base).unwrap();
|
| 213 |
+
|
| 214 |
+
let mut rec = sample();
|
| 215 |
+
rec.payload = b"DSV4 opaque native payload\0\xff".to_vec();
|
| 216 |
+
rec.trailer = b"KVT1 tool-map trailer".to_vec();
|
| 217 |
+
rec.header.ext_flags = ds4_kv::EXT_TOOL_MAP;
|
| 218 |
+
rec.header.text_bytes = rec.text.len() as u32;
|
| 219 |
+
rec.header.payload_bytes = rec.payload.len() as u64;
|
| 220 |
+
let payload_path = ds4_kv::path_for_sha(&store_dir, &ds4_kv::text_sha_hex(&rec.text))
|
| 221 |
+
.with_extension(format!("kv.tmp.{}", std::process::id()));
|
| 222 |
+
fs::create_dir_all(&store_dir).unwrap();
|
| 223 |
+
fs::write(&payload_path, &rec.payload).unwrap();
|
| 224 |
+
|
| 225 |
+
let mut store = Store::open(&store_dir, 16, false, Options::default()).unwrap();
|
| 226 |
+
let path = store
|
| 227 |
+
.write_payload_file(rec.header.clone(), &rec.text, &payload_path, &rec.trailer)
|
| 228 |
+
.unwrap();
|
| 229 |
+
assert_eq!(fs::read(&payload_path).unwrap(), rec.payload);
|
| 230 |
+
let got = c_read(&path);
|
| 231 |
+
same_envelope(&rec, &got);
|
| 232 |
+
|
| 233 |
+
let _ = fs::remove_dir_all(&base);
|
| 234 |
+
}
|
| 235 |
+
|
| 236 |
+
#[test]
|
| 237 |
+
fn sha1_matches_c() {
|
| 238 |
+
let text = b"hello world";
|
| 239 |
+
let out = Command::new(require_oracle())
|
| 240 |
+
.args(["sha1", "--text-hex", &hex(text)])
|
| 241 |
+
.output()
|
| 242 |
+
.unwrap();
|
| 243 |
+
assert!(out.status.success());
|
| 244 |
+
let c = String::from_utf8(out.stdout).unwrap();
|
| 245 |
+
assert_eq!(c.trim(), ds4_kv::text_sha_hex(text));
|
| 246 |
+
}
|
| 247 |
+
|
| 248 |
+
#[test]
|
| 249 |
+
fn store_len_matches_c() {
|
| 250 |
+
for tokens in [100, 512, 2048, 4096 + 40, 20000] {
|
| 251 |
+
let out = Command::new(require_oracle())
|
| 252 |
+
.args(["store-len", "--tokens", &tokens.to_string()])
|
| 253 |
+
.output()
|
| 254 |
+
.unwrap();
|
| 255 |
+
assert!(out.status.success());
|
| 256 |
+
let c: i32 = String::from_utf8(out.stdout)
|
| 257 |
+
.unwrap()
|
| 258 |
+
.trim()
|
| 259 |
+
.parse()
|
| 260 |
+
.unwrap();
|
| 261 |
+
assert_eq!(store_len(&Options::default(), tokens), c, "tokens={tokens}");
|
| 262 |
+
}
|
| 263 |
+
}
|
| 264 |
+
|
| 265 |
+
#[test]
|
| 266 |
+
fn continued_target_matches_c() {
|
| 267 |
+
for (name, min, interval, align, last, live, expected) in [
|
| 268 |
+
("default", 512, 10000, 2048, 0, 10240, 10240),
|
| 269 |
+
("disabled", 512, 0, 2048, 0, 10240, 0),
|
| 270 |
+
("custom-align", 512, 10000, 4096, 0, 12288, 12288),
|
| 271 |
+
("below-min", 20000, 10000, 2048, 0, 10240, 0),
|
| 272 |
+
("prior-marker", 512, 10000, 2048, 10240, 10240, 0),
|
| 273 |
+
("nonmultiple", 512, 10000, 2048, 0, 10239, 0),
|
| 274 |
+
("serial-align1", 512, 6800, 1, 0, 6800, 6800),
|
| 275 |
+
] {
|
| 276 |
+
let out = Command::new(require_oracle())
|
| 277 |
+
.args([
|
| 278 |
+
"continued-target",
|
| 279 |
+
"--min",
|
| 280 |
+
&min.to_string(),
|
| 281 |
+
"--interval",
|
| 282 |
+
&interval.to_string(),
|
| 283 |
+
"--align",
|
| 284 |
+
&align.to_string(),
|
| 285 |
+
"--last",
|
| 286 |
+
&last.to_string(),
|
| 287 |
+
"--live",
|
| 288 |
+
&live.to_string(),
|
| 289 |
+
])
|
| 290 |
+
.output()
|
| 291 |
+
.unwrap();
|
| 292 |
+
assert!(out.status.success(), "C continued-target failed for {name}");
|
| 293 |
+
let c: i32 = String::from_utf8(out.stdout)
|
| 294 |
+
.unwrap()
|
| 295 |
+
.trim()
|
| 296 |
+
.parse()
|
| 297 |
+
.unwrap();
|
| 298 |
+
assert_eq!(c, expected, "C case={name}");
|
| 299 |
+
let opt = Options {
|
| 300 |
+
min_tokens: min,
|
| 301 |
+
continued_interval_tokens: interval,
|
| 302 |
+
boundary_align_tokens: align,
|
| 303 |
+
..Options::default()
|
| 304 |
+
};
|
| 305 |
+
assert_eq!(continued_store_target(&opt, last, live), c, "case={name}");
|
| 306 |
+
}
|
| 307 |
+
}
|
| 308 |
+
|
| 309 |
+
#[test]
|
| 310 |
+
fn chat_anchor_matches_c() {
|
| 311 |
+
let mut tokens = vec![7; 600];
|
| 312 |
+
tokens[512] = 99;
|
| 313 |
+
tokens[530] = 99;
|
| 314 |
+
tokens[550] = 100;
|
| 315 |
+
tokens[560] = 99;
|
| 316 |
+
let mut args = vec!["chat-anchor".to_string(), "99".into(), "100".into()];
|
| 317 |
+
args.extend(tokens.iter().map(i32::to_string));
|
| 318 |
+
let out = Command::new(require_oracle()).args(&args).output().unwrap();
|
| 319 |
+
assert!(out.status.success());
|
| 320 |
+
let c: i32 = String::from_utf8(out.stdout)
|
| 321 |
+
.unwrap()
|
| 322 |
+
.trim()
|
| 323 |
+
.parse()
|
| 324 |
+
.unwrap();
|
| 325 |
+
assert_eq!(chat_anchor_pos(&Options::default(), &tokens, 99, 100), c);
|
| 326 |
+
assert_eq!(c, 530);
|
| 327 |
+
}
|
| 328 |
+
|
| 329 |
+
#[test]
|
| 330 |
+
fn eviction_score_matches_c() {
|
| 331 |
+
let out = Command::new(require_oracle())
|
| 332 |
+
.args([
|
| 333 |
+
"score",
|
| 334 |
+
"--hits",
|
| 335 |
+
"4",
|
| 336 |
+
"--tokens",
|
| 337 |
+
"512",
|
| 338 |
+
"--file-size",
|
| 339 |
+
"4096",
|
| 340 |
+
"--reason",
|
| 341 |
+
"1",
|
| 342 |
+
"--created",
|
| 343 |
+
"1000",
|
| 344 |
+
"--used",
|
| 345 |
+
"1000",
|
| 346 |
+
"--now",
|
| 347 |
+
"1000",
|
| 348 |
+
])
|
| 349 |
+
.output()
|
| 350 |
+
.unwrap();
|
| 351 |
+
assert!(out.status.success());
|
| 352 |
+
let c: f64 = String::from_utf8(out.stdout)
|
| 353 |
+
.unwrap()
|
| 354 |
+
.trim()
|
| 355 |
+
.parse()
|
| 356 |
+
.unwrap();
|
| 357 |
+
let rust = eviction_score(
|
| 358 |
+
&ScoreEntry {
|
| 359 |
+
sha: "x",
|
| 360 |
+
quant_bits: 2,
|
| 361 |
+
model_id: 0,
|
| 362 |
+
reason: Reason::Cold,
|
| 363 |
+
tokens: 512,
|
| 364 |
+
hits: 4,
|
| 365 |
+
ctx_size: 2048,
|
| 366 |
+
created_at: 1000,
|
| 367 |
+
last_used: 1000,
|
| 368 |
+
text_bytes: 11,
|
| 369 |
+
file_size: 4096,
|
| 370 |
+
},
|
| 371 |
+
1000,
|
| 372 |
+
None,
|
| 373 |
+
);
|
| 374 |
+
assert!((rust - c).abs() <= 1e-12, "rust={rust} c={c}");
|
| 375 |
+
}
|
crates/ds4-perf-gpu/Cargo.toml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[package]
|
| 2 |
+
name = "ds4-perf-gpu"
|
| 3 |
+
version.workspace = true
|
| 4 |
+
edition.workspace = true
|
| 5 |
+
license.workspace = true
|
| 6 |
+
publish.workspace = true
|
| 7 |
+
description = "Isolated CUDA calibration and CUPTI collection for ds4-perf"
|
| 8 |
+
|
| 9 |
+
[lib]
|
| 10 |
+
crate-type = ["rlib", "cdylib"]
|
| 11 |
+
|
| 12 |
+
[[bin]]
|
| 13 |
+
name = "ds4-perf-gpu"
|
| 14 |
+
path = "src/main.rs"
|
| 15 |
+
required-features = ["cuda"]
|
| 16 |
+
|
| 17 |
+
[features]
|
| 18 |
+
default = []
|
| 19 |
+
cuda = ["dep:cudarc", "dep:libc"]
|
| 20 |
+
|
| 21 |
+
[dependencies]
|
| 22 |
+
ds4-perf = { path = "../ds4-perf" }
|
| 23 |
+
serde_json = "1"
|
| 24 |
+
cudarc = { version = "0.19.9", optional = true, default-features = false, features = ["std", "driver", "nvrtc", "cupti", "dynamic-loading", "cuda-13030"] }
|
| 25 |
+
libc = { version = "0.2", optional = true }
|
crates/ds4-perf-gpu/src/calibration.rs
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
use cudarc::{
|
| 2 |
+
driver::{CudaContext, LaunchConfig, PushKernelArg},
|
| 3 |
+
nvrtc::{compile_ptx_with_opts, CompileOptions},
|
| 4 |
+
};
|
| 5 |
+
use ds4_perf::machine::Calibration;
|
| 6 |
+
use std::time::Instant;
|
| 7 |
+
|
| 8 |
+
const WARMUPS: u32 = 2;
|
| 9 |
+
const REPEATS: u32 = 7;
|
| 10 |
+
const FMA_ITERATIONS: i32 = 4096;
|
| 11 |
+
const LAUNCHES: u32 = 256;
|
| 12 |
+
const SOURCE: &str = r#"
|
| 13 |
+
extern "C" __global__ void copy_words(float *out, const float *input, int n) {
|
| 14 |
+
int i = blockIdx.x * blockDim.x + threadIdx.x;
|
| 15 |
+
if (i < n) out[i] = input[i];
|
| 16 |
+
}
|
| 17 |
+
extern "C" __global__ void fma_envelope(float *out, int iterations) {
|
| 18 |
+
int i = blockIdx.x * blockDim.x + threadIdx.x;
|
| 19 |
+
float a=0.1f,b=0.2f,c=0.3f,d=0.4f,e=0.5f,f=0.6f,g=0.7f,h=0.8f;
|
| 20 |
+
for (int j=0; j<iterations; ++j) {
|
| 21 |
+
a=fmaf(a,0.9999f,0.0001f); b=fmaf(b,0.9999f,0.0001f);
|
| 22 |
+
c=fmaf(c,0.9999f,0.0001f); d=fmaf(d,0.9999f,0.0001f);
|
| 23 |
+
e=fmaf(e,0.9999f,0.0001f); f=fmaf(f,0.9999f,0.0001f);
|
| 24 |
+
g=fmaf(g,0.9999f,0.0001f); h=fmaf(h,0.9999f,0.0001f);
|
| 25 |
+
}
|
| 26 |
+
out[i]=a+b+c+d+e+f+g+h;
|
| 27 |
+
}
|
| 28 |
+
extern "C" __global__ void empty_launch() {}
|
| 29 |
+
"#;
|
| 30 |
+
|
| 31 |
+
pub fn run(ordinal: usize) -> Result<Calibration, String> {
|
| 32 |
+
measure(ordinal).map_err(|e| e.to_string())
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
fn measure(ordinal: usize) -> Result<Calibration, Box<dyn std::error::Error>> {
|
| 36 |
+
let gpu = crate::device::inspect(ordinal)?;
|
| 37 |
+
let ctx = CudaContext::new(ordinal)?;
|
| 38 |
+
let stream = ctx.default_stream();
|
| 39 |
+
// Larger than L2; bounded to 512 MiB total so this is not a memory stress test.
|
| 40 |
+
let bytes = (64 * 1024 * 1024usize).max(gpu.l2_bytes.max(0) as usize * 2);
|
| 41 |
+
if bytes > 256 * 1024 * 1024 || gpu.free_memory_bytes < (bytes * 4) as u64 {
|
| 42 |
+
return Err("insufficient bounded calibration headroom".into());
|
| 43 |
+
}
|
| 44 |
+
let n = (bytes / 4) as i32;
|
| 45 |
+
let source = stream.clone_htod(&vec![1.0f32; n as usize])?;
|
| 46 |
+
let mut target = stream.alloc_zeros::<f32>(n as usize)?;
|
| 47 |
+
let workers = gpu.multiprocessors as u32 * 256 * 8;
|
| 48 |
+
let mut compute = stream.alloc_zeros::<f32>(workers as usize)?;
|
| 49 |
+
let ptx = compile_ptx_with_opts(
|
| 50 |
+
SOURCE,
|
| 51 |
+
CompileOptions {
|
| 52 |
+
options: vec![format!(
|
| 53 |
+
"--gpu-architecture=compute_{}{}",
|
| 54 |
+
gpu.compute_major, gpu.compute_minor
|
| 55 |
+
)],
|
| 56 |
+
..Default::default()
|
| 57 |
+
},
|
| 58 |
+
)?;
|
| 59 |
+
let ptx_sha256 = ds4_perf::artifact::hash_bytes(ptx.to_src().as_bytes());
|
| 60 |
+
let module = ctx.load_module(ptx)?;
|
| 61 |
+
let copy = module.load_function("copy_words")?;
|
| 62 |
+
let fma = module.load_function("fma_envelope")?;
|
| 63 |
+
let empty = module.load_function("empty_launch")?;
|
| 64 |
+
let mut copy_gb_s = Vec::new();
|
| 65 |
+
let mut fp32_gflop_s = Vec::new();
|
| 66 |
+
let mut launch_us = Vec::new();
|
| 67 |
+
let mut copy_times = Vec::new();
|
| 68 |
+
let mut compute_times = Vec::new();
|
| 69 |
+
let mut launch_times = Vec::new();
|
| 70 |
+
for repetition in 0..WARMUPS + REPEATS {
|
| 71 |
+
let start =
|
| 72 |
+
stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?;
|
| 73 |
+
// All pointers are cudarc-owned slices on this stream; n bounds the kernel.
|
| 74 |
+
unsafe {
|
| 75 |
+
stream
|
| 76 |
+
.launch_builder(©)
|
| 77 |
+
.arg(&mut target)
|
| 78 |
+
.arg(&source)
|
| 79 |
+
.arg(&n)
|
| 80 |
+
.launch(LaunchConfig::for_num_elems(n as u32))?;
|
| 81 |
+
}
|
| 82 |
+
let end =
|
| 83 |
+
stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?;
|
| 84 |
+
end.synchronize()?;
|
| 85 |
+
let copy_ms = start.elapsed_ms(&end)? as f64;
|
| 86 |
+
|
| 87 |
+
let start =
|
| 88 |
+
stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?;
|
| 89 |
+
// Exactly workers threads write an allocation of workers f32 elements.
|
| 90 |
+
unsafe {
|
| 91 |
+
stream
|
| 92 |
+
.launch_builder(&fma)
|
| 93 |
+
.arg(&mut compute)
|
| 94 |
+
.arg(&FMA_ITERATIONS)
|
| 95 |
+
.launch(LaunchConfig {
|
| 96 |
+
grid_dim: (workers / 256, 1, 1),
|
| 97 |
+
block_dim: (256, 1, 1),
|
| 98 |
+
shared_mem_bytes: 0,
|
| 99 |
+
})?;
|
| 100 |
+
}
|
| 101 |
+
let end =
|
| 102 |
+
stream.record_event(Some(cudarc::driver::sys::CUevent_flags::CU_EVENT_DEFAULT))?;
|
| 103 |
+
end.synchronize()?;
|
| 104 |
+
let compute_ms = start.elapsed_ms(&end)? as f64;
|
| 105 |
+
|
| 106 |
+
let host_start = Instant::now();
|
| 107 |
+
for _ in 0..LAUNCHES {
|
| 108 |
+
// This kernel has no arguments or memory accesses.
|
| 109 |
+
unsafe {
|
| 110 |
+
stream.launch_builder(&empty).launch(LaunchConfig {
|
| 111 |
+
grid_dim: (1, 1, 1),
|
| 112 |
+
block_dim: (1, 1, 1),
|
| 113 |
+
shared_mem_bytes: 0,
|
| 114 |
+
})?;
|
| 115 |
+
}
|
| 116 |
+
}
|
| 117 |
+
stream.synchronize()?;
|
| 118 |
+
let launch_ms = host_start.elapsed().as_secs_f64() * 1000.0;
|
| 119 |
+
let latency = launch_ms * 1000.0 / LAUNCHES as f64;
|
| 120 |
+
if repetition >= WARMUPS {
|
| 121 |
+
copy_gb_s.push(2.0 * bytes as f64 / (copy_ms * 1e6));
|
| 122 |
+
fp32_gflop_s.push(workers as f64 * FMA_ITERATIONS as f64 * 16.0 / (compute_ms * 1e6));
|
| 123 |
+
launch_us.push(latency);
|
| 124 |
+
copy_times.push(copy_ms);
|
| 125 |
+
compute_times.push(compute_ms);
|
| 126 |
+
launch_times.push(launch_ms);
|
| 127 |
+
}
|
| 128 |
+
}
|
| 129 |
+
if stream.clone_dtoh(&target)?.iter().any(|v| *v != 1.0) {
|
| 130 |
+
return Err("copy calibration validation failed".into());
|
| 131 |
+
}
|
| 132 |
+
let compute_result = stream.clone_dtoh(&compute)?;
|
| 133 |
+
if !valid_compute(&compute_result) {
|
| 134 |
+
return Err("FMA calibration validation failed".into());
|
| 135 |
+
}
|
| 136 |
+
if [©_gb_s, &fp32_gflop_s, &launch_us]
|
| 137 |
+
.iter()
|
| 138 |
+
.any(|values| values.iter().any(|v| !v.is_finite() || *v <= 0.0))
|
| 139 |
+
{
|
| 140 |
+
return Err("invalid calibration timing".into());
|
| 141 |
+
}
|
| 142 |
+
Ok(Calibration { ptx_sha256, method:"cuda-event-copy-fp32-fma-v1".into(),source_sha256:ds4_perf::artifact::hash_bytes(SOURCE.as_bytes()),
|
| 143 |
+
copy_ms:copy_times,compute_ms:compute_times,launch_batch_ms:launch_times,
|
| 144 |
+
compute_threads:workers,compute_block_threads:256,fma_iterations:FMA_ITERATIONS as u32,launch_count:LAUNCHES,
|
| 145 |
+
gpu, transfer_bytes: bytes as u64, warmups: WARMUPS, repeats: REPEATS, copy_gb_s, fp32_gflop_s, launch_us,
|
| 146 |
+
notes: vec!["Copy counts read plus write bytes, uses CUDA event time, and exceeds L2 capacity.".into(),
|
| 147 |
+
"FP32 measures eight independent SIMT FMA chains; it is not Tensor Core peak throughput.".into(),
|
| 148 |
+
"Launch latency is host wall time for 256 empty kernels plus completion synchronization, per kernel.".into(),
|
| 149 |
+
"Observed rates depend on current clocks, temperature and other processes; this is a measured envelope, not a hardware guarantee.".into()] })
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
+
fn valid_compute(values: &[f32]) -> bool {
|
| 153 |
+
let mut expected = [0.1f32, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8];
|
| 154 |
+
for _ in 0..FMA_ITERATIONS {
|
| 155 |
+
for value in &mut expected {
|
| 156 |
+
*value = value.mul_add(0.9999, 0.0001);
|
| 157 |
+
}
|
| 158 |
+
}
|
| 159 |
+
let expected: f32 = expected.iter().sum();
|
| 160 |
+
!values.is_empty()
|
| 161 |
+
&& values
|
| 162 |
+
.iter()
|
| 163 |
+
.all(|v| v.is_finite() && (*v - expected).abs() <= 0.00002)
|
| 164 |
+
}
|
| 165 |
+
|
| 166 |
+
#[cfg(test)]
|
| 167 |
+
mod tests {
|
| 168 |
+
#[test]
|
| 169 |
+
fn rejects_short_fma_work() {
|
| 170 |
+
let mut one_step = [0.1f32, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8];
|
| 171 |
+
for value in &mut one_step {
|
| 172 |
+
*value = value.mul_add(0.9999, 0.0001);
|
| 173 |
+
}
|
| 174 |
+
assert!(!super::valid_compute(&[one_step.iter().sum()]));
|
| 175 |
+
}
|
| 176 |
+
}
|
crates/ds4-perf-gpu/src/collector.rs
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// CUPTI injection follows NVIDIA's cupti_trace_injection sample. NVTX injection
|
| 2 |
+
// points directly to libcupti; all application annotations remain NVIDIA nvtx.
|
| 3 |
+
use cudarc::{
|
| 4 |
+
cupti::{
|
| 5 |
+
result::activity,
|
| 6 |
+
sys::{self, CUpti_ActivityKind::*},
|
| 7 |
+
},
|
| 8 |
+
driver,
|
| 9 |
+
};
|
| 10 |
+
use ds4_perf::trace::Event;
|
| 11 |
+
use std::{
|
| 12 |
+
alloc::{alloc, dealloc, Layout},
|
| 13 |
+
ffi::CStr,
|
| 14 |
+
fs::OpenOptions,
|
| 15 |
+
io::{BufWriter, Write},
|
| 16 |
+
sync::{
|
| 17 |
+
atomic::{AtomicU64, Ordering},
|
| 18 |
+
Mutex, OnceLock,
|
| 19 |
+
},
|
| 20 |
+
};
|
| 21 |
+
|
| 22 |
+
const BUFFER_BYTES: usize = 8 * 1024 * 1024;
|
| 23 |
+
const BUFFER_ALIGN: usize = 8;
|
| 24 |
+
static OUTPUT: OnceLock<Mutex<BufWriter<std::fs::File>>> = OnceLock::new();
|
| 25 |
+
static INITIALIZED: OnceLock<Result<(), String>> = OnceLock::new();
|
| 26 |
+
static ERRORS: AtomicU64 = AtomicU64::new(0);
|
| 27 |
+
static DROPPED: AtomicU64 = AtomicU64::new(0);
|
| 28 |
+
|
| 29 |
+
fn emit(event: Event) {
|
| 30 |
+
let Some(output) = OUTPUT.get() else {
|
| 31 |
+
return;
|
| 32 |
+
};
|
| 33 |
+
let Ok(mut output) = output.lock() else {
|
| 34 |
+
ERRORS.fetch_add(1, Ordering::Relaxed);
|
| 35 |
+
return;
|
| 36 |
+
};
|
| 37 |
+
if serde_json::to_writer(&mut *output, &event).is_err() || output.write_all(b"\n").is_err() {
|
| 38 |
+
ERRORS.fetch_add(1, Ordering::Relaxed);
|
| 39 |
+
}
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
fn error(message: String) {
|
| 43 |
+
ERRORS.fetch_add(1, Ordering::Relaxed);
|
| 44 |
+
emit(Event::Error { message });
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
// These callbacks are called by CUPTI with its documented output pointers.
|
| 48 |
+
extern "C" fn request_buffer(buffer: *mut *mut u8, size: *mut usize, records: *mut usize) {
|
| 49 |
+
let layout = Layout::from_size_align(BUFFER_BYTES, BUFFER_ALIGN).expect("constant layout");
|
| 50 |
+
unsafe {
|
| 51 |
+
*buffer = alloc(layout);
|
| 52 |
+
*size = if (*buffer).is_null() { 0 } else { BUFFER_BYTES };
|
| 53 |
+
*records = 0;
|
| 54 |
+
if (*buffer).is_null() {
|
| 55 |
+
ERRORS.fetch_add(1, Ordering::Relaxed);
|
| 56 |
+
}
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
extern "C" fn complete_buffer(
|
| 61 |
+
context: driver::sys::CUcontext,
|
| 62 |
+
stream: u32,
|
| 63 |
+
buffer: *mut u8,
|
| 64 |
+
_size: usize,
|
| 65 |
+
valid: usize,
|
| 66 |
+
) {
|
| 67 |
+
// No panic may cross CUPTI's C boundary. Record failures for the parent.
|
| 68 |
+
let result = std::panic::catch_unwind(|| unsafe { consume(buffer, valid) });
|
| 69 |
+
match result {
|
| 70 |
+
Ok(Ok(())) => {}
|
| 71 |
+
Ok(Err(message)) => error(message),
|
| 72 |
+
Err(_) => error("activity callback panicked".into()),
|
| 73 |
+
}
|
| 74 |
+
let mut dropped = 0;
|
| 75 |
+
// CUPTI owns context lifetime throughout this callback.
|
| 76 |
+
match unsafe { activity::get_num_dropped_records(context, stream, &mut dropped) } {
|
| 77 |
+
Ok(()) => {
|
| 78 |
+
DROPPED.fetch_add(dropped as u64, Ordering::Relaxed);
|
| 79 |
+
}
|
| 80 |
+
Err(e) => error(e.to_string()),
|
| 81 |
+
}
|
| 82 |
+
if !buffer.is_null() {
|
| 83 |
+
// This is exactly the allocation made by request_buffer.
|
| 84 |
+
unsafe {
|
| 85 |
+
dealloc(
|
| 86 |
+
buffer,
|
| 87 |
+
Layout::from_size_align(BUFFER_BYTES, BUFFER_ALIGN).expect("constant layout"),
|
| 88 |
+
);
|
| 89 |
+
}
|
| 90 |
+
}
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
unsafe fn string(pointer: *const std::ffi::c_char) -> String {
|
| 94 |
+
if pointer.is_null() {
|
| 95 |
+
return String::new();
|
| 96 |
+
}
|
| 97 |
+
// CUPTI guarantees activity strings remain valid until this callback returns.
|
| 98 |
+
unsafe { CStr::from_ptr(pointer) }
|
| 99 |
+
.to_string_lossy()
|
| 100 |
+
.into_owned()
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
unsafe fn consume(buffer: *mut u8, valid: usize) -> Result<(), String> {
|
| 104 |
+
if valid == 0 {
|
| 105 |
+
return Ok(());
|
| 106 |
+
}
|
| 107 |
+
if buffer.is_null() || valid > BUFFER_BYTES {
|
| 108 |
+
return Err("invalid CUPTI buffer length".into());
|
| 109 |
+
}
|
| 110 |
+
let mut record = std::ptr::null_mut();
|
| 111 |
+
loop {
|
| 112 |
+
match unsafe { activity::get_next_record(buffer, valid, &mut record) } {
|
| 113 |
+
Ok(()) => {}
|
| 114 |
+
Err(cudarc::cupti::result::CuptiError(
|
| 115 |
+
sys::CUptiResult::CUPTI_ERROR_MAX_LIMIT_REACHED,
|
| 116 |
+
)) => return Ok(()),
|
| 117 |
+
Err(error) => return Err(error.to_string()),
|
| 118 |
+
}
|
| 119 |
+
if record.is_null() {
|
| 120 |
+
return Err("null activity record".into());
|
| 121 |
+
}
|
| 122 |
+
// Runtime API version is checked before enable. Cast only the documented
|
| 123 |
+
// activity kind to its CUDA 13.3 record; copy values before buffer release.
|
| 124 |
+
match unsafe { (*record).kind } {
|
| 125 |
+
CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL => {
|
| 126 |
+
let k = unsafe { &*record.cast::<sys::CUpti_ActivityKernel12>() };
|
| 127 |
+
if [k.gridX, k.gridY, k.gridZ, k.blockX, k.blockY, k.blockZ]
|
| 128 |
+
.iter()
|
| 129 |
+
.any(|v| *v < 0)
|
| 130 |
+
{
|
| 131 |
+
return Err("negative launch dimension".into());
|
| 132 |
+
}
|
| 133 |
+
emit(Event::Kernel {
|
| 134 |
+
name: unsafe { string(k.name) },
|
| 135 |
+
start: k.start,
|
| 136 |
+
end: k.end,
|
| 137 |
+
device: k.deviceId,
|
| 138 |
+
context: k.contextId,
|
| 139 |
+
stream: k.streamId,
|
| 140 |
+
correlation: k.correlationId,
|
| 141 |
+
grid: [k.gridX as u32, k.gridY as u32, k.gridZ as u32],
|
| 142 |
+
block: [k.blockX as u32, k.blockY as u32, k.blockZ as u32],
|
| 143 |
+
registers: k.registersPerThread as u32,
|
| 144 |
+
shared_bytes: k.staticSharedMemory.max(0) as u64
|
| 145 |
+
+ k.dynamicSharedMemory.max(0) as u64,
|
| 146 |
+
graph_id: k.graphId,
|
| 147 |
+
});
|
| 148 |
+
}
|
| 149 |
+
CUPTI_ACTIVITY_KIND_MARKER => {
|
| 150 |
+
let m = unsafe { &*record.cast::<sys::CUpti_ActivityMarker2>() };
|
| 151 |
+
if !matches!(
|
| 152 |
+
m.objectKind,
|
| 153 |
+
sys::CUpti_ActivityObjectKind::CUPTI_ACTIVITY_OBJECT_THREAD
|
| 154 |
+
| sys::CUpti_ActivityObjectKind::CUPTI_ACTIVITY_OBJECT_PROCESS
|
| 155 |
+
) {
|
| 156 |
+
continue;
|
| 157 |
+
}
|
| 158 |
+
let id = unsafe { m.objectId.pt };
|
| 159 |
+
emit(Event::Marker {
|
| 160 |
+
name: unsafe { string(m.name) },
|
| 161 |
+
domain: unsafe { string(m.domain) },
|
| 162 |
+
timestamp: m.timestamp,
|
| 163 |
+
id: m.id,
|
| 164 |
+
flags: m.flags as u32,
|
| 165 |
+
pid: id.processId,
|
| 166 |
+
tid: id.threadId,
|
| 167 |
+
});
|
| 168 |
+
}
|
| 169 |
+
CUPTI_ACTIVITY_KIND_MEMCPY => {
|
| 170 |
+
let m = unsafe { &*record.cast::<sys::CUpti_ActivityMemcpy6>() };
|
| 171 |
+
emit(Event::Memop {
|
| 172 |
+
operation: "memcpy".into(),
|
| 173 |
+
start: m.start,
|
| 174 |
+
end: m.end,
|
| 175 |
+
bytes: m.bytes,
|
| 176 |
+
device: m.deviceId,
|
| 177 |
+
context: m.contextId,
|
| 178 |
+
stream: m.streamId,
|
| 179 |
+
});
|
| 180 |
+
}
|
| 181 |
+
CUPTI_ACTIVITY_KIND_MEMSET => {
|
| 182 |
+
let m = unsafe { &*record.cast::<sys::CUpti_ActivityMemset4>() };
|
| 183 |
+
emit(Event::Memop {
|
| 184 |
+
operation: "memset".into(),
|
| 185 |
+
start: m.start,
|
| 186 |
+
end: m.end,
|
| 187 |
+
bytes: m.bytes,
|
| 188 |
+
device: m.deviceId,
|
| 189 |
+
context: m.contextId,
|
| 190 |
+
stream: m.streamId,
|
| 191 |
+
});
|
| 192 |
+
}
|
| 193 |
+
_ => {}
|
| 194 |
+
}
|
| 195 |
+
}
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
extern "C" fn finish() {
|
| 199 |
+
let _ = std::panic::catch_unwind(|| {
|
| 200 |
+
if let Err(e) =
|
| 201 |
+
activity::flush_all(sys::CUpti_ActivityFlag::CUPTI_ACTIVITY_FLAG_FLUSH_FORCED as u32)
|
| 202 |
+
{
|
| 203 |
+
error(e.to_string());
|
| 204 |
+
}
|
| 205 |
+
emit(Event::End {
|
| 206 |
+
dropped: DROPPED.load(Ordering::Relaxed),
|
| 207 |
+
errors: ERRORS.load(Ordering::Relaxed),
|
| 208 |
+
});
|
| 209 |
+
if let Some(output) = OUTPUT.get() {
|
| 210 |
+
if let Ok(mut output) = output.lock() {
|
| 211 |
+
let _ = output.flush();
|
| 212 |
+
let _ = output.get_ref().sync_all();
|
| 213 |
+
}
|
| 214 |
+
}
|
| 215 |
+
});
|
| 216 |
+
}
|
| 217 |
+
|
| 218 |
+
fn initialize() -> Result<(), String> {
|
| 219 |
+
let path =
|
| 220 |
+
std::env::var_os("DS4_PERF_CUPTI_OUTPUT").ok_or("DS4_PERF_CUPTI_OUTPUT is required")?;
|
| 221 |
+
let file = OpenOptions::new()
|
| 222 |
+
.create_new(true)
|
| 223 |
+
.write(true)
|
| 224 |
+
.open(path)
|
| 225 |
+
.map_err(|e| e.to_string())?;
|
| 226 |
+
OUTPUT
|
| 227 |
+
.set(Mutex::new(BufWriter::new(file)))
|
| 228 |
+
.map_err(|_| "collector already initialized")?;
|
| 229 |
+
let mut version = 0;
|
| 230 |
+
// cudarc supplies the binding; the output pointer is a live u32 local.
|
| 231 |
+
unsafe { sys::cuptiGetVersion(&mut version) }
|
| 232 |
+
.result()
|
| 233 |
+
.map_err(|e| e.to_string())?;
|
| 234 |
+
emit(Event::Start {
|
| 235 |
+
schema_version: 1,
|
| 236 |
+
api_version: version,
|
| 237 |
+
pid: std::process::id(),
|
| 238 |
+
});
|
| 239 |
+
if version != sys::CUPTI_API_VERSION {
|
| 240 |
+
return Err(format!(
|
| 241 |
+
"CUPTI record ABI mismatch: runtime {version}, bindings {}",
|
| 242 |
+
sys::CUPTI_API_VERSION
|
| 243 |
+
));
|
| 244 |
+
}
|
| 245 |
+
activity::register_callbacks(Some(request_buffer), Some(complete_buffer))
|
| 246 |
+
.map_err(|e| e.to_string())?;
|
| 247 |
+
for kind in [
|
| 248 |
+
CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL,
|
| 249 |
+
CUPTI_ACTIVITY_KIND_MEMCPY,
|
| 250 |
+
CUPTI_ACTIVITY_KIND_MEMSET,
|
| 251 |
+
CUPTI_ACTIVITY_KIND_MARKER,
|
| 252 |
+
] {
|
| 253 |
+
activity::enable(kind).map_err(|e| e.to_string())?;
|
| 254 |
+
}
|
| 255 |
+
// Linux atexit is the documented CUPTI injection sample flush point.
|
| 256 |
+
if unsafe { libc::atexit(finish) } != 0 {
|
| 257 |
+
return Err("cannot register CUPTI exit flush".into());
|
| 258 |
+
}
|
| 259 |
+
Ok(())
|
| 260 |
+
}
|
| 261 |
+
|
| 262 |
+
#[unsafe(no_mangle)]
|
| 263 |
+
pub extern "C" fn InitializeInjection() -> i32 {
|
| 264 |
+
let result = std::panic::catch_unwind(|| INITIALIZED.get_or_init(initialize));
|
| 265 |
+
match result {
|
| 266 |
+
Ok(Ok(())) => 1,
|
| 267 |
+
Ok(Err(message)) => {
|
| 268 |
+
error(message.clone());
|
| 269 |
+
flush_output();
|
| 270 |
+
0
|
| 271 |
+
}
|
| 272 |
+
Err(_) => {
|
| 273 |
+
error("CUPTI initialization panicked".into());
|
| 274 |
+
flush_output();
|
| 275 |
+
0
|
| 276 |
+
}
|
| 277 |
+
}
|
| 278 |
+
}
|
| 279 |
+
|
| 280 |
+
fn flush_output() {
|
| 281 |
+
if let Some(output) = OUTPUT.get() {
|
| 282 |
+
if let Ok(mut output) = output.lock() {
|
| 283 |
+
if output.flush().is_err() || output.get_ref().sync_all().is_err() {
|
| 284 |
+
eprintln!("ds4-perf CUPTI: could not persist collector output");
|
| 285 |
+
}
|
| 286 |
+
}
|
| 287 |
+
}
|
| 288 |
+
}
|
crates/ds4-perf-gpu/src/device.rs
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
use cudarc::driver::{sys::CUdevice_attribute::*, CudaContext};
|
| 2 |
+
use ds4_perf::machine::Gpu;
|
| 3 |
+
|
| 4 |
+
pub fn inspect(ordinal: usize) -> Result<Gpu, String> {
|
| 5 |
+
let ctx = CudaContext::new(ordinal).map_err(|e| e.to_string())?;
|
| 6 |
+
let attr = |key| ctx.attribute(key).map_err(|e| e.to_string());
|
| 7 |
+
let (free, total) = ctx.mem_get_info().map_err(|e| e.to_string())?;
|
| 8 |
+
let uuid = ctx.uuid().map_err(|e| e.to_string())?;
|
| 9 |
+
let mut driver_version = 0;
|
| 10 |
+
// CUDA writes one integer to this live local, through cudarc's bindings.
|
| 11 |
+
unsafe { cudarc::driver::sys::cuDriverGetVersion(&mut driver_version) }
|
| 12 |
+
.result()
|
| 13 |
+
.map_err(|e| e.to_string())?;
|
| 14 |
+
let mut unavailable = std::collections::BTreeMap::new();
|
| 15 |
+
let mut optional = |name: &str, key| match attr(key) {
|
| 16 |
+
Ok(value) if value > 0 => Some(value),
|
| 17 |
+
other => {
|
| 18 |
+
unavailable.insert(name.into(), format!("{other:?}"));
|
| 19 |
+
None
|
| 20 |
+
}
|
| 21 |
+
};
|
| 22 |
+
let memory_bus_bits = optional(
|
| 23 |
+
"memory_bus_bits",
|
| 24 |
+
CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH,
|
| 25 |
+
);
|
| 26 |
+
let memory_clock_khz = optional("memory_clock_khz", CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE);
|
| 27 |
+
let clock_khz = optional("clock_khz", CU_DEVICE_ATTRIBUTE_CLOCK_RATE);
|
| 28 |
+
Ok(Gpu {
|
| 29 |
+
ordinal,
|
| 30 |
+
name: ctx.name().map_err(|e| e.to_string())?,
|
| 31 |
+
uuid: uuid.bytes.iter().map(|b| format!("{:02x}", *b)).collect(),
|
| 32 |
+
compute_major: attr(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR)?,
|
| 33 |
+
compute_minor: attr(CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR)?,
|
| 34 |
+
driver_version,
|
| 35 |
+
total_memory_bytes: total as u64,
|
| 36 |
+
free_memory_bytes: free as u64,
|
| 37 |
+
multiprocessors: attr(CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT)?,
|
| 38 |
+
warp_size: attr(CU_DEVICE_ATTRIBUTE_WARP_SIZE)?,
|
| 39 |
+
max_threads_per_block: attr(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK)?,
|
| 40 |
+
max_threads_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR)?,
|
| 41 |
+
max_blocks_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_BLOCKS_PER_MULTIPROCESSOR)?,
|
| 42 |
+
registers_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_MULTIPROCESSOR)?,
|
| 43 |
+
shared_bytes_per_sm: attr(CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR)?,
|
| 44 |
+
shared_bytes_per_block: attr(CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN)?,
|
| 45 |
+
l2_bytes: attr(CU_DEVICE_ATTRIBUTE_L2_CACHE_SIZE)?,
|
| 46 |
+
memory_bus_bits,
|
| 47 |
+
memory_clock_khz,
|
| 48 |
+
clock_khz,
|
| 49 |
+
unavailable,
|
| 50 |
+
})
|
| 51 |
+
}
|
crates/ds4-perf-gpu/src/lib.rs
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This helper owns profiler CUDA calls; it is never an inference dependency.
|
| 2 |
+
#![cfg_attr(not(feature = "cuda"), forbid(unsafe_code))]
|
| 3 |
+
|
| 4 |
+
#[cfg(feature = "cuda")]
|
| 5 |
+
pub mod device;
|
| 6 |
+
|
| 7 |
+
#[cfg(feature = "cuda")]
|
| 8 |
+
pub mod calibration;
|
| 9 |
+
|
| 10 |
+
#[cfg(feature = "cuda")]
|
| 11 |
+
mod collector;
|
crates/ds4-perf-gpu/src/main.rs
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
fn main() {
|
| 2 |
+
let args: Vec<_> = std::env::args().skip(1).collect();
|
| 3 |
+
let result = match args.as_slice() {
|
| 4 |
+
[command, ordinal] if command == "inspect" => ordinal
|
| 5 |
+
.parse::<usize>()
|
| 6 |
+
.map_err(|e| e.to_string())
|
| 7 |
+
.and_then(ds4_perf_gpu::device::inspect)
|
| 8 |
+
.and_then(|gpu| serde_json::to_string(&gpu).map_err(|e| e.to_string())),
|
| 9 |
+
[command, ordinal] if command == "calibrate" => ordinal
|
| 10 |
+
.parse::<usize>()
|
| 11 |
+
.map_err(|e| e.to_string())
|
| 12 |
+
.and_then(ds4_perf_gpu::calibration::run)
|
| 13 |
+
.and_then(|calibration| serde_json::to_string(&calibration).map_err(|e| e.to_string())),
|
| 14 |
+
_ => Err("usage: ds4-perf-gpu inspect|calibrate DEVICE".into()),
|
| 15 |
+
};
|
| 16 |
+
match result {
|
| 17 |
+
Ok(text) => println!("{text}"),
|
| 18 |
+
Err(error) => {
|
| 19 |
+
eprintln!("ds4-perf-gpu: {error}");
|
| 20 |
+
std::process::exit(1);
|
| 21 |
+
}
|
| 22 |
+
}
|
| 23 |
+
}
|
crates/ds4-perf-gpu/tests/live.rs
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#![cfg(feature = "cuda")]
|
| 2 |
+
|
| 3 |
+
#[test]
|
| 4 |
+
#[ignore = "requires an idle CUDA device and NVRTC"]
|
| 5 |
+
fn measured_envelope() {
|
| 6 |
+
let result = ds4_perf_gpu::calibration::run(0).unwrap();
|
| 7 |
+
for samples in [&result.copy_gb_s, &result.fp32_gflop_s, &result.launch_us] {
|
| 8 |
+
assert_eq!(samples.len(), result.repeats as usize);
|
| 9 |
+
assert!(samples
|
| 10 |
+
.iter()
|
| 11 |
+
.all(|value| value.is_finite() && *value > 0.0));
|
| 12 |
+
}
|
| 13 |
+
assert!(result.transfer_bytes > result.gpu.l2_bytes as u64);
|
| 14 |
+
}
|
crates/ds4-perf/Cargo.toml
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[package]
|
| 2 |
+
name = "ds4-perf"
|
| 3 |
+
version.workspace = true
|
| 4 |
+
edition.workspace = true
|
| 5 |
+
license.workspace = true
|
| 6 |
+
publish.workspace = true
|
| 7 |
+
description = "Model-free performance orchestration and Nsight diagnostics for ds4"
|
| 8 |
+
|
| 9 |
+
[dependencies]
|
| 10 |
+
clap = { version = "4.6", features = ["derive"] }
|
| 11 |
+
serde = { version = "1", features = ["derive"] }
|
| 12 |
+
# Evidence summaries must retain the exact f64 samples used for decisions.
|
| 13 |
+
serde_json = { version = "1", features = ["float_roundtrip"] }
|
| 14 |
+
sha2 = { version = "0.11", default-features = false }
|
| 15 |
+
ctrlc = { version = "3.5", features = ["termination"] }
|
| 16 |
+
|
| 17 |
+
[target.'cfg(unix)'.dependencies]
|
| 18 |
+
nix = { version = "0.31", default-features = false, features = ["signal", "process"] }
|
crates/ds4-perf/src/artifact.rs
ADDED
|
@@ -0,0 +1,564 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
| 2 |
+
use sha2::{Digest, Sha256};
|
| 3 |
+
use std::{
|
| 4 |
+
fs,
|
| 5 |
+
io::{Read, Write},
|
| 6 |
+
path::Path,
|
| 7 |
+
time::SystemTime,
|
| 8 |
+
};
|
| 9 |
+
|
| 10 |
+
/// Each payload evolves independently. Unknown versions fail closed.
|
| 11 |
+
pub trait Payload {
|
| 12 |
+
const KIND: &'static str;
|
| 13 |
+
const SCHEMA: u32 = 1;
|
| 14 |
+
fn validate(&self) -> Result<(), String>;
|
| 15 |
+
}
|
| 16 |
+
|
| 17 |
+
#[derive(Clone, Debug, Serialize, Deserialize)]
|
| 18 |
+
#[serde(deny_unknown_fields)]
|
| 19 |
+
pub struct Reference {
|
| 20 |
+
pub path: String,
|
| 21 |
+
pub sha256: String,
|
| 22 |
+
}
|
| 23 |
+
|
| 24 |
+
#[derive(Debug, Serialize, Deserialize)]
|
| 25 |
+
#[serde(deny_unknown_fields)]
|
| 26 |
+
pub struct Artifact<T> {
|
| 27 |
+
pub schema_version: u32,
|
| 28 |
+
pub kind: String,
|
| 29 |
+
pub producer_version: String,
|
| 30 |
+
pub created_unix: u64,
|
| 31 |
+
pub complete: bool,
|
| 32 |
+
pub inputs: Vec<Reference>,
|
| 33 |
+
pub warnings: Vec<String>,
|
| 34 |
+
pub data: Option<T>,
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
impl<T: Payload> Artifact<T> {
|
| 38 |
+
pub fn new(data: T) -> Self {
|
| 39 |
+
Self {
|
| 40 |
+
schema_version: T::SCHEMA,
|
| 41 |
+
kind: T::KIND.into(),
|
| 42 |
+
producer_version: env!("CARGO_PKG_VERSION").into(),
|
| 43 |
+
created_unix: SystemTime::now()
|
| 44 |
+
.duration_since(SystemTime::UNIX_EPOCH)
|
| 45 |
+
.unwrap_or_default()
|
| 46 |
+
.as_secs(),
|
| 47 |
+
complete: true,
|
| 48 |
+
inputs: Vec::new(),
|
| 49 |
+
warnings: Vec::new(),
|
| 50 |
+
data: Some(data),
|
| 51 |
+
}
|
| 52 |
+
}
|
| 53 |
+
pub fn failed(message: String) -> Self {
|
| 54 |
+
Self {
|
| 55 |
+
schema_version: T::SCHEMA,
|
| 56 |
+
kind: T::KIND.into(),
|
| 57 |
+
producer_version: env!("CARGO_PKG_VERSION").into(),
|
| 58 |
+
created_unix: SystemTime::now()
|
| 59 |
+
.duration_since(SystemTime::UNIX_EPOCH)
|
| 60 |
+
.unwrap_or_default()
|
| 61 |
+
.as_secs(),
|
| 62 |
+
complete: false,
|
| 63 |
+
inputs: Vec::new(),
|
| 64 |
+
warnings: vec![message],
|
| 65 |
+
data: None,
|
| 66 |
+
}
|
| 67 |
+
}
|
| 68 |
+
pub fn require(&self) -> Result<&T, String> {
|
| 69 |
+
if !self.complete {
|
| 70 |
+
return Err(format!("incomplete {} evidence", T::KIND));
|
| 71 |
+
}
|
| 72 |
+
self.data
|
| 73 |
+
.as_ref()
|
| 74 |
+
.ok_or_else(|| format!("missing {} payload", T::KIND))
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
fn hex(bytes: impl IntoIterator<Item = u8>) -> String {
|
| 79 |
+
bytes
|
| 80 |
+
.into_iter()
|
| 81 |
+
.map(|byte| format!("{byte:02x}"))
|
| 82 |
+
.collect()
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
pub fn hash_bytes(bytes: &[u8]) -> String {
|
| 86 |
+
hex(Sha256::digest(bytes))
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
pub fn hash(path: &Path) -> Result<String, String> {
|
| 90 |
+
let mut file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
|
| 91 |
+
hash_file(&mut file)
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
fn hash_file(file: &mut fs::File) -> Result<String, String> {
|
| 95 |
+
let mut state = Sha256::new();
|
| 96 |
+
let mut buffer = [0u8; 64 * 1024];
|
| 97 |
+
loop {
|
| 98 |
+
let count = file.read(&mut buffer).map_err(|e| e.to_string())?;
|
| 99 |
+
if count == 0 {
|
| 100 |
+
break;
|
| 101 |
+
}
|
| 102 |
+
state.update(&buffer[..count]);
|
| 103 |
+
}
|
| 104 |
+
Ok(hex(state.finalize()))
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
#[derive(Clone, PartialEq, Eq)]
|
| 108 |
+
struct FileStamp {
|
| 109 |
+
size: u64,
|
| 110 |
+
modified: SystemTime,
|
| 111 |
+
#[cfg(unix)]
|
| 112 |
+
identity: [u64; 6],
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
impl FileStamp {
|
| 116 |
+
fn read(metadata: &fs::Metadata) -> Result<Self, String> {
|
| 117 |
+
#[cfg(unix)]
|
| 118 |
+
use std::os::unix::fs::MetadataExt;
|
| 119 |
+
if !metadata.is_file() {
|
| 120 |
+
return Err("evidence input must be a regular file".into());
|
| 121 |
+
}
|
| 122 |
+
Ok(Self {
|
| 123 |
+
size: metadata.len(),
|
| 124 |
+
modified: metadata.modified().map_err(|e| e.to_string())?,
|
| 125 |
+
#[cfg(unix)]
|
| 126 |
+
identity: [
|
| 127 |
+
metadata.dev(),
|
| 128 |
+
metadata.ino(),
|
| 129 |
+
metadata.mtime() as u64,
|
| 130 |
+
metadata.mtime_nsec() as u64,
|
| 131 |
+
metadata.ctime() as u64,
|
| 132 |
+
metadata.ctime_nsec() as u64,
|
| 133 |
+
],
|
| 134 |
+
})
|
| 135 |
+
}
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
/// Share full-byte verification only within one operation. Reuse never trusts
|
| 139 |
+
/// an expected hash alone: replacement or metadata drift invalidates the read.
|
| 140 |
+
#[derive(Default)]
|
| 141 |
+
pub struct Verification {
|
| 142 |
+
files: std::collections::BTreeMap<std::path::PathBuf, (FileStamp, String)>,
|
| 143 |
+
aliases: std::collections::BTreeMap<std::path::PathBuf, std::path::PathBuf>,
|
| 144 |
+
#[cfg(test)]
|
| 145 |
+
full_reads: usize,
|
| 146 |
+
}
|
| 147 |
+
|
| 148 |
+
impl Verification {
|
| 149 |
+
pub fn verify(&mut self, path: &Path, expected: &str) -> Result<(), String> {
|
| 150 |
+
let alias = std::env::current_dir()
|
| 151 |
+
.map_err(|e| e.to_string())?
|
| 152 |
+
.join(path);
|
| 153 |
+
let canonical = path.canonicalize().map_err(|e| e.to_string())?;
|
| 154 |
+
if self
|
| 155 |
+
.aliases
|
| 156 |
+
.get(&alias)
|
| 157 |
+
.is_some_and(|previous| previous != &canonical)
|
| 158 |
+
{
|
| 159 |
+
return Err(format!("evidence path changed: {}", path.display()));
|
| 160 |
+
}
|
| 161 |
+
let mut file = fs::File::open(&canonical).map_err(|e| e.to_string())?;
|
| 162 |
+
let before = FileStamp::read(&file.metadata().map_err(|e| e.to_string())?)?;
|
| 163 |
+
let digest = match self.files.get(&canonical) {
|
| 164 |
+
Some((stamp, digest)) => {
|
| 165 |
+
if *stamp != before {
|
| 166 |
+
return Err(format!(
|
| 167 |
+
"evidence changed during verification: {}",
|
| 168 |
+
path.display()
|
| 169 |
+
));
|
| 170 |
+
}
|
| 171 |
+
digest.clone()
|
| 172 |
+
}
|
| 173 |
+
None => {
|
| 174 |
+
#[cfg(test)]
|
| 175 |
+
{
|
| 176 |
+
self.full_reads += 1;
|
| 177 |
+
}
|
| 178 |
+
hash_file(&mut file)?
|
| 179 |
+
}
|
| 180 |
+
};
|
| 181 |
+
if digest != expected {
|
| 182 |
+
return Err(format!("evidence changed: {}", path.display()));
|
| 183 |
+
}
|
| 184 |
+
let after = FileStamp::read(&file.metadata().map_err(|e| e.to_string())?)?;
|
| 185 |
+
let current = FileStamp::read(&fs::metadata(&canonical).map_err(|e| e.to_string())?)?;
|
| 186 |
+
if before != after
|
| 187 |
+
|| before != current
|
| 188 |
+
|| path.canonicalize().ok().as_ref() != Some(&canonical)
|
| 189 |
+
{
|
| 190 |
+
return Err(format!(
|
| 191 |
+
"evidence changed during verification: {}",
|
| 192 |
+
path.display()
|
| 193 |
+
));
|
| 194 |
+
}
|
| 195 |
+
self.aliases.insert(alias, canonical.clone());
|
| 196 |
+
self.files.insert(canonical, (before, digest));
|
| 197 |
+
Ok(())
|
| 198 |
+
}
|
| 199 |
+
|
| 200 |
+
pub fn finish(&self) -> Result<(), String> {
|
| 201 |
+
for (alias, target) in &self.aliases {
|
| 202 |
+
if alias.canonicalize().ok().as_ref() != Some(target) {
|
| 203 |
+
return Err(format!("evidence path changed: {}", alias.display()));
|
| 204 |
+
}
|
| 205 |
+
}
|
| 206 |
+
for (path, (before, _)) in &self.files {
|
| 207 |
+
let current = FileStamp::read(&fs::metadata(path).map_err(|e| e.to_string())?)?;
|
| 208 |
+
if *before != current {
|
| 209 |
+
return Err(format!(
|
| 210 |
+
"evidence changed during verification: {}",
|
| 211 |
+
path.display()
|
| 212 |
+
));
|
| 213 |
+
}
|
| 214 |
+
}
|
| 215 |
+
Ok(())
|
| 216 |
+
}
|
| 217 |
+
}
|
| 218 |
+
|
| 219 |
+
pub fn reference(path: &Path, directory: &Path) -> Result<Reference, String> {
|
| 220 |
+
let canonical = path.canonicalize().map_err(|e| e.to_string())?;
|
| 221 |
+
let base = directory.canonicalize().map_err(|e| e.to_string())?;
|
| 222 |
+
let relative = canonical.strip_prefix(base).unwrap_or(&canonical);
|
| 223 |
+
Ok(Reference {
|
| 224 |
+
path: relative.to_string_lossy().into(),
|
| 225 |
+
sha256: hash(&canonical)?,
|
| 226 |
+
})
|
| 227 |
+
}
|
| 228 |
+
|
| 229 |
+
pub fn save<T: Serialize + Payload>(path: &Path, value: &Artifact<T>) -> Result<(), String> {
|
| 230 |
+
if value.kind != T::KIND || value.schema_version != T::SCHEMA {
|
| 231 |
+
return Err("artifact type/schema mismatch".into());
|
| 232 |
+
}
|
| 233 |
+
if let Some(data) = &value.data {
|
| 234 |
+
data.validate()?;
|
| 235 |
+
}
|
| 236 |
+
if value.complete && value.data.is_none() {
|
| 237 |
+
return Err("complete artifact has no data".into());
|
| 238 |
+
}
|
| 239 |
+
let mut bytes = serde_json::to_vec_pretty(value).map_err(|e| e.to_string())?;
|
| 240 |
+
bytes.push(b'\n');
|
| 241 |
+
let nonce = SystemTime::now()
|
| 242 |
+
.duration_since(SystemTime::UNIX_EPOCH)
|
| 243 |
+
.unwrap_or_default()
|
| 244 |
+
.as_nanos();
|
| 245 |
+
let temporary = path.with_extension(format!("tmp-{}-{nonce}", std::process::id()));
|
| 246 |
+
let result = (|| {
|
| 247 |
+
let mut file = fs::OpenOptions::new()
|
| 248 |
+
.create_new(true)
|
| 249 |
+
.write(true)
|
| 250 |
+
.open(&temporary)?;
|
| 251 |
+
file.write_all(&bytes)?;
|
| 252 |
+
file.sync_all()?;
|
| 253 |
+
// Atomic publication with no overwrite, even if another writer wins.
|
| 254 |
+
fs::hard_link(&temporary, path)
|
| 255 |
+
})();
|
| 256 |
+
let _ = fs::remove_file(temporary);
|
| 257 |
+
result.map_err(|e| format!("{}: {e}", path.display()))
|
| 258 |
+
}
|
| 259 |
+
|
| 260 |
+
pub fn load<T: DeserializeOwned + Payload>(path: &Path) -> Result<Artifact<T>, String> {
|
| 261 |
+
let mut checked = Verification::default();
|
| 262 |
+
let value = load_verified(path, &mut checked)?;
|
| 263 |
+
checked.finish()?;
|
| 264 |
+
Ok(value)
|
| 265 |
+
}
|
| 266 |
+
|
| 267 |
+
pub fn load_verified<T: DeserializeOwned + Payload>(
|
| 268 |
+
path: &Path,
|
| 269 |
+
checked: &mut Verification,
|
| 270 |
+
) -> Result<Artifact<T>, String> {
|
| 271 |
+
let file = fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
|
| 272 |
+
const MAX_JSON: u64 = 64 * 1024 * 1024;
|
| 273 |
+
if file.metadata().map_err(|e| e.to_string())?.len() > MAX_JSON {
|
| 274 |
+
return Err("artifact exceeds 64 MiB".into());
|
| 275 |
+
}
|
| 276 |
+
let mut bytes = Vec::new();
|
| 277 |
+
file.take(MAX_JSON + 1)
|
| 278 |
+
.read_to_end(&mut bytes)
|
| 279 |
+
.map_err(|e| e.to_string())?;
|
| 280 |
+
if bytes.len() as u64 > MAX_JSON {
|
| 281 |
+
return Err("artifact exceeds 64 MiB".into());
|
| 282 |
+
}
|
| 283 |
+
checked.verify(path, &hash_bytes(&bytes))?;
|
| 284 |
+
let value: Artifact<T> =
|
| 285 |
+
serde_json::from_slice(&bytes).map_err(|e| format!("{}: {e}", path.display()))?;
|
| 286 |
+
if value.schema_version != T::SCHEMA || value.kind != T::KIND {
|
| 287 |
+
return Err(format!(
|
| 288 |
+
"{}: expected {} schema {}",
|
| 289 |
+
path.display(),
|
| 290 |
+
T::KIND,
|
| 291 |
+
T::SCHEMA
|
| 292 |
+
));
|
| 293 |
+
}
|
| 294 |
+
if let Some(data) = &value.data {
|
| 295 |
+
data.validate()?;
|
| 296 |
+
}
|
| 297 |
+
if value.complete {
|
| 298 |
+
value.require()?;
|
| 299 |
+
}
|
| 300 |
+
verify_refs(
|
| 301 |
+
path,
|
| 302 |
+
&value.inputs,
|
| 303 |
+
checked,
|
| 304 |
+
&mut std::collections::BTreeSet::new(),
|
| 305 |
+
0,
|
| 306 |
+
)?;
|
| 307 |
+
Ok(value)
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
fn verify_refs(
|
| 311 |
+
path: &Path,
|
| 312 |
+
refs: &[Reference],
|
| 313 |
+
checked: &mut Verification,
|
| 314 |
+
traversed: &mut std::collections::BTreeSet<std::path::PathBuf>,
|
| 315 |
+
depth: usize,
|
| 316 |
+
) -> Result<(), String> {
|
| 317 |
+
if depth > 32 {
|
| 318 |
+
return Err("evidence reference depth exceeds 32".into());
|
| 319 |
+
}
|
| 320 |
+
for input in refs {
|
| 321 |
+
let alias = path.parent().unwrap_or(Path::new(".")).join(&input.path);
|
| 322 |
+
let target = alias.canonicalize().map_err(|e| e.to_string())?;
|
| 323 |
+
checked.verify(&alias, &input.sha256)?;
|
| 324 |
+
if !traversed.insert(target.clone()) {
|
| 325 |
+
continue;
|
| 326 |
+
}
|
| 327 |
+
if target.extension().is_none_or(|e| e != "json") {
|
| 328 |
+
continue;
|
| 329 |
+
}
|
| 330 |
+
let file = fs::File::open(&target).map_err(|e| e.to_string())?;
|
| 331 |
+
if file.metadata().map_err(|e| e.to_string())?.len() > 64 * 1024 * 1024 {
|
| 332 |
+
return Err("JSON evidence exceeds 64 MiB".into());
|
| 333 |
+
}
|
| 334 |
+
let value: serde_json::Value = serde_json::from_reader(file).map_err(|e| e.to_string())?;
|
| 335 |
+
if value.get("kind").is_none() || value.get("schema_version").is_none() {
|
| 336 |
+
continue;
|
| 337 |
+
}
|
| 338 |
+
let refs: Vec<Reference> = serde_json::from_value(
|
| 339 |
+
value
|
| 340 |
+
.get("inputs")
|
| 341 |
+
.ok_or("nested artifact inputs missing")?
|
| 342 |
+
.clone(),
|
| 343 |
+
)
|
| 344 |
+
.map_err(|e| e.to_string())?;
|
| 345 |
+
verify_refs(&target, &refs, checked, traversed, depth + 1)?;
|
| 346 |
+
}
|
| 347 |
+
Ok(())
|
| 348 |
+
}
|
| 349 |
+
|
| 350 |
+
pub fn directory(path: &Path) -> Result<(), String> {
|
| 351 |
+
if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) {
|
| 352 |
+
fs::create_dir_all(parent).map_err(|e| e.to_string())?;
|
| 353 |
+
}
|
| 354 |
+
fs::create_dir(path).map_err(|e| {
|
| 355 |
+
format!(
|
| 356 |
+
"cannot create new evidence directory {}: {e}",
|
| 357 |
+
path.display()
|
| 358 |
+
)
|
| 359 |
+
})
|
| 360 |
+
}
|
| 361 |
+
|
| 362 |
+
/// Import an inspected machine/calibration bundle with its relative raw inputs.
|
| 363 |
+
/// External references require re-inspection into a self-contained directory.
|
| 364 |
+
pub fn bundle<T: DeserializeOwned + Payload>(
|
| 365 |
+
path: &Path,
|
| 366 |
+
out: &Path,
|
| 367 |
+
) -> Result<std::path::PathBuf, String> {
|
| 368 |
+
load::<T>(path)?;
|
| 369 |
+
let source = path.canonicalize().map_err(|e| e.to_string())?;
|
| 370 |
+
let root = source.parent().ok_or("bundle has no parent")?;
|
| 371 |
+
directory(out)?;
|
| 372 |
+
let mut visited = std::collections::BTreeSet::new();
|
| 373 |
+
let mut pending = vec![source.clone()];
|
| 374 |
+
let mut total = 0u64;
|
| 375 |
+
while let Some(path) = pending.pop() {
|
| 376 |
+
let path = path.canonicalize().map_err(|e| e.to_string())?;
|
| 377 |
+
if !visited.insert(path.clone()) {
|
| 378 |
+
continue;
|
| 379 |
+
}
|
| 380 |
+
let relative = path
|
| 381 |
+
.strip_prefix(root)
|
| 382 |
+
.map_err(|_| "bundle has external raw inputs; create a self-contained inspection")?;
|
| 383 |
+
total = total
|
| 384 |
+
.checked_add(fs::metadata(&path).map_err(|e| e.to_string())?.len())
|
| 385 |
+
.ok_or("bundle size overflow")?;
|
| 386 |
+
if total > 256 * 1024 * 1024 {
|
| 387 |
+
return Err("machine/calibration bundle exceeds 256 MiB".into());
|
| 388 |
+
}
|
| 389 |
+
let destination = out.join(relative);
|
| 390 |
+
if let Some(parent) = destination.parent() {
|
| 391 |
+
fs::create_dir_all(parent).map_err(|e| e.to_string())?;
|
| 392 |
+
}
|
| 393 |
+
let mut source_file = fs::File::open(&path).map_err(|e| e.to_string())?;
|
| 394 |
+
let mut copy = fs::OpenOptions::new()
|
| 395 |
+
.create_new(true)
|
| 396 |
+
.write(true)
|
| 397 |
+
.open(&destination)
|
| 398 |
+
.map_err(|e| e.to_string())?;
|
| 399 |
+
std::io::copy(&mut source_file, &mut copy).map_err(|e| e.to_string())?;
|
| 400 |
+
if path.extension().is_none_or(|e| e != "json") {
|
| 401 |
+
continue;
|
| 402 |
+
}
|
| 403 |
+
let value: serde_json::Value =
|
| 404 |
+
serde_json::from_reader(fs::File::open(&path).map_err(|e| e.to_string())?)
|
| 405 |
+
.map_err(|e| e.to_string())?;
|
| 406 |
+
if value.get("kind").is_none() || value.get("schema_version").is_none() {
|
| 407 |
+
continue;
|
| 408 |
+
}
|
| 409 |
+
let inputs: Vec<Reference> =
|
| 410 |
+
serde_json::from_value(value.get("inputs").ok_or("bundle inputs missing")?.clone())
|
| 411 |
+
.map_err(|e| e.to_string())?;
|
| 412 |
+
pending.extend(
|
| 413 |
+
inputs
|
| 414 |
+
.into_iter()
|
| 415 |
+
.map(|r| path.parent().unwrap().join(r.path)),
|
| 416 |
+
);
|
| 417 |
+
}
|
| 418 |
+
let imported = out.join(source.file_name().unwrap());
|
| 419 |
+
load::<T>(&imported)?;
|
| 420 |
+
Ok(imported)
|
| 421 |
+
}
|
| 422 |
+
|
| 423 |
+
#[cfg(test)]
|
| 424 |
+
mod tests {
|
| 425 |
+
use super::*;
|
| 426 |
+
#[derive(Debug, Serialize, Deserialize)]
|
| 427 |
+
struct Example {
|
| 428 |
+
value: u32,
|
| 429 |
+
}
|
| 430 |
+
impl Payload for Example {
|
| 431 |
+
const KIND: &'static str = "example";
|
| 432 |
+
fn validate(&self) -> Result<(), String> {
|
| 433 |
+
Ok(())
|
| 434 |
+
}
|
| 435 |
+
}
|
| 436 |
+
#[test]
|
| 437 |
+
fn verification_rejects_drift_and_conflicting_digest() {
|
| 438 |
+
let root = std::env::temp_dir().join(format!("ds4-verified-{}", std::process::id()));
|
| 439 |
+
directory(&root).unwrap();
|
| 440 |
+
let path = root.join("input");
|
| 441 |
+
fs::write(&path, b"first").unwrap();
|
| 442 |
+
let expected = hash(&path).unwrap();
|
| 443 |
+
let mut checked = Verification::default();
|
| 444 |
+
checked.verify(&path, &expected).unwrap();
|
| 445 |
+
checked.verify(&path, &expected).unwrap();
|
| 446 |
+
assert_eq!(checked.full_reads, 1);
|
| 447 |
+
assert!(checked.verify(&path, &hash_bytes(b"other")).is_err());
|
| 448 |
+
fs::write(&path, b"other").unwrap();
|
| 449 |
+
assert!(checked.verify(&path, &expected).is_err());
|
| 450 |
+
assert!(checked.finish().is_err());
|
| 451 |
+
|
| 452 |
+
let mut replaced = Verification::default();
|
| 453 |
+
replaced.verify(&path, &hash_bytes(b"other")).unwrap();
|
| 454 |
+
let replacement = root.join("replacement");
|
| 455 |
+
fs::write(&replacement, b"other").unwrap();
|
| 456 |
+
fs::rename(replacement, &path).unwrap();
|
| 457 |
+
assert!(
|
| 458 |
+
replaced.finish().is_err(),
|
| 459 |
+
"same bytes on another inode must fail"
|
| 460 |
+
);
|
| 461 |
+
fs::remove_dir_all(root).unwrap();
|
| 462 |
+
}
|
| 463 |
+
|
| 464 |
+
#[test]
|
| 465 |
+
fn operation_reuses_shared_inputs_across_artifacts() {
|
| 466 |
+
let root = std::env::temp_dir().join(format!("ds4-shared-{}", std::process::id()));
|
| 467 |
+
directory(&root).unwrap();
|
| 468 |
+
let input = root.join("weights");
|
| 469 |
+
fs::write(&input, b"same input").unwrap();
|
| 470 |
+
for index in 0..2 {
|
| 471 |
+
let mut evidence = Artifact::new(Example { value: index });
|
| 472 |
+
evidence.inputs.push(reference(&input, &root).unwrap());
|
| 473 |
+
save(&root.join(format!("{index}.json")), &evidence).unwrap();
|
| 474 |
+
}
|
| 475 |
+
let mut checked = Verification::default();
|
| 476 |
+
for index in 0..2 {
|
| 477 |
+
load_verified::<Example>(&root.join(format!("{index}.json")), &mut checked).unwrap();
|
| 478 |
+
}
|
| 479 |
+
assert_eq!(checked.full_reads, 3, "two envelopes and one shared input");
|
| 480 |
+
checked.finish().unwrap();
|
| 481 |
+
let envelope = root.join("0.json");
|
| 482 |
+
let bytes = fs::read(&envelope).unwrap();
|
| 483 |
+
fs::write(&envelope, [bytes.as_slice(), b"\n"].concat()).unwrap();
|
| 484 |
+
assert!(
|
| 485 |
+
checked.finish().is_err(),
|
| 486 |
+
"root envelope must remain pinned"
|
| 487 |
+
);
|
| 488 |
+
fs::write(&input, b"new input!").unwrap();
|
| 489 |
+
assert!(load_verified::<Example>(&root.join("1.json"), &mut checked).is_err());
|
| 490 |
+
assert!(checked.finish().is_err());
|
| 491 |
+
fs::remove_dir_all(root).unwrap();
|
| 492 |
+
}
|
| 493 |
+
|
| 494 |
+
#[cfg(unix)]
|
| 495 |
+
#[test]
|
| 496 |
+
fn verification_rejects_retargeted_alias() {
|
| 497 |
+
let root = std::env::temp_dir().join(format!("ds4-alias-{}", std::process::id()));
|
| 498 |
+
directory(&root).unwrap();
|
| 499 |
+
fs::write(root.join("a"), b"same").unwrap();
|
| 500 |
+
fs::write(root.join("b"), b"same").unwrap();
|
| 501 |
+
let alias = root.join("alias");
|
| 502 |
+
std::os::unix::fs::symlink(root.join("a"), &alias).unwrap();
|
| 503 |
+
let mut checked = Verification::default();
|
| 504 |
+
checked.verify(&alias, &hash_bytes(b"same")).unwrap();
|
| 505 |
+
fs::remove_file(&alias).unwrap();
|
| 506 |
+
std::os::unix::fs::symlink(root.join("b"), &alias).unwrap();
|
| 507 |
+
assert!(checked.finish().is_err());
|
| 508 |
+
assert!(checked.verify(&alias, &hash_bytes(b"same")).is_err());
|
| 509 |
+
fs::remove_dir_all(root).unwrap();
|
| 510 |
+
}
|
| 511 |
+
#[test]
|
| 512 |
+
fn transitive_hashes() {
|
| 513 |
+
let root = std::env::temp_dir().join(format!("ds4-tree-{}", std::process::id()));
|
| 514 |
+
directory(&root).unwrap();
|
| 515 |
+
fs::write(root.join("raw"), "original").unwrap();
|
| 516 |
+
let mut first = Artifact::new(Example { value: 1 });
|
| 517 |
+
first
|
| 518 |
+
.inputs
|
| 519 |
+
.push(reference(&root.join("raw"), &root).unwrap());
|
| 520 |
+
save(&root.join("first.json"), &first).unwrap();
|
| 521 |
+
let mut second = Artifact::new(Example { value: 2 });
|
| 522 |
+
second
|
| 523 |
+
.inputs
|
| 524 |
+
.push(reference(&root.join("first.json"), &root).unwrap());
|
| 525 |
+
save(&root.join("second.json"), &second).unwrap();
|
| 526 |
+
fs::write(root.join("raw"), "altered").unwrap();
|
| 527 |
+
let rejected = load::<Example>(&root.join("second.json")).is_err();
|
| 528 |
+
fs::remove_dir_all(root).unwrap();
|
| 529 |
+
assert!(rejected, "nested evidence tampering must fail");
|
| 530 |
+
}
|
| 531 |
+
|
| 532 |
+
#[test]
|
| 533 |
+
fn moved_evidence_and_tampering() {
|
| 534 |
+
let root = std::env::temp_dir().join(format!("ds4-evidence-{}", std::process::id()));
|
| 535 |
+
let source = root.join("source");
|
| 536 |
+
directory(&source).unwrap();
|
| 537 |
+
fs::write(source.join("raw"), "original").unwrap();
|
| 538 |
+
let mut artifact = Artifact::new(Example { value: 7 });
|
| 539 |
+
artifact
|
| 540 |
+
.inputs
|
| 541 |
+
.push(reference(&source.join("raw"), &source).unwrap());
|
| 542 |
+
save(&source.join("example.json"), &artifact).unwrap();
|
| 543 |
+
assert!(save(
|
| 544 |
+
&source.join("example.json"),
|
| 545 |
+
&Artifact::new(Example { value: 99 })
|
| 546 |
+
)
|
| 547 |
+
.is_err());
|
| 548 |
+
let moved = root.join("moved");
|
| 549 |
+
fs::rename(&source, &moved).unwrap();
|
| 550 |
+
assert_eq!(
|
| 551 |
+
load::<Example>(&moved.join("example.json"))
|
| 552 |
+
.unwrap()
|
| 553 |
+
.require()
|
| 554 |
+
.unwrap()
|
| 555 |
+
.value,
|
| 556 |
+
7
|
| 557 |
+
);
|
| 558 |
+
fs::write(moved.join("raw"), "changed").unwrap();
|
| 559 |
+
assert!(load::<Example>(&moved.join("example.json"))
|
| 560 |
+
.unwrap_err()
|
| 561 |
+
.contains("evidence changed"));
|
| 562 |
+
fs::remove_dir_all(root).unwrap();
|
| 563 |
+
}
|
| 564 |
+
}
|
crates/ds4-perf/src/bench.rs
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
use crate::csv;
|
| 2 |
+
use std::io::BufRead;
|
| 3 |
+
|
| 4 |
+
pub const HEADER: &str =
|
| 5 |
+
"ctx_tokens,prefill_tokens,prefill_tps,gen_tokens,gen_tps,first_token_sec,kvcache_bytes";
|
| 6 |
+
|
| 7 |
+
#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
|
| 8 |
+
#[serde(deny_unknown_fields)]
|
| 9 |
+
pub struct Row {
|
| 10 |
+
pub ctx: u64,
|
| 11 |
+
pub prefill_tokens: u64,
|
| 12 |
+
pub prefill_tps: f64,
|
| 13 |
+
pub gen_tokens: u64,
|
| 14 |
+
pub gen_tps: f64,
|
| 15 |
+
pub first_token_sec: f64,
|
| 16 |
+
pub kvcache_bytes: u64,
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
impl Row {
|
| 20 |
+
pub fn csv(&self) -> String {
|
| 21 |
+
format!(
|
| 22 |
+
"{},{},{},{},{},{},{}\n",
|
| 23 |
+
self.ctx,
|
| 24 |
+
self.prefill_tokens,
|
| 25 |
+
self.prefill_tps,
|
| 26 |
+
self.gen_tokens,
|
| 27 |
+
self.gen_tps,
|
| 28 |
+
self.first_token_sec,
|
| 29 |
+
self.kvcache_bytes
|
| 30 |
+
)
|
| 31 |
+
}
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
pub fn parse(reader: impl BufRead) -> Result<Vec<Row>, String> {
|
| 35 |
+
let mut header = Vec::new();
|
| 36 |
+
let mut rows = Vec::new();
|
| 37 |
+
csv::records(reader, |row| {
|
| 38 |
+
if csv::column(&row, &["prefill_tps"]).is_some() {
|
| 39 |
+
header = row;
|
| 40 |
+
return Ok(());
|
| 41 |
+
}
|
| 42 |
+
if header.is_empty() || row.iter().all(|v| v.trim().is_empty()) {
|
| 43 |
+
return Ok(());
|
| 44 |
+
}
|
| 45 |
+
let field = |name| {
|
| 46 |
+
csv::value(&row, &header, &[name])
|
| 47 |
+
.ok_or_else(|| format!("missing ds4-bench column: {name}"))
|
| 48 |
+
};
|
| 49 |
+
let integer = |name| {
|
| 50 |
+
field(name)?
|
| 51 |
+
.parse::<u64>()
|
| 52 |
+
.map_err(|_| format!("invalid {name}"))
|
| 53 |
+
};
|
| 54 |
+
rows.push(Row {
|
| 55 |
+
ctx: integer("ctx_tokens")?,
|
| 56 |
+
prefill_tokens: integer("prefill_tokens")?,
|
| 57 |
+
prefill_tps: csv::number(field("prefill_tps")?)?,
|
| 58 |
+
gen_tokens: integer("gen_tokens")?,
|
| 59 |
+
gen_tps: csv::number(field("gen_tps")?)?,
|
| 60 |
+
first_token_sec: csv::number(field("first_token_sec")?)?,
|
| 61 |
+
kvcache_bytes: integer("kvcache_bytes")?,
|
| 62 |
+
});
|
| 63 |
+
Ok(())
|
| 64 |
+
})?;
|
| 65 |
+
Ok(rows)
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
#[cfg(test)]
|
| 69 |
+
mod tests {
|
| 70 |
+
use super::*;
|
| 71 |
+
|
| 72 |
+
#[test]
|
| 73 |
+
fn bench_named_columns() {
|
| 74 |
+
let input = include_str!("../tests/fixtures/bench.csv");
|
| 75 |
+
let rows = parse(input.as_bytes()).unwrap();
|
| 76 |
+
assert_eq!(rows.len(), 1);
|
| 77 |
+
assert_eq!(rows[0].prefill_tps, 1502.8);
|
| 78 |
+
assert_eq!(rows[0].gen_tps, 24.3);
|
| 79 |
+
assert_eq!(rows[0].first_token_sec, 0.0512);
|
| 80 |
+
assert_eq!(rows[0].kvcache_bytes, 4294967296);
|
| 81 |
+
assert_eq!(rows[0].ctx, 8192);
|
| 82 |
+
assert!(parse(b"opaque output\n".as_slice()).unwrap().is_empty());
|
| 83 |
+
}
|
| 84 |
+
}
|
crates/ds4-perf/src/cli.rs
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
use clap::{Args, Parser, Subcommand, ValueEnum};
|
| 2 |
+
use std::{ffi::OsString, path::PathBuf};
|
| 3 |
+
|
| 4 |
+
#[derive(Parser, Debug)]
|
| 5 |
+
#[command(
|
| 6 |
+
version,
|
| 7 |
+
about = "Reproducible CUDA performance evidence and experiments"
|
| 8 |
+
)]
|
| 9 |
+
pub struct Cli {
|
| 10 |
+
#[command(subcommand)]
|
| 11 |
+
pub command: Command,
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
#[derive(Subcommand, Debug)]
|
| 15 |
+
pub enum Command {
|
| 16 |
+
/// Inspect machine capabilities; optionally measure the device envelope.
|
| 17 |
+
Inspect(Inspect),
|
| 18 |
+
/// Compatibility entry point for the v0.1.0 capability report.
|
| 19 |
+
#[command(hide = true)]
|
| 20 |
+
Doctor {
|
| 21 |
+
#[arg(long, default_value = "./ds4-bench")]
|
| 22 |
+
bench: String,
|
| 23 |
+
},
|
| 24 |
+
/// Profile a fresh benchmark and identify execution-path bottlenecks.
|
| 25 |
+
Scout(Scout),
|
| 26 |
+
/// Compare matched unprofiled experiments and correctness evidence.
|
| 27 |
+
Compare(Compare),
|
| 28 |
+
/// Choose experiments from evidence; --auto executes a bounded loop.
|
| 29 |
+
Optimize(Optimize),
|
| 30 |
+
/// Measure ordered HTTP conversation workloads and check their contracts.
|
| 31 |
+
Serving(Serving),
|
| 32 |
+
/// Propose one-control experiments from the server's capability report.
|
| 33 |
+
ServingControls(ServingControls),
|
| 34 |
+
/// Select a profile only from complete, repeated serving contracts.
|
| 35 |
+
ServingProfile(ServingProfile),
|
| 36 |
+
/// Check or execute a pinned serving profile; additional options are forbidden.
|
| 37 |
+
ApplyProfile(ApplyProfile),
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
#[derive(Args, Debug)]
|
| 41 |
+
pub struct ServingControls {
|
| 42 |
+
#[arg(long)]
|
| 43 |
+
pub plan: PathBuf,
|
| 44 |
+
#[arg(long)]
|
| 45 |
+
pub out: PathBuf,
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
#[derive(Args, Debug)]
|
| 49 |
+
pub struct ServingProfile {
|
| 50 |
+
#[arg(long)]
|
| 51 |
+
pub plan: PathBuf,
|
| 52 |
+
#[arg(long)]
|
| 53 |
+
pub out: PathBuf,
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
#[derive(Args, Debug)]
|
| 57 |
+
pub struct ApplyProfile {
|
| 58 |
+
#[arg(long)]
|
| 59 |
+
pub profile: PathBuf,
|
| 60 |
+
#[arg(long)]
|
| 61 |
+
pub out: PathBuf,
|
| 62 |
+
#[arg(long)]
|
| 63 |
+
pub execute: bool,
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
#[derive(Args, Debug)]
|
| 67 |
+
pub struct Serving {
|
| 68 |
+
#[command(flatten)]
|
| 69 |
+
pub budget: Budget,
|
| 70 |
+
/// Local server origin, e.g. http://127.0.0.1:8002.
|
| 71 |
+
#[arg(long)]
|
| 72 |
+
pub url: String,
|
| 73 |
+
#[arg(long)]
|
| 74 |
+
pub workload: PathBuf,
|
| 75 |
+
#[arg(long)]
|
| 76 |
+
pub out: PathBuf,
|
| 77 |
+
/// Replay the ordered workload; KV is deliberately preserved between repeats.
|
| 78 |
+
#[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..=100))]
|
| 79 |
+
pub repeats: u32,
|
| 80 |
+
/// Pin the local server executable, start time and command line.
|
| 81 |
+
#[arg(long)]
|
| 82 |
+
pub server_pid: Option<u32>,
|
| 83 |
+
/// Physical NVIDIA device ordinal used for clock/temperature snapshots.
|
| 84 |
+
#[arg(long, default_value_t = 0)]
|
| 85 |
+
pub device: usize,
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
#[derive(Args, Debug)]
|
| 89 |
+
pub struct Inspect {
|
| 90 |
+
#[arg(long)]
|
| 91 |
+
pub out: PathBuf,
|
| 92 |
+
#[arg(long)]
|
| 93 |
+
pub calibrate: bool,
|
| 94 |
+
#[arg(long, default_value_t = 0)]
|
| 95 |
+
pub device: usize,
|
| 96 |
+
#[arg(long)]
|
| 97 |
+
pub gpu_helper: Option<PathBuf>,
|
| 98 |
+
#[arg(long)]
|
| 99 |
+
pub bench: Option<String>,
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
#[derive(Clone, Copy, Debug, ValueEnum)]
|
| 103 |
+
pub enum Collector {
|
| 104 |
+
Nsys,
|
| 105 |
+
Cupti,
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
#[derive(Args, Clone, Debug)]
|
| 109 |
+
pub struct Scout {
|
| 110 |
+
#[arg(skip)]
|
| 111 |
+
pub exact_environment: bool,
|
| 112 |
+
#[command(flatten)]
|
| 113 |
+
pub budget: Budget,
|
| 114 |
+
#[arg(long)]
|
| 115 |
+
pub out: PathBuf,
|
| 116 |
+
#[arg(long, value_enum, default_value = "nsys")]
|
| 117 |
+
pub collector: Collector,
|
| 118 |
+
#[arg(long)]
|
| 119 |
+
pub fit: bool,
|
| 120 |
+
/// Request ds4-bench token and full-vocabulary frontier proof files.
|
| 121 |
+
#[arg(long)]
|
| 122 |
+
pub proof: bool,
|
| 123 |
+
/// CUPTI injection library (default: beside ds4-perf-gpu).
|
| 124 |
+
#[arg(long, requires = "collector")]
|
| 125 |
+
pub cupti_library: Option<PathBuf>,
|
| 126 |
+
/// NVIDIA libcupti.so used for NVTX injection and activity collection.
|
| 127 |
+
#[arg(long)]
|
| 128 |
+
pub cupti_sdk: Option<PathBuf>,
|
| 129 |
+
#[arg(long)]
|
| 130 |
+
pub ncu: bool,
|
| 131 |
+
#[arg(long)]
|
| 132 |
+
pub machine: Option<PathBuf>,
|
| 133 |
+
#[arg(long, requires = "fit")]
|
| 134 |
+
pub calibration: Option<PathBuf>,
|
| 135 |
+
#[arg(long)]
|
| 136 |
+
pub gpu_helper: Option<PathBuf>,
|
| 137 |
+
#[arg(long, default_value_t = 0)]
|
| 138 |
+
pub device: usize,
|
| 139 |
+
#[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..=100))]
|
| 140 |
+
pub repeats: u32,
|
| 141 |
+
/// Explicit workload/cache contract used when comparing runs.
|
| 142 |
+
#[arg(long)]
|
| 143 |
+
pub workload: Option<PathBuf>,
|
| 144 |
+
#[arg(long, default_value = "inherited", value_parser = ["inherited", "warmup-then-fresh"])]
|
| 145 |
+
pub cache_policy: String,
|
| 146 |
+
/// Allowlisted runtime control, passed directly to the child process.
|
| 147 |
+
#[arg(long = "env")]
|
| 148 |
+
pub environment: Vec<String>,
|
| 149 |
+
#[arg(last = true, required = true)]
|
| 150 |
+
pub command: Vec<OsString>,
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
#[derive(Args, Debug)]
|
| 154 |
+
pub struct Compare {
|
| 155 |
+
#[arg(long)]
|
| 156 |
+
pub baseline: PathBuf,
|
| 157 |
+
#[arg(long)]
|
| 158 |
+
pub candidate: PathBuf,
|
| 159 |
+
#[arg(long)]
|
| 160 |
+
pub out: PathBuf,
|
| 161 |
+
/// Exit unsuccessfully on regression or insufficient comparison evidence.
|
| 162 |
+
#[arg(long)]
|
| 163 |
+
pub regression: bool,
|
| 164 |
+
#[arg(long, default_value_t = 3.0)]
|
| 165 |
+
pub max_slowdown_percent: f64,
|
| 166 |
+
#[arg(long, default_value_t = 0.0001)]
|
| 167 |
+
pub logit_atol: f64,
|
| 168 |
+
#[arg(long, default_value_t = 0.0001)]
|
| 169 |
+
pub logit_rtol: f64,
|
| 170 |
+
/// Relaxed contract for summation-order changes: when positive, each
|
| 171 |
+
/// frontier must stay within this relative RMS of the baseline at 1024
|
| 172 |
+
/// tokens (the bound grows with log2 of the context) and keep its argmax;
|
| 173 |
+
/// greedy sequences may diverge. Zero keeps the exact contract.
|
| 174 |
+
#[arg(long, default_value_t = 0.0)]
|
| 175 |
+
pub logit_rel_rms: f64,
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
#[derive(Args, Debug)]
|
| 179 |
+
pub struct Optimize {
|
| 180 |
+
#[command(flatten)]
|
| 181 |
+
pub budget: Budget,
|
| 182 |
+
#[arg(long)]
|
| 183 |
+
pub scout: PathBuf,
|
| 184 |
+
#[arg(long)]
|
| 185 |
+
pub out: PathBuf,
|
| 186 |
+
/// Run evidence-selected experiments and compare each candidate.
|
| 187 |
+
#[arg(long = "auto")]
|
| 188 |
+
pub automatic: bool,
|
| 189 |
+
#[arg(long)]
|
| 190 |
+
pub plan: Option<PathBuf>,
|
| 191 |
+
#[arg(long, default_value_t = 3, value_parser = clap::value_parser!(u32).range(1..=32))]
|
| 192 |
+
pub rounds: u32,
|
| 193 |
+
#[arg(long, default_value_t = 3, value_parser = clap::value_parser!(u32).range(3..=100))]
|
| 194 |
+
pub repeats: u32,
|
| 195 |
+
}
|
| 196 |
+
|
| 197 |
+
#[derive(Args, Clone, Debug, serde::Serialize, serde::Deserialize)]
|
| 198 |
+
#[serde(deny_unknown_fields)]
|
| 199 |
+
pub struct Budget {
|
| 200 |
+
/// Deadline for each benchmark/profiler process, including descendants.
|
| 201 |
+
#[arg(long,default_value_t=1800,value_parser=clap::value_parser!(u64).range(1..=86400))]
|
| 202 |
+
pub timeout_seconds: u64,
|
| 203 |
+
/// Total raw output budget per scout; auto also bounds the campaign.
|
| 204 |
+
#[arg(long,default_value_t=2048,value_parser=clap::value_parser!(u64).range(1..=65536))]
|
| 205 |
+
pub max_output_mib: u64,
|
| 206 |
+
}
|
| 207 |
+
impl Budget {
|
| 208 |
+
pub fn limits(&self) -> crate::process::Limits {
|
| 209 |
+
crate::process::Limits {
|
| 210 |
+
timeout: std::time::Duration::from_secs(self.timeout_seconds),
|
| 211 |
+
bytes: self.max_output_mib.saturating_mul(1024 * 1024),
|
| 212 |
+
}
|
| 213 |
+
}
|
| 214 |
+
}
|
crates/ds4-perf/src/compare.rs
ADDED
|
@@ -0,0 +1,695 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
use crate::{
|
| 2 |
+
cli,
|
| 3 |
+
experiment::{self, Scout},
|
| 4 |
+
fit,
|
| 5 |
+
};
|
| 6 |
+
use ds4_perf::artifact::{self, Artifact, Payload, Reference};
|
| 7 |
+
use serde::{Deserialize, Serialize};
|
| 8 |
+
use std::{collections::BTreeMap, fs, path::Path};
|
| 9 |
+
|
| 10 |
+
#[derive(Debug, Serialize, Deserialize)]
|
| 11 |
+
#[serde(deny_unknown_fields)]
|
| 12 |
+
pub struct Delta {
|
| 13 |
+
pub metric: String,
|
| 14 |
+
pub ctx: u64,
|
| 15 |
+
pub unit: String,
|
| 16 |
+
pub baseline: Vec<f64>,
|
| 17 |
+
pub candidate: Vec<f64>,
|
| 18 |
+
pub baseline_median: f64,
|
| 19 |
+
pub candidate_median: f64,
|
| 20 |
+
pub median_percent: f64,
|
| 21 |
+
pub lower_percent: f64,
|
| 22 |
+
pub upper_percent: f64,
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
#[derive(Debug, Default, Serialize, Deserialize)]
|
| 26 |
+
#[serde(deny_unknown_fields)]
|
| 27 |
+
pub struct LogitError {
|
| 28 |
+
pub max_abs: f64,
|
| 29 |
+
pub max_scaled: f64,
|
| 30 |
+
pub bad: u64,
|
| 31 |
+
pub checked: u64,
|
| 32 |
+
/// Largest relative RMS of a candidate frontier against its reference
|
| 33 |
+
/// (sqrt(sum diff^2 / sum ref^2)); the relaxed contract's statistic.
|
| 34 |
+
#[serde(default)]
|
| 35 |
+
pub rel_rms: f64,
|
| 36 |
+
/// Largest `rel_rms / rel_rms_bound(ctx)`; above 1 the relaxed contract
|
| 37 |
+
/// is violated.
|
| 38 |
+
#[serde(default)]
|
| 39 |
+
pub rel_rms_scaled: f64,
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
/// Reference context of the relaxed contract: `--logit-rel-rms` is the bound
|
| 43 |
+
/// at this width and grows with log2(ctx) from it.
|
| 44 |
+
const REL_RMS_REFERENCE_CTX: f64 = 1024.0;
|
| 45 |
+
|
| 46 |
+
/// Context-dependent relative RMS bound. Reordering noise accumulates with
|
| 47 |
+
/// the attended length but the model's amplification saturates, and a
|
| 48 |
+
/// one-ulp perturbation of Inkling MQ85GB measured 0.063 at 16 tokens and
|
| 49 |
+
/// 0.105 at 515 (log-linear), so the bound grows logarithmically: at 2K it
|
| 50 |
+
/// is 1.1x the reference, at 8K 1.3x, at 64K 1.6x, never linear in ctx.
|
| 51 |
+
fn rel_rms_bound(reference: f64, ctx: u64) -> f64 {
|
| 52 |
+
reference * (ctx.max(2) as f64).log2() / REL_RMS_REFERENCE_CTX.log2()
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
|
| 56 |
+
#[serde(rename_all = "snake_case")]
|
| 57 |
+
pub enum Verdict {
|
| 58 |
+
Pass,
|
| 59 |
+
Improved,
|
| 60 |
+
Regressed,
|
| 61 |
+
Inconclusive,
|
| 62 |
+
Incorrect,
|
| 63 |
+
Incomparable,
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
#[derive(Debug, Serialize, Deserialize)]
|
| 67 |
+
#[serde(deny_unknown_fields)]
|
| 68 |
+
pub struct Comparison {
|
| 69 |
+
pub verdict: Verdict,
|
| 70 |
+
pub metrics: Vec<Delta>,
|
| 71 |
+
pub correctness: LogitError,
|
| 72 |
+
/// Number of proof comparisons with a greedy sequence or argmax mismatch.
|
| 73 |
+
pub token_mismatches: u64,
|
| 74 |
+
/// Number of proof comparisons whose frontier argmax differs (a subset of
|
| 75 |
+
/// `token_mismatches`).
|
| 76 |
+
#[serde(default)]
|
| 77 |
+
pub argmax_mismatches: u64,
|
| 78 |
+
pub max_slowdown_percent: f64,
|
| 79 |
+
pub logit_atol: f64,
|
| 80 |
+
pub logit_rtol: f64,
|
| 81 |
+
/// Relaxed contract when positive: a frontier is correct when its relative
|
| 82 |
+
/// RMS stays within `rel_rms_bound(ctx)` (this value at 1024 tokens,
|
| 83 |
+
/// growing with log2(ctx)) and its argmax matches; greedy sequences may
|
| 84 |
+
/// diverge. Zero keeps the per-logit atol/rtol and sequence contract.
|
| 85 |
+
#[serde(default)]
|
| 86 |
+
pub logit_rel_rms: f64,
|
| 87 |
+
pub changes: BTreeMap<String, String>,
|
| 88 |
+
pub reasons: Vec<String>,
|
| 89 |
+
pub method: String,
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
impl Payload for Comparison {
|
| 93 |
+
const KIND: &'static str = "compare";
|
| 94 |
+
fn validate(&self) -> Result<(), String> {
|
| 95 |
+
if self.method != "sample-extrema-envelope-v1" {
|
| 96 |
+
return Err("unsupported comparison method".into());
|
| 97 |
+
}
|
| 98 |
+
if [
|
| 99 |
+
self.max_slowdown_percent,
|
| 100 |
+
self.logit_atol,
|
| 101 |
+
self.logit_rtol,
|
| 102 |
+
self.logit_rel_rms,
|
| 103 |
+
self.correctness.max_abs,
|
| 104 |
+
self.correctness.max_scaled,
|
| 105 |
+
self.correctness.rel_rms,
|
| 106 |
+
self.correctness.rel_rms_scaled,
|
| 107 |
+
]
|
| 108 |
+
.into_iter()
|
| 109 |
+
.any(|v| !v.is_finite() || v < 0.0)
|
| 110 |
+
{
|
| 111 |
+
return Err("invalid comparison policy or result".into());
|
| 112 |
+
}
|
| 113 |
+
if self.verdict == Verdict::Incomparable {
|
| 114 |
+
if self.reasons.is_empty() || !self.metrics.is_empty() {
|
| 115 |
+
return Err("incomparable result lacks a reason".into());
|
| 116 |
+
}
|
| 117 |
+
return Ok(());
|
| 118 |
+
}
|
| 119 |
+
if self.metrics.is_empty()
|
| 120 |
+
|| self.correctness.checked == 0
|
| 121 |
+
|| self.correctness.bad > self.correctness.checked
|
| 122 |
+
|| self.argmax_mismatches > self.token_mismatches
|
| 123 |
+
{
|
| 124 |
+
return Err("conclusive comparison lacks timing/correctness evidence".into());
|
| 125 |
+
}
|
| 126 |
+
let expected = verdict(
|
| 127 |
+
&self.metrics,
|
| 128 |
+
&self.correctness,
|
| 129 |
+
self.token_mismatches,
|
| 130 |
+
self.argmax_mismatches,
|
| 131 |
+
self.max_slowdown_percent,
|
| 132 |
+
self.logit_rel_rms,
|
| 133 |
+
);
|
| 134 |
+
if self.verdict != expected {
|
| 135 |
+
return Err("comparison verdict disagrees with evidence".into());
|
| 136 |
+
}
|
| 137 |
+
for d in &self.metrics {
|
| 138 |
+
if [
|
| 139 |
+
d.baseline_median,
|
| 140 |
+
d.candidate_median,
|
| 141 |
+
d.median_percent,
|
| 142 |
+
d.lower_percent,
|
| 143 |
+
d.upper_percent,
|
| 144 |
+
]
|
| 145 |
+
.into_iter()
|
| 146 |
+
.any(|v| !v.is_finite())
|
| 147 |
+
|| d.baseline
|
| 148 |
+
.iter()
|
| 149 |
+
.chain(&d.candidate)
|
| 150 |
+
.any(|v| !v.is_finite() || *v <= 0.0)
|
| 151 |
+
{
|
| 152 |
+
return Err("invalid comparison samples".into());
|
| 153 |
+
}
|
| 154 |
+
let computed = delta(&d.metric, d.ctx, &d.baseline, &d.candidate)?;
|
| 155 |
+
if d.baseline.len() < 3
|
| 156 |
+
|| d.baseline.len() != d.candidate.len()
|
| 157 |
+
|| d.unit != computed.unit
|
| 158 |
+
|| [
|
| 159 |
+
d.baseline_median,
|
| 160 |
+
d.candidate_median,
|
| 161 |
+
d.median_percent,
|
| 162 |
+
d.lower_percent,
|
| 163 |
+
d.upper_percent,
|
| 164 |
+
] != [
|
| 165 |
+
computed.baseline_median,
|
| 166 |
+
computed.candidate_median,
|
| 167 |
+
computed.median_percent,
|
| 168 |
+
computed.lower_percent,
|
| 169 |
+
computed.upper_percent,
|
| 170 |
+
]
|
| 171 |
+
{
|
| 172 |
+
return Err("comparison summary disagrees with raw samples".into());
|
| 173 |
+
}
|
| 174 |
+
}
|
| 175 |
+
Ok(())
|
| 176 |
+
}
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
fn delta(metric: &str, ctx: u64, a: &[f64], b: &[f64]) -> Result<Delta, String> {
|
| 180 |
+
if a.is_empty() || b.is_empty() || a.iter().chain(b).any(|v| !v.is_finite() || *v <= 0.0) {
|
| 181 |
+
return Err("nonpositive or missing timing samples".into());
|
| 182 |
+
}
|
| 183 |
+
let baseline_median = fit::median(a);
|
| 184 |
+
let candidate_median = fit::median(b);
|
| 185 |
+
let extrema = |v: &[f64]| {
|
| 186 |
+
(
|
| 187 |
+
v.iter().copied().fold(f64::INFINITY, f64::min),
|
| 188 |
+
v.iter().copied().fold(0.0, f64::max),
|
| 189 |
+
)
|
| 190 |
+
};
|
| 191 |
+
let (amin, amax) = extrema(a);
|
| 192 |
+
let (bmin, bmax) = extrema(b);
|
| 193 |
+
Ok(Delta {
|
| 194 |
+
metric: metric.into(),
|
| 195 |
+
ctx,
|
| 196 |
+
unit: if metric == "first_token" {
|
| 197 |
+
"seconds"
|
| 198 |
+
} else {
|
| 199 |
+
"seconds/token"
|
| 200 |
+
}
|
| 201 |
+
.into(),
|
| 202 |
+
baseline: a.into(),
|
| 203 |
+
candidate: b.into(),
|
| 204 |
+
baseline_median,
|
| 205 |
+
candidate_median,
|
| 206 |
+
median_percent: 100.0 * (candidate_median / baseline_median - 1.0),
|
| 207 |
+
lower_percent: 100.0 * (bmin / amax - 1.0),
|
| 208 |
+
upper_percent: 100.0 * (bmax / amin - 1.0),
|
| 209 |
+
})
|
| 210 |
+
}
|
| 211 |
+
|
| 212 |
+
fn logit_error(a: &[f64], b: &[f64], atol: f64, rtol: f64) -> Result<LogitError, String> {
|
| 213 |
+
if a.is_empty() || a.len() != b.len() || a.iter().chain(b).any(|v| !v.is_finite()) {
|
| 214 |
+
return Err("invalid full-vocabulary logits".into());
|
| 215 |
+
}
|
| 216 |
+
let mut error = LogitError::default();
|
| 217 |
+
let (mut diff2, mut ref2) = (0.0f64, 0.0f64);
|
| 218 |
+
for (a, b) in a.iter().zip(b) {
|
| 219 |
+
let difference = (a - b).abs();
|
| 220 |
+
let tolerance = atol + rtol * a.abs();
|
| 221 |
+
error.max_abs = error.max_abs.max(difference);
|
| 222 |
+
// A zero tolerance is valid for an exact parity contract.
|
| 223 |
+
error.max_scaled = error
|
| 224 |
+
.max_scaled
|
| 225 |
+
.max((difference / tolerance.max(f64::MIN_POSITIVE)).min(f64::MAX));
|
| 226 |
+
error.bad += u64::from(difference > tolerance);
|
| 227 |
+
error.checked += 1;
|
| 228 |
+
diff2 += difference * difference;
|
| 229 |
+
ref2 += a * a;
|
| 230 |
+
}
|
| 231 |
+
error.rel_rms = (diff2 / ref2.max(f64::MIN_POSITIVE)).sqrt().min(f64::MAX);
|
| 232 |
+
Ok(error)
|
| 233 |
+
}
|
| 234 |
+
|
| 235 |
+
#[derive(Deserialize)]
|
| 236 |
+
#[serde(deny_unknown_fields)]
|
| 237 |
+
struct Frontier {
|
| 238 |
+
source: String,
|
| 239 |
+
model: String,
|
| 240 |
+
backend: String,
|
| 241 |
+
quality: bool,
|
| 242 |
+
quant_bits: u64,
|
| 243 |
+
prompt_tokens: u64,
|
| 244 |
+
frontier_tokens: u64,
|
| 245 |
+
prefill_tokens: u64,
|
| 246 |
+
ctx: u64,
|
| 247 |
+
vocab: usize,
|
| 248 |
+
argmax_id: usize,
|
| 249 |
+
argmax_logit: f64,
|
| 250 |
+
logits: Vec<f64>,
|
| 251 |
+
}
|
| 252 |
+
|
| 253 |
+
fn proof_file<T: serde::de::DeserializeOwned>(
|
| 254 |
+
root: &Path,
|
| 255 |
+
reference: &Reference,
|
| 256 |
+
) -> Result<T, String> {
|
| 257 |
+
let path = root.join(&reference.path);
|
| 258 |
+
if artifact::hash(&path)? != reference.sha256 {
|
| 259 |
+
return Err("correctness evidence changed".into());
|
| 260 |
+
}
|
| 261 |
+
let file = fs::File::open(&path).map_err(|e| e.to_string())?;
|
| 262 |
+
if file.metadata().map_err(|e| e.to_string())?.len() > 64 * 1024 * 1024 {
|
| 263 |
+
return Err("proof file exceeds 64 MiB".into());
|
| 264 |
+
}
|
| 265 |
+
serde_json::from_reader(file).map_err(|e| format!("{}: {e}", path.display()))
|
| 266 |
+
}
|
| 267 |
+
|
| 268 |
+
type FrontierProofs = BTreeMap<u64, Vec<(Frontier, Vec<i32>)>>;
|
| 269 |
+
|
| 270 |
+
fn proofs(root: &Path, scout: &Scout) -> Result<FrontierProofs, String> {
|
| 271 |
+
let mut proofs: BTreeMap<_, Vec<_>> = BTreeMap::new();
|
| 272 |
+
for sample in &scout.samples {
|
| 273 |
+
for row in &sample.rows {
|
| 274 |
+
let name = format!("frontier_{:06}.logits.json", row.ctx);
|
| 275 |
+
let frontier: Frontier = proof_file(
|
| 276 |
+
root,
|
| 277 |
+
sample
|
| 278 |
+
.proof_files
|
| 279 |
+
.get(&name)
|
| 280 |
+
.ok_or("missing full-vocabulary frontier; use scout --proof")?,
|
| 281 |
+
)?;
|
| 282 |
+
let name = format!("tokens-{}.json", row.ctx);
|
| 283 |
+
let tokens: Vec<i32> = proof_file(
|
| 284 |
+
root,
|
| 285 |
+
sample
|
| 286 |
+
.proof_files
|
| 287 |
+
.get(&name)
|
| 288 |
+
.ok_or("missing greedy-token evidence; use scout --proof")?,
|
| 289 |
+
)?;
|
| 290 |
+
if frontier.source != "ds4-bench"
|
| 291 |
+
|| frontier.backend != "cuda"
|
| 292 |
+
|| frontier.model.is_empty()
|
| 293 |
+
|| frontier.vocab == 0
|
| 294 |
+
|| frontier.logits.len() != frontier.vocab
|
| 295 |
+
|| frontier.argmax_id >= frontier.vocab
|
| 296 |
+
|| frontier.frontier_tokens != row.ctx
|
| 297 |
+
|| frontier.prompt_tokens != row.ctx
|
| 298 |
+
|| frontier.prefill_tokens != row.prefill_tokens
|
| 299 |
+
|| frontier.ctx <= row.ctx
|
| 300 |
+
|| tokens.len() as u64 != row.gen_tokens
|
| 301 |
+
|| tokens
|
| 302 |
+
.iter()
|
| 303 |
+
.any(|t| *t < 0 || *t as usize >= frontier.vocab)
|
| 304 |
+
|| frontier.logits.iter().any(|v| !v.is_finite())
|
| 305 |
+
|| !frontier.argmax_logit.is_finite()
|
| 306 |
+
|| frontier.logits[frontier.argmax_id] != frontier.argmax_logit
|
| 307 |
+
|| frontier.logits.iter().any(|v| *v > frontier.argmax_logit)
|
| 308 |
+
{
|
| 309 |
+
return Err("frontier/token proof disagrees with benchmark row".into());
|
| 310 |
+
}
|
| 311 |
+
if scout.workload.is_some()
|
| 312 |
+
&& scout.model_argument.as_deref() != Some(Path::new(&frontier.model))
|
| 313 |
+
{
|
| 314 |
+
return Err(
|
| 315 |
+
"proof model differs from the captured and hashed workload argument".into(),
|
| 316 |
+
);
|
| 317 |
+
}
|
| 318 |
+
proofs.entry(row.ctx).or_default().push((frontier, tokens));
|
| 319 |
+
}
|
| 320 |
+
}
|
| 321 |
+
Ok(proofs)
|
| 322 |
+
}
|
| 323 |
+
|
| 324 |
+
pub fn validate_proofs(root: &Path, scout: &Scout) -> Result<(), String> {
|
| 325 |
+
if scout.samples.len() != scout.requested_repeats as usize
|
| 326 |
+
|| scout.samples.iter().any(|s| s.rows.is_empty())
|
| 327 |
+
{
|
| 328 |
+
return Err("missing samples for requested proof".into());
|
| 329 |
+
}
|
| 330 |
+
proofs(root, scout).map(|_| ())
|
| 331 |
+
}
|
| 332 |
+
|
| 333 |
+
fn identity(a: &Scout, b: &Scout) -> Result<BTreeMap<String, String>, String> {
|
| 334 |
+
if !a.process_scope.verified
|
| 335 |
+
|| !b.process_scope.verified
|
| 336 |
+
|| a.process_scope.expected_owners != b.process_scope.expected_owners
|
| 337 |
+
{
|
| 338 |
+
return Err("GPU process snapshots do not match the intended owners for both runs".into());
|
| 339 |
+
}
|
| 340 |
+
if !a.unreviewed_environment.is_empty() || !b.unreviewed_environment.is_empty() {
|
| 341 |
+
return Err("unreviewed DS4/CUDA/LD environment controls prevent matched comparison; see scout.json".into());
|
| 342 |
+
}
|
| 343 |
+
let (Some(ga), Some(gb)) = (&a.gpu, &b.gpu) else {
|
| 344 |
+
return Err("comparison requires verified CUDA device identity".into());
|
| 345 |
+
};
|
| 346 |
+
if !ga.same_device(gb) {
|
| 347 |
+
return Err("device or driver differs".into());
|
| 348 |
+
}
|
| 349 |
+
if a.command[1..] != b.command[1..] || a.cwd != b.cwd {
|
| 350 |
+
return Err("benchmark arguments or working directory differ".into());
|
| 351 |
+
}
|
| 352 |
+
if a.cache_policy != "warmup-then-fresh" || a.cache_policy != b.cache_policy {
|
| 353 |
+
return Err("comparison requires matching explicit --cache-policy".into());
|
| 354 |
+
}
|
| 355 |
+
let (Some(wa), Some(wb)) = (&a.workload, &b.workload) else {
|
| 356 |
+
return Err(
|
| 357 |
+
"comparison requires --workload with hashed model/prompt and cache contract".into(),
|
| 358 |
+
);
|
| 359 |
+
};
|
| 360 |
+
let hashes = |w: &experiment::Workload| {
|
| 361 |
+
w.files
|
| 362 |
+
.iter()
|
| 363 |
+
.map(|(k, v)| (k.clone(), v.sha256.clone()))
|
| 364 |
+
.collect::<BTreeMap<_, _>>()
|
| 365 |
+
};
|
| 366 |
+
if wa.name != wb.name
|
| 367 |
+
|| wa.family != wb.family
|
| 368 |
+
|| hashes(wa) != hashes(wb)
|
| 369 |
+
|| wa.shape != wb.shape
|
| 370 |
+
|| wa.cache_state != wb.cache_state
|
| 371 |
+
{
|
| 372 |
+
return Err("workload model/prompt/shape/cache identity differs".into());
|
| 373 |
+
}
|
| 374 |
+
let mut changes = BTreeMap::new();
|
| 375 |
+
if a.executable_sha256 != b.executable_sha256 {
|
| 376 |
+
changes.insert(
|
| 377 |
+
"executable_sha256".into(),
|
| 378 |
+
format!("{} -> {}", a.executable_sha256, b.executable_sha256),
|
| 379 |
+
);
|
| 380 |
+
}
|
| 381 |
+
let keys: std::collections::BTreeSet<_> =
|
| 382 |
+
a.environment.keys().chain(b.environment.keys()).collect();
|
| 383 |
+
for key in keys {
|
| 384 |
+
if a.environment.get(key) == b.environment.get(key) {
|
| 385 |
+
continue;
|
| 386 |
+
}
|
| 387 |
+
if !crate::knobs::tunable(key) {
|
| 388 |
+
return Err(format!("non-tuning runtime/memory policy changed: {key}"));
|
| 389 |
+
}
|
| 390 |
+
for value in [a.environment.get(key), b.environment.get(key)]
|
| 391 |
+
.into_iter()
|
| 392 |
+
.flatten()
|
| 393 |
+
{
|
| 394 |
+
crate::knobs::validate(key, value, &wa.family)?;
|
| 395 |
+
}
|
| 396 |
+
changes.insert(
|
| 397 |
+
key.clone(),
|
| 398 |
+
format!(
|
| 399 |
+
"{:?} -> {:?}",
|
| 400 |
+
a.environment.get(key),
|
| 401 |
+
b.environment.get(key)
|
| 402 |
+
),
|
| 403 |
+
);
|
| 404 |
+
}
|
| 405 |
+
Ok(changes)
|
| 406 |
+
}
|
| 407 |
+
|
| 408 |
+
fn analyze(
|
| 409 |
+
a: &Scout,
|
| 410 |
+
ar: &Path,
|
| 411 |
+
b: &Scout,
|
| 412 |
+
br: &Path,
|
| 413 |
+
args: &cli::Compare,
|
| 414 |
+
) -> Result<Comparison, String> {
|
| 415 |
+
let changes = identity(a, b)?;
|
| 416 |
+
if a.requested_repeats != b.requested_repeats
|
| 417 |
+
|| a.samples.len() != a.requested_repeats as usize
|
| 418 |
+
|| b.samples.len() != b.requested_repeats as usize
|
| 419 |
+
{
|
| 420 |
+
return Err("comparison requires equal complete requested sample counts".into());
|
| 421 |
+
}
|
| 422 |
+
if a.samples.len() < 3 || b.samples.len() < 3 {
|
| 423 |
+
return Err("comparison requires at least three fresh unprofiled samples per side".into());
|
| 424 |
+
}
|
| 425 |
+
type TimingPair = (Vec<f64>, Vec<f64>);
|
| 426 |
+
let mut timing: BTreeMap<(String, u64), TimingPair> = BTreeMap::new();
|
| 427 |
+
let shape = |s: &experiment::Sample| {
|
| 428 |
+
s.rows
|
| 429 |
+
.iter()
|
| 430 |
+
.map(|r| (r.ctx, r.prefill_tokens, r.gen_tokens, r.kvcache_bytes))
|
| 431 |
+
.collect::<Vec<_>>()
|
| 432 |
+
};
|
| 433 |
+
let expected = shape(&a.samples[0]);
|
| 434 |
+
if expected.is_empty()
|
| 435 |
+
|| expected
|
| 436 |
+
.iter()
|
| 437 |
+
.map(|r| r.0)
|
| 438 |
+
.collect::<std::collections::BTreeSet<_>>()
|
| 439 |
+
.len()
|
| 440 |
+
!= expected.len()
|
| 441 |
+
{
|
| 442 |
+
return Err("empty or duplicate benchmark frontiers".into());
|
| 443 |
+
}
|
| 444 |
+
for (candidate, scout) in [(false, a), (true, b)] {
|
| 445 |
+
for sample in &scout.samples {
|
| 446 |
+
if shape(sample) != expected {
|
| 447 |
+
return Err("token counts/frontiers/KV allocation differ across samples".into());
|
| 448 |
+
}
|
| 449 |
+
for row in &sample.rows {
|
| 450 |
+
for (name, value) in [
|
| 451 |
+
("prefill", 1.0 / row.prefill_tps),
|
| 452 |
+
("decode", 1.0 / row.gen_tps),
|
| 453 |
+
("first_token", row.first_token_sec),
|
| 454 |
+
] {
|
| 455 |
+
let entry = timing.entry((name.into(), row.ctx)).or_default();
|
| 456 |
+
if candidate {
|
| 457 |
+
entry.1.push(value);
|
| 458 |
+
} else {
|
| 459 |
+
entry.0.push(value);
|
| 460 |
+
}
|
| 461 |
+
}
|
| 462 |
+
}
|
| 463 |
+
}
|
| 464 |
+
}
|
| 465 |
+
let baseline = proofs(ar, a)?;
|
| 466 |
+
let candidate = proofs(br, b)?;
|
| 467 |
+
let mut correctness = LogitError::default();
|
| 468 |
+
let mut token_mismatches = 0;
|
| 469 |
+
let mut argmax_mismatches = 0;
|
| 470 |
+
for (ctx, base) in &baseline {
|
| 471 |
+
let (reference, tokens) = &base[0];
|
| 472 |
+
let candidates = candidate.get(ctx).ok_or("missing candidate frontier")?;
|
| 473 |
+
for (frontier, ids) in base.iter().chain(candidates) {
|
| 474 |
+
if frontier.source != reference.source
|
| 475 |
+
|| frontier.backend != reference.backend
|
| 476 |
+
|| frontier.model != reference.model
|
| 477 |
+
|| frontier.quality != reference.quality
|
| 478 |
+
|| frontier.quant_bits != reference.quant_bits
|
| 479 |
+
|| frontier.ctx != reference.ctx
|
| 480 |
+
{
|
| 481 |
+
return Err("quality/quantization/context proof metadata differs".into());
|
| 482 |
+
}
|
| 483 |
+
token_mismatches +=
|
| 484 |
+
u64::from(ids != tokens || frontier.argmax_id != reference.argmax_id);
|
| 485 |
+
argmax_mismatches += u64::from(frontier.argmax_id != reference.argmax_id);
|
| 486 |
+
let error = logit_error(
|
| 487 |
+
&reference.logits,
|
| 488 |
+
&frontier.logits,
|
| 489 |
+
args.logit_atol,
|
| 490 |
+
args.logit_rtol,
|
| 491 |
+
)?;
|
| 492 |
+
correctness.max_abs = correctness.max_abs.max(error.max_abs);
|
| 493 |
+
correctness.max_scaled = correctness.max_scaled.max(error.max_scaled);
|
| 494 |
+
correctness.bad += error.bad;
|
| 495 |
+
correctness.checked += error.checked;
|
| 496 |
+
correctness.rel_rms = correctness.rel_rms.max(error.rel_rms);
|
| 497 |
+
if args.logit_rel_rms > 0.0 {
|
| 498 |
+
let scaled = error.rel_rms / rel_rms_bound(args.logit_rel_rms, *ctx);
|
| 499 |
+
correctness.rel_rms_scaled = correctness.rel_rms_scaled.max(scaled.min(f64::MAX));
|
| 500 |
+
}
|
| 501 |
+
}
|
| 502 |
+
}
|
| 503 |
+
let metrics = timing
|
| 504 |
+
.into_iter()
|
| 505 |
+
.map(|((name, ctx), (a, b))| delta(&name, ctx, &a, &b))
|
| 506 |
+
.collect::<Result<Vec<_>, _>>()?;
|
| 507 |
+
let verdict = verdict(
|
| 508 |
+
&metrics,
|
| 509 |
+
&correctness,
|
| 510 |
+
token_mismatches,
|
| 511 |
+
argmax_mismatches,
|
| 512 |
+
args.max_slowdown_percent,
|
| 513 |
+
args.logit_rel_rms,
|
| 514 |
+
);
|
| 515 |
+
Ok(Comparison {
|
| 516 |
+
verdict,
|
| 517 |
+
metrics,
|
| 518 |
+
correctness,
|
| 519 |
+
token_mismatches,
|
| 520 |
+
argmax_mismatches,
|
| 521 |
+
max_slowdown_percent: args.max_slowdown_percent,
|
| 522 |
+
logit_atol: args.logit_atol,
|
| 523 |
+
logit_rtol: args.logit_rtol,
|
| 524 |
+
logit_rel_rms: args.logit_rel_rms,
|
| 525 |
+
changes,
|
| 526 |
+
reasons: Vec::new(),
|
| 527 |
+
method: "sample-extrema-envelope-v1".into(),
|
| 528 |
+
})
|
| 529 |
+
}
|
| 530 |
+
|
| 531 |
+
fn verdict(
|
| 532 |
+
metrics: &[Delta],
|
| 533 |
+
correctness: &LogitError,
|
| 534 |
+
mismatches: u64,
|
| 535 |
+
argmax_mismatches: u64,
|
| 536 |
+
tolerance: f64,
|
| 537 |
+
logit_rel_rms: f64,
|
| 538 |
+
) -> Verdict {
|
| 539 |
+
// The relaxed contract accepts summation-order changes that a chaotic
|
| 540 |
+
// model amplifies into different greedy continuations: it bounds the
|
| 541 |
+
// frontier's relative RMS per context and keeps its argmax instead of
|
| 542 |
+
// every logit and the whole sequence.
|
| 543 |
+
let incorrect = if logit_rel_rms > 0.0 {
|
| 544 |
+
correctness.rel_rms_scaled > 1.0 || argmax_mismatches > 0
|
| 545 |
+
} else {
|
| 546 |
+
correctness.bad > 0 || mismatches > 0
|
| 547 |
+
};
|
| 548 |
+
if incorrect {
|
| 549 |
+
return Verdict::Incorrect;
|
| 550 |
+
}
|
| 551 |
+
if metrics.iter().any(|d| d.lower_percent > tolerance) {
|
| 552 |
+
return Verdict::Regressed;
|
| 553 |
+
}
|
| 554 |
+
if metrics.iter().any(|d| d.upper_percent > tolerance) {
|
| 555 |
+
return Verdict::Inconclusive;
|
| 556 |
+
}
|
| 557 |
+
if metrics
|
| 558 |
+
.iter()
|
| 559 |
+
.any(|d| d.metric != "first_token" && d.upper_percent < -1.0)
|
| 560 |
+
{
|
| 561 |
+
return Verdict::Improved;
|
| 562 |
+
}
|
| 563 |
+
Verdict::Pass
|
| 564 |
+
}
|
| 565 |
+
|
| 566 |
+
pub fn run(args: &cli::Compare) -> Result<(), String> {
|
| 567 |
+
if [
|
| 568 |
+
args.max_slowdown_percent,
|
| 569 |
+
args.logit_atol,
|
| 570 |
+
args.logit_rtol,
|
| 571 |
+
args.logit_rel_rms,
|
| 572 |
+
]
|
| 573 |
+
.into_iter()
|
| 574 |
+
.any(|v| !v.is_finite() || v < 0.0)
|
| 575 |
+
{
|
| 576 |
+
return Err("comparison tolerances must be finite and nonnegative".into());
|
| 577 |
+
}
|
| 578 |
+
artifact::directory(&args.out)?;
|
| 579 |
+
let result = (|| {
|
| 580 |
+
let a = experiment::load(&args.baseline)?;
|
| 581 |
+
let b = experiment::load(&args.candidate)?;
|
| 582 |
+
analyze(
|
| 583 |
+
a.require()?,
|
| 584 |
+
args.baseline.parent().unwrap_or(Path::new(".")),
|
| 585 |
+
b.require()?,
|
| 586 |
+
args.candidate.parent().unwrap_or(Path::new(".")),
|
| 587 |
+
args,
|
| 588 |
+
)
|
| 589 |
+
})();
|
| 590 |
+
let data = match result {
|
| 591 |
+
Ok(data) => data,
|
| 592 |
+
Err(error) => Comparison {
|
| 593 |
+
verdict: Verdict::Incomparable,
|
| 594 |
+
metrics: Vec::new(),
|
| 595 |
+
correctness: LogitError::default(),
|
| 596 |
+
token_mismatches: 0,
|
| 597 |
+
argmax_mismatches: 0,
|
| 598 |
+
max_slowdown_percent: args.max_slowdown_percent,
|
| 599 |
+
logit_atol: args.logit_atol,
|
| 600 |
+
logit_rtol: args.logit_rtol,
|
| 601 |
+
logit_rel_rms: args.logit_rel_rms,
|
| 602 |
+
changes: BTreeMap::new(),
|
| 603 |
+
reasons: vec![error],
|
| 604 |
+
method: "sample-extrema-envelope-v1".into(),
|
| 605 |
+
},
|
| 606 |
+
};
|
| 607 |
+
let passed = matches!(data.verdict, Verdict::Pass | Verdict::Improved);
|
| 608 |
+
let mut artifact = Artifact::new(data);
|
| 609 |
+
artifact.complete = artifact
|
| 610 |
+
.data
|
| 611 |
+
.as_ref()
|
| 612 |
+
.is_some_and(|d| d.verdict != Verdict::Incomparable);
|
| 613 |
+
for path in [&args.baseline, &args.candidate] {
|
| 614 |
+
if path.is_file() {
|
| 615 |
+
artifact.inputs.push(artifact::reference(path, &args.out)?);
|
| 616 |
+
}
|
| 617 |
+
}
|
| 618 |
+
artifact::save(&args.out.join("compare.json"), &artifact)?;
|
| 619 |
+
println!(
|
| 620 |
+
"compare: {:?}; {}",
|
| 621 |
+
artifact.data.as_ref().unwrap().verdict,
|
| 622 |
+
args.out.join("compare.json").display()
|
| 623 |
+
);
|
| 624 |
+
if args.regression && !passed {
|
| 625 |
+
return Err("regression gate did not pass; see compare.json".into());
|
| 626 |
+
}
|
| 627 |
+
Ok(())
|
| 628 |
+
}
|
| 629 |
+
|
| 630 |
+
#[cfg(test)]
|
| 631 |
+
mod tests {
|
| 632 |
+
use super::*;
|
| 633 |
+
#[test]
|
| 634 |
+
fn preserves_live_roundtrip() {
|
| 635 |
+
// These measured Qwen ratios exposed a lossy JSON parse on reload.
|
| 636 |
+
let recorded: Comparison =
|
| 637 |
+
serde_json::from_str(include_str!("../tests/fixtures/compare-live.json")).unwrap();
|
| 638 |
+
recorded.validate().unwrap();
|
| 639 |
+
let encoded = serde_json::to_vec(&recorded).unwrap();
|
| 640 |
+
let mut loaded: Comparison = serde_json::from_slice(&encoded).unwrap();
|
| 641 |
+
loaded.validate().unwrap();
|
| 642 |
+
loaded.metrics[0].median_percent += 0.01;
|
| 643 |
+
assert!(loaded.validate().is_err());
|
| 644 |
+
}
|
| 645 |
+
|
| 646 |
+
#[test]
|
| 647 |
+
fn separates_regression_noise() {
|
| 648 |
+
let stable = delta("prefill", 2048, &[1.0, 1.01, 0.99], &[1.1, 1.11, 1.09]).unwrap();
|
| 649 |
+
assert!(stable.lower_percent > 3.0);
|
| 650 |
+
let noise = delta("prefill", 2048, &[0.9, 1.0, 1.1], &[0.9, 1.03, 1.2]).unwrap();
|
| 651 |
+
assert!(noise.lower_percent < 3.0 && noise.upper_percent > 3.0);
|
| 652 |
+
assert!(delta("prefill", 2048, &[1.0], &[0.0]).is_err());
|
| 653 |
+
assert!(delta("prefill", 2048, &[1.0], &[f64::NAN]).is_err());
|
| 654 |
+
}
|
| 655 |
+
#[test]
|
| 656 |
+
fn checks_every_logit() {
|
| 657 |
+
let a = vec![0.0, 1.0, -2.0, 0.001];
|
| 658 |
+
assert!(logit_error(&a, &a, 1e-4, 1e-4).unwrap().bad == 0);
|
| 659 |
+
let mut b = a.clone();
|
| 660 |
+
b[3] = 0.01;
|
| 661 |
+
assert_eq!(logit_error(&a, &b, 1e-4, 1e-4).unwrap().bad, 1);
|
| 662 |
+
assert!(logit_error(&a, &[f64::NAN], 1e-4, 1e-4).is_err());
|
| 663 |
+
}
|
| 664 |
+
#[test]
|
| 665 |
+
fn relaxed_contract_bounds_rms_and_argmax() {
|
| 666 |
+
let a = vec![0.0, 3.0, -4.0];
|
| 667 |
+
let b = vec![0.3, 3.0, -4.0];
|
| 668 |
+
let mut error = logit_error(&a, &b, 1e-4, 1e-4).unwrap();
|
| 669 |
+
assert_eq!(error.bad, 1);
|
| 670 |
+
assert!((error.rel_rms - 0.06).abs() < 1e-12);
|
| 671 |
+
// The bound is the reference at 1024 tokens and grows with log2(ctx).
|
| 672 |
+
assert!((rel_rms_bound(0.1, 1024) - 0.1).abs() < 1e-12);
|
| 673 |
+
assert!((rel_rms_bound(0.1, 8192) - 0.13).abs() < 1e-12);
|
| 674 |
+
assert!(rel_rms_bound(0.1, 65536) < 0.17);
|
| 675 |
+
let stable = delta("prefill", 2048, &[1.0, 1.01, 0.99], &[0.9, 0.91, 0.89]).unwrap();
|
| 676 |
+
let metrics = [stable];
|
| 677 |
+
// Strict: any bad logit or sequence mismatch is incorrect.
|
| 678 |
+
assert_eq!(
|
| 679 |
+
verdict(&metrics, &error, 1, 0, 3.0, 0.0),
|
| 680 |
+
Verdict::Incorrect
|
| 681 |
+
);
|
| 682 |
+
// Relaxed: the sequence may diverge while the scaled RMS and argmax hold.
|
| 683 |
+
error.rel_rms_scaled = error.rel_rms / rel_rms_bound(0.1, 2048);
|
| 684 |
+
assert_eq!(verdict(&metrics, &error, 1, 0, 3.0, 0.1), Verdict::Improved);
|
| 685 |
+
assert_eq!(
|
| 686 |
+
verdict(&metrics, &error, 1, 1, 3.0, 0.1),
|
| 687 |
+
Verdict::Incorrect
|
| 688 |
+
);
|
| 689 |
+
error.rel_rms_scaled = error.rel_rms / rel_rms_bound(0.05, 2048);
|
| 690 |
+
assert_eq!(
|
| 691 |
+
verdict(&metrics, &error, 1, 0, 3.0, 0.05),
|
| 692 |
+
Verdict::Incorrect
|
| 693 |
+
);
|
| 694 |
+
}
|
| 695 |
+
}
|
crates/ds4-perf/src/csv.rs
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
use std::io::{self, BufRead};
|
| 2 |
+
|
| 3 |
+
// Read records incrementally: GPU traces can contain millions of launches.
|
| 4 |
+
// Quotes, escaped quotes and embedded newlines are legal in Nsight names.
|
| 5 |
+
pub fn records(
|
| 6 |
+
reader: impl BufRead,
|
| 7 |
+
mut visit: impl FnMut(Vec<String>) -> Result<(), String>,
|
| 8 |
+
) -> Result<(), String> {
|
| 9 |
+
let mut field = Vec::new();
|
| 10 |
+
let mut row = Vec::new();
|
| 11 |
+
let mut quoted = false;
|
| 12 |
+
let mut closed = false;
|
| 13 |
+
for byte in reader.bytes() {
|
| 14 |
+
let byte = byte.map_err(|e| e.to_string())?;
|
| 15 |
+
if quoted {
|
| 16 |
+
if byte == b'"' {
|
| 17 |
+
quoted = false;
|
| 18 |
+
closed = true;
|
| 19 |
+
} else {
|
| 20 |
+
field.push(byte);
|
| 21 |
+
}
|
| 22 |
+
continue;
|
| 23 |
+
}
|
| 24 |
+
match byte {
|
| 25 |
+
b'"' if closed => {
|
| 26 |
+
field.push(b'"');
|
| 27 |
+
quoted = true;
|
| 28 |
+
closed = false;
|
| 29 |
+
}
|
| 30 |
+
b'"' if field.is_empty() => quoted = true,
|
| 31 |
+
b',' | b'\n' => {
|
| 32 |
+
row.push(take_field(&mut field)?);
|
| 33 |
+
closed = false;
|
| 34 |
+
if byte == b'\n' {
|
| 35 |
+
visit(std::mem::take(&mut row))?;
|
| 36 |
+
}
|
| 37 |
+
}
|
| 38 |
+
b'\r' => {}
|
| 39 |
+
b' ' | b'\t' if closed => {}
|
| 40 |
+
_ if closed => return Err("characters after closing CSV quote".into()),
|
| 41 |
+
_ => field.push(byte),
|
| 42 |
+
}
|
| 43 |
+
}
|
| 44 |
+
if quoted {
|
| 45 |
+
return Err("unterminated CSV quote".into());
|
| 46 |
+
}
|
| 47 |
+
if !field.is_empty() || !row.is_empty() || closed {
|
| 48 |
+
row.push(take_field(&mut field)?);
|
| 49 |
+
visit(row)?;
|
| 50 |
+
}
|
| 51 |
+
Ok(())
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
fn take_field(bytes: &mut Vec<u8>) -> Result<String, String> {
|
| 55 |
+
String::from_utf8(std::mem::take(bytes)).map_err(|e| format!("CSV is not UTF-8: {e}"))
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
pub fn field(value: &str) -> String {
|
| 59 |
+
if value.contains([',', '"', '\r', '\n']) {
|
| 60 |
+
format!("\"{}\"", value.replace('"', "\"\""))
|
| 61 |
+
} else {
|
| 62 |
+
value.into()
|
| 63 |
+
}
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
pub fn key(value: &str) -> String {
|
| 67 |
+
value
|
| 68 |
+
.trim()
|
| 69 |
+
.trim_start_matches('\u{feff}')
|
| 70 |
+
.to_ascii_lowercase()
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
pub fn column(header: &[String], names: &[&str]) -> Option<usize> {
|
| 74 |
+
header.iter().position(|s| names.contains(&key(s).as_str()))
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
pub fn value<'a>(row: &'a [String], header: &[String], names: &[&str]) -> Option<&'a str> {
|
| 78 |
+
row.get(column(header, names)?)
|
| 79 |
+
.map(|s| s.trim())
|
| 80 |
+
.filter(|s| !s.is_empty())
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
pub fn number(value: &str) -> Result<f64, String> {
|
| 84 |
+
value
|
| 85 |
+
.trim()
|
| 86 |
+
.parse::<f64>()
|
| 87 |
+
.ok()
|
| 88 |
+
.filter(|v| v.is_finite() && *v >= 0.0)
|
| 89 |
+
.ok_or_else(|| format!("invalid nonnegative number: {value}"))
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
pub fn time(row: &[String], header: &[String], name: &str) -> Option<f64> {
|
| 93 |
+
for (unit, factor) in [
|
| 94 |
+
("ns", 1.0),
|
| 95 |
+
("us", 1_000.0),
|
| 96 |
+
("ms", 1_000_000.0),
|
| 97 |
+
("s", 1_000_000_000.0),
|
| 98 |
+
] {
|
| 99 |
+
let col = format!("{} ({unit})", name.to_ascii_lowercase());
|
| 100 |
+
if let Some(value) = value(row, header, &[&col]) {
|
| 101 |
+
return number(value)
|
| 102 |
+
.ok()
|
| 103 |
+
.map(|v| v * factor)
|
| 104 |
+
.filter(|v| v.is_finite());
|
| 105 |
+
}
|
| 106 |
+
}
|
| 107 |
+
None
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
pub fn open(path: &std::path::Path) -> io::Result<io::BufReader<std::fs::File>> {
|
| 111 |
+
std::fs::File::open(path).map(io::BufReader::new)
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
pub fn table(
|
| 115 |
+
reader: impl BufRead,
|
| 116 |
+
marker: &str,
|
| 117 |
+
mut visit: impl FnMut(&[String], &[String]) -> Result<(), String>,
|
| 118 |
+
) -> Result<(), String> {
|
| 119 |
+
let mut header = Vec::new();
|
| 120 |
+
records(reader, |row| {
|
| 121 |
+
if column(&row, &[marker]).is_some() {
|
| 122 |
+
header = row;
|
| 123 |
+
return Ok(());
|
| 124 |
+
}
|
| 125 |
+
if header.is_empty() || row.iter().all(|s| s.trim().is_empty()) {
|
| 126 |
+
return Ok(());
|
| 127 |
+
}
|
| 128 |
+
if row.len() != header.len() {
|
| 129 |
+
return Err("unexpected CSV record width after header".into());
|
| 130 |
+
}
|
| 131 |
+
visit(&header, &row)
|
| 132 |
+
})
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
#[cfg(test)]
|
| 136 |
+
mod tests {
|
| 137 |
+
use super::*;
|
| 138 |
+
|
| 139 |
+
#[test]
|
| 140 |
+
fn quoted_templates_and_newlines() {
|
| 141 |
+
let input = "Name,Count\r\n\"foo<int, 128, bar<float>>\",2\r\n\"line\n\"\"two\"\"\",3";
|
| 142 |
+
let mut rows = Vec::new();
|
| 143 |
+
records(input.as_bytes(), |row| {
|
| 144 |
+
rows.push(row);
|
| 145 |
+
Ok(())
|
| 146 |
+
})
|
| 147 |
+
.unwrap();
|
| 148 |
+
assert_eq!(rows[1], ["foo<int, 128, bar<float>>", "2"]);
|
| 149 |
+
assert_eq!(rows[2], ["line\n\"two\"", "3"]);
|
| 150 |
+
assert!(records(b"\"bad".as_slice(), |_| Ok(())).is_err());
|
| 151 |
+
for invalid in ["NaN", "inf", "-1"] {
|
| 152 |
+
assert!(number(invalid).is_err());
|
| 153 |
+
}
|
| 154 |
+
}
|
| 155 |
+
}
|